From 8d579966d2acc5fbb6dd2b87f2588388d2d972ea Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Thu, 11 Dec 2025 15:31:33 +0530 Subject: [PATCH 01/22] Added Endee VectorDB --- vectordb_bench/__init__.py | 2 +- vectordb_bench/backend/clients/__init__.py | 11 + .../backend/clients/endee/__init__.py | 0 vectordb_bench/backend/clients/endee/cli.py | 114 ++ .../backend/clients/endee/config.py | 32 + vectordb_bench/backend/clients/endee/endee.py | 206 ++++ .../backend/runner/serial_runner.py | 1048 ++++++++++++++++- vectordb_bench/cli/vectordbbench.py | 2 + vectordb_bench/frontend/config/styles.py | 1 + 9 files changed, 1378 insertions(+), 38 deletions(-) create mode 100644 vectordb_bench/backend/clients/endee/__init__.py create mode 100644 vectordb_bench/backend/clients/endee/cli.py create mode 100644 vectordb_bench/backend/clients/endee/config.py create mode 100644 vectordb_bench/backend/clients/endee/endee.py diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index 07f77bb02..0a2512f7c 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -19,7 +19,7 @@ class config: DEFAULT_DATASET_URL = env.str("DEFAULT_DATASET_URL", AWS_S3_URL) DATASET_SOURCE = env.str("DATASET_SOURCE", "S3") # Options "S3" or "AliyunOSS" DATASET_LOCAL_DIR = env.path("DATASET_LOCAL_DIR", "/tmp/vectordb_bench/dataset") - NUM_PER_BATCH = env.int("NUM_PER_BATCH", 100) + NUM_PER_BATCH = env.int("NUM_PER_BATCH", 1000) TIME_PER_BATCH = 1 # 1s. for streaming insertion. MAX_INSERT_RETRY = 5 MAX_SEARCH_RETRY = 5 diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 3bd2d3189..748f64707 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -56,6 +56,7 @@ class DB(Enum): AliSQL = "AlibabaCloudRDSMySQL" Doris = "Doris" TurboPuffer = "TurboPuffer" + Endee = "Endee" @property def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 @@ -227,6 +228,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 from .alisql.alisql import AliSQL return AliSQL + + if self == DB.Endee: + from .endee.endee import Endee + + return Endee msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -401,6 +407,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901, PLR0915 from .alisql.config import AliSQLConfig return AliSQLConfig + + if self == DB.Endee: + from .endee.config import EndeeConfig + + return EndeeConfig msg = f"Unknown DB: {self.name}" raise ValueError(msg) diff --git a/vectordb_bench/backend/clients/endee/__init__.py b/vectordb_bench/backend/clients/endee/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vectordb_bench/backend/clients/endee/cli.py b/vectordb_bench/backend/clients/endee/cli.py new file mode 100644 index 000000000..952f1b682 --- /dev/null +++ b/vectordb_bench/backend/clients/endee/cli.py @@ -0,0 +1,114 @@ +import click +import logging +import uuid +from typing import Annotated + +from vectordb_bench.cli.cli import ( + CommonTypedDict, + benchmark_runner, + click_parameter_decorators_from_typed_dict, + get_custom_case_config, + parse_task_stages, +) +from .. import DB +from .config import EndeeConfig +from ..api import EmptyDBCaseConfig + +log = logging.getLogger(__name__) + + +class EndeeTypedDict(CommonTypedDict): + token: Annotated[ + str, + click.option("--token", type=str, required=True, help="Endee API token") + ] + region: Annotated[ + str, + click.option("--region", type=str, default=None, help="Endee region", show_default=True) + ] + space_type: Annotated[ + str, + click.option("--space-type", type=click.Choice(["cosine", "l2", "dot_product"]), default="cosine", help="Distance metric", show_default=True) + ] + precision: Annotated[ + str, + click.option("--precision", type=click.Choice(["medium", "high", "ultra-high", "fp16"]), default="medium", help="Quant Level", show_default=True) + ] + version: Annotated[ + int, + click.option("--version", type=int, default=None, help="Index version", show_default=True) + ] + m: Annotated[ + int, + click.option("--m", type=int, default=None, help="HNSW M parameter", show_default=True) + ] + ef_con: Annotated[ + int, + click.option("--ef-con", type=int, default=None, help="HNSW construction parameter", show_default=True) + ] + ef_search: Annotated[ + int, + click.option("--ef-search", type=int, default=None, help="HNSW search parameter", show_default=True) + ] + index_name: Annotated[ + str, + click.option("--index-name", type=str, required=True, help="Endee index name (will use a random name if not provided)", show_default=True) + ] + + + +@click.command() +@click_parameter_decorators_from_typed_dict(EndeeTypedDict) +def Endee(**parameters): + stages = parse_task_stages( + parameters["drop_old"], + parameters["load"], + parameters["search_serial"], + parameters["search_concurrent"], + ) + + # Generate a random collection name if not provided + collection_name = parameters["index_name"] + if not collection_name: + collection_name = f"endee_bench_{uuid.uuid4().hex[:8]}" + + # Filter out None values before creating config + params_for_vecx = {k: v for k, v in parameters.items() if v is not None} + db_config = EndeeConfig(**params_for_vecx) + + custom_case_config = get_custom_case_config(parameters) + + # Create task config + from vectordb_bench.models import TaskConfig, CaseConfig, CaseType, ConcurrencySearchConfig + + # Create an instance of EmptyDBCaseConfig instead of passing None + db_case_config = EmptyDBCaseConfig() + + task = TaskConfig( + db=DB.Endee, + db_config=db_config, # Use the EndeeConfig instance directly + db_case_config=db_case_config, + case_config=CaseConfig( + case_id=CaseType[parameters["case_type"]], + k=parameters["k"], + concurrency_search_config=ConcurrencySearchConfig( + concurrency_duration=parameters["concurrency_duration"], + num_concurrency=[int(s) for s in parameters["num_concurrency"]], + ), + custom_case=custom_case_config, + ), + stages=stages, + ) + + # Use the run method of the benchmark_runner object + if not parameters["dry_run"]: + benchmark_runner.run([task]) + + # Wait for task to complete if needed + import time + from vectordb_bench.interface import global_result_future + from concurrent.futures import wait + + time.sleep(5) + if global_result_future: + wait([global_result_future]) \ No newline at end of file diff --git a/vectordb_bench/backend/clients/endee/config.py b/vectordb_bench/backend/clients/endee/config.py new file mode 100644 index 000000000..476f9b5e0 --- /dev/null +++ b/vectordb_bench/backend/clients/endee/config.py @@ -0,0 +1,32 @@ +from pydantic import SecretStr +from typing import Optional +from ..api import DBConfig + + +class EndeeConfig(DBConfig): + token: SecretStr + region: Optional[str] = "india-west-1" + space_type: str ="cosine" + # use_fp16: bool = False + precision: str = "medium" + version: Optional[int] = 1 + m: Optional[int] = 16 + ef_con: Optional[int] = 128 + ef_search: Optional[int] = 128 + # collection_name: str + index_name: str + + def to_dict(self) -> dict: + return { + "token": self.token.get_secret_value(), + "region": self.region, + "space_type": self.space_type, + # "use_fp16": self.use_fp16, + "precision": self.precision, + "version": self.version, + "m": self.m, + "ef_con": self.ef_con, + "ef_search": self.ef_search, + # "collection_name": self.collection_name, + "index_name": self.index_name, + } \ No newline at end of file diff --git a/vectordb_bench/backend/clients/endee/endee.py b/vectordb_bench/backend/clients/endee/endee.py new file mode 100644 index 000000000..5095acd13 --- /dev/null +++ b/vectordb_bench/backend/clients/endee/endee.py @@ -0,0 +1,206 @@ +import logging +from contextlib import contextmanager + +from endee import endee + +from ..api import DBCaseConfig, EmptyDBCaseConfig, IndexType, VectorDB +from .config import EndeeConfig + +log = logging.getLogger(__name__) + + +class Endee(VectorDB): + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: DBCaseConfig, + drop_old: bool = False, + **kwargs, + ): + print(db_config) + + self.token = db_config.get("token", "") + self.region = db_config.get("region", "india-west-1") + + # self.collection_name = db_config.get("collection_name", "") + self.collection_name = (db_config.get("collection_name") or db_config.get("index_name")) + + if not self.collection_name: + import uuid + self.collection_name = f"endee_bench_{uuid.uuid4().hex[:8]}" + + self.space_type = db_config.get("space_type", "cosine") + # self.use_fp16 = db_config.get("use_fp16", False) + self.precision = db_config.get("precision") + self.version = db_config.get("version") + self.M = db_config.get("m") + self.ef_con = db_config.get("ef_con") + self.ef_search = db_config.get("ef_search") + self.nd = endee.Endee(token=self.token) + + # Using Base url + # self.nd.base_url = "https://10.160.0.8:8080" + + + # try: + # indices = self.nd.list_indexes().get("indices", []) + # index_names = [index["name"] for index in indices] if indices else [] + + # if drop_old and self.collection_name in index_names: + # self._drop_index(self.collection_name) + # self._create_index(dim) + # elif self.collection_name not in index_names: + # self._create_index(dim) + # except Exception as e: + # log.error(f"Error connecting to Endee API: {e}") + # raise + + try: + index_name = self.collection_name # assign before use + indices = self.nd.list_indexes().get("indices", []) + index_names = [index["name"] for index in indices] if indices else [] + try: + self.index = self.nd.get_index(name=index_name) + log.info(f"Connected to existing Endee index: '{index_name}'") + + except Exception as fetch_error: + log.warning(f"Index '{index_name}' not found. Creating new index...") + try: + self._create_index(dim) + self.index = self.nd.get_index(name=index_name) + log.info(f"Successfully created and connected to index: '{index_name}'") + + except Exception as create_error: + # If error is "index already exists", just get the index again + if "already exists" in str(create_error).lower() or "conflict" in str(create_error).lower(): + log.warning(f"Index '{index_name}' already exists despite previous error. Fetching it again.") + self.index = self.nd.get_index(name=index_name) + else: + log.error(f"Failed to create Endee index: {create_error}") + raise + except Exception as e: + log.error(f"Error accessing or creating Endee index '{index_name}': {e}") + raise + + + + def _create_index(self, dim): + try: + resp = self.nd.create_index( + name=self.collection_name, + dimension=dim, + space_type=self.space_type, + # use_fp16=self.use_fp16, + precision=self.precision, + version=self.version, + M=self.M, + ef_con=self.ef_con + ) + log.info(f"Created new Endee index: {resp}") + except Exception as e: + log.error(f"Failed to create Endee index: {e}") + raise + + def _drop_index(self, collection_name): + try: + res = self.nd.delete_index(collection_name) + log.info(res) + except Exception as e: + log.error(f"Failed to drop Endee index: {e}") + raise + + @classmethod + def config_cls(cls) -> type[EndeeConfig]: + return EndeeConfig + + @classmethod + def case_config_cls(cls, index_type: IndexType | None = None) -> type[DBCaseConfig]: + return EmptyDBCaseConfig + + @contextmanager + def init(self): + try: + # log.info(f"Token: {self.token}") + nd = endee.Endee(token=self.token) + # Uncomment below to use base_url + # nd.base_url = "https://10.160.0.8:8080" + # self.nd = nd + self.index = nd.get_index(name=self.collection_name) + yield + except Exception as e: + if hasattr(e, 'response') and e.response is not None: + log.error(f"HTTP Status: {e.response.status_code}, Body: {e.response.text}") + log.error(f"Error initializing Endee client: {e}") + # log.error("Error initializing Endee client", exc_info=True) + raise + finally: + pass + + def optimize(self, data_size: int | None = None): + pass + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs, + ) -> (int, Exception): # type: ignore + assert len(embeddings) == len(metadata) + insert_count = 0 + try: + batch_vectors = [ + { + "id": str(metadata[i]), + "vector": embeddings[i], + "meta": {"id": str(metadata[i])} + } + for i in range(len(embeddings)) + ] + self.index.upsert(batch_vectors) + insert_count = len(batch_vectors) + + except Exception as e: + return (insert_count, e) + + return (len(embeddings), None) + + def search_embedding( + self, + query: list[float], + k: int = 30, + filters: dict | None = None, + **kwargs, + ) -> list[int]: + try: + filter_expr = None + if filters and "id" in filters: + filter_expr = {"id": filters["id"]} + + results = self.index.query( + vector=query, + top_k=k, + filter=filter_expr, + ef=self.ef_search, + include_vectors=False + ) + + return [int(result["id"]) for result in results] + + except Exception as e: + log.warning(f"Error querying Endee index: {e}") + raise + + def describe_index(self) -> dict: + """Get information about the current index.""" + try: + all_indices = self.nd.list_indexes().get("indices", []) + + for idx in all_indices: + if idx.get("name") == self.collection_name: + return idx + + return {} + except Exception as e: + log.warning(f"Error describing Endee index: {e}") + return {} diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index 300553a4e..eeb2e988b 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -17,6 +17,42 @@ from .. import utils from ..clients import api +import os +import json + +# CHECKPOINT_PATH = "/mnt/data/VectorDBBench/insert_checkpoint.json" + +# Get the path of the current script +CURRENT_FILE = os.path.abspath(__file__) + +# Go up 3 levels to reach the root benchmarking folder +ROOT_BENCHMARK_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(CURRENT_FILE)))) + +CHECKPOINT_FILENAME = "insert_checkpoint.json" +CHECKPOINT_PATH = os.path.join(ROOT_BENCHMARK_DIR, CHECKPOINT_FILENAME) + +# Ensure the root folder exists (should already exist, but safe) +os.makedirs(ROOT_BENCHMARK_DIR, exist_ok=True) + +def save_checkpoint(count: int): + with open(CHECKPOINT_PATH, "w") as f: + json.dump({"last_index": count}, f) + +def load_checkpoint() -> int: + if os.path.exists(CHECKPOINT_PATH): + try: + with open(CHECKPOINT_PATH, "r") as f: + data = json.load(f) + return data.get("last_index", 0) + except Exception as e: + log.warning(f"Failed to read checkpoint file: {e}") + return 0 + +def clear_checkpoint(): + if os.path.exists(CHECKPOINT_PATH): + os.remove(CHECKPOINT_PATH) + + NUM_PER_BATCH = config.NUM_PER_BATCH LOAD_MAX_TRY_COUNT = config.LOAD_MAX_TRY_COUNT @@ -52,20 +88,29 @@ def retry_insert(self, db: api.VectorDB, retry_idx: int = 0, **kwargs): def task(self) -> int: count = 0 + last_index = load_checkpoint() + log.info(f"Resuming insertion from checkpoint at index {last_index}") + with self.db.init(): log.info(f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}") start = time.perf_counter() + for data_df in self.dataset: - all_metadata = data_df[self.dataset.data.train_id_field].tolist() + batch_size = len(data_df) + if count + batch_size <= last_index: + count += batch_size + log.debug(f"Skipping batch of size {batch_size}, total skipped={count}") + continue + all_metadata = data_df[self.dataset.data.train_id_field].tolist() emb_np = np.stack(data_df[self.dataset.data.train_vector_field]) + if self.normalize: - log.debug("normalize the 100k train data") + log.debug("Normalizing train data") all_embeddings = (emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis]).tolist() else: all_embeddings = emb_np.tolist() del emb_np - log.debug(f"batch dataset size: {len(all_embeddings)}, {len(all_metadata)}") labels_data = None if self.filters.type == FilterOp.StrEqual: @@ -79,6 +124,7 @@ def task(self) -> int: metadata=all_metadata, labels_data=labels_data, ) + if error is not None: self.retry_insert( self.db, @@ -89,15 +135,19 @@ def task(self) -> int: assert insert_count == len(all_metadata) count += insert_count + save_checkpoint(count) # Save progress after each batch + if count % 100_000 == 0: log.info(f"({mp.current_process().name:16}) Loaded {count} embeddings into VectorDB") log.info( - f"({mp.current_process().name:16}) Finish loading all dataset into VectorDB, " + f"({mp.current_process().name:16}) Finished loading all dataset into VectorDB, " f"dur={time.perf_counter() - start}" ) + clear_checkpoint() # Clean checkpoint file after successful completion return count + def endless_insert_data(self, all_embeddings: list, all_metadata: list, left_id: int = 0) -> int: with self.db.init(): # unique id for endlessness insertion @@ -209,7 +259,80 @@ def run(self) -> int: return count +# ============================================================================= +# PARALLEL SEARCH WORKER FUNCTION +# ============================================================================= +def _parallel_search_worker(args: tuple) -> list[tuple[float, int, list[int]]]: + """ + Worker function executed in separate process for parallel search. + + Each worker: + 1. Receives a batch of queries (embeddings) with their indices + 2. Initializes its own DB connection + 3. Executes searches for all queries in the batch + 4. Returns results with latencies and original indices + + Args: + args: Tuple of (db_instance, batch_queries, batch_indices, k, filters) + + Returns: + List of tuples: [(latency, original_query_idx, search_results), ...] + + This function runs in a subprocess, so it needs to: + - Initialize DB connection independently + - Handle its own retry logic + - Return all results for aggregation in main process + """ + db_instance, batch_queries, batch_indices, k, filters = args + + results_batch = [] + + # Each worker initializes its own DB connection + with db_instance.init(): + # Prepare filters if needed (e.g., for filtered search scenarios) + db_instance.prepare_filter(filters) + + # Process each query in this worker's batch + for idx, emb in zip(batch_indices, batch_queries): + retry_count = 0 + max_retries = config.MAX_SEARCH_RETRY + + # Retry logic for each individual query + while retry_count <= max_retries: + try: + # Time the search operation + s = time.perf_counter() + results = db_instance.search_embedding(emb, k) + latency = time.perf_counter() - s + + # Store result with original index for proper ordering later + results_batch.append((latency, idx, results)) + break # Success, move to next query + + except Exception as e: + retry_count += 1 + if retry_count > max_retries: + log.error(f"Query {idx} failed after {max_retries} retries: {e}") + raise + log.warning(f"Query {idx} retry {retry_count}/{max_retries}: {e}") + time.sleep(0.1 * retry_count) # Exponential backoff + + return results_batch + + class SerialSearchRunner: + """ + Search runner that executes queries to measure recall and latency. + + PERFORMANCE ENHANCEMENT: + This class now uses parallel processing under the hood while maintaining + the same interface. Queries are distributed across multiple worker processes + for significant speedup (4-8x faster depending on CPU cores). + + The parallelization is transparent to callers - all existing code continues + to work without modification. + """ + def __init__( self, db: api.VectorDB, @@ -222,13 +345,25 @@ def __init__( self.k = k self.filters = filters + # Convert test data to list format if needed if isinstance(test_data[0], np.ndarray): self.test_data = [query.tolist() for query in test_data] else: self.test_data = test_data self.ground_truth = ground_truth + + # PARALLEL PROCESSING CONFIG: + # Auto-detect number of CPU cores, reserve 1 for main process + # This allows efficient parallel query execution + self.num_workers = max(1, mp.cpu_count() - 1) + log.info(f"SerialSearchRunner initialized with {self.num_workers} parallel workers") def _get_db_search_res(self, emb: list[float], retry_idx: int = 0) -> list[int]: + """ + LEGACY METHOD - Kept for backward compatibility. + This method is no longer used in the main search path but retained + in case any external code calls it directly. + """ try: results = self.db.search_embedding(emb, self.k) except Exception as e: @@ -241,47 +376,141 @@ def _get_db_search_res(self, emb: list[float], retry_idx: int = 0) -> list[int]: return results + def _create_query_batches(self) -> list[tuple[list[int], list]]: + """ + PARALLEL PROCESSING HELPER: + Splits test queries into batches for distribution to worker processes. + + Each batch contains: + - Original indices (to maintain ordering after parallel execution) + - Query embeddings for this batch + + Batch size is calculated as: total_queries / num_workers + This ensures even distribution of work across all available workers. + + Returns: + List of (indices, queries) tuples, one per worker + """ + total_queries = len(self.test_data) + batch_size = max(1, math.ceil(total_queries / self.num_workers)) + batches = [] + + for i in range(0, total_queries, batch_size): + end_idx = min(i + batch_size, total_queries) + indices = list(range(i, end_idx)) + queries = self.test_data[i:end_idx] + batches.append((indices, queries)) + + log.debug(f"Created {len(batches)} query batches, batch_size={batch_size}") + return batches + def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, float, float]: + """ + MODIFIED: Now uses parallel processing for significant speedup. + + Main changes from original: + 1. Splits queries into batches for parallel workers + 2. Spawns worker processes to execute searches concurrently + 3. Collects and re-orders results by original index + 4. Calculates metrics (recall, NDCG, latencies) from parallel results + + The interface and return values remain identical to the original version, + ensuring backward compatibility with existing code. + + Original single-threaded approach took ~164s for 1000 queries. + Parallel approach typically achieves 4-8x speedup depending on CPU cores. + + Args: + args: Tuple of (test_data, ground_truth) - kept for interface compatibility + + Returns: + Tuple of (avg_recall, avg_ndcg, p99_latency, p95_latency) + """ log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency") - with self.db.init(): - self.db.prepare_filter(self.filters) - test_data, ground_truth = args - ideal_dcg = get_ideal_dcg(self.k) - - log.debug(f"test dataset size: {len(test_data)}") - log.debug(f"ground truth size: {len(ground_truth)}") - - latencies, recalls, ndcgs = [], [], [] - for idx, emb in enumerate(test_data): - s = time.perf_counter() + + test_data, ground_truth = args + + # Validate input + if not test_data: + raise RuntimeError("empty test_data") + + log.debug(f"test dataset size: {len(test_data)}") + log.debug(f"ground truth size: {len(ground_truth) if ground_truth else 0}") + + # PARALLEL EXECUTION SECTION: + # Create batches for parallel processing + batches = self._create_query_batches() + + # Prepare arguments for each worker process + # Each worker gets: (db_instance, queries, indices, k, filters) + worker_args = [ + (self.db, queries, indices, self.k, self.filters) + for indices, queries in batches + ] + + log.info(f"Launching {len(worker_args)} parallel workers for search") + + # Execute searches in parallel using ProcessPoolExecutor + # Using 'spawn' context ensures clean process initialization + all_results = [] + with concurrent.futures.ProcessPoolExecutor( + max_workers=self.num_workers, + mp_context=mp.get_context("spawn") + ) as executor: + # Submit all batches to worker pool + futures = [ + executor.submit(_parallel_search_worker, args) + for args in worker_args + ] + + # Collect results as workers complete + # as_completed() allows processing results immediately when ready + for future in concurrent.futures.as_completed(futures): try: - results = self._get_db_search_res(emb) + batch_results = future.result() + all_results.extend(batch_results) except Exception as e: - log.warning(f"VectorDB search_embedding error: {e}") - raise e from None - - latencies.append(time.perf_counter() - s) - - if ground_truth is not None: - gt = ground_truth[idx] - recalls.append(calc_recall(self.k, gt[: self.k], results)) - ndcgs.append(calc_ndcg(gt[: self.k], results, ideal_dcg)) - else: - recalls.append(0) - ndcgs.append(0) - - if len(latencies) % 100 == 0: - log.debug( - f"({mp.current_process().name:14}) search_count={len(latencies):3}, " - f"latest_latency={latencies[-1]}, latest recall={recalls[-1]}" - ) - + log.error(f"Worker process failed: {e}") + raise + + # RESULTS PROCESSING SECTION: + # Sort results by original query index to maintain correct ordering + # This is crucial because workers complete in arbitrary order + all_results.sort(key=lambda x: x[1]) # Sort by index (2nd element) + + log.info(f"Collected {len(all_results)} search results from parallel workers") + + # Calculate performance metrics + latencies, recalls, ndcgs = [], [], [] + ideal_dcg = get_ideal_dcg(self.k) + + for latency, idx, results in all_results: + latencies.append(latency) + + # Calculate recall and NDCG if ground truth is available + if ground_truth is not None: + gt = ground_truth[idx] + recalls.append(calc_recall(self.k, gt[:self.k], results)) + ndcgs.append(calc_ndcg(gt[:self.k], results, ideal_dcg)) + else: + recalls.append(0) + ndcgs.append(0) + + # Progress logging every 100 queries + if len(latencies) % 100 == 0: + log.debug( + f"({mp.current_process().name:14}) search_count={len(latencies):3}, " + f"latest_latency={latencies[-1]}, latest recall={recalls[-1]}" + ) + + # Calculate aggregate statistics avg_latency = round(np.mean(latencies), 4) avg_recall = round(np.mean(recalls), 4) avg_ndcg = round(np.mean(ndcgs), 4) cost = round(np.sum(latencies), 4) p99 = round(np.percentile(latencies, 99), 4) p95 = round(np.percentile(latencies, 95), 4) + log.info( f"{mp.current_process().name:14} search entire test_data: " f"cost={cost}s, " @@ -292,15 +521,28 @@ def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, floa f"p99={p99}, " f"p95={p95}" ) + return (avg_recall, avg_ndcg, p99, p95) def _run_in_subprocess(self) -> tuple[float, float, float, float]: + """ + UNCHANGED: Wrapper to execute search in subprocess. + + This method is kept for interface compatibility. + The actual parallelization happens inside the search() method. + """ with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: future = executor.submit(self.search, (self.test_data, self.ground_truth)) return future.result() @utils.time_it def run(self) -> tuple[float, float, float, float]: + """ + UNCHANGED: Main entry point for search execution. + + Interface remains identical - callers don't need any changes. + Parallel processing is handled transparently. + """ log.info(f"{mp.current_process().name:14} start serial search") if self.test_data is None: msg = "empty test_data" @@ -311,9 +553,11 @@ def run(self) -> tuple[float, float, float, float]: @utils.time_it def run_with_cost(self) -> tuple[tuple[float, float, float, float], float]: """ - Search all test data in serial. + UNCHANGED: Search with cost tracking. + Returns: - tuple[tuple[float, float, float, float], float]: (avg_recall, avg_ndcg, p99_latency, p95_latency), cost + tuple[tuple[float, float, float, float], float]: + (avg_recall, avg_ndcg, p99_latency, p95_latency), cost """ log.info(f"{mp.current_process().name:14} start serial search") if self.test_data is None: @@ -321,3 +565,733 @@ def run_with_cost(self) -> tuple[tuple[float, float, float, float], float]: raise RuntimeError(msg) return self._run_in_subprocess() + + + + + + + + +# ======================================================================= + +# MODIFIED TO HAVE CHECKPOINT FOR RESUME LOGIC + +# ======================================================================= + + + + +# import concurrent +# import logging +# import math +# import multiprocessing as mp +# import time +# import traceback + +# import numpy as np +# import psutil + +# from vectordb_bench.backend.dataset import DatasetManager +# from vectordb_bench.backend.filter import Filter, FilterOp, non_filter + +# from ... import config +# from ...metric import calc_ndcg, calc_recall, get_ideal_dcg +# from ...models import LoadTimeoutError, PerformanceTimeoutError +# from .. import utils +# from ..clients import api + +# import os +# import json + + +# # Get the path of the current script +# CURRENT_FILE = os.path.abspath(__file__) + +# # Go up 3 levels to reach the root benchmarking folder +# ROOT_BENCHMARK_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(CURRENT_FILE)))) + +# CHECKPOINT_FILENAME = "insert_checkpoint.json" +# CHECKPOINT_PATH = os.path.join(ROOT_BENCHMARK_DIR, CHECKPOINT_FILENAME) + +# # Ensure the root folder exists (should already exist, but safe) +# os.makedirs(ROOT_BENCHMARK_DIR, exist_ok=True) + +# def save_checkpoint(count: int): +# with open(CHECKPOINT_PATH, "w") as f: +# json.dump({"last_index": count}, f) + +# def load_checkpoint() -> int: +# if os.path.exists(CHECKPOINT_PATH): +# try: +# with open(CHECKPOINT_PATH, "r") as f: +# data = json.load(f) +# return data.get("last_index", 0) +# except Exception as e: +# log.warning(f"Failed to read checkpoint file: {e}") +# return 0 + +# def clear_checkpoint(): +# if os.path.exists(CHECKPOINT_PATH): +# os.remove(CHECKPOINT_PATH) + + +# NUM_PER_BATCH = config.NUM_PER_BATCH +# LOAD_MAX_TRY_COUNT = config.LOAD_MAX_TRY_COUNT + +# log = logging.getLogger(__name__) + + +# class SerialInsertRunner: +# def __init__( +# self, +# db: api.VectorDB, +# dataset: DatasetManager, +# normalize: bool, +# filters: Filter = non_filter, +# timeout: float | None = None, +# ): +# self.timeout = timeout if isinstance(timeout, int | float) else None +# self.dataset = dataset +# self.db = db +# self.normalize = normalize +# self.filters = filters + +# def retry_insert(self, db: api.VectorDB, retry_idx: int = 0, **kwargs): +# _, error = db.insert_embeddings(**kwargs) +# if error is not None: +# log.warning(f"Insert Failed, try_idx={retry_idx}, Exception: {error}") +# retry_idx += 1 +# if retry_idx <= config.MAX_INSERT_RETRY: +# time.sleep(retry_idx) +# self.retry_insert(db, retry_idx=retry_idx, **kwargs) +# else: +# msg = f"Insert failed and retried more than {config.MAX_INSERT_RETRY} times" +# raise RuntimeError(msg) from None + +# def task(self) -> int: +# count = 0 +# last_index = load_checkpoint() +# log.info(f"Resuming insertion from checkpoint at index {last_index}") + +# with self.db.init(): +# log.info(f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}") +# start = time.perf_counter() + +# for data_df in self.dataset: +# batch_size = len(data_df) +# if count + batch_size <= last_index: +# count += batch_size +# log.debug(f"Skipping batch of size {batch_size}, total skipped={count}") +# continue + +# all_metadata = data_df[self.dataset.data.train_id_field].tolist() +# emb_np = np.stack(data_df[self.dataset.data.train_vector_field]) + +# if self.normalize: +# log.debug("Normalizing train data") +# all_embeddings = (emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis]).tolist() +# else: +# all_embeddings = emb_np.tolist() +# del emb_np + +# labels_data = None +# if self.filters.type == FilterOp.StrEqual: +# if self.dataset.data.scalar_labels_file_separated: +# labels_data = self.dataset.scalar_labels[self.filters.label_field][all_metadata].to_list() +# else: +# labels_data = data_df[self.filters.label_field].tolist() + +# insert_count, error = self.db.insert_embeddings( +# embeddings=all_embeddings, +# metadata=all_metadata, +# labels_data=labels_data, +# ) + +# if error is not None: +# self.retry_insert( +# self.db, +# embeddings=all_embeddings, +# metadata=all_metadata, +# labels_data=labels_data, +# ) + +# assert insert_count == len(all_metadata) +# count += insert_count +# save_checkpoint(count) # Save progress after each batch + +# if count % 100_000 == 0: +# log.info(f"({mp.current_process().name:16}) Loaded {count} embeddings into VectorDB") + +# log.info( +# f"({mp.current_process().name:16}) Finished loading all dataset into VectorDB, " +# f"dur={time.perf_counter() - start}" +# ) +# clear_checkpoint() # Clean checkpoint file after successful completion +# return count + + +# def endless_insert_data(self, all_embeddings: list, all_metadata: list, left_id: int = 0) -> int: +# with self.db.init(): +# # unique id for endlessness insertion +# all_metadata = [i + left_id for i in all_metadata] + +# num_batches = math.ceil(len(all_embeddings) / NUM_PER_BATCH) +# log.info( +# f"({mp.current_process().name:16}) Start inserting {len(all_embeddings)} " +# f"embeddings in batch {NUM_PER_BATCH}" +# ) +# count = 0 +# for batch_id in range(num_batches): +# retry_count = 0 +# already_insert_count = 0 +# metadata = all_metadata[batch_id * NUM_PER_BATCH : (batch_id + 1) * NUM_PER_BATCH] +# embeddings = all_embeddings[batch_id * NUM_PER_BATCH : (batch_id + 1) * NUM_PER_BATCH] + +# log.debug( +# f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_batches}], " +# f"Start inserting {len(metadata)} embeddings" +# ) +# while retry_count < LOAD_MAX_TRY_COUNT: +# insert_count, error = self.db.insert_embeddings( +# embeddings=embeddings[already_insert_count:], +# metadata=metadata[already_insert_count:], +# ) +# already_insert_count += insert_count +# if error is not None: +# retry_count += 1 +# time.sleep(10) + +# log.info(f"Failed to insert data, try {retry_count} time") +# if retry_count >= LOAD_MAX_TRY_COUNT: +# raise error +# else: +# break +# log.debug( +# f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_batches}], " +# f"Finish inserting {len(metadata)} embeddings" +# ) + +# assert already_insert_count == len(metadata) +# count += already_insert_count +# log.info( +# f"({mp.current_process().name:16}) Finish inserting {len(all_embeddings)} embeddings in " +# f"batch {NUM_PER_BATCH}" +# ) +# return count + +# @utils.time_it +# def _insert_all_batches(self) -> int: +# """Performance case only""" +# with concurrent.futures.ProcessPoolExecutor( +# mp_context=mp.get_context("spawn"), +# max_workers=1, +# ) as executor: +# future = executor.submit(self.task) +# try: +# count = future.result(timeout=self.timeout) +# except TimeoutError as e: +# msg = f"VectorDB load dataset timeout in {self.timeout}" +# log.warning(msg) +# for pid, _ in executor._processes.items(): +# psutil.Process(pid).kill() +# raise PerformanceTimeoutError(msg) from e +# except Exception as e: +# log.warning(f"VectorDB load dataset error: {e}") +# raise e from e +# else: +# return count + +# def run_endlessness(self) -> int: +# """run forever util DB raises exception or crash""" +# # datasets for load tests are quite small, can fit into memory +# # only 1 file +# data_df = next(iter(self.dataset)) +# all_embeddings, all_metadata = ( +# np.stack(data_df[self.dataset.data.train_vector_field]).tolist(), +# data_df[self.dataset.data.train_id_field].tolist(), +# ) + +# start_time = time.perf_counter() +# max_load_count, times = 0, 0 +# try: +# while time.perf_counter() - start_time < self.timeout: +# count = self.endless_insert_data( +# all_embeddings, +# all_metadata, +# left_id=max_load_count, +# ) +# max_load_count += count +# times += 1 +# log.info( +# f"Loaded {times} entire dataset, current max load counts={utils.numerize(max_load_count)}, " +# f"{max_load_count}" +# ) +# except Exception as e: +# log.info( +# f"Capacity case load reach limit, insertion counts={utils.numerize(max_load_count)}, " +# f"{max_load_count}, err={e}" +# ) +# traceback.print_exc() +# return max_load_count +# else: +# raise LoadTimeoutError(self.timeout) + +# def run(self) -> int: +# count, _ = self._insert_all_batches() +# return count + + +# class SerialSearchRunner: +# def __init__( +# self, +# db: api.VectorDB, +# test_data: list[list[float]], +# ground_truth: list[list[int]], +# k: int = 100, +# filters: Filter = non_filter, +# ): +# self.db = db +# self.k = k +# self.filters = filters + +# if isinstance(test_data[0], np.ndarray): +# self.test_data = [query.tolist() for query in test_data] +# else: +# self.test_data = test_data +# self.ground_truth = ground_truth + +# def _get_db_search_res(self, emb: list[float], retry_idx: int = 0) -> list[int]: +# try: +# results = self.db.search_embedding(emb, self.k) +# except Exception as e: +# log.warning(f"Serial search failed, retry_idx={retry_idx}, Exception: {e}") +# if retry_idx < config.MAX_SEARCH_RETRY: +# return self._get_db_search_res(emb=emb, retry_idx=retry_idx + 1) + +# msg = f"Serial search failed and retried more than {config.MAX_SEARCH_RETRY} times" +# raise RuntimeError(msg) from e + +# return results + +# def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, float, float]: +# log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency") +# with self.db.init(): +# self.db.prepare_filter(self.filters) +# test_data, ground_truth = args +# ideal_dcg = get_ideal_dcg(self.k) + +# log.debug(f"test dataset size: {len(test_data)}") +# log.debug(f"ground truth size: {len(ground_truth)}") + +# latencies, recalls, ndcgs = [], [], [] +# for idx, emb in enumerate(test_data): +# s = time.perf_counter() +# try: +# results = self._get_db_search_res(emb) +# except Exception as e: +# log.warning(f"VectorDB search_embedding error: {e}") +# raise e from None + +# latencies.append(time.perf_counter() - s) + +# if ground_truth is not None: +# gt = ground_truth[idx] +# recalls.append(calc_recall(self.k, gt[: self.k], results)) +# ndcgs.append(calc_ndcg(gt[: self.k], results, ideal_dcg)) +# else: +# recalls.append(0) +# ndcgs.append(0) + +# if len(latencies) % 100 == 0: +# log.debug( +# f"({mp.current_process().name:14}) search_count={len(latencies):3}, " +# f"latest_latency={latencies[-1]}, latest recall={recalls[-1]}" +# ) + +# avg_latency = round(np.mean(latencies), 4) +# avg_recall = round(np.mean(recalls), 4) +# avg_ndcg = round(np.mean(ndcgs), 4) +# cost = round(np.sum(latencies), 4) +# p99 = round(np.percentile(latencies, 99), 4) +# p95 = round(np.percentile(latencies, 95), 4) +# log.info( +# f"{mp.current_process().name:14} search entire test_data: " +# f"cost={cost}s, " +# f"queries={len(latencies)}, " +# f"avg_recall={avg_recall}, " +# f"avg_ndcg={avg_ndcg}, " +# f"avg_latency={avg_latency}, " +# f"p99={p99}, " +# f"p95={p95}" +# ) +# return (avg_recall, avg_ndcg, p99, p95) + +# def _run_in_subprocess(self) -> tuple[float, float, float, float]: +# with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: +# future = executor.submit(self.search, (self.test_data, self.ground_truth)) +# return future.result() + +# @utils.time_it +# def run(self) -> tuple[float, float, float, float]: +# log.info(f"{mp.current_process().name:14} start serial search") +# if self.test_data is None: +# msg = "empty test_data" +# raise RuntimeError(msg) + +# return self._run_in_subprocess() + +# @utils.time_it +# def run_with_cost(self) -> tuple[tuple[float, float, float, float], float]: +# """ +# Search all test data in serial. +# Returns: +# tuple[tuple[float, float, float, float], float]: (avg_recall, avg_ndcg, p99_latency, p95_latency), cost +# """ +# log.info(f"{mp.current_process().name:14} start serial search") +# if self.test_data is None: +# msg = "empty test_data" +# raise RuntimeError(msg) + +# return self._run_in_subprocess() + + + + + + +# ======================================================================= + +# ORIGINAL SERIAL RUNNER CODE + +# ======================================================================= + + + + + + + +# import concurrent +# import logging +# import math +# import multiprocessing as mp +# import time +# import traceback + +# import numpy as np +# import psutil + +# from vectordb_bench.backend.dataset import DatasetManager +# from vectordb_bench.backend.filter import Filter, FilterOp, non_filter + +# from ... import config +# from ...metric import calc_ndcg, calc_recall, get_ideal_dcg +# from ...models import LoadTimeoutError, PerformanceTimeoutError +# from .. import utils +# from ..clients import api + +# NUM_PER_BATCH = config.NUM_PER_BATCH +# LOAD_MAX_TRY_COUNT = config.LOAD_MAX_TRY_COUNT + +# log = logging.getLogger(__name__) + + +# class SerialInsertRunner: +# def __init__( +# self, +# db: api.VectorDB, +# dataset: DatasetManager, +# normalize: bool, +# filters: Filter = non_filter, +# timeout: float | None = None, +# ): +# self.timeout = timeout if isinstance(timeout, int | float) else None +# self.dataset = dataset +# self.db = db +# self.normalize = normalize +# self.filters = filters + +# def retry_insert(self, db: api.VectorDB, retry_idx: int = 0, **kwargs): +# _, error = db.insert_embeddings(**kwargs) +# if error is not None: +# log.warning(f"Insert Failed, try_idx={retry_idx}, Exception: {error}") +# retry_idx += 1 +# if retry_idx <= config.MAX_INSERT_RETRY: +# time.sleep(retry_idx) +# self.retry_insert(db, retry_idx=retry_idx, **kwargs) +# else: +# msg = f"Insert failed and retried more than {config.MAX_INSERT_RETRY} times" +# raise RuntimeError(msg) from None + +# def task(self) -> int: +# count = 0 +# with self.db.init(): +# log.info(f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}") +# start = time.perf_counter() +# for data_df in self.dataset: +# all_metadata = data_df[self.dataset.data.train_id_field].tolist() + +# emb_np = np.stack(data_df[self.dataset.data.train_vector_field]) +# if self.normalize: +# log.debug("normalize the 100k train data") +# all_embeddings = (emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis]).tolist() +# else: +# all_embeddings = emb_np.tolist() +# del emb_np +# log.debug(f"batch dataset size: {len(all_embeddings)}, {len(all_metadata)}") + +# labels_data = None +# if self.filters.type == FilterOp.StrEqual: +# if self.dataset.data.scalar_labels_file_separated: +# labels_data = self.dataset.scalar_labels[self.filters.label_field][all_metadata].to_list() +# else: +# labels_data = data_df[self.filters.label_field].tolist() + +# insert_count, error = self.db.insert_embeddings( +# embeddings=all_embeddings, +# metadata=all_metadata, +# labels_data=labels_data, +# ) +# if error is not None: +# self.retry_insert( +# self.db, +# embeddings=all_embeddings, +# metadata=all_metadata, +# labels_data=labels_data, +# ) + +# assert insert_count == len(all_metadata) +# count += insert_count +# if count % 100_000 == 0: +# log.info(f"({mp.current_process().name:16}) Loaded {count} embeddings into VectorDB") + +# log.info( +# f"({mp.current_process().name:16}) Finish loading all dataset into VectorDB, " +# f"dur={time.perf_counter() - start}" +# ) +# return count + +# def endless_insert_data(self, all_embeddings: list, all_metadata: list, left_id: int = 0) -> int: +# with self.db.init(): +# # unique id for endlessness insertion +# all_metadata = [i + left_id for i in all_metadata] + +# num_batches = math.ceil(len(all_embeddings) / NUM_PER_BATCH) +# log.info( +# f"({mp.current_process().name:16}) Start inserting {len(all_embeddings)} " +# f"embeddings in batch {NUM_PER_BATCH}" +# ) +# count = 0 +# for batch_id in range(num_batches): +# retry_count = 0 +# already_insert_count = 0 +# metadata = all_metadata[batch_id * NUM_PER_BATCH : (batch_id + 1) * NUM_PER_BATCH] +# embeddings = all_embeddings[batch_id * NUM_PER_BATCH : (batch_id + 1) * NUM_PER_BATCH] + +# log.debug( +# f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_batches}], " +# f"Start inserting {len(metadata)} embeddings" +# ) +# while retry_count < LOAD_MAX_TRY_COUNT: +# insert_count, error = self.db.insert_embeddings( +# embeddings=embeddings[already_insert_count:], +# metadata=metadata[already_insert_count:], +# ) +# already_insert_count += insert_count +# if error is not None: +# retry_count += 1 +# time.sleep(10) + +# log.info(f"Failed to insert data, try {retry_count} time") +# if retry_count >= LOAD_MAX_TRY_COUNT: +# raise error +# else: +# break +# log.debug( +# f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_batches}], " +# f"Finish inserting {len(metadata)} embeddings" +# ) + +# assert already_insert_count == len(metadata) +# count += already_insert_count +# log.info( +# f"({mp.current_process().name:16}) Finish inserting {len(all_embeddings)} embeddings in " +# f"batch {NUM_PER_BATCH}" +# ) +# return count + +# @utils.time_it +# def _insert_all_batches(self) -> int: +# """Performance case only""" +# with concurrent.futures.ProcessPoolExecutor( +# mp_context=mp.get_context("spawn"), +# max_workers=1, +# ) as executor: +# future = executor.submit(self.task) +# try: +# count = future.result(timeout=self.timeout) +# except TimeoutError as e: +# msg = f"VectorDB load dataset timeout in {self.timeout}" +# log.warning(msg) +# for pid, _ in executor._processes.items(): +# psutil.Process(pid).kill() +# raise PerformanceTimeoutError(msg) from e +# except Exception as e: +# log.warning(f"VectorDB load dataset error: {e}") +# raise e from e +# else: +# return count + +# def run_endlessness(self) -> int: +# """run forever util DB raises exception or crash""" +# # datasets for load tests are quite small, can fit into memory +# # only 1 file +# data_df = next(iter(self.dataset)) +# all_embeddings, all_metadata = ( +# np.stack(data_df[self.dataset.data.train_vector_field]).tolist(), +# data_df[self.dataset.data.train_id_field].tolist(), +# ) + +# start_time = time.perf_counter() +# max_load_count, times = 0, 0 +# try: +# while time.perf_counter() - start_time < self.timeout: +# count = self.endless_insert_data( +# all_embeddings, +# all_metadata, +# left_id=max_load_count, +# ) +# max_load_count += count +# times += 1 +# log.info( +# f"Loaded {times} entire dataset, current max load counts={utils.numerize(max_load_count)}, " +# f"{max_load_count}" +# ) +# except Exception as e: +# log.info( +# f"Capacity case load reach limit, insertion counts={utils.numerize(max_load_count)}, " +# f"{max_load_count}, err={e}" +# ) +# traceback.print_exc() +# return max_load_count +# else: +# raise LoadTimeoutError(self.timeout) + +# def run(self) -> int: +# count, _ = self._insert_all_batches() +# return count + + +# class SerialSearchRunner: +# def __init__( +# self, +# db: api.VectorDB, +# test_data: list[list[float]], +# ground_truth: list[list[int]], +# k: int = 100, +# filters: Filter = non_filter, +# ): +# self.db = db +# self.k = k +# self.filters = filters + +# if isinstance(test_data[0], np.ndarray): +# self.test_data = [query.tolist() for query in test_data] +# else: +# self.test_data = test_data +# self.ground_truth = ground_truth + +# def _get_db_search_res(self, emb: list[float], retry_idx: int = 0) -> list[int]: +# try: +# results = self.db.search_embedding(emb, self.k) +# except Exception as e: +# log.warning(f"Serial search failed, retry_idx={retry_idx}, Exception: {e}") +# if retry_idx < config.MAX_SEARCH_RETRY: +# return self._get_db_search_res(emb=emb, retry_idx=retry_idx + 1) + +# msg = f"Serial search failed and retried more than {config.MAX_SEARCH_RETRY} times" +# raise RuntimeError(msg) from e + +# return results + +# def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, float, float]: +# log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency") +# with self.db.init(): +# self.db.prepare_filter(self.filters) +# test_data, ground_truth = args +# ideal_dcg = get_ideal_dcg(self.k) + +# log.debug(f"test dataset size: {len(test_data)}") +# log.debug(f"ground truth size: {len(ground_truth)}") + +# latencies, recalls, ndcgs = [], [], [] +# for idx, emb in enumerate(test_data): +# s = time.perf_counter() +# try: +# results = self._get_db_search_res(emb) +# except Exception as e: +# log.warning(f"VectorDB search_embedding error: {e}") +# raise e from None + +# latencies.append(time.perf_counter() - s) + +# if ground_truth is not None: +# gt = ground_truth[idx] +# recalls.append(calc_recall(self.k, gt[: self.k], results)) +# ndcgs.append(calc_ndcg(gt[: self.k], results, ideal_dcg)) +# else: +# recalls.append(0) +# ndcgs.append(0) + +# if len(latencies) % 100 == 0: +# log.debug( +# f"({mp.current_process().name:14}) search_count={len(latencies):3}, " +# f"latest_latency={latencies[-1]}, latest recall={recalls[-1]}" +# ) + +# avg_latency = round(np.mean(latencies), 4) +# avg_recall = round(np.mean(recalls), 4) +# avg_ndcg = round(np.mean(ndcgs), 4) +# cost = round(np.sum(latencies), 4) +# p99 = round(np.percentile(latencies, 99), 4) +# p95 = round(np.percentile(latencies, 95), 4) +# log.info( +# f"{mp.current_process().name:14} search entire test_data: " +# f"cost={cost}s, " +# f"queries={len(latencies)}, " +# f"avg_recall={avg_recall}, " +# f"avg_ndcg={avg_ndcg}, " +# f"avg_latency={avg_latency}, " +# f"p99={p99}, " +# f"p95={p95}" +# ) +# return (avg_recall, avg_ndcg, p99, p95) + +# def _run_in_subprocess(self) -> tuple[float, float, float, float]: +# with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: +# future = executor.submit(self.search, (self.test_data, self.ground_truth)) +# return future.result() + +# @utils.time_it +# def run(self) -> tuple[float, float, float, float]: +# log.info(f"{mp.current_process().name:14} start serial search") +# if self.test_data is None: +# msg = "empty test_data" +# raise RuntimeError(msg) + +# return self._run_in_subprocess() + +# @utils.time_it +# def run_with_cost(self) -> tuple[tuple[float, float, float, float], float]: +# """ +# Search all test data in serial. +# Returns: +# tuple[tuple[float, float, float, float], float]: (avg_recall, avg_ndcg, p99_latency, p95_latency), cost +# """ +# log.info(f"{mp.current_process().name:14} start serial search") +# if self.test_data is None: +# msg = "empty test_data" +# raise RuntimeError(msg) + +# return self._run_in_subprocess() diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index a53f0d314..6a3765ff1 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -10,6 +10,7 @@ ElasticCloudHNSWInt4, ElasticCloudHNSWInt8, ) +from ..backend.clients.endee.cli import Endee from ..backend.clients.hologres.cli import HologresHGraph from ..backend.clients.lancedb.cli import LanceDB from ..backend.clients.mariadb.cli import MariaDBHNSW @@ -70,6 +71,7 @@ cli.add_command(AliSQLHNSW) cli.add_command(Doris) cli.add_command(TurboPuffer) +cli.add_command(Endee) if __name__ == "__main__": diff --git a/vectordb_bench/frontend/config/styles.py b/vectordb_bench/frontend/config/styles.py index bce4561fd..d14ca877e 100644 --- a/vectordb_bench/frontend/config/styles.py +++ b/vectordb_bench/frontend/config/styles.py @@ -71,6 +71,7 @@ def getPatternShape(i): DB.Doris: "https://doris.apache.org/images/logo.svg", DB.TurboPuffer: "https://turbopuffer.com/logo2.png", DB.CockroachDB: "https://raw.githubusercontent.com/cockroachdb/cockroach/master/docs/media/cockroach_db.png", + DB.Endee: "data:image/svg+xml,%3c?xml%20version=%271.0%27%20encoding=%27UTF-8%27?%3e%3csvg%20id=%27Layer_1%27%20xmlns=%27http://www.w3.org/2000/svg%27%20version=%271.1%27%20viewBox=%270%200%20600%20600%27%3e%3c!--%20Generator:%20Adobe%20Illustrator%2030.0.0,%20SVG%20Export%20Plug-In%20.%20SVG%20Version:%202.1.1%20Build%20123)%20--%3e%3cdefs%3e%3cstyle%3e%20.st0%20{%20fill:%20%233266a4;%20}%20%3c/style%3e%3c/defs%3e%3cpath%20class=%27st0%27%20d=%27M106.22,490.02H10.36l-.04-184.85c11.19-163.31,232.1-211.74,306.42-61.94,23.96,48.3,15.59,99.83,17,152.01h61.62c15.25,0,42.7-13.75,54.75-23.2,66.11-51.86,57.28-158.2-16.28-198.49-9.65-5.28-33.16-14.19-43.74-14.19h-154.31v-91.09c0-.87,2.55-1.86,3.63-1.63,104.05,4.23,201.15-21.64,284.48,55.84,119.18,110.8,69.12,325.47-91.33,362.6-28.65,6.63-85.47,7.76-115.02,4.8-19.71-1.97-43.29-16.57-55.97-31.45-37.98-44.56-20.77-98.07-24.7-151.16-5.18-69.99-100-85.31-125.9-20.47-1.16,2.92-4.76,13.88-4.76,16.3v186.91Z%27/%3e%3c/svg%3e", } # RedisCloud color: #0D6EFD From ae6c4937caa9f3b24f3f2409735bb53c61434ea4 Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Wed, 24 Dec 2025 17:56:40 +0530 Subject: [PATCH 02/22] Added base_url parameter and resume checkpoint with index name --- .allcommands | 70 ++ vectordb_bench/backend/clients/endee/cli.py | 4 + .../backend/clients/endee/config.py | 2 + vectordb_bench/backend/clients/endee/endee.py | 26 +- .../backend/runner/serial_runner.py | 939 +++++++++++++----- 5 files changed, 771 insertions(+), 270 deletions(-) create mode 100644 .allcommands diff --git a/.allcommands b/.allcommands new file mode 100644 index 000000000..355251929 --- /dev/null +++ b/.allcommands @@ -0,0 +1,70 @@ +# FOR CREATING A NEW INDEX AND START THE PROCESS +DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ + --token "as1prod:nb5l9VEBrgDEsvjut8Oawgcbf5jghoN5" \ + --region india-west-1 \ + --base-url "https://as1.endee.io/api/v1" \ + --index-name test_prod_10M_medium_3 \ + --task-label "20251224" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision medium \ + --version 1 \ + --case-type Performance768D10M \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + +PROD: "https://as1.endee.io/api/v1" +AWS: http://54.89.169.48:80/api/v1 + + DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ + --token "awsbench16:uVQMSo85De3k1lN5FiZI0xKcwXDlyBvu" \ + --region india-west-1 \ + --base-url "http://54.89.169.48:80/api/v1"\ + --index-name test_aws_10M_medium_1 \ + --task-label "20251223" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision medium \ + --version 1 \ + --case-type Performance768D10M \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + +# FOR USING THE EXISTING INDEX +DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ + --token "as1prod:nb5l9VEBrgDEsvjut8Oawgcbf5jghoN5" \ + --region india-west-1 \ + --base-url "https://as1.endee.io/api/v1" \ + --index-name test_prod_10M_medium_3 \ + --task-label "20251224" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision medium \ + --version 1 \ + --case-type Performance768D10M \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial diff --git a/vectordb_bench/backend/clients/endee/cli.py b/vectordb_bench/backend/clients/endee/cli.py index 952f1b682..e6096fc2e 100644 --- a/vectordb_bench/backend/clients/endee/cli.py +++ b/vectordb_bench/backend/clients/endee/cli.py @@ -26,6 +26,10 @@ class EndeeTypedDict(CommonTypedDict): str, click.option("--region", type=str, default=None, help="Endee region", show_default=True) ] + base_url: Annotated[ + str, + click.option("--base-url", type=str, default="http://127.0.0.1:8080/api/v1", help="API server URL", show_default=True) + ] space_type: Annotated[ str, click.option("--space-type", type=click.Choice(["cosine", "l2", "dot_product"]), default="cosine", help="Distance metric", show_default=True) diff --git a/vectordb_bench/backend/clients/endee/config.py b/vectordb_bench/backend/clients/endee/config.py index 476f9b5e0..aa4bf8c5d 100644 --- a/vectordb_bench/backend/clients/endee/config.py +++ b/vectordb_bench/backend/clients/endee/config.py @@ -6,6 +6,7 @@ class EndeeConfig(DBConfig): token: SecretStr region: Optional[str] = "india-west-1" + base_url: str = "http://127.0.0.1:8080/api/v1" # Default value space_type: str ="cosine" # use_fp16: bool = False precision: str = "medium" @@ -20,6 +21,7 @@ def to_dict(self) -> dict: return { "token": self.token.get_secret_value(), "region": self.region, + "base_url": self.base_url, "space_type": self.space_type, # "use_fp16": self.use_fp16, "precision": self.precision, diff --git a/vectordb_bench/backend/clients/endee/endee.py b/vectordb_bench/backend/clients/endee/endee.py index 5095acd13..a87efbf37 100644 --- a/vectordb_bench/backend/clients/endee/endee.py +++ b/vectordb_bench/backend/clients/endee/endee.py @@ -22,6 +22,7 @@ def __init__( self.token = db_config.get("token", "") self.region = db_config.get("region", "india-west-1") + self.base_url = db_config.get("base_url") # self.collection_name = db_config.get("collection_name", "") self.collection_name = (db_config.get("collection_name") or db_config.get("index_name")) @@ -39,9 +40,20 @@ def __init__( self.ef_search = db_config.get("ef_search") self.nd = endee.Endee(token=self.token) + # Dynamically set the URL + if self.base_url: + self.nd.set_base_url(self.base_url) + log.info(f"Targeting server: {self.base_url}") + # Using Base url - # self.nd.base_url = "https://10.160.0.8:8080" + # self.nd.base_url = "http://10.128.0.3:8080/api/v1" + # self.nd.set_base_url("http://10.128.0.3:8080/api/v1") + # self.nd.set_base_url("http://3.85.217.253:80/api/v1") + # self.nd.set_base_url("http://10.128.0.5:8080/api/v1") + # self.nd.set_base_url("http://54.89.169.48:80/api/v1") + + # BASE_URL="http://3.85.217.253:80/api/v1" # try: # indices = self.nd.list_indexes().get("indices", []) @@ -124,8 +136,16 @@ def init(self): # log.info(f"Token: {self.token}") nd = endee.Endee(token=self.token) # Uncomment below to use base_url - # nd.base_url = "https://10.160.0.8:8080" - # self.nd = nd + # nd.base_url = "http://10.128.0.3:8080/api/v1" + # nd.set_base_url("http://10.128.0.3:8080/api/v1") + # nd.set_base_url("http://3.85.217.253:80/api/v1") + # nd.set_base_url("http://10.128.0.5:8080/api/v1") + # nd.set_base_url("http://54.89.169.48:80/api/v1") + + if self.base_url: + nd.set_base_url(self.base_url) + self.nd = nd + self.index = nd.get_index(name=self.collection_name) yield except Exception as e: diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index eeb2e988b..5a4c2d0e3 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -4,7 +4,8 @@ import multiprocessing as mp import time import traceback - +import os +import json import numpy as np import psutil @@ -17,42 +18,15 @@ from .. import utils from ..clients import api -import os -import json - -# CHECKPOINT_PATH = "/mnt/data/VectorDBBench/insert_checkpoint.json" - -# Get the path of the current script +# Get the path of the current script to resolve root dir CURRENT_FILE = os.path.abspath(__file__) # Go up 3 levels to reach the root benchmarking folder ROOT_BENCHMARK_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(CURRENT_FILE)))) -CHECKPOINT_FILENAME = "insert_checkpoint.json" -CHECKPOINT_PATH = os.path.join(ROOT_BENCHMARK_DIR, CHECKPOINT_FILENAME) - # Ensure the root folder exists (should already exist, but safe) os.makedirs(ROOT_BENCHMARK_DIR, exist_ok=True) -def save_checkpoint(count: int): - with open(CHECKPOINT_PATH, "w") as f: - json.dump({"last_index": count}, f) - -def load_checkpoint() -> int: - if os.path.exists(CHECKPOINT_PATH): - try: - with open(CHECKPOINT_PATH, "r") as f: - data = json.load(f) - return data.get("last_index", 0) - except Exception as e: - log.warning(f"Failed to read checkpoint file: {e}") - return 0 - -def clear_checkpoint(): - if os.path.exists(CHECKPOINT_PATH): - os.remove(CHECKPOINT_PATH) - - NUM_PER_BATCH = config.NUM_PER_BATCH LOAD_MAX_TRY_COUNT = config.LOAD_MAX_TRY_COUNT @@ -74,6 +48,42 @@ def __init__( self.normalize = normalize self.filters = filters + # --- DYNAMIC CHECKPOINT LOGIC --- + # Generate a unique checkpoint file based on the index/collection name + # This prevents parallel processes from overwriting each other's progress. + collection_name = getattr(self.db, "collection_name", "unknown_index") + safe_name = "".join(c for c in collection_name if c.isalnum() or c in ('_', '-')) + self.checkpoint_path = os.path.join(ROOT_BENCHMARK_DIR, f"insert_checkpoint_{safe_name}.json") + log.info(f"Using unique checkpoint path: {self.checkpoint_path}") + + def _save_checkpoint(self, count: int): + """Saves current insertion progress to the dynamic JSON file.""" + try: + with open(self.checkpoint_path, "w") as f: + json.dump({"last_index": count}, f) + except Exception as e: + log.warning(f"Failed to save checkpoint: {e}") + + def _load_checkpoint(self) -> int: + """Reads progress from the dynamic JSON file if it exists.""" + if os.path.exists(self.checkpoint_path): + try: + with open(self.checkpoint_path, "r") as f: + data = json.load(f) + return data.get("last_index", 0) + except Exception as e: + log.warning(f"Failed to read checkpoint file: {e}") + return 0 + + def _clear_checkpoint(self): + """Removes the checkpoint file upon successful completion of the task.""" + if os.path.exists(self.checkpoint_path): + try: + os.remove(self.checkpoint_path) + log.info(f"Cleared checkpoint file: {self.checkpoint_path}") + except Exception as e: + log.warning(f"Failed to remove checkpoint file: {e}") + def retry_insert(self, db: api.VectorDB, retry_idx: int = 0, **kwargs): _, error = db.insert_embeddings(**kwargs) if error is not None: @@ -88,8 +98,9 @@ def retry_insert(self, db: api.VectorDB, retry_idx: int = 0, **kwargs): def task(self) -> int: count = 0 - last_index = load_checkpoint() - log.info(f"Resuming insertion from checkpoint at index {last_index}") + last_index = self._load_checkpoint() + if last_index > 0: + log.info(f"Resuming insertion from checkpoint at index {last_index}") with self.db.init(): log.info(f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}") @@ -99,7 +110,9 @@ def task(self) -> int: batch_size = len(data_df) if count + batch_size <= last_index: count += batch_size - log.debug(f"Skipping batch of size {batch_size}, total skipped={count}") + # Log skip progress occasionally to avoid silence + if count % 100_000 == 0: + log.debug(f"Skipping batch of size {batch_size}, total skipped={count}") continue all_metadata = data_df[self.dataset.data.train_id_field].tolist() @@ -135,7 +148,9 @@ def task(self) -> int: assert insert_count == len(all_metadata) count += insert_count - save_checkpoint(count) # Save progress after each batch + + # Save progress after each batch to the dynamic path + self._save_checkpoint(count) if count % 100_000 == 0: log.info(f"({mp.current_process().name:16}) Loaded {count} embeddings into VectorDB") @@ -144,7 +159,9 @@ def task(self) -> int: f"({mp.current_process().name:16}) Finished loading all dataset into VectorDB, " f"dur={time.perf_counter() - start}" ) - clear_checkpoint() # Clean checkpoint file after successful completion + + # Clean checkpoint file ONLY after successful completion + self._clear_checkpoint() return count @@ -259,80 +276,7 @@ def run(self) -> int: return count -# ============================================================================= -# PARALLEL SEARCH WORKER FUNCTION -# ============================================================================= -def _parallel_search_worker(args: tuple) -> list[tuple[float, int, list[int]]]: - """ - Worker function executed in separate process for parallel search. - - Each worker: - 1. Receives a batch of queries (embeddings) with their indices - 2. Initializes its own DB connection - 3. Executes searches for all queries in the batch - 4. Returns results with latencies and original indices - - Args: - args: Tuple of (db_instance, batch_queries, batch_indices, k, filters) - - Returns: - List of tuples: [(latency, original_query_idx, search_results), ...] - - This function runs in a subprocess, so it needs to: - - Initialize DB connection independently - - Handle its own retry logic - - Return all results for aggregation in main process - """ - db_instance, batch_queries, batch_indices, k, filters = args - - results_batch = [] - - # Each worker initializes its own DB connection - with db_instance.init(): - # Prepare filters if needed (e.g., for filtered search scenarios) - db_instance.prepare_filter(filters) - - # Process each query in this worker's batch - for idx, emb in zip(batch_indices, batch_queries): - retry_count = 0 - max_retries = config.MAX_SEARCH_RETRY - - # Retry logic for each individual query - while retry_count <= max_retries: - try: - # Time the search operation - s = time.perf_counter() - results = db_instance.search_embedding(emb, k) - latency = time.perf_counter() - s - - # Store result with original index for proper ordering later - results_batch.append((latency, idx, results)) - break # Success, move to next query - - except Exception as e: - retry_count += 1 - if retry_count > max_retries: - log.error(f"Query {idx} failed after {max_retries} retries: {e}") - raise - log.warning(f"Query {idx} retry {retry_count}/{max_retries}: {e}") - time.sleep(0.1 * retry_count) # Exponential backoff - - return results_batch - - class SerialSearchRunner: - """ - Search runner that executes queries to measure recall and latency. - - PERFORMANCE ENHANCEMENT: - This class now uses parallel processing under the hood while maintaining - the same interface. Queries are distributed across multiple worker processes - for significant speedup (4-8x faster depending on CPU cores). - - The parallelization is transparent to callers - all existing code continues - to work without modification. - """ - def __init__( self, db: api.VectorDB, @@ -345,25 +289,13 @@ def __init__( self.k = k self.filters = filters - # Convert test data to list format if needed if isinstance(test_data[0], np.ndarray): self.test_data = [query.tolist() for query in test_data] else: self.test_data = test_data self.ground_truth = ground_truth - - # PARALLEL PROCESSING CONFIG: - # Auto-detect number of CPU cores, reserve 1 for main process - # This allows efficient parallel query execution - self.num_workers = max(1, mp.cpu_count() - 1) - log.info(f"SerialSearchRunner initialized with {self.num_workers} parallel workers") def _get_db_search_res(self, emb: list[float], retry_idx: int = 0) -> list[int]: - """ - LEGACY METHOD - Kept for backward compatibility. - This method is no longer used in the main search path but retained - in case any external code calls it directly. - """ try: results = self.db.search_embedding(emb, self.k) except Exception as e: @@ -376,141 +308,47 @@ def _get_db_search_res(self, emb: list[float], retry_idx: int = 0) -> list[int]: return results - def _create_query_batches(self) -> list[tuple[list[int], list]]: - """ - PARALLEL PROCESSING HELPER: - Splits test queries into batches for distribution to worker processes. - - Each batch contains: - - Original indices (to maintain ordering after parallel execution) - - Query embeddings for this batch - - Batch size is calculated as: total_queries / num_workers - This ensures even distribution of work across all available workers. - - Returns: - List of (indices, queries) tuples, one per worker - """ - total_queries = len(self.test_data) - batch_size = max(1, math.ceil(total_queries / self.num_workers)) - batches = [] - - for i in range(0, total_queries, batch_size): - end_idx = min(i + batch_size, total_queries) - indices = list(range(i, end_idx)) - queries = self.test_data[i:end_idx] - batches.append((indices, queries)) - - log.debug(f"Created {len(batches)} query batches, batch_size={batch_size}") - return batches - def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, float, float]: - """ - MODIFIED: Now uses parallel processing for significant speedup. - - Main changes from original: - 1. Splits queries into batches for parallel workers - 2. Spawns worker processes to execute searches concurrently - 3. Collects and re-orders results by original index - 4. Calculates metrics (recall, NDCG, latencies) from parallel results - - The interface and return values remain identical to the original version, - ensuring backward compatibility with existing code. - - Original single-threaded approach took ~164s for 1000 queries. - Parallel approach typically achieves 4-8x speedup depending on CPU cores. - - Args: - args: Tuple of (test_data, ground_truth) - kept for interface compatibility - - Returns: - Tuple of (avg_recall, avg_ndcg, p99_latency, p95_latency) - """ log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency") - - test_data, ground_truth = args - - # Validate input - if not test_data: - raise RuntimeError("empty test_data") - - log.debug(f"test dataset size: {len(test_data)}") - log.debug(f"ground truth size: {len(ground_truth) if ground_truth else 0}") - - # PARALLEL EXECUTION SECTION: - # Create batches for parallel processing - batches = self._create_query_batches() - - # Prepare arguments for each worker process - # Each worker gets: (db_instance, queries, indices, k, filters) - worker_args = [ - (self.db, queries, indices, self.k, self.filters) - for indices, queries in batches - ] - - log.info(f"Launching {len(worker_args)} parallel workers for search") - - # Execute searches in parallel using ProcessPoolExecutor - # Using 'spawn' context ensures clean process initialization - all_results = [] - with concurrent.futures.ProcessPoolExecutor( - max_workers=self.num_workers, - mp_context=mp.get_context("spawn") - ) as executor: - # Submit all batches to worker pool - futures = [ - executor.submit(_parallel_search_worker, args) - for args in worker_args - ] - - # Collect results as workers complete - # as_completed() allows processing results immediately when ready - for future in concurrent.futures.as_completed(futures): + with self.db.init(): + self.db.prepare_filter(self.filters) + test_data, ground_truth = args + ideal_dcg = get_ideal_dcg(self.k) + + log.debug(f"test dataset size: {len(test_data)}") + log.debug(f"ground truth size: {len(ground_truth)}") + + latencies, recalls, ndcgs = [], [], [] + for idx, emb in enumerate(test_data): + s = time.perf_counter() try: - batch_results = future.result() - all_results.extend(batch_results) + results = self._get_db_search_res(emb) except Exception as e: - log.error(f"Worker process failed: {e}") - raise - - # RESULTS PROCESSING SECTION: - # Sort results by original query index to maintain correct ordering - # This is crucial because workers complete in arbitrary order - all_results.sort(key=lambda x: x[1]) # Sort by index (2nd element) - - log.info(f"Collected {len(all_results)} search results from parallel workers") - - # Calculate performance metrics - latencies, recalls, ndcgs = [], [], [] - ideal_dcg = get_ideal_dcg(self.k) - - for latency, idx, results in all_results: - latencies.append(latency) - - # Calculate recall and NDCG if ground truth is available - if ground_truth is not None: - gt = ground_truth[idx] - recalls.append(calc_recall(self.k, gt[:self.k], results)) - ndcgs.append(calc_ndcg(gt[:self.k], results, ideal_dcg)) - else: - recalls.append(0) - ndcgs.append(0) - - # Progress logging every 100 queries - if len(latencies) % 100 == 0: - log.debug( - f"({mp.current_process().name:14}) search_count={len(latencies):3}, " - f"latest_latency={latencies[-1]}, latest recall={recalls[-1]}" - ) - - # Calculate aggregate statistics + log.warning(f"VectorDB search_embedding error: {e}") + raise e from None + + latencies.append(time.perf_counter() - s) + + if ground_truth is not None: + gt = ground_truth[idx] + recalls.append(calc_recall(self.k, gt[: self.k], results)) + ndcgs.append(calc_ndcg(gt[: self.k], results, ideal_dcg)) + else: + recalls.append(0) + ndcgs.append(0) + + if len(latencies) % 100 == 0: + log.debug( + f"({mp.current_process().name:14}) search_count={len(latencies):3}, " + f"latest_latency={latencies[-1]}, latest recall={recalls[-1]}" + ) + avg_latency = round(np.mean(latencies), 4) avg_recall = round(np.mean(recalls), 4) avg_ndcg = round(np.mean(ndcgs), 4) cost = round(np.sum(latencies), 4) p99 = round(np.percentile(latencies, 99), 4) p95 = round(np.percentile(latencies, 95), 4) - log.info( f"{mp.current_process().name:14} search entire test_data: " f"cost={cost}s, " @@ -521,28 +359,15 @@ def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, floa f"p99={p99}, " f"p95={p95}" ) - return (avg_recall, avg_ndcg, p99, p95) def _run_in_subprocess(self) -> tuple[float, float, float, float]: - """ - UNCHANGED: Wrapper to execute search in subprocess. - - This method is kept for interface compatibility. - The actual parallelization happens inside the search() method. - """ with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: future = executor.submit(self.search, (self.test_data, self.ground_truth)) return future.result() @utils.time_it def run(self) -> tuple[float, float, float, float]: - """ - UNCHANGED: Main entry point for search execution. - - Interface remains identical - callers don't need any changes. - Parallel processing is handled transparently. - """ log.info(f"{mp.current_process().name:14} start serial search") if self.test_data is None: msg = "empty test_data" @@ -553,11 +378,9 @@ def run(self) -> tuple[float, float, float, float]: @utils.time_it def run_with_cost(self) -> tuple[tuple[float, float, float, float], float]: """ - UNCHANGED: Search with cost tracking. - + Search all test data in serial. Returns: - tuple[tuple[float, float, float, float], float]: - (avg_recall, avg_ndcg, p99_latency, p95_latency), cost + tuple[tuple[float, float, float, float], float]: (avg_recall, avg_ndcg, p99_latency, p95_latency), cost """ log.info(f"{mp.current_process().name:14} start serial search") if self.test_data is None: @@ -571,13 +394,595 @@ def run_with_cost(self) -> tuple[tuple[float, float, float, float], float]: +# # ======================================================================= + +# # PARALLEL PROCESSING WITH CHECKPOINT FOR RESUME LOGIC + +# # ======================================================================= + + + +# import concurrent +# import logging +# import math +# import multiprocessing as mp +# import time +# import traceback + +# import numpy as np +# import psutil + +# from vectordb_bench.backend.dataset import DatasetManager +# from vectordb_bench.backend.filter import Filter, FilterOp, non_filter + +# from ... import config +# from ...metric import calc_ndcg, calc_recall, get_ideal_dcg +# from ...models import LoadTimeoutError, PerformanceTimeoutError +# from .. import utils +# from ..clients import api + +# import os +# import json + +# # CHECKPOINT_PATH = "/mnt/data/VectorDBBench/insert_checkpoint.json" + +# # Get the path of the current script +# CURRENT_FILE = os.path.abspath(__file__) + +# # Go up 3 levels to reach the root benchmarking folder +# ROOT_BENCHMARK_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(CURRENT_FILE)))) + +# CHECKPOINT_FILENAME = "insert_checkpoint.json" +# CHECKPOINT_PATH = os.path.join(ROOT_BENCHMARK_DIR, CHECKPOINT_FILENAME) + +# # Ensure the root folder exists (should already exist, but safe) +# os.makedirs(ROOT_BENCHMARK_DIR, exist_ok=True) + +# def save_checkpoint(count: int): +# with open(CHECKPOINT_PATH, "w") as f: +# json.dump({"last_index": count}, f) + +# def load_checkpoint() -> int: +# if os.path.exists(CHECKPOINT_PATH): +# try: +# with open(CHECKPOINT_PATH, "r") as f: +# data = json.load(f) +# return data.get("last_index", 0) +# except Exception as e: +# log.warning(f"Failed to read checkpoint file: {e}") +# return 0 + +# def clear_checkpoint(): +# if os.path.exists(CHECKPOINT_PATH): +# os.remove(CHECKPOINT_PATH) + + +# NUM_PER_BATCH = config.NUM_PER_BATCH +# LOAD_MAX_TRY_COUNT = config.LOAD_MAX_TRY_COUNT + +# log = logging.getLogger(__name__) + + +# class SerialInsertRunner: +# def __init__( +# self, +# db: api.VectorDB, +# dataset: DatasetManager, +# normalize: bool, +# filters: Filter = non_filter, +# timeout: float | None = None, +# ): +# self.timeout = timeout if isinstance(timeout, int | float) else None +# self.dataset = dataset +# self.db = db +# self.normalize = normalize +# self.filters = filters + +# def retry_insert(self, db: api.VectorDB, retry_idx: int = 0, **kwargs): +# _, error = db.insert_embeddings(**kwargs) +# if error is not None: +# log.warning(f"Insert Failed, try_idx={retry_idx}, Exception: {error}") +# retry_idx += 1 +# if retry_idx <= config.MAX_INSERT_RETRY: +# time.sleep(retry_idx) +# self.retry_insert(db, retry_idx=retry_idx, **kwargs) +# else: +# msg = f"Insert failed and retried more than {config.MAX_INSERT_RETRY} times" +# raise RuntimeError(msg) from None + +# def task(self) -> int: +# count = 0 +# last_index = load_checkpoint() +# log.info(f"Resuming insertion from checkpoint at index {last_index}") + +# with self.db.init(): +# log.info(f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}") +# start = time.perf_counter() + +# for data_df in self.dataset: +# batch_size = len(data_df) +# if count + batch_size <= last_index: +# count += batch_size +# log.debug(f"Skipping batch of size {batch_size}, total skipped={count}") +# continue + +# all_metadata = data_df[self.dataset.data.train_id_field].tolist() +# emb_np = np.stack(data_df[self.dataset.data.train_vector_field]) + +# if self.normalize: +# log.debug("Normalizing train data") +# all_embeddings = (emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis]).tolist() +# else: +# all_embeddings = emb_np.tolist() +# del emb_np + +# labels_data = None +# if self.filters.type == FilterOp.StrEqual: +# if self.dataset.data.scalar_labels_file_separated: +# labels_data = self.dataset.scalar_labels[self.filters.label_field][all_metadata].to_list() +# else: +# labels_data = data_df[self.filters.label_field].tolist() + +# insert_count, error = self.db.insert_embeddings( +# embeddings=all_embeddings, +# metadata=all_metadata, +# labels_data=labels_data, +# ) + +# if error is not None: +# self.retry_insert( +# self.db, +# embeddings=all_embeddings, +# metadata=all_metadata, +# labels_data=labels_data, +# ) + +# assert insert_count == len(all_metadata) +# count += insert_count +# save_checkpoint(count) # Save progress after each batch + +# if count % 100_000 == 0: +# log.info(f"({mp.current_process().name:16}) Loaded {count} embeddings into VectorDB") + +# log.info( +# f"({mp.current_process().name:16}) Finished loading all dataset into VectorDB, " +# f"dur={time.perf_counter() - start}" +# ) +# clear_checkpoint() # Clean checkpoint file after successful completion +# return count + + +# def endless_insert_data(self, all_embeddings: list, all_metadata: list, left_id: int = 0) -> int: +# with self.db.init(): +# # unique id for endlessness insertion +# all_metadata = [i + left_id for i in all_metadata] + +# num_batches = math.ceil(len(all_embeddings) / NUM_PER_BATCH) +# log.info( +# f"({mp.current_process().name:16}) Start inserting {len(all_embeddings)} " +# f"embeddings in batch {NUM_PER_BATCH}" +# ) +# count = 0 +# for batch_id in range(num_batches): +# retry_count = 0 +# already_insert_count = 0 +# metadata = all_metadata[batch_id * NUM_PER_BATCH : (batch_id + 1) * NUM_PER_BATCH] +# embeddings = all_embeddings[batch_id * NUM_PER_BATCH : (batch_id + 1) * NUM_PER_BATCH] + +# log.debug( +# f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_batches}], " +# f"Start inserting {len(metadata)} embeddings" +# ) +# while retry_count < LOAD_MAX_TRY_COUNT: +# insert_count, error = self.db.insert_embeddings( +# embeddings=embeddings[already_insert_count:], +# metadata=metadata[already_insert_count:], +# ) +# already_insert_count += insert_count +# if error is not None: +# retry_count += 1 +# time.sleep(10) + +# log.info(f"Failed to insert data, try {retry_count} time") +# if retry_count >= LOAD_MAX_TRY_COUNT: +# raise error +# else: +# break +# log.debug( +# f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_batches}], " +# f"Finish inserting {len(metadata)} embeddings" +# ) + +# assert already_insert_count == len(metadata) +# count += already_insert_count +# log.info( +# f"({mp.current_process().name:16}) Finish inserting {len(all_embeddings)} embeddings in " +# f"batch {NUM_PER_BATCH}" +# ) +# return count + +# @utils.time_it +# def _insert_all_batches(self) -> int: +# """Performance case only""" +# with concurrent.futures.ProcessPoolExecutor( +# mp_context=mp.get_context("spawn"), +# max_workers=1, +# ) as executor: +# future = executor.submit(self.task) +# try: +# count = future.result(timeout=self.timeout) +# except TimeoutError as e: +# msg = f"VectorDB load dataset timeout in {self.timeout}" +# log.warning(msg) +# for pid, _ in executor._processes.items(): +# psutil.Process(pid).kill() +# raise PerformanceTimeoutError(msg) from e +# except Exception as e: +# log.warning(f"VectorDB load dataset error: {e}") +# raise e from e +# else: +# return count + +# def run_endlessness(self) -> int: +# """run forever util DB raises exception or crash""" +# # datasets for load tests are quite small, can fit into memory +# # only 1 file +# data_df = next(iter(self.dataset)) +# all_embeddings, all_metadata = ( +# np.stack(data_df[self.dataset.data.train_vector_field]).tolist(), +# data_df[self.dataset.data.train_id_field].tolist(), +# ) + +# start_time = time.perf_counter() +# max_load_count, times = 0, 0 +# try: +# while time.perf_counter() - start_time < self.timeout: +# count = self.endless_insert_data( +# all_embeddings, +# all_metadata, +# left_id=max_load_count, +# ) +# max_load_count += count +# times += 1 +# log.info( +# f"Loaded {times} entire dataset, current max load counts={utils.numerize(max_load_count)}, " +# f"{max_load_count}" +# ) +# except Exception as e: +# log.info( +# f"Capacity case load reach limit, insertion counts={utils.numerize(max_load_count)}, " +# f"{max_load_count}, err={e}" +# ) +# traceback.print_exc() +# return max_load_count +# else: +# raise LoadTimeoutError(self.timeout) + +# def run(self) -> int: +# count, _ = self._insert_all_batches() +# return count + + +# # ============================================================================= +# # PARALLEL SEARCH WORKER FUNCTION +# # ============================================================================= +# def _parallel_search_worker(args: tuple) -> list[tuple[float, int, list[int]]]: +# """ +# Worker function executed in separate process for parallel search. + +# Each worker: +# 1. Receives a batch of queries (embeddings) with their indices +# 2. Initializes its own DB connection +# 3. Executes searches for all queries in the batch +# 4. Returns results with latencies and original indices + +# Args: +# args: Tuple of (db_instance, batch_queries, batch_indices, k, filters) + +# Returns: +# List of tuples: [(latency, original_query_idx, search_results), ...] + +# This function runs in a subprocess, so it needs to: +# - Initialize DB connection independently +# - Handle its own retry logic +# - Return all results for aggregation in main process +# """ +# db_instance, batch_queries, batch_indices, k, filters = args + +# results_batch = [] + +# # Each worker initializes its own DB connection +# with db_instance.init(): +# # Prepare filters if needed (e.g., for filtered search scenarios) +# db_instance.prepare_filter(filters) + +# # Process each query in this worker's batch +# for idx, emb in zip(batch_indices, batch_queries): +# retry_count = 0 +# max_retries = config.MAX_SEARCH_RETRY + +# # Retry logic for each individual query +# while retry_count <= max_retries: +# try: +# # Time the search operation +# s = time.perf_counter() +# results = db_instance.search_embedding(emb, k) +# latency = time.perf_counter() - s + +# # Store result with original index for proper ordering later +# results_batch.append((latency, idx, results)) +# break # Success, move to next query + +# except Exception as e: +# retry_count += 1 +# if retry_count > max_retries: +# log.error(f"Query {idx} failed after {max_retries} retries: {e}") +# raise +# log.warning(f"Query {idx} retry {retry_count}/{max_retries}: {e}") +# time.sleep(0.1 * retry_count) # Exponential backoff + +# return results_batch + + +# class SerialSearchRunner: +# """ +# Search runner that executes queries to measure recall and latency. + +# PERFORMANCE ENHANCEMENT: +# This class now uses parallel processing under the hood while maintaining +# the same interface. Queries are distributed across multiple worker processes +# for significant speedup (4-8x faster depending on CPU cores). + +# The parallelization is transparent to callers - all existing code continues +# to work without modification. +# """ + +# def __init__( +# self, +# db: api.VectorDB, +# test_data: list[list[float]], +# ground_truth: list[list[int]], +# k: int = 100, +# filters: Filter = non_filter, +# ): +# self.db = db +# self.k = k +# self.filters = filters + +# # Convert test data to list format if needed +# if isinstance(test_data[0], np.ndarray): +# self.test_data = [query.tolist() for query in test_data] +# else: +# self.test_data = test_data +# self.ground_truth = ground_truth + +# # PARALLEL PROCESSING CONFIG: +# # Auto-detect number of CPU cores, reserve 1 for main process +# # This allows efficient parallel query execution +# # self.num_workers = max(1, mp.cpu_count() - 1) +# self.num_workers = max(1, 2) +# log.info(f"SerialSearchRunner initialized with {self.num_workers} parallel workers") + +# def _get_db_search_res(self, emb: list[float], retry_idx: int = 0) -> list[int]: +# """ +# LEGACY METHOD - Kept for backward compatibility. +# This method is no longer used in the main search path but retained +# in case any external code calls it directly. +# """ +# try: +# results = self.db.search_embedding(emb, self.k) +# except Exception as e: +# log.warning(f"Serial search failed, retry_idx={retry_idx}, Exception: {e}") +# if retry_idx < config.MAX_SEARCH_RETRY: +# return self._get_db_search_res(emb=emb, retry_idx=retry_idx + 1) + +# msg = f"Serial search failed and retried more than {config.MAX_SEARCH_RETRY} times" +# raise RuntimeError(msg) from e + +# return results + +# def _create_query_batches(self) -> list[tuple[list[int], list]]: +# """ +# PARALLEL PROCESSING HELPER: +# Splits test queries into batches for distribution to worker processes. + +# Each batch contains: +# - Original indices (to maintain ordering after parallel execution) +# - Query embeddings for this batch + +# Batch size is calculated as: total_queries / num_workers +# This ensures even distribution of work across all available workers. + +# Returns: +# List of (indices, queries) tuples, one per worker +# """ +# total_queries = len(self.test_data) +# batch_size = max(1, math.ceil(total_queries / self.num_workers)) +# batches = [] + +# for i in range(0, total_queries, batch_size): +# end_idx = min(i + batch_size, total_queries) +# indices = list(range(i, end_idx)) +# queries = self.test_data[i:end_idx] +# batches.append((indices, queries)) + +# log.debug(f"Created {len(batches)} query batches, batch_size={batch_size}") +# return batches + +# def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, float, float]: +# """ +# MODIFIED: Now uses parallel processing for significant speedup. + +# Main changes from original: +# 1. Splits queries into batches for parallel workers +# 2. Spawns worker processes to execute searches concurrently +# 3. Collects and re-orders results by original index +# 4. Calculates metrics (recall, NDCG, latencies) from parallel results + +# The interface and return values remain identical to the original version, +# ensuring backward compatibility with existing code. + +# Original single-threaded approach took ~164s for 1000 queries. +# Parallel approach typically achieves 4-8x speedup depending on CPU cores. + +# Args: +# args: Tuple of (test_data, ground_truth) - kept for interface compatibility + +# Returns: +# Tuple of (avg_recall, avg_ndcg, p99_latency, p95_latency) +# """ +# log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency") + +# test_data, ground_truth = args + +# # Validate input +# if not test_data: +# raise RuntimeError("empty test_data") + +# log.debug(f"test dataset size: {len(test_data)}") +# log.debug(f"ground truth size: {len(ground_truth) if ground_truth else 0}") + +# # PARALLEL EXECUTION SECTION: +# # Create batches for parallel processing +# batches = self._create_query_batches() + +# # Prepare arguments for each worker process +# # Each worker gets: (db_instance, queries, indices, k, filters) +# worker_args = [ +# (self.db, queries, indices, self.k, self.filters) +# for indices, queries in batches +# ] + +# log.info(f"Launching {len(worker_args)} parallel workers for search") + +# # Execute searches in parallel using ProcessPoolExecutor +# # Using 'spawn' context ensures clean process initialization +# all_results = [] +# with concurrent.futures.ProcessPoolExecutor( +# max_workers=self.num_workers, +# mp_context=mp.get_context("spawn") +# ) as executor: +# # Submit all batches to worker pool +# futures = [ +# executor.submit(_parallel_search_worker, args) +# for args in worker_args +# ] + +# # Collect results as workers complete +# # as_completed() allows processing results immediately when ready +# for future in concurrent.futures.as_completed(futures): +# try: +# batch_results = future.result() +# all_results.extend(batch_results) +# except Exception as e: +# log.error(f"Worker process failed: {e}") +# raise + +# # RESULTS PROCESSING SECTION: +# # Sort results by original query index to maintain correct ordering +# # This is crucial because workers complete in arbitrary order +# all_results.sort(key=lambda x: x[1]) # Sort by index (2nd element) + +# log.info(f"Collected {len(all_results)} search results from parallel workers") + +# # Calculate performance metrics +# latencies, recalls, ndcgs = [], [], [] +# ideal_dcg = get_ideal_dcg(self.k) + +# for latency, idx, results in all_results: +# latencies.append(latency) + +# # Calculate recall and NDCG if ground truth is available +# if ground_truth is not None: +# gt = ground_truth[idx] +# recalls.append(calc_recall(self.k, gt[:self.k], results)) +# ndcgs.append(calc_ndcg(gt[:self.k], results, ideal_dcg)) +# else: +# recalls.append(0) +# ndcgs.append(0) + +# # Progress logging every 100 queries +# if len(latencies) % 100 == 0: +# log.debug( +# f"({mp.current_process().name:14}) search_count={len(latencies):3}, " +# f"latest_latency={latencies[-1]}, latest recall={recalls[-1]}" +# ) + +# # Calculate aggregate statistics +# avg_latency = round(np.mean(latencies), 4) +# avg_recall = round(np.mean(recalls), 4) +# avg_ndcg = round(np.mean(ndcgs), 4) +# cost = round(np.sum(latencies), 4) +# p99 = round(np.percentile(latencies, 99), 4) +# p95 = round(np.percentile(latencies, 95), 4) + +# log.info( +# f"{mp.current_process().name:14} search entire test_data: " +# f"cost={cost}s, " +# f"queries={len(latencies)}, " +# f"avg_recall={avg_recall}, " +# f"avg_ndcg={avg_ndcg}, " +# f"avg_latency={avg_latency}, " +# f"p99={p99}, " +# f"p95={p95}" +# ) + +# return (avg_recall, avg_ndcg, p99, p95) + +# def _run_in_subprocess(self) -> tuple[float, float, float, float]: +# """ +# UNCHANGED: Wrapper to execute search in subprocess. + +# This method is kept for interface compatibility. +# The actual parallelization happens inside the search() method. +# """ +# with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: +# future = executor.submit(self.search, (self.test_data, self.ground_truth)) +# return future.result() + +# @utils.time_it +# def run(self) -> tuple[float, float, float, float]: +# """ +# UNCHANGED: Main entry point for search execution. + +# Interface remains identical - callers don't need any changes. +# Parallel processing is handled transparently. +# """ +# log.info(f"{mp.current_process().name:14} start serial search") +# if self.test_data is None: +# msg = "empty test_data" +# raise RuntimeError(msg) + +# return self._run_in_subprocess() + +# @utils.time_it +# def run_with_cost(self) -> tuple[tuple[float, float, float, float], float]: +# """ +# UNCHANGED: Search with cost tracking. + +# Returns: +# tuple[tuple[float, float, float, float], float]: +# (avg_recall, avg_ndcg, p99_latency, p95_latency), cost +# """ +# log.info(f"{mp.current_process().name:14} start serial search") +# if self.test_data is None: +# msg = "empty test_data" +# raise RuntimeError(msg) + +# return self._run_in_subprocess() + + + + + + -# ======================================================================= +# # ======================================================================= -# MODIFIED TO HAVE CHECKPOINT FOR RESUME LOGIC +# # MODIFIED TO HAVE CHECKPOINT FOR RESUME LOGIC -# ======================================================================= +# # ======================================================================= @@ -960,11 +1365,11 @@ def run_with_cost(self) -> tuple[tuple[float, float, float, float], float]: -# ======================================================================= +# # ======================================================================= -# ORIGINAL SERIAL RUNNER CODE +# # ORIGINAL SERIAL RUNNER CODE -# ======================================================================= +# # ======================================================================= From 621f9c8d78af0392b01cebc475947125cd688f21 Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Mon, 29 Dec 2025 09:29:21 +0530 Subject: [PATCH 03/22] Added automated script for the setup in server --- setup_bench.py | 112 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 setup_bench.py diff --git a/setup_bench.py b/setup_bench.py new file mode 100644 index 000000000..f03042409 --- /dev/null +++ b/setup_bench.py @@ -0,0 +1,112 @@ +import subprocess +import sys +import os +import shutil + +def run_command(command, shell=False): + """Runs a shell command and raises an exception on failure.""" + print(f"--> Running: {' '.join(command) if isinstance(command, list) else command}") + try: + subprocess.check_call(command, shell=shell) + except subprocess.CalledProcessError as e: + print(f"Error executing command: {e}") + sys.exit(1) + +def check_and_install_system_deps(): + print("### Checking System Dependencies ###") + + # 1. Update Apt + run_command("sudo apt-get update", shell=True) + + # 2. Check/Install Git + if shutil.which("git") is None: + print("Git not found. Installing...") + run_command("sudo apt-get install -y git", shell=True) + else: + print("Git is already installed.") + + # 3. Check/Install Python 3.11 + # We explicitly check for a python3.11 binary + if shutil.which("python3.11") is None: + print("Python 3.11 not found. Installing via deadsnakes PPA...") + run_command("sudo apt-get install -y software-properties-common", shell=True) + run_command("sudo add-apt-repository -y ppa:deadsnakes/ppa", shell=True) + run_command("sudo apt-get update", shell=True) + run_command("sudo apt-get install -y python3.11 python3.11-venv python3.11-dev", shell=True) + else: + print("Python 3.11 is already installed.") + +def setup_repo(): + repo_url = "https://github.com/EndeeLabs/VectorDBBench.git" + repo_dir = "VectorDBBench" + + print("\n### Setting up Repository ###") + + # 1. Clone + if not os.path.exists(repo_dir): + run_command(["git", "clone", repo_url]) + else: + print(f"Directory {repo_dir} already exists. Skipping clone.") + + # Change working directory to the repo + os.chdir(repo_dir) + + # 2. Switch Branch + print("Switching to 'Endee' branch...") + run_command(["git", "fetch", "origin"]) + run_command(["git", "checkout", "Endee"]) + + # Pull latest just in case + run_command(["git", "pull", "origin", "Endee"]) + + return os.getcwd() + +def setup_venv_and_install(repo_path): + print("\n### Setting up Virtual Environment & Installing Packages ###") + + venv_dir = os.path.join(repo_path, "venv") + + # 1. Create Venv using Python 3.11 specifically + if not os.path.exists(venv_dir): + print("Creating virtual environment...") + run_command(["python3.11", "-m", "venv", "venv"]) + else: + print("Virtual environment already exists.") + + # Define paths to the venv executables + venv_python = os.path.join(venv_dir, "bin", "python") + venv_pip = os.path.join(venv_dir, "bin", "pip") + + # Upgrade pip first + run_command([venv_python, "-m", "pip", "install", "--upgrade", "pip"]) + + # 2. pip install endee + print("Installing 'endee'...") + run_command([venv_pip, "install", "endee"]) + + # 3. pip install -e . + print("Installing editable project...") + run_command([venv_pip, "install", "-e", "."]) + + return venv_dir + +if __name__ == "__main__": + # Ensure we are on Linux (quick check) + if not sys.platform.startswith('linux'): + print("Warning: This script is optimized for Linux (AWS/GCP instances).") + + # Step 1: System Level Installs (Sudo required) + check_and_install_system_deps() + + # Step 2: Clone and Switch Branch + project_path = setup_repo() + + # Step 3 & 4: Create Venv and Install Pip packages + venv_path = setup_venv_and_install(project_path) + + print("\n" + "="*50) + print("SETUP COMPLETE!") + print("="*50) + print("To start using VectorDBBench, run the following command in your shell:") + print(f"\n source {os.path.join(project_path, 'venv/bin/activate')}\n") + print("="*50) \ No newline at end of file From 4da6487a0a32987c4b6fcab0b9215ad421c9aea0 Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Mon, 29 Dec 2025 10:29:06 +0530 Subject: [PATCH 04/22] Added small precision for cli --- vectordb_bench/backend/clients/endee/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordb_bench/backend/clients/endee/cli.py b/vectordb_bench/backend/clients/endee/cli.py index e6096fc2e..597ea9b4c 100644 --- a/vectordb_bench/backend/clients/endee/cli.py +++ b/vectordb_bench/backend/clients/endee/cli.py @@ -36,7 +36,7 @@ class EndeeTypedDict(CommonTypedDict): ] precision: Annotated[ str, - click.option("--precision", type=click.Choice(["medium", "high", "ultra-high", "fp16"]), default="medium", help="Quant Level", show_default=True) + click.option("--precision", type=click.Choice(["medium", "high", "ultra-high", "fp16", "small"]), default="medium", help="Quant Level", show_default=True) ] version: Annotated[ int, From d04d1eb5c8eed166fa5521df0a00bcaa7eeb1006 Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Mon, 29 Dec 2025 10:35:11 +0530 Subject: [PATCH 05/22] Modified demoCommands file --- .allcommands => .demoCommands | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) rename .allcommands => .demoCommands (87%) diff --git a/.allcommands b/.demoCommands similarity index 87% rename from .allcommands rename to .demoCommands index 355251929..38717a123 100644 --- a/.allcommands +++ b/.demoCommands @@ -1,6 +1,6 @@ # FOR CREATING A NEW INDEX AND START THE PROCESS DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ - --token "as1prod:nb5l9VEBrgDEsvjut8Oawgcbf5jghoN5" \ + --token "TOKEN" \ --region india-west-1 \ --base-url "https://as1.endee.io/api/v1" \ --index-name test_prod_10M_medium_3 \ @@ -21,11 +21,10 @@ DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ --search-concurrent \ --search-serial -PROD: "https://as1.endee.io/api/v1" -AWS: http://54.89.169.48:80/api/v1 +======================================================================== DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ - --token "awsbench16:uVQMSo85De3k1lN5FiZI0xKcwXDlyBvu" \ + --token "TOKEN" \ --region india-west-1 \ --base-url "http://54.89.169.48:80/api/v1"\ --index-name test_aws_10M_medium_1 \ @@ -46,9 +45,11 @@ AWS: http://54.89.169.48:80/api/v1 --search-concurrent \ --search-serial +======================================================================== + # FOR USING THE EXISTING INDEX DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ - --token "as1prod:nb5l9VEBrgDEsvjut8Oawgcbf5jghoN5" \ + --token "TOKEN" \ --region india-west-1 \ --base-url "https://as1.endee.io/api/v1" \ --index-name test_prod_10M_medium_3 \ From a6304ff680dfb49223e791e1532c980eb8eab11f Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Mon, 29 Dec 2025 11:17:04 +0530 Subject: [PATCH 06/22] Added download dataset file --- download_dataset.py | 177 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 download_dataset.py diff --git a/download_dataset.py b/download_dataset.py new file mode 100644 index 000000000..3fce58cde --- /dev/null +++ b/download_dataset.py @@ -0,0 +1,177 @@ +# =================================================================== # +# ==================== FOR COHERE 10M DATASET ======================= # +# =================================================================== # + +import os +import requests +from concurrent.futures import ThreadPoolExecutor +from tqdm import tqdm + +# --- CONFIGURATION --- +# Base URL for Cohere 10M dataset +BASE_URL = "https://assets.zilliz.com/benchmark/cohere_large_10m/" + +# Your SSD Path (Updated based on your terminal output) +LOCAL_DIR = "/home/admin/vectordataset/cohere/cohere_large_10m" + +# Number of parallel downloads +MAX_WORKERS = 10 + +# --- FILE LIST GENERATION --- +files = [ + "test.parquet", + "neighbors.parquet", + "scalar_labels.parquet" # Included based on your ls output +] + +# Generate the 10 training parts (00 to 09) +# Note: Cohere 10M uses the prefix 'shuffle_train-' and has 10 parts +for i in range(10): + files.append(f"shuffle_train-{i:02d}-of-10.parquet") + +def download_file(filename): + local_path = os.path.join(LOCAL_DIR, filename) + url = BASE_URL + filename + + # 1. Skip if fully downloaded (File > 1KB) + if os.path.exists(local_path) and os.path.getsize(local_path) > 1024: + return f"✅ Skipped (Exists): {filename}" + + try: + # 2. Stream download + with requests.get(url, stream=True, timeout=60) as r: + if r.status_code == 404: + return f"❌ Missing (404): {filename}" + + r.raise_for_status() + + # Write to file + with open(local_path, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024*1024): # 1MB chunks + f.write(chunk) + + return f"⬇️ Downloaded: {filename}" + + except Exception as e: + return f"⚠️ Error {filename}: {str(e)}" + +if __name__ == "__main__": + # Ensure directory exists + os.makedirs(LOCAL_DIR, exist_ok=True) + + print(f"--- Cohere 10M Downloader ---") + print(f"Source: {BASE_URL}") + print(f"Destination: {LOCAL_DIR}") + print(f"Files to check: {len(files)}") + print(f"Parallel threads: {MAX_WORKERS}\n") + + # Start Parallel Download + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: + # We use tqdm to show a progress bar for the *number of files* completed + results = list(tqdm(executor.map(download_file, files), total=len(files), unit="file")) + + # Print summary of missing or failed files + failures = [r for r in results if "Error" in r or "Missing" in r] + if failures: + print("\nSummary of Issues:") + for fail in failures: + print(fail) + else: + print("\n🎉 All files processed successfully!") + + + + + + + + + + +# # =================================================================== # +# # ==================== FOR LAION 100M DATASET ======================= # +# # =================================================================== # + +# import os +# import requests +# from concurrent.futures import ThreadPoolExecutor +# from tqdm import tqdm + +# # --- CONFIGURATION --- +# # Base URL for Laion 100M dataset +# BASE_URL = "https://assets.zilliz.com/benchmark/laion_large_100m/" + +# # Your SSD Path (Updated based on your terminal output) +# LOCAL_DIR = "/home/admin/vectordataset/laion/laion_large_100m" + +# # Number of parallel downloads +# MAX_WORKERS = 10 + +# # --- FILE LIST GENERATION --- +# files = [ +# "test.parquet", +# "neighbors.parquet", +# # "scalar_labels.parquet" # Uncomment if needed (often missing for 100M) +# ] + +# # Generate the 100 training parts (00 to 99) +# for i in range(100): +# files.append(f"train-{i:02d}-of-100.parquet") + + +# def download_file(filename): +# local_path = os.path.join(LOCAL_DIR, filename) +# url = BASE_URL + filename + +# # 1. Skip if fully downloaded (File > 1KB) +# if os.path.exists(local_path) and os.path.getsize(local_path) > 1024: +# return f"✅ Skipped (Exists): {filename}" + +# try: +# # 2. Stream download +# with requests.get(url, stream=True, timeout=60) as r: +# if r.status_code == 404: +# return f"❌ Missing (404): {filename}" + +# r.raise_for_status() + +# # Write to file +# with open(local_path, 'wb') as f: +# for chunk in r.iter_content(chunk_size=1024 * 1024): # 1MB chunks +# f.write(chunk) + +# return f"⬇️ Downloaded: {filename}" + +# except Exception as e: +# return f"⚠️ Error {filename}: {str(e)}" + + +# if __name__ == "__main__": +# # Ensure directory exists +# os.makedirs(LOCAL_DIR, exist_ok=True) + +# print(f"--- LAION 100M Downloader ---") +# print(f"Source: {BASE_URL}") +# print(f"Destination: {LOCAL_DIR}") +# print(f"Files to check: {len(files)}") +# print(f"Parallel threads: {MAX_WORKERS}\n") + +# # Start Parallel Download +# with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: +# # We use tqdm to show a progress bar for the *number of files* completed +# results = list( +# tqdm( +# executor.map(download_file, files), +# total=len(files), +# unit="file" +# ) +# ) + +# # Print summary of missing or failed files +# failures = [r for r in results if "Error" in r or "Missing" in r] +# if failures: +# print("\nSummary of Issues:") +# for fail in failures: +# print(fail) +# else: +# print("\n🎉 All files processed successfully!") From 1d1912d482f92377bcfbde6ee17e67c6a281cda8 Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Mon, 29 Dec 2025 12:08:51 +0530 Subject: [PATCH 07/22] Added Demo CLI Commands file --- .demoCommands => demo_cli_ommands.txt | 36 ++++++--------------------- 1 file changed, 8 insertions(+), 28 deletions(-) rename .demoCommands => demo_cli_ommands.txt (51%) diff --git a/.demoCommands b/demo_cli_ommands.txt similarity index 51% rename from .demoCommands rename to demo_cli_ommands.txt index 38717a123..4eea84e6d 100644 --- a/.demoCommands +++ b/demo_cli_ommands.txt @@ -1,9 +1,11 @@ +# ======================================================================== # FOR CREATING A NEW INDEX AND START THE PROCESS -DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ +# ======================================================================== +DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ --token "TOKEN" \ --region india-west-1 \ --base-url "https://as1.endee.io/api/v1" \ - --index-name test_prod_10M_medium_3 \ + --index-name test_index_name \ --task-label "20251224" \ --m 16 \ --ef-con 128 \ @@ -21,38 +23,16 @@ DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ --search-concurrent \ --search-serial -======================================================================== - DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ - --token "TOKEN" \ - --region india-west-1 \ - --base-url "http://54.89.169.48:80/api/v1"\ - --index-name test_aws_10M_medium_1 \ - --task-label "20251223" \ - --m 16 \ - --ef-con 128 \ - --ef-search 128 \ - --space-type cosine \ - --precision medium \ - --version 1 \ - --case-type Performance768D10M \ - --k 30 \ - --num-concurrency "8" \ - --concurrency-duration 30 \ - --concurrency-timeout 3600 \ - --drop-old \ - --load \ - --search-concurrent \ - --search-serial - -======================================================================== +# ======================================================================== # FOR USING THE EXISTING INDEX -DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ +# ======================================================================== +DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ --token "TOKEN" \ --region india-west-1 \ --base-url "https://as1.endee.io/api/v1" \ - --index-name test_prod_10M_medium_3 \ + --index-name test_index_name\ --task-label "20251224" \ --m 16 \ --ef-con 128 \ From 1599f221bfd21d9d2d7b1b94928cacefb439ef10 Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Mon, 29 Dec 2025 12:36:03 +0530 Subject: [PATCH 08/22] Modified the selecting options --- download_dataset.py | 209 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 169 insertions(+), 40 deletions(-) diff --git a/download_dataset.py b/download_dataset.py index 3fce58cde..cc04abd90 100644 --- a/download_dataset.py +++ b/download_dataset.py @@ -1,40 +1,72 @@ -# =================================================================== # -# ==================== FOR COHERE 10M DATASET ======================= # -# =================================================================== # - import os import requests from concurrent.futures import ThreadPoolExecutor from tqdm import tqdm -# --- CONFIGURATION --- -# Base URL for Cohere 10M dataset -BASE_URL = "https://assets.zilliz.com/benchmark/cohere_large_10m/" - -# Your SSD Path (Updated based on your terminal output) -LOCAL_DIR = "/home/admin/vectordataset/cohere/cohere_large_10m" - -# Number of parallel downloads -MAX_WORKERS = 10 - -# --- FILE LIST GENERATION --- -files = [ - "test.parquet", - "neighbors.parquet", - "scalar_labels.parquet" # Included based on your ls output -] +# =================================================================== +# ======================== USER CONFIGURATION ======================= + +# 1. SELECT DATASET +DATASET_NAME = "cohere" # Options: "cohere", "laion" + +# 2. ROOT DIRECTORY +ROOT_DIR = "/home/admin/vectordataset" + +# =================================================================== + +# 3. DATASET DETAILS +DATASET_CONFIG = { + "cohere": { + "base_url": "https://assets.zilliz.com/benchmark/cohere_large_10m/", + "rel_path": "cohere/cohere_large_10m", # Subfolder inside ROOT_DIR + "train_prefix": "shuffle_train", + "train_count": 10, + "train_suffix": "of-10.parquet", + "extra_files": ["scalar_labels.parquet"] + }, + "laion": { + "base_url": "https://assets.zilliz.com/benchmark/laion_large_100m/", + "rel_path": "laion/laion_large_100m", # Subfolder inside ROOT_DIR + "train_prefix": "train", + "train_count": 100, + "train_suffix": "of-100.parquet", + "extra_files": [] + } +} + +# 4. GLOBAL SETTINGS +MAX_WORKERS = 10 + + +def get_file_list(config): + """Generates the list of files based on the dataset config.""" + files = ["test.parquet", "neighbors.parquet"] + + # Add extra files (like scalar_labels) if they exist in config + files.extend(config["extra_files"]) + + # Generate Training Files + # Example: shuffle_train-00-of-10.parquet OR train-00-of-100.parquet + prefix = config["train_prefix"] + count = config["train_count"] + suffix = config["train_suffix"] + + for i in range(count): + # Format usually needs leading zeros (00, 01..99) + files.append(f"{prefix}-{i:02d}-{suffix}") + + return files -# Generate the 10 training parts (00 to 09) -# Note: Cohere 10M uses the prefix 'shuffle_train-' and has 10 parts -for i in range(10): - files.append(f"shuffle_train-{i:02d}-of-10.parquet") +def download_file(args): + """ + Downloads a single file. args is (filename, base_url, full_local_path) + """ + filename, base_url, dest_dir = args + local_file_path = os.path.join(dest_dir, filename) + url = base_url + filename -def download_file(filename): - local_path = os.path.join(LOCAL_DIR, filename) - url = BASE_URL + filename - # 1. Skip if fully downloaded (File > 1KB) - if os.path.exists(local_path) and os.path.getsize(local_path) > 1024: + if os.path.exists(local_file_path) and os.path.getsize(local_file_path) > 1024: return f"✅ Skipped (Exists): {filename}" try: @@ -46,7 +78,7 @@ def download_file(filename): r.raise_for_status() # Write to file - with open(local_path, 'wb') as f: + with open(local_file_path, 'wb') as f: for chunk in r.iter_content(chunk_size=1024*1024): # 1MB chunks f.write(chunk) @@ -56,21 +88,36 @@ def download_file(filename): return f"⚠️ Error {filename}: {str(e)}" if __name__ == "__main__": + if DATASET_NAME not in DATASET_CONFIG: + print(f"❌ Error: Invalid dataset name '{DATASET_NAME}'") + exit(1) + + cfg = DATASET_CONFIG[DATASET_NAME] + + # Construct the full destination path + full_dest_path = os.path.join(ROOT_DIR, cfg["rel_path"]) + # Ensure directory exists - os.makedirs(LOCAL_DIR, exist_ok=True) + os.makedirs(full_dest_path, exist_ok=True) - print(f"--- Cohere 10M Downloader ---") - print(f"Source: {BASE_URL}") - print(f"Destination: {LOCAL_DIR}") - print(f"Files to check: {len(files)}") - print(f"Parallel threads: {MAX_WORKERS}\n") + files_to_download = get_file_list(cfg) + + print(f"--- Unified Downloader: {DATASET_NAME.upper()} ---") + print(f"Source: {cfg['base_url']}") + print(f"Destination: {full_dest_path}") + print(f"File Count: {len(files_to_download)}") + print(f"Threads: {MAX_WORKERS}\n") - # Start Parallel Download + # Pack arguments: (filename, base_url, full_dest_path) + map_args = [(f, cfg["base_url"], full_dest_path) for f in files_to_download] + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: - # We use tqdm to show a progress bar for the *number of files* completed - results = list(tqdm(executor.map(download_file, files), total=len(files), unit="file")) + results = list(tqdm( + executor.map(download_file, map_args), + total=len(files_to_download), + unit="file" + )) - # Print summary of missing or failed files failures = [r for r in results if "Error" in r or "Missing" in r] if failures: print("\nSummary of Issues:") @@ -84,6 +131,88 @@ def download_file(filename): +# # =================================================================== # +# # ==================== FOR COHERE 10M DATASET ======================= # +# # =================================================================== # + +# import os +# import requests +# from concurrent.futures import ThreadPoolExecutor +# from tqdm import tqdm + +# # --- CONFIGURATION --- +# # Base URL for Cohere 10M dataset +# BASE_URL = "https://assets.zilliz.com/benchmark/cohere_large_10m/" + +# # Your SSD Path (Updated based on your terminal output) +# LOCAL_DIR = "/home/admin/vectordataset/cohere/cohere_large_10m" + +# # Number of parallel downloads +# MAX_WORKERS = 10 + +# # --- FILE LIST GENERATION --- +# files = [ +# "test.parquet", +# "neighbors.parquet", +# "scalar_labels.parquet" # Included based on your ls output +# ] + +# # Generate the 10 training parts (00 to 09) +# # Note: Cohere 10M uses the prefix 'shuffle_train-' and has 10 parts +# for i in range(10): +# files.append(f"shuffle_train-{i:02d}-of-10.parquet") + +# def download_file(filename): +# local_path = os.path.join(LOCAL_DIR, filename) +# url = BASE_URL + filename + +# # 1. Skip if fully downloaded (File > 1KB) +# if os.path.exists(local_path) and os.path.getsize(local_path) > 1024: +# return f"✅ Skipped (Exists): {filename}" + +# try: +# # 2. Stream download +# with requests.get(url, stream=True, timeout=60) as r: +# if r.status_code == 404: +# return f"❌ Missing (404): {filename}" + +# r.raise_for_status() + +# # Write to file +# with open(local_path, 'wb') as f: +# for chunk in r.iter_content(chunk_size=1024*1024): # 1MB chunks +# f.write(chunk) + +# return f"⬇️ Downloaded: {filename}" + +# except Exception as e: +# return f"⚠️ Error {filename}: {str(e)}" + +# if __name__ == "__main__": +# # Ensure directory exists +# os.makedirs(LOCAL_DIR, exist_ok=True) + +# print(f"--- Cohere 10M Downloader ---") +# print(f"Source: {BASE_URL}") +# print(f"Destination: {LOCAL_DIR}") +# print(f"Files to check: {len(files)}") +# print(f"Parallel threads: {MAX_WORKERS}\n") + +# # Start Parallel Download +# with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: +# # We use tqdm to show a progress bar for the *number of files* completed +# results = list(tqdm(executor.map(download_file, files), total=len(files), unit="file")) + +# # Print summary of missing or failed files +# failures = [r for r in results if "Error" in r or "Missing" in r] +# if failures: +# print("\nSummary of Issues:") +# for fail in failures: +# print(fail) +# else: +# print("\n🎉 All files processed successfully!") + + From b9c2e4b8256b0f9c2045d51b28665b11bd0557de Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Tue, 30 Dec 2025 17:04:22 +0530 Subject: [PATCH 09/22] Added new parameter for CLI command --- demo_cli_ommands.txt | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/demo_cli_ommands.txt b/demo_cli_ommands.txt index 4eea84e6d..48001e1f5 100644 --- a/demo_cli_ommands.txt +++ b/demo_cli_ommands.txt @@ -49,3 +49,29 @@ DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ --skip-load \ --search-concurrent \ --search-serial + + +# ======================================================================== +# WITH CHANGED BATCH SIZE, CREATING A NEW INDEX AND START THE PROCESS +# ======================================================================== +NUM_PER_BATCH=10000 DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ + --token "TOKEN" \ + --region india-west-1 \ + --base-url "https://as1.endee.io/api/v1" \ + --index-name test_index_name \ + --task-label "20251224" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision medium \ + --version 1 \ + --case-type Performance768D10M \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial From 71ddffd1e9b162da72e8eaf3f64306621ec685bc Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Tue, 6 Jan 2026 14:08:38 +0530 Subject: [PATCH 10/22] Updated precision types --- vectordb_bench/backend/clients/endee/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vectordb_bench/backend/clients/endee/cli.py b/vectordb_bench/backend/clients/endee/cli.py index 597ea9b4c..970e4ce7a 100644 --- a/vectordb_bench/backend/clients/endee/cli.py +++ b/vectordb_bench/backend/clients/endee/cli.py @@ -36,7 +36,8 @@ class EndeeTypedDict(CommonTypedDict): ] precision: Annotated[ str, - click.option("--precision", type=click.Choice(["medium", "high", "ultra-high", "fp16", "small"]), default="medium", help="Quant Level", show_default=True) + # click.option("--precision", type=click.Choice(["medium", "high", "ultra-high", "fp16", "small"]), default="medium", help="Quant Level", show_default=True) + click.option("--precision", type=click.Choice(["binary", "int4d", "int8d", "int16d", "float16", "float32"]), default="int8d", help="Quant Level", show_default=True) ] version: Annotated[ int, From a921498b22b7d9181bedd57a53f417cd241b2518 Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Tue, 6 Jan 2026 20:57:48 +0530 Subject: [PATCH 11/22] Modified setup bench script --- setup_bench.py | 183 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 182 insertions(+), 1 deletion(-) diff --git a/setup_bench.py b/setup_bench.py index f03042409..1353f4c76 100644 --- a/setup_bench.py +++ b/setup_bench.py @@ -3,6 +3,185 @@ import os import shutil +# ================= CONFIGURATION ================= +REPO_URL = "https://github.com/EndeeLabs/VectorDBBench.git" +REPO_DIR = "VectorDBBench" +PYTHON_VERSION = "3.11.9" +# ================================================= + +def run_command(command, shell=False, cwd=None): + """Runs a shell command and exits on failure.""" + cmd_str = ' '.join(command) if isinstance(command, list) else command + print(f"--> [EXEC]: {cmd_str}") + try: + subprocess.check_call(command, shell=shell, cwd=cwd) + except subprocess.CalledProcessError as e: + print(f"Error executing command: {e}") + sys.exit(1) + +def is_python311_installed(): + """Checks if python3.11 is currently available in the system PATH.""" + # Check standard PATH + if shutil.which("python3.11") is not None: + return True + + # Check common local install location (Source builds often go here) + if os.path.exists("/usr/local/bin/python3.11"): + return True + + return False + +def check_system_compatibility(): + """Ensures we are on a Debian-based system (Debian, Ubuntu, Mint, Kali, etc).""" + if shutil.which("apt-get") is None: + print("\n!!! CRITICAL ERROR !!!") + print("This script relies on 'apt-get'. It works on Debian, Ubuntu, Linux Mint, Kali, Pop!_OS, etc.") + sys.exit(1) + +def is_ubuntu(): + """Returns True only if we are confident this is Ubuntu.""" + try: + if os.path.exists("/etc/os-release"): + with open("/etc/os-release") as f: + if "ubuntu" in f.read().lower(): + return True + if shutil.which("lsb_release"): + out = subprocess.check_output(["lsb_release", "-i"]).decode().lower() + if "ubuntu" in out: + return True + except: + pass + return False + +def install_python_ubuntu_strategy(): + """Strategy A: Ubuntu (Fast PPA)""" + print("\n[Strategy] Detected Ubuntu. Using fast PPA installation...") + run_command("sudo apt-get update", shell=True) + run_command("sudo apt-get install -y software-properties-common git", shell=True) + print("Adding Deadsnakes PPA...") + run_command("sudo add-apt-repository -y ppa:deadsnakes/ppa", shell=True) + run_command("sudo apt-get update", shell=True) + run_command("sudo apt-get install -y python3.11 python3.11-venv python3.11-dev", shell=True) + +def install_python_debian_strategy(): + """Strategy B: Debian / Universal (Source Build Fallback)""" + print("\n[Strategy] Detected Debian/Other. Using robust compatibility mode...") + + # 1. Install Dependencies + run_command("sudo apt-get update", shell=True) + print("Installing build dependencies...") + deps = [ + "git", "wget", "build-essential", "zlib1g-dev", "libncurses5-dev", + "libgdbm-dev", "libnss3-dev", "libssl-dev", "libreadline-dev", + "libffi-dev", "libsqlite3-dev", "libbz2-dev", "pkg-config" + ] + run_command(f"sudo apt-get install -y {' '.join(deps)}", shell=True) + + # 2. Try APT first (Debian 12+) + print("Checking system repos for Python 3.11...") + try: + subprocess.check_call("apt-cache show python3.11", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + print("Python 3.11 found in APT. Installing...") + run_command("sudo apt-get install -y python3.11 python3.11-venv python3.11-dev", shell=True) + return + except subprocess.CalledProcessError: + print("Python 3.11 not in system repos. Proceeding to Source Build.") + + # 3. Source Build (Universal) + print("\n*** STARTING SOURCE BUILD ***") + tarball = f"Python-{PYTHON_VERSION}.tgz" + url = f"https://www.python.org/ftp/python/{PYTHON_VERSION}/{tarball}" + + if not os.path.exists(tarball): + run_command(f"wget {url}", shell=True) + + run_command(f"tar -xf {tarball}", shell=True) + src_dir = f"Python-{PYTHON_VERSION}" + + # Configure & Make + run_command("./configure --enable-optimizations", shell=True, cwd=src_dir) + nproc = subprocess.check_output("nproc", shell=True).decode().strip() + run_command(f"make -j {nproc}", shell=True, cwd=src_dir) + + # SAFE INSTALL (altinstall) + run_command("sudo make altinstall", shell=True, cwd=src_dir) + + # Cleanup + os.remove(tarball) + run_command(f"sudo rm -rf {src_dir}", shell=True) + +def setup_project_and_venv(): + print("\n[Project] Setting up VectorDBBench...") + + # 1. Clone + if not os.path.exists(REPO_DIR): + run_command(["git", "clone", REPO_URL]) + + os.chdir(REPO_DIR) + + # 2. Switch Branch + run_command(["git", "fetch", "origin"]) + run_command(["git", "checkout", "Endee"]) + run_command(["git", "pull", "origin", "Endee"]) + + # 3. Locate Python 3.11 + python_bin = "python3.11" + if shutil.which("python3.11") is None: + if os.path.exists("/usr/local/bin/python3.11"): + python_bin = "/usr/local/bin/python3.11" + else: + print("Error: Python 3.11 binary not found after installation attempt.") + sys.exit(1) + + print(f"Using Python binary: {python_bin}") + + # 4. Create Venv + if not os.path.exists("venv"): + run_command([python_bin, "-m", "venv", "venv"]) + else: + print("Virtual environment already exists.") + + # 5. Install Deps + venv_pip = os.path.join("venv", "bin", "pip") + run_command([venv_pip, "install", "--upgrade", "pip"]) + run_command([venv_pip, "install", "endee"]) + run_command([venv_pip, "install", "-e", "."]) + + return os.path.join(os.getcwd(), "venv") + +if __name__ == "__main__": + check_system_compatibility() + + # --- THE FIX IS HERE --- + if is_python311_installed(): + print("\n" + "="*50) + print("SKIP: Python 3.11 is already installed.") + print("="*50) + else: + # Only install if missing + if is_ubuntu(): + install_python_ubuntu_strategy() + else: + install_python_debian_strategy() + # ----------------------- + + venv_path = setup_project_and_venv() + + print("\n" + "="*50) + print("SETUP SUCCESSFUL!") + print(f"To start: source {os.path.join(venv_path, 'bin', 'activate')}") + print("="*50) + + + + + +''' +import subprocess +import sys +import os +import shutil + def run_command(command, shell=False): """Runs a shell command and raises an exception on failure.""" print(f"--> Running: {' '.join(command) if isinstance(command, list) else command}") @@ -109,4 +288,6 @@ def setup_venv_and_install(repo_path): print("="*50) print("To start using VectorDBBench, run the following command in your shell:") print(f"\n source {os.path.join(project_path, 'venv/bin/activate')}\n") - print("="*50) \ No newline at end of file + print("="*50) + +''' \ No newline at end of file From 737b11a4d02de9d3c7527ff26eee98c39a04c9ac Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Wed, 7 Jan 2026 12:30:13 +0530 Subject: [PATCH 12/22] Modified demo cli commands --- demo_cli_ommands.txt | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/demo_cli_ommands.txt b/demo_cli_ommands.txt index 48001e1f5..a770786bc 100644 --- a/demo_cli_ommands.txt +++ b/demo_cli_ommands.txt @@ -4,16 +4,16 @@ DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ --token "TOKEN" \ --region india-west-1 \ - --base-url "https://as1.endee.io/api/v1" \ + --base-url "http://127.0.0.1:8080/api/v1" \ --index-name test_index_name \ - --task-label "20251224" \ + --task-label "20260107" \ --m 16 \ --ef-con 128 \ --ef-search 128 \ --space-type cosine \ - --precision medium \ + --precision int8d \ --version 1 \ - --case-type Performance768D10M \ + --case-type Performance768D1M \ --k 30 \ --num-concurrency "8" \ --concurrency-duration 30 \ @@ -31,16 +31,16 @@ DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ --token "TOKEN" \ --region india-west-1 \ - --base-url "https://as1.endee.io/api/v1" \ + --base-url "http://127.0.0.1:8080/api/v1" \ --index-name test_index_name\ - --task-label "20251224" \ + --task-label "20260107" \ --m 16 \ --ef-con 128 \ --ef-search 128 \ --space-type cosine \ - --precision medium \ + --precision int8d \ --version 1 \ - --case-type Performance768D10M \ + --case-type Performance768D1M \ --k 30 \ --num-concurrency "8" \ --concurrency-duration 30 \ @@ -57,16 +57,16 @@ DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ NUM_PER_BATCH=10000 DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ --token "TOKEN" \ --region india-west-1 \ - --base-url "https://as1.endee.io/api/v1" \ + --base-url "http://127.0.0.1:8080/api/v1" \ --index-name test_index_name \ - --task-label "20251224" \ + --task-label "20260107" \ --m 16 \ --ef-con 128 \ --ef-search 128 \ --space-type cosine \ - --precision medium \ + --precision int8d \ --version 1 \ - --case-type Performance768D10M \ + --case-type Performance768D1M \ --k 30 \ --num-concurrency "8" \ --concurrency-duration 30 \ From 0aff8597b959a48e88028ebcc0cf8ba4985f6ead Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Wed, 7 Jan 2026 13:03:03 +0530 Subject: [PATCH 13/22] Modified setup bench for different OS --- setup_bench.py | 242 ++++++++++++++++++++++++++++++------------------- 1 file changed, 151 insertions(+), 91 deletions(-) diff --git a/setup_bench.py b/setup_bench.py index 1353f4c76..ad41055c1 100644 --- a/setup_bench.py +++ b/setup_bench.py @@ -1,7 +1,9 @@ -import subprocess import sys import os +import subprocess import shutil +import platform +import urllib.request # ================= CONFIGURATION ================= REPO_URL = "https://github.com/EndeeLabs/VectorDBBench.git" @@ -11,107 +13,145 @@ def run_command(command, shell=False, cwd=None): """Runs a shell command and exits on failure.""" + use_shell = shell + # On Windows, list commands usually need shell=True to find executables + if platform.system() == "Windows" and isinstance(command, list): + use_shell = True + cmd_str = ' '.join(command) if isinstance(command, list) else command print(f"--> [EXEC]: {cmd_str}") + try: - subprocess.check_call(command, shell=shell, cwd=cwd) + subprocess.check_call(command, shell=use_shell, cwd=cwd) except subprocess.CalledProcessError as e: print(f"Error executing command: {e}") sys.exit(1) -def is_python311_installed(): - """Checks if python3.11 is currently available in the system PATH.""" - # Check standard PATH - if shutil.which("python3.11") is not None: - return True +def get_os_type(): + """Returns 'windows', 'macos', or 'linux'.""" + os_name = platform.system().lower() - # Check common local install location (Source builds often go here) - if os.path.exists("/usr/local/bin/python3.11"): - return True - - return False - -def check_system_compatibility(): - """Ensures we are on a Debian-based system (Debian, Ubuntu, Mint, Kali, etc).""" - if shutil.which("apt-get") is None: - print("\n!!! CRITICAL ERROR !!!") - print("This script relies on 'apt-get'. It works on Debian, Ubuntu, Linux Mint, Kali, Pop!_OS, etc.") - sys.exit(1) + if "darwin" in os_name: + return "macos" + elif "win" in os_name: + return "windows" + return "linux" + +def find_python311(): + """Finds python3.11 executable path cross-platform.""" + candidates = [] + + if get_os_type() == "windows": + candidates = ["py", "python", "python3.11"] + else: + # Check standard PATH first, then explicit locations + candidates = ["python3.11", "/usr/bin/python3.11", "/usr/local/bin/python3.11", "/opt/homebrew/bin/python3.11"] + + for cmd in candidates: + path = shutil.which(cmd) + if path: + try: + # Verify it is actually 3.11 + # We use --version to check the actual output + ver = subprocess.check_output([path, "--version"]).decode().strip() + if "3.11" in ver: + return path + except: + continue + return None + +def install_linux_strategy(): + """Installs Python 3.11 on Linux (Ubuntu PPA or Debian Source).""" + print("\n[Linux] Python 3.11 not found. Determining installation strategy...") + + if not shutil.which("apt-get"): + print("Error: This script requires 'apt-get' (Debian/Ubuntu/Mint/Kali).") + sys.exit(1) -def is_ubuntu(): - """Returns True only if we are confident this is Ubuntu.""" + # 1. Detect Ubuntu (Use PPA for speed) + is_ubuntu = False try: if os.path.exists("/etc/os-release"): with open("/etc/os-release") as f: if "ubuntu" in f.read().lower(): - return True - if shutil.which("lsb_release"): - out = subprocess.check_output(["lsb_release", "-i"]).decode().lower() - if "ubuntu" in out: - return True + is_ubuntu = True except: pass - return False -def install_python_ubuntu_strategy(): - """Strategy A: Ubuntu (Fast PPA)""" - print("\n[Strategy] Detected Ubuntu. Using fast PPA installation...") - run_command("sudo apt-get update", shell=True) - run_command("sudo apt-get install -y software-properties-common git", shell=True) - print("Adding Deadsnakes PPA...") - run_command("sudo add-apt-repository -y ppa:deadsnakes/ppa", shell=True) run_command("sudo apt-get update", shell=True) - run_command("sudo apt-get install -y python3.11 python3.11-venv python3.11-dev", shell=True) -def install_python_debian_strategy(): - """Strategy B: Debian / Universal (Source Build Fallback)""" - print("\n[Strategy] Detected Debian/Other. Using robust compatibility mode...") - - # 1. Install Dependencies - run_command("sudo apt-get update", shell=True) - print("Installing build dependencies...") + if is_ubuntu: + print("Detected Ubuntu. Attempting PPA install...") + try: + run_command("sudo apt-get install -y software-properties-common", shell=True) + run_command("sudo add-apt-repository -y ppa:deadsnakes/ppa", shell=True) + run_command("sudo apt-get update", shell=True) + run_command("sudo apt-get install -y python3.11 python3.11-venv python3.11-dev", shell=True) + return + except Exception as e: + print(f"Ubuntu PPA failed ({e}). Falling back to source build.") + + # 2. Debian/Fallback Strategy (Source Build) + print("Detected Debian/Other. Using Source Build (Robust Method)...") + + # Install Build Dependencies deps = [ - "git", "wget", "build-essential", "zlib1g-dev", "libncurses5-dev", + "wget", "build-essential", "zlib1g-dev", "libncurses5-dev", "libgdbm-dev", "libnss3-dev", "libssl-dev", "libreadline-dev", "libffi-dev", "libsqlite3-dev", "libbz2-dev", "pkg-config" ] run_command(f"sudo apt-get install -y {' '.join(deps)}", shell=True) - - # 2. Try APT first (Debian 12+) - print("Checking system repos for Python 3.11...") - try: - subprocess.check_call("apt-cache show python3.11", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - print("Python 3.11 found in APT. Installing...") - run_command("sudo apt-get install -y python3.11 python3.11-venv python3.11-dev", shell=True) - return - except subprocess.CalledProcessError: - print("Python 3.11 not in system repos. Proceeding to Source Build.") - - # 3. Source Build (Universal) - print("\n*** STARTING SOURCE BUILD ***") + + # Download & Build tarball = f"Python-{PYTHON_VERSION}.tgz" url = f"https://www.python.org/ftp/python/{PYTHON_VERSION}/{tarball}" - + if not os.path.exists(tarball): run_command(f"wget {url}", shell=True) run_command(f"tar -xf {tarball}", shell=True) src_dir = f"Python-{PYTHON_VERSION}" - # Configure & Make + # Configure & Install + # --enable-optimizations is standard for production python builds run_command("./configure --enable-optimizations", shell=True, cwd=src_dir) nproc = subprocess.check_output("nproc", shell=True).decode().strip() run_command(f"make -j {nproc}", shell=True, cwd=src_dir) - # SAFE INSTALL (altinstall) + # CRITICAL: Use altinstall to avoid overwriting /usr/bin/python3 run_command("sudo make altinstall", shell=True, cwd=src_dir) # Cleanup os.remove(tarball) run_command(f"sudo rm -rf {src_dir}", shell=True) -def setup_project_and_venv(): - print("\n[Project] Setting up VectorDBBench...") +def install_macos_strategy(): + print("\n[macOS] Installing Python 3.11 via Homebrew...") + if shutil.which("brew") is None: + print("Error: Homebrew not found. Please install it from brew.sh") + sys.exit(1) + run_command("brew install python@3.11", shell=True) + +def install_windows_strategy(): + print("\n[Windows] Installing Python 3.11 via Winget/Installer...") + # Try Winget first (standard on Win 10/11) + if shutil.which("winget"): + try: + run_command("winget install -e --id Python.Python.3.11", shell=True) + return + except: + pass + + # Fallback to direct download + installer = "python-3.11.9-amd64.exe" + url = f"https://www.python.org/ftp/python/3.11.9/{installer}" + print(f"Downloading {url}...") + urllib.request.urlretrieve(url, installer) + run_command([installer, "/quiet", "InstallAllUsers=1", "PrependPath=1"]) + os.remove(installer) + +def setup_project(python_exe): + print(f"\n[Project] Setting up repo using found Python: {python_exe}") # 1. Clone if not os.path.exists(REPO_DIR): @@ -119,57 +159,77 @@ def setup_project_and_venv(): os.chdir(REPO_DIR) - # 2. Switch Branch + # 2. Checkout Branch run_command(["git", "fetch", "origin"]) run_command(["git", "checkout", "Endee"]) run_command(["git", "pull", "origin", "Endee"]) - # 3. Locate Python 3.11 - python_bin = "python3.11" - if shutil.which("python3.11") is None: - if os.path.exists("/usr/local/bin/python3.11"): - python_bin = "/usr/local/bin/python3.11" - else: - print("Error: Python 3.11 binary not found after installation attempt.") - sys.exit(1) - - print(f"Using Python binary: {python_bin}") - - # 4. Create Venv + # 3. Create Venv if not os.path.exists("venv"): - run_command([python_bin, "-m", "venv", "venv"]) + print("Creating virtual environment...") + run_command([python_exe, "-m", "venv", "venv"]) else: - print("Virtual environment already exists.") + print("Virtual environment already exists. Skipping creation.") + + # 4. Install Deps + # Check OS *again* here to determine correct PIP path + if get_os_type() == "windows": + venv_pip = os.path.join("venv", "Scripts", "pip.exe") + else: + # MacOS and Linux use this path + venv_pip = os.path.join("venv", "bin", "pip") - # 5. Install Deps - venv_pip = os.path.join("venv", "bin", "pip") + print(f"Installing dependencies using: {venv_pip}") run_command([venv_pip, "install", "--upgrade", "pip"]) run_command([venv_pip, "install", "endee"]) run_command([venv_pip, "install", "-e", "."]) - return os.path.join(os.getcwd(), "venv") + return venv_pip if __name__ == "__main__": - check_system_compatibility() + # 0. Check Git + if shutil.which("git") is None: + print("Error: Git is not installed.") + if get_os_type() == "linux": + run_command("sudo apt-get update && sudo apt-get install -y git", shell=True) + else: + sys.exit(1) - # --- THE FIX IS HERE --- - if is_python311_installed(): + # 1. Check for Existing Python 3.11 + py_path = find_python311() + + if py_path: print("\n" + "="*50) - print("SKIP: Python 3.11 is already installed.") + print(f"FOUND PYTHON 3.11: {py_path}") + print("Skipping OS installation steps.") print("="*50) else: - # Only install if missing - if is_ubuntu(): - install_python_ubuntu_strategy() - else: - install_python_debian_strategy() - # ----------------------- + # Install if missing + os_type = get_os_type() + if os_type == "linux": + install_linux_strategy() + elif os_type == "macos": + install_macos_strategy() + elif os_type == "windows": + install_windows_strategy() + + # Verify installation + py_path = find_python311() + if not py_path: + print("Error: Installation failed or binary not found.") + sys.exit(1) - venv_path = setup_project_and_venv() + # 2. Setup Project + setup_project(py_path) print("\n" + "="*50) print("SETUP SUCCESSFUL!") - print(f"To start: source {os.path.join(venv_path, 'bin', 'activate')}") + print("="*50) + + if get_os_type() == "windows": + print(f"To start: {os.path.join(os.getcwd(), 'venv', 'Scripts', 'activate')}") + else: + print(f"To start: source {os.path.join(os.getcwd(), 'venv', 'bin', 'activate')}") print("="*50) From e554491b6cc45128f882f4fa8105acad82487aec Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Wed, 7 Jan 2026 16:05:26 +0530 Subject: [PATCH 14/22] Modified cli implementation --- vectordb_bench/backend/clients/endee/cli.py | 47 ++++++++++++++++++--- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/vectordb_bench/backend/clients/endee/cli.py b/vectordb_bench/backend/clients/endee/cli.py index 970e4ce7a..57c96560c 100644 --- a/vectordb_bench/backend/clients/endee/cli.py +++ b/vectordb_bench/backend/clients/endee/cli.py @@ -78,20 +78,19 @@ def Endee(**parameters): collection_name = f"endee_bench_{uuid.uuid4().hex[:8]}" # Filter out None values before creating config - params_for_vecx = {k: v for k, v in parameters.items() if v is not None} - db_config = EndeeConfig(**params_for_vecx) + params_for_nd = {k: v for k, v in parameters.items() if v is not None} + db_config = EndeeConfig(**params_for_nd) custom_case_config = get_custom_case_config(parameters) # Create task config from vectordb_bench.models import TaskConfig, CaseConfig, CaseType, ConcurrencySearchConfig - # Create an instance of EmptyDBCaseConfig instead of passing None db_case_config = EmptyDBCaseConfig() task = TaskConfig( db=DB.Endee, - db_config=db_config, # Use the EndeeConfig instance directly + db_config=db_config, db_case_config=db_case_config, case_config=CaseConfig( case_id=CaseType[parameters["case_type"]], @@ -99,6 +98,7 @@ def Endee(**parameters): concurrency_search_config=ConcurrencySearchConfig( concurrency_duration=parameters["concurrency_duration"], num_concurrency=[int(s) for s in parameters["num_concurrency"]], + concurrency_timeout=parameters["concurrency_timeout"], # ========== Added this ========== ), custom_case=custom_case_config, ), @@ -107,13 +107,46 @@ def Endee(**parameters): # Use the run method of the benchmark_runner object if not parameters["dry_run"]: - benchmark_runner.run([task]) + # --- TASK LABEL LOGIC START --- + # 1. Generate a unique run ID + run_uuid = uuid.uuid4().hex + + # 2. Pick the base name: Use --task-label if provided, else --index-name + base_name = parameters.get("task_label") + if not base_name: + base_name = parameters.get("index_name", "endee") + + # 3. Combine into the final label: "MyLabel_UUID" + final_label = f"{base_name}_{run_uuid}" + + # 4. Pass the label to the runner to ensure the filename is correct + benchmark_runner.run([task], final_label) + # --- TASK LABEL LOGIC END --- - # Wait for task to complete if needed + # Wait for task to complete import time from vectordb_bench.interface import global_result_future from concurrent.futures import wait time.sleep(5) if global_result_future: - wait([global_result_future]) \ No newline at end of file + wait([global_result_future]) + + # Ensure CLI doesn't close while background processes are active + while benchmark_runner.has_running(): + time.sleep(1) + + + +# # Use the run method of the benchmark_runner object +# if not parameters["dry_run"]: +# benchmark_runner.run([task]) + +# # Wait for task to complete if needed +# import time +# from vectordb_bench.interface import global_result_future +# from concurrent.futures import wait + +# time.sleep(5) +# if global_result_future: +# wait([global_result_future]) \ No newline at end of file From 309020f603e4f059e0967abbbc6da4b1701a1126 Mon Sep 17 00:00:00 2001 From: darshan-endee Date: Sat, 10 Jan 2026 19:25:11 +0530 Subject: [PATCH 15/22] filter_endee --- demo_cli_ommands.txt | 77 +++- vectordb_bench/backend/clients/endee/endee.py | 372 +++++++++++++++--- 2 files changed, 394 insertions(+), 55 deletions(-) diff --git a/demo_cli_ommands.txt b/demo_cli_ommands.txt index a770786bc..8e7e55d7a 100644 --- a/demo_cli_ommands.txt +++ b/demo_cli_ommands.txt @@ -57,7 +57,7 @@ DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ NUM_PER_BATCH=10000 DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ --token "TOKEN" \ --region india-west-1 \ - --base-url "http://127.0.0.1:8080/api/v1" \ + --base-url "https://dev.endee.io/api/v1" \ --index-name test_index_name \ --task-label "20260107" \ --m 16 \ @@ -75,3 +75,78 @@ NUM_PER_BATCH=10000 DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench --load \ --search-concurrent \ --search-serial + + + +DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectordbbench endee \ + --token "x6zdfkgq:pd6fnXWS8e2ZkqAUNFBfirYcHqFShXgR:dev" \ + --region india-west-1 \ + --base-url "https://dev.endee.io/api/v1" \ + --index-name test_1M_numfilter_1p \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int8d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectordbbench endee \ + --token "x6zdfkgq:pd6fnXWS8e2ZkqAUNFBfirYcHqFShXgR:dev" \ + --region india-west-1 \ + --base-url "https://dev.endee.io/api/v1" \ + --index-name test_1M_numfilter_1p \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int8d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + +DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset_label_filter" vectordbbench endee \ + --token "x6zdfkgq:pd6fnXWS8e2ZkqAUNFBfirYcHqFShXgR:dev" \ + --region india-west-1 \ + --base-url "https://dev.endee.io/api/v1" \ + --index-name test_1M_labelfilter \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int8d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.01 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial diff --git a/vectordb_bench/backend/clients/endee/endee.py b/vectordb_bench/backend/clients/endee/endee.py index a87efbf37..fed841cfb 100644 --- a/vectordb_bench/backend/clients/endee/endee.py +++ b/vectordb_bench/backend/clients/endee/endee.py @@ -1,8 +1,238 @@ +# import logging +# from contextlib import contextmanager + +# from endee import endee + +# from ..api import DBCaseConfig, EmptyDBCaseConfig, IndexType, VectorDB +# from .config import EndeeConfig + +# log = logging.getLogger(__name__) + + +# class Endee(VectorDB): +# def __init__( +# self, +# dim: int, +# db_config: dict, +# db_case_config: DBCaseConfig, +# drop_old: bool = False, +# **kwargs, +# ): +# print(db_config) + +# self.token = db_config.get("token", "") +# self.region = db_config.get("region", "india-west-1") +# self.base_url = db_config.get("base_url") + +# # self.collection_name = db_config.get("collection_name", "") +# self.collection_name = (db_config.get("collection_name") or db_config.get("index_name")) + +# if not self.collection_name: +# import uuid +# self.collection_name = f"endee_bench_{uuid.uuid4().hex[:8]}" + +# self.space_type = db_config.get("space_type", "cosine") +# # self.use_fp16 = db_config.get("use_fp16", False) +# self.precision = db_config.get("precision") +# self.version = db_config.get("version") +# self.M = db_config.get("m") +# self.ef_con = db_config.get("ef_con") +# self.ef_search = db_config.get("ef_search") +# self.nd = endee.Endee(token=self.token) + +# # Dynamically set the URL +# if self.base_url: +# self.nd.set_base_url(self.base_url) +# log.info(f"Targeting server: {self.base_url}") + +# # Using Base url +# # self.nd.base_url = "http://10.128.0.3:8080/api/v1" +# # self.nd.set_base_url("http://10.128.0.3:8080/api/v1") +# # self.nd.set_base_url("http://3.85.217.253:80/api/v1") +# # self.nd.set_base_url("http://10.128.0.5:8080/api/v1") +# # self.nd.set_base_url("http://54.89.169.48:80/api/v1") + + +# # BASE_URL="http://3.85.217.253:80/api/v1" + +# # try: +# # indices = self.nd.list_indexes().get("indices", []) +# # index_names = [index["name"] for index in indices] if indices else [] + +# # if drop_old and self.collection_name in index_names: +# # self._drop_index(self.collection_name) +# # self._create_index(dim) +# # elif self.collection_name not in index_names: +# # self._create_index(dim) +# # except Exception as e: +# # log.error(f"Error connecting to Endee API: {e}") +# # raise + +# try: +# index_name = self.collection_name # assign before use +# indices = self.nd.list_indexes().get("indices", []) +# index_names = [index["name"] for index in indices] if indices else [] +# try: +# self.index = self.nd.get_index(name=index_name) +# log.info(f"Connected to existing Endee index: '{index_name}'") + +# except Exception as fetch_error: +# log.warning(f"Index '{index_name}' not found. Creating new index...") +# try: +# self._create_index(dim) +# self.index = self.nd.get_index(name=index_name) +# log.info(f"Successfully created and connected to index: '{index_name}'") + +# except Exception as create_error: +# # If error is "index already exists", just get the index again +# if "already exists" in str(create_error).lower() or "conflict" in str(create_error).lower(): +# log.warning(f"Index '{index_name}' already exists despite previous error. Fetching it again.") +# self.index = self.nd.get_index(name=index_name) +# else: +# log.error(f"Failed to create Endee index: {create_error}") +# raise +# except Exception as e: +# log.error(f"Error accessing or creating Endee index '{index_name}': {e}") +# raise + + + +# def _create_index(self, dim): +# try: +# resp = self.nd.create_index( +# name=self.collection_name, +# dimension=dim, +# space_type=self.space_type, +# # use_fp16=self.use_fp16, +# precision=self.precision, +# version=self.version, +# M=self.M, +# ef_con=self.ef_con +# ) +# log.info(f"Created new Endee index: {resp}") +# except Exception as e: +# log.error(f"Failed to create Endee index: {e}") +# raise + +# def _drop_index(self, collection_name): +# try: +# res = self.nd.delete_index(collection_name) +# log.info(res) +# except Exception as e: +# log.error(f"Failed to drop Endee index: {e}") +# raise + +# @classmethod +# def config_cls(cls) -> type[EndeeConfig]: +# return EndeeConfig + +# @classmethod +# def case_config_cls(cls, index_type: IndexType | None = None) -> type[DBCaseConfig]: +# return EmptyDBCaseConfig + +# @contextmanager +# def init(self): +# try: +# # log.info(f"Token: {self.token}") +# nd = endee.Endee(token=self.token) +# # Uncomment below to use base_url +# # nd.base_url = "http://10.128.0.3:8080/api/v1" +# # nd.set_base_url("http://10.128.0.3:8080/api/v1") +# # nd.set_base_url("http://3.85.217.253:80/api/v1") +# # nd.set_base_url("http://10.128.0.5:8080/api/v1") +# # nd.set_base_url("http://54.89.169.48:80/api/v1") + +# if self.base_url: +# nd.set_base_url(self.base_url) +# self.nd = nd + +# self.index = nd.get_index(name=self.collection_name) +# yield +# except Exception as e: +# if hasattr(e, 'response') and e.response is not None: +# log.error(f"HTTP Status: {e.response.status_code}, Body: {e.response.text}") +# log.error(f"Error initializing Endee client: {e}") +# # log.error("Error initializing Endee client", exc_info=True) +# raise +# finally: +# pass + +# def optimize(self, data_size: int | None = None): +# pass + +# def insert_embeddings( +# self, +# embeddings: list[list[float]], +# metadata: list[int], +# **kwargs, +# ) -> (int, Exception): # type: ignore +# assert len(embeddings) == len(metadata) +# insert_count = 0 +# try: +# batch_vectors = [ +# { +# "id": str(metadata[i]), +# "vector": embeddings[i], +# "meta": {"id": str(metadata[i])} +# } +# for i in range(len(embeddings)) +# ] +# self.index.upsert(batch_vectors) +# insert_count = len(batch_vectors) + +# except Exception as e: +# return (insert_count, e) + +# return (len(embeddings), None) + +# def search_embedding( +# self, +# query: list[float], +# k: int = 30, +# filters: dict | None = None, +# **kwargs, +# ) -> list[int]: +# try: +# filter_expr = None +# if filters and "id" in filters: +# filter_expr = {"id": filters["id"]} + +# results = self.index.query( +# vector=query, +# top_k=k, +# filter=filter_expr, +# ef=self.ef_search, +# include_vectors=False +# ) + +# return [int(result["id"]) for result in results] + +# except Exception as e: +# log.warning(f"Error querying Endee index: {e}") +# raise + +# def describe_index(self) -> dict: +# """Get information about the current index.""" +# try: +# all_indices = self.nd.list_indexes().get("indices", []) + +# for idx in all_indices: +# if idx.get("name") == self.collection_name: +# return idx + +# return {} +# except Exception as e: +# log.warning(f"Error describing Endee index: {e}") +# return {} + import logging from contextlib import contextmanager +from collections.abc import Iterable from endee import endee +from vectordb_bench.backend.filter import Filter, FilterOp + from ..api import DBCaseConfig, EmptyDBCaseConfig, IndexType, VectorDB from .config import EndeeConfig @@ -10,12 +240,20 @@ class Endee(VectorDB): + # Add supported filter types like Milvus + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + def __init__( self, dim: int, db_config: dict, db_case_config: DBCaseConfig, drop_old: bool = False, + with_scalar_labels: bool = False, **kwargs, ): print(db_config) @@ -24,7 +262,6 @@ def __init__( self.region = db_config.get("region", "india-west-1") self.base_url = db_config.get("base_url") - # self.collection_name = db_config.get("collection_name", "") self.collection_name = (db_config.get("collection_name") or db_config.get("index_name")) if not self.collection_name: @@ -32,12 +269,20 @@ def __init__( self.collection_name = f"endee_bench_{uuid.uuid4().hex[:8]}" self.space_type = db_config.get("space_type", "cosine") - # self.use_fp16 = db_config.get("use_fp16", False) self.precision = db_config.get("precision") self.version = db_config.get("version") self.M = db_config.get("m") self.ef_con = db_config.get("ef_con") self.ef_search = db_config.get("ef_search") + self.with_scalar_labels = with_scalar_labels + + # Initialize filter expression (similar to Milvus) + self.filter_expr = None + + # Field names to match Milvus convention + self._scalar_id_field = "id" + self._scalar_label_field = "label" + self.nd = endee.Endee(token=self.token) # Dynamically set the URL @@ -45,31 +290,8 @@ def __init__( self.nd.set_base_url(self.base_url) log.info(f"Targeting server: {self.base_url}") - # Using Base url - # self.nd.base_url = "http://10.128.0.3:8080/api/v1" - # self.nd.set_base_url("http://10.128.0.3:8080/api/v1") - # self.nd.set_base_url("http://3.85.217.253:80/api/v1") - # self.nd.set_base_url("http://10.128.0.5:8080/api/v1") - # self.nd.set_base_url("http://54.89.169.48:80/api/v1") - - - # BASE_URL="http://3.85.217.253:80/api/v1" - - # try: - # indices = self.nd.list_indexes().get("indices", []) - # index_names = [index["name"] for index in indices] if indices else [] - - # if drop_old and self.collection_name in index_names: - # self._drop_index(self.collection_name) - # self._create_index(dim) - # elif self.collection_name not in index_names: - # self._create_index(dim) - # except Exception as e: - # log.error(f"Error connecting to Endee API: {e}") - # raise - try: - index_name = self.collection_name # assign before use + index_name = self.collection_name indices = self.nd.list_indexes().get("indices", []) index_names = [index["name"] for index in indices] if indices else [] try: @@ -84,7 +306,6 @@ def __init__( log.info(f"Successfully created and connected to index: '{index_name}'") except Exception as create_error: - # If error is "index already exists", just get the index again if "already exists" in str(create_error).lower() or "conflict" in str(create_error).lower(): log.warning(f"Index '{index_name}' already exists despite previous error. Fetching it again.") self.index = self.nd.get_index(name=index_name) @@ -95,15 +316,12 @@ def __init__( log.error(f"Error accessing or creating Endee index '{index_name}': {e}") raise - - def _create_index(self, dim): try: resp = self.nd.create_index( name=self.collection_name, dimension=dim, space_type=self.space_type, - # use_fp16=self.use_fp16, precision=self.precision, version=self.version, M=self.M, @@ -133,14 +351,7 @@ def case_config_cls(cls, index_type: IndexType | None = None) -> type[DBCaseConf @contextmanager def init(self): try: - # log.info(f"Token: {self.token}") nd = endee.Endee(token=self.token) - # Uncomment below to use base_url - # nd.base_url = "http://10.128.0.3:8080/api/v1" - # nd.set_base_url("http://10.128.0.3:8080/api/v1") - # nd.set_base_url("http://3.85.217.253:80/api/v1") - # nd.set_base_url("http://10.128.0.5:8080/api/v1") - # nd.set_base_url("http://54.89.169.48:80/api/v1") if self.base_url: nd.set_base_url(self.base_url) @@ -152,7 +363,6 @@ def init(self): if hasattr(e, 'response') and e.response is not None: log.error(f"HTTP Status: {e.response.status_code}, Body: {e.response.text}") log.error(f"Error initializing Endee client: {e}") - # log.error("Error initializing Endee client", exc_info=True) raise finally: pass @@ -160,50 +370,104 @@ def init(self): def optimize(self, data_size: int | None = None): pass + # def prepare_filter(self, filters: Filter): + # """ + # Prepare filter expression for Endee based on filter type. + # Similar to Milvus's prepare_filter method. + # """ + # if filters.type == FilterOp.NonFilter: + # self.filter_expr = None + # elif filters.type == FilterOp.NumGE: + # # For numeric >= filter, use $range operator + # # filters.int_value is the minimum value + # self.filter_expr = [{self._scalar_id_field: {"$range": [filters.int_value, float('inf')]}}] + # elif filters.type == FilterOp.StrEqual: + # # For string equality filter, use $eq operator + # self.filter_expr = [{self._scalar_label_field: {"$eq": filters.label_value}}] + # else: + # msg = f"Not support Filter for Endee - {filters}" + # raise ValueError(msg) + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self.filter_expr = None + + elif filters.type == FilterOp.NumGE: + # Endee supports $range, NOT $gte + # Dataset size = 1 million → finite upper bound + self.filter_expr = [ + {self._scalar_id_field: {"$range": [filters.int_value, 1_000_000]}} + ] + + elif filters.type == FilterOp.StrEqual: + self.filter_expr = [ + {self._scalar_label_field: {"$eq": filters.label_value}} + ] + + else: + raise ValueError(f"Not support Filter for Endee - {filters}") + + def insert_embeddings( self, - embeddings: list[list[float]], + embeddings: Iterable[list[float]], metadata: list[int], + labels_data: list[str] | None = None, **kwargs, - ) -> (int, Exception): # type: ignore + ) -> tuple[int, Exception]: + """ + Insert embeddings with filter metadata. + Modified to include filter fields like Milvus does. + """ assert len(embeddings) == len(metadata) insert_count = 0 try: - batch_vectors = [ - { + batch_vectors = [] + for i in range(len(embeddings)): + vector_data = { "id": str(metadata[i]), "vector": embeddings[i], - "meta": {"id": str(metadata[i])} + "meta": {"id": metadata[i]}, # Store id in meta for reference + "filter": { + self._scalar_id_field: metadata[i] # Store id for numeric filtering + } } - for i in range(len(embeddings)) - ] + + # Add label field if using scalar labels + if self.with_scalar_labels and labels_data is not None: + vector_data["filter"][self._scalar_label_field] = labels_data[i] + + batch_vectors.append(vector_data) + self.index.upsert(batch_vectors) insert_count = len(batch_vectors) except Exception as e: - return (insert_count, e) + log.error(f"Failed to insert data: {e}") + return insert_count, e return (len(embeddings), None) def search_embedding( self, query: list[float], - k: int = 30, - filters: dict | None = None, + k: int = 100, + timeout: int | None = None, **kwargs, ) -> list[int]: + """ + Perform a search with filter expression. + Modified to use prepared filter like Milvus does. + """ try: - filter_expr = None - if filters and "id" in filters: - filter_expr = {"id": filters["id"]} - + # print("Filter expression:",self.filter_expr) results = self.index.query( vector=query, top_k=k, - filter=filter_expr, + filter=self.filter_expr, # Use prepared filter expression ef=self.ef_search, include_vectors=False ) + # print("Results:",results) return [int(result["id"]) for result in results] @@ -223,4 +487,4 @@ def describe_index(self) -> dict: return {} except Exception as e: log.warning(f"Error describing Endee index: {e}") - return {} + return {} \ No newline at end of file From d2e895d967e6f3908c2c0e9797a8d2586ba9ab2a Mon Sep 17 00:00:00 2001 From: darshan-endee Date: Thu, 22 Jan 2026 12:54:51 +0530 Subject: [PATCH 16/22] changed --- .gitignore | 7 ++- demo_cli_ommands.txt | 51 ++++++++++++++++++- vectordb_bench/backend/clients/endee/endee.py | 1 + 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index aa3a72f44..fe2e2988a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,9 @@ venv/ .venv/ .idea/ results/ -logs/ \ No newline at end of file +logs/ +neighbors_labels_label_1p.csv +vectordataset/ +vectordataset_label_filter/ +test_query.csv +test_ids.txt \ No newline at end of file diff --git a/demo_cli_ommands.txt b/demo_cli_ommands.txt index 8e7e55d7a..7ecf581ac 100644 --- a/demo_cli_ommands.txt +++ b/demo_cli_ommands.txt @@ -116,7 +116,7 @@ DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectord --version 1 \ --case-type NewIntFilterPerformanceCase \ --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ - --filter-rate 0.01 \ + --filter-rate 0.99 \ --k 30 \ --num-concurrency "8" \ --concurrency-duration 30 \ @@ -150,3 +150,52 @@ DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset_label_fi --load \ --search-concurrent \ --search-serial + + + DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectordbbench endee \ + --token "x6zdfkgq:pd6fnXWS8e2ZkqAUNFBfirYcHqFShXgR:dev" \ + --region india-west-1 \ + --base-url "https://dev.endee.io/api/v1" \ + --index-name test_1M_numfilter_1p \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int8d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset_label_filter" vectordbbench endee \ + --token "x6zdfkgq:pd6fnXWS8e2ZkqAUNFBfirYcHqFShXgR:dev" \ + --region india-west-1 \ + --base-url "https://dev.endee.io/api/v1" \ + --index-name test_1M_labelfilter \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int8d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.01 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial \ No newline at end of file diff --git a/vectordb_bench/backend/clients/endee/endee.py b/vectordb_bench/backend/clients/endee/endee.py index fed841cfb..bedec1a88 100644 --- a/vectordb_bench/backend/clients/endee/endee.py +++ b/vectordb_bench/backend/clients/endee/endee.py @@ -460,6 +460,7 @@ def search_embedding( """ try: # print("Filter expression:",self.filter_expr) + # print("Query:", query) results = self.index.query( vector=query, top_k=k, From 484b8be84109199ece56623cfbcf183d4d5bf50e Mon Sep 17 00:00:00 2001 From: darshan-endee Date: Mon, 23 Feb 2026 06:26:28 +0000 Subject: [PATCH 17/22] filter_parameters_addition_stability_test_function_insert_embeddings --- .gitignore | 6 +- vectordb_bench/backend/clients/endee/cli.py | 9 +- .../backend/clients/endee/config.py | 5 + vectordb_bench/backend/clients/endee/endee.py | 116 ++++++++++++++++-- 4 files changed, 123 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index fe2e2988a..c0f081862 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,8 @@ neighbors_labels_label_1p.csv vectordataset/ vectordataset_label_filter/ test_query.csv -test_ids.txt \ No newline at end of file +test_ids.txt +debug_logs/ +ground_truth_ids.csv +matched_metadata.txt +vectordataset_label/ \ No newline at end of file diff --git a/vectordb_bench/backend/clients/endee/cli.py b/vectordb_bench/backend/clients/endee/cli.py index 57c96560c..e51f605bf 100644 --- a/vectordb_bench/backend/clients/endee/cli.py +++ b/vectordb_bench/backend/clients/endee/cli.py @@ -59,7 +59,14 @@ class EndeeTypedDict(CommonTypedDict): str, click.option("--index-name", type=str, required=True, help="Endee index name (will use a random name if not provided)", show_default=True) ] - + prefilter_cardinality_threshold: Annotated[ + int, + click.option("--prefilter-cardinality-threshold", type=int, default=None, help="Switch to brute-force prefiltering when filter matches ≤N vectors. Range: 1,000–1,000,000 (default: 10,000).", show_default=True) + ] + filter_boost_percentage: Annotated[ + int, + click.option("--filter-boost-percentage", type=int, default=None, help="Expand HNSW candidate pool by N%% when a filter is active to compensate for filtered-out results. Range: 0–100 (default: 0).", show_default=True) + ] @click.command() diff --git a/vectordb_bench/backend/clients/endee/config.py b/vectordb_bench/backend/clients/endee/config.py index aa4bf8c5d..4bb6b50a3 100644 --- a/vectordb_bench/backend/clients/endee/config.py +++ b/vectordb_bench/backend/clients/endee/config.py @@ -16,6 +16,9 @@ class EndeeConfig(DBConfig): ef_search: Optional[int] = 128 # collection_name: str index_name: str + prefilter_cardinality_threshold: Optional[int] = 10000 + filter_boost_percentage: Optional[int] = 0 + def to_dict(self) -> dict: return { @@ -31,4 +34,6 @@ def to_dict(self) -> dict: "ef_search": self.ef_search, # "collection_name": self.collection_name, "index_name": self.index_name, + "prefilter_cardinality_threshold": self.prefilter_cardinality_threshold, + "filter_boost_percentage": self.filter_boost_percentage, } \ No newline at end of file diff --git a/vectordb_bench/backend/clients/endee/endee.py b/vectordb_bench/backend/clients/endee/endee.py index bedec1a88..dae5d1ed3 100644 --- a/vectordb_bench/backend/clients/endee/endee.py +++ b/vectordb_bench/backend/clients/endee/endee.py @@ -225,6 +225,7 @@ # log.warning(f"Error describing Endee index: {e}") # return {} +import time import logging from contextlib import contextmanager from collections.abc import Iterable @@ -274,6 +275,8 @@ def __init__( self.M = db_config.get("m") self.ef_con = db_config.get("ef_con") self.ef_search = db_config.get("ef_search") + self.prefilter_cardinality_threshold = db_config.get("prefilter_cardinality_threshold") + self.filter_boost_percentage = db_config.get("filter_boost_percentage") self.with_scalar_labels = with_scalar_labels # Initialize filter expression (similar to Milvus) @@ -407,6 +410,91 @@ def prepare_filter(self, filters: Filter): raise ValueError(f"Not support Filter for Endee - {filters}") + # def insert_embeddings( + # self, + # embeddings: Iterable[list[float]], + # metadata: list[int], + # labels_data: list[str] | None = None, + # **kwargs, + # ) -> tuple[int, Exception]: + # """ + # Insert embeddings with filter metadata. + # Modified to include filter fields like Milvus does. + # """ + # assert len(embeddings) == len(metadata) + # insert_count = 0 + # try: + # batch_vectors = [] + # for i in range(len(embeddings)): + # vector_data = { + # "id": str(metadata[i]), + # "vector": embeddings[i], + # "meta": {"id": metadata[i]}, # Store id in meta for reference + # "filter": { + # self._scalar_id_field: metadata[i] # Store id for numeric filtering + # } + # } + + # # Add label field if using scalar labels + # if self.with_scalar_labels and labels_data is not None: + # vector_data["filter"][self._scalar_label_field] = labels_data[i] + + # batch_vectors.append(vector_data) + + # self.index.upsert(batch_vectors) + # insert_count = len(batch_vectors) + + # except Exception as e: + # log.error(f"Failed to insert data: {e}") + # return insert_count, e + + # return (len(embeddings), None) + + + # def insert_embeddings( + # self, + # embeddings: Iterable[list[float]], + # metadata: list[int], + # labels_data: list[str] | None = None, + # **kwargs, + # ) -> tuple[int, Exception]: + # """ + # Insert embeddings with filter metadata. + # Modified to include filter fields like Milvus does. + # """ + # assert len(embeddings) == len(metadata) + # insert_count = 0 + # try: + # batch_vectors = [] + # for i in range(len(embeddings)): + # if labels_data is None or labels_data[i] != "label_1p": + # continue + # vector_data = { + # "id": str(metadata[i]), + # "vector": embeddings[i], + # "meta": {"id": metadata[i]}, # Store id in meta for reference + # "filter": { + # self._scalar_id_field: metadata[i] # Store id for numeric filtering + # } + # } + + # # Add label field if using scalar labels + # if self.with_scalar_labels and labels_data is not None: + # vector_data["filter"][self._scalar_label_field] = labels_data[i] + + # batch_vectors.append(vector_data) + + # if batch_vectors != []: + # self.index.upsert(batch_vectors) + # insert_count = len(batch_vectors) + + # except Exception as e: + # log.error(f"Failed to insert data: {e}") + # return insert_count, e + + # return (len(embeddings), None) + + def insert_embeddings( self, embeddings: Iterable[list[float]], @@ -414,32 +502,36 @@ def insert_embeddings( labels_data: list[str] | None = None, **kwargs, ) -> tuple[int, Exception]: - """ - Insert embeddings with filter metadata. - Modified to include filter fields like Milvus does. - """ assert len(embeddings) == len(metadata) insert_count = 0 try: batch_vectors = [] for i in range(len(embeddings)): + if metadata[i] > 9999 or metadata[i] < 0: + continue vector_data = { "id": str(metadata[i]), "vector": embeddings[i], - "meta": {"id": metadata[i]}, # Store id in meta for reference + "meta": {"id": metadata[i]}, "filter": { - self._scalar_id_field: metadata[i] # Store id for numeric filtering + self._scalar_id_field: metadata[i] } } - # Add label field if using scalar labels if self.with_scalar_labels and labels_data is not None: vector_data["filter"][self._scalar_label_field] = labels_data[i] batch_vectors.append(vector_data) - - self.index.upsert(batch_vectors) - insert_count = len(batch_vectors) + + # # Log matched metadata IDs to file + # with open("matched_metadata.txt", "a") as f: + # for v in batch_vectors: + # f.write(v["id"] + "\n") + + if batch_vectors != []: + self.index.upsert(batch_vectors) + insert_count = len(batch_vectors) + # time.sleep(20) except Exception as e: log.error(f"Failed to insert data: {e}") @@ -466,7 +558,9 @@ def search_embedding( top_k=k, filter=self.filter_expr, # Use prepared filter expression ef=self.ef_search, - include_vectors=False + include_vectors=False, + prefilter_cardinality_threshold=self.prefilter_cardinality_threshold, + filter_boost_percentage=self.filter_boost_percentage ) # print("Results:",results) From ec026bac4d36733fbe5d643a5d3ff05322bfa24f Mon Sep 17 00:00:00 2001 From: darshan-endee Date: Fri, 27 Feb 2026 08:57:37 +0000 Subject: [PATCH 18/22] scripts addition --- .gitignore | 3 +- demo_cli_ommands.txt | 911 +- intfilterscript.ipynb | 8347 +++++++++++++++++ script.ipynb | 329 + vectordb_bench/backend/clients/endee/endee.py | 137 +- 5 files changed, 9648 insertions(+), 79 deletions(-) create mode 100644 intfilterscript.ipynb create mode 100644 script.ipynb diff --git a/.gitignore b/.gitignore index c0f081862..727820272 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,5 @@ test_ids.txt debug_logs/ ground_truth_ids.csv matched_metadata.txt -vectordataset_label/ \ No newline at end of file +vectordataset_label/ +venv2/ \ No newline at end of file diff --git a/demo_cli_ommands.txt b/demo_cli_ommands.txt index 7ecf581ac..17e427063 100644 --- a/demo_cli_ommands.txt +++ b/demo_cli_ommands.txt @@ -78,11 +78,11 @@ NUM_PER_BATCH=10000 DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench -DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectordbbench endee \ - --token "x6zdfkgq:pd6fnXWS8e2ZkqAUNFBfirYcHqFShXgR:dev" \ +DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ --region india-west-1 \ - --base-url "https://dev.endee.io/api/v1" \ - --index-name test_1M_numfilter_1p \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_vnew \ --task-label "20260107" \ --m 16 \ --ef-con 128 \ @@ -92,7 +92,7 @@ DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectord --version 1 \ --case-type NewIntFilterPerformanceCase \ --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ - --filter-rate 0.01 \ + --filter-rate 0.99 \ --k 30 \ --num-concurrency "8" \ --concurrency-duration 30 \ @@ -102,11 +102,11 @@ DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectord --search-concurrent \ --search-serial - DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectordbbench endee \ - --token "x6zdfkgq:pd6fnXWS8e2ZkqAUNFBfirYcHqFShXgR:dev" \ + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ --region india-west-1 \ - --base-url "https://dev.endee.io/api/v1" \ - --index-name test_1M_numfilter_1p \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_vnew \ --task-label "20260107" \ --m 16 \ --ef-con 128 \ @@ -116,7 +116,7 @@ DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectord --version 1 \ --case-type NewIntFilterPerformanceCase \ --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ - --filter-rate 0.99 \ + --filter-rate 0.01 \ --k 30 \ --num-concurrency "8" \ --concurrency-duration 30 \ @@ -127,11 +127,11 @@ DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectord --search-serial -DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset_label_filter" vectordbbench endee \ - --token "x6zdfkgq:pd6fnXWS8e2ZkqAUNFBfirYcHqFShXgR:dev" \ +DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ --region india-west-1 \ - --base-url "https://dev.endee.io/api/v1" \ - --index-name test_1M_labelfilter \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_vnew \ --task-label "20260107" \ --m 16 \ --ef-con 128 \ @@ -141,7 +141,7 @@ DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset_label_fi --version 1 \ --case-type LabelFilterPerformanceCase \ --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ - --label-percentage 0.01 \ + --label-percentage 0.50 \ --k 30 \ --num-concurrency "8" \ --concurrency-duration 30 \ @@ -176,11 +176,11 @@ DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset_label_fi --search-concurrent \ --search-serial - DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset_label_filter" vectordbbench endee \ - --token "x6zdfkgq:pd6fnXWS8e2ZkqAUNFBfirYcHqFShXgR:dev" \ + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ --region india-west-1 \ - --base-url "https://dev.endee.io/api/v1" \ - --index-name test_1M_labelfilter \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_vnew \ --task-label "20260107" \ --m 16 \ --ef-con 128 \ @@ -190,11 +190,884 @@ DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset_label_fi --version 1 \ --case-type LabelFilterPerformanceCase \ --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.001 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_int16 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.80 \ + --k 100 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_int16 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_int16 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.50 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_int16 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.001 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_int16_graphbackfill \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 100 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_int16_graphbackfill \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.80 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_int16_graphbackfill \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.02 \ + --k 100 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_int16_graphbackfill \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.001 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_int16_vector_cache \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_int16_vector_cache \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_int16_vector_cache \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ --label-percentage 0.01 \ --k 30 \ --num-concurrency "8" \ --concurrency-duration 30 \ --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_int16_vector_cache \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.001 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + + DATASET_LOCAL_DIR="/home/debian/VectorDBBench/vectordataset_normal" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_normal_latest_master \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type Performance768D1M \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/VectorDBBench/vectordataset_normal" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_int16_latest_master \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type Performance768D1M \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/VectorDBBench/vectordataset_normal" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_normal_latest_master \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type Performance768D1M \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "q74ehlt6:2Imt8A5iBRz3XQ4mjdcd1Z5R081GgoYq:dev" \ + --region india-west-1 \ + --base-url "https://dev.endee.io/api/v1" \ + --index-name test_1M_numfilter_int16_dev \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "q74ehlt6:2Imt8A5iBRz3XQ4mjdcd1Z5R081GgoYq:dev" \ + --region india-west-1 \ + --base-url "https://dev.endee.io/api/v1" \ + --index-name test_1M_numfilter_int16_dev \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "q74ehlt6:2Imt8A5iBRz3XQ4mjdcd1Z5R081GgoYq:dev" \ + --region india-west-1 \ + --base-url "https://dev.endee.io/api/v1" \ + --index-name test_1M_labelfilter_int16_dev \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "q74ehlt6:2Imt8A5iBRz3XQ4mjdcd1Z5R081GgoYq:dev" \ + --region india-west-1 \ + --base-url "https://dev.endee.io/api/v1" \ + --index-name test_1M_labelfilter_int16_dev \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.002 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_int16_latest_master \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --prefilter-cardinality-threshold 10000 \ + --filter-boost-percentage 0 \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.01 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_int16_latest_master \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_int16_latest_master \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --prefilter-cardinality-threshold 10000 \ + --filter-boost-percentage 0 \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_int16_latest_master \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --prefilter-cardinality-threshold 10000 \ + --filter-boost-percentage 100 \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.001 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_latest_master_stability_bmw \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_latest_master_stability_bmw \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_dummy \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_latest_master_stability2 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.02 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_latest_master_stability2 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.001 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_latest_master_stability_bmw \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.02 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_latest_master_stability_vaib2 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_latest_master_stability_vaib2 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_reupsertcheck \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.50 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + test_1M_vaib_reupsertcheck_backup + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_reupsertcheck \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_reupsertcheck_new1 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + test_1M_vaib_reupsertcheck_new1 + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_reupsertcheck_new1 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ --skip-drop-old \ --skip-load \ --search-concurrent \ diff --git a/intfilterscript.ipynb b/intfilterscript.ipynb new file mode 100644 index 000000000..c4e00cf8f --- /dev/null +++ b/intfilterscript.ipynb @@ -0,0 +1,8347 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip install pandas pyarrow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip install endee" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Name: endee\n", + "Version: 0.1.13\n", + "Summary: Endee is the Next-Generation Vector Database for Scalable, High-Performance AI\n", + "Home-page: https://endee.io\n", + "Author: Endee Labs\n", + "Author-email: dev@endee.io\n", + "License: \n", + "Location: /home/debian/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages\n", + "Requires: httpx, msgpack, numpy, orjson, pydantic, requests\n", + "Required-by: \n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "pip show endee" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "base_path = \"/home/debian/latest_VDB/VectorDBBench/vectordataset\"\n", + "dataset_folder = \"cohere/cohere_medium_1m\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " id emb\n", + "0 322406 [0.19600096, -0.5270862, -0.29519123, 0.429556...\n", + "1 337610 [0.32829463, -0.055560444, -0.06252914, -0.101...\n", + "2 714728 [-0.054155815, 0.009554057, 0.24696879, -0.142...\n", + "3 354737 [0.2501778, 0.017853737, -0.43795395, 0.522256...\n", + "4 290979 [0.3444257, -0.62243223, -0.20940691, -0.08620...\n" + ] + } + ], + "source": [ + "#For checking parquet file contents\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "file_name = \"shuffle_train.parquet\" #input\n", + "\n", + "# Build full path\n", + "file_path = os.path.join(base_path, dataset_folder, file_name)\n", + "\n", + "\n", + "parquet_file = pq.ParquetFile(file_path)\n", + "\n", + "# Read only first batch of rows\n", + "first_batch = next(parquet_file.iter_batches(batch_size=5))\n", + "preview = first_batch.to_pandas()\n", + "\n", + "for col in preview.columns:\n", + " if preview[col].dtype == object and isinstance(preview[col].iloc[0], list):\n", + " preview[col] = preview[col].apply(lambda x: x[:5] if x is not None else x)\n", + "\n", + "print(preview)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from endee import Endee\n", + "client = Endee(token=\"localtest\")\n", + "client.set_base_url(\"http://148.113.54.173:8080/api/v1\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "test_1M_labelfilter_int16\n", + "1000000\n", + "\t\n", + "test_1M_labelfilter_int16_latest_master\n", + "1000000\n", + "\t\n", + "test_1M_labelfilter_latest_master_stability1\n", + "1000000\n", + "\t\n", + "test_1M_labelfilter_latest_master_stability2\n", + "990000\n", + "\t\n", + "test_1M_normal\n", + "1000000\n", + "\t\n", + "test_1M_normal_latest_master\n", + "1000000\n", + "\t\n", + "test_1M_numfilter_int16\n", + "1000000\n", + "\t\n", + "test_1M_numfilter_int16_latest_master\n", + "1000000\n", + "\t\n", + "test_1M_numfilter_latest_master_stability1\n", + "1000000\n", + "\t\n", + "test_1M_numfilter_latest_master_stability2\n", + "1000000\n", + "\t\n", + "test_1M_numfilter_latest_master_stability3\n", + "990000\n", + "\t\n", + "test_1M_numfilter_latest_master_stability_bmw\n", + "1000000\n", + "\t\n", + "test_1M_numfilter_latest_master_stability_vaib\n", + "1000000\n", + "\t\n", + "test_1M_numfilter_latest_master_stability_vaib2\n", + "200000\n", + "\t\n", + "test_1M_vaib_reupsertcheck\n", + "300000\n", + "\t\n", + "test_1M_vaib_reupsertcheck_new1\n", + "262392\n", + "\t\n" + ] + } + ], + "source": [ + "#For checking indexes\n", + "for obj in client.list_indexes().get('indexes'):\n", + " print(obj['name'])\n", + " print(obj['total_elements'])\n", + " print('\\t')" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "#give the index name\n", + "index_name = \"test_1M_vaib_reupsertcheck_new1\"\n", + "index = client.get_index(index_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_reupsertcheck_new1',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 262392,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Finding all query IDs with recall < 1.0 ...\n", + "\n", + "Query ID: 0\n", + "Returned IDs: 992169,992979,997002,993717,997637,997918,997822,993288,999453,992790,995401,994922,993289,994819,990399,993738,995994,998530,996438,991266,993032,999587,994334,991150,994964,994409,995701,995708,994804,999450\n", + "Ground Truth: 995888,992169,992979,997002,993717,991151,997637,997918,997822,993288,992933,999453,992790,995401,994922,993289,997852,994819,990399,993478,993738,995994,996777,998530,996438,991266,993412,990337,993032,999587\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 1\n", + "Returned IDs: 997420,995382,990103,993165,996814,996219,996878,992878,998104,994051,994199,990678,995438,992193,994119,997997,992780,993088,997331,992731,996811,998691,999191,991031,995783,999244,993324,996121,995562,993929\n", + "Ground Truth: 997420,995382,990103,993165,996814,996219,996878,992878,998104,994051,994199,990678,996062,995438,993727,992193,994119,997997,999441,996830,992780,993088,997331,990284,992731,997602,997098,996811,998691,999191\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 2\n", + "Returned IDs: 996312,991260,995701,992330,996592,993492,997913,992546,997976,990231,990949,994100,996881,992692,992969,993853,996267,994778,991045,994757,992931,990957,992961,990535,995793,997371,990588,996075,997771,995094\n", + "Ground Truth: 996312,991260,995701,992330,996592,993492,997449,997913,992546,997976,990231,990936,993517,990949,995014,994100,996881,992692,992969,993853,993005,996267,994778,991045,994757,992931,990957,999647,992961,990109\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 3\n", + "Returned IDs: 991304,990167,999555,992444,990627,995610,999994,999033,990194,990437,990434,996209,991798,997084,999131,994355,997086,993076,991667,992892,992517,990394,998335,997798,999280,994564,993480,992628,994836,992778\n", + "Ground Truth: 991304,990167,999555,992444,990627,995610,997415,999994,999033,990194,990437,990434,996209,991798,998279,997084,999131,994355,997086,993076,991667,992892,992517,990394,998335,994725,997798,999280,994564,993480\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 4\n", + "Returned IDs: 993262,991098,996599,992247,997534,990818,990802,992701,995626,999466,991236,997442,998303,991037,996216,997897,994310,995516,997825,999435,992573,990630,994098,997334,991212,993124,997165,994247,991148,993204\n", + "Ground Truth: 993262,992665,991098,996599,992247,997534,990818,990802,992701,995626,990402,999466,991236,997442,998303,991037,996216,997897,994310,998299,997825,995516,999435,992573,990630,994098,997334,991212,995991,994964\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 5\n", + "Returned IDs: 995274,997320,998779,992994,999144,992630,999026,998184,998365,995755,998951,999090,994100,993412,994964,998862,994757,997976,995336,990949,994730,994894,995438,997737,991790,999089,993529,997832,995824,993484\n", + "Ground Truth: 995274,997320,998779,992994,999144,992630,999026,998184,998365,995755,998951,999090,999647,994100,993412,996959,998633,997925,994964,995701,998862,994757,995702,997976,995336,990949,997377,993648,994730,994894\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 6\n", + "Returned IDs: 992384,995274,995204,993909,991743,992994,998184,999374,997645,990862,998500,995928,992582,990432,992147,995702,994850,998458,996521,992630,992637,998779,997301,996959,997192,994380,991585,997737,994197,995260\n", + "Ground Truth: 992384,995274,993648,995204,993909,991602,991743,993089,992994,998184,991866,993837,999374,997645,993412,990862,998500,995928,999533,992582,994636,990432,992147,995702,994850,998458,996521,992630,992637,991151\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 7\n", + "Returned IDs: 990956,992052,991185,993948,991451,995184,994488,991981,994742,997897,996277,996911,991111,995047,993991,994185,992973,995408,991769,999767,992595,994743,990336,994280,996328,995873,990887,998045,996657,995251\n", + "Ground Truth: 990956,992052,991185,993948,991451,995184,994488,991981,994742,997897,996220,996277,996911,991111,995047,993991,994185,992973,995408,991769,997059,999767,992595,990364,994743,990336,994280,996328,994134,996043\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 8\n", + "Returned IDs: 997913,998090,990949,990910,997320,995961,999710,997301,990231,996918,990560,996805,993504,998950,999639,998337,995336,997371,999806,994850,992748,994100,993529,998365,993696,991498,990957,992944,995802,999331\n", + "Ground Truth: 997913,998090,990949,990910,997320,995961,999710,997301,990231,996918,993412,990560,996805,993504,993515,998950,992277,999639,998337,995336,997371,999806,994850,992748,994100,993529,999900,998365,999338,991498\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 9\n", + "Returned IDs: 995274,998495,995755,992994,990925,992546,994100,994964,994266,990096,997320,999144,993168,991721,998779,996805,996368,996001,998862,993238,997976,993289,990957,992558,998458,990856,995701,994409,999026,993853\n", + "Ground Truth: 995274,998495,995755,992994,990925,993412,992546,994100,993648,994964,994266,990096,997320,999144,993168,991721,998779,996805,996368,996001,999024,998862,993238,997976,993289,990957,992558,998458,990856,994382\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 10\n", + "Returned IDs: 991067,996273,999530,993530,999193,998316,997057,994677,993223,994301,991190,997723,992545,998650,992493,999961,995963,997675,995723,997824,995685,993455,996046,990930,999646,991052,994948,996139,995507,995101\n", + "Ground Truth: 991067,996273,999530,993530,999193,998316,997057,994677,993223,994301,991190,997723,992545,998650,992493,999961,995963,997675,995723,997098,997824,992582,995685,993455,996046,990930,999646,991052,997083,991840\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 11\n", + "Returned IDs: 995459,992824,997775,990084,996742,991432,996380,991817,995503,990192,998754,993077,993891,992615,998061,990720,999639,997547,993583,990538,999564,994270,991272,990273,991871,997055,992095,998308,991070,998142\n", + "Ground Truth: 995459,992824,997775,990084,996742,991432,996380,991817,995503,990192,998754,993077,993891,992615,998061,990720,999639,997547,993583,990538,999564,994270,991272,990273,991064,991871,997055,992095,998308,991070\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 12\n", + "Returned IDs: 993497,993923,992784,992992,999726,992981,992934,992997,998293,990322,992052,996328,994802,998993,990391,992193,994280,997897,994196,999893,990306,991115,991285,995790,995890,990205,995058,996211,997244,991924\n", + "Ground Truth: 993497,990103,993923,992784,992992,999726,992981,992934,992997,998293,990322,992052,996328,994802,998993,990391,992193,994280,997897,994196,999893,997475,990306,991115,991285,995790,995890,990205,995058,996211\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 13\n", + "Returned IDs: \n", + "Ground Truth: 997135,998837,993237,993708,996100,994534,998057,997255,990043,990064,997275,996402,995210,999898,996419,993570,991801,992907,999592,990580,991840,995788,992328,994318,995657,993748,994722,991407,991999,998782\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 14\n", + "Returned IDs: 992561,995211,997733,990126,999985,990931,998251,999179,995633,994142,996411,990705,993219,992566,994783,995883,999191,990621,996431,994317,992415,998691,994587,996609,990854,990069,997823,997403,990340,995571\n", + "Ground Truth: 992561,995211,997733,990126,999985,990931,998251,999179,995633,994142,996411,990705,993219,992566,994783,998870,995883,999191,990621,996431,994317,992415,998691,994587,993775,996609,990854,990069,999352,997823\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 15\n", + "Returned IDs: \n", + "Ground Truth: 996960,995695,991755,995853,994280,998831,993805,997666,991459,992021,997506,998623,992429,998934,994531,990999,994105,990844,996328,993014,997790,994680,990479,990573,990998,996001,993848,995963,990771,995652\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 16\n", + "Returned IDs: 994222,996659,999277,995289,997406,995401,995144,998999,998045,991449,991463,995200,999770,996867,993433,995359,998530,990940,997723,992160,998297,992947,996302,991942,996769,993649,994409,992125,997319,998776\n", + "Ground Truth: 994222,996659,999277,995289,997406,996864,996015,995401,995144,998999,998045,991449,991463,995200,999770,996867,993433,995359,998530,990940,997723,999587,997473,992160,998297,992947,996302,991942,996769,993649\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 17\n", + "Returned IDs: 995886,991743,995893,998921,996351,994982,995515,991692,997218,996762,995968,990215,990561,999698,993250,994334,995154,997301,997002,993143,999075,990995,997571,993820,993237,998908,992384,997320,996903,992790\n", + "Ground Truth: 995886,991743,995893,998921,994850,996351,994982,995515,991692,997218,996762,994810,995968,990215,990561,999698,993250,994334,995154,999823,998445,997301,996438,997002,993143,999075,990995,997571,997370,993820\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 18\n", + "Returned IDs: 991640,996209,999909,999555,996566,997666,999576,990323,998006,997242,997685,995385,996733,995344,993513,991667,990173,993508,991385,995214,994727,994467,999246,991596,991093,994335,990627,996809,994181,990194\n", + "Ground Truth: 991640,996209,999909,999555,996566,997666,996702,999576,990323,998006,997242,997685,995385,996733,995344,993513,996454,991667,990173,990712,993508,991385,995214,994727,994467,999246,991596,991093,994335,990627\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 19\n", + "Returned IDs: 997370,993018,991437,997320,996001,996309,999586,998050,996810,993642,990532,990843,993840,994081,991943,991002,991787,998495,996219,990772,994964,994168,990974,993484,999977,998779,992792,999639,993053,993696\n", + "Ground Truth: 990338,997370,995899,993018,991437,997320,996001,996309,999586,998050,991224,996810,993642,990532,990843,993840,994081,991943,991002,991787,998495,996219,990772,994964,992655,991019,994168,995283,999561,992583\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 20\n", + "Returned IDs: 994351,996495,992237,995754,996809,991383,999435,992726,990194,994098,996428,995214,996774,997725,996328,990173,998335,996839,992193,997967,997331,999430,993224,994481,990705,990594,999832,999752,991723,990712\n", + "Ground Truth: 994351,990402,996495,992237,995754,996809,991383,991025,999435,992726,990194,994098,996428,995214,996774,997725,996328,990173,998335,996839,992193,997967,997331,999430,993224,994481,990705,990408,990594,999832\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 21\n", + "Returned IDs: 993018,997320,994171,993662,997913,990843,996075,994604,995596,995266,991142,997470,996296,990957,999077,998540,997457,993696,991999,997707,996469,992733,999784,999806,998950,990778,994440,995365,999426,999109\n", + "Ground Truth: 993018,997320,994171,993662,995351,997913,990843,996075,994604,995596,995266,994746,994366,991142,997470,996296,994341,990957,999077,998540,997457,993696,991999,997707,996469,995560,992733,999784,992277,997022\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 22\n", + "Returned IDs: 997290,998557,997517,994110,998543,991334,994893,995030,995159,996809,999924,997947,999788,995948,992566,996747,995190,990814,994422,998913,992289,990722,999942,992588,994541,998340,992364,992077,996302,995200\n", + "Ground Truth: 997290,998557,997517,994110,998543,991334,994893,995030,995159,996809,999924,997947,999788,994062,995948,992566,996747,995190,990814,994422,998913,992289,992468,990722,994824,999942,998194,992588,994541,998340\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 23\n", + "Returned IDs: 996053,990096,997913,997586,994850,995536,998090,995755,995927,998861,994894,999356,999711,997305,994618,998921,992848,997984,998550,994964,998495,992330,997320,992862,990866,999710,993431,990066,992384,996001\n", + "Ground Truth: 996053,990096,997913,997586,994850,995961,995536,998090,996143,995755,995927,998861,994894,999356,999711,993853,997305,994618,998921,992848,997984,992716,998550,994707,991582,995503,994964,998495,992330,997320\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 24\n", + "Returned IDs: 996093,997852,994922,992169,993717,991473,998523,997395,996438,992632,992152,990580,990399,994510,996727,991840,993143,998458,999676,997476,990080,995295,992813,997192,993781,996406,999453,999076,998671,993355\n", + "Ground Truth: 996093,997852,994922,992169,993717,991473,998523,997395,996438,992632,992152,990580,990399,994510,996727,991840,993143,998458,999676,997476,990080,991609,993859,995295,992813,997192,993781,996406,992044,999453\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 25\n", + "Returned IDs: \n", + "Ground Truth: 993412,998045,994419,996977,996818,996360,994057,995923,992129,992089,995702,999940,996769,994549,999450,998906,992413,991958,995289,991585,992660,991996,999530,993199,996867,995547,998545,996242,990975,999868\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 26\n", + "Returned IDs: 990487,993500,991977,990998,994254,992558,991447,993335,998686,992052,994148,996068,990684,998406,993975,992170,991620,994006,993583,998536,995408,992718,996262,998623,992280,993965,994080,993715,993681,994280\n", + "Ground Truth: 990487,993500,991977,990998,994254,992558,991447,993335,998686,992052,994148,996068,990684,998406,993975,992170,991620,994006,993583,998536,995408,998480,992718,996262,992933,992554,998623,992280,993965,994080\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 27\n", + "Returned IDs: 993721,996013,991409,993053,991382,999352,994983,998568,998495,993984,997270,996075,995596,993379,999806,999639,996650,994886,995535,994431,996001,997442,999302,993342,995352,996328,992515,993621,991128,995476\n", + "Ground Truth: 993721,996013,991409,991179,993053,991382,999352,994355,994983,996219,998568,998495,993984,997270,996075,997547,995596,993379,999806,999639,996650,994886,997448,995535,994431,996001,997442,999302,993342,995352\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 28\n", + "Returned IDs: 990615,998937,994703,996840,995788,994216,996915,991687,992662,999676,995515,990475,994601,998244,993495,991840,993190,992835,994612,998523,991485,990965,995080,993905,991392,997403,999613,998139,997220,994033\n", + "Ground Truth: 990615,998937,994703,996840,995788,994216,996915,991687,992662,999676,995515,990475,994601,998244,993495,991840,993190,992835,994612,998523,991485,990965,991499,995080,993905,993028,997496,991392,997403,999613\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 29\n", + "Returned IDs: 998276,993603,994829,997918,997002,995141,996996,996012,993139,996661,993289,991425,990051,996808,997977,992330,993504,994972,996576,994946,998401,992225,995855,997777,992979,990491,999111,994821,996001,996871\n", + "Ground Truth: 998276,993603,994829,997918,997002,995141,996996,992047,996012,993139,996661,993289,991425,997140,990051,996808,997977,992330,993504,992238,994972,996576,994946,998401,992225,994042,995855,997777,992979,990491\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 30\n", + "Returned IDs: 995257,992888,990402,997673,998570,997666,990158,996702,995231,995695,997369,992664,995349,997086,995393,999435,990041,990479,995142,998006,990194,991640,996960,994467,993461,996594,992945,995168,991755,993067\n", + "Ground Truth: 995257,992888,990402,997673,998570,997666,990158,996702,995231,992209,995695,997369,992664,995835,995349,997086,995393,999435,990041,990479,995142,998006,990194,991640,996960,994467,993461,996594,992945,995168\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 31\n", + "Returned IDs: 990811,998593,995526,992334,996298,997732,998039,992684,997969,990089,996834,990529,993642,994964,994783,995126,995679,997410,996001,999099,994925,992120,996873,993831,994293,999302,998382,990576,995861,993561\n", + "Ground Truth: 990811,998593,995526,992334,996298,997732,998039,992684,997969,997112,990089,996834,990529,993642,994964,994783,995126,995679,997410,995906,996001,999099,994925,992120,996873,993831,994293,999302,998382,991200\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 32\n", + "Returned IDs: 993781,999077,996592,998123,997918,993260,994552,998950,990580,995357,991791,990043,991157,992571,998180,998951,991015,991435,994964,995336,996196,993512,999144,994972,996658,991550,990339,992692,999676,993453\n", + "Ground Truth: 993781,999077,996592,992637,998123,997918,993260,999701,994552,998950,990580,995357,991791,990043,991157,992571,993633,995584,998180,998951,991015,991435,994964,995336,996196,993512,999144,994972,996658,990885\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 33\n", + "Returned IDs: 995211,994783,994587,994665,998913,995289,992827,995633,997823,992488,992219,993209,998823,996609,997209,995348,998188,999627,999957,993586,991269,991507,999759,992415,993857,997203,993642,995558,998396,992566\n", + "Ground Truth: 995211,994783,994587,994665,998913,995289,992827,995633,997823,992488,992219,993209,998823,996609,997209,995348,998188,999627,999957,993586,991269,991507,999759,992415,993857,997203,995401,993642,995558,998396\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 34\n", + "Returned IDs: \n", + "Ground Truth: 995130,994659,995332,993808,995750,997273,996128,997542,991961,997412,993347,993079,993304,991990,998860,997442,994452,994871,991343,998891,997199,993931,992567,995107,993499,992358,991061,993324,997555,993548\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 35\n", + "Returned IDs: 994397,996075,995274,991422,999289,995336,997571,993168,993280,999596,990585,997976,996108,994432,991125,990862,996714,997304,997377,994894,996368,993167,995502,995555,994100,992880,991721,997015,998779,990990\n", + "Ground Truth: 994397,996075,995274,991422,999289,996976,998011,995336,997571,993168,995374,990701,993280,999596,997759,990585,997976,996108,994432,991125,990862,996714,997304,994916,997377,994894,996368,999437,993167,997925\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 36\n", + "Returned IDs: \n", + "Ground Truth: 993381,996395,991513,998586,992721,999994,991945,992136,991610,990991,990177,997145,999555,990167,990292,991987,993122,997149,997009,995257,997506,998006,996623,995655,991038,995780,996571,990330,992444,991835\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 37\n", + "Returned IDs: 998143,991795,993379,995864,996759,993003,998205,997909,992448,996719,997868,993269,996256,994409,991969,997059,996162,998361,993561,991175,994140,998509,993537,998910,996187,996987,991357,994076,999362,991197\n", + "Ground Truth: 998143,991795,993379,995864,996759,993003,998205,997909,992448,996719,997868,993269,996256,994409,991969,997059,996162,998361,993561,991175,994140,998509,993537,998910,996187,996987,991357,994076,999866,999362\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 38\n", + "Returned IDs: 990210,999374,993860,999331,990545,999210,994964,995438,992147,994336,995928,992944,998302,995927,991743,993018,993484,991801,996805,997320,991498,999898,993877,992169,996219,997002,990083,990096,999122,990843\n", + "Ground Truth: 990210,999374,999338,993860,998160,999331,990545,999210,991720,994964,995438,992147,998294,994336,990648,995928,992944,993154,998302,995927,991743,993018,993484,991801,995075,996805,993238,997320,991498,999898\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 39\n", + "Returned IDs: 993433,997406,998165,995289,992125,993390,996112,996042,994419,992160,992576,996277,997822,991415,998564,996867,995439,992519,992925,996769,998045,997766,991145,992556,996322,997046,994571,990836,998486,995200\n", + "Ground Truth: 993433,997406,998165,995289,992125,993390,992881,996112,996042,994419,995401,992160,992576,996277,997822,991415,998564,996867,995439,992519,992925,996769,998045,997766,991145,996294,992556,996322,997046,994571\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 40\n", + "Returned IDs: 993484,997320,998246,996296,997370,995949,996452,997913,993885,990772,990957,997661,998302,990843,991801,993018,991498,990375,998950,993803,995932,993741,994366,997470,990949,991721,996143,994089,993154,995596\n", + "Ground Truth: 993484,997320,998246,996296,997370,995949,996452,997913,993885,990772,990957,997661,998302,990843,991801,993018,991498,990375,998950,998160,993803,995932,993741,994366,997470,990949,991721,996143,995265,991063\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 41\n", + "Returned IDs: \n", + "Ground Truth: 996395,990194,999555,998006,997205,997242,999994,999576,995167,990167,994467,993106,998122,990360,990230,993381,997149,999435,996209,991148,992771,993122,997086,998066,991767,993224,998429,994796,994638,995848\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 42\n", + "Returned IDs: 998951,993536,992646,993492,999144,992330,992969,992909,995833,994670,995336,992452,999089,993544,990275,996646,997918,998465,992183,998011,997192,999374,992384,997875,994100,996592,995793,994403,998401,994767\n", + "Ground Truth: 998951,993536,992646,993492,999144,992330,992969,992909,995833,994670,995336,992452,999089,993544,990275,996646,997918,998465,995374,992183,990171,998011,997192,999374,992384,997875,994100,996592,999343,992932\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 43\n", + "Returned IDs: 994541,999372,999830,995300,998055,998671,994322,994557,995280,992434,993437,995861,993929,993939,998484,995704,996057,990931,994558,991327,994119,993856,995191,993877,994782,998710,999416,996816,994692,994035\n", + "Ground Truth: 994541,999372,999830,995300,998055,998671,994322,998466,994557,995280,996939,992434,993437,995750,995861,993929,993939,998484,995704,991893,996057,990931,991537,994558,991327,994119,993856,995191,993877,994782\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 44\n", + "Returned IDs: 997320,990375,992041,993018,990843,994171,998950,991498,998246,990949,993785,998540,993484,997305,994081,991650,996075,992880,993260,993154,990083,997470,995438,999701,997370,996296,993504,999453,991860,991999\n", + "Ground Truth: 997320,990375,992041,993018,990843,994171,998950,991498,998246,990949,993785,999338,998540,993484,997305,994081,991650,996075,992880,993260,993154,990083,997470,995438,996143,999701,997370,996296,993504,999453\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 45\n", + "Returned IDs: 996001,990366,995536,998720,999216,992862,998934,993139,996500,993289,993055,993174,990449,994829,994672,997586,990399,993560,995672,995398,994167,999806,999639,997002,998593,995418,990352,990891,999111,993696\n", + "Ground Truth: 996001,990366,995536,998720,999216,992862,998934,993139,996500,993289,993055,993174,990449,994829,994672,997586,990399,993560,997531,995672,995398,994167,999806,999639,997002,998593,995418,990352,992617,990891\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 46\n", + "Returned IDs: 996504,998921,997889,998170,997192,994775,992147,990647,991743,995515,993518,996586,991127,999374,994939,996218,990515,991782,993250,992087,998721,999327,990275,998401,994919,999263,995621,995985,991479,993443\n", + "Ground Truth: 996504,998921,997889,990917,998170,997192,994775,992147,990647,991743,995515,993518,996586,991127,999374,994939,996218,990515,992379,991782,993250,992087,998721,999327,990275,992789,998401,994919,999263,995415\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 47\n", + "Returned IDs: 991403,999595,990325,993425,994446,999277,995144,997766,990751,997406,998999,990116,994222,995940,996302,995701,991478,998530,995401,997723,991630,991942,991603,996388,995478,995182,998045,995665,990242,999977\n", + "Ground Truth: 991403,999228,999595,990325,993425,994446,999277,995144,996015,993137,997766,990751,997406,998999,996864,990116,994222,995940,996302,995701,991478,998530,995401,997723,991630,991942,991603,995994,996388,995478\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 48\n", + "Returned IDs: 994925,993561,995895,991958,998382,998495,992448,993906,998593,993053,992521,999294,996219,998106,993637,997386,996001,992847,998649,994742,994419,995200,996397,998039,990435,997822,997529,996042,990096,992944\n", + "Ground Truth: 994925,995873,993561,995895,991958,998382,998495,992448,993906,998593,993053,992521,992120,999294,996219,998106,993637,994657,997386,996001,992847,998649,994742,997379,994419,995200,995899,996397,995401,998039\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 49\n", + "Returned IDs: 996523,992558,995552,990989,995599,995972,992411,997054,993534,999715,991550,996606,999860,996196,999564,993427,997523,990279,997681,998220,996655,994384,997547,990181,998417,993792,999540,993474,991953,997739\n", + "Ground Truth: 996523,992558,995552,990989,995599,995972,992411,997054,993534,999715,991550,996606,999860,996196,999564,993427,997523,990279,994006,997681,994404,998220,996655,994384,997547,990181,991595,998417,992141,993792\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 50\n", + "Returned IDs: 991081,992664,998076,993715,998716,994131,997230,993917,997197,997097,998949,996858,991663,992495,991684,996021,998440,990622,992735,998030,996183,998446,999139,991524,996986,995735,997673,993366,998812,993335\n", + "Ground Truth: 991081,992664,998076,993715,998716,994131,997230,993917,997197,997097,998949,996858,991663,992495,991684,996021,998440,990622,992735,998030,996183,998446,994918,999139,991524,998845,998680,991021,996986,995735\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 51\n", + "Returned IDs: 998794,993984,993184,994654,996739,998498,991877,999323,995596,994458,994886,993721,997661,994985,996219,995352,991409,997270,999722,993815,998588,994440,996936,994873,996616,999853,999430,999473,999551,997555\n", + "Ground Truth: 998794,994452,993984,993184,994654,996739,998498,991877,990780,999323,995596,994458,994886,993721,997661,994985,996219,995352,991409,997270,999722,993815,998588,990482,994440,996936,995622,998580,994873,996616\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 52\n", + "Returned IDs: 990096,998934,996001,998720,997906,998697,993174,998861,994167,997685,996164,993275,993154,996124,995536,999736,995203,992429,999639,998961,994718,993744,999831,999367,990991,996230,999499,998593,996893,995780\n", + "Ground Truth: 990096,998934,996001,998720,997906,998697,993174,998861,995855,994167,997685,996164,993275,993154,999343,996124,995536,999736,996894,999647,995203,992429,999639,999882,995401,998961,998514,994718,993744,999831\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 53\n", + "Returned IDs: 996436,992912,997640,990797,998779,991750,997355,993412,999144,993526,993725,995755,999740,997256,994990,995923,994436,994894,990388,999026,994964,996714,994838,999021,991173,997192,999916,990845,999004,995701\n", + "Ground Truth: 996436,992912,997640,990797,998779,991750,997355,993412,999144,991651,999587,993526,993725,995755,999740,997256,994990,994916,995923,994436,994894,990535,990388,999026,994964,996714,994838,999021,991173,999344\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 54\n", + "Returned IDs: \n", + "Ground Truth: 997320,999639,993738,990930,990532,995873,997822,996371,995395,996714,992546,990878,993293,992977,992521,991721,995375,999026,999527,990634,996777,990814,994140,991349,990585,992318,991224,995274,995813,993561\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 55\n", + "Returned IDs: 996627,992079,992680,994751,993205,996411,990528,993725,999302,999708,999740,991382,997933,992230,994783,995633,998055,990985,991669,992652,995456,995786,993053,991464,999126,995756,995076,994472,994567,993240\n", + "Ground Truth: 996627,992079,992680,994751,993205,996411,990528,997207,993725,999302,999708,999740,991382,997933,990743,992230,994783,995633,998055,990985,991669,991537,992652,995456,995786,993053,991464,999126,995756,995076\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 56\n", + "Returned IDs: \n", + "Ground Truth: 998416,991996,997014,997920,998221,997804,997900,995786,990126,994964,993378,997403,998495,991377,996305,990931,997477,991565,994466,992446,993199,998816,990158,996219,991179,997822,994419,993561,997586,993444\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 57\n", + "Returned IDs: 994724,998257,995207,999834,995936,998078,991987,992406,990450,996488,991513,995257,992856,999808,998959,996393,990657,998918,999994,992762,998005,992760,997801,995056,993115,999555,999117,993332,992623,990777\n", + "Ground Truth: 994724,998257,995207,999834,995936,998078,991987,992406,990450,996488,991513,995257,993381,992856,999808,998959,993884,996393,990657,998918,999994,992762,998005,992760,997801,996395,995056,993115,999555,999117\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 58\n", + "Returned IDs: 996196,990159,996690,997224,996523,994384,993670,994372,998286,990303,990824,996655,995503,991550,995290,990989,999860,994950,998411,994964,994912,993158,990538,995599,991458,995410,998697,991791,999716,998230\n", + "Ground Truth: 996196,990159,996690,997224,996523,994384,993670,994372,998286,990303,990824,996655,995503,991550,995290,990989,999860,994950,996744,998411,994964,994912,993158,990538,995599,991458,995410,998697,991791,999716\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 59\n", + "Returned IDs: 991451,991650,999698,995357,993721,993215,997002,994226,998951,993154,991997,992557,998961,992880,994449,990532,997918,996685,990083,994081,996178,994743,993018,995401,991535,999453,999831,999442,997571,995372\n", + "Ground Truth: 996280,991451,995374,991650,999698,995357,993721,994030,993215,997002,994226,998951,993154,991545,995181,991997,992557,999900,994850,998961,992880,994449,990532,997448,997918,994362,996685,990083,994081,993512\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 60\n", + "Returned IDs: 998224,999496,995831,995196,995407,994956,999138,994817,999244,990678,998779,995295,998691,996328,995861,993725,991319,991378,997732,992713,993678,996918,997714,995020,998067,994587,991192,990392,994119,994765\n", + "Ground Truth: 998224,999496,995831,995196,995407,994956,995087,999138,994817,999244,990678,998779,995295,998021,998691,996328,995861,993725,991390,991319,991378,997732,992713,992703,993678,996918,997714,995020,998067,994587\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 61\n", + "Returned IDs: 993738,992651,998831,995701,995184,991836,998697,997059,998060,993280,995079,995873,996828,992609,991530,996657,992293,995640,993948,991175,991007,997897,995216,994488,996001,990005,997554,992052,991576,998549\n", + "Ground Truth: 993738,992651,998831,995701,995184,991836,998697,996534,997059,998060,993280,995079,995873,996828,992609,991530,996657,992293,995640,993948,991175,991007,997897,995216,994488,996001,994163,990005,997554,992052\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 62\n", + "Returned IDs: 991157,999122,991015,998607,997305,992718,997192,993143,998180,996193,998123,995515,994046,992052,993260,998170,996504,992147,998950,991848,993077,998921,995408,995372,998106,993609,995154,999676,993781,998827\n", + "Ground Truth: 991157,999122,991015,997305,998607,991720,992718,997192,993143,998180,996193,998123,995515,994046,992052,997496,993260,998045,998170,996504,992147,998950,991848,995401,993077,998921,995408,995864,995372,998106\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 63\n", + "Returned IDs: 995059,990264,990516,991590,991960,998999,992415,995020,998525,993293,993079,990521,994435,995359,993180,992171,999981,995421,997319,994697,997718,990610,994349,992239,994409,991245,991867,991775,998961,994040\n", + "Ground Truth: 995059,990264,990516,991590,991960,998999,992415,995020,998525,993293,993079,990521,992318,994435,995359,993180,992171,999981,995421,997319,994817,994697,997718,990610,994349,992239,994409,991245,991867,991775\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 64\n", + "Returned IDs: 996184,999634,992754,996357,991089,991005,999722,993335,994958,995191,996794,995131,998794,995880,994370,992628,991382,995313,993479,998716,994769,998192,993073,998949,996153,999430,997331,996016,991554,999920\n", + "Ground Truth: 996184,999634,992754,996357,991089,991005,999722,993335,994958,995191,996794,995131,998794,995880,994370,992628,991382,995313,993479,998716,990402,994769,998192,993073,998949,996153,999430,997331,996016,991554\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 65\n", + "Returned IDs: 996809,998695,991571,993617,998790,999009,998786,991890,995461,990288,996370,999711,999430,998849,990538,995375,999194,990489,994451,997821,993515,995401,990930,996830,995560,991532,994643,993696,995559,996284\n", + "Ground Truth: 996809,998695,991571,993617,998790,999009,998786,991890,995461,990288,996370,999711,999430,991139,998849,990361,990538,995375,999194,990489,991052,994451,997821,993515,995401,992452,990930,996830,995560,991532\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 66\n", + "Returned IDs: 999526,997970,991245,999281,991629,997550,994016,997584,996075,993715,992351,992825,995375,995740,992747,993504,995372,998097,999724,993609,994531,996001,992052,992636,990096,992521,999338,999840,995203,994233\n", + "Ground Truth: 999526,997970,991245,999281,991629,997550,994016,997584,996075,993715,992351,992825,995375,995740,992747,993504,995372,998097,997691,999724,993609,992933,994531,996001,995727,994610,992052,992636,990096,992521\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 67\n", + "Returned IDs: 992664,994131,998446,998030,998716,991684,991486,993931,997453,996147,991663,998416,991081,996328,998949,996733,998440,993335,998536,998903,996830,998076,997073,993917,997944,991961,992408,993715,996858,990712\n", + "Ground Truth: 992664,994131,998446,998030,998716,997097,991684,991486,993931,997453,996147,991663,998416,991081,996328,998845,998949,996733,998440,993335,998536,998903,992780,996830,998076,997073,993917,997944,996028,991961\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 68\n", + "Returned IDs: 997547,992736,999908,998933,990538,995737,992824,994165,993706,997215,992170,990273,991070,995372,990192,997914,997775,999639,991432,991272,990279,995459,996196,993055,993611,993077,990885,996380,990930,997614\n", + "Ground Truth: 997547,992736,999908,998933,990538,995737,992824,994165,993706,997215,992170,990273,991070,995372,990192,997914,997775,999639,991432,997506,991272,990279,995459,996196,993055,993611,993077,997301,997821,990885\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 69\n", + "Returned IDs: 994100,993492,991673,996805,992330,995336,998495,995833,995968,990231,998921,992944,993512,995357,995748,996001,995274,998090,993154,995536,996219,997605,995200,997918,996790,996592,990957,999544,992443,992318\n", + "Ground Truth: 994100,993492,991673,996805,992330,995336,998495,994334,995833,995968,990231,998921,992944,990271,996769,993512,995357,995748,996001,995274,998090,993154,994088,995536,995826,996282,995162,995401,990648,996219\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 70\n", + "Returned IDs: 990086,998859,995851,996991,998748,997031,993908,998181,999805,990903,995268,997185,995778,994991,995106,993320,993466,998980,994945,991815,990435,999402,992805,990751,993883,998073,994759,995086,995324,998067\n", + "Ground Truth: 990086,992540,998859,995851,996991,998748,997031,993908,998181,999805,990903,995268,997185,995778,994991,995106,993320,993466,998980,994945,991815,990435,999402,992805,990751,993883,998073,994759,995086,995324\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 71\n", + "Returned IDs: 995841,995375,995372,992944,999711,996318,990043,994972,990585,997301,996193,999544,991977,997320,995560,996001,998180,995268,998097,995138,996609,992384,995515,991743,996219,992047,998401,993324,998921,990334\n", + "Ground Truth: 995841,995375,995372,992944,999711,996318,990538,990043,994972,998726,991052,995401,990585,997301,998445,996193,999544,991977,991540,997320,995560,996001,998180,995268,998097,995138,996609,992384,993533,999526\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 72\n", + "Returned IDs: 993738,995395,992790,990930,993409,997928,997320,991576,991349,997027,990784,996714,995007,992036,997218,997675,997168,996247,995701,998132,996918,994488,995521,992179,990561,998303,994170,994906,998723,995268\n", + "Ground Truth: 993738,995395,992790,990930,993409,997928,997320,991576,991349,997027,990006,990784,996714,995007,999820,992036,997218,995375,997675,997168,996247,995701,998132,995807,996918,994488,995521,992179,990561,998303\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 73\n", + "Returned IDs: 993113,994817,992832,991629,999827,993324,992636,994119,993917,995332,993533,997357,990993,990889,990392,996986,990814,992567,997309,993715,992026,990289,995746,993456,993304,999796,999791,992764,993742,995012\n", + "Ground Truth: 993113,996810,994817,992832,991629,999827,990468,999942,993324,992636,994119,993917,995332,993533,997357,990993,990889,990392,996986,990814,992567,997309,993715,992026,990289,995970,998848,995746,993456,993304\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 74\n", + "Returned IDs: \n", + "Ground Truth: 994102,994831,998764,995372,997913,991362,999982,991335,995374,998702,992880,992291,990585,993852,996143,995828,990538,990724,992748,994618,990949,997470,992114,990957,990275,996959,998500,998744,997192,996785\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 75\n", + "Returned IDs: 998981,998287,997320,998246,994225,996296,994889,997178,994450,991155,990020,992880,994214,994249,997470,998540,993484,992748,994542,993529,996075,996085,994416,990742,994964,995400,998550,993662,992811,990532\n", + "Ground Truth: 998981,998287,997320,998246,994225,996296,992902,994889,997178,994450,991155,990020,994214,992880,994249,997470,995116,998540,993484,992748,994542,998634,993529,996075,996085,995700,994416,999132,990742,994964\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 76\n", + "Returned IDs: 991212,995167,990557,993855,996001,994351,999880,990712,995214,996328,990752,993931,999435,993715,994280,990705,990314,999502,997897,997790,992864,993181,995873,991257,997002,991605,992726,991098,991184,999139\n", + "Ground Truth: 991212,995167,990557,993855,993981,996001,994351,999880,990712,995214,996328,990752,993931,997860,999435,993715,994280,990705,990314,999502,997897,997790,992864,993181,995873,991257,997002,991605,992726,991098\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 77\n", + "Returned IDs: 992713,998671,999830,994541,998279,990931,998256,994142,990705,991319,996189,990364,996328,992445,991327,991058,994891,998543,999191,994035,996816,991682,995191,996647,990712,997967,994098,999179,999775,999458\n", + "Ground Truth: 997669,998466,992713,998671,999830,994541,998279,990931,998256,994142,990705,991319,996189,990364,996328,992445,991327,991058,994891,998543,999191,994035,996816,997933,991682,995191,992780,996647,990712,997967\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 78\n", + "Returned IDs: 995618,990124,996158,996367,992306,993534,994398,994028,997505,996953,994080,999908,999022,996585,997467,998357,996606,992092,994760,991977,993427,995965,993583,990308,994679,992558,994011,993681,998652,992857\n", + "Ground Truth: 995618,990124,996158,996367,992306,993534,994398,994028,997505,996953,994080,999908,999022,996585,997467,993323,998357,996606,992092,994760,991977,993427,995965,993583,990308,994679,992558,994011,993491,993681\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 79\n", + "Returned IDs: 992931,997843,992514,996993,991260,998764,997483,995570,993484,991131,997913,990949,997320,996378,998195,999982,992748,998653,992105,992781,991872,992069,991752,995336,995596,995261,993492,994757,992980,993154\n", + "Ground Truth: 992931,997843,992514,990742,996993,991260,998764,997483,995570,990478,992961,993484,991131,997913,990949,997320,996378,998195,999982,992748,999838,999855,998653,992105,992781,991872,992069,995277,991752,995336\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 80\n", + "Returned IDs: 995841,995372,996785,997301,994964,996959,997796,991444,998921,997320,999374,990560,990585,992147,996218,998839,995949,995886,994972,991743,998797,992066,996805,990925,992944,998302,997015,996052,999263,997192\n", + "Ground Truth: 995841,995372,996785,998445,997301,994964,996959,997796,991444,998921,997320,999374,998644,990560,999533,990585,992147,993238,992789,996218,998839,993648,995949,996090,995886,994972,991743,998797,992066,996805\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 81\n", + "Returned IDs: 990146,993237,992085,997747,994318,992944,991727,991392,995443,990142,993253,997591,997585,990742,991121,994789,999784,998782,990822,994534,993570,990615,998550,990447,999676,999948,990650,992218,998139,990648\n", + "Ground Truth: 990146,993237,992085,997747,994318,992944,991727,992342,991392,995443,990142,993253,997591,997855,997585,990742,991121,994789,999784,998782,990822,995334,994534,993570,990615,998550,990447,999676,999948,996352\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 82\n", + "Returned IDs: 998874,996505,992521,996001,995110,997675,998536,995521,996715,990537,995873,990930,993931,996028,997067,996918,998593,993289,998397,997822,995268,992703,995536,996368,992488,992448,994380,996930,995701,992790\n", + "Ground Truth: 998874,996505,992521,995401,996001,995110,997675,998536,995521,996715,990537,995873,990930,995130,993931,996028,997067,996918,991459,998593,993289,998397,997822,995268,992703,995536,991052,996368,992488,992448\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 83\n", + "Returned IDs: 998779,994436,997640,997536,991028,991350,990406,991603,995702,997058,994161,997787,996379,997355,991958,998165,996388,994396,995052,990845,991011,993813,996719,990984,996042,993040,993831,997822,999196,996125\n", + "Ground Truth: 998779,994436,997640,997536,991028,991350,990406,991603,995702,997058,994161,997787,996379,997355,991958,998165,996388,994396,995052,990845,991011,993813,996719,990984,998999,996042,993040,993831,997822,999196\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 84\n", + "Returned IDs: 997913,994964,997371,993237,995515,994778,996727,996811,990142,996840,991727,995928,998359,996001,996438,994277,994415,997593,997291,999426,993143,996592,999290,995886,990580,997320,997370,994360,997301,990066\n", + "Ground Truth: 997913,994964,997371,993237,995515,994778,994475,991801,996727,997350,996811,990142,996840,991727,995928,998359,996001,996438,994277,994415,997593,997291,999426,992140,993143,994971,996592,999290,995886,990580\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 85\n", + "Returned IDs: 995207,991701,996395,998586,991987,996623,992136,995056,999117,995373,994063,990292,994413,998858,998257,994451,998066,991052,999249,994092,999096,997110,998659,992604,993115,997918,999834,994753,992638,999101\n", + "Ground Truth: 995207,991701,996395,998586,991987,996623,992136,995056,999117,995373,994063,990292,994413,998858,998257,994451,998066,991052,994187,999249,994092,995617,999096,997110,998659,992604,993115,997918,999834,994753\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 86\n", + "Returned IDs: 992752,998933,995011,995945,993037,998536,993931,990407,993055,995421,998341,992437,998250,998594,999139,996483,997926,998593,991245,998303,997019,991530,999111,994463,997506,996775,991633,991552,994259,996028\n", + "Ground Truth: 992752,998933,995011,995945,993037,998536,993931,991291,990407,993055,995421,998341,992437,998250,998594,999139,996483,997926,998593,991245,998303,997019,999265,991530,999111,994463,997506,996775,997531,991633\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 87\n", + "Returned IDs: 990146,991498,999263,998302,992994,990560,999533,994964,998644,998921,997301,998445,991444,990958,995306,995771,990925,993885,990096,999374,996785,992147,992546,994281,994922,994978,991406,999197,990224,996504\n", + "Ground Truth: 990146,997876,991498,999263,998302,992994,990560,999533,994964,993484,998644,998921,997301,998445,991444,990958,995306,995771,990925,993885,992814,990096,999374,996785,992147,992546,994281,994922,994978,991406\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 88\n", + "Returned IDs: 992931,995319,995942,997147,991593,997913,990922,992964,991250,993031,990114,993606,999559,995632,998899,996508,999945,994668,992255,993278,995984,990949,992228,998619,991502,990742,994633,997946,991456,995774\n", + "Ground Truth: 992931,995266,995319,994052,995942,997147,991616,991593,992954,997913,990922,992964,991250,993031,990114,993606,990653,996629,999559,995632,998899,996508,999945,994668,992255,993278,998562,995984,990949,992228\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 89\n", + "Returned IDs: 995755,997437,993484,995244,998649,994894,992296,997320,999740,998246,991991,994814,995435,994359,999599,997304,998855,993696,996053,997305,995841,990949,990786,996075,999867,992924,990307,993529,993818,990957\n", + "Ground Truth: 995755,997437,993630,993484,995244,998649,994894,996630,992296,997320,999740,991999,998246,991991,994814,995435,994359,999599,997304,998855,999708,993696,996053,996436,997305,995841,990949,990786,993412,996075\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 90\n", + "Returned IDs: 999451,997739,994852,997903,990376,992558,996693,995983,997301,995580,996894,998686,991923,999807,999639,997106,992383,998600,994195,994979,998079,999561,996262,992423,992718,990383,993283,996196,992521,994163\n", + "Ground Truth: 999451,997739,994852,997903,990376,992558,996693,995983,997301,995580,996894,998686,991923,999807,999639,997106,992383,998600,992612,994195,994979,998079,999561,996262,992423,992718,990383,993283,996196,992521\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 91\n", + "Returned IDs: 998105,994307,993053,997790,991021,993906,996028,995823,996147,990096,995586,995295,993289,992480,995456,998677,999444,994689,996062,996001,991685,994280,992408,990532,992072,993433,998382,993561,994531,993809\n", + "Ground Truth: 998105,994307,993053,997790,991021,993906,996028,995823,991451,996147,990096,995586,995295,993289,992480,995456,995401,998677,999444,994689,996062,996001,991685,994280,993778,992408,990532,992072,993433,998382\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 92\n", + "Returned IDs: 991148,992551,993472,990194,996216,995895,998122,998495,992892,997506,992209,990096,991411,997442,993184,995798,994661,999576,991382,997685,994682,996184,991881,993460,995535,992780,999430,990091,996001,999139\n", + "Ground Truth: 991148,992551,994355,993472,990194,996216,995895,999336,998122,998495,992892,997506,992209,990096,991411,997442,993184,995798,994661,999576,993121,991382,997685,994682,996184,991881,993460,997196,995535,992780\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 93\n", + "Returned IDs: 998779,991743,996727,993412,996976,992087,995274,997640,999026,998649,999074,992511,997355,995034,992685,998862,992147,998495,990406,992630,995755,996918,990949,999179,990606,994894,990957,999021,996890,990925\n", + "Ground Truth: 998779,991743,996727,993412,996976,990401,993673,991268,992087,995274,997640,999026,998649,999074,992511,997355,995034,992685,998862,997429,992147,998495,990406,994301,995616,996438,997799,992637,992630,995755\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 94\n", + "Returned IDs: 993785,997661,998921,992041,995515,995250,999806,995596,998498,992119,993447,990697,997550,997305,996075,992635,991984,991157,995372,994081,996001,993257,997320,993904,995203,996755,999426,993820,992944,996805\n", + "Ground Truth: 993785,997661,998921,992041,995515,995250,999806,995596,998498,992119,993447,990697,997550,997305,996075,992635,993022,991984,991157,992384,995372,994081,996001,993257,995622,997320,993904,995203,996755,999426\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 95\n", + "Returned IDs: 994964,993143,997320,996001,996657,996500,999219,992384,996219,999343,995928,991459,999144,991392,990999,999653,998303,996028,990366,998246,993174,993168,990771,998593,996576,998302,994280,996368,996147,993484\n", + "Ground Truth: 994964,993143,995855,997320,996001,996657,996500,999219,997920,992384,996219,999343,998029,995928,991459,999144,991392,990999,999653,998303,996028,990366,998246,993174,993168,990771,998593,996576,998302,994280\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 96\n", + "Returned IDs: 993741,992041,996909,991650,993785,995596,996741,990697,997836,992671,992635,999426,997661,994971,991372,995515,993904,996075,996875,997305,995372,990142,992733,998227,999439,993022,998498,990914,997291,994415\n", + "Ground Truth: 993741,992041,996909,991650,993785,995596,996741,990697,991218,997836,992671,992635,999426,997661,994971,991372,995515,993904,996075,996875,997305,995372,990142,992733,998227,999439,995995,993022,998498,990914\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 97\n", + "Returned IDs: 995357,991743,996658,994197,997571,998011,995336,994819,995274,993145,991131,991094,996976,996727,998779,990647,998495,999470,991361,992646,996714,999144,994334,994100,992718,990231,999089,991908,999021,999331\n", + "Ground Truth: 995357,995374,991743,996658,994197,997571,998011,995336,994819,995274,993145,991131,991094,996976,996727,998779,990647,998495,999470,991361,993096,992646,996714,995873,999144,994334,994100,992718,990231,999089\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 98\n", + "Returned IDs: \n", + "Ground Truth: 990712,996495,994351,999388,990705,993224,994481,993801,995214,991633,992726,994956,996728,996328,995754,996566,992445,994310,994378,992780,992154,990194,991296,994047,997086,993931,999852,992429,998389,996216\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 99\n", + "Returned IDs: 998697,999129,998536,991530,995536,993609,995672,992862,999639,995780,992268,991332,999806,997506,992617,990399,993055,995562,997775,990096,998303,999744,996001,995237,992423,996769,991157,994167,993342,998045\n", + "Ground Truth: 998697,999129,998536,991530,995536,993609,995672,992862,999639,995780,992268,991332,999806,997506,992617,990399,991630,993055,995562,999587,997775,990096,998303,996368,999744,995401,996001,995864,995237,992167\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 100\n", + "Returned IDs: 996351,998837,994703,998363,991193,999738,993708,994789,995515,993896,995489,991840,990138,991609,993143,999453,991485,999613,998380,992611,993577,994922,990310,993237,991999,995988,995267,995072,999676,995443\n", + "Ground Truth: 996351,998837,994703,998363,991193,999738,993708,994789,995515,997496,993896,992384,995489,991840,990138,990490,991609,993143,990134,999453,991485,999613,990809,998657,998380,994001,992611,993577,994922,991930\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 101\n", + "Returned IDs: 993813,991790,991163,996077,991466,997046,999731,990940,993561,997630,997289,997386,990751,998633,995679,998165,997822,994409,992919,990930,993293,993738,991958,999037,998593,995401,990634,995701,994140,992703\n", + "Ground Truth: 993813,991790,991163,996077,991466,997046,999731,990940,993561,990878,997630,997289,997386,999000,990751,998633,995679,998165,997822,994409,992919,990930,991603,990820,993293,993738,991958,999037,998593,995401\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 102\n", + "Returned IDs: 999639,998793,992824,996196,997775,994270,992306,990538,992802,999908,996742,994560,990084,997547,992030,995631,994305,995955,997215,990720,997864,990755,993077,999564,992190,991432,992458,991910,992558,994614\n", + "Ground Truth: 999639,998793,992824,996196,997775,994270,992306,990538,992802,999908,996742,994560,990084,997547,992030,995631,994305,991923,995955,997215,990720,999451,997864,990755,993077,999564,992190,990409,991432,990146\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 103\n", + "Returned IDs: 998105,999460,995906,990536,994419,997453,996974,990497,996062,998039,993906,993053,994307,992201,995823,996028,996647,991989,992622,993831,992408,991021,993561,992072,991958,990076,995295,997969,991168,990097\n", + "Ground Truth: 998105,999460,995906,990536,994419,997453,996974,990497,999083,996062,998039,993906,993053,994307,992201,995823,996028,996647,991989,992622,993831,992408,991021,993412,993561,992072,991958,990076,995295,997969\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 104\n", + "Returned IDs: 997554,999453,995216,997937,998495,996147,990845,996001,999402,992979,996368,991585,996810,991790,992792,990441,993561,991967,996759,990662,992364,997031,993289,996911,995289,992521,996657,998060,995963,996987\n", + "Ground Truth: 997554,999453,995216,997937,998495,996147,990845,996001,999402,992979,993412,994531,998166,996368,991585,996810,991790,993003,992792,992334,990441,993561,991943,991967,996759,990662,992364,997031,993289,996911\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 105\n", + "Returned IDs: 997913,991090,993834,999982,999247,992228,998500,993048,991626,995802,993796,997483,992066,992742,997320,990650,997470,995513,996959,990182,997759,999710,998090,991814,992443,991229,991250,998808,996785,990910\n", + "Ground Truth: 991873,997913,991090,993834,999982,992822,999247,992096,992228,998500,993048,991626,995802,993796,997483,991616,992066,999165,992742,997320,990650,993616,997470,995513,996959,990182,997321,997759,990588,999710\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 106\n", + "Returned IDs: 997320,992944,990772,991498,996001,999710,990949,991131,996075,992041,993484,995336,998550,995372,992443,997305,992748,992119,994100,996085,993388,996318,994964,990585,998246,990957,993785,999374,997661,994415\n", + "Ground Truth: 997320,992944,990772,991498,996001,999710,990949,991131,996075,992041,993484,998365,995336,998550,995372,992443,997305,992748,992119,994100,992692,998232,996085,993388,996318,994964,994449,990585,998246,990957\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 107\n", + "Returned IDs: 995274,994964,999144,999026,997015,999437,998779,992630,990535,993289,998951,993168,996959,997918,994197,993492,991467,992216,995755,992384,999374,997002,993484,990958,995336,990957,997320,997976,992558,998797\n", + "Ground Truth: 994916,995274,994964,999144,999026,997015,999437,998779,992630,990535,993289,998951,993168,996959,997918,992047,997920,994197,993492,991467,999916,992216,995755,999596,992384,999374,997002,993484,990958,995336\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 108\n", + "Returned IDs: 994379,997319,996411,990223,995611,990675,995819,991810,997822,991943,995289,994382,993831,998156,995456,991350,992561,993561,995633,994783,998750,990889,995020,998765,990277,992582,994742,990516,995702,994283\n", + "Ground Truth: 994379,997319,991973,996411,990223,995611,990675,995819,999627,991810,997822,991943,995289,994382,993909,993831,998156,995456,991350,992561,997732,993561,995633,994783,998750,996607,990889,996906,995020,998765\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 109\n", + "Returned IDs: 999898,997028,996996,991311,994541,993695,997410,997920,993472,990974,997345,991923,998302,994783,999343,998169,998039,998122,993807,995384,996062,996219,990505,997878,991776,992120,996216,995895,997320,999876\n", + "Ground Truth: 999898,997028,996996,991311,994541,993695,997410,997920,993472,990974,997345,991339,991923,998302,994783,996939,999343,998169,998039,998122,990210,992931,993807,995384,999009,996062,996219,990505,995620,997878\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 110\n", + "Returned IDs: 997028,990557,996001,996219,999948,998039,991498,999058,998765,991930,994964,992446,993885,990752,994487,995102,991523,991776,991212,997320,997332,996318,992182,991184,995216,997920,995107,997410,999344,991994\n", + "Ground Truth: 997028,990557,996001,997381,996219,999948,994777,998039,991498,999058,998765,994061,995053,991930,995424,994964,992446,993885,990752,994487,995102,997876,991523,991776,991212,997320,997332,996318,992182,991184\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 111\n", + "Returned IDs: 996328,991882,990133,992450,990531,998067,995326,998610,997529,995295,997860,993939,994310,991769,990712,998710,990027,997103,994830,994742,996301,990435,993079,991885,996001,992703,996752,991585,993856,998671\n", + "Ground Truth: 996328,991882,990133,992450,990531,998067,995326,998610,997529,995295,997860,993939,994310,991769,990712,998710,990027,997103,994830,997933,994742,996301,990435,993079,991885,996001,992703,996752,996611,991585\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 112\n", + "Returned IDs: 995981,992470,992874,992384,995701,990862,996001,998721,993848,999904,998246,991045,990588,998445,995467,994879,998549,993342,999977,998921,995158,992167,997984,991459,997437,991245,996714,995336,995886,996075\n", + "Ground Truth: 993154,995981,992470,992874,992384,995701,995792,990862,996001,998721,993432,993848,999904,998246,991045,990588,998445,995467,994879,998549,993342,999977,998921,995158,997984,992167,991459,997437,991245,996714\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 113\n", + "Returned IDs: 996425,994819,994334,995453,997918,994804,994088,996046,997002,993465,996903,990995,995200,999277,995373,993407,995444,994446,999089,999595,990791,995928,998530,991942,997637,996864,992933,998259,990459,995401\n", + "Ground Truth: 996425,994819,994334,995453,997918,994804,994088,996046,997002,993465,996903,994362,999964,990995,995200,999277,995373,995444,993407,995374,994446,999089,999595,991603,990791,995928,998530,991942,997637,992260\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 114\n", + "Returned IDs: 991721,990096,996237,999444,996368,998362,995016,991154,990856,998039,995899,991869,994964,994925,999781,993168,990930,991958,997320,994961,998862,992994,990925,998950,992546,995701,993389,995678,995126,990811\n", + "Ground Truth: 991721,990636,990096,996237,999444,996368,998362,995016,991154,990856,998039,995899,991869,994964,994925,999781,993168,997053,990930,991958,997320,994961,998862,996661,992994,996677,997920,994140,990925,998950\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 115\n", + "Returned IDs: 995941,991782,996592,999089,993145,997096,995793,995336,990647,999200,990180,998572,991456,990275,990994,994111,999700,991379,995465,995886,999374,997748,994757,997371,997194,997479,997571,997022,996521,995453\n", + "Ground Truth: 995941,991782,996592,999089,993145,997096,995793,995336,990917,990647,999200,990180,998572,991456,990275,990994,994281,994111,999700,991379,995465,995886,999374,997748,990478,994757,997371,997194,997479,997571\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 116\n", + "Returned IDs: 998523,994922,992907,993355,996840,990043,993237,990334,990332,990448,995004,999453,999620,997918,990432,990925,991510,996196,990483,992612,996438,997496,998401,999076,998837,993143,996402,999676,996915,993570\n", + "Ground Truth: 998523,994922,992907,993355,996840,990043,993237,990334,997876,990332,990448,995004,999453,999620,997918,990432,990925,991510,996196,990538,990483,992612,996438,991609,997496,998401,995401,999076,998837,993143\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 117\n", + "Returned IDs: 996194,999601,998823,993014,994541,995695,990521,990169,995421,996396,994119,996429,998934,994280,995401,999496,994051,997316,990011,998250,992052,998968,991245,992313,996633,998817,992437,993335,996435,998067\n", + "Ground Truth: 996194,999601,998823,993014,994541,995695,990521,990169,995421,996396,994119,998348,996429,998934,994280,995401,999496,994051,998546,994817,997316,990011,998250,992052,998968,991245,992313,992131,996633,998817\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 118\n", + "Returned IDs: \n", + "Ground Truth: 995961,994850,994081,997470,998839,997320,998170,990949,991379,997301,994290,995374,990066,995422,998500,990910,994100,991743,993852,994757,995372,993646,998090,999647,998950,990532,995949,992228,996629,997479\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 119\n", + "Returned IDs: 995614,993426,990191,997820,996516,997292,990855,994415,998921,997483,998721,996387,999109,999271,994618,995623,996993,996771,999792,994233,992853,991131,995306,993484,993785,995119,998445,996135,997550,999036\n", + "Ground Truth: 995614,993426,990191,997873,997820,996516,997292,990855,994415,998921,997483,998721,996387,999109,999271,994618,995623,996296,996993,996771,999792,994233,992853,991131,995306,993484,993785,995119,998445,996135\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 120\n", + "Returned IDs: 996658,993288,992933,997391,997309,997918,996282,990956,997575,995080,995421,997897,996633,995862,991666,994455,996867,992231,995357,994409,991950,996878,997125,993023,995159,992052,994073,999303,999660,990521\n", + "Ground Truth: 996658,991864,993288,992933,997391,997309,997918,993094,996282,990956,997575,995080,995421,997897,996633,993211,995862,991303,991666,994455,996867,992231,995357,994409,991950,997462,992779,996878,994300,997125\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 121\n", + "Returned IDs: 996075,992147,999374,995515,991545,991743,992087,991889,994850,999074,996504,994776,997305,993355,990957,990083,994266,997192,995372,997320,996521,992880,991372,998921,997913,995596,994894,998779,999426,995443\n", + "Ground Truth: 996075,992147,999374,995515,991545,991743,992087,990118,991889,994850,999074,996504,994776,997305,991801,993355,990957,999140,990083,994266,998045,996648,997192,995372,993053,995374,997320,995192,993186,996521\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 122\n", + "Returned IDs: 998019,995801,990834,995203,993504,996233,990557,991648,996884,999526,990203,993179,994955,996001,992944,998764,998855,996908,997332,995366,995562,999320,995130,997002,996075,994829,995500,992041,999019,991037\n", + "Ground Truth: 998019,995801,990834,995203,993504,996233,990557,991648,996884,999526,990203,994081,993179,994955,996001,992944,998764,992627,998855,996908,997332,995366,995562,999320,995130,994449,997002,996075,998849,994829\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 123\n", + "Returned IDs: 999902,996326,994467,999383,997618,993931,992209,990364,991515,992104,993965,993275,996124,998051,995408,999079,995630,990998,991685,993609,992407,993583,995780,998536,998006,998855,990784,991303,996209,991755\n", + "Ground Truth: 999902,996326,994467,999383,997618,993931,992209,990364,991515,992104,993965,993275,996124,998051,995408,999079,995630,990998,991685,993609,992407,993583,995780,998536,998006,998855,990784,994770,991303,996209\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 124\n", + "Returned IDs: 995985,990595,998263,992384,998500,993934,996714,998921,997192,997984,995793,995824,990275,991379,995554,997842,992994,995181,992546,996727,999599,991829,999327,992228,991749,995927,995090,996504,995274,997889\n", + "Ground Truth: 995985,990595,998263,992384,998500,993934,996714,998921,997192,997984,995374,995793,995824,990275,991379,993072,995554,997842,992994,995181,992546,996727,999599,991829,999327,992228,991749,995927,995090,996504\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 125\n", + "Returned IDs: 990940,995269,994446,999932,993465,990242,995289,995444,993831,999000,995685,992160,995204,996867,997723,997046,997937,998070,999770,996302,994108,998779,993433,996323,999037,999277,997822,994571,996166,995507\n", + "Ground Truth: 990940,995269,994446,999932,993465,990242,995289,995444,993831,999000,992160,995685,995204,996867,997723,997046,997937,993412,998070,999770,996302,994108,998779,994405,993433,996323,999037,999277,997822,994571\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 126\n", + "Returned IDs: 997550,992635,997584,996075,996387,994618,995515,999426,998498,998921,997301,996516,995596,994971,993447,990697,999453,995372,996741,997984,994415,999526,997227,995033,997470,995119,993785,990855,997320,994233\n", + "Ground Truth: 997550,992635,997584,996075,996387,994618,995515,999426,998498,998921,997301,996516,995596,994971,993447,990697,999453,995250,995372,996741,997448,997984,994415,999526,997227,995033,997470,995119,993785,990855\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 127\n", + "Returned IDs: 998634,996001,997320,998246,993484,992839,991787,993053,994964,993280,991155,997332,999099,990585,990146,993168,993885,991498,999453,991721,991184,996219,990772,990096,993389,991732,991609,992896,990949,998365\n", + "Ground Truth: 998634,996001,997320,998246,993484,992839,991787,993053,994964,993280,991155,994961,997332,999099,990585,990146,993168,993885,991498,995678,999453,991721,991184,996219,990772,990096,993389,993154,990269,991732\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 128\n", + "Returned IDs: 997809,996546,991923,992718,998975,999380,993681,998331,991977,996163,990908,994398,993786,995138,993572,994852,992170,996158,995578,991157,999387,993975,996894,995737,992349,996585,999806,993839,992258,998853\n", + "Ground Truth: 997809,996546,991923,992718,998975,999380,993681,998331,991977,996163,990908,994398,993786,995138,993572,995986,994852,996658,992170,996158,995578,991157,998921,999387,993975,998026,996894,995737,992349,996585\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 129\n", + "Returned IDs: 992255,990650,996876,995794,990114,992822,991873,998947,999948,993083,991616,993048,999110,998434,990742,990066,995109,998540,990699,994461,991396,990311,990030,998899,993616,997539,991900,997320,994934,991593\n", + "Ground Truth: 992255,990650,996876,995794,990114,992964,992822,991873,998947,997451,999948,995485,997102,997948,998003,993083,997966,991616,993048,999110,995367,998434,990742,990066,995109,998540,990699,994461,991396,990311\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 130\n", + "Returned IDs: 993100,998710,995658,990735,992449,994998,994830,994858,999472,992796,993856,997305,997529,990771,998105,992006,994663,999354,994558,993939,994799,996603,993053,994185,999372,995200,991938,999192,999302,994131\n", + "Ground Truth: 993100,998710,995658,990735,992449,994998,994830,994858,999472,992796,993856,999391,997305,997529,990771,998105,990439,992006,994663,996147,990041,999354,994558,995873,993939,994799,998379,996603,993053,994822\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 131\n", + "Returned IDs: 998787,993366,994834,990952,995393,999373,998565,990167,994833,999555,996553,998959,996408,990358,995299,990173,995231,995835,993528,993560,994756,991313,994374,993014,998536,995168,994682,994150,993948,992862\n", + "Ground Truth: 998787,993366,994834,990952,995393,999373,998565,990167,994833,995326,999555,996553,998959,996408,990358,995299,990173,995231,992282,995835,993528,993560,994756,991313,994374,993014,998536,995168,994682,994150\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 132\n", + "Returned IDs: 999236,996716,992790,999820,998723,991011,993281,996371,996067,992179,993504,994946,998303,996062,998401,992576,993071,998045,994113,991857,993979,997331,994188,997002,994964,993492,995289,995589,996198,999898\n", + "Ground Truth: 999236,996716,992790,999820,998723,991011,993281,996371,996067,992179,993504,994946,998303,996062,998401,992576,995401,993071,998045,994113,991857,990294,993979,992367,997331,996220,994188,990244,997002,994964\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 133\n", + "Returned IDs: 999503,997790,993055,999840,994280,997926,993489,995203,992862,997897,992521,990399,990005,992958,998250,997918,998303,991661,994042,991630,994946,991332,998933,992747,996657,998697,996368,993260,996733,990350\n", + "Ground Truth: 999503,997790,993055,999840,994280,997926,993489,991397,995203,992862,992979,997897,992521,990399,990005,992958,998250,990802,997918,998303,991661,994042,994531,991630,994946,991332,998933,992747,996657,993143\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 134\n", + "Returned IDs: 991227,992558,992052,993478,998536,996531,995783,994280,995214,994972,995408,990956,991052,998104,998397,993121,992997,993747,996438,996658,995401,997897,993727,992521,996328,999179,994119,990776,999514,995184\n", + "Ground Truth: 991227,992558,992052,993787,993478,992933,998536,996918,996531,995783,994280,995214,994972,995408,990956,992318,993673,991052,998104,998397,993121,992997,993747,996438,996658,995401,996930,997897,993727,992521\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 135\n", + "Returned IDs: 994406,992767,998431,997391,999960,998779,997634,994474,994671,996727,996903,995374,995453,997012,995034,997157,997640,993412,990647,993289,996172,995893,995369,998529,994092,997192,997702,997462,994794,997995\n", + "Ground Truth: 994406,992767,997799,998431,995403,997391,999960,990337,998779,997634,993026,994474,998080,994671,996727,996903,995374,996047,995453,997012,995034,997157,997754,997640,993412,990647,993289,996172,995893,995607\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 136\n", + "Returned IDs: 991324,999740,992698,993725,990011,990484,991390,994830,995756,990711,992479,997142,990743,994751,998710,995002,991885,996358,996633,997714,992664,991911,995435,992679,992480,996001,997933,999377,996275,990735\n", + "Ground Truth: 991324,999740,992698,993725,990011,990484,991390,994830,995756,990711,992479,997142,990743,994751,998710,990089,995002,991885,999555,996358,999922,996633,997714,992664,991911,995435,992679,990439,992480,996001\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 137\n", + "Returned IDs: 995771,998542,993364,997543,994584,999263,991533,993250,991889,990515,997301,992401,999291,991558,998211,992147,993581,994939,999968,992818,998539,999270,996218,996504,990958,994670,999089,992384,999145,999982\n", + "Ground Truth: 995771,998542,993364,997543,994584,995034,999263,991533,993250,991889,990515,997301,992401,999291,991558,998211,992147,993581,994939,999968,992818,998539,999270,996218,996504,990958,994670,999089,992384,999145\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 138\n", + "Returned IDs: 999350,994864,994488,995167,996001,992169,996406,997679,991212,992182,998122,994510,991596,990557,993931,999130,999304,999623,994467,999099,994119,990434,997332,997395,993032,998725,996996,990364,996028,994280\n", + "Ground Truth: 999350,994864,994488,995167,996001,992169,996406,999179,997679,991212,999679,992182,996803,998122,994510,991596,993931,990557,999130,999304,999623,994467,999099,994119,990434,997332,996075,997395,993032,994416\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 139\n", + "Returned IDs: 994168,992469,999500,995289,990126,990930,996811,998913,994783,991154,990340,993747,994742,997864,998495,990523,993642,996116,999977,990399,995401,996609,996062,994325,993561,995633,994431,994140,994190,990012\n", + "Ground Truth: 994168,992469,999500,995289,990126,997723,993656,990930,999586,996811,998913,995562,994783,991154,990340,993747,993400,994742,996219,995095,994759,997864,998633,998495,997797,990523,998156,990266,993642,996116\n", + "Recall: 0.6000\n", + "\n", + "Query ID: 140\n", + "Returned IDs: 999867,990957,990949,998246,997320,994964,993484,990736,991379,993853,992107,995949,995928,997913,998500,991895,995942,997371,999982,996785,994192,998170,995336,994100,995884,993518,997470,998365,999374,990925\n", + "Ground Truth: 999867,990957,990949,998246,998294,997320,994964,993484,990736,991379,993853,992107,995949,995928,997913,998500,991895,995942,997371,999710,999982,996785,990397,994192,993072,996560,998163,998170,995336,991743\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 141\n", + "Returned IDs: 995004,995639,995788,997288,996402,994922,994785,998657,992641,998241,994612,995588,993514,992302,995817,996840,991151,993708,996100,991172,990300,991473,993355,997486,993282,991193,997220,990902,993143,991510\n", + "Ground Truth: 995004,995639,999888,995788,997288,996402,994922,994785,998657,992641,993675,998241,998973,994612,995588,993514,992302,995817,996840,991151,993708,996100,991172,990300,991473,993355,993282,997486,991193,997220\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 142\n", + "Returned IDs: 993250,990515,998211,997543,999823,993364,997301,993669,999263,991743,996218,992401,999968,999089,990561,990760,997320,997192,995886,993798,998921,999291,999145,992384,999374,996504,995928,992147,999026,990585\n", + "Ground Truth: 993250,990515,998211,997543,999823,993364,997301,993669,999263,991743,996218,992401,999968,999089,990561,990760,997320,998445,995034,997192,993648,995886,993798,998921,997228,999291,999145,992384,999374,996504\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 143\n", + "Returned IDs: 999342,990662,996147,998878,994223,994708,996739,990771,992636,997550,996657,990573,994523,993344,999130,994830,994742,996001,998165,998803,993848,999570,999302,995873,995743,998949,994185,998076,998097,997565\n", + "Ground Truth: 999342,990662,996147,998878,994223,994708,996739,990771,992636,997550,996657,993599,991255,990573,994523,993344,992293,999130,994830,994742,996001,998165,998803,993848,999570,995827,999302,995873,995743,998949\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 144\n", + "Returned IDs: \n", + "Ground Truth: 995515,992835,990155,998244,996419,993582,993028,994216,998837,990142,991149,999053,990313,993237,991999,997591,994612,994281,992020,990652,997403,990943,994475,994497,990827,997465,990043,992662,994898,996485\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 145\n", + "Returned IDs: 990897,990942,995516,993055,997806,998758,996728,997506,995237,995268,993580,998346,993948,992636,991630,990005,991161,998067,995159,999775,993044,992052,992521,993275,990956,994380,993560,995184,990593,998697\n", + "Ground Truth: 990897,990942,995516,993055,997806,998758,996728,997506,995237,995268,993580,998346,993948,998045,992636,991630,990005,991161,998067,995159,999775,993044,992862,992052,994891,992521,993275,990956,994380,993560\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 146\n", + "Returned IDs: 994577,992423,999806,991157,998260,993552,998568,992617,992309,995515,997305,998495,999122,998921,999079,999526,993260,995080,992521,995372,998607,991743,999317,992944,997320,997227,999908,990908,999367,990843\n", + "Ground Truth: 994577,992423,999806,991157,998260,993552,998568,992617,992309,995515,997305,998495,999122,998921,999079,999526,997301,993260,995080,992521,995372,998607,991743,999317,992944,997320,997227,996281,999908,990908\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 147\n", + "Returned IDs: 990132,991664,993390,992582,997411,994717,990361,994656,995253,998779,991415,992989,996379,999646,991836,996867,994382,995701,996368,999402,995268,991350,997058,994092,997209,996659,994793,998922,993566,996388\n", + "Ground Truth: 990132,991664,993390,992582,997411,992604,992293,994717,990361,994656,995253,998257,998779,991603,991415,992989,996379,999646,991836,996867,994382,995701,996368,999402,995268,991350,997058,994092,997209,997723\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 148\n", + "Returned IDs: 997379,998455,993052,999077,996193,998180,993260,993696,999210,999784,998950,992571,993781,994341,992309,993504,996305,995203,997370,995443,990580,992944,991348,991157,996075,995365,990043,999122,994552,992169\n", + "Ground Truth: 997379,998455,993052,999077,996193,998180,993260,993696,999210,999784,998950,992571,993781,994341,992309,995094,993504,996305,995203,997370,995443,990580,992944,991348,991157,990414,996075,995365,990043,999122\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 149\n", + "Returned IDs: 997209,999843,992582,996220,994747,993857,997309,991189,994742,992460,990328,993549,995633,990132,991334,994119,998410,997544,991664,994835,997645,991269,992989,998910,999279,998495,993209,991075,996809,990126\n", + "Ground Truth: 997209,999843,992582,996220,994747,993857,997309,991189,994742,992460,990328,993549,995633,993948,990132,991334,994119,998410,997544,991664,994835,997645,991269,992989,992604,997208,992309,998910,999279,998495\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 150\n", + "Returned IDs: 992678,996848,998067,997915,991044,990331,991747,992097,999980,992521,997675,992569,998045,995268,991103,990314,996368,996277,996505,998060,993256,996719,995963,992187,994861,999402,993313,994817,995815,992448\n", + "Ground Truth: 992678,996848,998067,997915,991044,990331,991747,992097,999980,992521,997675,992569,998045,995268,998021,991103,990314,996368,996277,996505,998060,993256,992367,996719,995963,992187,994861,999402,997723,993313\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 151\n", + "Returned IDs: 999578,992448,994409,992660,992801,997832,999043,996038,999530,998714,991699,997868,997406,995139,998045,998979,999508,996607,991027,995507,994554,993433,996769,991671,993561,998545,999940,993421,999884,996867\n", + "Ground Truth: 999578,992448,994409,993468,992660,992801,996618,997832,999043,996038,999530,998714,991699,997868,996813,997406,995139,998045,998979,999508,996607,991027,995507,994554,993433,996769,991671,993561,993280,998545\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 152\n", + "Returned IDs: 997281,998720,992225,996857,992330,997305,990866,998514,997811,992068,998568,996230,995536,996075,993834,995638,994636,993551,999651,991373,991256,995701,996576,999636,993139,990051,993809,994572,992167,994531\n", + "Ground Truth: 997281,998720,992225,996857,992330,997305,990866,998514,997811,992068,998568,996230,995536,996075,993834,999987,995638,994636,993551,990642,999651,991373,991256,995701,996576,999636,993139,990051,993809,994572\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 153\n", + "Returned IDs: 992239,995637,990264,990814,991981,991590,999924,999767,992713,994891,993080,997494,994968,991428,990516,991769,995012,994094,992318,997698,990364,990270,992437,996728,999942,990853,992827,994742,999399,996911\n", + "Ground Truth: 992239,995637,990264,990814,991981,991590,999924,999767,992713,994891,993080,997494,994968,991428,990516,991769,995012,994094,998999,992318,997698,990364,990270,992437,996728,998869,999942,990853,992827,996658\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 154\n", + "Returned IDs: 994168,990080,990266,998633,991943,996093,999586,994742,999587,998156,991981,993080,993717,999453,992169,991585,995295,990957,998980,996867,991185,994431,996809,998256,995873,990860,997059,995274,994140,999957\n", + "Ground Truth: 994168,990080,990266,994131,992655,998633,991943,996093,999586,991255,994742,997723,999587,998156,991981,993080,993717,992582,999453,992169,997360,991585,991334,995295,994129,990957,998980,996867,991185,994431\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 155\n", + "Returned IDs: 995252,999024,993241,998462,999358,992066,990585,997015,998951,990817,996959,997771,999144,993544,990231,990535,995824,993562,994595,999650,999535,997320,999531,995716,997314,996785,996871,997256,996196,998090\n", + "Ground Truth: 995252,999024,993241,998462,999358,992066,990585,997015,998951,990817,996959,997771,999144,993544,990231,990535,995824,993562,994595,999650,998160,999535,997320,999531,995716,997314,996785,991362,996871,997256\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 156\n", + "Returned IDs: 999267,993637,991128,993053,995283,995659,990089,990368,999338,990629,995345,991412,991992,993410,999806,992636,997691,997052,994386,995638,998302,995467,998021,999446,996368,998215,999791,993561,995689,991795\n", + "Ground Truth: 999267,993637,991128,993053,995283,995659,990089,990368,999338,995274,990629,996701,990133,995345,991412,991992,993410,999806,992636,997691,997052,994386,995638,998302,995467,998021,999343,990260,997340,999446\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 157\n", + "Returned IDs: 990593,995907,998671,999934,995083,998315,992450,993476,995268,998537,998055,996431,995191,992106,994142,998256,990041,996495,997652,991769,992859,997025,997806,993466,994891,996328,990069,999360,996055,997185\n", + "Ground Truth: 990593,995907,998671,999934,995083,998315,992450,993476,995268,998537,998055,996431,995191,992106,994313,994142,999441,998256,990041,996495,997652,991769,998072,992859,997025,997806,993466,994891,996328,990069\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 158\n", + "Returned IDs: 990890,996219,991928,998899,992931,991249,998540,994069,999068,999343,994964,992330,991593,996343,997913,994486,998550,999620,997305,990210,998837,996646,993264,994366,991999,990957,994783,997022,998464,999815\n", + "Ground Truth: 990890,996219,991928,998899,992931,991249,998540,994831,994069,999300,990545,999068,999343,995942,994964,992330,991593,991743,996343,997913,993282,990404,994486,998550,990666,999620,991006,995210,997305,990210\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 159\n", + "Returned IDs: 992094,994119,996245,997944,998150,994193,990226,994280,992703,992052,992437,990790,994968,997309,997897,993915,997056,998355,995408,996277,990784,991011,994375,994472,997881,998165,995017,993778,999643,992576\n", + "Ground Truth: 992094,994119,996245,997944,998150,994177,994193,990226,990527,994280,999461,992703,992052,998755,993037,992437,990790,994968,997309,997897,993483,993915,992652,997056,993288,995401,998355,995408,996277,990784\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 160\n", + "Returned IDs: 992604,993754,995807,995527,999662,999445,997781,990915,990540,995953,991051,990657,990729,998297,991204,991110,994187,997918,995373,994753,998858,991987,996043,997158,999306,995805,997675,995105,998697,997723\n", + "Ground Truth: 992604,993754,995807,995527,999662,999445,997781,990915,990540,995953,991051,990657,990729,998297,991204,991110,991571,994187,997918,997212,995373,994753,998858,991987,996043,997158,999306,995805,997391,997675\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 161\n", + "Returned IDs: 998294,996504,999765,997913,994266,998921,993518,991406,997984,990760,999263,993669,998500,995755,990925,994978,995090,998987,997192,990595,997371,998540,995793,990958,994415,994775,995949,993143,990560,995942\n", + "Ground Truth: 998294,996504,999765,990917,997913,994266,998921,993518,991406,997984,991895,990760,999263,993669,998500,995755,990925,994978,990829,995090,998987,990936,997192,990595,996560,997371,998540,995793,998344,999647\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 162\n", + "Returned IDs: 995110,993478,991103,996505,995032,998397,995783,999646,991747,995401,997675,995701,993727,990537,995521,991413,990956,993738,997918,997752,995184,995886,990872,992790,994280,990930,993948,998874,996715,991789\n", + "Ground Truth: 995110,993478,991103,996505,995032,998237,998397,995783,999646,991747,995401,997675,995701,993727,990537,995521,991413,991010,990956,992457,993738,991630,997918,997752,991052,995184,991054,994232,992260,998045\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 163\n", + "Returned IDs: 998914,990678,996556,996028,998355,997380,995528,999891,990226,991426,996190,993725,994765,995317,996505,996016,992322,994119,998325,999479,992480,995449,995456,998982,999516,991747,997649,990776,994012,994587\n", + "Ground Truth: 998914,990678,996556,996028,998355,997380,995528,999891,998630,990226,991426,996190,993725,992521,994765,995317,996505,996016,992488,996728,992322,994119,998325,999479,992480,993820,995449,995456,993336,998982\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 164\n", + "Returned IDs: 999830,991540,990487,992280,994541,992934,995273,994661,993976,997374,993583,996194,995436,994913,992052,996178,990908,993931,994488,991977,992718,995159,998417,998370,997329,993256,998118,990364,995551,990270\n", + "Ground Truth: 999830,991540,990487,992280,994541,992934,995273,997960,994661,992467,993976,997374,993583,996194,998135,995436,994913,992052,997056,996178,993931,990908,994488,991977,992718,995159,998417,998370,996685,997329\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 165\n", + "Returned IDs: 991308,995120,994955,992958,992823,990364,997897,996001,997320,995167,990557,999526,990314,997790,997368,996657,995250,990096,992825,993375,998593,994081,994972,990409,995672,997926,995855,995740,997977,994809\n", + "Ground Truth: 991308,996764,995120,994955,992958,992823,990364,997897,996001,997320,995167,990557,999526,990314,991657,991898,994531,997790,997368,996657,995250,990096,992825,993375,998593,994972,994081,990409,995672,997926\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 166\n", + "Returned IDs: 994380,996064,996001,990392,990887,991866,991585,992832,990127,997332,994925,990712,993561,997732,998470,999126,992703,998122,992450,991831,998067,998416,993637,996505,994541,998862,998055,991185,993180,992595\n", + "Ground Truth: 994380,996064,996001,993710,990392,990887,991866,991585,992832,990127,997332,993648,994925,990712,993561,997732,998470,999126,993412,992703,998122,992450,991831,998067,998416,993637,996505,994541,998862,998055\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 167\n", + "Returned IDs: 997666,996733,999612,992234,996386,993132,993867,992209,999368,995593,998671,998781,994181,992716,992673,998598,991724,996209,990806,999388,995385,997693,993148,990423,994731,996893,995796,994335,999079,997685\n", + "Ground Truth: 997666,996733,999612,992234,996386,993132,993867,992209,999368,990712,995593,997807,998671,998781,990552,990630,994181,992716,992673,998598,991724,997442,996209,990806,999388,995385,997693,993148,992984,990423\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 168\n", + "Returned IDs: \n", + "Ground Truth: 993561,995923,996867,990975,996232,996769,997554,994419,998564,996001,993725,997406,992576,994925,996042,991958,996360,993813,998045,995963,994409,995216,992862,995401,993412,995052,997046,996277,993461,995873\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 169\n", + "Returned IDs: 994336,990103,996803,993727,998123,999244,993185,996806,996918,993805,993165,996535,994051,992558,997644,999296,991453,998968,995175,991227,999673,999029,994909,994035,996825,993478,991610,991054,997480,999191\n", + "Ground Truth: 994336,990103,996803,993727,998123,999244,993185,996806,996918,993805,993165,996535,994051,992558,997644,999296,991453,998968,995175,991227,999673,999029,994909,998279,994035,999098,996825,993478,996861,991610\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 170\n", + "Returned IDs: 999430,995332,995107,991961,991458,998416,998950,990377,993499,995560,996216,993807,995895,997331,995750,993617,993079,992780,997004,993479,996944,999210,992567,998157,996370,991382,993391,998732,990039,997586\n", + "Ground Truth: 999430,995332,995107,991179,991961,991458,994897,998416,998950,990377,993499,995560,996216,993807,995895,997331,990091,995750,993617,993079,990769,992780,997004,993479,996944,999210,992567,993335,998157,996370\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 171\n", + "Returned IDs: 996219,994990,994964,993378,991075,996052,997616,996593,997028,999453,995096,993342,996001,998039,990448,992780,990366,994777,990091,993648,999344,998464,998495,999151,990146,990632,999791,995107,998593,999261\n", + "Ground Truth: 996219,994990,994964,993378,991075,996052,994843,997616,992047,996593,997028,999453,995096,993342,996001,998039,990448,992780,990366,997920,994777,994472,999352,992261,994922,990091,993648,998633,999437,999344\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 172\n", + "Returned IDs: 990016,998105,992201,993588,994145,993053,995097,990435,990497,990536,994307,998548,991075,990662,994830,995289,994925,996974,999460,998165,993831,990811,996001,995823,996411,998950,998495,994730,995906,997691\n", + "Ground Truth: 990016,998105,992201,993588,994145,993053,995097,990435,990497,996147,990536,994307,998548,991075,990662,994830,995289,994925,990771,996974,999460,998165,993831,998097,990811,996001,993412,995823,996411,998950\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 173\n", + "Returned IDs: 992235,993832,996219,995483,993687,994288,993931,990705,994774,990931,994742,998080,993621,995455,997395,992645,996001,990722,997126,994062,998495,990126,995401,995932,990041,999961,999139,999430,992888,995134\n", + "Ground Truth: 992235,993122,993832,996219,995483,993687,997068,994288,999916,997149,993931,990705,994774,993566,990931,997790,997540,994742,998080,999551,993621,995455,999360,997395,992645,996001,999647,990722,997126,994062\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 174\n", + "Returned IDs: 997391,991698,997218,997918,990215,992933,995282,993933,992428,998199,997158,998022,994406,992179,996247,992790,995886,997514,994819,993288,990916,994982,992604,991459,991413,996792,994170,994280,992516,999154\n", + "Ground Truth: 997391,991698,997218,997918,990215,997799,992933,995282,993933,992428,998199,997158,998022,994406,992179,996247,992790,995886,997514,994819,991792,993288,990916,994982,992604,997901,991459,991413,996792,994170\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 175\n", + "Returned IDs: 994894,990689,997192,998974,995274,999444,994266,990957,993186,997035,997437,998697,990862,995536,996699,992811,990231,992357,997320,990958,996785,998649,998170,990146,998779,998500,995569,994100,995655,995701\n", + "Ground Truth: 994894,990689,997192,998974,995274,999444,994266,990957,993186,990498,997035,998326,997437,998697,991630,990337,990862,995536,996699,992811,990231,992357,997320,990958,996785,998649,993412,990799,998170,990146\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 176\n", + "Returned IDs: 994360,996178,996429,999676,991650,996219,995826,991409,999806,997301,990096,996001,998607,995672,996397,998180,993431,997456,998495,993696,997550,998401,999426,992423,998302,997586,999210,998921,993470,994742\n", + "Ground Truth: 992443,995401,994360,996178,996429,999676,995559,991650,996219,995826,991409,991607,999806,997301,990096,996001,994244,998607,995672,993831,998365,996397,997906,998180,993431,997456,995873,998495,993696,999900\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 177\n", + "Returned IDs: 999011,996730,992179,998536,996245,992790,991698,998874,994301,993673,993478,997918,995401,991459,992933,995154,995886,992558,999820,996040,994170,990214,993037,994334,998778,990623,992428,990479,994280,997702\n", + "Ground Truth: 999011,996730,992179,998536,996245,992790,991698,992685,998874,994301,993673,993478,997918,995401,991459,992933,995154,995886,992558,999820,998529,996040,994170,990214,991052,993037,994334,998778,990623,992428\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 178\n", + "Returned IDs: 994254,997809,998686,990487,991977,992889,997407,991923,998046,990776,995691,994165,998243,994760,990343,996068,999766,998975,993839,990522,997914,998357,993572,994010,995965,995506,996895,992645,998118,994119\n", + "Ground Truth: 994254,997809,998686,990487,991977,992889,997407,991923,998046,990776,995691,994165,998243,994760,990343,996068,999766,998975,993839,990522,993683,997914,998357,993572,994010,995965,995506,999913,996895,992645\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 179\n", + "Returned IDs: 993280,999453,996810,992395,998734,996001,991951,995988,990142,991517,995216,999076,991840,991248,999019,997320,998605,991999,992944,998697,995705,993018,997002,994280,996064,994081,993324,990652,994964,998416\n", + "Ground Truth: 990155,993280,999453,996810,992395,998734,990772,996001,991375,991951,995988,990142,994536,996915,996732,997876,991517,998567,995216,990778,995130,999076,991840,998523,991255,991248,999019,997320,999723,993154\n", + "Recall: 0.5333\n", + "\n", + "Query ID: 180\n", + "Returned IDs: 993324,999796,998104,999514,996918,990956,997420,995130,995401,992944,995332,993121,991961,995766,993347,994119,995438,998536,998180,991723,999881,995107,992052,994743,998950,995289,994541,996855,990776,999179\n", + "Ground Truth: 993324,999796,998104,999514,996918,990956,997420,995130,995401,992944,995332,993121,991961,995766,993347,994119,995438,998536,998180,991723,999881,995107,992052,994743,998950,995289,994541,996855,990776,993079\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 181\n", + "Returned IDs: \n", + "Ground Truth: 995895,996706,997442,998593,993805,993391,995438,992726,990844,998993,990133,999430,994431,994351,994742,996809,999435,999179,996930,998934,999587,998989,998990,994830,993335,997723,992864,997331,998055,994382\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 182\n", + "Returned IDs: 995289,995633,994742,995702,991585,997723,995200,990361,991185,993209,991664,996042,998162,995766,997182,998999,993831,996867,997822,996302,995648,992582,990940,992448,998165,997411,995401,996388,999862,995216\n", + "Ground Truth: 995289,995633,994742,995702,991585,997723,995200,990361,991185,993209,991664,996042,998162,995766,997182,998999,993831,996867,997822,996302,995940,999587,995648,992582,990940,992448,998165,997411,995401,996388\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 183\n", + "Returned IDs: 992002,991507,998162,995493,996607,998910,999054,997466,992448,990163,995458,996987,993046,990295,991671,991585,997529,994013,994865,998948,990924,993883,990804,996042,996815,995216,995766,993561,992801,992528\n", + "Ground Truth: 992002,991507,998162,995493,996607,998910,999054,997466,992448,990163,995458,996987,993046,990295,991671,991585,997529,992260,994013,994865,998948,990924,993883,990804,996042,996815,991603,998283,999475,995216\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 184\n", + "Returned IDs: 996689,990328,990600,994255,992445,999025,995456,994689,992052,995295,991277,996028,997329,991451,997309,994280,993560,997420,995166,990532,998589,995823,995273,997897,998182,995203,994946,998623,996580,997251\n", + "Ground Truth: 996689,990328,990600,994255,992445,999025,995456,994689,992052,995295,991277,992068,996028,997329,996110,991451,995832,997309,994280,998959,993560,997420,995166,990532,998589,995823,995273,997897,998182,995203\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 185\n", + "Returned IDs: 990193,996911,996277,993738,991769,995047,997059,999343,997554,991466,991349,991981,991185,995216,995184,992162,999402,998060,996719,992448,999731,991163,992521,997168,994140,991111,992526,991585,995052,998910\n", + "Ground Truth: 990193,996911,996277,993738,991769,995047,997059,999343,997554,991466,991349,991981,991185,995216,995184,992162,992879,994353,999402,998060,996719,991737,992448,999731,991163,992521,997168,990364,994140,991111\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 186\n", + "Returned IDs: 999858,994682,994233,994830,996183,997073,991885,998076,992372,997340,993101,993588,995299,993335,993917,998063,990582,992318,991409,990373,997109,991390,994269,990700,995735,993568,992026,996357,990392,992415\n", + "Ground Truth: 999858,994682,994233,994830,996183,997982,997073,991885,998076,992372,997340,993101,993588,993394,995299,993335,993917,998063,990582,992318,991409,990373,997109,991390,994269,990700,995735,993568,992026,996357\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 187\n", + "Returned IDs: 997391,997918,994334,990215,996282,994819,993492,995886,996903,997702,992979,992179,999820,995401,992211,993512,994088,994319,992428,999144,992933,994906,998892,990561,994670,996658,991698,992558,999804,993289\n", + "Ground Truth: 997391,997918,994334,990215,996282,994819,993492,995886,996903,999562,997702,992979,992047,992179,999820,995401,992211,993512,997782,994088,994319,992428,999144,994916,992933,994906,998892,990561,998011,994670\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 188\n", + "Returned IDs: 997106,992190,998409,995256,990675,992804,994852,991958,991923,999639,997864,994043,993342,991208,995701,996368,993283,992167,991530,997407,990622,998079,997739,995898,991350,996894,996127,997577,999216,992976\n", + "Ground Truth: 997106,992190,998409,995256,990675,992804,993791,998974,994852,991958,991923,999639,997864,994043,993342,991208,995701,996368,993283,992167,994796,991530,997407,990622,998079,995138,997739,995898,991350,994105\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 189\n", + "Returned IDs: 996062,992182,990096,992576,991857,996001,996277,998045,995130,991011,993561,995536,994964,995834,997586,991685,993433,993873,996884,994955,999740,996371,998039,991789,993906,998158,995289,996219,995052,998495\n", + "Ground Truth: 996062,992182,990096,992576,991857,997920,996001,996277,998045,995130,991011,993561,995536,994964,995834,993778,997586,991685,993433,993873,997320,996884,994955,999740,996371,998039,994419,991789,993906,998158\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 190\n", + "Returned IDs: 994105,994906,996050,992664,998855,991332,993866,995168,997332,999653,991038,997666,998623,995536,996075,995695,997906,996657,991691,999827,996001,999212,999543,999139,999724,990998,991630,996326,994280,995603\n", + "Ground Truth: 994105,994906,996050,992664,998855,991654,991332,993866,995168,998051,997332,999653,991038,997666,998623,995536,996075,995695,997906,996657,991691,999827,996001,999212,999543,999139,999724,990998,991630,996326\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 192\n", + "Returned IDs: 991372,993696,990628,994722,990885,996809,991109,992635,996452,990697,991172,994922,995210,995530,995596,998303,999337,996350,993504,991425,990627,996996,994964,993454,999210,993617,990957,999750,992041,991157\n", + "Ground Truth: 991372,993696,990628,994722,990885,996809,996296,997166,991109,992047,992635,997098,996452,990697,991172,998734,994922,995210,999219,992927,995530,994859,996875,995596,996913,998303,999337,996350,998790,993504\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 193\n", + "Returned IDs: 991515,990255,992015,995401,999595,998593,996930,990573,998687,998874,994119,993992,990412,996142,992367,991835,996657,996043,996759,999629,992104,995421,993275,996388,998990,995864,996001,990314,994743,998878\n", + "Ground Truth: 991515,993599,990255,992015,995401,999595,998593,997723,996930,990573,998687,998874,994119,992977,993992,990412,996142,992367,991835,996657,996043,996759,999629,992104,999669,995421,993275,996388,998990,995864\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 194\n", + "Returned IDs: 997319,995633,994283,991810,998750,995611,992561,994204,990223,997141,996411,991330,993261,995289,995401,995948,994148,992289,997155,994783,998765,994742,997117,990814,995190,998525,993747,999235,995274,994422\n", + "Ground Truth: 997319,995633,994283,991810,998750,995611,992561,994204,990223,997141,996411,991330,993261,995289,995401,995948,994148,992289,997155,994783,998765,994742,997117,990814,995190,998525,999627,993747,999235,995274\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 195\n", + "Returned IDs: 991890,996809,998786,999430,990361,996830,996284,998695,996930,998593,995461,994382,990255,994431,999009,990930,991846,995701,995401,990288,990167,995289,998874,995895,999994,995421,990941,993232,997630,991255\n", + "Ground Truth: 991890,996809,998786,999430,990361,996830,996284,998695,996930,998593,995461,994382,990255,996049,994431,999009,990930,991846,997723,995701,995401,990288,990167,995289,991025,998874,995895,999994,995421,990941\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 196\n", + "Returned IDs: 998697,992293,991530,993948,997884,995701,992862,992609,998758,999653,993738,995672,996828,995184,998831,990005,996368,991836,995536,996714,991844,993560,995516,997868,997928,990211,991630,998017,999639,993515\n", + "Ground Truth: 998697,992293,991530,993948,997884,995701,992862,992609,998758,999653,993738,995672,996828,995184,998831,990005,996368,991836,994125,995536,996714,991844,996657,998200,993560,995516,997868,997928,990211,991630\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 197\n", + "Returned IDs: 994111,999026,999144,997015,995274,997976,995336,998692,998365,994100,994964,992630,997320,997192,996959,991467,993561,995132,999374,995701,996714,993484,997371,995928,995502,998779,995204,997918,997571,998302\n", + "Ground Truth: 994111,999026,999144,997015,995274,997976,995336,998692,995374,995401,998365,994100,994964,992630,997320,992047,993412,997192,996959,991467,993561,995132,999374,992331,995701,998011,996714,993484,997371,994959\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 198\n", + "Returned IDs: 995645,991392,997291,998244,992612,990615,997227,990043,993143,991609,998837,999257,999076,998607,991999,992671,997496,999122,995515,990310,992146,996840,993237,997370,995203,990142,991013,998567,999784,991157\n", + "Ground Truth: 995645,991392,997291,998244,992612,992734,990615,997227,990043,993143,992336,991609,998837,999257,999076,998607,991999,992671,997496,999122,995515,990310,992146,996840,993237,997370,995203,990142,991013,996930\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 199\n", + "Returned IDs: \n", + "Ground Truth: 990382,997476,991581,999827,996062,992314,993981,994510,998095,994463,998436,992161,993289,990899,990520,991930,997790,996913,997642,993931,998158,992169,995954,997453,991596,995636,990557,990212,995185,991644\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 200\n", + "Returned IDs: 997320,997370,992944,997305,998950,991498,995886,993018,994266,995372,992041,990949,999453,995203,997301,995552,995927,993154,991801,999210,993260,990532,997470,995438,997913,993241,995841,997371,999701,998180\n", + "Ground Truth: 997320,997370,996811,992944,997305,998950,991498,995886,993018,994266,995372,992041,990949,995250,990772,999453,995203,997301,995552,990917,995927,993154,991801,999210,993260,990532,997470,995438,997913,993241\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 201\n", + "Returned IDs: 999082,991904,993879,997675,993385,997997,992801,991388,997588,999796,990875,994002,994413,996918,995159,990784,996505,995521,996386,993899,996472,998049,995080,994677,998405,998536,994835,992285,997459,993931\n", + "Ground Truth: 999082,991904,993879,997675,991061,993385,997997,992801,991388,997588,999796,990875,994002,994413,996918,995159,995200,990784,996505,995521,996386,993899,998107,996472,998049,995080,994677,997949,998405,998536\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 202\n", + "Returned IDs: 999267,998097,998021,990368,998076,993725,993715,996328,994016,998921,998812,995671,998848,999526,993719,991390,992825,999516,992068,990438,992636,990889,993053,990011,998536,992450,995705,994280,999338,995295\n", + "Ground Truth: 999267,998097,998021,990368,998076,993725,993715,996328,994016,998921,998812,995671,998848,999526,993719,991390,996429,992825,999516,990438,992068,992636,990889,993053,990011,998536,992450,995705,994280,999338\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 203\n", + "Returned IDs: 992927,993260,994988,994964,994537,996193,990786,997320,993055,992069,998568,994984,997918,993783,991795,992571,995536,992497,994789,992225,994552,999800,993948,993696,993834,992330,997909,999077,998790,999129\n", + "Ground Truth: 992927,993260,994988,990429,994964,994537,996193,990786,996053,997320,999216,996576,999362,993055,990885,999176,992069,998568,994984,997918,993783,991795,991675,992571,995536,992497,994789,992225,999711,994552\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 204\n", + "Returned IDs: 990487,994117,996391,998370,995896,991350,992929,997554,990211,997868,994793,994207,996368,996379,994809,996654,996657,990314,998990,996127,995873,993831,991585,992423,999639,990081,994032,996147,990662,993681\n", + "Ground Truth: 990487,994117,996391,998370,995896,991350,992929,997554,999587,990211,997868,994793,994207,996368,996379,993998,994809,996654,996657,997723,990314,998990,996127,991973,995873,998686,996181,993831,991585,992423\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 205\n", + "Returned IDs: 993637,997933,993906,992636,997052,995295,990392,993715,998039,998021,998097,990089,999138,990930,998593,997453,990845,991144,996001,993917,998076,990368,998440,995467,991021,997170,990027,999052,996906,990438\n", + "Ground Truth: 993637,997933,993906,993719,992636,997052,995295,990392,993715,998039,998021,994386,998097,990089,999138,990930,998593,997453,990845,991144,996147,996001,992547,993917,998076,990368,998440,995401,995467,991021\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 206\n", + "Returned IDs: 993885,993238,999533,997395,998029,995928,991743,997320,994266,992922,996805,991498,999827,994280,997732,998302,993250,997301,991932,993484,995961,997002,992182,994964,993080,996328,997822,990560,995861,990210\n", + "Ground Truth: 993885,993238,999533,997395,998029,995928,991743,997320,994266,992922,996805,991498,990814,999827,994280,997732,991611,993860,999290,998302,993250,997301,991932,993484,995961,991829,997002,992182,994964,993080\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 207\n", + "Returned IDs: 998165,992160,999782,995439,998045,996482,997406,992125,996769,992115,995289,996125,993830,994057,995052,991463,995182,990681,997046,990984,991603,993844,992925,995497,993390,998520,995285,992576,996277,991415\n", + "Ground Truth: 998165,992160,999782,995439,998045,996482,997406,992125,993236,996769,992115,995289,996125,993830,994057,995052,991463,995182,990681,997046,990984,991603,993844,992925,995497,993390,998520,996112,995285,992576\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 208\n", + "Returned IDs: 994273,999415,993127,995122,992449,995423,995702,994013,995701,999054,991636,996867,995182,995685,990082,995905,994954,990152,994409,998776,995401,997975,998518,997674,996232,992660,993856,995891,993232,998162\n", + "Ground Truth: 994273,999415,993127,995122,992449,995423,995702,994013,999745,995701,999054,991636,996867,995182,996897,995685,998636,990082,995905,997077,994954,990152,994409,998776,995401,997975,998518,997674,995303,996232\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 209\n", + "Returned IDs: 995167,996562,992199,993391,996216,997897,999748,998335,999829,990627,991037,993875,991596,999435,999093,992137,995626,992247,995214,997944,992151,990255,991148,997237,996501,993805,992825,993132,992056,992771\n", + "Ground Truth: 998731,995167,996562,992199,993391,996216,997897,999748,998335,999829,996986,990627,991037,993875,993195,991596,999435,993124,991170,999093,992137,995626,992247,999455,995214,997944,992151,990255,991148,997237\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 210\n", + "Returned IDs: 998749,992124,992213,992167,996576,992225,999041,991984,998232,993834,992238,996089,990550,996744,995516,996230,991157,995701,995897,994572,993766,990051,998950,990453,996138,993342,994809,996500,993902,999806\n", + "Ground Truth: 998749,992124,992213,992167,996576,992225,999041,991984,998232,993834,992238,990334,996089,990550,996744,995516,992276,996230,991157,995701,995897,994572,993766,998950,990051,990453,996138,993342,994809,996500\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 211\n", + "Returned IDs: 999453,994167,991987,991811,995596,996001,991459,999219,995899,990949,996657,990957,992979,996368,995281,997320,997258,992562,991835,993848,995866,999639,996744,992384,991392,994253,995455,998720,999586,998790\n", + "Ground Truth: 999453,994167,991987,990999,998567,991811,996161,991894,995596,996001,991459,993154,999219,995899,990949,996657,990957,991445,995855,992979,996368,995281,990542,997320,997258,992562,991835,993848,995866,999639\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 212\n", + "Returned IDs: 992824,995459,990894,996380,999639,991272,995631,995503,996742,990273,991432,996663,999908,998061,999387,997547,996894,991817,995728,993055,999674,993839,994560,993225,999346,998720,990061,992306,995724,999806\n", + "Ground Truth: 992824,995459,990894,996380,999639,991272,995631,995503,991064,996742,990273,991432,996663,999908,998061,999387,997547,996894,991817,995728,993055,999674,993839,994560,993225,999346,992752,998720,990061,992306\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 213\n", + "Returned IDs: 993879,995080,997675,995521,992801,996198,990244,999082,994170,994409,996505,993030,996277,998045,993561,992448,997822,998049,997832,996472,997020,993385,993738,994113,995648,995032,994589,996042,997723,993313\n", + "Ground Truth: 993879,995080,997675,995521,992801,991904,996198,990244,999082,994170,994409,996505,995401,993030,990527,996277,998045,993561,992448,995873,997822,998049,997832,991603,996472,994677,997020,997766,992260,990140\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 214\n", + "Returned IDs: 994280,991747,990434,997897,995184,993280,993931,993533,993738,990364,998067,996062,995047,990802,995876,992636,993324,993787,991255,990956,999827,994891,990951,998416,990993,998697,997679,996328,990784,997790\n", + "Ground Truth: 991860,994280,991747,990434,997897,990212,995184,998006,993280,993931,993533,993738,990364,998067,996062,995047,990802,995876,995281,992636,993324,993787,991255,990956,999827,994891,990951,998416,994389,990993\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 215\n", + "Returned IDs: 996062,998122,998495,999898,990194,990096,997242,990434,999827,991893,993106,999555,996219,996001,996028,998861,995655,995536,992056,994266,995562,999139,990167,990382,990796,998593,991296,991037,997586,996668\n", + "Ground Truth: 996062,998122,998495,999898,994869,990091,990194,990096,997242,990434,999827,991893,993106,999555,996219,996001,996028,998861,995655,995536,992056,994266,995562,999139,990167,990382,990796,998593,994200,997790\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 216\n", + "Returned IDs: 997429,995184,993238,991856,990958,992357,993648,999533,995963,994266,996861,993168,995800,998458,990925,995927,997852,995783,993885,999374,992147,995884,997309,992637,993014,998495,992994,995755,997803,995771\n", + "Ground Truth: 997429,995184,993238,991856,990958,992357,993648,994835,999533,995963,994266,995034,996861,991714,992008,993168,995800,994956,998458,990925,995927,997852,995783,993885,992526,999374,992147,995884,990399,997309\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 218\n", + "Returned IDs: 990711,990392,997529,991703,993053,998862,993725,996411,994730,995002,992156,993240,998913,999302,999377,995633,994742,990701,998495,998779,994830,990435,993166,993264,992680,996408,993856,995601,992024,998710\n", + "Ground Truth: 990711,990392,997529,991703,993053,998862,993725,996411,994730,995002,992156,993240,998913,999302,999377,995633,994742,990701,998495,998779,994830,995401,990435,993166,993264,995096,992680,996408,993856,991390\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 219\n", + "Returned IDs: 999343,993412,990366,996219,998779,995274,992630,998210,994964,998862,995336,990958,997479,998302,990974,998458,994100,999144,999026,996727,997320,990934,998090,990271,992384,996714,999647,990866,994990,998246\n", + "Ground Truth: 999343,993412,990366,996219,993909,998779,995274,992630,998210,994964,998862,995336,990958,990666,997920,997479,998302,990974,995374,990171,992008,998458,999331,994100,991311,999144,991362,999026,996727,997320\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 220\n", + "Returned IDs: 994643,998536,997752,993232,999117,998383,999155,990551,993995,996809,992288,998695,996797,991532,992558,997666,995963,990941,994063,991571,996043,993077,997131,994742,992604,992809,996395,990361,990991,995274\n", + "Ground Truth: 994643,998536,997752,993232,999117,998383,999155,990551,995625,993995,996809,992288,998695,996797,991532,992558,991630,997666,995963,990941,994063,991571,996043,993077,997131,991675,994742,995401,992604,992809\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 221\n", + "Returned IDs: 995596,995515,993143,991840,991848,999122,999784,999806,996840,990432,994709,997403,993748,995443,996219,999676,992979,999144,991395,991485,997135,997379,999613,992309,996062,997918,993237,994577,999453,994922\n", + "Ground Truth: 995596,995515,993143,991840,991848,999122,999784,999806,996840,990432,994709,997403,993748,995443,996219,995365,999676,992979,999144,991395,991485,992047,997135,997379,999613,991977,992309,996062,997918,994916\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 223\n", + "Returned IDs: 997030,990096,994717,994643,994413,993666,992638,993115,999530,998257,991309,994454,999898,991190,998277,999555,997211,993566,992369,995218,994677,998045,997568,990361,998536,994835,992161,994656,991937,990132\n", + "Ground Truth: 997030,990096,994717,994643,994413,993666,992638,993115,999530,998257,991309,994454,999898,990815,997077,997154,991190,997871,998277,999555,997211,993566,992369,995218,994677,998045,991596,991188,997568,990361\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 224\n", + "Returned IDs: 990049,995609,996194,999528,995043,995551,994326,993678,991241,996232,990417,993181,990011,996435,992566,994541,992068,999830,992473,993245,994317,995475,995537,993315,997906,996867,992563,995317,992951,995666\n", + "Ground Truth: 990049,995609,996194,999528,995043,995551,994326,993678,991241,996232,990417,993181,990011,996435,992566,994541,992068,999830,998769,992473,990121,993245,994317,995475,995537,993315,997906,990153,996867,992563\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 225\n", + "Returned IDs: 997806,991161,997251,993580,994280,994365,998067,994380,990294,991904,990942,991066,996960,999774,991103,996918,999294,992992,995268,994142,997918,993044,996328,991610,996683,994639,999994,997601,991038,993948\n", + "Ground Truth: 997806,991161,997251,993580,991273,994280,994365,998067,994380,990882,990294,991904,990942,991066,996960,992487,993439,999774,991103,996918,999294,992992,995268,994142,993727,997918,993044,996328,991610,990875\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 226\n", + "Returned IDs: 991140,999000,997723,996038,995685,998779,998999,995182,999412,991603,996968,998165,997832,997020,995648,994409,998297,996867,994948,996734,993103,990940,995507,995289,995421,997822,995401,998593,995052,996432\n", + "Ground Truth: 991140,999000,997723,996038,995685,998779,998999,995994,995182,999412,991603,996968,998165,997832,997020,995648,994409,998297,996867,994948,996734,993103,990940,995507,995289,997042,990406,995421,997822,995401\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 227\n", + "Returned IDs: 992511,998974,996252,997437,993457,994894,995838,997174,996917,997728,999599,993978,995755,998649,990957,997320,996192,995274,997304,994630,992531,993049,992296,997035,996700,997371,999333,998460,998090,999074\n", + "Ground Truth: 992511,998974,996252,997437,993457,990337,999867,994894,995838,997174,996917,997728,993222,999599,993978,995755,998649,994916,990957,997429,997320,996648,996192,995616,995274,995963,997304,994630,995374,991995\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 228\n", + "Returned IDs: 998060,993487,994203,992352,994106,992098,990573,992000,991057,996075,994488,996001,995079,995544,991212,994013,990662,990314,999172,997998,998593,996898,995873,994954,992569,995640,992097,991394,999590,997554\n", + "Ground Truth: 998060,993487,994203,992352,994106,992098,990573,992000,991057,996075,994488,996001,995079,995544,991212,994013,990662,990314,999172,996043,995515,997998,998593,996898,995873,994954,992569,995640,992097,991394\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 229\n", + "Returned IDs: 992558,990279,997547,997124,996196,990538,999715,995336,995886,996606,993948,994334,997859,992718,991157,996609,996918,999860,992857,995552,996721,994163,993363,990343,991498,994280,998697,991272,996318,992173\n", + "Ground Truth: 992558,990279,997547,997124,996196,990538,999715,995336,999908,995886,996606,993948,994334,997859,992718,990989,992971,991157,991052,996609,996918,999860,992857,990930,995552,996721,994163,993363,990343,991498\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 230\n", + "Returned IDs: \n", + "Ground Truth: 994266,995184,997320,997913,990231,998170,992546,991256,999647,992330,995701,993853,999041,997305,995417,996296,992637,996075,997554,999761,990958,996661,993834,997449,990650,996083,993168,990535,992748,991045\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 231\n", + "Returned IDs: 998913,994783,998823,998370,998196,995551,991428,994742,992566,991981,994806,994692,995289,995397,995348,992415,995948,995030,992827,997630,996411,991330,995633,998543,994110,996609,994587,997529,990126,997823\n", + "Ground Truth: 998913,994783,998823,998370,998196,995551,991428,994742,992566,991981,994806,994692,995289,995397,990846,995348,992415,995948,993939,995030,992827,997630,996411,991330,995633,998543,995300,994110,998028,996609\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 232\n", + "Returned IDs: 995841,997301,990405,995264,994942,999526,997292,990546,990191,992451,995372,991498,999523,998921,995623,996135,997470,996516,997320,998445,995306,993484,997333,999904,996771,993364,999338,998721,999594,999281\n", + "Ground Truth: 995841,997301,990405,995264,994942,999526,997292,990546,990191,992451,995372,991498,999523,998921,995623,996135,997470,996516,997320,998445,995306,993484,994741,997333,999904,996771,993387,993364,997876,999338\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 233\n", + "Returned IDs: 992981,992445,997806,996328,994336,990956,996728,996861,994125,993055,990005,997420,997884,996211,996918,997506,994142,999775,993948,996189,998959,993560,990961,993877,998495,998067,995516,994280,990226,994380\n", + "Ground Truth: 992981,992445,997806,996328,994336,990956,996728,996861,998017,992979,994125,993055,990005,997420,997884,996211,996918,997506,994142,999775,993948,996189,998959,993560,992293,990961,993877,998495,998067,995516\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 234\n", + "Returned IDs: 990067,994627,990092,994776,998550,992396,992147,992134,990470,996219,991015,991157,992942,999898,998899,996118,998090,990595,991014,997371,999144,995942,999784,998401,990231,998921,994964,990090,994757,993260\n", + "Ground Truth: 990067,994627,990092,994776,998550,992396,992147,992134,993412,990470,996219,998380,993633,991015,991157,992942,993852,999898,998899,996118,998090,990595,991014,997371,999144,995942,994831,999784,999102,998401\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 235\n", + "Returned IDs: 990435,994830,994742,995633,998913,995200,996132,993053,997529,992424,990012,996001,990621,999352,996219,995097,995289,994806,992446,999192,995401,991588,998495,990701,993335,998221,994307,996411,995002,993166\n", + "Ground Truth: 990435,994830,994742,995633,998913,995200,996132,993053,997529,992424,990012,996001,990621,999352,999261,996219,995097,995289,994806,992446,999367,999192,994145,995401,991588,998495,990701,993335,998221,994307\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 236\n", + "Returned IDs: \n", + "Ground Truth: 998971,991596,992673,999663,999139,994510,996406,997127,999898,996985,996706,996809,999555,996733,991420,997980,998149,999435,992365,998006,996209,994335,992234,997242,999498,990226,995104,997506,991589,999783\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 237\n", + "Returned IDs: 999093,991425,992571,991037,998303,993504,995839,994134,999840,990885,998447,990096,992168,994221,997977,990409,992351,992747,998154,997897,993260,992082,998495,995203,998732,999750,990212,991332,998299,991491\n", + "Ground Truth: 999093,991425,992571,991037,998303,993504,994468,995839,994134,999840,990885,998447,990096,992168,994221,997977,990409,992351,992747,998154,997897,993260,991596,992082,998495,998732,995203,999750,995167,990212\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 238\n", + "Returned IDs: 995289,998990,996930,995401,997822,997723,999587,993831,998874,993673,998165,996867,994343,997406,996379,995633,995190,998593,995685,992703,993412,995702,997046,997630,990517,992685,993848,997832,991603,997662\n", + "Ground Truth: 995289,998990,996930,995401,997822,997723,999587,993831,998874,993673,998165,996867,994343,997406,996379,995633,995190,998593,995685,996607,992703,993412,995702,997046,998323,996294,997630,990517,992685,990930\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 239\n", + "Returned IDs: 999676,998921,991485,995515,992612,994922,995788,998401,996840,999806,998950,999077,996351,991833,992944,996915,995983,998244,993260,998697,991157,990096,993143,999698,993289,993781,997320,995080,994195,995372\n", + "Ground Truth: 999676,995693,998033,998921,991485,995515,992612,994922,995788,998401,996840,999806,998950,999077,999261,995401,993470,996351,991833,992944,996915,995983,998244,993260,998697,991157,990096,996371,993143,999698\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 240\n", + "Returned IDs: 997397,999110,999523,997301,999471,992451,995623,999904,991916,993484,990146,995306,990546,998760,997470,990405,997820,993426,993083,997292,995264,997320,992066,999242,995045,990191,996516,998921,995454,991134\n", + "Ground Truth: 997397,999110,999523,997301,999471,992451,995623,999904,991916,993484,990146,995306,990546,998760,997470,990405,997820,993426,993083,997292,995264,997320,992066,999242,995045,999247,991582,990182,990191,996516\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 241\n", + "Returned IDs: 998697,991530,999639,995672,997586,995562,997506,991485,991256,998303,999783,995894,993055,996918,990399,996001,999806,995418,995455,993342,995536,999216,996406,997258,997320,990930,999041,994906,995701,993834\n", + "Ground Truth: 998697,991530,999639,995672,997586,995562,997506,991485,991256,998303,999783,997531,995894,993055,995855,996918,990399,996001,999806,995418,995455,990453,993342,995536,999216,995250,996406,997258,997320,990930\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 242\n", + "Returned IDs: 995659,999271,990585,997668,997286,990089,999338,990000,994942,994618,997292,997984,995841,997470,997301,996516,995306,992069,991503,997320,998721,998921,995372,998923,998445,993419,994964,992451,997113,997820\n", + "Ground Truth: 995659,999271,990585,997668,997286,990089,999338,990000,994942,994618,997292,997984,995841,997470,997301,996516,995306,992069,991503,997320,998721,998921,995372,991570,998923,998445,998461,993419,994964,992451\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 243\n", + "Returned IDs: 990467,994688,998649,996192,996869,990397,999301,998458,991268,995274,994630,990925,995942,996196,999500,996219,995502,997079,992630,997913,992291,998500,998797,991096,990949,996252,994783,998302,996727,992994\n", + "Ground Truth: 990467,994688,998649,996192,996869,990397,993656,999301,998458,991268,995274,992273,990315,993725,993412,991995,994630,996411,990925,995942,996196,999500,996219,995502,997079,992630,997913,992291,998500,998797\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 244\n", + "Returned IDs: 993158,990620,996389,998647,995410,995511,999715,996609,998827,992215,990731,995271,999240,997893,993473,997195,994334,993690,997859,992971,996476,990249,996127,999540,998213,992558,995388,993147,999562,995847\n", + "Ground Truth: 993158,990620,996389,998647,991809,995410,995511,999715,996609,998827,992215,990731,995271,999240,997893,994404,993473,997195,994334,993690,997859,992971,996476,990249,996127,993702,999540,990782,991447,998213\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 245\n", + "Returned IDs: 998913,995633,990126,990340,993219,996809,998495,990359,995289,995558,994382,993209,997975,994742,996226,994806,997209,996930,997023,996219,994835,995211,992566,996232,996328,992827,999646,993747,990621,995438\n", + "Ground Truth: 998913,995633,990126,990340,993219,996809,998495,990359,995289,995558,994382,997723,993209,997975,994742,998188,996226,994806,997209,996930,997023,996219,994835,996411,995211,992566,996232,996328,992827,995401\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 246\n", + "Returned IDs: 992646,999089,994670,996993,995336,999092,990595,998011,997479,996714,994197,991361,995793,992911,992183,998465,993596,997192,994757,998210,995824,999144,990275,994673,990647,995833,997605,994152,998951,995726\n", + "Ground Truth: 995374,990478,992646,999089,994670,996993,998385,995336,999092,990595,998011,995985,997479,996714,994197,991361,995793,992911,992183,998465,993596,997192,994757,998210,995824,999144,990275,994673,990647,995833\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 247\n", + "Returned IDs: 995473,992815,991045,992694,999616,998524,990588,995981,991900,997304,992874,997437,997320,993392,997913,998365,997353,999291,999725,998246,992074,998380,990535,998353,995701,991745,992384,994266,990862,995942\n", + "Ground Truth: 995473,992815,991045,992694,999616,998524,990588,997620,995981,991900,997304,992874,997437,993300,997320,993392,997913,998365,997353,999291,999725,998246,992074,998380,990535,998353,995701,991745,992384,992927\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 248\n", + "Returned IDs: 997377,995701,992851,994140,991349,995016,995813,996714,991154,991869,994922,995638,998831,994392,993738,991795,995245,999729,995184,990930,996368,995899,998200,991969,997928,995237,995555,999653,996001,994463\n", + "Ground Truth: 997377,995701,992851,997622,994140,991349,995016,995813,996714,991154,991869,994922,995638,998831,994392,993738,991795,995245,999729,995184,990930,996368,995899,998200,991969,997928,995237,990615,995555,999653\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 249\n", + "Returned IDs: 997529,995020,990711,998719,999500,992156,991189,994120,998495,994382,993725,996411,995633,990126,991881,991669,990435,991075,997485,998671,994730,998779,998982,993656,998862,990340,995002,993053,997914,998067\n", + "Ground Truth: 997529,995020,990711,998719,999500,992156,991189,994120,998495,994382,993412,993725,996411,997207,995633,990126,991881,991669,990435,991075,993439,997485,998671,994730,993747,998779,998982,993656,998862,990256\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 250\n", + "Returned IDs: 997723,995289,997046,993390,996867,998045,992160,995373,997466,993433,991603,996166,994409,993561,996322,997406,998070,995401,991415,996968,993831,997822,990836,999795,998165,996323,995702,996302,996360,991664\n", + "Ground Truth: 997723,995289,997046,993390,996867,998045,992160,993412,995373,997466,993433,991603,996166,994409,993561,996322,997406,998070,995401,996042,991415,996968,993831,997822,990836,999795,998165,996323,995702,996302\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 251\n", + "Returned IDs: 999826,997395,990790,999020,998080,994612,994170,996245,997184,993931,995807,997986,998820,994770,997301,995163,998049,992790,990243,997189,994280,993149,999967,991698,999898,995521,997918,993933,994774,997168\n", + "Ground Truth: 999826,997395,990790,999020,998080,994612,994170,996245,997184,993931,995807,997986,992879,998820,994770,997301,995163,998049,992790,990243,997189,994280,998014,993149,992683,999967,991698,999898,995521,997918\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 252\n", + "Returned IDs: \n", + "Ground Truth: 992198,990084,997407,991432,995631,997548,996742,992494,998754,995510,992459,992824,992462,996196,992095,997758,998693,997055,997547,998061,997931,997775,996065,990192,997744,996663,991977,993828,995955,998614\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 253\n", + "Returned IDs: 994020,995168,995231,990998,999783,993504,991691,995701,995695,997965,998959,998855,993342,991459,995780,999724,998623,994105,994906,997506,993076,995894,997618,997977,994365,998831,992862,997906,994125,999653\n", + "Ground Truth: 994020,995168,995231,990998,999783,993504,991691,995701,995695,997965,998959,998855,993342,991459,995780,993821,999724,998623,994105,994906,997506,993076,995894,997618,998514,997977,994809,994365,998831,992862\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 254\n", + "Returned IDs: 996633,994830,991885,992227,990735,992006,991390,993335,996603,996328,997026,998055,996314,995735,994806,996495,993725,994682,999496,993560,993100,998761,991382,994751,994960,990484,995002,995191,991610,998610\n", + "Ground Truth: 996633,994830,991885,992227,990735,992006,991390,993335,996603,996328,997026,998055,996314,998716,995735,994956,994806,996495,993725,994682,999496,993560,993100,998761,991382,994751,994960,990484,995002,995191\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 255\n", + "Returned IDs: 991817,998061,999763,994163,997614,996380,991432,999908,991871,993706,992824,997775,998308,992423,994270,991064,990538,990908,990894,995503,995459,990192,995983,999639,992462,995737,991550,991923,993839,990096\n", + "Ground Truth: 991817,998061,999763,994163,997614,996380,991432,999908,991871,993706,992824,997775,998308,992423,994270,991064,990538,990908,990894,995503,995459,992280,990192,995983,999639,992462,995737,991550,991923,993839\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 256\n", + "Returned IDs: 994125,995516,990897,990005,998697,995536,996918,993560,998758,997305,998536,994879,991530,997586,995672,991052,991630,993055,993948,998623,995963,991038,997752,997420,995701,993809,995455,997506,995237,995783\n", + "Ground Truth: 994125,995516,990897,990005,998697,995536,996918,993560,995401,997429,998758,997305,998536,994879,991530,997586,995672,991052,991630,993055,993948,998623,992293,995963,991038,997752,997420,995701,993809,995455\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 257\n", + "Returned IDs: 991720,993143,998921,995928,997370,991743,999077,992384,993785,992147,995336,995949,997918,996592,999710,997002,993260,995826,990366,997301,995515,996053,991249,994964,997661,993080,998090,995886,996504,994334\n", + "Ground Truth: 991720,993143,998921,995928,997370,991743,999077,992384,993785,998294,992147,995336,995949,997918,996592,999710,997002,993260,999109,995826,990366,997301,996829,995515,991626,996053,991249,994964,997661,993080\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 258\n", + "Returned IDs: 990827,990043,998983,998937,997370,996840,998551,997789,995988,999620,998734,991196,994922,999898,990858,995515,993282,998236,991348,997255,993647,992169,993143,993237,995443,996100,992641,996681,990332,998738\n", + "Ground Truth: 990827,990043,998983,998937,990849,997370,996840,998551,997789,995988,999620,994735,998734,991196,994922,999898,990858,997445,995515,993282,998236,991906,991348,997255,990500,993647,992169,993143,993237,995443\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 259\n", + "Returned IDs: 996219,993168,995876,993637,995001,998343,990173,998495,997331,999673,996067,990784,999430,997992,991037,995562,996549,996062,994279,992589,995438,990323,999748,990891,992576,990802,996408,997555,990958,993906\n", + "Ground Truth: 996219,993168,995876,995001,993637,999647,998343,993499,990173,999009,995689,998495,997331,999673,996067,990784,999430,997992,991037,995562,996549,991758,996283,996062,994279,991109,992589,995438,990323,991372\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 260\n", + "Returned IDs: \n", + "Ground Truth: 990796,995344,999829,992056,998855,990167,996249,999555,991038,999375,991037,998383,997332,997378,990891,992664,994193,999435,998335,991758,996702,994105,999430,996809,995895,995562,990194,998495,998594,996001\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 261\n", + "Returned IDs: 998917,997220,994617,994491,997826,995259,998782,998139,991727,995588,998983,994534,991999,992085,998928,995334,999535,991806,993340,990809,994528,994785,999147,995316,993143,999592,993282,995788,993570,994318\n", + "Ground Truth: 998917,997220,994617,994491,997826,997136,995259,998782,998139,991727,995588,998983,994534,992255,994741,990822,991999,999469,992085,998928,995334,999535,991806,991405,993340,990809,994528,998236,990890,994785\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 262\n", + "Returned IDs: 995345,993591,990368,990889,990928,995283,997933,994148,992415,990089,994379,999126,990918,997822,994663,997319,993053,997386,998165,996147,996906,998162,994783,992894,997529,994021,998765,995289,995295,994689\n", + "Ground Truth: 995345,993591,990368,990889,990928,995283,997933,994148,992415,990089,994379,999126,990918,997822,994663,997319,993053,997386,993886,998165,991923,991973,996147,996906,998162,994783,991537,992894,996382,997529\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 264\n", + "Returned IDs: 991137,996653,999533,998302,993845,997320,997921,999109,993579,996726,997381,995841,994978,999338,994332,999827,991714,997984,991787,992318,993807,992228,995659,992647,998797,990146,992896,992617,995372,993484\n", + "Ground Truth: 991137,996653,999533,998302,993845,997855,997320,997921,999109,993579,996726,990179,993066,997381,995569,995841,994978,999338,994332,999827,991714,997984,991787,992318,993807,992228,995659,998445,992647,998797\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 266\n", + "Returned IDs: 999356,998353,995981,998163,993853,998246,997728,992296,994879,990231,998662,992074,996075,993529,993186,990708,996609,994100,996384,992848,999740,997192,998950,998332,991721,992661,998119,997305,998697,992546\n", + "Ground Truth: 999356,998353,995981,998163,993853,998246,997728,992296,990453,994879,995755,990231,998662,991782,992074,996075,993529,993186,991913,990708,996609,994100,999761,996384,992848,990799,991895,999740,997906,997192\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 267\n", + "Returned IDs: \n", + "Ground Truth: 991685,991021,995945,997453,996062,997541,993472,998416,990899,990407,992182,997813,993665,992780,991633,990382,998593,998158,995895,995834,996566,996071,993931,990104,992664,994925,995421,999435,991857,999827\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 268\n", + "Returned IDs: 992944,995360,991848,993781,990043,999784,998991,994543,994341,991013,999210,997135,993708,998950,993260,996193,996802,998180,999122,997457,993279,995203,997379,992724,998123,993215,992907,997471,994171,992950\n", + "Ground Truth: 992944,995360,991848,993781,996829,990043,999784,998991,994543,994341,991013,999210,999577,997135,993708,998950,993260,996193,991416,996802,998180,990182,992168,995375,999122,996007,993365,997037,993633,997457\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 269\n", + "Returned IDs: 996429,997918,995701,998932,994276,996217,998303,991530,994221,993260,990409,993055,995932,990930,999384,994754,992168,998697,997897,990751,996196,993504,995203,992571,997586,990494,991332,999860,993087,992163\n", + "Ground Truth: 996429,997918,995701,998932,994276,995785,996217,992047,998303,991530,994832,994221,993260,990409,993055,995932,990930,998123,999384,994754,992168,998697,997897,990751,996196,993504,995203,994514,990425,992571\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 270\n", + "Returned IDs: 992562,997925,999639,995701,995005,999041,995886,991154,993809,996067,998200,991459,997337,996918,999219,991609,992927,994510,997918,995932,994081,993673,990897,995980,999444,999784,999122,998536,999384,997586\n", + "Ground Truth: 990530,992562,997925,999639,995701,995401,995005,999041,990588,993821,995886,990217,991154,993809,990930,996067,998200,991459,993834,997337,999680,996918,992933,999219,991609,992927,994510,997918,995636,995932\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 271\n", + "Returned IDs: \n", + "Ground Truth: 990736,995985,993387,995942,999263,997301,993250,996785,990595,991444,994978,997913,995841,998692,999291,997543,997972,993562,998170,990390,992709,997539,997704,991008,996218,992107,997192,997620,997320,998960\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 272\n", + "Returned IDs: 995208,996968,996867,998297,995401,994794,999530,992169,990432,995274,994409,995373,997191,990561,997871,998045,994301,995515,993143,996727,998080,991052,997723,996504,996769,995783,992364,992944,996903,990995\n", + "Ground Truth: 995208,996968,996867,998297,995401,990337,994794,999530,992169,993412,990432,995274,994409,998536,995373,990561,997191,994677,997871,998045,994301,995515,997775,997403,995374,993143,996727,998080,991052,996438\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 273\n", + "Returned IDs: 990072,999384,998861,997666,990096,997906,996623,997685,991459,995780,991471,993275,997265,999362,998623,996209,998536,995701,993561,999272,996733,994105,999827,999319,994955,997576,999736,995536,999194,990768\n", + "Ground Truth: 990072,999384,998861,997666,990096,997906,996623,991188,997685,991459,995780,998419,991471,993275,997265,999362,998623,993821,996209,998536,995701,990194,992977,998383,999653,993561,999272,996733,994105,994906\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 274\n", + "Returned IDs: 995702,995907,994380,996867,991161,993044,993188,997675,993597,996042,992576,994419,992212,993313,991794,999528,998106,991011,996388,997289,997822,992894,998165,995521,995666,999619,997020,991722,992413,993127\n", + "Ground Truth: 995702,995907,994380,996867,991161,993044,993188,997675,993597,996042,992576,994419,992212,993313,991794,999528,998106,991011,996388,997289,997822,992894,998165,995521,995666,999619,990725,997020,991722,992413\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 275\n", + "Returned IDs: 990020,997320,990532,997661,997918,993785,991795,992950,998302,996646,999446,995852,996075,993906,999444,994171,997227,991984,999990,994081,993561,999473,996001,996028,992546,995203,993157,993484,995993,994964\n", + "Ground Truth: 990020,997320,991989,990532,997661,997918,990207,993785,999219,991795,992950,991829,998634,998302,996646,999446,995852,996075,991311,993906,993412,999444,994171,997246,997227,997906,996735,994307,991984,999990\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 277\n", + "Returned IDs: 992546,990515,991431,999180,992339,992994,997614,990958,998246,990118,997305,994757,997437,996075,998797,991479,995981,994079,997320,990660,998562,994776,997543,997470,991045,997913,993662,998101,992119,997192\n", + "Ground Truth: 992546,990515,991431,993392,992583,999680,992238,999180,992339,992994,997614,990958,994653,998246,990118,997305,994757,997437,996075,998797,991479,995981,994079,997320,990660,998562,997976,994776,997543,997470\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 278\n", + "Returned IDs: 994427,990231,994403,992969,995697,998550,994171,996646,999710,994450,992119,993662,997320,994699,990135,999167,997354,990910,996540,990066,990182,997896,990957,999328,994961,991260,994604,997913,993264,990650\n", + "Ground Truth: 994427,990231,994403,996921,992969,993220,995697,992363,991743,998550,994171,996646,999710,994450,992119,993662,993278,997320,994699,991320,994048,995351,990135,995942,999167,994192,997354,990910,996540,990066\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 279\n", + "Returned IDs: 996388,993390,999080,995665,998979,996379,993561,991415,995289,993144,997406,995701,996719,991857,998128,992125,994811,991954,998045,998874,991630,997787,991113,996924,991011,997046,994113,996181,998165,991603\n", + "Ground Truth: 996388,993390,999080,995665,998979,996379,993561,991415,995289,993144,997406,995701,996719,991857,998128,992125,994811,995761,991954,996294,998045,998874,991630,995864,997787,991113,996924,991011,997046,994113\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 280\n", + "Returned IDs: 992628,996016,996794,995535,991382,993391,996474,999280,998803,996184,998084,994525,996938,995880,999205,997798,995735,992237,997084,996839,999634,992867,992073,998153,992754,995346,996650,994472,992100,995191\n", + "Ground Truth: 992628,996016,996794,995535,991382,993391,996474,999280,998803,996184,998084,994355,994525,996938,995880,999205,997798,995735,992237,997084,996839,999634,992867,992073,998153,992754,995346,996650,994472,992100\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 281\n", + "Returned IDs: 991269,993209,997023,993586,999957,996607,995778,997723,991507,997544,994496,999054,998188,991639,999415,999047,991140,998779,991136,999563,996867,997975,990372,992413,994881,992582,995289,997832,995401,995665\n", + "Ground Truth: 991269,993209,997023,993586,999957,996607,995778,997723,991507,997544,994496,999054,998188,991639,991943,999415,999047,991140,998779,997101,991136,999563,996867,992334,997975,990372,998295,992413,994881,992582\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 282\n", + "Returned IDs: 991820,993080,998720,991981,993342,995906,993809,990860,996911,999444,998593,995852,990532,999916,993290,997969,998710,995002,999922,996001,997790,993906,994636,990622,990078,992632,994783,993637,995184,995256\n", + "Ground Truth: 991820,993080,998720,991981,993342,995906,993809,998514,990860,996911,999444,995160,998593,995852,990532,996490,999916,990866,993290,999272,997969,998710,995002,999922,996001,991795,999647,997790,995961,995636\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 283\n", + "Returned IDs: 999912,998039,997920,996996,995384,995906,990737,997028,991565,997410,997529,990974,997345,995526,990315,999949,998169,993637,996219,994964,994293,999261,994783,999647,993378,991831,993168,998382,999151,990728\n", + "Ground Truth: 999912,998039,997920,996996,995384,995906,990737,997028,991565,997410,997529,990974,997345,995526,990315,999949,998169,993637,996219,994964,995222,994293,991359,999261,990891,994783,999647,993378,991831,995401\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 284\n", + "Returned IDs: \n", + "Ground Truth: 996524,993012,990366,997811,991373,997466,998568,993696,999202,992330,990866,996857,993628,992238,993834,997586,993823,997690,997806,995132,992019,998403,992068,995268,995516,999598,994879,990991,992521,997287\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 285\n", + "Returned IDs: 993805,990256,991052,995516,990897,994869,996406,999646,996811,994956,991982,996918,996695,995432,996762,997126,998318,990563,992795,993948,994125,991038,996861,991860,998123,992558,994510,993143,999820,994558\n", + "Ground Truth: 993805,990256,991052,993255,994714,995516,990897,994869,996406,999646,999676,993673,996811,994956,991982,996918,996695,992979,992147,995432,996762,997126,998318,991857,991824,990563,992795,993948,991977,994125\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 286\n", + "Returned IDs: 998779,998862,997640,993412,993293,997204,996727,999026,994382,996436,997319,999021,994409,995401,992169,995216,997355,994436,995374,995940,995274,997871,997832,995685,990675,990940,995200,996379,996368,997320\n", + "Ground Truth: 998779,998862,997640,993412,998999,993293,997204,996727,993831,999026,994382,996436,997319,997732,999021,991923,994409,995401,992169,995216,997355,994436,993280,995374,995940,991743,993226,996042,995274,997871\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 287\n", + "Returned IDs: 994313,990696,995783,999496,999871,990872,996087,997914,991319,990386,998293,999105,997919,997752,996362,990083,993678,999179,994051,993621,990061,995464,993727,993441,996814,993080,992571,996429,996825,991269\n", + "Ground Truth: 994313,990696,995783,999496,999871,990872,996087,997914,991319,990386,998293,999105,997919,997752,997506,996362,998388,990083,997429,993678,999179,994051,993621,992617,990061,995464,993727,993441,993080,996814\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 288\n", + "Returned IDs: 996249,991977,990796,994193,994605,995760,995727,998855,996882,999281,999375,998450,993316,990487,993681,992435,995357,993500,990776,992376,990206,992056,995618,992558,993715,990998,996809,995965,992052,997309\n", + "Ground Truth: 996249,991977,990796,994193,994605,995760,995727,998855,996882,999281,999375,998450,993316,990487,993681,992435,995357,993500,990776,992376,990206,992056,995618,992558,993715,990998,997104,996809,995965,992052\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 289\n", + "Returned IDs: 995289,993844,998014,992488,992289,994204,993433,995766,993143,998536,997319,993762,996867,994783,991840,997897,993232,995633,992052,998913,990941,999453,997822,996809,995788,996769,990882,990681,990340,998165\n", + "Ground Truth: 995289,993844,997403,998014,992488,992289,994204,993433,995401,995766,996438,993143,993673,998536,997319,993762,996867,994783,996930,991840,993232,997897,995633,992052,998913,990337,990941,992407,993209,999453\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 290\n", + "Returned IDs: 996657,993848,998593,990662,990811,994587,992293,992367,991185,998549,995289,991603,993390,990255,998779,993738,998165,996001,992703,991835,997822,994809,996759,996368,996147,998862,995158,991630,993561,996277\n", + "Ground Truth: 996657,993848,998593,990662,990811,994587,996864,992293,995401,992367,991185,998549,995289,991603,997723,993390,990255,998779,993738,998165,996001,992703,991835,997822,999587,990930,994809,996759,996368,996294\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 291\n", + "Returned IDs: 991795,996987,992448,996187,994409,998143,998361,998205,995864,994954,994872,996256,993003,999250,999926,998941,991969,998756,991349,997868,990341,993672,990603,995016,996162,998527,996919,991892,998278,994865\n", + "Ground Truth: 991795,996987,992448,996187,994409,998143,998361,998205,995864,994954,994872,996256,993003,999250,999926,998941,991969,998756,991349,997868,990341,999866,993672,990603,995016,996162,998527,996919,991892,998278\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 292\n", + "Returned IDs: 993127,993400,997674,995891,995470,999884,995423,998651,992448,995032,994273,991276,998147,999918,996621,999255,992967,996472,995122,994413,998321,999043,994113,995905,992048,994419,997675,994409,991699,992753\n", + "Ground Truth: 993127,993400,997674,995891,995470,999884,995423,998651,992448,995032,994273,991276,998147,999918,996621,999255,992967,996472,995122,994413,998321,999043,994113,995905,992048,994419,997675,994409,991699,998973\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 293\n", + "Returned IDs: 994672,994247,993715,999139,994577,995801,995349,993504,991552,993055,995839,993139,991245,995737,996884,993181,991648,994351,993609,999079,990885,995622,992747,992052,991332,999806,993489,999927,991605,999526\n", + "Ground Truth: 994672,994247,993715,999139,994577,995801,995349,993504,991552,993055,995839,993139,991245,995737,996884,993181,994743,991648,994351,993609,999079,990885,995622,997076,992747,992052,991332,999806,993489,999927\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 294\n", + "Returned IDs: 995139,995905,995507,999619,996038,991027,995470,996323,993400,997046,999578,997832,995547,992448,997139,993421,995923,997674,999508,990975,996867,994554,999530,993597,997252,996322,994278,991276,994409,994419\n", + "Ground Truth: 995139,995905,995507,999619,993468,992660,996038,991027,995470,996323,993400,997046,999578,997832,995547,997139,992448,993421,995923,997674,999508,990975,996867,994554,999530,990634,994002,993597,997252,996322\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 295\n", + "Returned IDs: 998294,998380,995949,999263,997301,994964,990561,998697,998090,997320,991530,995928,998029,997913,993342,999783,999444,995802,995672,998302,998365,997371,998692,990210,990366,998303,990146,999639,991743,990958\n", + "Ground Truth: 998294,998380,995949,999263,997301,994964,990561,998697,998090,997320,991530,991157,995928,998029,997913,993342,999783,999444,995802,995672,998302,997032,995638,998365,999331,997371,990310,998692,998607,997135\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 296\n", + "Returned IDs: 991720,992147,997842,998921,995372,992169,996915,991743,997192,995515,991889,998877,993080,991485,999698,997918,995927,992944,995443,994280,995771,995336,990432,998229,991503,994919,997984,995788,992942,993215\n", + "Ground Truth: 991720,999343,992147,997842,998921,995372,992169,996915,998259,993857,991743,997192,995515,991889,998877,993080,991485,999698,997918,995927,992944,991406,995443,994088,994280,995771,990648,995336,990432,992309\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 297\n", + "Returned IDs: 999616,995981,997437,999180,990588,999596,992296,993168,997320,995336,997759,997353,998353,995372,992748,994695,998828,991045,997875,990339,996075,998607,995184,993080,997309,994894,994964,990957,995927,997018\n", + "Ground Truth: 999616,995981,991720,995792,997437,999099,999180,990588,999596,992296,993168,998567,997320,994916,990778,995336,997759,997353,998353,995372,999527,992748,994695,993893,998828,991045,997875,990339,996075,998607\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 298\n", + "Returned IDs: 994745,991980,998716,998803,998903,991911,994830,993335,990730,996153,993101,998584,998030,991554,993568,996603,990678,991187,995735,997952,992495,996357,994683,992598,996183,991081,991297,993522,998440,998949\n", + "Ground Truth: 994745,991980,998716,998803,998903,991911,994830,993335,990730,996153,993101,998584,998030,991554,993568,996603,990678,991187,995735,997952,999786,992495,996357,994683,992598,996183,991081,998484,996503,991297\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 299\n", + "Returned IDs: 998039,991869,990076,993561,995923,996028,990845,995289,998633,990096,999729,993164,994925,990930,994853,994409,992851,994140,995813,997716,995526,991958,996001,990634,999516,995401,996062,991795,997386,997170\n", + "Ground Truth: 998039,991869,990076,993561,995923,996028,990845,995289,998633,990096,999729,993164,994925,990930,994853,994409,992851,994140,995813,997716,995526,991958,996001,990634,999516,995401,995678,996062,991795,997386\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 300\n", + "Returned IDs: 990602,996367,993583,993534,999305,990226,996016,992306,991432,993255,998357,999540,991605,992113,997547,993839,998652,993427,999479,997127,995618,995631,991977,993380,996742,999908,990423,994254,994560,994472\n", + "Ground Truth: 990602,996367,993583,993534,999305,990226,996016,992306,991432,993255,998357,999540,991605,992113,997547,993839,998652,993427,999479,997127,995618,995631,991977,993380,995542,996742,999908,990423,994254,994560\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 301\n", + "Returned IDs: 997585,994922,991392,995557,999620,996402,995057,990043,990090,994964,993484,996563,998837,993389,999437,993237,999484,994789,991150,992944,993355,993168,990229,990585,998765,992907,990690,994685,996219,990483\n", + "Ground Truth: 997585,994922,991392,995557,999620,996402,995057,990043,990090,994964,993484,996563,998837,993389,999437,991582,993237,999484,994789,991150,992944,993355,996419,993168,992346,993775,990229,990585,998765,997876\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 302\n", + "Returned IDs: 996606,993534,993427,995152,991983,995618,992113,994679,992857,993380,992558,990279,995710,999540,997849,992496,990345,997547,997917,991046,996750,998151,992092,998840,991026,990125,996367,993583,992306,999874\n", + "Ground Truth: 996606,993534,993427,995152,991983,995618,992113,994679,992857,993380,992558,990279,995710,999540,997849,992496,991058,990345,997547,997917,991046,996750,992335,998151,991871,992092,998840,991026,990125,996367\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 303\n", + "Returned IDs: 995459,998142,992824,997775,999171,994679,991432,991817,993706,993077,991550,994276,993583,997864,999908,992392,995503,993647,993828,995983,996219,992280,999639,995737,998061,990765,993260,990538,993891,999560\n", + "Ground Truth: 995459,998142,992824,997775,999171,994679,991432,991817,991370,993706,993077,991550,994276,993583,997864,999908,992392,995503,993647,993828,997906,995983,996219,992280,999639,995737,998061,990765,991923,990538\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 304\n", + "Returned IDs: 999504,997388,997157,996727,990797,991923,991082,998094,997640,999960,994406,999374,997083,995401,998529,997634,998559,999820,990475,998607,993412,990275,991977,994514,998779,997748,998179,999462,995369,991334\n", + "Ground Truth: 999504,997388,997157,996727,990797,991923,991082,998094,997640,999960,994406,999374,997083,995401,998529,997799,997634,998559,999820,990475,998607,995320,993412,990275,991977,994514,998779,997748,998179,999462\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 305\n", + "Returned IDs: 990644,997702,997002,991392,991485,990999,992979,999219,991459,993355,999453,995894,995875,993848,991157,998385,999977,996425,994946,998247,998401,995154,995515,994334,995016,998303,994906,996840,996368,992648\n", + "Ground Truth: 990644,997702,997002,991392,991485,990999,997533,994362,995374,992979,991894,999219,991459,993355,999453,995894,995875,993848,991157,998385,992750,999977,996425,994946,991151,998247,998401,995154,992047,995515\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 306\n", + "Returned IDs: 997475,994891,993611,994282,993821,991153,998065,996535,991066,990301,993046,990479,990961,996728,999775,999832,991597,994316,991111,994280,999205,996495,996431,990774,997919,995600,998221,991709,994196,998537\n", + "Ground Truth: 997475,994891,993611,994282,993821,991153,998065,996535,991066,990301,993046,990479,990961,996728,999775,999832,991597,994316,991111,998279,994280,998320,999205,996743,996495,996431,990774,997919,995600,998221\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 307\n", + "Returned IDs: 995260,995755,992546,994266,990925,995132,991829,997737,996001,997320,995927,993168,994776,997796,999444,991743,990146,996075,990958,992994,992147,996714,990096,993648,998326,996219,998365,993484,999647,998246\n", + "Ground Truth: 995260,995755,992546,994266,990925,995132,991829,997737,996805,996001,997320,995927,993168,994776,997796,999444,991743,990146,991582,996075,990958,992994,992147,997876,996714,990096,993648,998326,996219,998365\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 308\n", + "Returned IDs: 993426,990191,992451,991873,999271,999904,995623,995045,997292,997301,997984,996516,998921,993484,990546,995306,992709,990512,998760,991720,997397,996771,998425,999110,992107,992027,991916,999533,994618,997320\n", + "Ground Truth: 993426,990191,992451,991873,999271,999904,995623,995045,997292,997301,997984,996516,998921,998603,993484,990546,995306,995865,993048,992709,990512,998760,991720,997397,996771,998425,999110,992107,992027,991749\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 309\n", + "Returned IDs: 997233,997059,994205,998725,994013,996308,999130,990241,990662,996028,998982,999099,997452,999570,997554,996657,994382,991382,995295,995216,996001,998752,996147,999623,998055,996411,995544,995456,999342,995158\n", + "Ground Truth: 997233,997059,994205,998725,993857,994013,996308,999130,990241,990662,996028,995600,998982,999099,997452,999570,994131,997554,996657,993909,994382,991382,995295,995216,996001,998752,999623,996147,998055,998156\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 310\n", + "Returned IDs: 993567,998104,990872,994711,998293,994909,996435,993727,992457,992569,993799,992052,999796,990116,997897,992437,998162,997752,990956,992981,993180,993497,996825,991245,992028,995766,995421,994353,994435,993347\n", + "Ground Truth: 993567,998104,990872,994711,998293,999190,994909,996435,993727,994300,992457,992569,997949,993799,992052,999796,990116,997897,992437,998162,997752,990956,992981,992759,993180,993497,996825,991245,997506,992028\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 311\n", + "Returned IDs: 992041,992124,999639,997258,992238,990453,993696,998758,998260,992167,992225,991984,995516,993834,992927,994809,993283,990949,997370,997320,999041,998950,991157,992521,991459,990862,996500,995899,996075,992896\n", + "Ground Truth: 992041,992124,999639,997258,992238,990453,992497,993696,998758,998260,992167,992225,991984,995516,993834,992927,994809,993283,990949,997370,997320,999041,998950,991157,992521,991459,990862,996500,995899,996075\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 312\n", + "Returned IDs: 990821,990310,991840,995895,991157,993717,996809,991015,995633,993391,994922,993143,996408,998950,996840,994830,991848,993342,999453,999784,996328,994742,999122,992169,992612,997822,994964,993260,997918,990080\n", + "Ground Truth: 990821,990310,991840,995895,991157,990271,993717,996809,991015,995633,990941,993391,997331,994922,993642,993143,996408,998033,998950,996840,994830,991848,993342,999453,999784,996328,994742,999122,992169,992612\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 313\n", + "Returned IDs: 996936,999337,998794,991805,999157,993890,996944,998926,992499,998109,997047,991915,997270,995647,996006,996650,996796,992780,993472,995690,994558,992872,993721,994452,996475,994983,994832,991179,995596,996219\n", + "Ground Truth: 996936,999337,998794,991805,999157,993890,990780,996944,998926,992499,998109,997047,991915,997270,995647,996006,996650,997561,996796,992780,993472,995690,992986,994558,992872,993721,994452,996475,997257,993772\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 314\n", + "Returned IDs: \n", + "Ground Truth: 998728,998232,992733,994440,996075,990142,995596,992751,991984,993902,997291,997661,998498,995203,992671,991461,994626,999426,996317,994341,990123,996735,998654,999132,990083,996741,992119,990914,990697,994115\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 315\n", + "Returned IDs: 992771,997395,993566,990194,993106,996393,999994,993931,999673,995895,992685,996408,990951,999543,995620,999555,991037,999430,990407,993224,998536,998006,990796,990434,993011,996395,998080,991148,990167,991052\n", + "Ground Truth: 992771,997395,993566,990194,993106,996393,999994,993931,999673,995895,992685,991188,996408,990951,999543,995620,999555,991037,999430,990407,993224,998536,998006,996762,990796,990434,993011,995211,998383,995401\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 316\n", + "Returned IDs: 999932,995269,993465,996046,996903,995685,997046,994819,990459,997918,994804,999026,993407,993935,995444,995373,993433,997002,996734,999000,997822,996323,994334,990940,991451,995928,991415,995204,990215,998023\n", + "Ground Truth: 999932,995269,993465,996046,996903,995685,997046,994819,990459,997918,994804,999026,993407,990995,993935,995444,993412,995373,991698,993433,997002,992933,996734,999000,997822,996323,994334,990940,991451,999089\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 317\n", + "Returned IDs: 999139,991382,995895,998671,997506,995191,996328,994142,990392,999740,994280,994830,993053,998716,998495,991633,997933,996013,991672,992680,994472,993391,992636,992445,999086,999430,991390,995476,995295,996537\n", + "Ground Truth: 999139,991382,995895,998671,997506,995191,996328,994142,990133,990392,999740,994280,994830,999377,993053,998716,998495,991633,997933,996013,998030,991672,992680,994472,990293,993391,996775,992636,999441,992445\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 318\n", + "Returned IDs: 991144,993719,990257,993917,992636,993715,996986,990438,990951,997168,999334,996357,993150,993637,995991,998076,998716,998561,990089,997879,998348,996408,997790,996328,995002,994834,993139,996201,990289,999827\n", + "Ground Truth: 991144,993719,990257,993917,992636,993715,996986,990438,990951,997168,999334,996357,993150,993637,995991,998076,998716,998561,990089,997879,998348,996408,997790,996328,995002,994522,994834,991081,997669,993139\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 319\n", + "Returned IDs: 992747,998303,993489,991037,992571,990350,998401,999840,990409,991332,995536,990930,996217,997002,997977,992082,995894,996062,993504,994200,995203,993260,990885,990096,991530,997368,995025,995823,991425,992823\n", + "Ground Truth: 992747,998303,993489,991037,992571,990350,998401,999840,990409,991332,995536,990930,996217,994468,997002,997977,992082,995894,996062,993504,994200,995203,993260,990885,990096,991530,996677,997368,995025,995823\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 320\n", + "Returned IDs: 993809,998080,998495,996371,992571,993255,996370,992318,990083,998950,993696,992847,993515,995932,990532,999701,997586,992944,996805,997184,991412,990711,998416,990776,999179,992748,995552,996918,996809,995289\n", + "Ground Truth: 993809,998080,998495,995401,996371,992571,993255,991052,996370,992318,990083,998950,993696,992847,993515,997723,992753,995932,990532,993478,999701,998221,997586,992944,996805,997184,991412,993444,990711,998416\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 321\n", + "Returned IDs: 990999,998990,998179,995401,997258,991459,997191,996379,996198,996727,993561,990354,993848,999219,992684,991392,991630,999021,994946,994319,996247,999639,990838,995994,990561,992979,997640,993355,997158,997380\n", + "Ground Truth: 990999,998990,998179,995401,997258,991459,997191,996379,996198,996727,993561,990453,997799,990354,993848,999219,992684,991392,991630,999021,994946,993412,994319,996247,999639,990838,991792,999228,995994,990561\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 322\n", + "Returned IDs: 990866,997379,997586,993708,996397,993637,993552,999308,999647,999500,995701,997013,992068,991157,997811,992944,991808,997555,999598,998180,992682,995455,999473,999337,998550,991015,994341,991348,995360,994307\n", + "Ground Truth: 990866,997379,997586,993708,996397,993637,996429,993552,999308,999574,999647,999500,995701,997013,992068,990547,991157,997811,992944,991808,999216,992522,992047,997555,999598,997496,998180,998790,992682,995455\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 323\n", + "Returned IDs: 993160,995062,998080,999961,992150,997634,998806,997061,997395,998316,998650,992677,990131,993255,993263,994576,990789,995886,996273,992161,992604,994774,996139,993122,992493,991360,992558,992148,993740,990957\n", + "Ground Truth: 993160,995062,998080,993971,999961,992150,997634,998806,997061,997395,998316,998650,992677,997429,990131,993255,993263,994576,990789,995886,996273,990103,992161,992604,994774,996139,993122,992493,998528,991360\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 324\n", + "Returned IDs: 991825,996028,994689,995823,998140,998825,993378,997051,990811,997373,994012,991840,998982,993906,992201,997417,993890,998382,994307,995089,993725,991989,992780,999446,999740,995680,999286,996556,999613,991711\n", + "Ground Truth: 991825,996028,994689,995823,998140,998825,993378,997051,995649,990811,991426,997373,994012,991840,998982,993906,992201,997417,993890,998382,994307,995502,997541,995089,993725,991989,992780,999446,999740,995680\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 325\n", + "Returned IDs: 994630,996961,997525,994699,993518,991895,997913,998246,999710,992546,992709,994939,994932,995949,994757,998744,996785,997599,998332,990397,997535,990925,990509,991873,992119,991479,996890,991379,994427,992107\n", + "Ground Truth: 994630,996961,997525,999761,994699,993518,991895,996732,997913,998246,999710,992546,992709,994939,998016,994932,995949,994757,998744,997599,996785,998332,990397,992502,997535,990925,990509,991873,992119,991479\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 326\n", + "Returned IDs: 997976,998170,994894,997479,999182,994757,997320,999144,997918,995981,995279,995886,990231,994100,991379,990660,997571,995336,990958,994334,997192,990647,990588,995656,997605,997437,994280,993492,998951,995274\n", + "Ground Truth: 997976,998170,994894,997479,999182,994757,991782,997320,999144,997918,993673,995981,995279,995886,990231,994100,991379,990660,997571,998011,995336,990958,998163,994819,994334,997192,999527,991181,990647,990588\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 327\n", + "Returned IDs: 995274,996657,996001,995216,998302,999026,998779,996810,998952,990974,990845,996759,998862,995289,998058,990957,992792,999099,997529,996219,999343,993168,994730,996727,997059,999453,997554,993831,996147,995873\n", + "Ground Truth: 995274,996411,990293,996657,993412,996001,993909,995216,993003,998302,999026,998779,996810,998952,990974,991995,990845,996759,998862,995289,992120,995899,998058,999835,999587,990957,992792,997969,999099,997529\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 328\n", + "Returned IDs: 998257,995373,991204,997110,990657,993115,997568,994643,998858,994454,997589,999129,993795,991987,999530,991386,995401,993000,996393,997675,997683,992762,999136,998536,991052,990096,992861,998697,997918,990005\n", + "Ground Truth: 998257,995373,991204,997110,990657,993115,997568,994643,998858,994454,997030,997589,999129,993795,991987,999530,991386,995401,993000,996393,997675,997683,992762,999136,998536,991052,990096,992861,998697,995453\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 329\n", + "Returned IDs: 996001,992052,996657,996277,991977,997453,996505,991349,993280,999491,994743,998536,997168,993180,994409,990956,998593,993289,990930,994040,995200,992521,993561,994280,993931,992790,992718,995289,996042,990364\n", + "Ground Truth: 996001,992052,996657,996277,991977,997453,996505,991349,993280,999491,994743,998536,997168,993180,994409,995408,990956,998593,993289,990930,994040,995401,995200,992521,993561,994280,993931,999099,992790,992718\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 330\n", + "Returned IDs: 990255,997897,993232,990167,996657,998593,990361,991532,996395,998695,996809,993344,997752,990941,998786,992809,995167,993483,996001,996284,991255,990675,994587,996797,994451,996043,992437,999463,997131,990314\n", + "Ground Truth: 990255,997897,993232,990167,996657,998593,990361,991532,996395,996104,998695,996809,993344,997752,990941,998786,992809,995167,993483,996001,996284,991255,990675,994587,996797,994451,990211,996043,992437,999463\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 331\n", + "Returned IDs: 990154,998006,995815,999555,994181,997666,992771,995835,999673,990952,999543,992282,990951,994834,999388,994467,998861,996505,995655,994103,998067,995536,994756,997242,997427,994380,990537,997331,993931,995168\n", + "Ground Truth: 990154,998006,995815,999555,996728,994181,997666,992771,995835,999673,990952,990006,999543,997884,992282,990951,994834,999388,994467,998861,999350,999663,996505,995655,991596,994103,990429,998067,997358,995536\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 332\n", + "Returned IDs: 997452,991464,990614,998913,995295,993725,994742,991330,995633,997933,996116,994783,998779,994541,990701,995551,996411,995456,991303,998543,993053,990392,992434,997822,998495,992703,995289,998671,999302,997374\n", + "Ground Truth: 997452,991464,990614,998913,995295,993725,994742,991330,995837,995633,997933,996116,994783,998779,994541,990701,995551,996411,995456,991303,991537,999230,994580,998543,993053,990392,992434,997822,998495,992703\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 333\n", + "Returned IDs: 998109,999659,992780,997442,997296,992499,995476,991826,992070,991382,996809,995150,997270,996216,996650,993617,990039,998167,991633,998794,998495,991805,993342,999430,992872,998631,993391,998803,994927,994342\n", + "Ground Truth: 998109,999659,992780,997442,997296,992499,995476,991826,992070,991382,996809,995150,997270,996216,996650,993617,990039,998167,991633,998794,998495,991805,993342,999430,992872,999551,998631,993391,997318,998803\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 334\n", + "Returned IDs: 991377,992894,994419,995289,997386,997822,995702,995923,991923,993511,996080,990847,993561,996610,996867,991958,996360,997529,996745,996147,993433,994925,996322,995200,996127,994409,992130,997937,999782,996042\n", + "Ground Truth: 991377,992894,994419,995289,997386,997822,993412,995702,995923,991923,993511,996080,991603,990847,993561,996610,996867,991958,996360,997529,996745,996147,993433,994925,996322,995200,996127,994409,992130,997920\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 335\n", + "Returned IDs: 993079,993324,998289,999796,998543,997159,998104,994765,994119,992558,998180,990956,997420,990814,994742,993215,999179,990776,994541,995332,995438,995289,993293,990264,992832,997309,995401,998950,996210,994817\n", + "Ground Truth: 993079,993324,998289,999796,998543,997159,998104,994765,994119,992558,992933,998180,990956,993280,997420,990814,994742,993215,999179,998369,990776,994541,996370,995332,995438,995289,996178,993293,990264,992832\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 336\n", + "Returned IDs: 991454,998355,991177,993929,990103,990226,997732,994956,997244,998691,990712,993014,998631,991107,999304,994444,999673,996803,998343,994410,990678,990705,998495,997453,995382,999138,990951,991170,995919,992713\n", + "Ground Truth: 991454,998355,991177,993929,990103,990226,997732,994956,997244,993336,998691,996062,996362,990712,993014,998631,991107,999304,990408,994444,999673,997693,997807,998466,996803,991219,996942,998343,994410,994927\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 337\n", + "Returned IDs: 996531,998397,993747,990956,996276,999514,991747,998293,996867,992703,994119,994541,998165,996232,998486,992160,990116,997822,994444,999264,999646,990103,993727,997420,995159,999244,991145,995184,995144,994806\n", + "Ground Truth: 996531,998397,993747,990956,996276,999514,991747,998293,996867,998478,992703,994119,990466,994541,998165,996232,998486,992160,990116,997822,994444,999264,999646,990103,993727,996294,997420,995159,992992,999244\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 338\n", + "Returned IDs: 992112,990494,998715,992351,990885,997368,998018,997897,995932,996632,993504,992747,992825,997839,993171,997790,990818,995560,993260,995476,990697,990009,992780,991491,998473,992635,996216,994081,994280,993341\n", + "Ground Truth: 992112,990494,998715,992351,990885,997368,998018,997897,995932,996632,993504,992747,992825,997839,993171,997790,990818,992665,992137,995560,993260,995476,990697,990009,992780,995130,991491,998473,996424,992635\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 339\n", + "Returned IDs: 993209,997837,995778,995289,995766,995633,993997,994742,991185,992558,991507,994031,994743,999957,995106,996302,998748,992448,998188,993623,991044,997975,999415,997630,993003,995648,997319,994409,993265,997662\n", + "Ground Truth: 993209,997837,995778,995289,995766,995633,990361,993997,994742,991185,992558,995401,991507,994031,994743,999957,995106,996302,998748,995198,992448,998188,993623,991044,997975,991664,999377,999415,997630,993003\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 340\n", + "Returned IDs: 994662,994755,996276,995144,994073,995438,990956,999244,992315,993747,992934,997420,999601,992052,999514,998104,993805,996850,991747,993727,993742,993121,994541,999264,998293,999411,993448,999029,994336,993219\n", + "Ground Truth: 994662,994755,996276,995144,994073,995438,990956,999244,992315,993747,992934,997420,999601,992052,999514,998104,993805,996850,991747,993727,993742,993121,994541,999264,998293,999411,993448,999029,994336,995107\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 341\n", + "Returned IDs: 997386,995289,995923,994419,996080,993561,996867,998593,991603,998999,992894,997822,998165,997837,994925,995497,990811,995702,991958,992680,998382,993725,997046,998039,996322,995052,995200,996745,997406,993433\n", + "Ground Truth: 997386,995289,995923,994419,996080,993561,996867,998593,991603,998999,992894,997822,998165,997837,994925,995497,990811,995702,993412,991958,992680,998382,993725,995401,997046,998039,996322,995052,997630,995200\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 342\n", + "Returned IDs: 993364,992126,993885,990772,997380,990557,995841,994162,991212,991498,995372,997320,999982,995502,998444,994673,992788,995515,999453,996489,997973,997002,996001,996959,993032,994081,997301,997796,994088,990487\n", + "Ground Truth: 993364,992126,991994,993885,991792,990772,997876,997380,990557,993820,995841,994162,991212,991498,995372,997320,999982,995502,998444,994673,992788,995515,999453,996489,997973,999819,997002,996001,990724,996959\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 343\n", + "Returned IDs: 996442,991713,998260,996397,997158,999806,999731,996196,993785,998827,992423,998568,992927,994577,994163,990731,993071,991650,993504,997550,999598,996404,992041,995552,997977,991157,995820,999111,997320,997013\n", + "Ground Truth: 996442,991713,998260,996397,997158,999806,999731,996196,993785,998827,992423,998568,992927,994577,994334,994163,990731,993071,991650,993504,997550,999598,996404,992041,996979,995552,997977,993377,991157,995820\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 344\n", + "Returned IDs: 994277,993721,994233,994416,991409,996685,995899,992944,996024,997320,998950,991348,991416,998921,996210,996075,992318,995372,998913,996178,991650,991997,997305,995108,996001,991154,993018,991142,999701,995515\n", + "Ground Truth: 994277,993721,994233,994416,991409,996685,995401,997403,995899,992944,996024,997320,998950,991348,991416,998921,996210,993121,996075,994081,997448,990364,995372,992318,998913,996178,991650,991997,991743,997305\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 345\n", + "Returned IDs: 999827,996001,997320,994964,991498,993080,999453,994081,995841,992182,999099,994478,992395,993168,995216,997860,993280,997918,995274,990146,993289,995102,993484,991212,990532,993388,997371,990557,992469,999019\n", + "Ground Truth: 999827,996001,997320,994964,991498,993080,993454,997920,999453,994081,995841,992182,999099,999026,994478,992395,993168,995216,997860,998750,993280,997918,995274,990146,993477,993289,990772,993648,995102,991007\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 346\n", + "Returned IDs: 992907,995588,997465,998582,993091,999676,996840,995515,993622,999076,991840,990300,993355,990310,994922,991999,996686,993781,999077,994612,990432,993688,999620,991172,993143,991462,999784,993212,997496,999317\n", + "Ground Truth: 992907,995588,997465,998582,993091,999676,996840,995515,993028,993622,999076,991840,990300,993355,990310,994922,992979,999073,996352,999604,991999,996686,993781,999077,994612,990490,990432,993688,991160,992218\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 347\n", + "Returned IDs: 997909,997299,998890,996256,999925,993003,994409,997918,993537,991357,991795,992763,996815,996609,995864,994106,990087,990625,990432,999926,996613,994865,998205,991892,991927,997612,998361,991197,991220,994809\n", + "Ground Truth: 997909,997299,998890,996256,999925,993003,994409,995401,997918,993537,991357,991743,991795,992763,996815,996609,995864,997555,994106,990087,990625,994062,990432,999926,996613,997218,994865,996232,998205,991892\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 348\n", + "Returned IDs: 991487,991515,990630,994101,996562,996817,996496,999079,998250,993175,996461,992104,990203,995349,990834,993391,990160,991245,997666,995832,991897,998934,998022,997506,993483,995421,992407,990412,993489,996245\n", + "Ground Truth: 991487,991515,990630,994101,996562,996817,996496,999079,998250,993175,996461,992104,990203,995349,990834,993391,990160,991245,997666,999211,994952,998428,990712,995832,993599,999463,991897,998934,998022,997506\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 349\n", + "Returned IDs: 993754,997781,992604,995527,995807,999125,994774,999445,995105,999662,990915,998059,991051,999478,996691,992387,990131,998297,997212,990729,997462,997693,997110,993632,998528,992855,994539,995953,996206,996236\n", + "Ground Truth: 993754,997781,992604,995527,995807,999125,994774,999445,995105,999662,990915,998059,991051,999478,996691,992387,990131,998297,997212,990729,997462,997693,997110,993632,998528,992855,995625,994539,995953,996206\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 350\n", + "Returned IDs: 994142,997693,997309,990712,995214,991525,999179,990776,990956,997420,993453,996328,991257,994282,999091,997942,994193,993931,990705,993621,994119,995159,994092,992685,991723,994035,994835,996055,996633,993929\n", + "Ground Truth: 994142,997693,997309,990712,995214,999179,991525,990776,990956,997420,993453,996328,991257,994282,999091,997942,994193,993931,990705,993621,994119,995159,992441,994092,992685,991723,994351,994035,994835,996055\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 351\n", + "Returned IDs: 994964,995841,998445,990585,995372,997796,999026,998921,998797,991498,990925,993484,997320,990146,996001,998029,997301,994978,993845,997913,991444,996053,996959,992147,993579,995322,994584,999009,994776,998302\n", + "Ground Truth: 990936,993648,994964,995841,998445,990585,995372,997876,997796,999026,998921,998797,991498,990925,991582,990801,993484,997320,993402,990146,993663,993853,996001,998029,997301,994978,999533,993845,997913,991444\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 352\n", + "Returned IDs: 991498,993484,994503,997597,999005,999263,995372,997320,997301,995841,996560,992637,998634,991743,990829,990936,995034,998779,998839,990585,998445,999658,997305,997908,991626,995802,993443,995771,992811,997470\n", + "Ground Truth: 991498,993484,994503,997597,999005,999263,995372,997320,997301,995841,994917,996560,992637,998849,998966,998634,991743,990829,990936,995034,994183,998779,998839,990585,998445,992510,999658,997305,997908,994611\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 353\n", + "Returned IDs: 993237,998244,992252,998734,995515,998782,999453,990043,998837,990142,991999,994033,991307,990965,992835,993091,999793,996438,991485,993708,991840,992169,996686,996840,995443,991392,998363,993582,999620,991687\n", + "Ground Truth: 993237,998244,994439,992252,998734,995515,992342,998782,999453,990043,998837,990142,991999,994144,994033,991307,994785,993859,990965,992835,991149,993091,999793,996438,991609,991485,993708,991840,996686,992169\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 354\n", + "Returned IDs: 995097,993588,994145,994830,998440,991081,990622,993585,990016,994925,991921,990536,993342,998030,996858,993101,998710,994513,996974,992664,998903,990497,991075,998097,994689,995906,992201,997230,994682,990700\n", + "Ground Truth: 995097,993588,994145,994830,998440,991081,990622,993585,990016,994925,991921,990536,993342,994720,998030,996858,991087,993101,996147,998710,994513,996974,992664,998903,992174,994523,990497,991075,998097,994689\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 355\n", + "Returned IDs: 992169,995080,993143,997309,997918,991743,996918,991698,997002,995783,992558,990479,990561,990882,991257,994972,990897,996867,994334,998180,997586,993717,992944,994577,997822,999676,998726,995516,994409,993260\n", + "Ground Truth: 992169,995080,993143,997309,991052,997918,991743,996918,991698,997002,995783,992558,990479,990561,998045,995401,990882,991257,994972,991061,997723,990897,996867,994334,992933,998180,991447,997586,993717,992944\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 356\n", + "Returned IDs: 999587,992884,993289,999453,996657,997554,996001,990845,997002,990399,996810,992979,996028,994946,995401,996147,993032,996368,995216,992521,996759,997023,999827,994809,992558,998495,992862,999099,992395,999977\n", + "Ground Truth: 999587,992884,993289,999453,996657,997554,996001,990845,994531,997002,990399,996810,992979,996028,994946,995401,996147,993032,996368,995216,992521,995281,996759,997023,999827,994809,993280,992558,990956,998495\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 357\n", + "Returned IDs: 993809,998225,998302,994879,999444,997906,995638,990866,992748,996805,998950,997913,998123,996230,996075,999977,996811,995672,997320,996001,998580,995701,998697,992394,990096,990083,996761,994640,990949,993080\n", + "Ground Truth: 991063,993809,998225,998302,994879,999444,997906,995638,990866,992748,996805,998950,997913,998123,996230,996075,991927,999977,996811,995672,995755,997320,999831,993821,996001,998580,995701,998697,991311,992394\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 358\n", + "Returned IDs: 995580,991677,999333,999301,992668,999414,994464,996192,998170,992134,997913,998216,993222,997818,997192,990957,998600,997106,995870,997841,994748,992555,997071,997437,999624,992696,993853,993791,994197,996296\n", + "Ground Truth: 995580,991677,995775,999333,995266,999301,992668,999414,996499,998599,994464,996192,998170,992134,997913,998216,993222,992946,994904,997818,997192,991081,998152,990957,991422,995014,998686,990647,998600,997106\n", + "Recall: 0.6000\n", + "\n", + "Query ID: 359\n", + "Returned IDs: 990323,991037,999748,991596,995915,995385,990194,999576,995712,996702,997242,994955,995562,990148,994200,990096,997666,994231,993866,993458,999384,995655,990596,990876,999663,997685,999555,991640,991732,991093\n", + "Ground Truth: 990323,991037,999748,991596,995915,995385,990194,999576,995712,996702,997242,994955,995562,994020,990148,994200,995349,990096,997666,993179,994231,999435,998583,993866,992571,993458,999384,995655,990596,996424\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 360\n", + "Returned IDs: \n", + "Ground Truth: 995559,993695,991179,993038,995560,995946,998790,999898,996210,990435,991923,997920,992944,992047,998416,996042,995107,997586,993055,996370,990096,995720,995332,999430,994815,995401,993241,998349,998950,993324\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 361\n", + "Returned IDs: 995891,991544,998651,995946,993561,994413,996472,990096,991958,993738,993566,992448,991247,997252,997386,998545,998451,994285,995536,994365,995423,992631,995146,999673,994409,992801,998317,998703,995547,996733\n", + "Ground Truth: 995891,991544,998651,995946,993561,994413,996472,990096,991958,998973,993738,993566,992448,992260,991247,997252,997386,998545,998451,994285,995536,991447,994365,995423,995303,992631,995146,993433,999543,999673\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 362\n", + "Returned IDs: 994589,991666,995401,997723,991630,996659,990517,998530,999977,995701,996379,996864,998350,998874,991603,998990,992367,998045,999000,998536,994177,995994,994409,992790,992521,994946,999228,999277,991403,997658\n", + "Ground Truth: 994589,991666,995401,992933,997723,991630,996659,990517,998530,999977,995701,996379,996864,998350,998874,991603,999461,998990,992367,998045,999000,998536,994177,995994,994409,992977,992790,991151,992521,994946\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 363\n", + "Returned IDs: 994742,993080,995401,995932,999526,990096,996001,990289,998097,992052,996918,990468,993260,997168,996043,997320,992636,996387,996408,991245,992571,993637,997790,995740,997897,995536,998021,990814,991629,990392\n", + "Ground Truth: 994742,993080,995401,995932,999526,993719,990096,996001,990289,998097,992052,996918,990468,993260,990930,996516,997168,996043,994972,997320,992636,996387,993324,997103,996408,991245,992571,993637,997790,995203\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 364\n", + "Returned IDs: 996209,990423,991484,999555,994181,992234,998861,999576,997850,997666,995780,992716,998006,997242,997685,990194,992209,994335,992892,995209,992768,999273,996809,990167,993508,996386,999319,990154,998122,998598\n", + "Ground Truth: 996209,990423,991484,999555,994181,991003,992234,998861,999576,993615,997850,997666,995780,992716,998006,997242,997685,990194,992209,994335,992892,995209,992768,999273,996809,990167,993508,996386,999319,998419\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 365\n", + "Returned IDs: 994415,992041,993785,997320,996075,990914,992635,997550,996741,993741,991662,995596,995372,996875,992601,997661,996805,993447,997286,992119,997836,993125,997470,999982,997305,999701,991650,990697,991503,999473\n", + "Ground Truth: 994415,992041,993785,997320,996075,990914,992635,997550,996741,993741,991662,995596,995372,996875,995622,992601,995995,997661,996805,993447,997286,992942,992119,997836,993125,997470,999982,997305,999701,991650\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 366\n", + "Returned IDs: 995633,999676,997374,994317,992289,997319,994529,995289,990903,991743,996867,990126,995268,998221,995274,990432,994878,999453,994409,999587,993920,995020,998913,994850,994343,990435,998862,999801,996411,999026\n", + "Ground Truth: 995633,999676,997374,994317,992289,997319,995401,994529,995289,990903,991743,996867,990126,993412,995268,998221,995274,990432,993402,997723,994878,999453,994409,999587,993920,995020,998913,994850,994343,990435\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 367\n", + "Returned IDs: 994964,999151,990545,996219,995961,999331,999261,999367,990922,994896,999210,991382,993877,997720,993215,994705,995927,991378,994782,992446,990210,991751,996990,990957,996805,995861,996949,994783,998495,992114\n", + "Ground Truth: 994964,999151,990545,996219,995961,999331,991345,999261,994843,999352,999367,990922,994896,991921,994577,999210,991382,993238,992942,993877,998294,993154,997720,993215,994705,995927,991378,994782,990210,992446\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 368\n", + "Returned IDs: 995166,993857,998982,994689,991075,997208,996028,991585,995456,998427,993725,998182,990328,994307,995823,990097,993053,994765,995633,992480,996506,993906,995295,997309,995289,990811,992558,998180,991315,998105\n", + "Ground Truth: 995166,993857,998982,994689,991075,997208,996028,992374,991585,995456,998427,993725,998182,990328,994307,995823,990097,993053,995401,994765,995633,992480,996506,993906,995295,997309,995289,990811,992558,998180\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 369\n", + "Returned IDs: \n", + "Ground Truth: 995560,992944,995375,996370,996210,994972,991994,995372,995841,996001,998950,995559,995107,997320,999009,998567,990949,997155,995873,990930,991052,995536,990538,995641,997305,995332,998697,994081,992521,998849\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 370\n", + "Returned IDs: 997320,993484,997913,994604,996143,993154,998967,999948,995802,992442,990066,997470,998170,999658,990532,992119,995372,992330,990949,995865,999851,992546,993157,996075,998808,992896,999535,995351,990585,994363\n", + "Ground Truth: 997320,993484,997913,994604,996143,993154,998967,999948,995802,995228,992442,990066,997470,998170,999658,990532,992119,995372,992330,990949,995865,999851,992546,993157,996075,995942,998808,992896,999535,995351\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 371\n", + "Returned IDs: \n", + "Ground Truth: 991975,993641,990429,995168,992110,999176,995231,993629,998858,993381,995780,997078,991455,996623,991332,992365,991385,993560,999555,991987,996571,994413,990976,994834,992926,997546,990449,998731,993191,998149\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 372\n", + "Returned IDs: 990249,993811,992806,995967,993727,997195,992121,999240,999479,993121,993583,990121,996750,993473,992558,994163,999400,997897,998820,997918,993158,990117,995626,997124,994134,990395,999098,993839,994957,995309\n", + "Ground Truth: 990249,993811,992806,995967,993727,997195,992121,999240,999479,993121,993583,990121,996750,993473,992558,999887,994163,999400,997897,998493,992407,998820,991447,997918,998559,993158,993911,992864,990117,995626\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 373\n", + "Returned IDs: 992225,996089,993174,999041,999216,995005,998200,991984,993834,996230,990051,997258,992167,999219,993928,998934,992213,994902,994167,993322,994248,998989,993489,996657,993766,990472,995516,992238,991157,990547\n", + "Ground Truth: 992225,996089,993174,999041,999216,995005,998200,991984,993834,996230,990051,997258,992167,999219,993928,998934,992213,994902,994167,993322,994248,998989,993489,996657,993087,993766,990472,995516,992238,991157\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 374\n", + "Returned IDs: 998593,995421,991212,992437,993715,997897,991245,997790,995167,996001,990255,996043,998250,994743,993931,998037,998934,997453,990364,995945,996213,995126,991725,994925,996657,995873,993289,990314,991958,990811\n", + "Ground Truth: 998593,995421,991212,992437,993715,997897,991245,997790,995167,996001,990255,996043,998250,994743,993931,998037,998934,997453,995740,990364,995945,996213,995126,991725,994925,996657,995873,993289,990314,991958\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 375\n", + "Returned IDs: 998921,994964,996504,997984,994415,991720,996143,997320,997470,993518,994584,990958,999343,991582,993253,999647,990146,999676,997192,991498,990648,992147,998550,999102,995515,995336,995181,992942,993484,997305\n", + "Ground Truth: 998921,994889,994964,996504,997984,994415,991720,996143,997320,997470,993518,994584,995865,990958,998607,999343,991582,993253,999647,990146,999676,997192,991498,990648,990917,992147,998550,999102,995515,995336\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 376\n", + "Returned IDs: 995372,998849,992944,993241,990585,995375,996809,990489,999544,990226,995560,995841,999711,991498,999210,996318,997320,994972,994402,991994,993388,996001,990043,995641,997436,997113,995443,990532,999641,997002\n", + "Ground Truth: 995372,998849,992944,993241,990585,990060,995375,996809,990489,999544,998567,990226,995560,995841,999711,991498,999210,996318,997320,995401,994972,993533,994402,991994,997848,993388,996001,990043,995641,990894\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 377\n", + "Returned IDs: 998710,999377,994663,994307,998105,996132,994513,997434,993053,994830,999430,994185,999472,999613,996906,994742,992201,999500,994925,996147,992079,996411,994145,999064,991128,990435,992230,995852,991409,999302\n", + "Ground Truth: 998710,999377,994663,994307,998105,996132,995310,994513,990016,997434,993053,994830,999430,994185,999472,999613,990012,996906,994742,992201,999500,994925,996147,992079,996411,994145,999064,991128,990435,992230\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 378\n", + "Returned IDs: \n", + "Ground Truth: 995655,993948,991667,992517,998861,997145,995536,992595,990154,998536,990005,990330,993001,998122,998495,991610,995835,992561,995780,997884,996623,991490,997265,997242,997319,999555,998006,992110,999711,990956\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 379\n", + "Returned IDs: 992532,995974,998686,998334,991480,999451,992349,992645,999711,993976,995314,998417,998079,992383,997521,990383,996196,990401,996262,991015,991208,990376,997758,998061,999639,999913,991977,992718,998118,990908\n", + "Ground Truth: 992532,995974,998686,998334,991480,999451,995560,992349,992645,996894,993174,999711,993976,995314,998417,998079,992654,992383,997521,990383,996196,990401,998849,996262,991015,999216,991208,990376,997758,998061\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 380\n", + "Returned IDs: 997683,996393,998918,991309,994393,990657,993749,992623,992638,997110,997568,990361,998257,995044,998858,999156,993795,999555,995829,999312,995373,991204,998277,997114,990005,994901,993076,991987,992587,992861\n", + "Ground Truth: 997683,996393,998918,991309,994393,990657,993749,992623,992638,997110,997568,990361,998257,995044,998858,999156,993795,999555,991188,990554,995829,997030,999312,995373,997429,991204,998277,997114,991052,990005\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 381\n", + "Returned IDs: 995482,999179,990157,994806,997997,993466,995106,991664,995055,998256,996633,993731,990083,994166,997411,993820,998748,995778,995777,996328,993111,998543,998221,999759,990806,992364,990987,992285,995996,993931\n", + "Ground Truth: 995482,999179,990157,994806,997997,993466,995106,991664,995055,998256,996633,993731,996283,991061,990083,994166,997411,993820,998748,996386,995778,995777,996328,993111,998543,992563,998221,999759,990806,992364\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 382\n", + "Returned IDs: 996147,992161,990771,995167,996001,997790,997918,999898,991667,992950,993289,996657,997554,999150,990399,999099,999453,991212,994266,993143,991585,991485,992384,996549,990314,998720,993280,993157,995536,992367\n", + "Ground Truth: 996147,992161,990771,995167,996001,997790,997918,999898,991667,992950,993289,996657,997554,998215,999150,990399,999099,991311,999453,991212,994266,995855,993143,997163,991585,991485,992384,996549,995281,990314\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 383\n", + "Returned IDs: 992558,990538,998950,990279,999639,996196,997775,997918,997258,994163,992615,995552,999206,999860,996609,990273,997859,999564,996592,998411,991522,994280,995340,994742,995372,999715,992436,991498,991791,993077\n", + "Ground Truth: 992558,990538,998950,990279,999639,996196,997775,997918,995693,997258,994163,992615,995552,999206,999860,996609,990273,997859,999564,996592,998411,991522,994280,995340,994742,995372,999715,992436,991498,991791\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 384\n", + "Returned IDs: 998748,991981,991185,995083,993209,993920,990593,992595,993466,992450,995633,990887,996328,991585,990331,997374,995268,997710,990157,990126,990435,997185,994541,994742,994185,993997,995996,995106,998055,991044\n", + "Ground Truth: 998748,992091,991981,991185,995083,993209,993920,990593,992595,993466,992450,995633,990887,996328,990226,991585,990331,997374,995268,997710,990157,990126,990435,997185,994541,994472,994742,994185,993997,995996\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 385\n", + "Returned IDs: 997852,999077,999676,992169,994922,991334,996325,997204,992612,990432,993260,997822,993717,991840,991743,994964,992813,999453,998950,993355,994742,994318,998779,991435,993237,998123,997309,997918,997379,993143\n", + "Ground Truth: 997852,997476,999077,999676,992169,994922,991334,990490,996325,997204,992612,999916,990432,993260,997822,997496,993717,998041,993673,991840,991743,994964,992813,999453,998950,994044,999843,997135,993355,990214\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 386\n", + "Returned IDs: 998130,991982,993245,995449,993948,990005,993628,998758,991987,991630,993055,995516,998861,998959,994125,991530,997884,990897,995536,995780,992719,990991,996918,991052,992293,992862,995432,998623,998067,995184\n", + "Ground Truth: 998130,991613,991982,993245,995449,993948,990005,993628,998758,991987,991630,993055,995516,998861,998959,994125,991530,998383,997884,990897,995536,995780,992719,990991,996918,991052,992293,992862,995432,998586\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 387\n", + "Returned IDs: 993725,992424,990528,997927,991565,997079,995551,993705,999751,996928,992753,999922,996633,998348,995274,996411,993623,990392,992226,995435,993053,999083,999740,992680,995744,992636,994783,994878,993585,994730\n", + "Ground Truth: 993725,992424,990528,997927,991565,997079,995551,990315,997207,993705,998574,999751,996928,992753,999922,996633,998348,995274,996411,997714,993623,990392,992226,995435,993053,999083,999740,992680,995744,992636\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 388\n", + "Returned IDs: 994578,993856,996867,992448,994140,990751,995289,992239,991790,997319,999235,999474,995633,994409,999875,992364,998910,996368,993293,990814,997723,997046,995640,992289,995030,996277,990940,992521,997868,997466\n", + "Ground Truth: 994578,993856,996867,992448,992468,994140,990751,995289,994419,992239,991790,997319,999235,999474,995633,994409,999875,992364,998999,991185,998910,996368,993293,990814,997723,997046,995640,992289,995544,993280\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 389\n", + "Returned IDs: 990742,991593,991662,991016,997826,998899,999701,997913,995748,991502,998003,999374,991616,997320,995928,998170,994964,991479,993885,999948,993606,992978,991743,998663,997470,991528,990650,991999,999263,995771\n", + "Ground Truth: 990742,991593,991662,991016,997826,998899,999701,997913,995748,991502,998003,999374,991616,997320,999390,995928,990902,991237,998170,993238,994964,991479,993885,999948,993606,998294,992978,991743,990816,998663\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 390\n", + "Returned IDs: 990249,996609,998820,991248,995124,992558,993433,999240,996762,994457,995289,995377,997822,996606,992469,995354,995401,995119,991330,991449,998874,996750,993647,995766,991334,999400,993561,993673,993831,997374\n", + "Ground Truth: 990249,996609,998820,991248,995124,992558,993433,999240,996762,991350,994457,991266,995289,995377,997822,996606,993412,995332,996294,992469,995354,997723,999646,995401,995119,997498,991330,991449,998874,999898\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 391\n", + "Returned IDs: 992052,995873,994742,991451,994689,995295,993738,997897,996580,998256,990930,996328,992445,998623,992239,994280,990956,997329,993504,998045,991185,994255,996277,996001,996028,995823,990005,992636,994307,990096\n", + "Ground Truth: 992052,993456,998427,995873,994742,991451,994689,995295,993738,997897,995408,996580,998256,990930,996328,992445,991255,998623,992239,994280,990956,997329,993504,998045,991185,997918,994255,996277,996001,996028\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 392\n", + "Returned IDs: 996986,990951,992636,996408,999334,991393,990173,996183,997168,999791,992372,993917,998348,996728,993001,998076,999138,997838,998440,993715,996882,994351,992664,999435,992282,999373,994742,990918,997732,996062\n", + "Ground Truth: 996986,990951,992636,996408,999334,991393,990173,990027,996183,997168,999791,992372,993917,998348,996728,993001,998076,999138,997838,998440,993715,996882,998389,990712,994351,992664,999435,992282,999373,994742\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 393\n", + "Returned IDs: 990487,999774,999191,996194,998528,997770,991977,994254,997251,996042,999179,990102,997115,990678,992558,991447,992979,994387,992086,998104,991303,994280,993275,997918,991061,998703,997126,990249,991334,996429\n", + "Ground Truth: 990487,996451,999774,999191,996194,991720,998528,999343,997770,991977,994254,997251,996042,999179,990102,998968,997115,990678,992558,991447,992979,996658,994387,992086,998104,991303,990705,994280,993275,991743\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 394\n", + "Returned IDs: 990535,999144,994217,993168,992630,998951,993083,997015,992331,995336,997320,999437,997082,994964,990925,992748,993389,995252,997002,996001,999019,993484,994757,995824,994854,998090,990958,990231,992384,993050\n", + "Ground Truth: 990535,999144,994217,993168,992630,998951,993083,997015,992331,995336,999916,997320,995947,999437,997082,994964,990925,992748,993389,995252,997002,995216,996001,999019,993484,994757,994916,995824,994854,998090\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 395\n", + "Returned IDs: \n", + "Ground Truth: 994280,991334,997023,995968,990214,995377,995401,999701,991923,994835,995820,991743,998374,990921,995886,994655,992165,992582,995824,994982,992521,998302,992384,993561,997918,991269,994496,996762,995336,998295\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 396\n", + "Returned IDs: 996915,996001,996147,998697,997506,995536,997227,993157,998720,998303,995250,990771,993280,996657,991332,996368,993289,993561,990532,990096,991530,998060,999639,992950,995562,993055,995899,997554,990399,999453\n", + "Ground Truth: 996915,996001,996147,998697,997506,995536,997227,993157,998720,998303,995250,995401,990771,991585,993280,996657,991332,996368,993289,993561,999587,990532,990096,991530,998060,999639,992950,995562,993055,995899\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 397\n", + "Returned IDs: 996178,990532,996805,998950,990096,991981,997320,992318,994307,994742,997906,999444,993809,994185,990843,990364,990083,995401,993080,997913,997790,993342,993504,994416,996397,990814,996370,993621,991459,995438\n", + "Ground Truth: 996178,990532,996805,998950,990096,991981,997320,992318,994307,994742,997906,999444,993809,994185,990843,990364,990083,995401,992636,993080,997913,997790,998365,993342,993504,994416,996397,999647,990814,993834\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 399\n", + "Returned IDs: 998908,998921,997539,997301,993581,995154,995910,993143,999219,990760,996014,991157,993364,997370,990561,996040,998029,998726,997371,993250,998894,991392,997218,997984,992734,990053,991782,990146,992546,994675\n", + "Ground Truth: 998908,998921,994244,997539,997301,999995,993581,995154,990702,999426,995910,993143,999219,992461,998539,990760,996014,995554,991157,993900,993364,997158,993662,997777,997370,990561,992124,999358,996040,993005\n", + "Recall: 0.5333\n", + "\n", + "Query ID: 400\n", + "Returned IDs: 996042,995401,992685,997822,997723,996867,993433,997232,995289,998080,992169,995200,992558,997918,997675,993407,998913,998495,995105,990214,998528,990119,997466,995685,992582,996734,996903,996809,992604,993717\n", + "Ground Truth: 996042,991052,995401,992685,997822,997723,996867,993433,997232,995289,993412,998080,992169,995200,992558,997918,997675,993407,998913,998495,995105,990214,998528,990119,997466,995685,992582,996038,996734,996903\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 401\n", + "Returned IDs: 992978,991883,997320,990210,993431,994964,991721,997913,996904,990114,997470,995841,991249,999167,998550,992944,992546,998364,993072,997371,998365,991801,998540,992692,990096,993662,992310,990925,996143,992066\n", + "Ground Truth: 992978,991883,997320,990210,993431,994598,994964,991721,997913,996904,990114,997470,995841,991249,999167,998550,992114,992944,992546,998364,993072,998698,997371,998365,991801,998540,992692,990096,993662,992909\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 402\n", + "Returned IDs: 997643,994724,997210,992638,995044,997683,990657,997589,999156,999117,998918,992623,993076,998257,996393,998277,993795,994660,993666,991797,999808,995207,991309,998787,991204,990361,998383,995056,992926,991038\n", + "Ground Truth: 997643,994724,997210,992638,995044,997683,990657,997589,999156,999117,998918,992206,992623,993076,998257,996393,998277,993795,994660,993666,991797,999808,995207,991309,990167,998787,991204,994917,990361,998383\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 403\n", + "Returned IDs: 998149,990658,994869,999663,991038,999834,993055,999736,997506,999139,995780,994467,995237,991169,999555,992421,998831,993629,995516,996406,999896,996623,999806,995755,998720,991154,990292,997650,998586,998758\n", + "Ground Truth: 998149,990658,994869,999663,991038,999834,993055,994155,999736,997506,999139,998858,995780,994467,995237,991169,999555,992421,998831,993629,995516,996406,999896,996623,999806,995755,998383,998720,991154,991835\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 404\n", + "Returned IDs: 992493,995807,999826,997189,994677,994301,998820,993530,997232,993223,997420,996139,997184,995625,991051,998316,995521,993149,997075,998080,999475,997675,997634,997574,997061,997824,999961,995493,996009,992801\n", + "Ground Truth: 992493,995807,999826,997189,994677,994301,998820,993530,997232,993223,997420,996139,997184,995625,991051,998316,995521,993149,994141,997075,998080,999475,997675,997634,997574,997061,997824,992683,996903,995366\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 405\n", + "Returned IDs: 998999,995940,997662,993103,995289,996302,999957,991603,995401,995200,999875,990940,997630,996769,994409,997723,995182,996388,990033,997020,998530,994161,994948,991954,997319,998765,993144,995648,992619,998162\n", + "Ground Truth: 998999,995940,997662,993103,995289,996302,999957,997042,991603,995401,996864,995200,999875,990940,997630,996769,994409,997723,995182,996388,990033,997020,998530,994161,994948,991954,997319,998765,995701,993856\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 406\n", + "Returned IDs: 994577,992068,998816,991015,999806,999194,997379,995039,999122,999432,992423,993012,991157,991987,998568,998950,998858,993143,997466,990991,998437,998257,991713,999568,992330,991844,997320,991571,997918,996262\n", + "Ground Truth: 994577,992068,998816,991015,999806,999194,997379,995039,999122,993628,995453,999432,992423,993012,991157,991987,996281,998568,998950,998858,993143,997466,990991,998437,998257,999267,991713,999568,992330,990479\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 407\n", + "Returned IDs: 994717,994809,990255,990361,995401,994092,992293,996657,999629,998593,996809,995701,996759,990573,994431,996930,995289,992558,993848,992367,998536,997897,998990,996393,998250,996777,995963,997822,995873,995864\n", + "Ground Truth: 994717,994809,990255,990361,997088,995401,999587,993515,994092,992293,996657,999629,998593,996043,996809,995701,996759,990573,994431,996930,995289,992558,993848,997411,992367,998536,997897,998990,996393,998250\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 408\n", + "Returned IDs: 994794,993465,990242,993412,998297,999923,991603,997083,995269,998999,990385,990940,995208,991028,991790,997045,995702,995274,997723,995940,999374,999932,997787,998779,991350,997020,997832,995373,996379,991543\n", + "Ground Truth: 994794,993465,990242,994702,993412,998297,999923,991603,997083,995269,998999,990385,990940,995208,997473,991028,991790,997045,995702,995274,997723,995940,999374,999932,997787,998779,991350,997020,997832,995373\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 409\n", + "Returned IDs: 996690,993363,996963,991772,991157,991550,993370,993411,993706,999393,992383,994163,996196,999639,997775,998697,990159,996926,995013,992349,990279,992770,999908,992423,992738,997225,997258,998142,995599,999041\n", + "Ground Truth: 996690,993363,996963,991772,991157,991550,993370,993411,993706,999393,992383,994163,996196,999639,997775,998697,990159,996926,995013,992349,990279,992770,991706,999908,992423,992738,997225,997258,998142,995599\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 410\n", + "Returned IDs: 998170,994240,993431,996592,999077,992880,997470,993260,997320,997918,991801,991348,999701,996296,998180,995826,991928,993696,995357,991564,991131,994964,992748,994552,994360,990949,996761,995748,993646,993154\n", + "Ground Truth: 998170,994240,997699,993431,996592,994584,999077,992880,997470,993260,997320,997918,991801,995094,991348,993454,992927,994666,999701,996296,998180,994450,995826,991928,993696,995266,995357,991564,991131,994964\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 411\n", + "Returned IDs: 991593,992069,998899,994399,997320,991774,992255,990957,991662,999290,997470,997301,999077,993662,995826,991479,998090,999602,994604,990910,994290,996296,992978,990650,993034,993402,997371,993606,991743,995898\n", + "Ground Truth: 995266,991593,992069,998899,994399,997320,991774,992255,991580,990957,998967,991662,999290,997470,997301,999077,993662,995826,991479,993852,998090,999602,994604,990910,994290,996296,992978,992946,990650,998801\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 412\n", + "Returned IDs: 992521,997545,993269,997466,996552,991878,991969,993107,992097,994353,998238,993438,991555,993353,998075,998910,995193,994508,995813,998205,995644,992448,994636,994954,997723,991117,998509,999121,990603,995198\n", + "Ground Truth: 992521,997545,993269,997466,996552,991878,991969,993107,992097,994353,998238,993438,991555,998564,993353,998075,998910,998346,995193,994508,991760,997216,995813,998205,998272,995644,998568,992448,994636,994954\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 413\n", + "Returned IDs: 990956,992558,996809,994742,999179,992052,990776,992604,990361,998495,997752,994643,993077,994351,993948,990657,991257,995105,997914,993121,996043,991145,997110,999191,996277,994280,995373,996328,990705,990083\n", + "Ground Truth: 990956,992558,996809,994742,995963,991052,999179,992052,990776,992604,997723,990361,998495,997752,994643,993077,995408,994351,993948,990657,998543,991257,993799,995401,993079,991319,995105,992685,997914,993121\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 415\n", + "Returned IDs: 991643,992554,993325,997685,992209,993055,991724,999744,993965,996889,994834,991738,990096,999622,994420,996733,999319,993744,995385,991243,995402,991093,995736,996209,999771,999079,995796,990160,992237,996817\n", + "Ground Truth: 991643,992554,993325,997685,992209,993055,991724,999744,993965,996889,994834,991738,990096,999622,994420,996733,999319,993744,999273,995385,998419,995402,991243,991093,995736,993132,996209,999771,999079,995796\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 416\n", + "Returned IDs: 992713,994142,994891,991177,993567,995991,990784,990364,999775,998067,998256,996328,990956,991747,996728,995184,993621,994999,994444,992526,994280,998251,992450,998405,991111,996911,998671,990712,997506,993336\n", + "Ground Truth: 992713,994142,994891,991177,993567,995991,990784,990364,999775,998067,998256,996328,999441,990956,991747,996728,995184,993621,994999,991143,997860,994444,992526,994280,998251,992450,998405,991111,996911,993929\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 417\n", + "Returned IDs: 991198,997789,990043,999620,998232,991196,996681,998236,997275,998983,993582,990155,991801,990858,993570,998837,994922,994785,992907,999076,991510,991999,994964,990585,998280,993237,999053,994612,990142,997288\n", + "Ground Truth: 991198,997789,990043,999620,998232,991196,996681,998236,997275,998983,993582,990155,991801,990858,990809,993570,998837,996318,994922,994785,992907,999076,993675,991510,991999,994964,990585,997876,998280,993237\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 418\n", + "Returned IDs: 994106,994203,999731,991253,995216,990193,998205,998255,997059,995873,992521,992162,997554,998527,995813,996001,994488,990081,998910,993951,993738,990775,991163,992448,999402,991394,995079,990603,997928,997176\n", + "Ground Truth: 994106,994203,999731,991253,995216,990193,998205,994773,998255,997059,995873,992521,992162,997554,998527,995813,996001,994488,990081,998910,993951,993738,990775,991163,992448,999402,991394,995079,990603,997928\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 419\n", + "Returned IDs: \n", + "Ground Truth: 993478,994193,997276,999244,996743,997184,996181,993155,991052,996918,991799,990521,994280,997420,993727,996825,991837,995401,998267,992933,999259,997675,994092,992685,995783,996438,992133,993980,999264,994474\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 420\n", + "Returned IDs: 998886,992293,994382,990662,997868,990361,998017,996809,999922,997466,992582,998495,993390,992862,999587,996147,996042,994793,994689,995679,997529,998982,994730,995701,993561,995702,995289,996391,994431,995536\n", + "Ground Truth: 998886,992293,994382,990662,997868,990361,998017,990399,996809,999922,997466,995470,992582,998495,993390,992862,991603,999587,996147,996042,993831,994793,997723,994689,995679,997529,991578,998982,995562,994730\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 421\n", + "Returned IDs: 999374,998779,992685,993412,997529,996219,994382,997822,991958,995702,996379,995401,993096,995685,998045,998380,998495,995289,998123,992169,997723,994964,994057,992351,993355,996959,995633,996721,997002,993089\n", + "Ground Truth: 999374,998779,992685,991052,993412,997529,996219,994382,997822,999900,990337,991958,995702,996379,995401,993096,995685,998045,998380,995994,998495,995289,990701,998123,991743,992169,993725,993186,997723,994964\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 422\n", + "Returned IDs: 996328,999139,992726,999734,991652,998864,995705,994574,997944,993139,991605,990712,993665,996591,995541,995112,993715,991990,991685,996495,991037,994280,997847,999827,996733,990658,994510,992450,995655,997309\n", + "Ground Truth: 996328,999139,992726,999734,991652,998864,995705,994574,997944,993139,991605,990712,993665,996591,995541,995112,993715,991990,995103,991685,996495,991037,991596,994280,997847,999827,993289,993981,996733,990330\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 423\n", + "Returned IDs: 998470,995214,992558,999791,999767,998122,998279,998495,997732,990956,998765,996001,999191,999827,992713,995438,990089,996219,996647,998710,990364,994280,999138,999852,994119,996809,997586,992780,994742,997453\n", + "Ground Truth: 998470,995214,992558,999791,999767,998122,995222,998279,994972,998495,997732,990956,998765,996001,997603,999191,999827,992713,995438,990091,997920,990089,999916,996219,996647,998710,990364,994280,999138,995107\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 424\n", + "Returned IDs: 990811,998593,996147,999173,991000,996657,992326,996028,998878,990771,992727,996001,998934,995158,999740,990573,998250,993391,992289,993813,995289,995200,995182,993848,996302,997007,997753,999868,998999,991958\n", + "Ground Truth: 990811,998593,996147,999173,991000,996657,990130,992326,993483,996028,998878,990771,992727,996001,998934,994838,995158,999740,995864,990573,998250,993391,996864,992289,993813,995289,995200,995182,993848,996302\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 425\n", + "Returned IDs: 990382,999543,994510,993931,999139,990096,998593,998250,996062,991685,995889,994463,991596,995536,998495,997790,995895,990930,996733,994191,991011,998376,996001,996549,999994,995017,999091,991066,990255,997897\n", + "Ground Truth: 990382,999543,994510,993515,993931,999139,990096,998593,998250,996062,991685,995889,994463,991596,995536,998495,994796,997790,995895,990930,996733,994191,991011,998376,996001,996549,999994,995017,993778,999091\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 426\n", + "Returned IDs: 995591,994964,999367,991588,999210,992612,992944,997379,995200,997529,997822,996867,990690,995289,996990,993215,997406,995052,992169,995923,996219,996042,997723,993265,996328,996809,998495,994742,995360,998765\n", + "Ground Truth: 995591,994964,999367,991588,999210,992612,992944,997379,995200,997529,997822,996867,990690,995289,996990,993215,997406,995052,992169,995923,996219,996042,997723,993265,996328,991921,999352,996809,994307,998495\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 427\n", + "Returned IDs: 992862,997586,991490,995701,990786,999806,993809,995536,998708,999653,995455,996082,993637,993470,999444,990167,998200,997506,993834,996370,999087,993515,998697,995203,998536,993055,995672,991530,999216,990449\n", + "Ground Truth: 992862,997586,995005,991490,995701,990786,999806,993809,995536,998708,999653,995455,996082,993637,993470,999444,990167,998200,997506,993834,990271,996370,999087,993515,998697,993504,995203,993821,998536,997577\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 428\n", + "Returned IDs: 993586,996304,997544,993371,991639,996220,993209,990082,993206,998811,999759,995883,995778,990372,997975,990298,996102,994496,999646,997023,991269,999957,999054,992582,991507,997915,990638,994106,998910,997792\n", + "Ground Truth: 993586,996304,997544,993371,991639,996220,993209,990082,993206,998811,996595,999759,995883,995778,990372,997975,990298,996102,994496,999646,997023,991269,999957,999054,992582,991507,997915,990638,994106,999250\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 429\n", + "Returned IDs: 996734,995401,990337,997191,998297,995685,993412,995373,998045,991603,994409,993561,998999,997406,996968,992448,995940,993288,991140,997723,994948,995648,999530,997822,992685,995994,991630,995886,998779,996042\n", + "Ground Truth: 996734,995401,990337,997191,998297,995685,993412,995373,998045,991603,994409,993561,998999,991052,997406,996968,992448,995940,996277,993288,991140,997723,994948,990930,997918,995648,999530,997822,995873,992685\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 430\n", + "Returned IDs: 996219,996328,990214,999179,996811,997442,999500,994387,998256,998968,995438,997420,999560,994783,995562,995130,994233,995895,990039,992237,997945,994742,997485,993100,999430,993560,999139,994402,994280,996494\n", + "Ground Truth: 994956,996219,996328,990214,999179,996811,997442,999500,994387,996149,993727,995332,998256,998968,995438,991212,997420,999560,994783,995562,996071,995130,997002,994233,995895,999352,999540,990039,994206,992237\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 431\n", + "Returned IDs: 994809,997218,992790,991671,996762,991197,996930,993673,992749,992558,990312,992636,994835,995864,998855,993003,992448,998205,996162,996187,990181,993275,995873,997023,996001,993561,995536,998910,994409,998495\n", + "Ground Truth: 994809,997218,995401,992790,990841,991671,996762,991197,991334,996930,993673,992749,992558,990312,992636,991459,994835,995864,992685,998855,993003,992448,998205,996162,998104,996187,990181,993275,995873,996438\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 432\n", + "Returned IDs: 996279,994533,999301,996192,994688,994757,993852,999883,990215,990469,994854,990275,998190,996714,999089,992511,993457,999327,995034,991379,997012,992384,998170,992087,999548,998011,999953,992854,995875,997605\n", + "Ground Truth: 996279,994533,999301,996192,994688,994757,993852,999883,990215,990469,994854,990275,998190,996714,999089,992511,993457,999327,995034,991379,997012,999500,992384,998170,992087,991650,999548,998011,999953,992854\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 433\n", + "Returned IDs: 999820,990561,995327,992979,994906,997910,992790,994946,994673,993289,994819,995968,991074,996425,994982,994170,997218,994334,994188,997002,994922,999977,995875,999026,997380,999587,990215,997605,992884,990916\n", + "Ground Truth: 999820,990561,995327,992979,994906,997910,992790,994946,994673,993289,994819,990304,995968,998011,992047,991074,996425,994982,995401,996714,994170,997218,999715,994334,997396,994188,997002,994922,998210,999977\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 434\n", + "Returned IDs: \n", + "Ground Truth: 992950,997320,996904,993662,998540,999109,992451,998302,997470,993237,994964,997913,997379,992443,992140,996890,993791,998550,999710,999210,998837,991626,993796,994192,990615,992069,991801,993018,991582,997135\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 435\n", + "Returned IDs: 996802,994964,997320,993389,994341,993662,990146,993484,996296,991609,990949,990843,997379,993781,990585,995094,993696,990231,991743,991732,992950,998950,999210,999647,991154,993260,994742,993603,996469,999430\n", + "Ground Truth: 996802,994964,997320,997747,993389,994341,993662,990146,993484,996296,991609,990949,990843,997379,995560,993781,990930,992120,990585,995094,993696,990231,991743,991732,992950,998950,999210,991886,999647,991154\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 436\n", + "Returned IDs: 992237,999752,990806,998405,997392,993196,996839,991723,992827,993335,994472,995630,998055,994335,994101,994960,996016,992372,997527,993501,997442,990173,992892,992285,991418,990440,997997,991672,993460,991061\n", + "Ground Truth: 992237,999752,990806,991961,998405,997392,993196,996839,991723,992827,993335,994472,995630,998055,994335,994101,994960,996016,992372,997527,993501,997442,990173,992892,992285,991418,990440,997997,991672,993460\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 437\n", + "Returned IDs: 994749,999544,997249,995968,997498,999254,994081,997888,995372,990772,990043,995932,991440,993071,999676,996309,991787,991880,999453,995841,997661,996785,998734,994240,995250,991002,996217,993241,994789,993683\n", + "Ground Truth: 994749,999544,994826,997249,995968,997498,999254,994081,997888,995372,990772,990043,995932,991440,993071,999676,996309,996506,991787,998654,994171,992531,991880,999453,995841,997661,996785,998734,994240,994687\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 438\n", + "Returned IDs: 990550,998200,994624,993834,999087,998232,992862,996082,999651,997913,996661,999783,995296,991256,995701,996062,990096,993342,993504,996001,995184,995672,998458,997586,995132,992293,993489,998697,999647,990366\n", + "Ground Truth: 990550,998200,994624,993834,993902,999087,998232,992862,996082,999651,997913,996661,999783,995296,991256,993403,995701,996062,990096,993342,993504,996001,995184,995672,998458,990866,997586,995132,995927,992293\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 439\n", + "Returned IDs: 998536,992110,991644,990449,991620,993001,992282,992595,990399,992832,999827,997506,990154,998067,995655,990942,995326,991667,996328,992444,998625,996408,999994,992862,995030,994380,992761,991747,998623,992793\n", + "Ground Truth: 998536,992110,991644,990449,991620,993001,992282,992595,990399,992832,999827,997506,990154,998067,995655,990942,995326,997723,991667,996328,992444,998625,998861,992318,996408,999994,999555,992862,993993,995030\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 440\n", + "Returned IDs: 999594,992451,993419,990089,998163,993060,994325,995659,999791,996755,993848,994776,992470,993484,996001,996516,993490,997320,998921,996075,995454,994757,997984,992636,993637,996155,995981,994964,995306,999740\n", + "Ground Truth: 999594,992451,993419,990936,992583,990089,998163,999516,993060,994325,995659,999791,996279,990862,996755,993848,994776,992546,994199,995792,992470,999757,993630,993484,996001,996516,993490,992785,997320,998921\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 441\n", + "Returned IDs: \n", + "Ground Truth: 991178,990980,993914,990648,999343,996219,997984,994266,999404,993238,994964,998500,992546,995942,990146,993072,999824,990595,994415,993048,990650,990736,996028,993518,994889,998458,990545,998581,997855,991325\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 442\n", + "Returned IDs: 991856,993960,991202,994497,994604,999260,990224,995135,996854,991733,990958,999374,997470,999398,994922,990980,991593,995959,998921,995365,993640,994266,990660,998500,997192,997842,994584,998660,994819,995260\n", + "Ground Truth: 991856,993960,991202,994497,994604,992323,999260,999261,990224,995135,999633,996854,991733,990958,999374,999793,997470,991889,999398,998190,994922,995947,995259,990980,991593,995959,998921,995365,993640,993570\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 443\n", + "Returned IDs: 992944,998950,993018,997370,997320,995130,996693,995438,996075,993079,997113,995873,996140,990949,991142,996001,998302,998180,996318,993504,990022,995560,993053,996657,991498,995203,997775,994171,999210,996210\n", + "Ground Truth: 992944,998950,993018,997370,997320,995130,996693,995438,996075,993079,997113,995873,996140,990949,991142,996001,998302,998180,996318,993280,993504,990022,995560,993053,992521,996657,991498,995203,997775,994171\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 444\n", + "Returned IDs: 999868,996042,998866,993597,992125,991301,998564,993433,995905,996112,996781,998165,996769,997406,997046,995289,994419,995702,998593,999080,992160,995439,990681,997822,995401,996360,998045,990984,992519,991463\n", + "Ground Truth: 999868,996042,998866,993597,992125,991301,998564,993433,995905,996112,996781,998165,996769,997406,997046,995289,994419,995702,998593,999080,992160,995439,990706,990681,997822,995401,996360,998045,990984,992519\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 445\n", + "Returned IDs: 991266,990352,990751,996368,993561,993673,997918,999099,993289,998999,997002,998186,990517,993275,994664,999977,998593,996001,991958,991332,996379,992828,990930,996759,995701,992703,990035,997189,995679,993813\n", + "Ground Truth: 991266,990352,990751,996368,995421,993561,993673,997918,999099,993289,998999,997002,998186,991603,990517,999303,999866,993275,994664,995097,996028,997077,995274,999977,998593,994959,996001,991958,991332,996379\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 446\n", + "Returned IDs: 992169,996062,993717,997732,994280,999827,996328,998180,997852,997309,995861,991066,993215,993052,992703,995295,992944,992724,996733,997822,993080,997918,994964,996093,992713,990392,997790,990080,997002,998765\n", + "Ground Truth: 992169,996062,993717,997732,994280,999827,996328,998180,997852,998633,996728,997309,995861,991066,993215,993052,992703,990889,995295,992944,992724,996733,996216,997822,993080,993238,997918,994964,996093,992713\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 447\n", + "Returned IDs: 990366,990089,995569,999898,993401,990271,996219,997855,995306,994919,999610,994279,994792,999651,995095,995096,995551,991593,999192,999791,997340,995868,993402,990173,999617,992704,990179,995384,991840,996052\n", + "Ground Truth: 990366,990089,995569,994373,999898,993401,990271,996219,997855,995306,994919,999417,999610,994279,994792,991572,999651,992174,995095,995096,995210,990438,995551,991593,991172,999192,999216,999791,997340,995868\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 448\n", + "Returned IDs: 999658,994611,991088,998966,997320,993253,993484,991593,994249,990890,991895,996838,992546,999167,998246,990936,990949,993694,994964,996219,990210,999247,994573,991155,997372,990545,997470,994542,996143,999948\n", + "Ground Truth: 999658,994611,991088,998966,997320,990444,993253,995942,993484,991593,994249,990890,991895,996838,992546,999167,998246,990936,990949,993694,994964,996219,990210,997699,999247,994573,998287,991155,999991,997372\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 449\n", + "Returned IDs: 999343,990366,994964,995438,990934,995200,998921,996219,997371,997842,990448,990652,997192,999144,998401,999647,997370,996727,993289,997320,997403,993253,998779,995927,996053,997586,996001,995455,994789,996504\n", + "Ground Truth: 999343,990366,994964,995438,990934,995200,998921,997920,996219,997371,997842,992047,995401,990448,990652,997192,999144,990866,998401,999647,993412,997370,996727,993289,994670,995096,997320,997403,993237,993253\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 450\n", + "Returned IDs: 995981,998828,997584,991467,992671,997976,990066,996755,993696,992733,991131,993047,994894,994100,992972,992119,992692,991372,992635,995336,995596,996075,999883,992748,992041,992366,993484,991456,997426,991662\n", + "Ground Truth: 995417,997843,998653,995981,998828,997584,991467,992671,997976,992931,990066,996755,993696,992733,991131,993047,994894,994100,990780,992972,992119,997449,992692,991372,992635,995336,997256,995596,996075,996966\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 451\n", + "Returned IDs: 991392,997320,996247,999453,993300,999806,999639,996744,998232,990772,991984,993904,997777,996001,995250,997258,995515,992238,993484,997913,993848,990957,990352,992429,994081,990949,992384,997305,992750,991002\n", + "Ground Truth: 995855,999680,991392,997320,996247,999453,993300,999806,999639,996744,998232,990772,991984,993904,997777,996463,996001,995250,997258,995515,992238,993484,997380,997913,993848,990957,990352,992429,994081,993003\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 452\n", + "Returned IDs: 991939,993419,997785,995306,995659,998868,998921,997984,999516,996516,990089,990366,997301,997286,991479,998445,990512,991743,997292,998726,993143,997371,994942,992451,990560,994919,998401,991172,990585,998500\n", + "Ground Truth: 991939,993419,997785,995306,995659,998868,998921,997984,999516,996516,990089,990366,997301,997286,991479,991999,999824,998445,990512,991852,991743,997292,998726,993143,997371,997873,994942,991406,992451,996135\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 453\n", + "Returned IDs: 997411,996368,997868,993751,991002,998265,994730,999486,998370,998990,999586,991585,990974,990412,996379,999977,995633,999639,990132,994895,995679,997554,997822,994964,996160,997371,997320,994380,992423,994409\n", + "Ground Truth: 997411,996368,997868,996181,996145,993751,995702,991002,995937,998265,994730,999486,998370,997723,998990,999586,991585,990974,990412,996379,996959,999801,999977,999537,994826,991157,995633,999639,990132,994895\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 454\n", + "Returned IDs: 996328,994273,995315,994327,996633,996495,997193,995122,998067,999225,999528,994998,993167,992726,993466,998864,993825,997026,997847,999980,993308,994113,993936,993328,998518,993127,992576,990887,996170,996194\n", + "Ground Truth: 996328,994273,995315,994327,996633,996495,997193,993089,995122,999391,998067,999225,999528,994998,993167,992726,993466,998864,992091,990987,993825,997026,999980,997847,993308,994113,993936,993328,996277,998518\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 455\n", + "Returned IDs: 991925,996328,996506,991807,990776,997309,996318,995268,992726,996495,993121,994119,995080,992052,997380,997208,994895,994409,999259,995098,991712,993180,990887,996001,997675,998566,992558,996429,991747,999581\n", + "Ground Truth: 991925,996328,996506,991807,990776,997309,996318,998021,995268,990987,992726,991743,996495,993121,994119,995080,992052,997380,997208,995375,994895,994409,999259,995098,991712,993180,990887,997184,996001,997675\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 456\n", + "Returned IDs: \n", + "Ground Truth: 995080,990043,999784,999516,992318,995442,997918,993052,996489,996193,995119,996829,997155,997320,995375,998726,995336,992031,995246,999741,997301,996887,999453,995641,994972,991816,994162,993528,991348,991440\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 457\n", + "Returned IDs: 997024,995372,997470,997320,994618,995841,998951,999109,992451,992978,991498,990191,993241,991650,991999,992228,993788,990275,993853,993579,998764,998195,999929,997984,998921,991545,999317,996685,990957,992942\n", + "Ground Truth: 997024,995372,997470,997320,992789,994618,995841,998951,996143,998445,999109,990936,992451,992978,991362,991498,995422,990191,993241,991650,991999,992228,993788,990275,993853,993579,998764,998195,999929,997984\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 458\n", + "Returned IDs: 990854,991935,992450,995571,998671,996328,994142,998251,990931,992713,995083,991769,998055,999126,995861,999775,996431,996495,990593,997488,990887,991257,990712,990956,996728,991062,999830,992832,998691,994891\n", + "Ground Truth: 990854,991935,992450,995571,998671,996328,991731,994142,998251,990931,992713,995083,991769,998055,991521,999126,995861,999441,999775,996431,996495,992667,990593,997488,990887,991257,990712,990956,990889,991062\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 459\n", + "Returned IDs: 992604,997110,991052,994753,995373,998318,996488,991844,995807,991309,994643,998707,991110,992387,991987,999834,997675,999136,998858,995105,999478,995963,994677,992558,990915,990729,990657,990131,999445,997466\n", + "Ground Truth: 992604,997110,991052,994753,995373,998318,996488,991844,995807,991309,994643,998383,998707,991110,992387,991987,999834,997675,999136,998858,995105,999478,995963,994677,995449,992558,991571,990915,990729,990657\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 460\n", + "Returned IDs: 995942,999358,997468,999263,999075,990595,995771,993364,994964,996218,990560,998211,993492,990146,994741,995502,999291,998964,990958,999374,992968,998644,991593,994281,993484,996592,992401,994100,993250,997301\n", + "Ground Truth: 995942,999358,997468,999263,999075,990595,995771,993364,994964,996218,990560,998211,993492,990146,994741,995865,995502,999291,998964,996869,990958,999374,992968,999701,998644,991593,990886,994281,993484,996592\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 461\n", + "Returned IDs: 997023,991958,997544,995211,990887,992917,996987,993209,992521,993561,996719,995648,994690,990331,996001,998990,995815,998776,992862,994742,995515,998778,993289,996371,994280,996220,991389,991269,993939,994835\n", + "Ground Truth: 997023,991958,997544,995211,995281,990887,992917,996987,993209,992521,993561,996719,997163,995648,995198,998720,994690,995283,990331,996001,998990,995401,995815,998776,995911,992862,994742,997191,996777,991743\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 462\n", + "Returned IDs: 993269,992521,999990,991969,999516,995813,999866,993684,992426,993561,997554,999513,994949,991869,998361,990035,993738,997822,995177,993813,995216,990603,992448,991795,998910,990845,996368,998205,996719,991076\n", + "Ground Truth: 993269,992521,999990,991969,999516,995813,999866,995283,993684,992426,993561,997554,999513,994949,991869,998361,990035,993738,997216,997822,995177,993813,995216,990603,992448,991795,994386,998346,998910,990845\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 463\n", + "Returned IDs: 998507,999832,997070,992827,998405,998006,999179,995191,993121,997442,991723,998671,992652,991421,999933,992237,996428,994196,993501,993335,994510,996216,997160,995421,991596,994541,994869,996328,991672,994467\n", + "Ground Truth: 998507,999832,997070,992827,998405,990712,998006,999435,999179,995191,993121,997442,991723,998671,990039,992652,991421,999933,992237,996428,994196,993501,996775,993335,994510,997693,996216,997160,995421,991596\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 464\n", + "Returned IDs: 996101,991382,993053,996075,996001,994718,993696,994558,997442,991409,990536,998593,992551,998495,995895,993342,993444,996809,996028,993721,999570,997963,997270,990497,999430,991184,995203,993984,996013,990096\n", + "Ground Truth: 996101,991382,993053,996075,996001,994718,993696,994558,997442,991409,990536,998593,992551,998495,995895,993342,993444,996809,996028,993721,999570,997963,997270,990497,999430,991184,995203,997448,993984,996013\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 465\n", + "Returned IDs: 997768,996082,997586,996001,992862,999284,998269,995455,992446,997334,999413,991628,999963,998154,998303,995823,998708,995536,996915,994907,990366,994964,998447,995535,996216,990096,991751,994672,999651,995887\n", + "Ground Truth: 997768,996082,997586,996001,992862,999284,998269,995455,992446,997334,999413,991628,999963,995256,998154,998303,995823,998708,995536,998568,996915,994907,990366,994964,998447,995535,996856,998440,997531,998734\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 466\n", + "Returned IDs: \n", + "Ground Truth: 996042,997251,990624,993453,992483,998703,991110,992052,997766,997234,991447,997420,998758,995464,990963,990776,995516,996232,991977,992558,992487,996814,992147,998968,993948,995373,997309,998221,999529,993433\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 467\n", + "Returned IDs: 996406,992752,993347,995920,995961,993055,995600,996216,999205,999640,993993,992628,995051,993560,997151,996549,996483,999555,992867,998594,992293,993610,998589,998716,999139,996082,998752,995191,995895,998017\n", + "Ground Truth: 996406,992752,993347,995920,997506,995103,995961,993055,995600,996216,999205,999640,993993,992628,995051,993560,997151,996549,996483,999555,992867,998594,992293,993545,995185,993610,998589,998716,999139,996082\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 468\n", + "Returned IDs: 990866,994163,996429,991880,993948,999860,992558,999639,993831,996196,990930,995552,998495,998697,995289,996188,999977,996442,994742,991790,994140,992024,992423,992469,996379,991550,995633,995983,996178,992121\n", + "Ground Truth: 990866,994163,996918,996429,991880,993948,999860,992558,999639,993831,996196,995873,990930,995552,998495,998697,995289,996188,997723,999977,996442,991052,994742,991790,994140,999587,993725,992024,992423,992469\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 469\n", + "Returned IDs: 998697,995894,997586,990096,994972,991713,997320,995536,997002,999840,998303,991530,996062,992521,991332,996918,997918,997246,995080,992571,996368,995672,995560,993504,990538,998568,998260,991157,993071,996001\n", + "Ground Truth: 998697,995894,997586,991052,990096,994972,991713,997320,995536,997002,995740,999840,993280,998303,994040,991530,996062,992521,991332,997790,996918,997918,997246,995080,992571,994334,996368,995672,995560,999234\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 470\n", + "Returned IDs: 999590,996001,995167,992862,993157,998934,996147,993174,997586,990573,998593,997555,990399,997790,996075,998481,998228,996661,994964,995701,991212,996028,995536,995250,999099,994925,992395,997320,998549,995216\n", + "Ground Truth: 999590,996001,997920,995167,992665,992862,993157,998934,996147,994531,993174,997586,990573,998593,998299,997555,990399,997790,996075,998481,998228,996661,999216,993168,994964,995701,991212,996028,991609,997002\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 471\n", + "Returned IDs: 998186,993813,999099,990081,990576,990845,996368,990035,994140,991969,995245,991350,990820,997868,995813,991958,996379,990930,997716,994436,996719,995016,995538,991011,990751,994396,996039,996127,991790,997622\n", + "Ground Truth: 998186,993813,999099,990081,990576,990845,996368,990035,995547,994140,991969,995245,991350,990820,997868,995813,991958,995923,996379,991024,990930,997716,994436,996719,995016,995538,991011,994386,990751,994396\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 472\n", + "Returned IDs: 990557,996318,993572,997918,996429,996001,995455,992749,999898,993885,994188,998401,997113,995203,995932,992161,993504,991212,996210,996866,995596,992047,990409,990181,992466,991498,997305,998849,992790,998495\n", + "Ground Truth: 990557,996318,993572,997918,996429,996001,995455,992749,999898,993885,994188,998401,990448,997113,995203,995932,991571,990519,992161,993504,994832,991212,996210,999715,994155,990772,991311,996866,997395,995596\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 473\n", + "Returned IDs: 999691,991081,998864,998845,990072,993366,996446,999349,993715,996106,998934,999273,992752,994587,993055,997039,998314,995349,991629,999498,992664,998959,992975,995270,991170,995736,995299,999139,993502,994680\n", + "Ground Truth: 999691,991081,998864,998845,997019,990072,993366,996446,999349,993715,996106,998934,999273,995636,992752,994587,995258,993055,997039,998314,995349,991629,999498,992664,998959,999663,992975,995270,994096,991170\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 474\n", + "Returned IDs: 992398,995203,996429,996001,995932,994087,993300,998734,998568,996075,993918,995536,994221,999840,995358,990196,999898,999473,993260,997555,997894,998123,994972,999210,990409,990096,997977,999731,994789,997305\n", + "Ground Truth: 992398,995203,996429,996001,995932,994087,993300,998734,998568,996075,992484,993918,995536,994221,999840,995855,995358,997140,994042,990196,994832,999898,999473,993260,997555,999963,996764,994531,997894,998123\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 475\n", + "Returned IDs: 999384,990096,998495,996809,997918,993696,997906,998950,997320,991571,996805,992944,993018,998861,995438,993260,993215,993154,996623,999210,994661,999390,991801,993504,999009,990843,999320,994964,999831,996370\n", + "Ground Truth: 999384,990096,998495,996809,997918,993696,997906,998950,997320,998383,990210,991571,996805,992944,993018,998861,995438,993260,993154,993215,996623,999210,993744,994661,999390,991801,993504,999009,991743,990843\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 476\n", + "Returned IDs: 991819,992020,994922,994789,996402,995004,998837,997370,990155,995791,993570,990379,995515,998492,998734,990652,990229,998551,990300,995759,990334,999076,994647,990448,993477,992138,998936,990310,991999,999053\n", + "Ground Truth: 991819,992020,994922,994789,996402,995004,998837,997370,990155,995791,993570,990379,995515,998492,998734,990652,990229,999888,998551,990300,995759,990334,999076,994647,990448,993477,992138,998936,990310,991999\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 477\n", + "Returned IDs: 990866,997364,993168,997586,994782,996232,993642,993831,994964,997732,998107,998668,993878,995906,993689,995482,997268,995107,992862,990934,997305,995944,997004,998458,992052,993172,999676,995536,991999,990096\n", + "Ground Truth: 990866,997364,993168,997586,996352,994782,996232,993642,993831,994964,997732,998107,997493,997279,998668,993878,990690,995906,993689,995482,991325,998369,996647,997268,995107,993630,998837,992862,990934,997305\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 478\n", + "Returned IDs: 994925,998228,999729,998039,990845,991789,990820,993906,990856,991233,991958,996873,996001,996368,990811,996298,998593,991685,995923,991200,993637,996237,995899,990930,992408,996062,998549,991125,995679,992862\n", + "Ground Truth: 994925,998228,999729,998534,998039,990845,991789,990820,993906,990856,991233,991958,996873,996001,996368,990811,996298,998593,991685,995923,991200,999590,993637,996237,997920,999009,995899,990930,992408,996062\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 479\n", + "Returned IDs: 998837,995588,991172,999784,991840,990142,999898,997135,999077,991999,993633,997238,990822,993282,994922,990366,997747,999613,990432,994964,996402,993708,992907,997275,990615,994789,990310,998582,993748,991485\n", + "Ground Truth: 998837,995588,991172,996352,999784,991840,990142,999898,997135,999077,991999,993633,997238,990822,993282,994922,990366,997747,999613,990432,994964,996402,993708,992907,997275,990615,994789,990310,998582,993748\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 480\n", + "Returned IDs: 994964,997913,992546,994100,995981,995942,997320,997470,999444,994778,990231,992330,996219,991721,999374,999740,997301,993168,998540,993484,999328,998495,998797,990925,996075,996809,999710,990957,997614,992443\n", + "Ground Truth: 994964,997913,992546,994100,995981,995942,997320,997470,999444,994778,990231,992330,999109,995560,990588,996219,991721,999374,999740,997301,993168,998540,993484,999328,998495,998797,999900,990925,998163,996075\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 481\n", + "Returned IDs: 999860,992558,996609,996196,996523,993839,997002,991334,992411,992161,993583,996606,995618,994418,996389,995482,991977,998679,997864,999715,996127,999540,996882,997914,990249,990061,995332,994280,999711,991426\n", + "Ground Truth: 999860,992558,996609,996196,996523,993839,997002,997436,991334,992411,992161,993583,996606,995618,994418,996389,997003,995482,991977,990701,998679,997864,999240,999715,996429,996127,999540,996882,997457,997914\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 482\n", + "Returned IDs: 998559,990647,996291,998779,997192,990399,991630,996727,997918,992080,995886,992330,995401,995336,995373,998921,991743,994334,998536,997571,998697,998259,993696,992384,995453,994757,995701,993143,993443,998401\n", + "Ground Truth: 998559,990647,996291,998779,998407,995374,997192,990399,991630,996727,997918,992080,995886,992330,995401,995336,995373,998921,991743,994334,998536,997571,998697,998259,993696,992384,995453,994757,995701,999450\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 483\n", + "Returned IDs: 998950,993018,999019,994171,997320,999701,996626,990083,990532,992944,996210,990843,991951,995560,993686,993080,991142,994081,991503,993696,995372,996024,996075,996726,994280,991348,992521,991224,991372,990375\n", + "Ground Truth: 998950,993018,999019,994171,997320,999701,996626,990083,990532,992944,996210,990843,991951,995560,993686,993080,998567,992318,991142,994081,991503,993696,995372,996024,996075,996726,994280,991348,992521,991224\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 484\n", + "Returned IDs: 995633,995289,991463,993725,991989,995401,990033,998106,999739,995766,990392,992160,997822,993813,992595,994419,996867,993762,997155,999665,993561,991585,994742,998536,997141,994057,998070,992289,996001,992558\n", + "Ground Truth: 995633,995289,991463,993725,997682,997319,991989,995401,990562,990033,998106,999739,995766,990392,998574,992160,997822,993813,992595,994419,996867,993762,997155,999665,999052,993412,990930,993561,991585,996254\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 485\n", + "Returned IDs: 991610,992778,997009,990293,999555,999994,992136,992721,990167,995257,993076,991667,990177,992862,994374,996408,991513,991987,998586,997506,997752,993993,999216,997655,995536,998790,997328,993948,997666,996328\n", + "Ground Truth: 991610,992778,997009,990293,999555,999994,992136,992721,990167,995257,993076,991667,990177,992862,994374,998383,996408,991513,991987,998586,999430,994796,997506,990437,997752,991304,991569,993993,999216,996633\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 486\n", + "Returned IDs: 998913,993696,992469,998950,994742,992661,994163,998790,997117,992289,999235,996809,991157,990340,992124,993809,996178,996609,996196,997320,992558,999639,992423,998662,999676,993355,994964,990866,998196,998827\n", + "Ground Truth: 998913,993696,992469,993529,998950,994742,992661,994163,998790,997117,991445,992289,999235,996809,991157,990340,992124,993809,990930,996178,996609,995873,996196,997320,992558,994317,991559,999639,994529,992423\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 487\n", + "Returned IDs: 991858,998906,990033,999450,991603,999037,996048,998045,993144,990940,991463,992519,994409,992125,995444,996388,990984,991958,991747,993813,998165,994140,998999,996719,994057,995702,999665,995052,995289,996482\n", + "Ground Truth: 991858,998906,990033,999450,991603,999037,996048,998045,993144,990940,991463,992519,994409,992125,995444,996388,990984,991958,991747,993813,995864,998165,994140,998999,997187,996719,994057,995702,999665,995052\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 488\n", + "Returned IDs: 997270,990039,996475,998794,996650,997047,992780,990377,992100,993479,995750,999430,999323,996013,996830,999659,993890,999551,990718,991959,993391,992872,998599,992754,993721,991382,999722,994558,997331,996016\n", + "Ground Truth: 997270,990039,996475,998794,996650,997047,992780,990377,992100,993479,995750,999430,999323,996013,996830,999659,997939,993890,999551,990718,991959,993391,992872,998599,992754,993721,991382,999722,994558,997331\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 489\n", + "Returned IDs: 999946,997476,990043,996093,997135,992169,992907,995791,999204,993570,990483,992612,996438,994922,993237,994144,999453,992632,990580,991146,994703,995489,992994,998734,998523,991473,995004,998837,993355,992813\n", + "Ground Truth: 999946,997476,990043,996093,993675,997135,992169,992907,995791,999204,993570,990483,992612,996438,994922,993237,994144,999453,992632,990580,994827,993238,991146,994703,995489,992994,992342,998734,994629,998523\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 490\n", + "Returned IDs: 994384,996140,991550,991817,990279,994732,998697,997739,996380,995983,992558,991458,995552,992661,998950,994341,996926,994171,994163,993686,990068,991791,990383,996196,991713,998827,997258,990456,990343,993018\n", + "Ground Truth: 994384,996140,991550,991817,990279,994732,998697,997739,996380,995983,992558,991458,995552,992661,998950,994341,996926,994171,994163,993686,990068,996894,991791,990383,995372,996196,998567,991713,998827,990769\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 491\n", + "Returned IDs: 990096,998968,993624,994636,992716,996386,999103,997884,999711,999119,996209,999282,998861,990072,990154,999287,995536,997803,991451,999319,993055,993744,999443,991724,996283,999384,996893,999744,994834,992068\n", + "Ground Truth: 990096,998968,998419,993624,994636,992716,996386,991936,999103,991167,997884,999711,999119,996209,999282,998861,990072,990154,999287,995536,997803,991451,999319,993055,999443,993744,991724,996283,996429,999384\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 492\n", + "Returned IDs: 996284,997182,996809,996830,994465,994382,991382,997318,990167,995895,991881,992551,999555,994927,990361,996706,993391,993890,998495,990145,992752,990226,994431,996580,992780,995978,998006,996494,991767,998895\n", + "Ground Truth: 996284,997182,996809,996830,994465,994382,991382,997318,990167,995895,991881,992551,999555,994927,990361,996706,993391,993890,998495,990145,992752,990226,994431,996580,992780,999886,994355,995978,998006,996494\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 493\n", + "Returned IDs: 993283,991485,992309,999639,994042,991157,995562,990399,995516,998232,995443,991015,999806,998180,992124,996744,994809,998697,993260,999041,996368,991984,993055,993289,996657,992862,992905,991002,996138,998758\n", + "Ground Truth: 993283,991485,992309,999639,994042,991157,995562,990399,995516,998232,995443,991015,999806,998180,992124,996744,994809,998697,993260,999041,996368,991984,993055,994922,993289,996657,992147,992862,992905,991002\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 494\n", + "Returned IDs: 992169,996325,997852,992152,997476,992632,992813,994922,993717,996093,999676,990266,994318,999453,994761,996438,992612,997822,994148,999077,997797,994956,990268,991334,998999,998765,993080,998138,998523,990080\n", + "Ground Truth: 992169,996325,997852,992152,997476,992632,992813,992309,994922,993717,996093,990490,999676,990266,994318,999843,999453,994761,996438,992612,997645,990399,997822,994148,994161,999077,995210,997797,998171,994956\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 495\n", + "Returned IDs: 994960,991383,992164,999138,996537,990089,996011,998812,992026,996037,996301,992827,996328,990730,992372,998348,993335,993715,991390,991885,999227,995191,998076,997357,998671,992832,993568,992237,999430,993413\n", + "Ground Truth: 994960,991383,992164,999138,996537,990089,996011,998812,992026,996037,996301,992827,996328,990730,992372,998348,993335,993715,991390,991885,999227,995191,998076,997357,998671,992832,993568,992237,999430,990040\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 496\n", + "Returned IDs: 992124,991984,995516,998697,994742,999136,995701,998758,993186,993727,995672,995562,992747,993834,996075,993342,996657,991530,994082,994809,999783,994879,998950,995203,995981,995119,994280,994325,996884,992480\n", + "Ground Truth: 992124,991984,995516,998697,994742,999136,995701,998758,993186,993727,995672,995562,993470,992747,993834,996075,994374,993342,996657,999041,991530,994082,991052,994809,999783,997377,994879,998586,998950,995203\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 497\n", + "Returned IDs: 997969,993765,993642,997732,999151,994964,990226,996216,993929,990091,994927,993080,998436,998107,998495,998416,997410,997004,993906,991633,997453,990690,996647,990632,997586,990845,992820,993831,996171,998950\n", + "Ground Truth: 997969,993765,993642,997732,999151,994964,997111,990226,996216,995920,993929,990091,994927,990441,993080,998436,998107,998495,998416,997410,997004,993906,991633,997453,997112,990690,996647,990632,991239,990580\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 498\n", + "Returned IDs: \n", + "Ground Truth: 999899,994906,996759,992648,992651,999977,998083,990304,990260,991175,998301,992979,993713,992126,993275,996368,997059,999820,996714,996425,993032,996075,997554,993280,994860,997910,999587,999453,994334,994946\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 499\n", + "Returned IDs: 998080,991334,999479,998820,991025,995190,993931,997395,994222,990126,998493,994204,991449,991743,990733,996609,992685,995401,999098,990249,992604,997515,995289,999788,990131,994431,999961,999533,993566,997209\n", + "Ground Truth: 998080,991334,999479,998820,991025,995190,993473,993931,997395,994222,990126,997030,998493,999586,994204,995612,991449,991743,990733,996609,993929,997212,992685,995401,999098,990249,992604,997515,995332,995289\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 500\n", + "Returned IDs: 999009,991776,995895,990096,993397,997586,990930,990786,999430,995455,994742,996624,990866,990701,990271,997028,996830,997768,999599,991154,995701,996216,998495,996192,995435,999639,995899,996001,997378,999261\n", + "Ground Truth: 999009,991776,995895,990096,993397,997586,990930,990786,999430,995455,994742,996624,990866,990701,990271,997028,996830,997768,999599,991154,995701,996216,998495,996192,995435,999639,995899,991572,996001,999647\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 501\n", + "Returned IDs: 998158,997453,994463,992488,996062,998874,992322,990930,996915,991021,994783,995289,999139,993561,990076,990104,991685,990226,999091,992326,995783,996028,991857,992703,992182,990899,995126,991633,998302,998593\n", + "Ground Truth: 998158,997453,994463,992488,996062,998874,992322,990930,996915,991021,994783,995289,999139,991154,994777,993561,993673,996071,990076,990104,991685,990226,995113,999091,992326,995783,997797,996028,991857,992703\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 503\n", + "Returned IDs: 994894,998353,995981,994100,997913,995942,992228,997946,999243,991045,997301,994266,998090,997728,998246,999374,996219,996805,993186,992384,997320,999331,997614,997304,999444,992848,993529,992124,998417,995701\n", + "Ground Truth: 994894,998353,999599,995981,994100,997913,995942,992228,997946,999243,999658,991045,990701,997301,994266,998090,997728,998246,999374,996219,991468,996805,993186,992384,997320,999331,997614,997304,999444,992848\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 504\n", + "Returned IDs: 990532,999453,995852,994081,994955,993053,995295,996147,992408,994042,995203,995456,990392,990771,991609,999052,998039,997002,999639,996918,997918,993809,998855,993906,996368,996001,992944,998536,996879,993157\n", + "Ground Truth: 990532,999453,995852,994081,995130,994955,993053,995295,996147,992408,994042,995203,995456,990771,990392,991609,999052,998039,997002,999639,996918,995401,997918,993809,998855,993906,994226,996368,992832,992562\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 505\n", + "Returned IDs: 991698,997395,993407,992790,990215,994671,991404,993409,992832,995375,995080,991607,997515,990889,995685,992576,995648,996245,992749,999052,990930,998536,995968,998158,997192,992036,994280,999967,996918,994271\n", + "Ground Truth: 991698,997395,993407,992790,990215,994671,991404,993409,992832,995375,995080,991607,994855,997515,990889,995685,992576,991576,995648,996245,992749,999052,990930,997391,998536,993412,995968,998158,997192,992036\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 507\n", + "Returned IDs: 994472,999791,991921,990089,992322,994402,990091,992820,995720,998107,997939,999876,996355,998355,990226,998630,995535,996761,993775,993584,997349,995638,993929,991751,998169,993378,991382,993637,994927,999294\n", + "Ground Truth: 994472,999791,991921,990089,992322,994402,990091,995222,992820,995720,998107,997939,999876,998765,996355,990622,996050,998355,990226,999642,998630,995535,998594,996761,993775,993584,997349,995638,993929,991325\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 508\n", + "Returned IDs: 992582,991743,990949,991782,990958,995274,999533,997614,994850,994266,995949,995981,999331,990957,997913,990493,991889,999343,990910,998921,998779,998302,997301,994919,997320,991984,995515,994894,997571,995886\n", + "Ground Truth: 992582,991743,990949,991782,990958,995274,995374,999533,993961,997614,994850,990490,994266,995949,995981,997645,991479,996648,999331,990957,997913,990588,990359,990493,991889,999328,993431,999343,990910,998294\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 509\n", + "Returned IDs: 998982,990328,992226,990528,990701,998671,992230,991334,990711,998719,994751,999500,994783,994689,995990,995295,997586,990126,996116,990864,991075,991464,996645,994558,997485,999430,998543,995020,993656,996481\n", + "Ground Truth: 998982,990328,992226,990528,990701,996110,992068,998671,996939,992230,991334,990468,990711,998719,994751,999500,994783,994689,991239,999916,995990,995295,991771,997379,997586,990126,996116,990011,990864,991075\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 510\n", + "Returned IDs: 997913,990650,991593,998500,994964,997301,997320,998170,999982,997192,992978,990066,999948,999167,992087,998899,995802,996219,991335,997470,998808,999647,996504,998550,995961,990210,995949,998302,998921,991479\n", + "Ground Truth: 997913,990650,991593,998500,994964,997301,994831,997320,998170,999982,997192,992978,994618,990066,995942,999948,999167,992087,998899,995802,996219,991335,997470,998808,999647,996504,998550,995961,996053,990210\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 511\n", + "Returned IDs: \n", + "Ground Truth: 993387,994192,998899,996946,992464,992119,997913,996592,990957,991479,990890,997320,993662,992134,993174,995336,994978,999658,996296,996075,993853,992330,991260,999559,998550,998090,991131,994432,999948,996397\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 512\n", + "Returned IDs: 997168,996277,999281,991245,993275,999526,992315,994743,993715,990409,991685,992636,996001,998623,999724,990956,991349,991037,998536,993560,994016,990951,998045,995701,993561,995562,993738,994409,993948,994353\n", + "Ground Truth: 997168,996277,999281,991245,993275,999526,992315,994743,993715,990409,991685,992636,996001,998623,999724,990956,991349,991037,998536,993560,994016,990951,998045,995701,993561,995562,991755,995835,993738,994409\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 513\n", + "Returned IDs: 991375,993906,996216,996082,997790,995991,994925,996062,998950,996599,996178,995107,991769,995895,996001,999827,990364,993561,992780,995705,994280,993931,991142,992847,990096,994742,992322,992680,990930,996196\n", + "Ground Truth: 991375,993906,996216,996082,997790,995991,994925,996062,998950,996599,996178,995107,991769,995895,992473,996001,995961,999827,990786,990364,999009,993561,990886,992780,995705,994280,993931,990887,991142,995586\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 514\n", + "Returned IDs: 991396,990925,998797,990595,997972,999262,996959,995884,994964,997102,993914,996532,990146,998501,997040,999600,994673,993389,997320,993761,990863,991008,993083,991086,990797,992284,992147,996714,999444,993484\n", + "Ground Truth: 991396,994160,990925,998797,990595,997972,999262,996959,995884,994964,997102,993914,995588,996532,990146,998501,993168,997040,997028,999600,994673,993389,997320,995909,993761,997189,990863,991008,993083,991086\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 515\n", + "Returned IDs: 996379,991255,995268,990930,993561,992367,997925,990984,993673,992703,996198,996277,992294,994589,992115,995701,990133,996867,990899,994113,995289,991011,998779,996328,996918,999977,990940,990033,996930,996678\n", + "Ground Truth: 996379,991255,995268,990930,995963,993561,992367,991747,991603,996913,991459,997925,990984,993673,992703,996198,996277,992294,992660,994589,990897,992115,995701,990133,996867,990899,994113,995289,991011,998779\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 516\n", + "Returned IDs: 996187,997868,996605,998941,991349,991795,998527,995814,998255,990732,999121,999926,994409,998205,991878,997998,998796,990035,999731,991163,992858,991969,996618,992311,998143,995813,994638,994773,994140,995856\n", + "Ground Truth: 996187,997868,996605,998941,991349,991795,998527,995814,998255,990732,999121,999926,994409,998205,991878,997998,998796,990035,999731,991163,992858,991969,996618,992311,998143,995813,994638,994773,990481,994140\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 517\n", + "Returned IDs: 996038,998832,995507,997723,995139,997046,996323,995905,999619,997406,996867,992545,993433,999000,998165,993421,999530,995289,992130,997832,996769,997020,996112,991027,993390,995753,992605,995052,997607,996968\n", + "Ground Truth: 996038,998832,995507,997723,995139,997046,996323,995905,999619,997406,996867,992545,993433,999000,998165,993421,999530,995289,992130,997832,996769,997020,996112,991027,993390,996166,992660,996815,995753,992605\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 518\n", + "Returned IDs: 993501,999179,997733,998671,992193,992652,996803,996211,999832,990784,993931,997068,992827,994929,992237,997967,992981,998870,996809,997126,991672,994402,990186,990852,994119,993335,996328,994541,993929,997331\n", + "Ground Truth: 993501,999179,997733,998671,992193,992652,996803,996211,999832,990784,993931,997068,992827,994929,993439,992237,997967,992981,994094,998870,996809,997126,995790,991672,994402,990186,990852,994119,993335,996328\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 519\n", + "Returned IDs: 992315,991037,990596,990834,996562,994302,995562,994123,995349,996286,991648,995500,990630,998358,998052,996908,999435,995740,994955,990203,995366,998855,992199,996884,998303,993504,991212,996001,990409,994743\n", + "Ground Truth: 992315,991037,990596,990834,996562,994302,995562,994123,995349,996286,991648,995500,990630,998358,998052,996908,996677,999435,995740,994955,990203,995366,998855,992199,996884,998303,991956,993504,991212,996001\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 520\n", + "Returned IDs: 997320,997918,993512,992521,998260,995336,999731,993071,994964,993492,994360,999516,996397,994081,998365,994290,995826,992124,995274,997977,992330,992909,991880,998568,992423,995560,996085,994577,998950,998550\n", + "Ground Truth: 992692,997320,997918,993512,990171,992521,998260,995336,999731,993071,994964,993492,994360,999516,996397,994661,994081,998365,998246,994290,995826,992124,996417,995274,997977,992330,992909,991883,991880,992047\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 521\n", + "Returned IDs: 999453,994081,990661,990532,996075,997370,990652,998567,998492,998734,996309,990882,995515,997320,996626,992521,995372,993280,995203,991869,994789,990083,996368,995245,993241,994449,990043,996001,992617,991348\n", + "Ground Truth: 999453,994081,990661,990532,996075,997370,990652,998567,998492,998734,996309,990882,995515,997320,996626,992521,996193,995372,991713,993280,995203,991869,999123,994789,990083,999954,996368,995401,992031,995245\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 522\n", + "Returned IDs: 997822,993433,995289,995633,996042,998106,996494,996867,998165,992160,995401,997046,993390,997766,995040,990252,995200,994742,992558,993053,996769,992521,994783,995052,990435,994529,992448,997961,991301,992576\n", + "Ground Truth: 997822,993433,995289,995633,996042,998106,997797,993857,996494,996867,998165,992160,995401,995873,997046,993390,997918,997766,995040,990252,995200,994742,992558,993053,996769,992521,994783,996028,995052,990435\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 523\n", + "Returned IDs: 998246,992896,990066,997320,993484,994225,996560,998540,998634,999005,995400,992839,997102,998384,997610,995841,998603,996598,997178,994164,994214,991999,994964,991016,994183,991281,994611,999810,994873,994632\n", + "Ground Truth: 998246,992896,990066,997320,993484,994225,996560,999757,998540,999102,998634,999128,995904,995224,999005,992986,995400,994542,992839,991567,997102,998384,997610,995841,998603,990269,996598,997178,994164,994214\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 524\n", + "Returned IDs: \n", + "Ground Truth: 990882,994612,997465,999698,998937,997403,993845,998607,995515,999036,996309,992228,994922,995788,997164,995154,993002,992309,996840,993028,998921,999671,998244,992843,993143,997591,991510,995893,996653,998401\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 525\n", + "Returned IDs: 998225,990022,996075,997305,991348,993809,993341,996626,990697,995203,996001,994416,998359,992635,990532,996538,993721,993696,990083,995250,990573,990786,999289,993637,995899,990866,992557,994233,994081,999453\n", + "Ground Truth: 998225,990022,996075,997305,991348,993809,993341,996626,990697,995203,996001,994416,998359,992635,990532,996538,993721,993696,995613,990083,995250,990573,990786,999289,993637,995899,990866,992557,994233,994081\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 526\n", + "Returned IDs: 998536,995655,995385,996245,991332,995168,992832,991630,995589,994280,995231,996760,993901,991654,998565,995536,992110,996501,999443,993489,992793,997965,993194,997145,993037,994380,998697,997158,998303,999994\n", + "Ground Truth: 998536,995655,997702,995385,996245,991332,995461,995168,992832,991630,995811,995589,994280,993381,995231,996760,993901,991654,990429,998565,995536,992110,996501,999443,993489,992793,997965,993194,993580,997145\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 527\n", + "Returned IDs: 995596,993341,996741,990697,997305,996075,994886,992041,994985,993785,996178,993119,994859,996755,993154,997661,994440,994971,993904,992661,990022,993696,996210,997291,992635,993741,991650,997320,998715,996616\n", + "Ground Truth: 995596,993341,996741,990697,997305,996075,994886,992041,994985,993785,996178,990571,993119,994859,996755,993154,997661,994440,994971,993904,992661,990022,997699,993696,996210,997291,992635,993741,991218,991650\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 528\n", + "Returned IDs: 991795,999800,997868,996187,994140,996867,993003,990341,998205,992448,991969,990845,992994,998633,994419,994409,996042,993561,993813,995289,995864,996169,998361,997089,995702,996719,991790,997909,994386,999250\n", + "Ground Truth: 991795,999800,997868,993412,996187,994140,996867,993003,990341,998205,992448,991969,990845,992994,998633,994419,994409,996042,993561,993813,995289,995864,996169,998361,997089,995702,996719,991790,997909,994386\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 529\n", + "Returned IDs: 992636,997168,995837,997201,995295,993715,994621,999052,998076,995159,994488,997838,991451,990956,996505,994380,998021,990336,990887,990889,996147,995836,993448,996277,992973,998174,995521,991007,997329,992450\n", + "Ground Truth: 992636,997168,995837,997201,995295,993715,994621,999052,998076,995159,994488,997838,991451,990956,992832,992595,996505,994380,998021,990336,990887,990889,996147,995836,993448,996277,992973,998174,995521,991007\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 530\n", + "Returned IDs: 990072,990096,990154,999663,992716,995536,994103,998861,994834,994756,993055,994636,999711,999677,999898,992862,996406,993139,996733,996623,997506,992293,995963,997480,994640,994771,997154,996386,996209,995701\n", + "Ground Truth: 991167,990072,990096,990154,999663,992716,995536,994103,998861,994834,994756,993055,998419,994636,999711,995562,999677,999898,992862,996406,993139,993624,996733,996623,995103,997506,992293,992110,995963,997480\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 531\n", + "Returned IDs: 994749,991673,992685,992165,990772,995886,999182,993840,992677,999453,994655,997433,999586,992052,991943,995274,995401,991787,990432,995932,991585,995968,999820,997370,997574,990957,993260,995515,990103,990214\n", + "Ground Truth: 994749,991673,992685,992165,990772,995886,999182,995374,996949,993840,992677,997498,999453,991110,994655,997433,999586,992087,991052,992052,991943,995274,993673,995401,991787,990432,995932,991585,993412,995968\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 532\n", + "Returned IDs: 996216,990930,999009,998950,994964,995961,999453,993484,997320,995895,999716,993906,999294,993080,999430,990096,997379,994925,991498,998593,997918,990560,992780,999849,996911,998455,994742,993389,995438,998495\n", + "Ground Truth: 996216,990930,999009,998950,994964,995961,999453,993484,997320,995895,998123,999716,993906,999294,993080,999430,992497,990096,997379,994925,990445,991498,998593,997918,990560,992780,999849,996911,998455,992847\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 533\n", + "Returned IDs: 997506,992154,996775,995226,994891,997944,997050,991633,998933,998177,999139,999079,993665,999576,993347,991712,998536,992793,990194,996817,990959,992170,997666,997906,994955,994280,996733,995536,993821,990196\n", + "Ground Truth: 997506,992154,996775,996055,995226,994891,997944,997050,991633,998933,998177,999139,999079,993665,999576,993347,991712,998536,992793,990194,996817,990959,992170,997666,997906,994955,994280,996733,995536,999091\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 534\n", + "Returned IDs: 999876,993484,993504,998302,990092,993803,997320,998294,996869,997370,997913,998029,991498,993741,998580,992384,993885,999535,999374,990146,991043,991593,999982,998246,994089,994366,995949,995372,997661,992733\n", + "Ground Truth: 999876,993484,993504,998302,990092,993803,997320,998294,996869,997370,997913,998029,999363,991498,993741,994826,998580,992384,993885,999535,999374,990146,991043,991593,999982,998246,994089,994366,995949,995372\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 535\n", + "Returned IDs: 994685,994160,996343,990936,998634,990925,993484,998966,990146,992546,999262,998644,998797,995884,993885,990958,997239,997102,991558,996532,993389,992147,999374,991396,993083,994964,996295,998153,990224,996874\n", + "Ground Truth: 994685,994160,996343,990936,994758,998634,990925,997876,993484,998966,990146,992546,999262,999620,998644,998797,995884,993885,990958,997239,997102,991558,996532,993389,992147,999374,991396,993083,994964,996295\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 536\n", + "Returned IDs: \n", + "Ground Truth: 993497,996763,998293,990956,995058,994073,995783,995159,996276,990149,992997,990872,995184,997713,991850,992981,997897,990306,992526,997746,993727,996189,992052,994280,993478,997752,996911,994315,992784,999514\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 537\n", + "Returned IDs: 997958,993767,997185,992272,994742,993997,999235,999279,998733,994730,993209,999402,993856,997566,995268,995633,999805,998162,999160,998962,991585,994140,996867,998748,993466,997660,995640,992289,993751,998898\n", + "Ground Truth: 997958,993767,997185,992272,994742,993997,999235,999279,998733,994730,993488,993209,999402,993856,992063,997566,995268,995633,999805,998162,999160,998962,991255,991585,994140,996867,998748,993466,997660,995640\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 538\n", + "Returned IDs: 990589,999555,990167,992778,992234,994210,992353,996209,995610,991610,990194,994756,997242,998006,998495,992628,999634,994355,997850,993224,995879,998122,999994,998861,996794,991484,995209,997415,990107,998435\n", + "Ground Truth: 990589,999555,990167,992778,992234,994210,992353,996209,995610,991610,990194,994756,997242,998006,998495,996342,992628,999634,993055,994355,997850,993224,995879,998122,999994,998861,999771,996794,992745,994927\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 539\n", + "Returned IDs: 990157,995482,995083,992859,992450,992434,999934,994428,996306,991769,993731,992449,999240,994185,991935,995784,999695,998671,991185,992595,990593,994998,990331,993777,990784,993437,993856,998256,990126,990887\n", + "Ground Truth: 990157,995482,994976,995083,992859,992450,992434,999934,994428,996306,991769,993731,992449,999240,994185,991935,995784,999695,998671,991185,992595,990593,994998,990331,993777,990784,993437,993856,998256,990126\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 540\n", + "Returned IDs: \n", + "Ground Truth: 995300,999981,994742,995633,999474,992468,999372,998913,995012,990905,998765,992415,999788,995494,998543,998450,995289,996302,998557,992289,996057,998895,991590,991185,990126,990392,992598,999235,992239,995030\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 541\n", + "Returned IDs: 998965,996219,994452,992998,995339,990780,999683,995455,996111,994777,991370,996178,997004,995460,990886,996612,998364,991373,990931,990366,993128,999343,998390,998593,999876,991179,998157,996564,995438,995332\n", + "Ground Truth: 998965,996219,994452,992998,995339,990780,999683,995455,996111,994777,991370,996178,992070,997032,997004,995460,994981,990886,996612,998364,991373,990931,990366,993128,999343,997813,993690,998390,999013,997706\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 542\n", + "Returned IDs: 997131,993185,994063,999155,993232,995461,996809,994643,990255,997462,990941,994122,998632,995963,998855,990930,995421,998695,992809,998786,996930,997723,998919,996797,992558,998593,995766,991767,990956,991532\n", + "Ground Truth: 997131,993185,994063,999155,993232,995461,996809,994643,990255,997462,990941,994122,998632,995963,998855,990361,990930,995421,998695,992809,998786,996930,997723,998919,996797,992558,998593,995766,991767,990956\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 543\n", + "Returned IDs: 992944,993018,995560,998950,991142,995438,996726,998160,996140,990375,990843,991650,993324,998364,997320,991951,995107,999210,994171,994964,995130,995332,996370,998697,996653,990585,991206,997002,992318,996805\n", + "Ground Truth: 992944,993018,995560,998950,991142,995438,996726,998160,996140,990375,990843,991650,993324,998364,997320,991951,995107,999210,994171,994964,995130,999647,997379,995332,996370,999109,998697,998180,993499,996653\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 544\n", + "Returned IDs: 993755,992761,991850,998778,991779,992595,995536,994356,992832,999107,999256,993055,995385,996001,996216,998536,990127,995501,999384,991644,995655,996328,999798,992561,997586,997141,991667,994076,992517,990392\n", + "Ground Truth: 993755,992761,991850,998778,991779,995222,997155,992595,995536,994356,992832,999107,999256,993055,995385,996001,996216,998536,992110,997158,990127,995501,991474,999384,991644,995655,996328,992224,999798,992561\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 545\n", + "Returned IDs: 997391,992428,991698,996930,992933,998080,998855,994193,993255,991257,993288,997918,991337,992179,995357,991743,996762,995401,999144,992516,990459,996903,992531,998121,991110,992604,997045,991459,997942,993673\n", + "Ground Truth: 997391,992428,991698,996930,992933,990438,998080,998855,994193,991619,993255,991257,993288,997918,991337,992179,999527,995357,991743,992850,996762,995511,995401,994062,999144,992685,998180,992516,990459,996903\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 546\n", + "Returned IDs: 994419,996745,992894,990252,997776,996322,996148,998593,992660,995643,997046,998486,997940,995289,992125,991301,999482,995923,996867,992048,997822,998045,995702,991958,995063,998564,996323,997937,994173,998165\n", + "Ground Truth: 994419,996745,992894,990252,997776,996322,996148,998593,992660,995643,997046,998486,997940,995289,992125,995470,991301,999482,995923,996867,992048,997822,998045,993412,995702,991958,995063,998564,998105,995401\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 547\n", + "Returned IDs: 991630,992862,990005,999450,998536,994409,996368,992521,991844,998779,997918,991987,999653,991332,995701,995373,993948,995536,991667,995401,996809,995200,996657,994955,996328,991795,993813,998495,994946,995702\n", + "Ground Truth: 991630,992862,990005,999450,998536,998999,994409,996368,992521,991844,998779,997918,991987,999653,991332,995701,993412,995373,993948,998858,998697,991667,995536,997630,995401,996809,995864,995200,996657,994955\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 548\n", + "Returned IDs: 995716,999261,994922,999620,990701,990934,997371,990648,996916,997770,999613,991588,994990,995693,992944,992384,997403,996219,991999,995551,999343,996840,996990,997529,995387,993215,990366,998921,999610,998523\n", + "Ground Truth: 990092,995716,998991,990315,999261,994922,999620,997920,990701,990934,997371,990648,996916,991921,997770,991196,999613,991588,994990,995222,995401,995693,998641,992944,992384,996419,997403,997855,996219,999151\n", + "Recall: 0.6000\n", + "\n", + "Query ID: 549\n", + "Returned IDs: 997897,998250,993483,999463,992437,995421,992727,998593,990255,990521,992367,998934,993175,995626,998628,996181,991245,998162,994280,995401,990844,998032,996001,994809,997918,993727,996657,993232,996043,995823\n", + "Ground Truth: 997897,998250,993483,999463,992437,995421,992727,998593,990255,990521,992367,998934,993175,995626,998628,992407,996181,991245,998162,993489,994280,995182,995401,990844,998032,996001,994809,997918,993727,996657\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 550\n", + "Returned IDs: 997320,991498,995841,995372,990585,993484,992944,996959,993018,997305,994964,996001,990949,997301,998302,998950,997370,995203,990096,993885,998246,998495,998849,995438,997913,995375,992119,997796,996318,990957\n", + "Ground Truth: 997320,991498,995841,995372,990585,993484,992944,996959,993018,997305,998445,994964,996001,990949,991052,997301,998302,998950,997370,995203,990096,993885,991743,998246,998495,998849,995438,993412,990772,997913\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 551\n", + "Returned IDs: 991335,998500,997688,999647,998139,998329,990092,994699,997192,992357,996727,997371,995370,992546,990934,990366,999898,994778,996576,992994,990315,990505,999343,995203,990231,992330,996646,999201,994831,990957\n", + "Ground Truth: 991335,998500,990650,997006,995985,997688,993174,999647,995942,994776,998139,998329,990092,994699,997192,992357,996727,997371,995370,992546,990934,990366,999898,995906,990980,992822,993034,994778,990866,996576\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 552\n", + "Returned IDs: 995655,991667,997973,992517,997145,993001,991685,999107,990330,990458,991093,998861,998122,997732,997679,992561,998536,997265,991569,996733,996028,995835,997332,999827,995780,995536,990096,997319,995167,997685\n", + "Ground Truth: 995655,991667,997973,992517,997145,993001,991685,999107,990330,990458,991093,995222,993280,998861,990091,998122,997732,997679,992561,998536,997265,991569,996733,996028,998710,992862,995835,997332,999827,993006\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 553\n", + "Returned IDs: 997698,993715,998716,998030,990392,993917,991663,992664,996183,995705,992636,991685,995837,997278,996986,990364,999052,998623,991620,998903,990998,997944,992407,992026,993366,991920,998440,994510,995735,991451\n", + "Ground Truth: 997698,993715,998716,998030,990392,993917,991663,992664,996183,995705,992636,991685,995837,997278,996986,990364,999052,998623,991620,998903,998174,992547,990998,997097,997944,992407,992026,993366,991920,998440\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 554\n", + "Returned IDs: 993305,992350,992258,991152,996163,996669,995248,997540,996620,997395,998892,995542,991705,997689,999579,994301,995772,995986,990019,999685,993255,997538,999298,997547,992150,990569,997169,994072,991977,990093\n", + "Ground Truth: 993305,992350,992258,991152,996163,996669,995248,997540,996620,997395,998892,995542,991705,997689,999579,994301,995772,995986,990019,999685,993255,997538,999298,997547,992150,990569,997169,994072,997523,991977\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 556\n", + "Returned IDs: 990399,993834,999041,993055,994237,998697,995184,999087,995701,996075,996230,999639,998758,992225,991042,991530,995672,998720,993727,995418,999724,995203,994167,994809,991256,994280,991485,995516,993283,998514\n", + "Ground Truth: 990399,993834,999041,993055,995855,994237,998697,995184,999087,995701,992774,991300,996075,996230,999639,998758,992225,999651,991042,993902,991530,995672,998720,993727,995418,991557,997531,999724,995203,994167\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 557\n", + "Returned IDs: 995336,998090,992384,996296,990949,994100,996657,996001,997918,997320,998951,990343,995873,997571,993143,993809,997554,997913,990772,992330,990339,996592,992880,992052,993492,999144,992692,990897,990910,993289\n", + "Ground Truth: 995336,999900,992277,998090,992384,996296,990949,994100,996657,996001,997918,997320,998951,990343,995873,997571,993143,995374,993809,997554,997913,990772,992330,993512,990339,996592,992880,992052,993492,999144\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 558\n", + "Returned IDs: 998934,998720,991459,994167,994280,991130,996001,998593,992429,999639,999173,999216,993834,992437,992441,993993,997337,995158,995701,993375,996328,997790,992225,996526,993848,998864,997847,995895,994610,990096\n", + "Ground Truth: 998934,998720,991657,991459,994167,994280,991130,996001,992429,998593,999639,999173,999216,993834,999408,995103,996408,992437,992441,993993,997337,995158,995701,993375,996328,997790,992225,996526,997166,993848\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 559\n", + "Returned IDs: 996809,998495,990340,995895,997442,994529,990435,999500,993391,999377,994783,998913,997182,994830,999430,994269,999352,992827,990126,994402,996411,995633,991382,995535,992551,991128,996819,992566,993219,995476\n", + "Ground Truth: 996809,998495,990340,995895,997442,994529,990435,999500,993391,999377,994783,998913,997182,994830,999430,996049,994269,999352,992827,990126,994402,996411,995633,991382,995535,992551,991128,996819,992566,993219\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 560\n", + "Returned IDs: 997247,997977,995203,996868,999111,992082,993504,995740,992241,993157,996075,992747,995372,993174,999735,994167,995855,991425,993609,994672,996196,998303,992825,999526,993489,997368,996217,990196,998715,992571\n", + "Ground Truth: 997247,997977,995203,996868,999111,992082,993504,995740,992241,993157,996075,992747,995372,993174,999735,994167,995855,991425,993609,994672,996196,998303,992825,999526,993489,997368,996217,990196,998715,994694\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 561\n", + "Returned IDs: 991753,995066,995401,996607,993003,991680,994106,996813,992448,997975,996304,990319,999054,993597,991639,995864,998910,997544,993642,997091,996695,999759,997176,999646,997075,991795,995490,992653,993209,991592\n", + "Ground Truth: 991753,995066,995401,996607,993003,991680,994106,996813,992448,997975,996304,990319,999054,993597,991639,995864,998910,997544,993642,997091,996695,999759,999646,997176,997075,991795,995702,995490,992653,993209\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 562\n", + "Returned IDs: 996043,997723,995401,991630,998267,990033,991954,997380,990525,991403,999595,993275,991074,990325,995940,991666,998166,994178,999277,993412,999587,995200,994409,999977,991794,994474,999228,997918,996903,998045\n", + "Ground Truth: 997957,996043,997723,995401,991630,998267,990033,991954,997380,991603,990525,991403,999595,993275,991074,990325,995940,992582,991666,998166,994178,999277,993412,999587,995200,996181,994409,999977,991794,997630\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 563\n", + "Returned IDs: 990508,998359,999717,995206,991323,991650,993603,997581,991348,996178,992001,996075,999258,993686,996626,994233,993809,999806,999473,996458,990022,992557,990866,997367,999480,994331,995596,995108,999230,990456\n", + "Ground Truth: 990508,998359,999717,990031,995206,991323,991650,993603,997581,991348,996178,992001,999132,996075,999258,993686,996803,996626,994233,993809,999806,999473,994341,996458,990022,992557,990866,997442,990912,997367\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 564\n", + "Returned IDs: \n", + "Ground Truth: 993591,997036,993015,993553,994275,997170,991989,996906,990368,994021,995923,995345,992585,998039,994663,990955,992798,994386,993561,993164,994379,992014,990845,993725,997357,993813,997920,991168,993697,997933\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 565\n", + "Returned IDs: 992286,993030,995854,997634,997020,994301,998386,995723,996273,998080,993263,998650,990591,992531,994677,997723,991423,996139,997395,993717,990189,991404,995062,993223,995625,993127,990178,999961,999020,992558\n", + "Ground Truth: 992286,993030,995854,997634,997020,994301,998386,995723,997114,996273,998080,993263,992683,998650,995282,990591,992531,994677,997723,994360,991423,996139,997395,990103,993717,990189,991404,995062,993223,995625\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 566\n", + "Returned IDs: 991384,998049,996019,997675,998536,996501,995521,998933,995080,996506,999448,996245,996505,996918,996062,996313,995589,990479,999091,990537,997395,999079,993931,995536,998874,992211,994955,994280,995655,995408\n", + "Ground Truth: 991384,998049,996019,997675,998536,996501,995521,998933,991904,995474,995080,996506,999448,996245,996505,996918,991052,993251,998331,996062,996313,995589,990479,999091,990537,997395,999079,993931,995536,991758\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 567\n", + "Returned IDs: 990974,994541,993637,999009,996996,992551,999898,993807,997320,990096,997004,996216,993695,998633,996830,991179,992927,995895,990786,997410,996564,996944,998302,996809,997126,994782,995638,998950,993053,991154\n", + "Ground Truth: 990974,994541,994557,993637,999009,996996,992551,999898,994638,992553,993807,997320,990096,997004,996216,993695,995906,993928,998633,996830,991179,992927,998006,997577,995895,990786,997410,995755,996564,996944\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 568\n", + "Returned IDs: 993883,992859,990416,991815,992272,996898,994013,993908,990903,997566,999402,996109,998162,997185,990462,995268,993209,999279,998980,995544,991585,999172,997384,997710,998898,991507,999672,997031,998060,994991\n", + "Ground Truth: 993883,992859,990416,991815,992272,996898,994013,993908,990903,997566,999402,996109,998162,997185,998609,990462,995268,993209,999279,998980,995544,991585,999172,997384,993166,997710,998898,991507,995633,999672\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 569\n", + "Returned IDs: 994145,993561,993993,993939,996328,992862,996001,990887,994853,998710,992450,997878,996219,996132,997529,994742,990096,994682,996408,991759,999444,998382,993342,996042,995895,994830,990918,998221,996884,998495\n", + "Ground Truth: 994145,993561,993993,993939,996328,992862,996001,990887,994853,998710,995326,995535,992450,997878,996219,996132,997529,994742,999472,990133,990096,994682,996408,991759,999444,998382,993342,996042,995895,994830\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 570\n", + "Returned IDs: 992293,990662,990097,996001,996657,998549,999853,992521,996028,996147,995283,998878,997305,993848,996984,997554,995401,996368,999099,995873,993053,992862,998593,998017,992395,995158,990811,990211,993725,990573\n", + "Ground Truth: 992293,990662,990097,996001,996657,998549,999853,992473,992521,996028,996147,995283,998878,997305,993433,993848,996984,997554,995401,996368,999099,995873,993053,997466,992862,995002,994531,999587,998593,998017\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 571\n", + "Returned IDs: 993997,998962,990663,999279,997031,996786,997844,990758,996131,995633,996867,996302,993920,993767,995640,995268,990157,994120,993209,997566,995766,995289,996346,999415,996254,996109,994409,999672,994730,995240\n", + "Ground Truth: 993997,998962,990663,999279,997031,996786,997844,990758,996131,995633,996867,998609,996302,993920,993767,995640,995268,990157,994120,999996,993412,993209,993488,997566,995766,995289,996346,999415,996254,996109\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 572\n", + "Returned IDs: 996001,995823,998593,991789,997768,996028,990930,992862,997586,994307,995536,997790,995002,993053,996368,995289,990811,998495,996147,996062,993342,999444,999413,993906,998874,990096,990399,998039,990382,993168\n", + "Ground Truth: 996001,995823,998593,991789,997768,996028,990930,992862,997586,994307,995536,997790,997920,995002,993053,996368,995289,990811,998495,995097,996147,991747,996062,993342,999444,999413,997691,993906,998874,995096\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 573\n", + "Returned IDs: 998171,998866,995923,993390,993561,991996,997822,995547,992660,998045,996042,995052,995701,991958,994409,997289,990137,997058,996867,995289,994396,998165,991011,993433,994140,999868,992576,996125,991415,997191\n", + "Ground Truth: 998171,998866,995923,993390,993561,991996,997822,995547,992660,998045,996042,995052,995701,991958,998323,994409,997289,990137,997058,996867,995289,990706,994396,998165,991011,993433,994140,999868,992576,996125\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 574\n", + "Returned IDs: 991348,995206,996626,992318,993603,993053,995438,990083,998950,996024,991763,993621,994233,992944,991142,993166,993747,996178,994742,993154,995107,996001,993696,999426,995669,990883,996653,996210,991997,990508\n", + "Ground Truth: 991348,995206,995108,996626,992318,993603,993053,995438,990083,998950,996024,991763,993621,994233,992944,991142,993166,993747,996178,994742,993154,992617,998567,990031,995107,998180,996001,993696,997448,999426\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 575\n", + "Returned IDs: 999136,998570,998586,991844,999306,995695,995257,996001,995142,994643,998593,993948,990991,994185,991987,994187,995455,999117,995701,991038,992945,995755,996623,996147,993628,995200,993342,992760,997369,993289\n", + "Ground Truth: 990450,998383,999136,998570,998586,994105,991844,999306,996446,995695,995257,996001,995142,994643,998593,996043,993948,990991,994185,991987,999653,994187,997766,995455,997191,998493,999117,995701,999872,991038\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 576\n", + "Returned IDs: 998623,990998,998536,995408,992052,994743,990096,999287,990255,996809,995203,997618,995536,994682,992558,994742,994510,991629,991515,994955,993910,999724,996483,994954,991977,992636,993948,996930,995237,994906\n", + "Ground Truth: 998623,990998,998536,995408,992052,994743,990096,999287,990255,996809,995203,997618,995536,994682,992558,991630,999107,995864,994742,995401,994510,991629,991515,992933,994955,993910,999724,996483,995198,994954\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 577\n", + "Returned IDs: 997918,996903,995373,995453,994088,991887,995444,991844,995080,994334,992604,994092,991698,995807,994804,997675,998318,995685,991110,997515,997637,992933,999144,997110,999478,994819,996867,993754,994170,996658\n", + "Ground Truth: 997918,993223,996903,995373,993512,995453,994088,991887,995444,996282,991844,995080,994334,992604,994092,991698,995807,993412,994804,997675,998318,995685,991110,997515,997637,992933,999144,997110,999478,994819\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 578\n", + "Returned IDs: 997110,991987,995268,999478,994948,990991,997723,994643,993122,998078,991630,999994,990361,990930,994125,997191,996968,991052,999117,996867,999530,995963,994253,999555,992604,993232,992124,992861,995373,997406\n", + "Ground Truth: 997110,991987,995268,999478,994948,990991,997723,994643,993122,995848,998078,990805,991630,999994,990361,993381,990930,994125,997191,996968,991052,999117,996867,991954,999530,995963,994253,995701,992556,999555\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 579\n", + "Returned IDs: 990195,991585,995216,999302,998060,996147,996001,998549,990975,991958,995873,998200,996219,994925,994013,997554,990887,995701,996657,993813,993561,994076,995079,999343,995555,997377,992384,991125,995016,999402\n", + "Ground Truth: 990195,991585,995216,999302,998060,996147,996001,998549,993377,990975,991958,995873,998200,996219,994925,994013,997920,997554,990887,995701,996657,993813,993561,994076,995079,999343,995555,997377,992384,991125\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 580\n", + "Returned IDs: \n", + "Ground Truth: 997126,997415,990893,994467,994903,993501,993460,997084,991303,998588,997340,990998,993084,993931,992056,991383,999430,999323,997196,990173,990579,994510,998716,991672,996706,999139,995895,997565,996184,994098\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 581\n", + "Returned IDs: 995899,991154,998697,992167,998720,995536,993139,994167,995562,996075,999806,999724,997305,999639,996001,992225,994577,997586,992423,993560,991157,996894,993055,995203,998514,998568,994636,990051,991332,995516\n", + "Ground Truth: 995899,991154,998697,992167,998720,995536,993139,994167,995562,996075,999806,999724,996811,997305,999639,996001,992225,994577,997586,992423,993560,991157,995963,996894,999107,993055,998105,995203,998514,998568\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 582\n", + "Returned IDs: 994806,992598,999852,998256,999192,998903,990069,994175,996633,998716,994830,996960,997202,990806,996495,996537,996301,993335,999628,999126,990998,996328,999372,996153,998761,996011,994608,993502,990985,991769\n", + "Ground Truth: 994806,992598,999852,998256,999192,998903,990069,994175,996633,998716,994830,996960,999441,997202,990806,996495,996537,996301,993335,999628,999126,990998,996328,999372,996153,998761,996011,994608,993502,990985\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 583\n", + "Returned IDs: 992680,995756,996645,993725,991669,999740,999356,991075,992090,993444,999302,990743,995435,991464,996411,993645,990528,992024,990701,999280,996627,993205,996969,998940,993053,996384,994783,995126,992679,995906\n", + "Ground Truth: 992680,995756,996645,993725,991669,999740,999356,991075,997798,992090,993444,999302,990743,999922,995435,991464,996411,991624,993645,990528,992024,999261,990701,999280,992783,996627,993205,996969,998940,993053\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 584\n", + "Returned IDs: 998182,994799,990328,998105,993857,999460,991075,990980,995166,998427,998197,996689,995295,996028,997961,995823,990145,999843,992480,999230,993475,994119,998982,996494,990536,990097,990811,993053,993221,997822\n", + "Ground Truth: 998182,994799,990328,998105,993857,999460,991075,990980,998017,995166,998427,998197,996689,995295,996028,997961,995823,990145,999843,993055,992480,999230,993475,992374,994119,998982,996494,990536,990854,990097\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 585\n", + "Returned IDs: 995755,997301,993484,994742,990096,998246,995942,990958,991743,996001,997614,997371,995701,993853,994682,992862,993048,997320,998294,998921,990949,990515,996805,999801,993342,991498,996521,990146,995132,997913\n", + "Ground Truth: 995755,997301,999533,993484,994742,990096,998246,995942,990958,991743,996001,997614,997371,995701,993853,994682,993412,992862,993048,997320,998294,993948,998921,990949,990515,998002,996805,999801,997533,990545\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 586\n", + "Returned IDs: 992183,991479,990109,995802,998011,994778,993853,998550,994699,995396,990231,990647,994450,993662,999243,993403,997913,999710,995793,991502,998899,998828,998951,991249,995336,992147,990146,995824,990910,992442\n", + "Ground Truth: 992183,991479,990109,995802,998011,992946,994778,999561,993853,998550,994699,995396,990231,997439,991207,990647,994450,993662,999243,993403,995465,996912,997913,999647,996735,991147,990478,999710,995793,997194\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 587\n", + "Returned IDs: 993562,990535,994685,997015,995947,991008,998992,991068,993083,997306,993484,992681,990936,991444,990925,999644,994234,995129,999531,992274,997543,993050,991396,996343,996504,994997,990146,990585,996959,998797\n", + "Ground Truth: 991433,993562,990535,994685,997015,995947,991008,998992,991068,993083,997306,993484,992681,990936,991444,990925,999644,994234,995129,990539,999531,992274,997543,993050,991396,999012,996343,996504,994997,990146\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 588\n", + "Returned IDs: 990845,994925,996906,998593,992703,997453,998039,993561,991958,996062,996001,999729,991789,993906,991989,992334,996237,990899,991011,990096,990076,995289,996311,996277,994424,996834,993064,991869,999491,992576\n", + "Ground Truth: 990845,994925,996906,998593,992703,997453,998039,993561,991958,995162,996062,996001,999729,995401,991789,993906,991989,992334,996237,997028,990899,991011,990096,990076,995289,996311,996277,999340,998207,995560\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 589\n", + "Returned IDs: 990271,997878,990179,991776,995868,994448,992193,996219,994145,990369,994689,994373,990272,997349,996593,995096,991075,990077,999102,990315,995438,995690,994682,994742,993725,991283,999331,999144,992337,995220\n", + "Ground Truth: 990271,997878,990179,995222,991776,995868,990621,994448,992193,996219,998269,995719,994145,990369,996580,994689,994373,990272,997349,996593,995096,991075,998204,990077,999102,990315,995438,995690,994682,994742\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 590\n", + "Returned IDs: 998792,990690,992169,990226,996028,994964,997822,998039,993717,995526,993906,999898,992944,992408,993197,996494,993807,994925,995200,993561,990845,995861,997320,996328,993637,993869,998593,990210,994340,999613\n", + "Ground Truth: 998792,990690,995360,992169,990226,996028,994964,997822,998039,993717,995526,993906,997732,999898,992944,992408,993197,990580,996494,993807,994925,995200,993561,990845,995906,995861,997320,996328,991780,993637\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 591\n", + "Returned IDs: 993433,996042,995289,996607,992448,998910,995146,993561,995820,995018,997023,995200,996769,995521,998162,995864,996112,996815,991671,999611,992576,991699,996867,991958,994409,997822,995766,999054,997406,998564\n", + "Ground Truth: 993433,996042,995289,996607,992448,998910,995401,995146,993561,996809,995820,995018,997023,995200,996769,995521,998162,995864,996112,996815,991671,999611,992576,991699,996867,997723,991958,994409,997822,995766\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 592\n", + "Returned IDs: 994315,997059,990853,993991,995808,996763,995159,994354,990956,995184,992526,996189,997949,995873,992052,994175,992427,991111,994621,994377,996277,991981,997041,990611,992981,994488,995047,991769,994185,992285\n", + "Ground Truth: 994315,997059,990853,993991,995808,996763,995159,994354,990956,995184,992526,996189,991864,997949,995873,992052,994175,992427,991111,997233,994621,994377,996277,991981,997041,990611,992981,994488,995047,991950\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 593\n", + "Returned IDs: 992558,993255,994280,992790,995401,997395,995886,995134,990214,991404,999715,996762,993478,991977,997918,990538,992685,999305,995119,995154,995968,990969,992933,993143,999179,999154,991382,998950,999961,997822\n", + "Ground Truth: 992558,993255,994280,992790,995401,990790,997395,995374,995886,995134,990214,991404,999715,995145,996762,993478,991977,997918,990538,992685,999305,995119,996721,995154,995968,991052,998045,990969,992933,999461\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 594\n", + "Returned IDs: 998314,990096,999282,990583,993086,999079,996733,999194,996209,999384,993744,993965,997821,999287,991243,992407,996893,993595,990072,999831,990920,998207,998536,995736,999411,997442,998510,991093,996124,999320\n", + "Ground Truth: 998314,990096,999282,990583,993086,999079,996733,999194,996209,999384,993744,993965,997821,999287,991243,992407,996893,993595,990072,999831,990920,998207,998419,998536,995736,999411,992112,997442,998510,993609\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 595\n", + "Returned IDs: 991828,992186,990580,991840,990310,997539,994922,999076,996402,996996,993036,993577,994005,990138,995515,996449,996478,991609,991172,999846,996840,993143,998455,995156,998711,990615,990957,993598,992611,995525\n", + "Ground Truth: 991828,992186,990580,991840,990310,997539,994922,999076,996402,996996,993036,993577,994005,990138,995515,996449,996478,994700,991609,991172,999846,993278,996840,993143,992207,991519,998455,995156,995767,998711\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 596\n", + "Returned IDs: 995981,997620,999374,997304,997437,998246,995596,992357,994266,999331,997913,998697,992748,994100,995132,990588,997429,993186,997320,998974,994894,995638,990949,997614,996075,992384,992124,998353,991045,990866\n", + "Ground Truth: 995981,997620,999374,997304,997437,998246,995792,995596,992357,994266,999331,997913,998697,992748,992047,999710,994100,995132,990588,997429,993412,993186,994916,992583,997320,998974,994894,995638,990949,997614\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 597\n", + "Returned IDs: 995144,990116,996878,990325,997420,991145,999144,993727,995130,995357,993805,998045,997766,996811,995289,995562,996877,994051,996918,999365,993121,993433,997822,992558,993275,992718,990956,991131,992052,998758\n", + "Ground Truth: 995144,990116,996878,990325,997420,991145,999144,997723,993727,995130,993504,995357,993137,992315,993805,998045,997766,996811,995289,995562,996877,999527,995374,994051,996918,990527,999365,993121,993433,992933\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 598\n", + "Returned IDs: 992636,996986,995467,997340,999334,990438,990257,993715,993917,990951,999724,992282,991144,998671,996594,996183,993139,993055,997546,999373,997168,993342,993448,995326,990353,996408,995746,997577,998348,993224\n", + "Ground Truth: 992636,996986,995467,997340,999334,994680,990438,990257,993715,993719,993917,990951,999724,992282,991144,998671,996594,996183,993139,993055,997546,999373,997168,993342,993448,995326,990353,996408,995746,997577\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 599\n", + "Returned IDs: 996298,991464,999922,992240,992408,993725,990145,994472,996627,990531,990091,995295,993444,993906,997453,994765,990226,991021,993053,995126,998055,999126,999139,997944,992322,994663,992703,992169,992910,995680\n", + "Ground Truth: 996298,991464,999922,992240,991663,992408,993725,990145,994472,996627,990531,990091,995295,993444,993906,997453,994765,990226,991021,993053,995126,998055,999126,999139,997944,992322,994663,992703,991168,990712\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 600\n", + "Returned IDs: 996610,997787,991603,998906,996047,996388,990033,993390,995289,991463,997406,990681,998045,990940,994161,998165,993561,996112,992125,991367,991954,996769,998999,995182,992160,997046,995497,997832,995318,995052\n", + "Ground Truth: 996610,997787,991603,998906,996047,996388,990033,993390,995289,991463,997406,990681,998045,990940,994161,998165,995040,993561,996112,992125,991367,991954,996769,998999,995182,992160,997046,995497,997832,995318\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 601\n", + "Returned IDs: 995819,994119,995401,991943,994148,992832,994817,990814,991151,993857,996760,996328,999099,991747,999711,992595,992582,992289,996064,995295,990127,998067,990531,991569,994280,998536,997319,999107,995476,996689\n", + "Ground Truth: 995819,994119,995401,991079,991943,994148,992832,994817,992318,990814,991151,993857,996760,996328,999099,990769,991747,999711,992595,992582,992289,996064,997452,995295,990127,990531,998067,991569,995536,994280\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 603\n", + "Returned IDs: 997878,991464,993561,996834,998705,994838,998382,990315,993725,993166,993053,991958,994730,995679,994145,995435,995283,999151,999173,995384,994338,990621,990441,999740,990614,999302,992261,997529,996411,999126\n", + "Ground Truth: 991024,997878,991464,993561,996834,998705,994838,998382,996752,990315,993725,999472,996565,993166,993053,991958,994730,995679,994145,995435,995283,999151,999173,996943,995384,999352,994338,990621,990441,999740\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 604\n", + "Returned IDs: 994297,995244,994964,993452,994919,998500,999821,998837,999444,992169,995135,998909,991991,999545,999740,994922,991999,990736,990432,997984,996351,991840,998553,995336,998523,997192,999676,997320,994318,998921\n", + "Ground Truth: 994297,995244,994964,993452,991438,994919,998500,999261,999821,998837,999444,990064,992169,995135,996854,995591,998909,991991,999545,999740,994922,991999,990736,990432,997984,996351,991840,998553,995336,991751\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 605\n", + "Returned IDs: 999375,997152,991960,998196,994193,991758,994119,999788,998274,992056,991515,998543,994742,994541,997452,990814,991334,999981,993180,997967,992077,998913,990796,996302,999543,990722,998855,994893,993744,995200\n", + "Ground Truth: 999375,997152,992468,991960,998196,994193,991758,994119,999788,998274,992056,991447,998466,995438,991515,998543,994742,994541,995494,997452,990814,991334,999981,990127,993180,997967,992077,998913,990796,996302\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 606\n", + "Returned IDs: 992195,993708,992907,994922,997135,995588,995693,995365,999784,991157,993696,999676,992039,991015,992612,998837,997379,995203,990310,998123,993143,996840,993355,991485,999077,994964,996727,998950,991848,997305\n", + "Ground Truth: 992195,993708,992907,994922,997135,995588,995693,995365,999784,991157,993696,999676,992039,991015,992612,998837,997379,995203,990310,998123,993143,996840,993355,991485,999077,997037,994964,993633,996727,998950\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 607\n", + "Returned IDs: 990146,999600,990958,990863,993083,993562,994964,994685,992301,994217,994425,996959,996532,990925,992968,995947,995884,993238,997320,999262,998302,994673,995716,999374,994922,991743,993168,995372,991008,998170\n", + "Ground Truth: 990146,999600,990958,991086,990863,993083,993562,994964,994685,992301,994217,995909,994425,996959,996532,999444,990652,990925,992968,997876,995947,995884,993238,997320,999262,998302,993412,994673,995716,999374\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 608\n", + "Returned IDs: 991569,997009,992561,996328,990437,993931,990040,997925,997611,990293,993001,991667,995159,997513,998720,990330,997973,990956,994509,998593,997204,995655,990364,995701,992832,990951,996408,991679,998067,998156\n", + "Ground Truth: 991569,997009,992561,996328,990437,993931,990040,997925,997611,990293,993001,993778,996706,991667,995159,997513,998720,990330,994796,997973,990956,994509,998593,997204,995655,990364,995701,992832,992850,990133\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 609\n", + "Returned IDs: 993053,998153,993721,994751,992680,993444,999740,998435,995861,998495,996685,992679,994558,991382,991128,995735,998055,998454,993984,999356,999864,993725,999280,998761,997933,996645,992079,998221,991075,994742\n", + "Ground Truth: 993053,998153,993721,994751,992680,993444,999740,998435,995861,998495,996685,992679,994558,991382,991128,995735,999295,998055,998454,993984,999356,999864,993725,999280,998761,997933,996645,992079,998221,993123\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 610\n", + "Returned IDs: 991638,997732,994956,993929,993765,992655,998107,992120,995861,992813,999827,992169,998458,992408,990226,997852,991334,994131,993877,990889,990961,997453,996328,994557,998495,997822,993080,997969,990382,998171\n", + "Ground Truth: 991638,997732,994956,993929,997112,993765,992655,998107,992120,995861,992813,999985,999827,992169,998458,992408,990226,997852,990441,991334,994131,993877,990889,990961,997453,990266,996328,994557,998495,997822\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 611\n", + "Returned IDs: 999056,999426,996744,996500,992167,996075,991984,992041,992927,993766,991823,993283,997661,995596,996761,995132,993237,999041,996169,996001,995516,994809,993834,995016,991392,998232,995855,996657,996576,992351\n", + "Ground Truth: 999056,999426,996744,996500,992167,996075,991984,997777,992041,992927,993766,995291,991823,993283,993156,997661,995596,996761,995132,993237,995375,999041,999132,996169,996001,995516,995898,994809,993834,998990\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 612\n", + "Returned IDs: 994830,994742,998950,993588,996809,992807,998913,998823,994806,995552,994269,992318,996805,991154,999627,990392,993342,998495,999430,992598,992551,992427,993053,995899,995002,994682,992415,997073,995289,999377\n", + "Ground Truth: 994830,994742,998950,993588,996809,992807,998913,998823,994806,995552,994269,992318,996805,991154,996301,999627,990392,991390,993342,998495,995299,999430,992598,992551,994751,992427,993053,995899,995401,995002\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 613\n", + "Returned IDs: 994051,990256,991860,995516,992915,991799,995755,990862,992596,992165,997437,993396,999725,997305,996811,999251,991052,994879,995963,992546,991690,995184,996406,990958,996942,991982,996918,990096,994266,997154\n", + "Ground Truth: 994051,990256,991860,995516,997645,992915,997811,991799,995755,990862,992596,992165,997437,995132,997429,993396,999725,993336,992238,997305,996856,996811,999251,994335,991052,994879,992357,995963,990178,992546\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 614\n", + "Returned IDs: 995769,995850,999606,998837,997275,993237,995004,990043,996686,994612,991407,990514,997486,994785,992612,995557,998657,991307,994967,995515,990580,991510,999714,993091,993570,998244,990965,995788,996402,992907\n", + "Ground Truth: 995769,998035,995850,999606,998837,997275,993237,995004,990043,996686,994612,991407,999888,998101,990514,997486,994785,992612,995557,998657,991307,994967,996561,995515,990580,991510,999714,993091,993570,998244\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 615\n", + "Returned IDs: 996116,990701,997374,997207,998913,997209,992024,994120,990126,995704,994168,995482,997628,992599,993240,993384,990614,994730,999402,992156,993857,994728,995633,998982,991943,996481,990012,998495,998156,998722\n", + "Ground Truth: 996116,990701,997374,994350,997207,998913,997209,992024,994120,990126,995704,999586,991156,994168,995482,997628,992599,992251,992871,993240,993384,990614,994730,999402,992156,993857,994728,995633,998982,991943\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 616\n", + "Returned IDs: 998566,997193,995691,997126,990305,993566,996194,990184,991636,996421,993127,998466,995482,990189,991157,990776,993678,991938,995883,992944,990948,991783,993920,999158,993181,996222,997674,998104,991052,994835\n", + "Ground Truth: 998566,997193,995691,997126,990305,994064,993566,996194,990184,996451,991636,996421,993127,998466,995482,990189,991157,996170,990776,993678,999025,999586,992704,991938,992841,995883,990827,990052,995620,992944\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 617\n", + "Returned IDs: 991128,999282,990096,992557,999045,999079,999831,994661,999411,993721,997442,998510,997766,993744,996209,998302,997821,993139,992209,997906,999806,996494,999384,996893,990584,995206,991650,993215,994416,998359\n", + "Ground Truth: 991128,999282,990096,992557,997418,999045,999079,999831,994661,999411,993721,997442,998510,997524,997766,993744,996209,998302,997821,993139,992209,997906,998419,999806,996494,999384,996893,990584,995206,991650\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 618\n", + "Returned IDs: \n", + "Ground Truth: 998888,999555,991944,990991,999584,991100,992638,991188,997154,997850,996393,995655,990167,998006,993948,993749,990423,990558,990154,994727,994467,993381,997030,993956,990361,994374,991836,994643,991385,991596\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 619\n", + "Returned IDs: 997995,997455,995095,996609,997993,998221,990432,993219,995401,992619,991836,990435,996088,999820,990126,994587,995354,999923,992979,997185,995289,993412,999160,993656,995633,995374,999240,998297,994993,999021\n", + "Ground Truth: 997995,997455,995095,996609,992281,997993,998221,992087,990432,993219,991445,995401,992619,991836,990435,996088,993209,999820,990126,994587,995354,999923,992979,997185,995289,993412,999160,990157,993656,995633\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 620\n", + "Returned IDs: 997652,997897,998428,992052,993621,998522,993171,990956,999002,999179,998250,995421,992028,992747,990409,990626,994280,994765,995213,994742,991605,996328,996461,999927,990364,993175,991212,996001,992437,993347\n", + "Ground Truth: 997652,997897,998428,992052,993621,998522,993171,990956,999002,999179,998250,995421,992028,992747,990409,997368,990626,994280,994765,995213,994743,997723,994742,997494,991605,996328,996461,999587,999927,990364\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 621\n", + "Returned IDs: 995941,997371,990645,993507,994757,993818,998246,991782,995367,995755,996890,997192,994281,990994,997194,992053,990863,999374,991379,993518,997984,995353,999290,992978,991498,993852,995090,990094,990917,990647\n", + "Ground Truth: 995941,997371,990645,993507,994757,993818,998246,991782,995367,995755,996890,997192,993005,994281,990994,997194,994183,992053,990863,999374,991379,993326,991059,993518,997984,995353,999247,997022,990969,993739\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 622\n", + "Returned IDs: 993539,993564,992167,992225,998758,999216,995701,994167,999144,999219,991459,993834,997258,995562,992429,993766,999783,992384,996368,991256,999041,993342,994906,996744,995132,999899,998232,993139,998720,992069\n", + "Ground Truth: 993539,993564,992167,992225,998758,999216,995701,994167,999144,999219,991459,993834,997258,995562,992429,993766,999783,992384,996368,991256,990041,999041,993342,996283,993517,994906,996744,995132,999899,999677\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 623\n", + "Returned IDs: 992558,999564,992738,995724,998827,994372,997124,990273,993077,995983,991009,997158,999146,996918,990538,992170,992179,994163,991260,991905,991157,997775,999639,995401,993706,992615,990479,998142,997859,999908\n", + "Ground Truth: 992558,999564,992738,995724,998827,994372,993140,997124,990273,993077,995983,991009,995281,997158,991585,999146,996918,990538,992170,992179,997411,994163,991260,991905,991157,997775,995411,999639,994742,995401\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 624\n", + "Returned IDs: 992832,998543,992595,997319,992713,990889,995633,991850,996328,998750,994891,992239,997698,999788,992077,991981,994742,999833,990516,992561,990270,992318,999827,998525,995030,990814,991769,999767,991590,999924\n", + "Ground Truth: 992832,998543,992595,997319,992713,990889,995633,991850,996328,998750,994891,992239,997698,991731,999788,992077,991981,994742,999833,990516,992561,990270,995222,992318,999827,998525,995030,990814,991769,999767\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 625\n", + "Returned IDs: 997977,992823,993504,995168,994468,991425,992082,997618,994672,996217,998303,995536,995231,993260,990196,997002,993071,997790,997918,995740,999111,996001,994955,995203,991863,995167,996884,990350,992862,993489\n", + "Ground Truth: 997977,992823,993504,995168,994468,991425,992082,997618,994672,996217,998303,997140,998007,995536,995231,993260,990196,997002,993071,997790,997918,995740,999111,996001,994955,995203,991863,995167,996884,990350\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 626\n", + "Returned IDs: 994906,993289,993342,999587,992664,993820,991459,995401,994946,992636,998855,998536,993275,990399,998623,999977,996764,997002,996001,999820,999724,992179,996147,997910,994743,997790,993715,998831,999453,994955\n", + "Ground Truth: 994906,993289,993342,999587,992664,993820,991459,995401,994946,992636,998855,998536,993275,990399,998623,999977,997332,996764,997002,996001,999820,999724,996371,992179,996147,997910,994743,997790,993715,998831\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 627\n", + "Returned IDs: 992636,990998,996183,996408,998379,993335,994682,996594,991919,992026,999334,996328,993715,990096,990392,994830,993342,990173,997340,991007,993448,992764,994742,995695,995299,998716,990027,990257,993001,992479\n", + "Ground Truth: 992636,990998,996183,996408,998379,993335,994682,996594,991919,992026,991620,999334,996328,993715,990096,990392,994830,992933,995326,993342,990173,997340,991007,993448,992764,994742,995695,995299,998716,990027\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 628\n", + "Returned IDs: 999457,990194,998279,999555,999994,999205,993461,990167,999223,998674,999430,992628,993076,994109,993224,998586,998570,991610,995234,995848,991025,998086,998078,995907,997506,994731,990402,997205,991987,997084\n", + "Ground Truth: 999457,990194,998279,999555,999994,999205,993461,990167,999223,998674,999430,992628,998072,993076,994109,993224,998586,998570,991038,997415,991610,995234,995848,991025,999985,998086,998078,995907,995082,997506\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 629\n", + "Returned IDs: 995135,990898,998751,999649,998660,991172,991840,996854,998011,995365,999676,998915,997291,993640,990965,993253,995693,993404,991999,994415,998540,997192,994497,996976,993143,993809,997403,991833,992087,995090\n", + "Ground Truth: 993250,995135,990898,998751,999649,998660,995865,991582,991172,991840,996854,994819,998011,998996,996352,997641,995365,999676,998915,997291,993640,997479,990965,990310,993253,995693,993404,990979,991999,994415\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 630\n", + "Returned IDs: 994852,992190,999451,997106,990884,999797,994622,996894,995138,995752,992349,993789,997818,993355,999645,997076,990096,996884,993706,998974,992668,996196,999807,999639,996262,990522,995210,993222,990957,995436\n", + "Ground Truth: 994852,992190,999451,997106,990884,999797,994622,996894,995138,998455,995752,992349,993789,997818,993355,999645,997076,990096,996884,995769,993706,998974,992668,996196,999807,991637,999639,992612,996262,990351\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 631\n", + "Returned IDs: 999934,999225,994529,997837,999372,990593,993856,990226,990126,995083,990340,998338,995300,992859,995633,990435,994742,992450,996042,995861,993100,992434,993731,996328,999830,991746,993433,999672,992449,993561\n", + "Ground Truth: 999934,999225,994529,997837,999372,990593,993856,990226,990126,995083,996372,990340,998338,995300,992859,995633,990435,994742,992450,996042,996306,995861,993100,992434,997185,993731,996328,999830,991746,993433\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 632\n", + "Returned IDs: 995210,996808,997918,996062,998436,993512,990366,993289,990891,997002,991425,997332,994821,990885,994042,993260,994959,996001,993504,990622,995826,994694,991833,991311,993157,998303,990491,996551,992571,994200\n", + "Ground Truth: 995210,996808,997918,996062,998436,993512,990366,993289,990891,997002,991425,997332,994821,992711,990885,994468,994042,993260,992612,994959,997531,996001,993504,990622,995826,998611,994694,991833,993268,999091\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 633\n", + "Returned IDs: 999854,998228,993637,998436,997790,999827,996062,990557,995678,994472,997918,990091,997002,998039,999516,996001,993906,995502,997586,992576,991994,992161,995107,993484,997453,992395,990080,994964,998416,993875\n", + "Ground Truth: 999854,990584,998228,993637,998436,997790,999827,996062,990557,995678,994472,997918,990091,997002,998039,993238,999516,996001,993906,993765,995502,997586,992576,997920,991994,992161,995107,993484,997453,992395\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 634\n", + "Returned IDs: 990545,995260,992119,995942,997320,997913,994964,994100,998550,996143,996351,999647,992546,993877,995886,998495,998500,990958,991743,990648,998302,996219,997371,996554,998458,997984,994164,990936,999331,999374\n", + "Ground Truth: 990545,995260,992119,995942,997320,997913,994964,993154,994100,998550,996143,996351,999647,992546,993877,995886,998495,991720,998500,990958,991743,990648,998302,996219,997371,993238,994898,996554,998458,997984\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 635\n", + "Returned IDs: 995702,991973,992521,992703,997822,993293,998162,994409,990930,995766,994140,994742,998779,996368,993738,993561,991185,997723,995648,993831,998633,998165,997320,990845,995289,992448,995216,995052,995679,994711\n", + "Ground Truth: 995702,991973,992521,992703,997822,993293,998162,994409,990930,995766,994140,994742,999134,998779,996368,993738,993561,991185,997723,995648,993831,991603,998633,998165,997320,996042,990845,998999,995289,992448\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 636\n", + "Returned IDs: 995557,996596,990310,993748,998899,993237,995365,991593,993279,996748,995588,993091,998917,995794,993675,992195,994856,998997,998057,999039,994263,992085,995072,994964,997275,992059,996007,999784,993083,991157\n", + "Ground Truth: 995557,999604,996596,990310,993748,999592,998899,993237,995365,991593,993279,996748,995588,993091,992207,998917,995952,995794,993675,992195,994856,998997,998057,999039,998654,994263,992085,995072,994005,991519\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 637\n", + "Returned IDs: 998357,995965,993500,991977,993975,993583,994614,992376,993077,994760,996663,999171,998406,998142,999763,995408,992535,991740,996158,991432,994165,990249,993786,990908,993609,993839,992494,998879,995459,993810\n", + "Ground Truth: 998357,995965,993500,991977,993975,993583,994614,992376,993077,994760,996663,999171,998406,998142,999763,995408,992535,991740,996158,991432,994165,990249,993786,999008,990908,993609,993839,992494,998879,995459\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 638\n", + "Returned IDs: 999638,990926,992349,992718,997816,997859,994163,991540,996894,995138,996318,994010,991498,991977,998697,998417,997113,998118,998334,999387,994964,992124,994165,990908,990343,995536,994972,992558,990083,990772\n", + "Ground Truth: 999638,990926,992349,992718,997816,997859,994163,991540,993948,996894,995138,996318,994010,991498,991977,998697,998417,997113,998118,998334,999387,994964,992124,994165,990908,990343,995536,994972,991637,999908\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 639\n", + "Returned IDs: 994137,999358,995378,990560,990390,999363,991900,998462,995942,998966,992310,991396,993403,990146,991444,997972,993484,990742,994964,994479,999110,999390,997539,999710,990311,996629,998365,995949,996959,994978\n", + "Ground Truth: 994137,999358,995378,990560,990390,999363,991900,998462,995942,998966,992310,991396,993403,990146,993072,991444,997972,993484,991178,999229,990742,994964,997880,994479,999110,993284,999390,997539,999710,990311\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 640\n", + "Returned IDs: 996127,993839,999860,999639,993561,991923,990550,994117,995013,991958,992473,996196,998593,991332,993489,990167,996391,994372,991350,996379,995562,998186,995983,998697,995894,996001,998142,998303,990096,995701\n", + "Ground Truth: 996127,993839,998220,999860,999639,993561,991923,990550,990303,994117,995013,991958,992473,996181,996196,998593,991332,993489,990167,996391,994372,999393,991350,991212,996379,995562,994796,998186,995983,998514\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 641\n", + "Returned IDs: 991810,996809,990127,993817,999788,994682,994742,992827,998063,992372,997349,997319,993744,995633,999852,990223,995895,990173,998536,998495,990986,990722,996408,997401,990392,997529,991620,998525,992832,997630\n", + "Ground Truth: 991810,996809,990127,993817,999788,994682,994742,992827,998063,999183,992372,997349,997319,993744,995633,999852,990223,995222,992318,995895,990173,995401,998536,998495,990986,990722,996408,997401,990392,997529\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 642\n", + "Returned IDs: \n", + "Ground Truth: 999338,998921,997292,997301,994942,995659,996516,997873,992451,997984,998721,998445,990512,997668,997320,995306,999271,990560,990405,995045,995961,994964,997168,997286,999904,995623,992176,999526,995874,999533\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 643\n", + "Returned IDs: 997749,999411,994909,995421,994765,993727,992934,990521,994280,991491,996276,994541,998934,995963,997897,995110,994697,994809,993448,999601,992457,996657,995203,990205,992052,995456,991850,998522,992933,998779\n", + "Ground Truth: 997749,999411,994909,995421,994765,993727,992934,998123,990521,994280,999453,991491,996276,990844,994541,993678,998934,995963,997897,990998,995110,994697,998510,994809,992112,990021,993448,999601,992457,999179\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 644\n", + "Returned IDs: \n", + "Ground Truth: 996296,991456,990176,997022,994842,996075,990742,997699,992931,992880,997178,990957,997918,991131,990949,995417,998540,993253,996592,993696,998967,992442,991634,995336,993529,992942,994100,999710,992277,995981\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 645\n", + "Returned IDs: 997867,991740,993839,992376,998357,991977,993583,998879,995965,990908,993647,997407,994614,999008,993975,990894,991225,995491,997547,996663,992333,995459,993681,996585,999022,994254,993500,994398,999763,999387\n", + "Ground Truth: 997867,991740,993839,992376,998357,991977,993583,998879,995965,990908,993647,997407,994614,991064,999008,993975,990894,991225,995491,997547,996663,992333,995459,993681,996585,999022,994254,993500,994398,999763\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 646\n", + "Returned IDs: 997643,991309,994660,992638,991797,998277,994393,992943,997683,999530,993566,993749,994294,997568,993666,999156,997110,994656,997030,995044,997429,996393,990554,997114,998888,995253,990132,991937,998858,992369\n", + "Ground Truth: 997643,991309,994660,992638,991797,998277,994393,992943,997683,999530,993566,993749,994294,997568,993666,999156,997110,994656,997030,992926,995044,997429,996393,990554,995994,997114,998888,995253,990132,991937\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 647\n", + "Returned IDs: 990679,998734,995250,999473,993154,997370,994789,995203,999898,996075,996623,997918,990352,993484,990652,994829,990470,993646,996452,999876,999384,990957,992548,996001,990123,999653,999017,990448,997688,994268\n", + "Ground Truth: 990679,994552,998734,995855,995250,999473,993154,997370,994789,995203,999954,994212,999898,996075,996623,997918,993243,990352,993484,990652,994829,990470,993646,990809,996452,999876,999807,999384,999132,997920\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 648\n", + "Returned IDs: 994125,990641,996609,990655,992604,997918,995401,991844,995289,998913,991996,996809,995718,996411,996196,996075,993725,995274,990126,998999,994603,993561,998495,995881,995633,996918,995886,990096,995200,995455\n", + "Ground Truth: 994125,990641,996609,990655,992604,997918,995401,998383,997403,991844,998913,995289,991996,996809,993529,993412,994317,995718,993433,996411,996196,991179,996075,993725,995274,990126,997906,998999,994603,993561\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 649\n", + "Returned IDs: 996360,998045,993561,998545,991052,991958,993433,996042,995702,996322,994409,997822,991923,996867,994419,996769,995679,995648,994549,994280,997868,997386,998564,995536,997674,997675,997289,990137,999530,997838\n", + "Ground Truth: 996360,998045,993561,998545,991052,991958,999809,993433,996042,995702,996322,994409,990882,997822,991923,993412,996867,994419,996769,995679,995648,994549,994280,997868,997386,995873,998564,995536,997674,997675\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 650\n", + "Returned IDs: 993839,996127,996658,991432,998372,995340,990061,990487,991028,997864,991923,993976,998370,991743,991350,996391,990638,995691,994514,995515,993583,998779,998879,997914,995453,992929,990602,998853,996742,996379\n", + "Ground Truth: 993839,996127,996658,991432,998372,993718,995340,990061,990487,991028,998079,997864,991923,993976,998370,991743,991350,996391,992813,990638,995691,995515,994514,993583,998779,998879,997914,998686,995453,992929\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 651\n", + "Returned IDs: 995562,992315,990596,998303,991037,999144,997420,991332,993489,996562,992819,990834,997926,996908,996286,992199,993275,999435,998331,995894,998495,996233,993609,995349,998855,990630,997506,991425,995536,993560\n", + "Ground Truth: 995562,992315,990596,998303,991037,999144,997420,991332,993489,997531,996562,992819,991212,990834,997926,996908,996286,995740,992199,993275,999435,998331,995894,998495,996233,994946,993609,995835,995349,998855\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 652\n", + "Returned IDs: 996646,997320,999647,997913,994450,993253,999109,997470,999948,999324,998550,990650,992950,992969,993387,998302,997826,999701,994266,991593,993031,990066,992972,992147,994604,993579,999036,991479,993662,999535\n", + "Ground Truth: 999328,996646,997320,999647,997913,994450,993253,999109,997470,999948,991743,999324,998967,998550,998287,990182,994192,990650,997921,996296,992950,992969,999390,999258,993387,998302,997826,999701,994266,998644\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 653\n", + "Returned IDs: 996690,992608,994384,996196,997775,992215,997902,991772,996140,993670,994163,992661,995983,993706,991550,998950,996726,995019,991009,996262,990538,998697,990824,999639,998364,991822,995552,995972,997368,993500\n", + "Ground Truth: 996690,992608,994384,996196,997775,992215,997902,991772,996140,993670,994163,992661,995983,993706,991550,999908,998950,996726,995019,991009,996262,990538,998697,990824,999639,998364,991822,995552,995972,997368\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 654\n", + "Returned IDs: 997192,993852,997571,998844,994281,999374,996554,994850,999953,995833,990275,992147,992183,994218,990432,992309,998011,998263,995354,999144,993529,995336,999317,993443,999883,999804,992854,996075,992452,996721\n", + "Ground Truth: 995374,993636,997192,993852,997571,998844,994281,999374,996554,994850,999953,995833,990275,992147,992183,994218,990432,992309,998011,998263,995354,999144,993529,995336,999317,993443,999883,999562,999804,992854\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 655\n", + "Returned IDs: 996247,991459,999804,990294,999639,995894,997918,991979,994125,996407,991630,996500,994911,998022,992473,996918,992516,992429,995932,990897,992558,993188,999306,993906,992293,995701,995336,998758,998697,993848\n", + "Ground Truth: 996247,991459,999804,990294,999639,995894,997918,991979,994125,996407,995132,991630,996500,997957,994911,998022,992748,992473,993948,996918,992516,992429,995932,990897,992927,992558,993188,999306,993906,991657\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 656\n", + "Returned IDs: 999544,999676,992348,999453,994800,991787,999452,995968,992612,994195,993683,991673,996309,992907,992309,992739,990080,993781,991485,995372,999686,998607,999254,994964,991999,991510,999210,990334,990233,991427\n", + "Ground Truth: 999544,999676,992348,999453,994800,991787,999452,995968,992612,994195,993683,991673,996309,992907,992309,992739,990080,993781,991485,995372,996352,999686,998607,992298,999254,994964,993633,991999,995365,999951\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 657\n", + "Returned IDs: 995861,994751,990091,998055,990133,993939,992780,999673,997103,999740,991334,993730,995191,999378,993877,992240,990145,997026,998610,995895,990712,990531,992652,996328,994472,991390,995735,999458,990705,995326\n", + "Ground Truth: 995861,994751,990091,998055,990133,993939,992780,999673,997103,999740,991334,993730,995191,999441,996752,999378,993877,992240,997077,997933,990145,997026,999436,998610,995895,996566,990712,990531,993118,992652\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 658\n", + "Returned IDs: 994488,994416,992318,999060,993772,996809,997332,990843,998950,991650,994541,997679,996001,993603,994955,993347,993721,991142,990930,993079,995899,995678,991409,995167,999320,996024,993293,999643,990096,992944\n", + "Ground Truth: 994488,994416,992318,999060,993772,996809,999179,997332,990843,998950,991650,994541,997679,996001,993603,998543,994955,998218,993347,993721,991142,990930,993079,995899,995678,991409,995167,999320,996503,996024\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 660\n", + "Returned IDs: 990516,995948,994893,995633,995030,998557,997319,996747,996867,994009,992289,999235,996672,994283,994742,995190,995289,991810,996302,999957,997401,995558,999875,997406,999160,992239,998196,998750,994204,995302\n", + "Ground Truth: 990516,995948,994893,995633,995030,998557,997319,996747,996867,994009,992289,999235,996672,994283,994742,995190,995289,991810,996302,993856,999957,997401,995558,999875,997406,999805,995348,999160,992239,998196\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 661\n", + "Returned IDs: 993461,990479,996918,991052,996535,990897,998267,993014,999646,997723,997586,990041,991758,998564,998518,999980,995963,997910,994282,992979,997675,997216,991891,992068,991666,992558,990942,998067,990256,997420\n", + "Ground Truth: 993461,990479,996918,991052,996535,990897,998267,993014,999646,997723,997586,990041,991758,998564,992933,998518,999980,995963,997910,998924,994282,992979,997675,997216,991891,998105,993412,992068,991666,992558\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 663\n", + "Returned IDs: 990225,999555,990167,993076,997506,999373,994834,990361,993366,996393,994374,995257,996408,993001,998495,996797,993224,995218,998536,999430,998888,996209,991100,999994,997154,991610,992470,997182,990173,990589\n", + "Ground Truth: 990225,999555,990167,993076,997506,999373,994834,995082,990361,993381,991881,993366,996393,994374,995257,996408,993001,998495,996797,999841,997142,993224,995218,998536,995222,999430,998888,996209,991100,995326\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 664\n", + "Returned IDs: 994978,992228,993579,991137,995841,999109,993845,991615,997984,998477,990585,990512,991479,995372,997024,998445,995668,999036,999024,995641,995623,993387,997018,996653,999358,999982,996959,997320,991450,993066\n", + "Ground Truth: 994978,992228,993579,992789,991137,995841,999109,993845,991615,997984,998477,990585,990512,991479,995372,997024,991090,998445,995668,999036,999024,995641,991468,991209,995623,993387,997018,996653,999358,993853\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 665\n", + "Returned IDs: 997777,990949,991593,997320,999290,993484,997370,992839,993253,999535,990957,997539,995515,997372,995203,999453,996002,994486,999982,995400,992069,995596,994964,998950,994366,999586,997913,994789,995826,991131\n", + "Ground Truth: 997777,990949,991593,997320,999290,993484,997370,992839,993253,999855,999535,990957,997539,995515,997372,995203,999453,990742,990890,996002,994486,995942,999982,995400,999680,994826,992069,993389,999437,995596\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 666\n", + "Returned IDs: 994954,992521,993438,994409,999866,992448,995640,996368,996769,997868,993561,996987,991969,995820,998255,996187,997091,990005,998205,999250,997466,991349,998045,992426,991967,997554,992937,997612,995536,998075\n", + "Ground Truth: 994954,992521,993438,994409,999866,992448,995640,995873,996368,996769,997868,993561,996987,991969,995820,998255,996187,997091,990005,995701,998205,999250,997466,991349,998045,992426,991967,997723,997554,993517\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 667\n", + "Returned IDs: 997652,993324,993347,995332,991255,990646,991961,993180,993079,996591,997752,990776,997309,996370,999305,996777,998272,996455,994765,996210,997914,992052,994119,994092,995766,993621,998416,997698,996633,994435\n", + "Ground Truth: 997652,993324,993347,995332,991255,990646,999190,991961,993180,996810,993079,996591,997752,990776,997309,994300,996370,999305,996777,998272,996455,994765,996210,997914,991990,992052,994119,994092,995766,993621\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 668\n", + "Returned IDs: \n", + "Ground Truth: 999117,993122,994467,995625,995044,991188,994643,997149,997154,992809,995207,999555,990991,999155,996395,992945,993795,999663,995257,993469,993995,993232,998861,996209,996393,992638,998536,995401,998006,999431\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 669\n", + "Returned IDs: 996489,990442,996981,990249,990256,990209,992558,994334,992017,995332,997380,991426,996811,995552,990538,997155,995449,992179,999860,993473,995401,992170,996196,993515,990620,999715,999908,994661,996429,991337\n", + "Ground Truth: 999901,996489,990442,996981,996186,990249,990256,990209,992558,994334,992017,995332,997380,991426,996811,995552,999480,990538,997155,995449,992179,999860,993473,995401,992170,998213,999859,996196,993515,990620\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 670\n", + "Returned IDs: 998289,998104,993324,992558,994732,990264,998950,996140,995289,994697,993079,997752,994514,997630,998697,996194,995963,995783,996210,996370,994817,993018,997723,992052,997713,991981,993280,993948,993478,998160\n", + "Ground Truth: 998289,998104,993324,992558,994732,990264,998950,996140,995289,996810,990769,995928,994697,993079,997752,994514,997630,998697,996852,996194,992988,995963,995783,995281,996210,996370,994146,995359,994817,993018\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 671\n", + "Returned IDs: 991925,991743,998334,992702,993721,994119,990585,993229,991977,999601,995375,996318,998495,997481,999453,998849,992944,999711,995130,996219,996001,993342,990489,997309,993241,996328,991994,994199,992169,998045\n", + "Ground Truth: 991925,996811,991743,998334,992702,993721,994119,990585,996918,993229,991977,999601,995375,996318,994662,998495,997481,999453,998849,994081,992944,997732,995873,999711,999707,995130,990640,996219,997436,996001\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 672\n", + "Returned IDs: 991618,991163,991969,991349,999731,993813,992521,995820,995813,990603,994140,991117,993738,991795,997868,996277,990930,993561,995701,997928,990193,991922,998205,996169,995864,995640,992448,995395,992851,994409\n", + "Ground Truth: 991618,991163,991969,991349,999731,993813,992521,995820,995813,990603,994140,991117,993738,991795,997868,996277,990930,993561,995701,997928,990193,991922,994353,998205,996169,995864,995640,992448,995395,999866\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 673\n", + "Returned IDs: 999703,992640,993314,993895,991161,993580,992728,990479,998463,992598,994672,993433,994280,998045,990942,994844,993609,995815,996867,999373,994920,995196,996328,996861,993313,990069,997506,998331,995835,994838\n", + "Ground Truth: 999703,992640,993314,993895,991161,993580,992728,990479,999520,998463,992598,994672,993433,994280,991731,998045,990942,990882,994844,993315,998812,993609,995815,999441,992617,996867,999052,999373,997192,994920\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 674\n", + "Returned IDs: 996913,994136,996761,996067,997839,997790,990550,996089,998473,993834,998934,993766,990897,998185,993478,994910,992750,999216,999041,996547,993809,999384,994915,991795,993832,991692,991657,994280,998200,995855\n", + "Ground Truth: 996913,994136,996761,996067,997839,996144,997790,990550,996089,998473,999363,993834,998399,998934,993766,990897,998185,993478,994910,992645,992429,992750,996795,990139,999216,999041,996547,993809,999384,996803\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 675\n", + "Returned IDs: 995467,997506,997168,992636,998536,993715,992282,998045,996769,990438,992521,998021,990951,995375,998697,999338,999526,990096,993637,998921,992825,997586,994016,990930,996408,996594,998097,991245,992576,993001\n", + "Ground Truth: 995467,997506,997168,992636,998536,993715,992282,998045,996769,990438,992521,993681,998021,990951,995375,998697,999338,999526,990096,993637,998921,992825,993719,997586,994016,996918,997723,992933,990930,995562\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 676\n", + "Returned IDs: 999290,997370,996840,992944,993484,998734,995886,993143,999367,996949,996438,993052,992558,993080,995203,991840,990334,999426,991348,990697,998455,997839,990580,995596,993504,996397,992169,999453,990142,998244\n", + "Ground Truth: 999290,997370,996840,992944,993484,998734,996762,995886,993143,999367,996949,996438,993052,992558,993080,995203,991840,990334,999426,991348,990697,998455,990351,997839,990580,995596,993504,996397,992169,999453\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 678\n", + "Returned IDs: 997348,999647,994450,991428,996805,997913,993853,998426,991550,990893,999444,997918,993264,990231,994964,990797,997320,996041,998921,998029,991721,997301,993529,997586,994503,996809,994266,990647,992862,999982\n", + "Ground Truth: 997348,999647,996904,991582,994450,991428,996805,997913,993853,998426,993141,991550,990893,997918,999444,993264,990231,994964,993412,990797,997320,997880,996041,996296,998921,998029,997772,998550,991721,999683\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 679\n", + "Returned IDs: 992604,997598,994187,998697,991844,995455,992128,999136,996645,995755,990540,995435,999306,992133,999867,995409,992051,993560,998022,999740,994125,999478,995780,995268,995527,993820,996649,999356,999795,993561\n", + "Ground Truth: 992604,997598,994187,997212,998383,998697,991844,995455,992128,999136,996645,995755,990540,995435,999306,992133,999867,995409,996129,992051,993560,998022,996915,999740,994125,999478,995780,995268,995527,993820\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 680\n", + "Returned IDs: 996232,996219,994894,998649,999144,995274,991029,992337,999343,994757,996504,996001,995502,999026,993168,990949,999647,997479,995200,990975,990934,991096,999740,995569,998106,996721,992446,997192,998572,998495\n", + "Ground Truth: 997920,996232,996219,993412,994894,998649,999144,995274,991029,992337,999343,994757,996052,999261,999311,996504,991445,995881,995401,996001,995502,994819,999026,993168,993529,990949,999647,997479,995200,990975\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 681\n", + "Returned IDs: 997975,999137,999415,998188,993209,991671,993586,995778,994730,996302,991639,999581,991507,996226,992289,999875,997544,999054,996867,996852,995268,996607,996187,999957,998910,992448,994742,990631,995640,996633\n", + "Ground Truth: 997975,999137,999415,998188,993209,991671,993586,995778,994730,996302,991639,997723,999581,991507,996226,992289,999875,997544,993412,999054,996867,996852,995268,996607,996187,990361,999957,998910,999980,995963\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 682\n", + "Returned IDs: 998104,993799,998536,993609,995963,990392,994119,994711,992521,993324,996760,997752,995783,994409,999129,998162,996987,996328,997309,994280,997420,996867,995401,994541,991712,993180,992569,992832,997320,997176\n", + "Ground Truth: 998104,994300,993799,998536,993609,995963,990392,994119,990879,992617,994711,998568,992521,993324,999840,996760,992131,997723,997752,992289,995783,994409,999129,998162,996987,996328,997309,994280,997420,999886\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 683\n", + "Returned IDs: 999466,998993,995027,995890,996960,996611,998335,995626,992864,998279,990712,996537,990806,992151,998935,992726,991098,996082,996232,991243,999880,994280,995214,990705,996328,994098,993219,996599,993224,994351\n", + "Ground Truth: 999466,998993,995027,995890,996960,996611,998335,995626,992864,998279,990712,996537,990806,992151,998935,992726,995167,991098,996082,991232,996232,991243,999880,994280,995214,996328,990705,994098,993219,996599\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 684\n", + "Returned IDs: 994742,997319,999367,995107,995401,996001,996219,995633,990089,995289,990096,998495,996411,992827,994532,992558,995200,999791,995536,990585,999026,997301,998950,991968,994972,994783,991588,994743,999009,992944\n", + "Ground Truth: 994742,997319,999367,995222,999377,995107,995401,996001,996219,995633,990089,995289,995611,993080,999647,990096,998495,996411,992827,994532,992558,990077,993725,995200,999791,991921,990585,995536,999026,997301\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 685\n", + "Returned IDs: 998184,994964,990361,998779,995204,995289,993238,995274,995702,996867,997822,997320,998999,991588,995633,995216,990845,992703,993648,999160,995052,995766,997192,993831,994742,993848,999587,992994,998458,998165\n", + "Ground Truth: 998184,994964,990361,998779,995204,995289,993238,995274,995702,996867,993673,997822,996959,997320,998999,993209,992334,991588,995633,999026,993389,993412,992582,995216,990845,992703,993648,999160,995052,995766\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 686\n", + "Returned IDs: 994776,999982,999180,998302,992848,992852,992384,998246,997320,997661,994415,997053,990275,997913,992119,991545,993853,999343,998660,994266,996075,990949,998580,992994,990958,993484,992896,998458,991264,991335\n", + "Ground Truth: 994776,999982,990020,999180,998302,992848,992852,997620,992384,998246,997320,997661,994415,997053,990275,997913,992119,991545,993853,997895,999343,998660,994266,996075,990949,998580,992994,990958,996143,993154\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 687\n", + "Returned IDs: 997584,992635,994971,993785,992041,997550,992601,995596,992671,990914,996075,997661,993447,998227,998498,993125,992733,994886,995372,996755,999426,992617,996186,996631,992515,995515,990142,993129,996875,999317\n", + "Ground Truth: 997584,992635,994971,993785,992041,997550,992601,995596,992671,990914,996075,997661,993447,998227,998498,993125,991218,992733,994886,999439,995372,996755,999426,992617,996186,996631,992515,992277,995515,990142\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 688\n", + "Returned IDs: 999812,993504,993024,996429,994964,990818,999339,993260,999111,995895,998303,998715,998950,993391,996216,992747,997586,992571,994672,990409,997977,991458,993489,996196,991946,990494,991206,993906,999840,995560\n", + "Ground Truth: 999812,993504,993024,996429,994964,990818,999339,993260,999111,995895,998303,998715,998950,993391,996216,992747,997586,992571,996677,993805,992979,994672,995622,999735,990409,997977,991458,993489,996196,995887\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 689\n", + "Returned IDs: 992493,997634,995163,992677,998316,992150,992148,993223,997824,998650,995854,994620,993263,995723,992700,995060,991067,993530,995091,997750,996139,991190,996273,995062,991100,998080,999961,993160,997723,994576\n", + "Ground Truth: 992493,997634,995163,992677,998316,992150,992148,993223,997824,998650,995854,994620,993263,995723,992700,995060,991067,995963,993530,995091,997750,996139,991190,996273,995062,991100,998080,999961,993160,991052\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 690\n", + "Returned IDs: 998536,999759,994835,999259,993014,995175,996877,994119,990103,999533,995783,991052,994742,999847,994522,999244,997309,994280,998021,995685,995110,991747,996328,995328,993180,997675,993256,999601,995017,996087\n", + "Ground Truth: 992933,998536,999759,994835,992166,999259,993014,995175,996877,994119,990103,999533,995783,991052,994742,999847,994522,999244,995963,997309,999527,997184,991712,994280,998021,991891,995685,997723,990184,995110\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 691\n", + "Returned IDs: 999045,993721,994718,992551,990096,991409,998495,994269,991650,993053,996001,996075,990843,996936,995361,990584,991382,996219,996631,998803,997550,991451,999302,990091,996101,994472,996016,996013,995476,990786\n", + "Ground Truth: 999045,993721,994718,992551,990096,991409,998495,994269,991650,993053,996001,999336,996075,990843,996936,995361,994355,990584,991382,996219,999577,996631,998803,997550,991451,999302,990091,996101,999343,994472\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 692\n", + "Returned IDs: 991977,993839,993583,996158,994398,992718,996585,993077,994080,993681,994119,992558,990894,992104,994334,999079,999771,994254,998357,990273,995459,992211,996733,992052,992824,995503,999565,990908,990776,998061\n", + "Ground Truth: 991977,993839,993583,996158,994398,992718,996585,993077,994080,993681,994119,992095,992558,990894,992104,994334,999079,999771,994254,998357,990273,995459,992211,996733,992052,992824,995503,990279,999565,990908\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 693\n", + "Returned IDs: 991462,992724,997486,992739,992632,993395,997326,998359,999015,992612,997772,996693,995119,991485,999453,992309,997539,992152,998401,990580,998950,995515,990334,999077,991402,994964,997775,994922,993781,990043\n", + "Ground Truth: 997448,991462,992724,997486,992739,992632,993395,997326,998359,999015,992612,997772,996693,995119,991485,999453,992309,993369,996829,997539,992152,998401,995721,990580,998950,995515,990334,999077,995767,991402\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 694\n", + "Returned IDs: 991425,999794,993504,990956,993665,991552,998615,995349,992457,998303,999111,992137,991648,994672,998109,996884,992351,997506,991037,996012,990658,993609,990409,991257,993429,990959,992640,991779,990885,992819\n", + "Ground Truth: 990010,991425,999794,999211,993504,990956,993665,991552,998615,995349,992457,998303,999111,992137,991648,994672,998109,996884,995541,992110,995103,992351,990752,997506,991037,996012,990658,993609,990409,994681\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 696\n", + "Returned IDs: 993003,992267,996256,992917,994409,997466,991357,999250,995401,992448,997909,990295,992763,995242,996815,994508,994865,998910,998278,994106,993537,992521,997299,996968,998756,991197,995268,999530,995864,998361\n", + "Ground Truth: 993003,992267,996256,992917,994409,997466,999250,991357,995401,992448,997909,990295,992763,995242,996815,994508,994865,998910,998278,994106,993537,992521,996223,997299,996968,998756,991197,995268,992260,999530\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 697\n", + "Returned IDs: 993512,995336,997571,993096,999144,999331,998090,990647,992134,993853,992384,994197,992692,994334,993492,991131,993153,995357,998562,993785,996296,992453,994227,994100,999453,992748,992337,999639,997918,991094\n", + "Ground Truth: 993512,995336,997571,993096,999144,999331,998090,992134,990647,993853,992384,994197,990214,994810,992692,995748,994334,993492,991131,993153,995357,995374,994819,993925,991923,996882,990654,998562,993785,998410\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 699\n", + "Returned IDs: 999984,995589,995408,999543,993936,991011,998855,993778,991977,994113,993433,991962,994191,997944,999643,992576,995582,992179,998623,991556,994254,996277,992052,991515,996733,995113,998536,993281,999091,993979\n", + "Ground Truth: 999984,995589,995408,999543,993936,991011,998855,993778,991977,999461,994113,993433,991962,994191,997944,999643,992576,995582,992179,998623,991556,995401,994254,996277,992052,991515,996733,995113,998536,993281\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 700\n", + "Returned IDs: 990956,996016,991382,992652,992827,993501,994927,998543,994541,998495,996809,995191,999179,994765,995895,993460,998167,996830,997270,991469,996706,999830,993219,996301,993391,997506,992628,999192,997442,994307\n", + "Ground Truth: 990956,996016,991382,992652,992827,993501,994927,998543,994541,998495,996809,994378,995191,999179,994765,995895,993460,998167,996830,997270,995401,991469,996706,999830,993219,996301,998206,993948,990712,993391\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 701\n", + "Returned IDs: 990930,991459,992036,992790,999639,998874,995536,995401,996001,994987,990412,993515,993673,996247,995932,996371,991530,997918,995672,998593,996918,998303,996181,991721,994280,992749,990794,994809,993504,997191\n", + "Ground Truth: 990930,991459,992036,992790,999639,998874,995536,995401,996001,994987,990412,993515,993673,990337,996247,995932,991408,996371,991530,997918,998990,995672,998593,991052,996918,998303,991392,996181,997238,991721\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 702\n", + "Returned IDs: 998039,990146,994925,994964,990925,990845,997969,992334,998593,998302,990560,994419,996001,996028,994853,998921,995923,993831,993561,997453,997822,994307,997586,992944,997320,997046,996785,999374,990529,991685\n", + "Ground Truth: 998039,990146,994925,995162,994964,990925,990845,997969,992334,998593,998302,990560,994419,996001,996028,993648,994853,993412,997920,998921,995923,993831,998228,995906,993561,997453,997822,994307,995401,997586\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 703\n", + "Returned IDs: 998668,994557,991377,991179,990506,995547,995289,998416,999294,993561,991923,996042,995923,997969,997732,994782,998292,997822,995643,993378,996494,996298,998133,996216,996080,997386,998039,993929,998382,990137\n", + "Ground Truth: 998668,994557,991377,991179,999785,990506,995547,995289,995162,998416,999294,993561,991923,996042,995923,997969,991790,997732,994782,998292,997822,995643,993378,993412,996494,996298,994026,998133,996216,996080\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 704\n", + "Returned IDs: 990003,997913,999647,990210,990545,996053,999331,994873,998302,990958,993031,990231,998540,995438,990650,994450,993484,997320,991370,998550,999167,999263,994102,992147,990957,998899,995802,996075,992546,990949\n", + "Ground Truth: 990003,997913,999647,990210,990545,992946,996053,998344,999331,994873,998302,990958,993031,991479,997439,990231,998540,991429,995438,990650,991852,991782,994450,993484,997320,993990,991370,994898,996296,998550\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 705\n", + "Returned IDs: 997897,994280,995421,991605,991447,991232,993489,992052,990627,997442,990844,992864,997926,995626,995203,992457,991332,996817,995630,998428,990885,997790,991218,991212,993175,993391,994738,994531,990630,994351\n", + "Ground Truth: 997897,994280,995421,991605,991447,991232,993489,992052,990627,997442,990844,992864,997926,998071,995626,993087,995203,992457,991332,996817,997531,995630,998428,990885,997790,991218,991212,993175,999550,993391\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 706\n", + "Returned IDs: 998798,995374,996703,990992,997015,991467,992854,996353,998951,990585,994197,991504,997609,990535,999026,996959,994595,997009,991173,991498,999089,996714,992189,996658,993562,995824,995034,998844,999024,995274\n", + "Ground Truth: 993368,998798,995374,996703,990992,997015,991467,992854,996353,998951,990585,994197,991504,997609,990535,993909,999026,996959,994595,997009,991173,991498,999089,996714,992189,996658,993562,995824,995034,998844\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 707\n", + "Returned IDs: 997485,997529,993939,998610,995861,996752,996328,990133,992450,993538,997103,997544,994830,991114,998710,993079,990359,994185,999372,995295,990889,991334,998484,994835,993127,997933,992434,998671,995211,995620\n", + "Ground Truth: 997485,997529,993939,998610,995861,996752,996328,990133,992450,993538,997103,996042,997544,994830,991114,998710,993079,994131,990359,994185,999372,995295,990889,991334,998484,994835,993127,997933,992434,998671\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 708\n", + "Returned IDs: 996052,996219,990448,999998,998464,991999,994964,991844,995581,990538,999876,997010,992604,990063,990366,996001,998593,999791,995596,992944,992337,992558,998495,991052,997470,995693,996959,996871,998758,990886\n", + "Ground Truth: 996052,996219,998383,990448,999998,998464,991743,995716,991999,994964,993412,991844,996232,995581,990538,999876,997010,992604,990063,991650,990366,996001,998593,999791,995596,992944,992337,995107,992558,990091\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 709\n", + "Returned IDs: 997103,999980,990537,999581,993939,999054,992448,991722,990220,990251,998910,994861,999279,990372,998733,997544,991389,993127,992449,993856,995268,995778,997176,993308,991507,994380,991664,992097,991560,996505\n", + "Ground Truth: 997103,999980,990537,998283,999581,993939,999054,992448,991722,996897,990220,990251,998910,994861,999279,990372,998733,997544,991389,993127,992449,993856,995268,995778,997176,993308,993046,993003,991507,994380\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 710\n", + "Returned IDs: 994922,991840,990958,992169,996001,998593,997822,990076,996351,992994,999729,996867,999676,990064,993143,992692,996854,997320,992944,990455,994380,992622,999453,991392,993168,998458,991015,995245,995638,992448\n", + "Ground Truth: 994922,991840,997591,990958,999261,992169,996001,998593,997822,990076,996351,992994,999729,990648,990399,996867,999676,991087,990064,993143,992692,996854,997320,992944,990866,990455,997620,994380,992622,996438\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 711\n", + "Returned IDs: 993698,991066,995007,992576,991011,997331,997168,995589,998723,996505,994191,996067,995245,994380,991857,992472,996277,998045,996716,995815,996062,990537,995907,994113,991747,996769,993561,993813,997675,991958\n", + "Ground Truth: 993698,991066,995007,992576,991011,997331,997168,995589,998723,996505,994191,996932,996067,995245,990951,994380,993778,991857,992472,996277,994419,995431,998045,996716,995815,996062,990537,995907,994113,991747\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 712\n", + "Returned IDs: 991720,990545,998170,993143,991999,993253,990432,999851,994964,995588,995515,995927,998294,997908,993484,997320,998550,992384,993885,994100,998101,999453,994561,997002,996001,992942,995400,994346,993877,991743\n", + "Ground Truth: 991720,990545,996296,998170,993143,999437,999764,991999,993253,991715,998562,994251,992979,990432,999077,999851,994964,995588,995515,995927,997876,998294,997908,993484,999343,997320,998550,992384,995748,993885\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 713\n", + "Returned IDs: 999384,994972,990096,995536,998697,997320,999898,999647,996001,996062,999144,997913,991154,995701,996219,995130,993504,998401,993809,997775,993906,996370,995596,993561,991459,995560,999653,997918,994964,991157\n", + "Ground Truth: 999384,994972,997811,990096,995536,998697,991179,997320,999898,999647,996001,996062,999144,997913,991154,995701,996219,995130,993504,998401,993809,992692,997775,993906,996370,992047,995596,993561,991459,995560\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 714\n", + "Returned IDs: \n", + "Ground Truth: 992669,990063,997526,998858,992604,999961,997110,992439,993115,995949,991309,998707,994724,994774,995982,993122,999478,990111,998383,990657,994569,999231,997205,991987,994454,997107,993976,998806,995455,992051\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 715\n", + "Returned IDs: 996042,997046,997822,993642,996328,997386,990137,993433,994419,996322,998545,997453,998165,996494,995289,998039,995200,997969,993561,993831,996867,995401,996062,991301,995685,998633,997289,994783,994742,994409\n", + "Ground Truth: 996042,997046,997822,993642,996328,997386,993827,990137,993433,993412,994419,996322,998545,997453,998165,996494,995289,998039,995200,997969,993561,993831,996867,995526,997732,994964,995401,993400,996062,991301\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 717\n", + "Returned IDs: 993179,992635,990697,991260,994886,991372,996884,996755,990532,995596,996001,992515,997550,992041,993820,990083,993071,998097,995515,996075,996805,999426,997002,990834,994862,997584,995358,993129,992733,994972\n", + "Ground Truth: 993179,992635,990697,991260,994886,991372,996884,996755,990532,995596,995622,996001,992515,997550,992041,993820,990083,990640,993071,998097,995515,996075,991142,996805,999426,997002,990834,994862,997584,991048\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 718\n", + "Returned IDs: 996536,993588,994897,994830,993342,997952,992664,998716,993335,998092,995097,998623,998903,998440,997296,998256,997073,998030,996183,992427,998097,994513,993866,990392,992458,994682,993715,994906,997002,996001\n", + "Ground Truth: 996536,993588,994897,994830,993342,997952,992664,998716,997301,993335,998092,995097,998623,998903,998440,997296,998256,997073,998030,990334,996183,992427,998097,994513,993866,990392,999963,992458,994682,992174\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 719\n", + "Returned IDs: 992330,997371,997913,995985,990397,990469,990560,998500,992384,992443,996193,990595,998921,995260,996840,994360,992546,999122,997320,999676,997984,997305,997831,993492,991626,996504,999603,995949,999165,998445\n", + "Ground Truth: 992330,997371,997913,995985,993072,990397,990469,990560,993048,998500,991468,990092,992384,992443,997889,996193,990595,998921,995260,996840,994360,992546,994281,999613,991631,999122,995365,997320,999676,997984\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 720\n", + "Returned IDs: 998555,998169,999781,990271,990096,997529,998122,999343,995638,992551,993800,997732,994145,992120,995535,999103,993637,996219,990089,999647,999319,990690,994532,998934,998314,990918,996328,990072,990366,999791\n", + "Ground Truth: 998555,998169,999781,990271,990096,997529,998122,999343,995638,992551,993800,997732,990091,994145,992120,995535,999103,993637,996219,995216,990089,999647,999319,990690,997918,994532,998934,997920,998314,990918\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 721\n", + "Returned IDs: 990242,998162,992937,990940,995444,993831,997046,996388,992160,997466,991163,991603,991973,993390,995702,998503,996867,997723,992521,998238,998267,998165,999595,997868,997822,993678,996769,990110,994954,991415\n", + "Ground Truth: 990242,998162,992937,990940,995444,993831,993799,997046,996388,992160,997466,991163,991603,991973,993390,996038,995702,998503,996867,997723,992521,998238,998267,998165,999595,992977,997868,998941,997822,993678\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 722\n", + "Returned IDs: 998536,993965,992407,999448,995461,998331,992052,999079,994119,995408,991707,997897,998051,996817,999212,996633,999433,996245,995110,996733,998177,992320,990551,998874,997881,991515,990169,994743,996134,991245\n", + "Ground Truth: 998536,993965,992407,999448,995461,998331,992052,999079,994119,995408,991707,997897,998051,996817,999212,991093,996633,999433,996245,995110,996733,998177,992320,990551,998874,997881,991515,990169,994743,999179\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 723\n", + "Returned IDs: 992636,995328,994522,999767,994742,992832,991850,991998,995012,992764,991007,992973,996911,998174,999759,990336,994835,992239,997153,994891,993747,990270,991629,998536,993621,993448,993959,996183,994175,996050\n", + "Ground Truth: 992636,995328,992933,994522,999767,994742,992832,996557,991850,991998,995012,992764,991007,992973,996911,998174,992779,999759,990336,994835,990364,992239,997153,994891,993747,990270,991629,998536,993621,993448\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 724\n", + "Returned IDs: 993775,998765,999791,995477,991921,990089,994742,997529,999058,996633,997401,995057,997698,996328,996302,999788,997309,999875,991588,990179,999192,999261,995438,994806,993053,993265,995289,995551,999138,991315\n", + "Ground Truth: 993775,998765,999791,995477,991921,990089,994742,997529,999058,996633,997401,995057,997698,996328,996302,999788,997309,999875,991588,990179,999192,999261,995438,994806,993053,993265,995289,995551,999138,995600\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 725\n", + "Returned IDs: 998168,992517,995768,995501,994356,995476,993755,990851,991382,991667,995479,992953,999425,990335,997340,995879,993319,991826,999130,998794,996184,994205,990145,998625,999107,999256,995073,995191,991779,992868\n", + "Ground Truth: 998168,992517,995768,995501,994356,995476,993755,990851,991382,991667,995479,999634,992953,999425,990335,995222,997340,995879,993319,991826,999130,998794,996184,994205,990145,998625,999107,999256,995073,995191\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 726\n", + "Returned IDs: \n", + "Ground Truth: 993035,995791,995004,995365,993570,993633,995588,999039,999592,993282,998837,990043,999620,993028,991999,997496,995155,994545,995657,994666,991407,991593,993505,993675,994543,994922,993708,993946,991196,990155\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 727\n", + "Returned IDs: 995865,990612,994611,996143,996318,991872,994214,997320,990210,997178,994812,992909,996805,997022,993885,998550,997305,991498,994964,999374,994699,994249,996755,992978,992066,998540,999982,994164,994778,992993\n", + "Ground Truth: 993990,995865,990612,994670,994611,996143,993745,996318,993154,991872,994214,997320,990210,997178,993579,994812,992909,996805,997022,993885,999351,998550,998287,997305,991498,990772,998344,994964,999374,994699\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 728\n", + "Returned IDs: 995932,993504,999516,990196,993071,993906,991440,991648,997555,991206,993515,992082,990350,997918,992571,993783,997790,992747,993696,993260,997002,992351,991435,995375,991994,996429,998726,996949,997965,993637\n", + "Ground Truth: 995932,993504,999516,990196,993071,993906,991440,997758,991648,997555,991206,994081,997135,995622,993515,990682,992082,993821,990350,997918,992571,993783,997790,992747,993696,993260,997002,994146,992351,991435\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 729\n", + "Returned IDs: 995655,998625,990154,993001,992110,992517,999555,990167,992752,992444,991667,998536,994465,999433,998006,993948,991620,990173,996408,998888,995879,999256,994423,995768,992595,993641,999319,991421,999994,995936\n", + "Ground Truth: 995655,998625,990154,993001,992110,992517,999555,990167,992752,992444,991667,998536,994465,999433,998006,993948,991620,990173,991188,990429,996408,998888,995879,994796,999256,994423,995768,993641,992595,999319\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 730\n", + "Returned IDs: 997025,995521,995907,998703,999653,995303,991161,992448,997251,995891,995537,993275,993313,993342,994380,990593,995778,994125,991844,997675,993561,996042,993127,998855,991459,991447,991794,995815,992576,994419\n", + "Ground Truth: 997025,995521,995907,998703,999653,995303,991161,992448,997251,995891,995537,997909,993275,993313,993342,994380,990593,995778,990725,994125,991844,997723,991338,997675,993561,996042,993127,998855,991459,991447\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 731\n", + "Returned IDs: 999374,999710,997320,997913,996504,992748,991860,999331,991801,998495,998540,993484,995942,991743,997301,996178,992147,995886,994964,993809,995961,991498,998246,990958,990957,998550,990210,996318,998029,991593\n", + "Ground Truth: 999374,999710,997320,997913,999701,991052,996504,992748,991860,999331,991801,998294,998495,998540,993484,995942,991743,995632,997301,996178,992789,992147,995886,994964,993809,995961,994850,991498,998246,990958\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 732\n", + "Returned IDs: 993561,996783,996313,996733,997023,996062,991958,993931,994444,990537,990096,998207,992322,999673,990382,991093,998536,994380,997453,998024,990899,990531,992752,995648,995521,993289,996001,998314,998416,997790\n", + "Ground Truth: 993561,996783,996313,996733,997023,996062,991958,993931,999436,994444,990537,990096,998207,992322,999673,990382,990072,991093,998536,994380,997453,995401,998024,990899,990531,992752,995648,995521,993289,996001\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 733\n", + "Returned IDs: 992037,998022,996277,998934,994906,996181,990400,996001,998720,998080,991459,994444,999020,992685,999026,999543,997204,993652,995780,994589,994424,992648,996379,992576,999759,998990,991248,991574,996861,992516\n", + "Ground Truth: 992037,998022,996277,998934,994906,995401,993880,996181,990400,996001,998720,998080,991459,999259,994444,999020,992685,999026,998221,999543,997204,994105,995091,996408,993652,995780,994589,994424,992648,996379\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 734\n", + "Returned IDs: 992712,995855,995356,998622,992163,996105,994167,990472,994955,993283,999216,997368,993300,998513,991232,992484,999041,993174,999219,998697,997977,999750,995672,998934,997417,995203,993918,998447,993260,998393\n", + "Ground Truth: 992712,995855,995356,995563,998622,992163,996105,994167,990472,994955,993283,999216,997368,993300,998513,991232,996144,992484,999041,993174,999219,998697,997977,990688,999750,995672,998934,997417,995284,995203\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 735\n", + "Returned IDs: 996196,997775,998950,994163,996140,994661,995596,997320,995560,990538,990532,996178,995983,990096,992124,990901,992558,991923,997913,997227,995552,995974,999564,996380,998697,996498,995289,995536,999639,998779\n", + "Ground Truth: 996196,997775,998950,995873,994163,996140,994661,995596,997320,995560,990538,992521,990532,996178,998417,991157,995983,990930,990096,992124,992277,990901,992558,991923,997913,997227,995552,997379,995974,999587\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 736\n", + "Returned IDs: 995507,999619,997832,996038,999530,993421,996323,996618,995905,995139,991027,995470,992448,996867,994409,997359,993400,995685,997046,996322,992130,997289,990836,992605,998045,992169,998520,997723,999482,997466\n", + "Ground Truth: 995507,999619,997832,996038,999530,993421,996323,996618,995905,995139,991027,995470,992448,996867,994409,997359,993400,995685,997046,996322,992130,997289,990836,992605,998045,993468,992169,998520,997723,999482\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 737\n", + "Returned IDs: 998682,999066,995372,996309,991787,995968,999544,993840,995250,999254,996452,993241,992944,991673,998849,995515,991248,997498,991650,992348,999453,997320,998921,997301,993447,998734,999686,991157,997453,996805\n", + "Ground Truth: 998682,999066,995372,996309,991787,995968,999544,993840,995250,999254,996452,993241,992944,991673,998849,995515,991248,997498,991650,992348,999453,997320,998921,994826,997301,997906,993447,998734,999686,995154\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 738\n", + "Returned IDs: 990115,998315,998925,992193,996431,995191,998055,994472,995223,990712,998671,996357,991682,994299,996016,992237,999205,991723,990187,997573,995789,999752,990091,996153,995919,991153,999832,992652,990931,996537\n", + "Ground Truth: 990115,998315,998925,992193,996431,995191,998055,994472,995223,990712,998251,998671,997954,996357,991682,994299,996016,992237,999205,999441,991723,990187,997573,995789,999752,996942,990091,997602,998084,998122\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 739\n", + "Returned IDs: 992932,990535,997304,998246,995981,998119,999761,992296,991900,998353,992815,997320,995473,991045,991534,992238,990588,993392,990949,998225,997353,992074,998950,990057,992357,999357,993809,992443,997913,995336\n", + "Ground Truth: 992932,990535,997304,996732,998246,995981,998119,999761,992296,991900,998353,992815,997320,995473,991045,995132,991534,992238,995792,990588,993392,993154,996950,992927,990949,997428,998225,998361,997353,992074\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 740\n", + "Returned IDs: 992862,998156,993256,994564,991428,996232,997173,990328,997374,992979,994742,996328,995819,997452,997309,999843,997918,995633,995295,998708,997914,990814,991189,996411,997208,993909,994382,990127,995274,999099\n", + "Ground Truth: 992862,998156,993857,993256,994564,994940,991428,996232,997173,990328,993440,997374,992979,994742,996328,995819,997452,997309,999843,997918,995633,995295,994850,998708,996752,997852,997914,990814,999453,998073\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 741\n", + "Returned IDs: 994672,992351,990885,993504,998732,996062,991425,998303,999811,990409,990196,997918,991037,997227,997977,995740,993289,994964,997790,990557,997002,995203,999093,990096,991212,999840,995932,994221,996196,992571\n", + "Ground Truth: 994672,992351,990885,993504,998732,996062,991425,998303,994468,999811,990409,990196,997918,991037,997227,997977,995740,993289,994964,997790,990557,993609,997002,995203,999093,990096,991212,999840,995932,993333\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 742\n", + "Returned IDs: 995231,997369,995168,994105,997666,992275,995385,990096,999176,996702,994779,995257,993866,997965,990658,992945,997847,998536,995393,997086,998177,992664,999129,996050,994834,991654,997142,995695,997576,997673\n", + "Ground Truth: 995231,997369,995168,994105,997666,992275,995385,990096,999176,996702,994779,995257,992987,993866,997965,990429,990658,999576,992945,991313,997847,998536,995393,997086,998177,992664,999129,996050,994834,990670\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 743\n", + "Returned IDs: 991630,997191,991140,998297,996734,993738,995401,990005,999530,992448,994319,998045,997868,998205,991969,995395,994019,991958,996277,997928,999820,992521,999731,995940,995813,995507,995815,995994,992919,999653\n", + "Ground Truth: 995121,991630,997191,991140,998297,992977,996734,995268,993738,995401,990005,999530,992448,994319,998045,997868,998205,995864,991969,995395,994019,997957,991958,993381,998346,996277,997928,999820,992521,999731\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 744\n", + "Returned IDs: 993885,998294,997320,990898,999374,993364,990917,991900,992896,993853,994964,990760,994476,996218,993484,990863,998899,998246,990296,992931,993662,990560,990146,991498,991593,990925,992546,999263,996279,999710\n", + "Ground Truth: 997006,993885,998294,997876,997320,990898,999374,996296,993364,990917,991059,991900,995266,992896,993048,993853,994964,994776,990760,994476,999328,996218,993484,999247,990863,998899,999647,999823,998246,990296\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 745\n", + "Returned IDs: 995793,991022,992518,995833,990862,998822,990535,996713,995771,998951,997976,994266,997320,994100,995336,998263,996714,998721,996961,992646,992183,992384,995438,999089,995279,997571,991889,993711,992609,992874\n", + "Ground Truth: 995793,991022,992518,995833,991782,990862,998822,990535,996713,995771,998951,997976,994266,997320,993735,994100,995336,998263,996714,998721,996961,992646,992183,992384,995438,999089,995279,997571,991889,999533\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 746\n", + "Returned IDs: 995030,990516,991590,998557,991968,992827,994716,994742,995633,999957,997662,996302,995289,994258,998196,992239,997319,992973,994893,992415,998999,991810,995200,993078,995012,998525,998543,999875,992364,999981\n", + "Ground Truth: 995030,990516,991590,998557,991968,992827,994716,994742,995633,999957,997662,996302,994317,995289,995239,994258,998196,992239,997319,992973,994893,992415,998999,991810,995200,993078,995012,998525,998543,999875\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 747\n", + "Returned IDs: 998132,992037,997320,996277,992294,996497,996247,991011,999820,990409,997918,998874,996368,991349,995130,993433,992979,996181,992759,999019,995873,992165,997027,996127,993652,997897,990899,990885,999026,992576\n", + "Ground Truth: 998132,992037,997320,993542,996277,992294,996497,995701,996247,991011,999820,990409,997918,994113,998874,996368,990442,991349,995130,993433,992979,996181,996294,999804,995876,991556,995047,992759,999019,990723\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 748\n", + "Returned IDs: 999453,991930,990772,996001,991248,991440,991609,993280,990314,995250,992395,995968,997332,991787,990399,991581,995372,990532,993032,994449,991158,994478,991542,991994,992182,992469,998567,999019,991537,994081\n", + "Ground Truth: 999453,991930,990772,996001,991248,991440,991609,993280,990314,995250,994484,992395,995968,997332,991787,998127,992884,990399,991581,995372,990532,993032,994449,999575,991158,994478,991542,991994,992182,992469\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 749\n", + "Returned IDs: 997032,995482,994925,996232,998221,996328,994742,999647,990096,993053,990435,994998,992450,996196,996178,998649,990887,998495,996001,999898,998067,996884,993783,991769,997126,998710,996062,993168,998550,996216\n", + "Ground Truth: 996111,997032,995482,994925,996232,998221,996328,994742,994798,999647,990096,993053,990435,994998,992450,996196,996178,998649,990887,998495,999586,996001,991743,997379,999898,998067,996884,993783,991769,997126\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 750\n", + "Returned IDs: \n", + "Ground Truth: 992701,995626,996082,991098,996599,995991,991236,995167,993375,994259,998389,990802,992437,995047,991327,990382,991741,990212,992247,998593,996001,997790,995568,997860,997494,991212,996733,997897,992862,991308\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 751\n", + "Returned IDs: 997015,991568,993168,993050,998992,990535,992744,995336,993562,996714,998951,995252,992646,999531,997082,994964,999026,995824,992911,992331,999144,994197,998495,999437,991444,997709,990925,993484,990797,997918\n", + "Ground Truth: 997015,991568,993168,993050,998992,990535,992744,995336,993562,996714,998951,995252,992646,999531,997082,994964,999026,990936,995824,992911,995536,992331,993648,999144,994197,998495,999437,991444,997709,990925\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 752\n", + "Returned IDs: 994259,995626,998376,994280,997790,998639,999093,995167,993262,993024,990212,991212,995343,997897,997534,990877,992864,995047,992168,996767,996216,992701,993187,995991,996599,991994,998032,992780,991098,991661\n", + "Ground Truth: 994259,995626,998376,994280,997790,996424,998639,999093,995167,993262,993024,990212,991212,995343,997897,997534,990877,992864,994815,995047,992168,990494,996767,997494,996216,992701,995705,993187,995991,996599\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 753\n", + "Returned IDs: 994140,993390,993831,991958,997046,991415,996042,998776,996719,995289,993561,994409,997822,995052,994783,993813,999099,991790,990940,996322,996388,998633,997554,998165,996867,997630,996924,993738,995679,994743\n", + "Ground Truth: 994140,993390,993831,991958,997046,991415,996042,998776,996719,995289,993561,994409,997822,995052,994783,990193,993813,999099,991790,990940,995873,996322,996388,998633,993856,997554,998165,996867,997630,996924\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 754\n", + "Returned IDs: 998950,996805,992944,993696,992041,998180,999210,999426,997470,991650,997371,995633,997403,997913,993504,994742,999676,998495,998855,999639,993747,993529,998790,992520,996196,996219,990096,996397,999367,998913\n", + "Ground Truth: 998950,996805,999647,992944,993696,992041,998180,997906,999210,999426,994089,997470,991650,997371,995633,997403,997913,993504,994742,997291,995401,999676,998495,998855,999639,993747,993529,998790,992520,996196\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 755\n", + "Returned IDs: 993791,999247,997320,998302,998090,996296,997661,993260,999290,994360,991774,998246,997470,993845,998337,997371,996592,990949,991743,992950,990231,997913,993741,995949,995260,997586,995596,993484,992896,990957\n", + "Ground Truth: 993791,999247,997320,998302,998090,996296,997661,993260,999290,994360,991774,998246,997470,993845,998337,990467,997371,996592,990949,991743,992950,990231,997913,993741,995949,995260,997586,995596,993484,993143\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 756\n", + "Returned IDs: 991507,991591,999957,998162,995778,999054,995958,997723,994861,998910,993209,997700,997020,999137,992448,991401,990105,994013,997046,999415,991671,995665,995493,996079,996867,998267,999787,997544,992002,999646\n", + "Ground Truth: 991507,991591,999957,998162,995778,999054,995958,997723,994861,998910,993209,997700,997020,999137,990836,992448,991401,999412,993899,990105,994013,997046,999415,991671,995665,995493,998609,996079,996867,998267\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 757\n", + "Returned IDs: 992372,995313,997856,998063,993335,997340,991620,998379,994682,995299,992026,990712,999381,991382,992882,990851,990173,999798,999348,995768,998625,997527,990579,994356,995501,992953,991667,992832,990392,996301\n", + "Ground Truth: 992372,995313,997856,998063,993335,997340,991620,998379,994682,995299,992026,990712,999381,991382,992882,990851,990173,999798,999348,995768,998625,995222,996037,997527,990579,994356,995501,992953,991667,992832\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 758\n", + "Returned IDs: 996001,991119,995455,995755,999009,998861,997337,990862,998122,990096,996809,994879,995701,990866,995425,999480,998514,999388,997586,995401,990091,991459,999289,991154,991903,995536,997305,999663,999898,990974\n", + "Ground Truth: 996001,991119,995455,995755,999009,998861,991881,999916,997337,990862,990505,998122,990096,996809,996406,994879,995701,990866,995425,999480,996408,998514,999388,997586,995401,990091,995132,991459,999289,991154\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 759\n", + "Returned IDs: 994833,998959,993560,994834,999373,995393,996797,998623,999136,993055,995237,998787,993366,998831,997210,993948,995936,998383,993014,990952,991844,998758,998565,995780,998861,999724,995695,990005,992903,992636\n", + "Ground Truth: 994833,998959,993560,994834,999373,995393,996797,998623,999136,993055,995237,998787,993366,998831,997210,993948,995936,998383,993014,990952,994906,991844,998758,998565,995780,998861,999724,995695,990005,992903\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 760\n", + "Returned IDs: 990061,990273,995309,993583,990206,994254,993500,996380,997547,992052,990042,994679,993077,995459,999022,993811,991977,991432,997866,995728,993742,990776,996328,992558,990894,993879,994280,997494,999639,994891\n", + "Ground Truth: 990061,990273,995309,993583,990206,994254,993500,996380,991064,997547,992052,990042,994679,993077,995459,999022,993811,991977,991432,997866,995728,993742,990776,996328,995600,992558,990894,993879,994280,997494\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 761\n", + "Returned IDs: 990127,990331,993209,995778,992862,994409,995268,990887,995536,999179,993313,994488,992260,990975,996328,994954,990930,992052,994835,992636,993809,992595,991958,993561,992917,996001,996232,994742,995907,990593\n", + "Ground Truth: 990127,995198,990331,993209,995873,995778,992862,994409,995268,995401,990887,995536,999179,993313,994488,992260,990975,997184,996328,994954,992293,990930,992052,994835,992636,993809,992595,995122,999581,991958\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 762\n", + "Returned IDs: 997550,995596,994618,998921,999338,999426,996075,997661,997113,991992,995033,992041,995515,996318,993785,992944,997584,996516,994971,997984,991984,994233,996741,996631,996387,999698,999588,995467,992636,997291\n", + "Ground Truth: 997550,995596,994618,998921,999338,999426,996075,997661,997113,991992,995033,992041,995515,997448,996318,993785,992944,997584,996516,994971,997984,991984,993125,995401,994233,996741,996631,996387,999698,999588\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 763\n", + "Returned IDs: 995317,993678,996194,992992,997854,993256,994541,993044,990987,997251,990069,999287,995159,996438,994064,997806,993621,993014,992313,996232,991285,994639,999179,990776,991172,993948,994169,993039,990953,991319\n", + "Ground Truth: 995317,993558,993678,996194,992992,997854,993256,994541,991075,993044,990987,997251,990069,995401,999287,997429,995159,996438,991863,994064,999386,997806,994428,993621,993014,992313,996232,991285,993454,997770\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 764\n", + "Returned IDs: 994955,998933,993194,999798,991685,993665,998536,997944,992825,994280,995168,995932,992561,993715,995589,997453,992823,995945,993931,993037,994167,990387,999840,990364,999543,991712,995834,997506,990409,990658\n", + "Ground Truth: 994955,998933,993194,999798,991685,993665,999052,998536,997944,992825,994280,995168,997155,992850,995932,992561,993715,995589,995366,997453,992823,998778,995945,993931,997926,993037,994167,990387,999840,990364\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 765\n", + "Returned IDs: 994557,991058,997638,993929,994119,995332,990345,992704,997822,997309,990987,990678,999210,997693,991107,995620,999643,990226,992147,994280,997945,994336,997126,992962,994541,991177,999612,991378,991848,994142\n", + "Ground Truth: 994557,991058,997638,997807,993929,994119,995332,990879,990345,992704,997822,997309,990987,993014,990678,999210,997693,996071,991107,995620,999643,990226,992147,994280,997945,994336,997126,992962,994541,991177\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 766\n", + "Returned IDs: 992293,990662,990771,998878,998017,998549,996147,996657,995158,999302,993848,996001,996759,999853,999922,997554,990811,999342,990573,992862,994223,991588,993344,996368,990211,997079,995575,998710,991585,993168\n", + "Ground Truth: 992293,990662,990771,998878,998017,998549,996147,996657,995158,999302,993848,996001,996759,995002,999853,999922,997554,990811,999342,990573,992862,994223,991588,999587,993344,993374,996368,990211,997079,995575\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 768\n", + "Returned IDs: 999117,994643,993122,993608,995625,992677,999155,993232,990361,991987,996395,998078,999231,991630,997149,995807,999834,996809,993795,998257,997110,994092,997752,997568,995373,998536,995105,992288,993076,992685\n", + "Ground Truth: 999117,994643,993122,993608,995625,992677,999155,993232,990361,996395,991987,998078,999231,991630,997149,995807,995401,999834,991052,996809,993795,998257,997110,994092,992206,997752,997568,995373,998536,995105\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 769\n", + "Returned IDs: 999388,990154,998971,999663,994636,999555,992282,992403,998701,991596,996406,997803,992155,993132,991385,999153,994181,990394,992716,994335,999898,994834,990305,994638,993458,994155,993560,999677,991154,997154\n", + "Ground Truth: 999388,990154,998971,999663,994636,999555,992282,992403,998701,991596,996406,992445,997803,992155,993132,991385,999153,994181,990394,992716,994335,999898,994834,990305,994638,993458,994155,993560,999677,991154\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 770\n", + "Returned IDs: 991889,995949,998294,995260,996053,998692,999374,991705,991406,992309,995928,993143,997627,996334,995693,994919,990432,996854,995354,992169,997429,990760,999432,990146,990958,998170,992252,994195,990189,996351\n", + "Ground Truth: 991889,995949,998294,995260,996053,998692,999374,991705,991406,996554,993837,992309,990026,995928,993143,997627,996334,992979,995693,994160,994919,990432,990545,990103,993412,996854,990490,995354,996202,992169\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 771\n", + "Returned IDs: 993416,992684,994855,991958,998158,998382,990794,993561,994987,996040,996062,990254,996371,998593,995923,995547,993738,994076,996834,994419,997453,999868,992260,998297,995373,998891,997406,998416,995401,996198\n", + "Ground Truth: 993416,992684,994855,991958,998158,998382,990794,993561,994987,996040,996062,990254,993673,991052,996371,999236,998593,992017,995923,994946,995547,993738,994076,996834,998323,994419,997453,999868,992260,998297\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 772\n", + "Returned IDs: 997002,998902,994906,992790,994334,995894,997301,996658,999442,998951,995560,991621,994661,992239,996001,995401,992165,995836,992759,996220,995357,997918,999587,994088,992641,995536,990561,991485,997320,994946\n", + "Ground Truth: 997002,998902,994906,992790,994334,995894,997301,996658,999442,998951,995560,991621,994661,992239,996001,995401,992165,995374,995836,992759,996220,994362,993820,995357,998045,997918,991989,990140,999587,994088\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 773\n", + "Returned IDs: 992041,996075,990022,990366,993785,992167,990697,991372,995515,994440,999426,991984,996001,991109,999473,997661,995596,992872,998498,996219,993484,992733,992927,999053,990934,992617,996996,999806,990957,993807\n", + "Ground Truth: 992041,996075,990022,990366,993785,992167,990697,991372,995515,994440,999426,991984,996001,991109,999473,997836,994776,998654,997661,993928,995596,996227,992872,998498,996219,993484,992733,992927,999053,990934\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 774\n", + "Returned IDs: 992383,990383,991550,990884,999387,999451,990376,994964,996894,995983,992824,996196,998600,998401,990538,993781,995427,995372,999639,993706,997903,994979,994852,995459,991817,993492,992190,996085,998697,995841\n", + "Ground Truth: 992383,990383,991550,990884,999387,996445,999451,990376,991923,994964,996894,995983,992824,996196,998600,998401,990538,993781,995427,995372,999639,993706,997903,994979,994852,995459,991817,993492,992190,998545\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 775\n", + "Returned IDs: 990832,995100,990194,993435,997725,997331,996475,996809,996219,996830,999435,995880,995131,999331,996216,993342,998084,998122,999430,999388,999647,992100,997047,992237,999898,993148,993391,993931,998387,990957\n", + "Ground Truth: 990832,995100,990194,993435,997725,997331,996475,996809,996219,996830,992892,999435,991187,995880,991459,995131,999331,999384,996067,996216,993342,998084,998122,995130,999430,999388,999647,993192,996913,998925\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 776\n", + "Returned IDs: 999159,991582,995632,997284,997354,991348,996296,997320,992691,993018,991763,993454,999777,995194,991879,996199,996387,991131,992896,990456,996075,996626,999658,994415,990375,998950,997370,994341,997315,993253\n", + "Ground Truth: 999159,991582,995632,997284,997354,994225,991348,995878,992022,996296,997320,992691,993018,991763,991992,993454,999777,995194,991879,996199,996387,991131,992896,990456,991662,996075,995108,996626,999658,994415\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 777\n", + "Returned IDs: 999663,995536,992225,998514,998861,999009,999711,990096,998720,991459,992293,996809,996001,992553,993139,994434,995701,992862,998200,999216,998082,997586,993948,997506,991893,998495,996996,991154,990786,999898\n", + "Ground Truth: 999663,995536,992225,998514,998861,999009,999711,990096,998720,991459,992293,996809,996001,992553,993139,994434,995401,993821,995701,992862,998200,996406,999216,998082,998568,997586,993948,995961,997506,991893\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 778\n", + "Returned IDs: 994401,993039,998972,995065,992985,993931,995159,990184,998851,998566,996277,993799,994835,992576,997181,995648,995760,996223,995268,993256,996505,999759,995790,995815,993879,997126,990956,990391,995685,996173\n", + "Ground Truth: 994401,993039,997077,998972,995065,992985,993931,995159,990184,998851,998566,996277,993799,994835,992576,997181,995648,995760,996223,995268,993256,996505,999759,990291,990942,995790,995815,993879,997126,990956\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 779\n", + "Returned IDs: \n", + "Ground Truth: 998496,995840,994879,996075,991166,992442,991984,993834,997258,992041,998708,993902,998200,997586,997305,997770,993766,992330,996230,993174,993809,996933,991476,995701,994558,996001,999701,996657,993696,999041\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 780\n", + "Returned IDs: \n", + "Ground Truth: 998428,996270,991605,997151,990010,999139,994196,993715,993181,991633,996483,990705,990712,994101,995421,994280,995349,999435,992779,993460,990998,993347,994351,998250,992441,994510,994193,991212,999205,997733\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 781\n", + "Returned IDs: 994273,994098,999430,998671,999144,990711,993100,996816,991840,998416,993161,996222,994801,991672,999225,997331,996328,999532,991378,993501,993127,990392,997193,993929,994351,990226,993805,997967,990408,990340\n", + "Ground Truth: 992226,994273,990815,994098,999430,998671,995704,999144,990711,993100,999435,996816,994558,991840,997955,998416,993725,993161,996222,994801,991672,999225,997331,996328,999171,998142,991938,999532,991378,998729\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 782\n", + "Returned IDs: 999530,998070,992448,995685,991301,994409,996323,991699,994002,996987,998714,998545,997046,997675,995507,992605,999578,999043,996322,991671,996607,991247,999884,991583,996042,995139,993561,998910,998045,996390\n", + "Ground Truth: 999530,998070,992448,995685,991301,994409,996323,991699,994002,996987,998714,998545,997046,997675,995507,992605,999578,999043,996322,993412,991671,996607,998697,991247,990337,999884,991583,993400,996042,995401\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 783\n", + "Returned IDs: 990089,995454,997292,995659,999791,995045,993637,992334,999444,990918,996147,998076,998097,992636,998302,999302,995306,992415,996516,997490,991144,996001,997349,997933,995295,995345,999099,999338,997533,998921\n", + "Ground Truth: 990089,995454,995720,997292,995659,999791,995045,993637,999858,992334,991537,999444,990918,996147,998076,998097,992636,998302,999302,995306,992415,996516,997490,994373,995133,991144,996001,995755,993663,997349\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 784\n", + "Returned IDs: 998256,999852,990998,998279,996986,996147,993335,996189,999342,999334,998343,995326,999273,999924,998959,996960,997790,994830,990402,994185,992636,990771,990956,998335,994891,990434,992664,996328,990027,996594\n", + "Ground Truth: 998256,999852,990998,998279,996986,991347,996147,993335,996189,999342,999334,998343,995326,999273,999924,999555,998959,996960,997790,998122,994830,990402,998680,994185,997506,993545,992636,990771,990956,997494\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 786\n", + "Returned IDs: 990635,998143,998205,993672,996187,997868,995864,994954,998941,999926,992521,991969,998587,991892,993269,990341,995814,999250,998527,996256,990603,996987,994409,997909,997466,992448,996169,998278,992426,996162\n", + "Ground Truth: 990635,998143,998205,993672,996187,997868,995864,994954,998941,999926,992521,991969,998587,991892,993269,990341,995814,999250,998527,996256,990603,996987,994409,997909,997466,992448,996383,996169,998278,992426\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 787\n", + "Returned IDs: 994670,991511,996219,992630,995274,990679,992931,992124,996714,990271,992277,993154,994778,999243,997976,990910,997404,992748,999596,991456,995981,995502,990448,994894,992134,997571,997348,998337,999804,993342\n", + "Ground Truth: 994670,991511,996219,993517,992630,995274,990679,994916,990909,992931,992124,999723,996714,990271,991085,992277,993154,994778,999243,997976,990910,995161,999340,997404,992748,999596,991456,995981,993481,997018\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 788\n", + "Returned IDs: 997411,994730,992293,997436,993725,997925,997529,990435,999922,994606,992605,992582,995701,993342,996368,999740,998000,997723,996028,990811,995435,992024,995016,998934,992473,994382,994636,993168,996001,990012\n", + "Ground Truth: 997411,994730,992293,997436,993725,990815,997925,997529,990435,999922,994606,992605,998105,992582,993856,995701,993342,996368,995906,999740,998000,997723,991301,994674,996028,994728,990811,998708,995435,998181\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 789\n", + "Returned IDs: 999240,993473,997195,993986,992558,999860,993811,991595,991550,993077,995552,991025,992121,998929,990078,999479,999715,996750,993158,995967,996196,990249,997054,991426,994679,992375,999400,992161,999540,996523\n", + "Ground Truth: 999240,993473,997195,993986,992558,999860,993811,991595,991550,993077,995552,991025,992121,998929,990078,995972,999479,999715,996918,991817,996750,993158,993400,995967,996196,990249,999631,994168,997054,991426\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 790\n", + "Returned IDs: 992147,994964,994850,995516,999210,995927,995181,997305,996219,999374,998302,995132,994778,994584,990843,991650,998123,994789,991999,995755,998950,996075,991801,996504,991348,996186,996805,991498,997831,995656\n", + "Ground Truth: 992147,994964,994850,995516,998464,998287,999210,995927,993154,995181,997305,996219,991582,999374,998302,995132,994778,994584,990843,991650,998123,994789,991999,995755,998950,996075,997906,991801,996504,991348\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 791\n", + "Returned IDs: 996633,992598,994639,996155,999059,999192,994541,998761,995551,990392,998348,993825,997026,994169,996960,996194,994356,993448,993314,994742,993691,992832,994327,993725,992595,994861,995122,999774,999225,997340\n", + "Ground Truth: 996633,992598,994639,996155,999059,999192,994541,998761,995551,990392,998348,993825,997026,994169,996960,992992,996194,997838,994356,993793,993448,993314,995401,994742,993691,992832,992540,994327,993725,992595\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 792\n", + "Returned IDs: 998902,998236,991131,998967,996030,994604,991999,998001,993399,995515,997918,998492,995357,998734,990176,999698,991157,992944,994329,995865,995988,994789,991609,995372,991801,993244,999453,990843,994216,992510\n", + "Ground Truth: 998902,998236,991131,998967,996030,994604,991999,991359,990444,998001,993399,995515,991499,997918,998492,995357,998734,990176,999698,992839,994449,991157,997831,992944,994329,990219,995865,995988,995364,994789\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 793\n", + "Returned IDs: 995047,997790,996001,995184,998593,999827,990399,993289,993280,990364,992862,994743,996657,998697,991212,994280,998250,999453,997897,997002,990096,992395,996043,994742,996062,996368,999019,998303,993931,995536\n", + "Ground Truth: 995047,997790,996001,995184,998593,999827,990399,993289,993280,990364,992862,995401,996657,994743,998697,991212,994280,998250,996277,999453,997897,997002,990096,992395,996043,994742,997494,996062,996368,999019\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 794\n", + "Returned IDs: 998679,990930,991585,993293,993813,998779,995633,995289,991141,994742,994730,994140,994436,993997,999402,998980,990435,993856,994409,991664,993831,995702,995648,995216,997554,995052,999163,990940,995268,995016\n", + "Ground Truth: 998679,990930,991585,993293,993813,998779,995633,995289,991141,994742,994730,994140,994436,993997,999402,998980,990435,993856,994409,991664,993831,995702,995648,997723,990361,995216,997554,999805,995052,996277\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 795\n", + "Returned IDs: 997586,995887,994832,990605,992068,996429,997379,996217,999711,993260,990866,993114,992112,997334,992983,990547,999284,992661,991880,999176,996105,996305,995430,999210,994558,999430,998513,994247,998359,995672\n", + "Ground Truth: 997586,995887,994832,990605,992068,999467,996429,997379,996217,996901,999711,993260,990866,993114,999216,992112,997334,992983,994200,990547,999284,992661,991880,999176,996424,996105,996305,995321,995430,998825\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 796\n", + "Returned IDs: 995252,992932,997320,997483,990231,997771,992228,995336,992692,991900,992330,998090,992069,992969,998951,990885,998246,998365,996296,993696,991131,990172,992978,995802,996196,992942,995942,996646,994757,993529\n", + "Ground Truth: 995252,992932,997320,997483,992096,990231,990541,997771,992228,995336,992277,992692,991900,992330,998090,992069,992969,998951,990885,998246,998365,998653,996296,993696,991131,990172,992978,995802,996196,996246\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 797\n", + "Returned IDs: 990050,996825,999206,993165,992358,996087,998507,993155,994051,998267,996918,997420,991319,997806,997438,996394,999296,990116,995516,997752,999881,998537,991799,993727,998778,999646,993453,991107,999871,997942\n", + "Ground Truth: 990050,996825,999206,993165,992358,996087,998507,993155,994051,998267,996918,997420,991319,997806,997438,999105,998021,996394,999296,990116,990386,994301,995516,997752,999881,998537,991799,997416,993727,998778\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 798\n", + "Returned IDs: 992832,995551,996633,994806,995326,992598,997814,999126,997026,993314,996155,994541,998348,993448,990127,993715,991850,997838,998536,991712,994908,994835,996328,993014,997340,998073,995159,995122,993046,997282\n", + "Ground Truth: 992832,995551,996633,994806,995326,992598,997814,999126,997026,993314,995401,996155,994541,998348,993448,990127,993715,991850,997838,998536,991712,994908,994835,996328,993014,997340,990956,998073,993560,995159\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 799\n", + "Returned IDs: 990941,996809,993727,990255,991471,999158,995963,998536,991890,996930,990930,999906,990194,998250,993232,992809,995144,994351,999706,996797,990116,994431,995701,990521,990956,993293,993980,996408,991245,996531\n", + "Ground Truth: 990941,996809,993727,990255,996043,991471,999158,990361,995963,998536,991890,996930,990930,997723,999906,990194,998250,993232,992809,995144,994351,999706,996797,993137,990116,994431,995701,990521,990956,993293\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 800\n", + "Returned IDs: 993708,995357,992134,992692,997918,998967,993145,991015,996592,993696,991131,999077,995336,993492,993781,990957,993512,998245,999144,996075,993683,999784,995210,996840,996085,994552,993529,998734,990949,994046\n", + "Ground Truth: 993708,995357,992134,993633,992692,997918,998967,993145,998996,991015,996592,993696,991131,998011,999077,995336,995693,993492,993781,992946,990957,993512,998245,999144,996075,992277,993683,997470,999784,995210\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 801\n", + "Returned IDs: \n", + "Ground Truth: 995203,993053,990918,993561,996645,990096,998416,995283,995873,990271,991464,994558,997370,994145,999653,994742,996147,993154,995310,997906,992334,998382,991168,994853,991565,999806,992408,995756,997920,990827\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 802\n", + "Returned IDs: 998500,991720,998921,998839,996218,994281,995181,996504,996890,992147,999263,995212,996785,993048,998029,993648,993443,991743,999374,995985,997470,995372,997371,994850,996805,997301,994497,992789,999483,995884\n", + "Ground Truth: 998500,991720,995621,998921,998839,996218,994281,995181,996504,996890,992147,999263,995212,996785,993048,990362,998029,993648,993443,991743,999374,991626,995985,997470,995372,994179,997371,994850,996805,997301\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 803\n", + "Returned IDs: 992321,995404,994163,990159,996196,993612,991009,999564,990548,996766,996690,994384,994950,996637,994957,992608,992215,995013,996127,992558,993670,992661,997775,994912,998220,996963,997902,990303,991550,994000\n", + "Ground Truth: 992321,995404,994163,990159,996196,993612,992458,991009,999564,990548,996766,996690,994384,994950,991601,996637,994957,992608,992215,995013,996127,992558,993670,999639,992661,997775,994912,998220,996963,992650\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 804\n", + "Returned IDs: 997309,997914,990705,992169,998123,990776,999273,993677,994119,998495,996328,994765,994280,992364,996001,999179,998180,990328,993993,991882,994817,992944,991334,994310,994835,996809,994587,997480,995295,995326\n", + "Ground Truth: 997309,997914,990705,992169,991720,998123,990776,999273,993677,994119,998495,996328,994765,994280,992364,996001,999179,995372,998180,990328,993993,991882,998466,994817,992944,991334,994310,994835,996809,994587\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 805\n", + "Returned IDs: 991642,990011,993906,995295,993725,994879,995852,997002,994689,991315,997790,996811,995823,998359,996328,995586,997918,999500,994307,994012,994742,990532,998067,999025,997379,990600,996132,990174,994817,999891\n", + "Ground Truth: 991642,990011,993906,995295,991435,993725,994879,995852,997002,994689,991315,998848,997790,992161,996811,990954,994206,994964,995823,992226,998359,991989,996328,995586,998105,990494,997918,999500,994307,994012\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 806\n", + "Returned IDs: 998343,990976,992056,995185,990255,990449,993875,995695,992293,998831,999334,995672,994510,998453,990399,999498,998589,999663,991836,999087,999502,997790,996082,994815,991038,992862,990959,993515,994365,990292\n", + "Ground Truth: 998343,990976,992056,995185,990255,990449,993875,997088,991256,995695,992293,998831,999334,995672,993545,994510,998453,990399,998834,999498,998589,999663,998959,991836,999087,999502,997790,996082,995074,994815\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 807\n", + "Returned IDs: 993052,999453,996333,999122,992571,993260,990196,998123,996193,997918,999210,991713,993504,993080,992944,993071,992169,994107,993143,990882,995203,993355,998180,997002,996981,994200,997370,998950,993280,993781\n", + "Ground Truth: 993052,999453,996333,999122,992571,993260,990196,998123,996193,997918,999210,991713,993504,993080,990098,992944,993071,992169,994107,993143,991052,990882,995203,993355,998180,996676,997002,994162,996981,994200\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 808\n", + "Returned IDs: 999707,991783,997331,993192,996475,990039,995298,991382,997481,993617,994558,999430,997436,996830,999435,994830,993435,992628,991595,997442,996495,994098,994113,999556,995342,998926,997270,994897,992479,990377\n", + "Ground Truth: 999707,991783,997331,993192,996475,990039,995298,991382,997481,993617,990640,994558,999430,997436,990527,996830,999435,994830,993435,992628,996219,993566,991595,997442,996495,993260,994098,994113,999556,995342\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 809\n", + "Returned IDs: 994964,996219,991582,993168,993289,999827,995927,997370,993143,997822,999104,997732,997918,997586,994922,990866,994280,990862,990146,999343,992169,992147,996001,995536,990382,990096,990224,996062,997002,997284\n", + "Ground Truth: 994964,996219,991582,993168,993289,999827,995927,997370,993143,997920,997822,999104,991052,997732,993412,997918,997586,994922,990866,994280,995210,990862,990146,999343,992169,992147,997028,999289,996001,995536\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 810\n", + "Returned IDs: 998314,999831,995385,993086,990096,998536,996918,992407,998177,998405,999287,993965,999079,994661,999194,992747,991052,996733,991093,998968,991724,991243,993239,999320,991571,993609,993715,997897,999384,990583\n", + "Ground Truth: 998314,999831,995385,993086,990096,998536,996918,992407,998177,998405,999287,993965,999079,994661,999194,992747,991052,996733,991093,998968,991724,991243,993239,999320,991571,993609,997775,993715,997897,999384\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 811\n", + "Returned IDs: 992215,998213,992615,998827,994287,997859,996196,990068,991550,994163,995886,992747,997124,995552,996442,992558,990343,995372,995983,999639,993693,992170,991817,993130,997258,999385,993260,996127,996404,990620\n", + "Ground Truth: 992215,998213,992615,998827,994287,997859,996196,990068,991550,994163,995886,992747,997124,990486,995552,996442,992558,990343,995372,995983,993693,999639,992170,990910,991817,993130,997258,999385,993260,996127\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 812\n", + "Returned IDs: 990925,994160,996343,990958,998797,999262,995884,992546,995947,990535,991396,995485,990461,992994,993083,990146,992284,997102,991444,998458,995771,993562,999600,990936,994922,994685,998927,999644,997192,996959\n", + "Ground Truth: 990925,994160,996343,990958,998797,999262,995884,992546,995947,990535,991396,995485,990461,992994,993083,990146,992284,997102,997876,991444,998458,995771,993562,999600,993914,997053,990936,994922,994685,998927\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 813\n", + "Returned IDs: 994334,999144,997012,994819,994088,996658,995374,999820,998011,996903,995279,990215,997571,997918,999089,996425,992087,997910,999374,997002,996282,998385,995453,991131,991977,991074,993407,992147,990275,999026\n", + "Ground Truth: 994334,999144,997012,994819,994088,996658,995374,999820,998011,996903,995279,990215,997571,993868,997918,999089,996425,992087,997910,999374,993412,997002,996282,998385,995453,999562,991131,991977,991074,993407\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 815\n", + "Returned IDs: 991269,990392,995283,992713,994853,999126,998811,991168,993678,991189,995295,993561,991464,994742,996411,996752,991769,993813,997141,991789,997452,993725,995633,999496,991981,996628,993209,994090,994119,992832\n", + "Ground Truth: 991269,990392,995283,997682,992713,994853,999126,998811,991168,993678,991189,995295,993561,991464,994742,997933,996411,996752,991769,994386,993813,997141,996055,991789,990614,997452,993725,999496,995633,991981\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 816\n", + "Returned IDs: 993392,992027,995771,993620,997984,996504,998921,996218,993518,997704,994281,994939,991782,998692,995985,994922,998822,992709,991889,999374,990595,997192,992546,994775,992147,993161,999327,994641,995554,994630\n", + "Ground Truth: 993392,992027,995771,993620,997984,996504,998921,996218,993518,997704,994281,994939,991782,998692,993457,995985,991873,994922,998822,992709,991889,992932,999374,990595,997192,997006,992546,994775,992147,993161\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 817\n", + "Returned IDs: 994304,997868,997586,992521,995814,994636,994954,996918,992293,997723,993438,997466,998260,995701,999731,991790,996386,998255,993831,997305,998346,990097,991967,990078,996187,997216,995963,996368,993053,991163\n", + "Ground Truth: 994304,997868,997586,992521,995814,994636,994954,996918,992293,997723,993438,997466,998260,995701,990866,999731,991790,996386,998255,993831,997305,998346,995873,990097,991967,990078,996360,996187,994419,998564\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 818\n", + "Returned IDs: \n", + "Ground Truth: 992909,994214,995090,997320,998808,998090,992978,994290,991479,999621,990588,994427,991883,995372,995802,998364,992692,995826,995961,995228,994775,993662,992069,991503,991873,992969,994280,999761,997913,998821\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 819\n", + "Returned IDs: 992293,995701,993848,993834,992225,998549,996657,992862,998593,992367,990811,995158,990255,998886,990211,998200,994809,994587,998878,992773,993937,998215,999853,990662,990573,990399,990097,997687,999977,994409\n", + "Ground Truth: 992293,995701,993848,993834,992225,998549,996657,992862,998593,992367,990811,995158,990255,998886,990211,998200,994809,994587,998878,992773,993937,991603,998215,999853,990662,992473,990573,990399,991630,990097\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 820\n", + "Returned IDs: 997331,998147,994113,995382,992433,993281,998770,996702,999643,991011,996494,996722,993778,993939,996067,995620,991148,998484,996219,991179,994692,997837,996080,991377,994589,994382,993936,996371,996475,995032\n", + "Ground Truth: 997331,998147,994113,995382,992433,993281,998770,996702,999643,991011,996494,996722,993778,993939,996067,996866,995620,992840,991148,998484,990527,996219,991179,994692,994984,995401,992987,997837,996080,991377\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 821\n", + "Returned IDs: 999837,992752,991633,998594,991552,991605,998279,997673,993611,998389,997151,996082,995421,998108,992864,993715,999139,997050,995705,995626,994734,996216,995232,994196,993347,996270,991469,993001,997897,996328\n", + "Ground Truth: 999837,992752,991633,998594,991242,991552,990133,991605,998279,997673,993611,998389,997151,995167,996082,995421,998108,992864,990950,993715,998716,999139,997050,995705,995626,990017,996216,994734,995232,994196\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 822\n", + "Returned IDs: 991765,993319,999798,993396,996914,997340,996535,995768,998525,990408,997714,998168,994356,999601,991779,992250,995501,992517,994841,993993,996918,999886,999052,999496,998224,991712,991850,992561,999256,992868\n", + "Ground Truth: 991765,993319,999798,993396,996914,997340,996535,995768,998525,990408,997714,998168,994356,999601,990183,991779,997925,995222,992250,993817,995501,992517,994841,992131,993993,996918,995755,990711,999886,999052\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 823\n", + "Returned IDs: 991356,992558,991977,997547,993255,990602,990226,994560,999908,993583,990894,999305,993077,997859,992113,993742,992827,993839,994472,998892,996318,995886,995552,995542,999026,990279,997918,998495,992824,994254\n", + "Ground Truth: 991356,992558,991977,997547,993255,990602,990226,994560,999908,993583,997806,990894,999305,993077,997859,992113,993742,998151,992827,993839,994472,990995,998892,995374,996318,995886,995552,995542,999026,990279\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 824\n", + "Returned IDs: 993324,991258,998272,994349,996600,993079,994765,994587,996810,995130,992223,993347,992178,996175,996777,995401,994092,994235,993548,998543,994743,995017,991255,996314,994119,996930,998950,997993,990930,990668\n", + "Ground Truth: 993324,991258,998272,994349,996600,993079,994765,994587,996810,995130,992223,995375,993347,992178,996175,996277,996777,995401,994092,994235,993548,998543,994743,995017,991255,996314,998355,994119,996930,996318\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 825\n", + "Returned IDs: 994682,998006,997666,995209,990712,990998,992209,995798,993877,997527,998122,998084,992652,992237,990091,996209,992892,998055,990194,993224,996960,999273,990226,994280,997618,995223,991382,998671,990806,997506\n", + "Ground Truth: 994682,998006,997666,995209,990712,990998,992209,995798,993877,997527,998122,998084,992652,992237,995859,990091,996209,992892,998055,990194,993224,996960,999273,990226,994280,997618,995223,991382,999441,998671\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 826\n", + "Returned IDs: 996362,993478,997020,992169,997723,992161,997918,998758,997420,997395,993030,995516,997675,995203,997693,999075,995783,996305,998593,994248,990103,999179,994409,996001,996918,994280,992124,996867,993809,996075\n", + "Ground Truth: 996362,993478,997020,992169,997723,992161,997918,998758,997420,997531,997395,993030,995516,997675,991799,995203,997693,999075,995783,996305,998593,992933,994248,990103,990279,999179,994409,996001,996918,994280\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 827\n", + "Returned IDs: 994051,993396,996899,999601,991399,994801,991052,996042,995963,996918,995342,997723,990189,994280,998123,993805,993727,996315,996127,995592,998267,993014,998778,993165,994817,998397,990103,992719,991765,997693\n", + "Ground Truth: 994051,993396,996899,999601,991399,994801,998766,997714,991052,996042,995963,996918,995342,997723,990189,994280,992569,998123,999596,993578,993805,993727,992933,996315,999820,996127,997803,995592,998267,993014\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 828\n", + "Returned IDs: \n", + "Ground Truth: 997116,996495,994142,999273,997506,998456,993139,996328,990712,995571,996408,993993,994680,990408,994196,990293,998168,999985,991667,990091,999139,996431,991633,994012,993560,990705,994296,991552,990479,990449\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 829\n", + "Returned IDs: \n", + "Ground Truth: 990310,993143,998033,991157,996840,999257,991392,995584,999122,999144,991609,991015,991840,995515,990580,999676,996829,997593,992950,992734,997379,998909,993355,997320,996007,994757,999546,992979,997037,997918\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 830\n", + "Returned IDs: 993324,995375,990930,995130,990646,994092,991712,998272,998349,997675,996277,996328,993561,996591,994743,993293,996809,994409,992790,995080,993931,993347,999533,997309,993738,995408,995560,994488,999091,993280\n", + "Ground Truth: 993324,995375,997184,990930,995130,990646,994092,991712,992933,993533,998272,998349,997522,996277,997675,996328,993561,996591,991052,994743,993293,996809,994409,992790,995080,993931,993347,999533,997309,993738\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 831\n", + "Returned IDs: 992809,998593,996001,990255,998103,993232,990314,997374,994467,997332,996657,995158,990364,993819,991212,999099,998006,998878,993931,991610,998536,999555,997009,994667,995167,990288,991453,996809,990167,990573\n", + "Ground Truth: 992809,998593,996001,990255,998103,993232,990314,997374,994467,997332,996657,995158,990364,993819,991062,991212,999099,998006,992966,998878,993931,991610,998536,999555,990786,997009,994667,995167,990288,991453\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 832\n", + "Returned IDs: 990934,997529,996219,996507,999876,990974,996328,999343,995384,991335,996075,995107,996727,998668,999898,992147,990866,998500,999144,995455,995536,999374,997381,998950,990275,992384,992291,997918,997192,993168\n", + "Ground Truth: 990934,995471,997529,996219,996507,999876,990974,996328,999343,995384,991335,996075,995107,996727,991743,990052,998668,999898,995096,992147,990866,999784,998500,999144,995455,995536,999374,991096,993335,997381\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 833\n", + "Returned IDs: \n", + "Ground Truth: 996339,998410,992582,996310,998011,997645,996220,992989,990284,992738,999240,998521,992558,992134,994819,997942,999820,995811,999400,997745,991389,996903,993296,990054,990921,994804,992428,999144,991680,992638\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 834\n", + "Returned IDs: 998002,990167,999273,992862,994640,995835,995780,993231,993560,995231,996104,990551,992752,990449,998536,995257,998861,995695,995168,998716,998959,996209,997506,997965,998787,993866,992721,998623,999555,996406\n", + "Ground Truth: 998002,990167,999273,995326,992862,994640,995835,995780,993231,993560,995231,996104,990551,992752,990449,998536,995257,998861,995695,995168,998716,998959,996209,997506,997965,998787,993866,992721,998623,999555\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 835\n", + "Returned IDs: 997301,997320,995274,998302,990231,995961,997913,999338,997470,998779,990585,990949,991498,991743,995045,993484,990957,990096,999290,992509,999710,999801,994964,999026,993431,998950,993845,990560,997775,998921\n", + "Ground Truth: 997301,997320,995274,998302,990231,995961,997913,999338,999109,997470,998779,990585,990949,991498,991743,995045,993484,990957,999533,990096,999290,992509,999710,994831,999801,999432,994964,999647,999026,993431\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 836\n", + "Returned IDs: 995515,996840,997192,998921,993143,998401,991157,999122,991848,991392,994216,991840,999613,998607,996504,999327,999053,997496,997465,991687,999257,999676,995154,994800,991015,990142,997486,998915,994922,997403\n", + "Ground Truth: 995515,996840,997192,998921,993143,993028,998401,991157,999122,991848,991392,994216,991840,999613,998607,996504,999327,999053,997496,997465,991687,998711,999257,999676,995154,994800,997591,991015,997889,999785\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 837\n", + "Returned IDs: 993544,992646,998465,999144,993941,992183,997605,998951,998170,997571,990647,996658,996961,995824,994757,990509,990478,999883,994778,990994,998011,994100,990934,995336,998210,991511,999092,995793,991379,990275\n", + "Ground Truth: 993544,992646,998465,999144,993941,992183,997605,998951,998170,997571,990647,995014,996658,996961,995824,994757,990509,990478,999883,994778,992040,990994,998011,994100,990934,995336,998210,991511,999092,995793\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 838\n", + "Returned IDs: 991459,990096,997906,998934,999384,999724,992211,999079,998623,997884,993564,990072,993342,999622,994434,998720,995402,991451,993275,999736,994906,997506,998022,994147,996164,997666,996817,993848,997685,994682\n", + "Ground Truth: 991459,990096,997906,998934,999384,992429,999724,992211,999079,998623,997884,993564,990072,993342,999622,994434,998720,993609,995402,991451,998419,993275,999736,994906,997506,998022,994147,996164,997666,996817\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 839\n", + "Returned IDs: \n", + "Ground Truth: 999216,992068,994988,995560,990786,994638,992423,996262,999806,999639,999711,990957,990167,998383,992617,994852,993504,994598,994660,990769,996809,996053,996857,999210,998292,993617,991157,998708,992979,997864\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 841\n", + "Returned IDs: 995443,994046,998097,998726,995080,998623,993143,999784,992636,999122,991999,998950,993512,990096,995806,999676,991801,992944,999210,993917,999701,998180,993260,997309,997301,995515,993866,995097,994640,997550\n", + "Ground Truth: 995443,990648,994046,998097,998726,998550,995080,998623,992933,993143,996292,999784,992636,999122,991999,998950,990257,993512,992318,990096,995806,993719,999676,994835,991801,992944,999210,993917,999701,998180\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 842\n", + "Returned IDs: 999334,993535,993715,996594,996050,998536,995746,999373,991436,995655,999711,992458,997685,996989,990293,992664,997617,995969,995385,994723,991654,990622,993917,992157,995393,996501,998330,993001,994226,999724\n", + "Ground Truth: 990670,999334,993535,993715,998468,996594,996050,998536,995746,999373,991436,995655,999711,992458,997685,996989,990293,992664,992933,997617,995969,995385,994723,991654,992110,990622,993917,992157,990942,997470\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 843\n", + "Returned IDs: 992664,994280,991000,996147,998934,998716,994682,998593,996830,997790,999852,999342,994382,993335,996153,990771,998030,990675,996809,994131,990091,991081,993715,996001,995200,995326,999922,990811,993931,993391\n", + "Ground Truth: 992664,994280,991000,996147,998934,998716,994682,996028,998593,996830,997790,999130,999852,999342,994382,993335,996153,990771,998030,990675,996809,994131,990091,991081,993715,992780,996001,995200,995326,999922\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 844\n", + "Returned IDs: 991470,996760,991257,997506,997019,998536,995168,995257,998281,992110,997265,996594,996395,992721,992793,993194,998331,990292,993715,999373,996245,993866,993936,990942,997666,991610,999129,999994,995231,990194\n", + "Ground Truth: 991470,996760,991257,997506,997019,998536,993381,995168,995257,998281,992110,997265,993076,996594,996395,995998,992721,990976,992793,998468,994374,993194,998331,998214,990292,991945,993715,999373,996245,993936\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 845\n", + "Returned IDs: 991389,993314,997103,995268,998733,998851,999279,999241,994991,993466,997185,991664,990132,991585,993751,990251,999759,994380,997411,995633,999085,992940,993467,998067,990537,993883,993209,993939,996505,992448\n", + "Ground Truth: 991389,993314,997103,995268,998733,998851,999279,999241,994991,999436,993466,997185,991664,990132,991585,992091,993751,990251,999759,994380,998536,997411,996752,995633,999085,992940,993467,998067,990537,993883\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 846\n", + "Returned IDs: 990624,997251,992483,992992,999529,996042,995464,992844,999845,996687,997344,998838,991334,997234,991161,993857,997918,994928,997601,990686,993948,993313,995763,995537,992639,993335,999392,997746,997840,994139\n", + "Ground Truth: 990624,997251,992483,992992,999529,996042,995464,992844,999845,996687,997344,999244,998838,991334,997234,991161,993857,995550,997918,994928,997601,992487,990686,993948,993313,990487,995763,995537,996220,992639\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 847\n", + "Returned IDs: 999682,991374,992740,993530,992493,997232,990733,995807,995105,997574,992677,997675,997515,993263,997634,995521,995625,998806,999961,998080,996695,997149,990243,996094,994774,993149,994677,997723,993160,994301\n", + "Ground Truth: 999682,991374,992740,993530,992493,997232,990733,995807,995105,997574,992677,997675,997515,993263,996579,997634,995521,994823,995625,998806,999961,998080,996695,997149,990243,999327,996094,994774,991061,991052\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 848\n", + "Returned IDs: 998960,992511,999374,991444,996959,998211,990469,992147,998644,999263,992416,996218,997704,998500,993250,992854,997192,998921,992546,994850,990760,997543,997301,993457,999968,993364,994673,994939,990515,990215\n", + "Ground Truth: 998960,992511,999374,991444,996959,998211,990469,992147,998644,999263,992416,996218,997704,991362,998500,993250,992854,997192,998921,992546,994850,990760,997543,993457,997301,999968,993364,994673,994939,990515\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 849\n", + "Returned IDs: 990266,993656,992161,992147,990483,992169,994761,994922,992994,997429,996494,991464,990328,997852,998500,999122,999374,992979,990957,990080,996609,998458,992637,996438,991984,999453,996093,990678,995279,996871\n", + "Ground Truth: 990266,991720,993656,992161,992147,997476,991377,990483,992169,994761,997053,994922,992994,997429,996494,991464,990328,999343,997852,992226,996721,990441,998500,995210,999122,999374,992979,990957,990080,992068\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 850\n", + "Returned IDs: 993319,999798,994465,995501,995476,992372,999886,992595,997331,991667,995768,992868,993001,997565,998006,995073,991779,996809,999430,994356,994682,996580,990194,999256,999425,990173,996328,998536,997340,992832\n", + "Ground Truth: 993319,999798,994465,995501,995476,992372,999886,992595,997331,992752,991667,995768,992868,993001,997565,998006,995073,991779,996809,992110,999430,994356,994682,996580,990194,999256,999425,990173,996328,998536\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 851\n", + "Returned IDs: 996277,992115,997168,997522,994488,990364,994574,992576,990956,999091,998102,993931,998045,992182,991977,991011,995047,993561,991712,998536,992521,992052,995184,991747,999543,995589,991459,998623,994770,996001\n", + "Ground Truth: 996277,992115,997168,997522,994353,994488,990364,994574,992576,990956,995408,999091,998102,993931,998045,992182,991977,991011,995047,993561,995326,991712,998536,992521,992052,995184,993778,991747,999543,995589\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 852\n", + "Returned IDs: 992617,992692,999806,992228,997320,995820,991137,992330,998255,991015,992927,992521,991157,996903,997605,994954,996592,999109,992124,999639,999711,996262,998075,999731,999144,994757,994577,993845,995372,995640\n", + "Ground Truth: 992617,992692,999806,992228,997320,995820,991137,992330,998255,991015,992927,992521,991157,996903,997605,995536,994954,996592,999109,992124,999639,999711,992277,995374,996262,998075,999731,999144,994757,994577\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 853\n", + "Returned IDs: 996695,990319,996813,992801,997742,997075,999646,995493,995521,996607,995146,996472,997077,999475,995450,996762,997675,992952,995423,992605,994554,995702,993385,993899,994835,995891,991699,998910,997975,996112\n", + "Ground Truth: 996695,990319,996813,992801,997742,997075,999646,995493,995521,996607,995146,996472,994823,997077,999475,995450,996762,997675,992952,995423,992605,994554,995702,993385,993899,994835,995891,991699,998910,997975\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 854\n", + "Returned IDs: 997383,999915,996001,998097,992395,995873,996685,995476,993053,994742,992792,995073,997320,990930,999099,992568,996809,999560,998779,999399,997814,992318,991007,995216,999453,998725,991245,993342,996328,994793\n", + "Ground Truth: 997383,999915,996001,991175,998097,992395,995873,996685,995476,993053,994742,992792,995073,997320,990930,999099,992568,996809,999560,998779,999399,997814,992318,991007,996043,995216,999453,998725,991245,993342\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 855\n", + "Returned IDs: 994080,994760,992558,991977,992170,994119,992718,990776,990487,994254,996633,996809,992052,995503,995159,998243,990908,994334,998331,990345,998372,999179,996328,996391,993839,998370,990998,992615,993500,997309\n", + "Ground Truth: 994080,994760,992558,991977,992170,994119,992718,990776,990487,994254,996633,996809,995179,992052,998686,998480,995503,994956,995159,998243,995491,990908,994334,998331,990345,998372,999179,996328,996391,993839\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 856\n", + "Returned IDs: 993289,993845,993637,990438,993301,999338,992825,993719,994081,998021,995401,990855,992176,993715,998726,998536,996918,997002,990289,997918,990096,998950,999724,996328,999791,999516,995932,997970,999453,992318\n", + "Ground Truth: 993289,992933,993845,993637,990438,993301,999338,992825,993719,994081,998021,995401,990855,999698,992176,993715,998726,998536,996918,995080,997002,990289,997918,990096,998950,991758,999533,999724,996328,999791\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 857\n", + "Returned IDs: 990275,996053,998713,995260,996554,991018,990231,992363,997371,990366,992228,991479,999243,995755,992931,997913,992453,993387,997508,999938,992087,994266,992942,999144,998090,992370,994100,991379,997880,997192\n", + "Ground Truth: 990275,997439,996053,998713,995260,996554,994618,991018,990231,992363,990650,997371,990366,997451,992228,991479,997432,999243,995755,993048,996219,995210,998744,992931,997913,992453,993387,997508,999938,992087\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 858\n", + "Returned IDs: 995679,994140,992448,994409,990081,999099,993813,997868,991585,990751,995702,991790,996042,998633,993561,992858,997832,997289,995216,991125,996169,993831,997822,996719,990576,996368,996038,998941,994419,997046\n", + "Ground Truth: 995679,994140,992448,994409,990081,999099,993813,997868,991585,990751,995702,991790,991603,996042,998633,993412,991024,993561,992858,997832,997289,995216,991125,996169,993831,997822,996719,990576,996368,996038\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 859\n", + "Returned IDs: 991618,998075,996987,997998,992448,994954,991117,998403,999421,996187,991357,998979,996621,992521,997176,998255,998910,997478,999121,999800,994106,998162,995813,991253,996169,990341,997909,992763,993672,999731\n", + "Ground Truth: 991618,998075,996987,997998,992448,994954,991117,998403,994773,999421,996187,991357,998979,996621,992521,997176,998255,998910,997478,999121,999800,994106,998162,995813,991253,996169,990341,997909,992763,993738\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 860\n", + "Returned IDs: 990275,993853,991260,992384,992452,999647,997976,992969,998899,994992,991893,993168,998514,990925,996727,997571,996714,990957,992994,999089,995034,992911,990505,995279,995755,995336,992546,994917,998951,995793\n", + "Ground Truth: 990275,996336,993853,991260,992384,992452,999647,995014,997976,994776,991829,992969,998899,994992,994192,990724,991893,990862,990893,993168,998514,990925,997022,996727,997571,992909,996714,990356,990957,992994\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 861\n", + "Returned IDs: 998302,994382,997937,991790,995204,990560,997320,997301,992862,995981,998294,994964,995949,997913,998779,990252,998807,996391,998495,995895,997969,995755,990096,995976,991696,997822,990224,995886,995274,998776\n", + "Ground Truth: 998302,994382,999755,997937,991790,995204,991923,994707,990560,995672,997320,993412,997301,995702,992862,995981,991995,998294,994964,995949,997913,998779,990252,998807,996391,998495,995895,997969,995755,990096\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 862\n", + "Returned IDs: 996219,997913,990843,997301,994266,993662,998495,995942,990958,999542,999374,998540,995927,990776,997420,995802,995949,997320,998921,994964,991650,999331,994850,995260,997470,994277,998550,999908,998758,990366\n", + "Ground Truth: 996219,997913,990843,997301,991085,994266,993049,993662,998495,992946,995942,990958,999533,998380,997741,999542,999374,998540,995927,990776,997420,995802,995949,997320,998921,994964,991650,999331,994850,997855\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 863\n", + "Returned IDs: 999912,992337,998115,994972,990366,997453,999544,999876,999949,996052,990179,995107,998122,990585,990089,992047,999791,992907,995716,993139,999711,990043,995818,998349,996001,999898,997796,993484,995372,998849\n", + "Ground Truth: 999912,992337,998115,994972,990366,997453,999544,999876,999949,996052,990179,995107,998122,990585,995222,990089,992047,999791,996947,992907,995716,993139,999711,990043,995818,998349,996001,999898,997301,997796\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 864\n", + "Returned IDs: 992069,998779,998950,997918,996262,995694,994360,990949,993781,993504,993071,994450,993260,997002,995602,990957,997320,993492,999639,997640,991677,998170,993484,996140,994964,995560,996001,996196,999453,994742\n", + "Ground Truth: 992069,998779,998950,997918,996262,995694,994360,990949,994334,993781,993504,993071,994449,994450,993260,997002,995602,990957,997320,990930,993492,997739,999639,997640,991677,998170,993484,995826,996140,992532\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 865\n", + "Returned IDs: 995550,996455,991870,992973,991990,994175,995239,992981,993357,999775,991769,996328,992239,992784,999399,990956,995699,991111,992450,995159,997244,995637,999660,997153,993991,999924,997041,992713,994315,995184\n", + "Ground Truth: 995550,996455,991870,992973,991990,994175,995239,993980,992981,993357,999775,991769,996328,992784,992239,999399,990956,995699,991111,992450,995159,997244,995637,999660,997153,993991,999924,997041,993006,992713\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 866\n", + "Returned IDs: 998558,994679,994098,995691,996882,999435,998993,990796,998652,994193,990765,999171,994605,992056,994080,995618,990852,993718,996428,993267,997864,990487,996413,999158,998855,996930,993976,991377,990998,993500\n", + "Ground Truth: 998558,994679,994098,995691,996882,999435,998993,990796,998652,994193,990765,991541,999171,994605,992056,994080,995618,990852,993718,996428,993267,994689,997864,990487,996413,990133,999158,998855,996930,990189\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 867\n", + "Returned IDs: 999805,996867,998162,994409,997844,993767,997868,994730,990631,995702,996786,992582,997031,997723,995052,994954,993209,995355,998181,993649,999459,991508,998962,993856,994742,990435,996038,997544,992448,999787\n", + "Ground Truth: 999805,996867,998162,994409,997844,998943,993767,993466,997868,990361,994730,990631,995702,994060,995766,996633,996786,992582,997031,997723,995052,994954,993209,992063,995355,998181,993649,999459,991603,991508\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 868\n", + "Returned IDs: 997554,994793,998060,990845,995216,993422,990081,992792,991585,995575,999099,996147,993841,996657,996001,990662,990255,996368,999402,994106,993344,990314,995679,990211,998878,993804,993032,998593,994013,990751\n", + "Ground Truth: 997554,994793,998060,990845,995216,993422,990081,992792,991585,995575,999099,996147,993841,996657,996001,994764,990662,990255,995963,996368,999402,994106,993344,990314,995679,990211,998878,993804,993032,998593\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 869\n", + "Returned IDs: 996411,995482,997374,993466,995020,990435,999627,998156,990293,999599,994142,993209,994730,991439,992024,998221,994802,999179,995274,995633,998067,990758,998913,995289,994817,993747,996867,999833,993725,994541\n", + "Ground Truth: 996411,995482,997374,993466,995020,995002,990435,999627,998156,990293,999599,994142,993209,999785,994730,991439,992024,998221,991480,994802,998586,999179,995274,995633,998067,991075,995401,990758,998913,995374\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 870\n", + "Returned IDs: 997730,994409,998941,996077,993561,998697,997868,997832,996368,994636,992858,992169,991958,997723,993717,996867,991530,993813,998186,995894,995702,999343,993143,996042,990634,995679,998545,995701,998382,993882\n", + "Ground Truth: 997730,994409,998941,996077,993561,998697,997868,997832,996368,997630,994636,992858,992169,991958,997723,993717,996867,991530,993813,998186,995894,992268,995702,999343,993143,996042,994107,990634,990399,995679\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 871\n", + "Returned IDs: 999851,995336,997320,999883,997371,994415,996143,993364,997571,999982,997842,990231,995372,991782,992027,996516,999263,995841,999810,994164,998921,998170,995865,991379,991662,997913,996504,995515,994854,990958\n", + "Ground Truth: 999851,995336,997320,999883,997371,994415,996143,993364,997571,998344,999982,997842,990231,995372,991782,992027,996516,999263,995841,999810,994164,998921,998170,995865,991379,991662,995422,992789,997913,996504\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 872\n", + "Returned IDs: 996362,995783,994313,996087,992997,993497,997746,995184,990872,997997,995464,991285,994280,997244,990021,994196,996911,991111,999759,996276,990103,997713,993256,993478,993621,991981,994541,995696,992526,998201\n", + "Ground Truth: 996362,995783,994313,996087,992997,993497,997746,995184,990872,990149,997997,995464,991285,994280,990343,997244,990021,994196,999893,996911,991111,999759,996276,990103,997713,993256,993478,993621,991981,992784\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 873\n", + "Returned IDs: 998382,996371,993561,999068,995289,997386,995401,995200,990137,996219,996042,992068,993433,999367,995535,994307,994577,996769,997406,991650,991996,998045,998495,995923,996805,990435,990096,992944,999780,998039\n", + "Ground Truth: 998382,996371,993561,999068,995289,997386,995401,995200,990137,996219,991179,996042,992068,993433,995535,999367,994307,994577,996769,997406,991650,991996,998045,998495,999598,995923,991921,996805,990435,990096\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 874\n", + "Returned IDs: 991255,997914,997542,993738,997918,990814,998697,992979,994552,991447,994280,996328,994163,993256,996305,993079,997251,997555,990776,998860,996848,996721,998123,993976,995401,999820,994040,993062,995559,990930\n", + "Ground Truth: 991255,997914,997542,993738,997918,991948,990814,998697,995281,998567,992979,994552,992012,998097,991447,998427,994280,994972,993143,996328,994163,993256,996305,993079,995600,997251,997806,997555,990776,998860\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 875\n", + "Returned IDs: 998586,998790,994537,996395,991610,992136,997009,990912,991667,990167,991571,993995,990194,999216,996209,992721,994643,996809,991513,991332,991975,990991,994451,991608,996960,999555,990786,997145,991139,997328\n", + "Ground Truth: 995922,998586,998790,994537,996395,991624,993381,991610,996424,990429,992136,997009,990912,991667,990167,991571,993995,990194,999216,994981,996209,992721,990450,994643,996809,993194,991513,991332,991975,990991\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 876\n", + "Returned IDs: 998921,995372,990585,997286,997320,998721,996318,991503,995841,999338,999526,997984,990083,997113,998445,991348,991498,995515,993018,994618,995659,995375,997301,999698,998950,996653,998302,991545,996516,997470\n", + "Ground Truth: 998921,995372,990585,997286,997320,998721,996318,991503,995841,999338,999526,997984,990083,997113,998445,991348,991498,995515,993018,994618,995659,995375,997301,999698,998950,996653,999019,998302,991545,996516\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 877\n", + "Returned IDs: 997320,994964,997920,992331,992744,995357,994873,995274,993168,998764,999437,995336,998228,995438,998365,991776,990210,999740,997178,990535,996219,997330,992692,993484,997028,999344,992942,994432,991883,996001\n", + "Ground Truth: 990269,997320,994964,997920,992331,992744,999159,995357,994873,998470,994225,995274,993168,998764,999437,995336,991433,993066,998228,999340,995438,993909,998365,991776,990210,997178,999740,994584,990535,996219\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 878\n", + "Returned IDs: 995955,998242,997215,990125,994305,995503,992306,992824,991333,995804,990538,992170,990721,990192,991034,993140,991834,998260,999908,992490,992113,997849,997547,999430,999639,996742,996510,995724,991977,997055\n", + "Ground Truth: 995955,998242,997215,990125,994305,995503,992306,992824,991333,995804,990538,992170,990721,990192,991034,993140,996629,991834,998260,999908,992490,992113,997849,997547,999430,999639,996742,994155,996510,995724\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 879\n", + "Returned IDs: 997529,993725,994730,998710,994742,991185,995633,994830,999377,992424,993166,990887,998106,993561,993053,995289,998913,999740,997167,994409,996411,993920,996328,993209,994013,994419,996867,992230,996132,995401\n", + "Ground Truth: 997529,993725,994730,998710,994742,991185,995633,994830,999377,992424,993166,990887,998106,993561,993053,995289,998913,999740,990681,997167,994409,996411,993920,996328,992260,993209,994013,994419,996867,992230\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 880\n", + "Returned IDs: 992411,995552,992558,993920,995309,991851,998652,999206,996953,990843,994418,993474,996878,993077,995618,999305,991337,990279,996140,990345,994817,998950,999540,993742,990657,993456,999908,990989,993534,990308\n", + "Ground Truth: 992411,995552,992558,993920,995309,991851,998652,999206,996953,990843,994418,993474,996878,993077,995618,999305,991337,990279,996140,990345,994817,998950,999540,992966,993742,991263,990657,993456,999908,990989\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 882\n", + "Returned IDs: 992032,992293,991076,991203,991954,996028,990662,993561,990811,993725,998476,993857,994835,998823,997529,991958,999286,990097,992636,998197,991301,999922,998593,997554,995701,997822,992605,998105,995216,997079\n", + "Ground Truth: 992032,992293,991076,991203,996225,991954,996028,990662,993561,990811,993725,991087,998476,993857,994835,998823,991603,997529,991958,999286,990399,990097,990033,992636,994677,991771,998197,991301,991466,999922\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 883\n", + "Returned IDs: 997362,990958,996504,993813,997192,990925,996316,999789,998692,995755,999483,998294,990118,997984,997371,995090,997320,999327,993518,991958,995132,995985,995367,998649,995884,997620,991332,998303,996785,992994\n", + "Ground Truth: 997362,990958,996504,993813,997192,990925,996316,999789,998692,995755,992204,999483,998294,990118,997984,997371,995090,997320,999327,993518,991958,995132,995985,991009,995367,998649,997911,997889,993902,993982\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 884\n", + "Returned IDs: 992809,991767,993232,998565,998623,996797,994643,998536,997019,993995,994839,993727,994901,999555,998787,995695,992110,993366,990255,998006,996395,994844,993306,997502,991257,993715,991228,990741,994374,992926\n", + "Ground Truth: 992809,991767,993232,999498,998565,998623,994274,996797,994643,998536,997019,993995,994839,993727,994901,999663,999555,998787,995695,992110,993366,990255,998006,996395,994844,993306,997502,991257,999458,993715\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 885\n", + "Returned IDs: 999869,994449,993260,995261,992748,992041,995336,996296,990532,994955,998951,994100,997498,997113,999698,993512,998550,997320,993529,990772,997918,996592,996075,991951,994171,992880,998090,990949,998246,996210\n", + "Ground Truth: 990778,999869,994449,993260,995261,992748,992041,995336,996296,990532,992933,994955,998951,994100,997498,999019,999351,997113,990519,999698,993512,998550,997320,993529,990772,997918,996592,996075,991951,994171\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 886\n", + "Returned IDs: 994463,992703,991734,994444,994424,991685,995126,995678,995945,995421,990520,990899,990255,992322,996062,998593,999491,991857,998250,990382,991789,994510,998039,993931,997453,996001,993289,992182,991093,993715\n", + "Ground Truth: 994463,992703,991734,994444,994424,991685,995126,995678,995945,995421,990520,990899,990255,992322,996062,998593,999491,991857,990133,998250,990382,991789,994510,998039,993931,997453,996001,993289,992182,991093\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 887\n", + "Returned IDs: 991432,995955,995562,999639,997775,992435,996065,998260,990721,991157,994305,998142,994163,997739,995618,995983,996510,992824,997614,990413,996884,990538,997420,991817,992095,991923,990192,991015,997055,992306\n", + "Ground Truth: 995156,991432,995955,995562,999639,997032,997775,992435,996065,998260,990721,991157,994305,998142,996511,990210,994163,996592,997739,992497,995618,992350,995983,996283,991801,996510,992824,997614,990413,996884\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 888\n", + "Returned IDs: 996759,999977,997002,996810,999587,995401,999595,991666,997554,998593,998779,998530,998123,992790,992884,997910,994819,999026,993032,994474,997012,991968,996657,990399,996088,996001,993289,996864,994946,994409\n", + "Ground Truth: 990304,996759,999977,997002,996810,992637,999587,995401,999595,991666,997554,998593,994178,998779,998530,998123,992790,992884,997910,994819,999026,993032,994474,997012,991968,996657,993820,990399,996088,995374\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 889\n", + "Returned IDs: 995159,990355,998507,994929,996211,994541,996413,997746,998755,997722,996344,996276,992489,990931,994094,997290,992445,994336,992313,994913,996430,993621,997814,998174,992052,997420,992832,999179,992981,992934\n", + "Ground Truth: 995159,990355,998507,994929,996211,994541,996413,997746,998755,997722,996344,996276,992489,990931,994094,997879,997290,992445,994336,992313,999974,994913,996430,993621,997814,998174,992052,997420,992832,999179\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 890\n", + "Returned IDs: 999547,997723,992796,996867,994730,996633,990631,993209,995702,997868,998162,994380,995958,998067,995907,999805,999402,993466,994409,996042,992448,997411,991301,992959,991664,992051,992599,999241,994945,996109\n", + "Ground Truth: 999547,997723,992796,996867,994730,996633,990631,993209,995702,995470,997868,998162,994380,995958,998067,995907,999805,999402,993466,994409,992091,996042,991943,992448,997411,992024,991301,992959,991664,992051\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 891\n", + "Returned IDs: 990957,993283,992668,995752,991677,999561,996894,994852,999806,995974,998600,995580,990949,995562,999714,995443,991801,992309,993781,998495,999898,993222,994979,995519,997903,991923,992124,992423,997864,990383\n", + "Ground Truth: 990957,993283,992668,995752,991677,999561,996894,994852,999806,995974,998600,995580,998232,998455,990949,995562,999714,995443,991801,992309,993781,998495,999898,993222,997032,994979,995519,997903,991923,992124\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 892\n", + "Returned IDs: 994742,999377,990435,994730,990096,993053,999279,998495,999402,995633,991185,990930,993642,998980,993623,992680,994759,992024,995289,990758,996001,993920,993725,999302,994830,990012,993209,996809,995435,999009\n", + "Ground Truth: 994742,999377,990435,994730,990096,993053,999279,998495,999402,995633,991185,990930,993642,998980,993623,992680,994759,998188,995198,992024,995289,990758,996001,997723,993920,993725,999302,994830,990012,995873\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 893\n", + "Returned IDs: 992748,995203,994081,997320,990083,996075,993809,999453,990949,991984,991002,992880,998697,998741,998951,995596,995250,993781,990532,995016,999317,998090,999639,990697,999698,991609,998232,999701,990339,999076\n", + "Ground Truth: 992748,995203,994081,997320,992497,990083,996075,993809,990778,999453,990949,991984,991002,992880,998697,998741,999338,998951,995596,995250,993781,995398,990532,995016,999317,998090,999639,990697,999698,991609\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 894\n", + "Returned IDs: 994336,997309,993877,994280,993775,995861,996328,991378,994119,994956,991334,992147,993335,998123,997822,998950,996855,995438,999533,995125,993929,996050,991007,997918,994964,992052,996521,998495,997002,998779\n", + "Ground Truth: 994336,997309,993877,994280,993775,992933,995861,996328,991378,994119,994956,999338,991334,991720,998076,992147,990998,993335,998123,997822,997159,998950,996855,995438,999533,992789,995125,998369,993929,996050\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 895\n", + "Returned IDs: 995964,991430,997495,998417,991923,994126,997864,994543,995138,996163,990776,997407,999716,990894,999826,995510,994165,993572,993828,998142,996882,994163,994904,995580,994213,990908,990930,999171,995986,998061\n", + "Ground Truth: 995964,991430,997495,998417,991923,994126,997864,994543,995138,996163,990776,997407,999716,990894,999826,995510,994165,993572,993828,998142,992280,996882,994163,994904,995580,994213,990908,996216,990930,998904\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 896\n", + "Returned IDs: 991924,996328,998279,996362,991111,994742,991885,994496,998067,990854,996816,996728,991981,995735,993621,993450,994806,996495,992450,998251,992713,998256,998671,991428,994428,996189,991769,999852,995831,999812\n", + "Ground Truth: 991924,996328,998279,996362,991111,994742,991885,994496,998067,990854,996816,996728,995735,991981,993621,993450,994806,996495,992450,998251,992713,998256,998671,991428,994428,996189,991769,995315,999852,995831\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 897\n", + "Returned IDs: 997475,997652,991153,991075,997480,994891,993621,993611,990696,999496,990600,992480,994765,993055,994558,995600,997714,999205,998405,993165,992449,990776,996429,994307,994280,991723,998072,990097,992327,993004\n", + "Ground Truth: 997475,997652,991153,991075,997480,994891,993475,993621,993611,990696,999496,990600,992480,994765,991426,993055,995145,994558,995600,994313,997714,999205,998405,993165,992449,996856,990776,996429,994051,994307\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 898\n", + "Returned IDs: \n", + "Ground Truth: 991593,995260,992432,999374,991999,994922,990278,998734,997620,998899,996351,990545,994497,991352,991889,999676,997985,990958,991172,999810,993885,992203,997591,994776,990760,990026,991406,991720,994266,999658\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 899\n", + "Returned IDs: 997992,993504,995918,991595,997790,999735,996067,999811,992082,996082,990494,992747,993737,995167,993024,991212,994134,993071,990434,992864,993611,991741,996196,996599,999093,991425,998732,990196,992247,994537\n", + "Ground Truth: 997992,993504,995918,991595,997790,999735,996067,999811,996986,992082,993609,996082,990494,992747,993737,995167,993024,991212,992665,994134,993071,990434,992864,993611,991741,994815,996196,996599,994651,999093\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 900\n", + "Returned IDs: 995193,996581,997998,998527,997176,998255,991769,999047,992311,998075,995397,992521,996277,995640,995661,991922,991981,995820,993379,992097,998670,994742,996728,994954,998910,997612,998637,997091,990712,999208\n", + "Ground Truth: 995193,996581,997998,998527,997176,998255,991769,999047,992311,998075,995397,994773,992521,996277,995640,995661,991922,991981,995820,993379,992097,998670,994742,994690,996728,991057,994954,998910,997612,998637\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 901\n", + "Returned IDs: 995986,992718,992258,994163,997689,998694,999359,996163,991637,996926,995301,994165,993463,992645,996669,992558,997864,998079,997692,996214,991430,999432,995436,990367,997558,997538,999826,991977,996620,996127\n", + "Ground Truth: 995986,992718,992258,994163,994072,997689,998694,999908,999359,996163,991637,996926,997395,995301,994165,994195,993463,992645,991172,998758,996669,992558,997864,998079,997692,996214,991430,999432,999385,993609\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 902\n", + "Returned IDs: 992289,995948,995289,995030,995633,996867,995190,998196,993261,997319,997630,991810,996411,997947,999957,993433,999375,994110,990516,998913,995401,990814,992364,996302,998765,994204,992827,994283,994783,995078\n", + "Ground Truth: 992289,995948,995289,995030,995633,996867,995190,998196,993261,997319,998557,995494,997797,997630,991810,996411,997947,999957,991447,993433,999375,994110,990516,998913,995401,990814,992364,996302,998765,994204\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 903\n", + "Returned IDs: \n", + "Ground Truth: 999426,991984,990142,999769,993257,991013,998753,994971,997551,993785,992328,994489,998498,994789,996075,993582,991218,992635,995515,998734,990648,997584,995033,997255,996631,995108,993675,996829,996840,998227\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 904\n", + "Returned IDs: 992827,998779,994382,993744,999788,995300,995200,994595,990722,998543,997529,994682,998765,990934,996219,991590,995401,994742,991885,995861,990814,994783,999138,990701,998862,995633,999144,997918,998055,998671\n", + "Ground Truth: 992827,998779,994382,993744,999788,995300,992748,999916,995200,994595,990722,998543,997529,994682,991334,998765,997348,990934,996219,993856,993412,991590,995401,994742,991885,995374,995861,990814,997933,991986\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 905\n", + "Returned IDs: 995113,994191,991962,993931,992771,996062,998593,999543,996001,994280,997675,995168,997453,990407,996733,995536,992437,999984,990802,998723,997506,998250,994463,999091,995017,999994,996028,996505,995421,994424\n", + "Ground Truth: 995113,994191,991962,993931,992771,996062,998593,999543,995401,996001,994280,997675,992793,995168,997453,990407,996733,996071,995536,999461,992437,990998,999984,990802,998723,997506,998250,994463,997732,999091\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 906\n", + "Returned IDs: 996890,994641,993518,992709,992107,997709,990595,995353,994630,995949,994192,993734,997535,993005,994939,990863,995367,997371,990397,995090,997972,994281,997525,996961,994757,995910,998378,996785,992511,997880\n", + "Ground Truth: 996890,994641,993518,992709,992107,997709,990595,995353,994630,995949,994192,993734,997535,993005,999112,994939,990863,992004,995367,997371,990397,995090,997972,994281,997525,996961,994757,995910,994953,998378\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 907\n", + "Returned IDs: \n", + "Ground Truth: 995802,994778,999710,993492,992442,997006,991009,992443,999982,993504,997483,993852,993048,995826,992309,997913,992119,994670,992931,992228,994831,990231,995161,997605,990957,996993,997579,992066,990890,996785\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 908\n", + "Returned IDs: 992020,993570,998551,996570,998492,998936,994922,995791,991806,994777,998983,997789,998734,997474,991405,990334,992138,992186,999453,999972,994438,995004,992146,994318,990043,991198,994089,990652,992944,992360\n", + "Ground Truth: 992020,993570,998551,996570,998492,998936,994922,995791,991806,990155,994777,998983,997789,998734,997474,991405,990334,992138,992186,999892,999453,998232,999972,994438,995004,992146,994318,990043,990849,990490\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 909\n", + "Returned IDs: 998303,998377,997926,995025,991332,999032,992866,999265,995894,992241,993489,994906,993260,998697,991630,991691,991295,996217,999840,995740,995203,995672,991530,993504,996001,998401,999977,996725,992163,997653\n", + "Ground Truth: 998303,998377,997926,995025,991332,999032,992866,999265,995894,992241,993489,994906,993260,998697,991630,991691,991295,996217,999840,995740,995203,995672,991530,993504,997531,996001,998401,999977,996725,992163\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 911\n", + "Returned IDs: 994783,990340,990852,999126,990126,991330,992827,995633,992566,992598,998913,993725,994878,990985,990069,996809,996302,995861,995289,999957,997733,994742,998804,994541,991464,999372,990929,991418,999740,996411\n", + "Ground Truth: 994783,990340,990852,999126,990126,991330,992827,995633,992566,992598,998913,993725,994878,990985,990069,996809,996302,995861,995289,999957,997733,994742,994317,998804,994541,991464,994751,999372,990929,991418\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 912\n", + "Returned IDs: 999079,990096,999831,996918,999771,997906,998849,997305,999367,994955,992052,992944,996001,995203,995737,994718,995932,995536,998495,996124,997320,994643,996915,994661,991498,994901,999711,992407,995372,998302\n", + "Ground Truth: 999079,990096,999831,996918,999771,997906,998849,997305,999367,994955,992052,992944,996001,995203,995737,994718,994226,995932,995536,998495,996124,995332,994081,997320,994643,996915,995401,994661,991498,994901\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 913\n", + "Returned IDs: 996868,996001,993174,998934,992072,995701,998593,995216,992610,994419,994610,995823,998186,998710,996147,998349,995855,997897,991958,999590,994280,996075,997687,998382,993280,996368,998039,997320,997586,993293\n", + "Ground Truth: 996868,996001,993174,998934,992072,995701,997920,998593,995216,992610,994419,994610,995823,998186,998710,996147,998349,995855,996401,991000,997897,991958,999590,993738,994280,996075,997687,993559,998382,990557\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 914\n", + "Returned IDs: 991527,992448,991969,998205,997868,991795,990845,996987,996368,998278,999250,991220,996187,997909,992521,995864,993537,996256,997176,993003,999453,994409,995813,991892,990603,994106,994140,993813,997727,998910\n", + "Ground Truth: 991527,992448,991969,998205,997868,991795,990845,999866,996987,996368,998278,997216,999250,991220,996187,997909,992521,995864,993537,996256,997176,993003,993412,990341,999453,994409,995813,991892,990603,994106\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 915\n", + "Returned IDs: 992515,997215,999426,993125,990697,993447,991503,995372,994886,996805,997914,996075,997550,997584,998498,991070,993721,990538,992843,993341,995459,991426,991409,997775,994233,991272,992661,990192,998950,998933\n", + "Ground Truth: 992515,997215,999426,993125,997448,994626,990697,993447,991503,995372,994886,996805,997914,995398,996075,997550,997584,998498,991070,993721,990538,992843,993341,995459,991426,991409,997775,995622,994233,991272\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 916\n", + "Returned IDs: 994673,992833,997192,993648,998458,991585,999328,991361,992284,991743,991534,995702,996721,992994,999263,996532,997880,999374,992147,991213,990432,991958,997913,992384,992922,992087,994383,990958,994919,993402\n", + "Ground Truth: 994673,992833,994160,991900,997192,990724,991782,997826,993648,993412,998458,991585,999328,996111,991361,990760,992284,991743,991534,990462,995702,996721,992994,999263,996532,991602,997880,999374,992147,998215\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 917\n", + "Returned IDs: 994013,994203,995544,992098,996898,998060,993377,991057,999689,997059,998162,994412,991507,999023,995216,993217,996328,992000,994409,994488,996001,999402,998045,997554,996765,995766,997466,996322,997566,997020\n", + "Ground Truth: 994013,994203,995544,992098,996898,998060,993377,991057,999689,997059,998162,994412,991507,999023,995216,993217,996328,992000,994409,994488,996001,999402,998045,997554,991255,996765,995766,997466,996322,997566\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 918\n", + "Returned IDs: 991075,999472,990315,992156,995455,995633,995613,998017,996411,994783,995289,994431,995131,999064,999500,994558,998105,998982,994130,997079,992449,991315,993725,992079,995002,995831,999302,994307,999922,999356\n", + "Ground Truth: 991075,999472,990315,992156,997798,996275,995455,995633,995613,998017,996411,995600,994783,994879,995289,994431,995131,999064,999500,994558,998105,998982,994130,999949,997079,992449,991315,993725,992079,995002\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 919\n", + "Returned IDs: 994342,992780,998631,994577,998109,995776,992070,998568,994984,999806,997331,999780,996219,994558,999430,998495,997909,995895,997480,992628,992423,992655,997529,993053,996216,999294,993617,992617,990435,995476\n", + "Ground Truth: 994342,992780,998631,994577,998109,996475,995776,992068,992070,990271,998568,994984,999806,997331,999780,996219,993055,997990,994558,999430,998495,997909,995895,997116,997554,997480,992628,992423,993678,992655\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 920\n", + "Returned IDs: 993727,990391,998175,997693,997942,992809,998507,994407,991052,993980,998537,999773,992295,996098,993453,990297,996918,993155,995048,991054,998481,990479,994051,995790,994280,999105,997806,998267,994187,996395\n", + "Ground Truth: 993727,990391,998175,997693,998430,997942,992809,998507,994407,991052,993980,998537,999773,992295,996098,993453,990297,996918,993155,995048,991054,998481,990479,994051,995790,994280,999105,997429,997806,998267\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 921\n", + "Returned IDs: 997261,994389,993357,990610,996591,994435,993079,997168,998049,996245,999438,997393,991850,993280,993738,993715,990810,993931,992239,994040,991551,997494,990264,993293,992832,998891,995059,996277,996505,994076\n", + "Ground Truth: 997261,994389,993357,991904,990610,996591,994435,995361,993079,997168,998049,996245,999438,997393,991850,993280,993738,990769,995359,993715,990810,993931,992239,994040,991551,997494,990264,993293,992832,998891\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 922\n", + "Returned IDs: 996840,996686,995495,998582,998523,995515,997486,991392,993091,998244,999524,994195,990026,998692,995004,991485,999453,993143,991462,990300,992309,999613,998996,996351,992328,998837,991149,995588,991889,994922\n", + "Ground Truth: 996840,996686,995495,998582,998523,995515,993028,997486,991392,993091,998244,999524,994195,990026,998692,995004,991485,999453,993143,991462,990300,992309,999613,998996,996351,995401,992328,990702,998837,991149\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 923\n", + "Returned IDs: 995289,996610,997787,995182,997406,991603,993561,996769,997630,996388,995497,992125,998165,999080,994783,997822,997386,996042,991954,997058,996080,994742,995200,996473,991923,994396,993725,991958,993844,990984\n", + "Ground Truth: 995289,996610,997787,995182,997406,991603,993561,996769,997630,995497,996388,992125,998165,999080,994783,997822,997386,996042,991954,997058,996080,994742,995200,998999,996473,991923,994396,993725,991958,993412\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 924\n", + "Returned IDs: 998996,999453,996485,990965,995850,996419,998001,990043,995515,991485,991407,998837,997486,993237,992835,991131,994439,990827,997219,990858,998244,991157,993282,994800,996840,992944,998937,998782,994216,995372\n", + "Ground Truth: 998996,999453,996485,990965,995850,996419,998001,990043,995515,991485,991407,998837,997486,993237,992835,991131,994439,990827,997219,998236,996652,990858,998244,991157,993282,994800,998116,996840,992944,999671\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 926\n", + "Returned IDs: 994830,996132,994204,990340,990711,994742,994783,995633,998913,990012,998710,997319,993053,998196,996411,991330,999377,995289,994529,992289,996302,993433,996049,995401,997529,995200,996809,990435,993166,997797\n", + "Ground Truth: 994830,996132,994204,990340,990711,994742,994783,995633,998913,990012,998710,997319,993053,998196,996411,991330,999377,995289,994529,992289,996302,993433,996049,995401,997529,995948,995200,996809,990435,993166\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 927\n", + "Returned IDs: 997513,991667,998335,992778,991610,990330,997145,990177,994467,994996,995655,992517,992561,998781,992721,990437,997009,991470,990293,992209,996134,996501,990292,990194,998720,997242,997328,991569,993381,997666\n", + "Ground Truth: 997513,991667,998335,992778,991610,990330,997145,990177,994467,992136,994996,995655,992517,992561,998781,992721,990437,997009,991470,991038,990293,992573,992209,996134,996501,990292,990194,996395,998720,997242\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 928\n", + "Returned IDs: 991037,991212,991227,993855,999510,991956,994221,996213,992315,997897,993504,995562,995213,998250,994510,992819,992168,990596,995500,996884,995167,992176,994302,991648,991425,994743,999139,990409,994101,991245\n", + "Ground Truth: 991037,991212,991227,993855,999510,991956,994221,996213,992315,997897,993504,995562,995213,998250,994510,992819,992168,996677,994468,990596,995500,991596,996884,995167,992176,994302,991648,991425,995421,993981\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 929\n", + "Returned IDs: 993209,995289,995633,996815,994529,995766,995401,997262,998913,991507,991958,994742,995268,995778,992488,993831,998165,998476,991954,993997,996302,999957,990126,993219,996867,994783,998196,996388,994013,993586\n", + "Ground Truth: 993209,995289,995198,995633,996815,996930,994529,995766,995401,997262,998913,997797,991507,991958,994742,999587,995268,996411,995778,990157,992488,993831,998165,998476,991954,993997,996302,999957,990126,993219\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 930\n", + "Returned IDs: 993426,998921,990898,996516,997984,997320,998445,995515,998721,996135,994415,997292,997192,997301,994757,993253,998294,993419,992451,997470,993484,995306,997571,997291,998246,991720,995841,999327,995372,992942\n", + "Ground Truth: 993426,998921,990898,996516,997984,997320,997873,998445,995515,998721,996135,994415,999036,997292,997192,997301,994757,993253,995401,998294,993419,996560,992451,997470,993484,998741,995306,997571,997291,998246\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 931\n", + "Returned IDs: 995299,990074,998803,993413,992456,996183,998949,997340,996986,990730,996408,992372,998903,993335,994682,990998,998716,990700,991919,997577,995393,996357,992628,995313,990173,992664,994269,993917,993101,991382\n", + "Ground Truth: 995299,990074,998803,993413,992456,996183,998949,997340,996986,990730,996408,992372,998903,993335,994682,990998,998716,990700,991919,997577,995393,996357,992628,995313,990173,992664,994269,993917,997116,993101\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 932\n", + "Returned IDs: 997918,993289,997002,995826,993504,993492,993512,996511,992571,993781,994959,999840,997790,994268,990096,998090,996062,999019,990399,997320,997977,999355,991648,994280,997586,991206,996592,998401,994661,995672\n", + "Ground Truth: 997918,993289,997002,995826,993504,993492,993512,999900,996511,994449,992571,993781,992029,994959,999840,997790,994268,990096,998090,996062,999019,990399,997320,997977,990930,999355,991648,994280,997586,991206\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 933\n", + "Returned IDs: 999784,996001,996028,998302,997371,997002,997320,998921,999806,994266,995560,993820,995596,997913,994964,993342,996840,993902,997301,990532,993906,995886,993260,994689,992330,997506,994360,990771,999453,996075\n", + "Ground Truth: 999784,996001,996028,998302,997371,997002,997320,998921,999806,994266,995560,993820,995596,998720,997913,993834,994964,993342,996840,993902,997301,990532,995375,993906,996096,995886,993260,994689,995852,992330\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 934\n", + "Returned IDs: 999867,994964,995435,999302,996645,992546,990957,997371,995942,997913,998163,997320,995949,999374,994783,993853,995755,993725,990925,993484,998500,994831,995928,998294,993885,994179,999740,990736,999356,998326\n", + "Ground Truth: 999867,994964,995435,999302,996645,992546,990957,997371,995942,997913,998163,997320,995949,999374,994783,993853,995755,998365,990504,993725,990925,993484,998500,992107,994831,995928,993412,998294,999444,993885\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 935\n", + "Returned IDs: 995981,996609,990444,996648,991721,994069,999374,991593,990990,998302,999046,997320,998246,996805,997301,998339,998849,991422,990585,998974,990096,990146,992291,990957,999533,994409,997614,998156,998294,995884\n", + "Ground Truth: 995981,996609,990444,996648,991721,994069,999374,991593,993412,990990,996085,998302,998777,999046,997320,998246,996805,997301,995266,998339,993186,998849,991422,990585,998974,991602,993049,990096,990146,992291\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 936\n", + "Returned IDs: 997700,999581,999054,991507,990537,991722,995106,994861,995268,993209,992448,993313,995778,996607,990372,997738,995820,998162,998910,999279,995518,996505,994380,990924,991424,996813,995401,991699,995521,999047\n", + "Ground Truth: 997700,999581,999054,991507,990537,991722,995106,994861,995268,993209,992448,993313,995778,996607,990372,997738,995820,998162,998910,998283,999279,995518,996505,994380,990924,991424,993046,996813,995401,991699\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 937\n", + "Returned IDs: 992751,995443,996609,992944,997918,991801,996075,999355,995203,997002,998232,995336,994552,996871,993781,996193,999620,990043,995515,998180,996001,993283,996735,997305,995536,992880,991498,991131,994042,995372\n", + "Ground Truth: 992751,995443,996609,992944,997918,991801,996075,999355,995203,997002,998232,995336,994552,996871,993781,996193,999620,994162,990043,995515,998180,996744,996001,993283,994449,995365,996735,997305,995536,992880\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 938\n", + "Returned IDs: 998777,996219,999676,995401,999453,998530,998797,992309,998495,993355,994318,990843,996840,998302,992950,990974,995016,991585,994195,993431,994964,990432,992216,999302,997554,991743,992169,990856,991732,993717\n", + "Ground Truth: 998777,996219,999676,995401,999453,998530,998797,992309,998495,993355,994318,990843,996840,998302,992950,990974,992328,994850,995016,991585,994195,993431,993412,994964,995899,990432,992216,995374,999302,997554\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 940\n", + "Returned IDs: 994897,993342,999806,993617,999430,992628,993528,992807,995393,990173,990194,991846,999555,994155,998787,994150,996809,999834,995895,994122,993076,994269,996184,996944,994834,991148,997906,997084,997378,994682\n", + "Ground Truth: 994897,993342,999806,993617,997335,999430,992628,995039,991271,993528,992807,995393,990173,990194,991846,999555,994155,998214,998787,997126,994150,996809,999834,996660,995895,997196,994122,993076,994269,996184\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 941\n", + "Returned IDs: 993143,997370,990580,994789,995515,999077,993237,991840,999898,998837,999784,998401,991999,998734,992612,996840,994964,993260,990096,997918,992944,998180,991172,990142,996693,999210,999676,994195,997586,998455\n", + "Ground Truth: 993143,997370,990580,994789,995515,999077,993237,991840,999898,998837,999784,998401,991999,997135,998734,992612,996840,994964,993260,990096,997918,992944,995401,998180,991172,990142,996693,999210,999676,994195\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 942\n", + "Returned IDs: 998793,999476,997215,992802,997744,990150,995552,990755,994965,995724,990192,990908,992608,999639,999164,998242,996196,990930,997548,991791,998495,990538,994661,999908,995979,995430,995372,995983,998754,994165\n", + "Ground Truth: 998793,999476,997215,992802,997744,990782,990150,995552,990755,994965,994536,995724,990192,990908,992608,999639,990334,999164,998242,996196,990930,997548,991791,998495,990538,994661,999908,995979,995430,995372\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 943\n", + "Returned IDs: 994879,992582,997728,996673,997353,998225,992502,991431,992293,992862,997304,995981,999761,992481,998708,994304,998986,994894,998697,999243,993809,995927,992183,997154,992852,998200,997913,996714,993970,995701\n", + "Ground Truth: 994879,998347,993222,992582,997728,996673,992760,994265,997353,998225,992502,991431,992293,996948,992862,997304,991680,995981,999761,992481,990862,998708,994304,998986,994894,998697,993072,999243,993809,997620\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 944\n", + "Returned IDs: 992868,995768,991490,997340,999798,991850,991779,992761,999256,991667,995655,990173,992517,995385,995501,994356,995476,993055,990154,990127,995879,993560,992372,995299,992561,998495,991313,997506,996580,993391\n", + "Ground Truth: 992868,995768,991490,997340,999798,991850,991779,992761,999256,991667,995655,990173,992110,992517,995385,995501,994356,999601,995476,993055,990154,995222,990127,991494,995879,993560,998167,992372,995299,992561\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 945\n", + "Returned IDs: 993209,993299,996867,990638,993586,997544,990082,994742,997319,992582,990126,993549,995078,995030,994587,991943,995211,996302,998913,995633,990493,998910,992289,995268,993920,996411,991639,995274,999091,997746\n", + "Ground Truth: 993209,993299,996867,994317,990638,993586,997544,990082,994742,997723,997319,992582,990126,997077,993549,995078,995030,994587,991943,995211,996302,992667,998913,995633,996277,991664,990493,998910,992289,995268\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 946\n", + "Returned IDs: 995547,999530,994409,999367,998545,997723,993717,993561,991923,997191,996042,994948,996769,996038,996918,997406,995401,995507,997832,991459,994002,997675,993143,997289,991052,995923,993271,996048,992130,998045\n", + "Ground Truth: 995547,999530,994409,999367,995994,998545,991061,997723,993717,993561,991923,997191,996042,994948,996769,994026,996038,996918,997406,995401,995507,997832,991459,993122,994002,993143,997675,997289,998250,991052\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 947\n", + "Returned IDs: 991298,992680,992469,995126,993053,993164,994140,993561,990529,990096,990845,996411,994925,996001,996062,996028,999729,990076,994783,999302,991789,995289,995216,998039,995823,994964,999099,997386,990930,994293\n", + "Ground Truth: 991298,992680,992469,995126,998633,993053,992334,993164,994140,993561,990529,990096,990845,996411,994925,996001,996062,996028,999729,990076,994783,999302,991789,995401,995289,997822,998940,995216,993433,998039\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 948\n", + "Returned IDs: 996001,995536,990532,996661,997586,997320,992862,996368,994200,999444,996178,999639,999587,995672,990814,993504,992469,999974,995983,993157,990399,991981,998827,999453,995401,994334,995535,991157,993289,996147\n", + "Ground Truth: 996001,994449,995536,990532,996661,997586,997320,996824,996511,992862,996368,994200,999444,996178,999639,991630,992521,992635,999587,995672,990814,993504,992469,999974,994343,995983,995250,993157,990399,991981\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 949\n", + "Returned IDs: 993209,995268,990361,999581,995106,995289,991630,991664,993920,990593,994742,995766,996867,998748,993586,999047,995778,997411,995052,993751,998188,993053,999653,997915,992521,998045,994409,993997,999279,991507\n", + "Ground Truth: 993209,995268,990361,997723,999581,995198,995106,995289,991630,991664,993920,990593,994742,995766,996867,998748,993586,996277,999047,995778,997411,992293,995052,996042,998999,993751,998188,997612,993053,999653\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 950\n", + "Returned IDs: 995226,993609,992457,997506,999139,996915,990169,990203,992640,995459,992170,995349,998933,996233,992824,991245,994672,995839,994743,993595,996742,997926,997442,992736,995740,992052,991138,993715,993055,992429\n", + "Ground Truth: 995226,993609,992457,997506,999139,996915,990169,990203,992640,995459,992170,995349,998933,996233,992824,991245,994672,995839,994743,993595,993077,996742,997926,997442,992736,995740,992052,991138,995408,993715\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 951\n", + "Returned IDs: 999076,993355,997539,994922,990300,998424,990809,990580,995210,995515,997747,999582,993688,998236,998751,993237,998837,994789,992612,991392,997291,998401,997772,997818,992950,990615,993091,993143,997852,991833\n", + "Ground Truth: 999076,993355,997539,994922,990300,998424,990809,990580,995210,995515,997747,999582,995430,993688,998236,998751,993237,998837,999469,994789,992612,993222,991609,991392,997291,996561,992979,999097,996793,998401\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 952\n", + "Returned IDs: 995358,990075,998568,994897,996217,999963,997617,996429,993866,993260,995203,996764,993055,999079,997965,997906,994832,992082,996075,993696,993342,990697,998697,993504,993560,991517,994680,992617,995455,999653\n", + "Ground Truth: 995358,990075,998568,994897,996217,999963,997617,996429,993866,993260,995203,996764,995622,993055,999079,997965,997906,997577,994832,992082,997506,996075,991630,993696,993342,990697,998697,993504,993560,991517\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 953\n", + "Returned IDs: 997320,990560,997371,998950,996075,994964,997913,991743,990949,997301,991260,998090,996805,993529,990910,998550,992443,995826,990231,999374,997192,993264,998365,995942,996219,992041,998246,995336,993504,992635\n", + "Ground Truth: 997320,990560,997371,998950,996075,994964,997913,991743,990949,997301,991260,998090,996805,993529,990910,998550,992443,992277,995826,990231,999374,997192,993264,998365,995942,996219,992041,998246,995336,993504\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 954\n", + "Returned IDs: 995840,997807,994782,998668,997586,998496,997004,995927,999210,999941,993114,995944,993689,993878,997364,990866,999898,993624,997305,998814,998708,990373,993831,999218,997552,994815,996219,991938,991373,998495\n", + "Ground Truth: 995840,997807,998414,994782,998668,997586,998496,997004,994756,995927,999210,999941,998068,993114,995944,999931,993689,993878,997364,990866,999898,998861,991006,995971,993624,997305,991259,998814,998708,990373\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 955\n", + "Returned IDs: 998921,997913,995693,998302,998950,990432,994964,994318,996053,997320,996219,992277,992944,996915,997470,998170,993431,997371,991999,996075,999210,999374,991743,996178,992978,999343,999077,992309,993143,994919\n", + "Ground Truth: 998921,995826,997913,995693,998302,998950,990432,994964,994318,996053,997320,996219,992277,998294,992944,996915,997470,998170,993431,997371,991999,992328,992140,996075,999210,999374,997379,991743,996178,992978\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 956\n", + "Returned IDs: 992630,999343,991335,992384,996219,990934,992931,994100,990974,990448,996143,999331,998779,997571,998170,997371,997920,996790,999102,990275,999647,991862,996727,998500,994831,994964,995274,990271,998263,999144\n", + "Ground Truth: 992630,999343,991335,992384,996219,990934,990171,992931,996401,994100,990974,990448,996143,999331,998779,997571,998170,997371,997920,996790,999102,998550,997006,990275,999647,996777,991862,990315,996727,998500\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 957\n", + "Returned IDs: 993561,991958,997406,998382,996080,995126,992334,992680,998593,993738,995923,997753,992469,998039,995216,997386,991125,999729,998165,995289,994925,991857,993645,990096,993390,996001,993433,992322,998999,995182\n", + "Ground Truth: 993561,991958,997406,998382,996080,995126,992334,992680,998593,993738,995923,997753,992469,998039,995216,997386,991125,999729,998165,995289,994925,991857,993645,990930,990096,993390,993725,996001,993433,992322\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 958\n", + "Returned IDs: 998827,999122,992558,990620,996140,999806,991009,990538,997913,997124,997859,991157,995596,995737,998892,997227,996404,998950,995983,991272,999562,992423,999804,996918,997320,994577,996196,993492,995203,997305\n", + "Ground Truth: 998827,999122,992558,990620,996140,999806,991009,990538,997913,997124,997859,991157,995596,995737,998892,997227,996404,998950,995983,991272,999562,992423,992971,999804,996918,997320,992612,994577,996196,993492\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 959\n", + "Returned IDs: 993764,996472,991987,994365,990942,990593,997884,991610,995891,996971,990331,995537,992631,995268,994002,998586,992136,991038,997506,992123,999129,997675,996623,998067,993111,992721,993956,990005,994809,994125\n", + "Ground Truth: 993764,996472,991987,994365,990942,993381,990593,997884,999906,991610,995891,996971,990331,995537,992631,995864,990998,995268,994002,991709,993629,998586,992136,991038,997506,992123,999129,997675,998959,992589\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 960\n", + "Returned IDs: 998335,990627,992862,991304,996456,996104,994796,995991,996594,992270,994564,990802,995231,994742,995168,990998,996408,999334,995780,996611,998279,992636,996728,991098,997506,995626,993638,995530,999216,990844\n", + "Ground Truth: 998335,990627,992862,991304,996456,996104,994796,995991,996594,992270,994564,990802,991447,995231,994742,995168,990998,996408,999319,993076,999334,995780,996611,998279,992636,996728,991098,994498,997506,995626\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 961\n", + "Returned IDs: 995935,997221,991463,993433,999742,998045,992125,998906,996125,991367,992556,996994,995439,998520,992521,996769,996719,996112,996323,997359,993844,995052,996277,997675,997406,990984,996048,992115,992089,997046\n", + "Ground Truth: 995935,997221,991463,993433,999742,998045,997187,992125,998906,996125,991367,992556,996994,995439,998520,992521,996769,996719,996112,996323,997359,993844,995052,996277,997675,997406,990984,996048,992115,992089\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 962\n", + "Returned IDs: 995336,997301,995438,994964,995961,994757,998246,999144,994360,999331,999710,997320,990949,995942,993492,992384,997470,990231,998302,997775,999301,994100,998090,990958,996592,995886,993241,992228,994081,990957\n", + "Ground Truth: 995336,997301,995438,994964,995961,994757,998246,999144,994360,999331,993154,999710,997320,990949,995942,993492,992384,997470,990231,998302,997775,999301,994100,998090,990958,996592,995886,993241,992228,994081\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 963\n", + "Returned IDs: 992135,991459,998623,990537,990682,996001,995968,996060,990502,996247,990005,996812,994809,994380,999453,999543,996759,999724,995932,998068,994835,995268,991389,999653,996256,991175,990251,995815,990294,999977\n", + "Ground Truth: 992135,991459,998623,990537,998720,990682,996001,995968,990457,997140,996060,990502,991154,996247,990005,993673,996812,995027,993131,994809,997091,994380,999453,992124,999543,996417,990616,997965,996759,998029\n", + "Recall: 0.5667\n", + "\n", + "Query ID: 964\n", + "Returned IDs: 991168,991464,992547,995283,995295,999519,995861,994742,995456,992334,991789,999126,994817,994853,999302,998224,993725,997933,992408,993831,998779,994765,993717,994783,996028,990266,990392,990889,993561,999341\n", + "Ground Truth: 991168,991464,992547,995283,995295,999519,995861,994742,995456,992334,990928,991789,999126,994817,994853,999302,991334,998224,996906,997933,993725,996308,992408,993831,998779,994765,993717,996147,994783,996028\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 965\n", + "Returned IDs: 994964,997379,995791,993035,992944,994777,999210,998950,996811,993696,993215,992612,990310,998665,994922,996915,991999,999676,990690,993355,992169,993809,996093,994318,993561,995596,997320,993906,996867,999076\n", + "Ground Truth: 994964,997379,995791,993035,992944,994777,996397,999210,998950,996811,993831,993696,993633,994089,993215,992612,990310,998665,995365,994922,996915,991999,999676,990690,993355,992169,993809,996093,994318,993561\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 966\n", + "Returned IDs: 998797,999995,997040,990146,999262,995884,997320,990925,994964,998302,995873,992546,996959,990936,990958,995947,997586,992896,997911,993484,998246,997913,996075,995336,999343,999242,999374,992330,993168,991498\n", + "Ground Truth: 998797,999995,997040,990146,999262,995884,997320,990925,994964,998302,995873,992546,996959,990936,990958,991852,995947,997586,992896,996785,997911,993484,998246,997913,996075,995336,997822,999343,996296,999242\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 967\n", + "Returned IDs: 995349,997685,999079,992664,997673,991148,993866,997906,999443,999576,992209,995969,995231,992275,994661,997369,990096,997666,993965,995064,995895,996733,991477,995695,993445,996124,995735,995393,995168,990622\n", + "Ground Truth: 995349,997685,999079,992664,997673,991148,993866,997906,998468,999443,999576,992209,995969,995231,992275,994661,997369,990096,997666,993965,995064,995895,996733,991477,995695,993445,996124,995735,992752,995393\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 968\n", + "Returned IDs: 992664,994105,998959,995168,991477,990998,994834,995231,994906,998623,997666,996050,998565,992760,993948,997332,999373,990261,998536,990952,992862,995695,997506,995835,993866,996393,995969,998352,995536,990622\n", + "Ground Truth: 992664,994105,998959,995168,991477,990998,994834,995231,994906,998623,997666,996050,998383,998565,991654,992760,993948,997332,999373,990261,998536,990952,999555,992862,995695,997506,995835,991188,993866,996393\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 969\n", + "Returned IDs: 991110,990729,999737,998022,991767,998318,994063,991887,994088,995373,999662,994956,995449,997131,995480,992604,990256,999445,994122,992133,996236,990577,996730,998629,993137,997942,995080,990345,994539,993185\n", + "Ground Truth: 991110,990729,999737,998022,991767,998318,994063,991887,994088,995373,999662,994956,995449,997131,995480,992604,990256,999445,994122,992133,996236,990577,996730,994577,998629,993137,997942,995080,990345,994539\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 970\n", + "Returned IDs: 993484,994964,993648,993642,993238,997320,999374,990399,999453,995755,993389,990224,997737,996785,993143,993885,995961,998294,991743,994922,995886,990958,994673,998692,996854,997796,999827,997822,994266,997301\n", + "Ground Truth: 993484,994964,993648,993642,993238,997320,999374,990995,992047,990399,999453,995755,993389,996959,990936,990224,997737,996785,993143,993885,993412,995961,993663,992334,993400,998294,991743,994922,995886,990958\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 971\n", + "Returned IDs: 995940,991478,992364,997319,997723,998999,995685,991140,991590,995200,995190,996747,990223,991810,999788,997832,995289,993412,992289,992917,990940,995994,990986,998750,993293,991968,994948,995269,996302,993831\n", + "Ground Truth: 995940,991478,992364,997319,997723,998999,995685,991140,991590,995200,995190,996747,990223,991810,999788,997832,995289,993412,992289,992917,990940,995994,990986,998750,993293,991403,991968,991603,994948,995269\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 972\n", + "Returned IDs: 996001,993053,994106,992182,995200,996219,995873,994382,997909,990435,998725,999653,995216,994689,990557,991464,990752,995823,995289,993438,996657,993984,994529,994783,998982,991212,997529,994558,999402,999028\n", + "Ground Truth: 996001,993053,994300,999151,994106,992182,999352,995200,998568,996219,993883,995873,994382,997909,990435,995384,998725,999653,995216,994689,996411,997837,990557,991464,992384,995823,990752,995289,995198,993438\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 973\n", + "Returned IDs: 990296,992161,998221,998649,998495,990957,999898,990399,997192,996869,992107,995942,990435,993656,996001,990934,991335,997582,992087,996219,999144,995289,992124,993168,991096,995755,995536,990096,994961,993648\n", + "Ground Truth: 990296,992161,998221,993412,998649,998495,990957,999898,990399,997920,997192,996869,992107,995942,990435,993656,996001,990934,991335,997582,991311,992087,996219,999144,999710,995289,992124,995899,998039,993168\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 975\n", + "Returned IDs: 995332,995560,991994,996609,995107,998950,997002,994280,998697,994783,999653,995535,994964,990435,995536,996429,992558,996219,991212,991382,992446,993504,999210,993079,994972,994742,999740,997305,996871,991571\n", + "Ground Truth: 995332,995560,991994,996609,995107,998950,997002,994280,991921,998697,994783,999653,996706,995535,994964,995401,990435,995536,996429,992558,996219,991212,991382,992446,993504,999210,993079,994972,994742,995873\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 976\n", + "Returned IDs: 995958,995778,997023,992473,993209,995401,996429,995701,999675,995536,998067,997204,997216,991507,991044,999977,996232,997732,997544,991747,993313,996328,991958,996506,998045,996220,990537,992293,993561,995702\n", + "Ground Truth: 995958,995778,997023,992473,997723,993209,996376,994453,995401,996429,996226,995701,999675,995536,998067,997601,997204,997216,991507,991044,999977,996232,998495,997732,997544,993948,990814,991747,993313,996328\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 977\n", + "Returned IDs: 996379,998186,994589,999977,998045,997058,994946,996368,992197,997554,993561,995401,998779,996391,991630,995268,995873,993412,995701,993275,993390,996028,994409,996169,990525,992367,993813,999587,991415,992362\n", + "Ground Truth: 996379,998186,994589,999977,998045,997058,994946,996368,992197,997554,993561,995401,998779,996391,991630,995268,995873,993412,995701,993275,993390,996028,997723,994409,996169,990525,992367,993813,999587,991415\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 978\n", + "Returned IDs: 993504,993071,992823,997977,995203,992635,992082,999111,991408,991648,999840,991009,990358,991157,991206,997320,997965,992747,994694,997327,999750,992119,998401,993515,990885,992351,990196,998950,994134,992036\n", + "Ground Truth: 993504,993071,992823,997977,995203,992635,992082,999111,992336,991408,991648,999840,991009,990358,992750,991157,991206,997320,997965,992747,994694,997327,999750,992119,998401,993515,990885,992351,990196,991542\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 980\n", + "Returned IDs: 995643,996322,997776,990252,994419,995547,996818,992894,998545,996042,996254,996148,991301,994173,998486,998866,996569,992660,997046,997822,990137,996719,997477,996360,998564,996745,990975,997937,997289,991011\n", + "Ground Truth: 995643,996322,997776,990252,994419,995547,996818,992894,998545,996042,996254,996148,991301,994173,998486,998866,996569,992660,997046,997822,990137,996719,998973,997477,993416,996360,998416,998564,996745,990975\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 981\n", + "Returned IDs: 993857,997309,994119,994742,995166,994564,995295,990328,993384,997914,991334,997452,995456,999635,998180,997822,996689,993180,997173,997374,990392,998779,999843,993256,992703,998224,992582,999533,995633,994199\n", + "Ground Truth: 993857,997309,994119,994742,995166,994564,995295,990328,993909,993384,997208,997914,991334,997452,995456,999635,998180,997822,996689,993180,997173,991447,997374,990392,990133,998779,999843,992473,993256,992703\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 982\n", + "Returned IDs: 997544,990082,992582,990298,996220,997723,994453,993471,996269,998910,999646,991269,996304,998776,991136,995702,991334,995958,995685,999054,991639,999759,991507,993586,996226,990372,997466,990804,993209,990638\n", + "Ground Truth: 997544,990082,992582,990298,996220,998310,997723,994453,993471,991603,996269,998910,999646,991269,996304,998776,991136,995702,990035,991334,996038,995958,995685,999054,991639,999759,996777,991507,993586,996226\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 983\n", + "Returned IDs: 995295,996580,995283,994730,990528,992226,991464,993725,990531,990930,993585,990315,996411,994742,994682,993623,990614,990076,990133,995289,994783,999798,990811,992680,993053,997529,999916,997340,990392,991330\n", + "Ground Truth: 995295,996580,995283,994730,990528,992226,991464,993725,990531,990930,993585,990315,996411,994742,994682,993623,990614,990076,990133,995289,994783,999798,990811,992680,996136,993053,997141,991629,993705,997529\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 984\n", + "Returned IDs: 991471,998086,995701,997576,991459,998861,999555,999994,990194,998212,999384,996623,993561,991630,994884,990167,999663,991038,997906,999977,996649,999724,998022,993275,992384,993342,997154,997242,996406,996809\n", + "Ground Truth: 991471,998086,995701,997576,991459,998861,999555,999994,990194,995319,998212,999384,996623,993561,991630,990708,994884,999653,990167,999663,991038,997906,999977,999647,991154,996649,999958,999724,994906,998022\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 985\n", + "Returned IDs: 990096,999831,995638,993499,998934,992551,991154,998968,999647,990583,998314,999384,990584,999194,998697,996216,999045,998495,995899,996811,996564,997586,993809,999499,990083,994661,993686,993458,997685,993055\n", + "Ground Truth: 990096,999831,995638,993499,998934,992551,993928,997524,991154,998968,999647,990583,998314,999384,990584,999194,998697,996216,999045,998495,995899,996811,996564,997586,993809,999499,992661,999343,990780,990083\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 986\n", + "Returned IDs: 991348,994552,997918,993260,993241,997320,992041,994290,999453,995372,999210,999077,997113,999317,990843,998950,999784,991999,993484,996318,993696,992944,995108,996075,991743,993018,996592,998464,994886,991131\n", + "Ground Truth: 991348,994552,997918,993260,993241,995622,997320,992041,994290,999453,995372,999210,999077,997113,999317,990843,998950,999784,991999,993484,996318,993696,992944,993365,995108,996075,991743,993018,992942,997135\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 987\n", + "Returned IDs: 997204,992169,993288,995289,999587,995359,999099,992685,995685,997918,998530,993412,991810,994409,996328,996220,999820,992979,992239,995401,997395,997002,997630,994906,996903,995994,990940,999977,995611,990223\n", + "Ground Truth: 997204,992169,993288,995289,995374,999587,995359,999099,992685,995685,991151,997918,998530,993909,993412,991810,994409,996328,996220,999820,992979,992239,995401,997395,997002,997630,994906,990337,996903,997447\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 988\n", + "Returned IDs: 998992,997015,993562,990535,991068,993050,994685,999531,991568,994997,994234,992744,993484,994964,991008,997306,994217,995336,990673,995824,996785,997320,993648,999757,991467,992709,996959,997314,993168,994111\n", + "Ground Truth: 998992,997015,993562,990535,991068,993050,991433,994685,999531,991568,994997,994234,992744,993484,994964,991008,997306,994217,995336,990673,995824,996785,997320,993648,999757,991467,992709,996959,997314,993168\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 989\n", + "Returned IDs: 997267,990129,991544,995891,996472,992448,992801,996042,991958,995521,994409,997884,998045,998703,990681,995200,993433,994002,994742,991671,994954,994413,993127,993561,998623,995702,996695,996607,994365,998910\n", + "Ground Truth: 997267,990129,991544,995891,996472,992448,992801,996042,991958,995521,994409,997884,995864,995873,998045,997909,998703,990681,995200,995198,993433,994002,994742,991671,994954,994413,993127,993561,998623,992123\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 990\n", + "Returned IDs: 999139,992457,993567,994280,995600,997480,992824,998109,994130,994196,997442,998405,993055,992480,992237,992736,990929,996016,998251,991382,996270,999985,992052,991723,998631,995823,991128,999205,997050,990705\n", + "Ground Truth: 999139,992457,993567,994280,995600,997480,992824,998109,994130,994196,997442,998405,993055,992480,992237,992736,990929,996016,998251,991382,996270,999985,995459,992052,991723,998631,995823,991128,999205,993621\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 991\n", + "Returned IDs: 996571,991975,997965,994724,991987,993641,991513,991038,999653,997369,995780,996920,997906,991844,997265,992945,998586,994374,998861,995168,992233,996209,990991,995231,992862,996326,994451,998858,990167,995536\n", + "Ground Truth: 996571,991975,996120,993381,997145,997965,994724,991987,993641,991513,991038,999653,997369,995780,996920,997906,991844,997265,993629,992945,998586,994374,998861,990450,995168,992233,996209,990991,995231,992862\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 992\n", + "Returned IDs: 993562,997015,993050,998992,990535,999531,997314,991568,990673,999026,993168,991444,992911,992744,995034,991391,990797,994964,993648,996714,994595,992646,999596,990817,999024,991467,991086,998951,995077,994100\n", + "Ground Truth: 993562,997015,993050,998992,990535,999531,997314,991568,991433,990673,999026,993168,991444,992911,992744,994265,996336,995034,991391,990797,994425,994964,993648,996714,994595,992646,999596,990817,999024,991467\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 993\n", + "Returned IDs: 993018,998950,997320,991157,992423,992617,990843,996262,992521,992944,997370,993840,995560,999432,999806,995596,999453,995515,995552,999210,995203,998255,996140,998232,990083,991801,994746,991348,991224,994577\n", + "Ground Truth: 993018,998950,997320,991157,992423,992617,990843,996262,992521,992944,997370,993840,995560,999432,999806,995596,999453,995515,995552,999210,995203,996368,998255,995250,996140,998232,990083,991801,995899,994746\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 994\n", + "Returned IDs: \n", + "Ground Truth: 994102,990092,991862,991335,996052,999647,990957,990231,996219,992134,994831,990545,997439,997913,996143,992114,995942,991743,995844,999343,999993,994192,993264,997320,990171,999144,993135,992931,996053,994898\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 995\n", + "Returned IDs: 996627,994751,996411,997729,990011,990012,990528,990743,996906,994663,995295,993725,992230,992680,992408,999740,994830,990392,993240,994730,995076,996011,992024,996147,993705,993906,995435,998030,991464,999302\n", + "Ground Truth: 996627,994751,996411,993070,997729,990011,990012,990528,990743,996906,994663,995295,993725,992230,992680,992408,999740,999922,994830,990392,993240,994730,995076,996011,992024,996147,993705,993906,995435,998030\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 996\n", + "Returned IDs: 995193,992521,998527,992046,992657,993021,995820,995640,997023,996581,999047,999394,991922,991102,992917,994508,992311,999731,998075,993107,994185,998255,999992,990295,992595,998910,993998,993379,998060,994954\n", + "Ground Truth: 995193,992521,998527,992046,992657,993021,995820,995640,997023,996581,999047,999394,991922,991102,992917,994508,990099,992311,999731,998075,993107,994185,994690,998255,999992,990295,992595,997554,998910,993998\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 997\n", + "Returned IDs: 991052,997675,996918,993805,996506,991061,991529,993014,996505,995521,996879,998397,999646,991747,993931,998593,994280,995080,993313,990537,996042,992521,995685,992558,998536,995110,999179,992457,992801,999139\n", + "Ground Truth: 991052,997675,996918,993805,996506,991061,997723,995401,991529,993014,996505,995521,996879,998397,999646,991747,993931,998593,994280,995080,993313,990537,994662,993600,998021,996042,992521,995994,994095,992933\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 998\n", + "Returned IDs: 990011,996275,990484,998710,998108,994830,994951,991885,996906,991826,994858,996025,993064,993070,993335,996925,997529,996301,998435,991383,995002,995293,994351,998642,991382,991313,991390,995476,993568,990186\n", + "Ground Truth: 990011,996275,997733,990484,998710,998108,994830,994951,991885,996906,991826,994858,996025,993064,993070,993335,996925,997529,996301,998435,991383,995002,995293,994351,998642,991382,991313,991390,995476,993568\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 999\n", + "Returned IDs: 999162,992654,991977,999638,998334,999387,994010,994672,998118,992718,991540,998142,995503,993018,997407,991432,991157,997320,990843,990926,999954,997816,993976,996262,992124,990487,992944,999019,995019,990532\n", + "Ground Truth: 999162,992654,991977,999638,995560,998334,999387,994010,994672,998118,992718,991540,998142,995503,993018,998567,998280,990930,997407,991432,991157,997320,990843,990926,999954,997816,993976,996262,992124,990487\n", + "Recall: 0.8667\n", + "\n", + "Total query IDs with recall < 1.0: 972\n", + "IDs: [np.int64(0), np.int64(1), np.int64(2), np.int64(3), np.int64(4), np.int64(5), np.int64(6), np.int64(7), np.int64(8), np.int64(9), np.int64(10), np.int64(11), np.int64(12), np.int64(13), np.int64(14), np.int64(15), np.int64(16), np.int64(17), np.int64(18), np.int64(19), np.int64(20), np.int64(21), np.int64(22), np.int64(23), np.int64(24), np.int64(25), np.int64(26), np.int64(27), np.int64(28), np.int64(29), np.int64(30), np.int64(31), np.int64(32), np.int64(33), np.int64(34), np.int64(35), np.int64(36), np.int64(37), np.int64(38), np.int64(39), np.int64(40), np.int64(41), np.int64(42), np.int64(43), np.int64(44), np.int64(45), np.int64(46), np.int64(47), np.int64(48), np.int64(49), np.int64(50), np.int64(51), np.int64(52), np.int64(53), np.int64(54), np.int64(55), np.int64(56), np.int64(57), np.int64(58), np.int64(59), np.int64(60), np.int64(61), np.int64(62), np.int64(63), np.int64(64), np.int64(65), np.int64(66), np.int64(67), np.int64(68), np.int64(69), np.int64(70), np.int64(71), np.int64(72), np.int64(73), np.int64(74), np.int64(75), np.int64(76), np.int64(77), np.int64(78), np.int64(79), np.int64(80), np.int64(81), np.int64(82), np.int64(83), np.int64(84), np.int64(85), np.int64(86), np.int64(87), np.int64(88), np.int64(89), np.int64(90), np.int64(91), np.int64(92), np.int64(93), np.int64(94), np.int64(95), np.int64(96), np.int64(97), np.int64(98), np.int64(99), np.int64(100), np.int64(101), np.int64(102), np.int64(103), np.int64(104), np.int64(105), np.int64(106), np.int64(107), np.int64(108), np.int64(109), np.int64(110), np.int64(111), np.int64(112), np.int64(113), np.int64(114), np.int64(115), np.int64(116), np.int64(117), np.int64(118), np.int64(119), np.int64(120), np.int64(121), np.int64(122), np.int64(123), np.int64(124), np.int64(125), np.int64(126), np.int64(127), np.int64(128), np.int64(129), np.int64(130), np.int64(131), np.int64(132), np.int64(133), np.int64(134), np.int64(135), np.int64(136), np.int64(137), np.int64(138), np.int64(139), np.int64(140), np.int64(141), np.int64(142), np.int64(143), np.int64(144), np.int64(145), np.int64(146), np.int64(147), np.int64(148), np.int64(149), np.int64(150), np.int64(151), np.int64(152), np.int64(153), np.int64(154), np.int64(155), np.int64(156), np.int64(157), np.int64(158), np.int64(159), np.int64(160), np.int64(161), np.int64(162), np.int64(163), np.int64(164), np.int64(165), np.int64(166), np.int64(167), np.int64(168), np.int64(169), np.int64(170), np.int64(171), np.int64(172), np.int64(173), np.int64(174), np.int64(175), np.int64(176), np.int64(177), np.int64(178), np.int64(179), np.int64(180), np.int64(181), np.int64(182), np.int64(183), np.int64(184), np.int64(185), np.int64(186), np.int64(187), np.int64(188), np.int64(189), np.int64(190), np.int64(192), np.int64(193), np.int64(194), np.int64(195), np.int64(196), np.int64(197), np.int64(198), np.int64(199), np.int64(200), np.int64(201), np.int64(202), np.int64(203), np.int64(204), np.int64(205), np.int64(206), np.int64(207), np.int64(208), np.int64(209), np.int64(210), np.int64(211), np.int64(212), np.int64(213), np.int64(214), np.int64(215), np.int64(216), np.int64(218), np.int64(219), np.int64(220), np.int64(221), np.int64(223), np.int64(224), np.int64(225), np.int64(226), np.int64(227), np.int64(228), np.int64(229), np.int64(230), np.int64(231), np.int64(232), np.int64(233), np.int64(234), np.int64(235), np.int64(236), np.int64(237), np.int64(238), np.int64(239), np.int64(240), np.int64(241), np.int64(242), np.int64(243), np.int64(244), np.int64(245), np.int64(246), np.int64(247), np.int64(248), np.int64(249), np.int64(250), np.int64(251), np.int64(252), np.int64(253), np.int64(254), np.int64(255), np.int64(256), np.int64(257), np.int64(258), np.int64(259), np.int64(260), np.int64(261), np.int64(262), np.int64(264), np.int64(266), np.int64(267), np.int64(268), np.int64(269), np.int64(270), np.int64(271), np.int64(272), np.int64(273), np.int64(274), np.int64(275), np.int64(277), np.int64(278), np.int64(279), np.int64(280), np.int64(281), np.int64(282), np.int64(283), np.int64(284), np.int64(285), np.int64(286), np.int64(287), np.int64(288), np.int64(289), np.int64(290), np.int64(291), np.int64(292), np.int64(293), np.int64(294), np.int64(295), np.int64(296), np.int64(297), np.int64(298), np.int64(299), np.int64(300), np.int64(301), np.int64(302), np.int64(303), np.int64(304), np.int64(305), np.int64(306), np.int64(307), np.int64(308), np.int64(309), np.int64(310), np.int64(311), np.int64(312), np.int64(313), np.int64(314), np.int64(315), np.int64(316), np.int64(317), np.int64(318), np.int64(319), np.int64(320), np.int64(321), np.int64(322), np.int64(323), np.int64(324), np.int64(325), np.int64(326), np.int64(327), np.int64(328), np.int64(329), np.int64(330), np.int64(331), np.int64(332), np.int64(333), np.int64(334), np.int64(335), np.int64(336), np.int64(337), np.int64(338), np.int64(339), np.int64(340), np.int64(341), np.int64(342), np.int64(343), np.int64(344), np.int64(345), np.int64(346), np.int64(347), np.int64(348), np.int64(349), np.int64(350), np.int64(351), np.int64(352), np.int64(353), np.int64(354), np.int64(355), np.int64(356), np.int64(357), np.int64(358), np.int64(359), np.int64(360), np.int64(361), np.int64(362), np.int64(363), np.int64(364), np.int64(365), np.int64(366), np.int64(367), np.int64(368), np.int64(369), np.int64(370), np.int64(371), np.int64(372), np.int64(373), np.int64(374), np.int64(375), np.int64(376), np.int64(377), np.int64(378), np.int64(379), np.int64(380), np.int64(381), np.int64(382), np.int64(383), np.int64(384), np.int64(385), np.int64(386), np.int64(387), np.int64(388), np.int64(389), np.int64(390), np.int64(391), np.int64(392), np.int64(393), np.int64(394), np.int64(395), np.int64(396), np.int64(397), np.int64(399), np.int64(400), np.int64(401), np.int64(402), np.int64(403), np.int64(404), np.int64(405), np.int64(406), np.int64(407), np.int64(408), np.int64(409), np.int64(410), np.int64(411), np.int64(412), np.int64(413), np.int64(415), np.int64(416), np.int64(417), np.int64(418), np.int64(419), np.int64(420), np.int64(421), np.int64(422), np.int64(423), np.int64(424), np.int64(425), np.int64(426), np.int64(427), np.int64(428), np.int64(429), np.int64(430), np.int64(431), np.int64(432), np.int64(433), np.int64(434), np.int64(435), np.int64(436), np.int64(437), np.int64(438), np.int64(439), np.int64(440), np.int64(441), np.int64(442), np.int64(443), np.int64(444), np.int64(445), np.int64(446), np.int64(447), np.int64(448), np.int64(449), np.int64(450), np.int64(451), np.int64(452), np.int64(453), np.int64(454), np.int64(455), np.int64(456), np.int64(457), np.int64(458), np.int64(459), np.int64(460), np.int64(461), np.int64(462), np.int64(463), np.int64(464), np.int64(465), np.int64(466), np.int64(467), np.int64(468), np.int64(469), np.int64(470), np.int64(471), np.int64(472), np.int64(473), np.int64(474), np.int64(475), np.int64(476), np.int64(477), np.int64(478), np.int64(479), np.int64(480), np.int64(481), np.int64(482), np.int64(483), np.int64(484), np.int64(485), np.int64(486), np.int64(487), np.int64(488), np.int64(489), np.int64(490), np.int64(491), np.int64(492), np.int64(493), np.int64(494), np.int64(495), np.int64(496), np.int64(497), np.int64(498), np.int64(499), np.int64(500), np.int64(501), np.int64(503), np.int64(504), np.int64(505), np.int64(507), np.int64(508), np.int64(509), np.int64(510), np.int64(511), np.int64(512), np.int64(513), np.int64(514), np.int64(515), np.int64(516), np.int64(517), np.int64(518), np.int64(519), np.int64(520), np.int64(521), np.int64(522), np.int64(523), np.int64(524), np.int64(525), np.int64(526), np.int64(527), np.int64(528), np.int64(529), np.int64(530), np.int64(531), np.int64(532), np.int64(533), np.int64(534), np.int64(535), np.int64(536), np.int64(537), np.int64(538), np.int64(539), np.int64(540), np.int64(541), np.int64(542), np.int64(543), np.int64(544), np.int64(545), np.int64(546), np.int64(547), np.int64(548), np.int64(549), np.int64(550), np.int64(551), np.int64(552), np.int64(553), np.int64(554), np.int64(556), np.int64(557), np.int64(558), np.int64(559), np.int64(560), np.int64(561), np.int64(562), np.int64(563), np.int64(564), np.int64(565), np.int64(566), np.int64(567), np.int64(568), np.int64(569), np.int64(570), np.int64(571), np.int64(572), np.int64(573), np.int64(574), np.int64(575), np.int64(576), np.int64(577), np.int64(578), np.int64(579), np.int64(580), np.int64(581), np.int64(582), np.int64(583), np.int64(584), np.int64(585), np.int64(586), np.int64(587), np.int64(588), np.int64(589), np.int64(590), np.int64(591), np.int64(592), np.int64(593), np.int64(594), np.int64(595), np.int64(596), np.int64(597), np.int64(598), np.int64(599), np.int64(600), np.int64(601), np.int64(603), np.int64(604), np.int64(605), np.int64(606), np.int64(607), np.int64(608), np.int64(609), np.int64(610), np.int64(611), np.int64(612), np.int64(613), np.int64(614), np.int64(615), np.int64(616), np.int64(617), np.int64(618), np.int64(619), np.int64(620), np.int64(621), np.int64(622), np.int64(623), np.int64(624), np.int64(625), np.int64(626), np.int64(627), np.int64(628), np.int64(629), np.int64(630), np.int64(631), np.int64(632), np.int64(633), np.int64(634), np.int64(635), np.int64(636), np.int64(637), np.int64(638), np.int64(639), np.int64(640), np.int64(641), np.int64(642), np.int64(643), np.int64(644), np.int64(645), np.int64(646), np.int64(647), np.int64(648), np.int64(649), np.int64(650), np.int64(651), np.int64(652), np.int64(653), np.int64(654), np.int64(655), np.int64(656), np.int64(657), np.int64(658), np.int64(660), np.int64(661), np.int64(663), np.int64(664), np.int64(665), np.int64(666), np.int64(667), np.int64(668), np.int64(669), np.int64(670), np.int64(671), np.int64(672), np.int64(673), np.int64(674), np.int64(675), np.int64(676), np.int64(678), np.int64(679), np.int64(680), np.int64(681), np.int64(682), np.int64(683), np.int64(684), np.int64(685), np.int64(686), np.int64(687), np.int64(688), np.int64(689), np.int64(690), np.int64(691), np.int64(692), np.int64(693), np.int64(694), np.int64(696), np.int64(697), np.int64(699), np.int64(700), np.int64(701), np.int64(702), np.int64(703), np.int64(704), np.int64(705), np.int64(706), np.int64(707), np.int64(708), np.int64(709), np.int64(710), np.int64(711), np.int64(712), np.int64(713), np.int64(714), np.int64(715), np.int64(717), np.int64(718), np.int64(719), np.int64(720), np.int64(721), np.int64(722), np.int64(723), np.int64(724), np.int64(725), np.int64(726), np.int64(727), np.int64(728), np.int64(729), np.int64(730), np.int64(731), np.int64(732), np.int64(733), np.int64(734), np.int64(735), np.int64(736), np.int64(737), np.int64(738), np.int64(739), np.int64(740), np.int64(741), np.int64(742), np.int64(743), np.int64(744), np.int64(745), np.int64(746), np.int64(747), np.int64(748), np.int64(749), np.int64(750), np.int64(751), np.int64(752), np.int64(753), np.int64(754), np.int64(755), np.int64(756), np.int64(757), np.int64(758), np.int64(759), np.int64(760), np.int64(761), np.int64(762), np.int64(763), np.int64(764), np.int64(765), np.int64(766), np.int64(768), np.int64(769), np.int64(770), np.int64(771), np.int64(772), np.int64(773), np.int64(774), np.int64(775), np.int64(776), np.int64(777), np.int64(778), np.int64(779), np.int64(780), np.int64(781), np.int64(782), np.int64(783), np.int64(784), np.int64(786), np.int64(787), np.int64(788), np.int64(789), np.int64(790), np.int64(791), np.int64(792), np.int64(793), np.int64(794), np.int64(795), np.int64(796), np.int64(797), np.int64(798), np.int64(799), np.int64(800), np.int64(801), np.int64(802), np.int64(803), np.int64(804), np.int64(805), np.int64(806), np.int64(807), np.int64(808), np.int64(809), np.int64(810), np.int64(811), np.int64(812), np.int64(813), np.int64(815), np.int64(816), np.int64(817), np.int64(818), np.int64(819), np.int64(820), np.int64(821), np.int64(822), np.int64(823), np.int64(824), np.int64(825), np.int64(826), np.int64(827), np.int64(828), np.int64(829), np.int64(830), np.int64(831), np.int64(832), np.int64(833), np.int64(834), np.int64(835), np.int64(836), np.int64(837), np.int64(838), np.int64(839), np.int64(841), np.int64(842), np.int64(843), np.int64(844), np.int64(845), np.int64(846), np.int64(847), np.int64(848), np.int64(849), np.int64(850), np.int64(851), np.int64(852), np.int64(853), np.int64(854), np.int64(855), np.int64(856), np.int64(857), np.int64(858), np.int64(859), np.int64(860), np.int64(861), np.int64(862), np.int64(863), np.int64(864), np.int64(865), np.int64(866), np.int64(867), np.int64(868), np.int64(869), np.int64(870), np.int64(871), np.int64(872), np.int64(873), np.int64(874), np.int64(875), np.int64(876), np.int64(877), np.int64(878), np.int64(879), np.int64(880), np.int64(882), np.int64(883), np.int64(884), np.int64(885), np.int64(886), np.int64(887), np.int64(888), np.int64(889), np.int64(890), np.int64(891), np.int64(892), np.int64(893), np.int64(894), np.int64(895), np.int64(896), np.int64(897), np.int64(898), np.int64(899), np.int64(900), np.int64(901), np.int64(902), np.int64(903), np.int64(904), np.int64(905), np.int64(906), np.int64(907), np.int64(908), np.int64(909), np.int64(911), np.int64(912), np.int64(913), np.int64(914), np.int64(915), np.int64(916), np.int64(917), np.int64(918), np.int64(919), np.int64(920), np.int64(921), np.int64(922), np.int64(923), np.int64(924), np.int64(926), np.int64(927), np.int64(928), np.int64(929), np.int64(930), np.int64(931), np.int64(932), np.int64(933), np.int64(934), np.int64(935), np.int64(936), np.int64(937), np.int64(938), np.int64(940), np.int64(941), np.int64(942), np.int64(943), np.int64(944), np.int64(945), np.int64(946), np.int64(947), np.int64(948), np.int64(949), np.int64(950), np.int64(951), np.int64(952), np.int64(953), np.int64(954), np.int64(955), np.int64(956), np.int64(957), np.int64(958), np.int64(959), np.int64(960), np.int64(961), np.int64(962), np.int64(963), np.int64(964), np.int64(965), np.int64(966), np.int64(967), np.int64(968), np.int64(969), np.int64(970), np.int64(971), np.int64(972), np.int64(973), np.int64(975), np.int64(976), np.int64(977), np.int64(978), np.int64(980), np.int64(981), np.int64(982), np.int64(983), np.int64(984), np.int64(985), np.int64(986), np.int64(987), np.int64(988), np.int64(989), np.int64(990), np.int64(991), np.int64(992), np.int64(993), np.int64(994), np.int64(995), np.int64(996), np.int64(997), np.int64(998), np.int64(999)]\n", + "Average Recall across all 1000 queries: 0.7946\n" + ] + } + ], + "source": [ + "#For querying (int filter)- range filter\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "#inputs\n", + "int_rate_query = \"99p\" # change to 1p, 50p, 80p, 99p\n", + "top_k = 30\n", + "\n", + "# Mode 1: Single query id\n", + "# Mode 2: Find all query ids with recall < 1\n", + "mode = 2\n", + "query_id = 457 # only used in mode 1\n", + "\n", + "# Map int_rate_query to filter range\n", + "range_map = {\n", + " \"1p\": [10000, 1000000],\n", + " \"50p\": [500000, 1000000],\n", + " \"80p\": [800000, 1000000],\n", + " \"99p\": [990000, 1000000],\n", + "}\n", + "filter_range = range_map[int_rate_query]\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "test_parquet_path = os.path.join(dataset_path, \"test.parquet\")\n", + "gt_parquet_path = os.path.join(dataset_path, f\"neighbors_int_{int_rate_query}.parquet\")\n", + "\n", + "# Read parquet files\n", + "test_df = pq.ParquetFile(test_parquet_path).read().to_pandas()\n", + "gt_df = pq.ParquetFile(gt_parquet_path).read().to_pandas()\n", + "\n", + "def run_query(query_id):\n", + " query_row = test_df[test_df[\"id\"] == query_id]\n", + " if query_row.empty:\n", + " print(f\"Query ID {query_id} not found in test.parquet\")\n", + " return None\n", + "\n", + " query_vector = query_row[\"emb\"].values[0]\n", + "\n", + " gt_row = gt_df[gt_df[\"id\"] == query_id]\n", + " if gt_row.empty:\n", + " print(f\"Query ID {query_id} not found in ground truth file\")\n", + " return None\n", + "\n", + " ground_truth = gt_row[\"neighbors_id\"].values[0][:top_k]\n", + "\n", + " results = index.query(\n", + " vector=query_vector,\n", + " top_k=top_k,\n", + " filter=[{\"id\": {\"$range\": filter_range}}],\n", + " include_vectors=False,\n", + " )\n", + " returned_ids = [int(r[\"id\"]) for r in results]\n", + "\n", + " intersection = len(set(returned_ids) & set(ground_truth))\n", + " recall = intersection / len(ground_truth)\n", + "\n", + " return returned_ids, ground_truth, recall\n", + "\n", + "\n", + "if mode == 1:\n", + " result = run_query(query_id)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " print(f\"Query ID: {query_id}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\")\n", + "\n", + "elif mode == 2:\n", + " print(f\"Finding all query IDs with recall < 1.0 ...\\n\")\n", + " low_recall_ids = []\n", + " all_recalls = []\n", + "\n", + " for qid in test_df[\"id\"].values:\n", + " result = run_query(qid)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " all_recalls.append(recall)\n", + " if recall < 1.0:\n", + " low_recall_ids.append(qid)\n", + " print(f\"Query ID: {qid}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\\n\")\n", + "\n", + " print(f\"Total query IDs with recall < 1.0: {len(low_recall_ids)}\")\n", + " print(f\"IDs: {low_recall_ids}\")\n", + " print(f\"Average Recall across all {len(all_recalls)} queries: {sum(all_recalls)/len(all_recalls):.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Finding all query IDs with recall < 1.0 ...\n", + "\n", + "Query ID: 1\n", + "Returned IDs: 997420,995382,990103,993165,996814,996219,996878,992878,998104,994051,994199,990678,996062,995438,993727,992193,994119,997997,999441,992780,993088,997331,990284,992731,997602,997098,996811,998691,999191,991031\n", + "Ground Truth: 997420,995382,990103,993165,996814,996219,996878,992878,998104,994051,994199,990678,996062,995438,993727,992193,994119,997997,999441,996830,992780,993088,997331,990284,992731,997602,997098,996811,998691,999191\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 2\n", + "Returned IDs: 996312,991260,995701,992330,996592,993492,997449,997913,992546,997976,990231,990936,990949,995014,994100,996881,992692,992969,993853,993005,996267,994778,991045,994757,992931,990957,999647,992961,990109,993970\n", + "Ground Truth: 996312,991260,995701,992330,996592,993492,997449,997913,992546,997976,990231,990936,993517,990949,995014,994100,996881,992692,992969,993853,993005,996267,994778,991045,994757,992931,990957,999647,992961,990109\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 4\n", + "Returned IDs: 993262,992665,991098,996599,992247,997534,990818,990802,992701,995626,990402,999466,991236,998303,991037,996216,997897,994310,998299,995516,997825,999435,992573,994098,997334,991212,995991,999216,993124,996424\n", + "Ground Truth: 993262,992665,991098,996599,992247,997534,990818,990802,992701,995626,990402,999466,991236,997442,998303,991037,996216,997897,994310,998299,997825,995516,999435,992573,990630,994098,997334,991212,995991,994964\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 5\n", + "Returned IDs: 995274,997320,998779,992994,999144,992630,999026,998184,998365,995755,998951,999090,999647,994100,993412,996959,994964,995701,998862,994757,995702,997976,995336,990949,993648,994730,994894,995438,992331,997737\n", + "Ground Truth: 995274,997320,998779,992994,999144,992630,999026,998184,998365,995755,998951,999090,999647,994100,993412,996959,998633,997925,994964,995701,998862,994757,995702,997976,995336,990949,997377,993648,994730,994894\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 6\n", + "Returned IDs: 992384,995274,993648,995204,993909,991743,992994,998184,993837,999374,993412,998500,995928,999533,992582,994636,990432,992147,995702,994850,998458,996521,992630,992637,998779,995961,991852,997301,996732,996959\n", + "Ground Truth: 992384,995274,993648,995204,993909,991602,991743,993089,992994,998184,991866,993837,999374,997645,993412,990862,998500,995928,999533,992582,994636,990432,992147,995702,994850,998458,996521,992630,992637,991151\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 8\n", + "Returned IDs: 997913,998090,990949,990910,997320,995961,999710,997301,990231,996918,990560,996805,993504,993515,998950,992277,999639,998337,995336,997371,999806,994850,992748,994100,993529,999900,998365,999338,993696,991498\n", + "Ground Truth: 997913,998090,990949,990910,997320,995961,999710,997301,990231,996918,993412,990560,996805,993504,993515,998950,992277,999639,998337,995336,997371,999806,994850,992748,994100,993529,999900,998365,999338,991498\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 10\n", + "Returned IDs: 991067,996273,999530,993530,999193,998316,997057,994677,993223,994301,991190,997723,992545,998650,992493,999961,995963,997675,995723,997098,997824,992582,995685,993455,996046,990930,999646,991052,997083,994948\n", + "Ground Truth: 991067,996273,999530,993530,999193,998316,997057,994677,993223,994301,991190,997723,992545,998650,992493,999961,995963,997675,995723,997098,997824,992582,995685,993455,996046,990930,999646,991052,997083,991840\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 19\n", + "Returned IDs: 990338,997370,993018,991437,997320,996001,996309,999586,998050,991224,996810,993642,990532,990843,993840,994081,991943,991002,991787,998495,996219,990772,994964,992655,994168,999561,992583,990974,990585,990786\n", + "Ground Truth: 990338,997370,995899,993018,991437,997320,996001,996309,999586,998050,991224,996810,993642,990532,990843,993840,994081,991943,991002,991787,998495,996219,990772,994964,992655,991019,994168,995283,999561,992583\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 21\n", + "Returned IDs: 993018,997320,994171,993662,995351,997913,990843,996075,995596,995266,994746,991142,994366,997470,996296,994341,990957,999077,998540,997457,993696,991999,997707,996469,995560,992733,999784,992277,997022,999806\n", + "Ground Truth: 993018,997320,994171,993662,995351,997913,990843,996075,994604,995596,995266,994746,994366,991142,997470,996296,994341,990957,999077,998540,997457,993696,991999,997707,996469,995560,992733,999784,992277,997022\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 22\n", + "Returned IDs: 997290,998557,997517,994110,998543,991334,994893,995030,995159,996809,999924,997947,999788,995948,992566,996747,995190,990814,994422,998913,992289,992468,990722,994824,999942,998194,992588,994541,998340,992364\n", + "Ground Truth: 997290,998557,997517,994110,998543,991334,994893,995030,995159,996809,999924,997947,999788,994062,995948,992566,996747,995190,990814,994422,998913,992289,992468,990722,994824,999942,998194,992588,994541,998340\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 23\n", + "Returned IDs: 996053,990096,997913,997586,994850,995961,995536,998090,996143,995755,995927,998861,994894,999356,999711,993853,997305,994618,998921,992848,997984,998550,994707,991582,994964,998495,992330,997320,992862,993948\n", + "Ground Truth: 996053,990096,997913,997586,994850,995961,995536,998090,996143,995755,995927,998861,994894,999356,999711,993853,997305,994618,998921,992848,997984,992716,998550,994707,991582,995503,994964,998495,992330,997320\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 24\n", + "Returned IDs: 996093,997852,994922,992169,993717,991473,998523,997395,996438,992632,992152,990580,990399,994510,996727,991840,993143,998458,999676,997476,990080,991609,993859,995295,992813,997192,993781,999453,992044,990490\n", + "Ground Truth: 996093,997852,994922,992169,993717,991473,998523,997395,996438,992632,992152,990580,990399,994510,996727,991840,993143,998458,999676,997476,990080,991609,993859,995295,992813,997192,993781,996406,992044,999453\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 26\n", + "Returned IDs: 990487,993500,991977,990998,994254,992558,991447,993335,998686,992052,994148,996068,990684,998406,993975,992170,991620,994006,993583,998536,995408,998480,992718,996262,992933,998623,992280,993965,994080,993715\n", + "Ground Truth: 990487,993500,991977,990998,994254,992558,991447,993335,998686,992052,994148,996068,990684,998406,993975,992170,991620,994006,993583,998536,995408,998480,992718,996262,992933,992554,998623,992280,993965,994080\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 29\n", + "Returned IDs: 998276,993603,994829,997918,997002,995141,992047,996012,993139,996661,993289,991425,997140,990051,996808,997977,992330,993504,992238,994972,996576,994946,998401,992225,994042,995855,997777,992979,990491,999111\n", + "Ground Truth: 998276,993603,994829,997918,997002,995141,996996,992047,996012,993139,996661,993289,991425,997140,990051,996808,997977,992330,993504,992238,994972,996576,994946,998401,992225,994042,995855,997777,992979,990491\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 31\n", + "Returned IDs: 990811,998593,995526,992334,996298,997732,998039,992684,997969,990089,996834,990529,993642,994964,994783,995126,995679,997410,995906,996001,999099,994925,992120,996873,993831,994293,999302,998382,991200,990576\n", + "Ground Truth: 990811,998593,995526,992334,996298,997732,998039,992684,997969,997112,990089,996834,990529,993642,994964,994783,995126,995679,997410,995906,996001,999099,994925,992120,996873,993831,994293,999302,998382,991200\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 34\n", + "Returned IDs: 995130,994659,995332,993808,995750,997273,996128,997542,991961,997412,993347,993079,993304,991990,998860,997442,994871,991343,998891,997199,993931,992567,995107,993499,992358,991061,993324,997555,993548,994410\n", + "Ground Truth: 995130,994659,995332,993808,995750,997273,996128,997542,991961,997412,993347,993079,993304,991990,998860,997442,994452,994871,991343,998891,997199,993931,992567,995107,993499,992358,991061,993324,997555,993548\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 35\n", + "Returned IDs: 994397,996075,995274,991422,999289,996976,998011,995336,997571,993168,995374,990701,993280,999596,997759,990585,997976,996108,994432,991125,990862,996714,997304,994916,997377,994894,996368,999437,993167,994100\n", + "Ground Truth: 994397,996075,995274,991422,999289,996976,998011,995336,997571,993168,995374,990701,993280,999596,997759,990585,997976,996108,994432,991125,990862,996714,997304,994916,997377,994894,996368,999437,993167,997925\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 40\n", + "Returned IDs: 993484,997320,998246,996296,997370,995949,997913,993885,990772,990957,997661,998302,990843,991801,993018,991498,990375,998950,993803,995932,993741,994366,997470,990949,991721,996143,991063,994089,993154,995596\n", + "Ground Truth: 993484,997320,998246,996296,997370,995949,996452,997913,993885,990772,990957,997661,998302,990843,991801,993018,991498,990375,998950,998160,993803,995932,993741,994366,997470,990949,991721,996143,995265,991063\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 41\n", + "Returned IDs: 996395,990194,999555,998006,997205,997242,999994,999576,990167,994467,993106,998122,990360,990230,993381,997149,999435,996209,991148,992771,993122,997086,998066,991767,993224,998429,994796,994638,995848,992517\n", + "Ground Truth: 996395,990194,999555,998006,997205,997242,999994,999576,995167,990167,994467,993106,998122,990360,990230,993381,997149,999435,996209,991148,992771,993122,997086,998066,991767,993224,998429,994796,994638,995848\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 43\n", + "Returned IDs: 994541,999372,999830,995300,998055,998671,994322,998466,994557,995280,996939,992434,993437,995750,995861,993929,993939,998484,995704,991893,996057,990931,994558,991327,994119,993856,995191,993877,994782,998710\n", + "Ground Truth: 994541,999372,999830,995300,998055,998671,994322,998466,994557,995280,996939,992434,993437,995750,995861,993929,993939,998484,995704,991893,996057,990931,991537,994558,991327,994119,993856,995191,993877,994782\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 45\n", + "Returned IDs: 996001,990366,995536,998720,999216,992862,998934,993139,996500,993289,993055,993174,990449,994829,994672,997586,990399,993560,997531,995672,995398,994167,999806,999639,997002,998593,990352,990891,999111,993696\n", + "Ground Truth: 996001,990366,995536,998720,999216,992862,998934,993139,996500,993289,993055,993174,990449,994829,994672,997586,990399,993560,997531,995672,995398,994167,999806,999639,997002,998593,995418,990352,992617,990891\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 46\n", + "Returned IDs: 996504,998921,997889,990917,998170,997192,994775,992147,990647,991743,995515,993518,996586,991127,999374,994939,996218,990515,991782,993250,992087,998721,999327,990275,992789,998401,994919,999263,995415,995621\n", + "Ground Truth: 996504,998921,997889,990917,998170,997192,994775,992147,990647,991743,995515,993518,996586,991127,999374,994939,996218,990515,992379,991782,993250,992087,998721,999327,990275,992789,998401,994919,999263,995415\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 48\n", + "Returned IDs: 994925,995873,993561,995895,991958,998382,998495,992448,993906,998593,993053,992521,992120,999294,996219,998106,993637,997386,996001,992847,998649,994742,997379,994419,995200,996397,995401,998039,990435,997822\n", + "Ground Truth: 994925,995873,993561,995895,991958,998382,998495,992448,993906,998593,993053,992521,992120,999294,996219,998106,993637,994657,997386,996001,992847,998649,994742,997379,994419,995200,995899,996397,995401,998039\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 52\n", + "Returned IDs: 990096,998934,996001,998720,997906,998697,993174,998861,995855,994167,997685,996164,993275,993154,999343,996124,995536,999736,996894,999647,995203,992429,999639,999882,998961,998514,994718,993744,999831,990991\n", + "Ground Truth: 990096,998934,996001,998720,997906,998697,993174,998861,995855,994167,997685,996164,993275,993154,999343,996124,995536,999736,996894,999647,995203,992429,999639,999882,995401,998961,998514,994718,993744,999831\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 53\n", + "Returned IDs: 996436,992912,997640,990797,998779,991750,997355,993412,999144,991651,993526,993725,995755,999740,997256,994990,994916,994436,994894,990535,990388,999026,994964,996714,994838,999021,991173,999344,997192,999596\n", + "Ground Truth: 996436,992912,997640,990797,998779,991750,997355,993412,999144,991651,999587,993526,993725,995755,999740,997256,994990,994916,995923,994436,994894,990535,990388,999026,994964,996714,994838,999021,991173,999344\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 54\n", + "Returned IDs: 997320,999639,993738,990930,990532,995873,997822,996371,995395,996714,992546,990878,993293,992977,992521,991721,995375,999026,990634,996777,994140,991349,990585,992318,991224,995274,995813,993561,996277,999587\n", + "Ground Truth: 997320,999639,993738,990930,990532,995873,997822,996371,995395,996714,992546,990878,993293,992977,992521,991721,995375,999026,999527,990634,996777,990814,994140,991349,990585,992318,991224,995274,995813,993561\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 56\n", + "Returned IDs: 998416,991996,997014,997920,998221,997804,997900,995786,990126,994964,993378,997403,998495,991377,990931,997477,991565,994466,992446,993199,998816,990158,996219,991179,997822,994419,993561,997586,993444,991831\n", + "Ground Truth: 998416,991996,997014,997920,998221,997804,997900,995786,990126,994964,993378,997403,998495,991377,996305,990931,997477,991565,994466,992446,993199,998816,990158,996219,991179,997822,994419,993561,997586,993444\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 58\n", + "Returned IDs: 996196,990159,996690,997224,996523,994384,993670,994372,998286,990303,990824,996655,995503,991550,995290,990989,999860,994950,998411,994964,994912,993158,990538,995599,991458,995410,998697,991791,999716,993809\n", + "Ground Truth: 996196,990159,996690,997224,996523,994384,993670,994372,998286,990303,990824,996655,995503,991550,995290,990989,999860,994950,996744,998411,994964,994912,993158,990538,995599,991458,995410,998697,991791,999716\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 59\n", + "Returned IDs: 996280,991451,995374,991650,999698,995357,993721,994030,993215,997002,998951,993154,991545,995181,991997,992557,999900,994850,998961,992880,994449,990532,997448,997918,994362,996685,990083,994081,993512,998779\n", + "Ground Truth: 996280,991451,995374,991650,999698,995357,993721,994030,993215,997002,994226,998951,993154,991545,995181,991997,992557,999900,994850,998961,992880,994449,990532,997448,997918,994362,996685,990083,994081,993512\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 60\n", + "Returned IDs: 998224,999496,995831,995196,995407,994956,995087,999138,994817,999244,990678,998779,995295,998691,996328,995861,993725,991390,991319,991378,997732,992713,992703,993678,996918,997714,995020,998067,994587,991192\n", + "Ground Truth: 998224,999496,995831,995196,995407,994956,995087,999138,994817,999244,990678,998779,995295,998021,998691,996328,995861,993725,991390,991319,991378,997732,992713,992703,993678,996918,997714,995020,998067,994587\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 61\n", + "Returned IDs: 993738,992651,998831,995701,995184,991836,998697,996534,997059,998060,993280,995079,995873,996828,992609,991530,996657,992293,995640,993948,991175,991007,997897,995216,994488,996001,990005,997554,992052,991576\n", + "Ground Truth: 993738,992651,998831,995701,995184,991836,998697,996534,997059,998060,993280,995079,995873,996828,992609,991530,996657,992293,995640,993948,991175,991007,997897,995216,994488,996001,994163,990005,997554,992052\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 62\n", + "Returned IDs: 991157,999122,991015,998607,997305,991720,992718,997192,993143,998180,996193,998123,995515,994046,992052,997496,993260,998045,998170,996504,992147,998950,991848,995401,993077,998921,995408,995372,998106,993948\n", + "Ground Truth: 991157,999122,991015,997305,998607,991720,992718,997192,993143,998180,996193,998123,995515,994046,992052,997496,993260,998045,998170,996504,992147,998950,991848,995401,993077,998921,995408,995864,995372,998106\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 63\n", + "Returned IDs: 995059,990264,990516,991590,991960,998999,992415,995020,998525,993293,993079,990521,992318,994435,995359,993180,992171,999981,995421,997319,994817,994697,997718,990610,994349,992239,991245,991867,991775,998961\n", + "Ground Truth: 995059,990264,990516,991590,991960,998999,992415,995020,998525,993293,993079,990521,992318,994435,995359,993180,992171,999981,995421,997319,994817,994697,997718,990610,994349,992239,994409,991245,991867,991775\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 65\n", + "Returned IDs: 996809,998695,991571,993617,998790,999009,998786,991890,995461,990288,996370,999711,999430,991139,998849,990361,990538,995375,999194,990489,994451,997821,993515,992452,990930,996830,995560,991532,994643,993696\n", + "Ground Truth: 996809,998695,991571,993617,998790,999009,998786,991890,995461,990288,996370,999711,999430,991139,998849,990361,990538,995375,999194,990489,991052,994451,997821,993515,995401,992452,990930,996830,995560,991532\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 66\n", + "Returned IDs: 999526,997970,991245,999281,991629,997550,994016,997584,996075,993715,992351,992825,995375,995740,992747,993504,995372,998097,997691,999724,993609,992933,994531,996001,995727,992052,994610,992636,990096,999338\n", + "Ground Truth: 999526,997970,991245,999281,991629,997550,994016,997584,996075,993715,992351,992825,995375,995740,992747,993504,995372,998097,997691,999724,993609,992933,994531,996001,995727,994610,992052,992636,990096,992521\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 67\n", + "Returned IDs: 992664,994131,998446,998030,998716,997097,991684,991486,993931,997453,996147,991663,998416,991081,996328,998845,998949,996733,998440,993335,998536,992780,998903,998076,997073,993917,997944,996028,991961,992408\n", + "Ground Truth: 992664,994131,998446,998030,998716,997097,991684,991486,993931,997453,996147,991663,998416,991081,996328,998845,998949,996733,998440,993335,998536,998903,992780,996830,998076,997073,993917,997944,996028,991961\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 69\n", + "Returned IDs: 994100,993492,991673,996805,992330,995336,998495,994334,995833,995968,990231,998921,992944,990271,993512,995357,996001,995274,998090,993154,994088,995536,995826,996282,995401,990648,996219,995260,994449,997605\n", + "Ground Truth: 994100,993492,991673,996805,992330,995336,998495,994334,995833,995968,990231,998921,992944,990271,996769,993512,995357,995748,996001,995274,998090,993154,994088,995536,995826,996282,995162,995401,990648,996219\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 71\n", + "Returned IDs: 995841,995375,995372,992944,999711,996318,990538,990043,994972,998726,991052,995401,990585,997301,998445,996193,999544,991977,991540,997320,995560,996001,998180,995268,995138,996609,993533,999526,999019,995515\n", + "Ground Truth: 995841,995375,995372,992944,999711,996318,990538,990043,994972,998726,991052,995401,990585,997301,998445,996193,999544,991977,991540,997320,995560,996001,998180,995268,998097,995138,996609,992384,993533,999526\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 74\n", + "Returned IDs: 994102,994831,998764,995372,997913,991362,999982,991335,995374,998702,992880,992291,990585,993852,996143,995828,990724,992748,994618,990949,997470,992114,990957,990275,996959,998500,998744,997192,996785,997320\n", + "Ground Truth: 994102,994831,998764,995372,997913,991362,999982,991335,995374,998702,992880,992291,990585,993852,996143,995828,990538,990724,992748,994618,990949,997470,992114,990957,990275,996959,998500,998744,997192,996785\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 75\n", + "Returned IDs: 998981,998287,997320,998246,994225,996296,992902,994889,997178,994450,991155,990020,992880,994214,994249,997470,995116,998540,993484,992748,994542,998634,993529,996075,996085,995700,999132,990742,994964,995400\n", + "Ground Truth: 998981,998287,997320,998246,994225,996296,992902,994889,997178,994450,991155,990020,994214,992880,994249,997470,995116,998540,993484,992748,994542,998634,993529,996075,996085,995700,994416,999132,990742,994964\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 79\n", + "Returned IDs: 992931,997843,992514,990742,996993,991260,998764,997483,995570,990478,992961,993484,991131,997913,990949,997320,998195,999982,992748,999838,999855,998653,992105,992781,991872,992069,995277,991752,995336,997307\n", + "Ground Truth: 992931,997843,992514,990742,996993,991260,998764,997483,995570,990478,992961,993484,991131,997913,990949,997320,996378,998195,999982,992748,999838,999855,998653,992105,992781,991872,992069,995277,991752,995336\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 81\n", + "Returned IDs: 990146,993237,992085,997747,994318,992944,991727,992342,991392,995443,990142,993253,997591,997855,997585,990742,991121,994789,999784,998782,990822,995334,994534,993570,990615,998550,999676,999948,994366,995693\n", + "Ground Truth: 990146,993237,992085,997747,994318,992944,991727,992342,991392,995443,990142,993253,997591,997855,997585,990742,991121,994789,999784,998782,990822,995334,994534,993570,990615,998550,990447,999676,999948,996352\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 84\n", + "Returned IDs: 997913,994964,997371,993237,995515,994778,991801,996727,996811,990142,996840,991727,995928,998359,996001,996438,994277,994415,997593,997291,992140,993143,994971,996592,999290,995886,990580,997320,997370,994360\n", + "Ground Truth: 997913,994964,997371,993237,995515,994778,994475,991801,996727,997350,996811,990142,996840,991727,995928,998359,996001,996438,994277,994415,997593,997291,999426,992140,993143,994971,996592,999290,995886,990580\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 85\n", + "Returned IDs: 995207,991701,996395,998586,991987,996623,992136,995056,999117,995373,994063,990292,994413,998858,998257,994451,998066,991052,994187,999249,994092,999096,997110,998659,992604,993115,997918,999834,994753,993515\n", + "Ground Truth: 995207,991701,996395,998586,991987,996623,992136,995056,999117,995373,994063,990292,994413,998858,998257,994451,998066,991052,994187,999249,994092,995617,999096,997110,998659,992604,993115,997918,999834,994753\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 88\n", + "Returned IDs: 992931,995266,995319,994052,995942,997147,991616,991593,992954,997913,990922,992964,991250,993031,990114,993606,999559,995632,998899,996508,999945,994668,992255,993278,998562,995984,990949,992228,998619,998056\n", + "Ground Truth: 992931,995266,995319,994052,995942,997147,991616,991593,992954,997913,990922,992964,991250,993031,990114,993606,990653,996629,999559,995632,998899,996508,999945,994668,992255,993278,998562,995984,990949,992228\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 89\n", + "Returned IDs: 995755,997437,993484,995244,998649,994894,996630,992296,997320,999740,991999,998246,991991,994814,995435,994359,999599,997304,999708,993696,996053,996436,997305,995841,990949,990786,996075,992924,993410,990307\n", + "Ground Truth: 995755,997437,993630,993484,995244,998649,994894,996630,992296,997320,999740,991999,998246,991991,994814,995435,994359,999599,997304,998855,999708,993696,996053,996436,997305,995841,990949,990786,993412,996075\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 91\n", + "Returned IDs: 998105,994307,993053,997790,991021,993906,996028,995823,991451,996147,990096,995586,995295,993289,992480,995456,995401,998677,999444,994689,996062,996001,991685,994280,992408,990532,992072,993433,998382,993561\n", + "Ground Truth: 998105,994307,993053,997790,991021,993906,996028,995823,991451,996147,990096,995586,995295,993289,992480,995456,995401,998677,999444,994689,996062,996001,991685,994280,993778,992408,990532,992072,993433,998382\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 93\n", + "Returned IDs: 998779,991743,996727,993412,996976,993673,991268,992087,995274,997640,999026,998649,999074,992511,997355,995034,992685,998862,997429,992147,998495,990406,994301,995616,997799,992637,992630,995755,996918,990949\n", + "Ground Truth: 998779,991743,996727,993412,996976,990401,993673,991268,992087,995274,997640,999026,998649,999074,992511,997355,995034,992685,998862,997429,992147,998495,990406,994301,995616,996438,997799,992637,992630,995755\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 100\n", + "Returned IDs: 996351,998837,994703,998363,991193,999738,993708,994789,995515,997496,993896,992384,995489,991840,990138,990490,991609,993143,990134,999453,991485,999613,990809,998657,998380,994001,992611,993577,994922,990310\n", + "Ground Truth: 996351,998837,994703,998363,991193,999738,993708,994789,995515,997496,993896,992384,995489,991840,990138,990490,991609,993143,990134,999453,991485,999613,990809,998657,998380,994001,992611,993577,994922,991930\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 102\n", + "Returned IDs: 999639,998793,992824,996196,997775,994270,992306,990538,992802,999908,996742,994560,990084,997547,992030,995631,994305,991923,995955,997215,990720,999451,997864,990755,993077,999564,992190,991432,992458,991910\n", + "Ground Truth: 999639,998793,992824,996196,997775,994270,992306,990538,992802,999908,996742,994560,990084,997547,992030,995631,994305,991923,995955,997215,990720,999451,997864,990755,993077,999564,992190,990409,991432,990146\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 104\n", + "Returned IDs: 997554,999453,995216,998495,996147,990845,996001,999402,992979,993412,994531,996368,991585,996810,991790,993003,992792,992334,990441,993561,991943,991967,996759,990662,992364,997031,993289,996911,995289,999866\n", + "Ground Truth: 997554,999453,995216,997937,998495,996147,990845,996001,999402,992979,993412,994531,998166,996368,991585,996810,991790,993003,992792,992334,990441,993561,991943,991967,996759,990662,992364,997031,993289,996911\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 105\n", + "Returned IDs: 991873,997913,991090,993834,999982,992822,992096,992228,998500,991626,995802,993796,997483,991616,992066,999165,992742,997320,990650,993616,997470,995513,996959,990182,997321,997759,990588,999710,998090,995949\n", + "Ground Truth: 991873,997913,991090,993834,999982,992822,999247,992096,992228,998500,993048,991626,995802,993796,997483,991616,992066,999165,992742,997320,990650,993616,997470,995513,996959,990182,997321,997759,990588,999710\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 107\n", + "Returned IDs: 994916,995274,994964,999144,999026,997015,999437,998779,992630,990535,993289,998951,993168,996959,997918,992047,994197,993492,991467,992216,995755,999596,992384,999374,997002,993484,990958,995336,990957,997320\n", + "Ground Truth: 994916,995274,994964,999144,999026,997015,999437,998779,992630,990535,993289,998951,993168,996959,997918,992047,997920,994197,993492,991467,999916,992216,995755,999596,992384,999374,997002,993484,990958,995336\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 108\n", + "Returned IDs: 994379,997319,991973,996411,990223,995611,990675,995819,991810,997822,991943,995289,994382,993909,993831,998156,995456,991350,992561,997732,993561,995633,994783,998750,990889,996906,995020,998765,997797,990277\n", + "Ground Truth: 994379,997319,991973,996411,990223,995611,990675,995819,999627,991810,997822,991943,995289,994382,993909,993831,998156,995456,991350,992561,997732,993561,995633,994783,998750,996607,990889,996906,995020,998765\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 109\n", + "Returned IDs: 999898,997028,996996,991311,994541,993695,997410,997920,993472,990974,997345,991339,998302,994783,996939,999343,998169,998039,998122,990210,993807,995384,999009,996062,996219,990505,997878,991776,992120,996216\n", + "Ground Truth: 999898,997028,996996,991311,994541,993695,997410,997920,993472,990974,997345,991339,991923,998302,994783,996939,999343,998169,998039,998122,990210,992931,993807,995384,999009,996062,996219,990505,995620,997878\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 110\n", + "Returned IDs: 997028,990557,996001,996219,999948,994777,998039,991498,999058,998765,995053,991930,994964,992446,993885,990752,994487,995102,997876,991523,991776,991212,997320,997332,996318,992182,991184,991921,995216,999617\n", + "Ground Truth: 997028,990557,996001,997381,996219,999948,994777,998039,991498,999058,998765,994061,995053,991930,995424,994964,992446,993885,990752,994487,995102,997876,991523,991776,991212,997320,997332,996318,992182,991184\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 112\n", + "Returned IDs: 995981,992470,992874,992384,995701,995792,990862,996001,998721,993432,993848,998246,991045,990588,998445,995467,994879,998549,993342,998921,995158,992167,997984,991459,997437,991245,996714,995336,995886,996075\n", + "Ground Truth: 993154,995981,992470,992874,992384,995701,995792,990862,996001,998721,993432,993848,999904,998246,991045,990588,998445,995467,994879,998549,993342,999977,998921,995158,997984,992167,991459,997437,991245,996714\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 113\n", + "Returned IDs: 996425,994819,994334,995453,997918,994804,994088,996046,997002,993465,996903,994362,999964,990995,995200,999277,995373,993407,995444,995374,994446,999089,999595,991603,990791,995928,998530,991942,997637,996864\n", + "Ground Truth: 996425,994819,994334,995453,997918,994804,994088,996046,997002,993465,996903,994362,999964,990995,995200,999277,995373,995444,993407,995374,994446,999089,999595,991603,990791,995928,998530,991942,997637,992260\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 116\n", + "Returned IDs: 998523,994922,992907,993355,996840,990043,993237,990334,997876,990332,990448,995004,999453,999620,997918,990432,990925,991510,996196,990538,990483,992612,996438,991609,997496,998401,999076,998837,993143,996402\n", + "Ground Truth: 998523,994922,992907,993355,996840,990043,993237,990334,997876,990332,990448,995004,999453,999620,997918,990432,990925,991510,996196,990538,990483,992612,996438,991609,997496,998401,995401,999076,998837,993143\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 117\n", + "Returned IDs: 996194,999601,998823,993014,994541,995695,990521,990169,995421,996396,994119,996429,998934,994280,995401,999496,994051,998546,994817,997316,990011,998250,992052,998968,991245,992313,992131,996633,998817,992437\n", + "Ground Truth: 996194,999601,998823,993014,994541,995695,990521,990169,995421,996396,994119,998348,996429,998934,994280,995401,999496,994051,998546,994817,997316,990011,998250,992052,998968,991245,992313,992131,996633,998817\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 119\n", + "Returned IDs: 995614,993426,990191,997873,997820,996516,997292,990855,994415,998921,997483,998721,996387,999109,999271,994618,995623,996296,996771,999792,994233,992853,991131,995306,993484,993785,995119,998445,996135,997550\n", + "Ground Truth: 995614,993426,990191,997873,997820,996516,997292,990855,994415,998921,997483,998721,996387,999109,999271,994618,995623,996296,996993,996771,999792,994233,992853,991131,995306,993484,993785,995119,998445,996135\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 127\n", + "Returned IDs: 998634,996001,997320,998246,993484,992839,991787,993053,994964,993280,991155,994961,997332,999099,990585,990146,993168,993885,991498,995678,999453,991721,996219,990772,990096,993389,990269,991732,998927,991609\n", + "Ground Truth: 998634,996001,997320,998246,993484,992839,991787,993053,994964,993280,991155,994961,997332,999099,990585,990146,993168,993885,991498,995678,999453,991721,991184,996219,990772,990096,993389,993154,990269,991732\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 128\n", + "Returned IDs: 997809,996546,991923,992718,998975,999380,993681,998331,991977,996163,990908,994398,993786,995138,993572,995986,994852,996658,992170,996158,995578,991157,999387,993975,998026,996894,995737,992349,996585,999806\n", + "Ground Truth: 997809,996546,991923,992718,998975,999380,993681,998331,991977,996163,990908,994398,993786,995138,993572,995986,994852,996658,992170,996158,995578,991157,998921,999387,993975,998026,996894,995737,992349,996585\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 129\n", + "Returned IDs: 992255,990650,996876,995794,990114,992964,992822,991873,998947,997451,999948,995485,997102,997948,998003,993083,997966,991616,999110,998434,990742,990066,995109,998540,990699,994461,990311,991178,995904,990030\n", + "Ground Truth: 992255,990650,996876,995794,990114,992964,992822,991873,998947,997451,999948,995485,997102,997948,998003,993083,997966,991616,993048,999110,995367,998434,990742,990066,995109,998540,990699,994461,991396,990311\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 131\n", + "Returned IDs: 998787,993366,994834,990952,995393,999373,998565,990167,994833,995326,999555,996553,998959,996408,990358,995299,990173,995231,992282,995835,993528,993560,994756,994374,993014,998536,995168,994682,994150,993948\n", + "Ground Truth: 998787,993366,994834,990952,995393,999373,998565,990167,994833,995326,999555,996553,998959,996408,990358,995299,990173,995231,992282,995835,993528,993560,994756,991313,994374,993014,998536,995168,994682,994150\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 132\n", + "Returned IDs: 999236,996716,992790,999820,998723,991011,993281,996371,996067,992179,993504,994946,998303,996062,998401,992576,995401,993071,998045,994113,991857,993979,992367,997331,996220,994188,997002,994964,993492,995289\n", + "Ground Truth: 999236,996716,992790,999820,998723,991011,993281,996371,996067,992179,993504,994946,998303,996062,998401,992576,995401,993071,998045,994113,991857,990294,993979,992367,997331,996220,994188,990244,997002,994964\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 133\n", + "Returned IDs: 999503,997790,993055,999840,994280,997926,993489,991397,995203,992862,997897,992521,990399,990005,992958,998250,990802,997918,998303,991661,994042,994531,991630,994946,991332,998933,992747,996657,998697,996368\n", + "Ground Truth: 999503,997790,993055,999840,994280,997926,993489,991397,995203,992862,992979,997897,992521,990399,990005,992958,998250,990802,997918,998303,991661,994042,994531,991630,994946,991332,998933,992747,996657,993143\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 134\n", + "Returned IDs: 991227,992558,992052,993787,993478,992933,998536,996918,996531,995783,994280,995214,994972,995408,990956,992318,993673,991052,998104,998397,993121,992997,993747,996438,996658,995401,996930,997897,993727,996328\n", + "Ground Truth: 991227,992558,992052,993787,993478,992933,998536,996918,996531,995783,994280,995214,994972,995408,990956,992318,993673,991052,998104,998397,993121,992997,993747,996438,996658,995401,996930,997897,993727,992521\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 135\n", + "Returned IDs: 994406,992767,997799,998431,995403,997391,999960,990337,998779,997634,993026,994474,998080,994671,996727,996903,995374,995453,997012,995034,997157,997754,997640,993412,990647,993289,996172,995893,995607,998915\n", + "Ground Truth: 994406,992767,997799,998431,995403,997391,999960,990337,998779,997634,993026,994474,998080,994671,996727,996903,995374,996047,995453,997012,995034,997157,997754,997640,993412,990647,993289,996172,995893,995607\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 136\n", + "Returned IDs: 991324,999740,992698,993725,990011,990484,991390,994830,995756,990711,992479,997142,990743,994751,998710,990089,995002,991885,996358,999922,996633,997714,992664,991911,995435,992679,990439,992480,996001,997933\n", + "Ground Truth: 991324,999740,992698,993725,990011,990484,991390,994830,995756,990711,992479,997142,990743,994751,998710,990089,995002,991885,999555,996358,999922,996633,997714,992664,991911,995435,992679,990439,992480,996001\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 138\n", + "Returned IDs: 999350,994864,994488,995167,996001,992169,996406,999179,997679,991212,992182,998122,994510,991596,990557,993931,999130,999304,999623,994467,994119,990434,997332,996075,997395,993032,994416,998725,996775,991066\n", + "Ground Truth: 999350,994864,994488,995167,996001,992169,996406,999179,997679,991212,999679,992182,996803,998122,994510,991596,993931,990557,999130,999304,999623,994467,999099,994119,990434,997332,996075,997395,993032,994416\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 139\n", + "Returned IDs: 994168,992469,999500,995289,990126,997723,993656,990930,999586,996811,998913,995562,994783,991154,990340,993747,994742,996219,995095,994759,998495,997797,990523,998156,990266,993642,996116,999977,990399,998855\n", + "Ground Truth: 994168,992469,999500,995289,990126,997723,993656,990930,999586,996811,998913,995562,994783,991154,990340,993747,993400,994742,996219,995095,994759,997864,998633,998495,997797,990523,998156,990266,993642,996116\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 144\n", + "Returned IDs: 995515,992835,990155,998244,996419,993582,993028,994216,998837,990142,991149,999053,990313,993237,991999,997591,994612,994281,992020,990652,997403,994475,994497,990827,997465,990043,992662,994898,996485,991196\n", + "Ground Truth: 995515,992835,990155,998244,996419,993582,993028,994216,998837,990142,991149,999053,990313,993237,991999,997591,994612,994281,992020,990652,997403,990943,994475,994497,990827,997465,990043,992662,994898,996485\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 150\n", + "Returned IDs: 992678,996848,998067,997915,991044,990331,991747,992097,999980,992521,997675,992569,998045,995268,991103,990314,996368,996277,996505,998060,993256,992367,996719,995963,992187,994861,999402,997723,993313,994817\n", + "Ground Truth: 992678,996848,998067,997915,991044,990331,991747,992097,999980,992521,997675,992569,998045,995268,998021,991103,990314,996368,996277,996505,998060,993256,992367,996719,995963,992187,994861,999402,997723,993313\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 151\n", + "Returned IDs: 999578,992448,994409,993468,992660,992801,996618,997832,999043,996038,999530,998714,991699,997868,996813,997406,995139,998045,998979,999508,996607,991027,995507,994554,993433,996769,991671,993561,998545,999940\n", + "Ground Truth: 999578,992448,994409,993468,992660,992801,996618,997832,999043,996038,999530,998714,991699,997868,996813,997406,995139,998045,998979,999508,996607,991027,995507,994554,993433,996769,991671,993561,993280,998545\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 153\n", + "Returned IDs: 992239,995637,990264,990814,991981,991590,999924,999767,992713,994891,993080,997494,994968,991428,990516,991769,995012,994094,998999,992318,997698,990364,990270,992437,996728,998869,999942,992827,996658,994742\n", + "Ground Truth: 992239,995637,990264,990814,991981,991590,999924,999767,992713,994891,993080,997494,994968,991428,990516,991769,995012,994094,998999,992318,997698,990364,990270,992437,996728,998869,999942,990853,992827,996658\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 154\n", + "Returned IDs: 994168,990080,990266,994131,998633,991943,996093,999586,991255,994742,997723,999587,998156,991981,993080,993717,992582,999453,992169,991585,995295,994129,990957,998980,996867,991185,994431,996809,998256,995702\n", + "Ground Truth: 994168,990080,990266,994131,992655,998633,991943,996093,999586,991255,994742,997723,999587,998156,991981,993080,993717,992582,999453,992169,997360,991585,991334,995295,994129,990957,998980,996867,991185,994431\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 156\n", + "Returned IDs: 999267,993637,991128,993053,995283,995659,990089,990368,999338,995274,990629,996701,995345,991412,991992,993410,999806,992636,997691,997052,994386,995638,998302,995467,998021,999343,997340,999446,996368,998215\n", + "Ground Truth: 999267,993637,991128,993053,995283,995659,990089,990368,999338,995274,990629,996701,990133,995345,991412,991992,993410,999806,992636,997691,997052,994386,995638,998302,995467,998021,999343,990260,997340,999446\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 158\n", + "Returned IDs: 990890,996219,991928,998899,992931,991249,998540,994831,994069,999300,990545,999068,999343,995942,994964,992330,991593,991743,996343,997913,993282,990404,994486,998550,999620,997305,990210,998837,990444,996646\n", + "Ground Truth: 990890,996219,991928,998899,992931,991249,998540,994831,994069,999300,990545,999068,999343,995942,994964,992330,991593,991743,996343,997913,993282,990404,994486,998550,990666,999620,991006,995210,997305,990210\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 159\n", + "Returned IDs: 992094,994119,996245,997944,998150,994177,994193,990226,990527,994280,999461,992703,992052,998755,993037,992437,994968,997309,997897,993483,993915,992652,997056,993288,995401,998355,995408,996277,990784,996775\n", + "Ground Truth: 992094,994119,996245,997944,998150,994177,994193,990226,990527,994280,999461,992703,992052,998755,993037,992437,990790,994968,997309,997897,993483,993915,992652,997056,993288,995401,998355,995408,996277,990784\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 160\n", + "Returned IDs: 992604,993754,995807,995527,999662,999445,997781,990915,990540,995953,991051,990657,990729,991204,991110,991571,994187,997918,997212,995373,994753,998858,991987,996043,999306,995805,997391,997675,995105,998697\n", + "Ground Truth: 992604,993754,995807,995527,999662,999445,997781,990915,990540,995953,991051,990657,990729,998297,991204,991110,991571,994187,997918,997212,995373,994753,998858,991987,996043,997158,999306,995805,997391,997675\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 161\n", + "Returned IDs: 998294,996504,999765,990917,997913,994266,998921,993518,991406,997984,991895,990760,999263,993669,998500,995755,990925,994978,990829,995090,998987,990936,997192,990595,996560,997371,998540,995793,998344,992789\n", + "Ground Truth: 998294,996504,999765,990917,997913,994266,998921,993518,991406,997984,991895,990760,999263,993669,998500,995755,990925,994978,990829,995090,998987,990936,997192,990595,996560,997371,998540,995793,998344,999647\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 162\n", + "Returned IDs: 995110,993478,991103,996505,995032,998237,998397,995783,999646,991747,995401,997675,995701,993727,990537,995521,991413,991010,990956,992457,997918,997752,991052,995184,991054,994232,998045,990872,992790,993412\n", + "Ground Truth: 995110,993478,991103,996505,995032,998237,998397,995783,999646,991747,995401,997675,995701,993727,990537,995521,991413,991010,990956,992457,993738,991630,997918,997752,991052,995184,991054,994232,992260,998045\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 163\n", + "Returned IDs: 998914,990678,996556,996028,998355,997380,995528,999891,998630,990226,991426,996190,993725,992521,994765,995317,996505,996016,992488,996728,994119,998325,999479,992480,993820,995449,995456,998982,999516,991747\n", + "Ground Truth: 998914,990678,996556,996028,998355,997380,995528,999891,998630,990226,991426,996190,993725,992521,994765,995317,996505,996016,992488,996728,992322,994119,998325,999479,992480,993820,995449,995456,993336,998982\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 170\n", + "Returned IDs: 999430,995332,995107,991179,991961,991458,994897,998416,998950,990377,993499,995560,996216,993807,995895,997331,990091,995750,993617,993079,990769,992780,997004,993479,996944,999210,992567,998157,996370,991382\n", + "Ground Truth: 999430,995332,995107,991179,991961,991458,994897,998416,998950,990377,993499,995560,996216,993807,995895,997331,990091,995750,993617,993079,990769,992780,997004,993479,996944,999210,992567,993335,998157,996370\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 171\n", + "Returned IDs: 996219,994990,994964,993378,991075,996052,994843,992047,996593,997028,999453,995096,993342,996001,998039,990448,992780,990366,997920,994777,994472,999352,992261,994922,990091,993648,999437,999344,998464,998495\n", + "Ground Truth: 996219,994990,994964,993378,991075,996052,994843,997616,992047,996593,997028,999453,995096,993342,996001,998039,990448,992780,990366,997920,994777,994472,999352,992261,994922,990091,993648,998633,999437,999344\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 172\n", + "Returned IDs: 990016,998105,992201,993588,994145,993053,995097,990435,990497,996147,990536,994307,998548,991075,990662,994830,995289,994925,990771,996974,999460,993831,998097,990811,996001,993412,995823,996411,998950,998495\n", + "Ground Truth: 990016,998105,992201,993588,994145,993053,995097,990435,990497,996147,990536,994307,998548,991075,990662,994830,995289,994925,990771,996974,999460,998165,993831,998097,990811,996001,993412,995823,996411,998950\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 175\n", + "Returned IDs: 994894,990689,997192,998974,995274,999444,994266,990957,993186,990498,997035,998326,997437,998697,991630,990337,995536,996699,992811,990231,992357,997320,990958,996785,998649,993412,990799,998170,990146,992692\n", + "Ground Truth: 994894,990689,997192,998974,995274,999444,994266,990957,993186,990498,997035,998326,997437,998697,991630,990337,990862,995536,996699,992811,990231,992357,997320,990958,996785,998649,993412,990799,998170,990146\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 178\n", + "Returned IDs: 994254,997809,998686,990487,991977,992889,997407,991923,998046,990776,995691,994165,998243,994760,990343,996068,999766,998975,993839,990522,993683,997914,998357,993572,994010,995965,995506,996895,992645,998118\n", + "Ground Truth: 994254,997809,998686,990487,991977,992889,997407,991923,998046,990776,995691,994165,998243,994760,990343,996068,999766,998975,993839,990522,993683,997914,998357,993572,994010,995965,995506,999913,996895,992645\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 179\n", + "Returned IDs: 990155,993280,999453,996810,992395,998734,990772,996001,991375,991951,995988,990142,994536,996915,996732,997876,991517,998567,995216,990778,995130,999076,991840,991255,991248,999019,997320,998605,991999,992944\n", + "Ground Truth: 990155,993280,999453,996810,992395,998734,990772,996001,991375,991951,995988,990142,994536,996915,996732,997876,991517,998567,995216,990778,995130,999076,991840,998523,991255,991248,999019,997320,999723,993154\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 181\n", + "Returned IDs: 995895,996706,997442,998593,993805,993391,995438,992726,990844,998993,990133,999430,994431,994351,994742,996809,999435,999179,996930,998934,998989,994830,993335,997723,992864,997331,998055,994382,996830,992850\n", + "Ground Truth: 995895,996706,997442,998593,993805,993391,995438,992726,990844,998993,990133,999430,994431,994351,994742,996809,999435,999179,996930,998934,999587,998989,998990,994830,993335,997723,992864,997331,998055,994382\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 183\n", + "Returned IDs: 992002,991507,998162,995493,996607,998910,999054,997466,992448,990163,995458,996987,993046,990295,991671,991585,992260,994013,994865,998948,990924,990804,996042,996815,991603,998283,999475,995216,995766,993561\n", + "Ground Truth: 992002,991507,998162,995493,996607,998910,999054,997466,992448,990163,995458,996987,993046,990295,991671,991585,997529,992260,994013,994865,998948,990924,993883,990804,996042,996815,991603,998283,999475,995216\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 184\n", + "Returned IDs: 996689,990328,990600,994255,992445,999025,995456,994689,992052,995295,991277,992068,996028,997329,991451,995832,997309,994280,998959,993560,997420,995166,990532,998589,995823,995273,997897,998182,995203,994946\n", + "Ground Truth: 996689,990328,990600,994255,992445,999025,995456,994689,992052,995295,991277,992068,996028,997329,996110,991451,995832,997309,994280,998959,993560,997420,995166,990532,998589,995823,995273,997897,998182,995203\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 187\n", + "Returned IDs: 997391,997918,994334,990215,996282,994819,993492,995886,996903,999562,997702,992979,992179,999820,995401,992211,993512,997782,994088,994319,992428,999144,994916,992933,994906,998892,990561,998011,994670,996658\n", + "Ground Truth: 997391,997918,994334,990215,996282,994819,993492,995886,996903,999562,997702,992979,992047,992179,999820,995401,992211,993512,997782,994088,994319,992428,999144,994916,992933,994906,998892,990561,998011,994670\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 188\n", + "Returned IDs: 997106,992190,998409,995256,990675,992804,993791,994852,991958,991923,999639,997864,994043,993342,991208,995701,996368,993283,992167,991530,997407,990622,998079,995138,997739,995898,991350,994105,996894,997577\n", + "Ground Truth: 997106,992190,998409,995256,990675,992804,993791,998974,994852,991958,991923,999639,997864,994043,993342,991208,995701,996368,993283,992167,994796,991530,997407,990622,998079,995138,997739,995898,991350,994105\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 191\n", + "Returned IDs: 999172,991585,998060,994013,997554,999570,999402,995544,996147,993377,993053,996001,993217,993209,991185,990887,997059,990331,997529,992260,997566,994203,996752,993561,992450,995873,998162,993021,995216,994488\n", + "Ground Truth: 999172,991585,998060,994013,997554,999570,999402,995544,996147,993377,993053,996001,993217,993209,991185,990887,997059,990331,997529,992260,997566,994203,996752,993561,992450,993166,995873,998162,993021,995216\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 192\n", + "Returned IDs: 991372,993696,990628,994722,990885,996809,996296,991109,992047,992635,997098,996452,990697,991172,998734,994922,995210,999219,992927,995530,996875,995596,996913,999337,996350,998790,993504,993038,991425,990627\n", + "Ground Truth: 991372,993696,990628,994722,990885,996809,996296,997166,991109,992047,992635,997098,996452,990697,991172,998734,994922,995210,999219,992927,995530,994859,996875,995596,996913,998303,999337,996350,998790,993504\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 199\n", + "Returned IDs: 990382,997476,991581,999827,996062,992314,993981,994510,998095,994463,998436,992161,993289,990899,990520,991930,997790,997642,993931,998158,992169,995954,997453,991596,995636,990557,990212,995185,991644,994411\n", + "Ground Truth: 990382,997476,991581,999827,996062,992314,993981,994510,998095,994463,998436,992161,993289,990899,990520,991930,997790,996913,997642,993931,998158,992169,995954,997453,991596,995636,990557,990212,995185,991644\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 200\n", + "Returned IDs: 997320,997370,996811,992944,997305,998950,991498,995886,993018,994266,995372,992041,990949,995250,990772,999453,995203,997301,995927,993154,991801,999210,993260,990532,997470,995438,997913,993241,995841,997371\n", + "Ground Truth: 997320,997370,996811,992944,997305,998950,991498,995886,993018,994266,995372,992041,990949,995250,990772,999453,995203,997301,995552,990917,995927,993154,991801,999210,993260,990532,997470,995438,997913,993241\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 203\n", + "Returned IDs: 992927,993260,994988,990429,994964,994537,996193,990786,996053,997320,999216,996576,993055,990885,999176,992069,998568,997918,993783,991795,991675,992571,995536,992497,994789,992225,999711,994552,998299,999800\n", + "Ground Truth: 992927,993260,994988,990429,994964,994537,996193,990786,996053,997320,999216,996576,999362,993055,990885,999176,992069,998568,994984,997918,993783,991795,991675,992571,995536,992497,994789,992225,999711,994552\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 204\n", + "Returned IDs: 990487,994117,996391,998370,995896,991350,992929,997554,999587,990211,997868,994793,994207,996368,996379,993998,994809,996654,996657,997723,990314,998990,996127,991973,995873,998686,996181,993831,991585,999639\n", + "Ground Truth: 990487,994117,996391,998370,995896,991350,992929,997554,999587,990211,997868,994793,994207,996368,996379,993998,994809,996654,996657,997723,990314,998990,996127,991973,995873,998686,996181,993831,991585,992423\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 208\n", + "Returned IDs: 994273,999415,993127,995122,992449,995423,995702,994013,999745,995701,999054,991636,996867,995182,996897,995685,998636,990082,995905,997077,994954,990152,994409,998776,995401,997975,998518,997674,996232,993856\n", + "Ground Truth: 994273,999415,993127,995122,992449,995423,995702,994013,999745,995701,999054,991636,996867,995182,996897,995685,998636,990082,995905,997077,994954,990152,994409,998776,995401,997975,998518,997674,995303,996232\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 211\n", + "Returned IDs: 999453,994167,991987,990999,991811,996161,991894,995596,996001,991459,999219,995899,990949,996657,990957,991445,995855,992979,996368,995281,990542,997320,997258,992562,991835,993848,995866,999639,996744,992384\n", + "Ground Truth: 999453,994167,991987,990999,998567,991811,996161,991894,995596,996001,991459,993154,999219,995899,990949,996657,990957,991445,995855,992979,996368,995281,990542,997320,997258,992562,991835,993848,995866,999639\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 212\n", + "Returned IDs: 992824,995459,990894,996380,999639,991272,995631,995503,991064,996742,990273,991432,996663,999908,998061,999387,997547,996894,991817,995728,993055,999674,993839,994560,993225,999346,998720,990061,992306,995724\n", + "Ground Truth: 992824,995459,990894,996380,999639,991272,995631,995503,991064,996742,990273,991432,996663,999908,998061,999387,997547,996894,991817,995728,993055,999674,993839,994560,993225,999346,992752,998720,990061,992306\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 213\n", + "Returned IDs: 993879,995080,997675,995521,992801,991904,996198,990244,999082,994170,994409,996505,995401,993030,990527,996277,998045,993561,992448,995873,997822,998049,997832,991603,996472,994677,997020,997766,992260,999461\n", + "Ground Truth: 993879,995080,997675,995521,992801,991904,996198,990244,999082,994170,994409,996505,995401,993030,990527,996277,998045,993561,992448,995873,997822,998049,997832,991603,996472,994677,997020,997766,992260,990140\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 215\n", + "Returned IDs: 996062,998122,998495,999898,994869,990091,990194,990096,997242,990434,999827,991893,993106,999555,996219,996001,996028,995655,995536,992056,994266,995562,999139,990167,990382,990796,998593,994200,997790,996677\n", + "Ground Truth: 996062,998122,998495,999898,994869,990091,990194,990096,997242,990434,999827,991893,993106,999555,996219,996001,996028,998861,995655,995536,992056,994266,995562,999139,990167,990382,990796,998593,994200,997790\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 216\n", + "Returned IDs: 997429,995184,993238,991856,990958,992357,993648,994835,999533,995963,994266,995034,996861,992008,993168,995800,994956,998458,990925,995927,997852,995783,993885,992526,999374,992147,995884,990399,997309,991468\n", + "Ground Truth: 997429,995184,993238,991856,990958,992357,993648,994835,999533,995963,994266,995034,996861,991714,992008,993168,995800,994956,998458,990925,995927,997852,995783,993885,992526,999374,992147,995884,990399,997309\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 218\n", + "Returned IDs: 990711,990392,997529,991703,993053,998862,993725,996411,994730,995002,992156,993240,998913,999302,999377,995633,994742,990701,998495,998779,994830,995401,990435,993166,993264,995096,992680,996408,991390,995601\n", + "Ground Truth: 990711,990392,997529,991703,993053,998862,993725,996411,994730,995002,992156,993240,998913,999302,999377,995633,994742,990701,998495,998779,994830,995401,990435,993166,993264,995096,992680,996408,993856,991390\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 220\n", + "Returned IDs: 994643,998536,997752,993232,999117,998383,999155,990551,995625,993995,996809,992288,998695,996797,991532,992558,991630,997666,995963,990941,994063,991571,996043,997131,991675,994742,995401,992604,992809,996395\n", + "Ground Truth: 994643,998536,997752,993232,999117,998383,999155,990551,995625,993995,996809,992288,998695,996797,991532,992558,991630,997666,995963,990941,994063,991571,996043,993077,997131,991675,994742,995401,992604,992809\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 221\n", + "Returned IDs: 995596,995515,993143,991840,991848,999122,999784,999806,996840,990432,994709,997403,993748,995443,996219,995365,999676,992979,999144,991395,991485,997135,997379,999613,991977,992309,996062,997918,994916,993237\n", + "Ground Truth: 995596,995515,993143,991840,991848,999122,999784,999806,996840,990432,994709,997403,993748,995443,996219,995365,999676,992979,999144,991395,991485,992047,997135,997379,999613,991977,992309,996062,997918,994916\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 223\n", + "Returned IDs: 997030,990096,994717,994643,994413,993666,992638,993115,998257,991309,994454,999898,990815,997077,997154,991190,998277,999555,997211,993566,992369,995218,994677,998045,991596,991188,997568,990361,994569,998536\n", + "Ground Truth: 997030,990096,994717,994643,994413,993666,992638,993115,999530,998257,991309,994454,999898,990815,997077,997154,991190,997871,998277,999555,997211,993566,992369,995218,994677,998045,991596,991188,997568,990361\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 224\n", + "Returned IDs: 990049,995609,996194,999528,995043,995551,994326,993678,991241,996232,990417,993181,990011,996435,992566,994541,992068,999830,998769,992473,990121,993245,994317,995475,995537,993315,996867,992563,995317,992951\n", + "Ground Truth: 990049,995609,996194,999528,995043,995551,994326,993678,991241,996232,990417,993181,990011,996435,992566,994541,992068,999830,998769,992473,990121,993245,994317,995475,995537,993315,997906,990153,996867,992563\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 227\n", + "Returned IDs: 992511,998974,996252,997437,993457,990337,999867,994894,995838,997174,996917,997728,999599,993978,995755,998649,994916,990957,997320,996648,996192,995616,995274,995963,997304,994630,995374,991995,992531,993049\n", + "Ground Truth: 992511,998974,996252,997437,993457,990337,999867,994894,995838,997174,996917,997728,993222,999599,993978,995755,998649,994916,990957,997429,997320,996648,996192,995616,995274,995963,997304,994630,995374,991995\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 228\n", + "Returned IDs: 998060,993487,994203,992352,994106,992098,990573,992000,991057,996075,994488,996001,995079,995544,991212,994013,990662,990314,999172,995515,997998,998593,996898,995873,994954,992569,995640,992097,991394,999590\n", + "Ground Truth: 998060,993487,994203,992352,994106,992098,990573,992000,991057,996075,994488,996001,995079,995544,991212,994013,990662,990314,999172,996043,995515,997998,998593,996898,995873,994954,992569,995640,992097,991394\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 229\n", + "Returned IDs: 992558,990279,997547,997124,996196,990538,999715,995336,999908,995886,996606,993948,994334,997859,992718,990989,992971,991157,991052,996609,996918,999860,992857,990930,995552,996721,994163,993363,990343,992521\n", + "Ground Truth: 992558,990279,997547,997124,996196,990538,999715,995336,999908,995886,996606,993948,994334,997859,992718,990989,992971,991157,991052,996609,996918,999860,992857,990930,995552,996721,994163,993363,990343,991498\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 230\n", + "Returned IDs: 994266,995184,997320,997913,990231,998170,992546,991256,999647,992330,995701,993853,997305,995417,996296,992637,996075,997554,999761,990958,996661,993834,997449,990650,996083,990535,993168,992748,991045,994476\n", + "Ground Truth: 994266,995184,997320,997913,990231,998170,992546,991256,999647,992330,995701,993853,999041,997305,995417,996296,992637,996075,997554,999761,990958,996661,993834,997449,990650,996083,993168,990535,992748,991045\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 231\n", + "Returned IDs: 998913,994783,998823,998370,998196,995551,991428,994742,992566,991981,994806,994692,995289,995397,995348,992415,995948,993939,995030,992827,997630,996411,991330,995633,998543,995300,994110,998028,996609,994587\n", + "Ground Truth: 998913,994783,998823,998370,998196,995551,991428,994742,992566,991981,994806,994692,995289,995397,990846,995348,992415,995948,993939,995030,992827,997630,996411,991330,995633,998543,995300,994110,998028,996609\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 233\n", + "Returned IDs: 992981,992445,997806,996328,994336,990956,996728,996861,998017,994125,993055,990005,997420,997884,996211,996918,997506,994142,999775,993948,996189,998959,993560,992293,990961,993877,998495,998067,995516,994280\n", + "Ground Truth: 992981,992445,997806,996328,994336,990956,996728,996861,998017,992979,994125,993055,990005,997420,997884,996211,996918,997506,994142,999775,993948,996189,998959,993560,992293,990961,993877,998495,998067,995516\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 236\n", + "Returned IDs: 998971,991596,992673,999663,999139,994510,996406,997127,999898,996985,996706,996809,999555,996733,991420,997980,998149,999435,992365,998006,996209,994335,992234,997242,999498,990226,995104,997506,991589,994103\n", + "Ground Truth: 998971,991596,992673,999663,999139,994510,996406,997127,999898,996985,996706,996809,999555,996733,991420,997980,998149,999435,992365,998006,996209,994335,992234,997242,999498,990226,995104,997506,991589,999783\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 240\n", + "Returned IDs: 997397,999110,999523,997301,999471,992451,995623,999904,991916,993484,990146,995306,990546,998760,997470,990405,997820,993426,993083,997292,995264,997320,992066,999242,995045,991582,990182,990191,996516,993853\n", + "Ground Truth: 997397,999110,999523,997301,999471,992451,995623,999904,991916,993484,990146,995306,990546,998760,997470,990405,997820,993426,993083,997292,995264,997320,992066,999242,995045,999247,991582,990182,990191,996516\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 242\n", + "Returned IDs: 995659,999271,990585,997668,997286,990089,999338,990000,994942,994618,997292,997984,995841,997470,997301,996516,995306,992069,991503,997320,998721,998921,995372,991570,998923,998445,998461,993419,994964,997113\n", + "Ground Truth: 995659,999271,990585,997668,997286,990089,999338,990000,994942,994618,997292,997984,995841,997470,997301,996516,995306,992069,991503,997320,998721,998921,995372,991570,998923,998445,998461,993419,994964,992451\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 247\n", + "Returned IDs: 995473,992815,991045,992694,999616,998524,990588,997620,995981,991900,997304,992874,997437,993300,997320,993392,997913,998365,997353,999291,999725,998246,992074,990535,998353,995701,991745,992384,992927,994266\n", + "Ground Truth: 995473,992815,991045,992694,999616,998524,990588,997620,995981,991900,997304,992874,997437,993300,997320,993392,997913,998365,997353,999291,999725,998246,992074,998380,990535,998353,995701,991745,992384,992927\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 249\n", + "Returned IDs: 997529,995020,990711,998719,999500,992156,991189,994120,998495,994382,993412,993725,996411,997207,995633,990126,991881,991669,990435,991075,993439,998671,994730,993747,998779,998982,998862,990256,995131,990340\n", + "Ground Truth: 997529,995020,990711,998719,999500,992156,991189,994120,998495,994382,993412,993725,996411,997207,995633,990126,991881,991669,990435,991075,993439,997485,998671,994730,993747,998779,998982,993656,998862,990256\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 251\n", + "Returned IDs: 999826,997395,990790,999020,998080,994612,994170,996245,997184,993931,995807,997986,992879,998820,994770,995163,998049,992790,990243,997189,994280,998014,993149,992683,999967,991698,999898,995521,997918,993933\n", + "Ground Truth: 999826,997395,990790,999020,998080,994612,994170,996245,997184,993931,995807,997986,992879,998820,994770,997301,995163,998049,992790,990243,997189,994280,998014,993149,992683,999967,991698,999898,995521,997918\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 258\n", + "Returned IDs: 990827,990043,998983,998937,990849,997370,996840,998551,997789,995988,999620,998734,991196,994922,999898,990858,997445,995515,993282,998236,991906,991348,997255,990500,993647,992169,993143,993237,995443,990701\n", + "Ground Truth: 990827,990043,998983,998937,990849,997370,996840,998551,997789,995988,999620,994735,998734,991196,994922,999898,990858,997445,995515,993282,998236,991906,991348,997255,990500,993647,992169,993143,993237,995443\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 259\n", + "Returned IDs: 996219,993168,995876,993637,995001,999647,998343,993499,990173,999009,995689,998495,997331,999673,996067,990784,999430,991037,995562,996549,991758,996062,994279,991109,992589,995438,990323,991372,999748,992960\n", + "Ground Truth: 996219,993168,995876,995001,993637,999647,998343,993499,990173,999009,995689,998495,997331,999673,996067,990784,999430,997992,991037,995562,996549,991758,996283,996062,994279,991109,992589,995438,990323,991372\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 261\n", + "Returned IDs: 998917,997220,994617,994491,997136,995259,998782,998139,991727,995588,998983,994534,992255,994741,990822,991999,999469,992085,998928,995334,991806,991405,993340,990809,994528,998236,990890,994785,999147,995316\n", + "Ground Truth: 998917,997220,994617,994491,997826,997136,995259,998782,998139,991727,995588,998983,994534,992255,994741,990822,991999,999469,992085,998928,995334,999535,991806,991405,993340,990809,994528,998236,990890,994785\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 264\n", + "Returned IDs: 991137,996653,999533,998302,993845,997855,997320,999109,993579,996726,993066,997381,995569,995841,994978,999338,994332,999827,991714,997984,992318,993807,992228,995659,998445,992647,998797,998349,996729,990146\n", + "Ground Truth: 991137,996653,999533,998302,993845,997855,997320,997921,999109,993579,996726,990179,993066,997381,995569,995841,994978,999338,994332,999827,991714,997984,991787,992318,993807,992228,995659,998445,992647,998797\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 266\n", + "Returned IDs: 999356,998353,995981,998163,993853,998246,997728,992296,990453,994879,995755,990231,998662,992074,996075,993529,993186,990708,996609,994100,999761,996384,992848,990799,991895,999740,997192,998950,998332,991721\n", + "Ground Truth: 999356,998353,995981,998163,993853,998246,997728,992296,990453,994879,995755,990231,998662,991782,992074,996075,993529,993186,991913,990708,996609,994100,999761,996384,992848,990799,991895,999740,997906,997192\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 268\n", + "Returned IDs: 992944,995360,991848,993781,996829,990043,999784,998991,994543,994341,991013,999210,997135,993708,998950,993260,996193,991416,996802,998180,990182,995375,999122,996007,993365,997037,993633,997457,998726,993279\n", + "Ground Truth: 992944,995360,991848,993781,996829,990043,999784,998991,994543,994341,991013,999210,999577,997135,993708,998950,993260,996193,991416,996802,998180,990182,992168,995375,999122,996007,993365,997037,993633,997457\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 269\n", + "Returned IDs: 996429,997918,995701,998932,994276,995785,996217,992047,998303,991530,994832,994221,993260,990409,993055,995932,990930,998123,999384,994754,992168,998697,997897,990751,996196,993504,995203,992571,997586,990494\n", + "Ground Truth: 996429,997918,995701,998932,994276,995785,996217,992047,998303,991530,994832,994221,993260,990409,993055,995932,990930,998123,999384,994754,992168,998697,997897,990751,996196,993504,995203,994514,990425,992571\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 270\n", + "Returned IDs: 990530,992562,997925,999639,995701,995401,995005,999041,990588,993821,995886,990217,991154,993809,990930,996067,998200,991459,993834,997337,999680,996918,992933,999219,991609,992927,994510,997918,995932,994081\n", + "Ground Truth: 990530,992562,997925,999639,995701,995401,995005,999041,990588,993821,995886,990217,991154,993809,990930,996067,998200,991459,993834,997337,999680,996918,992933,999219,991609,992927,994510,997918,995636,995932\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 272\n", + "Returned IDs: 995208,996968,996867,998297,995401,990337,994794,999530,992169,993412,990432,995274,994409,998536,995373,997191,990561,994677,997871,998045,994301,995515,995374,993143,996727,998080,991052,997723,997319,996769\n", + "Ground Truth: 995208,996968,996867,998297,995401,990337,994794,999530,992169,993412,990432,995274,994409,998536,995373,990561,997191,994677,997871,998045,994301,995515,997775,997403,995374,993143,996727,998080,991052,996438\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 273\n", + "Returned IDs: 990072,999384,998861,997666,990096,997906,996623,991188,997685,991459,995780,998419,991471,993275,997265,999362,998623,993821,996209,998536,995701,990194,998383,999653,993561,996733,994105,994906,999827,999319\n", + "Ground Truth: 990072,999384,998861,997666,990096,997906,996623,991188,997685,991459,995780,998419,991471,993275,997265,999362,998623,993821,996209,998536,995701,990194,992977,998383,999653,993561,999272,996733,994105,994906\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 275\n", + "Returned IDs: 990020,997320,991989,990532,997661,997918,990207,993785,999219,991795,992950,991829,998634,998302,996646,999446,995852,996075,991311,993906,999444,994171,997227,997906,996735,994307,991984,999990,994081,993561\n", + "Ground Truth: 990020,997320,991989,990532,997661,997918,990207,993785,999219,991795,992950,991829,998634,998302,996646,999446,995852,996075,991311,993906,993412,999444,994171,997246,997227,997906,996735,994307,991984,999990\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 277\n", + "Returned IDs: 992546,990515,991431,993392,999180,992339,992994,997614,990958,994653,998246,990118,997305,994757,997437,996075,998797,991479,995981,994079,997320,990660,998562,997976,994776,997543,997470,991045,997913,993662\n", + "Ground Truth: 992546,990515,991431,993392,992583,999680,992238,999180,992339,992994,997614,990958,994653,998246,990118,997305,994757,997437,996075,998797,991479,995981,994079,997320,990660,998562,997976,994776,997543,997470\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 278\n", + "Returned IDs: 994427,990231,994403,996921,992969,993220,995697,992363,991743,998550,994171,996646,999710,994450,992119,993662,993278,997320,994699,994048,995351,990135,995942,999167,994192,997354,990910,996540,990066,990182\n", + "Ground Truth: 994427,990231,994403,996921,992969,993220,995697,992363,991743,998550,994171,996646,999710,994450,992119,993662,993278,997320,994699,991320,994048,995351,990135,995942,999167,994192,997354,990910,996540,990066\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 281\n", + "Returned IDs: 991269,993209,997023,993586,999957,995778,997723,991507,997544,994496,999054,998188,991639,991943,999415,999047,991140,998779,991136,999563,996867,992334,997975,990372,998295,994881,992582,995289,997832,990361\n", + "Ground Truth: 991269,993209,997023,993586,999957,996607,995778,997723,991507,997544,994496,999054,998188,991639,991943,999415,999047,991140,998779,997101,991136,999563,996867,992334,997975,990372,998295,992413,994881,992582\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 282\n", + "Returned IDs: 991820,993080,998720,991981,993342,995906,993809,998514,990860,996911,999444,995160,998593,995852,990532,999916,990866,998710,995002,999922,996001,991795,999647,997790,995961,995636,993906,994636,990622,992314\n", + "Ground Truth: 991820,993080,998720,991981,993342,995906,993809,998514,990860,996911,999444,995160,998593,995852,990532,996490,999916,990866,993290,999272,997969,998710,995002,999922,996001,991795,999647,997790,995961,995636\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 283\n", + "Returned IDs: 999912,998039,997920,996996,995384,995906,990737,997028,991565,997410,997529,990974,997345,995526,990315,999949,998169,993637,996219,994964,995222,994293,999261,990891,994783,999647,993378,993168,998382,999151\n", + "Ground Truth: 999912,998039,997920,996996,995384,995906,990737,997028,991565,997410,997529,990974,997345,995526,990315,999949,998169,993637,996219,994964,995222,994293,991359,999261,990891,994783,999647,993378,991831,995401\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 284\n", + "Returned IDs: 996524,993012,990366,997811,991373,997466,998568,993696,999202,992330,990866,996857,993628,992238,993834,997586,993823,997806,995132,992019,998403,992068,995268,995516,999598,994879,990991,992521,997287,994125\n", + "Ground Truth: 996524,993012,990366,997811,991373,997466,998568,993696,999202,992330,990866,996857,993628,992238,993834,997586,993823,997690,997806,995132,992019,998403,992068,995268,995516,999598,994879,990991,992521,997287\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 285\n", + "Returned IDs: 993805,990256,991052,994714,995516,990897,994869,996406,999646,993673,996811,994956,991982,996918,992979,992147,995432,996762,997126,998318,991824,990563,992795,993948,991977,994125,991038,997765,996168,990209\n", + "Ground Truth: 993805,990256,991052,993255,994714,995516,990897,994869,996406,999646,999676,993673,996811,994956,991982,996918,996695,992979,992147,995432,996762,997126,998318,991857,991824,990563,992795,993948,991977,994125\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 286\n", + "Returned IDs: 998779,998862,997640,993412,998999,993293,997204,996727,993831,999026,994382,996436,997319,997732,999021,991923,994409,995401,992169,995216,997355,993280,994436,995374,995940,991743,996042,995274,997871,997832\n", + "Ground Truth: 998779,998862,997640,993412,998999,993293,997204,996727,993831,999026,994382,996436,997319,997732,999021,991923,994409,995401,992169,995216,997355,994436,993280,995374,995940,991743,993226,996042,995274,997871\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 287\n", + "Returned IDs: 994313,990696,995783,999496,999871,990872,996087,997914,991319,990386,998293,999105,997919,997752,997506,996362,998388,990083,997429,993678,999179,994051,993621,990061,995464,993727,993441,996814,993080,996429\n", + "Ground Truth: 994313,990696,995783,999496,999871,990872,996087,997914,991319,990386,998293,999105,997919,997752,997506,996362,998388,990083,997429,993678,999179,994051,993621,992617,990061,995464,993727,993441,993080,996814\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 289\n", + "Returned IDs: 995289,993844,997403,998014,992488,992289,994204,993433,995401,995766,996438,993143,993673,998536,997319,993762,996867,994783,996930,991840,997897,993232,995633,992052,998913,990941,992407,993209,999453,997822\n", + "Ground Truth: 995289,993844,997403,998014,992488,992289,994204,993433,995401,995766,996438,993143,993673,998536,997319,993762,996867,994783,996930,991840,993232,997897,995633,992052,998913,990337,990941,992407,993209,999453\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 293\n", + "Returned IDs: 994672,994247,993715,999139,994577,995801,995349,993504,991552,993055,995839,993139,991245,995737,996884,993181,994743,991648,994351,993609,999079,990885,995622,997076,992747,992052,991332,999806,993489,991605\n", + "Ground Truth: 994672,994247,993715,999139,994577,995801,995349,993504,991552,993055,995839,993139,991245,995737,996884,993181,994743,991648,994351,993609,999079,990885,995622,997076,992747,992052,991332,999806,993489,999927\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 294\n", + "Returned IDs: 995139,995905,995507,999619,993468,996038,991027,995470,996323,993400,997046,999578,997832,995547,992448,997139,993421,995923,997674,999508,990975,996867,994554,999530,990634,994002,993597,997252,996322,994278\n", + "Ground Truth: 995139,995905,995507,999619,993468,992660,996038,991027,995470,996323,993400,997046,999578,997832,995547,997139,992448,993421,995923,997674,999508,990975,996867,994554,999530,990634,994002,993597,997252,996322\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 295\n", + "Returned IDs: 998294,998380,995949,999263,997301,994964,990561,998697,998090,997320,991530,991157,995928,998029,997913,993342,999783,999444,995802,995672,998302,997032,995638,998365,999331,997371,998692,998607,990366,998303\n", + "Ground Truth: 998294,998380,995949,999263,997301,994964,990561,998697,998090,997320,991530,991157,995928,998029,997913,993342,999783,999444,995802,995672,998302,997032,995638,998365,999331,997371,990310,998692,998607,997135\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 296\n", + "Returned IDs: 991720,999343,992147,997842,998921,995372,992169,996915,998259,991743,997192,995515,991889,998877,993080,991485,999698,997918,995927,992944,991406,995443,994280,995771,990648,995336,990432,992309,998229,990917\n", + "Ground Truth: 991720,999343,992147,997842,998921,995372,992169,996915,998259,993857,991743,997192,995515,991889,998877,993080,991485,999698,997918,995927,992944,991406,995443,994088,994280,995771,990648,995336,990432,992309\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 297\n", + "Returned IDs: 999616,995981,991720,995792,997437,999099,999180,990588,999596,992296,993168,998567,997320,994916,995336,997759,997353,998353,995372,999527,992748,993893,998828,991045,997875,990339,996075,998607,995913,995184\n", + "Ground Truth: 999616,995981,991720,995792,997437,999099,999180,990588,999596,992296,993168,998567,997320,994916,990778,995336,997759,997353,998353,995372,999527,992748,994695,993893,998828,991045,997875,990339,996075,998607\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 302\n", + "Returned IDs: 996606,993534,993427,995152,991983,995618,992113,994679,992857,993380,992558,990279,995710,999540,997849,992496,990345,997547,997917,991046,996750,992335,998151,991871,992092,998840,991026,990125,996367,993583\n", + "Ground Truth: 996606,993534,993427,995152,991983,995618,992113,994679,992857,993380,992558,990279,995710,999540,997849,992496,991058,990345,997547,997917,991046,996750,992335,998151,991871,992092,998840,991026,990125,996367\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 303\n", + "Returned IDs: 995459,998142,992824,997775,999171,994679,991432,991817,991370,993706,993077,991550,994276,993583,997864,999908,992392,995503,993647,993828,997906,995983,996219,992280,999639,995737,998061,990765,991923,993260\n", + "Ground Truth: 995459,998142,992824,997775,999171,994679,991432,991817,991370,993706,993077,991550,994276,993583,997864,999908,992392,995503,993647,993828,997906,995983,996219,992280,999639,995737,998061,990765,991923,990538\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 304\n", + "Returned IDs: 999504,997388,997157,996727,990797,991923,991082,998094,997640,999960,994406,999374,997083,995401,998529,997799,997634,998559,999820,990475,998607,993412,990275,991977,994514,998779,997748,998179,999462,995369\n", + "Ground Truth: 999504,997388,997157,996727,990797,991923,991082,998094,997640,999960,994406,999374,997083,995401,998529,997799,997634,998559,999820,990475,998607,995320,993412,990275,991977,994514,998779,997748,998179,999462\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 305\n", + "Returned IDs: 990644,997702,997002,991392,991485,990999,997533,994362,995374,992979,991894,999219,991459,993355,999453,995894,995875,993848,991157,998385,992750,999977,996425,994946,991151,998247,998401,995154,995515,994334\n", + "Ground Truth: 990644,997702,997002,991392,991485,990999,997533,994362,995374,992979,991894,999219,991459,993355,999453,995894,995875,993848,991157,998385,992750,999977,996425,994946,991151,998247,998401,995154,992047,995515\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 306\n", + "Returned IDs: 997475,994891,993611,994282,993821,991153,998065,996535,991066,990301,993046,990479,990961,996728,999775,999832,991597,991111,998279,994280,998320,999205,996743,996495,996431,990774,997919,995600,998221,991273\n", + "Ground Truth: 997475,994891,993611,994282,993821,991153,998065,996535,991066,990301,993046,990479,990961,996728,999775,999832,991597,994316,991111,998279,994280,998320,999205,996743,996495,996431,990774,997919,995600,998221\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 308\n", + "Returned IDs: 993426,990191,992451,991873,999271,999904,995623,995045,997292,997301,997984,996516,998921,998603,993484,990546,995306,995865,993048,992709,990512,998760,991720,997397,996771,998425,999110,992107,991916,999533\n", + "Ground Truth: 993426,990191,992451,991873,999271,999904,995623,995045,997292,997301,997984,996516,998921,998603,993484,990546,995306,995865,993048,992709,990512,998760,991720,997397,996771,998425,999110,992107,992027,991749\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 309\n", + "Returned IDs: 997233,997059,994205,998725,993857,994013,996308,999130,990241,990662,996028,998982,999099,997452,999570,994131,997554,996657,993909,994382,991382,995295,995216,996001,998752,996147,999623,998055,998156,996411\n", + "Ground Truth: 997233,997059,994205,998725,993857,994013,996308,999130,990241,990662,996028,995600,998982,999099,997452,999570,994131,997554,996657,993909,994382,991382,995295,995216,996001,998752,999623,996147,998055,998156\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 311\n", + "Returned IDs: 992041,992124,999639,997258,992238,990453,992497,993696,998758,998260,992167,992225,991984,995516,993834,992927,994809,993283,990949,997370,997320,999041,998950,991157,992521,991459,996500,995899,996075,992896\n", + "Ground Truth: 992041,992124,999639,997258,992238,990453,992497,993696,998758,998260,992167,992225,991984,995516,993834,992927,994809,993283,990949,997370,997320,999041,998950,991157,992521,991459,990862,996500,995899,996075\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 312\n", + "Returned IDs: 990821,990310,991840,995895,991157,990271,993717,996809,991015,995633,990941,993391,997331,994922,993642,993143,998033,998950,996840,994830,991848,993342,999453,999784,996328,994742,999122,992169,992612,997822\n", + "Ground Truth: 990821,990310,991840,995895,991157,990271,993717,996809,991015,995633,990941,993391,997331,994922,993642,993143,996408,998033,998950,996840,994830,991848,993342,999453,999784,996328,994742,999122,992169,992612\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 315\n", + "Returned IDs: 992771,997395,993566,990194,993106,996393,999994,993931,999673,995895,992685,991188,996408,990951,999543,995620,999555,991037,999430,990407,993224,998536,998006,996762,990796,990434,993011,998383,996395,998855\n", + "Ground Truth: 992771,997395,993566,990194,993106,996393,999994,993931,999673,995895,992685,991188,996408,990951,999543,995620,999555,991037,999430,990407,993224,998536,998006,996762,990796,990434,993011,995211,998383,995401\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 318\n", + "Returned IDs: 991144,993719,990257,993917,992636,993715,996986,990438,990951,997168,999334,996357,993150,993637,995991,998076,998716,998561,990089,997879,998348,996408,997790,996328,994522,994834,991081,997669,993139,996201\n", + "Ground Truth: 991144,993719,990257,993917,992636,993715,996986,990438,990951,997168,999334,996357,993150,993637,995991,998076,998716,998561,990089,997879,998348,996408,997790,996328,995002,994522,994834,991081,997669,993139\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 319\n", + "Returned IDs: 992747,998303,993489,991037,992571,990350,998401,999840,990409,991332,995536,990930,996217,994468,997002,997977,992082,995894,996062,993504,994200,995203,993260,990885,990096,991530,996677,997368,995025,998007\n", + "Ground Truth: 992747,998303,993489,991037,992571,990350,998401,999840,990409,991332,995536,990930,996217,994468,997002,997977,992082,995894,996062,993504,994200,995203,993260,990885,990096,991530,996677,997368,995025,995823\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 320\n", + "Returned IDs: 993809,998080,998495,995401,996371,992571,993255,991052,996370,992318,990083,998950,993696,992847,993515,997723,995932,990532,993478,999701,998221,997586,992944,996805,997184,991412,993444,998416,998968,990776\n", + "Ground Truth: 993809,998080,998495,995401,996371,992571,993255,991052,996370,992318,990083,998950,993696,992847,993515,997723,992753,995932,990532,993478,999701,998221,997586,992944,996805,997184,991412,993444,990711,998416\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 321\n", + "Returned IDs: 990999,998990,998179,995401,997258,991459,997191,996379,996198,996727,993561,990453,997799,990354,993848,999219,992684,991392,991630,999021,994946,993412,994319,996247,999639,990838,991792,995994,990561,992979\n", + "Ground Truth: 990999,998990,998179,995401,997258,991459,997191,996379,996198,996727,993561,990453,997799,990354,993848,999219,992684,991392,991630,999021,994946,993412,994319,996247,999639,990838,991792,999228,995994,990561\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 322\n", + "Returned IDs: 990866,997379,997586,993708,996397,993637,996429,993552,999308,999574,999647,999500,995701,997013,992068,990547,991157,997811,992944,991808,999216,992522,992047,997555,999598,997496,998180,992682,995455,999473\n", + "Ground Truth: 990866,997379,997586,993708,996397,993637,996429,993552,999308,999574,999647,999500,995701,997013,992068,990547,991157,997811,992944,991808,999216,992522,992047,997555,999598,997496,998180,998790,992682,995455\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 323\n", + "Returned IDs: 993160,995062,998080,993971,999961,992150,997634,998806,997061,997395,998316,998650,992677,990131,993255,993263,994576,990789,995886,996273,990103,992604,994774,996139,993122,992493,998528,991360,992558,992148\n", + "Ground Truth: 993160,995062,998080,993971,999961,992150,997634,998806,997061,997395,998316,998650,992677,997429,990131,993255,993263,994576,990789,995886,996273,990103,992161,992604,994774,996139,993122,992493,998528,991360\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 324\n", + "Returned IDs: 991825,996028,994689,995823,998140,998825,993378,997051,995649,990811,991426,997373,994012,991840,998982,993906,992201,997417,993890,998382,994307,997541,995089,993725,991989,992780,999446,999740,995680,999286\n", + "Ground Truth: 991825,996028,994689,995823,998140,998825,993378,997051,995649,990811,991426,997373,994012,991840,998982,993906,992201,997417,993890,998382,994307,995502,997541,995089,993725,991989,992780,999446,999740,995680\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 325\n", + "Returned IDs: 994630,996961,997525,999761,994699,993518,991895,996732,997913,998246,999710,992546,992709,994939,994932,995949,994757,998744,996785,997599,998332,990397,992502,997535,990925,990509,991873,992119,991479,996890\n", + "Ground Truth: 994630,996961,997525,999761,994699,993518,991895,996732,997913,998246,999710,992546,992709,994939,998016,994932,995949,994757,998744,997599,996785,998332,990397,992502,997535,990925,990509,991873,992119,991479\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 326\n", + "Returned IDs: 997976,998170,994894,997479,999182,994757,991782,997320,999144,997918,993673,995981,995279,995886,990231,994100,991379,990660,997571,998011,995336,990958,994819,994334,997192,999527,991181,990647,990588,997777\n", + "Ground Truth: 997976,998170,994894,997479,999182,994757,991782,997320,999144,997918,993673,995981,995279,995886,990231,994100,991379,990660,997571,998011,995336,990958,998163,994819,994334,997192,999527,991181,990647,990588\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 327\n", + "Returned IDs: 995274,996411,990293,996657,993412,996001,993909,995216,993003,998302,999026,998779,996810,998952,990974,991995,990845,996759,998862,995289,998058,999835,999587,990957,992792,999099,997529,996219,999343,993168\n", + "Ground Truth: 995274,996411,990293,996657,993412,996001,993909,995216,993003,998302,999026,998779,996810,998952,990974,991995,990845,996759,998862,995289,992120,995899,998058,999835,999587,990957,992792,997969,999099,997529\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 328\n", + "Returned IDs: 998257,995373,991204,997110,990657,993115,997568,994643,998858,994454,997030,997589,999129,993795,991987,991386,995401,993000,996393,997675,997683,992762,999136,998536,991052,990096,992861,995453,997918,990005\n", + "Ground Truth: 998257,995373,991204,997110,990657,993115,997568,994643,998858,994454,997030,997589,999129,993795,991987,999530,991386,995401,993000,996393,997675,997683,992762,999136,998536,991052,990096,992861,998697,995453\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 331\n", + "Returned IDs: 990154,998006,995815,999555,996728,994181,997666,992771,995835,999673,990952,990006,999543,997884,992282,990951,994834,999388,994467,998861,999663,996505,995655,991596,994103,990429,998067,997358,995536,994756\n", + "Ground Truth: 990154,998006,995815,999555,996728,994181,997666,992771,995835,999673,990952,990006,999543,997884,992282,990951,994834,999388,994467,998861,999350,999663,996505,995655,991596,994103,990429,998067,997358,995536\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 334\n", + "Returned IDs: 991377,992894,994419,995289,997386,997822,993412,995702,995923,991923,993511,996080,991603,990847,993561,996610,996867,991958,996360,997529,996745,993433,994925,996322,995200,994409,992130,995401,997937,999782\n", + "Ground Truth: 991377,992894,994419,995289,997386,997822,993412,995702,995923,991923,993511,996080,991603,990847,993561,996610,996867,991958,996360,997529,996745,996147,993433,994925,996322,995200,996127,994409,992130,997920\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 337\n", + "Returned IDs: 996531,998397,993747,990956,996276,999514,991747,998293,996867,998478,992703,994119,990466,994541,996232,998486,992160,990116,997822,994444,999264,999646,990103,993727,996294,997420,995159,992992,999244,991145\n", + "Ground Truth: 996531,998397,993747,990956,996276,999514,991747,998293,996867,998478,992703,994119,990466,994541,998165,996232,998486,992160,990116,997822,994444,999264,999646,990103,993727,996294,997420,995159,992992,999244\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 342\n", + "Returned IDs: 993364,992126,991994,993885,991792,990772,997876,990557,993820,995841,994162,991212,991498,995372,997320,999982,998444,994673,992788,995515,999453,996489,997973,999819,997002,996001,990724,996959,993032,994081\n", + "Ground Truth: 993364,992126,991994,993885,991792,990772,997876,997380,990557,993820,995841,994162,991212,991498,995372,997320,999982,995502,998444,994673,992788,995515,999453,996489,997973,999819,997002,996001,990724,996959\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 344\n", + "Returned IDs: 994277,993721,994233,994416,991409,996685,995401,997403,995899,992944,996024,997320,998950,991348,991416,998921,996210,996075,994081,997448,990364,992318,995372,998913,996178,991650,991997,991743,997305,999159\n", + "Ground Truth: 994277,993721,994233,994416,991409,996685,995401,997403,995899,992944,996024,997320,998950,991348,991416,998921,996210,993121,996075,994081,997448,990364,995372,992318,998913,996178,991650,991997,991743,997305\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 345\n", + "Returned IDs: 999827,996001,997320,994964,991498,993080,997920,999453,994081,995841,992182,999099,994478,992395,993168,995216,997860,998750,993280,997918,995274,990146,993477,993289,990772,993648,995102,993484,991212,990532\n", + "Ground Truth: 999827,996001,997320,994964,991498,993080,993454,997920,999453,994081,995841,992182,999099,999026,994478,992395,993168,995216,997860,998750,993280,997918,995274,990146,993477,993289,990772,993648,995102,991007\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 346\n", + "Returned IDs: 992907,995588,997465,998582,993091,999676,996840,995515,993028,993622,999076,991840,990300,993355,990310,994922,992979,999073,996352,991999,996686,993781,999077,994612,990490,990432,993688,991160,992218,999620\n", + "Ground Truth: 992907,995588,997465,998582,993091,999676,996840,995515,993028,993622,999076,991840,990300,993355,990310,994922,992979,999073,996352,999604,991999,996686,993781,999077,994612,990490,990432,993688,991160,992218\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 347\n", + "Returned IDs: 997909,997299,998890,996256,999925,993003,994409,995401,997918,993537,991357,991743,991795,992763,996815,996609,995864,994106,990087,990625,994062,999926,996613,997218,994865,996232,998205,991892,996576,997612\n", + "Ground Truth: 997909,997299,998890,996256,999925,993003,994409,995401,997918,993537,991357,991743,991795,992763,996815,996609,995864,997555,994106,990087,990625,994062,990432,999926,996613,997218,994865,996232,998205,991892\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 349\n", + "Returned IDs: 993754,997781,992604,995527,995807,999125,994774,999445,995105,999662,990915,998059,991051,999478,996691,992387,990131,997212,990729,997462,997693,997110,993632,998528,992855,995625,994539,995953,996206,996236\n", + "Ground Truth: 993754,997781,992604,995527,995807,999125,994774,999445,995105,999662,990915,998059,991051,999478,996691,992387,990131,998297,997212,990729,997462,997693,997110,993632,998528,992855,995625,994539,995953,996206\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 352\n", + "Returned IDs: 991498,993484,994503,997597,999005,999263,995372,997320,997301,995841,994917,996560,992637,998849,998966,998634,991743,990829,990936,995034,994183,998779,998839,990585,998445,992510,999658,997305,994611,998163\n", + "Ground Truth: 991498,993484,994503,997597,999005,999263,995372,997320,997301,995841,994917,996560,992637,998849,998966,998634,991743,990829,990936,995034,994183,998779,998839,990585,998445,992510,999658,997305,997908,994611\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 356\n", + "Returned IDs: 999587,992884,993289,999453,996657,997554,996001,990845,994531,997002,990399,996810,992979,996028,994946,995401,996147,993032,996368,995216,992521,995281,996759,999827,994809,993280,992558,990956,998495,990255\n", + "Ground Truth: 999587,992884,993289,999453,996657,997554,996001,990845,994531,997002,990399,996810,992979,996028,994946,995401,996147,993032,996368,995216,992521,995281,996759,997023,999827,994809,993280,992558,990956,998495\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 357\n", + "Returned IDs: 991063,993809,998225,998302,994879,999444,997906,995638,990866,992748,996805,998950,997913,998123,996230,996075,999977,996811,995672,995755,997320,999831,996001,998580,995701,998697,992394,990096,990083,996761\n", + "Ground Truth: 991063,993809,998225,998302,994879,999444,997906,995638,990866,992748,996805,998950,997913,998123,996230,996075,991927,999977,996811,995672,995755,997320,999831,993821,996001,998580,995701,998697,991311,992394\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 358\n", + "Returned IDs: 995580,991677,995775,999333,995266,999301,992668,999414,996499,994464,996192,998170,992134,997913,998216,993222,992946,997818,997192,998152,990957,991422,995014,998686,990647,998600,997106,995870,997841,994748\n", + "Ground Truth: 995580,991677,995775,999333,995266,999301,992668,999414,996499,998599,994464,996192,998170,992134,997913,998216,993222,992946,994904,997818,997192,991081,998152,990957,991422,995014,998686,990647,998600,997106\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 359\n", + "Returned IDs: 990323,991037,999748,991596,995915,995385,990194,999576,995712,996702,997242,994955,995562,990148,994200,995349,990096,997666,993179,994231,999435,998583,993866,992571,993458,999384,995655,990596,996424,990876\n", + "Ground Truth: 990323,991037,999748,991596,995915,995385,990194,999576,995712,996702,997242,994955,995562,994020,990148,994200,995349,990096,997666,993179,994231,999435,998583,993866,992571,993458,999384,995655,990596,996424\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 360\n", + "Returned IDs: 995559,993695,991179,993038,995560,995946,998790,999898,996210,990435,991923,997920,992944,992047,998416,996042,995107,997586,993055,996370,990096,995720,995332,999430,994815,993241,998349,998950,993324,999806\n", + "Ground Truth: 995559,993695,991179,993038,995560,995946,998790,999898,996210,990435,991923,997920,992944,992047,998416,996042,995107,997586,993055,996370,990096,995720,995332,999430,994815,995401,993241,998349,998950,993324\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 361\n", + "Returned IDs: 995891,991544,998651,995946,993561,994413,996472,990096,991958,998973,993566,992448,992260,991247,997252,997386,998545,998451,994285,995536,991447,994365,995423,995303,992631,995146,993433,999543,994409,992801\n", + "Ground Truth: 995891,991544,998651,995946,993561,994413,996472,990096,991958,998973,993738,993566,992448,992260,991247,997252,997386,998545,998451,994285,995536,991447,994365,995423,995303,992631,995146,993433,999543,999673\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 362\n", + "Returned IDs: 994589,991666,995401,992933,997723,991630,996659,990517,998530,999977,995701,996379,996864,998350,998874,991603,998990,999461,992367,998045,999000,998536,994177,995994,994409,992977,992790,992521,994946,999228\n", + "Ground Truth: 994589,991666,995401,992933,997723,991630,996659,990517,998530,999977,995701,996379,996864,998350,998874,991603,999461,998990,992367,998045,999000,998536,994177,995994,994409,992977,992790,991151,992521,994946\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 366\n", + "Returned IDs: 995633,999676,997374,994317,992289,997319,995401,994529,995289,990903,991743,996867,990126,995268,998221,995274,990432,993402,997723,994878,999453,994409,999587,993920,995020,998913,994850,994343,990435,999533\n", + "Ground Truth: 995633,999676,997374,994317,992289,997319,995401,994529,995289,990903,991743,996867,990126,993412,995268,998221,995274,990432,993402,997723,994878,999453,994409,999587,993920,995020,998913,994850,994343,990435\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 367\n", + "Returned IDs: 994964,999151,990545,996219,995961,999331,991345,999261,994843,999352,999367,990922,994896,991921,994577,999210,991382,993238,993877,997720,993215,994705,995927,991378,994782,992446,990210,995739,994850,991751\n", + "Ground Truth: 994964,999151,990545,996219,995961,999331,991345,999261,994843,999352,999367,990922,994896,991921,994577,999210,991382,993238,992942,993877,998294,993154,997720,993215,994705,995927,991378,994782,990210,992446\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 368\n", + "Returned IDs: 995166,993857,998982,994689,991075,997208,996028,992374,991585,995456,998427,993725,998182,990328,994307,995823,990097,993053,994765,995633,992480,996506,993906,995295,997309,995289,990811,992558,998180,991315\n", + "Ground Truth: 995166,993857,998982,994689,991075,997208,996028,992374,991585,995456,998427,993725,998182,990328,994307,995823,990097,993053,995401,994765,995633,992480,996506,993906,995295,997309,995289,990811,992558,998180\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 370\n", + "Returned IDs: 997320,993484,997913,994604,996143,993154,998967,999948,995802,992442,990066,997470,998170,999658,990532,992119,995372,992330,990949,995865,999851,992546,993157,996075,995942,998808,992896,999535,995351,990585\n", + "Ground Truth: 997320,993484,997913,994604,996143,993154,998967,999948,995802,995228,992442,990066,997470,998170,999658,990532,992119,995372,992330,990949,995865,999851,992546,993157,996075,995942,998808,992896,999535,995351\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 371\n", + "Returned IDs: 991975,993641,990429,995168,992110,999176,995231,993629,998858,993381,995780,997078,991455,996623,991332,992365,991385,993560,999555,991987,996571,994413,990976,994834,992926,990449,993191,998149,991038,995655\n", + "Ground Truth: 991975,993641,990429,995168,992110,999176,995231,993629,998858,993381,995780,997078,991455,996623,991332,992365,991385,993560,999555,991987,996571,994413,990976,994834,992926,997546,990449,998731,993191,998149\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 372\n", + "Returned IDs: 990249,993811,992806,995967,993727,997195,992121,999240,999479,993121,993583,990121,996750,993473,992558,994163,999400,998493,998820,991447,997918,993158,993911,992864,990117,995626,997124,994168,990395,999098\n", + "Ground Truth: 990249,993811,992806,995967,993727,997195,992121,999240,999479,993121,993583,990121,996750,993473,992558,999887,994163,999400,997897,998493,992407,998820,991447,997918,998559,993158,993911,992864,990117,995626\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 376\n", + "Returned IDs: 995372,998849,992944,993241,990585,990060,995375,996809,990489,999544,998567,990226,995560,995841,999711,991498,999210,996318,997320,994972,994402,991994,997848,996001,990043,995641,990894,997436,997113,995443\n", + "Ground Truth: 995372,998849,992944,993241,990585,990060,995375,996809,990489,999544,998567,990226,995560,995841,999711,991498,999210,996318,997320,995401,994972,993533,994402,991994,997848,993388,996001,990043,995641,990894\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 379\n", + "Returned IDs: 992532,995974,998686,998334,991480,999451,995560,992349,992645,996894,999711,993976,995314,998417,998079,992654,992383,997521,990383,996196,990401,996262,991015,991208,990376,997758,998061,999639,999913,991977\n", + "Ground Truth: 992532,995974,998686,998334,991480,999451,995560,992349,992645,996894,993174,999711,993976,995314,998417,998079,992654,992383,997521,990383,996196,990401,998849,996262,991015,999216,991208,990376,997758,998061\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 381\n", + "Returned IDs: 995482,999179,990157,994806,997997,993466,995106,991664,995055,998256,996633,993731,996283,991061,994166,997411,993820,998748,996386,995778,995777,996328,993111,998543,992563,998221,999759,990806,992364,990987\n", + "Ground Truth: 995482,999179,990157,994806,997997,993466,995106,991664,995055,998256,996633,993731,996283,991061,990083,994166,997411,993820,998748,996386,995778,995777,996328,993111,998543,992563,998221,999759,990806,992364\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 382\n", + "Returned IDs: 996147,992161,990771,995167,996001,997790,997918,999898,991667,992950,993289,996657,997554,998215,999150,990399,999099,991311,999453,991212,994266,995855,997163,991585,992384,996549,995281,990314,993168,998720\n", + "Ground Truth: 996147,992161,990771,995167,996001,997790,997918,999898,991667,992950,993289,996657,997554,998215,999150,990399,999099,991311,999453,991212,994266,995855,993143,997163,991585,991485,992384,996549,995281,990314\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 383\n", + "Returned IDs: 992558,990538,998950,990279,999639,996196,997775,997918,997258,994163,992615,995552,999206,999860,996609,990273,997859,999564,996592,998411,991522,994280,995340,994742,995372,999715,992436,991498,991791,993077\n", + "Ground Truth: 992558,990538,998950,990279,999639,996196,997775,997918,995693,997258,994163,992615,995552,999206,999860,996609,990273,997859,999564,996592,998411,991522,994280,995340,994742,995372,999715,992436,991498,991791\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 385\n", + "Returned IDs: 997852,997476,999077,999676,992169,994922,991334,990490,996325,992612,999916,990432,993260,997822,997496,993717,998041,993673,991840,991743,994964,992813,999453,998950,999843,997135,993355,994742,999533,994318\n", + "Ground Truth: 997852,997476,999077,999676,992169,994922,991334,990490,996325,997204,992612,999916,990432,993260,997822,997496,993717,998041,993673,991840,991743,994964,992813,999453,998950,994044,999843,997135,993355,990214\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 389\n", + "Returned IDs: 990742,991593,991662,991016,997826,998899,999701,997913,995748,991502,998003,999374,991616,997320,995928,998170,993238,994964,991479,993885,999948,993606,998294,992978,991743,990816,999262,994735,995917,992255\n", + "Ground Truth: 990742,991593,991662,991016,997826,998899,999701,997913,995748,991502,998003,999374,991616,997320,999390,995928,990902,991237,998170,993238,994964,991479,993885,999948,993606,998294,992978,991743,990816,998663\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 390\n", + "Returned IDs: 990249,996609,998820,991248,995124,992558,999240,996762,991350,994457,991266,995289,995377,997822,996606,993412,995332,995354,997723,999646,995401,995119,997498,991330,991449,998874,999898,992576,998529,996750\n", + "Ground Truth: 990249,996609,998820,991248,995124,992558,993433,999240,996762,991350,994457,991266,995289,995377,997822,996606,993412,995332,996294,992469,995354,997723,999646,995401,995119,997498,991330,991449,998874,999898\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 393\n", + "Returned IDs: 990487,996451,999774,999191,996194,997770,991977,994254,997251,996042,999179,990102,998968,997115,990678,992558,991447,992979,996658,994387,992086,998104,991303,990705,994280,993275,991743,997918,997529,991061\n", + "Ground Truth: 990487,996451,999774,999191,996194,991720,998528,999343,997770,991977,994254,997251,996042,999179,990102,998968,997115,990678,992558,991447,992979,996658,994387,992086,998104,991303,990705,994280,993275,991743\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 394\n", + "Returned IDs: 990535,999144,994217,993168,992630,998951,993083,997015,992331,995336,997320,995947,999437,997082,994964,990925,992748,993389,995252,997002,996001,999019,993484,994757,994916,995824,994854,998090,990958,992047\n", + "Ground Truth: 990535,999144,994217,993168,992630,998951,993083,997015,992331,995336,999916,997320,995947,999437,997082,994964,990925,992748,993389,995252,997002,995216,996001,999019,993484,994757,994916,995824,994854,998090\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 395\n", + "Returned IDs: 994280,991334,997023,995968,990214,995377,995401,999701,991923,994835,995820,991743,990921,995886,994655,992165,992582,995824,994982,992521,998302,992384,993561,997918,991269,994496,996762,995336,998295,995211\n", + "Ground Truth: 994280,991334,997023,995968,990214,995377,995401,999701,991923,994835,995820,991743,998374,990921,995886,994655,992165,992582,995824,994982,992521,998302,992384,993561,997918,991269,994496,996762,995336,998295\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 399\n", + "Returned IDs: 998908,998921,994244,997539,997301,999995,993581,995154,995910,993143,999219,998539,990760,996014,995554,991157,993364,993662,997777,997370,990561,992124,999358,996040,998029,998726,997371,993250,996092,998894\n", + "Ground Truth: 998908,998921,994244,997539,997301,999995,993581,995154,990702,999426,995910,993143,999219,992461,998539,990760,996014,995554,991157,993900,993364,997158,993662,997777,997370,990561,992124,999358,996040,993005\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 402\n", + "Returned IDs: 997643,994724,997210,992638,995044,997683,990657,997589,999156,999117,998918,992206,992623,993076,998257,996393,998277,993795,994660,993666,991797,999808,995207,991309,990167,998787,991204,990361,998383,992760\n", + "Ground Truth: 997643,994724,997210,992638,995044,997683,990657,997589,999156,999117,998918,992206,992623,993076,998257,996393,998277,993795,994660,993666,991797,999808,995207,991309,990167,998787,991204,994917,990361,998383\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 403\n", + "Returned IDs: 998149,990658,994869,999663,991038,999834,993055,994155,999736,997506,999139,998858,995780,994467,995237,991169,999555,992421,998831,993629,995516,996406,999896,996623,999806,995755,998383,998720,991154,996446\n", + "Ground Truth: 998149,990658,994869,999663,991038,999834,993055,994155,999736,997506,999139,998858,995780,994467,995237,991169,999555,992421,998831,993629,995516,996406,999896,996623,999806,995755,998383,998720,991154,991835\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 406\n", + "Returned IDs: 994577,992068,998816,991015,999806,999194,997379,995039,999122,993628,999432,992423,993012,991157,991987,996281,998568,998950,998858,993143,997466,990991,998437,998257,999267,991713,999568,992330,990479,991844\n", + "Ground Truth: 994577,992068,998816,991015,999806,999194,997379,995039,999122,993628,995453,999432,992423,993012,991157,991987,996281,998568,998950,998858,993143,997466,990991,998437,998257,999267,991713,999568,992330,990479\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 407\n", + "Returned IDs: 994717,994809,990255,990361,997088,995401,999587,993515,994092,992293,996657,999629,998593,996043,996809,995701,996759,990573,994431,996930,995289,992558,993848,992367,998536,997897,998990,996393,998250,996777\n", + "Ground Truth: 994717,994809,990255,990361,997088,995401,999587,993515,994092,992293,996657,999629,998593,996043,996809,995701,996759,990573,994431,996930,995289,992558,993848,997411,992367,998536,997897,998990,996393,998250\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 409\n", + "Returned IDs: 996690,993363,996963,991772,991157,991550,993370,993411,993706,999393,994163,996196,999639,997775,998697,990159,996926,995013,992349,990279,992770,999908,992423,992738,997225,997258,998142,995599,999041,995804\n", + "Ground Truth: 996690,993363,996963,991772,991157,991550,993370,993411,993706,999393,992383,994163,996196,999639,997775,998697,990159,996926,995013,992349,990279,992770,991706,999908,992423,992738,997225,997258,998142,995599\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 410\n", + "Returned IDs: 998170,994240,997699,993431,996592,994584,999077,992880,997470,993260,997320,997918,991801,995094,991348,992927,994666,999701,996296,998180,994450,995826,991928,993696,995266,995357,991564,991131,994964,992748\n", + "Ground Truth: 998170,994240,997699,993431,996592,994584,999077,992880,997470,993260,997320,997918,991801,995094,991348,993454,992927,994666,999701,996296,998180,994450,995826,991928,993696,995266,995357,991564,991131,994964\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 411\n", + "Returned IDs: 995266,991593,992069,998899,994399,997320,991774,992255,991580,990957,998967,991662,999290,997470,997301,999077,993662,995826,991479,993852,998090,999602,994604,990910,994290,996296,992978,990650,993034,993402\n", + "Ground Truth: 995266,991593,992069,998899,994399,997320,991774,992255,991580,990957,998967,991662,999290,997470,997301,999077,993662,995826,991479,993852,998090,999602,994604,990910,994290,996296,992978,992946,990650,998801\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 412\n", + "Returned IDs: 992521,997545,993269,997466,996552,991878,991969,993107,992097,994353,998238,993438,991555,993353,998075,998910,998346,995193,994508,991760,997216,995813,998205,998568,992448,994636,994954,993738,997723,995864\n", + "Ground Truth: 992521,997545,993269,997466,996552,991878,991969,993107,992097,994353,998238,993438,991555,998564,993353,998075,998910,998346,995193,994508,991760,997216,995813,998205,998272,995644,998568,992448,994636,994954\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 413\n", + "Returned IDs: 990956,992558,996809,994742,995963,991052,999179,992052,990776,992604,997723,990361,998495,997752,994643,995408,994351,993948,990657,998543,993799,995401,993079,991319,995105,992685,997914,993121,996043,993820\n", + "Ground Truth: 990956,992558,996809,994742,995963,991052,999179,992052,990776,992604,997723,990361,998495,997752,994643,993077,995408,994351,993948,990657,998543,991257,993799,995401,993079,991319,995105,992685,997914,993121\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 417\n", + "Returned IDs: 991198,997789,990043,999620,998232,991196,996681,998236,997275,998983,993582,990155,991801,990858,990809,993570,998837,996318,994922,994785,992907,999076,993675,991510,991999,994964,990585,997876,993237,999053\n", + "Ground Truth: 991198,997789,990043,999620,998232,991196,996681,998236,997275,998983,993582,990155,991801,990858,990809,993570,998837,996318,994922,994785,992907,999076,993675,991510,991999,994964,990585,997876,998280,993237\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 420\n", + "Returned IDs: 998886,992293,994382,990662,997868,990361,998017,990399,996809,999922,997466,995470,992582,998495,993390,992862,991603,999587,996042,993831,994793,997723,994689,995679,997529,991578,998982,995562,994730,990638\n", + "Ground Truth: 998886,992293,994382,990662,997868,990361,998017,990399,996809,999922,997466,995470,992582,998495,993390,992862,991603,999587,996147,996042,993831,994793,997723,994689,995679,997529,991578,998982,995562,994730\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 423\n", + "Returned IDs: 998470,995214,992558,999791,999767,998122,995222,998279,994972,998495,997732,990956,998765,996001,997603,999191,999827,992713,995438,990091,990089,996219,996647,998710,990364,994280,999138,995107,999852,994119\n", + "Ground Truth: 998470,995214,992558,999791,999767,998122,995222,998279,994972,998495,997732,990956,998765,996001,997603,999191,999827,992713,995438,990091,997920,990089,999916,996219,996647,998710,990364,994280,999138,995107\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 424\n", + "Returned IDs: 990811,998593,996147,999173,991000,996657,990130,992326,993483,996028,998878,990771,992727,996001,994838,998934,995158,999740,995864,990573,998250,993391,992289,993813,995289,995200,995182,993848,996302,997007\n", + "Ground Truth: 990811,998593,996147,999173,991000,996657,990130,992326,993483,996028,998878,990771,992727,996001,998934,994838,995158,999740,995864,990573,998250,993391,996864,992289,993813,995289,995200,995182,993848,996302\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 425\n", + "Returned IDs: 990382,999543,994510,993515,993931,999139,990096,998593,998250,996062,991685,995889,994463,991596,995536,998495,997790,995895,990930,996733,994191,991011,998376,996001,996549,999994,995017,993778,999091,991066\n", + "Ground Truth: 990382,999543,994510,993515,993931,999139,990096,998593,998250,996062,991685,995889,994463,991596,995536,998495,994796,997790,995895,990930,996733,994191,991011,998376,996001,996549,999994,995017,993778,999091\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 427\n", + "Returned IDs: 992862,997586,995005,991490,995701,990786,999806,993809,995536,998708,999653,995455,996082,993637,993470,999444,990167,998200,997506,993834,990271,999087,993515,998697,993504,995203,998536,991038,993055,995672\n", + "Ground Truth: 992862,997586,995005,991490,995701,990786,999806,993809,995536,998708,999653,995455,996082,993637,993470,999444,990167,998200,997506,993834,990271,996370,999087,993515,998697,993504,995203,993821,998536,997577\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 429\n", + "Returned IDs: 996734,995401,990337,997191,998297,995685,993412,995373,998045,991603,994409,993561,998999,991052,997406,996968,992448,995940,996277,993288,991140,997723,994948,990930,997918,995648,999530,997822,992685,995994\n", + "Ground Truth: 996734,995401,990337,997191,998297,995685,993412,995373,998045,991603,994409,993561,998999,991052,997406,996968,992448,995940,996277,993288,991140,997723,994948,990930,997918,995648,999530,997822,995873,992685\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 430\n", + "Returned IDs: 994956,996219,996328,990214,999179,996811,997442,999500,994387,996149,993727,995332,998256,998968,995438,997420,999560,994783,995562,995130,997002,994233,995895,999540,999352,990039,994206,992237,994452,997945\n", + "Ground Truth: 994956,996219,996328,990214,999179,996811,997442,999500,994387,996149,993727,995332,998256,998968,995438,991212,997420,999560,994783,995562,996071,995130,997002,994233,995895,999352,999540,990039,994206,992237\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 431\n", + "Returned IDs: 994809,997218,995401,992790,990841,991671,996762,991197,991334,996930,993673,992749,992558,990312,992636,991459,994835,995864,992685,993003,992448,998205,996162,998104,996187,990181,993275,995873,997023,996001\n", + "Ground Truth: 994809,997218,995401,992790,990841,991671,996762,991197,991334,996930,993673,992749,992558,990312,992636,991459,994835,995864,992685,998855,993003,992448,998205,996162,998104,996187,990181,993275,995873,996438\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 432\n", + "Returned IDs: 996279,994533,999301,996192,994688,994757,993852,999883,990215,990469,994854,990275,998190,996714,999089,992511,993457,999327,995034,991379,997012,999500,992384,998170,992087,999548,998011,999953,992854,997605\n", + "Ground Truth: 996279,994533,999301,996192,994688,994757,993852,999883,990215,990469,994854,990275,998190,996714,999089,992511,993457,999327,995034,991379,997012,999500,992384,998170,992087,991650,999548,998011,999953,992854\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 433\n", + "Returned IDs: 999820,990561,995327,992979,994906,997910,992790,994946,994673,993289,994819,990304,995968,998011,991074,996425,994982,995401,996714,994170,997218,999715,994334,997396,994188,997002,994922,998210,999977,991607\n", + "Ground Truth: 999820,990561,995327,992979,994906,997910,992790,994946,994673,993289,994819,990304,995968,998011,992047,991074,996425,994982,995401,996714,994170,997218,999715,994334,997396,994188,997002,994922,998210,999977\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 434\n", + "Returned IDs: 992950,997320,996904,993662,998540,999109,992451,998302,997470,993237,994964,997913,997379,992443,996890,993791,998550,999710,999210,998837,991626,993796,994192,992069,991801,993018,991582,997135,992733,998950\n", + "Ground Truth: 992950,997320,996904,993662,998540,999109,992451,998302,997470,993237,994964,997913,997379,992443,992140,996890,993791,998550,999710,999210,998837,991626,993796,994192,990615,992069,991801,993018,991582,997135\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 437\n", + "Returned IDs: 994749,999544,994826,997249,995968,997498,999254,994081,997888,995372,990772,990043,995932,991440,993071,999676,996309,996506,991787,994171,992531,991880,999453,995841,997661,998734,994687,997189,995250,991002\n", + "Ground Truth: 994749,999544,994826,997249,995968,997498,999254,994081,997888,995372,990772,990043,995932,991440,993071,999676,996309,996506,991787,998654,994171,992531,991880,999453,995841,997661,996785,998734,994240,994687\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 439\n", + "Returned IDs: 998536,992110,991644,990449,991620,993001,992282,992595,990399,992832,999827,997506,990154,998067,995655,990942,995326,991667,996328,992444,998625,992318,996408,999994,999555,992862,993993,995030,994380,997494\n", + "Ground Truth: 998536,992110,991644,990449,991620,993001,992282,992595,990399,992832,999827,997506,990154,998067,995655,990942,995326,997723,991667,996328,992444,998625,998861,992318,996408,999994,999555,992862,993993,995030\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 440\n", + "Returned IDs: 999594,992451,993419,990936,992583,990089,998163,999516,993060,995659,999791,996279,996755,993848,994776,992546,995792,992470,999757,993630,993484,996001,996516,993490,992785,997320,998921,996075,995454,999900\n", + "Ground Truth: 999594,992451,993419,990936,992583,990089,998163,999516,993060,994325,995659,999791,996279,990862,996755,993848,994776,992546,994199,995792,992470,999757,993630,993484,996001,996516,993490,992785,997320,998921\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 441\n", + "Returned IDs: 991178,990980,993914,990648,999343,996219,997984,994266,999404,993238,994964,998500,992546,995942,990146,999824,993072,990595,994415,990650,990736,996028,993518,994889,998458,990545,998581,997855,991325,998921\n", + "Ground Truth: 991178,990980,993914,990648,999343,996219,997984,994266,999404,993238,994964,998500,992546,995942,990146,993072,999824,990595,994415,993048,990650,990736,996028,993518,994889,998458,990545,998581,997855,991325\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 442\n", + "Returned IDs: 991856,993960,991202,994497,994604,999260,990224,995135,999633,996854,990958,999374,999793,991889,998190,994922,995947,995259,990980,991593,998921,995365,993640,994266,990660,995443,994195,998500,997192,991196\n", + "Ground Truth: 991856,993960,991202,994497,994604,992323,999260,999261,990224,995135,999633,996854,991733,990958,999374,999793,997470,991889,999398,998190,994922,995947,995259,990980,991593,995959,998921,995365,993640,993570\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 447\n", + "Returned IDs: 990366,990089,995569,994373,999898,993401,990271,996219,997855,995306,994919,999417,999610,994279,994792,999651,992174,995095,995096,995210,990438,995551,991593,991172,999192,999791,995868,993402,990173,991751\n", + "Ground Truth: 990366,990089,995569,994373,999898,993401,990271,996219,997855,995306,994919,999417,999610,994279,994792,991572,999651,992174,995095,995096,995210,990438,995551,991593,991172,999192,999216,999791,997340,995868\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 448\n", + "Returned IDs: 999658,994611,991088,998966,997320,990444,993253,995942,993484,991593,994249,990890,991895,996838,992546,999167,998246,990936,990949,993694,994964,996219,990210,997699,994573,998287,991155,997372,990545,997470\n", + "Ground Truth: 999658,994611,991088,998966,997320,990444,993253,995942,993484,991593,994249,990890,991895,996838,992546,999167,998246,990936,990949,993694,994964,996219,990210,997699,999247,994573,998287,991155,999991,997372\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 449\n", + "Returned IDs: 999343,990366,994964,995438,990934,995200,998921,997920,996219,997371,997842,992047,995401,990448,997192,999144,990866,998401,999647,993412,997370,996727,993289,994670,995096,997320,993237,993253,998779,995927\n", + "Ground Truth: 999343,990366,994964,995438,990934,995200,998921,997920,996219,997371,997842,992047,995401,990448,990652,997192,999144,990866,998401,999647,993412,997370,996727,993289,994670,995096,997320,997403,993237,993253\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 450\n", + "Returned IDs: 995417,997843,998653,995981,998828,997584,991467,992671,997976,992931,990066,996755,993696,992733,991131,993047,994894,994100,990780,992972,992119,992692,992635,995336,997256,995596,996075,996710,999883,992748\n", + "Ground Truth: 995417,997843,998653,995981,998828,997584,991467,992671,997976,992931,990066,996755,993696,992733,991131,993047,994894,994100,990780,992972,992119,997449,992692,991372,992635,995336,997256,995596,996075,996966\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 451\n", + "Returned IDs: 995855,999680,991392,997320,996247,999453,993300,999806,999639,996744,998232,990772,991984,993904,997777,996463,996001,995250,997258,995515,992238,993484,997380,997913,993848,990957,990352,992429,994081,990949\n", + "Ground Truth: 995855,999680,991392,997320,996247,999453,993300,999806,999639,996744,998232,990772,991984,993904,997777,996463,996001,995250,997258,995515,992238,993484,997380,997913,993848,990957,990352,992429,994081,993003\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 452\n", + "Returned IDs: 991939,993419,997785,995306,995659,998868,998921,997984,996516,990089,990366,997301,997286,991479,991999,999824,998445,990512,991852,991743,997292,998726,993143,997371,997873,994942,991406,992451,996135,990560\n", + "Ground Truth: 991939,993419,997785,995306,995659,998868,998921,997984,999516,996516,990089,990366,997301,997286,991479,991999,999824,998445,990512,991852,991743,997292,998726,993143,997371,997873,994942,991406,992451,996135\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 453\n", + "Returned IDs: 997411,996368,997868,996181,996145,993751,995702,991002,995937,998265,994730,999486,998370,997723,998990,999586,991585,990974,990412,996379,996959,999977,999537,994826,991157,995633,990132,994895,995679,995095\n", + "Ground Truth: 997411,996368,997868,996181,996145,993751,995702,991002,995937,998265,994730,999486,998370,997723,998990,999586,991585,990974,990412,996379,996959,999801,999977,999537,994826,991157,995633,999639,990132,994895\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 455\n", + "Returned IDs: 991925,996328,996506,991807,990776,997309,996318,998021,995268,990987,992726,996495,993121,994119,995080,992052,997208,995375,994895,999259,995098,991712,993180,997184,996001,997675,998566,995438,997732,992558\n", + "Ground Truth: 991925,996328,996506,991807,990776,997309,996318,998021,995268,990987,992726,991743,996495,993121,994119,995080,992052,997380,997208,995375,994895,994409,999259,995098,991712,993180,990887,997184,996001,997675\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 456\n", + "Returned IDs: 995080,990043,999784,999516,992318,995442,997918,993052,996489,996193,995119,997155,997320,995375,998726,995336,992031,997301,999453,995641,994972,991816,994162,991348,991440,990882,996840,991609,995515,993260\n", + "Ground Truth: 995080,990043,999784,999516,992318,995442,997918,993052,996489,996193,995119,996829,997155,997320,995375,998726,995336,992031,995246,999741,997301,996887,999453,995641,994972,991816,994162,993528,991348,991440\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 457\n", + "Returned IDs: 997024,995372,997470,997320,992789,994618,995841,998951,996143,998445,999109,990936,992451,992978,991362,991498,995422,990191,993241,991650,991999,992228,993788,990275,993853,993579,998764,998195,997984,998921\n", + "Ground Truth: 997024,995372,997470,997320,992789,994618,995841,998951,996143,998445,999109,990936,992451,992978,991362,991498,995422,990191,993241,991650,991999,992228,993788,990275,993853,993579,998764,998195,999929,997984\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 458\n", + "Returned IDs: 990854,991935,992450,995571,998671,996328,991731,994142,998251,990931,992713,995083,991769,998055,991521,999126,995861,999441,999775,996431,996495,992667,990593,997488,990887,991257,990712,990956,990889,996728\n", + "Ground Truth: 990854,991935,992450,995571,998671,996328,991731,994142,998251,990931,992713,995083,991769,998055,991521,999126,995861,999441,999775,996431,996495,992667,990593,997488,990887,991257,990712,990956,990889,991062\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 459\n", + "Returned IDs: 992604,997110,991052,994753,995373,998318,996488,991844,995807,991309,994643,998383,998707,991110,992387,991987,999834,997675,999136,998858,995105,999478,994677,995449,992558,991571,990915,990729,990657,993433\n", + "Ground Truth: 992604,997110,991052,994753,995373,998318,996488,991844,995807,991309,994643,998383,998707,991110,992387,991987,999834,997675,999136,998858,995105,999478,995963,994677,995449,992558,991571,990915,990729,990657\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 460\n", + "Returned IDs: 995942,999358,997468,999263,999075,990595,995771,993364,994964,996218,990560,998211,993492,990146,994741,995865,999291,998964,990958,999374,992968,999701,998644,991593,994281,993484,996592,992401,994100,993250\n", + "Ground Truth: 995942,999358,997468,999263,999075,990595,995771,993364,994964,996218,990560,998211,993492,990146,994741,995865,995502,999291,998964,996869,990958,999374,992968,999701,998644,991593,990886,994281,993484,996592\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 461\n", + "Returned IDs: 997023,991958,997544,995211,995281,990887,992917,996987,993209,992521,993561,996719,995648,995198,998720,994690,990331,996001,998990,995401,995815,998776,994742,996777,995515,993289,996371,999587,994280,996220\n", + "Ground Truth: 997023,991958,997544,995211,995281,990887,992917,996987,993209,992521,993561,996719,997163,995648,995198,998720,994690,995283,990331,996001,998990,995401,995815,998776,995911,992862,994742,997191,996777,991743\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 462\n", + "Returned IDs: 993269,992521,999990,991969,999516,995813,999866,995283,993684,992426,993561,997554,999513,994949,991869,998361,990035,993738,997822,995177,993813,995216,990603,992448,991795,994386,998910,990845,996368,998205\n", + "Ground Truth: 993269,992521,999990,991969,999516,995813,999866,995283,993684,992426,993561,997554,999513,994949,991869,998361,990035,993738,997216,997822,995177,993813,995216,990603,992448,991795,994386,998346,998910,990845\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 467\n", + "Returned IDs: 996406,992752,993347,995920,997506,995103,995961,993055,995600,996216,999205,999640,993993,992628,995051,993560,997151,996549,996483,999555,992867,998594,992293,993545,993610,998589,998716,999139,996082,998752\n", + "Ground Truth: 996406,992752,993347,995920,997506,995103,995961,993055,995600,996216,999205,999640,993993,992628,995051,993560,997151,996549,996483,999555,992867,998594,992293,993545,995185,993610,998589,998716,999139,996082\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 469\n", + "Returned IDs: 998697,995894,997586,991052,990096,994972,991713,997320,995536,997002,995740,999840,993280,998303,994040,991530,996062,992521,991332,997790,996918,997918,997246,995080,992571,994334,996368,995672,995560,993504\n", + "Ground Truth: 998697,995894,997586,991052,990096,994972,991713,997320,995536,997002,995740,999840,993280,998303,994040,991530,996062,992521,991332,997790,996918,997918,997246,995080,992571,994334,996368,995672,995560,999234\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 470\n", + "Returned IDs: 999590,996001,997920,995167,992665,992862,993157,998934,996147,994531,993174,997586,990573,998593,998299,990399,997790,996075,998481,998228,996661,999216,993168,994964,995701,991212,996028,991609,997002,995536\n", + "Ground Truth: 999590,996001,997920,995167,992665,992862,993157,998934,996147,994531,993174,997586,990573,998593,998299,997555,990399,997790,996075,998481,998228,996661,999216,993168,994964,995701,991212,996028,991609,997002\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 473\n", + "Returned IDs: 999691,991081,998864,998845,997019,990072,993366,996446,999349,993715,996106,998934,999273,995636,992752,994587,995258,993055,997039,998314,995349,991629,999498,992664,998959,999663,995270,994096,995736,995299\n", + "Ground Truth: 999691,991081,998864,998845,997019,990072,993366,996446,999349,993715,996106,998934,999273,995636,992752,994587,995258,993055,997039,998314,995349,991629,999498,992664,998959,999663,992975,995270,994096,991170\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 475\n", + "Returned IDs: 999384,990096,998495,996809,997918,993696,997906,998950,997320,998383,990210,991571,996805,992944,993018,995438,993260,993215,993154,996623,999210,993744,994661,999390,991801,993504,999009,991743,990843,999320\n", + "Ground Truth: 999384,990096,998495,996809,997918,993696,997906,998950,997320,998383,990210,991571,996805,992944,993018,998861,995438,993260,993154,993215,996623,999210,993744,994661,999390,991801,993504,999009,991743,990843\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 477\n", + "Returned IDs: 990866,997364,993168,997586,994782,996232,993642,993831,994964,997732,998107,997493,997279,998668,993878,995906,993689,995482,991325,998369,997268,995107,993630,998837,992862,997305,997091,991595,995944,997004\n", + "Ground Truth: 990866,997364,993168,997586,996352,994782,996232,993642,993831,994964,997732,998107,997493,997279,998668,993878,990690,995906,993689,995482,991325,998369,996647,997268,995107,993630,998837,992862,990934,997305\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 480\n", + "Returned IDs: 994964,997913,992546,994100,995981,995942,997320,997470,999444,994778,990231,992330,999109,995560,990588,996219,991721,999374,999740,997301,993168,998540,993484,999328,998495,998797,999900,990925,996075,996809\n", + "Ground Truth: 994964,997913,992546,994100,995981,995942,997320,997470,999444,994778,990231,992330,999109,995560,990588,996219,991721,999374,999740,997301,993168,998540,993484,999328,998495,998797,999900,990925,998163,996075\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 481\n", + "Returned IDs: 999860,992558,996609,996196,996523,993839,997002,997436,991334,992411,992161,993583,996606,995618,994418,996389,997003,991977,990701,997864,999240,999715,996429,996127,999540,996882,997457,997914,999374,990249\n", + "Ground Truth: 999860,992558,996609,996196,996523,993839,997002,997436,991334,992411,992161,993583,996606,995618,994418,996389,997003,995482,991977,990701,998679,997864,999240,999715,996429,996127,999540,996882,997457,997914\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 485\n", + "Returned IDs: 991610,992778,997009,990293,999555,999994,992136,992721,990167,995257,993076,991667,990177,992862,994374,998383,996408,991513,991987,998586,999430,994796,997506,990437,997752,991304,991569,996633,997655,995536\n", + "Ground Truth: 991610,992778,997009,990293,999555,999994,992136,992721,990167,995257,993076,991667,990177,992862,994374,998383,996408,991513,991987,998586,999430,994796,997506,990437,997752,991304,991569,993993,999216,996633\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 488\n", + "Returned IDs: 997270,990039,996475,998794,996650,997047,992780,990377,992100,993479,995750,999430,999323,996013,996830,999659,993890,999551,990718,991959,993391,992872,998599,992754,993721,991382,999722,994558,997331,996016\n", + "Ground Truth: 997270,990039,996475,998794,996650,997047,992780,990377,992100,993479,995750,999430,999323,996013,996830,999659,997939,993890,999551,990718,991959,993391,992872,998599,992754,993721,991382,999722,994558,997331\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 491\n", + "Returned IDs: 990096,998968,998419,993624,994636,992716,996386,991936,999103,991167,997884,999711,999119,996209,999282,998861,990072,990154,999287,995536,997803,991451,999319,993055,993744,999443,991724,996283,999384,996893\n", + "Ground Truth: 990096,998968,998419,993624,994636,992716,996386,991936,999103,991167,997884,999711,999119,996209,999282,998861,990072,990154,999287,995536,997803,991451,999319,993055,999443,993744,991724,996283,996429,999384\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 494\n", + "Returned IDs: 992169,996325,997852,992152,997476,992632,992813,992309,994922,993717,996093,990490,999676,990266,994318,999843,999453,994761,996438,992612,990399,997822,994148,999077,995210,997797,998171,994956,990268,991334\n", + "Ground Truth: 992169,996325,997852,992152,997476,992632,992813,992309,994922,993717,996093,990490,999676,990266,994318,999843,999453,994761,996438,992612,997645,990399,997822,994148,994161,999077,995210,997797,998171,994956\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 496\n", + "Returned IDs: 992124,991984,995516,998697,994742,999136,995701,998758,993186,993727,995672,995562,993470,992747,993834,996075,994374,993342,996657,999041,991530,994082,991052,994809,999783,994879,998586,998950,995203,995981\n", + "Ground Truth: 992124,991984,995516,998697,994742,999136,995701,998758,993186,993727,995672,995562,993470,992747,993834,996075,994374,993342,996657,999041,991530,994082,991052,994809,999783,997377,994879,998586,998950,995203\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 498\n", + "Returned IDs: 999899,994906,996759,992648,992651,999977,998083,990304,991175,998301,992979,993713,992126,993275,996368,997059,999820,996714,996425,993032,996075,997554,993280,994860,997910,999587,999453,994334,994946,997002\n", + "Ground Truth: 999899,994906,996759,992648,992651,999977,998083,990304,990260,991175,998301,992979,993713,992126,993275,996368,997059,999820,996714,996425,993032,996075,997554,993280,994860,997910,999587,999453,994334,994946\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 499\n", + "Returned IDs: 998080,991334,999479,998820,991025,995190,993473,993931,997395,994222,990126,998493,994204,995612,991449,991743,990733,996609,993929,997212,992685,995401,999098,990249,992604,997515,995332,995289,997723,999788\n", + "Ground Truth: 998080,991334,999479,998820,991025,995190,993473,993931,997395,994222,990126,997030,998493,999586,994204,995612,991449,991743,990733,996609,993929,997212,992685,995401,999098,990249,992604,997515,995332,995289\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 500\n", + "Returned IDs: 999009,991776,995895,990096,993397,997586,990930,990786,999430,995455,994742,996624,990866,990701,990271,997028,996830,997768,999599,991154,995701,996216,998495,996192,995435,999639,991572,996001,999647,997378\n", + "Ground Truth: 999009,991776,995895,990096,993397,997586,990930,990786,999430,995455,994742,996624,990866,990701,990271,997028,996830,997768,999599,991154,995701,996216,998495,996192,995435,999639,995899,991572,996001,999647\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 501\n", + "Returned IDs: 998158,997453,994463,992488,996062,998874,992322,990930,996915,991021,994783,995289,999139,991154,993561,993673,996071,990076,990104,991685,990226,995113,999091,995783,997797,996028,991857,992703,995586,992182\n", + "Ground Truth: 998158,997453,994463,992488,996062,998874,992322,990930,996915,991021,994783,995289,999139,991154,994777,993561,993673,996071,990076,990104,991685,990226,995113,999091,992326,995783,997797,996028,991857,992703\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 503\n", + "Returned IDs: 994894,998353,999599,995981,994100,997913,995942,992228,997946,999243,999658,991045,990701,997301,994266,998090,997728,998246,999374,996219,996805,993186,992384,997320,999331,997614,997304,999444,992848,993529\n", + "Ground Truth: 994894,998353,999599,995981,994100,997913,995942,992228,997946,999243,999658,991045,990701,997301,994266,998090,997728,998246,999374,996219,991468,996805,993186,992384,997320,999331,997614,997304,999444,992848\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 504\n", + "Returned IDs: 990532,999453,995852,994081,995130,994955,993053,995295,996147,992408,994042,995203,995456,990392,990771,991609,999052,998039,997002,999639,996918,997918,993809,998855,993906,994226,996368,992832,992562,996001\n", + "Ground Truth: 990532,999453,995852,994081,995130,994955,993053,995295,996147,992408,994042,995203,995456,990771,990392,991609,999052,998039,997002,999639,996918,995401,997918,993809,998855,993906,994226,996368,992832,992562\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 505\n", + "Returned IDs: 991698,997395,992790,990215,994671,991404,993409,992832,995375,995080,991607,994855,997515,990889,995685,992576,991576,995648,996245,992749,999052,990930,997391,998536,993412,995968,998158,997192,992036,995994\n", + "Ground Truth: 991698,997395,993407,992790,990215,994671,991404,993409,992832,995375,995080,991607,994855,997515,990889,995685,992576,991576,995648,996245,992749,999052,990930,997391,998536,993412,995968,998158,997192,992036\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 507\n", + "Returned IDs: 994472,999791,991921,990089,992322,994402,990091,995222,992820,995720,998107,997939,999876,998765,996355,990622,998355,990226,999642,998630,995535,998594,996761,993775,993584,997349,995638,993929,991325,991751\n", + "Ground Truth: 994472,999791,991921,990089,992322,994402,990091,995222,992820,995720,998107,997939,999876,998765,996355,990622,996050,998355,990226,999642,998630,995535,998594,996761,993775,993584,997349,995638,993929,991325\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 508\n", + "Returned IDs: 992582,991743,990949,991782,990958,995274,995374,999533,993961,997614,994850,990490,994266,995949,995981,997645,991479,996648,999331,990957,997913,990588,990359,990493,991889,993431,999343,990910,998294,993412\n", + "Ground Truth: 992582,991743,990949,991782,990958,995274,995374,999533,993961,997614,994850,990490,994266,995949,995981,997645,991479,996648,999331,990957,997913,990588,990359,990493,991889,999328,993431,999343,990910,998294\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 509\n", + "Returned IDs: 998982,990328,992226,990528,990701,992068,998671,996939,992230,991334,990468,990711,998719,994751,999500,994783,994689,999916,995295,991771,997379,997586,990126,996116,990011,990864,991075,991464,996645,994558\n", + "Ground Truth: 998982,990328,992226,990528,990701,996110,992068,998671,996939,992230,991334,990468,990711,998719,994751,999500,994783,994689,991239,999916,995990,995295,991771,997379,997586,990126,996116,990011,990864,991075\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 510\n", + "Returned IDs: 997913,990650,991593,998500,994964,997301,994831,997320,998170,999982,997192,992978,994618,990066,995942,999948,999167,992087,998899,995802,996219,991335,997470,999647,996504,998550,995961,996053,990210,995949\n", + "Ground Truth: 997913,990650,991593,998500,994964,997301,994831,997320,998170,999982,997192,992978,994618,990066,995942,999948,999167,992087,998899,995802,996219,991335,997470,998808,999647,996504,998550,995961,996053,990210\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 511\n", + "Returned IDs: 993387,994192,998899,992464,992119,997913,996592,990957,991479,990890,997320,993662,995336,994978,999658,996296,996075,993853,992330,999559,998550,998090,991131,994432,999948,990545,995826,992969,992255,991379\n", + "Ground Truth: 993387,994192,998899,996946,992464,992119,997913,996592,990957,991479,990890,997320,993662,992134,993174,995336,994978,999658,996296,996075,993853,992330,991260,999559,998550,998090,991131,994432,999948,996397\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 512\n", + "Returned IDs: 997168,996277,999281,991245,993275,999526,992315,994743,993715,990409,991685,992636,996001,998623,999724,990956,991349,991037,998536,993560,994016,990951,998045,995701,993561,995562,995835,993738,996677,995408\n", + "Ground Truth: 997168,996277,999281,991245,993275,999526,992315,994743,993715,990409,991685,992636,996001,998623,999724,990956,991349,991037,998536,993560,994016,990951,998045,995701,993561,995562,991755,995835,993738,994409\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 513\n", + "Returned IDs: 991375,993906,996216,996082,997790,995991,994925,996062,998950,996599,996178,995107,991769,995895,996001,995961,999827,990786,990364,999009,993561,990886,992780,995705,994280,993931,991142,995586,992847,990096\n", + "Ground Truth: 991375,993906,996216,996082,997790,995991,994925,996062,998950,996599,996178,995107,991769,995895,992473,996001,995961,999827,990786,990364,999009,993561,990886,992780,995705,994280,993931,990887,991142,995586\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 514\n", + "Returned IDs: 991396,994160,990925,998797,990595,997972,999262,996959,995884,994964,997102,993914,995588,996532,990146,998501,993168,997040,999600,994673,993389,997320,993761,997189,990863,991008,993083,991086,990797,992284\n", + "Ground Truth: 991396,994160,990925,998797,990595,997972,999262,996959,995884,994964,997102,993914,995588,996532,990146,998501,993168,997040,997028,999600,994673,993389,997320,995909,993761,997189,990863,991008,993083,991086\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 515\n", + "Returned IDs: 996379,991255,995268,990930,995963,993561,992367,991747,991603,996913,991459,997925,990984,993673,992703,996198,996277,992294,994589,992115,995701,990133,996867,990899,994113,995289,991011,998779,996328,996918\n", + "Ground Truth: 996379,991255,995268,990930,995963,993561,992367,991747,991603,996913,991459,997925,990984,993673,992703,996198,996277,992294,992660,994589,990897,992115,995701,990133,996867,990899,994113,995289,991011,998779\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 517\n", + "Returned IDs: 996038,998832,995507,997723,995139,997046,996323,995905,999619,997406,996867,992545,993433,999000,998165,993421,999530,995289,992130,997832,996769,997020,996112,991027,993390,996166,996815,995753,992605,995052\n", + "Ground Truth: 996038,998832,995507,997723,995139,997046,996323,995905,999619,997406,996867,992545,993433,999000,998165,993421,999530,995289,992130,997832,996769,997020,996112,991027,993390,996166,992660,996815,995753,992605\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 520\n", + "Returned IDs: 992692,997320,997918,993512,990171,992521,998260,995336,999731,993071,994964,993492,994360,999516,996397,994081,998365,998246,994290,995826,992124,996417,995274,997977,992330,992909,991883,991880,992047,998568\n", + "Ground Truth: 992692,997320,997918,993512,990171,992521,998260,995336,999731,993071,994964,993492,994360,999516,996397,994661,994081,998365,998246,994290,995826,992124,996417,995274,997977,992330,992909,991883,991880,992047\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 521\n", + "Returned IDs: 999453,994081,990661,990532,996075,997370,990652,998567,998492,998734,996309,990882,995515,997320,996626,992521,995372,991713,993280,995203,991869,994789,990083,999954,996368,995401,992031,995245,998607,996966\n", + "Ground Truth: 999453,994081,990661,990532,996075,997370,990652,998567,998492,998734,996309,990882,995515,997320,996626,992521,996193,995372,991713,993280,995203,991869,999123,994789,990083,999954,996368,995401,992031,995245\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 523\n", + "Returned IDs: 998246,992896,990066,997320,993484,994225,996560,998540,999102,998634,999128,995904,995224,999005,992986,995400,994542,992839,998384,997610,995841,998603,990269,996598,997178,994164,994214,991999,994964,992054\n", + "Ground Truth: 998246,992896,990066,997320,993484,994225,996560,999757,998540,999102,998634,999128,995904,995224,999005,992986,995400,994542,992839,991567,997102,998384,997610,995841,998603,990269,996598,997178,994164,994214\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 524\n", + "Returned IDs: 990882,994612,997465,999698,998937,997403,993845,998607,995515,999036,996309,992228,994922,995788,997164,995154,993002,992309,996840,993028,998921,999671,998244,992843,993143,991510,995893,996653,998401,997486\n", + "Ground Truth: 990882,994612,997465,999698,998937,997403,993845,998607,995515,999036,996309,992228,994922,995788,997164,995154,993002,992309,996840,993028,998921,999671,998244,992843,993143,997591,991510,995893,996653,998401\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 527\n", + "Returned IDs: 995596,993341,996741,990697,997305,996075,994886,992041,994985,993785,996178,990571,993119,994859,996755,997661,994440,994971,993904,992661,990022,993696,996210,997291,992635,993741,991218,991650,997320,998715\n", + "Ground Truth: 995596,993341,996741,990697,997305,996075,994886,992041,994985,993785,996178,990571,993119,994859,996755,993154,997661,994440,994971,993904,992661,990022,997699,993696,996210,997291,992635,993741,991218,991650\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 529\n", + "Returned IDs: 992636,997168,995837,997201,995295,993715,994621,999052,998076,995159,994488,997838,991451,990956,992832,992595,996505,994380,990336,990887,990889,996147,995836,993448,996277,992973,998174,995521,991007,997329\n", + "Ground Truth: 992636,997168,995837,997201,995295,993715,994621,999052,998076,995159,994488,997838,991451,990956,992832,992595,996505,994380,998021,990336,990887,990889,996147,995836,993448,996277,992973,998174,995521,991007\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 533\n", + "Returned IDs: 997506,992154,996775,996055,995226,994891,997944,997050,991633,998933,998177,999139,999079,993665,999576,993347,991712,998536,992793,990194,996817,992170,997666,997906,994955,994280,996733,995536,999091,993074\n", + "Ground Truth: 997506,992154,996775,996055,995226,994891,997944,997050,991633,998933,998177,999139,999079,993665,999576,993347,991712,998536,992793,990194,996817,990959,992170,997666,997906,994955,994280,996733,995536,999091\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 534\n", + "Returned IDs: 999876,993484,993504,998302,990092,993803,997320,998294,996869,997370,997913,998029,999363,991498,993741,994826,998580,992384,993885,999535,999374,990146,991043,991593,998246,994089,994366,995949,995372,997661\n", + "Ground Truth: 999876,993484,993504,998302,990092,993803,997320,998294,996869,997370,997913,998029,999363,991498,993741,994826,998580,992384,993885,999535,999374,990146,991043,991593,999982,998246,994089,994366,995949,995372\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 541\n", + "Returned IDs: 998965,996219,994452,992998,995339,990780,999683,995455,996111,994777,991370,996178,992070,997032,997004,995460,990886,996612,998364,990931,990366,993128,997813,999343,998390,999013,997706,998593,999876,991179\n", + "Ground Truth: 998965,996219,994452,992998,995339,990780,999683,995455,996111,994777,991370,996178,992070,997032,997004,995460,994981,990886,996612,998364,991373,990931,990366,993128,999343,997813,993690,998390,999013,997706\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 543\n", + "Returned IDs: 992944,993018,995560,998950,991142,995438,996726,998160,996140,990375,990843,991650,993324,998364,997320,991951,995107,999210,994171,994964,995130,999647,997379,995332,996370,998697,998180,993499,996653,996810\n", + "Ground Truth: 992944,993018,995560,998950,991142,995438,996726,998160,996140,990375,990843,991650,993324,998364,997320,991951,995107,999210,994171,994964,995130,999647,997379,995332,996370,999109,998697,998180,993499,996653\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 544\n", + "Returned IDs: 993755,992761,991850,998778,991779,995222,997155,992595,995536,994356,992832,999107,999256,993055,995385,996001,996216,998536,992110,997158,990127,995501,991474,991644,995655,996328,999798,992561,997586,997141\n", + "Ground Truth: 993755,992761,991850,998778,991779,995222,997155,992595,995536,994356,992832,999107,999256,993055,995385,996001,996216,998536,992110,997158,990127,995501,991474,999384,991644,995655,996328,992224,999798,992561\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 545\n", + "Returned IDs: 997391,992428,991698,996930,992933,990438,998080,998855,994193,991619,993255,991257,993288,997918,991337,992179,995357,991743,992850,996762,995511,995401,994062,992685,999144,998180,992516,990459,996903,992531\n", + "Ground Truth: 997391,992428,991698,996930,992933,990438,998080,998855,994193,991619,993255,991257,993288,997918,991337,992179,999527,995357,991743,992850,996762,995511,995401,994062,999144,992685,998180,992516,990459,996903\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 546\n", + "Returned IDs: 994419,996745,992894,990252,997776,996322,996148,998593,992660,995643,997046,998486,997940,995289,992125,995470,991301,999482,995923,996867,992048,997822,998045,993412,995702,991958,995063,998564,995401,996323\n", + "Ground Truth: 994419,996745,992894,990252,997776,996322,996148,998593,992660,995643,997046,998486,997940,995289,992125,995470,991301,999482,995923,996867,992048,997822,998045,993412,995702,991958,995063,998564,998105,995401\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 548\n", + "Returned IDs: 990092,995716,990315,999261,994922,999620,990701,990934,997371,990648,996916,991921,997770,991196,999613,991588,994990,995222,995401,995693,998641,992944,992384,997403,997855,996219,991999,995551,999343,996840\n", + "Ground Truth: 990092,995716,998991,990315,999261,994922,999620,997920,990701,990934,997371,990648,996916,991921,997770,991196,999613,991588,994990,995222,995401,995693,998641,992944,992384,996419,997403,997855,996219,999151\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 549\n", + "Returned IDs: 997897,998250,993483,999463,992437,995421,992727,998593,990255,990521,992367,998934,993175,995626,998628,992407,996181,991245,998162,993489,994280,995401,990844,998032,996001,994809,997918,993727,996657,993232\n", + "Ground Truth: 997897,998250,993483,999463,992437,995421,992727,998593,990255,990521,992367,998934,993175,995626,998628,992407,996181,991245,998162,993489,994280,995182,995401,990844,998032,996001,994809,997918,993727,996657\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 551\n", + "Returned IDs: 991335,998500,990650,997006,995985,997688,999647,995942,994776,998139,998329,990092,994699,997192,992357,996727,997371,995370,992546,990934,990366,999898,995906,990980,993034,994778,990866,992994,994889,990315\n", + "Ground Truth: 991335,998500,990650,997006,995985,997688,993174,999647,995942,994776,998139,998329,990092,994699,997192,992357,996727,997371,995370,992546,990934,990366,999898,995906,990980,992822,993034,994778,990866,996576\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 552\n", + "Returned IDs: 995655,991667,997973,992517,997145,993001,991685,999107,990330,990458,991093,995222,993280,998861,990091,998122,997732,997679,992561,998536,997265,991569,996733,996028,992862,995835,997332,999827,995780,995536\n", + "Ground Truth: 995655,991667,997973,992517,997145,993001,991685,999107,990330,990458,991093,995222,993280,998861,990091,998122,997732,997679,992561,998536,997265,991569,996733,996028,998710,992862,995835,997332,999827,993006\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 557\n", + "Returned IDs: 995336,999900,992277,998090,992384,996296,990949,994100,996657,996001,997918,997320,998951,995873,997571,993143,995374,993809,997554,997913,990772,992330,993512,990339,996592,992880,992052,993492,999144,997156\n", + "Ground Truth: 995336,999900,992277,998090,992384,996296,990949,994100,996657,996001,997918,997320,998951,990343,995873,997571,993143,995374,993809,997554,997913,990772,992330,993512,990339,996592,992880,992052,993492,999144\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 558\n", + "Returned IDs: 998934,998720,991657,991459,994167,994280,991130,996001,998593,992429,999639,999173,999216,993834,999408,995103,996408,992437,992441,993993,995158,995701,993375,996328,997790,992225,996526,993848,998864,997847\n", + "Ground Truth: 998934,998720,991657,991459,994167,994280,991130,996001,992429,998593,999639,999173,999216,993834,999408,995103,996408,992437,992441,993993,997337,995158,995701,993375,996328,997790,992225,996526,997166,993848\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 561\n", + "Returned IDs: 991753,995066,995401,996607,993003,991680,994106,996813,992448,997975,996304,990319,999054,991639,995864,998910,997544,993642,997091,996695,999759,997176,999646,997075,991795,995702,995490,992653,993209,991592\n", + "Ground Truth: 991753,995066,995401,996607,993003,991680,994106,996813,992448,997975,996304,990319,999054,993597,991639,995864,998910,997544,993642,997091,996695,999759,999646,997176,997075,991795,995702,995490,992653,993209\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 563\n", + "Returned IDs: 990508,998359,999717,990031,995206,991323,991650,993603,997581,991348,996178,992001,999132,996075,999258,993686,996626,994233,993809,999806,999473,994341,996458,990022,992557,990866,997442,990912,997367,999480\n", + "Ground Truth: 990508,998359,999717,990031,995206,991323,991650,993603,997581,991348,996178,992001,999132,996075,999258,993686,996803,996626,994233,993809,999806,999473,994341,996458,990022,992557,990866,997442,990912,997367\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 564\n", + "Returned IDs: 993591,997036,993015,993553,994275,997170,991989,996906,990368,994021,995923,995345,992585,998039,994663,990955,992798,994386,993561,993164,994379,992014,990845,993725,997357,993813,991168,993697,997933,995076\n", + "Ground Truth: 993591,997036,993015,993553,994275,997170,991989,996906,990368,994021,995923,995345,992585,998039,994663,990955,992798,994386,993561,993164,994379,992014,990845,993725,997357,993813,997920,991168,993697,997933\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 565\n", + "Returned IDs: 992286,993030,995854,997634,997020,994301,998386,995723,997114,996273,998080,993263,992683,998650,990591,992531,994677,997723,991423,996139,997395,990103,993717,990189,991404,995062,993223,995625,993127,990178\n", + "Ground Truth: 992286,993030,995854,997634,997020,994301,998386,995723,997114,996273,998080,993263,992683,998650,995282,990591,992531,994677,997723,994360,991423,996139,997395,990103,993717,990189,991404,995062,993223,995625\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 567\n", + "Returned IDs: 990974,994541,994557,993637,999009,996996,992551,999898,994638,992553,993807,997320,990096,997004,996216,993695,995906,993928,998633,991179,992927,998006,995895,990786,997410,996564,996944,995222,997732,998302\n", + "Ground Truth: 990974,994541,994557,993637,999009,996996,992551,999898,994638,992553,993807,997320,990096,997004,996216,993695,995906,993928,998633,996830,991179,992927,998006,997577,995895,990786,997410,995755,996564,996944\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 569\n", + "Returned IDs: 994145,993561,993993,993939,996328,992862,996001,990887,994853,998710,995326,995535,992450,997878,996219,996132,997529,994742,999472,990096,994682,996408,991759,999444,998382,993342,996042,995895,994830,990918\n", + "Ground Truth: 994145,993561,993993,993939,996328,992862,996001,990887,994853,998710,995326,995535,992450,997878,996219,996132,997529,994742,999472,990133,990096,994682,996408,991759,999444,998382,993342,996042,995895,994830\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 571\n", + "Returned IDs: 993997,998962,990663,999279,997031,996786,997844,990758,996131,995633,996867,998609,996302,993920,993767,995640,995268,990157,994120,999996,993209,993488,997566,995766,995289,996346,999415,996109,991214,997223\n", + "Ground Truth: 993997,998962,990663,999279,997031,996786,997844,990758,996131,995633,996867,998609,996302,993920,993767,995640,995268,990157,994120,999996,993412,993209,993488,997566,995766,995289,996346,999415,996254,996109\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 574\n", + "Returned IDs: 991348,995206,995108,996626,992318,993603,993053,995438,990083,998950,996024,991763,993621,994233,992944,991142,993166,996178,994742,993154,992617,998567,990031,995107,998180,996001,993696,997448,999426,995669\n", + "Ground Truth: 991348,995206,995108,996626,992318,993603,993053,995438,990083,998950,996024,991763,993621,994233,992944,991142,993166,993747,996178,994742,993154,992617,998567,990031,995107,998180,996001,993696,997448,999426\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 575\n", + "Returned IDs: 990450,998383,999136,998570,998586,994105,991844,999306,996446,995695,995257,996001,995142,994643,998593,996043,993948,990991,994185,991987,999653,994187,995455,997191,999117,995701,999872,991038,992945,995755\n", + "Ground Truth: 990450,998383,999136,998570,998586,994105,991844,999306,996446,995695,995257,996001,995142,994643,998593,996043,993948,990991,994185,991987,999653,994187,997766,995455,997191,998493,999117,995701,999872,991038\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 576\n", + "Returned IDs: 998623,990998,998536,995408,992052,994743,990096,999287,990255,996809,995203,997618,995536,994682,992558,991630,999107,994742,995401,994510,991629,991515,992933,994955,993910,999724,996483,995198,994954,991977\n", + "Ground Truth: 998623,990998,998536,995408,992052,994743,990096,999287,990255,996809,995203,997618,995536,994682,992558,991630,999107,995864,994742,995401,994510,991629,991515,992933,994955,993910,999724,996483,995198,994954\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 577\n", + "Returned IDs: 997918,993223,996903,995373,993512,995453,994088,991887,995444,996282,991844,994334,992604,994092,991698,995807,993412,994804,997675,998318,995685,991110,997515,997637,992933,999144,997110,999478,994819,996867\n", + "Ground Truth: 997918,993223,996903,995373,993512,995453,994088,991887,995444,996282,991844,995080,994334,992604,994092,991698,995807,993412,994804,997675,998318,995685,991110,997515,997637,992933,999144,997110,999478,994819\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 578\n", + "Returned IDs: 997110,991987,995268,999478,994948,990991,997723,994643,993122,995848,998078,990805,991630,999994,990361,993381,990930,994125,997191,996968,991052,999117,996867,991954,999530,995963,994253,995701,999555,998257\n", + "Ground Truth: 997110,991987,995268,999478,994948,990991,997723,994643,993122,995848,998078,990805,991630,999994,990361,993381,990930,994125,997191,996968,991052,999117,996867,991954,999530,995963,994253,995701,992556,999555\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 581\n", + "Returned IDs: 995899,991154,998697,992167,998720,995536,993139,994167,995562,996075,999806,999724,996811,997305,999639,996001,992225,994577,997586,992423,993560,991157,995963,996894,999107,993055,995203,998514,998568,994636\n", + "Ground Truth: 995899,991154,998697,992167,998720,995536,993139,994167,995562,996075,999806,999724,996811,997305,999639,996001,992225,994577,997586,992423,993560,991157,995963,996894,999107,993055,998105,995203,998514,998568\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 583\n", + "Returned IDs: 992680,995756,996645,993725,991669,999740,999356,991075,997798,992090,993444,999302,990743,999922,995435,991464,996411,993645,990528,992024,999261,990701,999280,992783,996627,993205,996969,998940,993053,996384\n", + "Ground Truth: 992680,995756,996645,993725,991669,999740,999356,991075,997798,992090,993444,999302,990743,999922,995435,991464,996411,991624,993645,990528,992024,999261,990701,999280,992783,996627,993205,996969,998940,993053\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 584\n", + "Returned IDs: 998182,994799,990328,998105,993857,999460,991075,990980,998017,995166,998427,998197,996689,995295,996028,997961,995823,990145,999843,993055,992480,999230,993475,992374,994119,998982,996494,990536,990097,990811\n", + "Ground Truth: 998182,994799,990328,998105,993857,999460,991075,990980,998017,995166,998427,998197,996689,995295,996028,997961,995823,990145,999843,993055,992480,999230,993475,992374,994119,998982,996494,990536,990854,990097\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 586\n", + "Returned IDs: 992183,991479,995802,998011,992946,994778,993853,998550,994699,995396,990231,997439,991207,990647,994450,993662,999243,993403,995465,996912,997913,999647,996735,991147,990478,999710,995793,997194,991502,998899\n", + "Ground Truth: 992183,991479,990109,995802,998011,992946,994778,999561,993853,998550,994699,995396,990231,997439,991207,990647,994450,993662,999243,993403,995465,996912,997913,999647,996735,991147,990478,999710,995793,997194\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 588\n", + "Returned IDs: 990845,994925,996906,998593,992703,997453,998039,993561,991958,995162,996062,996001,999729,995401,991789,993906,991989,992334,996237,997028,990899,991011,990096,990076,995289,996311,996277,999340,998207,994424\n", + "Ground Truth: 990845,994925,996906,998593,992703,997453,998039,993561,991958,995162,996062,996001,999729,995401,991789,993906,991989,992334,996237,997028,990899,991011,990096,990076,995289,996311,996277,999340,998207,995560\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 589\n", + "Returned IDs: 990271,997878,990179,995222,991776,995868,990621,994448,992193,996219,998269,995719,994145,990369,996580,994689,994373,990272,997349,995096,991075,998204,990077,999102,990315,995438,995690,994682,994742,993725\n", + "Ground Truth: 990271,997878,990179,995222,991776,995868,990621,994448,992193,996219,998269,995719,994145,990369,996580,994689,994373,990272,997349,996593,995096,991075,998204,990077,999102,990315,995438,995690,994682,994742\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 591\n", + "Returned IDs: 993433,996042,995289,996607,992448,998910,995401,995146,993561,995820,995018,995200,996769,995521,998162,995864,996112,996815,991671,999611,992576,991699,996867,997723,991958,994409,997822,995766,999054,997406\n", + "Ground Truth: 993433,996042,995289,996607,992448,998910,995401,995146,993561,996809,995820,995018,997023,995200,996769,995521,998162,995864,996112,996815,991671,999611,992576,991699,996867,997723,991958,994409,997822,995766\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 593\n", + "Returned IDs: 992558,993255,994280,992790,995401,990790,997395,995374,995886,995134,990214,991404,999715,995145,996762,993478,991977,997918,990538,992685,999305,995119,995154,995968,991052,990969,992933,996803,999179,999154\n", + "Ground Truth: 992558,993255,994280,992790,995401,990790,997395,995374,995886,995134,990214,991404,999715,995145,996762,993478,991977,997918,990538,992685,999305,995119,996721,995154,995968,991052,998045,990969,992933,999461\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 595\n", + "Returned IDs: 991828,992186,990580,991840,990310,997539,994922,999076,996402,996996,993036,993577,994005,990138,995515,996449,996478,991609,991172,999846,996840,993143,992207,991519,998455,995156,995767,998711,991072,994917\n", + "Ground Truth: 991828,992186,990580,991840,990310,997539,994922,999076,996402,996996,993036,993577,994005,990138,995515,996449,996478,994700,991609,991172,999846,993278,996840,993143,992207,991519,998455,995156,995767,998711\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 596\n", + "Returned IDs: 995981,997620,999374,997304,997437,998246,995792,995596,992357,994266,999331,997913,998697,992748,994100,995132,990588,997429,993186,994916,997320,994894,995638,990949,997614,996075,992384,999815,998353,991045\n", + "Ground Truth: 995981,997620,999374,997304,997437,998246,995792,995596,992357,994266,999331,997913,998697,992748,992047,999710,994100,995132,990588,997429,993412,993186,994916,992583,997320,998974,994894,995638,990949,997614\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 600\n", + "Returned IDs: 996610,997787,991603,998906,996047,996388,990033,993390,995289,991463,997406,990681,998045,990940,994161,998165,993561,996112,992125,991367,991954,996769,998999,995182,992160,997046,995497,997832,995318,995052\n", + "Ground Truth: 996610,997787,991603,998906,996047,996388,990033,993390,995289,991463,997406,990681,998045,990940,994161,998165,995040,993561,996112,992125,991367,991954,996769,998999,995182,992160,997046,995497,997832,995318\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 604\n", + "Returned IDs: 994297,995244,994964,993452,991438,994919,998500,999261,999821,998837,999444,992169,995135,996854,995591,998909,991991,999545,999740,994922,991999,990736,990432,997984,996351,991840,998553,995336,994898,997720\n", + "Ground Truth: 994297,995244,994964,993452,991438,994919,998500,999261,999821,998837,999444,990064,992169,995135,996854,995591,998909,991991,999545,999740,994922,991999,990736,990432,997984,996351,991840,998553,995336,991751\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 605\n", + "Returned IDs: 999375,997152,992468,991960,998196,994193,991758,994119,999788,998274,992056,991447,995438,991515,998543,994742,994541,995494,997452,990814,991334,999981,990127,993180,997967,992077,998913,990796,996302,999543\n", + "Ground Truth: 999375,997152,992468,991960,998196,994193,991758,994119,999788,998274,992056,991447,998466,995438,991515,998543,994742,994541,995494,997452,990814,991334,999981,990127,993180,997967,992077,998913,990796,996302\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 610\n", + "Returned IDs: 991638,997732,994956,993929,997112,993765,992655,998107,992120,995861,992813,999827,992169,998458,992408,990226,997852,990441,991334,994131,993877,990889,990961,997453,990266,996328,994557,998495,997822,993080\n", + "Ground Truth: 991638,997732,994956,993929,997112,993765,992655,998107,992120,995861,992813,999985,999827,992169,998458,992408,990226,997852,990441,991334,994131,993877,990889,990961,997453,990266,996328,994557,998495,997822\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 611\n", + "Returned IDs: 999056,999426,996744,996500,992167,996075,991984,997777,992041,992927,993766,995291,991823,993283,993156,997661,995596,996761,995132,993237,999041,999132,996169,996001,995516,995898,994809,993834,998990,999219\n", + "Ground Truth: 999056,999426,996744,996500,992167,996075,991984,997777,992041,992927,993766,995291,991823,993283,993156,997661,995596,996761,995132,993237,995375,999041,999132,996169,996001,995516,995898,994809,993834,998990\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 612\n", + "Returned IDs: 994830,994742,998950,993588,996809,992807,998913,998823,994806,995552,994269,992318,996805,991154,996301,999627,990392,991390,993342,998495,995299,999430,992598,992551,994751,992427,993053,995899,995002,994682\n", + "Ground Truth: 994830,994742,998950,993588,996809,992807,998913,998823,994806,995552,994269,992318,996805,991154,996301,999627,990392,991390,993342,998495,995299,999430,992598,992551,994751,992427,993053,995899,995401,995002\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 613\n", + "Returned IDs: 994051,990256,991860,995516,997645,992915,997811,991799,995755,990862,992596,992165,995132,997429,993396,993336,997305,996856,996811,999251,991052,994879,992357,995963,991690,995184,996406,990958,996942,991982\n", + "Ground Truth: 994051,990256,991860,995516,997645,992915,997811,991799,995755,990862,992596,992165,997437,995132,997429,993396,999725,993336,992238,997305,996856,996811,999251,994335,991052,994879,992357,995963,990178,992546\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 615\n", + "Returned IDs: 996116,990701,997374,994350,997207,998913,997209,992024,994120,990126,995704,991156,994168,995482,997628,992599,992251,992871,993240,993384,990614,994730,999402,992156,993857,994728,995633,998982,991943,996481\n", + "Ground Truth: 996116,990701,997374,994350,997207,998913,997209,992024,994120,990126,995704,999586,991156,994168,995482,997628,992599,992251,992871,993240,993384,990614,994730,999402,992156,993857,994728,995633,998982,991943\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 616\n", + "Returned IDs: 998566,997193,995691,997126,990305,994064,993566,996194,990184,996451,991636,996421,993127,998466,995482,990189,996170,990776,993678,999025,992841,995883,995620,992944,990948,993920,999158,992067,993181,994351\n", + "Ground Truth: 998566,997193,995691,997126,990305,994064,993566,996194,990184,996451,991636,996421,993127,998466,995482,990189,991157,996170,990776,993678,999025,999586,992704,991938,992841,995883,990827,990052,995620,992944\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 618\n", + "Returned IDs: 998888,999555,991944,990991,999584,991100,992638,991188,997154,997850,996393,995655,990167,998006,993948,993749,990423,990558,990154,994727,994467,993381,997030,993956,990361,994374,994643,991385,991596,998257\n", + "Ground Truth: 998888,999555,991944,990991,999584,991100,992638,991188,997154,997850,996393,995655,990167,998006,993948,993749,990423,990558,990154,994727,994467,993381,997030,993956,990361,994374,991836,994643,991385,991596\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 619\n", + "Returned IDs: 997995,997455,995095,996609,992281,997993,998221,990432,993219,995401,992619,991836,990435,993209,999820,990126,994587,995354,999923,992979,997185,995289,993412,999160,990157,993656,995633,995374,999240,998297\n", + "Ground Truth: 997995,997455,995095,996609,992281,997993,998221,992087,990432,993219,991445,995401,992619,991836,990435,996088,993209,999820,990126,994587,995354,999923,992979,997185,995289,993412,999160,990157,993656,995633\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 620\n", + "Returned IDs: 997652,997897,998428,992052,993621,998522,993171,990956,999002,999179,998250,995421,992028,992747,990409,997368,990626,994280,994765,995213,994743,994742,997494,991605,996328,996461,999587,999927,990364,993175\n", + "Ground Truth: 997652,997897,998428,992052,993621,998522,993171,990956,999002,999179,998250,995421,992028,992747,990409,997368,990626,994280,994765,995213,994743,997723,994742,997494,991605,996328,996461,999587,999927,990364\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 621\n", + "Returned IDs: 995941,997371,990645,993507,994757,993818,998246,991782,995367,995755,996890,997192,993005,994281,990994,997194,992053,990863,999374,991379,991059,993518,997984,995353,999247,990969,993739,999290,992978,991456\n", + "Ground Truth: 995941,997371,990645,993507,994757,993818,998246,991782,995367,995755,996890,997192,993005,994281,990994,997194,994183,992053,990863,999374,991379,993326,991059,993518,997984,995353,999247,997022,990969,993739\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 623\n", + "Returned IDs: 992558,999564,992738,995724,998827,994372,993140,997124,990273,993077,995983,991009,997158,999146,996918,990538,992170,992179,994163,991260,991905,991157,997775,995411,999639,994742,993706,992615,998142,997859\n", + "Ground Truth: 992558,999564,992738,995724,998827,994372,993140,997124,990273,993077,995983,991009,995281,997158,991585,999146,996918,990538,992170,992179,997411,994163,991260,991905,991157,997775,995411,999639,994742,995401\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 625\n", + "Returned IDs: 997977,992823,993504,995168,994468,991425,992082,997618,994672,996217,998303,997140,998007,995536,995231,993260,990196,997002,993071,997790,997918,995740,999111,996001,994955,995203,991863,996884,990350,997913\n", + "Ground Truth: 997977,992823,993504,995168,994468,991425,992082,997618,994672,996217,998303,997140,998007,995536,995231,993260,990196,997002,993071,997790,997918,995740,999111,996001,994955,995203,991863,995167,996884,990350\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 626\n", + "Returned IDs: 994906,993289,993342,999587,992664,993820,991459,995401,994946,992636,998855,998536,993275,990399,998623,999977,997332,996764,997002,996001,999820,999724,992179,996147,997910,994743,997790,993715,998831,999453\n", + "Ground Truth: 994906,993289,993342,999587,992664,993820,991459,995401,994946,992636,998855,998536,993275,990399,998623,999977,997332,996764,997002,996001,999820,999724,996371,992179,996147,997910,994743,997790,993715,998831\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 628\n", + "Returned IDs: 999457,990194,998279,999555,999994,999205,993461,990167,999223,998674,999430,992628,998072,993076,994109,993224,998586,998570,991038,997415,991610,995234,995848,991025,999985,998086,998078,995907,997506,994731\n", + "Ground Truth: 999457,990194,998279,999555,999994,999205,993461,990167,999223,998674,999430,992628,998072,993076,994109,993224,998586,998570,991038,997415,991610,995234,995848,991025,999985,998086,998078,995907,995082,997506\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 629\n", + "Returned IDs: 993250,995135,990898,998751,999649,998660,995865,991582,991172,991840,996854,994819,998011,998996,996352,995365,999676,998915,993640,997479,990965,990310,993253,995693,990979,991999,994415,998540,997192,994497\n", + "Ground Truth: 993250,995135,990898,998751,999649,998660,995865,991582,991172,991840,996854,994819,998011,998996,996352,997641,995365,999676,998915,997291,993640,997479,990965,990310,993253,995693,993404,990979,991999,994415\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 630\n", + "Returned IDs: 994852,992190,999451,997106,990884,999797,994622,996894,995138,998455,995752,992349,993789,997818,993355,999645,997076,990096,996884,995769,993706,998974,992668,996196,999807,999639,996262,990522,993222,991609\n", + "Ground Truth: 994852,992190,999451,997106,990884,999797,994622,996894,995138,998455,995752,992349,993789,997818,993355,999645,997076,990096,996884,995769,993706,998974,992668,996196,999807,991637,999639,992612,996262,990351\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 632\n", + "Returned IDs: 995210,996808,997918,996062,998436,993512,990366,993289,990891,997002,991425,994821,992711,990885,994468,994042,993260,994959,996001,993504,990622,995826,994694,991833,993268,999091,991311,992420,993157,994538\n", + "Ground Truth: 995210,996808,997918,996062,998436,993512,990366,993289,990891,997002,991425,997332,994821,992711,990885,994468,994042,993260,992612,994959,997531,996001,993504,990622,995826,998611,994694,991833,993268,999091\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 633\n", + "Returned IDs: 999854,990584,998228,993637,998436,997790,999827,996062,990557,995678,994472,997918,990091,997002,998039,999516,996001,993906,993765,997586,997920,992576,991994,992161,995107,993484,997453,992395,990080,998734\n", + "Ground Truth: 999854,990584,998228,993637,998436,997790,999827,996062,990557,995678,994472,997918,990091,997002,998039,993238,999516,996001,993906,993765,995502,997586,992576,997920,991994,992161,995107,993484,997453,992395\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 636\n", + "Returned IDs: 995557,999604,996596,990310,993748,999592,998899,993237,995365,991593,993279,996748,995588,993091,992207,998917,995952,993675,992195,994856,998997,998057,999039,998654,994263,992085,995072,991519,994964,997275\n", + "Ground Truth: 995557,999604,996596,990310,993748,999592,998899,993237,995365,991593,993279,996748,995588,993091,992207,998917,995952,995794,993675,992195,994856,998997,998057,999039,998654,994263,992085,995072,994005,991519\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 638\n", + "Returned IDs: 999638,990926,992349,992718,997816,997859,994163,991540,996894,995138,996318,994010,991498,991977,998697,998417,997113,998118,998334,999387,994964,992124,994165,990908,990343,995536,994972,991637,999908,995986\n", + "Ground Truth: 999638,990926,992349,992718,997816,997859,994163,991540,993948,996894,995138,996318,994010,991498,991977,998697,998417,997113,998118,998334,999387,994964,992124,994165,990908,990343,995536,994972,991637,999908\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 639\n", + "Returned IDs: 994137,999358,995378,990560,990390,999363,991900,998462,995942,998966,992310,991396,993403,990146,993072,991444,997972,993484,999229,990742,994964,997880,994479,999110,997539,999710,990311,996629,998365,995949\n", + "Ground Truth: 994137,999358,995378,990560,990390,999363,991900,998462,995942,998966,992310,991396,993403,990146,993072,991444,997972,993484,991178,999229,990742,994964,997880,994479,999110,993284,999390,997539,999710,990311\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 640\n", + "Returned IDs: 996127,993839,998220,999860,999639,993561,991923,990550,990303,994117,995013,991958,996181,996196,998593,991332,993489,990167,996391,994372,999393,991350,996379,995562,994796,998186,995983,998514,995855,998697\n", + "Ground Truth: 996127,993839,998220,999860,999639,993561,991923,990550,990303,994117,995013,991958,992473,996181,996196,998593,991332,993489,990167,996391,994372,999393,991350,991212,996379,995562,994796,998186,995983,998514\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 641\n", + "Returned IDs: 991810,996809,990127,993817,999788,994682,994742,992827,998063,999183,992372,997349,997319,993744,995633,999852,990223,995222,995895,990173,995401,998536,998495,990986,990722,996408,997401,990392,991620,998525\n", + "Ground Truth: 991810,996809,990127,993817,999788,994682,994742,992827,998063,999183,992372,997349,997319,993744,995633,999852,990223,995222,992318,995895,990173,995401,998536,998495,990986,990722,996408,997401,990392,997529\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 643\n", + "Returned IDs: 997749,999411,994909,995421,994765,993727,992934,998123,990521,994280,991491,996276,990844,994541,993678,998934,995963,997897,990998,995110,994697,998510,994809,992112,990021,993448,999601,992457,999179,995401\n", + "Ground Truth: 997749,999411,994909,995421,994765,993727,992934,998123,990521,994280,999453,991491,996276,990844,994541,993678,998934,995963,997897,990998,995110,994697,998510,994809,992112,990021,993448,999601,992457,999179\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 644\n", + "Returned IDs: 996296,991456,990176,997022,994842,996075,990742,997699,992931,992880,997178,990957,997918,991131,990949,995417,998540,993253,996592,993696,998967,992442,991634,995336,992942,993529,994100,999710,992277,992978\n", + "Ground Truth: 996296,991456,990176,997022,994842,996075,990742,997699,992931,992880,997178,990957,997918,991131,990949,995417,998540,993253,996592,993696,998967,992442,991634,995336,993529,992942,994100,999710,992277,995981\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 645\n", + "Returned IDs: 997867,991740,993839,992376,998357,991977,993583,998879,995965,990908,993647,994614,991064,999008,993975,990894,991225,995491,997547,996663,992333,995459,993681,996585,999022,994254,993500,994398,999763,999387\n", + "Ground Truth: 997867,991740,993839,992376,998357,991977,993583,998879,995965,990908,993647,997407,994614,991064,999008,993975,990894,991225,995491,997547,996663,992333,995459,993681,996585,999022,994254,993500,994398,999763\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 646\n", + "Returned IDs: 997643,991309,994660,992638,991797,998277,994393,992943,997683,993566,993749,994294,997568,993666,999156,997110,994656,997030,992926,995044,997429,996393,990554,995994,997114,998888,995253,990132,991937,998858\n", + "Ground Truth: 997643,991309,994660,992638,991797,998277,994393,992943,997683,999530,993566,993749,994294,997568,993666,999156,997110,994656,997030,992926,995044,997429,996393,990554,995994,997114,998888,995253,990132,991937\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 647\n", + "Returned IDs: 990679,994552,998734,995855,995250,999473,993154,997370,994789,995203,999954,994212,999898,996075,996623,997918,993243,990352,993484,990652,994829,990470,993646,999876,999384,999132,997920,990957,992548,996001\n", + "Ground Truth: 990679,994552,998734,995855,995250,999473,993154,997370,994789,995203,999954,994212,999898,996075,996623,997918,993243,990352,993484,990652,994829,990470,993646,990809,996452,999876,999807,999384,999132,997920\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 648\n", + "Returned IDs: 994125,990641,996609,990655,992604,997918,995401,997403,991844,995289,998913,991996,996809,993412,994317,995718,993433,996411,996196,991179,996075,993725,995274,990126,997906,998999,994603,993561,998495,995881\n", + "Ground Truth: 994125,990641,996609,990655,992604,997918,995401,998383,997403,991844,998913,995289,991996,996809,993529,993412,994317,995718,993433,996411,996196,991179,996075,993725,995274,990126,997906,998999,994603,993561\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 649\n", + "Returned IDs: 996360,998045,993561,998545,991052,991958,993433,996042,995702,996322,994409,997822,991923,993412,996867,994419,996769,995679,995648,994549,994280,997868,997386,995873,998564,995536,997674,997675,997289,990137\n", + "Ground Truth: 996360,998045,993561,998545,991052,991958,999809,993433,996042,995702,996322,994409,990882,997822,991923,993412,996867,994419,996769,995679,995648,994549,994280,997868,997386,995873,998564,995536,997674,997675\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 650\n", + "Returned IDs: 993839,996127,996658,991432,998372,993718,995340,990061,990487,991028,998079,997864,991923,993976,998370,991743,991350,996391,992813,990638,995691,994514,995515,993583,998779,998879,997914,998686,992929,990602\n", + "Ground Truth: 993839,996127,996658,991432,998372,993718,995340,990061,990487,991028,998079,997864,991923,993976,998370,991743,991350,996391,992813,990638,995691,995515,994514,993583,998779,998879,997914,998686,995453,992929\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 652\n", + "Returned IDs: 999328,996646,997320,999647,997913,994450,993253,999109,997470,999948,991743,999324,998967,998550,998287,990182,994192,990650,996296,992969,999390,993387,998302,997826,999701,994266,998644,991593,993031,991616\n", + "Ground Truth: 999328,996646,997320,999647,997913,994450,993253,999109,997470,999948,991743,999324,998967,998550,998287,990182,994192,990650,997921,996296,992950,992969,999390,999258,993387,998302,997826,999701,994266,998644\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 654\n", + "Returned IDs: 995374,993636,997192,993852,997571,998844,994281,999374,996554,994850,999953,995833,990275,992147,992183,994218,990432,992309,998011,998263,995354,999144,993529,995336,999317,993443,999883,999804,992854,996075\n", + "Ground Truth: 995374,993636,997192,993852,997571,998844,994281,999374,996554,994850,999953,995833,990275,992147,992183,994218,990432,992309,998011,998263,995354,999144,993529,995336,999317,993443,999883,999562,999804,992854\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 655\n", + "Returned IDs: 996247,991459,999804,990294,999639,995894,997918,991979,994125,996407,991630,996500,997957,998022,992748,992473,993948,996918,992516,992429,995932,990897,992558,993188,999306,991657,992293,995701,995336,998758\n", + "Ground Truth: 996247,991459,999804,990294,999639,995894,997918,991979,994125,996407,995132,991630,996500,997957,994911,998022,992748,992473,993948,996918,992516,992429,995932,990897,992927,992558,993188,999306,993906,991657\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 656\n", + "Returned IDs: 999544,999676,992348,999453,994800,991787,999452,995968,992612,994195,993683,991673,996309,992907,992309,992739,990080,993781,991485,995372,999686,998607,992298,999254,994964,993633,991999,995365,999951,997498\n", + "Ground Truth: 999544,999676,992348,999453,994800,991787,999452,995968,992612,994195,993683,991673,996309,992907,992309,992739,990080,993781,991485,995372,996352,999686,998607,992298,999254,994964,993633,991999,995365,999951\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 658\n", + "Returned IDs: 994488,994416,992318,999060,993772,996809,999179,997332,990843,998950,991650,994541,997679,996001,993603,998543,994955,998218,993347,993721,991142,990930,995899,995678,991409,999320,996024,993293,997442,999009\n", + "Ground Truth: 994488,994416,992318,999060,993772,996809,999179,997332,990843,998950,991650,994541,997679,996001,993603,998543,994955,998218,993347,993721,991142,990930,993079,995899,995678,991409,995167,999320,996503,996024\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 661\n", + "Returned IDs: 993461,990479,996918,991052,996535,990897,998267,993014,999646,997723,997586,990041,991758,998564,992933,998518,999980,995963,997910,994282,992979,997675,997216,991891,993412,992068,991666,992558,990942,998067\n", + "Ground Truth: 993461,990479,996918,991052,996535,990897,998267,993014,999646,997723,997586,990041,991758,998564,992933,998518,999980,995963,997910,998924,994282,992979,997675,997216,991891,998105,993412,992068,991666,992558\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 665\n", + "Returned IDs: 997777,990949,991593,997320,999290,993484,997370,992839,993253,999855,999535,990957,997539,995515,997372,995203,999453,990742,990890,994486,995942,999982,995400,999680,994826,992069,993389,999437,995596,993278\n", + "Ground Truth: 997777,990949,991593,997320,999290,993484,997370,992839,993253,999855,999535,990957,997539,995515,997372,995203,999453,990742,990890,996002,994486,995942,999982,995400,999680,994826,992069,993389,999437,995596\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 666\n", + "Returned IDs: 994954,992521,993438,994409,999866,992448,995640,995873,996368,997868,993561,996987,991969,995820,998255,996187,997091,990005,995701,998205,999250,997466,991349,998045,992426,991967,997723,997554,992937,997612\n", + "Ground Truth: 994954,992521,993438,994409,999866,992448,995640,995873,996368,996769,997868,993561,996987,991969,995820,998255,996187,997091,990005,995701,998205,999250,997466,991349,998045,992426,991967,997723,997554,993517\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 668\n", + "Returned IDs: 999117,993122,994467,995625,995044,991188,994643,997149,997154,992809,995207,999555,990991,999155,996395,992945,993795,999663,995257,993469,993995,993232,998861,996209,996393,992638,998536,998006,991385,997205\n", + "Ground Truth: 999117,993122,994467,995625,995044,991188,994643,997149,997154,992809,995207,999555,990991,999155,996395,992945,993795,999663,995257,993469,993995,993232,998861,996209,996393,992638,998536,995401,998006,999431\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 669\n", + "Returned IDs: 996489,990442,996981,996186,990249,990256,990209,992558,994334,992017,995332,997380,991426,996811,995552,990538,997155,995449,992179,999860,993473,995401,992170,999859,996196,993515,990620,995528,999715,999908\n", + "Ground Truth: 999901,996489,990442,996981,996186,990249,990256,990209,992558,994334,992017,995332,997380,991426,996811,995552,999480,990538,997155,995449,992179,999860,993473,995401,992170,998213,999859,996196,993515,990620\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 670\n", + "Returned IDs: 998289,998104,993324,992558,994732,990264,998950,996140,995289,996810,990769,994697,993079,997752,998697,996194,992988,995783,995281,996210,996370,994146,995359,994817,993018,996276,990956,997723,992052,997713\n", + "Ground Truth: 998289,998104,993324,992558,994732,990264,998950,996140,995289,996810,990769,995928,994697,993079,997752,994514,997630,998697,996852,996194,992988,995963,995783,995281,996210,996370,994146,995359,994817,993018\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 671\n", + "Returned IDs: 991925,996811,991743,998334,992702,993721,994119,990585,996918,993229,991977,999601,995375,996318,994662,998495,997481,999453,998849,992944,995873,999711,999707,995130,990640,996219,997436,996001,993121,991807\n", + "Ground Truth: 991925,996811,991743,998334,992702,993721,994119,990585,996918,993229,991977,999601,995375,996318,994662,998495,997481,999453,998849,994081,992944,997732,995873,999711,999707,995130,990640,996219,997436,996001\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 673\n", + "Returned IDs: 999703,992640,993314,993895,991161,993580,992728,990479,999520,998463,992598,994672,994280,991731,998045,990942,990882,994844,993315,998812,993609,995815,999441,996867,999052,999373,997192,994920,992752,995196\n", + "Ground Truth: 999703,992640,993314,993895,991161,993580,992728,990479,999520,998463,992598,994672,993433,994280,991731,998045,990942,990882,994844,993315,998812,993609,995815,999441,992617,996867,999052,999373,997192,994920\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 674\n", + "Returned IDs: 996913,994136,996761,996067,997839,997790,990550,996089,998473,999363,993834,998934,993766,990897,998185,993478,994910,992645,992429,992750,996795,990139,999216,999041,996547,993809,999384,993143,994915,996611\n", + "Ground Truth: 996913,994136,996761,996067,997839,996144,997790,990550,996089,998473,999363,993834,998399,998934,993766,990897,998185,993478,994910,992645,992429,992750,996795,990139,999216,999041,996547,993809,999384,996803\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 676\n", + "Returned IDs: 999290,997370,996840,992944,993484,998734,996762,995886,993143,999367,996949,996438,993052,992558,993080,995203,991840,990334,999426,991348,990697,998455,997839,990580,995596,993504,996397,992169,999453,993630\n", + "Ground Truth: 999290,997370,996840,992944,993484,998734,996762,995886,993143,999367,996949,996438,993052,992558,993080,995203,991840,990334,999426,991348,990697,998455,990351,997839,990580,995596,993504,996397,992169,999453\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 678\n", + "Returned IDs: 997348,999647,991582,994450,991428,996805,997913,993853,998426,991550,990893,999444,997918,993264,990231,994964,990797,997320,997880,996041,998921,998029,998550,990271,997301,993529,997586,994503,996809,994266\n", + "Ground Truth: 997348,999647,996904,991582,994450,991428,996805,997913,993853,998426,993141,991550,990893,997918,999444,993264,990231,994964,993412,990797,997320,997880,996041,996296,998921,998029,997772,998550,991721,999683\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 680\n", + "Returned IDs: 997920,996232,996219,993412,994894,998649,999144,995274,991029,992337,999343,994757,996052,999261,999311,996504,995881,996001,995502,994819,999026,993168,993529,990949,999647,997479,995200,990975,990934,996869\n", + "Ground Truth: 997920,996232,996219,993412,994894,998649,999144,995274,991029,992337,999343,994757,996052,999261,999311,996504,991445,995881,995401,996001,995502,994819,999026,993168,993529,990949,999647,997479,995200,990975\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 681\n", + "Returned IDs: 997975,999137,999415,998188,993209,991671,993586,995778,994730,996302,991639,997723,999581,991507,996226,992289,999875,997544,993412,999054,996867,996852,995268,996187,990361,999957,998910,999980,992448,994742\n", + "Ground Truth: 997975,999137,999415,998188,993209,991671,993586,995778,994730,996302,991639,997723,999581,991507,996226,992289,999875,997544,993412,999054,996867,996852,995268,996607,996187,990361,999957,998910,999980,995963\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 682\n", + "Returned IDs: 998104,994300,993799,998536,993609,995963,990392,994119,990879,992617,994711,998568,992521,993324,996760,992131,997723,997752,995783,994409,999129,998162,996987,996328,997309,994280,997420,999886,996867,995401\n", + "Ground Truth: 998104,994300,993799,998536,993609,995963,990392,994119,990879,992617,994711,998568,992521,993324,999840,996760,992131,997723,997752,992289,995783,994409,999129,998162,996987,996328,997309,994280,997420,999886\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 688\n", + "Returned IDs: 999812,993504,993024,996429,994964,990818,999339,993260,999111,995895,998303,998715,998950,993391,996216,992747,997586,992571,993805,994672,995622,999735,990409,997977,991458,993489,996196,995887,991946,990494\n", + "Ground Truth: 999812,993504,993024,996429,994964,990818,999339,993260,999111,995895,998303,998715,998950,993391,996216,992747,997586,992571,996677,993805,992979,994672,995622,999735,990409,997977,991458,993489,996196,995887\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 689\n", + "Returned IDs: 992493,997634,995163,992677,998316,992150,992148,993223,997824,998650,995854,994620,993263,995723,992700,995060,991067,993530,995091,997750,996139,991190,996273,995062,991100,998080,999961,993160,991052,997723\n", + "Ground Truth: 992493,997634,995163,992677,998316,992150,992148,993223,997824,998650,995854,994620,993263,995723,992700,995060,991067,995963,993530,995091,997750,996139,991190,996273,995062,991100,998080,999961,993160,991052\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 690\n", + "Returned IDs: 992933,998536,999759,994835,992166,999259,993014,995175,996877,994119,990103,999533,995783,991052,994742,999847,994522,999244,995963,997309,997184,991712,994280,998021,991891,995685,997723,990184,995110,991747\n", + "Ground Truth: 992933,998536,999759,994835,992166,999259,993014,995175,996877,994119,990103,999533,995783,991052,994742,999847,994522,999244,995963,997309,999527,997184,991712,994280,998021,991891,995685,997723,990184,995110\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 691\n", + "Returned IDs: 999045,993721,994718,992551,990096,991409,998495,994269,991650,993053,996001,999336,996075,990843,996936,995361,994355,990584,991382,996219,999577,996631,998803,997550,991451,990091,996101,999343,994472,996016\n", + "Ground Truth: 999045,993721,994718,992551,990096,991409,998495,994269,991650,993053,996001,999336,996075,990843,996936,995361,994355,990584,991382,996219,999577,996631,998803,997550,991451,999302,990091,996101,999343,994472\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 694\n", + "Returned IDs: 990010,991425,999794,999211,993504,990956,993665,991552,998615,995349,992457,998303,999111,992137,991648,994672,996884,995541,992110,995103,992351,990752,997506,991037,996012,990658,993609,990409,994681,991257\n", + "Ground Truth: 990010,991425,999794,999211,993504,990956,993665,991552,998615,995349,992457,998303,999111,992137,991648,994672,998109,996884,995541,992110,995103,992351,990752,997506,991037,996012,990658,993609,990409,994681\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 696\n", + "Returned IDs: 993003,992267,996256,992917,994409,997466,991357,999250,995401,992448,997909,990295,992763,995242,996815,994508,994865,998910,998278,994106,993537,992521,997299,996968,998756,991197,995268,992260,999530,995864\n", + "Ground Truth: 993003,992267,996256,992917,994409,997466,999250,991357,995401,992448,997909,990295,992763,995242,996815,994508,994865,998910,998278,994106,993537,992521,996223,997299,996968,998756,991197,995268,992260,999530\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 697\n", + "Returned IDs: 993512,995336,997571,993096,999144,999331,998090,990647,992134,993853,992384,994197,990214,994810,992692,994334,993492,991131,993153,995357,995374,994819,993925,996882,998562,993785,998410,996296,992453,997876\n", + "Ground Truth: 993512,995336,997571,993096,999144,999331,998090,992134,990647,993853,992384,994197,990214,994810,992692,995748,994334,993492,991131,993153,995357,995374,994819,993925,991923,996882,990654,998562,993785,998410\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 700\n", + "Returned IDs: 990956,996016,991382,992652,992827,993501,994927,998543,994541,998495,996809,994378,995191,999179,994765,995895,993460,998167,996830,997270,991469,996706,999830,993219,996301,998206,993948,990712,993391,997506\n", + "Ground Truth: 990956,996016,991382,992652,992827,993501,994927,998543,994541,998495,996809,994378,995191,999179,994765,995895,993460,998167,996830,997270,995401,991469,996706,999830,993219,996301,998206,993948,990712,993391\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 701\n", + "Returned IDs: 990930,991459,992036,992790,999639,998874,995536,995401,996001,994987,990412,993515,993673,996247,995932,991408,991530,997918,998990,995672,998593,991052,996918,998303,991392,996181,997238,991721,994280,992749\n", + "Ground Truth: 990930,991459,992036,992790,999639,998874,995536,995401,996001,994987,990412,993515,993673,990337,996247,995932,991408,996371,991530,997918,998990,995672,998593,991052,996918,998303,991392,996181,997238,991721\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 704\n", + "Returned IDs: 990003,997913,999647,990210,990545,996053,998344,999331,994873,998302,990958,993031,991479,990231,998540,991429,995438,990650,991852,991782,994450,993484,997320,993990,996296,998550,992789,999167,999263,999109\n", + "Ground Truth: 990003,997913,999647,990210,990545,992946,996053,998344,999331,994873,998302,990958,993031,991479,997439,990231,998540,991429,995438,990650,991852,991782,994450,993484,997320,993990,991370,994898,996296,998550\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 705\n", + "Returned IDs: 997897,994280,995421,991605,991447,991232,993489,992052,990627,997442,990844,992864,997926,998071,995626,995203,992457,991332,996817,997531,995630,998428,990885,997790,991218,991212,993175,999550,993391,994738\n", + "Ground Truth: 997897,994280,995421,991605,991447,991232,993489,992052,990627,997442,990844,992864,997926,998071,995626,993087,995203,992457,991332,996817,997531,995630,998428,990885,997790,991218,991212,993175,999550,993391\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 706\n", + "Returned IDs: 993368,998798,995374,996703,990992,997015,991467,992854,996353,998951,990585,994197,991504,997609,990535,999026,996959,994595,997009,991173,991498,999089,996714,992189,996658,993562,995824,995034,998844,999024\n", + "Ground Truth: 993368,998798,995374,996703,990992,997015,991467,992854,996353,998951,990585,994197,991504,997609,990535,993909,999026,996959,994595,997009,991173,991498,999089,996714,992189,996658,993562,995824,995034,998844\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 708\n", + "Returned IDs: 996052,996219,998383,990448,999998,998464,991743,995716,994964,993412,991844,996232,995581,990538,999876,997010,992604,990063,991650,990366,996001,998593,999791,992944,992337,995107,992558,998495,990783,991052\n", + "Ground Truth: 996052,996219,998383,990448,999998,998464,991743,995716,991999,994964,993412,991844,996232,995581,990538,999876,997010,992604,990063,991650,990366,996001,998593,999791,995596,992944,992337,995107,992558,990091\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 710\n", + "Returned IDs: 994922,991840,997591,990958,999261,992169,996001,998593,997822,990076,996351,992994,999729,990648,990399,996867,999676,991087,990064,993143,992692,996854,997320,992944,990866,990455,994380,992622,996438,995401\n", + "Ground Truth: 994922,991840,997591,990958,999261,992169,996001,998593,997822,990076,996351,992994,999729,990648,990399,996867,999676,991087,990064,993143,992692,996854,997320,992944,990866,990455,997620,994380,992622,996438\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 712\n", + "Returned IDs: 991720,990545,996296,998170,993143,999437,999764,991999,993253,998562,994251,990432,999077,999851,994964,995588,995515,995927,997876,998294,997908,993484,999343,997320,998550,992384,993885,990504,993238,994100\n", + "Ground Truth: 991720,990545,996296,998170,993143,999437,999764,991999,993253,991715,998562,994251,992979,990432,999077,999851,994964,995588,995515,995927,997876,998294,997908,993484,999343,997320,998550,992384,995748,993885\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 713\n", + "Returned IDs: 999384,994972,997811,990096,995536,998697,991179,997320,999898,999647,996001,996062,999144,997913,991154,995701,996219,995130,993504,998401,993809,997775,993906,996370,992047,995596,993561,991459,995560,995401\n", + "Ground Truth: 999384,994972,997811,990096,995536,998697,991179,997320,999898,999647,996001,996062,999144,997913,991154,995701,996219,995130,993504,998401,993809,992692,997775,993906,996370,992047,995596,993561,991459,995560\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 714\n", + "Returned IDs: 992669,990063,997526,998858,992604,999961,997110,992439,993115,991309,998707,994724,994774,995982,993122,999478,990111,998383,990657,994569,999231,997205,991987,994454,997107,998806,995455,992051,991052,998297\n", + "Ground Truth: 992669,990063,997526,998858,992604,999961,997110,992439,993115,995949,991309,998707,994724,994774,995982,993122,999478,990111,998383,990657,994569,999231,997205,991987,994454,997107,993976,998806,995455,992051\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 718\n", + "Returned IDs: 996536,993588,994897,994830,993342,997952,992664,998716,993335,998092,995097,998623,998903,998440,997296,998256,997073,998030,990334,996183,992427,998097,994513,993866,990392,999963,992458,994682,992174,993715\n", + "Ground Truth: 996536,993588,994897,994830,993342,997952,992664,998716,997301,993335,998092,995097,998623,998903,998440,997296,998256,997073,998030,990334,996183,992427,998097,994513,993866,990392,999963,992458,994682,992174\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 719\n", + "Returned IDs: 992330,997371,997913,995985,993072,990397,990469,990560,993048,998500,991468,990092,992384,992443,997889,996193,990595,998921,995260,996840,994360,992546,994281,999122,995365,997320,999676,997984,997305,997831\n", + "Ground Truth: 992330,997371,997913,995985,993072,990397,990469,990560,993048,998500,991468,990092,992384,992443,997889,996193,990595,998921,995260,996840,994360,992546,994281,999613,991631,999122,995365,997320,999676,997984\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 720\n", + "Returned IDs: 998555,998169,999781,990271,990096,997529,998122,999343,995638,992551,993800,997732,990091,994145,992120,995535,999103,993637,996219,990089,999647,999319,990690,997918,994532,997920,998314,990918,996328,990072\n", + "Ground Truth: 998555,998169,999781,990271,990096,997529,998122,999343,995638,992551,993800,997732,990091,994145,992120,995535,999103,993637,996219,995216,990089,999647,999319,990690,997918,994532,998934,997920,998314,990918\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 724\n", + "Returned IDs: 993775,998765,999791,995477,991921,990089,994742,997529,999058,996633,997401,995057,997698,996328,996302,999788,997309,999875,991588,990179,999192,999261,995438,994806,993053,993265,995289,995551,995600,999886\n", + "Ground Truth: 993775,998765,999791,995477,991921,990089,994742,997529,999058,996633,997401,995057,997698,996328,996302,999788,997309,999875,991588,990179,999192,999261,995438,994806,993053,993265,995289,995551,999138,995600\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 726\n", + "Returned IDs: 993035,995791,995004,995365,993570,993633,995588,999039,999592,993282,998837,990043,999620,993028,991999,997496,995155,994545,995657,994666,991407,991593,993675,994543,994922,993708,993946,991196,990155,996402\n", + "Ground Truth: 993035,995791,995004,995365,993570,993633,995588,999039,999592,993282,998837,990043,999620,993028,991999,997496,995155,994545,995657,994666,991407,991593,993505,993675,994543,994922,993708,993946,991196,990155\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 727\n", + "Returned IDs: 995865,994611,996143,993745,996318,993154,991872,994214,997320,990210,997178,993579,994812,992909,996805,997022,993885,999351,998550,998287,997305,991498,990772,998344,994964,999374,994699,994249,996755,992978\n", + "Ground Truth: 993990,995865,990612,994670,994611,996143,993745,996318,993154,991872,994214,997320,990210,997178,993579,994812,992909,996805,997022,993885,999351,998550,998287,997305,991498,990772,998344,994964,999374,994699\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 728\n", + "Returned IDs: 995932,993504,999516,990196,993071,993906,991440,997758,991648,997555,991206,994081,997135,995622,993515,990682,992082,993821,990350,997918,992571,993783,997790,992747,993260,997002,992351,991435,997158,995375\n", + "Ground Truth: 995932,993504,999516,990196,993071,993906,991440,997758,991648,997555,991206,994081,997135,995622,993515,990682,992082,993821,990350,997918,992571,993783,997790,992747,993696,993260,997002,994146,992351,991435\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 730\n", + "Returned IDs: 997025,995521,995907,998703,999653,995303,991161,992448,997251,995891,995537,997909,993275,993313,993342,994380,990593,995778,990725,994125,991844,997723,991338,997675,993561,996042,993127,991459,991447,991794\n", + "Ground Truth: 997025,995521,995907,998703,999653,995303,991161,992448,997251,995891,995537,997909,993275,993313,993342,994380,990593,995778,990725,994125,991844,997723,991338,997675,993561,996042,993127,998855,991459,991447\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 731\n", + "Returned IDs: 999374,999710,997320,997913,999701,996504,992748,991860,999331,991801,998294,998495,998540,993484,995942,991743,997301,996178,992789,992147,995886,994964,993809,995961,994850,991498,998246,990958,990957,998550\n", + "Ground Truth: 999374,999710,997320,997913,999701,991052,996504,992748,991860,999331,991801,998294,998495,998540,993484,995942,991743,995632,997301,996178,992789,992147,995886,994964,993809,995961,994850,991498,998246,990958\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 733\n", + "Returned IDs: 992037,998022,996277,998934,994906,995401,996181,990400,996001,998720,991459,994444,999020,992685,999026,999543,997204,994105,995091,993652,994589,994424,992648,996379,992576,999759,998990,990453,991392,991542\n", + "Ground Truth: 992037,998022,996277,998934,994906,995401,993880,996181,990400,996001,998720,998080,991459,999259,994444,999020,992685,999026,998221,999543,997204,994105,995091,996408,993652,995780,994589,994424,992648,996379\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 734\n", + "Returned IDs: 992712,995855,995356,995563,998622,992163,996105,994167,990472,994955,993283,999216,997368,993300,998513,991232,996144,992484,999041,993174,999219,998697,997977,990688,999750,995672,998934,997417,995203,993821\n", + "Ground Truth: 992712,995855,995356,995563,998622,992163,996105,994167,990472,994955,993283,999216,997368,993300,998513,991232,996144,992484,999041,993174,999219,998697,997977,990688,999750,995672,998934,997417,995284,995203\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 735\n", + "Returned IDs: 996196,997775,998950,995873,994163,996140,994661,995596,997320,995560,990538,990532,998417,996178,991157,995983,990930,990096,992124,992277,990901,992558,991923,997913,997227,995552,997379,999564,996380,996721\n", + "Ground Truth: 996196,997775,998950,995873,994163,996140,994661,995596,997320,995560,990538,992521,990532,996178,998417,991157,995983,990930,990096,992124,992277,990901,992558,991923,997913,997227,995552,997379,995974,999587\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 739\n", + "Returned IDs: 992932,990535,997304,998246,995981,998119,999761,992296,991900,998353,992815,997320,995473,991045,995132,991534,992238,995792,990588,993392,993154,990949,997428,998225,997353,992074,998950,990057,992357,999357\n", + "Ground Truth: 992932,990535,997304,996732,998246,995981,998119,999761,992296,991900,998353,992815,997320,995473,991045,995132,991534,992238,995792,990588,993392,993154,996950,992927,990949,997428,998225,998361,997353,992074\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 740\n", + "Returned IDs: 992862,998156,993857,993256,994564,991428,997173,990328,997374,994742,996328,995819,997452,997309,999843,997918,995633,995295,998708,996752,997852,997914,990814,998073,992473,991189,996411,997409,995600,997208\n", + "Ground Truth: 992862,998156,993857,993256,994564,994940,991428,996232,997173,990328,993440,997374,992979,994742,996328,995819,997452,997309,999843,997918,995633,995295,994850,998708,996752,997852,997914,990814,999453,998073\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 742\n", + "Returned IDs: 995231,997369,995168,994105,997666,992275,995385,990096,999176,996702,994779,995257,992987,993866,997965,990429,990658,999576,992945,997847,998536,995393,997086,998177,992664,999129,996050,994834,990670,991654\n", + "Ground Truth: 995231,997369,995168,994105,997666,992275,995385,990096,999176,996702,994779,995257,992987,993866,997965,990429,990658,999576,992945,991313,997847,998536,995393,997086,998177,992664,999129,996050,994834,990670\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 743\n", + "Returned IDs: 995121,991630,997191,991140,998297,992977,996734,993738,995401,990005,999530,992448,994319,998045,997868,998205,995864,991969,995395,994019,997957,991958,993381,998346,996277,999820,992521,999731,995940,997723\n", + "Ground Truth: 995121,991630,997191,991140,998297,992977,996734,995268,993738,995401,990005,999530,992448,994319,998045,997868,998205,995864,991969,995395,994019,997957,991958,993381,998346,996277,997928,999820,992521,999731\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 744\n", + "Returned IDs: 997006,993885,998294,997876,997320,990898,999374,996296,993364,990917,991059,991900,995266,992896,993048,993853,994964,994776,990760,994476,999328,996218,993484,990863,999647,999823,998246,992931,990560,990146\n", + "Ground Truth: 997006,993885,998294,997876,997320,990898,999374,996296,993364,990917,991059,991900,995266,992896,993048,993853,994964,994776,990760,994476,999328,996218,993484,999247,990863,998899,999647,999823,998246,990296\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 745\n", + "Returned IDs: 995793,991022,992518,995833,991782,990862,998822,990535,996713,995771,998951,997976,994266,997320,993735,994100,995336,998263,996714,998721,996961,992646,992183,992384,999089,995279,997571,991889,993711,992609\n", + "Ground Truth: 995793,991022,992518,995833,991782,990862,998822,990535,996713,995771,998951,997976,994266,997320,993735,994100,995336,998263,996714,998721,996961,992646,992183,992384,995438,999089,995279,997571,991889,999533\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 748\n", + "Returned IDs: 999453,991930,990772,996001,991248,991440,991609,993280,990314,995250,994484,992395,995968,997332,991787,992884,990399,991581,995372,990532,993032,994449,999575,991158,994478,991542,991994,992182,992469,998567\n", + "Ground Truth: 999453,991930,990772,996001,991248,991440,991609,993280,990314,995250,994484,992395,995968,997332,991787,998127,992884,990399,991581,995372,990532,993032,994449,999575,991158,994478,991542,991994,992182,992469\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 749\n", + "Returned IDs: 996111,997032,995482,994925,996232,998221,996328,994742,994798,999647,990096,993053,990435,994998,992450,996196,996178,998649,990887,998495,996001,991743,997379,999898,998067,996884,993783,991769,997126,998710\n", + "Ground Truth: 996111,997032,995482,994925,996232,998221,996328,994742,994798,999647,990096,993053,990435,994998,992450,996196,996178,998649,990887,998495,999586,996001,991743,997379,999898,998067,996884,993783,991769,997126\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 751\n", + "Returned IDs: 997015,991568,993168,993050,998992,990535,992744,995336,993562,996714,998951,995252,992646,999531,997082,994964,999026,990936,995824,992911,995536,992331,993648,999144,994197,999437,991444,997709,990925,993484\n", + "Ground Truth: 997015,991568,993168,993050,998992,990535,992744,995336,993562,996714,998951,995252,992646,999531,997082,994964,999026,990936,995824,992911,995536,992331,993648,999144,994197,998495,999437,991444,997709,990925\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 754\n", + "Returned IDs: 998950,996805,999647,992944,993696,992041,998180,997906,999210,999426,997470,991650,997371,995633,997403,997913,993504,994742,997291,995401,999676,998495,999639,993747,993529,998790,992520,996196,996219,990096\n", + "Ground Truth: 998950,996805,999647,992944,993696,992041,998180,997906,999210,999426,994089,997470,991650,997371,995633,997403,997913,993504,994742,997291,995401,999676,998495,998855,999639,993747,993529,998790,992520,996196\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 758\n", + "Returned IDs: 996001,991119,995455,995755,999009,998861,991881,997337,990505,998122,990096,996809,996406,994879,995701,990866,995425,999480,996408,998514,999388,997586,995401,990091,995132,991459,999289,991154,995536,997305\n", + "Ground Truth: 996001,991119,995455,995755,999009,998861,991881,999916,997337,990862,990505,998122,990096,996809,996406,994879,995701,990866,995425,999480,996408,998514,999388,997586,995401,990091,995132,991459,999289,991154\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 760\n", + "Returned IDs: 990061,990273,995309,993583,990206,994254,993500,996380,991064,997547,992052,990042,994679,993077,995459,999022,993811,991977,991432,997866,995728,993742,990776,996328,992558,990894,993879,994280,997494,991447\n", + "Ground Truth: 990061,990273,995309,993583,990206,994254,993500,996380,991064,997547,992052,990042,994679,993077,995459,999022,993811,991977,991432,997866,995728,993742,990776,996328,995600,992558,990894,993879,994280,997494\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 761\n", + "Returned IDs: 990127,995198,990331,993209,995873,995778,992862,994409,995268,995401,990887,999179,993313,994488,992260,990975,996328,994954,992293,990930,992052,994835,992636,992595,995122,999581,991958,995864,998623,993561\n", + "Ground Truth: 990127,995198,990331,993209,995873,995778,992862,994409,995268,995401,990887,995536,999179,993313,994488,992260,990975,997184,996328,994954,992293,990930,992052,994835,992636,993809,992595,995122,999581,991958\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 762\n", + "Returned IDs: 997550,995596,994618,998921,999338,999426,996075,997661,997113,991992,995033,992041,995515,997448,996318,993785,992944,997584,996516,994971,997984,991984,993125,994233,996741,996631,996387,999698,999588,995398\n", + "Ground Truth: 997550,995596,994618,998921,999338,999426,996075,997661,997113,991992,995033,992041,995515,997448,996318,993785,992944,997584,996516,994971,997984,991984,993125,995401,994233,996741,996631,996387,999698,999588\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 763\n", + "Returned IDs: 995317,993558,993678,996194,992992,997854,993256,994541,993044,990987,997251,990069,999287,997429,995159,996438,991863,994064,999386,997806,994428,993621,993014,992313,996232,991285,997770,994639,999179,990776\n", + "Ground Truth: 995317,993558,993678,996194,992992,997854,993256,994541,991075,993044,990987,997251,990069,995401,999287,997429,995159,996438,991863,994064,999386,997806,994428,993621,993014,992313,996232,991285,993454,997770\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 764\n", + "Returned IDs: 994955,998933,993194,999798,991685,993665,999052,998536,997944,992825,994280,995168,997155,992850,995932,992561,993715,995589,995366,997453,992823,998778,995945,993931,993037,994167,990387,999840,990364,999543\n", + "Ground Truth: 994955,998933,993194,999798,991685,993665,999052,998536,997944,992825,994280,995168,997155,992850,995932,992561,993715,995589,995366,997453,992823,998778,995945,993931,997926,993037,994167,990387,999840,990364\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 765\n", + "Returned IDs: 994557,991058,997638,997807,993929,994119,995332,990879,990345,992704,997822,997309,990987,993014,990678,999210,997693,996071,991107,999643,990226,992147,994280,997945,994336,997126,992962,994541,991177,991487\n", + "Ground Truth: 994557,991058,997638,997807,993929,994119,995332,990879,990345,992704,997822,997309,990987,993014,990678,999210,997693,996071,991107,995620,999643,990226,992147,994280,997945,994336,997126,992962,994541,991177\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 768\n", + "Returned IDs: 999117,994643,993122,993608,995625,992677,999155,993232,990361,991987,996395,998078,999231,991630,997149,995807,999834,991052,996809,993795,998257,997110,994092,992206,997752,997568,995373,998536,995105,992288\n", + "Ground Truth: 999117,994643,993122,993608,995625,992677,999155,993232,990361,996395,991987,998078,999231,991630,997149,995807,995401,999834,991052,996809,993795,998257,997110,994092,992206,997752,997568,995373,998536,995105\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 770\n", + "Returned IDs: 991889,995949,998294,995260,996053,998692,999374,991705,991406,996554,993837,992309,995928,993143,997627,996334,992979,995693,994160,994919,990432,996854,990490,995354,996202,992169,991445,997429,997083,994850\n", + "Ground Truth: 991889,995949,998294,995260,996053,998692,999374,991705,991406,996554,993837,992309,990026,995928,993143,997627,996334,992979,995693,994160,994919,990432,990545,990103,993412,996854,990490,995354,996202,992169\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 771\n", + "Returned IDs: 993416,992684,994855,991958,998158,998382,990794,993561,994987,996040,996062,990254,993673,991052,996371,999236,998593,992017,995923,994946,995547,993738,998323,994419,997453,999868,992260,998297,995373,998891\n", + "Ground Truth: 993416,992684,994855,991958,998158,998382,990794,993561,994987,996040,996062,990254,993673,991052,996371,999236,998593,992017,995923,994946,995547,993738,994076,996834,998323,994419,997453,999868,992260,998297\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 772\n", + "Returned IDs: 997002,998902,994906,992790,994334,997301,996658,999442,998951,995560,991621,994661,992239,996001,995401,992165,995374,995836,992759,996220,994362,993820,995357,998045,997918,990140,999587,994088,992641,995536\n", + "Ground Truth: 997002,998902,994906,992790,994334,995894,997301,996658,999442,998951,995560,991621,994661,992239,996001,995401,992165,995374,995836,992759,996220,994362,993820,995357,998045,997918,991989,990140,999587,994088\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 773\n", + "Returned IDs: 992041,996075,990022,990366,993785,992167,990697,991372,995515,994440,999426,991984,996001,991109,999473,997836,998654,997661,993928,995596,996227,992872,998498,996219,993484,992733,992927,999053,992617,999806\n", + "Ground Truth: 992041,996075,990022,990366,993785,992167,990697,991372,995515,994440,999426,991984,996001,991109,999473,997836,994776,998654,997661,993928,995596,996227,992872,998498,996219,993484,992733,992927,999053,990934\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 774\n", + "Returned IDs: 992383,990383,991550,990884,999387,996445,999451,990376,991923,994964,996894,995983,992824,996196,998600,998401,990538,993781,995427,995372,999639,993706,997903,994979,994852,995459,991817,993492,992190,996085\n", + "Ground Truth: 992383,990383,991550,990884,999387,996445,999451,990376,991923,994964,996894,995983,992824,996196,998600,998401,990538,993781,995427,995372,999639,993706,997903,994979,994852,995459,991817,993492,992190,998545\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 775\n", + "Returned IDs: 990832,995100,990194,993435,997725,997331,996475,996809,996219,996830,992892,999435,991187,995880,991459,995131,999331,999384,996067,996216,993342,998122,995130,999430,999388,999647,993192,996913,992100,997047\n", + "Ground Truth: 990832,995100,990194,993435,997725,997331,996475,996809,996219,996830,992892,999435,991187,995880,991459,995131,999331,999384,996067,996216,993342,998084,998122,995130,999430,999388,999647,993192,996913,998925\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 776\n", + "Returned IDs: 999159,991582,995632,997284,997354,994225,991348,996296,997320,992691,993018,991763,991992,993454,999777,995194,991879,996199,996387,991131,992896,990456,991662,996075,995108,996626,999658,994415,990375,998950\n", + "Ground Truth: 999159,991582,995632,997284,997354,994225,991348,995878,992022,996296,997320,992691,993018,991763,991992,993454,999777,995194,991879,996199,996387,991131,992896,990456,991662,996075,995108,996626,999658,994415\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 777\n", + "Returned IDs: 999663,995536,992225,998514,998861,999009,999711,990096,998720,991459,992293,996809,996001,992553,993139,994434,995401,995701,992862,998200,996406,999216,998082,998568,997586,993948,995961,997506,991893,998495\n", + "Ground Truth: 999663,995536,992225,998514,998861,999009,999711,990096,998720,991459,992293,996809,996001,992553,993139,994434,995401,993821,995701,992862,998200,996406,999216,998082,998568,997586,993948,995961,997506,991893\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 778\n", + "Returned IDs: 994401,993039,997077,998972,995065,992985,993931,995159,990184,998851,998566,996277,993799,994835,992576,997181,995760,995268,993256,996505,999759,990291,990942,995790,995815,993879,997126,990956,990391,995685\n", + "Ground Truth: 994401,993039,997077,998972,995065,992985,993931,995159,990184,998851,998566,996277,993799,994835,992576,997181,995648,995760,996223,995268,993256,996505,999759,990291,990942,995790,995815,993879,997126,990956\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 779\n", + "Returned IDs: 998496,995840,994879,996075,991166,992442,991984,993834,997258,992041,998708,993902,998200,997586,997305,993766,992330,996230,993809,996933,991476,995701,994558,996001,999701,996657,993696,999041,993283,991823\n", + "Ground Truth: 998496,995840,994879,996075,991166,992442,991984,993834,997258,992041,998708,993902,998200,997586,997305,997770,993766,992330,996230,993174,993809,996933,991476,995701,994558,996001,999701,996657,993696,999041\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 781\n", + "Returned IDs: 992226,994273,990815,994098,999430,998671,995704,999144,990711,993100,999435,996816,994558,997955,998416,993725,996222,994801,991672,999225,997331,996328,991938,999532,991378,998729,992951,993501,996856,993127\n", + "Ground Truth: 992226,994273,990815,994098,999430,998671,995704,999144,990711,993100,999435,996816,994558,991840,997955,998416,993725,993161,996222,994801,991672,999225,997331,996328,999171,998142,991938,999532,991378,998729\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 782\n", + "Returned IDs: 999530,998070,992448,995685,991301,994409,996323,991699,994002,996987,998714,998545,997046,997675,995507,992605,999578,999043,996322,993412,991671,996607,991247,999884,991583,993400,996042,995401,991052,995139\n", + "Ground Truth: 999530,998070,992448,995685,991301,994409,996323,991699,994002,996987,998714,998545,997046,997675,995507,992605,999578,999043,996322,993412,991671,996607,998697,991247,990337,999884,991583,993400,996042,995401\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 783\n", + "Returned IDs: 990089,995454,995720,997292,995659,999791,995045,993637,999858,992334,991537,999444,990918,996147,998076,998097,992636,998302,999302,995306,992415,996516,997490,994373,995133,991144,996001,995755,997349,997933\n", + "Ground Truth: 990089,995454,995720,997292,995659,999791,995045,993637,999858,992334,991537,999444,990918,996147,998076,998097,992636,998302,999302,995306,992415,996516,997490,994373,995133,991144,996001,995755,993663,997349\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 787\n", + "Returned IDs: 994670,991511,996219,992630,995274,990679,994916,992931,996714,990271,991085,992277,993154,994778,999243,997976,990910,995161,999340,997404,992748,999596,991456,995981,997018,995502,995901,990448,994894,990612\n", + "Ground Truth: 994670,991511,996219,993517,992630,995274,990679,994916,990909,992931,992124,999723,996714,990271,991085,992277,993154,994778,999243,997976,990910,995161,999340,997404,992748,999596,991456,995981,993481,997018\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 789\n", + "Returned IDs: 999240,993473,997195,993986,992558,999860,993811,991550,993077,995552,991025,992121,998929,990078,995972,999479,999715,996918,991817,996750,993158,993400,995967,996196,990249,994168,997054,991426,994679,992375\n", + "Ground Truth: 999240,993473,997195,993986,992558,999860,993811,991595,991550,993077,995552,991025,992121,998929,990078,995972,999479,999715,996918,991817,996750,993158,993400,995967,996196,990249,999631,994168,997054,991426\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 791\n", + "Returned IDs: 996633,992598,994639,996155,999059,999192,994541,998761,995551,990392,998348,993825,997026,994169,996960,992992,996194,997838,994356,993793,993448,993314,994742,993691,992832,992540,994327,993725,992595,994861\n", + "Ground Truth: 996633,992598,994639,996155,999059,999192,994541,998761,995551,990392,998348,993825,997026,994169,996960,992992,996194,997838,994356,993793,993448,993314,995401,994742,993691,992832,992540,994327,993725,992595\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 792\n", + "Returned IDs: 998902,998236,991131,998967,994604,991999,991359,990444,998001,993399,995515,991499,997918,998492,995357,998734,990176,999698,992839,991157,997831,992944,994329,995865,995988,995364,994789,994049,991609,991407\n", + "Ground Truth: 998902,998236,991131,998967,996030,994604,991999,991359,990444,998001,993399,995515,991499,997918,998492,995357,998734,990176,999698,992839,994449,991157,997831,992944,994329,990219,995865,995988,995364,994789\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 794\n", + "Returned IDs: 998679,990930,991585,993293,993813,998779,995633,995289,991141,994742,994730,994140,994436,993997,999402,998980,990435,993856,994409,991664,993831,995702,995648,997723,995216,997554,999805,995052,996277,999163\n", + "Ground Truth: 998679,990930,991585,993293,993813,998779,995633,995289,991141,994742,994730,994140,994436,993997,999402,998980,990435,993856,994409,991664,993831,995702,995648,997723,990361,995216,997554,999805,995052,996277\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 796\n", + "Returned IDs: 995252,992932,997320,997483,992096,990231,990541,997771,992228,995336,992277,992692,991900,992330,998090,992069,992969,998951,990885,998246,998365,998653,996296,993696,991131,990172,992978,995802,996196,992942\n", + "Ground Truth: 995252,992932,997320,997483,992096,990231,990541,997771,992228,995336,992277,992692,991900,992330,998090,992069,992969,998951,990885,998246,998365,998653,996296,993696,991131,990172,992978,995802,996196,996246\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 800\n", + "Returned IDs: 993708,995357,992134,993633,992692,997918,998967,993145,998996,991015,996592,991131,998011,999077,995336,995693,993492,993781,992946,990957,993512,998245,999144,996075,992277,993683,997470,999784,995210,996840\n", + "Ground Truth: 993708,995357,992134,993633,992692,997918,998967,993145,998996,991015,996592,993696,991131,998011,999077,995336,995693,993492,993781,992946,990957,993512,998245,999144,996075,992277,993683,997470,999784,995210\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 801\n", + "Returned IDs: 995203,993053,990918,993561,996645,990096,998416,995283,995873,990271,991464,994558,997370,994145,999653,994742,995310,997906,992334,998382,991168,994853,991565,999806,992408,995756,997920,990827,997305,999384\n", + "Ground Truth: 995203,993053,990918,993561,996645,990096,998416,995283,995873,990271,991464,994558,997370,994145,999653,994742,996147,993154,995310,997906,992334,998382,991168,994853,991565,999806,992408,995756,997920,990827\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 804\n", + "Returned IDs: 997309,997914,990705,992169,991720,998123,990776,999273,993677,994119,998495,996328,994765,994280,992364,996001,999179,995372,998180,990328,993993,991882,992944,991334,994310,994835,996809,994587,994531,997480\n", + "Ground Truth: 997309,997914,990705,992169,991720,998123,990776,999273,993677,994119,998495,996328,994765,994280,992364,996001,999179,995372,998180,990328,993993,991882,998466,994817,992944,991334,994310,994835,996809,994587\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 805\n", + "Returned IDs: 991642,990011,993906,995295,991435,993725,994879,995852,997002,994689,991315,998848,997790,992161,996811,990954,994206,994964,995823,992226,998359,996328,995586,998105,990494,997918,999500,994307,994012,994742\n", + "Ground Truth: 991642,990011,993906,995295,991435,993725,994879,995852,997002,994689,991315,998848,997790,992161,996811,990954,994206,994964,995823,992226,998359,991989,996328,995586,998105,990494,997918,999500,994307,994012\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 806\n", + "Returned IDs: 998343,990976,992056,995185,990255,990449,993875,997088,991256,995695,992293,998831,999334,995672,993545,994510,990399,998834,999498,998589,999663,998959,991836,999087,999502,997790,994815,995074,991038,992862\n", + "Ground Truth: 998343,990976,992056,995185,990255,990449,993875,997088,991256,995695,992293,998831,999334,995672,993545,994510,998453,990399,998834,999498,998589,999663,998959,991836,999087,999502,997790,996082,995074,994815\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 809\n", + "Returned IDs: 994964,996219,991582,993168,993289,999827,995927,997370,993143,997920,997822,999104,991052,997732,993412,997918,997586,994922,990866,994280,995210,990146,999343,992169,992147,997028,996001,995536,990382,990096\n", + "Ground Truth: 994964,996219,991582,993168,993289,999827,995927,997370,993143,997920,997822,999104,991052,997732,993412,997918,997586,994922,990866,994280,995210,990862,990146,999343,992169,992147,997028,999289,996001,995536\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 810\n", + "Returned IDs: 998314,999831,995385,993086,990096,998536,996918,992407,998177,998405,999287,993965,999079,994661,999194,992747,991052,996733,991093,998968,991724,991243,993239,999320,991571,993609,993715,997897,999384,990583\n", + "Ground Truth: 998314,999831,995385,993086,990096,998536,996918,992407,998177,998405,999287,993965,999079,994661,999194,992747,991052,996733,991093,998968,991724,991243,993239,999320,991571,993609,997775,993715,997897,999384\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 816\n", + "Returned IDs: 993392,992027,995771,993620,997984,996504,998921,996218,993518,997704,994281,994939,991782,998692,993457,995985,991873,994922,998822,992709,991889,999374,990595,997192,997006,992546,994775,992147,993161,994808\n", + "Ground Truth: 993392,992027,995771,993620,997984,996504,998921,996218,993518,997704,994281,994939,991782,998692,993457,995985,991873,994922,998822,992709,991889,992932,999374,990595,997192,997006,992546,994775,992147,993161\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 817\n", + "Returned IDs: 994304,997868,997586,992521,995814,994636,994954,996918,992293,997723,993438,997466,998260,995701,990866,999731,991790,998255,993831,997305,998346,995873,990097,991967,990078,996360,996187,994419,998934,997216\n", + "Ground Truth: 994304,997868,997586,992521,995814,994636,994954,996918,992293,997723,993438,997466,998260,995701,990866,999731,991790,996386,998255,993831,997305,998346,995873,990097,991967,990078,996360,996187,994419,998564\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 818\n", + "Returned IDs: 992909,994214,997320,998808,998090,992978,994290,991479,999621,990588,994427,991883,995372,995802,998364,992692,995826,995961,995228,994775,993662,992069,991503,991873,992969,994280,999761,997913,990910,993512\n", + "Ground Truth: 992909,994214,995090,997320,998808,998090,992978,994290,991479,999621,990588,994427,991883,995372,995802,998364,992692,995826,995961,995228,994775,993662,992069,991503,991873,992969,994280,999761,997913,998821\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 820\n", + "Returned IDs: 997331,998147,994113,995382,992433,993281,998770,999643,991011,996494,996722,993778,993939,996067,996866,995620,992840,991148,998484,990527,996219,991179,994692,995401,997837,996080,991377,994589,998855,994557\n", + "Ground Truth: 997331,998147,994113,995382,992433,993281,998770,996702,999643,991011,996494,996722,993778,993939,996067,996866,995620,992840,991148,998484,990527,996219,991179,994692,994984,995401,992987,997837,996080,991377\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 821\n", + "Returned IDs: 999837,992752,991633,998594,991242,991552,991605,998279,997673,993611,998389,997151,995167,996082,995421,998108,992864,990950,993715,999139,997050,995705,995626,990017,994734,996216,995232,994196,993347,995103\n", + "Ground Truth: 999837,992752,991633,998594,991242,991552,990133,991605,998279,997673,993611,998389,997151,995167,996082,995421,998108,992864,990950,993715,998716,999139,997050,995705,995626,990017,996216,994734,995232,994196\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 822\n", + "Returned IDs: 991765,993319,999798,996914,997340,996535,995768,998525,990408,997714,998168,994356,999601,990183,991779,997925,995222,992250,995501,992517,994841,992131,993993,996918,995755,990711,999886,999052,999496,998224\n", + "Ground Truth: 991765,993319,999798,993396,996914,997340,996535,995768,998525,990408,997714,998168,994356,999601,990183,991779,997925,995222,992250,993817,995501,992517,994841,992131,993993,996918,995755,990711,999886,999052\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 823\n", + "Returned IDs: 991356,992558,991977,997547,993255,990602,990226,994560,999908,993583,990894,999305,993077,997859,992113,993742,998151,992827,993839,994472,998892,995374,996318,995886,995552,995542,999026,990279,997918,998495\n", + "Ground Truth: 991356,992558,991977,997547,993255,990602,990226,994560,999908,993583,997806,990894,999305,993077,997859,992113,993742,998151,992827,993839,994472,990995,998892,995374,996318,995886,995552,995542,999026,990279\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 827\n", + "Returned IDs: 994051,993396,996899,999601,991399,994801,998766,997714,991052,996042,995963,996918,995342,997723,990189,994280,998123,993578,993805,993727,992933,996315,999820,996127,998267,993014,998778,993165,998397,990103\n", + "Ground Truth: 994051,993396,996899,999601,991399,994801,998766,997714,991052,996042,995963,996918,995342,997723,990189,994280,992569,998123,999596,993578,993805,993727,992933,996315,999820,996127,997803,995592,998267,993014\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 828\n", + "Returned IDs: 997116,996495,994142,999273,997506,998456,993139,996328,990712,995571,996408,993993,994680,990408,994196,998168,999985,991667,990091,999139,996431,991633,994012,993560,990705,994296,991552,990479,990449,998594\n", + "Ground Truth: 997116,996495,994142,999273,997506,998456,993139,996328,990712,995571,996408,993993,994680,990408,994196,990293,998168,999985,991667,990091,999139,996431,991633,994012,993560,990705,994296,991552,990479,990449\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 829\n", + "Returned IDs: 990310,993143,998033,991157,996840,999257,991392,995584,999122,999144,991609,991015,991840,995515,990580,999676,996829,997593,992950,992734,997379,993355,997320,996007,999546,997037,997918,998607,996282,992612\n", + "Ground Truth: 990310,993143,998033,991157,996840,999257,991392,995584,999122,999144,991609,991015,991840,995515,990580,999676,996829,997593,992950,992734,997379,998909,993355,997320,996007,994757,999546,992979,997037,997918\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 830\n", + "Returned IDs: 993324,995375,997184,990930,995130,990646,994092,991712,992933,993533,998272,998349,997675,996277,996328,993561,996591,991052,994743,993293,994409,992790,995080,993931,993347,999533,997309,993738,995401,995408\n", + "Ground Truth: 993324,995375,997184,990930,995130,990646,994092,991712,992933,993533,998272,998349,997522,996277,997675,996328,993561,996591,991052,994743,993293,996809,994409,992790,995080,993931,993347,999533,997309,993738\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 832\n", + "Returned IDs: 990934,995471,997529,996219,996507,999876,990974,996328,999343,995384,991335,996075,995107,996727,991743,990052,998668,999898,995096,992147,990866,998500,999144,995455,999374,997381,999417,999261,993412,991565\n", + "Ground Truth: 990934,995471,997529,996219,996507,999876,990974,996328,999343,995384,991335,996075,995107,996727,991743,990052,998668,999898,995096,992147,990866,999784,998500,999144,995455,995536,999374,991096,993335,997381\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 833\n", + "Returned IDs: 996339,998410,992582,996310,998011,997645,996220,992989,990284,998521,992558,992134,994819,997942,999820,995811,999400,997745,991389,996903,993296,990054,990921,994804,992428,999144,991680,995968,995886,997209\n", + "Ground Truth: 996339,998410,992582,996310,998011,997645,996220,992989,990284,992738,999240,998521,992558,992134,994819,997942,999820,995811,999400,997745,991389,996903,993296,990054,990921,994804,992428,999144,991680,992638\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 835\n", + "Returned IDs: 997301,997320,995274,998302,990231,995961,997913,999338,999109,997470,998779,990585,990949,991498,991743,995045,993484,990957,999533,990096,999290,992509,999710,994831,999801,994964,999647,999026,993431,998950\n", + "Ground Truth: 997301,997320,995274,998302,990231,995961,997913,999338,999109,997470,998779,990585,990949,991498,991743,995045,993484,990957,999533,990096,999290,992509,999710,994831,999801,999432,994964,999647,999026,993431\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 839\n", + "Returned IDs: 999216,992068,994988,995560,990786,992423,994638,996262,999806,999639,999711,990167,998383,992617,994852,993504,990769,996809,996053,999210,998292,993617,991157,998708,997864,994964,992282,991995,993355,990675\n", + "Ground Truth: 999216,992068,994988,995560,990786,994638,992423,996262,999806,999639,999711,990957,990167,998383,992617,994852,993504,994598,994660,990769,996809,996053,996857,999210,998292,993617,991157,998708,992979,997864\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 841\n", + "Returned IDs: 995443,990648,994046,998097,998726,998550,995080,998623,992933,993143,996292,999784,992636,999122,991999,998950,990257,992318,990096,995806,993719,999676,991801,992944,999210,993917,999701,998180,993260,997309\n", + "Ground Truth: 995443,990648,994046,998097,998726,998550,995080,998623,992933,993143,996292,999784,992636,999122,991999,998950,990257,993512,992318,990096,995806,993719,999676,994835,991801,992944,999210,993917,999701,998180\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 842\n", + "Returned IDs: 990670,999334,993535,993715,998468,996594,996050,998536,995746,999373,991436,995655,999711,992458,997685,996989,990293,992664,992933,997617,995969,995385,994723,991654,992110,990622,993917,992157,990942,995393\n", + "Ground Truth: 990670,999334,993535,993715,998468,996594,996050,998536,995746,999373,991436,995655,999711,992458,997685,996989,990293,992664,992933,997617,995969,995385,994723,991654,992110,990622,993917,992157,990942,997470\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 844\n", + "Returned IDs: 991470,996760,991257,997506,997019,998536,993381,995168,995257,998281,992110,997265,993076,996594,996395,995998,992721,990976,992793,998468,994374,993194,998331,998214,990292,991945,993715,999373,996245,993866\n", + "Ground Truth: 991470,996760,991257,997506,997019,998536,993381,995168,995257,998281,992110,997265,993076,996594,996395,995998,992721,990976,992793,998468,994374,993194,998331,998214,990292,991945,993715,999373,996245,993936\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 846\n", + "Returned IDs: 990624,997251,992483,992992,999529,996042,995464,992844,999845,996687,997344,999244,998838,991334,997234,991161,993857,995550,997918,994928,997601,992487,990686,993948,993313,990487,995763,995537,992639,993335\n", + "Ground Truth: 990624,997251,992483,992992,999529,996042,995464,992844,999845,996687,997344,999244,998838,991334,997234,991161,993857,995550,997918,994928,997601,992487,990686,993948,993313,990487,995763,995537,996220,992639\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 847\n", + "Returned IDs: 999682,991374,992740,993530,992493,997232,990733,995807,995105,997574,992677,997675,997515,993263,996579,997634,995521,994823,995625,998806,999961,998080,996695,997149,990243,994774,991061,991052,993149,994677\n", + "Ground Truth: 999682,991374,992740,993530,992493,997232,990733,995807,995105,997574,992677,997675,997515,993263,996579,997634,995521,994823,995625,998806,999961,998080,996695,997149,990243,999327,996094,994774,991061,991052\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 848\n", + "Returned IDs: 998960,992511,999374,991444,996959,998211,990469,992147,998644,999263,992416,996218,997704,991362,998500,993250,992854,997192,998921,992546,994850,997543,997301,993457,999968,993364,994673,994939,990515,990215\n", + "Ground Truth: 998960,992511,999374,991444,996959,998211,990469,992147,998644,999263,992416,996218,997704,991362,998500,993250,992854,997192,998921,992546,994850,990760,997543,993457,997301,999968,993364,994673,994939,990515\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 849\n", + "Returned IDs: 990266,991720,993656,992161,992147,997476,991377,990483,992169,994761,994922,992994,997429,996494,991464,990328,999343,997852,992226,996721,990441,998500,999122,999374,992979,990957,990080,992068,995332,996609\n", + "Ground Truth: 990266,991720,993656,992161,992147,997476,991377,990483,992169,994761,997053,994922,992994,997429,996494,991464,990328,999343,997852,992226,996721,990441,998500,995210,999122,999374,992979,990957,990080,992068\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 854\n", + "Returned IDs: 997383,999915,996001,991175,998097,992395,995873,996685,995476,993053,994742,992792,995073,997320,990930,999099,992568,996809,999560,998779,999399,997814,992318,991007,996043,995216,999453,998725,991245,996328\n", + "Ground Truth: 997383,999915,996001,991175,998097,992395,995873,996685,995476,993053,994742,992792,995073,997320,990930,999099,992568,996809,999560,998779,999399,997814,992318,991007,996043,995216,999453,998725,991245,993342\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 855\n", + "Returned IDs: 994080,994760,992558,991977,992170,994119,992718,990776,990487,994254,996633,996809,995179,992052,998686,998480,995503,994956,995159,998243,995491,990908,994334,998331,990345,998372,996328,996391,993839,998370\n", + "Ground Truth: 994080,994760,992558,991977,992170,994119,992718,990776,990487,994254,996633,996809,995179,992052,998686,998480,995503,994956,995159,998243,995491,990908,994334,998331,990345,998372,999179,996328,996391,993839\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 856\n", + "Returned IDs: 993289,992933,993845,993637,990438,993301,999338,992825,993719,994081,998021,995401,990855,999698,992176,993715,998726,998536,996918,995080,997002,990289,997918,990096,998950,999533,999724,996328,999791,999516\n", + "Ground Truth: 993289,992933,993845,993637,990438,993301,999338,992825,993719,994081,998021,995401,990855,999698,992176,993715,998726,998536,996918,995080,997002,990289,997918,990096,998950,991758,999533,999724,996328,999791\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 857\n", + "Returned IDs: 990275,997439,996053,998713,995260,996554,994618,991018,990231,992363,990650,997371,990366,997451,992228,991479,997432,999243,995755,996219,995210,998744,992931,997913,992453,993387,999938,992087,994266,992942\n", + "Ground Truth: 990275,997439,996053,998713,995260,996554,994618,991018,990231,992363,990650,997371,990366,997451,992228,991479,997432,999243,995755,993048,996219,995210,998744,992931,997913,992453,993387,997508,999938,992087\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 860\n", + "Returned IDs: 990275,996336,993853,991260,992384,992452,999647,995014,997976,994776,991829,992969,998899,994992,994192,990724,990862,993168,998514,990925,996727,997571,996714,990356,990957,992994,999089,995034,992911,995279\n", + "Ground Truth: 990275,996336,993853,991260,992384,992452,999647,995014,997976,994776,991829,992969,998899,994992,994192,990724,991893,990862,990893,993168,998514,990925,997022,996727,997571,992909,996714,990356,990957,992994\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 861\n", + "Returned IDs: 998302,994382,997937,991790,995204,991923,994707,990560,995672,997320,993412,997301,995702,992862,995981,991995,998294,994964,995949,997913,998779,990252,998807,996391,998495,995895,995755,990096,995976,997822\n", + "Ground Truth: 998302,994382,999755,997937,991790,995204,991923,994707,990560,995672,997320,993412,997301,995702,992862,995981,991995,998294,994964,995949,997913,998779,990252,998807,996391,998495,995895,997969,995755,990096\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 862\n", + "Returned IDs: 996219,997913,990843,997301,991085,994266,993662,998495,995942,990958,999533,998380,999542,999374,998540,995927,990776,997420,995802,995949,997320,998921,994964,991650,999331,994850,997855,995260,996318,996811\n", + "Ground Truth: 996219,997913,990843,997301,991085,994266,993049,993662,998495,992946,995942,990958,999533,998380,997741,999542,999374,998540,995927,990776,997420,995802,995949,997320,998921,994964,991650,999331,994850,997855\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 863\n", + "Returned IDs: 999912,992337,998115,994972,990366,997453,999544,999876,999949,996052,990179,995107,998122,990585,995222,990089,992047,999791,992907,995716,993139,999711,990043,995818,998349,996001,999898,997301,997796,997028\n", + "Ground Truth: 999912,992337,998115,994972,990366,997453,999544,999876,999949,996052,990179,995107,998122,990585,995222,990089,992047,999791,996947,992907,995716,993139,999711,990043,995818,998349,996001,999898,997301,997796\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 866\n", + "Returned IDs: 998558,994679,994098,995691,996882,999435,998993,990796,998652,994193,990765,991541,999171,994605,992056,994080,995618,990852,993718,996428,993267,994689,997864,990487,996413,990133,999158,998855,996930,992435\n", + "Ground Truth: 998558,994679,994098,995691,996882,999435,998993,990796,998652,994193,990765,991541,999171,994605,992056,994080,995618,990852,993718,996428,993267,994689,997864,990487,996413,990133,999158,998855,996930,990189\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 867\n", + "Returned IDs: 999805,996867,998162,994409,997844,998943,993767,997868,990361,994730,990631,995702,994060,995766,996786,992582,997031,997723,995052,994954,993209,992063,995355,998181,993649,999459,991603,991508,998962,993856\n", + "Ground Truth: 999805,996867,998162,994409,997844,998943,993767,993466,997868,990361,994730,990631,995702,994060,995766,996633,996786,992582,997031,997723,995052,994954,993209,992063,995355,998181,993649,999459,991603,991508\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 869\n", + "Returned IDs: 996411,995482,997374,993466,995020,995002,990435,999627,998156,990293,999599,993209,999785,994730,992024,998221,991480,994802,999179,995274,995633,998067,991075,995401,990758,998913,995374,995289,994817,993747\n", + "Ground Truth: 996411,995482,997374,993466,995020,995002,990435,999627,998156,990293,999599,994142,993209,999785,994730,991439,992024,998221,991480,994802,998586,999179,995274,995633,998067,991075,995401,990758,998913,995374\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 872\n", + "Returned IDs: 996362,995783,994313,996087,992997,993497,997746,995184,990872,990149,995464,991285,994280,990343,997244,990021,994196,999893,996911,991111,999759,996276,990103,997713,993256,993478,993621,991981,992784,994541\n", + "Ground Truth: 996362,995783,994313,996087,992997,993497,997746,995184,990872,990149,997997,995464,991285,994280,990343,997244,990021,994196,999893,996911,991111,999759,996276,990103,997713,993256,993478,993621,991981,992784\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 873\n", + "Returned IDs: 998382,996371,993561,995289,997386,995401,995200,990137,996219,991179,996042,992068,993433,999367,995535,994307,994577,996769,997406,991650,991996,998045,998495,999598,995923,991921,996805,990435,990096,990628\n", + "Ground Truth: 998382,996371,993561,999068,995289,997386,995401,995200,990137,996219,991179,996042,992068,993433,995535,999367,994307,994577,996769,997406,991650,991996,998045,998495,999598,995923,991921,996805,990435,990096\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 874\n", + "Returned IDs: 991255,997914,997542,993738,997918,991948,990814,998697,995281,992979,994552,992012,998427,994280,993143,996328,994163,993256,996305,993079,997251,997806,997555,990776,998860,990634,998123,993976,995401,999820\n", + "Ground Truth: 991255,997914,997542,993738,997918,991948,990814,998697,995281,998567,992979,994552,992012,998097,991447,998427,994280,994972,993143,996328,994163,993256,996305,993079,995600,997251,997806,997555,990776,998860\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 875\n", + "Returned IDs: 995922,998586,998790,994537,996395,991624,993381,991610,996424,990429,992136,997009,990912,991667,990167,991571,993995,990194,999216,996209,992721,990450,994643,996809,993194,991513,991332,991975,990991,994451\n", + "Ground Truth: 995922,998586,998790,994537,996395,991624,993381,991610,996424,990429,992136,997009,990912,991667,990167,991571,993995,990194,999216,994981,996209,992721,990450,994643,996809,993194,991513,991332,991975,990991\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 877\n", + "Returned IDs: 990269,997320,994964,992331,992744,999159,995357,994873,998470,994225,995274,993168,998764,999437,995336,991433,993066,998228,999340,995438,993909,998365,991776,990210,999740,997178,994584,990535,996219,996401\n", + "Ground Truth: 990269,997320,994964,997920,992331,992744,999159,995357,994873,998470,994225,995274,993168,998764,999437,995336,991433,993066,998228,999340,995438,993909,998365,991776,990210,997178,999740,994584,990535,996219\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 878\n", + "Returned IDs: 995955,998242,997215,990125,994305,995503,992306,992824,991333,995804,990538,992170,990721,990192,991034,993140,991834,998260,999908,992490,992113,997849,997547,999430,999639,996742,996510,995724,991977,997055\n", + "Ground Truth: 995955,998242,997215,990125,994305,995503,992306,992824,991333,995804,990538,992170,990721,990192,991034,993140,996629,991834,998260,999908,992490,992113,997849,997547,999430,999639,996742,994155,996510,995724\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 879\n", + "Returned IDs: 997529,993725,994730,998710,994742,991185,995633,994830,999377,992424,993166,990887,998106,993561,993053,995289,998913,999740,990681,994409,996411,993920,996328,992260,993209,994013,994419,996867,992230,996132\n", + "Ground Truth: 997529,993725,994730,998710,994742,991185,995633,994830,999377,992424,993166,990887,998106,993561,993053,995289,998913,999740,990681,997167,994409,996411,993920,996328,992260,993209,994013,994419,996867,992230\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 880\n", + "Returned IDs: 992411,995552,992558,993920,995309,991851,998652,999206,996953,990843,994418,993474,996878,993077,995618,999305,991337,990279,996140,990345,998950,999540,993742,991263,993456,999908,990989,993534,990308,992318\n", + "Ground Truth: 992411,995552,992558,993920,995309,991851,998652,999206,996953,990843,994418,993474,996878,993077,995618,999305,991337,990279,996140,990345,994817,998950,999540,992966,993742,991263,990657,993456,999908,990989\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 882\n", + "Returned IDs: 992032,992293,991076,991203,996225,991954,996028,990662,993561,990811,993725,991087,998476,993857,994835,998823,991603,997529,991958,999286,990399,990097,990033,992636,994677,998197,991301,999922,998593,997554\n", + "Ground Truth: 992032,992293,991076,991203,996225,991954,996028,990662,993561,990811,993725,991087,998476,993857,994835,998823,991603,997529,991958,999286,990399,990097,990033,992636,994677,991771,998197,991301,991466,999922\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 883\n", + "Returned IDs: 997362,990958,996504,993813,997192,990925,996316,999789,998692,995755,992204,999483,998294,990118,997984,997371,995090,997320,999327,993518,991958,995132,995985,995367,998649,997911,997889,992789,995884,997620\n", + "Ground Truth: 997362,990958,996504,993813,997192,990925,996316,999789,998692,995755,992204,999483,998294,990118,997984,997371,995090,997320,999327,993518,991958,995132,995985,991009,995367,998649,997911,997889,993902,993982\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 887\n", + "Returned IDs: 995156,991432,995955,995562,999639,997032,997775,992435,996065,998260,990721,991157,994305,998142,994163,996592,997739,992497,995618,995983,991801,996510,992824,997614,990413,996884,990538,997420,991817,992095\n", + "Ground Truth: 995156,991432,995955,995562,999639,997032,997775,992435,996065,998260,990721,991157,994305,998142,996511,990210,994163,996592,997739,992497,995618,992350,995983,996283,991801,996510,992824,997614,990413,996884\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 889\n", + "Returned IDs: 995159,990355,998507,994929,996211,994541,996413,997746,998755,997722,996344,996276,992489,990931,994094,997879,997290,992445,994336,992313,994913,996430,997814,998174,992052,997420,992832,999179,992981,992934\n", + "Ground Truth: 995159,990355,998507,994929,996211,994541,996413,997746,998755,997722,996344,996276,992489,990931,994094,997879,997290,992445,994336,992313,999974,994913,996430,993621,997814,998174,992052,997420,992832,999179\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 891\n", + "Returned IDs: 990957,993283,992668,995752,991677,999561,996894,994852,999806,995974,998600,995580,998455,990949,995562,999714,995443,991801,992309,993781,998495,999898,993222,994979,995519,997903,991923,992124,992423,997864\n", + "Ground Truth: 990957,993283,992668,995752,991677,999561,996894,994852,999806,995974,998600,995580,998232,998455,990949,995562,999714,995443,991801,992309,993781,998495,999898,993222,997032,994979,995519,997903,991923,992124\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 894\n", + "Returned IDs: 994336,997309,993877,994280,992933,995861,996328,991378,994119,994956,999338,991334,991720,998076,992147,990998,993335,998123,997822,997159,998950,996855,995438,999533,992789,995125,998369,993929,996050,991007\n", + "Ground Truth: 994336,997309,993877,994280,993775,992933,995861,996328,991378,994119,994956,999338,991334,991720,998076,992147,990998,993335,998123,997822,997159,998950,996855,995438,999533,992789,995125,998369,993929,996050\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 897\n", + "Returned IDs: 997475,997652,991153,991075,997480,994891,993475,993621,993611,990696,999496,990600,992480,994765,993055,994558,995600,994313,997714,999205,998405,993165,992449,996856,990776,996429,994051,994307,994280,991723\n", + "Ground Truth: 997475,997652,991153,991075,997480,994891,993475,993621,993611,990696,999496,990600,992480,994765,991426,993055,995145,994558,995600,994313,997714,999205,998405,993165,992449,996856,990776,996429,994051,994307\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 898\n", + "Returned IDs: 991593,995260,999374,991999,994922,990278,998734,997620,998899,996351,990545,994497,991352,991889,999676,997985,990958,991172,999810,993885,992203,994776,990760,990026,991406,991720,994266,999658,995443,993387\n", + "Ground Truth: 991593,995260,992432,999374,991999,994922,990278,998734,997620,998899,996351,990545,994497,991352,991889,999676,997985,990958,991172,999810,993885,992203,997591,994776,990760,990026,991406,991720,994266,999658\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 899\n", + "Returned IDs: 997992,993504,995918,991595,997790,999735,996067,999811,992082,993609,996082,990494,992747,993737,995167,993024,991212,992665,994134,993071,990434,992864,991741,996196,996599,994651,999093,991425,998732,991596\n", + "Ground Truth: 997992,993504,995918,991595,997790,999735,996067,999811,996986,992082,993609,996082,990494,992747,993737,995167,993024,991212,992665,994134,993071,990434,992864,993611,991741,994815,996196,996599,994651,999093\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 901\n", + "Returned IDs: 995986,992718,992258,994163,994072,997689,998694,999908,999359,996163,991637,996926,997395,995301,994165,994195,993463,992645,996669,992558,997864,998079,997692,996214,991430,999432,999385,993609,995436,990367\n", + "Ground Truth: 995986,992718,992258,994163,994072,997689,998694,999908,999359,996163,991637,996926,997395,995301,994165,994195,993463,992645,991172,998758,996669,992558,997864,998079,997692,996214,991430,999432,999385,993609\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 903\n", + "Returned IDs: 999426,991984,990142,999769,993257,991013,998753,994971,997551,993785,992328,994489,998498,994789,996075,993582,991218,992635,995515,998734,990648,997584,995033,996631,995108,993675,996829,996840,998227,999132\n", + "Ground Truth: 999426,991984,990142,999769,993257,991013,998753,994971,997551,993785,992328,994489,998498,994789,996075,993582,991218,992635,995515,998734,990648,997584,995033,997255,996631,995108,993675,996829,996840,998227\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 904\n", + "Returned IDs: 992827,998779,994382,993744,999788,995300,992748,995200,994595,990722,998543,997529,994682,991334,998765,990934,996219,993856,991590,995401,994742,991885,995374,995861,990814,997933,994783,990701,998557,998862\n", + "Ground Truth: 992827,998779,994382,993744,999788,995300,992748,999916,995200,994595,990722,998543,997529,994682,991334,998765,997348,990934,996219,993856,993412,991590,995401,994742,991885,995374,995861,990814,997933,991986\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 907\n", + "Returned IDs: 995802,994778,999710,993492,992442,997006,992443,999982,993504,997483,993852,995826,992309,997913,992119,994670,992931,992228,994831,990231,995161,997605,990957,996993,992066,990890,996785,997320,998170,991873\n", + "Ground Truth: 995802,994778,999710,993492,992442,997006,991009,992443,999982,993504,997483,993852,993048,995826,992309,997913,992119,994670,992931,992228,994831,990231,995161,997605,990957,996993,997579,992066,990890,996785\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 912\n", + "Returned IDs: 999079,990096,999831,996918,999771,997906,998849,997305,999367,994955,992052,992944,996001,995203,994718,994226,995932,995536,998495,996124,995332,994081,997320,994643,996915,995401,994661,991498,994901,993609\n", + "Ground Truth: 999079,990096,999831,996918,999771,997906,998849,997305,999367,994955,992052,992944,996001,995203,995737,994718,994226,995932,995536,998495,996124,995332,994081,997320,994643,996915,995401,994661,991498,994901\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 913\n", + "Returned IDs: 996868,996001,993174,998934,992072,995701,997920,998593,995216,992610,994419,994610,995823,998186,998710,996147,998349,995855,996401,991000,997897,991958,999590,993738,994280,996075,997687,993559,998382,993280\n", + "Ground Truth: 996868,996001,993174,998934,992072,995701,997920,998593,995216,992610,994419,994610,995823,998186,998710,996147,998349,995855,996401,991000,997897,991958,999590,993738,994280,996075,997687,993559,998382,990557\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 916\n", + "Returned IDs: 994673,992833,994160,997192,990724,991782,997826,993648,993412,998458,991585,999328,996111,991361,990760,992284,991743,995702,996721,992994,999263,996532,991602,997880,999374,992147,991213,990432,991958,998899\n", + "Ground Truth: 994673,992833,994160,991900,997192,990724,991782,997826,993648,993412,998458,991585,999328,996111,991361,990760,992284,991743,991534,990462,995702,996721,992994,999263,996532,991602,997880,999374,992147,998215\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 917\n", + "Returned IDs: 994013,994203,995544,992098,996898,998060,993377,991057,999689,997059,998162,994412,991507,999023,995216,993217,996328,992000,994409,994488,996001,999402,998045,997554,991255,996765,995766,997466,997566,997020\n", + "Ground Truth: 994013,994203,995544,992098,996898,998060,993377,991057,999689,997059,998162,994412,991507,999023,995216,993217,996328,992000,994409,994488,996001,999402,998045,997554,991255,996765,995766,997466,996322,997566\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 918\n", + "Returned IDs: 991075,999472,990315,992156,997798,995455,996275,995633,995613,998017,996411,995600,994783,994879,995289,994431,995131,999064,999500,994558,998105,998982,999949,997079,992449,991315,993725,992079,995002,995831\n", + "Ground Truth: 991075,999472,990315,992156,997798,996275,995455,995633,995613,998017,996411,995600,994783,994879,995289,994431,995131,999064,999500,994558,998105,998982,994130,999949,997079,992449,991315,993725,992079,995002\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 919\n", + "Returned IDs: 994342,992780,998631,994577,998109,996475,995776,992068,992070,990271,998568,994984,999806,997331,999780,996219,993055,997990,994558,999430,998495,997909,995895,997554,997480,992628,992423,992655,997529,993053\n", + "Ground Truth: 994342,992780,998631,994577,998109,996475,995776,992068,992070,990271,998568,994984,999806,997331,999780,996219,993055,997990,994558,999430,998495,997909,995895,997116,997554,997480,992628,992423,993678,992655\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 922\n", + "Returned IDs: 996840,996686,995495,998582,998523,995515,993028,997486,991392,993091,998244,999524,994195,990026,998692,995004,991485,999453,993143,991462,990300,992309,999613,998996,996351,992328,990702,998837,991149,995588\n", + "Ground Truth: 996840,996686,995495,998582,998523,995515,993028,997486,991392,993091,998244,999524,994195,990026,998692,995004,991485,999453,993143,991462,990300,992309,999613,998996,996351,995401,992328,990702,998837,991149\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 925\n", + "Returned IDs: 992449,990220,993856,994998,993437,997975,997103,993939,990531,990887,997544,998067,997529,998710,998733,999532,997459,999054,995778,994830,999496,994185,999279,998910,995083,993100,990435,991769,999118,995873\n", + "Ground Truth: 992449,990220,993856,994998,993437,997975,997103,993939,990531,990887,997544,998067,997529,998710,998733,999532,997459,999054,995778,994830,999496,994185,999279,998910,995083,993100,990435,991769,999118,997554\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 926\n", + "Returned IDs: 994830,996132,994204,990340,990711,994742,994783,995633,998913,990012,998710,997319,993053,998196,996411,991330,999377,995289,994529,992289,996302,993433,996049,997529,995948,995200,996809,990435,993166,997797\n", + "Ground Truth: 994830,996132,994204,990340,990711,994742,994783,995633,998913,990012,998710,997319,993053,998196,996411,991330,999377,995289,994529,992289,996302,993433,996049,995401,997529,995948,995200,996809,990435,993166\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 931\n", + "Returned IDs: 995299,990074,998803,993413,992456,996183,998949,997340,996986,990730,996408,992372,998903,993335,994682,990998,998716,990700,991919,997577,995393,996357,992628,995313,990173,992664,994269,993917,993101,991382\n", + "Ground Truth: 995299,990074,998803,993413,992456,996183,998949,997340,996986,990730,996408,992372,998903,993335,994682,990998,998716,990700,991919,997577,995393,996357,992628,995313,990173,992664,994269,993917,997116,993101\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 932\n", + "Returned IDs: 997918,993289,997002,995826,993504,993492,993512,999900,996511,994449,992571,993781,992029,994959,999840,997790,994268,990096,998090,996062,999019,990399,997320,997977,990930,999355,994280,997586,991206,996592\n", + "Ground Truth: 997918,993289,997002,995826,993504,993492,993512,999900,996511,994449,992571,993781,992029,994959,999840,997790,994268,990096,998090,996062,999019,990399,997320,997977,990930,999355,991648,994280,997586,991206\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 933\n", + "Returned IDs: 999784,996001,996028,998302,997371,997002,997320,998921,999806,994266,995560,993820,995596,998720,997913,993834,994964,993342,996840,993902,997301,990532,993906,996096,995886,993260,994689,995852,992330,997506\n", + "Ground Truth: 999784,996001,996028,998302,997371,997002,997320,998921,999806,994266,995560,993820,995596,998720,997913,993834,994964,993342,996840,993902,997301,990532,995375,993906,996096,995886,993260,994689,995852,992330\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 935\n", + "Returned IDs: 995981,996609,990444,996648,991721,994069,999374,991593,993412,990990,996085,998302,998777,997320,998246,996805,997301,995266,998339,993186,998849,991422,990585,998974,991602,993049,990096,990146,992291,991809\n", + "Ground Truth: 995981,996609,990444,996648,991721,994069,999374,991593,993412,990990,996085,998302,998777,999046,997320,998246,996805,997301,995266,998339,993186,998849,991422,990585,998974,991602,993049,990096,990146,992291\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 936\n", + "Returned IDs: 997700,999581,999054,991507,990537,991722,995106,994861,995268,993209,992448,993313,995778,996607,990372,997738,995820,998162,998910,998283,999279,995518,996505,994380,990924,991424,996813,995401,991699,995521\n", + "Ground Truth: 997700,999581,999054,991507,990537,991722,995106,994861,995268,993209,992448,993313,995778,996607,990372,997738,995820,998162,998910,998283,999279,995518,996505,994380,990924,991424,993046,996813,995401,991699\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 938\n", + "Returned IDs: 998777,996219,999676,995401,999453,998530,998797,992309,998495,993355,994318,990843,996840,998302,992950,990974,992328,994850,995016,991585,994195,993412,994964,990432,995899,992216,995374,999302,997554,996033\n", + "Ground Truth: 998777,996219,999676,995401,999453,998530,998797,992309,998495,993355,994318,990843,996840,998302,992950,990974,992328,994850,995016,991585,994195,993431,993412,994964,995899,990432,992216,995374,999302,997554\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 940\n", + "Returned IDs: 994897,993342,999806,993617,997335,999430,992628,991271,993528,992807,995393,990173,990194,991846,999555,994155,998787,997126,994150,996809,999834,996660,995895,997196,994122,993076,994269,996184,996944,994834\n", + "Ground Truth: 994897,993342,999806,993617,997335,999430,992628,995039,991271,993528,992807,995393,990173,990194,991846,999555,994155,998214,998787,997126,994150,996809,999834,996660,995895,997196,994122,993076,994269,996184\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 942\n", + "Returned IDs: 998793,999476,997215,992802,997744,990782,990150,995552,990755,994965,995724,990192,990908,992608,999639,990334,999164,998242,996196,990930,997548,991791,998495,990538,994661,999908,995979,995430,995372,999294\n", + "Ground Truth: 998793,999476,997215,992802,997744,990782,990150,995552,990755,994965,994536,995724,990192,990908,992608,999639,990334,999164,998242,996196,990930,997548,991791,998495,990538,994661,999908,995979,995430,995372\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 943\n", + "Returned IDs: 994879,998347,992582,997728,996673,992760,994265,997353,998225,992502,991431,992293,996948,992862,997304,995981,999761,992481,990862,998708,994304,998986,994894,998697,993072,999243,993809,997620,995927,992183\n", + "Ground Truth: 994879,998347,993222,992582,997728,996673,992760,994265,997353,998225,992502,991431,992293,996948,992862,997304,991680,995981,999761,992481,990862,998708,994304,998986,994894,998697,993072,999243,993809,997620\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 945\n", + "Returned IDs: 993209,993299,996867,994317,990638,993586,997544,990082,994742,997723,997319,992582,990126,997077,993549,995030,994587,991943,995211,996302,998913,995633,991664,990493,998910,992289,995268,993920,996411,991639\n", + "Ground Truth: 993209,993299,996867,994317,990638,993586,997544,990082,994742,997723,997319,992582,990126,997077,993549,995078,995030,994587,991943,995211,996302,992667,998913,995633,996277,991664,990493,998910,992289,995268\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 946\n", + "Returned IDs: 995547,999530,994409,999367,995994,998545,991061,997723,993717,993561,991923,997191,996042,994948,996769,994026,996038,996918,997406,995401,995507,997832,991459,994002,997675,993143,997289,991052,995923,993271\n", + "Ground Truth: 995547,999530,994409,999367,995994,998545,991061,997723,993717,993561,991923,997191,996042,994948,996769,994026,996038,996918,997406,995401,995507,997832,991459,993122,994002,993143,997675,997289,998250,991052\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 948\n", + "Returned IDs: 996001,994449,995536,990532,996661,997586,997320,996824,992862,994200,999444,996178,999639,991630,992635,999587,995672,990814,993504,992469,999974,995983,995250,993157,990399,991981,998827,999453,995401,994334\n", + "Ground Truth: 996001,994449,995536,990532,996661,997586,997320,996824,996511,992862,996368,994200,999444,996178,999639,991630,992521,992635,999587,995672,990814,993504,992469,999974,994343,995983,995250,993157,990399,991981\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 951\n", + "Returned IDs: 999076,993355,997539,994922,990300,998424,990809,990580,995210,995515,997747,999582,993688,998236,998751,993237,998837,999469,994789,992612,991609,991392,997291,996561,992979,999097,998401,997772,997818,992950\n", + "Ground Truth: 999076,993355,997539,994922,990300,998424,990809,990580,995210,995515,997747,999582,995430,993688,998236,998751,993237,998837,999469,994789,992612,993222,991609,991392,997291,996561,992979,999097,996793,998401\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 954\n", + "Returned IDs: 995840,997807,998414,994782,998668,997586,998496,997004,995927,999210,999941,998068,993114,995944,999931,993689,993878,997364,990866,999898,995971,993624,997305,991259,998814,998708,990373,993831,999218,997552\n", + "Ground Truth: 995840,997807,998414,994782,998668,997586,998496,997004,994756,995927,999210,999941,998068,993114,995944,999931,993689,993878,997364,990866,999898,998861,991006,995971,993624,997305,991259,998814,998708,990373\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 956\n", + "Returned IDs: 992630,999343,991335,992384,996219,990934,990171,996401,994100,990974,990448,996143,999331,998779,997571,998170,997371,997920,996790,999102,998550,997006,990275,999647,996777,991862,990315,996727,998500,990666\n", + "Ground Truth: 992630,999343,991335,992384,996219,990934,990171,992931,996401,994100,990974,990448,996143,999331,998779,997571,998170,997371,997920,996790,999102,998550,997006,990275,999647,996777,991862,990315,996727,998500\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 960\n", + "Returned IDs: 998335,990627,992862,991304,996456,996104,995991,996594,992270,994564,990802,991447,995231,994742,995168,990998,996408,999319,993076,999334,995780,996611,998279,992636,996728,991098,994498,997506,995626,993638\n", + "Ground Truth: 998335,990627,992862,991304,996456,996104,994796,995991,996594,992270,994564,990802,991447,995231,994742,995168,990998,996408,999319,993076,999334,995780,996611,998279,992636,996728,991098,994498,997506,995626\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 962\n", + "Returned IDs: 995336,997301,995438,994964,995961,994757,998246,999144,994360,999331,999710,997320,990949,995942,993492,992384,997470,990231,998302,994100,998090,990958,996592,995886,993241,992228,994081,990957,990096,991883\n", + "Ground Truth: 995336,997301,995438,994964,995961,994757,998246,999144,994360,999331,993154,999710,997320,990949,995942,993492,992384,997470,990231,998302,997775,999301,994100,998090,990958,996592,995886,993241,992228,994081\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 963\n", + "Returned IDs: 992135,991459,998623,990537,998720,990682,996001,995968,990457,996060,990502,991154,996247,990005,993673,996812,995027,994809,997091,994380,999453,999543,996417,990616,997965,996759,999724,995932,998068,995937\n", + "Ground Truth: 992135,991459,998623,990537,998720,990682,996001,995968,990457,997140,996060,990502,991154,996247,990005,993673,996812,995027,993131,994809,997091,994380,999453,992124,999543,996417,990616,997965,996759,998029\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 965\n", + "Returned IDs: 994964,997379,995791,993035,992944,994777,996397,999210,998950,996811,993696,993633,993215,992612,990310,998665,995365,994922,996915,991999,999676,990690,993355,992169,993809,996093,994318,993561,992334,995596\n", + "Ground Truth: 994964,997379,995791,993035,992944,994777,996397,999210,998950,996811,993831,993696,993633,994089,993215,992612,990310,998665,995365,994922,996915,991999,999676,990690,993355,992169,993809,996093,994318,993561\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 966\n", + "Returned IDs: 998797,999995,997040,990146,999262,995884,997320,990925,994964,998302,992546,996959,990936,990958,991852,995947,997586,992896,996785,997911,993484,998246,997913,996075,995336,997822,999343,999242,999374,994366\n", + "Ground Truth: 998797,999995,997040,990146,999262,995884,997320,990925,994964,998302,995873,992546,996959,990936,990958,991852,995947,997586,992896,996785,997911,993484,998246,997913,996075,995336,997822,999343,996296,999242\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 967\n", + "Returned IDs: 995349,997685,999079,992664,997673,991148,993866,997906,998468,999443,999576,992209,995969,995231,992275,994661,997369,990096,997666,993965,995064,995895,996733,991477,995695,993445,996124,995735,995393,994751\n", + "Ground Truth: 995349,997685,999079,992664,997673,991148,993866,997906,998468,999443,999576,992209,995969,995231,992275,994661,997369,990096,997666,993965,995064,995895,996733,991477,995695,993445,996124,995735,992752,995393\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 969\n", + "Returned IDs: 991110,990729,999737,998022,991767,998318,994063,991887,994088,995373,999662,995449,997131,995480,992604,990256,999445,994122,992133,996236,990577,996730,994577,998629,993137,997942,995080,990345,994539,993185\n", + "Ground Truth: 991110,990729,999737,998022,991767,998318,994063,991887,994088,995373,999662,994956,995449,997131,995480,992604,990256,999445,994122,992133,996236,990577,996730,994577,998629,993137,997942,995080,990345,994539\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 970\n", + "Returned IDs: 993484,994964,993648,993642,993238,997320,999374,990995,992047,990399,999453,995755,993389,996959,990936,990224,997737,996785,993143,993885,993412,995961,993663,998294,991743,994922,995886,990958,994673,998692\n", + "Ground Truth: 993484,994964,993648,993642,993238,997320,999374,990995,992047,990399,999453,995755,993389,996959,990936,990224,997737,996785,993143,993885,993412,995961,993663,992334,993400,998294,991743,994922,995886,990958\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 972\n", + "Returned IDs: 996001,993053,994106,992182,999352,995200,996219,993883,995873,994382,997909,990435,998725,999653,995216,994689,990557,991464,990752,995823,995289,993438,996657,994529,994783,998982,991212,999862,995222,996277\n", + "Ground Truth: 996001,993053,994300,999151,994106,992182,999352,995200,998568,996219,993883,995873,994382,997909,990435,995384,998725,999653,995216,994689,996411,997837,990557,991464,992384,995823,990752,995289,995198,993438\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 973\n", + "Returned IDs: 990296,992161,998221,993412,998649,998495,990957,999898,990399,997192,996869,992107,995942,990435,993656,996001,990934,991335,997582,991311,992087,996219,999144,999710,995289,998039,993168,991096,995755,995536\n", + "Ground Truth: 990296,992161,998221,993412,998649,998495,990957,999898,990399,997920,997192,996869,992107,995942,990435,993656,996001,990934,991335,997582,991311,992087,996219,999144,999710,995289,992124,995899,998039,993168\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 976\n", + "Returned IDs: 995958,995778,997023,992473,997723,993209,996376,994453,995401,996429,996226,995701,999675,998067,997601,997204,997216,991507,991044,996232,998495,997732,997544,993948,991747,993313,996328,991958,996506,998045\n", + "Ground Truth: 995958,995778,997023,992473,997723,993209,996376,994453,995401,996429,996226,995701,999675,995536,998067,997601,997204,997216,991507,991044,999977,996232,998495,997732,997544,993948,990814,991747,993313,996328\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 983\n", + "Returned IDs: 995295,996580,995283,994730,990528,992226,991464,993725,990531,990930,993585,990315,996411,994742,994682,993623,990614,990076,990133,995289,994783,999798,990811,992680,996136,993053,997141,993705,997529,997340\n", + "Ground Truth: 995295,996580,995283,994730,990528,992226,991464,993725,990531,990930,993585,990315,996411,994742,994682,993623,990614,990076,990133,995289,994783,999798,990811,992680,996136,993053,997141,991629,993705,997529\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 987\n", + "Returned IDs: 997204,992169,993288,995289,995374,999587,995359,999099,992685,995685,991151,997918,998530,993909,993412,991810,994409,996328,999820,992979,992239,995401,997395,997002,997630,994906,990337,996903,997447,995994\n", + "Ground Truth: 997204,992169,993288,995289,995374,999587,995359,999099,992685,995685,991151,997918,998530,993909,993412,991810,994409,996328,996220,999820,992979,992239,995401,997395,997002,997630,994906,990337,996903,997447\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 989\n", + "Returned IDs: 997267,990129,991544,995891,996472,992448,992801,996042,991958,995521,994409,997884,995864,998045,997909,998703,995200,995198,993433,994002,994742,991671,994954,994413,993127,993561,998623,992123,995702,997077\n", + "Ground Truth: 997267,990129,991544,995891,996472,992448,992801,996042,991958,995521,994409,997884,995864,995873,998045,997909,998703,990681,995200,995198,993433,994002,994742,991671,994954,994413,993127,993561,998623,992123\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 991\n", + "Returned IDs: 996571,991975,996120,993381,997145,997965,994724,991987,993641,991513,991038,999653,997369,995780,996920,997906,991844,997265,993629,992945,998586,994374,998861,990450,995168,996209,990991,995231,992862,996326\n", + "Ground Truth: 996571,991975,996120,993381,997145,997965,994724,991987,993641,991513,991038,999653,997369,995780,996920,997906,991844,997265,993629,992945,998586,994374,998861,990450,995168,992233,996209,990991,995231,992862\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 994\n", + "Returned IDs: 994102,990092,991862,991335,996052,999647,990957,990231,996219,992134,994831,990545,997439,997913,996143,992114,995942,991743,999343,994192,993264,997320,990171,999144,993135,992931,996053,995865,998302,990949\n", + "Ground Truth: 994102,990092,991862,991335,996052,999647,990957,990231,996219,992134,994831,990545,997439,997913,996143,992114,995942,991743,995844,999343,999993,994192,993264,997320,990171,999144,993135,992931,996053,994898\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 995\n", + "Returned IDs: 996627,994751,996411,993070,997729,990011,990012,990528,990743,996906,994663,995295,993725,992230,992680,992408,999740,999922,994830,990392,993240,994730,995076,996011,992024,993705,993906,995435,998030,991464\n", + "Ground Truth: 996627,994751,996411,993070,997729,990011,990012,990528,990743,996906,994663,995295,993725,992230,992680,992408,999740,999922,994830,990392,993240,994730,995076,996011,992024,996147,993705,993906,995435,998030\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 996\n", + "Returned IDs: 995193,992521,998527,992046,992657,993021,995820,995640,997023,996581,999047,999394,991922,991102,992917,994508,990099,992311,999731,998075,993107,994185,994690,998255,990295,992595,997554,998910,993998,997320\n", + "Ground Truth: 995193,992521,998527,992046,992657,993021,995820,995640,997023,996581,999047,999394,991922,991102,992917,994508,990099,992311,999731,998075,993107,994185,994690,998255,999992,990295,992595,997554,998910,993998\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 997\n", + "Returned IDs: 991052,997675,996918,993805,996506,991061,997723,995401,991529,993014,996505,995521,996879,998397,999646,991747,993931,998593,994280,995080,993313,990537,994662,993600,996042,992521,995994,994095,992933,997411\n", + "Ground Truth: 991052,997675,996918,993805,996506,991061,997723,995401,991529,993014,996505,995521,996879,998397,999646,991747,993931,998593,994280,995080,993313,990537,994662,993600,998021,996042,992521,995994,994095,992933\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 999\n", + "Returned IDs: 999162,992654,991977,999638,995560,998334,999387,994010,994672,998118,992718,991540,998142,995503,993018,998280,990930,997407,991432,991157,997320,990843,990926,999954,997816,993976,996262,992124,990487,992944\n", + "Ground Truth: 999162,992654,991977,999638,995560,998334,999387,994010,994672,998118,992718,991540,998142,995503,993018,998567,998280,990930,997407,991432,991157,997320,990843,990926,999954,997816,993976,996262,992124,990487\n", + "Recall: 0.9667\n", + "\n", + "Total query IDs with recall < 1.0: 542\n", + "IDs: [np.int64(1), np.int64(2), np.int64(4), np.int64(5), np.int64(6), np.int64(8), np.int64(10), np.int64(19), np.int64(21), np.int64(22), np.int64(23), np.int64(24), np.int64(26), np.int64(29), np.int64(31), np.int64(34), np.int64(35), np.int64(40), np.int64(41), np.int64(43), np.int64(45), np.int64(46), np.int64(48), np.int64(52), np.int64(53), np.int64(54), np.int64(56), np.int64(58), np.int64(59), np.int64(60), np.int64(61), np.int64(62), np.int64(63), np.int64(65), np.int64(66), np.int64(67), np.int64(69), np.int64(71), np.int64(74), np.int64(75), np.int64(79), np.int64(81), np.int64(84), np.int64(85), np.int64(88), np.int64(89), np.int64(91), np.int64(93), np.int64(100), np.int64(102), np.int64(104), np.int64(105), np.int64(107), np.int64(108), np.int64(109), np.int64(110), np.int64(112), np.int64(113), np.int64(116), np.int64(117), np.int64(119), np.int64(127), np.int64(128), np.int64(129), np.int64(131), np.int64(132), np.int64(133), np.int64(134), np.int64(135), np.int64(136), np.int64(138), np.int64(139), np.int64(144), np.int64(150), np.int64(151), np.int64(153), np.int64(154), np.int64(156), np.int64(158), np.int64(159), np.int64(160), np.int64(161), np.int64(162), np.int64(163), np.int64(170), np.int64(171), np.int64(172), np.int64(175), np.int64(178), np.int64(179), np.int64(181), np.int64(183), np.int64(184), np.int64(187), np.int64(188), np.int64(191), np.int64(192), np.int64(199), np.int64(200), np.int64(203), np.int64(204), np.int64(208), np.int64(211), np.int64(212), np.int64(213), np.int64(215), np.int64(216), np.int64(218), np.int64(220), np.int64(221), np.int64(223), np.int64(224), np.int64(227), np.int64(228), np.int64(229), np.int64(230), np.int64(231), np.int64(233), np.int64(236), np.int64(240), np.int64(242), np.int64(247), np.int64(249), np.int64(251), np.int64(258), np.int64(259), np.int64(261), np.int64(264), np.int64(266), np.int64(268), np.int64(269), np.int64(270), np.int64(272), np.int64(273), np.int64(275), np.int64(277), np.int64(278), np.int64(281), np.int64(282), np.int64(283), np.int64(284), np.int64(285), np.int64(286), np.int64(287), np.int64(289), np.int64(293), np.int64(294), np.int64(295), np.int64(296), np.int64(297), np.int64(302), np.int64(303), np.int64(304), np.int64(305), np.int64(306), np.int64(308), np.int64(309), np.int64(311), np.int64(312), np.int64(315), np.int64(318), np.int64(319), np.int64(320), np.int64(321), np.int64(322), np.int64(323), np.int64(324), np.int64(325), np.int64(326), np.int64(327), np.int64(328), np.int64(331), np.int64(334), np.int64(337), np.int64(342), np.int64(344), np.int64(345), np.int64(346), np.int64(347), np.int64(349), np.int64(352), np.int64(356), np.int64(357), np.int64(358), np.int64(359), np.int64(360), np.int64(361), np.int64(362), np.int64(366), np.int64(367), np.int64(368), np.int64(370), np.int64(371), np.int64(372), np.int64(376), np.int64(379), np.int64(381), np.int64(382), np.int64(383), np.int64(385), np.int64(389), np.int64(390), np.int64(393), np.int64(394), np.int64(395), np.int64(399), np.int64(402), np.int64(403), np.int64(406), np.int64(407), np.int64(409), np.int64(410), np.int64(411), np.int64(412), np.int64(413), np.int64(417), np.int64(420), np.int64(423), np.int64(424), np.int64(425), np.int64(427), np.int64(429), np.int64(430), np.int64(431), np.int64(432), np.int64(433), np.int64(434), np.int64(437), np.int64(439), np.int64(440), np.int64(441), np.int64(442), np.int64(447), np.int64(448), np.int64(449), np.int64(450), np.int64(451), np.int64(452), np.int64(453), np.int64(455), np.int64(456), np.int64(457), np.int64(458), np.int64(459), np.int64(460), np.int64(461), np.int64(462), np.int64(467), np.int64(469), np.int64(470), np.int64(473), np.int64(475), np.int64(477), np.int64(480), np.int64(481), np.int64(485), np.int64(488), np.int64(491), np.int64(494), np.int64(496), np.int64(498), np.int64(499), np.int64(500), np.int64(501), np.int64(503), np.int64(504), np.int64(505), np.int64(507), np.int64(508), np.int64(509), np.int64(510), np.int64(511), np.int64(512), np.int64(513), np.int64(514), np.int64(515), np.int64(517), np.int64(520), np.int64(521), np.int64(523), np.int64(524), np.int64(527), np.int64(529), np.int64(533), np.int64(534), np.int64(541), np.int64(543), np.int64(544), np.int64(545), np.int64(546), np.int64(548), np.int64(549), np.int64(551), np.int64(552), np.int64(557), np.int64(558), np.int64(561), np.int64(563), np.int64(564), np.int64(565), np.int64(567), np.int64(569), np.int64(571), np.int64(574), np.int64(575), np.int64(576), np.int64(577), np.int64(578), np.int64(581), np.int64(583), np.int64(584), np.int64(586), np.int64(588), np.int64(589), np.int64(591), np.int64(593), np.int64(595), np.int64(596), np.int64(600), np.int64(604), np.int64(605), np.int64(610), np.int64(611), np.int64(612), np.int64(613), np.int64(615), np.int64(616), np.int64(618), np.int64(619), np.int64(620), np.int64(621), np.int64(623), np.int64(625), np.int64(626), np.int64(628), np.int64(629), np.int64(630), np.int64(632), np.int64(633), np.int64(636), np.int64(638), np.int64(639), np.int64(640), np.int64(641), np.int64(643), np.int64(644), np.int64(645), np.int64(646), np.int64(647), np.int64(648), np.int64(649), np.int64(650), np.int64(652), np.int64(654), np.int64(655), np.int64(656), np.int64(658), np.int64(661), np.int64(665), np.int64(666), np.int64(668), np.int64(669), np.int64(670), np.int64(671), np.int64(673), np.int64(674), np.int64(676), np.int64(678), np.int64(680), np.int64(681), np.int64(682), np.int64(688), np.int64(689), np.int64(690), np.int64(691), np.int64(694), np.int64(696), np.int64(697), np.int64(700), np.int64(701), np.int64(704), np.int64(705), np.int64(706), np.int64(708), np.int64(710), np.int64(712), np.int64(713), np.int64(714), np.int64(718), np.int64(719), np.int64(720), np.int64(724), np.int64(726), np.int64(727), np.int64(728), np.int64(730), np.int64(731), np.int64(733), np.int64(734), np.int64(735), np.int64(739), np.int64(740), np.int64(742), np.int64(743), np.int64(744), np.int64(745), np.int64(748), np.int64(749), np.int64(751), np.int64(754), np.int64(758), np.int64(760), np.int64(761), np.int64(762), np.int64(763), np.int64(764), np.int64(765), np.int64(768), np.int64(770), np.int64(771), np.int64(772), np.int64(773), np.int64(774), np.int64(775), np.int64(776), np.int64(777), np.int64(778), np.int64(779), np.int64(781), np.int64(782), np.int64(783), np.int64(787), np.int64(789), np.int64(791), np.int64(792), np.int64(794), np.int64(796), np.int64(800), np.int64(801), np.int64(804), np.int64(805), np.int64(806), np.int64(809), np.int64(810), np.int64(816), np.int64(817), np.int64(818), np.int64(820), np.int64(821), np.int64(822), np.int64(823), np.int64(827), np.int64(828), np.int64(829), np.int64(830), np.int64(832), np.int64(833), np.int64(835), np.int64(839), np.int64(841), np.int64(842), np.int64(844), np.int64(846), np.int64(847), np.int64(848), np.int64(849), np.int64(854), np.int64(855), np.int64(856), np.int64(857), np.int64(860), np.int64(861), np.int64(862), np.int64(863), np.int64(866), np.int64(867), np.int64(869), np.int64(872), np.int64(873), np.int64(874), np.int64(875), np.int64(877), np.int64(878), np.int64(879), np.int64(880), np.int64(882), np.int64(883), np.int64(887), np.int64(889), np.int64(891), np.int64(894), np.int64(897), np.int64(898), np.int64(899), np.int64(901), np.int64(903), np.int64(904), np.int64(907), np.int64(912), np.int64(913), np.int64(916), np.int64(917), np.int64(918), np.int64(919), np.int64(922), np.int64(925), np.int64(926), np.int64(931), np.int64(932), np.int64(933), np.int64(935), np.int64(936), np.int64(938), np.int64(940), np.int64(942), np.int64(943), np.int64(945), np.int64(946), np.int64(948), np.int64(951), np.int64(954), np.int64(956), np.int64(960), np.int64(962), np.int64(963), np.int64(965), np.int64(966), np.int64(967), np.int64(969), np.int64(970), np.int64(972), np.int64(973), np.int64(976), np.int64(983), np.int64(987), np.int64(989), np.int64(991), np.int64(994), np.int64(995), np.int64(996), np.int64(997), np.int64(999)]\n", + "Average Recall across all 1000 queries: 0.9664\n" + ] + } + ], + "source": [ + "#For querying (int filter)- gte filter\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "#inputs\n", + "int_rate_query = \"99p\" # change to 1p, 50p, 80p, 99p\n", + "top_k = 30\n", + "\n", + "# Mode 1: Single query id\n", + "# Mode 2: Find all query ids with recall < 1\n", + "mode = 2\n", + "query_id = 457 # only used in mode 1\n", + "\n", + "# Map int_rate_query to filter range\n", + "range_map = {\n", + " \"1p\": [10000, 1000000],\n", + " \"50p\": [500000, 1000000],\n", + " \"80p\": [800000, 1000000],\n", + " \"99p\": [990000, 1000000],\n", + "}\n", + "filter_range = range_map[int_rate_query]\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "test_parquet_path = os.path.join(dataset_path, \"test.parquet\")\n", + "gt_parquet_path = os.path.join(dataset_path, f\"neighbors_int_{int_rate_query}.parquet\")\n", + "\n", + "# Read parquet files\n", + "test_df = pq.ParquetFile(test_parquet_path).read().to_pandas()\n", + "gt_df = pq.ParquetFile(gt_parquet_path).read().to_pandas()\n", + "\n", + "def run_query(query_id):\n", + " query_row = test_df[test_df[\"id\"] == query_id]\n", + " if query_row.empty:\n", + " print(f\"Query ID {query_id} not found in test.parquet\")\n", + " return None\n", + "\n", + " query_vector = query_row[\"emb\"].values[0]\n", + "\n", + " gt_row = gt_df[gt_df[\"id\"] == query_id]\n", + " if gt_row.empty:\n", + " print(f\"Query ID {query_id} not found in ground truth file\")\n", + " return None\n", + "\n", + " ground_truth = gt_row[\"neighbors_id\"].values[0][:top_k]\n", + "\n", + " results = index.query(\n", + " vector=query_vector,\n", + " top_k=top_k,\n", + " filter=[{\"id\": {\"$gte\": filter_range[0]}}],\n", + " include_vectors=False,\n", + " )\n", + " returned_ids = [int(r[\"id\"]) for r in results]\n", + "\n", + " intersection = len(set(returned_ids) & set(ground_truth))\n", + " recall = intersection / len(ground_truth)\n", + "\n", + " return returned_ids, ground_truth, recall\n", + "\n", + "\n", + "if mode == 1:\n", + " result = run_query(query_id)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " print(f\"Query ID: {query_id}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\")\n", + "\n", + "elif mode == 2:\n", + " print(f\"Finding all query IDs with recall < 1.0 ...\\n\")\n", + " low_recall_ids = []\n", + " all_recalls = []\n", + "\n", + " for qid in test_df[\"id\"].values:\n", + " result = run_query(qid)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " all_recalls.append(recall)\n", + " if recall < 1.0:\n", + " low_recall_ids.append(qid)\n", + " print(f\"Query ID: {qid}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\\n\")\n", + "\n", + " print(f\"Total query IDs with recall < 1.0: {len(low_recall_ids)}\")\n", + " print(f\"IDs: {low_recall_ids}\")\n", + " print(f\"Average Recall across all {len(all_recalls)} queries: {sum(all_recalls)/len(all_recalls):.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_numfilter_latest_master_stability_vaib2',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 200000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For deleting (int filter) - range operator\n", + "int_rate_delete = \"1p\" # change to 1p, 50p, 80p, 99p\n", + "\n", + "# Map int_rate to range\n", + "range_map = {\n", + " \"1p\": [0, 9999],\n", + " \"50p\": [0, 499999],\n", + " \"80p\": [0, 799999],\n", + " \"99p\": [0, 989999],\n", + "}\n", + "filter_range = range_map[int_rate_delete]\n", + "\n", + "result = index.delete_with_filter([{\"id\": {\"$range\": filter_range}}])\n", + "print(f\"Deleted vectors with id range: {filter_range}\")\n", + "print(result)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Deleted vectors with id < 800000\n", + "800000 vectors deleted\n" + ] + } + ], + "source": [ + "#For deleting (int filter) - gte\n", + "int_rate_delete = \"80p\" # change to 1p, 50p, 80p, 99p\n", + "\n", + "# Map int_rate to lower bound (gte value)\n", + "range_map = {\n", + " \"1p\": 10000,\n", + " \"50p\": 500000,\n", + " \"60p\": 600000,\n", + " \"70p\": 700000,\n", + " \"80p\": 800000,\n", + " \"99p\": 990000,\n", + "}\n", + "gte_value = range_map[int_rate_delete]\n", + "\n", + "result = index.delete_with_filter([{\"id\": {\"$lt\": gte_value}}])\n", + "print(f\"Deleted vectors with id < {gte_value}\")\n", + "print(result)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_reupsertcheck',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 300000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Upserted 798 vectors, total so far: 798\n", + "Upserted 812 vectors, total so far: 1610\n", + "Upserted 785 vectors, total so far: 2395\n", + "Upserted 813 vectors, total so far: 3208\n", + "Upserted 789 vectors, total so far: 3997\n", + "Upserted 827 vectors, total so far: 4824\n", + "Upserted 796 vectors, total so far: 5620\n", + "Upserted 804 vectors, total so far: 6424\n", + "Upserted 793 vectors, total so far: 7217\n", + "Upserted 787 vectors, total so far: 8004\n", + "Upserted 794 vectors, total so far: 8798\n", + "Upserted 797 vectors, total so far: 9595\n", + "Upserted 797 vectors, total so far: 10392\n", + "Upserted 817 vectors, total so far: 11209\n", + "Upserted 787 vectors, total so far: 11996\n", + "Upserted 802 vectors, total so far: 12798\n", + "Upserted 800 vectors, total so far: 13598\n", + "Upserted 786 vectors, total so far: 14384\n", + "Upserted 811 vectors, total so far: 15195\n", + "Upserted 772 vectors, total so far: 15967\n", + "Upserted 816 vectors, total so far: 16783\n", + "Upserted 796 vectors, total so far: 17579\n", + "Upserted 811 vectors, total so far: 18390\n", + "Upserted 767 vectors, total so far: 19157\n", + "Upserted 816 vectors, total so far: 19973\n", + "Upserted 800 vectors, total so far: 20773\n", + "Upserted 792 vectors, total so far: 21565\n", + "Upserted 800 vectors, total so far: 22365\n", + "Upserted 806 vectors, total so far: 23171\n", + "Upserted 792 vectors, total so far: 23963\n", + "Upserted 799 vectors, total so far: 24762\n", + "Upserted 798 vectors, total so far: 25560\n", + "Upserted 817 vectors, total so far: 26377\n", + "Upserted 813 vectors, total so far: 27190\n", + "Upserted 809 vectors, total so far: 27999\n", + "Upserted 810 vectors, total so far: 28809\n", + "Upserted 805 vectors, total so far: 29614\n", + "Upserted 803 vectors, total so far: 30417\n", + "Upserted 790 vectors, total so far: 31207\n", + "Upserted 818 vectors, total so far: 32025\n", + "Upserted 805 vectors, total so far: 32830\n", + "Upserted 790 vectors, total so far: 33620\n", + "Upserted 795 vectors, total so far: 34415\n", + "Upserted 801 vectors, total so far: 35216\n", + "Upserted 789 vectors, total so far: 36005\n", + "Upserted 810 vectors, total so far: 36815\n", + "Upserted 796 vectors, total so far: 37611\n", + "Upserted 791 vectors, total so far: 38402\n", + "Upserted 785 vectors, total so far: 39187\n", + "Upserted 820 vectors, total so far: 40007\n", + "Upserted 812 vectors, total so far: 40819\n", + "Upserted 789 vectors, total so far: 41608\n", + "Upserted 796 vectors, total so far: 42404\n", + "Upserted 807 vectors, total so far: 43211\n", + "Upserted 805 vectors, total so far: 44016\n", + "Upserted 792 vectors, total so far: 44808\n", + "Upserted 788 vectors, total so far: 45596\n", + "Upserted 785 vectors, total so far: 46381\n", + "Upserted 818 vectors, total so far: 47199\n", + "Upserted 807 vectors, total so far: 48006\n", + "Upserted 818 vectors, total so far: 48824\n", + "Upserted 790 vectors, total so far: 49614\n", + "Upserted 797 vectors, total so far: 50411\n", + "Upserted 816 vectors, total so far: 51227\n", + "Upserted 795 vectors, total so far: 52022\n", + "Upserted 807 vectors, total so far: 52829\n", + "Upserted 778 vectors, total so far: 53607\n", + "Upserted 785 vectors, total so far: 54392\n", + "Upserted 807 vectors, total so far: 55199\n", + "Upserted 792 vectors, total so far: 55991\n", + "Upserted 822 vectors, total so far: 56813\n", + "Upserted 785 vectors, total so far: 57598\n", + "Upserted 789 vectors, total so far: 58387\n", + "Upserted 807 vectors, total so far: 59194\n", + "Upserted 811 vectors, total so far: 60005\n", + "Upserted 801 vectors, total so far: 60806\n", + "Upserted 780 vectors, total so far: 61586\n", + "Upserted 806 vectors, total so far: 62392\n", + "Upserted 791 vectors, total so far: 63183\n", + "Upserted 787 vectors, total so far: 63970\n", + "Upserted 794 vectors, total so far: 64764\n" + ] + }, + { + "ename": "ConnectionError", + "evalue": "HTTPConnectionPool(host='148.113.54.173', port=8080): Max retries exceeded with url: /api/v1/index/test_1M_vaib_reupsertcheck_new1/vector/insert (Caused by NewConnectionError(\"HTTPConnection(host='148.113.54.173', port=8080): Failed to establish a new connection: [Errno 111] Connection refused\"))", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mConnectionRefusedError\u001b[39m Traceback (most recent call last)", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connection.py:204\u001b[39m, in \u001b[36mHTTPConnection._new_conn\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 203\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m204\u001b[39m sock = \u001b[43mconnection\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcreate_connection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 205\u001b[39m \u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_dns_host\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mport\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 206\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 207\u001b[39m \u001b[43m \u001b[49m\u001b[43msource_address\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msource_address\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 208\u001b[39m \u001b[43m \u001b[49m\u001b[43msocket_options\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msocket_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 209\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 210\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m socket.gaierror \u001b[38;5;28;01mas\u001b[39;00m e:\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/util/connection.py:85\u001b[39m, in \u001b[36mcreate_connection\u001b[39m\u001b[34m(address, timeout, source_address, socket_options)\u001b[39m\n\u001b[32m 84\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m85\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[32m 86\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m 87\u001b[39m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/util/connection.py:73\u001b[39m, in \u001b[36mcreate_connection\u001b[39m\u001b[34m(address, timeout, source_address, socket_options)\u001b[39m\n\u001b[32m 72\u001b[39m sock.bind(source_address)\n\u001b[32m---> \u001b[39m\u001b[32m73\u001b[39m \u001b[43msock\u001b[49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43msa\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 74\u001b[39m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n", + "\u001b[31mConnectionRefusedError\u001b[39m: [Errno 111] Connection refused", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[31mNewConnectionError\u001b[39m Traceback (most recent call last)", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connectionpool.py:787\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 786\u001b[39m \u001b[38;5;66;03m# Make the request on the HTTPConnection object\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m787\u001b[39m response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 788\u001b[39m \u001b[43m \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 789\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 790\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 791\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 792\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 793\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 794\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 795\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 796\u001b[39m \u001b[43m \u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 797\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 798\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 799\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 800\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 802\u001b[39m \u001b[38;5;66;03m# Everything went great!\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connectionpool.py:493\u001b[39m, in \u001b[36mHTTPConnectionPool._make_request\u001b[39m\u001b[34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 492\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m493\u001b[39m \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 494\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 495\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 496\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 497\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 498\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 499\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 500\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 501\u001b[39m \u001b[43m \u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[43m=\u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 502\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 504\u001b[39m \u001b[38;5;66;03m# We are swallowing BrokenPipeError (errno.EPIPE) since the server is\u001b[39;00m\n\u001b[32m 505\u001b[39m \u001b[38;5;66;03m# legitimately able to close the connection after sending a valid response.\u001b[39;00m\n\u001b[32m 506\u001b[39m \u001b[38;5;66;03m# With this behaviour, the received response is still readable.\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connection.py:500\u001b[39m, in \u001b[36mHTTPConnection.request\u001b[39m\u001b[34m(self, method, url, body, headers, chunked, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 499\u001b[39m \u001b[38;5;28mself\u001b[39m.putheader(header, value)\n\u001b[32m--> \u001b[39m\u001b[32m500\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mendheaders\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 502\u001b[39m \u001b[38;5;66;03m# If we're given a body we start sending that in chunks.\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/http/client.py:1298\u001b[39m, in \u001b[36mHTTPConnection.endheaders\u001b[39m\u001b[34m(self, message_body, encode_chunked)\u001b[39m\n\u001b[32m 1297\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m CannotSendHeader()\n\u001b[32m-> \u001b[39m\u001b[32m1298\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_send_output\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/http/client.py:1058\u001b[39m, in \u001b[36mHTTPConnection._send_output\u001b[39m\u001b[34m(self, message_body, encode_chunked)\u001b[39m\n\u001b[32m 1057\u001b[39m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m._buffer[:]\n\u001b[32m-> \u001b[39m\u001b[32m1058\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1060\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m message_body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 1061\u001b[39m \n\u001b[32m 1062\u001b[39m \u001b[38;5;66;03m# create a consistent interface to message_body\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/http/client.py:996\u001b[39m, in \u001b[36mHTTPConnection.send\u001b[39m\u001b[34m(self, data)\u001b[39m\n\u001b[32m 995\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.auto_open:\n\u001b[32m--> \u001b[39m\u001b[32m996\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 997\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connection.py:331\u001b[39m, in \u001b[36mHTTPConnection.connect\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 330\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mconnect\u001b[39m(\u001b[38;5;28mself\u001b[39m) -> \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m331\u001b[39m \u001b[38;5;28mself\u001b[39m.sock = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_new_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 332\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._tunnel_host:\n\u001b[32m 333\u001b[39m \u001b[38;5;66;03m# If we're tunneling it means we're connected to our proxy.\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connection.py:219\u001b[39m, in \u001b[36mHTTPConnection._new_conn\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 218\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m--> \u001b[39m\u001b[32m219\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m NewConnectionError(\n\u001b[32m 220\u001b[39m \u001b[38;5;28mself\u001b[39m, \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mFailed to establish a new connection: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m 221\u001b[39m ) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01me\u001b[39;00m\n\u001b[32m 223\u001b[39m sys.audit(\u001b[33m\"\u001b[39m\u001b[33mhttp.client.connect\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28mself\u001b[39m, \u001b[38;5;28mself\u001b[39m.host, \u001b[38;5;28mself\u001b[39m.port)\n", + "\u001b[31mNewConnectionError\u001b[39m: HTTPConnection(host='148.113.54.173', port=8080): Failed to establish a new connection: [Errno 111] Connection refused", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[31mMaxRetryError\u001b[39m Traceback (most recent call last)", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/requests/adapters.py:644\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 643\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m644\u001b[39m resp = \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 645\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 646\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 647\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 648\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 649\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 650\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 651\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 652\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 653\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 654\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 655\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 656\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 658\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connectionpool.py:871\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 868\u001b[39m log.warning(\n\u001b[32m 869\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mRetrying (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m) after connection broken by \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m\"\u001b[39m, retries, err, url\n\u001b[32m 870\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m871\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 872\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 873\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 874\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 875\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 876\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 877\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 878\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 879\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 880\u001b[39m \u001b[43m \u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 881\u001b[39m \u001b[43m \u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 882\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 883\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 884\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 885\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 886\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 887\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 889\u001b[39m \u001b[38;5;66;03m# Handle redirect?\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connectionpool.py:871\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 868\u001b[39m log.warning(\n\u001b[32m 869\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mRetrying (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m) after connection broken by \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m\"\u001b[39m, retries, err, url\n\u001b[32m 870\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m871\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 872\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 873\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 874\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 875\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 876\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 877\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 878\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 879\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 880\u001b[39m \u001b[43m \u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 881\u001b[39m \u001b[43m \u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 882\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 883\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 884\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 885\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 886\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 887\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 889\u001b[39m \u001b[38;5;66;03m# Handle redirect?\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connectionpool.py:871\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 868\u001b[39m log.warning(\n\u001b[32m 869\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mRetrying (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m) after connection broken by \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m\"\u001b[39m, retries, err, url\n\u001b[32m 870\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m871\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 872\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 873\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 874\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 875\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 876\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 877\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 878\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 879\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 880\u001b[39m \u001b[43m \u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 881\u001b[39m \u001b[43m \u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 882\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 883\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 884\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 885\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 886\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 887\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 889\u001b[39m \u001b[38;5;66;03m# Handle redirect?\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connectionpool.py:841\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 839\u001b[39m new_e = ProtocolError(\u001b[33m\"\u001b[39m\u001b[33mConnection aborted.\u001b[39m\u001b[33m\"\u001b[39m, new_e)\n\u001b[32m--> \u001b[39m\u001b[32m841\u001b[39m retries = \u001b[43mretries\u001b[49m\u001b[43m.\u001b[49m\u001b[43mincrement\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 842\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnew_e\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_pool\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[43m=\u001b[49m\u001b[43msys\u001b[49m\u001b[43m.\u001b[49m\u001b[43mexc_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[32;43m2\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[32m 843\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 844\u001b[39m retries.sleep()\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/util/retry.py:535\u001b[39m, in \u001b[36mRetry.increment\u001b[39m\u001b[34m(self, method, url, response, error, _pool, _stacktrace)\u001b[39m\n\u001b[32m 534\u001b[39m reason = error \u001b[38;5;129;01mor\u001b[39;00m ResponseError(cause)\n\u001b[32m--> \u001b[39m\u001b[32m535\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m MaxRetryError(_pool, url, reason) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mreason\u001b[39;00m \u001b[38;5;66;03m# type: ignore[arg-type]\u001b[39;00m\n\u001b[32m 537\u001b[39m log.debug(\u001b[33m\"\u001b[39m\u001b[33mIncremented Retry for (url=\u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m): \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m\"\u001b[39m, url, new_retry)\n", + "\u001b[31mMaxRetryError\u001b[39m: HTTPConnectionPool(host='148.113.54.173', port=8080): Max retries exceeded with url: /api/v1/index/test_1M_vaib_reupsertcheck_new1/vector/insert (Caused by NewConnectionError(\"HTTPConnection(host='148.113.54.173', port=8080): Failed to establish a new connection: [Errno 111] Connection refused\"))", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[31mConnectionError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[8]\u001b[39m\u001b[32m, line 52\u001b[39m\n\u001b[32m 42\u001b[39m vector_data = {\n\u001b[32m 43\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mid\u001b[39m\u001b[33m\"\u001b[39m: \u001b[38;5;28mstr\u001b[39m(row[\u001b[33m\"\u001b[39m\u001b[33mid\u001b[39m\u001b[33m\"\u001b[39m]),\n\u001b[32m 44\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mvector\u001b[39m\u001b[33m\"\u001b[39m: row[\u001b[33m\"\u001b[39m\u001b[33memb\u001b[39m\u001b[33m\"\u001b[39m],\n\u001b[32m (...)\u001b[39m\u001b[32m 48\u001b[39m }\n\u001b[32m 49\u001b[39m }\n\u001b[32m 50\u001b[39m batch_vectors.append(vector_data)\n\u001b[32m---> \u001b[39m\u001b[32m52\u001b[39m \u001b[43mindex\u001b[49m\u001b[43m.\u001b[49m\u001b[43mupsert\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch_vectors\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 53\u001b[39m total_inserted += \u001b[38;5;28mlen\u001b[39m(batch_vectors)\n\u001b[32m 54\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mUpserted \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(batch_vectors)\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m vectors, total so far: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtotal_inserted\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/endee/index.py:338\u001b[39m, in \u001b[36mIndex.upsert\u001b[39m\u001b[34m(self, input_array)\u001b[39m\n\u001b[32m 335\u001b[39m http_client = \u001b[38;5;28mself\u001b[39m._get_session_client()\n\u001b[32m 337\u001b[39m \u001b[38;5;66;03m# Sending the batch to the server\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m338\u001b[39m response = \u001b[43mhttp_client\u001b[49m\u001b[43m.\u001b[49m\u001b[43mpost\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 339\u001b[39m \u001b[43m \u001b[49m\u001b[33;43mf\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[33;43m/index/\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[33;43m/vector/insert\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 340\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 341\u001b[39m \u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m=\u001b[49m\u001b[43mserialized_data\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 342\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 344\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m response.status_code != \u001b[32m200\u001b[39m:\n\u001b[32m 345\u001b[39m raise_exception(response.status_code, response.text)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/requests/sessions.py:637\u001b[39m, in \u001b[36mSession.post\u001b[39m\u001b[34m(self, url, data, json, **kwargs)\u001b[39m\n\u001b[32m 626\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mpost\u001b[39m(\u001b[38;5;28mself\u001b[39m, url, data=\u001b[38;5;28;01mNone\u001b[39;00m, json=\u001b[38;5;28;01mNone\u001b[39;00m, **kwargs):\n\u001b[32m 627\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33mr\u001b[39m\u001b[33;03m\"\"\"Sends a POST request. Returns :class:`Response` object.\u001b[39;00m\n\u001b[32m 628\u001b[39m \n\u001b[32m 629\u001b[39m \u001b[33;03m :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 634\u001b[39m \u001b[33;03m :rtype: requests.Response\u001b[39;00m\n\u001b[32m 635\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m637\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mPOST\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[43m=\u001b[49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/requests/sessions.py:589\u001b[39m, in \u001b[36mSession.request\u001b[39m\u001b[34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[39m\n\u001b[32m 584\u001b[39m send_kwargs = {\n\u001b[32m 585\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mtimeout\u001b[39m\u001b[33m\"\u001b[39m: timeout,\n\u001b[32m 586\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mallow_redirects\u001b[39m\u001b[33m\"\u001b[39m: allow_redirects,\n\u001b[32m 587\u001b[39m }\n\u001b[32m 588\u001b[39m send_kwargs.update(settings)\n\u001b[32m--> \u001b[39m\u001b[32m589\u001b[39m resp = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 591\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/requests/sessions.py:703\u001b[39m, in \u001b[36mSession.send\u001b[39m\u001b[34m(self, request, **kwargs)\u001b[39m\n\u001b[32m 700\u001b[39m start = preferred_clock()\n\u001b[32m 702\u001b[39m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m703\u001b[39m r = \u001b[43madapter\u001b[49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 705\u001b[39m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[32m 706\u001b[39m elapsed = preferred_clock() - start\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/requests/adapters.py:677\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 673\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e.reason, _SSLError):\n\u001b[32m 674\u001b[39m \u001b[38;5;66;03m# This branch is for urllib3 v1.22 and later.\u001b[39;00m\n\u001b[32m 675\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m SSLError(e, request=request)\n\u001b[32m--> \u001b[39m\u001b[32m677\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request=request)\n\u001b[32m 679\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m ClosedPoolError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 680\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request=request)\n", + "\u001b[31mConnectionError\u001b[39m: HTTPConnectionPool(host='148.113.54.173', port=8080): Max retries exceeded with url: /api/v1/index/test_1M_vaib_reupsertcheck_new1/vector/insert (Caused by NewConnectionError(\"HTTPConnection(host='148.113.54.173', port=8080): Failed to establish a new connection: [Errno 111] Connection refused\"))" + ] + } + ], + "source": [ + "#For reinserting deleted vectors (int filter)\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "import time\n", + "\n", + "# User inputs\n", + "int_rate_insert = \"80p\" # change to 1p, 50p, 80p, 99p\n", + "batch_size = 1000\n", + "\n", + "# Map int_rate to range\n", + "range_map = {\n", + " \"1p\": [0, 9999],\n", + " \"50p\": [0, 499999],\n", + " \"60p\": [0, 599999],\n", + " \"70p\": [0, 699999],\n", + " \"80p\": [0, 799999],\n", + " \"99p\": [0, 989999],\n", + "}\n", + "filter_range = range_map[int_rate_insert]\n", + "low, high = filter_range\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "emb_path = os.path.join(dataset_path, \"shuffle_train.parquet\")\n", + "\n", + "# Process shuffle_train in batches to avoid RAM issues\n", + "emb_file = pq.ParquetFile(emb_path)\n", + "total_inserted = 0\n", + "\n", + "start = time.perf_counter()\n", + "for batch in emb_file.iter_batches(batch_size=batch_size):\n", + " batch_df = batch.to_pandas()\n", + "\n", + " # Filter by id range\n", + " batch_df = batch_df[(batch_df[\"id\"] >= low) & (batch_df[\"id\"] <= high)]\n", + "\n", + " if batch_df.empty:\n", + " continue\n", + "\n", + " batch_vectors = []\n", + " for _, row in batch_df.iterrows():\n", + " vector_data = {\n", + " \"id\": str(row[\"id\"]),\n", + " \"vector\": row[\"emb\"],\n", + " \"meta\": {\"id\": row[\"id\"]},\n", + " \"filter\": {\n", + " \"id\": row[\"id\"],\n", + " }\n", + " }\n", + " batch_vectors.append(vector_data)\n", + "\n", + " index.upsert(batch_vectors)\n", + " total_inserted += len(batch_vectors)\n", + " print(f\"Upserted {len(batch_vectors)} vectors, total so far: {total_inserted}\")\n", + "end = time.perf_counter()\n", + "print(f\"Done! Total inserted: {total_inserted} vectors with id range {filter_range}\")\n", + "print(\"Time taken:\", end-start)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index.describe()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv2", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/script.ipynb b/script.ipynb new file mode 100644 index 000000000..520dc89c0 --- /dev/null +++ b/script.ipynb @@ -0,0 +1,329 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip install pandas pyarrow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip install endee" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip show endee" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "base_path = \"/home/debian/latest_VDB/VectorDBBench/vectordataset_label\"\n", + "dataset_folder = \"cohere/cohere_medium_1m\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For checking parquet file contents\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "file_name = \"shuffle_train.parquet\" #input\n", + "\n", + "# Build full path\n", + "file_path = os.path.join(base_path, dataset_folder, file_name)\n", + "\n", + "\n", + "parquet_file = pq.ParquetFile(file_path)\n", + "\n", + "# Read only first batch of rows\n", + "first_batch = next(parquet_file.iter_batches(batch_size=5))\n", + "preview = first_batch.to_pandas()\n", + "\n", + "for col in preview.columns:\n", + " if preview[col].dtype == object and isinstance(preview[col].iloc[0], list):\n", + " preview[col] = preview[col].apply(lambda x: x[:5] if x is not None else x)\n", + "\n", + "print(preview)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from endee import Endee\n", + "client = Endee(token=\"localtest\")\n", + "client.set_base_url(\"http://148.113.54.173:8080/api/v1\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For checking indexes\n", + "for obj in client.list_indexes().get('indexes'):\n", + " print(obj['name'])\n", + " print(obj['total_elements'])\n", + " print('\\t')" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "#give the index name\n", + "index_name = \"test_1M_labelfilter_int16_latest_master\"\n", + "index = client.get_index(index_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For querying\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "#inputs\n", + "label_to_query = \"label_1p\"\n", + "top_k = 30\n", + "\n", + "# Mode 1: Single query id\n", + "# Mode 2: Find all query ids with recall < 1\n", + "mode = 2 # change to 1 or 2\n", + "query_id = 457 # only used in mode 1\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "test_parquet_path = os.path.join(dataset_path, \"test.parquet\")\n", + "gt_parquet_path = os.path.join(dataset_path, f\"neighbors_labels_{label_to_query}.parquet\")\n", + "\n", + "# Read parquet files\n", + "test_df = pq.ParquetFile(test_parquet_path).read().to_pandas()\n", + "gt_df = pq.ParquetFile(gt_parquet_path).read().to_pandas()\n", + "\n", + "def run_query(query_id):\n", + " query_row = test_df[test_df[\"id\"] == query_id]\n", + " if query_row.empty:\n", + " print(f\"Query ID {query_id} not found in test.parquet\")\n", + " return None\n", + "\n", + " query_vector = query_row[\"emb\"].values[0]\n", + "\n", + " gt_row = gt_df[gt_df[\"id\"] == query_id]\n", + " if gt_row.empty:\n", + " print(f\"Query ID {query_id} not found in ground truth file\")\n", + " return None\n", + "\n", + " ground_truth = gt_row[\"neighbors_id\"].values[0][:top_k]\n", + "\n", + " results = index.query(\n", + " vector=query_vector,\n", + " top_k=top_k,\n", + " filter=[{\"label\": {\"$eq\": label_to_query}}],\n", + " include_vectors=False,\n", + " )\n", + " returned_ids = [int(r[\"id\"]) for r in results]\n", + "\n", + " intersection = len(set(returned_ids) & set(ground_truth))\n", + " recall = intersection / len(ground_truth)\n", + "\n", + " return returned_ids, ground_truth, recall\n", + "\n", + "\n", + "if mode == 1:\n", + " result = run_query(query_id)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " print(f\"Query ID: {query_id}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\")\n", + "\n", + "elif mode == 2:\n", + " print(f\"Finding all query IDs with recall < 1.0 ...\\n\")\n", + " low_recall_ids = []\n", + " all_recalls = []\n", + " for query_id in test_df[\"id\"].values:\n", + " result = run_query(query_id)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " all_recalls.append(recall)\n", + " if recall < 1.0:\n", + " low_recall_ids.append(query_id)\n", + " print(f\"Query ID: {query_id}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\\n\")\n", + "\n", + " print(f\"Total query IDs with recall < 1.0: {len(low_recall_ids)}\")\n", + " print(f\"IDs: {low_recall_ids}\")\n", + " print(f\"Average Recall across all {len(all_recalls)} queries: {sum(all_recalls)/len(all_recalls):.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For deleting\n", + "# Delete by label\n", + "label_to_delete = \"label_1p\" #input\n", + "\n", + "result = index.delete_with_filter([{\"label\": {\"$eq\": label_to_delete}}])\n", + "print(f\"Deleted vectors with label: {label_to_delete}\")\n", + "print(result)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For reinserting delted vectors\n", + "import pyarrow.parquet as pq\n", + "import pandas as pd\n", + "import os\n", + "\n", + "# User inputs\n", + "label_to_insert = \"label_1p\"\n", + "batch_size = 1000\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "labels_path = os.path.join(dataset_path, \"scalar_labels.parquet\")\n", + "emb_path = os.path.join(dataset_path, \"shuffle_train.parquet\")\n", + "\n", + "# Read labels file fully (small file, just id + label)\n", + "labels_df = pq.ParquetFile(labels_path).read().to_pandas()\n", + "\n", + "# Filter only the label we need\n", + "filtered_labels = labels_df[labels_df[\"labels\"] == label_to_insert]\n", + "valid_ids = set(filtered_labels[\"id\"].values)\n", + "\n", + "print(f\"Total vectors to insert: {len(valid_ids)}\")\n", + "\n", + "# Process shuffle_train in batches to avoid RAM issues\n", + "emb_file = pq.ParquetFile(emb_path)\n", + "total_inserted = 0\n", + "\n", + "for batch in emb_file.iter_batches(batch_size=batch_size):\n", + " batch_df = batch.to_pandas()\n", + " \n", + " # Keep only rows whose id is in filtered labels\n", + " batch_df = batch_df[batch_df[\"id\"].isin(valid_ids)]\n", + " \n", + " if batch_df.empty:\n", + " continue\n", + " \n", + " # Merge to get labels\n", + " batch_df = batch_df.merge(filtered_labels[[\"id\", \"labels\"]], on=\"id\")\n", + " \n", + " batch_vectors = []\n", + " for _, row in batch_df.iterrows():\n", + " vector_data = {\n", + " \"id\": str(row[\"id\"]),\n", + " \"vector\": row[\"emb\"],\n", + " \"meta\": {\"id\": row[\"id\"]},\n", + " \"filter\": {\n", + " \"id\": row[\"id\"],\n", + " \"label\": row[\"labels\"]\n", + " }\n", + " }\n", + " batch_vectors.append(vector_data)\n", + " \n", + " index.upsert(batch_vectors)\n", + " total_inserted += len(batch_vectors)\n", + " print(f\"Upserted {len(batch_vectors)} vectors, total so far: {total_inserted}\")\n", + "\n", + "print(f\"Done! Total inserted: {total_inserted} vectors with label '{label_to_insert}'\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index.describe()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv2", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/vectordb_bench/backend/clients/endee/endee.py b/vectordb_bench/backend/clients/endee/endee.py index dae5d1ed3..6f7d3dea9 100644 --- a/vectordb_bench/backend/clients/endee/endee.py +++ b/vectordb_bench/backend/clients/endee/endee.py @@ -390,15 +390,34 @@ def optimize(self, data_size: int | None = None): # else: # msg = f"Not support Filter for Endee - {filters}" # raise ValueError(msg) + #--------------------------------------------------------------------------- + # def prepare_filter(self, filters: Filter): + # if filters.type == FilterOp.NonFilter: + # self.filter_expr = None + + # elif filters.type == FilterOp.NumGE: + # # Endee supports $range, NOT $gte + # # Dataset size = 1 million → finite upper bound + # self.filter_expr = [ + # {self._scalar_id_field: {"$range": [filters.int_value, 1_000_000]}} + # ] + + # elif filters.type == FilterOp.StrEqual: + # self.filter_expr = [ + # {self._scalar_label_field: {"$eq": filters.label_value}} + # ] + + # else: + # raise ValueError(f"Not support Filter for Endee - {filters}") + #----------------------------------------------------------------------- def prepare_filter(self, filters: Filter): if filters.type == FilterOp.NonFilter: self.filter_expr = None elif filters.type == FilterOp.NumGE: - # Endee supports $range, NOT $gte # Dataset size = 1 million → finite upper bound self.filter_expr = [ - {self._scalar_id_field: {"$range": [filters.int_value, 1_000_000]}} + {self._scalar_id_field: {"$gte": filters.int_value}} ] elif filters.type == FilterOp.StrEqual: @@ -410,6 +429,47 @@ def prepare_filter(self, filters: Filter): raise ValueError(f"Not support Filter for Endee - {filters}") + def insert_embeddings( + self, + embeddings: Iterable[list[float]], + metadata: list[int], + labels_data: list[str] | None = None, + **kwargs, + ) -> tuple[int, Exception]: + """ + Insert embeddings with filter metadata. + Modified to include filter fields like Milvus does. + """ + assert len(embeddings) == len(metadata) + insert_count = 0 + try: + batch_vectors = [] + for i in range(len(embeddings)): + vector_data = { + "id": str(metadata[i]), + "vector": embeddings[i], + "meta": {"id": metadata[i]}, # Store id in meta for reference + "filter": { + self._scalar_id_field: metadata[i] # Store id for numeric filtering + } + } + + # Add label field if using scalar labels + if self.with_scalar_labels and labels_data is not None: + vector_data["filter"][self._scalar_label_field] = labels_data[i] + + batch_vectors.append(vector_data) + + self.index.upsert(batch_vectors) + insert_count = len(batch_vectors) + + except Exception as e: + log.error(f"Failed to insert data: {e}") + return insert_count, e + + return (len(embeddings), None) + + # def insert_embeddings( # self, # embeddings: Iterable[list[float]], @@ -426,6 +486,8 @@ def prepare_filter(self, filters: Filter): # try: # batch_vectors = [] # for i in range(len(embeddings)): + # if labels_data is None or labels_data[i] != "label_1p": + # continue # vector_data = { # "id": str(metadata[i]), # "vector": embeddings[i], @@ -440,9 +502,10 @@ def prepare_filter(self, filters: Filter): # vector_data["filter"][self._scalar_label_field] = labels_data[i] # batch_vectors.append(vector_data) - - # self.index.upsert(batch_vectors) - # insert_count = len(batch_vectors) + + # if batch_vectors != []: + # self.index.upsert(batch_vectors) + # insert_count = len(batch_vectors) # except Exception as e: # log.error(f"Failed to insert data: {e}") @@ -458,35 +521,36 @@ def prepare_filter(self, filters: Filter): # labels_data: list[str] | None = None, # **kwargs, # ) -> tuple[int, Exception]: - # """ - # Insert embeddings with filter metadata. - # Modified to include filter fields like Milvus does. - # """ # assert len(embeddings) == len(metadata) # insert_count = 0 # try: # batch_vectors = [] # for i in range(len(embeddings)): - # if labels_data is None or labels_data[i] != "label_1p": + # if metadata[i] > 9999 or metadata[i] < 0: # continue # vector_data = { # "id": str(metadata[i]), # "vector": embeddings[i], - # "meta": {"id": metadata[i]}, # Store id in meta for reference + # "meta": {"id": metadata[i]}, # "filter": { - # self._scalar_id_field: metadata[i] # Store id for numeric filtering + # self._scalar_id_field: metadata[i] # } # } - # # Add label field if using scalar labels # if self.with_scalar_labels and labels_data is not None: # vector_data["filter"][self._scalar_label_field] = labels_data[i] # batch_vectors.append(vector_data) - + + # # # Log matched metadata IDs to file + # # with open("matched_metadata.txt", "a") as f: + # # for v in batch_vectors: + # # f.write(v["id"] + "\n") + # if batch_vectors != []: # self.index.upsert(batch_vectors) # insert_count = len(batch_vectors) + # # time.sleep(20) # except Exception as e: # log.error(f"Failed to insert data: {e}") @@ -494,51 +558,6 @@ def prepare_filter(self, filters: Filter): # return (len(embeddings), None) - - def insert_embeddings( - self, - embeddings: Iterable[list[float]], - metadata: list[int], - labels_data: list[str] | None = None, - **kwargs, - ) -> tuple[int, Exception]: - assert len(embeddings) == len(metadata) - insert_count = 0 - try: - batch_vectors = [] - for i in range(len(embeddings)): - if metadata[i] > 9999 or metadata[i] < 0: - continue - vector_data = { - "id": str(metadata[i]), - "vector": embeddings[i], - "meta": {"id": metadata[i]}, - "filter": { - self._scalar_id_field: metadata[i] - } - } - - if self.with_scalar_labels and labels_data is not None: - vector_data["filter"][self._scalar_label_field] = labels_data[i] - - batch_vectors.append(vector_data) - - # # Log matched metadata IDs to file - # with open("matched_metadata.txt", "a") as f: - # for v in batch_vectors: - # f.write(v["id"] + "\n") - - if batch_vectors != []: - self.index.upsert(batch_vectors) - insert_count = len(batch_vectors) - # time.sleep(20) - - except Exception as e: - log.error(f"Failed to insert data: {e}") - return insert_count, e - - return (len(embeddings), None) - def search_embedding( self, query: list[float], From ae4ef174ad6f291493cb36e6d75e171e0de11e66 Mon Sep 17 00:00:00 2001 From: darshan-endee Date: Fri, 27 Feb 2026 09:21:25 +0000 Subject: [PATCH 19/22] sh file added --- run.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 run.sh diff --git a/run.sh b/run.sh new file mode 100644 index 000000000..35e33f4e4 --- /dev/null +++ b/run.sh @@ -0,0 +1,23 @@ + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_reupsertcheck \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.50 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial \ No newline at end of file From ec69debb1ab6a5d2cd11daf8556221169ade0522 Mon Sep 17 00:00:00 2001 From: darshan-endee Date: Fri, 27 Feb 2026 11:00:50 +0000 Subject: [PATCH 20/22] final_intfilterscript added --- intfilterscript_final.ipynb | 337 ++++++++++++++++++++++++++++++++++++ 1 file changed, 337 insertions(+) create mode 100644 intfilterscript_final.ipynb diff --git a/intfilterscript_final.ipynb b/intfilterscript_final.ipynb new file mode 100644 index 000000000..cb399efe4 --- /dev/null +++ b/intfilterscript_final.ipynb @@ -0,0 +1,337 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip install pandas pyarrow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip install endee" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip show endee" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "base_path = \"/home/debian/latest_VDB/VectorDBBench/vectordataset\"\n", + "dataset_folder = \"cohere/cohere_medium_1m\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For checking parquet file contents\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "file_name = \"shuffle_train.parquet\" #input\n", + "\n", + "# Build full path\n", + "file_path = os.path.join(base_path, dataset_folder, file_name)\n", + "\n", + "\n", + "parquet_file = pq.ParquetFile(file_path)\n", + "\n", + "# Read only first batch of rows\n", + "first_batch = next(parquet_file.iter_batches(batch_size=5))\n", + "preview = first_batch.to_pandas()\n", + "\n", + "for col in preview.columns:\n", + " if preview[col].dtype == object and isinstance(preview[col].iloc[0], list):\n", + " preview[col] = preview[col].apply(lambda x: x[:5] if x is not None else x)\n", + "\n", + "print(preview)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from endee import Endee\n", + "client = Endee(token=\"localtest\")\n", + "client.set_base_url(\"http://148.113.54.173:8080/api/v1\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For checking indexes\n", + "for obj in client.list_indexes().get('indexes'):\n", + " print(obj['name'])\n", + " print(obj['total_elements'])\n", + " print('\\t')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#give the index name\n", + "index_name = \"test_1M_vaib_reupsertcheck_new1\"\n", + "index = client.get_index(index_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For querying (int filter)- gte filter\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "#inputs\n", + "int_rate_query = \"99p\" # change to 1p, 50p, 80p, 99p\n", + "top_k = 30\n", + "\n", + "# Mode 1: Single query id\n", + "# Mode 2: Find all query ids with recall < 1\n", + "mode = 2\n", + "query_id = 457 # only used in mode 1\n", + "\n", + "# Map int_rate_query to filter range\n", + "range_map = {\n", + " \"1p\": [10000, 1000000],\n", + " \"50p\": [500000, 1000000],\n", + " \"80p\": [800000, 1000000],\n", + " \"99p\": [990000, 1000000],\n", + "}\n", + "filter_range = range_map[int_rate_query]\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "test_parquet_path = os.path.join(dataset_path, \"test.parquet\")\n", + "gt_parquet_path = os.path.join(dataset_path, f\"neighbors_int_{int_rate_query}.parquet\")\n", + "\n", + "# Read parquet files\n", + "test_df = pq.ParquetFile(test_parquet_path).read().to_pandas()\n", + "gt_df = pq.ParquetFile(gt_parquet_path).read().to_pandas()\n", + "\n", + "def run_query(query_id):\n", + " query_row = test_df[test_df[\"id\"] == query_id]\n", + " if query_row.empty:\n", + " print(f\"Query ID {query_id} not found in test.parquet\")\n", + " return None\n", + "\n", + " query_vector = query_row[\"emb\"].values[0]\n", + "\n", + " gt_row = gt_df[gt_df[\"id\"] == query_id]\n", + " if gt_row.empty:\n", + " print(f\"Query ID {query_id} not found in ground truth file\")\n", + " return None\n", + "\n", + " ground_truth = gt_row[\"neighbors_id\"].values[0][:top_k]\n", + "\n", + " results = index.query(\n", + " vector=query_vector,\n", + " top_k=top_k,\n", + " filter=[{\"id\": {\"$gte\": filter_range[0]}}],\n", + " include_vectors=False,\n", + " )\n", + " returned_ids = [int(r[\"id\"]) for r in results]\n", + "\n", + " intersection = len(set(returned_ids) & set(ground_truth))\n", + " recall = intersection / len(ground_truth)\n", + "\n", + " return returned_ids, ground_truth, recall\n", + "\n", + "\n", + "if mode == 1:\n", + " result = run_query(query_id)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " print(f\"Query ID: {query_id}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\")\n", + "\n", + "elif mode == 2:\n", + " print(f\"Finding all query IDs with recall < 1.0 ...\\n\")\n", + " low_recall_ids = []\n", + " all_recalls = []\n", + "\n", + " for qid in test_df[\"id\"].values:\n", + " result = run_query(qid)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " all_recalls.append(recall)\n", + " if recall < 1.0:\n", + " low_recall_ids.append(qid)\n", + " print(f\"Query ID: {qid}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\\n\")\n", + "\n", + " print(f\"Total query IDs with recall < 1.0: {len(low_recall_ids)}\")\n", + " print(f\"IDs: {low_recall_ids}\")\n", + " print(f\"Average Recall across all {len(all_recalls)} queries: {sum(all_recalls)/len(all_recalls):.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For deleting (int filter) - lt\n", + "int_rate_delete = \"99p\" # change to 1p, 50p, 80p, 99p\n", + "\n", + "# Map int_rate to lower bound (gte value)\n", + "range_map = {\n", + " \"1p\": 10000,\n", + " \"50p\": 500000,\n", + " \"80p\": 800000,\n", + " \"99p\": 990000,\n", + "}\n", + "gte_value = range_map[int_rate_delete]\n", + "\n", + "result = index.delete_with_filter([{\"id\": {\"$lt\": gte_value}}])\n", + "print(f\"Deleted vectors with id < {gte_value}\")\n", + "print(result)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For reinserting deleted vectors (int filter)\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "import time\n", + "\n", + "# User inputs\n", + "int_rate_insert = \"99p\" # change to 1p, 50p, 80p, 99p\n", + "batch_size = 1000\n", + "\n", + "# Map int_rate to range\n", + "range_map = {\n", + " \"1p\": [0, 9999],\n", + " \"50p\": [0, 499999],\n", + " \"80p\": [0, 799999],\n", + " \"99p\": [0, 989999],\n", + "}\n", + "filter_range = range_map[int_rate_insert]\n", + "low, high = filter_range\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "emb_path = os.path.join(dataset_path, \"shuffle_train.parquet\")\n", + "\n", + "# Process shuffle_train in batches to avoid RAM issues\n", + "emb_file = pq.ParquetFile(emb_path)\n", + "total_inserted = 0\n", + "\n", + "start = time.perf_counter()\n", + "for batch in emb_file.iter_batches(batch_size=batch_size):\n", + " batch_df = batch.to_pandas()\n", + "\n", + " # Filter by id range\n", + " batch_df = batch_df[(batch_df[\"id\"] >= low) & (batch_df[\"id\"] <= high)]\n", + "\n", + " if batch_df.empty:\n", + " continue\n", + "\n", + " batch_vectors = []\n", + " for _, row in batch_df.iterrows():\n", + " vector_data = {\n", + " \"id\": str(row[\"id\"]),\n", + " \"vector\": row[\"emb\"],\n", + " \"meta\": {\"id\": row[\"id\"]},\n", + " \"filter\": {\n", + " \"id\": row[\"id\"],\n", + " }\n", + " }\n", + " batch_vectors.append(vector_data)\n", + "\n", + " index.upsert(batch_vectors)\n", + " total_inserted += len(batch_vectors)\n", + " print(f\"Upserted {len(batch_vectors)} vectors, total so far: {total_inserted}\")\n", + "end = time.perf_counter()\n", + "print(f\"Done! Total inserted: {total_inserted} vectors with id range {filter_range}\")\n", + "print(\"Time taken:\", end-start)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index.describe()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 26d939ec60225d2b78ef6998530e2901cf5dc6d2 Mon Sep 17 00:00:00 2001 From: darshan-endee Date: Fri, 27 Feb 2026 11:31:26 +0000 Subject: [PATCH 21/22] 0.13b1 version added --- endee-0.1.13b1-py3-none-any.whl | Bin 0 -> 28543 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 endee-0.1.13b1-py3-none-any.whl diff --git a/endee-0.1.13b1-py3-none-any.whl b/endee-0.1.13b1-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..82625a03a05d77eebf173f89c54ab8424497913a GIT binary patch literal 28543 zcmYhhV~j3Lu&zC}%{8`d+qUgzY}>YN+qP|cW{vGN-+R83?0u51AN{AQyHiQuxvx@` z0R=+?0s?{pvQ**Jm14F%Wc+W({a+ye7iIucGcyJQ11o@)i-7^XgQpD5)V$h)%-lQ; zz0?d1mHNaC-2?-*5)`Gh+#K!P%>1!5D2?Q-jFeocR96tEE|B8?q~FyN(S1_r!8`;8 z0s?>m0sWt%|2Msfy`6)TnX|K%J>b6zsmgrz+l)xNA2bMAv;%A)E-Z?mVmYOPDg|=g z5GfvM49HS}B{>^H|GeW%$fhG%M1w_@-S74wql=K%)!h+#DQphx3(`Oe!(&|?OwfqG zhq>>~>4IxYBsu`la!5x2nLjORzR?R@-6W%SbZp!AArJ5`U>yRWdt{P@Op>Dwut}9>?rE{bIBeHnFR}`z!)8pV!Ve$Y?Scv-2NE*j}cT zj5%wQ;W+U`&MDAe`fF$41&9e3--xpECph;fTG#a>Xh5sB-0rWN(^2V#mQ-U%(CCWh zjOQZZUB7;Bnnm0_t%(Q*3rM#pUX|sqX?_hB#GDfZ?CRAV#nrSEj-)+tB5Lo~wT5!W zQ6z9qdGlBy??Yrd#~$IRTrq2f>CeP#!{#y^1Nk%HedK-QI5-2P89F;-nbRsso%dT+ zb4^J^yiG(EldmgdUWdxQ*vpeo=5L0aP3dDZS~kvVPqs$X@Bg+0mV&ZwP3}u`5EKy5 z5DE~`f9ro*0^sao1aNWwuOV_Zx1F{pkbTEBWSJP0L%R|lxucXl0|Y08v57mtE?PNS zLvcVvoTdl|MuA#R%YVYZjrvR#<`2XtH*LkJqaSrYy*^#y#fuY9(K@R<^r>D|OI*n} z$|y6l%Y_>di7W5(vFqQnXa;j+qk5WZEGf{pvzIG3CUhysnq$avE3_k1)ViB0+PjPD z?yD_L$+aP-Pr6P{pEi;_lU6Hd+FiU!+Fd7Uv`$FV*R@K=WLLvQl+;P@KLoQLQ}8{> zRh$`#LWk?#l?d|CMW$iV3XOei$s*TQ(+tSgvA(6?P>YtdtdAeT> zB3@fQzQ)Pz=`S?OsShAp@}r?#@^ZO>9&#)TaG%W**{neir)@+zb@)c_Zfj(73xHB= z2y9x`?lKnI*OZvmdOo@}__5rq*_WB`ZU{{;FneaZRkz<9(XG*5G0bd6o-~7y^WWEC zTu;jrgGy1wxbv2uMAoN0U2xwGuc87BWH0ti(C9K)M9sQ8Fjwm&*)F0PfM83P^*w6P z`qzZjrixS^l5zPUni<=C1>Psgs{gThor@PU;nD>y_|!d*&%EqTtirmT@W%OFt`z50 zX&169!&Ag5Q=eimnf25~@zqEdpxYEF;fTAnz?e{EbP`$lu>5>w{5bf^36wn_hxzp^ z;$WC5p-UwqZ*V-)uqKdHm&PB}IY-5%VUIl*?A1gioVr2Z2^{6?`Q+h?3}(eviUs>*8ymM(+WgbD#onx6vC^bL@Ec@hjRn zdD-wl1&@CJ8Av7|UoLcLj6^RPY5-;60)mlCnH~3hM+K!G#Nd?*I6l#eb|yGlXic9x zmU1Yw4`XIYw6D&^D3%F1MS|1ublY(BAdxP(tFIZ;uS^1=&$Zdy^Eva2Ie$&Y{kr|u zV_kVTQ$Kezzrg1QL@G&%sgv@8-Vrn%VRPLGHoNQ-g@a-H4jC3x{EetUr|AUpjd(=x)xcR>Qjb9 zfFFx@Fygs=3o^sQV<8ZAHXdS7m$2-!`ljVnyiq(-mv$N1oCAmGa_OUK4%<(=ji=#= z8QOTeO}`*H(ew5;Wjl({>2qu9lMKk9H;|*JIB6i~BZ>3#iJ^G^GqJqx`078E?WeKz zoLXb05Q(U4N|mibq+~<=>K)Xt&$?PFL9sB~kgr?Vf&c}4yFtU6TKd+Ek8Dh!=Y2*m z6|YxJmO2~4+~o4W=TTRmO(gXP{&DT@qGv2+J#qEPtbMn!x~~_^`*>mUMmZ9zTs#eP zEI$7>eXnLeje3u-UM7iT+Fm*Hk_NeiSk2CZ^#wSdD{v0gP2Zl{0jstp+8U*hSb@j~LRsh)x7; zzrsx~1fA{!i7PTWCliaRwP`wtR#_C#&+%5`S(15{;1wOX6tG4*E0udGu$8C*Oy|m4 zE>5^%=-zyt`NN%#_zjp`LDk4V#Z2}7L0*^^R}&eUMB;vz;?i=9i0nEq0k=YSg+Pr& zgvpL&p}glBSXEgaHLm~;Gp2@SYU;dP(Rd28!kOWq9k!QWCcnU_^vnb+a((i{W+cdo zLft93&XiFl6C)8(*_o)h!T3Cu?F2^G=ns-AKBN!(sB1y(1b1bY| z8+fW}x9@h@YJrKs3+UMHKw@i;q?r{+u&6onZ;cDzDVE+OE1jp#hM<&Qknh(;CEXVZR@$sre@y zy{N=kMCuZ(@_=#(W9^H|j4ao1_V22?Gg`ecpIg#oR22tZ-$^-$HVKmhmsd=ttbKSH z4#Lg3rkfjlIafl4v5+pB=NjMhScJJZ5!L)& zz{EUKE8CKAF&YAri!X%fV8!fXfA)#9=X_uxg2nVSm*uM~kH77*;>Y$wGO9IFimk!@ z-NZ6?89SDLeEHt*tm-IW8{q?E(JC3ElZnuORz+}m%&$-opI}vKCJI$5kXP6o)B>?3 zd4c;;!|9HH$$G`frtn^kk1ifB z!4rV)%icOZZCDO3X!g?!=BkE6Cj_B%nH*gOqKYZPUHIxNcm6%PUM|I5{lq?MS^jb5 z%>(ZZ`P^_Q_>PD`!VA=RDqCY+d-oPv2{}NG)fG051`ulDz9Bx1=kOzOLy5fM?%#p( zW?crJc05h^om;#7BK)sn!Te9LbWJxi^ZX}O+~t6P5dKH8{Qpw+KeckhYvX(@?zI0w zL;f6ITGz31OkJ6w<{(D1sK0uNp4gPzp-YK`6~!3{6a(CHWcS)%?)_(=M^LGIZmbSW zhbB=_K|!Iv+oIsmfivS2y7Yur#!5eV+C@uj)s0cJMn#-mY{@hwMeQMlJSEXfZjxNu z83p!&wce^V%2--mjI2-0G4~AdL`1uui*uacqsS2+NlH~>Iznm4@+D==MVKB*Du*p1 zS7~gCI2`-&A`oC0#KbFm!9pc1s5{KUjtv)hBGg3{T}2L#TT^8$qArK<9vy@HL0Fx| z7*&tJ0$>QB;&>Y<-puUq^>lXg!tUb%58K_> z`}=llF3>WQJ%2VKg<0+8;O*>unPkix+V~Xxn-fXVjvlL`+VyZCxDNh_x122CtQu5vm6eo4YqO?g%OwxIX=LUb6!*3(Hjs>Zl~2=^pZ)_0TCUKkHF*u^+?4kBf&h! zL(PZ54xV@Z6j~gb2xpshROWArI4(1G!Y+5!E1fPYlf8#zAmzKZG&cqFyEK%p%brEq zP^ozaPu;Ov0Y98G7B;_+-nU&m45hCR`;EEqFhXm5)8e~PlSrPB zd!7*ti5jTB;M!%yP4b31ZEn1X3=$z#Mdo6v;H43ya^Si;0+n{PiHR@Hp{-0X6duTj z$%2RT`z#%V;|B1Cql1sBtSNAV*h;CcyQl=Y6-?C>u`aVMWqisLi~bbNnT`F=HtD5SYO^ zI8Y2oUV#JUtri^NcyemlRDX*IYmmY%^VSYKtPq>C>co1dx8^#Jvq{o@mKEvS4rlU^&Q>Lh`XbmI52-Eks;1I_40g3h z6R}Pjt1BsoX}L%Il)q_K3O zFY2mtnwCdy1-Ir`yd0R=ZfWf;+@J9gA=pGJYHI{VW!Z_3pCQ-}hn^qa&kdIfG@$%u zG&+gFh+iH!50h`m>iLyRA|UYOUy$IaK`aT|ohAjr;so0x8U~uD7Jovj7kECqI9Q&+ zY&hA_qzv2?N2p#~{2fS6#~#5vK$wUv%8SaGVc4NlipV?P@Es(r8}eX2DuUBbDi;on z|4Bc5l$pwto~|K03Cj-bH*6tS(O^34wSenj0_^4j@u&b?#2Bdx3KFg>XfNouoTxxp zO>2{id3%s@3wp17+_HH>$3Ce0pz4E-9sIsTr-1IEx3wW4&s{t4U`^k1)&r6*Dofc$ z@942AhHn;lgRl#IpY$OCB}R|dFt~V<|Knh31`jL5VUiW-2FaJ2<}0S9?o%Mu5ooAH zN_)ZZ{Crgq?!6-gnFh&WL?@?I)=ibw^Cxq(L^&`RQ+&o(q=u<$2ofe7QhWpc8;{dZ zc6{-pnUkh~o`pAS-#eG+08sC$V^mDE4O0RaYH|XffdBZQok%(xu0)5rVAS!J>jC}9 z71UGF$C+)W?*H ziV=&A)JB>%16(5=TsHg|F`i;P=k$9T8X~SMf(d;j`(GDoANuTY7ReGwY?YXMO40>B zZ)%A&LMH(eY-9&@#sY3lou)z{gq4)pUkqB`A0HfSF0{tKpqH{|f)2OU3bhWa>DL0Hv1PX7tE$p>n>EQ$IjNJd; zf*t-nE%S%lW*{3b*Au;||GVW!-;iYn{LQ<#Am1!sEv-ro()#Dj#3(Va(X09$?ZLu2 zq((rsl^XO!gGPP!qpwFD{oSu!S*_rTT1^j7U!yD*hkT_4qiS#8Mv14ImzGEhH*L*j6eUF;jJf+ zbmJz(ElnT+>@3E9hO$VB7}nLjw>T^W2$O$RUYK>ZSen#v|v->yCux{dN3(vVVOB$_oxe{TGZ zm5N20Nk0reM1Dd~ASE1$to~LMI9y1#llZpK%nRbp0vy7$PI89FlcapOYznQ&ed6P* zTV`%_c^h_Tiw@Qhke)9nHEv_CHBOk&vl=w`+PBFMyhe zF`zpYOR_j@U3AZcd3kdavX#j%@tr3nCwE?Q@V6%=U+@s%peMd4Aa6GS5R9^1kdY&4 zDvPS7QcoW6Wgk$|R55?sl{1t+ZrTBaK>casI*({(@aW*4DRwXuzt|&Tv`Mj8>v-cK z=m>3yJ5bo1&=dHM$6dHkw@{TjZ0ImfQdg1-s_0U~c56awB-cEBBGyq~5+#1>YGR}9 zCfT~&XxI=b&j}_=k!1R=sD&}Oc7!dXKBTj}j0kZVgtbPHYmvzS@co4CiXp>MilJ{E z5BO6t!lD$cpvSo@L%@XE*6Je?3{W15>xhD8X6cJ1Cw|r^s3vn(e2Wp}36B%AtEPRu zDVr#J)8*lzgxx&@MZaIx1Ss{Wk!o4E!nlY4{pj-Qf}q3mRe7c3@%y^MaNwlk=_`nn zEx)2LJRJ6w?OKn7YS00ZxzQMXc1F1RIP!WnQl1R#$U_&<{w>(2+uk34^QNclTnVea zDqFFuE4>I%tDY2Q8vuW=C^9u#VC!4&xZC8z;>Hf_AnvF6xPkW)XVweTAeX(l^B`q7 zc=aj_;|UeVMV*GEluq4Zh$q*ZhRu`3YybL`LZ9g_g+ZTn;-h;xx&FZfjdec%MVg5( zm%waBD)7@VT~@POI#bWOuKRXP|0I#Y+c_O?M2Eb^2842qCEe8ILQ(6qeV@a+Hg=oj zSA)Yv`)7ThS`DUV+E>QK75|3+@-$WZmSBCp+yQ=R*$Qk;B&xk#)J&xvwy%D-xSBBk zWrop#jhm!PX9k48V%lHH(srypEzmm#q-Zn~Wj#D1(wFY-WM>CtUg(sM7|G%2)jHF? zY1a%Zm{Q6SM+Na;Vta5^8BVk}qJFj}n~XlQ0Ekt-WPbqmP`u947SH~kRRzasFzJ+T zwe-n})9Sjfde1eTjC);oc2TX3*+=C)XZUu`i_K;EX9YHhePvir_(FSTss!%T`o^yG z68vnaOemYnn-=ycYfXD8-+{E9h!(~DLlI3(GJAlZEkV6~p=ChPM6GHCpXK)r8Um3$~b6Cq#Kup&E`e z#qz*#)F^mc1uWDq@9@ynu@JlaGi*ub*+EsBPEfWF=K6Va{_xduwU!50_dlZteS9S* z8f>`)fvMNjE>Qj6y(DD$NGKiscjU_?0fmQ~n2aR&GIl2H-bh$VV=8jI7GR0gfTs}# zD`){Ld<9wE0@}O!a{*)|M8>kSOJ8?=&&)o#C27uG%HR`ne>5>~ebP04F8#Y@$ers{ zSI@4&F7z*1rZgy-K8A4dfrKqN;1A2;0iAA-ExJSXV0fg`ksODAe$uNG1EsY=1fL;n zq&i4k4&)F_<`aPrfIwYhaHilXcgBh-fpP_-+4q35Mc9wTYXmr@FVXwDG zI881JGCOTzPh$`LSnhrOJzSyQ-3<;2{dstn-H&T%V+U(Tt$ly6(MuIEH>;Bpb@|S` zB?m@P7V4CX^@-ktyAvOMIk)r)D z72aynk&@I>Ved zTaY`Z9kNN0+i+uX1>LI`RFc@X-?OEyA0j_tF&()I$w+S`?>U}`dWZy&d|Vu_T*)ol zWTO2+hls}V`RBl<&7UYRbxV+oS}3XasTuX{=b;nwl&_TTsU4Ys<60f)>bQq7Ikkr= zp123fCmAt%c@Fdm!gZ9*CGozNNyH7qku!-jx3HOc=zuI5cD+Fvzs+-4HywhyM+Hl7 zplh|4dMo=M*6%`FSuLRjp#q+TG>Te95I2bQg_t(8-V5~>HJ|vK+$W+0-D|yWeUnvc zZ^fYr!Z&o5-`mS8nYIlO1I*S{T7#XOeZfoDw8e@g%Tv0U>2B`_?IMp;1*E(ErSGV; zALNg(OGU*e3RXv<_usD6JJMrBvx}t|4dyJE4;s5?@Kt}CU+b}oZ7<$|(UVd8#grZ% z?tg8r5ED)79w66GG#mv8Bag)mEki84TZ<$=@xB)rh>b6i680pIZP^eWj8W>l`N#uL zD4mUZJc|uXlqq7VQF;lPwR}MI2wlf3qoz z2LF!GSepC+SP33BaVp=)uqo08t?e1$r<^Hb&b~aHUwHAJ(|@ z8^!DFMnNWrpqDAjPWZq>O1+o7du_3 zjqY~G`I9=wuQ#Xrl6nX7j$9hm{P3R@k8M}-{WRK-%qzDiY>+e$6%`)0*{2PFhUUR_ zO;}=hh4#>vp<l2l1F|Yr`eO zA%!`P`?@co?B1o(U1wA*xx^96WF&qSN_Yjm(fDqm6dHdQOT|#*bD3(Dewq!qz3JR&jYBivQhV zHi}^n)~pu&?A;CG!EhdR!8;GD#Smty%wgnpqwGim07r`kUO)Z=3;{fPD%kNlT*)(gU{9Ep#hPq=n+e?* z3Z}is2Dca8+qtkC-U~^6|3KrPk}WvpmFYiPkSO1ZDg%ezNFJf2we(sHzGrnZasA1o z3rUQdg{yJJ0^72bwKk;a71lUx3)*(2!x;QM5uc;=3Evpq34|=KjLVmyox|pu&28AD z{-zCW^`X}^&Be}b+K!Q}U}uNoXE}f}et@s}04FYstCaK{}># z=sxDGIZ%R@U}?0fQ^2{P8@x!4XQZsIgrm@%+c54*uC&|b`*=B!kB8rQfvPYd=YKxb zmAGHU>&a2?lUaIHGuDV{s5!(8cvlvDZ4}hX8;aBpf*zv@veAn^# zbGR9x<~MVjQ2t4=MjX7@M0!~y;x_rknEu6$12O7@c_N;P%Xx^7-moWu$1#o`rC#Gw z2QPwOg&b9>HyNtKfg|J_30huXH~$+iA0Ib%{d|Gt?=wI^Q7o52X=wePXq^`W9TfvU zjqZQQJW47V0%d7C+aS_hbb8nusWk3E2tbSbOxbo7-~qH|PLU#8O9bt}Hm9r+TO%dC zEOiEwab=7aP*dOdaD}8NzyML_8EPx%7^pfy1`td=v6ykW?A6YEE^fHUjPqQ0ZAfH|Z|x6R?R;M7dT_#{?lM2nzSkr~Q#1_I2y@S-kkg1M% zMBN~>8=_2&vitd({%0{FdZljDC{#9et0;CuK{?chfb60-+zpgmdU)OWm~IJiBB@D- z{K$S3?b59zpGaf|eVBy>?axZ8bwOhohl}7u;AdB_O)f%50}+u#csGE$v0p{u#lA+p zmZ5>!Wh5Snt##?o>3@&TKS?!qUnR*$ba!MXtbw9dp(gMpBWdD%_a3#uYy?`OXCNId zD>O{w)hHuytB$mh%w`GOLdb~&x_14ut$V1>o|_ADg6o=Y&BXgBz`zN2$tVYwgE_W; z9JhFKTa6tTFnTJzbU5-9KpXVDAG4rdD13THbeJH*gWma2)Lr|L0%5?k1EAYVyB1p# z>%j3;R}h8d?_)z~3Pb9sQe~jkTtMaGwOkAed?*#;BzOB(T`J`gcfevxRV~R{A$@E@ z2u+Wx^jKoX8w*%QZ)43_6jO^|$fCLU>Ua(pID-2u@B9TSBw(Gy_5}Dgj?)x1c4YKj zLWH#Wz?;OdMp$}`Vlz?e6=!Q%8ul@3MZ3j|NzDoKgpLB-utl`x;2mPHK{oTCG4?)Z zShiUhM8m)!jB|9Fp#&6tTsa$`wPso=7*7x4KJgVnB^Ps`yR+Hyg}46wToD+Ijm`{T zm5_-jbDFYHEuCNr5Nj1qcB1=*9@5meW?Zlx-QsVu*A0bYexryXJ<|JA4$84TC%?yt z$@H142x(I3mEdKAdazZMhHt&T6pmr*@4HLl z6yYvI`*o$4gxMZ9VTEw>$%&LL9)oAfo9rf38sKOLp^{eC8R&Ufx1`3Uf`!yRT4%1CW(b$x6L~Jcs`jR*zMJaI_WpxzA{y ziq0tUlM>!bS@x5qX|RZk}PQS5MTgEm~-Fwi?wj1g!mN7MYh^q|`2xZXBe8I!K3eiq_AD@p41U2Mm|APLn1i<-^G^wN#bCqEM z0kInVzog0PznLtL|9<~b-faM`l-u2B6bwb+A#yBfeQ2wxSW+(!5*Q#^ zFU3pN&ABVPjWsBM7kt9z5%#0$BPmB_w_8$rZxaq!$MmIArQKPeyleT=7XLrr5tYKM zEk}xpj>ZuTUfCxh8Nv^*OpaULh6=&FlQyfic6z%9!#PY}^|YKfxz!R-YRH?crM(_v zPDJz0G9Qim5b2n@kizb%wL;85-fRXj-%t&4WOPNY zmhp|^)JJKuD~yQE4~&2=+Tc-aX#X%fkp3?@ouL*?TQ}K(^ZnVy!93=j0?NOa+n>wR ziHv3yE_`iSqO>f9}V? zxw&|>oxy(}hYy$kwvVR?B+~8Y;_3H#U?IDF++QtLoqU{Jyd1x8S90ecMlWXL1#$=o zx?(B5{PFR7c=-6ZJHYOv?-wC0{oVZnpKyN~{43WTy8N9^f`(WTfK>V+y9p<^vd!@9rg=&ipr` zjx&xV=Q=t%Gmu~c1(k%Qhx>?Oae+Ds3fAwDDd8zHvH=1oKC~<+hy_aeE#_?DkqsQp z@n%ijj@b2c=O47Sk7xQT%(aGeZ1!-Cnd{R=oZ)8Y*0sKli{@=NO&PnWT@()Z2`{Yu zS~zpXeqZo#ociodXL3UdiF2hFYjUf&l$k+vXO1doS5|H*zkrsb>*-f333G6YEKQm)?G4A zkCWJk**`g^suR+$&AKSlM~lxNH`xjv1twt2B&ezQ3JWrknnyawDr}rY;ssta>$qhU zKW9ktc2Va?+(t`jorn4)MqoDceG3@$TwsG~@rOS-wH+qJDd1;$X3YhJ$xH9SpXN4*d=uPnCDB6jWXcr13=WS#A%ny8KaecJ-Hh=mpf|pbywMn=y`d?JgAEm! z6raHBXkRKOTBT8n$P)}(>&Qd-6G=ilw;7=LN2m)3(n#slip28+MusB?I?iF6HM*PM zIla`Ieb#=K{;wmZq28dv6X%fAzm_yLZHNqq;@sLBF9oafIM{}X0jSlKY6%B5e49wKSam>oY>=en zyrPvz4&kPr-lu;EO2_%z4U^C}t}-vy$y*rNlh;TZ6N_=qBGI*9I3E1m3im8^Bw_z`S`ZV6nj*HVUdxHo zO485KhDBSlfJPe-=PP{D4rs*;&LNxQCZKvh5b|I5*iieHc4C7w0|{33s_vc^tkRw&-eaf23YB z!yDR9!a#>KJqsR`#KOPWLn_<}_OE!8LiD^brM#YStF|a)DkR)FFK{ElKG)ChJ0;cY zvrIIW3-WPRNx0>;288b4%Zk^9Zi}&CzOCM|h(0W5ognsGTwOXq>I=jnsxAts4+qCS z*JMX!wj;qD-zKU);MLhh{v?~zwY#V20f=q<_1MD6>h<|^=DUi(A=MW?is3v|flq_A zd~yolp`uXRzT??5R0&kDfMIboIl=t3NPEq48&fL_a@cfBuUi!DwKGK*EE@_3_XLFu zq^tbuGFX2z{Q$?En-c-D4~6aeM|tC8N-x^|*#P;cXZm~*hN1Kbz0O`(;dWhi5$nrC z<9TYvyn(HWM+MuexVuq?ChFpz2d4?=ZX8k&-`*>>u4OGS!L?mue3DR*m4K>B8)0i{ zG|b&R5{>a`pERnSV#h66fP^5yliD9RI z3zj!Lgk~LU`2T><>6HW124@=B>4M0+)K#lfDd#443=1I3Qq!W&kX~P<0*|9@S`G;S zafyN2ZEFaHrts{C4DCs>-fiGcc1E@fguEl+4kCG_6fLt4q7yeMJZh$> z!#WV0I&+vWP=^6MTnPK+?4S=aeGBt)6?y_9GGcte^=mhR>I;Y$<7pDjB#LMv=mj}BA_3&M2ypCKMDQDmutE% zYrt@yoa=^H#aStsx(VRfEgs4F>(dteqxw{fvukr<%=X7P1`Jp(APK8^;HXR7^?p-) z%$Vn4;WII9h>SSlzw3=5Q)~X5t}?P7LDup`YD!`LgAwvBT)csawPbr(9Wi`F3bPSUZYwwD(vXCf4!v?vLNzi&5dbSO`BiZLiBo$Ur%#LE1fvWh*UI#n(_yA zt)6b%20DN(EcP%z=4DJ4POu@)$~{TFdTWikP9PyB2{XX3RyZ2Q`mxWC?2++GfV(;O zM{J1>mKN<6Q&OH%dJJNCgTyPR*o)S`2k_97NXX@#wImk6{>G`F6+W>@6=t~#=%F^* zJek|SEJ%yl<8a+KDprxZ~8uIDQ3C;$3fhgCV+8;bpu;-~p*NkwU z<8jr}VJsNE)((2=7q9*(SIRtmsAHI>NuKU)ec%UeXd3i-m15~jAsg@nd=~U0 zE~N|-9EkF_c_*Wx(G9c!^7o4ov_CfdIIK-v*cyJ zR_s8Ij>>9o8?XG%nYPMzy>_EvW^*(kTjB&=^FRo@gUr3?y!jLevr+^EMN*J%Oj$kMP~y=(#*RK(@hl zptz_8$F^MXV683vxI;waBo*aNNj2aK;bH!N9C3CG3?#@hj5LB(iYHDHANYWyiPeZ2S8KO8 zLI3VlD{?CC&xJwi*rq9eqDC;7F$~mzXdj?8vWue|87$CL#QQ1PRp%a6RB%_*2liRv z(%G3!(Wb>ebhv!yH+l$BA z3RT=OG_7bY=9}bqsnnEp%BLI%omc4!o{rNH=E3+q9qj-Z|d6I0A6oObDlNsEmS8;ELX(=04VBhfS| z!?4LqSxDO_-V|3xfx7lw?s(JPFhY&3DCJXYIQ0sQG}xl#$;|!1Fm-GCi!*ux zv-`;m6^0EYbv!||=``FjVp`03c z!WhU7hQ55H6#`1Ab?0hmO8~bFt+~+bTh**NpY{ zVP)JdZA;{AJd|l!)D1A8y+4?!-*tf+&Ghb{%WT{$@j-_Q5P)MQ-18Z8)(Q-rHP$rm z9(49KWNWW&vz-P%psS5bZ9!0Lw{_RM@vwK|zy-ZpP6Xh5M9_k3Q(k85z{!T9E8kxp zf_9Xlo|IrNjZe*pxK-v+9iLi3dZmsobk7=;<#+U8^wos!_l^{V;KZ%#2pRLbvwu&t z`nfsY{77#5Si5;Y7KpovkOzE7MV(oK-r?>-X0oy~RJN1WZ2MlbZSK}t>Q{ON3YZz=qzeXF4$#8+P}F4XF}zc@2Mz3(SFmm2L1!byrU8p!NIJWBYXLmJTWH@ zEu&W{gxo-F&fMiImY^&d`n?c!1E>ZuA;SlC@Eo32>@OOe{%f^#)k2x!3YKoPm%1N@ z)<-%nQuYK@@wWUP>?jUeZMCxIM^L`JdJ;afrmeDMa?Rt}DQPb6t zr+m5t64syo&O$mvW!T#ukD6X$(6Ng(vQAziqU?1I{&xtnTLx&;0M0FkH>=ajq1tbYZmFJvGAKa?YV^hl27fQ`+**?qUYAV?d;nMOEOrX0*ZY zmA|_?KMr!a6kA$Li4>|=7Aq!Q8%6AsoP{&eWDrQ(KR@xujB_hTl0Xs^!JjMtZ^WNb z|DOm4%gN6`&Muu2O_QM_Y(qu{1qrS|H#bsDGcFMUsA1k464nWiaAgykK*@4=1);nkOBU|}`n;p&ktz5*UCV+y_WEi@ z2i`>bP&P*7TV0jBgQ@Oz+Jx{PwJL=e-BxfwVkT?@%xWEnC0sb2W`oVbDmb#kA{HUa{$bJ(3vNPk=25FLPcc3IYP@(hYAgb5>b7 zZ?CK6<_L_f$@fK1TT@JaimgTPNB6sk8IytM<#YI)z;wQv_KzdDdUak6@E5*x9i_{( z=x#L77|Rn)l9z2~BbTRJeyK<4+Os3*M?)@Phc}HEr>enty9d{amV!7kyy_1F<3O0x zUyjgcjPKV#&~;-y@T@yakA9y4IsNgl^TyV8zBu%oMOh@Skxptt66t-2V`mU!$D;YFQfn28Intc-skv&8bV8D|7DH`u$B?y-vW!V#Q zLO0EoL_`BgIgzTQXKuiX@Ox^?J62Are`8b$W3SV2oX^B! zh!mKkq^zIxUn*4H+{7rkmGCy;G&V}_>P5z6Cx=j%QVlMg;mRq=VYP0eTKFrqIx|z{ z8xIV%Z#R-%9kKq#nARjP>DM*D8cuV#Df4QBCt8t3cFkUtoK#^!CO90*A|H{XfJ~LE za3{L)Hsu~(+Tk0F?L>RC;zb9pN0ML4t(Y}UNuIVq0Z$DLOuz?eZ45uSemEY5i}^Fe zGo26B{F_h4yl4CfvFI4zu3W5B67sKb1hGLW%_NhYDNKwgQu~LH z^9w)NmJ+XeOz+t3_%X_X)7G*$fr*uY317%qrnCmmkcdQeO*SSeNhK6H!0nO=2Q zY1+v(+7{?{RUXMO7Hybwq5DuwySfm+q8F*WD#KB5$g;6_@N_MX@TC1VLErX5lgzxA z$A(<8pMSJcI_;WMwGEBNUI*Xbk^E!Vb-qteyfp)-qSeH&-%XZ66BXgZ*Jk3be%nlA z4F;&C4qPwTl`HLP4?L71=NK-?{liTSxO1sk^9qm6dNIV@<*%A<#z0ej_L`-lb|(i0 zs$VL#8PhvI1{tOuj_ERNWnSeJN6+aR2*799req$ocV!j#5!+rTc9ZIiUmtV7YYBFr z$9T_Pz1zbi^R~PPn`7xFlcAy&U!$+;RSHwFyj1BT>`{Zt$>%u5k6;_rt=CH^+Skzz zAntaA!;d51&M^6B12gZViNQzgKU8V>IvBj}g&3rv*6QHRu+`RqPWEvyxIG9tI0{9u z;ko`X;8|ROh}MJVgVgi13>nuKuItL>*c$M!Yy@|+>KF2Vi3aX}L<7Zy_1+X52&nYG zi3Y^~8ESAgu{5(Y`VVQWYU;$}aG?2}YjhM#2TUnHVv!a#F=mS$sv9nsRM(#7G8vj- zDcFjLsDJHy+afv}K@62uRn?@J8WZGyz2SRrk{<~Z3DL~>P4?E%g6|Wy#%p#6KR9vh za46<`Rt1L14(%sPGW|&^2)q@3uhQ<{EJT0B@7}(<1MhXKp=#mK!Qbv zUSv-~qaGxoj-dBOMNp4{IBQQ-1#z|YwV5?<4rHet0fyJG-t|=DFjhC)exCbbJ`m;0 zc!5_d$$ehx1vGdot}pz&-X~bTTqfCqhNg>FZ@T*=G)JY=Dyn|dpfeK-E6)gP7BLr3 zaZU@HwZwQzHXkjtZx^+|(%?>(F zI<{@6W81cE+sVth_nmvrJMX;TcvYizjs3@*`>Qc(ueHanx#r9Wu9#DLCx(baX~Y}UXR6A}Cz+8awt?8ku>l%5w74x zRypQC_*_89k;KtSIHGBZ@SWWViMDtpjGkje;Z z7(=dzIuSz~&F zdd*^)DVhGgD7L}p3?zUa6)f|?tavC)$Lw?NY0kwkEwp}AeEg0)ND73f+uV|6ZRrs?xs&fdF%N07=FA zpme3cVq^EDYCb(qrKq^Ft6|(Q@x42^FN%en_whb1_MgMTdNFLaG$;33I_PraC-erFCi=V~$d4u&Pj!P4X z>qfBK3nX8<&U4%f}Mw~v~ z?XrTUse#@bK-~#>;CfHS5+obnDxQlM`0E7C*pHUNrqikB-Zm51F8D z7p9goZO)XoajukUOjc@*No8Co(OrrLrI@#TloUou?M#TGxH!zZVxMt+#VlJRrWo-W`wg?+y!du+IaohYTTi9_>a@nzo~-GUfRPS0rD#VN+-{>&TsB+$5zasL<4Z5 zigB-inJ{e)^5|GFv7mrev>|AwX23(12tLm=))y{-Q5Xul)aGMUifmwl)g6%8=!jH!)c4KamDOFr<#u~{2 zSSv+Mgm^|j0*4Gs!|kP*b1Ul3%y8rtUodGRcx$?j%hw6lO%a7YUbhk)8W|nxfQp*v z6CA`!c{iwowOd$M2xd|R&w+(NC;5iu1XeaCF>qFU@y=lDiohh%R(_}6gfVwy;6)CU zVyfSIN90~kB~!8150??4T|Xeu_&5r)-J?oas^Xps1144jjSwXAc+M?GA%E& zxin;}>PF|e+b2|KjZ5Jcu5~d)ZjD{Xrn=ml2EQ2rzT&7vI-LZI# zf2w$b=0o#~`;$R7Aq*TSUUsEb8oX@uCBE#4()iND1DvRzdr9#Gq)cZ@!HSmg*r}ju zEs|9&$-vc|B0m%V_z0&Lp} z^D;hK+mKw0DR6BoN5LB=iyLky4j=JG&*b$+BMp8NeIVS@hsXoj0-<|Q@iWq}<{lSH z(Hw(E$r%Yo5RAI=?g#xq$B)`#OphG&v$0-z-VnN(EhfK8^h_-YiuF{8e#(ltZ;io1 z2Ma4DW-o%xo2Q~KVQ83MoZJ(>A#7%Nz1c&oZ$VB+rRbQh_O|y0JI~IRzdRzaDkiOa z1*q+&Lcph+0TcsV((N4}mw8nc#c!QQ{EU6vdBfM-1VDcHo1lSPMN#Xk;{$x%Z2z@U ze!c$J&@3$j9V6Y>gDplnBXdV5T5}r{TY4*VLt`69V@G;v2_X?#B@y7i8Oi)Xq|nG# zWI+f3AY%mp@OAos6Um4u3kVA+3!L+OnTUwHTt0=-xi?B%@N2}<6EpXh3?*`_R?v9T zE-PUtNr8ltg|b5E0mvhn?#=NQP{~OcKT^^U3-J8tx6i3kOA6cr`jq67tiY#`~Hnb==d|2u==N zfGNh{a>_G>&WIk`gyFu8rq1AM3y?MkzhGmdHod4h^M(Qg{X{{I&YuD=L_YJg;`v1; z^8>N)vfa~@1>yDBs5x3fV)O>~h#>>Tn|a68!8ztYpAq4(?Lxj{H7UFXrk5fG^@AvR zEKfL09n2j_?yX&@GXzZmXHTIS36wi?>^*6aeDfLdjV4@?1nC>6>}`)H9=Hgi3^8Qu zQqm!lEVEVuyI&wJG$TfcST1=cAP-d*9vLYe6N=|3Lu^QiA11w}p4===>Xc{w(?cUU zGJtZ=?C{T@Mqt>n99CePXZv^TPzI@Nk$IT1-vNP}GWNR-{DdO?jT%7wx0l$c3B>&Q zc-l@M-^pF^uJ1F%&NkfpX1Su4HdPN<@D2#wcfu;sC3E4nR+j#04EC86%Y@z4dQywSpQIc z=LtL62Y+)s+-2*feKTqOkg3%&q6Px+D0{F`7^~I0IZh*_s3f~%`813nYpOUu|JeN+ zQAPzLAc-S{3o6k+EhAAEj`M>Mzn=3k6*0<^h3Po2n=+`n*SF1;*8q|{M|Z5iIM0)l z)Lka8>3Kq7u*ou!M5c2zHE-UqfY7vdPe&X+5BEpyI9ykHXS z=JXsK)GmnD2l0okg^?E)&6d{`0UVmWMiMPWy@p_ftkuKmSHSGv3NKC%C*U4T=!ME_ zBJn;B53J4b&7zUIohcHTRNKW#EFvrjw-0CldI%8=(?$6njgAE=fg30SWRp&qS9;lL)jQQv5EN=+2UJpT$rJu|X1e z??-L-yEh~p=?XVmzjr?$B?qs4YZ^w=P^()Y+S3@)Bn5q(o!lRDXA7DPt9NNO-YbVP zRDR=V+)s2wEpY^fhSWPWBqUExG2#btC;s!5T6~1tdEVw;cKLqDG_*7Y&NBZ~l|Tpj zH2h*UJM26h5}Z5aCEzdqgJidF8CVWexg=qja6mlVzf4AP!2JPZ;vjIpf#{pX92af1 z`kr=yXb*{`R^frfK}KYh3#WkRNEglev)?cP0`)OMMjZYUC61nJP7=XH2-~3eMA(>m z?9>&FlRFyUBlWI~A7YV1e=7(<6ZTYudBd5*RcxlYuCo$XLN zWmyy0Govuj989k&$9O8?JAU>>T@00hyTuuHJh)%k6;$rJ*d#Q zsk^6fn+y1{42hgmgWnKgisBRrrdM1@sPyAYMs>m4>!#M!M%K-aykIY@0HXB0l3TF1 zv376$Cs`(t_tfy-ChUcYVov33mrzo3UUakS8r0*vaS%KgzF7sIhleA!LK7nb_STIp zHOY(;9<(hlztyggZ{1HH?mLmr&CesYXvmemNRf9SVxKksPH;bbEih=5dig+X-&I%! zHYAiwsLixrHeN1?U1320^*%82BYLV^LWzh7NxvpRw(1^Z7_L1PzmE_oI|ob~;w_=d zgAJucP|i4xnQ|LEbzk_DeGUl#^&VE4@u!%WW(BQkw!vo_KIPQ@#Ao9dTWENHCKlwP zbey`>JN&I-5(zNqN(B9T?6)lZGMV5ai{xFM-%b|=CD-h)Uhibk&L=xQs7s%>vv6cX zq;YIZ((UkHHlV)W$PNriD18c@SL(6>1l-iJPxmtltI^bL85F`GoWVRfFpDMT9(bvW zUbC@~=PpT4u_Edbm`CuyLsfLV?KuapFnvVTAXyM$sY!~BFsp=76!RMl3cw2=kRcgU z;9R&AxN@0-RALgIj^%v(jAeI>%~;|H<^{W}42;p@{H@U)fSx$Ef^mupbn-QnP0y+eAa$SIv)Mnst5scJa2qNnsDXJaUkzrBA@y+q9 zI+j)*q&7dJeKeW!K}lExTgR|^^sIJE7K#<_EeFyHbOC!t{QBY?Q9n45CnZmaj58-4 zY$*TE&ehVjZ1uBqy{h4|`ivRIWr6_ZbZs!(nmpQv_q`fPzj0S<>qgkIEk^Me4-4`Trt6Rp2zZ(V8 zeg$+O*zqa~7Tu3j+;JF-Zx9nuxVxsKj%m@e1s2tt4Xg=RfEE!8!^zGS+B9wOsw`;m z{XS0Hr-ELFUI(`El(+db!>3FlHk%0JQ><^C zJfj4sK)RnQ*d=?>UN=(aK|e^!5IBzmCEj`N%0xI_53@LU_NC=E3}f37%~Y&kPzuGq zmEyfYOb}BCJmYQcSR$#==7#?##BWl(uF7DZD;*^i!?#~<;GaRuV%Q1|j7S-hnf_bt zKn(FFDcWM|c*9_Cc6fbKGwefwpdRGKg#B*AITRrKiv0&cLBIwO$zjg|lWK_PmL?kV zlIk(rucUeKhtv!^{Q zs%N?^FPGB!O_mafKzReTy11ao4{|4F4;Wz zM|g$}aRF5R2+a{}uK`Ic(@A!Wdz$z~0(`2RV6Jw1i(CaRk^F#bMi%> z5g3wmIYdq7E-DB>5LK$d+0>CG?bmi={i^OR7SvxjViM6+rN}WiZ4Sz3s341QTht?2 z&@{zelU2zUC+8k%pn+xyj`)m^5H4e2u12I6k0~T3DAsA=SLWC#+*wG2?^l7`- z+=|Qf&Dt~sGD!5_bRzBO?;K+>Iye;Ic1|w)7Cb|AfpT}0$^o}pEUPq5RmWz!QV{BL z&fUsBz8E#mvUvlOmi9U(l$MrFYI+n|&6R;>1xm)hk<0715Zi>#+RtploV<4&_o2i* zj(AYFefe{gDGA!?xC^X{x}uK2B7Il~kR)bY5l9-^5;&gh=v`;n|ZmNE=ERgXDM0njvNOm`S>~)#mhtqJAu{d6X6`k#3Ob zM<(;k3Uw?++1|#Z4s+-~(}XlfI+cn1nqTZZ@(|zQCP(W3?cs@GQFoj`;|H|@lonJ} zZmd{ly)>jdAHt~nO@vQH#o0Q@LWbhfQU6!lH@orfz^8s8kF~e^)h3b99He)|B}2Pb zRjQ&wm(vTxy)NNCyrM)*W32`JQQ60XY{=IN1}Q^BOZ5&i_(-o&NL&oE2P8%!pr|H< z;Iw%_uBpKJ(y>WK(Hi?IN#hCr{p_P2ts`>^yplQDbeupLad3SK3R|4q$~hgg!3R-i zM!%UJw)nOA5?CYrIc;B08Xtob{(QqTIRbjNEkSnxKf?ZKs%PBcHS)2h`txEMBW_Ta z8Mv0pM+bvm?gJ#?H`3i`X4zR%H2h>-{TAJH*lro2^YXKUil@k}sh@bf#dD2lEdgaQ zk#1;)FdnulxBmK;7CP1Qwt$To#(;<;-Z%T zK}7N>auz@%oqtmx3X7t4y=7?KhXiFC5zP}Q&QW{LJ=ErRD!cc99(n^$OxQzplbErd z3m_W))O-T2htnS!VBkH{%0M6R3N593*BNFc>|yn?Ntr!iDflvKVO`%Cn%*x&_!d4v zSoyqB;-|k`S*lQ>`!-r@Ghf^y2~ae{c0ftQUDfC=ohz3~G6A?CfA!b$bYQwe z^N5Dy<_RVpJ+$D_auC>fP8Vn2ND$0J6UjN=_#Y<8Gh;A&*oUKv4|)p7l8@iHvf~M_ znIhFAp|2J*jBXR*M>f#s#K1duZ28`gC-2xkI#05YLnvt{Uj!4N|8$I&i@Y0NRi-#87_AL-J`Xal(`}FmgD*zT>pzy)33cWSsf#!k zYH&CsKr?K5>S-9Mh?zRsev!Y#CH0>=l?*|7E|${fOL$P9^jw~qqK2NQ zE=$w{CY)wGuYrqHR#&wyJukjk23OZH9wylF$0$k1T=bXTrLC_WKrql}YA;Y(4mxH) zEZ2jq8>DX3&^7m`I}{ksklc;;7p=Fc98I7AovT6-R1TJrOE7+Ef#KoT@oaspIBMJX zWexH499{PR$?Zu{Obx|p>8suJA}(8CsP>2ySMI?)7s#6%KB)i4UosF0OT(+D0FPLYrt(C0zjSb}3B({o&e(tLK2MJ!5Vz9gsOb-LBg4B+}-l`mmI?+w((^^WG zu_sJC>2**H-w`7`GibZ%#NIy-{s0UszlG5Hc)8p-KAmmPt9t~Wl+*3x0n|7Hz;~Z~ zkj?7k=Hhs31wM=FB7Y68+7jE6yWAV!SdIJ1d=IcZam&tt8wQWhb0I^ji(>6s1k^dI zUHaJ9F*>(EfuY?8BXP%143d4Wb5#W=y?G7}%48W#^l8&6T?1ayD>TooQYEjF75XWm zYuQ#>rK0EqAPyjue)l8VT@v4cj!fbQ0Z4Ogk@rH*R5B?7?nYa`%HrU*wrlZ?0=8!F zrOg_a^Cuj(drk=wmhj4Q)U?`amS&tVSj|JhGoT<}#(+)JB$~4G*PpE~UAA7st@Zdf zm$bKOZ}IMea?Z6DEjjrc$-^(-FtOwF@Ir=oZ6_^Mg6j*LK$?VOo+_3QdvlhM^ms_x zcEMI*p*QM$X7EW+yzQvEs-2>B=$;FS5+#CpkBQX9cEo>lAgw+%5%B;stC2Pi^N(?@ zcaK{j6+h)=9M_Q*@z7)ka}bs1UWd7~YmeB-L`D{@b7ECswu{=i<+m_X$5KR;3Zp8i zOoCXln2Yi-SvB}YtQ3IX8mf;g}mgmdXDs&t{_JlrnHU%b^7M$@ZQY9)*n&=Yp3K4 zvor_{zd!>H6_94)w0(VBYX9M$=-_#%Kd(IV6FVZi!6LyJYjxpX{rF~;Yngd+i4pHC zmGJ_BM06*~sImobwe);&1AV0=OFz#hX30suW?-`)qoRKT&eBAbJ<`S-qRM;8z}9JD zPR6%VMz503&rr^-obP9^?HpzL3TE0!49@#DE&Tq`$Ygs;_A`D+|#e-`ELj9 zIs*8c#mT6h8>;j^MR$x@K+XgRN>#zLV|SI49#zAanaaqkvUyh-4)u$sEuA#9ucRE3 z8Ch?Z8aIn+iOT-Zi)VY#$|Y?UhQsePR9J| zvaoPAWCdvXn&bNOr#&>`U#XtIrZ;7-6P3~}Axu++Pg3ODcL2%!9Iw0&yWNE&Dcc_z z=~3=uON8Z-oyE?0gPU9gj?Ivk_l&2_i zC_7^y)M*+$LzNe#3D>F`7XsLx94~!z53ClUVa=cW@?%YSWI1#tQ|o@8vxwX88us`1 zNM4vj@t*%dSpzOv0+;~=r`O+9uc#5?-IhVLFc9NtP1Pgst!>YlcQM|+>m3q*b#x>M zR%$MJ-cy2MXrHLG;8wUmmkyI#UQ8q094Squ-#>W@p&rHEmSddtID7w_m9y2lvuj;@ z?L@&j4vFDi@}R9{c6--I$s$AGz%qgNd5j|_%7(tMSx0!iLHKxeR&}945wUu`psqr% z&JR1B#l8|>V1zc0-Kl>tbF+gj*Ep-tuI+es5NUC3!6rzO0i&) z$f2ru4g$5U&5tD@m3!0cyW`oYv@~yf_WfCp{KWlzVMzzq8m{;#E61~_b_(Ac5ipPV zmDPm2sr{Lt78rnaR^Gf{(77M3K$Vybff-3QJ>Y770!EVSWDT*obV(+ySo-XJlS7YCT%&U(7UCjUWM+PoVBwRH(ulkeKPi|kQGnruV^!m{*PQc3v%frOTDZMf z@dUP4kxGV6&0&TOQ^b9}h=CR!2T4mi?a{>{rauHULsr%YeQ0VF?bEKMi`VF97F+rC z+a&KtnAIzczoBxG?VVFO-dHHAGj3P&mumh(I$j=|!jVsCMFV(5Pl`v2qW^uC?*m-y zywI$<*du&$vn!6^DrK> zd{X}%NK5SZLEU$n@47L8gg!VXzFQL@AH`(L3Wq-jy71gwSVLZmMTeGh>#stiLtP5a z$=I6_MZ0p`ZiNPo1L+Q+qkTxoGfQp***APOxuhmf-^ z8FiSGQ9y$KnUcSQ(;+W0`0+Bo+K1#JNv)N7f>oMdb&^8O%W32e*2`%n9jl+EC`1s? z_ukvNrESfUM_-Hy`Jfam49kFeJh07W@ur;HcW(8b=sqow!^2#GF4V1~Vo?1VLh9|S zVFn#lStQls7n|BaEp@Eg{0J)ZUTtgwoJMm&Ks$GNnuXtG7ZSsqM7IGw&3bmIw|A`gaVW8^0TW zd~4G&Rj@PBCNJ>1n6InE6e=_8=tS2q*rsXXTr9dJo(H*mOiW8>e(eaX&?m&ryp||a zXvRHnk!Lbg2gk!4SoNv(-X1)aWFek>7wW}>LOT0mc;&PD<`RwTaYJugxcx#a3kyG) z8!A9%#vcGK;bf#sdYIt@nD%WnuNpDW8<~7CmA~kaiW2(!2fe zQUf5zj?wR7Y1R#s3vHRT(F34$@$WV=xeK~-k(UH~L(`8QB2bCBT zE}VbBMONe19LS;(8F$*s0%6YTi;)FSYr?Mr>-%YjJYqXzPf86+<{d3^vWY!3gJC=K zelNeQ^{AFycx;0AThiq>wCS77VeftOa1@-H{H8-7sn&Y{YL7_tm9;I+^;vh5MTl*@ zQz8zE2$U0zvkfH`>aTO!h|QXaUYpCh$#32+pf<#OT0mdBdYktCkP2sq0%*(e7%Z9e5~d zofAtNs6tJpYh=5&gd8P?lx!IF`{=H-WKYsJvDc#08p6R%9>_JFw-RYP?JV~4!Q`IZ zgK{fWEgELJIYNXix_+B|^ZHKy{)&`LYA{)*<5vwDRAzF&bp~@O#LcC&BX+nT>I5%$ z3cE$7WNNd!mk*~?Bhkr(17uGmb$H;P2zVeX`~DNNaH4JpV+zO178>ff)qJd^ooCW3bh+6oxQT1pP;8?i% zM2<`xr}oB`cd)o)IDM!@w~FQ^ccV=0yswo~{eBTNBMg)hlDodZybd%;Trv%?d(}*M zr1_@()bL98qAO_7U?mhLqYQZ_f!X?HQML><^y3ZfMVv8s{|T7evG-fy58aGrT_};t z;G~86Pe*VI^9x)>U6)@OEv>ox7mb$mK9zeN&(R5Ne1xO=SNe`_TM4@cxrfLrq*nL>jc45vrE~26RfhXleg!!KjeU<|8j(_AvH_>uitG$enMGWit9jh<(8KQ?C?T$|A3Veww}I zeO4UnPc%6E?0TEPs9QPNr6?jKrzq?^GaQ@Cj{-X7?G3F#^0O-! z9j0UqE~1tY14%-JAM^gSH8=0fZufw0vXyNohg0l`E7JbgE4xkud@eO|nD+a!$bE(* zXyp`j9Y)~X3gH>|6Qd`)Vk@tNC6%!kk0jv{MB=C5XL@;agX^4gQ^_U;Ya?vYc%$Vy zzF`~3i+@teA!vv4@2iga_7x<1;0g*!_ujId7jDZ1p4OQ zlUN)H_Lvd6D?s|ya24|5oq1x6%={lEKF#et3mVkY=#pcE49tQT28z z^@eV#$G+EJ$xYb86df<^i*_Xw|E3WFBC6tg_Zt z&9%G%TNb)(YOQm_S>sfNwmy&B>QNDb7^nwe}qWP570F|vcKmYUBxLU zDnwE86AVC$FJ%$`ni(EZMW&b3piuA9uq_y#gy3)2(G#HSMw+5jKWI8dZ=y@RgfcC@ zkbktcVET>w)Y4@Kwbp2=laOwN*t){1UfmKA#KWmoS#DkLfna?!gSmhw@Z*5?C@kzK zjQ)YdI%UShcxyb^TeVb?Ogq3%nJ1M@$+)Rnc~m{~Nf{_6n?2HiZ`Kbg=h-33ff1K& zH81dcdwOw?A>M$gABHsnecpV)+_qQS`Xt~nCXxG z0eQJM2HwpjYjJd&X>7JGj7c#bHG@$xPlNkqUVe8^d5E1;dU+{8AQa&LcgWY*E&E?v zzTY3G|10wA@9=*M@%nEt06@Oqmrc$87N9DTJ-!K`)~61 zzp#3|e`5bf4F7lVzezR!0$&UL3I0>K`FHHU3Ge>G>Wcnv?EjAp@1L~)l;QnFTl}AB z{}ko@lkT6*;x9S{v;VNl|7sooa{l&XHUAIm`|q+Z WF9rHXvjKqp+JnA2#jfQa@BSCw(4%z# literal 0 HcmV?d00001 From fc977131a4f2b34042c9ebb07298f0d0e4555cee Mon Sep 17 00:00:00 2001 From: darshan-endee Date: Wed, 25 Mar 2026 09:44:42 +0000 Subject: [PATCH 22/22] automation scripts added --- .gitignore | 6 +- demo_cli_ommands.txt | 525 +- intfilterscript.ipynb | 6921 ++++++++++++++--- intfilterscript_final.ipynb | 107 +- intfilterscript_final_gt.ipynb | 1539 ++++ intfilterscript_final_lt.ipynb | 3075 ++++++++ intfilterscript_final_lte.ipynb | 6551 ++++++++++++++++ labelfilter_bench.py | 288 + prepare_groundtruth.py | 88 + prepare_groundtruth_gt.py | 88 + prepare_groundtruth_lt.py | 88 + range_filter_bench.py | 287 + script.ipynb | 4389 ++++++++++- stability_test_gt.py | 472 ++ stability_test_gte.py | 434 ++ stability_test_labelfilter.py | 393 + stability_test_lte.py | 440 ++ vectordb_bench/backend/clients/endee/cli.py | 2 +- .../backend/clients/endee/config.py | 2 +- vectordb_bench/backend/clients/endee/endee.py | 318 +- .../clients/qdrant_local/qdrant_local.py | 350 +- vectordb_bench/backend/data_source.py | 16 +- 22 files changed, 24897 insertions(+), 1482 deletions(-) create mode 100644 intfilterscript_final_gt.ipynb create mode 100644 intfilterscript_final_lt.ipynb create mode 100644 intfilterscript_final_lte.ipynb create mode 100644 labelfilter_bench.py create mode 100644 prepare_groundtruth.py create mode 100644 prepare_groundtruth_gt.py create mode 100644 prepare_groundtruth_lt.py create mode 100644 range_filter_bench.py create mode 100644 stability_test_gt.py create mode 100644 stability_test_gte.py create mode 100644 stability_test_labelfilter.py create mode 100644 stability_test_lte.py diff --git a/.gitignore b/.gitignore index 727820272..a347f6ec0 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,8 @@ debug_logs/ ground_truth_ids.csv matched_metadata.txt vectordataset_label/ -venv2/ \ No newline at end of file +venv2/ +vectordataset_gt/ +vectordataset_lt/ +vectordataset_lte/ +venv3/ \ No newline at end of file diff --git a/demo_cli_ommands.txt b/demo_cli_ommands.txt index 17e427063..2e3903e55 100644 --- a/demo_cli_ommands.txt +++ b/demo_cli_ommands.txt @@ -28,21 +28,21 @@ DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ # ======================================================================== # FOR USING THE EXISTING INDEX # ======================================================================== -DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ - --token "TOKEN" \ + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ --region india-west-1 \ - --base-url "http://127.0.0.1:8080/api/v1" \ - --index-name test_index_name\ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_filter_1803_adup6\ --task-label "20260107" \ --m 16 \ --ef-con 128 \ - --ef-search 128 \ + --ef-search 256 \ --space-type cosine \ - --precision int8d \ + --precision float32 \ --version 1 \ --case-type Performance768D1M \ - --k 30 \ - --num-concurrency "8" \ + --k 1000 \ + --num-concurrency "16" \ --concurrency-duration 30 \ --concurrency-timeout 3600 \ --skip-drop-old \ @@ -1071,4 +1071,513 @@ DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" ve --skip-drop-old \ --skip-load \ --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_labelfilter_reupsertcheck \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_labelfilter_reupsertcheck \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.001 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + +curl -X POST "http://localhost:8080/api/v1/index/test_1M_vaib_labelfilter_reupsertcheck/backup" \ + -H "Content-Type: application/json" \ + -d '{"name": "labelfilter_1M_backup"}' + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_labelfilter_0503_1 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_labelfilter_0503_1_duplicate1 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_labelfilter_0503_1 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_labelfilter_0503_1_adup5 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --prefilter-cardinality-threshold 20000 \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_labelfilter_0503_1_adup4 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.001 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_lte" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_labelfilter_0503_1_adup6 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_lte" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_labelfilter_0503_1_adup6 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --prefilter-cardinality-threshold 20000 \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_gt" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_labelfilter_0503_1_adup7 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_gt" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_labelfilter_0503_1_adup7 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --prefilter-cardinality-threshold 9999 \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.80 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_lt" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_labelfilter_0503_1_adup8 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_lt" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_labelfilter_0503_1_adup8 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --prefilter-cardinality-threshold 9999 \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_labelfilter_0503_1_adup_original \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.001 \ + --k 100 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_labelfilter_0503_1 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.02 \ + --k 100 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_lt" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_filter_1503_adup1 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_filter_1503 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16 \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_filter_1503 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --prefilter-cardinality-threshold 20000 \ + --precision int16 \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0. \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_latest_master_1803_2 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16 \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "TOKEN" \ + --region location \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test \ + --task-label "test" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16 \ + --version 1 \ + --case-type Performance768D1M \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ --search-serial \ No newline at end of file diff --git a/intfilterscript.ipynb b/intfilterscript.ipynb index c4e00cf8f..4c067ecdd 100644 --- a/intfilterscript.ipynb +++ b/intfilterscript.ipynb @@ -117,53 +117,23 @@ "name": "stdout", "output_type": "stream", "text": [ - "test_1M_labelfilter_int16\n", + "test_1M_vaib_labelfilter_0503_1\n", "1000000\n", "\t\n", - "test_1M_labelfilter_int16_latest_master\n", + "test_1M_vaib_labelfilter_0503_1_adup1\n", "1000000\n", "\t\n", - "test_1M_labelfilter_latest_master_stability1\n", + "test_1M_vaib_labelfilter_0503_1_adup3\n", "1000000\n", "\t\n", - "test_1M_labelfilter_latest_master_stability2\n", - "990000\n", - "\t\n", - "test_1M_normal\n", - "1000000\n", - "\t\n", - "test_1M_normal_latest_master\n", - "1000000\n", - "\t\n", - "test_1M_numfilter_int16\n", - "1000000\n", - "\t\n", - "test_1M_numfilter_int16_latest_master\n", - "1000000\n", - "\t\n", - "test_1M_numfilter_latest_master_stability1\n", - "1000000\n", - "\t\n", - "test_1M_numfilter_latest_master_stability2\n", - "1000000\n", - "\t\n", - "test_1M_numfilter_latest_master_stability3\n", - "990000\n", - "\t\n", - "test_1M_numfilter_latest_master_stability_bmw\n", - "1000000\n", - "\t\n", - "test_1M_numfilter_latest_master_stability_vaib\n", + "test_1M_vaib_labelfilter_0503_1_adup4\n", "1000000\n", "\t\n", - "test_1M_numfilter_latest_master_stability_vaib2\n", + "test_1M_vaib_labelfilter_0503_1_adup5\n", "200000\n", "\t\n", - "test_1M_vaib_reupsertcheck\n", - "300000\n", - "\t\n", - "test_1M_vaib_reupsertcheck_new1\n", - "262392\n", + "test_1M_vaib_labelfilter_0503_1_adup6\n", + "1000000\n", "\t\n" ] } @@ -183,34 +153,29 @@ "outputs": [], "source": [ "#give the index name\n", - "index_name = \"test_1M_vaib_reupsertcheck_new1\"\n", + "index_name = \"test_1M_vaib_labelfilter_0503_1_adup6\"\n", "index = client.get_index(index_name)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'name': 'test_1M_vaib_reupsertcheck_new1',\n", + "{'name': 'test_1M_vaib_labelfilter_0503_1_adup6',\n", " 'space_type': 'cosine',\n", " 'dimension': 768,\n", " 'sparse_dim': 0,\n", " 'is_hybrid': False,\n", - " 'count': 262392,\n", + " 'count': 1000000,\n", " 'precision': 'int16',\n", " 'M': 16}" ] }, - "execution_count": 8, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -5190,7 +5155,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -5199,1284 +5164,2404 @@ "text": [ "Finding all query IDs with recall < 1.0 ...\n", "\n", + "Query ID: 0\n", + "Returned IDs: 992169,992979,997002,993717,997637,997918,997822,993288,999453,992790,995401,994922,993289,994819,990399,993738,995994,998530,996438,991266,993032,999587,994334,991150,994964,994409,995701,995708,994804,999450\n", + "Ground Truth: 995888,992169,992979,997002,993717,991151,997637,997918,997822,993288,992933,999453,992790,995401,994922,993289,997852,994819,990399,993478,993738,995994,996777,998530,996438,991266,993412,990337,993032,999587\n", + "Recall: 0.7333\n", + "\n", "Query ID: 1\n", - "Returned IDs: 997420,995382,990103,993165,996814,996219,996878,992878,998104,994051,994199,990678,996062,995438,993727,992193,994119,997997,999441,992780,993088,997331,990284,992731,997602,997098,996811,998691,999191,991031\n", + "Returned IDs: 997420,995382,990103,993165,996814,996219,996878,992878,998104,994051,994199,990678,995438,992193,994119,997997,992780,993088,997331,992731,996811,998691,999191,991031,995783,999244,993324,996121,995562,993929\n", "Ground Truth: 997420,995382,990103,993165,996814,996219,996878,992878,998104,994051,994199,990678,996062,995438,993727,992193,994119,997997,999441,996830,992780,993088,997331,990284,992731,997602,997098,996811,998691,999191\n", - "Recall: 0.9667\n", + "Recall: 0.7667\n", "\n", "Query ID: 2\n", - "Returned IDs: 996312,991260,995701,992330,996592,993492,997449,997913,992546,997976,990231,990936,990949,995014,994100,996881,992692,992969,993853,993005,996267,994778,991045,994757,992931,990957,999647,992961,990109,993970\n", + "Returned IDs: 996312,991260,995701,992330,996592,993492,997913,992546,997976,990231,990949,994100,996881,992692,992969,993853,996267,994778,991045,994757,992931,990957,992961,990535,995793,997371,990588,996075,997771,995094\n", "Ground Truth: 996312,991260,995701,992330,996592,993492,997449,997913,992546,997976,990231,990936,993517,990949,995014,994100,996881,992692,992969,993853,993005,996267,994778,991045,994757,992931,990957,999647,992961,990109\n", - "Recall: 0.9667\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 3\n", + "Returned IDs: 991304,990167,999555,992444,990627,995610,999994,999033,990194,990437,990434,996209,991798,997084,999131,994355,997086,993076,991667,992892,992517,990394,998335,997798,999280,994564,993480,992628,994836,992778\n", + "Ground Truth: 991304,990167,999555,992444,990627,995610,997415,999994,999033,990194,990437,990434,996209,991798,998279,997084,999131,994355,997086,993076,991667,992892,992517,990394,998335,994725,997798,999280,994564,993480\n", + "Recall: 0.9000\n", "\n", "Query ID: 4\n", - "Returned IDs: 993262,992665,991098,996599,992247,997534,990818,990802,992701,995626,990402,999466,991236,998303,991037,996216,997897,994310,998299,995516,997825,999435,992573,994098,997334,991212,995991,999216,993124,996424\n", + "Returned IDs: 993262,991098,996599,992247,997534,990818,990802,992701,995626,999466,991236,997442,998303,991037,996216,997897,994310,995516,997825,999435,992573,990630,994098,997334,991212,993124,997165,994247,991148,993204\n", "Ground Truth: 993262,992665,991098,996599,992247,997534,990818,990802,992701,995626,990402,999466,991236,997442,998303,991037,996216,997897,994310,998299,997825,995516,999435,992573,990630,994098,997334,991212,995991,994964\n", - "Recall: 0.9000\n", + "Recall: 0.8333\n", "\n", "Query ID: 5\n", - "Returned IDs: 995274,997320,998779,992994,999144,992630,999026,998184,998365,995755,998951,999090,999647,994100,993412,996959,994964,995701,998862,994757,995702,997976,995336,990949,993648,994730,994894,995438,992331,997737\n", + "Returned IDs: 995274,997320,998779,992994,999144,992630,999026,998184,998365,995755,998951,999090,994100,993412,994964,998862,994757,997976,995336,990949,994730,994894,995438,997737,991790,999089,993529,997832,995824,993484\n", "Ground Truth: 995274,997320,998779,992994,999144,992630,999026,998184,998365,995755,998951,999090,999647,994100,993412,996959,998633,997925,994964,995701,998862,994757,995702,997976,995336,990949,997377,993648,994730,994894\n", - "Recall: 0.9000\n", + "Recall: 0.7333\n", "\n", "Query ID: 6\n", - "Returned IDs: 992384,995274,993648,995204,993909,991743,992994,998184,993837,999374,993412,998500,995928,999533,992582,994636,990432,992147,995702,994850,998458,996521,992630,992637,998779,995961,991852,997301,996732,996959\n", + "Returned IDs: 992384,995274,995204,993909,991743,992994,998184,999374,997645,990862,998500,995928,992582,990432,992147,995702,994850,998458,996521,992630,992637,998779,997301,996959,997192,994380,991585,997737,994197,995260\n", "Ground Truth: 992384,995274,993648,995204,993909,991602,991743,993089,992994,998184,991866,993837,999374,997645,993412,990862,998500,995928,999533,992582,994636,990432,992147,995702,994850,998458,996521,992630,992637,991151\n", - "Recall: 0.8000\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 7\n", + "Returned IDs: 990956,992052,991185,993948,991451,995184,994488,991981,994742,997897,996277,996911,991111,995047,993991,994185,992973,995408,991769,999767,992595,994743,990336,994280,996328,995873,990887,998045,996657,995251\n", + "Ground Truth: 990956,992052,991185,993948,991451,995184,994488,991981,994742,997897,996220,996277,996911,991111,995047,993991,994185,992973,995408,991769,997059,999767,992595,990364,994743,990336,994280,996328,994134,996043\n", + "Recall: 0.8333\n", "\n", "Query ID: 8\n", - "Returned IDs: 997913,998090,990949,990910,997320,995961,999710,997301,990231,996918,990560,996805,993504,993515,998950,992277,999639,998337,995336,997371,999806,994850,992748,994100,993529,999900,998365,999338,993696,991498\n", + "Returned IDs: 997913,998090,990949,990910,997320,995961,999710,997301,990231,996918,990560,996805,993504,998950,999639,998337,995336,997371,999806,994850,992748,994100,993529,998365,993696,991498,990957,992944,995802,999331\n", "Ground Truth: 997913,998090,990949,990910,997320,995961,999710,997301,990231,996918,993412,990560,996805,993504,993515,998950,992277,999639,998337,995336,997371,999806,994850,992748,994100,993529,999900,998365,999338,991498\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 9\n", + "Returned IDs: 995274,998495,995755,992994,990925,992546,994100,994964,994266,990096,997320,999144,993168,991721,998779,996805,996368,996001,998862,993238,997976,993289,990957,992558,998458,990856,995701,994409,999026,993853\n", + "Ground Truth: 995274,998495,995755,992994,990925,993412,992546,994100,993648,994964,994266,990096,997320,999144,993168,991721,998779,996805,996368,996001,999024,998862,993238,997976,993289,990957,992558,998458,990856,994382\n", + "Recall: 0.8667\n", "\n", "Query ID: 10\n", - "Returned IDs: 991067,996273,999530,993530,999193,998316,997057,994677,993223,994301,991190,997723,992545,998650,992493,999961,995963,997675,995723,997098,997824,992582,995685,993455,996046,990930,999646,991052,997083,994948\n", + "Returned IDs: 991067,996273,999530,993530,999193,998316,997057,994677,993223,994301,991190,997723,992545,998650,992493,999961,995963,997675,995723,997824,995685,993455,996046,990930,999646,991052,994948,996139,995507,995101\n", "Ground Truth: 991067,996273,999530,993530,999193,998316,997057,994677,993223,994301,991190,997723,992545,998650,992493,999961,995963,997675,995723,997098,997824,992582,995685,993455,996046,990930,999646,991052,997083,991840\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 11\n", + "Returned IDs: 995459,992824,997775,990084,996742,991432,996380,991817,995503,990192,998754,993077,993891,992615,998061,990720,999639,997547,993583,990538,999564,994270,991272,990273,991871,997055,992095,998308,991070,998142\n", + "Ground Truth: 995459,992824,997775,990084,996742,991432,996380,991817,995503,990192,998754,993077,993891,992615,998061,990720,999639,997547,993583,990538,999564,994270,991272,990273,991064,991871,997055,992095,998308,991070\n", "Recall: 0.9667\n", "\n", + "Query ID: 12\n", + "Returned IDs: 993497,993923,992784,992992,999726,992981,992934,992997,998293,990322,992052,996328,994802,998993,990391,992193,994280,997897,994196,999893,990306,991115,991285,995790,995890,990205,995058,996211,997244,991924\n", + "Ground Truth: 993497,990103,993923,992784,992992,999726,992981,992934,992997,998293,990322,992052,996328,994802,998993,990391,992193,994280,997897,994196,999893,997475,990306,991115,991285,995790,995890,990205,995058,996211\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 13\n", + "Returned IDs: \n", + "Ground Truth: 997135,998837,993237,993708,996100,994534,998057,997255,990043,990064,997275,996402,995210,999898,996419,993570,991801,992907,999592,990580,991840,995788,992328,994318,995657,993748,994722,991407,991999,998782\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 14\n", + "Returned IDs: 992561,995211,997733,990126,999985,990931,998251,999179,995633,994142,996411,990705,993219,992566,994783,995883,999191,990621,996431,994317,992415,998691,994587,996609,990854,990069,997823,997403,990340,995571\n", + "Ground Truth: 992561,995211,997733,990126,999985,990931,998251,999179,995633,994142,996411,990705,993219,992566,994783,998870,995883,999191,990621,996431,994317,992415,998691,994587,993775,996609,990854,990069,999352,997823\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 15\n", + "Returned IDs: \n", + "Ground Truth: 996960,995695,991755,995853,994280,998831,993805,997666,991459,992021,997506,998623,992429,998934,994531,990999,994105,990844,996328,993014,997790,994680,990479,990573,990998,996001,993848,995963,990771,995652\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 16\n", + "Returned IDs: 994222,996659,999277,995289,997406,995401,995144,998999,998045,991449,991463,995200,999770,996867,993433,995359,998530,990940,997723,992160,998297,992947,996302,991942,996769,993649,994409,992125,997319,998776\n", + "Ground Truth: 994222,996659,999277,995289,997406,996864,996015,995401,995144,998999,998045,991449,991463,995200,999770,996867,993433,995359,998530,990940,997723,999587,997473,992160,998297,992947,996302,991942,996769,993649\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 17\n", + "Returned IDs: 995886,991743,995893,998921,996351,994982,995515,991692,997218,996762,995968,990215,990561,999698,993250,994334,995154,997301,997002,993143,999075,990995,997571,993820,993237,998908,992384,997320,996903,992790\n", + "Ground Truth: 995886,991743,995893,998921,994850,996351,994982,995515,991692,997218,996762,994810,995968,990215,990561,999698,993250,994334,995154,999823,998445,997301,996438,997002,993143,999075,990995,997571,997370,993820\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 18\n", + "Returned IDs: 991640,996209,999909,999555,996566,997666,999576,990323,998006,997242,997685,995385,996733,995344,993513,991667,990173,993508,991385,995214,994727,994467,999246,991596,991093,994335,990627,996809,994181,990194\n", + "Ground Truth: 991640,996209,999909,999555,996566,997666,996702,999576,990323,998006,997242,997685,995385,996733,995344,993513,996454,991667,990173,990712,993508,991385,995214,994727,994467,999246,991596,991093,994335,990627\n", + "Recall: 0.9000\n", + "\n", "Query ID: 19\n", - "Returned IDs: 990338,997370,993018,991437,997320,996001,996309,999586,998050,991224,996810,993642,990532,990843,993840,994081,991943,991002,991787,998495,996219,990772,994964,992655,994168,999561,992583,990974,990585,990786\n", + "Returned IDs: 997370,993018,991437,997320,996001,996309,999586,998050,996810,993642,990532,990843,993840,994081,991943,991002,991787,998495,996219,990772,994964,994168,990974,993484,999977,998779,992792,999639,993053,993696\n", "Ground Truth: 990338,997370,995899,993018,991437,997320,996001,996309,999586,998050,991224,996810,993642,990532,990843,993840,994081,991943,991002,991787,998495,996219,990772,994964,992655,991019,994168,995283,999561,992583\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 20\n", + "Returned IDs: 994351,996495,992237,995754,996809,991383,999435,992726,990194,994098,996428,995214,996774,997725,996328,990173,998335,996839,992193,997967,997331,999430,993224,994481,990705,990594,999832,999752,991723,990712\n", + "Ground Truth: 994351,990402,996495,992237,995754,996809,991383,991025,999435,992726,990194,994098,996428,995214,996774,997725,996328,990173,998335,996839,992193,997967,997331,999430,993224,994481,990705,990408,990594,999832\n", "Recall: 0.9000\n", "\n", "Query ID: 21\n", - "Returned IDs: 993018,997320,994171,993662,995351,997913,990843,996075,995596,995266,994746,991142,994366,997470,996296,994341,990957,999077,998540,997457,993696,991999,997707,996469,995560,992733,999784,992277,997022,999806\n", + "Returned IDs: 993018,997320,994171,993662,997913,990843,996075,994604,995596,995266,991142,997470,996296,990957,999077,998540,997457,993696,991999,997707,996469,992733,999784,999806,998950,990778,994440,995365,999426,999109\n", "Ground Truth: 993018,997320,994171,993662,995351,997913,990843,996075,994604,995596,995266,994746,994366,991142,997470,996296,994341,990957,999077,998540,997457,993696,991999,997707,996469,995560,992733,999784,992277,997022\n", - "Recall: 0.9667\n", + "Recall: 0.7667\n", "\n", "Query ID: 22\n", - "Returned IDs: 997290,998557,997517,994110,998543,991334,994893,995030,995159,996809,999924,997947,999788,995948,992566,996747,995190,990814,994422,998913,992289,992468,990722,994824,999942,998194,992588,994541,998340,992364\n", + "Returned IDs: 997290,998557,997517,994110,998543,991334,994893,995030,995159,996809,999924,997947,999788,995948,992566,996747,995190,990814,994422,998913,992289,990722,999942,992588,994541,998340,992364,992077,996302,995200\n", "Ground Truth: 997290,998557,997517,994110,998543,991334,994893,995030,995159,996809,999924,997947,999788,994062,995948,992566,996747,995190,990814,994422,998913,992289,992468,990722,994824,999942,998194,992588,994541,998340\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 23\n", - "Returned IDs: 996053,990096,997913,997586,994850,995961,995536,998090,996143,995755,995927,998861,994894,999356,999711,993853,997305,994618,998921,992848,997984,998550,994707,991582,994964,998495,992330,997320,992862,993948\n", + "Returned IDs: 996053,990096,997913,997586,994850,995536,998090,995755,995927,998861,994894,999356,999711,997305,994618,998921,992848,997984,998550,994964,998495,992330,997320,992862,990866,999710,993431,990066,992384,996001\n", "Ground Truth: 996053,990096,997913,997586,994850,995961,995536,998090,996143,995755,995927,998861,994894,999356,999711,993853,997305,994618,998921,992848,997984,992716,998550,994707,991582,995503,994964,998495,992330,997320\n", - "Recall: 0.9333\n", + "Recall: 0.7667\n", "\n", "Query ID: 24\n", - "Returned IDs: 996093,997852,994922,992169,993717,991473,998523,997395,996438,992632,992152,990580,990399,994510,996727,991840,993143,998458,999676,997476,990080,991609,993859,995295,992813,997192,993781,999453,992044,990490\n", + "Returned IDs: 996093,997852,994922,992169,993717,991473,998523,997395,996438,992632,992152,990580,990399,994510,996727,991840,993143,998458,999676,997476,990080,995295,992813,997192,993781,996406,999453,999076,998671,993355\n", "Ground Truth: 996093,997852,994922,992169,993717,991473,998523,997395,996438,992632,992152,990580,990399,994510,996727,991840,993143,998458,999676,997476,990080,991609,993859,995295,992813,997192,993781,996406,992044,999453\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 25\n", + "Returned IDs: \n", + "Ground Truth: 993412,998045,994419,996977,996818,996360,994057,995923,992129,992089,995702,999940,996769,994549,999450,998906,992413,991958,995289,991585,992660,991996,999530,993199,996867,995547,998545,996242,990975,999868\n", + "Recall: 0.0000\n", "\n", "Query ID: 26\n", - "Returned IDs: 990487,993500,991977,990998,994254,992558,991447,993335,998686,992052,994148,996068,990684,998406,993975,992170,991620,994006,993583,998536,995408,998480,992718,996262,992933,998623,992280,993965,994080,993715\n", + "Returned IDs: 990487,993500,991977,990998,994254,992558,991447,993335,998686,992052,994148,996068,990684,998406,993975,992170,991620,994006,993583,998536,995408,992718,996262,998623,992280,993965,994080,993715,993681,994280\n", "Ground Truth: 990487,993500,991977,990998,994254,992558,991447,993335,998686,992052,994148,996068,990684,998406,993975,992170,991620,994006,993583,998536,995408,998480,992718,996262,992933,992554,998623,992280,993965,994080\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 27\n", + "Returned IDs: 993721,996013,991409,993053,991382,999352,994983,998568,998495,993984,997270,996075,995596,993379,999806,999639,996650,994886,995535,994431,996001,997442,999302,993342,995352,996328,992515,993621,991128,995476\n", + "Ground Truth: 993721,996013,991409,991179,993053,991382,999352,994355,994983,996219,998568,998495,993984,997270,996075,997547,995596,993379,999806,999639,996650,994886,997448,995535,994431,996001,997442,999302,993342,995352\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 28\n", + "Returned IDs: 990615,998937,994703,996840,995788,994216,996915,991687,992662,999676,995515,990475,994601,998244,993495,991840,993190,992835,994612,998523,991485,990965,995080,993905,991392,997403,999613,998139,997220,994033\n", + "Ground Truth: 990615,998937,994703,996840,995788,994216,996915,991687,992662,999676,995515,990475,994601,998244,993495,991840,993190,992835,994612,998523,991485,990965,991499,995080,993905,993028,997496,991392,997403,999613\n", + "Recall: 0.9000\n", "\n", "Query ID: 29\n", - "Returned IDs: 998276,993603,994829,997918,997002,995141,992047,996012,993139,996661,993289,991425,997140,990051,996808,997977,992330,993504,992238,994972,996576,994946,998401,992225,994042,995855,997777,992979,990491,999111\n", + "Returned IDs: 998276,993603,994829,997918,997002,995141,996996,996012,993139,996661,993289,991425,990051,996808,997977,992330,993504,994972,996576,994946,998401,992225,995855,997777,992979,990491,999111,994821,996001,996871\n", "Ground Truth: 998276,993603,994829,997918,997002,995141,996996,992047,996012,993139,996661,993289,991425,997140,990051,996808,997977,992330,993504,992238,994972,996576,994946,998401,992225,994042,995855,997777,992979,990491\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 30\n", + "Returned IDs: 995257,992888,990402,997673,998570,997666,990158,996702,995231,995695,997369,992664,995349,997086,995393,999435,990041,990479,995142,998006,990194,991640,996960,994467,993461,996594,992945,995168,991755,993067\n", + "Ground Truth: 995257,992888,990402,997673,998570,997666,990158,996702,995231,992209,995695,997369,992664,995835,995349,997086,995393,999435,990041,990479,995142,998006,990194,991640,996960,994467,993461,996594,992945,995168\n", + "Recall: 0.9333\n", "\n", "Query ID: 31\n", - "Returned IDs: 990811,998593,995526,992334,996298,997732,998039,992684,997969,990089,996834,990529,993642,994964,994783,995126,995679,997410,995906,996001,999099,994925,992120,996873,993831,994293,999302,998382,991200,990576\n", + "Returned IDs: 990811,998593,995526,992334,996298,997732,998039,992684,997969,990089,996834,990529,993642,994964,994783,995126,995679,997410,996001,999099,994925,992120,996873,993831,994293,999302,998382,990576,995861,993561\n", "Ground Truth: 990811,998593,995526,992334,996298,997732,998039,992684,997969,997112,990089,996834,990529,993642,994964,994783,995126,995679,997410,995906,996001,999099,994925,992120,996873,993831,994293,999302,998382,991200\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 32\n", + "Returned IDs: 993781,999077,996592,998123,997918,993260,994552,998950,990580,995357,991791,990043,991157,992571,998180,998951,991015,991435,994964,995336,996196,993512,999144,994972,996658,991550,990339,992692,999676,993453\n", + "Ground Truth: 993781,999077,996592,992637,998123,997918,993260,999701,994552,998950,990580,995357,991791,990043,991157,992571,993633,995584,998180,998951,991015,991435,994964,995336,996196,993512,999144,994972,996658,990885\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 33\n", + "Returned IDs: 995211,994783,994587,994665,998913,995289,992827,995633,997823,992488,992219,993209,998823,996609,997209,995348,998188,999627,999957,993586,991269,991507,999759,992415,993857,997203,993642,995558,998396,992566\n", + "Ground Truth: 995211,994783,994587,994665,998913,995289,992827,995633,997823,992488,992219,993209,998823,996609,997209,995348,998188,999627,999957,993586,991269,991507,999759,992415,993857,997203,995401,993642,995558,998396\n", "Recall: 0.9667\n", "\n", "Query ID: 34\n", - "Returned IDs: 995130,994659,995332,993808,995750,997273,996128,997542,991961,997412,993347,993079,993304,991990,998860,997442,994871,991343,998891,997199,993931,992567,995107,993499,992358,991061,993324,997555,993548,994410\n", + "Returned IDs: \n", "Ground Truth: 995130,994659,995332,993808,995750,997273,996128,997542,991961,997412,993347,993079,993304,991990,998860,997442,994452,994871,991343,998891,997199,993931,992567,995107,993499,992358,991061,993324,997555,993548\n", - "Recall: 0.9667\n", + "Recall: 0.0000\n", "\n", "Query ID: 35\n", - "Returned IDs: 994397,996075,995274,991422,999289,996976,998011,995336,997571,993168,995374,990701,993280,999596,997759,990585,997976,996108,994432,991125,990862,996714,997304,994916,997377,994894,996368,999437,993167,994100\n", + "Returned IDs: 994397,996075,995274,991422,999289,995336,997571,993168,993280,999596,990585,997976,996108,994432,991125,990862,996714,997304,997377,994894,996368,993167,995502,995555,994100,992880,991721,997015,998779,990990\n", "Ground Truth: 994397,996075,995274,991422,999289,996976,998011,995336,997571,993168,995374,990701,993280,999596,997759,990585,997976,996108,994432,991125,990862,996714,997304,994916,997377,994894,996368,999437,993167,997925\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 36\n", + "Returned IDs: \n", + "Ground Truth: 993381,996395,991513,998586,992721,999994,991945,992136,991610,990991,990177,997145,999555,990167,990292,991987,993122,997149,997009,995257,997506,998006,996623,995655,991038,995780,996571,990330,992444,991835\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 37\n", + "Returned IDs: 998143,991795,993379,995864,996759,993003,998205,997909,992448,996719,997868,993269,996256,994409,991969,997059,996162,998361,993561,991175,994140,998509,993537,998910,996187,996987,991357,994076,999362,991197\n", + "Ground Truth: 998143,991795,993379,995864,996759,993003,998205,997909,992448,996719,997868,993269,996256,994409,991969,997059,996162,998361,993561,991175,994140,998509,993537,998910,996187,996987,991357,994076,999866,999362\n", "Recall: 0.9667\n", "\n", + "Query ID: 38\n", + "Returned IDs: 990210,999374,993860,999331,990545,999210,994964,995438,992147,994336,995928,992944,998302,995927,991743,993018,993484,991801,996805,997320,991498,999898,993877,992169,996219,997002,990083,990096,999122,990843\n", + "Ground Truth: 990210,999374,999338,993860,998160,999331,990545,999210,991720,994964,995438,992147,998294,994336,990648,995928,992944,993154,998302,995927,991743,993018,993484,991801,995075,996805,993238,997320,991498,999898\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 39\n", + "Returned IDs: 993433,997406,998165,995289,992125,993390,996112,996042,994419,992160,992576,996277,997822,991415,998564,996867,995439,992519,992925,996769,998045,997766,991145,992556,996322,997046,994571,990836,998486,995200\n", + "Ground Truth: 993433,997406,998165,995289,992125,993390,992881,996112,996042,994419,995401,992160,992576,996277,997822,991415,998564,996867,995439,992519,992925,996769,998045,997766,991145,996294,992556,996322,997046,994571\n", + "Recall: 0.9000\n", + "\n", "Query ID: 40\n", - "Returned IDs: 993484,997320,998246,996296,997370,995949,997913,993885,990772,990957,997661,998302,990843,991801,993018,991498,990375,998950,993803,995932,993741,994366,997470,990949,991721,996143,991063,994089,993154,995596\n", + "Returned IDs: 993484,997320,998246,996296,997370,995949,996452,997913,993885,990772,990957,997661,998302,990843,991801,993018,991498,990375,998950,993803,995932,993741,994366,997470,990949,991721,996143,994089,993154,995596\n", "Ground Truth: 993484,997320,998246,996296,997370,995949,996452,997913,993885,990772,990957,997661,998302,990843,991801,993018,991498,990375,998950,998160,993803,995932,993741,994366,997470,990949,991721,996143,995265,991063\n", "Recall: 0.9000\n", "\n", "Query ID: 41\n", - "Returned IDs: 996395,990194,999555,998006,997205,997242,999994,999576,990167,994467,993106,998122,990360,990230,993381,997149,999435,996209,991148,992771,993122,997086,998066,991767,993224,998429,994796,994638,995848,992517\n", + "Returned IDs: \n", "Ground Truth: 996395,990194,999555,998006,997205,997242,999994,999576,995167,990167,994467,993106,998122,990360,990230,993381,997149,999435,996209,991148,992771,993122,997086,998066,991767,993224,998429,994796,994638,995848\n", - "Recall: 0.9667\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 42\n", + "Returned IDs: 998951,993536,992646,993492,999144,992330,992969,992909,995833,994670,995336,992452,999089,993544,990275,996646,997918,998465,992183,998011,997192,999374,992384,997875,994100,996592,995793,994403,998401,994767\n", + "Ground Truth: 998951,993536,992646,993492,999144,992330,992969,992909,995833,994670,995336,992452,999089,993544,990275,996646,997918,998465,995374,992183,990171,998011,997192,999374,992384,997875,994100,996592,999343,992932\n", + "Recall: 0.8667\n", "\n", "Query ID: 43\n", - "Returned IDs: 994541,999372,999830,995300,998055,998671,994322,998466,994557,995280,996939,992434,993437,995750,995861,993929,993939,998484,995704,991893,996057,990931,994558,991327,994119,993856,995191,993877,994782,998710\n", + "Returned IDs: 994541,999372,999830,995300,998055,998671,994322,994557,995280,992434,993437,995861,993929,993939,998484,995704,996057,990931,994558,991327,994119,993856,995191,993877,994782,998710,999416,996816,994692,994035\n", "Ground Truth: 994541,999372,999830,995300,998055,998671,994322,998466,994557,995280,996939,992434,993437,995750,995861,993929,993939,998484,995704,991893,996057,990931,991537,994558,991327,994119,993856,995191,993877,994782\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 44\n", + "Returned IDs: 997320,990375,992041,993018,990843,994171,998950,991498,998246,990949,993785,998540,993484,997305,994081,991650,996075,992880,993260,993154,990083,997470,995438,999701,997370,996296,993504,999453,991860,991999\n", + "Ground Truth: 997320,990375,992041,993018,990843,994171,998950,991498,998246,990949,993785,999338,998540,993484,997305,994081,991650,996075,992880,993260,993154,990083,997470,995438,996143,999701,997370,996296,993504,999453\n", + "Recall: 0.9333\n", "\n", "Query ID: 45\n", - "Returned IDs: 996001,990366,995536,998720,999216,992862,998934,993139,996500,993289,993055,993174,990449,994829,994672,997586,990399,993560,997531,995672,995398,994167,999806,999639,997002,998593,990352,990891,999111,993696\n", + "Returned IDs: 996001,990366,995536,998720,999216,992862,998934,993139,996500,993289,993055,993174,990449,994829,994672,997586,990399,993560,995672,995398,994167,999806,999639,997002,998593,995418,990352,990891,999111,993696\n", "Ground Truth: 996001,990366,995536,998720,999216,992862,998934,993139,996500,993289,993055,993174,990449,994829,994672,997586,990399,993560,997531,995672,995398,994167,999806,999639,997002,998593,995418,990352,992617,990891\n", "Recall: 0.9333\n", "\n", "Query ID: 46\n", - "Returned IDs: 996504,998921,997889,990917,998170,997192,994775,992147,990647,991743,995515,993518,996586,991127,999374,994939,996218,990515,991782,993250,992087,998721,999327,990275,992789,998401,994919,999263,995415,995621\n", + "Returned IDs: 996504,998921,997889,998170,997192,994775,992147,990647,991743,995515,993518,996586,991127,999374,994939,996218,990515,991782,993250,992087,998721,999327,990275,998401,994919,999263,995621,995985,991479,993443\n", "Ground Truth: 996504,998921,997889,990917,998170,997192,994775,992147,990647,991743,995515,993518,996586,991127,999374,994939,996218,990515,992379,991782,993250,992087,998721,999327,990275,992789,998401,994919,999263,995415\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 47\n", + "Returned IDs: 991403,999595,990325,993425,994446,999277,995144,997766,990751,997406,998999,990116,994222,995940,996302,995701,991478,998530,995401,997723,991630,991942,991603,996388,995478,995182,998045,995665,990242,999977\n", + "Ground Truth: 991403,999228,999595,990325,993425,994446,999277,995144,996015,993137,997766,990751,997406,998999,996864,990116,994222,995940,996302,995701,991478,998530,995401,997723,991630,991942,991603,995994,996388,995478\n", + "Recall: 0.8333\n", "\n", "Query ID: 48\n", - "Returned IDs: 994925,995873,993561,995895,991958,998382,998495,992448,993906,998593,993053,992521,992120,999294,996219,998106,993637,997386,996001,992847,998649,994742,997379,994419,995200,996397,995401,998039,990435,997822\n", + "Returned IDs: 994925,993561,995895,991958,998382,998495,992448,993906,998593,993053,992521,999294,996219,998106,993637,997386,996001,992847,998649,994742,994419,995200,996397,998039,990435,997822,997529,996042,990096,992944\n", "Ground Truth: 994925,995873,993561,995895,991958,998382,998495,992448,993906,998593,993053,992521,992120,999294,996219,998106,993637,994657,997386,996001,992847,998649,994742,997379,994419,995200,995899,996397,995401,998039\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 49\n", + "Returned IDs: 996523,992558,995552,990989,995599,995972,992411,997054,993534,999715,991550,996606,999860,996196,999564,993427,997523,990279,997681,998220,996655,994384,997547,990181,998417,993792,999540,993474,991953,997739\n", + "Ground Truth: 996523,992558,995552,990989,995599,995972,992411,997054,993534,999715,991550,996606,999860,996196,999564,993427,997523,990279,994006,997681,994404,998220,996655,994384,997547,990181,991595,998417,992141,993792\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 50\n", + "Returned IDs: 991081,992664,998076,993715,998716,994131,997230,993917,997197,997097,998949,996858,991663,992495,991684,996021,998440,990622,992735,998030,996183,998446,999139,991524,996986,995735,997673,993366,998812,993335\n", + "Ground Truth: 991081,992664,998076,993715,998716,994131,997230,993917,997197,997097,998949,996858,991663,992495,991684,996021,998440,990622,992735,998030,996183,998446,994918,999139,991524,998845,998680,991021,996986,995735\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 51\n", + "Returned IDs: 998794,993984,993184,994654,996739,998498,991877,999323,995596,994458,994886,993721,997661,994985,996219,995352,991409,997270,999722,993815,998588,994440,996936,994873,996616,999853,999430,999473,999551,997555\n", + "Ground Truth: 998794,994452,993984,993184,994654,996739,998498,991877,990780,999323,995596,994458,994886,993721,997661,994985,996219,995352,991409,997270,999722,993815,998588,990482,994440,996936,995622,998580,994873,996616\n", + "Recall: 0.8333\n", "\n", "Query ID: 52\n", - "Returned IDs: 990096,998934,996001,998720,997906,998697,993174,998861,995855,994167,997685,996164,993275,993154,999343,996124,995536,999736,996894,999647,995203,992429,999639,999882,998961,998514,994718,993744,999831,990991\n", + "Returned IDs: 990096,998934,996001,998720,997906,998697,993174,998861,994167,997685,996164,993275,993154,996124,995536,999736,995203,992429,999639,998961,994718,993744,999831,999367,990991,996230,999499,998593,996893,995780\n", "Ground Truth: 990096,998934,996001,998720,997906,998697,993174,998861,995855,994167,997685,996164,993275,993154,999343,996124,995536,999736,996894,999647,995203,992429,999639,999882,995401,998961,998514,994718,993744,999831\n", - "Recall: 0.9667\n", + "Recall: 0.7667\n", "\n", "Query ID: 53\n", - "Returned IDs: 996436,992912,997640,990797,998779,991750,997355,993412,999144,991651,993526,993725,995755,999740,997256,994990,994916,994436,994894,990535,990388,999026,994964,996714,994838,999021,991173,999344,997192,999596\n", + "Returned IDs: 996436,992912,997640,990797,998779,991750,997355,993412,999144,993526,993725,995755,999740,997256,994990,995923,994436,994894,990388,999026,994964,996714,994838,999021,991173,997192,999916,990845,999004,995701\n", "Ground Truth: 996436,992912,997640,990797,998779,991750,997355,993412,999144,991651,999587,993526,993725,995755,999740,997256,994990,994916,995923,994436,994894,990535,990388,999026,994964,996714,994838,999021,991173,999344\n", - "Recall: 0.9333\n", + "Recall: 0.8333\n", "\n", "Query ID: 54\n", - "Returned IDs: 997320,999639,993738,990930,990532,995873,997822,996371,995395,996714,992546,990878,993293,992977,992521,991721,995375,999026,990634,996777,994140,991349,990585,992318,991224,995274,995813,993561,996277,999587\n", + "Returned IDs: \n", "Ground Truth: 997320,999639,993738,990930,990532,995873,997822,996371,995395,996714,992546,990878,993293,992977,992521,991721,995375,999026,999527,990634,996777,990814,994140,991349,990585,992318,991224,995274,995813,993561\n", - "Recall: 0.9333\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 55\n", + "Returned IDs: 996627,992079,992680,994751,993205,996411,990528,993725,999302,999708,999740,991382,997933,992230,994783,995633,998055,990985,991669,992652,995456,995786,993053,991464,999126,995756,995076,994472,994567,993240\n", + "Ground Truth: 996627,992079,992680,994751,993205,996411,990528,997207,993725,999302,999708,999740,991382,997933,990743,992230,994783,995633,998055,990985,991669,991537,992652,995456,995786,993053,991464,999126,995756,995076\n", + "Recall: 0.9000\n", "\n", "Query ID: 56\n", - "Returned IDs: 998416,991996,997014,997920,998221,997804,997900,995786,990126,994964,993378,997403,998495,991377,990931,997477,991565,994466,992446,993199,998816,990158,996219,991179,997822,994419,993561,997586,993444,991831\n", + "Returned IDs: \n", "Ground Truth: 998416,991996,997014,997920,998221,997804,997900,995786,990126,994964,993378,997403,998495,991377,996305,990931,997477,991565,994466,992446,993199,998816,990158,996219,991179,997822,994419,993561,997586,993444\n", - "Recall: 0.9667\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 57\n", + "Returned IDs: 994724,998257,995207,999834,995936,998078,991987,992406,990450,996488,991513,995257,992856,999808,998959,996393,990657,998918,999994,992762,998005,992760,997801,995056,993115,999555,999117,993332,992623,990777\n", + "Ground Truth: 994724,998257,995207,999834,995936,998078,991987,992406,990450,996488,991513,995257,993381,992856,999808,998959,993884,996393,990657,998918,999994,992762,998005,992760,997801,996395,995056,993115,999555,999117\n", + "Recall: 0.9000\n", "\n", "Query ID: 58\n", - "Returned IDs: 996196,990159,996690,997224,996523,994384,993670,994372,998286,990303,990824,996655,995503,991550,995290,990989,999860,994950,998411,994964,994912,993158,990538,995599,991458,995410,998697,991791,999716,993809\n", + "Returned IDs: 996196,990159,996690,997224,996523,994384,993670,994372,998286,990303,990824,996655,995503,991550,995290,990989,999860,994950,998411,994964,994912,993158,990538,995599,991458,995410,998697,991791,999716,998230\n", "Ground Truth: 996196,990159,996690,997224,996523,994384,993670,994372,998286,990303,990824,996655,995503,991550,995290,990989,999860,994950,996744,998411,994964,994912,993158,990538,995599,991458,995410,998697,991791,999716\n", "Recall: 0.9667\n", "\n", "Query ID: 59\n", - "Returned IDs: 996280,991451,995374,991650,999698,995357,993721,994030,993215,997002,998951,993154,991545,995181,991997,992557,999900,994850,998961,992880,994449,990532,997448,997918,994362,996685,990083,994081,993512,998779\n", + "Returned IDs: 991451,991650,999698,995357,993721,993215,997002,994226,998951,993154,991997,992557,998961,992880,994449,990532,997918,996685,990083,994081,996178,994743,993018,995401,991535,999453,999831,999442,997571,995372\n", "Ground Truth: 996280,991451,995374,991650,999698,995357,993721,994030,993215,997002,994226,998951,993154,991545,995181,991997,992557,999900,994850,998961,992880,994449,990532,997448,997918,994362,996685,990083,994081,993512\n", - "Recall: 0.9667\n", + "Recall: 0.6667\n", "\n", "Query ID: 60\n", - "Returned IDs: 998224,999496,995831,995196,995407,994956,995087,999138,994817,999244,990678,998779,995295,998691,996328,995861,993725,991390,991319,991378,997732,992713,992703,993678,996918,997714,995020,998067,994587,991192\n", + "Returned IDs: 998224,999496,995831,995196,995407,994956,999138,994817,999244,990678,998779,995295,998691,996328,995861,993725,991319,991378,997732,992713,993678,996918,997714,995020,998067,994587,991192,990392,994119,994765\n", "Ground Truth: 998224,999496,995831,995196,995407,994956,995087,999138,994817,999244,990678,998779,995295,998021,998691,996328,995861,993725,991390,991319,991378,997732,992713,992703,993678,996918,997714,995020,998067,994587\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 61\n", - "Returned IDs: 993738,992651,998831,995701,995184,991836,998697,996534,997059,998060,993280,995079,995873,996828,992609,991530,996657,992293,995640,993948,991175,991007,997897,995216,994488,996001,990005,997554,992052,991576\n", + "Returned IDs: 993738,992651,998831,995701,995184,991836,998697,997059,998060,993280,995079,995873,996828,992609,991530,996657,992293,995640,993948,991175,991007,997897,995216,994488,996001,990005,997554,992052,991576,998549\n", "Ground Truth: 993738,992651,998831,995701,995184,991836,998697,996534,997059,998060,993280,995079,995873,996828,992609,991530,996657,992293,995640,993948,991175,991007,997897,995216,994488,996001,994163,990005,997554,992052\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 62\n", - "Returned IDs: 991157,999122,991015,998607,997305,991720,992718,997192,993143,998180,996193,998123,995515,994046,992052,997496,993260,998045,998170,996504,992147,998950,991848,995401,993077,998921,995408,995372,998106,993948\n", + "Returned IDs: 991157,999122,991015,998607,997305,992718,997192,993143,998180,996193,998123,995515,994046,992052,993260,998170,996504,992147,998950,991848,993077,998921,995408,995372,998106,993609,995154,999676,993781,998827\n", "Ground Truth: 991157,999122,991015,997305,998607,991720,992718,997192,993143,998180,996193,998123,995515,994046,992052,997496,993260,998045,998170,996504,992147,998950,991848,995401,993077,998921,995408,995864,995372,998106\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 63\n", - "Returned IDs: 995059,990264,990516,991590,991960,998999,992415,995020,998525,993293,993079,990521,992318,994435,995359,993180,992171,999981,995421,997319,994817,994697,997718,990610,994349,992239,991245,991867,991775,998961\n", + "Returned IDs: 995059,990264,990516,991590,991960,998999,992415,995020,998525,993293,993079,990521,994435,995359,993180,992171,999981,995421,997319,994697,997718,990610,994349,992239,994409,991245,991867,991775,998961,994040\n", "Ground Truth: 995059,990264,990516,991590,991960,998999,992415,995020,998525,993293,993079,990521,992318,994435,995359,993180,992171,999981,995421,997319,994817,994697,997718,990610,994349,992239,994409,991245,991867,991775\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 64\n", + "Returned IDs: 996184,999634,992754,996357,991089,991005,999722,993335,994958,995191,996794,995131,998794,995880,994370,992628,991382,995313,993479,998716,994769,998192,993073,998949,996153,999430,997331,996016,991554,999920\n", + "Ground Truth: 996184,999634,992754,996357,991089,991005,999722,993335,994958,995191,996794,995131,998794,995880,994370,992628,991382,995313,993479,998716,990402,994769,998192,993073,998949,996153,999430,997331,996016,991554\n", "Recall: 0.9667\n", "\n", "Query ID: 65\n", - "Returned IDs: 996809,998695,991571,993617,998790,999009,998786,991890,995461,990288,996370,999711,999430,991139,998849,990361,990538,995375,999194,990489,994451,997821,993515,992452,990930,996830,995560,991532,994643,993696\n", + "Returned IDs: 996809,998695,991571,993617,998790,999009,998786,991890,995461,990288,996370,999711,999430,998849,990538,995375,999194,990489,994451,997821,993515,995401,990930,996830,995560,991532,994643,993696,995559,996284\n", "Ground Truth: 996809,998695,991571,993617,998790,999009,998786,991890,995461,990288,996370,999711,999430,991139,998849,990361,990538,995375,999194,990489,991052,994451,997821,993515,995401,992452,990930,996830,995560,991532\n", - "Recall: 0.9333\n", + "Recall: 0.8667\n", "\n", "Query ID: 66\n", - "Returned IDs: 999526,997970,991245,999281,991629,997550,994016,997584,996075,993715,992351,992825,995375,995740,992747,993504,995372,998097,997691,999724,993609,992933,994531,996001,995727,992052,994610,992636,990096,999338\n", + "Returned IDs: 999526,997970,991245,999281,991629,997550,994016,997584,996075,993715,992351,992825,995375,995740,992747,993504,995372,998097,999724,993609,994531,996001,992052,992636,990096,992521,999338,999840,995203,994233\n", "Ground Truth: 999526,997970,991245,999281,991629,997550,994016,997584,996075,993715,992351,992825,995375,995740,992747,993504,995372,998097,997691,999724,993609,992933,994531,996001,995727,994610,992052,992636,990096,992521\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 67\n", - "Returned IDs: 992664,994131,998446,998030,998716,997097,991684,991486,993931,997453,996147,991663,998416,991081,996328,998845,998949,996733,998440,993335,998536,992780,998903,998076,997073,993917,997944,996028,991961,992408\n", + "Returned IDs: 992664,994131,998446,998030,998716,991684,991486,993931,997453,996147,991663,998416,991081,996328,998949,996733,998440,993335,998536,998903,996830,998076,997073,993917,997944,991961,992408,993715,996858,990712\n", "Ground Truth: 992664,994131,998446,998030,998716,997097,991684,991486,993931,997453,996147,991663,998416,991081,996328,998845,998949,996733,998440,993335,998536,998903,992780,996830,998076,997073,993917,997944,996028,991961\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 68\n", + "Returned IDs: 997547,992736,999908,998933,990538,995737,992824,994165,993706,997215,992170,990273,991070,995372,990192,997914,997775,999639,991432,991272,990279,995459,996196,993055,993611,993077,990885,996380,990930,997614\n", + "Ground Truth: 997547,992736,999908,998933,990538,995737,992824,994165,993706,997215,992170,990273,991070,995372,990192,997914,997775,999639,991432,997506,991272,990279,995459,996196,993055,993611,993077,997301,997821,990885\n", + "Recall: 0.9000\n", "\n", "Query ID: 69\n", - "Returned IDs: 994100,993492,991673,996805,992330,995336,998495,994334,995833,995968,990231,998921,992944,990271,993512,995357,996001,995274,998090,993154,994088,995536,995826,996282,995401,990648,996219,995260,994449,997605\n", + "Returned IDs: 994100,993492,991673,996805,992330,995336,998495,995833,995968,990231,998921,992944,993512,995357,995748,996001,995274,998090,993154,995536,996219,997605,995200,997918,996790,996592,990957,999544,992443,992318\n", "Ground Truth: 994100,993492,991673,996805,992330,995336,998495,994334,995833,995968,990231,998921,992944,990271,996769,993512,995357,995748,996001,995274,998090,993154,994088,995536,995826,996282,995162,995401,990648,996219\n", - "Recall: 0.9000\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 70\n", + "Returned IDs: 990086,998859,995851,996991,998748,997031,993908,998181,999805,990903,995268,997185,995778,994991,995106,993320,993466,998980,994945,991815,990435,999402,992805,990751,993883,998073,994759,995086,995324,998067\n", + "Ground Truth: 990086,992540,998859,995851,996991,998748,997031,993908,998181,999805,990903,995268,997185,995778,994991,995106,993320,993466,998980,994945,991815,990435,999402,992805,990751,993883,998073,994759,995086,995324\n", + "Recall: 0.9667\n", "\n", "Query ID: 71\n", - "Returned IDs: 995841,995375,995372,992944,999711,996318,990538,990043,994972,998726,991052,995401,990585,997301,998445,996193,999544,991977,991540,997320,995560,996001,998180,995268,995138,996609,993533,999526,999019,995515\n", + "Returned IDs: 995841,995375,995372,992944,999711,996318,990043,994972,990585,997301,996193,999544,991977,997320,995560,996001,998180,995268,998097,995138,996609,992384,995515,991743,996219,992047,998401,993324,998921,990334\n", "Ground Truth: 995841,995375,995372,992944,999711,996318,990538,990043,994972,998726,991052,995401,990585,997301,998445,996193,999544,991977,991540,997320,995560,996001,998180,995268,998097,995138,996609,992384,993533,999526\n", - "Recall: 0.9333\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 72\n", + "Returned IDs: 993738,995395,992790,990930,993409,997928,997320,991576,991349,997027,990784,996714,995007,992036,997218,997675,997168,996247,995701,998132,996918,994488,995521,992179,990561,998303,994170,994906,998723,995268\n", + "Ground Truth: 993738,995395,992790,990930,993409,997928,997320,991576,991349,997027,990006,990784,996714,995007,999820,992036,997218,995375,997675,997168,996247,995701,998132,995807,996918,994488,995521,992179,990561,998303\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 73\n", + "Returned IDs: 993113,994817,992832,991629,999827,993324,992636,994119,993917,995332,993533,997357,990993,990889,990392,996986,990814,992567,997309,993715,992026,990289,995746,993456,993304,999796,999791,992764,993742,995012\n", + "Ground Truth: 993113,996810,994817,992832,991629,999827,990468,999942,993324,992636,994119,993917,995332,993533,997357,990993,990889,990392,996986,990814,992567,997309,993715,992026,990289,995970,998848,995746,993456,993304\n", + "Recall: 0.8333\n", "\n", "Query ID: 74\n", - "Returned IDs: 994102,994831,998764,995372,997913,991362,999982,991335,995374,998702,992880,992291,990585,993852,996143,995828,990724,992748,994618,990949,997470,992114,990957,990275,996959,998500,998744,997192,996785,997320\n", + "Returned IDs: \n", "Ground Truth: 994102,994831,998764,995372,997913,991362,999982,991335,995374,998702,992880,992291,990585,993852,996143,995828,990538,990724,992748,994618,990949,997470,992114,990957,990275,996959,998500,998744,997192,996785\n", - "Recall: 0.9667\n", + "Recall: 0.0000\n", "\n", "Query ID: 75\n", - "Returned IDs: 998981,998287,997320,998246,994225,996296,992902,994889,997178,994450,991155,990020,992880,994214,994249,997470,995116,998540,993484,992748,994542,998634,993529,996075,996085,995700,999132,990742,994964,995400\n", + "Returned IDs: 998981,998287,997320,998246,994225,996296,994889,997178,994450,991155,990020,992880,994214,994249,997470,998540,993484,992748,994542,993529,996075,996085,994416,990742,994964,995400,998550,993662,992811,990532\n", "Ground Truth: 998981,998287,997320,998246,994225,996296,992902,994889,997178,994450,991155,990020,994214,992880,994249,997470,995116,998540,993484,992748,994542,998634,993529,996075,996085,995700,994416,999132,990742,994964\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 76\n", + "Returned IDs: 991212,995167,990557,993855,996001,994351,999880,990712,995214,996328,990752,993931,999435,993715,994280,990705,990314,999502,997897,997790,992864,993181,995873,991257,997002,991605,992726,991098,991184,999139\n", + "Ground Truth: 991212,995167,990557,993855,993981,996001,994351,999880,990712,995214,996328,990752,993931,997860,999435,993715,994280,990705,990314,999502,997897,997790,992864,993181,995873,991257,997002,991605,992726,991098\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 77\n", + "Returned IDs: 992713,998671,999830,994541,998279,990931,998256,994142,990705,991319,996189,990364,996328,992445,991327,991058,994891,998543,999191,994035,996816,991682,995191,996647,990712,997967,994098,999179,999775,999458\n", + "Ground Truth: 997669,998466,992713,998671,999830,994541,998279,990931,998256,994142,990705,991319,996189,990364,996328,992445,991327,991058,994891,998543,999191,994035,996816,997933,991682,995191,992780,996647,990712,997967\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 78\n", + "Returned IDs: 995618,990124,996158,996367,992306,993534,994398,994028,997505,996953,994080,999908,999022,996585,997467,998357,996606,992092,994760,991977,993427,995965,993583,990308,994679,992558,994011,993681,998652,992857\n", + "Ground Truth: 995618,990124,996158,996367,992306,993534,994398,994028,997505,996953,994080,999908,999022,996585,997467,993323,998357,996606,992092,994760,991977,993427,995965,993583,990308,994679,992558,994011,993491,993681\n", + "Recall: 0.9333\n", "\n", "Query ID: 79\n", - "Returned IDs: 992931,997843,992514,990742,996993,991260,998764,997483,995570,990478,992961,993484,991131,997913,990949,997320,998195,999982,992748,999838,999855,998653,992105,992781,991872,992069,995277,991752,995336,997307\n", + "Returned IDs: 992931,997843,992514,996993,991260,998764,997483,995570,993484,991131,997913,990949,997320,996378,998195,999982,992748,998653,992105,992781,991872,992069,991752,995336,995596,995261,993492,994757,992980,993154\n", "Ground Truth: 992931,997843,992514,990742,996993,991260,998764,997483,995570,990478,992961,993484,991131,997913,990949,997320,996378,998195,999982,992748,999838,999855,998653,992105,992781,991872,992069,995277,991752,995336\n", - "Recall: 0.9667\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 80\n", + "Returned IDs: 995841,995372,996785,997301,994964,996959,997796,991444,998921,997320,999374,990560,990585,992147,996218,998839,995949,995886,994972,991743,998797,992066,996805,990925,992944,998302,997015,996052,999263,997192\n", + "Ground Truth: 995841,995372,996785,998445,997301,994964,996959,997796,991444,998921,997320,999374,998644,990560,999533,990585,992147,993238,992789,996218,998839,993648,995949,996090,995886,994972,991743,998797,992066,996805\n", + "Recall: 0.7667\n", "\n", "Query ID: 81\n", - "Returned IDs: 990146,993237,992085,997747,994318,992944,991727,992342,991392,995443,990142,993253,997591,997855,997585,990742,991121,994789,999784,998782,990822,995334,994534,993570,990615,998550,999676,999948,994366,995693\n", + "Returned IDs: 990146,993237,992085,997747,994318,992944,991727,991392,995443,990142,993253,997591,997585,990742,991121,994789,999784,998782,990822,994534,993570,990615,998550,990447,999676,999948,990650,992218,998139,990648\n", "Ground Truth: 990146,993237,992085,997747,994318,992944,991727,992342,991392,995443,990142,993253,997591,997855,997585,990742,991121,994789,999784,998782,990822,995334,994534,993570,990615,998550,990447,999676,999948,996352\n", - "Recall: 0.9333\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 82\n", + "Returned IDs: 998874,996505,992521,996001,995110,997675,998536,995521,996715,990537,995873,990930,993931,996028,997067,996918,998593,993289,998397,997822,995268,992703,995536,996368,992488,992448,994380,996930,995701,992790\n", + "Ground Truth: 998874,996505,992521,995401,996001,995110,997675,998536,995521,996715,990537,995873,990930,995130,993931,996028,997067,996918,991459,998593,993289,998397,997822,995268,992703,995536,991052,996368,992488,992448\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 83\n", + "Returned IDs: 998779,994436,997640,997536,991028,991350,990406,991603,995702,997058,994161,997787,996379,997355,991958,998165,996388,994396,995052,990845,991011,993813,996719,990984,996042,993040,993831,997822,999196,996125\n", + "Ground Truth: 998779,994436,997640,997536,991028,991350,990406,991603,995702,997058,994161,997787,996379,997355,991958,998165,996388,994396,995052,990845,991011,993813,996719,990984,998999,996042,993040,993831,997822,999196\n", + "Recall: 0.9667\n", "\n", "Query ID: 84\n", - "Returned IDs: 997913,994964,997371,993237,995515,994778,991801,996727,996811,990142,996840,991727,995928,998359,996001,996438,994277,994415,997593,997291,992140,993143,994971,996592,999290,995886,990580,997320,997370,994360\n", + "Returned IDs: 997913,994964,997371,993237,995515,994778,996727,996811,990142,996840,991727,995928,998359,996001,996438,994277,994415,997593,997291,999426,993143,996592,999290,995886,990580,997320,997370,994360,997301,990066\n", "Ground Truth: 997913,994964,997371,993237,995515,994778,994475,991801,996727,997350,996811,990142,996840,991727,995928,998359,996001,996438,994277,994415,997593,997291,999426,992140,993143,994971,996592,999290,995886,990580\n", - "Recall: 0.9000\n", + "Recall: 0.8333\n", "\n", "Query ID: 85\n", - "Returned IDs: 995207,991701,996395,998586,991987,996623,992136,995056,999117,995373,994063,990292,994413,998858,998257,994451,998066,991052,994187,999249,994092,999096,997110,998659,992604,993115,997918,999834,994753,993515\n", + "Returned IDs: 995207,991701,996395,998586,991987,996623,992136,995056,999117,995373,994063,990292,994413,998858,998257,994451,998066,991052,999249,994092,999096,997110,998659,992604,993115,997918,999834,994753,992638,999101\n", "Ground Truth: 995207,991701,996395,998586,991987,996623,992136,995056,999117,995373,994063,990292,994413,998858,998257,994451,998066,991052,994187,999249,994092,995617,999096,997110,998659,992604,993115,997918,999834,994753\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 86\n", + "Returned IDs: 992752,998933,995011,995945,993037,998536,993931,990407,993055,995421,998341,992437,998250,998594,999139,996483,997926,998593,991245,998303,997019,991530,999111,994463,997506,996775,991633,991552,994259,996028\n", + "Ground Truth: 992752,998933,995011,995945,993037,998536,993931,991291,990407,993055,995421,998341,992437,998250,998594,999139,996483,997926,998593,991245,998303,997019,999265,991530,999111,994463,997506,996775,997531,991633\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 87\n", + "Returned IDs: 990146,991498,999263,998302,992994,990560,999533,994964,998644,998921,997301,998445,991444,990958,995306,995771,990925,993885,990096,999374,996785,992147,992546,994281,994922,994978,991406,999197,990224,996504\n", + "Ground Truth: 990146,997876,991498,999263,998302,992994,990560,999533,994964,993484,998644,998921,997301,998445,991444,990958,995306,995771,990925,993885,992814,990096,999374,996785,992147,992546,994281,994922,994978,991406\n", + "Recall: 0.9000\n", "\n", "Query ID: 88\n", - "Returned IDs: 992931,995266,995319,994052,995942,997147,991616,991593,992954,997913,990922,992964,991250,993031,990114,993606,999559,995632,998899,996508,999945,994668,992255,993278,998562,995984,990949,992228,998619,998056\n", + "Returned IDs: 992931,995319,995942,997147,991593,997913,990922,992964,991250,993031,990114,993606,999559,995632,998899,996508,999945,994668,992255,993278,995984,990949,992228,998619,991502,990742,994633,997946,991456,995774\n", "Ground Truth: 992931,995266,995319,994052,995942,997147,991616,991593,992954,997913,990922,992964,991250,993031,990114,993606,990653,996629,999559,995632,998899,996508,999945,994668,992255,993278,998562,995984,990949,992228\n", - "Recall: 0.9333\n", + "Recall: 0.7667\n", "\n", "Query ID: 89\n", - "Returned IDs: 995755,997437,993484,995244,998649,994894,996630,992296,997320,999740,991999,998246,991991,994814,995435,994359,999599,997304,999708,993696,996053,996436,997305,995841,990949,990786,996075,992924,993410,990307\n", + "Returned IDs: 995755,997437,993484,995244,998649,994894,992296,997320,999740,998246,991991,994814,995435,994359,999599,997304,998855,993696,996053,997305,995841,990949,990786,996075,999867,992924,990307,993529,993818,990957\n", "Ground Truth: 995755,997437,993630,993484,995244,998649,994894,996630,992296,997320,999740,991999,998246,991991,994814,995435,994359,999599,997304,998855,999708,993696,996053,996436,997305,995841,990949,990786,993412,996075\n", - "Recall: 0.9000\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 90\n", + "Returned IDs: 999451,997739,994852,997903,990376,992558,996693,995983,997301,995580,996894,998686,991923,999807,999639,997106,992383,998600,994195,994979,998079,999561,996262,992423,992718,990383,993283,996196,992521,994163\n", + "Ground Truth: 999451,997739,994852,997903,990376,992558,996693,995983,997301,995580,996894,998686,991923,999807,999639,997106,992383,998600,992612,994195,994979,998079,999561,996262,992423,992718,990383,993283,996196,992521\n", + "Recall: 0.9667\n", "\n", "Query ID: 91\n", - "Returned IDs: 998105,994307,993053,997790,991021,993906,996028,995823,991451,996147,990096,995586,995295,993289,992480,995456,995401,998677,999444,994689,996062,996001,991685,994280,992408,990532,992072,993433,998382,993561\n", + "Returned IDs: 998105,994307,993053,997790,991021,993906,996028,995823,996147,990096,995586,995295,993289,992480,995456,998677,999444,994689,996062,996001,991685,994280,992408,990532,992072,993433,998382,993561,994531,993809\n", "Ground Truth: 998105,994307,993053,997790,991021,993906,996028,995823,991451,996147,990096,995586,995295,993289,992480,995456,995401,998677,999444,994689,996062,996001,991685,994280,993778,992408,990532,992072,993433,998382\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 92\n", + "Returned IDs: 991148,992551,993472,990194,996216,995895,998122,998495,992892,997506,992209,990096,991411,997442,993184,995798,994661,999576,991382,997685,994682,996184,991881,993460,995535,992780,999430,990091,996001,999139\n", + "Ground Truth: 991148,992551,994355,993472,990194,996216,995895,999336,998122,998495,992892,997506,992209,990096,991411,997442,993184,995798,994661,999576,993121,991382,997685,994682,996184,991881,993460,997196,995535,992780\n", + "Recall: 0.8667\n", "\n", "Query ID: 93\n", - "Returned IDs: 998779,991743,996727,993412,996976,993673,991268,992087,995274,997640,999026,998649,999074,992511,997355,995034,992685,998862,997429,992147,998495,990406,994301,995616,997799,992637,992630,995755,996918,990949\n", + "Returned IDs: 998779,991743,996727,993412,996976,992087,995274,997640,999026,998649,999074,992511,997355,995034,992685,998862,992147,998495,990406,992630,995755,996918,990949,999179,990606,994894,990957,999021,996890,990925\n", "Ground Truth: 998779,991743,996727,993412,996976,990401,993673,991268,992087,995274,997640,999026,998649,999074,992511,997355,995034,992685,998862,997429,992147,998495,990406,994301,995616,996438,997799,992637,992630,995755\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 94\n", + "Returned IDs: 993785,997661,998921,992041,995515,995250,999806,995596,998498,992119,993447,990697,997550,997305,996075,992635,991984,991157,995372,994081,996001,993257,997320,993904,995203,996755,999426,993820,992944,996805\n", + "Ground Truth: 993785,997661,998921,992041,995515,995250,999806,995596,998498,992119,993447,990697,997550,997305,996075,992635,993022,991984,991157,992384,995372,994081,996001,993257,995622,997320,993904,995203,996755,999426\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 95\n", + "Returned IDs: 994964,993143,997320,996001,996657,996500,999219,992384,996219,999343,995928,991459,999144,991392,990999,999653,998303,996028,990366,998246,993174,993168,990771,998593,996576,998302,994280,996368,996147,993484\n", + "Ground Truth: 994964,993143,995855,997320,996001,996657,996500,999219,997920,992384,996219,999343,998029,995928,991459,999144,991392,990999,999653,998303,996028,990366,998246,993174,993168,990771,998593,996576,998302,994280\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 96\n", + "Returned IDs: 993741,992041,996909,991650,993785,995596,996741,990697,997836,992671,992635,999426,997661,994971,991372,995515,993904,996075,996875,997305,995372,990142,992733,998227,999439,993022,998498,990914,997291,994415\n", + "Ground Truth: 993741,992041,996909,991650,993785,995596,996741,990697,991218,997836,992671,992635,999426,997661,994971,991372,995515,993904,996075,996875,997305,995372,990142,992733,998227,999439,995995,993022,998498,990914\n", "Recall: 0.9333\n", "\n", + "Query ID: 97\n", + "Returned IDs: 995357,991743,996658,994197,997571,998011,995336,994819,995274,993145,991131,991094,996976,996727,998779,990647,998495,999470,991361,992646,996714,999144,994334,994100,992718,990231,999089,991908,999021,999331\n", + "Ground Truth: 995357,995374,991743,996658,994197,997571,998011,995336,994819,995274,993145,991131,991094,996976,996727,998779,990647,998495,999470,991361,993096,992646,996714,995873,999144,994334,994100,992718,990231,999089\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 98\n", + "Returned IDs: \n", + "Ground Truth: 990712,996495,994351,999388,990705,993224,994481,993801,995214,991633,992726,994956,996728,996328,995754,996566,992445,994310,994378,992780,992154,990194,991296,994047,997086,993931,999852,992429,998389,996216\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 99\n", + "Returned IDs: 998697,999129,998536,991530,995536,993609,995672,992862,999639,995780,992268,991332,999806,997506,992617,990399,993055,995562,997775,990096,998303,999744,996001,995237,992423,996769,991157,994167,993342,998045\n", + "Ground Truth: 998697,999129,998536,991530,995536,993609,995672,992862,999639,995780,992268,991332,999806,997506,992617,990399,991630,993055,995562,999587,997775,990096,998303,996368,999744,995401,996001,995864,995237,992167\n", + "Recall: 0.8000\n", + "\n", "Query ID: 100\n", - "Returned IDs: 996351,998837,994703,998363,991193,999738,993708,994789,995515,997496,993896,992384,995489,991840,990138,990490,991609,993143,990134,999453,991485,999613,990809,998657,998380,994001,992611,993577,994922,990310\n", + "Returned IDs: 996351,998837,994703,998363,991193,999738,993708,994789,995515,993896,995489,991840,990138,991609,993143,999453,991485,999613,998380,992611,993577,994922,990310,993237,991999,995988,995267,995072,999676,995443\n", "Ground Truth: 996351,998837,994703,998363,991193,999738,993708,994789,995515,997496,993896,992384,995489,991840,990138,990490,991609,993143,990134,999453,991485,999613,990809,998657,998380,994001,992611,993577,994922,991930\n", - "Recall: 0.9667\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 101\n", + "Returned IDs: 993813,991790,991163,996077,991466,997046,999731,990940,993561,997630,997289,997386,990751,998633,995679,998165,997822,994409,992919,990930,993293,993738,991958,999037,998593,995401,990634,995701,994140,992703\n", + "Ground Truth: 993813,991790,991163,996077,991466,997046,999731,990940,993561,990878,997630,997289,997386,999000,990751,998633,995679,998165,997822,994409,992919,990930,991603,990820,993293,993738,991958,999037,998593,995401\n", + "Recall: 0.8667\n", "\n", "Query ID: 102\n", - "Returned IDs: 999639,998793,992824,996196,997775,994270,992306,990538,992802,999908,996742,994560,990084,997547,992030,995631,994305,991923,995955,997215,990720,999451,997864,990755,993077,999564,992190,991432,992458,991910\n", + "Returned IDs: 999639,998793,992824,996196,997775,994270,992306,990538,992802,999908,996742,994560,990084,997547,992030,995631,994305,995955,997215,990720,997864,990755,993077,999564,992190,991432,992458,991910,992558,994614\n", "Ground Truth: 999639,998793,992824,996196,997775,994270,992306,990538,992802,999908,996742,994560,990084,997547,992030,995631,994305,991923,995955,997215,990720,999451,997864,990755,993077,999564,992190,990409,991432,990146\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 103\n", + "Returned IDs: 998105,999460,995906,990536,994419,997453,996974,990497,996062,998039,993906,993053,994307,992201,995823,996028,996647,991989,992622,993831,992408,991021,993561,992072,991958,990076,995295,997969,991168,990097\n", + "Ground Truth: 998105,999460,995906,990536,994419,997453,996974,990497,999083,996062,998039,993906,993053,994307,992201,995823,996028,996647,991989,992622,993831,992408,991021,993412,993561,992072,991958,990076,995295,997969\n", "Recall: 0.9333\n", "\n", "Query ID: 104\n", - "Returned IDs: 997554,999453,995216,998495,996147,990845,996001,999402,992979,993412,994531,996368,991585,996810,991790,993003,992792,992334,990441,993561,991943,991967,996759,990662,992364,997031,993289,996911,995289,999866\n", + "Returned IDs: 997554,999453,995216,997937,998495,996147,990845,996001,999402,992979,996368,991585,996810,991790,992792,990441,993561,991967,996759,990662,992364,997031,993289,996911,995289,992521,996657,998060,995963,996987\n", "Ground Truth: 997554,999453,995216,997937,998495,996147,990845,996001,999402,992979,993412,994531,998166,996368,991585,996810,991790,993003,992792,992334,990441,993561,991943,991967,996759,990662,992364,997031,993289,996911\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", "\n", "Query ID: 105\n", - "Returned IDs: 991873,997913,991090,993834,999982,992822,992096,992228,998500,991626,995802,993796,997483,991616,992066,999165,992742,997320,990650,993616,997470,995513,996959,990182,997321,997759,990588,999710,998090,995949\n", + "Returned IDs: 997913,991090,993834,999982,999247,992228,998500,993048,991626,995802,993796,997483,992066,992742,997320,990650,997470,995513,996959,990182,997759,999710,998090,991814,992443,991229,991250,998808,996785,990910\n", "Ground Truth: 991873,997913,991090,993834,999982,992822,999247,992096,992228,998500,993048,991626,995802,993796,997483,991616,992066,999165,992742,997320,990650,993616,997470,995513,996959,990182,997321,997759,990588,999710\n", - "Recall: 0.9333\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 106\n", + "Returned IDs: 997320,992944,990772,991498,996001,999710,990949,991131,996075,992041,993484,995336,998550,995372,992443,997305,992748,992119,994100,996085,993388,996318,994964,990585,998246,990957,993785,999374,997661,994415\n", + "Ground Truth: 997320,992944,990772,991498,996001,999710,990949,991131,996075,992041,993484,998365,995336,998550,995372,992443,997305,992748,992119,994100,992692,998232,996085,993388,996318,994964,994449,990585,998246,990957\n", + "Recall: 0.8667\n", "\n", "Query ID: 107\n", - "Returned IDs: 994916,995274,994964,999144,999026,997015,999437,998779,992630,990535,993289,998951,993168,996959,997918,992047,994197,993492,991467,992216,995755,999596,992384,999374,997002,993484,990958,995336,990957,997320\n", + "Returned IDs: 995274,994964,999144,999026,997015,999437,998779,992630,990535,993289,998951,993168,996959,997918,994197,993492,991467,992216,995755,992384,999374,997002,993484,990958,995336,990957,997320,997976,992558,998797\n", "Ground Truth: 994916,995274,994964,999144,999026,997015,999437,998779,992630,990535,993289,998951,993168,996959,997918,992047,997920,994197,993492,991467,999916,992216,995755,999596,992384,999374,997002,993484,990958,995336\n", - "Recall: 0.9333\n", + "Recall: 0.8333\n", "\n", "Query ID: 108\n", - "Returned IDs: 994379,997319,991973,996411,990223,995611,990675,995819,991810,997822,991943,995289,994382,993909,993831,998156,995456,991350,992561,997732,993561,995633,994783,998750,990889,996906,995020,998765,997797,990277\n", + "Returned IDs: 994379,997319,996411,990223,995611,990675,995819,991810,997822,991943,995289,994382,993831,998156,995456,991350,992561,993561,995633,994783,998750,990889,995020,998765,990277,992582,994742,990516,995702,994283\n", "Ground Truth: 994379,997319,991973,996411,990223,995611,990675,995819,999627,991810,997822,991943,995289,994382,993909,993831,998156,995456,991350,992561,997732,993561,995633,994783,998750,996607,990889,996906,995020,998765\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", "\n", "Query ID: 109\n", - "Returned IDs: 999898,997028,996996,991311,994541,993695,997410,997920,993472,990974,997345,991339,998302,994783,996939,999343,998169,998039,998122,990210,993807,995384,999009,996062,996219,990505,997878,991776,992120,996216\n", + "Returned IDs: 999898,997028,996996,991311,994541,993695,997410,997920,993472,990974,997345,991923,998302,994783,999343,998169,998039,998122,993807,995384,996062,996219,990505,997878,991776,992120,996216,995895,997320,999876\n", "Ground Truth: 999898,997028,996996,991311,994541,993695,997410,997920,993472,990974,997345,991339,991923,998302,994783,996939,999343,998169,998039,998122,990210,992931,993807,995384,999009,996062,996219,990505,995620,997878\n", - "Recall: 0.9000\n", + "Recall: 0.8000\n", "\n", "Query ID: 110\n", - "Returned IDs: 997028,990557,996001,996219,999948,994777,998039,991498,999058,998765,995053,991930,994964,992446,993885,990752,994487,995102,997876,991523,991776,991212,997320,997332,996318,992182,991184,991921,995216,999617\n", + "Returned IDs: 997028,990557,996001,996219,999948,998039,991498,999058,998765,991930,994964,992446,993885,990752,994487,995102,991523,991776,991212,997320,997332,996318,992182,991184,995216,997920,995107,997410,999344,991994\n", "Ground Truth: 997028,990557,996001,997381,996219,999948,994777,998039,991498,999058,998765,994061,995053,991930,995424,994964,992446,993885,990752,994487,995102,997876,991523,991776,991212,997320,997332,996318,992182,991184\n", - "Recall: 0.9000\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 111\n", + "Returned IDs: 996328,991882,990133,992450,990531,998067,995326,998610,997529,995295,997860,993939,994310,991769,990712,998710,990027,997103,994830,994742,996301,990435,993079,991885,996001,992703,996752,991585,993856,998671\n", + "Ground Truth: 996328,991882,990133,992450,990531,998067,995326,998610,997529,995295,997860,993939,994310,991769,990712,998710,990027,997103,994830,997933,994742,996301,990435,993079,991885,996001,992703,996752,996611,991585\n", + "Recall: 0.9333\n", "\n", "Query ID: 112\n", - "Returned IDs: 995981,992470,992874,992384,995701,995792,990862,996001,998721,993432,993848,998246,991045,990588,998445,995467,994879,998549,993342,998921,995158,992167,997984,991459,997437,991245,996714,995336,995886,996075\n", + "Returned IDs: 995981,992470,992874,992384,995701,990862,996001,998721,993848,999904,998246,991045,990588,998445,995467,994879,998549,993342,999977,998921,995158,992167,997984,991459,997437,991245,996714,995336,995886,996075\n", "Ground Truth: 993154,995981,992470,992874,992384,995701,995792,990862,996001,998721,993432,993848,999904,998246,991045,990588,998445,995467,994879,998549,993342,999977,998921,995158,997984,992167,991459,997437,991245,996714\n", "Recall: 0.9000\n", "\n", "Query ID: 113\n", - "Returned IDs: 996425,994819,994334,995453,997918,994804,994088,996046,997002,993465,996903,994362,999964,990995,995200,999277,995373,993407,995444,995374,994446,999089,999595,991603,990791,995928,998530,991942,997637,996864\n", + "Returned IDs: 996425,994819,994334,995453,997918,994804,994088,996046,997002,993465,996903,990995,995200,999277,995373,993407,995444,994446,999089,999595,990791,995928,998530,991942,997637,996864,992933,998259,990459,995401\n", "Ground Truth: 996425,994819,994334,995453,997918,994804,994088,996046,997002,993465,996903,994362,999964,990995,995200,999277,995373,995444,993407,995374,994446,999089,999595,991603,990791,995928,998530,991942,997637,992260\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 114\n", + "Returned IDs: 991721,990096,996237,999444,996368,998362,995016,991154,990856,998039,995899,991869,994964,994925,999781,993168,990930,991958,997320,994961,998862,992994,990925,998950,992546,995701,993389,995678,995126,990811\n", + "Ground Truth: 991721,990636,990096,996237,999444,996368,998362,995016,991154,990856,998039,995899,991869,994964,994925,999781,993168,997053,990930,991958,997320,994961,998862,996661,992994,996677,997920,994140,990925,998950\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 115\n", + "Returned IDs: 995941,991782,996592,999089,993145,997096,995793,995336,990647,999200,990180,998572,991456,990275,990994,994111,999700,991379,995465,995886,999374,997748,994757,997371,997194,997479,997571,997022,996521,995453\n", + "Ground Truth: 995941,991782,996592,999089,993145,997096,995793,995336,990917,990647,999200,990180,998572,991456,990275,990994,994281,994111,999700,991379,995465,995886,999374,997748,990478,994757,997371,997194,997479,997571\n", + "Recall: 0.9000\n", "\n", "Query ID: 116\n", - "Returned IDs: 998523,994922,992907,993355,996840,990043,993237,990334,997876,990332,990448,995004,999453,999620,997918,990432,990925,991510,996196,990538,990483,992612,996438,991609,997496,998401,999076,998837,993143,996402\n", + "Returned IDs: 998523,994922,992907,993355,996840,990043,993237,990334,990332,990448,995004,999453,999620,997918,990432,990925,991510,996196,990483,992612,996438,997496,998401,999076,998837,993143,996402,999676,996915,993570\n", "Ground Truth: 998523,994922,992907,993355,996840,990043,993237,990334,997876,990332,990448,995004,999453,999620,997918,990432,990925,991510,996196,990538,990483,992612,996438,991609,997496,998401,995401,999076,998837,993143\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 117\n", - "Returned IDs: 996194,999601,998823,993014,994541,995695,990521,990169,995421,996396,994119,996429,998934,994280,995401,999496,994051,998546,994817,997316,990011,998250,992052,998968,991245,992313,992131,996633,998817,992437\n", + "Returned IDs: 996194,999601,998823,993014,994541,995695,990521,990169,995421,996396,994119,996429,998934,994280,995401,999496,994051,997316,990011,998250,992052,998968,991245,992313,996633,998817,992437,993335,996435,998067\n", "Ground Truth: 996194,999601,998823,993014,994541,995695,990521,990169,995421,996396,994119,998348,996429,998934,994280,995401,999496,994051,998546,994817,997316,990011,998250,992052,998968,991245,992313,992131,996633,998817\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 118\n", + "Returned IDs: \n", + "Ground Truth: 995961,994850,994081,997470,998839,997320,998170,990949,991379,997301,994290,995374,990066,995422,998500,990910,994100,991743,993852,994757,995372,993646,998090,999647,998950,990532,995949,992228,996629,997479\n", + "Recall: 0.0000\n", "\n", "Query ID: 119\n", - "Returned IDs: 995614,993426,990191,997873,997820,996516,997292,990855,994415,998921,997483,998721,996387,999109,999271,994618,995623,996296,996771,999792,994233,992853,991131,995306,993484,993785,995119,998445,996135,997550\n", + "Returned IDs: 995614,993426,990191,997820,996516,997292,990855,994415,998921,997483,998721,996387,999109,999271,994618,995623,996993,996771,999792,994233,992853,991131,995306,993484,993785,995119,998445,996135,997550,999036\n", "Ground Truth: 995614,993426,990191,997873,997820,996516,997292,990855,994415,998921,997483,998721,996387,999109,999271,994618,995623,996296,996993,996771,999792,994233,992853,991131,995306,993484,993785,995119,998445,996135\n", - "Recall: 0.9667\n", - "\n", - "Query ID: 127\n", - "Returned IDs: 998634,996001,997320,998246,993484,992839,991787,993053,994964,993280,991155,994961,997332,999099,990585,990146,993168,993885,991498,995678,999453,991721,996219,990772,990096,993389,990269,991732,998927,991609\n", - "Ground Truth: 998634,996001,997320,998246,993484,992839,991787,993053,994964,993280,991155,994961,997332,999099,990585,990146,993168,993885,991498,995678,999453,991721,991184,996219,990772,990096,993389,993154,990269,991732\n", "Recall: 0.9333\n", "\n", - "Query ID: 128\n", - "Returned IDs: 997809,996546,991923,992718,998975,999380,993681,998331,991977,996163,990908,994398,993786,995138,993572,995986,994852,996658,992170,996158,995578,991157,999387,993975,998026,996894,995737,992349,996585,999806\n", - "Ground Truth: 997809,996546,991923,992718,998975,999380,993681,998331,991977,996163,990908,994398,993786,995138,993572,995986,994852,996658,992170,996158,995578,991157,998921,999387,993975,998026,996894,995737,992349,996585\n", - "Recall: 0.9667\n", + "Query ID: 120\n", + "Returned IDs: 996658,993288,992933,997391,997309,997918,996282,990956,997575,995080,995421,997897,996633,995862,991666,994455,996867,992231,995357,994409,991950,996878,997125,993023,995159,992052,994073,999303,999660,990521\n", + "Ground Truth: 996658,991864,993288,992933,997391,997309,997918,993094,996282,990956,997575,995080,995421,997897,996633,993211,995862,991303,991666,994455,996867,992231,995357,994409,991950,997462,992779,996878,994300,997125\n", + "Recall: 0.7667\n", "\n", - "Query ID: 129\n", - "Returned IDs: 992255,990650,996876,995794,990114,992964,992822,991873,998947,997451,999948,995485,997102,997948,998003,993083,997966,991616,999110,998434,990742,990066,995109,998540,990699,994461,990311,991178,995904,990030\n", - "Ground Truth: 992255,990650,996876,995794,990114,992964,992822,991873,998947,997451,999948,995485,997102,997948,998003,993083,997966,991616,993048,999110,995367,998434,990742,990066,995109,998540,990699,994461,991396,990311\n", - "Recall: 0.9000\n", + "Query ID: 121\n", + "Returned IDs: 996075,992147,999374,995515,991545,991743,992087,991889,994850,999074,996504,994776,997305,993355,990957,990083,994266,997192,995372,997320,996521,992880,991372,998921,997913,995596,994894,998779,999426,995443\n", + "Ground Truth: 996075,992147,999374,995515,991545,991743,992087,990118,991889,994850,999074,996504,994776,997305,991801,993355,990957,999140,990083,994266,998045,996648,997192,995372,993053,995374,997320,995192,993186,996521\n", + "Recall: 0.7000\n", "\n", - "Query ID: 131\n", - "Returned IDs: 998787,993366,994834,990952,995393,999373,998565,990167,994833,995326,999555,996553,998959,996408,990358,995299,990173,995231,992282,995835,993528,993560,994756,994374,993014,998536,995168,994682,994150,993948\n", - "Ground Truth: 998787,993366,994834,990952,995393,999373,998565,990167,994833,995326,999555,996553,998959,996408,990358,995299,990173,995231,992282,995835,993528,993560,994756,991313,994374,993014,998536,995168,994682,994150\n", + "Query ID: 122\n", + "Returned IDs: 998019,995801,990834,995203,993504,996233,990557,991648,996884,999526,990203,993179,994955,996001,992944,998764,998855,996908,997332,995366,995562,999320,995130,997002,996075,994829,995500,992041,999019,991037\n", + "Ground Truth: 998019,995801,990834,995203,993504,996233,990557,991648,996884,999526,990203,994081,993179,994955,996001,992944,998764,992627,998855,996908,997332,995366,995562,999320,995130,994449,997002,996075,998849,994829\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 123\n", + "Returned IDs: 999902,996326,994467,999383,997618,993931,992209,990364,991515,992104,993965,993275,996124,998051,995408,999079,995630,990998,991685,993609,992407,993583,995780,998536,998006,998855,990784,991303,996209,991755\n", + "Ground Truth: 999902,996326,994467,999383,997618,993931,992209,990364,991515,992104,993965,993275,996124,998051,995408,999079,995630,990998,991685,993609,992407,993583,995780,998536,998006,998855,990784,994770,991303,996209\n", "Recall: 0.9667\n", "\n", - "Query ID: 132\n", - "Returned IDs: 999236,996716,992790,999820,998723,991011,993281,996371,996067,992179,993504,994946,998303,996062,998401,992576,995401,993071,998045,994113,991857,993979,992367,997331,996220,994188,997002,994964,993492,995289\n", - "Ground Truth: 999236,996716,992790,999820,998723,991011,993281,996371,996067,992179,993504,994946,998303,996062,998401,992576,995401,993071,998045,994113,991857,990294,993979,992367,997331,996220,994188,990244,997002,994964\n", + "Query ID: 124\n", + "Returned IDs: 995985,990595,998263,992384,998500,993934,996714,998921,997192,997984,995793,995824,990275,991379,995554,997842,992994,995181,992546,996727,999599,991829,999327,992228,991749,995927,995090,996504,995274,997889\n", + "Ground Truth: 995985,990595,998263,992384,998500,993934,996714,998921,997192,997984,995374,995793,995824,990275,991379,993072,995554,997842,992994,995181,992546,996727,999599,991829,999327,992228,991749,995927,995090,996504\n", "Recall: 0.9333\n", "\n", + "Query ID: 125\n", + "Returned IDs: 990940,995269,994446,999932,993465,990242,995289,995444,993831,999000,995685,992160,995204,996867,997723,997046,997937,998070,999770,996302,994108,998779,993433,996323,999037,999277,997822,994571,996166,995507\n", + "Ground Truth: 990940,995269,994446,999932,993465,990242,995289,995444,993831,999000,992160,995685,995204,996867,997723,997046,997937,993412,998070,999770,996302,994108,998779,994405,993433,996323,999037,999277,997822,994571\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 126\n", + "Returned IDs: 997550,992635,997584,996075,996387,994618,995515,999426,998498,998921,997301,996516,995596,994971,993447,990697,999453,995372,996741,997984,994415,999526,997227,995033,997470,995119,993785,990855,997320,994233\n", + "Ground Truth: 997550,992635,997584,996075,996387,994618,995515,999426,998498,998921,997301,996516,995596,994971,993447,990697,999453,995250,995372,996741,997448,997984,994415,999526,997227,995033,997470,995119,993785,990855\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 127\n", + "Returned IDs: 998634,996001,997320,998246,993484,992839,991787,993053,994964,993280,991155,997332,999099,990585,990146,993168,993885,991498,999453,991721,991184,996219,990772,990096,993389,991732,991609,992896,990949,998365\n", + "Ground Truth: 998634,996001,997320,998246,993484,992839,991787,993053,994964,993280,991155,994961,997332,999099,990585,990146,993168,993885,991498,995678,999453,991721,991184,996219,990772,990096,993389,993154,990269,991732\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 128\n", + "Returned IDs: 997809,996546,991923,992718,998975,999380,993681,998331,991977,996163,990908,994398,993786,995138,993572,994852,992170,996158,995578,991157,999387,993975,996894,995737,992349,996585,999806,993839,992258,998853\n", + "Ground Truth: 997809,996546,991923,992718,998975,999380,993681,998331,991977,996163,990908,994398,993786,995138,993572,995986,994852,996658,992170,996158,995578,991157,998921,999387,993975,998026,996894,995737,992349,996585\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 129\n", + "Returned IDs: 992255,990650,996876,995794,990114,992822,991873,998947,999948,993083,991616,993048,999110,998434,990742,990066,995109,998540,990699,994461,991396,990311,990030,998899,993616,997539,991900,997320,994934,991593\n", + "Ground Truth: 992255,990650,996876,995794,990114,992964,992822,991873,998947,997451,999948,995485,997102,997948,998003,993083,997966,991616,993048,999110,995367,998434,990742,990066,995109,998540,990699,994461,991396,990311\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 130\n", + "Returned IDs: 993100,998710,995658,990735,992449,994998,994830,994858,999472,992796,993856,997305,997529,990771,998105,992006,994663,999354,994558,993939,994799,996603,993053,994185,999372,995200,991938,999192,999302,994131\n", + "Ground Truth: 993100,998710,995658,990735,992449,994998,994830,994858,999472,992796,993856,999391,997305,997529,990771,998105,990439,992006,994663,996147,990041,999354,994558,995873,993939,994799,998379,996603,993053,994822\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 131\n", + "Returned IDs: 998787,993366,994834,990952,995393,999373,998565,990167,994833,999555,996553,998959,996408,990358,995299,990173,995231,995835,993528,993560,994756,991313,994374,993014,998536,995168,994682,994150,993948,992862\n", + "Ground Truth: 998787,993366,994834,990952,995393,999373,998565,990167,994833,995326,999555,996553,998959,996408,990358,995299,990173,995231,992282,995835,993528,993560,994756,991313,994374,993014,998536,995168,994682,994150\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 132\n", + "Returned IDs: 999236,996716,992790,999820,998723,991011,993281,996371,996067,992179,993504,994946,998303,996062,998401,992576,993071,998045,994113,991857,993979,997331,994188,997002,994964,993492,995289,995589,996198,999898\n", + "Ground Truth: 999236,996716,992790,999820,998723,991011,993281,996371,996067,992179,993504,994946,998303,996062,998401,992576,995401,993071,998045,994113,991857,990294,993979,992367,997331,996220,994188,990244,997002,994964\n", + "Recall: 0.8333\n", + "\n", "Query ID: 133\n", - "Returned IDs: 999503,997790,993055,999840,994280,997926,993489,991397,995203,992862,997897,992521,990399,990005,992958,998250,990802,997918,998303,991661,994042,994531,991630,994946,991332,998933,992747,996657,998697,996368\n", + "Returned IDs: 999503,997790,993055,999840,994280,997926,993489,995203,992862,997897,992521,990399,990005,992958,998250,997918,998303,991661,994042,991630,994946,991332,998933,992747,996657,998697,996368,993260,996733,990350\n", "Ground Truth: 999503,997790,993055,999840,994280,997926,993489,991397,995203,992862,992979,997897,992521,990399,990005,992958,998250,990802,997918,998303,991661,994042,994531,991630,994946,991332,998933,992747,996657,993143\n", - "Recall: 0.9333\n", + "Recall: 0.8333\n", "\n", "Query ID: 134\n", - "Returned IDs: 991227,992558,992052,993787,993478,992933,998536,996918,996531,995783,994280,995214,994972,995408,990956,992318,993673,991052,998104,998397,993121,992997,993747,996438,996658,995401,996930,997897,993727,996328\n", + "Returned IDs: 991227,992558,992052,993478,998536,996531,995783,994280,995214,994972,995408,990956,991052,998104,998397,993121,992997,993747,996438,996658,995401,997897,993727,992521,996328,999179,994119,990776,999514,995184\n", "Ground Truth: 991227,992558,992052,993787,993478,992933,998536,996918,996531,995783,994280,995214,994972,995408,990956,992318,993673,991052,998104,998397,993121,992997,993747,996438,996658,995401,996930,997897,993727,992521\n", - "Recall: 0.9667\n", + "Recall: 0.8000\n", "\n", "Query ID: 135\n", - "Returned IDs: 994406,992767,997799,998431,995403,997391,999960,990337,998779,997634,993026,994474,998080,994671,996727,996903,995374,995453,997012,995034,997157,997754,997640,993412,990647,993289,996172,995893,995607,998915\n", + "Returned IDs: 994406,992767,998431,997391,999960,998779,997634,994474,994671,996727,996903,995374,995453,997012,995034,997157,997640,993412,990647,993289,996172,995893,995369,998529,994092,997192,997702,997462,994794,997995\n", "Ground Truth: 994406,992767,997799,998431,995403,997391,999960,990337,998779,997634,993026,994474,998080,994671,996727,996903,995374,996047,995453,997012,995034,997157,997754,997640,993412,990647,993289,996172,995893,995607\n", - "Recall: 0.9667\n", + "Recall: 0.7333\n", "\n", "Query ID: 136\n", - "Returned IDs: 991324,999740,992698,993725,990011,990484,991390,994830,995756,990711,992479,997142,990743,994751,998710,990089,995002,991885,996358,999922,996633,997714,992664,991911,995435,992679,990439,992480,996001,997933\n", + "Returned IDs: 991324,999740,992698,993725,990011,990484,991390,994830,995756,990711,992479,997142,990743,994751,998710,995002,991885,996358,996633,997714,992664,991911,995435,992679,992480,996001,997933,999377,996275,990735\n", "Ground Truth: 991324,999740,992698,993725,990011,990484,991390,994830,995756,990711,992479,997142,990743,994751,998710,990089,995002,991885,999555,996358,999922,996633,997714,992664,991911,995435,992679,990439,992480,996001\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 137\n", + "Returned IDs: 995771,998542,993364,997543,994584,999263,991533,993250,991889,990515,997301,992401,999291,991558,998211,992147,993581,994939,999968,992818,998539,999270,996218,996504,990958,994670,999089,992384,999145,999982\n", + "Ground Truth: 995771,998542,993364,997543,994584,995034,999263,991533,993250,991889,990515,997301,992401,999291,991558,998211,992147,993581,994939,999968,992818,998539,999270,996218,996504,990958,994670,999089,992384,999145\n", "Recall: 0.9667\n", "\n", "Query ID: 138\n", - "Returned IDs: 999350,994864,994488,995167,996001,992169,996406,999179,997679,991212,992182,998122,994510,991596,990557,993931,999130,999304,999623,994467,994119,990434,997332,996075,997395,993032,994416,998725,996775,991066\n", + "Returned IDs: 999350,994864,994488,995167,996001,992169,996406,997679,991212,992182,998122,994510,991596,990557,993931,999130,999304,999623,994467,999099,994119,990434,997332,997395,993032,998725,996996,990364,996028,994280\n", "Ground Truth: 999350,994864,994488,995167,996001,992169,996406,999179,997679,991212,999679,992182,996803,998122,994510,991596,993931,990557,999130,999304,999623,994467,999099,994119,990434,997332,996075,997395,993032,994416\n", - "Recall: 0.9000\n", + "Recall: 0.8333\n", "\n", "Query ID: 139\n", - "Returned IDs: 994168,992469,999500,995289,990126,997723,993656,990930,999586,996811,998913,995562,994783,991154,990340,993747,994742,996219,995095,994759,998495,997797,990523,998156,990266,993642,996116,999977,990399,998855\n", + "Returned IDs: 994168,992469,999500,995289,990126,990930,996811,998913,994783,991154,990340,993747,994742,997864,998495,990523,993642,996116,999977,990399,995401,996609,996062,994325,993561,995633,994431,994140,994190,990012\n", "Ground Truth: 994168,992469,999500,995289,990126,997723,993656,990930,999586,996811,998913,995562,994783,991154,990340,993747,993400,994742,996219,995095,994759,997864,998633,998495,997797,990523,998156,990266,993642,996116\n", + "Recall: 0.6000\n", + "\n", + "Query ID: 140\n", + "Returned IDs: 999867,990957,990949,998246,997320,994964,993484,990736,991379,993853,992107,995949,995928,997913,998500,991895,995942,997371,999982,996785,994192,998170,995336,994100,995884,993518,997470,998365,999374,990925\n", + "Ground Truth: 999867,990957,990949,998246,998294,997320,994964,993484,990736,991379,993853,992107,995949,995928,997913,998500,991895,995942,997371,999710,999982,996785,990397,994192,993072,996560,998163,998170,995336,991743\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 141\n", + "Returned IDs: 995004,995639,995788,997288,996402,994922,994785,998657,992641,998241,994612,995588,993514,992302,995817,996840,991151,993708,996100,991172,990300,991473,993355,997486,993282,991193,997220,990902,993143,991510\n", + "Ground Truth: 995004,995639,999888,995788,997288,996402,994922,994785,998657,992641,993675,998241,998973,994612,995588,993514,992302,995817,996840,991151,993708,996100,991172,990300,991473,993355,993282,997486,991193,997220\n", "Recall: 0.9000\n", "\n", + "Query ID: 142\n", + "Returned IDs: 993250,990515,998211,997543,999823,993364,997301,993669,999263,991743,996218,992401,999968,999089,990561,990760,997320,997192,995886,993798,998921,999291,999145,992384,999374,996504,995928,992147,999026,990585\n", + "Ground Truth: 993250,990515,998211,997543,999823,993364,997301,993669,999263,991743,996218,992401,999968,999089,990561,990760,997320,998445,995034,997192,993648,995886,993798,998921,997228,999291,999145,992384,999374,996504\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 143\n", + "Returned IDs: 999342,990662,996147,998878,994223,994708,996739,990771,992636,997550,996657,990573,994523,993344,999130,994830,994742,996001,998165,998803,993848,999570,999302,995873,995743,998949,994185,998076,998097,997565\n", + "Ground Truth: 999342,990662,996147,998878,994223,994708,996739,990771,992636,997550,996657,993599,991255,990573,994523,993344,992293,999130,994830,994742,996001,998165,998803,993848,999570,995827,999302,995873,995743,998949\n", + "Recall: 0.8667\n", + "\n", "Query ID: 144\n", - "Returned IDs: 995515,992835,990155,998244,996419,993582,993028,994216,998837,990142,991149,999053,990313,993237,991999,997591,994612,994281,992020,990652,997403,994475,994497,990827,997465,990043,992662,994898,996485,991196\n", + "Returned IDs: \n", "Ground Truth: 995515,992835,990155,998244,996419,993582,993028,994216,998837,990142,991149,999053,990313,993237,991999,997591,994612,994281,992020,990652,997403,990943,994475,994497,990827,997465,990043,992662,994898,996485\n", - "Recall: 0.9667\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 145\n", + "Returned IDs: 990897,990942,995516,993055,997806,998758,996728,997506,995237,995268,993580,998346,993948,992636,991630,990005,991161,998067,995159,999775,993044,992052,992521,993275,990956,994380,993560,995184,990593,998697\n", + "Ground Truth: 990897,990942,995516,993055,997806,998758,996728,997506,995237,995268,993580,998346,993948,998045,992636,991630,990005,991161,998067,995159,999775,993044,992862,992052,994891,992521,993275,990956,994380,993560\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 146\n", + "Returned IDs: 994577,992423,999806,991157,998260,993552,998568,992617,992309,995515,997305,998495,999122,998921,999079,999526,993260,995080,992521,995372,998607,991743,999317,992944,997320,997227,999908,990908,999367,990843\n", + "Ground Truth: 994577,992423,999806,991157,998260,993552,998568,992617,992309,995515,997305,998495,999122,998921,999079,999526,997301,993260,995080,992521,995372,998607,991743,999317,992944,997320,997227,996281,999908,990908\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 147\n", + "Returned IDs: 990132,991664,993390,992582,997411,994717,990361,994656,995253,998779,991415,992989,996379,999646,991836,996867,994382,995701,996368,999402,995268,991350,997058,994092,997209,996659,994793,998922,993566,996388\n", + "Ground Truth: 990132,991664,993390,992582,997411,992604,992293,994717,990361,994656,995253,998257,998779,991603,991415,992989,996379,999646,991836,996867,994382,995701,996368,999402,995268,991350,997058,994092,997209,997723\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 148\n", + "Returned IDs: 997379,998455,993052,999077,996193,998180,993260,993696,999210,999784,998950,992571,993781,994341,992309,993504,996305,995203,997370,995443,990580,992944,991348,991157,996075,995365,990043,999122,994552,992169\n", + "Ground Truth: 997379,998455,993052,999077,996193,998180,993260,993696,999210,999784,998950,992571,993781,994341,992309,995094,993504,996305,995203,997370,995443,990580,992944,991348,991157,990414,996075,995365,990043,999122\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 149\n", + "Returned IDs: 997209,999843,992582,996220,994747,993857,997309,991189,994742,992460,990328,993549,995633,990132,991334,994119,998410,997544,991664,994835,997645,991269,992989,998910,999279,998495,993209,991075,996809,990126\n", + "Ground Truth: 997209,999843,992582,996220,994747,993857,997309,991189,994742,992460,990328,993549,995633,993948,990132,991334,994119,998410,997544,991664,994835,997645,991269,992989,992604,997208,992309,998910,999279,998495\n", + "Recall: 0.8667\n", "\n", "Query ID: 150\n", - "Returned IDs: 992678,996848,998067,997915,991044,990331,991747,992097,999980,992521,997675,992569,998045,995268,991103,990314,996368,996277,996505,998060,993256,992367,996719,995963,992187,994861,999402,997723,993313,994817\n", + "Returned IDs: 992678,996848,998067,997915,991044,990331,991747,992097,999980,992521,997675,992569,998045,995268,991103,990314,996368,996277,996505,998060,993256,996719,995963,992187,994861,999402,993313,994817,995815,992448\n", "Ground Truth: 992678,996848,998067,997915,991044,990331,991747,992097,999980,992521,997675,992569,998045,995268,998021,991103,990314,996368,996277,996505,998060,993256,992367,996719,995963,992187,994861,999402,997723,993313\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", "\n", "Query ID: 151\n", - "Returned IDs: 999578,992448,994409,993468,992660,992801,996618,997832,999043,996038,999530,998714,991699,997868,996813,997406,995139,998045,998979,999508,996607,991027,995507,994554,993433,996769,991671,993561,998545,999940\n", + "Returned IDs: 999578,992448,994409,992660,992801,997832,999043,996038,999530,998714,991699,997868,997406,995139,998045,998979,999508,996607,991027,995507,994554,993433,996769,991671,993561,998545,999940,993421,999884,996867\n", "Ground Truth: 999578,992448,994409,993468,992660,992801,996618,997832,999043,996038,999530,998714,991699,997868,996813,997406,995139,998045,998979,999508,996607,991027,995507,994554,993433,996769,991671,993561,993280,998545\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 152\n", + "Returned IDs: 997281,998720,992225,996857,992330,997305,990866,998514,997811,992068,998568,996230,995536,996075,993834,995638,994636,993551,999651,991373,991256,995701,996576,999636,993139,990051,993809,994572,992167,994531\n", + "Ground Truth: 997281,998720,992225,996857,992330,997305,990866,998514,997811,992068,998568,996230,995536,996075,993834,999987,995638,994636,993551,990642,999651,991373,991256,995701,996576,999636,993139,990051,993809,994572\n", + "Recall: 0.9333\n", "\n", "Query ID: 153\n", - "Returned IDs: 992239,995637,990264,990814,991981,991590,999924,999767,992713,994891,993080,997494,994968,991428,990516,991769,995012,994094,998999,992318,997698,990364,990270,992437,996728,998869,999942,992827,996658,994742\n", + "Returned IDs: 992239,995637,990264,990814,991981,991590,999924,999767,992713,994891,993080,997494,994968,991428,990516,991769,995012,994094,992318,997698,990364,990270,992437,996728,999942,990853,992827,994742,999399,996911\n", "Ground Truth: 992239,995637,990264,990814,991981,991590,999924,999767,992713,994891,993080,997494,994968,991428,990516,991769,995012,994094,998999,992318,997698,990364,990270,992437,996728,998869,999942,990853,992827,996658\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", "\n", "Query ID: 154\n", - "Returned IDs: 994168,990080,990266,994131,998633,991943,996093,999586,991255,994742,997723,999587,998156,991981,993080,993717,992582,999453,992169,991585,995295,994129,990957,998980,996867,991185,994431,996809,998256,995702\n", + "Returned IDs: 994168,990080,990266,998633,991943,996093,999586,994742,999587,998156,991981,993080,993717,999453,992169,991585,995295,990957,998980,996867,991185,994431,996809,998256,995873,990860,997059,995274,994140,999957\n", "Ground Truth: 994168,990080,990266,994131,992655,998633,991943,996093,999586,991255,994742,997723,999587,998156,991981,993080,993717,992582,999453,992169,997360,991585,991334,995295,994129,990957,998980,996867,991185,994431\n", - "Recall: 0.9000\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 155\n", + "Returned IDs: 995252,999024,993241,998462,999358,992066,990585,997015,998951,990817,996959,997771,999144,993544,990231,990535,995824,993562,994595,999650,999535,997320,999531,995716,997314,996785,996871,997256,996196,998090\n", + "Ground Truth: 995252,999024,993241,998462,999358,992066,990585,997015,998951,990817,996959,997771,999144,993544,990231,990535,995824,993562,994595,999650,998160,999535,997320,999531,995716,997314,996785,991362,996871,997256\n", + "Recall: 0.9333\n", "\n", "Query ID: 156\n", - "Returned IDs: 999267,993637,991128,993053,995283,995659,990089,990368,999338,995274,990629,996701,995345,991412,991992,993410,999806,992636,997691,997052,994386,995638,998302,995467,998021,999343,997340,999446,996368,998215\n", + "Returned IDs: 999267,993637,991128,993053,995283,995659,990089,990368,999338,990629,995345,991412,991992,993410,999806,992636,997691,997052,994386,995638,998302,995467,998021,999446,996368,998215,999791,993561,995689,991795\n", "Ground Truth: 999267,993637,991128,993053,995283,995659,990089,990368,999338,995274,990629,996701,990133,995345,991412,991992,993410,999806,992636,997691,997052,994386,995638,998302,995467,998021,999343,990260,997340,999446\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 157\n", + "Returned IDs: 990593,995907,998671,999934,995083,998315,992450,993476,995268,998537,998055,996431,995191,992106,994142,998256,990041,996495,997652,991769,992859,997025,997806,993466,994891,996328,990069,999360,996055,997185\n", + "Ground Truth: 990593,995907,998671,999934,995083,998315,992450,993476,995268,998537,998055,996431,995191,992106,994313,994142,999441,998256,990041,996495,997652,991769,998072,992859,997025,997806,993466,994891,996328,990069\n", + "Recall: 0.9000\n", "\n", "Query ID: 158\n", - "Returned IDs: 990890,996219,991928,998899,992931,991249,998540,994831,994069,999300,990545,999068,999343,995942,994964,992330,991593,991743,996343,997913,993282,990404,994486,998550,999620,997305,990210,998837,990444,996646\n", + "Returned IDs: 990890,996219,991928,998899,992931,991249,998540,994069,999068,999343,994964,992330,991593,996343,997913,994486,998550,999620,997305,990210,998837,996646,993264,994366,991999,990957,994783,997022,998464,999815\n", "Ground Truth: 990890,996219,991928,998899,992931,991249,998540,994831,994069,999300,990545,999068,999343,995942,994964,992330,991593,991743,996343,997913,993282,990404,994486,998550,990666,999620,991006,995210,997305,990210\n", - "Recall: 0.9000\n", + "Recall: 0.6667\n", "\n", "Query ID: 159\n", - "Returned IDs: 992094,994119,996245,997944,998150,994177,994193,990226,990527,994280,999461,992703,992052,998755,993037,992437,994968,997309,997897,993483,993915,992652,997056,993288,995401,998355,995408,996277,990784,996775\n", + "Returned IDs: 992094,994119,996245,997944,998150,994193,990226,994280,992703,992052,992437,990790,994968,997309,997897,993915,997056,998355,995408,996277,990784,991011,994375,994472,997881,998165,995017,993778,999643,992576\n", "Ground Truth: 992094,994119,996245,997944,998150,994177,994193,990226,990527,994280,999461,992703,992052,998755,993037,992437,990790,994968,997309,997897,993483,993915,992652,997056,993288,995401,998355,995408,996277,990784\n", - "Recall: 0.9667\n", + "Recall: 0.7000\n", "\n", "Query ID: 160\n", - "Returned IDs: 992604,993754,995807,995527,999662,999445,997781,990915,990540,995953,991051,990657,990729,991204,991110,991571,994187,997918,997212,995373,994753,998858,991987,996043,999306,995805,997391,997675,995105,998697\n", + "Returned IDs: 992604,993754,995807,995527,999662,999445,997781,990915,990540,995953,991051,990657,990729,998297,991204,991110,994187,997918,995373,994753,998858,991987,996043,997158,999306,995805,997675,995105,998697,997723\n", "Ground Truth: 992604,993754,995807,995527,999662,999445,997781,990915,990540,995953,991051,990657,990729,998297,991204,991110,991571,994187,997918,997212,995373,994753,998858,991987,996043,997158,999306,995805,997391,997675\n", - "Recall: 0.9333\n", + "Recall: 0.9000\n", "\n", "Query ID: 161\n", - "Returned IDs: 998294,996504,999765,990917,997913,994266,998921,993518,991406,997984,991895,990760,999263,993669,998500,995755,990925,994978,990829,995090,998987,990936,997192,990595,996560,997371,998540,995793,998344,992789\n", + "Returned IDs: 998294,996504,999765,997913,994266,998921,993518,991406,997984,990760,999263,993669,998500,995755,990925,994978,995090,998987,997192,990595,997371,998540,995793,990958,994415,994775,995949,993143,990560,995942\n", "Ground Truth: 998294,996504,999765,990917,997913,994266,998921,993518,991406,997984,991895,990760,999263,993669,998500,995755,990925,994978,990829,995090,998987,990936,997192,990595,996560,997371,998540,995793,998344,999647\n", - "Recall: 0.9667\n", + "Recall: 0.7667\n", "\n", "Query ID: 162\n", - "Returned IDs: 995110,993478,991103,996505,995032,998237,998397,995783,999646,991747,995401,997675,995701,993727,990537,995521,991413,991010,990956,992457,997918,997752,991052,995184,991054,994232,998045,990872,992790,993412\n", + "Returned IDs: 995110,993478,991103,996505,995032,998397,995783,999646,991747,995401,997675,995701,993727,990537,995521,991413,990956,993738,997918,997752,995184,995886,990872,992790,994280,990930,993948,998874,996715,991789\n", "Ground Truth: 995110,993478,991103,996505,995032,998237,998397,995783,999646,991747,995401,997675,995701,993727,990537,995521,991413,991010,990956,992457,993738,991630,997918,997752,991052,995184,991054,994232,992260,998045\n", - "Recall: 0.9000\n", + "Recall: 0.7000\n", "\n", "Query ID: 163\n", - "Returned IDs: 998914,990678,996556,996028,998355,997380,995528,999891,998630,990226,991426,996190,993725,992521,994765,995317,996505,996016,992488,996728,994119,998325,999479,992480,993820,995449,995456,998982,999516,991747\n", + "Returned IDs: 998914,990678,996556,996028,998355,997380,995528,999891,990226,991426,996190,993725,994765,995317,996505,996016,992322,994119,998325,999479,992480,995449,995456,998982,999516,991747,997649,990776,994012,994587\n", "Ground Truth: 998914,990678,996556,996028,998355,997380,995528,999891,998630,990226,991426,996190,993725,992521,994765,995317,996505,996016,992488,996728,992322,994119,998325,999479,992480,993820,995449,995456,993336,998982\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 164\n", + "Returned IDs: 999830,991540,990487,992280,994541,992934,995273,994661,993976,997374,993583,996194,995436,994913,992052,996178,990908,993931,994488,991977,992718,995159,998417,998370,997329,993256,998118,990364,995551,990270\n", + "Ground Truth: 999830,991540,990487,992280,994541,992934,995273,997960,994661,992467,993976,997374,993583,996194,998135,995436,994913,992052,997056,996178,993931,990908,994488,991977,992718,995159,998417,998370,996685,997329\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 165\n", + "Returned IDs: 991308,995120,994955,992958,992823,990364,997897,996001,997320,995167,990557,999526,990314,997790,997368,996657,995250,990096,992825,993375,998593,994081,994972,990409,995672,997926,995855,995740,997977,994809\n", + "Ground Truth: 991308,996764,995120,994955,992958,992823,990364,997897,996001,997320,995167,990557,999526,990314,991657,991898,994531,997790,997368,996657,995250,990096,992825,993375,998593,994972,994081,990409,995672,997926\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 166\n", + "Returned IDs: 994380,996064,996001,990392,990887,991866,991585,992832,990127,997332,994925,990712,993561,997732,998470,999126,992703,998122,992450,991831,998067,998416,993637,996505,994541,998862,998055,991185,993180,992595\n", + "Ground Truth: 994380,996064,996001,993710,990392,990887,991866,991585,992832,990127,997332,993648,994925,990712,993561,997732,998470,999126,993412,992703,998122,992450,991831,998067,998416,993637,996505,994541,998862,998055\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 167\n", + "Returned IDs: 997666,996733,999612,992234,996386,993132,993867,992209,999368,995593,998671,998781,994181,992716,992673,998598,991724,996209,990806,999388,995385,997693,993148,990423,994731,996893,995796,994335,999079,997685\n", + "Ground Truth: 997666,996733,999612,992234,996386,993132,993867,992209,999368,990712,995593,997807,998671,998781,990552,990630,994181,992716,992673,998598,991724,997442,996209,990806,999388,995385,997693,993148,992984,990423\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 168\n", + "Returned IDs: \n", + "Ground Truth: 993561,995923,996867,990975,996232,996769,997554,994419,998564,996001,993725,997406,992576,994925,996042,991958,996360,993813,998045,995963,994409,995216,992862,995401,993412,995052,997046,996277,993461,995873\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 169\n", + "Returned IDs: 994336,990103,996803,993727,998123,999244,993185,996806,996918,993805,993165,996535,994051,992558,997644,999296,991453,998968,995175,991227,999673,999029,994909,994035,996825,993478,991610,991054,997480,999191\n", + "Ground Truth: 994336,990103,996803,993727,998123,999244,993185,996806,996918,993805,993165,996535,994051,992558,997644,999296,991453,998968,995175,991227,999673,999029,994909,998279,994035,999098,996825,993478,996861,991610\n", + "Recall: 0.9000\n", "\n", "Query ID: 170\n", - "Returned IDs: 999430,995332,995107,991179,991961,991458,994897,998416,998950,990377,993499,995560,996216,993807,995895,997331,990091,995750,993617,993079,990769,992780,997004,993479,996944,999210,992567,998157,996370,991382\n", + "Returned IDs: 999430,995332,995107,991961,991458,998416,998950,990377,993499,995560,996216,993807,995895,997331,995750,993617,993079,992780,997004,993479,996944,999210,992567,998157,996370,991382,993391,998732,990039,997586\n", "Ground Truth: 999430,995332,995107,991179,991961,991458,994897,998416,998950,990377,993499,995560,996216,993807,995895,997331,990091,995750,993617,993079,990769,992780,997004,993479,996944,999210,992567,993335,998157,996370\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 171\n", - "Returned IDs: 996219,994990,994964,993378,991075,996052,994843,992047,996593,997028,999453,995096,993342,996001,998039,990448,992780,990366,997920,994777,994472,999352,992261,994922,990091,993648,999437,999344,998464,998495\n", + "Returned IDs: 996219,994990,994964,993378,991075,996052,997616,996593,997028,999453,995096,993342,996001,998039,990448,992780,990366,994777,990091,993648,999344,998464,998495,999151,990146,990632,999791,995107,998593,999261\n", "Ground Truth: 996219,994990,994964,993378,991075,996052,994843,997616,992047,996593,997028,999453,995096,993342,996001,998039,990448,992780,990366,997920,994777,994472,999352,992261,994922,990091,993648,998633,999437,999344\n", - "Recall: 0.9333\n", + "Recall: 0.7000\n", "\n", "Query ID: 172\n", - "Returned IDs: 990016,998105,992201,993588,994145,993053,995097,990435,990497,996147,990536,994307,998548,991075,990662,994830,995289,994925,990771,996974,999460,993831,998097,990811,996001,993412,995823,996411,998950,998495\n", + "Returned IDs: 990016,998105,992201,993588,994145,993053,995097,990435,990497,990536,994307,998548,991075,990662,994830,995289,994925,996974,999460,998165,993831,990811,996001,995823,996411,998950,998495,994730,995906,997691\n", "Ground Truth: 990016,998105,992201,993588,994145,993053,995097,990435,990497,996147,990536,994307,998548,991075,990662,994830,995289,994925,990771,996974,999460,998165,993831,998097,990811,996001,993412,995823,996411,998950\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 173\n", + "Returned IDs: 992235,993832,996219,995483,993687,994288,993931,990705,994774,990931,994742,998080,993621,995455,997395,992645,996001,990722,997126,994062,998495,990126,995401,995932,990041,999961,999139,999430,992888,995134\n", + "Ground Truth: 992235,993122,993832,996219,995483,993687,997068,994288,999916,997149,993931,990705,994774,993566,990931,997790,997540,994742,998080,999551,993621,995455,999360,997395,992645,996001,999647,990722,997126,994062\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 174\n", + "Returned IDs: 997391,991698,997218,997918,990215,992933,995282,993933,992428,998199,997158,998022,994406,992179,996247,992790,995886,997514,994819,993288,990916,994982,992604,991459,991413,996792,994170,994280,992516,999154\n", + "Ground Truth: 997391,991698,997218,997918,990215,997799,992933,995282,993933,992428,998199,997158,998022,994406,992179,996247,992790,995886,997514,994819,991792,993288,990916,994982,992604,997901,991459,991413,996792,994170\n", + "Recall: 0.9000\n", "\n", "Query ID: 175\n", - "Returned IDs: 994894,990689,997192,998974,995274,999444,994266,990957,993186,990498,997035,998326,997437,998697,991630,990337,995536,996699,992811,990231,992357,997320,990958,996785,998649,993412,990799,998170,990146,992692\n", + "Returned IDs: 994894,990689,997192,998974,995274,999444,994266,990957,993186,997035,997437,998697,990862,995536,996699,992811,990231,992357,997320,990958,996785,998649,998170,990146,998779,998500,995569,994100,995655,995701\n", "Ground Truth: 994894,990689,997192,998974,995274,999444,994266,990957,993186,990498,997035,998326,997437,998697,991630,990337,990862,995536,996699,992811,990231,992357,997320,990958,996785,998649,993412,990799,998170,990146\n", - "Recall: 0.9667\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 176\n", + "Returned IDs: 994360,996178,996429,999676,991650,996219,995826,991409,999806,997301,990096,996001,998607,995672,996397,998180,993431,997456,998495,993696,997550,998401,999426,992423,998302,997586,999210,998921,993470,994742\n", + "Ground Truth: 992443,995401,994360,996178,996429,999676,995559,991650,996219,995826,991409,991607,999806,997301,990096,996001,994244,998607,995672,993831,998365,996397,997906,998180,993431,997456,995873,998495,993696,999900\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 177\n", + "Returned IDs: 999011,996730,992179,998536,996245,992790,991698,998874,994301,993673,993478,997918,995401,991459,992933,995154,995886,992558,999820,996040,994170,990214,993037,994334,998778,990623,992428,990479,994280,997702\n", + "Ground Truth: 999011,996730,992179,998536,996245,992790,991698,992685,998874,994301,993673,993478,997918,995401,991459,992933,995154,995886,992558,999820,998529,996040,994170,990214,991052,993037,994334,998778,990623,992428\n", + "Recall: 0.9000\n", "\n", "Query ID: 178\n", - "Returned IDs: 994254,997809,998686,990487,991977,992889,997407,991923,998046,990776,995691,994165,998243,994760,990343,996068,999766,998975,993839,990522,993683,997914,998357,993572,994010,995965,995506,996895,992645,998118\n", + "Returned IDs: 994254,997809,998686,990487,991977,992889,997407,991923,998046,990776,995691,994165,998243,994760,990343,996068,999766,998975,993839,990522,997914,998357,993572,994010,995965,995506,996895,992645,998118,994119\n", "Ground Truth: 994254,997809,998686,990487,991977,992889,997407,991923,998046,990776,995691,994165,998243,994760,990343,996068,999766,998975,993839,990522,993683,997914,998357,993572,994010,995965,995506,999913,996895,992645\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 179\n", - "Returned IDs: 990155,993280,999453,996810,992395,998734,990772,996001,991375,991951,995988,990142,994536,996915,996732,997876,991517,998567,995216,990778,995130,999076,991840,991255,991248,999019,997320,998605,991999,992944\n", + "Returned IDs: 993280,999453,996810,992395,998734,996001,991951,995988,990142,991517,995216,999076,991840,991248,999019,997320,998605,991999,992944,998697,995705,993018,997002,994280,996064,994081,993324,990652,994964,998416\n", "Ground Truth: 990155,993280,999453,996810,992395,998734,990772,996001,991375,991951,995988,990142,994536,996915,996732,997876,991517,998567,995216,990778,995130,999076,991840,998523,991255,991248,999019,997320,999723,993154\n", - "Recall: 0.9000\n", + "Recall: 0.5333\n", + "\n", + "Query ID: 180\n", + "Returned IDs: 993324,999796,998104,999514,996918,990956,997420,995130,995401,992944,995332,993121,991961,995766,993347,994119,995438,998536,998180,991723,999881,995107,992052,994743,998950,995289,994541,996855,990776,999179\n", + "Ground Truth: 993324,999796,998104,999514,996918,990956,997420,995130,995401,992944,995332,993121,991961,995766,993347,994119,995438,998536,998180,991723,999881,995107,992052,994743,998950,995289,994541,996855,990776,993079\n", + "Recall: 0.9667\n", "\n", "Query ID: 181\n", - "Returned IDs: 995895,996706,997442,998593,993805,993391,995438,992726,990844,998993,990133,999430,994431,994351,994742,996809,999435,999179,996930,998934,998989,994830,993335,997723,992864,997331,998055,994382,996830,992850\n", + "Returned IDs: \n", "Ground Truth: 995895,996706,997442,998593,993805,993391,995438,992726,990844,998993,990133,999430,994431,994351,994742,996809,999435,999179,996930,998934,999587,998989,998990,994830,993335,997723,992864,997331,998055,994382\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 182\n", + "Returned IDs: 995289,995633,994742,995702,991585,997723,995200,990361,991185,993209,991664,996042,998162,995766,997182,998999,993831,996867,997822,996302,995648,992582,990940,992448,998165,997411,995401,996388,999862,995216\n", + "Ground Truth: 995289,995633,994742,995702,991585,997723,995200,990361,991185,993209,991664,996042,998162,995766,997182,998999,993831,996867,997822,996302,995940,999587,995648,992582,990940,992448,998165,997411,995401,996388\n", "Recall: 0.9333\n", "\n", "Query ID: 183\n", - "Returned IDs: 992002,991507,998162,995493,996607,998910,999054,997466,992448,990163,995458,996987,993046,990295,991671,991585,992260,994013,994865,998948,990924,990804,996042,996815,991603,998283,999475,995216,995766,993561\n", + "Returned IDs: 992002,991507,998162,995493,996607,998910,999054,997466,992448,990163,995458,996987,993046,990295,991671,991585,997529,994013,994865,998948,990924,993883,990804,996042,996815,995216,995766,993561,992801,992528\n", "Ground Truth: 992002,991507,998162,995493,996607,998910,999054,997466,992448,990163,995458,996987,993046,990295,991671,991585,997529,992260,994013,994865,998948,990924,993883,990804,996042,996815,991603,998283,999475,995216\n", - "Recall: 0.9333\n", + "Recall: 0.8667\n", "\n", "Query ID: 184\n", - "Returned IDs: 996689,990328,990600,994255,992445,999025,995456,994689,992052,995295,991277,992068,996028,997329,991451,995832,997309,994280,998959,993560,997420,995166,990532,998589,995823,995273,997897,998182,995203,994946\n", + "Returned IDs: 996689,990328,990600,994255,992445,999025,995456,994689,992052,995295,991277,996028,997329,991451,997309,994280,993560,997420,995166,990532,998589,995823,995273,997897,998182,995203,994946,998623,996580,997251\n", "Ground Truth: 996689,990328,990600,994255,992445,999025,995456,994689,992052,995295,991277,992068,996028,997329,996110,991451,995832,997309,994280,998959,993560,997420,995166,990532,998589,995823,995273,997897,998182,995203\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 185\n", + "Returned IDs: 990193,996911,996277,993738,991769,995047,997059,999343,997554,991466,991349,991981,991185,995216,995184,992162,999402,998060,996719,992448,999731,991163,992521,997168,994140,991111,992526,991585,995052,998910\n", + "Ground Truth: 990193,996911,996277,993738,991769,995047,997059,999343,997554,991466,991349,991981,991185,995216,995184,992162,992879,994353,999402,998060,996719,991737,992448,999731,991163,992521,997168,990364,994140,991111\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 186\n", + "Returned IDs: 999858,994682,994233,994830,996183,997073,991885,998076,992372,997340,993101,993588,995299,993335,993917,998063,990582,992318,991409,990373,997109,991390,994269,990700,995735,993568,992026,996357,990392,992415\n", + "Ground Truth: 999858,994682,994233,994830,996183,997982,997073,991885,998076,992372,997340,993101,993588,993394,995299,993335,993917,998063,990582,992318,991409,990373,997109,991390,994269,990700,995735,993568,992026,996357\n", + "Recall: 0.9333\n", "\n", "Query ID: 187\n", - "Returned IDs: 997391,997918,994334,990215,996282,994819,993492,995886,996903,999562,997702,992979,992179,999820,995401,992211,993512,997782,994088,994319,992428,999144,994916,992933,994906,998892,990561,998011,994670,996658\n", + "Returned IDs: 997391,997918,994334,990215,996282,994819,993492,995886,996903,997702,992979,992179,999820,995401,992211,993512,994088,994319,992428,999144,992933,994906,998892,990561,994670,996658,991698,992558,999804,993289\n", "Ground Truth: 997391,997918,994334,990215,996282,994819,993492,995886,996903,999562,997702,992979,992047,992179,999820,995401,992211,993512,997782,994088,994319,992428,999144,994916,992933,994906,998892,990561,998011,994670\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 188\n", - "Returned IDs: 997106,992190,998409,995256,990675,992804,993791,994852,991958,991923,999639,997864,994043,993342,991208,995701,996368,993283,992167,991530,997407,990622,998079,995138,997739,995898,991350,994105,996894,997577\n", + "Returned IDs: 997106,992190,998409,995256,990675,992804,994852,991958,991923,999639,997864,994043,993342,991208,995701,996368,993283,992167,991530,997407,990622,998079,997739,995898,991350,996894,996127,997577,999216,992976\n", "Ground Truth: 997106,992190,998409,995256,990675,992804,993791,998974,994852,991958,991923,999639,997864,994043,993342,991208,995701,996368,993283,992167,994796,991530,997407,990622,998079,995138,997739,995898,991350,994105\n", - "Recall: 0.9333\n", + "Recall: 0.8333\n", "\n", - "Query ID: 191\n", - "Returned IDs: 999172,991585,998060,994013,997554,999570,999402,995544,996147,993377,993053,996001,993217,993209,991185,990887,997059,990331,997529,992260,997566,994203,996752,993561,992450,995873,998162,993021,995216,994488\n", - "Ground Truth: 999172,991585,998060,994013,997554,999570,999402,995544,996147,993377,993053,996001,993217,993209,991185,990887,997059,990331,997529,992260,997566,994203,996752,993561,992450,993166,995873,998162,993021,995216\n", - "Recall: 0.9667\n", + "Query ID: 189\n", + "Returned IDs: 996062,992182,990096,992576,991857,996001,996277,998045,995130,991011,993561,995536,994964,995834,997586,991685,993433,993873,996884,994955,999740,996371,998039,991789,993906,998158,995289,996219,995052,998495\n", + "Ground Truth: 996062,992182,990096,992576,991857,997920,996001,996277,998045,995130,991011,993561,995536,994964,995834,993778,997586,991685,993433,993873,997320,996884,994955,999740,996371,998039,994419,991789,993906,998158\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 190\n", + "Returned IDs: 994105,994906,996050,992664,998855,991332,993866,995168,997332,999653,991038,997666,998623,995536,996075,995695,997906,996657,991691,999827,996001,999212,999543,999139,999724,990998,991630,996326,994280,995603\n", + "Ground Truth: 994105,994906,996050,992664,998855,991654,991332,993866,995168,998051,997332,999653,991038,997666,998623,995536,996075,995695,997906,996657,991691,999827,996001,999212,999543,999139,999724,990998,991630,996326\n", + "Recall: 0.9333\n", "\n", "Query ID: 192\n", - "Returned IDs: 991372,993696,990628,994722,990885,996809,996296,991109,992047,992635,997098,996452,990697,991172,998734,994922,995210,999219,992927,995530,996875,995596,996913,999337,996350,998790,993504,993038,991425,990627\n", + "Returned IDs: 991372,993696,990628,994722,990885,996809,991109,992635,996452,990697,991172,994922,995210,995530,995596,998303,999337,996350,993504,991425,990627,996996,994964,993454,999210,993617,990957,999750,992041,991157\n", "Ground Truth: 991372,993696,990628,994722,990885,996809,996296,997166,991109,992047,992635,997098,996452,990697,991172,998734,994922,995210,999219,992927,995530,994859,996875,995596,996913,998303,999337,996350,998790,993504\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 193\n", + "Returned IDs: 991515,990255,992015,995401,999595,998593,996930,990573,998687,998874,994119,993992,990412,996142,992367,991835,996657,996043,996759,999629,992104,995421,993275,996388,998990,995864,996001,990314,994743,998878\n", + "Ground Truth: 991515,993599,990255,992015,995401,999595,998593,997723,996930,990573,998687,998874,994119,992977,993992,990412,996142,992367,991835,996657,996043,996759,999629,992104,999669,995421,993275,996388,998990,995864\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 194\n", + "Returned IDs: 997319,995633,994283,991810,998750,995611,992561,994204,990223,997141,996411,991330,993261,995289,995401,995948,994148,992289,997155,994783,998765,994742,997117,990814,995190,998525,993747,999235,995274,994422\n", + "Ground Truth: 997319,995633,994283,991810,998750,995611,992561,994204,990223,997141,996411,991330,993261,995289,995401,995948,994148,992289,997155,994783,998765,994742,997117,990814,995190,998525,999627,993747,999235,995274\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 195\n", + "Returned IDs: 991890,996809,998786,999430,990361,996830,996284,998695,996930,998593,995461,994382,990255,994431,999009,990930,991846,995701,995401,990288,990167,995289,998874,995895,999994,995421,990941,993232,997630,991255\n", + "Ground Truth: 991890,996809,998786,999430,990361,996830,996284,998695,996930,998593,995461,994382,990255,996049,994431,999009,990930,991846,997723,995701,995401,990288,990167,995289,991025,998874,995895,999994,995421,990941\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 196\n", + "Returned IDs: 998697,992293,991530,993948,997884,995701,992862,992609,998758,999653,993738,995672,996828,995184,998831,990005,996368,991836,995536,996714,991844,993560,995516,997868,997928,990211,991630,998017,999639,993515\n", + "Ground Truth: 998697,992293,991530,993948,997884,995701,992862,992609,998758,999653,993738,995672,996828,995184,998831,990005,996368,991836,994125,995536,996714,991844,996657,998200,993560,995516,997868,997928,990211,991630\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 197\n", + "Returned IDs: 994111,999026,999144,997015,995274,997976,995336,998692,998365,994100,994964,992630,997320,997192,996959,991467,993561,995132,999374,995701,996714,993484,997371,995928,995502,998779,995204,997918,997571,998302\n", + "Ground Truth: 994111,999026,999144,997015,995274,997976,995336,998692,995374,995401,998365,994100,994964,992630,997320,992047,993412,997192,996959,991467,993561,995132,999374,992331,995701,998011,996714,993484,997371,994959\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 198\n", + "Returned IDs: 995645,991392,997291,998244,992612,990615,997227,990043,993143,991609,998837,999257,999076,998607,991999,992671,997496,999122,995515,990310,992146,996840,993237,997370,995203,990142,991013,998567,999784,991157\n", + "Ground Truth: 995645,991392,997291,998244,992612,992734,990615,997227,990043,993143,992336,991609,998837,999257,999076,998607,991999,992671,997496,999122,995515,990310,992146,996840,993237,997370,995203,990142,991013,996930\n", "Recall: 0.9000\n", "\n", "Query ID: 199\n", - "Returned IDs: 990382,997476,991581,999827,996062,992314,993981,994510,998095,994463,998436,992161,993289,990899,990520,991930,997790,997642,993931,998158,992169,995954,997453,991596,995636,990557,990212,995185,991644,994411\n", + "Returned IDs: \n", "Ground Truth: 990382,997476,991581,999827,996062,992314,993981,994510,998095,994463,998436,992161,993289,990899,990520,991930,997790,996913,997642,993931,998158,992169,995954,997453,991596,995636,990557,990212,995185,991644\n", - "Recall: 0.9667\n", + "Recall: 0.0000\n", "\n", "Query ID: 200\n", - "Returned IDs: 997320,997370,996811,992944,997305,998950,991498,995886,993018,994266,995372,992041,990949,995250,990772,999453,995203,997301,995927,993154,991801,999210,993260,990532,997470,995438,997913,993241,995841,997371\n", + "Returned IDs: 997320,997370,992944,997305,998950,991498,995886,993018,994266,995372,992041,990949,999453,995203,997301,995552,995927,993154,991801,999210,993260,990532,997470,995438,997913,993241,995841,997371,999701,998180\n", "Ground Truth: 997320,997370,996811,992944,997305,998950,991498,995886,993018,994266,995372,992041,990949,995250,990772,999453,995203,997301,995552,990917,995927,993154,991801,999210,993260,990532,997470,995438,997913,993241\n", - "Recall: 0.9333\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 201\n", + "Returned IDs: 999082,991904,993879,997675,993385,997997,992801,991388,997588,999796,990875,994002,994413,996918,995159,990784,996505,995521,996386,993899,996472,998049,995080,994677,998405,998536,994835,992285,997459,993931\n", + "Ground Truth: 999082,991904,993879,997675,991061,993385,997997,992801,991388,997588,999796,990875,994002,994413,996918,995159,995200,990784,996505,995521,996386,993899,998107,996472,998049,995080,994677,997949,998405,998536\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 202\n", + "Returned IDs: 999267,998097,998021,990368,998076,993725,993715,996328,994016,998921,998812,995671,998848,999526,993719,991390,992825,999516,992068,990438,992636,990889,993053,990011,998536,992450,995705,994280,999338,995295\n", + "Ground Truth: 999267,998097,998021,990368,998076,993725,993715,996328,994016,998921,998812,995671,998848,999526,993719,991390,996429,992825,999516,990438,992068,992636,990889,993053,990011,998536,992450,995705,994280,999338\n", + "Recall: 0.9667\n", "\n", "Query ID: 203\n", - "Returned IDs: 992927,993260,994988,990429,994964,994537,996193,990786,996053,997320,999216,996576,993055,990885,999176,992069,998568,997918,993783,991795,991675,992571,995536,992497,994789,992225,999711,994552,998299,999800\n", + "Returned IDs: 992927,993260,994988,994964,994537,996193,990786,997320,993055,992069,998568,994984,997918,993783,991795,992571,995536,992497,994789,992225,994552,999800,993948,993696,993834,992330,997909,999077,998790,999129\n", "Ground Truth: 992927,993260,994988,990429,994964,994537,996193,990786,996053,997320,999216,996576,999362,993055,990885,999176,992069,998568,994984,997918,993783,991795,991675,992571,995536,992497,994789,992225,999711,994552\n", - "Recall: 0.9333\n", + "Recall: 0.7000\n", "\n", "Query ID: 204\n", - "Returned IDs: 990487,994117,996391,998370,995896,991350,992929,997554,999587,990211,997868,994793,994207,996368,996379,993998,994809,996654,996657,997723,990314,998990,996127,991973,995873,998686,996181,993831,991585,999639\n", + "Returned IDs: 990487,994117,996391,998370,995896,991350,992929,997554,990211,997868,994793,994207,996368,996379,994809,996654,996657,990314,998990,996127,995873,993831,991585,992423,999639,990081,994032,996147,990662,993681\n", "Ground Truth: 990487,994117,996391,998370,995896,991350,992929,997554,999587,990211,997868,994793,994207,996368,996379,993998,994809,996654,996657,997723,990314,998990,996127,991973,995873,998686,996181,993831,991585,992423\n", - "Recall: 0.9667\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 205\n", + "Returned IDs: 993637,997933,993906,992636,997052,995295,990392,993715,998039,998021,998097,990089,999138,990930,998593,997453,990845,991144,996001,993917,998076,990368,998440,995467,991021,997170,990027,999052,996906,990438\n", + "Ground Truth: 993637,997933,993906,993719,992636,997052,995295,990392,993715,998039,998021,994386,998097,990089,999138,990930,998593,997453,990845,991144,996147,996001,992547,993917,998076,990368,998440,995401,995467,991021\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 206\n", + "Returned IDs: 993885,993238,999533,997395,998029,995928,991743,997320,994266,992922,996805,991498,999827,994280,997732,998302,993250,997301,991932,993484,995961,997002,992182,994964,993080,996328,997822,990560,995861,990210\n", + "Ground Truth: 993885,993238,999533,997395,998029,995928,991743,997320,994266,992922,996805,991498,990814,999827,994280,997732,991611,993860,999290,998302,993250,997301,991932,993484,995961,991829,997002,992182,994964,993080\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 207\n", + "Returned IDs: 998165,992160,999782,995439,998045,996482,997406,992125,996769,992115,995289,996125,993830,994057,995052,991463,995182,990681,997046,990984,991603,993844,992925,995497,993390,998520,995285,992576,996277,991415\n", + "Ground Truth: 998165,992160,999782,995439,998045,996482,997406,992125,993236,996769,992115,995289,996125,993830,994057,995052,991463,995182,990681,997046,990984,991603,993844,992925,995497,993390,998520,996112,995285,992576\n", + "Recall: 0.9333\n", "\n", "Query ID: 208\n", - "Returned IDs: 994273,999415,993127,995122,992449,995423,995702,994013,999745,995701,999054,991636,996867,995182,996897,995685,998636,990082,995905,997077,994954,990152,994409,998776,995401,997975,998518,997674,996232,993856\n", + "Returned IDs: 994273,999415,993127,995122,992449,995423,995702,994013,995701,999054,991636,996867,995182,995685,990082,995905,994954,990152,994409,998776,995401,997975,998518,997674,996232,992660,993856,995891,993232,998162\n", "Ground Truth: 994273,999415,993127,995122,992449,995423,995702,994013,999745,995701,999054,991636,996867,995182,996897,995685,998636,990082,995905,997077,994954,990152,994409,998776,995401,997975,998518,997674,995303,996232\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 209\n", + "Returned IDs: 995167,996562,992199,993391,996216,997897,999748,998335,999829,990627,991037,993875,991596,999435,999093,992137,995626,992247,995214,997944,992151,990255,991148,997237,996501,993805,992825,993132,992056,992771\n", + "Ground Truth: 998731,995167,996562,992199,993391,996216,997897,999748,998335,999829,996986,990627,991037,993875,993195,991596,999435,993124,991170,999093,992137,995626,992247,999455,995214,997944,992151,990255,991148,997237\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 210\n", + "Returned IDs: 998749,992124,992213,992167,996576,992225,999041,991984,998232,993834,992238,996089,990550,996744,995516,996230,991157,995701,995897,994572,993766,990051,998950,990453,996138,993342,994809,996500,993902,999806\n", + "Ground Truth: 998749,992124,992213,992167,996576,992225,999041,991984,998232,993834,992238,990334,996089,990550,996744,995516,992276,996230,991157,995701,995897,994572,993766,998950,990051,990453,996138,993342,994809,996500\n", + "Recall: 0.9333\n", "\n", "Query ID: 211\n", - "Returned IDs: 999453,994167,991987,990999,991811,996161,991894,995596,996001,991459,999219,995899,990949,996657,990957,991445,995855,992979,996368,995281,990542,997320,997258,992562,991835,993848,995866,999639,996744,992384\n", + "Returned IDs: 999453,994167,991987,991811,995596,996001,991459,999219,995899,990949,996657,990957,992979,996368,995281,997320,997258,992562,991835,993848,995866,999639,996744,992384,991392,994253,995455,998720,999586,998790\n", "Ground Truth: 999453,994167,991987,990999,998567,991811,996161,991894,995596,996001,991459,993154,999219,995899,990949,996657,990957,991445,995855,992979,996368,995281,990542,997320,997258,992562,991835,993848,995866,999639\n", - "Recall: 0.9333\n", + "Recall: 0.7333\n", "\n", "Query ID: 212\n", - "Returned IDs: 992824,995459,990894,996380,999639,991272,995631,995503,991064,996742,990273,991432,996663,999908,998061,999387,997547,996894,991817,995728,993055,999674,993839,994560,993225,999346,998720,990061,992306,995724\n", + "Returned IDs: 992824,995459,990894,996380,999639,991272,995631,995503,996742,990273,991432,996663,999908,998061,999387,997547,996894,991817,995728,993055,999674,993839,994560,993225,999346,998720,990061,992306,995724,999806\n", "Ground Truth: 992824,995459,990894,996380,999639,991272,995631,995503,991064,996742,990273,991432,996663,999908,998061,999387,997547,996894,991817,995728,993055,999674,993839,994560,993225,999346,992752,998720,990061,992306\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 213\n", - "Returned IDs: 993879,995080,997675,995521,992801,991904,996198,990244,999082,994170,994409,996505,995401,993030,990527,996277,998045,993561,992448,995873,997822,998049,997832,991603,996472,994677,997020,997766,992260,999461\n", + "Returned IDs: 993879,995080,997675,995521,992801,996198,990244,999082,994170,994409,996505,993030,996277,998045,993561,992448,997822,998049,997832,996472,997020,993385,993738,994113,995648,995032,994589,996042,997723,993313\n", "Ground Truth: 993879,995080,997675,995521,992801,991904,996198,990244,999082,994170,994409,996505,995401,993030,990527,996277,998045,993561,992448,995873,997822,998049,997832,991603,996472,994677,997020,997766,992260,990140\n", - "Recall: 0.9667\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 214\n", + "Returned IDs: 994280,991747,990434,997897,995184,993280,993931,993533,993738,990364,998067,996062,995047,990802,995876,992636,993324,993787,991255,990956,999827,994891,990951,998416,990993,998697,997679,996328,990784,997790\n", + "Ground Truth: 991860,994280,991747,990434,997897,990212,995184,998006,993280,993931,993533,993738,990364,998067,996062,995047,990802,995876,995281,992636,993324,993787,991255,990956,999827,994891,990951,998416,994389,990993\n", + "Recall: 0.8333\n", "\n", "Query ID: 215\n", - "Returned IDs: 996062,998122,998495,999898,994869,990091,990194,990096,997242,990434,999827,991893,993106,999555,996219,996001,996028,995655,995536,992056,994266,995562,999139,990167,990382,990796,998593,994200,997790,996677\n", + "Returned IDs: 996062,998122,998495,999898,990194,990096,997242,990434,999827,991893,993106,999555,996219,996001,996028,998861,995655,995536,992056,994266,995562,999139,990167,990382,990796,998593,991296,991037,997586,996668\n", "Ground Truth: 996062,998122,998495,999898,994869,990091,990194,990096,997242,990434,999827,991893,993106,999555,996219,996001,996028,998861,995655,995536,992056,994266,995562,999139,990167,990382,990796,998593,994200,997790\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 216\n", - "Returned IDs: 997429,995184,993238,991856,990958,992357,993648,994835,999533,995963,994266,995034,996861,992008,993168,995800,994956,998458,990925,995927,997852,995783,993885,992526,999374,992147,995884,990399,997309,991468\n", + "Returned IDs: 997429,995184,993238,991856,990958,992357,993648,999533,995963,994266,996861,993168,995800,998458,990925,995927,997852,995783,993885,999374,992147,995884,997309,992637,993014,998495,992994,995755,997803,995771\n", "Ground Truth: 997429,995184,993238,991856,990958,992357,993648,994835,999533,995963,994266,995034,996861,991714,992008,993168,995800,994956,998458,990925,995927,997852,995783,993885,992526,999374,992147,995884,990399,997309\n", - "Recall: 0.9667\n", + "Recall: 0.7667\n", "\n", "Query ID: 218\n", - "Returned IDs: 990711,990392,997529,991703,993053,998862,993725,996411,994730,995002,992156,993240,998913,999302,999377,995633,994742,990701,998495,998779,994830,995401,990435,993166,993264,995096,992680,996408,991390,995601\n", + "Returned IDs: 990711,990392,997529,991703,993053,998862,993725,996411,994730,995002,992156,993240,998913,999302,999377,995633,994742,990701,998495,998779,994830,990435,993166,993264,992680,996408,993856,995601,992024,998710\n", "Ground Truth: 990711,990392,997529,991703,993053,998862,993725,996411,994730,995002,992156,993240,998913,999302,999377,995633,994742,990701,998495,998779,994830,995401,990435,993166,993264,995096,992680,996408,993856,991390\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 219\n", + "Returned IDs: 999343,993412,990366,996219,998779,995274,992630,998210,994964,998862,995336,990958,997479,998302,990974,998458,994100,999144,999026,996727,997320,990934,998090,990271,992384,996714,999647,990866,994990,998246\n", + "Ground Truth: 999343,993412,990366,996219,993909,998779,995274,992630,998210,994964,998862,995336,990958,990666,997920,997479,998302,990974,995374,990171,992008,998458,999331,994100,991311,999144,991362,999026,996727,997320\n", + "Recall: 0.7000\n", "\n", "Query ID: 220\n", - "Returned IDs: 994643,998536,997752,993232,999117,998383,999155,990551,995625,993995,996809,992288,998695,996797,991532,992558,991630,997666,995963,990941,994063,991571,996043,997131,991675,994742,995401,992604,992809,996395\n", + "Returned IDs: 994643,998536,997752,993232,999117,998383,999155,990551,993995,996809,992288,998695,996797,991532,992558,997666,995963,990941,994063,991571,996043,993077,997131,994742,992604,992809,996395,990361,990991,995274\n", "Ground Truth: 994643,998536,997752,993232,999117,998383,999155,990551,995625,993995,996809,992288,998695,996797,991532,992558,991630,997666,995963,990941,994063,991571,996043,993077,997131,991675,994742,995401,992604,992809\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 221\n", - "Returned IDs: 995596,995515,993143,991840,991848,999122,999784,999806,996840,990432,994709,997403,993748,995443,996219,995365,999676,992979,999144,991395,991485,997135,997379,999613,991977,992309,996062,997918,994916,993237\n", + "Returned IDs: 995596,995515,993143,991840,991848,999122,999784,999806,996840,990432,994709,997403,993748,995443,996219,999676,992979,999144,991395,991485,997135,997379,999613,992309,996062,997918,993237,994577,999453,994922\n", "Ground Truth: 995596,995515,993143,991840,991848,999122,999784,999806,996840,990432,994709,997403,993748,995443,996219,995365,999676,992979,999144,991395,991485,992047,997135,997379,999613,991977,992309,996062,997918,994916\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 223\n", - "Returned IDs: 997030,990096,994717,994643,994413,993666,992638,993115,998257,991309,994454,999898,990815,997077,997154,991190,998277,999555,997211,993566,992369,995218,994677,998045,991596,991188,997568,990361,994569,998536\n", + "Returned IDs: 997030,990096,994717,994643,994413,993666,992638,993115,999530,998257,991309,994454,999898,991190,998277,999555,997211,993566,992369,995218,994677,998045,997568,990361,998536,994835,992161,994656,991937,990132\n", "Ground Truth: 997030,990096,994717,994643,994413,993666,992638,993115,999530,998257,991309,994454,999898,990815,997077,997154,991190,997871,998277,999555,997211,993566,992369,995218,994677,998045,991596,991188,997568,990361\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", "\n", "Query ID: 224\n", - "Returned IDs: 990049,995609,996194,999528,995043,995551,994326,993678,991241,996232,990417,993181,990011,996435,992566,994541,992068,999830,998769,992473,990121,993245,994317,995475,995537,993315,996867,992563,995317,992951\n", + "Returned IDs: 990049,995609,996194,999528,995043,995551,994326,993678,991241,996232,990417,993181,990011,996435,992566,994541,992068,999830,992473,993245,994317,995475,995537,993315,997906,996867,992563,995317,992951,995666\n", "Ground Truth: 990049,995609,996194,999528,995043,995551,994326,993678,991241,996232,990417,993181,990011,996435,992566,994541,992068,999830,998769,992473,990121,993245,994317,995475,995537,993315,997906,990153,996867,992563\n", - "Recall: 0.9333\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 225\n", + "Returned IDs: 997806,991161,997251,993580,994280,994365,998067,994380,990294,991904,990942,991066,996960,999774,991103,996918,999294,992992,995268,994142,997918,993044,996328,991610,996683,994639,999994,997601,991038,993948\n", + "Ground Truth: 997806,991161,997251,993580,991273,994280,994365,998067,994380,990882,990294,991904,990942,991066,996960,992487,993439,999774,991103,996918,999294,992992,995268,994142,993727,997918,993044,996328,991610,990875\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 226\n", + "Returned IDs: 991140,999000,997723,996038,995685,998779,998999,995182,999412,991603,996968,998165,997832,997020,995648,994409,998297,996867,994948,996734,993103,990940,995507,995289,995421,997822,995401,998593,995052,996432\n", + "Ground Truth: 991140,999000,997723,996038,995685,998779,998999,995994,995182,999412,991603,996968,998165,997832,997020,995648,994409,998297,996867,994948,996734,993103,990940,995507,995289,997042,990406,995421,997822,995401\n", + "Recall: 0.9000\n", "\n", "Query ID: 227\n", - "Returned IDs: 992511,998974,996252,997437,993457,990337,999867,994894,995838,997174,996917,997728,999599,993978,995755,998649,994916,990957,997320,996648,996192,995616,995274,995963,997304,994630,995374,991995,992531,993049\n", + "Returned IDs: 992511,998974,996252,997437,993457,994894,995838,997174,996917,997728,999599,993978,995755,998649,990957,997320,996192,995274,997304,994630,992531,993049,992296,997035,996700,997371,999333,998460,998090,999074\n", "Ground Truth: 992511,998974,996252,997437,993457,990337,999867,994894,995838,997174,996917,997728,993222,999599,993978,995755,998649,994916,990957,997429,997320,996648,996192,995616,995274,995963,997304,994630,995374,991995\n", - "Recall: 0.9333\n", + "Recall: 0.6667\n", "\n", "Query ID: 228\n", - "Returned IDs: 998060,993487,994203,992352,994106,992098,990573,992000,991057,996075,994488,996001,995079,995544,991212,994013,990662,990314,999172,995515,997998,998593,996898,995873,994954,992569,995640,992097,991394,999590\n", + "Returned IDs: 998060,993487,994203,992352,994106,992098,990573,992000,991057,996075,994488,996001,995079,995544,991212,994013,990662,990314,999172,997998,998593,996898,995873,994954,992569,995640,992097,991394,999590,997554\n", "Ground Truth: 998060,993487,994203,992352,994106,992098,990573,992000,991057,996075,994488,996001,995079,995544,991212,994013,990662,990314,999172,996043,995515,997998,998593,996898,995873,994954,992569,995640,992097,991394\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 229\n", - "Returned IDs: 992558,990279,997547,997124,996196,990538,999715,995336,999908,995886,996606,993948,994334,997859,992718,990989,992971,991157,991052,996609,996918,999860,992857,990930,995552,996721,994163,993363,990343,992521\n", + "Returned IDs: 992558,990279,997547,997124,996196,990538,999715,995336,995886,996606,993948,994334,997859,992718,991157,996609,996918,999860,992857,995552,996721,994163,993363,990343,991498,994280,998697,991272,996318,992173\n", "Ground Truth: 992558,990279,997547,997124,996196,990538,999715,995336,999908,995886,996606,993948,994334,997859,992718,990989,992971,991157,991052,996609,996918,999860,992857,990930,995552,996721,994163,993363,990343,991498\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 230\n", - "Returned IDs: 994266,995184,997320,997913,990231,998170,992546,991256,999647,992330,995701,993853,997305,995417,996296,992637,996075,997554,999761,990958,996661,993834,997449,990650,996083,990535,993168,992748,991045,994476\n", + "Returned IDs: \n", "Ground Truth: 994266,995184,997320,997913,990231,998170,992546,991256,999647,992330,995701,993853,999041,997305,995417,996296,992637,996075,997554,999761,990958,996661,993834,997449,990650,996083,993168,990535,992748,991045\n", - "Recall: 0.9667\n", + "Recall: 0.0000\n", "\n", "Query ID: 231\n", - "Returned IDs: 998913,994783,998823,998370,998196,995551,991428,994742,992566,991981,994806,994692,995289,995397,995348,992415,995948,993939,995030,992827,997630,996411,991330,995633,998543,995300,994110,998028,996609,994587\n", + "Returned IDs: 998913,994783,998823,998370,998196,995551,991428,994742,992566,991981,994806,994692,995289,995397,995348,992415,995948,995030,992827,997630,996411,991330,995633,998543,994110,996609,994587,997529,990126,997823\n", "Ground Truth: 998913,994783,998823,998370,998196,995551,991428,994742,992566,991981,994806,994692,995289,995397,990846,995348,992415,995948,993939,995030,992827,997630,996411,991330,995633,998543,995300,994110,998028,996609\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 232\n", + "Returned IDs: 995841,997301,990405,995264,994942,999526,997292,990546,990191,992451,995372,991498,999523,998921,995623,996135,997470,996516,997320,998445,995306,993484,997333,999904,996771,993364,999338,998721,999594,999281\n", + "Ground Truth: 995841,997301,990405,995264,994942,999526,997292,990546,990191,992451,995372,991498,999523,998921,995623,996135,997470,996516,997320,998445,995306,993484,994741,997333,999904,996771,993387,993364,997876,999338\n", + "Recall: 0.9000\n", "\n", "Query ID: 233\n", - "Returned IDs: 992981,992445,997806,996328,994336,990956,996728,996861,998017,994125,993055,990005,997420,997884,996211,996918,997506,994142,999775,993948,996189,998959,993560,992293,990961,993877,998495,998067,995516,994280\n", + "Returned IDs: 992981,992445,997806,996328,994336,990956,996728,996861,994125,993055,990005,997420,997884,996211,996918,997506,994142,999775,993948,996189,998959,993560,990961,993877,998495,998067,995516,994280,990226,994380\n", "Ground Truth: 992981,992445,997806,996328,994336,990956,996728,996861,998017,992979,994125,993055,990005,997420,997884,996211,996918,997506,994142,999775,993948,996189,998959,993560,992293,990961,993877,998495,998067,995516\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 234\n", + "Returned IDs: 990067,994627,990092,994776,998550,992396,992147,992134,990470,996219,991015,991157,992942,999898,998899,996118,998090,990595,991014,997371,999144,995942,999784,998401,990231,998921,994964,990090,994757,993260\n", + "Ground Truth: 990067,994627,990092,994776,998550,992396,992147,992134,993412,990470,996219,998380,993633,991015,991157,992942,993852,999898,998899,996118,998090,990595,991014,997371,999144,995942,994831,999784,999102,998401\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 235\n", + "Returned IDs: 990435,994830,994742,995633,998913,995200,996132,993053,997529,992424,990012,996001,990621,999352,996219,995097,995289,994806,992446,999192,995401,991588,998495,990701,993335,998221,994307,996411,995002,993166\n", + "Ground Truth: 990435,994830,994742,995633,998913,995200,996132,993053,997529,992424,990012,996001,990621,999352,999261,996219,995097,995289,994806,992446,999367,999192,994145,995401,991588,998495,990701,993335,998221,994307\n", + "Recall: 0.9000\n", "\n", "Query ID: 236\n", - "Returned IDs: 998971,991596,992673,999663,999139,994510,996406,997127,999898,996985,996706,996809,999555,996733,991420,997980,998149,999435,992365,998006,996209,994335,992234,997242,999498,990226,995104,997506,991589,994103\n", + "Returned IDs: \n", "Ground Truth: 998971,991596,992673,999663,999139,994510,996406,997127,999898,996985,996706,996809,999555,996733,991420,997980,998149,999435,992365,998006,996209,994335,992234,997242,999498,990226,995104,997506,991589,999783\n", - "Recall: 0.9667\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 237\n", + "Returned IDs: 999093,991425,992571,991037,998303,993504,995839,994134,999840,990885,998447,990096,992168,994221,997977,990409,992351,992747,998154,997897,993260,992082,998495,995203,998732,999750,990212,991332,998299,991491\n", + "Ground Truth: 999093,991425,992571,991037,998303,993504,994468,995839,994134,999840,990885,998447,990096,992168,994221,997977,990409,992351,992747,998154,997897,993260,991596,992082,998495,998732,995203,999750,995167,990212\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 238\n", + "Returned IDs: 995289,998990,996930,995401,997822,997723,999587,993831,998874,993673,998165,996867,994343,997406,996379,995633,995190,998593,995685,992703,993412,995702,997046,997630,990517,992685,993848,997832,991603,997662\n", + "Ground Truth: 995289,998990,996930,995401,997822,997723,999587,993831,998874,993673,998165,996867,994343,997406,996379,995633,995190,998593,995685,996607,992703,993412,995702,997046,998323,996294,997630,990517,992685,990930\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 239\n", + "Returned IDs: 999676,998921,991485,995515,992612,994922,995788,998401,996840,999806,998950,999077,996351,991833,992944,996915,995983,998244,993260,998697,991157,990096,993143,999698,993289,993781,997320,995080,994195,995372\n", + "Ground Truth: 999676,995693,998033,998921,991485,995515,992612,994922,995788,998401,996840,999806,998950,999077,999261,995401,993470,996351,991833,992944,996915,995983,998244,993260,998697,991157,990096,996371,993143,999698\n", + "Recall: 0.8000\n", "\n", "Query ID: 240\n", - "Returned IDs: 997397,999110,999523,997301,999471,992451,995623,999904,991916,993484,990146,995306,990546,998760,997470,990405,997820,993426,993083,997292,995264,997320,992066,999242,995045,991582,990182,990191,996516,993853\n", + "Returned IDs: 997397,999110,999523,997301,999471,992451,995623,999904,991916,993484,990146,995306,990546,998760,997470,990405,997820,993426,993083,997292,995264,997320,992066,999242,995045,990191,996516,998921,995454,991134\n", "Ground Truth: 997397,999110,999523,997301,999471,992451,995623,999904,991916,993484,990146,995306,990546,998760,997470,990405,997820,993426,993083,997292,995264,997320,992066,999242,995045,999247,991582,990182,990191,996516\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 241\n", + "Returned IDs: 998697,991530,999639,995672,997586,995562,997506,991485,991256,998303,999783,995894,993055,996918,990399,996001,999806,995418,995455,993342,995536,999216,996406,997258,997320,990930,999041,994906,995701,993834\n", + "Ground Truth: 998697,991530,999639,995672,997586,995562,997506,991485,991256,998303,999783,997531,995894,993055,995855,996918,990399,996001,999806,995418,995455,990453,993342,995536,999216,995250,996406,997258,997320,990930\n", + "Recall: 0.8667\n", "\n", "Query ID: 242\n", - "Returned IDs: 995659,999271,990585,997668,997286,990089,999338,990000,994942,994618,997292,997984,995841,997470,997301,996516,995306,992069,991503,997320,998721,998921,995372,991570,998923,998445,998461,993419,994964,997113\n", + "Returned IDs: 995659,999271,990585,997668,997286,990089,999338,990000,994942,994618,997292,997984,995841,997470,997301,996516,995306,992069,991503,997320,998721,998921,995372,998923,998445,993419,994964,992451,997113,997820\n", "Ground Truth: 995659,999271,990585,997668,997286,990089,999338,990000,994942,994618,997292,997984,995841,997470,997301,996516,995306,992069,991503,997320,998721,998921,995372,991570,998923,998445,998461,993419,994964,992451\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 243\n", + "Returned IDs: 990467,994688,998649,996192,996869,990397,999301,998458,991268,995274,994630,990925,995942,996196,999500,996219,995502,997079,992630,997913,992291,998500,998797,991096,990949,996252,994783,998302,996727,992994\n", + "Ground Truth: 990467,994688,998649,996192,996869,990397,993656,999301,998458,991268,995274,992273,990315,993725,993412,991995,994630,996411,990925,995942,996196,999500,996219,995502,997079,992630,997913,992291,998500,998797\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 244\n", + "Returned IDs: 993158,990620,996389,998647,995410,995511,999715,996609,998827,992215,990731,995271,999240,997893,993473,997195,994334,993690,997859,992971,996476,990249,996127,999540,998213,992558,995388,993147,999562,995847\n", + "Ground Truth: 993158,990620,996389,998647,991809,995410,995511,999715,996609,998827,992215,990731,995271,999240,997893,994404,993473,997195,994334,993690,997859,992971,996476,990249,996127,993702,999540,990782,991447,998213\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 245\n", + "Returned IDs: 998913,995633,990126,990340,993219,996809,998495,990359,995289,995558,994382,993209,997975,994742,996226,994806,997209,996930,997023,996219,994835,995211,992566,996232,996328,992827,999646,993747,990621,995438\n", + "Ground Truth: 998913,995633,990126,990340,993219,996809,998495,990359,995289,995558,994382,997723,993209,997975,994742,998188,996226,994806,997209,996930,997023,996219,994835,996411,995211,992566,996232,996328,992827,995401\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 246\n", + "Returned IDs: 992646,999089,994670,996993,995336,999092,990595,998011,997479,996714,994197,991361,995793,992911,992183,998465,993596,997192,994757,998210,995824,999144,990275,994673,990647,995833,997605,994152,998951,995726\n", + "Ground Truth: 995374,990478,992646,999089,994670,996993,998385,995336,999092,990595,998011,995985,997479,996714,994197,991361,995793,992911,992183,998465,993596,997192,994757,998210,995824,999144,990275,994673,990647,995833\n", + "Recall: 0.8667\n", "\n", "Query ID: 247\n", - "Returned IDs: 995473,992815,991045,992694,999616,998524,990588,997620,995981,991900,997304,992874,997437,993300,997320,993392,997913,998365,997353,999291,999725,998246,992074,990535,998353,995701,991745,992384,992927,994266\n", + "Returned IDs: 995473,992815,991045,992694,999616,998524,990588,995981,991900,997304,992874,997437,997320,993392,997913,998365,997353,999291,999725,998246,992074,998380,990535,998353,995701,991745,992384,994266,990862,995942\n", "Ground Truth: 995473,992815,991045,992694,999616,998524,990588,997620,995981,991900,997304,992874,997437,993300,997320,993392,997913,998365,997353,999291,999725,998246,992074,998380,990535,998353,995701,991745,992384,992927\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 248\n", + "Returned IDs: 997377,995701,992851,994140,991349,995016,995813,996714,991154,991869,994922,995638,998831,994392,993738,991795,995245,999729,995184,990930,996368,995899,998200,991969,997928,995237,995555,999653,996001,994463\n", + "Ground Truth: 997377,995701,992851,997622,994140,991349,995016,995813,996714,991154,991869,994922,995638,998831,994392,993738,991795,995245,999729,995184,990930,996368,995899,998200,991969,997928,995237,990615,995555,999653\n", + "Recall: 0.9333\n", "\n", "Query ID: 249\n", - "Returned IDs: 997529,995020,990711,998719,999500,992156,991189,994120,998495,994382,993412,993725,996411,997207,995633,990126,991881,991669,990435,991075,993439,998671,994730,993747,998779,998982,998862,990256,995131,990340\n", + "Returned IDs: 997529,995020,990711,998719,999500,992156,991189,994120,998495,994382,993725,996411,995633,990126,991881,991669,990435,991075,997485,998671,994730,998779,998982,993656,998862,990340,995002,993053,997914,998067\n", "Ground Truth: 997529,995020,990711,998719,999500,992156,991189,994120,998495,994382,993412,993725,996411,997207,995633,990126,991881,991669,990435,991075,993439,997485,998671,994730,993747,998779,998982,993656,998862,990256\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 250\n", + "Returned IDs: 997723,995289,997046,993390,996867,998045,992160,995373,997466,993433,991603,996166,994409,993561,996322,997406,998070,995401,991415,996968,993831,997822,990836,999795,998165,996323,995702,996302,996360,991664\n", + "Ground Truth: 997723,995289,997046,993390,996867,998045,992160,993412,995373,997466,993433,991603,996166,994409,993561,996322,997406,998070,995401,996042,991415,996968,993831,997822,990836,999795,998165,996323,995702,996302\n", "Recall: 0.9333\n", "\n", "Query ID: 251\n", - "Returned IDs: 999826,997395,990790,999020,998080,994612,994170,996245,997184,993931,995807,997986,992879,998820,994770,995163,998049,992790,990243,997189,994280,998014,993149,992683,999967,991698,999898,995521,997918,993933\n", + "Returned IDs: 999826,997395,990790,999020,998080,994612,994170,996245,997184,993931,995807,997986,998820,994770,997301,995163,998049,992790,990243,997189,994280,993149,999967,991698,999898,995521,997918,993933,994774,997168\n", "Ground Truth: 999826,997395,990790,999020,998080,994612,994170,996245,997184,993931,995807,997986,992879,998820,994770,997301,995163,998049,992790,990243,997189,994280,998014,993149,992683,999967,991698,999898,995521,997918\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", "\n", - "Query ID: 258\n", - "Returned IDs: 990827,990043,998983,998937,990849,997370,996840,998551,997789,995988,999620,998734,991196,994922,999898,990858,997445,995515,993282,998236,991906,991348,997255,990500,993647,992169,993143,993237,995443,990701\n", - "Ground Truth: 990827,990043,998983,998937,990849,997370,996840,998551,997789,995988,999620,994735,998734,991196,994922,999898,990858,997445,995515,993282,998236,991906,991348,997255,990500,993647,992169,993143,993237,995443\n", - "Recall: 0.9667\n", + "Query ID: 252\n", + "Returned IDs: \n", + "Ground Truth: 992198,990084,997407,991432,995631,997548,996742,992494,998754,995510,992459,992824,992462,996196,992095,997758,998693,997055,997547,998061,997931,997775,996065,990192,997744,996663,991977,993828,995955,998614\n", + "Recall: 0.0000\n", "\n", - "Query ID: 259\n", - "Returned IDs: 996219,993168,995876,993637,995001,999647,998343,993499,990173,999009,995689,998495,997331,999673,996067,990784,999430,991037,995562,996549,991758,996062,994279,991109,992589,995438,990323,991372,999748,992960\n", - "Ground Truth: 996219,993168,995876,995001,993637,999647,998343,993499,990173,999009,995689,998495,997331,999673,996067,990784,999430,997992,991037,995562,996549,991758,996283,996062,994279,991109,992589,995438,990323,991372\n", - "Recall: 0.9333\n", + "Query ID: 253\n", + "Returned IDs: 994020,995168,995231,990998,999783,993504,991691,995701,995695,997965,998959,998855,993342,991459,995780,999724,998623,994105,994906,997506,993076,995894,997618,997977,994365,998831,992862,997906,994125,999653\n", + "Ground Truth: 994020,995168,995231,990998,999783,993504,991691,995701,995695,997965,998959,998855,993342,991459,995780,993821,999724,998623,994105,994906,997506,993076,995894,997618,998514,997977,994809,994365,998831,992862\n", + "Recall: 0.9000\n", "\n", - "Query ID: 261\n", - "Returned IDs: 998917,997220,994617,994491,997136,995259,998782,998139,991727,995588,998983,994534,992255,994741,990822,991999,999469,992085,998928,995334,991806,991405,993340,990809,994528,998236,990890,994785,999147,995316\n", - "Ground Truth: 998917,997220,994617,994491,997826,997136,995259,998782,998139,991727,995588,998983,994534,992255,994741,990822,991999,999469,992085,998928,995334,999535,991806,991405,993340,990809,994528,998236,990890,994785\n", + "Query ID: 254\n", + "Returned IDs: 996633,994830,991885,992227,990735,992006,991390,993335,996603,996328,997026,998055,996314,995735,994806,996495,993725,994682,999496,993560,993100,998761,991382,994751,994960,990484,995002,995191,991610,998610\n", + "Ground Truth: 996633,994830,991885,992227,990735,992006,991390,993335,996603,996328,997026,998055,996314,998716,995735,994956,994806,996495,993725,994682,999496,993560,993100,998761,991382,994751,994960,990484,995002,995191\n", "Recall: 0.9333\n", "\n", + "Query ID: 255\n", + "Returned IDs: 991817,998061,999763,994163,997614,996380,991432,999908,991871,993706,992824,997775,998308,992423,994270,991064,990538,990908,990894,995503,995459,990192,995983,999639,992462,995737,991550,991923,993839,990096\n", + "Ground Truth: 991817,998061,999763,994163,997614,996380,991432,999908,991871,993706,992824,997775,998308,992423,994270,991064,990538,990908,990894,995503,995459,992280,990192,995983,999639,992462,995737,991550,991923,993839\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 256\n", + "Returned IDs: 994125,995516,990897,990005,998697,995536,996918,993560,998758,997305,998536,994879,991530,997586,995672,991052,991630,993055,993948,998623,995963,991038,997752,997420,995701,993809,995455,997506,995237,995783\n", + "Ground Truth: 994125,995516,990897,990005,998697,995536,996918,993560,995401,997429,998758,997305,998536,994879,991530,997586,995672,991052,991630,993055,993948,998623,992293,995963,991038,997752,997420,995701,993809,995455\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 257\n", + "Returned IDs: 991720,993143,998921,995928,997370,991743,999077,992384,993785,992147,995336,995949,997918,996592,999710,997002,993260,995826,990366,997301,995515,996053,991249,994964,997661,993080,998090,995886,996504,994334\n", + "Ground Truth: 991720,993143,998921,995928,997370,991743,999077,992384,993785,998294,992147,995336,995949,997918,996592,999710,997002,993260,999109,995826,990366,997301,996829,995515,991626,996053,991249,994964,997661,993080\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 258\n", + "Returned IDs: 990827,990043,998983,998937,997370,996840,998551,997789,995988,999620,998734,991196,994922,999898,990858,995515,993282,998236,991348,997255,993647,992169,993143,993237,995443,996100,992641,996681,990332,998738\n", + "Ground Truth: 990827,990043,998983,998937,990849,997370,996840,998551,997789,995988,999620,994735,998734,991196,994922,999898,990858,997445,995515,993282,998236,991906,991348,997255,990500,993647,992169,993143,993237,995443\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 259\n", + "Returned IDs: 996219,993168,995876,993637,995001,998343,990173,998495,997331,999673,996067,990784,999430,997992,991037,995562,996549,996062,994279,992589,995438,990323,999748,990891,992576,990802,996408,997555,990958,993906\n", + "Ground Truth: 996219,993168,995876,995001,993637,999647,998343,993499,990173,999009,995689,998495,997331,999673,996067,990784,999430,997992,991037,995562,996549,991758,996283,996062,994279,991109,992589,995438,990323,991372\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 260\n", + "Returned IDs: \n", + "Ground Truth: 990796,995344,999829,992056,998855,990167,996249,999555,991038,999375,991037,998383,997332,997378,990891,992664,994193,999435,998335,991758,996702,994105,999430,996809,995895,995562,990194,998495,998594,996001\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 261\n", + "Returned IDs: 998917,997220,994617,994491,997826,995259,998782,998139,991727,995588,998983,994534,991999,992085,998928,995334,999535,991806,993340,990809,994528,994785,999147,995316,993143,999592,993282,995788,993570,994318\n", + "Ground Truth: 998917,997220,994617,994491,997826,997136,995259,998782,998139,991727,995588,998983,994534,992255,994741,990822,991999,999469,992085,998928,995334,999535,991806,991405,993340,990809,994528,998236,990890,994785\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 262\n", + "Returned IDs: 995345,993591,990368,990889,990928,995283,997933,994148,992415,990089,994379,999126,990918,997822,994663,997319,993053,997386,998165,996147,996906,998162,994783,992894,997529,994021,998765,995289,995295,994689\n", + "Ground Truth: 995345,993591,990368,990889,990928,995283,997933,994148,992415,990089,994379,999126,990918,997822,994663,997319,993053,997386,993886,998165,991923,991973,996147,996906,998162,994783,991537,992894,996382,997529\n", + "Recall: 0.8333\n", + "\n", "Query ID: 264\n", - "Returned IDs: 991137,996653,999533,998302,993845,997855,997320,999109,993579,996726,993066,997381,995569,995841,994978,999338,994332,999827,991714,997984,992318,993807,992228,995659,998445,992647,998797,998349,996729,990146\n", + "Returned IDs: 991137,996653,999533,998302,993845,997320,997921,999109,993579,996726,997381,995841,994978,999338,994332,999827,991714,997984,991787,992318,993807,992228,995659,992647,998797,990146,992896,992617,995372,993484\n", "Ground Truth: 991137,996653,999533,998302,993845,997855,997320,997921,999109,993579,996726,990179,993066,997381,995569,995841,994978,999338,994332,999827,991714,997984,991787,992318,993807,992228,995659,998445,992647,998797\n", - "Recall: 0.9000\n", + "Recall: 0.8333\n", "\n", "Query ID: 266\n", - "Returned IDs: 999356,998353,995981,998163,993853,998246,997728,992296,990453,994879,995755,990231,998662,992074,996075,993529,993186,990708,996609,994100,999761,996384,992848,990799,991895,999740,997192,998950,998332,991721\n", + "Returned IDs: 999356,998353,995981,998163,993853,998246,997728,992296,994879,990231,998662,992074,996075,993529,993186,990708,996609,994100,996384,992848,999740,997192,998950,998332,991721,992661,998119,997305,998697,992546\n", "Ground Truth: 999356,998353,995981,998163,993853,998246,997728,992296,990453,994879,995755,990231,998662,991782,992074,996075,993529,993186,991913,990708,996609,994100,999761,996384,992848,990799,991895,999740,997906,997192\n", - "Recall: 0.9000\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 267\n", + "Returned IDs: \n", + "Ground Truth: 991685,991021,995945,997453,996062,997541,993472,998416,990899,990407,992182,997813,993665,992780,991633,990382,998593,998158,995895,995834,996566,996071,993931,990104,992664,994925,995421,999435,991857,999827\n", + "Recall: 0.0000\n", "\n", "Query ID: 268\n", - "Returned IDs: 992944,995360,991848,993781,996829,990043,999784,998991,994543,994341,991013,999210,997135,993708,998950,993260,996193,991416,996802,998180,990182,995375,999122,996007,993365,997037,993633,997457,998726,993279\n", + "Returned IDs: 992944,995360,991848,993781,990043,999784,998991,994543,994341,991013,999210,997135,993708,998950,993260,996193,996802,998180,999122,997457,993279,995203,997379,992724,998123,993215,992907,997471,994171,992950\n", "Ground Truth: 992944,995360,991848,993781,996829,990043,999784,998991,994543,994341,991013,999210,999577,997135,993708,998950,993260,996193,991416,996802,998180,990182,992168,995375,999122,996007,993365,997037,993633,997457\n", - "Recall: 0.9333\n", + "Recall: 0.6667\n", "\n", "Query ID: 269\n", - "Returned IDs: 996429,997918,995701,998932,994276,995785,996217,992047,998303,991530,994832,994221,993260,990409,993055,995932,990930,998123,999384,994754,992168,998697,997897,990751,996196,993504,995203,992571,997586,990494\n", + "Returned IDs: 996429,997918,995701,998932,994276,996217,998303,991530,994221,993260,990409,993055,995932,990930,999384,994754,992168,998697,997897,990751,996196,993504,995203,992571,997586,990494,991332,999860,993087,992163\n", "Ground Truth: 996429,997918,995701,998932,994276,995785,996217,992047,998303,991530,994832,994221,993260,990409,993055,995932,990930,998123,999384,994754,992168,998697,997897,990751,996196,993504,995203,994514,990425,992571\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", "\n", "Query ID: 270\n", - "Returned IDs: 990530,992562,997925,999639,995701,995401,995005,999041,990588,993821,995886,990217,991154,993809,990930,996067,998200,991459,993834,997337,999680,996918,992933,999219,991609,992927,994510,997918,995932,994081\n", + "Returned IDs: 992562,997925,999639,995701,995005,999041,995886,991154,993809,996067,998200,991459,997337,996918,999219,991609,992927,994510,997918,995932,994081,993673,990897,995980,999444,999784,999122,998536,999384,997586\n", "Ground Truth: 990530,992562,997925,999639,995701,995401,995005,999041,990588,993821,995886,990217,991154,993809,990930,996067,998200,991459,993834,997337,999680,996918,992933,999219,991609,992927,994510,997918,995636,995932\n", - "Recall: 0.9667\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 271\n", + "Returned IDs: \n", + "Ground Truth: 990736,995985,993387,995942,999263,997301,993250,996785,990595,991444,994978,997913,995841,998692,999291,997543,997972,993562,998170,990390,992709,997539,997704,991008,996218,992107,997192,997620,997320,998960\n", + "Recall: 0.0000\n", "\n", "Query ID: 272\n", - "Returned IDs: 995208,996968,996867,998297,995401,990337,994794,999530,992169,993412,990432,995274,994409,998536,995373,997191,990561,994677,997871,998045,994301,995515,995374,993143,996727,998080,991052,997723,997319,996769\n", + "Returned IDs: 995208,996968,996867,998297,995401,994794,999530,992169,990432,995274,994409,995373,997191,990561,997871,998045,994301,995515,993143,996727,998080,991052,997723,996504,996769,995783,992364,992944,996903,990995\n", "Ground Truth: 995208,996968,996867,998297,995401,990337,994794,999530,992169,993412,990432,995274,994409,998536,995373,990561,997191,994677,997871,998045,994301,995515,997775,997403,995374,993143,996727,998080,991052,996438\n", - "Recall: 0.9000\n", + "Recall: 0.7333\n", "\n", "Query ID: 273\n", - "Returned IDs: 990072,999384,998861,997666,990096,997906,996623,991188,997685,991459,995780,998419,991471,993275,997265,999362,998623,993821,996209,998536,995701,990194,998383,999653,993561,996733,994105,994906,999827,999319\n", + "Returned IDs: 990072,999384,998861,997666,990096,997906,996623,997685,991459,995780,991471,993275,997265,999362,998623,996209,998536,995701,993561,999272,996733,994105,999827,999319,994955,997576,999736,995536,999194,990768\n", "Ground Truth: 990072,999384,998861,997666,990096,997906,996623,991188,997685,991459,995780,998419,991471,993275,997265,999362,998623,993821,996209,998536,995701,990194,992977,998383,999653,993561,999272,996733,994105,994906\n", - "Recall: 0.9333\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 274\n", + "Returned IDs: 995702,995907,994380,996867,991161,993044,993188,997675,993597,996042,992576,994419,992212,993313,991794,999528,998106,991011,996388,997289,997822,992894,998165,995521,995666,999619,997020,991722,992413,993127\n", + "Ground Truth: 995702,995907,994380,996867,991161,993044,993188,997675,993597,996042,992576,994419,992212,993313,991794,999528,998106,991011,996388,997289,997822,992894,998165,995521,995666,999619,990725,997020,991722,992413\n", + "Recall: 0.9667\n", "\n", "Query ID: 275\n", - "Returned IDs: 990020,997320,991989,990532,997661,997918,990207,993785,999219,991795,992950,991829,998634,998302,996646,999446,995852,996075,991311,993906,999444,994171,997227,997906,996735,994307,991984,999990,994081,993561\n", + "Returned IDs: 990020,997320,990532,997661,997918,993785,991795,992950,998302,996646,999446,995852,996075,993906,999444,994171,997227,991984,999990,994081,993561,999473,996001,996028,992546,995203,993157,993484,995993,994964\n", "Ground Truth: 990020,997320,991989,990532,997661,997918,990207,993785,999219,991795,992950,991829,998634,998302,996646,999446,995852,996075,991311,993906,993412,999444,994171,997246,997227,997906,996735,994307,991984,999990\n", - "Recall: 0.9333\n", + "Recall: 0.6333\n", "\n", "Query ID: 277\n", - "Returned IDs: 992546,990515,991431,993392,999180,992339,992994,997614,990958,994653,998246,990118,997305,994757,997437,996075,998797,991479,995981,994079,997320,990660,998562,997976,994776,997543,997470,991045,997913,993662\n", + "Returned IDs: 992546,990515,991431,999180,992339,992994,997614,990958,998246,990118,997305,994757,997437,996075,998797,991479,995981,994079,997320,990660,998562,994776,997543,997470,991045,997913,993662,998101,992119,997192\n", "Ground Truth: 992546,990515,991431,993392,992583,999680,992238,999180,992339,992994,997614,990958,994653,998246,990118,997305,994757,997437,996075,998797,991479,995981,994079,997320,990660,998562,997976,994776,997543,997470\n", - "Recall: 0.9000\n", + "Recall: 0.8000\n", "\n", "Query ID: 278\n", - "Returned IDs: 994427,990231,994403,996921,992969,993220,995697,992363,991743,998550,994171,996646,999710,994450,992119,993662,993278,997320,994699,994048,995351,990135,995942,999167,994192,997354,990910,996540,990066,990182\n", + "Returned IDs: 994427,990231,994403,992969,995697,998550,994171,996646,999710,994450,992119,993662,997320,994699,990135,999167,997354,990910,996540,990066,990182,997896,990957,999328,994961,991260,994604,997913,993264,990650\n", "Ground Truth: 994427,990231,994403,996921,992969,993220,995697,992363,991743,998550,994171,996646,999710,994450,992119,993662,993278,997320,994699,991320,994048,995351,990135,995942,999167,994192,997354,990910,996540,990066\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 279\n", + "Returned IDs: 996388,993390,999080,995665,998979,996379,993561,991415,995289,993144,997406,995701,996719,991857,998128,992125,994811,991954,998045,998874,991630,997787,991113,996924,991011,997046,994113,996181,998165,991603\n", + "Ground Truth: 996388,993390,999080,995665,998979,996379,993561,991415,995289,993144,997406,995701,996719,991857,998128,992125,994811,995761,991954,996294,998045,998874,991630,995864,997787,991113,996924,991011,997046,994113\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 280\n", + "Returned IDs: 992628,996016,996794,995535,991382,993391,996474,999280,998803,996184,998084,994525,996938,995880,999205,997798,995735,992237,997084,996839,999634,992867,992073,998153,992754,995346,996650,994472,992100,995191\n", + "Ground Truth: 992628,996016,996794,995535,991382,993391,996474,999280,998803,996184,998084,994355,994525,996938,995880,999205,997798,995735,992237,997084,996839,999634,992867,992073,998153,992754,995346,996650,994472,992100\n", "Recall: 0.9667\n", "\n", "Query ID: 281\n", - "Returned IDs: 991269,993209,997023,993586,999957,995778,997723,991507,997544,994496,999054,998188,991639,991943,999415,999047,991140,998779,991136,999563,996867,992334,997975,990372,998295,994881,992582,995289,997832,990361\n", + "Returned IDs: 991269,993209,997023,993586,999957,996607,995778,997723,991507,997544,994496,999054,998188,991639,999415,999047,991140,998779,991136,999563,996867,997975,990372,992413,994881,992582,995289,997832,995401,995665\n", "Ground Truth: 991269,993209,997023,993586,999957,996607,995778,997723,991507,997544,994496,999054,998188,991639,991943,999415,999047,991140,998779,997101,991136,999563,996867,992334,997975,990372,998295,992413,994881,992582\n", - "Recall: 0.9000\n", + "Recall: 0.8667\n", "\n", "Query ID: 282\n", - "Returned IDs: 991820,993080,998720,991981,993342,995906,993809,998514,990860,996911,999444,995160,998593,995852,990532,999916,990866,998710,995002,999922,996001,991795,999647,997790,995961,995636,993906,994636,990622,992314\n", + "Returned IDs: 991820,993080,998720,991981,993342,995906,993809,990860,996911,999444,998593,995852,990532,999916,993290,997969,998710,995002,999922,996001,997790,993906,994636,990622,990078,992632,994783,993637,995184,995256\n", "Ground Truth: 991820,993080,998720,991981,993342,995906,993809,998514,990860,996911,999444,995160,998593,995852,990532,996490,999916,990866,993290,999272,997969,998710,995002,999922,996001,991795,999647,997790,995961,995636\n", - "Recall: 0.8667\n", + "Recall: 0.7000\n", "\n", "Query ID: 283\n", - "Returned IDs: 999912,998039,997920,996996,995384,995906,990737,997028,991565,997410,997529,990974,997345,995526,990315,999949,998169,993637,996219,994964,995222,994293,999261,990891,994783,999647,993378,993168,998382,999151\n", + "Returned IDs: 999912,998039,997920,996996,995384,995906,990737,997028,991565,997410,997529,990974,997345,995526,990315,999949,998169,993637,996219,994964,994293,999261,994783,999647,993378,991831,993168,998382,999151,990728\n", "Ground Truth: 999912,998039,997920,996996,995384,995906,990737,997028,991565,997410,997529,990974,997345,995526,990315,999949,998169,993637,996219,994964,995222,994293,991359,999261,990891,994783,999647,993378,991831,995401\n", - "Recall: 0.9000\n", + "Recall: 0.8667\n", "\n", "Query ID: 284\n", - "Returned IDs: 996524,993012,990366,997811,991373,997466,998568,993696,999202,992330,990866,996857,993628,992238,993834,997586,993823,997806,995132,992019,998403,992068,995268,995516,999598,994879,990991,992521,997287,994125\n", + "Returned IDs: \n", "Ground Truth: 996524,993012,990366,997811,991373,997466,998568,993696,999202,992330,990866,996857,993628,992238,993834,997586,993823,997690,997806,995132,992019,998403,992068,995268,995516,999598,994879,990991,992521,997287\n", - "Recall: 0.9667\n", + "Recall: 0.0000\n", "\n", "Query ID: 285\n", - "Returned IDs: 993805,990256,991052,994714,995516,990897,994869,996406,999646,993673,996811,994956,991982,996918,992979,992147,995432,996762,997126,998318,991824,990563,992795,993948,991977,994125,991038,997765,996168,990209\n", + "Returned IDs: 993805,990256,991052,995516,990897,994869,996406,999646,996811,994956,991982,996918,996695,995432,996762,997126,998318,990563,992795,993948,994125,991038,996861,991860,998123,992558,994510,993143,999820,994558\n", "Ground Truth: 993805,990256,991052,993255,994714,995516,990897,994869,996406,999646,999676,993673,996811,994956,991982,996918,996695,992979,992147,995432,996762,997126,998318,991857,991824,990563,992795,993948,991977,994125\n", - "Recall: 0.8667\n", + "Recall: 0.7000\n", "\n", "Query ID: 286\n", - "Returned IDs: 998779,998862,997640,993412,998999,993293,997204,996727,993831,999026,994382,996436,997319,997732,999021,991923,994409,995401,992169,995216,997355,993280,994436,995374,995940,991743,996042,995274,997871,997832\n", + "Returned IDs: 998779,998862,997640,993412,993293,997204,996727,999026,994382,996436,997319,999021,994409,995401,992169,995216,997355,994436,995374,995940,995274,997871,997832,995685,990675,990940,995200,996379,996368,997320\n", "Ground Truth: 998779,998862,997640,993412,998999,993293,997204,996727,993831,999026,994382,996436,997319,997732,999021,991923,994409,995401,992169,995216,997355,994436,993280,995374,995940,991743,993226,996042,995274,997871\n", - "Recall: 0.9667\n", + "Recall: 0.7333\n", "\n", "Query ID: 287\n", - "Returned IDs: 994313,990696,995783,999496,999871,990872,996087,997914,991319,990386,998293,999105,997919,997752,997506,996362,998388,990083,997429,993678,999179,994051,993621,990061,995464,993727,993441,996814,993080,996429\n", + "Returned IDs: 994313,990696,995783,999496,999871,990872,996087,997914,991319,990386,998293,999105,997919,997752,996362,990083,993678,999179,994051,993621,990061,995464,993727,993441,996814,993080,992571,996429,996825,991269\n", "Ground Truth: 994313,990696,995783,999496,999871,990872,996087,997914,991319,990386,998293,999105,997919,997752,997506,996362,998388,990083,997429,993678,999179,994051,993621,992617,990061,995464,993727,993441,993080,996814\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 288\n", + "Returned IDs: 996249,991977,990796,994193,994605,995760,995727,998855,996882,999281,999375,998450,993316,990487,993681,992435,995357,993500,990776,992376,990206,992056,995618,992558,993715,990998,996809,995965,992052,997309\n", + "Ground Truth: 996249,991977,990796,994193,994605,995760,995727,998855,996882,999281,999375,998450,993316,990487,993681,992435,995357,993500,990776,992376,990206,992056,995618,992558,993715,990998,997104,996809,995965,992052\n", "Recall: 0.9667\n", "\n", "Query ID: 289\n", - "Returned IDs: 995289,993844,997403,998014,992488,992289,994204,993433,995401,995766,996438,993143,993673,998536,997319,993762,996867,994783,996930,991840,997897,993232,995633,992052,998913,990941,992407,993209,999453,997822\n", + "Returned IDs: 995289,993844,998014,992488,992289,994204,993433,995766,993143,998536,997319,993762,996867,994783,991840,997897,993232,995633,992052,998913,990941,999453,997822,996809,995788,996769,990882,990681,990340,998165\n", "Ground Truth: 995289,993844,997403,998014,992488,992289,994204,993433,995401,995766,996438,993143,993673,998536,997319,993762,996867,994783,996930,991840,993232,997897,995633,992052,998913,990337,990941,992407,993209,999453\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 290\n", + "Returned IDs: 996657,993848,998593,990662,990811,994587,992293,992367,991185,998549,995289,991603,993390,990255,998779,993738,998165,996001,992703,991835,997822,994809,996759,996368,996147,998862,995158,991630,993561,996277\n", + "Ground Truth: 996657,993848,998593,990662,990811,994587,996864,992293,995401,992367,991185,998549,995289,991603,997723,993390,990255,998779,993738,998165,996001,992703,991835,997822,999587,990930,994809,996759,996368,996294\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 291\n", + "Returned IDs: 991795,996987,992448,996187,994409,998143,998361,998205,995864,994954,994872,996256,993003,999250,999926,998941,991969,998756,991349,997868,990341,993672,990603,995016,996162,998527,996919,991892,998278,994865\n", + "Ground Truth: 991795,996987,992448,996187,994409,998143,998361,998205,995864,994954,994872,996256,993003,999250,999926,998941,991969,998756,991349,997868,990341,999866,993672,990603,995016,996162,998527,996919,991892,998278\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 292\n", + "Returned IDs: 993127,993400,997674,995891,995470,999884,995423,998651,992448,995032,994273,991276,998147,999918,996621,999255,992967,996472,995122,994413,998321,999043,994113,995905,992048,994419,997675,994409,991699,992753\n", + "Ground Truth: 993127,993400,997674,995891,995470,999884,995423,998651,992448,995032,994273,991276,998147,999918,996621,999255,992967,996472,995122,994413,998321,999043,994113,995905,992048,994419,997675,994409,991699,998973\n", "Recall: 0.9667\n", "\n", "Query ID: 293\n", - "Returned IDs: 994672,994247,993715,999139,994577,995801,995349,993504,991552,993055,995839,993139,991245,995737,996884,993181,994743,991648,994351,993609,999079,990885,995622,997076,992747,992052,991332,999806,993489,991605\n", + "Returned IDs: 994672,994247,993715,999139,994577,995801,995349,993504,991552,993055,995839,993139,991245,995737,996884,993181,991648,994351,993609,999079,990885,995622,992747,992052,991332,999806,993489,999927,991605,999526\n", "Ground Truth: 994672,994247,993715,999139,994577,995801,995349,993504,991552,993055,995839,993139,991245,995737,996884,993181,994743,991648,994351,993609,999079,990885,995622,997076,992747,992052,991332,999806,993489,999927\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 294\n", - "Returned IDs: 995139,995905,995507,999619,993468,996038,991027,995470,996323,993400,997046,999578,997832,995547,992448,997139,993421,995923,997674,999508,990975,996867,994554,999530,990634,994002,993597,997252,996322,994278\n", + "Returned IDs: 995139,995905,995507,999619,996038,991027,995470,996323,993400,997046,999578,997832,995547,992448,997139,993421,995923,997674,999508,990975,996867,994554,999530,993597,997252,996322,994278,991276,994409,994419\n", "Ground Truth: 995139,995905,995507,999619,993468,992660,996038,991027,995470,996323,993400,997046,999578,997832,995547,997139,992448,993421,995923,997674,999508,990975,996867,994554,999530,990634,994002,993597,997252,996322\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 295\n", - "Returned IDs: 998294,998380,995949,999263,997301,994964,990561,998697,998090,997320,991530,991157,995928,998029,997913,993342,999783,999444,995802,995672,998302,997032,995638,998365,999331,997371,998692,998607,990366,998303\n", + "Returned IDs: 998294,998380,995949,999263,997301,994964,990561,998697,998090,997320,991530,995928,998029,997913,993342,999783,999444,995802,995672,998302,998365,997371,998692,990210,990366,998303,990146,999639,991743,990958\n", "Ground Truth: 998294,998380,995949,999263,997301,994964,990561,998697,998090,997320,991530,991157,995928,998029,997913,993342,999783,999444,995802,995672,998302,997032,995638,998365,999331,997371,990310,998692,998607,997135\n", - "Recall: 0.9333\n", + "Recall: 0.7667\n", "\n", "Query ID: 296\n", - "Returned IDs: 991720,999343,992147,997842,998921,995372,992169,996915,998259,991743,997192,995515,991889,998877,993080,991485,999698,997918,995927,992944,991406,995443,994280,995771,990648,995336,990432,992309,998229,990917\n", + "Returned IDs: 991720,992147,997842,998921,995372,992169,996915,991743,997192,995515,991889,998877,993080,991485,999698,997918,995927,992944,995443,994280,995771,995336,990432,998229,991503,994919,997984,995788,992942,993215\n", "Ground Truth: 991720,999343,992147,997842,998921,995372,992169,996915,998259,993857,991743,997192,995515,991889,998877,993080,991485,999698,997918,995927,992944,991406,995443,994088,994280,995771,990648,995336,990432,992309\n", - "Recall: 0.9333\n", + "Recall: 0.7667\n", "\n", "Query ID: 297\n", - "Returned IDs: 999616,995981,991720,995792,997437,999099,999180,990588,999596,992296,993168,998567,997320,994916,995336,997759,997353,998353,995372,999527,992748,993893,998828,991045,997875,990339,996075,998607,995913,995184\n", + "Returned IDs: 999616,995981,997437,999180,990588,999596,992296,993168,997320,995336,997759,997353,998353,995372,992748,994695,998828,991045,997875,990339,996075,998607,995184,993080,997309,994894,994964,990957,995927,997018\n", "Ground Truth: 999616,995981,991720,995792,997437,999099,999180,990588,999596,992296,993168,998567,997320,994916,990778,995336,997759,997353,998353,995372,999527,992748,994695,993893,998828,991045,997875,990339,996075,998607\n", - "Recall: 0.9333\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 298\n", + "Returned IDs: 994745,991980,998716,998803,998903,991911,994830,993335,990730,996153,993101,998584,998030,991554,993568,996603,990678,991187,995735,997952,992495,996357,994683,992598,996183,991081,991297,993522,998440,998949\n", + "Ground Truth: 994745,991980,998716,998803,998903,991911,994830,993335,990730,996153,993101,998584,998030,991554,993568,996603,990678,991187,995735,997952,999786,992495,996357,994683,992598,996183,991081,998484,996503,991297\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 299\n", + "Returned IDs: 998039,991869,990076,993561,995923,996028,990845,995289,998633,990096,999729,993164,994925,990930,994853,994409,992851,994140,995813,997716,995526,991958,996001,990634,999516,995401,996062,991795,997386,997170\n", + "Ground Truth: 998039,991869,990076,993561,995923,996028,990845,995289,998633,990096,999729,993164,994925,990930,994853,994409,992851,994140,995813,997716,995526,991958,996001,990634,999516,995401,995678,996062,991795,997386\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 300\n", + "Returned IDs: 990602,996367,993583,993534,999305,990226,996016,992306,991432,993255,998357,999540,991605,992113,997547,993839,998652,993427,999479,997127,995618,995631,991977,993380,996742,999908,990423,994254,994560,994472\n", + "Ground Truth: 990602,996367,993583,993534,999305,990226,996016,992306,991432,993255,998357,999540,991605,992113,997547,993839,998652,993427,999479,997127,995618,995631,991977,993380,995542,996742,999908,990423,994254,994560\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 301\n", + "Returned IDs: 997585,994922,991392,995557,999620,996402,995057,990043,990090,994964,993484,996563,998837,993389,999437,993237,999484,994789,991150,992944,993355,993168,990229,990585,998765,992907,990690,994685,996219,990483\n", + "Ground Truth: 997585,994922,991392,995557,999620,996402,995057,990043,990090,994964,993484,996563,998837,993389,999437,991582,993237,999484,994789,991150,992944,993355,996419,993168,992346,993775,990229,990585,998765,997876\n", + "Recall: 0.8333\n", "\n", "Query ID: 302\n", - "Returned IDs: 996606,993534,993427,995152,991983,995618,992113,994679,992857,993380,992558,990279,995710,999540,997849,992496,990345,997547,997917,991046,996750,992335,998151,991871,992092,998840,991026,990125,996367,993583\n", + "Returned IDs: 996606,993534,993427,995152,991983,995618,992113,994679,992857,993380,992558,990279,995710,999540,997849,992496,990345,997547,997917,991046,996750,998151,992092,998840,991026,990125,996367,993583,992306,999874\n", "Ground Truth: 996606,993534,993427,995152,991983,995618,992113,994679,992857,993380,992558,990279,995710,999540,997849,992496,991058,990345,997547,997917,991046,996750,992335,998151,991871,992092,998840,991026,990125,996367\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", "\n", "Query ID: 303\n", - "Returned IDs: 995459,998142,992824,997775,999171,994679,991432,991817,991370,993706,993077,991550,994276,993583,997864,999908,992392,995503,993647,993828,997906,995983,996219,992280,999639,995737,998061,990765,991923,993260\n", + "Returned IDs: 995459,998142,992824,997775,999171,994679,991432,991817,993706,993077,991550,994276,993583,997864,999908,992392,995503,993647,993828,995983,996219,992280,999639,995737,998061,990765,993260,990538,993891,999560\n", "Ground Truth: 995459,998142,992824,997775,999171,994679,991432,991817,991370,993706,993077,991550,994276,993583,997864,999908,992392,995503,993647,993828,997906,995983,996219,992280,999639,995737,998061,990765,991923,990538\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", "\n", "Query ID: 304\n", - "Returned IDs: 999504,997388,997157,996727,990797,991923,991082,998094,997640,999960,994406,999374,997083,995401,998529,997799,997634,998559,999820,990475,998607,993412,990275,991977,994514,998779,997748,998179,999462,995369\n", + "Returned IDs: 999504,997388,997157,996727,990797,991923,991082,998094,997640,999960,994406,999374,997083,995401,998529,997634,998559,999820,990475,998607,993412,990275,991977,994514,998779,997748,998179,999462,995369,991334\n", "Ground Truth: 999504,997388,997157,996727,990797,991923,991082,998094,997640,999960,994406,999374,997083,995401,998529,997799,997634,998559,999820,990475,998607,995320,993412,990275,991977,994514,998779,997748,998179,999462\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 305\n", - "Returned IDs: 990644,997702,997002,991392,991485,990999,997533,994362,995374,992979,991894,999219,991459,993355,999453,995894,995875,993848,991157,998385,992750,999977,996425,994946,991151,998247,998401,995154,995515,994334\n", + "Returned IDs: 990644,997702,997002,991392,991485,990999,992979,999219,991459,993355,999453,995894,995875,993848,991157,998385,999977,996425,994946,998247,998401,995154,995515,994334,995016,998303,994906,996840,996368,992648\n", "Ground Truth: 990644,997702,997002,991392,991485,990999,997533,994362,995374,992979,991894,999219,991459,993355,999453,995894,995875,993848,991157,998385,992750,999977,996425,994946,991151,998247,998401,995154,992047,995515\n", - "Recall: 0.9667\n", + "Recall: 0.7667\n", "\n", "Query ID: 306\n", - "Returned IDs: 997475,994891,993611,994282,993821,991153,998065,996535,991066,990301,993046,990479,990961,996728,999775,999832,991597,991111,998279,994280,998320,999205,996743,996495,996431,990774,997919,995600,998221,991273\n", + "Returned IDs: 997475,994891,993611,994282,993821,991153,998065,996535,991066,990301,993046,990479,990961,996728,999775,999832,991597,994316,991111,994280,999205,996495,996431,990774,997919,995600,998221,991709,994196,998537\n", "Ground Truth: 997475,994891,993611,994282,993821,991153,998065,996535,991066,990301,993046,990479,990961,996728,999775,999832,991597,994316,991111,998279,994280,998320,999205,996743,996495,996431,990774,997919,995600,998221\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 307\n", + "Returned IDs: 995260,995755,992546,994266,990925,995132,991829,997737,996001,997320,995927,993168,994776,997796,999444,991743,990146,996075,990958,992994,992147,996714,990096,993648,998326,996219,998365,993484,999647,998246\n", + "Ground Truth: 995260,995755,992546,994266,990925,995132,991829,997737,996805,996001,997320,995927,993168,994776,997796,999444,991743,990146,991582,996075,990958,992994,992147,997876,996714,990096,993648,998326,996219,998365\n", + "Recall: 0.9000\n", "\n", "Query ID: 308\n", - "Returned IDs: 993426,990191,992451,991873,999271,999904,995623,995045,997292,997301,997984,996516,998921,998603,993484,990546,995306,995865,993048,992709,990512,998760,991720,997397,996771,998425,999110,992107,991916,999533\n", + "Returned IDs: 993426,990191,992451,991873,999271,999904,995623,995045,997292,997301,997984,996516,998921,993484,990546,995306,992709,990512,998760,991720,997397,996771,998425,999110,992107,992027,991916,999533,994618,997320\n", "Ground Truth: 993426,990191,992451,991873,999271,999904,995623,995045,997292,997301,997984,996516,998921,998603,993484,990546,995306,995865,993048,992709,990512,998760,991720,997397,996771,998425,999110,992107,992027,991749\n", - "Recall: 0.9333\n", + "Recall: 0.8667\n", "\n", "Query ID: 309\n", - "Returned IDs: 997233,997059,994205,998725,993857,994013,996308,999130,990241,990662,996028,998982,999099,997452,999570,994131,997554,996657,993909,994382,991382,995295,995216,996001,998752,996147,999623,998055,998156,996411\n", + "Returned IDs: 997233,997059,994205,998725,994013,996308,999130,990241,990662,996028,998982,999099,997452,999570,997554,996657,994382,991382,995295,995216,996001,998752,996147,999623,998055,996411,995544,995456,999342,995158\n", "Ground Truth: 997233,997059,994205,998725,993857,994013,996308,999130,990241,990662,996028,995600,998982,999099,997452,999570,994131,997554,996657,993909,994382,991382,995295,995216,996001,998752,999623,996147,998055,998156\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 310\n", + "Returned IDs: 993567,998104,990872,994711,998293,994909,996435,993727,992457,992569,993799,992052,999796,990116,997897,992437,998162,997752,990956,992981,993180,993497,996825,991245,992028,995766,995421,994353,994435,993347\n", + "Ground Truth: 993567,998104,990872,994711,998293,999190,994909,996435,993727,994300,992457,992569,997949,993799,992052,999796,990116,997897,992437,998162,997752,990956,992981,992759,993180,993497,996825,991245,997506,992028\n", + "Recall: 0.8333\n", "\n", "Query ID: 311\n", - "Returned IDs: 992041,992124,999639,997258,992238,990453,992497,993696,998758,998260,992167,992225,991984,995516,993834,992927,994809,993283,990949,997370,997320,999041,998950,991157,992521,991459,996500,995899,996075,992896\n", + "Returned IDs: 992041,992124,999639,997258,992238,990453,993696,998758,998260,992167,992225,991984,995516,993834,992927,994809,993283,990949,997370,997320,999041,998950,991157,992521,991459,990862,996500,995899,996075,992896\n", "Ground Truth: 992041,992124,999639,997258,992238,990453,992497,993696,998758,998260,992167,992225,991984,995516,993834,992927,994809,993283,990949,997370,997320,999041,998950,991157,992521,991459,990862,996500,995899,996075\n", "Recall: 0.9667\n", "\n", "Query ID: 312\n", - "Returned IDs: 990821,990310,991840,995895,991157,990271,993717,996809,991015,995633,990941,993391,997331,994922,993642,993143,998033,998950,996840,994830,991848,993342,999453,999784,996328,994742,999122,992169,992612,997822\n", + "Returned IDs: 990821,990310,991840,995895,991157,993717,996809,991015,995633,993391,994922,993143,996408,998950,996840,994830,991848,993342,999453,999784,996328,994742,999122,992169,992612,997822,994964,993260,997918,990080\n", "Ground Truth: 990821,990310,991840,995895,991157,990271,993717,996809,991015,995633,990941,993391,997331,994922,993642,993143,996408,998033,998950,996840,994830,991848,993342,999453,999784,996328,994742,999122,992169,992612\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 313\n", + "Returned IDs: 996936,999337,998794,991805,999157,993890,996944,998926,992499,998109,997047,991915,997270,995647,996006,996650,996796,992780,993472,995690,994558,992872,993721,994452,996475,994983,994832,991179,995596,996219\n", + "Ground Truth: 996936,999337,998794,991805,999157,993890,990780,996944,998926,992499,998109,997047,991915,997270,995647,996006,996650,997561,996796,992780,993472,995690,992986,994558,992872,993721,994452,996475,997257,993772\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 314\n", + "Returned IDs: \n", + "Ground Truth: 998728,998232,992733,994440,996075,990142,995596,992751,991984,993902,997291,997661,998498,995203,992671,991461,994626,999426,996317,994341,990123,996735,998654,999132,990083,996741,992119,990914,990697,994115\n", + "Recall: 0.0000\n", "\n", "Query ID: 315\n", - "Returned IDs: 992771,997395,993566,990194,993106,996393,999994,993931,999673,995895,992685,991188,996408,990951,999543,995620,999555,991037,999430,990407,993224,998536,998006,996762,990796,990434,993011,998383,996395,998855\n", + "Returned IDs: 992771,997395,993566,990194,993106,996393,999994,993931,999673,995895,992685,996408,990951,999543,995620,999555,991037,999430,990407,993224,998536,998006,990796,990434,993011,996395,998080,991148,990167,991052\n", "Ground Truth: 992771,997395,993566,990194,993106,996393,999994,993931,999673,995895,992685,991188,996408,990951,999543,995620,999555,991037,999430,990407,993224,998536,998006,996762,990796,990434,993011,995211,998383,995401\n", - "Recall: 0.9333\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 316\n", + "Returned IDs: 999932,995269,993465,996046,996903,995685,997046,994819,990459,997918,994804,999026,993407,993935,995444,995373,993433,997002,996734,999000,997822,996323,994334,990940,991451,995928,991415,995204,990215,998023\n", + "Ground Truth: 999932,995269,993465,996046,996903,995685,997046,994819,990459,997918,994804,999026,993407,990995,993935,995444,993412,995373,991698,993433,997002,992933,996734,999000,997822,996323,994334,990940,991451,999089\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 317\n", + "Returned IDs: 999139,991382,995895,998671,997506,995191,996328,994142,990392,999740,994280,994830,993053,998716,998495,991633,997933,996013,991672,992680,994472,993391,992636,992445,999086,999430,991390,995476,995295,996537\n", + "Ground Truth: 999139,991382,995895,998671,997506,995191,996328,994142,990133,990392,999740,994280,994830,999377,993053,998716,998495,991633,997933,996013,998030,991672,992680,994472,990293,993391,996775,992636,999441,992445\n", + "Recall: 0.8000\n", "\n", "Query ID: 318\n", - "Returned IDs: 991144,993719,990257,993917,992636,993715,996986,990438,990951,997168,999334,996357,993150,993637,995991,998076,998716,998561,990089,997879,998348,996408,997790,996328,994522,994834,991081,997669,993139,996201\n", + "Returned IDs: 991144,993719,990257,993917,992636,993715,996986,990438,990951,997168,999334,996357,993150,993637,995991,998076,998716,998561,990089,997879,998348,996408,997790,996328,995002,994834,993139,996201,990289,999827\n", "Ground Truth: 991144,993719,990257,993917,992636,993715,996986,990438,990951,997168,999334,996357,993150,993637,995991,998076,998716,998561,990089,997879,998348,996408,997790,996328,995002,994522,994834,991081,997669,993139\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", "\n", "Query ID: 319\n", - "Returned IDs: 992747,998303,993489,991037,992571,990350,998401,999840,990409,991332,995536,990930,996217,994468,997002,997977,992082,995894,996062,993504,994200,995203,993260,990885,990096,991530,996677,997368,995025,998007\n", + "Returned IDs: 992747,998303,993489,991037,992571,990350,998401,999840,990409,991332,995536,990930,996217,997002,997977,992082,995894,996062,993504,994200,995203,993260,990885,990096,991530,997368,995025,995823,991425,992823\n", "Ground Truth: 992747,998303,993489,991037,992571,990350,998401,999840,990409,991332,995536,990930,996217,994468,997002,997977,992082,995894,996062,993504,994200,995203,993260,990885,990096,991530,996677,997368,995025,995823\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 320\n", - "Returned IDs: 993809,998080,998495,995401,996371,992571,993255,991052,996370,992318,990083,998950,993696,992847,993515,997723,995932,990532,993478,999701,998221,997586,992944,996805,997184,991412,993444,998416,998968,990776\n", + "Returned IDs: 993809,998080,998495,996371,992571,993255,996370,992318,990083,998950,993696,992847,993515,995932,990532,999701,997586,992944,996805,997184,991412,990711,998416,990776,999179,992748,995552,996918,996809,995289\n", "Ground Truth: 993809,998080,998495,995401,996371,992571,993255,991052,996370,992318,990083,998950,993696,992847,993515,997723,992753,995932,990532,993478,999701,998221,997586,992944,996805,997184,991412,993444,990711,998416\n", - "Recall: 0.9333\n", + "Recall: 0.7667\n", "\n", "Query ID: 321\n", - "Returned IDs: 990999,998990,998179,995401,997258,991459,997191,996379,996198,996727,993561,990453,997799,990354,993848,999219,992684,991392,991630,999021,994946,993412,994319,996247,999639,990838,991792,995994,990561,992979\n", + "Returned IDs: 990999,998990,998179,995401,997258,991459,997191,996379,996198,996727,993561,990354,993848,999219,992684,991392,991630,999021,994946,994319,996247,999639,990838,995994,990561,992979,997640,993355,997158,997380\n", "Ground Truth: 990999,998990,998179,995401,997258,991459,997191,996379,996198,996727,993561,990453,997799,990354,993848,999219,992684,991392,991630,999021,994946,993412,994319,996247,999639,990838,991792,999228,995994,990561\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 322\n", - "Returned IDs: 990866,997379,997586,993708,996397,993637,996429,993552,999308,999574,999647,999500,995701,997013,992068,990547,991157,997811,992944,991808,999216,992522,992047,997555,999598,997496,998180,992682,995455,999473\n", + "Returned IDs: 990866,997379,997586,993708,996397,993637,993552,999308,999647,999500,995701,997013,992068,991157,997811,992944,991808,997555,999598,998180,992682,995455,999473,999337,998550,991015,994341,991348,995360,994307\n", "Ground Truth: 990866,997379,997586,993708,996397,993637,996429,993552,999308,999574,999647,999500,995701,997013,992068,990547,991157,997811,992944,991808,999216,992522,992047,997555,999598,997496,998180,998790,992682,995455\n", - "Recall: 0.9667\n", + "Recall: 0.7333\n", "\n", "Query ID: 323\n", - "Returned IDs: 993160,995062,998080,993971,999961,992150,997634,998806,997061,997395,998316,998650,992677,990131,993255,993263,994576,990789,995886,996273,990103,992604,994774,996139,993122,992493,998528,991360,992558,992148\n", + "Returned IDs: 993160,995062,998080,999961,992150,997634,998806,997061,997395,998316,998650,992677,990131,993255,993263,994576,990789,995886,996273,992161,992604,994774,996139,993122,992493,991360,992558,992148,993740,990957\n", "Ground Truth: 993160,995062,998080,993971,999961,992150,997634,998806,997061,997395,998316,998650,992677,997429,990131,993255,993263,994576,990789,995886,996273,990103,992161,992604,994774,996139,993122,992493,998528,991360\n", - "Recall: 0.9333\n", + "Recall: 0.8667\n", "\n", "Query ID: 324\n", - "Returned IDs: 991825,996028,994689,995823,998140,998825,993378,997051,995649,990811,991426,997373,994012,991840,998982,993906,992201,997417,993890,998382,994307,997541,995089,993725,991989,992780,999446,999740,995680,999286\n", + "Returned IDs: 991825,996028,994689,995823,998140,998825,993378,997051,990811,997373,994012,991840,998982,993906,992201,997417,993890,998382,994307,995089,993725,991989,992780,999446,999740,995680,999286,996556,999613,991711\n", "Ground Truth: 991825,996028,994689,995823,998140,998825,993378,997051,995649,990811,991426,997373,994012,991840,998982,993906,992201,997417,993890,998382,994307,995502,997541,995089,993725,991989,992780,999446,999740,995680\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 325\n", - "Returned IDs: 994630,996961,997525,999761,994699,993518,991895,996732,997913,998246,999710,992546,992709,994939,994932,995949,994757,998744,996785,997599,998332,990397,992502,997535,990925,990509,991873,992119,991479,996890\n", + "Returned IDs: 994630,996961,997525,994699,993518,991895,997913,998246,999710,992546,992709,994939,994932,995949,994757,998744,996785,997599,998332,990397,997535,990925,990509,991873,992119,991479,996890,991379,994427,992107\n", "Ground Truth: 994630,996961,997525,999761,994699,993518,991895,996732,997913,998246,999710,992546,992709,994939,998016,994932,995949,994757,998744,997599,996785,998332,990397,992502,997535,990925,990509,991873,992119,991479\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 326\n", - "Returned IDs: 997976,998170,994894,997479,999182,994757,991782,997320,999144,997918,993673,995981,995279,995886,990231,994100,991379,990660,997571,998011,995336,990958,994819,994334,997192,999527,991181,990647,990588,997777\n", + "Returned IDs: 997976,998170,994894,997479,999182,994757,997320,999144,997918,995981,995279,995886,990231,994100,991379,990660,997571,995336,990958,994334,997192,990647,990588,995656,997605,997437,994280,993492,998951,995274\n", "Ground Truth: 997976,998170,994894,997479,999182,994757,991782,997320,999144,997918,993673,995981,995279,995886,990231,994100,991379,990660,997571,998011,995336,990958,998163,994819,994334,997192,999527,991181,990647,990588\n", - "Recall: 0.9667\n", + "Recall: 0.7667\n", "\n", "Query ID: 327\n", - "Returned IDs: 995274,996411,990293,996657,993412,996001,993909,995216,993003,998302,999026,998779,996810,998952,990974,991995,990845,996759,998862,995289,998058,999835,999587,990957,992792,999099,997529,996219,999343,993168\n", + "Returned IDs: 995274,996657,996001,995216,998302,999026,998779,996810,998952,990974,990845,996759,998862,995289,998058,990957,992792,999099,997529,996219,999343,993168,994730,996727,997059,999453,997554,993831,996147,995873\n", "Ground Truth: 995274,996411,990293,996657,993412,996001,993909,995216,993003,998302,999026,998779,996810,998952,990974,991995,990845,996759,998862,995289,992120,995899,998058,999835,999587,990957,992792,997969,999099,997529\n", - "Recall: 0.9000\n", + "Recall: 0.6333\n", "\n", "Query ID: 328\n", - "Returned IDs: 998257,995373,991204,997110,990657,993115,997568,994643,998858,994454,997030,997589,999129,993795,991987,991386,995401,993000,996393,997675,997683,992762,999136,998536,991052,990096,992861,995453,997918,990005\n", + "Returned IDs: 998257,995373,991204,997110,990657,993115,997568,994643,998858,994454,997589,999129,993795,991987,999530,991386,995401,993000,996393,997675,997683,992762,999136,998536,991052,990096,992861,998697,997918,990005\n", "Ground Truth: 998257,995373,991204,997110,990657,993115,997568,994643,998858,994454,997030,997589,999129,993795,991987,999530,991386,995401,993000,996393,997675,997683,992762,999136,998536,991052,990096,992861,998697,995453\n", "Recall: 0.9333\n", "\n", + "Query ID: 329\n", + "Returned IDs: 996001,992052,996657,996277,991977,997453,996505,991349,993280,999491,994743,998536,997168,993180,994409,990956,998593,993289,990930,994040,995200,992521,993561,994280,993931,992790,992718,995289,996042,990364\n", + "Ground Truth: 996001,992052,996657,996277,991977,997453,996505,991349,993280,999491,994743,998536,997168,993180,994409,995408,990956,998593,993289,990930,994040,995401,995200,992521,993561,994280,993931,999099,992790,992718\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 330\n", + "Returned IDs: 990255,997897,993232,990167,996657,998593,990361,991532,996395,998695,996809,993344,997752,990941,998786,992809,995167,993483,996001,996284,991255,990675,994587,996797,994451,996043,992437,999463,997131,990314\n", + "Ground Truth: 990255,997897,993232,990167,996657,998593,990361,991532,996395,996104,998695,996809,993344,997752,990941,998786,992809,995167,993483,996001,996284,991255,990675,994587,996797,994451,990211,996043,992437,999463\n", + "Recall: 0.9333\n", + "\n", "Query ID: 331\n", - "Returned IDs: 990154,998006,995815,999555,996728,994181,997666,992771,995835,999673,990952,990006,999543,997884,992282,990951,994834,999388,994467,998861,999663,996505,995655,991596,994103,990429,998067,997358,995536,994756\n", + "Returned IDs: 990154,998006,995815,999555,994181,997666,992771,995835,999673,990952,999543,992282,990951,994834,999388,994467,998861,996505,995655,994103,998067,995536,994756,997242,997427,994380,990537,997331,993931,995168\n", "Ground Truth: 990154,998006,995815,999555,996728,994181,997666,992771,995835,999673,990952,990006,999543,997884,992282,990951,994834,999388,994467,998861,999350,999663,996505,995655,991596,994103,990429,998067,997358,995536\n", - "Recall: 0.9667\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 332\n", + "Returned IDs: 997452,991464,990614,998913,995295,993725,994742,991330,995633,997933,996116,994783,998779,994541,990701,995551,996411,995456,991303,998543,993053,990392,992434,997822,998495,992703,995289,998671,999302,997374\n", + "Ground Truth: 997452,991464,990614,998913,995295,993725,994742,991330,995837,995633,997933,996116,994783,998779,994541,990701,995551,996411,995456,991303,991537,999230,994580,998543,993053,990392,992434,997822,998495,992703\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 333\n", + "Returned IDs: 998109,999659,992780,997442,997296,992499,995476,991826,992070,991382,996809,995150,997270,996216,996650,993617,990039,998167,991633,998794,998495,991805,993342,999430,992872,998631,993391,998803,994927,994342\n", + "Ground Truth: 998109,999659,992780,997442,997296,992499,995476,991826,992070,991382,996809,995150,997270,996216,996650,993617,990039,998167,991633,998794,998495,991805,993342,999430,992872,999551,998631,993391,997318,998803\n", + "Recall: 0.9333\n", "\n", "Query ID: 334\n", - "Returned IDs: 991377,992894,994419,995289,997386,997822,993412,995702,995923,991923,993511,996080,991603,990847,993561,996610,996867,991958,996360,997529,996745,993433,994925,996322,995200,994409,992130,995401,997937,999782\n", + "Returned IDs: 991377,992894,994419,995289,997386,997822,995702,995923,991923,993511,996080,990847,993561,996610,996867,991958,996360,997529,996745,996147,993433,994925,996322,995200,996127,994409,992130,997937,999782,996042\n", "Ground Truth: 991377,992894,994419,995289,997386,997822,993412,995702,995923,991923,993511,996080,991603,990847,993561,996610,996867,991958,996360,997529,996745,996147,993433,994925,996322,995200,996127,994409,992130,997920\n", "Recall: 0.9000\n", "\n", + "Query ID: 335\n", + "Returned IDs: 993079,993324,998289,999796,998543,997159,998104,994765,994119,992558,998180,990956,997420,990814,994742,993215,999179,990776,994541,995332,995438,995289,993293,990264,992832,997309,995401,998950,996210,994817\n", + "Ground Truth: 993079,993324,998289,999796,998543,997159,998104,994765,994119,992558,992933,998180,990956,993280,997420,990814,994742,993215,999179,998369,990776,994541,996370,995332,995438,995289,996178,993293,990264,992832\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 336\n", + "Returned IDs: 991454,998355,991177,993929,990103,990226,997732,994956,997244,998691,990712,993014,998631,991107,999304,994444,999673,996803,998343,994410,990678,990705,998495,997453,995382,999138,990951,991170,995919,992713\n", + "Ground Truth: 991454,998355,991177,993929,990103,990226,997732,994956,997244,993336,998691,996062,996362,990712,993014,998631,991107,999304,990408,994444,999673,997693,997807,998466,996803,991219,996942,998343,994410,994927\n", + "Recall: 0.6667\n", + "\n", "Query ID: 337\n", - "Returned IDs: 996531,998397,993747,990956,996276,999514,991747,998293,996867,998478,992703,994119,990466,994541,996232,998486,992160,990116,997822,994444,999264,999646,990103,993727,996294,997420,995159,992992,999244,991145\n", + "Returned IDs: 996531,998397,993747,990956,996276,999514,991747,998293,996867,992703,994119,994541,998165,996232,998486,992160,990116,997822,994444,999264,999646,990103,993727,997420,995159,999244,991145,995184,995144,994806\n", "Ground Truth: 996531,998397,993747,990956,996276,999514,991747,998293,996867,998478,992703,994119,990466,994541,998165,996232,998486,992160,990116,997822,994444,999264,999646,990103,993727,996294,997420,995159,992992,999244\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 338\n", + "Returned IDs: 992112,990494,998715,992351,990885,997368,998018,997897,995932,996632,993504,992747,992825,997839,993171,997790,990818,995560,993260,995476,990697,990009,992780,991491,998473,992635,996216,994081,994280,993341\n", + "Ground Truth: 992112,990494,998715,992351,990885,997368,998018,997897,995932,996632,993504,992747,992825,997839,993171,997790,990818,992665,992137,995560,993260,995476,990697,990009,992780,995130,991491,998473,996424,992635\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 339\n", + "Returned IDs: 993209,997837,995778,995289,995766,995633,993997,994742,991185,992558,991507,994031,994743,999957,995106,996302,998748,992448,998188,993623,991044,997975,999415,997630,993003,995648,997319,994409,993265,997662\n", + "Ground Truth: 993209,997837,995778,995289,995766,995633,990361,993997,994742,991185,992558,995401,991507,994031,994743,999957,995106,996302,998748,995198,992448,998188,993623,991044,997975,991664,999377,999415,997630,993003\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 340\n", + "Returned IDs: 994662,994755,996276,995144,994073,995438,990956,999244,992315,993747,992934,997420,999601,992052,999514,998104,993805,996850,991747,993727,993742,993121,994541,999264,998293,999411,993448,999029,994336,993219\n", + "Ground Truth: 994662,994755,996276,995144,994073,995438,990956,999244,992315,993747,992934,997420,999601,992052,999514,998104,993805,996850,991747,993727,993742,993121,994541,999264,998293,999411,993448,999029,994336,995107\n", "Recall: 0.9667\n", "\n", + "Query ID: 341\n", + "Returned IDs: 997386,995289,995923,994419,996080,993561,996867,998593,991603,998999,992894,997822,998165,997837,994925,995497,990811,995702,991958,992680,998382,993725,997046,998039,996322,995052,995200,996745,997406,993433\n", + "Ground Truth: 997386,995289,995923,994419,996080,993561,996867,998593,991603,998999,992894,997822,998165,997837,994925,995497,990811,995702,993412,991958,992680,998382,993725,995401,997046,998039,996322,995052,997630,995200\n", + "Recall: 0.9000\n", + "\n", "Query ID: 342\n", - "Returned IDs: 993364,992126,991994,993885,991792,990772,997876,990557,993820,995841,994162,991212,991498,995372,997320,999982,998444,994673,992788,995515,999453,996489,997973,999819,997002,996001,990724,996959,993032,994081\n", + "Returned IDs: 993364,992126,993885,990772,997380,990557,995841,994162,991212,991498,995372,997320,999982,995502,998444,994673,992788,995515,999453,996489,997973,997002,996001,996959,993032,994081,997301,997796,994088,990487\n", "Ground Truth: 993364,992126,991994,993885,991792,990772,997876,997380,990557,993820,995841,994162,991212,991498,995372,997320,999982,995502,998444,994673,992788,995515,999453,996489,997973,999819,997002,996001,990724,996959\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 343\n", + "Returned IDs: 996442,991713,998260,996397,997158,999806,999731,996196,993785,998827,992423,998568,992927,994577,994163,990731,993071,991650,993504,997550,999598,996404,992041,995552,997977,991157,995820,999111,997320,997013\n", + "Ground Truth: 996442,991713,998260,996397,997158,999806,999731,996196,993785,998827,992423,998568,992927,994577,994334,994163,990731,993071,991650,993504,997550,999598,996404,992041,996979,995552,997977,993377,991157,995820\n", + "Recall: 0.9000\n", "\n", "Query ID: 344\n", - "Returned IDs: 994277,993721,994233,994416,991409,996685,995401,997403,995899,992944,996024,997320,998950,991348,991416,998921,996210,996075,994081,997448,990364,992318,995372,998913,996178,991650,991997,991743,997305,999159\n", + "Returned IDs: 994277,993721,994233,994416,991409,996685,995899,992944,996024,997320,998950,991348,991416,998921,996210,996075,992318,995372,998913,996178,991650,991997,997305,995108,996001,991154,993018,991142,999701,995515\n", "Ground Truth: 994277,993721,994233,994416,991409,996685,995401,997403,995899,992944,996024,997320,998950,991348,991416,998921,996210,993121,996075,994081,997448,990364,995372,992318,998913,996178,991650,991997,991743,997305\n", - "Recall: 0.9667\n", + "Recall: 0.7667\n", "\n", "Query ID: 345\n", - "Returned IDs: 999827,996001,997320,994964,991498,993080,997920,999453,994081,995841,992182,999099,994478,992395,993168,995216,997860,998750,993280,997918,995274,990146,993477,993289,990772,993648,995102,993484,991212,990532\n", + "Returned IDs: 999827,996001,997320,994964,991498,993080,999453,994081,995841,992182,999099,994478,992395,993168,995216,997860,993280,997918,995274,990146,993289,995102,993484,991212,990532,993388,997371,990557,992469,999019\n", "Ground Truth: 999827,996001,997320,994964,991498,993080,993454,997920,999453,994081,995841,992182,999099,999026,994478,992395,993168,995216,997860,998750,993280,997918,995274,990146,993477,993289,990772,993648,995102,991007\n", - "Recall: 0.9000\n", + "Recall: 0.7333\n", "\n", "Query ID: 346\n", - "Returned IDs: 992907,995588,997465,998582,993091,999676,996840,995515,993028,993622,999076,991840,990300,993355,990310,994922,992979,999073,996352,991999,996686,993781,999077,994612,990490,990432,993688,991160,992218,999620\n", + "Returned IDs: 992907,995588,997465,998582,993091,999676,996840,995515,993622,999076,991840,990300,993355,990310,994922,991999,996686,993781,999077,994612,990432,993688,999620,991172,993143,991462,999784,993212,997496,999317\n", "Ground Truth: 992907,995588,997465,998582,993091,999676,996840,995515,993028,993622,999076,991840,990300,993355,990310,994922,992979,999073,996352,999604,991999,996686,993781,999077,994612,990490,990432,993688,991160,992218\n", - "Recall: 0.9667\n", + "Recall: 0.7333\n", "\n", "Query ID: 347\n", - "Returned IDs: 997909,997299,998890,996256,999925,993003,994409,995401,997918,993537,991357,991743,991795,992763,996815,996609,995864,994106,990087,990625,994062,999926,996613,997218,994865,996232,998205,991892,996576,997612\n", + "Returned IDs: 997909,997299,998890,996256,999925,993003,994409,997918,993537,991357,991795,992763,996815,996609,995864,994106,990087,990625,990432,999926,996613,994865,998205,991892,991927,997612,998361,991197,991220,994809\n", "Ground Truth: 997909,997299,998890,996256,999925,993003,994409,995401,997918,993537,991357,991743,991795,992763,996815,996609,995864,997555,994106,990087,990625,994062,990432,999926,996613,997218,994865,996232,998205,991892\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 348\n", + "Returned IDs: 991487,991515,990630,994101,996562,996817,996496,999079,998250,993175,996461,992104,990203,995349,990834,993391,990160,991245,997666,995832,991897,998934,998022,997506,993483,995421,992407,990412,993489,996245\n", + "Ground Truth: 991487,991515,990630,994101,996562,996817,996496,999079,998250,993175,996461,992104,990203,995349,990834,993391,990160,991245,997666,999211,994952,998428,990712,995832,993599,999463,991897,998934,998022,997506\n", + "Recall: 0.8000\n", "\n", "Query ID: 349\n", - "Returned IDs: 993754,997781,992604,995527,995807,999125,994774,999445,995105,999662,990915,998059,991051,999478,996691,992387,990131,997212,990729,997462,997693,997110,993632,998528,992855,995625,994539,995953,996206,996236\n", + "Returned IDs: 993754,997781,992604,995527,995807,999125,994774,999445,995105,999662,990915,998059,991051,999478,996691,992387,990131,998297,997212,990729,997462,997693,997110,993632,998528,992855,994539,995953,996206,996236\n", "Ground Truth: 993754,997781,992604,995527,995807,999125,994774,999445,995105,999662,990915,998059,991051,999478,996691,992387,990131,998297,997212,990729,997462,997693,997110,993632,998528,992855,995625,994539,995953,996206\n", "Recall: 0.9667\n", "\n", + "Query ID: 350\n", + "Returned IDs: 994142,997693,997309,990712,995214,991525,999179,990776,990956,997420,993453,996328,991257,994282,999091,997942,994193,993931,990705,993621,994119,995159,994092,992685,991723,994035,994835,996055,996633,993929\n", + "Ground Truth: 994142,997693,997309,990712,995214,999179,991525,990776,990956,997420,993453,996328,991257,994282,999091,997942,994193,993931,990705,993621,994119,995159,992441,994092,992685,991723,994351,994035,994835,996055\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 351\n", + "Returned IDs: 994964,995841,998445,990585,995372,997796,999026,998921,998797,991498,990925,993484,997320,990146,996001,998029,997301,994978,993845,997913,991444,996053,996959,992147,993579,995322,994584,999009,994776,998302\n", + "Ground Truth: 990936,993648,994964,995841,998445,990585,995372,997876,997796,999026,998921,998797,991498,990925,991582,990801,993484,997320,993402,990146,993663,993853,996001,998029,997301,994978,999533,993845,997913,991444\n", + "Recall: 0.7000\n", + "\n", "Query ID: 352\n", - "Returned IDs: 991498,993484,994503,997597,999005,999263,995372,997320,997301,995841,994917,996560,992637,998849,998966,998634,991743,990829,990936,995034,994183,998779,998839,990585,998445,992510,999658,997305,994611,998163\n", + "Returned IDs: 991498,993484,994503,997597,999005,999263,995372,997320,997301,995841,996560,992637,998634,991743,990829,990936,995034,998779,998839,990585,998445,999658,997305,997908,991626,995802,993443,995771,992811,997470\n", "Ground Truth: 991498,993484,994503,997597,999005,999263,995372,997320,997301,995841,994917,996560,992637,998849,998966,998634,991743,990829,990936,995034,994183,998779,998839,990585,998445,992510,999658,997305,997908,994611\n", - "Recall: 0.9667\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 353\n", + "Returned IDs: 993237,998244,992252,998734,995515,998782,999453,990043,998837,990142,991999,994033,991307,990965,992835,993091,999793,996438,991485,993708,991840,992169,996686,996840,995443,991392,998363,993582,999620,991687\n", + "Ground Truth: 993237,998244,994439,992252,998734,995515,992342,998782,999453,990043,998837,990142,991999,994144,994033,991307,994785,993859,990965,992835,991149,993091,999793,996438,991609,991485,993708,991840,996686,992169\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 354\n", + "Returned IDs: 995097,993588,994145,994830,998440,991081,990622,993585,990016,994925,991921,990536,993342,998030,996858,993101,998710,994513,996974,992664,998903,990497,991075,998097,994689,995906,992201,997230,994682,990700\n", + "Ground Truth: 995097,993588,994145,994830,998440,991081,990622,993585,990016,994925,991921,990536,993342,994720,998030,996858,991087,993101,996147,998710,994513,996974,992664,998903,992174,994523,990497,991075,998097,994689\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 355\n", + "Returned IDs: 992169,995080,993143,997309,997918,991743,996918,991698,997002,995783,992558,990479,990561,990882,991257,994972,990897,996867,994334,998180,997586,993717,992944,994577,997822,999676,998726,995516,994409,993260\n", + "Ground Truth: 992169,995080,993143,997309,991052,997918,991743,996918,991698,997002,995783,992558,990479,990561,998045,995401,990882,991257,994972,991061,997723,990897,996867,994334,992933,998180,991447,997586,993717,992944\n", + "Recall: 0.7667\n", "\n", "Query ID: 356\n", - "Returned IDs: 999587,992884,993289,999453,996657,997554,996001,990845,994531,997002,990399,996810,992979,996028,994946,995401,996147,993032,996368,995216,992521,995281,996759,999827,994809,993280,992558,990956,998495,990255\n", + "Returned IDs: 999587,992884,993289,999453,996657,997554,996001,990845,997002,990399,996810,992979,996028,994946,995401,996147,993032,996368,995216,992521,996759,997023,999827,994809,992558,998495,992862,999099,992395,999977\n", "Ground Truth: 999587,992884,993289,999453,996657,997554,996001,990845,994531,997002,990399,996810,992979,996028,994946,995401,996147,993032,996368,995216,992521,995281,996759,997023,999827,994809,993280,992558,990956,998495\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 357\n", - "Returned IDs: 991063,993809,998225,998302,994879,999444,997906,995638,990866,992748,996805,998950,997913,998123,996230,996075,999977,996811,995672,995755,997320,999831,996001,998580,995701,998697,992394,990096,990083,996761\n", + "Returned IDs: 993809,998225,998302,994879,999444,997906,995638,990866,992748,996805,998950,997913,998123,996230,996075,999977,996811,995672,997320,996001,998580,995701,998697,992394,990096,990083,996761,994640,990949,993080\n", "Ground Truth: 991063,993809,998225,998302,994879,999444,997906,995638,990866,992748,996805,998950,997913,998123,996230,996075,991927,999977,996811,995672,995755,997320,999831,993821,996001,998580,995701,998697,991311,992394\n", - "Recall: 0.9000\n", + "Recall: 0.8000\n", "\n", "Query ID: 358\n", - "Returned IDs: 995580,991677,995775,999333,995266,999301,992668,999414,996499,994464,996192,998170,992134,997913,998216,993222,992946,997818,997192,998152,990957,991422,995014,998686,990647,998600,997106,995870,997841,994748\n", + "Returned IDs: 995580,991677,999333,999301,992668,999414,994464,996192,998170,992134,997913,998216,993222,997818,997192,990957,998600,997106,995870,997841,994748,992555,997071,997437,999624,992696,993853,993791,994197,996296\n", "Ground Truth: 995580,991677,995775,999333,995266,999301,992668,999414,996499,998599,994464,996192,998170,992134,997913,998216,993222,992946,994904,997818,997192,991081,998152,990957,991422,995014,998686,990647,998600,997106\n", - "Recall: 0.9000\n", + "Recall: 0.6000\n", "\n", "Query ID: 359\n", - "Returned IDs: 990323,991037,999748,991596,995915,995385,990194,999576,995712,996702,997242,994955,995562,990148,994200,995349,990096,997666,993179,994231,999435,998583,993866,992571,993458,999384,995655,990596,996424,990876\n", + "Returned IDs: 990323,991037,999748,991596,995915,995385,990194,999576,995712,996702,997242,994955,995562,990148,994200,990096,997666,994231,993866,993458,999384,995655,990596,990876,999663,997685,999555,991640,991732,991093\n", "Ground Truth: 990323,991037,999748,991596,995915,995385,990194,999576,995712,996702,997242,994955,995562,994020,990148,994200,995349,990096,997666,993179,994231,999435,998583,993866,992571,993458,999384,995655,990596,996424\n", - "Recall: 0.9667\n", + "Recall: 0.7667\n", "\n", "Query ID: 360\n", - "Returned IDs: 995559,993695,991179,993038,995560,995946,998790,999898,996210,990435,991923,997920,992944,992047,998416,996042,995107,997586,993055,996370,990096,995720,995332,999430,994815,993241,998349,998950,993324,999806\n", + "Returned IDs: \n", "Ground Truth: 995559,993695,991179,993038,995560,995946,998790,999898,996210,990435,991923,997920,992944,992047,998416,996042,995107,997586,993055,996370,990096,995720,995332,999430,994815,995401,993241,998349,998950,993324\n", - "Recall: 0.9667\n", + "Recall: 0.0000\n", "\n", "Query ID: 361\n", - "Returned IDs: 995891,991544,998651,995946,993561,994413,996472,990096,991958,998973,993566,992448,992260,991247,997252,997386,998545,998451,994285,995536,991447,994365,995423,995303,992631,995146,993433,999543,994409,992801\n", + "Returned IDs: 995891,991544,998651,995946,993561,994413,996472,990096,991958,993738,993566,992448,991247,997252,997386,998545,998451,994285,995536,994365,995423,992631,995146,999673,994409,992801,998317,998703,995547,996733\n", "Ground Truth: 995891,991544,998651,995946,993561,994413,996472,990096,991958,998973,993738,993566,992448,992260,991247,997252,997386,998545,998451,994285,995536,991447,994365,995423,995303,992631,995146,993433,999543,999673\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", "\n", "Query ID: 362\n", - "Returned IDs: 994589,991666,995401,992933,997723,991630,996659,990517,998530,999977,995701,996379,996864,998350,998874,991603,998990,999461,992367,998045,999000,998536,994177,995994,994409,992977,992790,992521,994946,999228\n", + "Returned IDs: 994589,991666,995401,997723,991630,996659,990517,998530,999977,995701,996379,996864,998350,998874,991603,998990,992367,998045,999000,998536,994177,995994,994409,992790,992521,994946,999228,999277,991403,997658\n", "Ground Truth: 994589,991666,995401,992933,997723,991630,996659,990517,998530,999977,995701,996379,996864,998350,998874,991603,999461,998990,992367,998045,999000,998536,994177,995994,994409,992977,992790,991151,992521,994946\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 363\n", + "Returned IDs: 994742,993080,995401,995932,999526,990096,996001,990289,998097,992052,996918,990468,993260,997168,996043,997320,992636,996387,996408,991245,992571,993637,997790,995740,997897,995536,998021,990814,991629,990392\n", + "Ground Truth: 994742,993080,995401,995932,999526,993719,990096,996001,990289,998097,992052,996918,990468,993260,990930,996516,997168,996043,994972,997320,992636,996387,993324,997103,996408,991245,992571,993637,997790,995203\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 364\n", + "Returned IDs: 996209,990423,991484,999555,994181,992234,998861,999576,997850,997666,995780,992716,998006,997242,997685,990194,992209,994335,992892,995209,992768,999273,996809,990167,993508,996386,999319,990154,998122,998598\n", + "Ground Truth: 996209,990423,991484,999555,994181,991003,992234,998861,999576,993615,997850,997666,995780,992716,998006,997242,997685,990194,992209,994335,992892,995209,992768,999273,996809,990167,993508,996386,999319,998419\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 365\n", + "Returned IDs: 994415,992041,993785,997320,996075,990914,992635,997550,996741,993741,991662,995596,995372,996875,992601,997661,996805,993447,997286,992119,997836,993125,997470,999982,997305,999701,991650,990697,991503,999473\n", + "Ground Truth: 994415,992041,993785,997320,996075,990914,992635,997550,996741,993741,991662,995596,995372,996875,995622,992601,995995,997661,996805,993447,997286,992942,992119,997836,993125,997470,999982,997305,999701,991650\n", + "Recall: 0.9000\n", "\n", "Query ID: 366\n", - "Returned IDs: 995633,999676,997374,994317,992289,997319,995401,994529,995289,990903,991743,996867,990126,995268,998221,995274,990432,993402,997723,994878,999453,994409,999587,993920,995020,998913,994850,994343,990435,999533\n", + "Returned IDs: 995633,999676,997374,994317,992289,997319,994529,995289,990903,991743,996867,990126,995268,998221,995274,990432,994878,999453,994409,999587,993920,995020,998913,994850,994343,990435,998862,999801,996411,999026\n", "Ground Truth: 995633,999676,997374,994317,992289,997319,995401,994529,995289,990903,991743,996867,990126,993412,995268,998221,995274,990432,993402,997723,994878,999453,994409,999587,993920,995020,998913,994850,994343,990435\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 367\n", - "Returned IDs: 994964,999151,990545,996219,995961,999331,991345,999261,994843,999352,999367,990922,994896,991921,994577,999210,991382,993238,993877,997720,993215,994705,995927,991378,994782,992446,990210,995739,994850,991751\n", + "Returned IDs: 994964,999151,990545,996219,995961,999331,999261,999367,990922,994896,999210,991382,993877,997720,993215,994705,995927,991378,994782,992446,990210,991751,996990,990957,996805,995861,996949,994783,998495,992114\n", "Ground Truth: 994964,999151,990545,996219,995961,999331,991345,999261,994843,999352,999367,990922,994896,991921,994577,999210,991382,993238,992942,993877,998294,993154,997720,993215,994705,995927,991378,994782,990210,992446\n", - "Recall: 0.9000\n", + "Recall: 0.7000\n", "\n", "Query ID: 368\n", - "Returned IDs: 995166,993857,998982,994689,991075,997208,996028,992374,991585,995456,998427,993725,998182,990328,994307,995823,990097,993053,994765,995633,992480,996506,993906,995295,997309,995289,990811,992558,998180,991315\n", + "Returned IDs: 995166,993857,998982,994689,991075,997208,996028,991585,995456,998427,993725,998182,990328,994307,995823,990097,993053,994765,995633,992480,996506,993906,995295,997309,995289,990811,992558,998180,991315,998105\n", "Ground Truth: 995166,993857,998982,994689,991075,997208,996028,992374,991585,995456,998427,993725,998182,990328,994307,995823,990097,993053,995401,994765,995633,992480,996506,993906,995295,997309,995289,990811,992558,998180\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 369\n", + "Returned IDs: \n", + "Ground Truth: 995560,992944,995375,996370,996210,994972,991994,995372,995841,996001,998950,995559,995107,997320,999009,998567,990949,997155,995873,990930,991052,995536,990538,995641,997305,995332,998697,994081,992521,998849\n", + "Recall: 0.0000\n", "\n", "Query ID: 370\n", - "Returned IDs: 997320,993484,997913,994604,996143,993154,998967,999948,995802,992442,990066,997470,998170,999658,990532,992119,995372,992330,990949,995865,999851,992546,993157,996075,995942,998808,992896,999535,995351,990585\n", + "Returned IDs: 997320,993484,997913,994604,996143,993154,998967,999948,995802,992442,990066,997470,998170,999658,990532,992119,995372,992330,990949,995865,999851,992546,993157,996075,998808,992896,999535,995351,990585,994363\n", "Ground Truth: 997320,993484,997913,994604,996143,993154,998967,999948,995802,995228,992442,990066,997470,998170,999658,990532,992119,995372,992330,990949,995865,999851,992546,993157,996075,995942,998808,992896,999535,995351\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 371\n", - "Returned IDs: 991975,993641,990429,995168,992110,999176,995231,993629,998858,993381,995780,997078,991455,996623,991332,992365,991385,993560,999555,991987,996571,994413,990976,994834,992926,990449,993191,998149,991038,995655\n", + "Returned IDs: \n", "Ground Truth: 991975,993641,990429,995168,992110,999176,995231,993629,998858,993381,995780,997078,991455,996623,991332,992365,991385,993560,999555,991987,996571,994413,990976,994834,992926,997546,990449,998731,993191,998149\n", - "Recall: 0.9333\n", + "Recall: 0.0000\n", "\n", "Query ID: 372\n", - "Returned IDs: 990249,993811,992806,995967,993727,997195,992121,999240,999479,993121,993583,990121,996750,993473,992558,994163,999400,998493,998820,991447,997918,993158,993911,992864,990117,995626,997124,994168,990395,999098\n", + "Returned IDs: 990249,993811,992806,995967,993727,997195,992121,999240,999479,993121,993583,990121,996750,993473,992558,994163,999400,997897,998820,997918,993158,990117,995626,997124,994134,990395,999098,993839,994957,995309\n", "Ground Truth: 990249,993811,992806,995967,993727,997195,992121,999240,999479,993121,993583,990121,996750,993473,992558,999887,994163,999400,997897,998493,992407,998820,991447,997918,998559,993158,993911,992864,990117,995626\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 373\n", + "Returned IDs: 992225,996089,993174,999041,999216,995005,998200,991984,993834,996230,990051,997258,992167,999219,993928,998934,992213,994902,994167,993322,994248,998989,993489,996657,993766,990472,995516,992238,991157,990547\n", + "Ground Truth: 992225,996089,993174,999041,999216,995005,998200,991984,993834,996230,990051,997258,992167,999219,993928,998934,992213,994902,994167,993322,994248,998989,993489,996657,993087,993766,990472,995516,992238,991157\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 374\n", + "Returned IDs: 998593,995421,991212,992437,993715,997897,991245,997790,995167,996001,990255,996043,998250,994743,993931,998037,998934,997453,990364,995945,996213,995126,991725,994925,996657,995873,993289,990314,991958,990811\n", + "Ground Truth: 998593,995421,991212,992437,993715,997897,991245,997790,995167,996001,990255,996043,998250,994743,993931,998037,998934,997453,995740,990364,995945,996213,995126,991725,994925,996657,995873,993289,990314,991958\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 375\n", + "Returned IDs: 998921,994964,996504,997984,994415,991720,996143,997320,997470,993518,994584,990958,999343,991582,993253,999647,990146,999676,997192,991498,990648,992147,998550,999102,995515,995336,995181,992942,993484,997305\n", + "Ground Truth: 998921,994889,994964,996504,997984,994415,991720,996143,997320,997470,993518,994584,995865,990958,998607,999343,991582,993253,999647,990146,999676,997192,991498,990648,990917,992147,998550,999102,995515,995336\n", "Recall: 0.8667\n", "\n", "Query ID: 376\n", - "Returned IDs: 995372,998849,992944,993241,990585,990060,995375,996809,990489,999544,998567,990226,995560,995841,999711,991498,999210,996318,997320,994972,994402,991994,997848,996001,990043,995641,990894,997436,997113,995443\n", + "Returned IDs: 995372,998849,992944,993241,990585,995375,996809,990489,999544,990226,995560,995841,999711,991498,999210,996318,997320,994972,994402,991994,993388,996001,990043,995641,997436,997113,995443,990532,999641,997002\n", "Ground Truth: 995372,998849,992944,993241,990585,990060,995375,996809,990489,999544,998567,990226,995560,995841,999711,991498,999210,996318,997320,995401,994972,993533,994402,991994,997848,993388,996001,990043,995641,990894\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 377\n", + "Returned IDs: 998710,999377,994663,994307,998105,996132,994513,997434,993053,994830,999430,994185,999472,999613,996906,994742,992201,999500,994925,996147,992079,996411,994145,999064,991128,990435,992230,995852,991409,999302\n", + "Ground Truth: 998710,999377,994663,994307,998105,996132,995310,994513,990016,997434,993053,994830,999430,994185,999472,999613,990012,996906,994742,992201,999500,994925,996147,992079,996411,994145,999064,991128,990435,992230\n", "Recall: 0.9000\n", "\n", + "Query ID: 378\n", + "Returned IDs: \n", + "Ground Truth: 995655,993948,991667,992517,998861,997145,995536,992595,990154,998536,990005,990330,993001,998122,998495,991610,995835,992561,995780,997884,996623,991490,997265,997242,997319,999555,998006,992110,999711,990956\n", + "Recall: 0.0000\n", + "\n", "Query ID: 379\n", - "Returned IDs: 992532,995974,998686,998334,991480,999451,995560,992349,992645,996894,999711,993976,995314,998417,998079,992654,992383,997521,990383,996196,990401,996262,991015,991208,990376,997758,998061,999639,999913,991977\n", + "Returned IDs: 992532,995974,998686,998334,991480,999451,992349,992645,999711,993976,995314,998417,998079,992383,997521,990383,996196,990401,996262,991015,991208,990376,997758,998061,999639,999913,991977,992718,998118,990908\n", "Ground Truth: 992532,995974,998686,998334,991480,999451,995560,992349,992645,996894,993174,999711,993976,995314,998417,998079,992654,992383,997521,990383,996196,990401,998849,996262,991015,999216,991208,990376,997758,998061\n", - "Recall: 0.9000\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 380\n", + "Returned IDs: 997683,996393,998918,991309,994393,990657,993749,992623,992638,997110,997568,990361,998257,995044,998858,999156,993795,999555,995829,999312,995373,991204,998277,997114,990005,994901,993076,991987,992587,992861\n", + "Ground Truth: 997683,996393,998918,991309,994393,990657,993749,992623,992638,997110,997568,990361,998257,995044,998858,999156,993795,999555,991188,990554,995829,997030,999312,995373,997429,991204,998277,997114,991052,990005\n", + "Recall: 0.8333\n", "\n", "Query ID: 381\n", - "Returned IDs: 995482,999179,990157,994806,997997,993466,995106,991664,995055,998256,996633,993731,996283,991061,994166,997411,993820,998748,996386,995778,995777,996328,993111,998543,992563,998221,999759,990806,992364,990987\n", + "Returned IDs: 995482,999179,990157,994806,997997,993466,995106,991664,995055,998256,996633,993731,990083,994166,997411,993820,998748,995778,995777,996328,993111,998543,998221,999759,990806,992364,990987,992285,995996,993931\n", "Ground Truth: 995482,999179,990157,994806,997997,993466,995106,991664,995055,998256,996633,993731,996283,991061,990083,994166,997411,993820,998748,996386,995778,995777,996328,993111,998543,992563,998221,999759,990806,992364\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 382\n", - "Returned IDs: 996147,992161,990771,995167,996001,997790,997918,999898,991667,992950,993289,996657,997554,998215,999150,990399,999099,991311,999453,991212,994266,995855,997163,991585,992384,996549,995281,990314,993168,998720\n", + "Returned IDs: 996147,992161,990771,995167,996001,997790,997918,999898,991667,992950,993289,996657,997554,999150,990399,999099,999453,991212,994266,993143,991585,991485,992384,996549,990314,998720,993280,993157,995536,992367\n", "Ground Truth: 996147,992161,990771,995167,996001,997790,997918,999898,991667,992950,993289,996657,997554,998215,999150,990399,999099,991311,999453,991212,994266,995855,993143,997163,991585,991485,992384,996549,995281,990314\n", - "Recall: 0.9333\n", + "Recall: 0.8333\n", "\n", "Query ID: 383\n", "Returned IDs: 992558,990538,998950,990279,999639,996196,997775,997918,997258,994163,992615,995552,999206,999860,996609,990273,997859,999564,996592,998411,991522,994280,995340,994742,995372,999715,992436,991498,991791,993077\n", "Ground Truth: 992558,990538,998950,990279,999639,996196,997775,997918,995693,997258,994163,992615,995552,999206,999860,996609,990273,997859,999564,996592,998411,991522,994280,995340,994742,995372,999715,992436,991498,991791\n", "Recall: 0.9667\n", "\n", + "Query ID: 384\n", + "Returned IDs: 998748,991981,991185,995083,993209,993920,990593,992595,993466,992450,995633,990887,996328,991585,990331,997374,995268,997710,990157,990126,990435,997185,994541,994742,994185,993997,995996,995106,998055,991044\n", + "Ground Truth: 998748,992091,991981,991185,995083,993209,993920,990593,992595,993466,992450,995633,990887,996328,990226,991585,990331,997374,995268,997710,990157,990126,990435,997185,994541,994472,994742,994185,993997,995996\n", + "Recall: 0.9000\n", + "\n", "Query ID: 385\n", - "Returned IDs: 997852,997476,999077,999676,992169,994922,991334,990490,996325,992612,999916,990432,993260,997822,997496,993717,998041,993673,991840,991743,994964,992813,999453,998950,999843,997135,993355,994742,999533,994318\n", + "Returned IDs: 997852,999077,999676,992169,994922,991334,996325,997204,992612,990432,993260,997822,993717,991840,991743,994964,992813,999453,998950,993355,994742,994318,998779,991435,993237,998123,997309,997918,997379,993143\n", "Ground Truth: 997852,997476,999077,999676,992169,994922,991334,990490,996325,997204,992612,999916,990432,993260,997822,997496,993717,998041,993673,991840,991743,994964,992813,999453,998950,994044,999843,997135,993355,990214\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 386\n", + "Returned IDs: 998130,991982,993245,995449,993948,990005,993628,998758,991987,991630,993055,995516,998861,998959,994125,991530,997884,990897,995536,995780,992719,990991,996918,991052,992293,992862,995432,998623,998067,995184\n", + "Ground Truth: 998130,991613,991982,993245,995449,993948,990005,993628,998758,991987,991630,993055,995516,998861,998959,994125,991530,998383,997884,990897,995536,995780,992719,990991,996918,991052,992293,992862,995432,998586\n", "Recall: 0.9000\n", "\n", + "Query ID: 387\n", + "Returned IDs: 993725,992424,990528,997927,991565,997079,995551,993705,999751,996928,992753,999922,996633,998348,995274,996411,993623,990392,992226,995435,993053,999083,999740,992680,995744,992636,994783,994878,993585,994730\n", + "Ground Truth: 993725,992424,990528,997927,991565,997079,995551,990315,997207,993705,998574,999751,996928,992753,999922,996633,998348,995274,996411,997714,993623,990392,992226,995435,993053,999083,999740,992680,995744,992636\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 388\n", + "Returned IDs: 994578,993856,996867,992448,994140,990751,995289,992239,991790,997319,999235,999474,995633,994409,999875,992364,998910,996368,993293,990814,997723,997046,995640,992289,995030,996277,990940,992521,997868,997466\n", + "Ground Truth: 994578,993856,996867,992448,992468,994140,990751,995289,994419,992239,991790,997319,999235,999474,995633,994409,999875,992364,998999,991185,998910,996368,993293,990814,997723,997046,995640,992289,995544,993280\n", + "Recall: 0.8000\n", + "\n", "Query ID: 389\n", - "Returned IDs: 990742,991593,991662,991016,997826,998899,999701,997913,995748,991502,998003,999374,991616,997320,995928,998170,993238,994964,991479,993885,999948,993606,998294,992978,991743,990816,999262,994735,995917,992255\n", + "Returned IDs: 990742,991593,991662,991016,997826,998899,999701,997913,995748,991502,998003,999374,991616,997320,995928,998170,994964,991479,993885,999948,993606,992978,991743,998663,997470,991528,990650,991999,999263,995771\n", "Ground Truth: 990742,991593,991662,991016,997826,998899,999701,997913,995748,991502,998003,999374,991616,997320,999390,995928,990902,991237,998170,993238,994964,991479,993885,999948,993606,998294,992978,991743,990816,998663\n", - "Recall: 0.8667\n", + "Recall: 0.8000\n", "\n", "Query ID: 390\n", - "Returned IDs: 990249,996609,998820,991248,995124,992558,999240,996762,991350,994457,991266,995289,995377,997822,996606,993412,995332,995354,997723,999646,995401,995119,997498,991330,991449,998874,999898,992576,998529,996750\n", + "Returned IDs: 990249,996609,998820,991248,995124,992558,993433,999240,996762,994457,995289,995377,997822,996606,992469,995354,995401,995119,991330,991449,998874,996750,993647,995766,991334,999400,993561,993673,993831,997374\n", "Ground Truth: 990249,996609,998820,991248,995124,992558,993433,999240,996762,991350,994457,991266,995289,995377,997822,996606,993412,995332,996294,992469,995354,997723,999646,995401,995119,997498,991330,991449,998874,999898\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 391\n", + "Returned IDs: 992052,995873,994742,991451,994689,995295,993738,997897,996580,998256,990930,996328,992445,998623,992239,994280,990956,997329,993504,998045,991185,994255,996277,996001,996028,995823,990005,992636,994307,990096\n", + "Ground Truth: 992052,993456,998427,995873,994742,991451,994689,995295,993738,997897,995408,996580,998256,990930,996328,992445,991255,998623,992239,994280,990956,997329,993504,998045,991185,997918,994255,996277,996001,996028\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 392\n", + "Returned IDs: 996986,990951,992636,996408,999334,991393,990173,996183,997168,999791,992372,993917,998348,996728,993001,998076,999138,997838,998440,993715,996882,994351,992664,999435,992282,999373,994742,990918,997732,996062\n", + "Ground Truth: 996986,990951,992636,996408,999334,991393,990173,990027,996183,997168,999791,992372,993917,998348,996728,993001,998076,999138,997838,998440,993715,996882,998389,990712,994351,992664,999435,992282,999373,994742\n", "Recall: 0.9000\n", "\n", "Query ID: 393\n", - "Returned IDs: 990487,996451,999774,999191,996194,997770,991977,994254,997251,996042,999179,990102,998968,997115,990678,992558,991447,992979,996658,994387,992086,998104,991303,990705,994280,993275,991743,997918,997529,991061\n", + "Returned IDs: 990487,999774,999191,996194,998528,997770,991977,994254,997251,996042,999179,990102,997115,990678,992558,991447,992979,994387,992086,998104,991303,994280,993275,997918,991061,998703,997126,990249,991334,996429\n", "Ground Truth: 990487,996451,999774,999191,996194,991720,998528,999343,997770,991977,994254,997251,996042,999179,990102,998968,997115,990678,992558,991447,992979,996658,994387,992086,998104,991303,990705,994280,993275,991743\n", - "Recall: 0.9000\n", + "Recall: 0.7667\n", "\n", "Query ID: 394\n", - "Returned IDs: 990535,999144,994217,993168,992630,998951,993083,997015,992331,995336,997320,995947,999437,997082,994964,990925,992748,993389,995252,997002,996001,999019,993484,994757,994916,995824,994854,998090,990958,992047\n", + "Returned IDs: 990535,999144,994217,993168,992630,998951,993083,997015,992331,995336,997320,999437,997082,994964,990925,992748,993389,995252,997002,996001,999019,993484,994757,995824,994854,998090,990958,990231,992384,993050\n", "Ground Truth: 990535,999144,994217,993168,992630,998951,993083,997015,992331,995336,999916,997320,995947,999437,997082,994964,990925,992748,993389,995252,997002,995216,996001,999019,993484,994757,994916,995824,994854,998090\n", - "Recall: 0.9333\n", + "Recall: 0.8667\n", "\n", "Query ID: 395\n", - "Returned IDs: 994280,991334,997023,995968,990214,995377,995401,999701,991923,994835,995820,991743,990921,995886,994655,992165,992582,995824,994982,992521,998302,992384,993561,997918,991269,994496,996762,995336,998295,995211\n", + "Returned IDs: \n", "Ground Truth: 994280,991334,997023,995968,990214,995377,995401,999701,991923,994835,995820,991743,998374,990921,995886,994655,992165,992582,995824,994982,992521,998302,992384,993561,997918,991269,994496,996762,995336,998295\n", - "Recall: 0.9667\n", + "Recall: 0.0000\n", "\n", - "Query ID: 399\n", - "Returned IDs: 998908,998921,994244,997539,997301,999995,993581,995154,995910,993143,999219,998539,990760,996014,995554,991157,993364,993662,997777,997370,990561,992124,999358,996040,998029,998726,997371,993250,996092,998894\n", - "Ground Truth: 998908,998921,994244,997539,997301,999995,993581,995154,990702,999426,995910,993143,999219,992461,998539,990760,996014,995554,991157,993900,993364,997158,993662,997777,997370,990561,992124,999358,996040,993005\n", - "Recall: 0.8000\n", + "Query ID: 396\n", + "Returned IDs: 996915,996001,996147,998697,997506,995536,997227,993157,998720,998303,995250,990771,993280,996657,991332,996368,993289,993561,990532,990096,991530,998060,999639,992950,995562,993055,995899,997554,990399,999453\n", + "Ground Truth: 996915,996001,996147,998697,997506,995536,997227,993157,998720,998303,995250,995401,990771,991585,993280,996657,991332,996368,993289,993561,999587,990532,990096,991530,998060,999639,992950,995562,993055,995899\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 397\n", + "Returned IDs: 996178,990532,996805,998950,990096,991981,997320,992318,994307,994742,997906,999444,993809,994185,990843,990364,990083,995401,993080,997913,997790,993342,993504,994416,996397,990814,996370,993621,991459,995438\n", + "Ground Truth: 996178,990532,996805,998950,990096,991981,997320,992318,994307,994742,997906,999444,993809,994185,990843,990364,990083,995401,992636,993080,997913,997790,998365,993342,993504,994416,996397,999647,990814,993834\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 399\n", + "Returned IDs: 998908,998921,997539,997301,993581,995154,995910,993143,999219,990760,996014,991157,993364,997370,990561,996040,998029,998726,997371,993250,998894,991392,997218,997984,992734,990053,991782,990146,992546,994675\n", + "Ground Truth: 998908,998921,994244,997539,997301,999995,993581,995154,990702,999426,995910,993143,999219,992461,998539,990760,996014,995554,991157,993900,993364,997158,993662,997777,997370,990561,992124,999358,996040,993005\n", + "Recall: 0.5333\n", + "\n", + "Query ID: 400\n", + "Returned IDs: 996042,995401,992685,997822,997723,996867,993433,997232,995289,998080,992169,995200,992558,997918,997675,993407,998913,998495,995105,990214,998528,990119,997466,995685,992582,996734,996903,996809,992604,993717\n", + "Ground Truth: 996042,991052,995401,992685,997822,997723,996867,993433,997232,995289,993412,998080,992169,995200,992558,997918,997675,993407,998913,998495,995105,990214,998528,990119,997466,995685,992582,996038,996734,996903\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 401\n", + "Returned IDs: 992978,991883,997320,990210,993431,994964,991721,997913,996904,990114,997470,995841,991249,999167,998550,992944,992546,998364,993072,997371,998365,991801,998540,992692,990096,993662,992310,990925,996143,992066\n", + "Ground Truth: 992978,991883,997320,990210,993431,994598,994964,991721,997913,996904,990114,997470,995841,991249,999167,998550,992114,992944,992546,998364,993072,998698,997371,998365,991801,998540,992692,990096,993662,992909\n", + "Recall: 0.8667\n", "\n", "Query ID: 402\n", - "Returned IDs: 997643,994724,997210,992638,995044,997683,990657,997589,999156,999117,998918,992206,992623,993076,998257,996393,998277,993795,994660,993666,991797,999808,995207,991309,990167,998787,991204,990361,998383,992760\n", + "Returned IDs: 997643,994724,997210,992638,995044,997683,990657,997589,999156,999117,998918,992623,993076,998257,996393,998277,993795,994660,993666,991797,999808,995207,991309,998787,991204,990361,998383,995056,992926,991038\n", "Ground Truth: 997643,994724,997210,992638,995044,997683,990657,997589,999156,999117,998918,992206,992623,993076,998257,996393,998277,993795,994660,993666,991797,999808,995207,991309,990167,998787,991204,994917,990361,998383\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", "\n", "Query ID: 403\n", - "Returned IDs: 998149,990658,994869,999663,991038,999834,993055,994155,999736,997506,999139,998858,995780,994467,995237,991169,999555,992421,998831,993629,995516,996406,999896,996623,999806,995755,998383,998720,991154,996446\n", + "Returned IDs: 998149,990658,994869,999663,991038,999834,993055,999736,997506,999139,995780,994467,995237,991169,999555,992421,998831,993629,995516,996406,999896,996623,999806,995755,998720,991154,990292,997650,998586,998758\n", "Ground Truth: 998149,990658,994869,999663,991038,999834,993055,994155,999736,997506,999139,998858,995780,994467,995237,991169,999555,992421,998831,993629,995516,996406,999896,996623,999806,995755,998383,998720,991154,991835\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 404\n", + "Returned IDs: 992493,995807,999826,997189,994677,994301,998820,993530,997232,993223,997420,996139,997184,995625,991051,998316,995521,993149,997075,998080,999475,997675,997634,997574,997061,997824,999961,995493,996009,992801\n", + "Ground Truth: 992493,995807,999826,997189,994677,994301,998820,993530,997232,993223,997420,996139,997184,995625,991051,998316,995521,993149,994141,997075,998080,999475,997675,997634,997574,997061,997824,992683,996903,995366\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 405\n", + "Returned IDs: 998999,995940,997662,993103,995289,996302,999957,991603,995401,995200,999875,990940,997630,996769,994409,997723,995182,996388,990033,997020,998530,994161,994948,991954,997319,998765,993144,995648,992619,998162\n", + "Ground Truth: 998999,995940,997662,993103,995289,996302,999957,997042,991603,995401,996864,995200,999875,990940,997630,996769,994409,997723,995182,996388,990033,997020,998530,994161,994948,991954,997319,998765,995701,993856\n", + "Recall: 0.8667\n", "\n", "Query ID: 406\n", - "Returned IDs: 994577,992068,998816,991015,999806,999194,997379,995039,999122,993628,999432,992423,993012,991157,991987,996281,998568,998950,998858,993143,997466,990991,998437,998257,999267,991713,999568,992330,990479,991844\n", + "Returned IDs: 994577,992068,998816,991015,999806,999194,997379,995039,999122,999432,992423,993012,991157,991987,998568,998950,998858,993143,997466,990991,998437,998257,991713,999568,992330,991844,997320,991571,997918,996262\n", "Ground Truth: 994577,992068,998816,991015,999806,999194,997379,995039,999122,993628,995453,999432,992423,993012,991157,991987,996281,998568,998950,998858,993143,997466,990991,998437,998257,999267,991713,999568,992330,990479\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 407\n", - "Returned IDs: 994717,994809,990255,990361,997088,995401,999587,993515,994092,992293,996657,999629,998593,996043,996809,995701,996759,990573,994431,996930,995289,992558,993848,992367,998536,997897,998990,996393,998250,996777\n", + "Returned IDs: 994717,994809,990255,990361,995401,994092,992293,996657,999629,998593,996809,995701,996759,990573,994431,996930,995289,992558,993848,992367,998536,997897,998990,996393,998250,996777,995963,997822,995873,995864\n", "Ground Truth: 994717,994809,990255,990361,997088,995401,999587,993515,994092,992293,996657,999629,998593,996043,996809,995701,996759,990573,994431,996930,995289,992558,993848,997411,992367,998536,997897,998990,996393,998250\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 408\n", + "Returned IDs: 994794,993465,990242,993412,998297,999923,991603,997083,995269,998999,990385,990940,995208,991028,991790,997045,995702,995274,997723,995940,999374,999932,997787,998779,991350,997020,997832,995373,996379,991543\n", + "Ground Truth: 994794,993465,990242,994702,993412,998297,999923,991603,997083,995269,998999,990385,990940,995208,997473,991028,991790,997045,995702,995274,997723,995940,999374,999932,997787,998779,991350,997020,997832,995373\n", + "Recall: 0.9333\n", "\n", "Query ID: 409\n", - "Returned IDs: 996690,993363,996963,991772,991157,991550,993370,993411,993706,999393,994163,996196,999639,997775,998697,990159,996926,995013,992349,990279,992770,999908,992423,992738,997225,997258,998142,995599,999041,995804\n", + "Returned IDs: 996690,993363,996963,991772,991157,991550,993370,993411,993706,999393,992383,994163,996196,999639,997775,998697,990159,996926,995013,992349,990279,992770,999908,992423,992738,997225,997258,998142,995599,999041\n", "Ground Truth: 996690,993363,996963,991772,991157,991550,993370,993411,993706,999393,992383,994163,996196,999639,997775,998697,990159,996926,995013,992349,990279,992770,991706,999908,992423,992738,997225,997258,998142,995599\n", - "Recall: 0.9333\n", + "Recall: 0.9667\n", "\n", "Query ID: 410\n", - "Returned IDs: 998170,994240,997699,993431,996592,994584,999077,992880,997470,993260,997320,997918,991801,995094,991348,992927,994666,999701,996296,998180,994450,995826,991928,993696,995266,995357,991564,991131,994964,992748\n", + "Returned IDs: 998170,994240,993431,996592,999077,992880,997470,993260,997320,997918,991801,991348,999701,996296,998180,995826,991928,993696,995357,991564,991131,994964,992748,994552,994360,990949,996761,995748,993646,993154\n", "Ground Truth: 998170,994240,997699,993431,996592,994584,999077,992880,997470,993260,997320,997918,991801,995094,991348,993454,992927,994666,999701,996296,998180,994450,995826,991928,993696,995266,995357,991564,991131,994964\n", - "Recall: 0.9667\n", + "Recall: 0.7333\n", "\n", "Query ID: 411\n", - "Returned IDs: 995266,991593,992069,998899,994399,997320,991774,992255,991580,990957,998967,991662,999290,997470,997301,999077,993662,995826,991479,993852,998090,999602,994604,990910,994290,996296,992978,990650,993034,993402\n", + "Returned IDs: 991593,992069,998899,994399,997320,991774,992255,990957,991662,999290,997470,997301,999077,993662,995826,991479,998090,999602,994604,990910,994290,996296,992978,990650,993034,993402,997371,993606,991743,995898\n", "Ground Truth: 995266,991593,992069,998899,994399,997320,991774,992255,991580,990957,998967,991662,999290,997470,997301,999077,993662,995826,991479,993852,998090,999602,994604,990910,994290,996296,992978,992946,990650,998801\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", "\n", "Query ID: 412\n", - "Returned IDs: 992521,997545,993269,997466,996552,991878,991969,993107,992097,994353,998238,993438,991555,993353,998075,998910,998346,995193,994508,991760,997216,995813,998205,998568,992448,994636,994954,993738,997723,995864\n", + "Returned IDs: 992521,997545,993269,997466,996552,991878,991969,993107,992097,994353,998238,993438,991555,993353,998075,998910,995193,994508,995813,998205,995644,992448,994636,994954,997723,991117,998509,999121,990603,995198\n", "Ground Truth: 992521,997545,993269,997466,996552,991878,991969,993107,992097,994353,998238,993438,991555,998564,993353,998075,998910,998346,995193,994508,991760,997216,995813,998205,998272,995644,998568,992448,994636,994954\n", - "Recall: 0.9000\n", + "Recall: 0.8000\n", "\n", "Query ID: 413\n", - "Returned IDs: 990956,992558,996809,994742,995963,991052,999179,992052,990776,992604,997723,990361,998495,997752,994643,995408,994351,993948,990657,998543,993799,995401,993079,991319,995105,992685,997914,993121,996043,993820\n", + "Returned IDs: 990956,992558,996809,994742,999179,992052,990776,992604,990361,998495,997752,994643,993077,994351,993948,990657,991257,995105,997914,993121,996043,991145,997110,999191,996277,994280,995373,996328,990705,990083\n", "Ground Truth: 990956,992558,996809,994742,995963,991052,999179,992052,990776,992604,997723,990361,998495,997752,994643,993077,995408,994351,993948,990657,998543,991257,993799,995401,993079,991319,995105,992685,997914,993121\n", - "Recall: 0.9333\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 415\n", + "Returned IDs: 991643,992554,993325,997685,992209,993055,991724,999744,993965,996889,994834,991738,990096,999622,994420,996733,999319,993744,995385,991243,995402,991093,995736,996209,999771,999079,995796,990160,992237,996817\n", + "Ground Truth: 991643,992554,993325,997685,992209,993055,991724,999744,993965,996889,994834,991738,990096,999622,994420,996733,999319,993744,999273,995385,998419,995402,991243,991093,995736,993132,996209,999771,999079,995796\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 416\n", + "Returned IDs: 992713,994142,994891,991177,993567,995991,990784,990364,999775,998067,998256,996328,990956,991747,996728,995184,993621,994999,994444,992526,994280,998251,992450,998405,991111,996911,998671,990712,997506,993336\n", + "Ground Truth: 992713,994142,994891,991177,993567,995991,990784,990364,999775,998067,998256,996328,999441,990956,991747,996728,995184,993621,994999,991143,997860,994444,992526,994280,998251,992450,998405,991111,996911,993929\n", + "Recall: 0.8667\n", "\n", "Query ID: 417\n", - "Returned IDs: 991198,997789,990043,999620,998232,991196,996681,998236,997275,998983,993582,990155,991801,990858,990809,993570,998837,996318,994922,994785,992907,999076,993675,991510,991999,994964,990585,997876,993237,999053\n", + "Returned IDs: 991198,997789,990043,999620,998232,991196,996681,998236,997275,998983,993582,990155,991801,990858,993570,998837,994922,994785,992907,999076,991510,991999,994964,990585,998280,993237,999053,994612,990142,997288\n", "Ground Truth: 991198,997789,990043,999620,998232,991196,996681,998236,997275,998983,993582,990155,991801,990858,990809,993570,998837,996318,994922,994785,992907,999076,993675,991510,991999,994964,990585,997876,998280,993237\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 418\n", + "Returned IDs: 994106,994203,999731,991253,995216,990193,998205,998255,997059,995873,992521,992162,997554,998527,995813,996001,994488,990081,998910,993951,993738,990775,991163,992448,999402,991394,995079,990603,997928,997176\n", + "Ground Truth: 994106,994203,999731,991253,995216,990193,998205,994773,998255,997059,995873,992521,992162,997554,998527,995813,996001,994488,990081,998910,993951,993738,990775,991163,992448,999402,991394,995079,990603,997928\n", "Recall: 0.9667\n", "\n", + "Query ID: 419\n", + "Returned IDs: \n", + "Ground Truth: 993478,994193,997276,999244,996743,997184,996181,993155,991052,996918,991799,990521,994280,997420,993727,996825,991837,995401,998267,992933,999259,997675,994092,992685,995783,996438,992133,993980,999264,994474\n", + "Recall: 0.0000\n", + "\n", "Query ID: 420\n", - "Returned IDs: 998886,992293,994382,990662,997868,990361,998017,990399,996809,999922,997466,995470,992582,998495,993390,992862,991603,999587,996042,993831,994793,997723,994689,995679,997529,991578,998982,995562,994730,990638\n", + "Returned IDs: 998886,992293,994382,990662,997868,990361,998017,996809,999922,997466,992582,998495,993390,992862,999587,996147,996042,994793,994689,995679,997529,998982,994730,995701,993561,995702,995289,996391,994431,995536\n", "Ground Truth: 998886,992293,994382,990662,997868,990361,998017,990399,996809,999922,997466,995470,992582,998495,993390,992862,991603,999587,996147,996042,993831,994793,997723,994689,995679,997529,991578,998982,995562,994730\n", - "Recall: 0.9667\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 421\n", + "Returned IDs: 999374,998779,992685,993412,997529,996219,994382,997822,991958,995702,996379,995401,993096,995685,998045,998380,998495,995289,998123,992169,997723,994964,994057,992351,993355,996959,995633,996721,997002,993089\n", + "Ground Truth: 999374,998779,992685,991052,993412,997529,996219,994382,997822,999900,990337,991958,995702,996379,995401,993096,995685,998045,998380,995994,998495,995289,990701,998123,991743,992169,993725,993186,997723,994964\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 422\n", + "Returned IDs: 996328,999139,992726,999734,991652,998864,995705,994574,997944,993139,991605,990712,993665,996591,995541,995112,993715,991990,991685,996495,991037,994280,997847,999827,996733,990658,994510,992450,995655,997309\n", + "Ground Truth: 996328,999139,992726,999734,991652,998864,995705,994574,997944,993139,991605,990712,993665,996591,995541,995112,993715,991990,995103,991685,996495,991037,991596,994280,997847,999827,993289,993981,996733,990330\n", + "Recall: 0.8333\n", "\n", "Query ID: 423\n", - "Returned IDs: 998470,995214,992558,999791,999767,998122,995222,998279,994972,998495,997732,990956,998765,996001,997603,999191,999827,992713,995438,990091,990089,996219,996647,998710,990364,994280,999138,995107,999852,994119\n", + "Returned IDs: 998470,995214,992558,999791,999767,998122,998279,998495,997732,990956,998765,996001,999191,999827,992713,995438,990089,996219,996647,998710,990364,994280,999138,999852,994119,996809,997586,992780,994742,997453\n", "Ground Truth: 998470,995214,992558,999791,999767,998122,995222,998279,994972,998495,997732,990956,998765,996001,997603,999191,999827,992713,995438,990091,997920,990089,999916,996219,996647,998710,990364,994280,999138,995107\n", - "Recall: 0.9333\n", + "Recall: 0.7667\n", "\n", "Query ID: 424\n", - "Returned IDs: 990811,998593,996147,999173,991000,996657,990130,992326,993483,996028,998878,990771,992727,996001,994838,998934,995158,999740,995864,990573,998250,993391,992289,993813,995289,995200,995182,993848,996302,997007\n", + "Returned IDs: 990811,998593,996147,999173,991000,996657,992326,996028,998878,990771,992727,996001,998934,995158,999740,990573,998250,993391,992289,993813,995289,995200,995182,993848,996302,997007,997753,999868,998999,991958\n", "Ground Truth: 990811,998593,996147,999173,991000,996657,990130,992326,993483,996028,998878,990771,992727,996001,998934,994838,995158,999740,995864,990573,998250,993391,996864,992289,993813,995289,995200,995182,993848,996302\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 425\n", - "Returned IDs: 990382,999543,994510,993515,993931,999139,990096,998593,998250,996062,991685,995889,994463,991596,995536,998495,997790,995895,990930,996733,994191,991011,998376,996001,996549,999994,995017,993778,999091,991066\n", + "Returned IDs: 990382,999543,994510,993931,999139,990096,998593,998250,996062,991685,995889,994463,991596,995536,998495,997790,995895,990930,996733,994191,991011,998376,996001,996549,999994,995017,999091,991066,990255,997897\n", "Ground Truth: 990382,999543,994510,993515,993931,999139,990096,998593,998250,996062,991685,995889,994463,991596,995536,998495,994796,997790,995895,990930,996733,994191,991011,998376,996001,996549,999994,995017,993778,999091\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 426\n", + "Returned IDs: 995591,994964,999367,991588,999210,992612,992944,997379,995200,997529,997822,996867,990690,995289,996990,993215,997406,995052,992169,995923,996219,996042,997723,993265,996328,996809,998495,994742,995360,998765\n", + "Ground Truth: 995591,994964,999367,991588,999210,992612,992944,997379,995200,997529,997822,996867,990690,995289,996990,993215,997406,995052,992169,995923,996219,996042,997723,993265,996328,991921,999352,996809,994307,998495\n", + "Recall: 0.9000\n", "\n", "Query ID: 427\n", - "Returned IDs: 992862,997586,995005,991490,995701,990786,999806,993809,995536,998708,999653,995455,996082,993637,993470,999444,990167,998200,997506,993834,990271,999087,993515,998697,993504,995203,998536,991038,993055,995672\n", + "Returned IDs: 992862,997586,991490,995701,990786,999806,993809,995536,998708,999653,995455,996082,993637,993470,999444,990167,998200,997506,993834,996370,999087,993515,998697,995203,998536,993055,995672,991530,999216,990449\n", "Ground Truth: 992862,997586,995005,991490,995701,990786,999806,993809,995536,998708,999653,995455,996082,993637,993470,999444,990167,998200,997506,993834,990271,996370,999087,993515,998697,993504,995203,993821,998536,997577\n", - "Recall: 0.9000\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 428\n", + "Returned IDs: 993586,996304,997544,993371,991639,996220,993209,990082,993206,998811,999759,995883,995778,990372,997975,990298,996102,994496,999646,997023,991269,999957,999054,992582,991507,997915,990638,994106,998910,997792\n", + "Ground Truth: 993586,996304,997544,993371,991639,996220,993209,990082,993206,998811,996595,999759,995883,995778,990372,997975,990298,996102,994496,999646,997023,991269,999957,999054,992582,991507,997915,990638,994106,999250\n", + "Recall: 0.9333\n", "\n", "Query ID: 429\n", - "Returned IDs: 996734,995401,990337,997191,998297,995685,993412,995373,998045,991603,994409,993561,998999,991052,997406,996968,992448,995940,996277,993288,991140,997723,994948,990930,997918,995648,999530,997822,992685,995994\n", + "Returned IDs: 996734,995401,990337,997191,998297,995685,993412,995373,998045,991603,994409,993561,998999,997406,996968,992448,995940,993288,991140,997723,994948,995648,999530,997822,992685,995994,991630,995886,998779,996042\n", "Ground Truth: 996734,995401,990337,997191,998297,995685,993412,995373,998045,991603,994409,993561,998999,991052,997406,996968,992448,995940,996277,993288,991140,997723,994948,990930,997918,995648,999530,997822,995873,992685\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 430\n", - "Returned IDs: 994956,996219,996328,990214,999179,996811,997442,999500,994387,996149,993727,995332,998256,998968,995438,997420,999560,994783,995562,995130,997002,994233,995895,999540,999352,990039,994206,992237,994452,997945\n", + "Returned IDs: 996219,996328,990214,999179,996811,997442,999500,994387,998256,998968,995438,997420,999560,994783,995562,995130,994233,995895,990039,992237,997945,994742,997485,993100,999430,993560,999139,994402,994280,996494\n", "Ground Truth: 994956,996219,996328,990214,999179,996811,997442,999500,994387,996149,993727,995332,998256,998968,995438,991212,997420,999560,994783,995562,996071,995130,997002,994233,995895,999352,999540,990039,994206,992237\n", - "Recall: 0.9333\n", + "Recall: 0.6667\n", "\n", "Query ID: 431\n", - "Returned IDs: 994809,997218,995401,992790,990841,991671,996762,991197,991334,996930,993673,992749,992558,990312,992636,991459,994835,995864,992685,993003,992448,998205,996162,998104,996187,990181,993275,995873,997023,996001\n", + "Returned IDs: 994809,997218,992790,991671,996762,991197,996930,993673,992749,992558,990312,992636,994835,995864,998855,993003,992448,998205,996162,996187,990181,993275,995873,997023,996001,993561,995536,998910,994409,998495\n", "Ground Truth: 994809,997218,995401,992790,990841,991671,996762,991197,991334,996930,993673,992749,992558,990312,992636,991459,994835,995864,992685,998855,993003,992448,998205,996162,998104,996187,990181,993275,995873,996438\n", - "Recall: 0.9333\n", + "Recall: 0.7667\n", "\n", "Query ID: 432\n", - "Returned IDs: 996279,994533,999301,996192,994688,994757,993852,999883,990215,990469,994854,990275,998190,996714,999089,992511,993457,999327,995034,991379,997012,999500,992384,998170,992087,999548,998011,999953,992854,997605\n", + "Returned IDs: 996279,994533,999301,996192,994688,994757,993852,999883,990215,990469,994854,990275,998190,996714,999089,992511,993457,999327,995034,991379,997012,992384,998170,992087,999548,998011,999953,992854,995875,997605\n", "Ground Truth: 996279,994533,999301,996192,994688,994757,993852,999883,990215,990469,994854,990275,998190,996714,999089,992511,993457,999327,995034,991379,997012,999500,992384,998170,992087,991650,999548,998011,999953,992854\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 433\n", - "Returned IDs: 999820,990561,995327,992979,994906,997910,992790,994946,994673,993289,994819,990304,995968,998011,991074,996425,994982,995401,996714,994170,997218,999715,994334,997396,994188,997002,994922,998210,999977,991607\n", + "Returned IDs: 999820,990561,995327,992979,994906,997910,992790,994946,994673,993289,994819,995968,991074,996425,994982,994170,997218,994334,994188,997002,994922,999977,995875,999026,997380,999587,990215,997605,992884,990916\n", "Ground Truth: 999820,990561,995327,992979,994906,997910,992790,994946,994673,993289,994819,990304,995968,998011,992047,991074,996425,994982,995401,996714,994170,997218,999715,994334,997396,994188,997002,994922,998210,999977\n", - "Recall: 0.9667\n", + "Recall: 0.7333\n", "\n", "Query ID: 434\n", - "Returned IDs: 992950,997320,996904,993662,998540,999109,992451,998302,997470,993237,994964,997913,997379,992443,996890,993791,998550,999710,999210,998837,991626,993796,994192,992069,991801,993018,991582,997135,992733,998950\n", + "Returned IDs: \n", "Ground Truth: 992950,997320,996904,993662,998540,999109,992451,998302,997470,993237,994964,997913,997379,992443,992140,996890,993791,998550,999710,999210,998837,991626,993796,994192,990615,992069,991801,993018,991582,997135\n", - "Recall: 0.9333\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 435\n", + "Returned IDs: 996802,994964,997320,993389,994341,993662,990146,993484,996296,991609,990949,990843,997379,993781,990585,995094,993696,990231,991743,991732,992950,998950,999210,999647,991154,993260,994742,993603,996469,999430\n", + "Ground Truth: 996802,994964,997320,997747,993389,994341,993662,990146,993484,996296,991609,990949,990843,997379,995560,993781,990930,992120,990585,995094,993696,990231,991743,991732,992950,998950,999210,991886,999647,991154\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 436\n", + "Returned IDs: 992237,999752,990806,998405,997392,993196,996839,991723,992827,993335,994472,995630,998055,994335,994101,994960,996016,992372,997527,993501,997442,990173,992892,992285,991418,990440,997997,991672,993460,991061\n", + "Ground Truth: 992237,999752,990806,991961,998405,997392,993196,996839,991723,992827,993335,994472,995630,998055,994335,994101,994960,996016,992372,997527,993501,997442,990173,992892,992285,991418,990440,997997,991672,993460\n", + "Recall: 0.9667\n", "\n", "Query ID: 437\n", - "Returned IDs: 994749,999544,994826,997249,995968,997498,999254,994081,997888,995372,990772,990043,995932,991440,993071,999676,996309,996506,991787,994171,992531,991880,999453,995841,997661,998734,994687,997189,995250,991002\n", + "Returned IDs: 994749,999544,997249,995968,997498,999254,994081,997888,995372,990772,990043,995932,991440,993071,999676,996309,991787,991880,999453,995841,997661,996785,998734,994240,995250,991002,996217,993241,994789,993683\n", "Ground Truth: 994749,999544,994826,997249,995968,997498,999254,994081,997888,995372,990772,990043,995932,991440,993071,999676,996309,996506,991787,998654,994171,992531,991880,999453,995841,997661,996785,998734,994240,994687\n", - "Recall: 0.9000\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 438\n", + "Returned IDs: 990550,998200,994624,993834,999087,998232,992862,996082,999651,997913,996661,999783,995296,991256,995701,996062,990096,993342,993504,996001,995184,995672,998458,997586,995132,992293,993489,998697,999647,990366\n", + "Ground Truth: 990550,998200,994624,993834,993902,999087,998232,992862,996082,999651,997913,996661,999783,995296,991256,993403,995701,996062,990096,993342,993504,996001,995184,995672,998458,990866,997586,995132,995927,992293\n", + "Recall: 0.8667\n", "\n", "Query ID: 439\n", - "Returned IDs: 998536,992110,991644,990449,991620,993001,992282,992595,990399,992832,999827,997506,990154,998067,995655,990942,995326,991667,996328,992444,998625,992318,996408,999994,999555,992862,993993,995030,994380,997494\n", + "Returned IDs: 998536,992110,991644,990449,991620,993001,992282,992595,990399,992832,999827,997506,990154,998067,995655,990942,995326,991667,996328,992444,998625,996408,999994,992862,995030,994380,992761,991747,998623,992793\n", "Ground Truth: 998536,992110,991644,990449,991620,993001,992282,992595,990399,992832,999827,997506,990154,998067,995655,990942,995326,997723,991667,996328,992444,998625,998861,992318,996408,999994,999555,992862,993993,995030\n", - "Recall: 0.9333\n", + "Recall: 0.8333\n", "\n", "Query ID: 440\n", - "Returned IDs: 999594,992451,993419,990936,992583,990089,998163,999516,993060,995659,999791,996279,996755,993848,994776,992546,995792,992470,999757,993630,993484,996001,996516,993490,992785,997320,998921,996075,995454,999900\n", + "Returned IDs: 999594,992451,993419,990089,998163,993060,994325,995659,999791,996755,993848,994776,992470,993484,996001,996516,993490,997320,998921,996075,995454,994757,997984,992636,993637,996155,995981,994964,995306,999740\n", "Ground Truth: 999594,992451,993419,990936,992583,990089,998163,999516,993060,994325,995659,999791,996279,990862,996755,993848,994776,992546,994199,995792,992470,999757,993630,993484,996001,996516,993490,992785,997320,998921\n", - "Recall: 0.9000\n", + "Recall: 0.6333\n", "\n", "Query ID: 441\n", - "Returned IDs: 991178,990980,993914,990648,999343,996219,997984,994266,999404,993238,994964,998500,992546,995942,990146,999824,993072,990595,994415,990650,990736,996028,993518,994889,998458,990545,998581,997855,991325,998921\n", + "Returned IDs: \n", "Ground Truth: 991178,990980,993914,990648,999343,996219,997984,994266,999404,993238,994964,998500,992546,995942,990146,993072,999824,990595,994415,993048,990650,990736,996028,993518,994889,998458,990545,998581,997855,991325\n", - "Recall: 0.9667\n", + "Recall: 0.0000\n", "\n", "Query ID: 442\n", - "Returned IDs: 991856,993960,991202,994497,994604,999260,990224,995135,999633,996854,990958,999374,999793,991889,998190,994922,995947,995259,990980,991593,998921,995365,993640,994266,990660,995443,994195,998500,997192,991196\n", + "Returned IDs: 991856,993960,991202,994497,994604,999260,990224,995135,996854,991733,990958,999374,997470,999398,994922,990980,991593,995959,998921,995365,993640,994266,990660,998500,997192,997842,994584,998660,994819,995260\n", "Ground Truth: 991856,993960,991202,994497,994604,992323,999260,999261,990224,995135,999633,996854,991733,990958,999374,999793,997470,991889,999398,998190,994922,995947,995259,990980,991593,995959,998921,995365,993640,993570\n", - "Recall: 0.7667\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 443\n", + "Returned IDs: 992944,998950,993018,997370,997320,995130,996693,995438,996075,993079,997113,995873,996140,990949,991142,996001,998302,998180,996318,993504,990022,995560,993053,996657,991498,995203,997775,994171,999210,996210\n", + "Ground Truth: 992944,998950,993018,997370,997320,995130,996693,995438,996075,993079,997113,995873,996140,990949,991142,996001,998302,998180,996318,993280,993504,990022,995560,993053,992521,996657,991498,995203,997775,994171\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 444\n", + "Returned IDs: 999868,996042,998866,993597,992125,991301,998564,993433,995905,996112,996781,998165,996769,997406,997046,995289,994419,995702,998593,999080,992160,995439,990681,997822,995401,996360,998045,990984,992519,991463\n", + "Ground Truth: 999868,996042,998866,993597,992125,991301,998564,993433,995905,996112,996781,998165,996769,997406,997046,995289,994419,995702,998593,999080,992160,995439,990706,990681,997822,995401,996360,998045,990984,992519\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 445\n", + "Returned IDs: 991266,990352,990751,996368,993561,993673,997918,999099,993289,998999,997002,998186,990517,993275,994664,999977,998593,996001,991958,991332,996379,992828,990930,996759,995701,992703,990035,997189,995679,993813\n", + "Ground Truth: 991266,990352,990751,996368,995421,993561,993673,997918,999099,993289,998999,997002,998186,991603,990517,999303,999866,993275,994664,995097,996028,997077,995274,999977,998593,994959,996001,991958,991332,996379\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 446\n", + "Returned IDs: 992169,996062,993717,997732,994280,999827,996328,998180,997852,997309,995861,991066,993215,993052,992703,995295,992944,992724,996733,997822,993080,997918,994964,996093,992713,990392,997790,990080,997002,998765\n", + "Ground Truth: 992169,996062,993717,997732,994280,999827,996328,998180,997852,998633,996728,997309,995861,991066,993215,993052,992703,990889,995295,992944,992724,996733,996216,997822,993080,993238,997918,994964,996093,992713\n", + "Recall: 0.8333\n", "\n", "Query ID: 447\n", - "Returned IDs: 990366,990089,995569,994373,999898,993401,990271,996219,997855,995306,994919,999417,999610,994279,994792,999651,992174,995095,995096,995210,990438,995551,991593,991172,999192,999791,995868,993402,990173,991751\n", + "Returned IDs: 990366,990089,995569,999898,993401,990271,996219,997855,995306,994919,999610,994279,994792,999651,995095,995096,995551,991593,999192,999791,997340,995868,993402,990173,999617,992704,990179,995384,991840,996052\n", "Ground Truth: 990366,990089,995569,994373,999898,993401,990271,996219,997855,995306,994919,999417,999610,994279,994792,991572,999651,992174,995095,995096,995210,990438,995551,991593,991172,999192,999216,999791,997340,995868\n", - "Recall: 0.9000\n", + "Recall: 0.7333\n", "\n", "Query ID: 448\n", - "Returned IDs: 999658,994611,991088,998966,997320,990444,993253,995942,993484,991593,994249,990890,991895,996838,992546,999167,998246,990936,990949,993694,994964,996219,990210,997699,994573,998287,991155,997372,990545,997470\n", + "Returned IDs: 999658,994611,991088,998966,997320,993253,993484,991593,994249,990890,991895,996838,992546,999167,998246,990936,990949,993694,994964,996219,990210,999247,994573,991155,997372,990545,997470,994542,996143,999948\n", "Ground Truth: 999658,994611,991088,998966,997320,990444,993253,995942,993484,991593,994249,990890,991895,996838,992546,999167,998246,990936,990949,993694,994964,996219,990210,997699,999247,994573,998287,991155,999991,997372\n", - "Recall: 0.9333\n", + "Recall: 0.8333\n", "\n", "Query ID: 449\n", - "Returned IDs: 999343,990366,994964,995438,990934,995200,998921,997920,996219,997371,997842,992047,995401,990448,997192,999144,990866,998401,999647,993412,997370,996727,993289,994670,995096,997320,993237,993253,998779,995927\n", + "Returned IDs: 999343,990366,994964,995438,990934,995200,998921,996219,997371,997842,990448,990652,997192,999144,998401,999647,997370,996727,993289,997320,997403,993253,998779,995927,996053,997586,996001,995455,994789,996504\n", "Ground Truth: 999343,990366,994964,995438,990934,995200,998921,997920,996219,997371,997842,992047,995401,990448,990652,997192,999144,990866,998401,999647,993412,997370,996727,993289,994670,995096,997320,997403,993237,993253\n", - "Recall: 0.9333\n", + "Recall: 0.7333\n", "\n", "Query ID: 450\n", - "Returned IDs: 995417,997843,998653,995981,998828,997584,991467,992671,997976,992931,990066,996755,993696,992733,991131,993047,994894,994100,990780,992972,992119,992692,992635,995336,997256,995596,996075,996710,999883,992748\n", + "Returned IDs: 995981,998828,997584,991467,992671,997976,990066,996755,993696,992733,991131,993047,994894,994100,992972,992119,992692,991372,992635,995336,995596,996075,999883,992748,992041,992366,993484,991456,997426,991662\n", "Ground Truth: 995417,997843,998653,995981,998828,997584,991467,992671,997976,992931,990066,996755,993696,992733,991131,993047,994894,994100,990780,992972,992119,997449,992692,991372,992635,995336,997256,995596,996075,996966\n", - "Recall: 0.9000\n", + "Recall: 0.7333\n", "\n", "Query ID: 451\n", - "Returned IDs: 995855,999680,991392,997320,996247,999453,993300,999806,999639,996744,998232,990772,991984,993904,997777,996463,996001,995250,997258,995515,992238,993484,997380,997913,993848,990957,990352,992429,994081,990949\n", + "Returned IDs: 991392,997320,996247,999453,993300,999806,999639,996744,998232,990772,991984,993904,997777,996001,995250,997258,995515,992238,993484,997913,993848,990957,990352,992429,994081,990949,992384,997305,992750,991002\n", "Ground Truth: 995855,999680,991392,997320,996247,999453,993300,999806,999639,996744,998232,990772,991984,993904,997777,996463,996001,995250,997258,995515,992238,993484,997380,997913,993848,990957,990352,992429,994081,993003\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 452\n", - "Returned IDs: 991939,993419,997785,995306,995659,998868,998921,997984,996516,990089,990366,997301,997286,991479,991999,999824,998445,990512,991852,991743,997292,998726,993143,997371,997873,994942,991406,992451,996135,990560\n", + "Returned IDs: 991939,993419,997785,995306,995659,998868,998921,997984,999516,996516,990089,990366,997301,997286,991479,998445,990512,991743,997292,998726,993143,997371,994942,992451,990560,994919,998401,991172,990585,998500\n", "Ground Truth: 991939,993419,997785,995306,995659,998868,998921,997984,999516,996516,990089,990366,997301,997286,991479,991999,999824,998445,990512,991852,991743,997292,998726,993143,997371,997873,994942,991406,992451,996135\n", - "Recall: 0.9667\n", + "Recall: 0.8000\n", "\n", "Query ID: 453\n", - "Returned IDs: 997411,996368,997868,996181,996145,993751,995702,991002,995937,998265,994730,999486,998370,997723,998990,999586,991585,990974,990412,996379,996959,999977,999537,994826,991157,995633,990132,994895,995679,995095\n", + "Returned IDs: 997411,996368,997868,993751,991002,998265,994730,999486,998370,998990,999586,991585,990974,990412,996379,999977,995633,999639,990132,994895,995679,997554,997822,994964,996160,997371,997320,994380,992423,994409\n", "Ground Truth: 997411,996368,997868,996181,996145,993751,995702,991002,995937,998265,994730,999486,998370,997723,998990,999586,991585,990974,990412,996379,996959,999801,999977,999537,994826,991157,995633,999639,990132,994895\n", - "Recall: 0.9333\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 454\n", + "Returned IDs: 996328,994273,995315,994327,996633,996495,997193,995122,998067,999225,999528,994998,993167,992726,993466,998864,993825,997026,997847,999980,993308,994113,993936,993328,998518,993127,992576,990887,996170,996194\n", + "Ground Truth: 996328,994273,995315,994327,996633,996495,997193,993089,995122,999391,998067,999225,999528,994998,993167,992726,993466,998864,992091,990987,993825,997026,999980,997847,993308,994113,993936,993328,996277,998518\n", + "Recall: 0.8333\n", "\n", "Query ID: 455\n", - "Returned IDs: 991925,996328,996506,991807,990776,997309,996318,998021,995268,990987,992726,996495,993121,994119,995080,992052,997208,995375,994895,999259,995098,991712,993180,997184,996001,997675,998566,995438,997732,992558\n", + "Returned IDs: 991925,996328,996506,991807,990776,997309,996318,995268,992726,996495,993121,994119,995080,992052,997380,997208,994895,994409,999259,995098,991712,993180,990887,996001,997675,998566,992558,996429,991747,999581\n", "Ground Truth: 991925,996328,996506,991807,990776,997309,996318,998021,995268,990987,992726,991743,996495,993121,994119,995080,992052,997380,997208,995375,994895,994409,999259,995098,991712,993180,990887,997184,996001,997675\n", - "Recall: 0.8667\n", + "Recall: 0.8333\n", "\n", "Query ID: 456\n", - "Returned IDs: 995080,990043,999784,999516,992318,995442,997918,993052,996489,996193,995119,997155,997320,995375,998726,995336,992031,997301,999453,995641,994972,991816,994162,991348,991440,990882,996840,991609,995515,993260\n", + "Returned IDs: \n", "Ground Truth: 995080,990043,999784,999516,992318,995442,997918,993052,996489,996193,995119,996829,997155,997320,995375,998726,995336,992031,995246,999741,997301,996887,999453,995641,994972,991816,994162,993528,991348,991440\n", - "Recall: 0.8333\n", + "Recall: 0.0000\n", "\n", "Query ID: 457\n", - "Returned IDs: 997024,995372,997470,997320,992789,994618,995841,998951,996143,998445,999109,990936,992451,992978,991362,991498,995422,990191,993241,991650,991999,992228,993788,990275,993853,993579,998764,998195,997984,998921\n", + "Returned IDs: 997024,995372,997470,997320,994618,995841,998951,999109,992451,992978,991498,990191,993241,991650,991999,992228,993788,990275,993853,993579,998764,998195,999929,997984,998921,991545,999317,996685,990957,992942\n", "Ground Truth: 997024,995372,997470,997320,992789,994618,995841,998951,996143,998445,999109,990936,992451,992978,991362,991498,995422,990191,993241,991650,991999,992228,993788,990275,993853,993579,998764,998195,999929,997984\n", - "Recall: 0.9667\n", + "Recall: 0.8000\n", "\n", "Query ID: 458\n", - "Returned IDs: 990854,991935,992450,995571,998671,996328,991731,994142,998251,990931,992713,995083,991769,998055,991521,999126,995861,999441,999775,996431,996495,992667,990593,997488,990887,991257,990712,990956,990889,996728\n", + "Returned IDs: 990854,991935,992450,995571,998671,996328,994142,998251,990931,992713,995083,991769,998055,999126,995861,999775,996431,996495,990593,997488,990887,991257,990712,990956,996728,991062,999830,992832,998691,994891\n", "Ground Truth: 990854,991935,992450,995571,998671,996328,991731,994142,998251,990931,992713,995083,991769,998055,991521,999126,995861,999441,999775,996431,996495,992667,990593,997488,990887,991257,990712,990956,990889,991062\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 459\n", - "Returned IDs: 992604,997110,991052,994753,995373,998318,996488,991844,995807,991309,994643,998383,998707,991110,992387,991987,999834,997675,999136,998858,995105,999478,994677,995449,992558,991571,990915,990729,990657,993433\n", + "Returned IDs: 992604,997110,991052,994753,995373,998318,996488,991844,995807,991309,994643,998707,991110,992387,991987,999834,997675,999136,998858,995105,999478,995963,994677,992558,990915,990729,990657,990131,999445,997466\n", "Ground Truth: 992604,997110,991052,994753,995373,998318,996488,991844,995807,991309,994643,998383,998707,991110,992387,991987,999834,997675,999136,998858,995105,999478,995963,994677,995449,992558,991571,990915,990729,990657\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", "\n", "Query ID: 460\n", - "Returned IDs: 995942,999358,997468,999263,999075,990595,995771,993364,994964,996218,990560,998211,993492,990146,994741,995865,999291,998964,990958,999374,992968,999701,998644,991593,994281,993484,996592,992401,994100,993250\n", + "Returned IDs: 995942,999358,997468,999263,999075,990595,995771,993364,994964,996218,990560,998211,993492,990146,994741,995502,999291,998964,990958,999374,992968,998644,991593,994281,993484,996592,992401,994100,993250,997301\n", "Ground Truth: 995942,999358,997468,999263,999075,990595,995771,993364,994964,996218,990560,998211,993492,990146,994741,995865,995502,999291,998964,996869,990958,999374,992968,999701,998644,991593,990886,994281,993484,996592\n", - "Recall: 0.9000\n", + "Recall: 0.8667\n", "\n", "Query ID: 461\n", - "Returned IDs: 997023,991958,997544,995211,995281,990887,992917,996987,993209,992521,993561,996719,995648,995198,998720,994690,990331,996001,998990,995401,995815,998776,994742,996777,995515,993289,996371,999587,994280,996220\n", + "Returned IDs: 997023,991958,997544,995211,990887,992917,996987,993209,992521,993561,996719,995648,994690,990331,996001,998990,995815,998776,992862,994742,995515,998778,993289,996371,994280,996220,991389,991269,993939,994835\n", "Ground Truth: 997023,991958,997544,995211,995281,990887,992917,996987,993209,992521,993561,996719,997163,995648,995198,998720,994690,995283,990331,996001,998990,995401,995815,998776,995911,992862,994742,997191,996777,991743\n", - "Recall: 0.8000\n", + "Recall: 0.6667\n", "\n", "Query ID: 462\n", - "Returned IDs: 993269,992521,999990,991969,999516,995813,999866,995283,993684,992426,993561,997554,999513,994949,991869,998361,990035,993738,997822,995177,993813,995216,990603,992448,991795,994386,998910,990845,996368,998205\n", + "Returned IDs: 993269,992521,999990,991969,999516,995813,999866,993684,992426,993561,997554,999513,994949,991869,998361,990035,993738,997822,995177,993813,995216,990603,992448,991795,998910,990845,996368,998205,996719,991076\n", "Ground Truth: 993269,992521,999990,991969,999516,995813,999866,995283,993684,992426,993561,997554,999513,994949,991869,998361,990035,993738,997216,997822,995177,993813,995216,990603,992448,991795,994386,998346,998910,990845\n", - "Recall: 0.9333\n", + "Recall: 0.8667\n", "\n", - "Query ID: 467\n", - "Returned IDs: 996406,992752,993347,995920,997506,995103,995961,993055,995600,996216,999205,999640,993993,992628,995051,993560,997151,996549,996483,999555,992867,998594,992293,993545,993610,998589,998716,999139,996082,998752\n", - "Ground Truth: 996406,992752,993347,995920,997506,995103,995961,993055,995600,996216,999205,999640,993993,992628,995051,993560,997151,996549,996483,999555,992867,998594,992293,993545,995185,993610,998589,998716,999139,996082\n", - "Recall: 0.9667\n", + "Query ID: 463\n", + "Returned IDs: 998507,999832,997070,992827,998405,998006,999179,995191,993121,997442,991723,998671,992652,991421,999933,992237,996428,994196,993501,993335,994510,996216,997160,995421,991596,994541,994869,996328,991672,994467\n", + "Ground Truth: 998507,999832,997070,992827,998405,990712,998006,999435,999179,995191,993121,997442,991723,998671,990039,992652,991421,999933,992237,996428,994196,993501,996775,993335,994510,997693,996216,997160,995421,991596\n", + "Recall: 0.8333\n", "\n", - "Query ID: 469\n", - "Returned IDs: 998697,995894,997586,991052,990096,994972,991713,997320,995536,997002,995740,999840,993280,998303,994040,991530,996062,992521,991332,997790,996918,997918,997246,995080,992571,994334,996368,995672,995560,993504\n", - "Ground Truth: 998697,995894,997586,991052,990096,994972,991713,997320,995536,997002,995740,999840,993280,998303,994040,991530,996062,992521,991332,997790,996918,997918,997246,995080,992571,994334,996368,995672,995560,999234\n", + "Query ID: 464\n", + "Returned IDs: 996101,991382,993053,996075,996001,994718,993696,994558,997442,991409,990536,998593,992551,998495,995895,993342,993444,996809,996028,993721,999570,997963,997270,990497,999430,991184,995203,993984,996013,990096\n", + "Ground Truth: 996101,991382,993053,996075,996001,994718,993696,994558,997442,991409,990536,998593,992551,998495,995895,993342,993444,996809,996028,993721,999570,997963,997270,990497,999430,991184,995203,997448,993984,996013\n", "Recall: 0.9667\n", "\n", - "Query ID: 470\n", - "Returned IDs: 999590,996001,997920,995167,992665,992862,993157,998934,996147,994531,993174,997586,990573,998593,998299,990399,997790,996075,998481,998228,996661,999216,993168,994964,995701,991212,996028,991609,997002,995536\n", - "Ground Truth: 999590,996001,997920,995167,992665,992862,993157,998934,996147,994531,993174,997586,990573,998593,998299,997555,990399,997790,996075,998481,998228,996661,999216,993168,994964,995701,991212,996028,991609,997002\n", - "Recall: 0.9667\n", + "Query ID: 465\n", + "Returned IDs: 997768,996082,997586,996001,992862,999284,998269,995455,992446,997334,999413,991628,999963,998154,998303,995823,998708,995536,996915,994907,990366,994964,998447,995535,996216,990096,991751,994672,999651,995887\n", + "Ground Truth: 997768,996082,997586,996001,992862,999284,998269,995455,992446,997334,999413,991628,999963,995256,998154,998303,995823,998708,995536,998568,996915,994907,990366,994964,998447,995535,996856,998440,997531,998734\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 466\n", + "Returned IDs: \n", + "Ground Truth: 996042,997251,990624,993453,992483,998703,991110,992052,997766,997234,991447,997420,998758,995464,990963,990776,995516,996232,991977,992558,992487,996814,992147,998968,993948,995373,997309,998221,999529,993433\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 467\n", + "Returned IDs: 996406,992752,993347,995920,995961,993055,995600,996216,999205,999640,993993,992628,995051,993560,997151,996549,996483,999555,992867,998594,992293,993610,998589,998716,999139,996082,998752,995191,995895,998017\n", + "Ground Truth: 996406,992752,993347,995920,997506,995103,995961,993055,995600,996216,999205,999640,993993,992628,995051,993560,997151,996549,996483,999555,992867,998594,992293,993545,995185,993610,998589,998716,999139,996082\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 468\n", + "Returned IDs: 990866,994163,996429,991880,993948,999860,992558,999639,993831,996196,990930,995552,998495,998697,995289,996188,999977,996442,994742,991790,994140,992024,992423,992469,996379,991550,995633,995983,996178,992121\n", + "Ground Truth: 990866,994163,996918,996429,991880,993948,999860,992558,999639,993831,996196,995873,990930,995552,998495,998697,995289,996188,997723,999977,996442,991052,994742,991790,994140,999587,993725,992024,992423,992469\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 469\n", + "Returned IDs: 998697,995894,997586,990096,994972,991713,997320,995536,997002,999840,998303,991530,996062,992521,991332,996918,997918,997246,995080,992571,996368,995672,995560,993504,990538,998568,998260,991157,993071,996001\n", + "Ground Truth: 998697,995894,997586,991052,990096,994972,991713,997320,995536,997002,995740,999840,993280,998303,994040,991530,996062,992521,991332,997790,996918,997918,997246,995080,992571,994334,996368,995672,995560,999234\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 470\n", + "Returned IDs: 999590,996001,995167,992862,993157,998934,996147,993174,997586,990573,998593,997555,990399,997790,996075,998481,998228,996661,994964,995701,991212,996028,995536,995250,999099,994925,992395,997320,998549,995216\n", + "Ground Truth: 999590,996001,997920,995167,992665,992862,993157,998934,996147,994531,993174,997586,990573,998593,998299,997555,990399,997790,996075,998481,998228,996661,999216,993168,994964,995701,991212,996028,991609,997002\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 471\n", + "Returned IDs: 998186,993813,999099,990081,990576,990845,996368,990035,994140,991969,995245,991350,990820,997868,995813,991958,996379,990930,997716,994436,996719,995016,995538,991011,990751,994396,996039,996127,991790,997622\n", + "Ground Truth: 998186,993813,999099,990081,990576,990845,996368,990035,995547,994140,991969,995245,991350,990820,997868,995813,991958,995923,996379,991024,990930,997716,994436,996719,995016,995538,991011,994386,990751,994396\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 472\n", + "Returned IDs: 990557,996318,993572,997918,996429,996001,995455,992749,999898,993885,994188,998401,997113,995203,995932,992161,993504,991212,996210,996866,995596,992047,990409,990181,992466,991498,997305,998849,992790,998495\n", + "Ground Truth: 990557,996318,993572,997918,996429,996001,995455,992749,999898,993885,994188,998401,990448,997113,995203,995932,991571,990519,992161,993504,994832,991212,996210,999715,994155,990772,991311,996866,997395,995596\n", + "Recall: 0.7000\n", "\n", "Query ID: 473\n", - "Returned IDs: 999691,991081,998864,998845,997019,990072,993366,996446,999349,993715,996106,998934,999273,995636,992752,994587,995258,993055,997039,998314,995349,991629,999498,992664,998959,999663,995270,994096,995736,995299\n", + "Returned IDs: 999691,991081,998864,998845,990072,993366,996446,999349,993715,996106,998934,999273,992752,994587,993055,997039,998314,995349,991629,999498,992664,998959,992975,995270,991170,995736,995299,999139,993502,994680\n", "Ground Truth: 999691,991081,998864,998845,997019,990072,993366,996446,999349,993715,996106,998934,999273,995636,992752,994587,995258,993055,997039,998314,995349,991629,999498,992664,998959,999663,992975,995270,994096,991170\n", - "Recall: 0.9333\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 474\n", + "Returned IDs: 992398,995203,996429,996001,995932,994087,993300,998734,998568,996075,993918,995536,994221,999840,995358,990196,999898,999473,993260,997555,997894,998123,994972,999210,990409,990096,997977,999731,994789,997305\n", + "Ground Truth: 992398,995203,996429,996001,995932,994087,993300,998734,998568,996075,992484,993918,995536,994221,999840,995855,995358,997140,994042,990196,994832,999898,999473,993260,997555,999963,996764,994531,997894,998123\n", + "Recall: 0.7333\n", "\n", "Query ID: 475\n", - "Returned IDs: 999384,990096,998495,996809,997918,993696,997906,998950,997320,998383,990210,991571,996805,992944,993018,995438,993260,993215,993154,996623,999210,993744,994661,999390,991801,993504,999009,991743,990843,999320\n", + "Returned IDs: 999384,990096,998495,996809,997918,993696,997906,998950,997320,991571,996805,992944,993018,998861,995438,993260,993215,993154,996623,999210,994661,999390,991801,993504,999009,990843,999320,994964,999831,996370\n", "Ground Truth: 999384,990096,998495,996809,997918,993696,997906,998950,997320,998383,990210,991571,996805,992944,993018,998861,995438,993260,993154,993215,996623,999210,993744,994661,999390,991801,993504,999009,991743,990843\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 476\n", + "Returned IDs: 991819,992020,994922,994789,996402,995004,998837,997370,990155,995791,993570,990379,995515,998492,998734,990652,990229,998551,990300,995759,990334,999076,994647,990448,993477,992138,998936,990310,991999,999053\n", + "Ground Truth: 991819,992020,994922,994789,996402,995004,998837,997370,990155,995791,993570,990379,995515,998492,998734,990652,990229,999888,998551,990300,995759,990334,999076,994647,990448,993477,992138,998936,990310,991999\n", "Recall: 0.9667\n", "\n", "Query ID: 477\n", - "Returned IDs: 990866,997364,993168,997586,994782,996232,993642,993831,994964,997732,998107,997493,997279,998668,993878,995906,993689,995482,991325,998369,997268,995107,993630,998837,992862,997305,997091,991595,995944,997004\n", + "Returned IDs: 990866,997364,993168,997586,994782,996232,993642,993831,994964,997732,998107,998668,993878,995906,993689,995482,997268,995107,992862,990934,997305,995944,997004,998458,992052,993172,999676,995536,991999,990096\n", "Ground Truth: 990866,997364,993168,997586,996352,994782,996232,993642,993831,994964,997732,998107,997493,997279,998668,993878,990690,995906,993689,995482,991325,998369,996647,997268,995107,993630,998837,992862,990934,997305\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 478\n", + "Returned IDs: 994925,998228,999729,998039,990845,991789,990820,993906,990856,991233,991958,996873,996001,996368,990811,996298,998593,991685,995923,991200,993637,996237,995899,990930,992408,996062,998549,991125,995679,992862\n", + "Ground Truth: 994925,998228,999729,998534,998039,990845,991789,990820,993906,990856,991233,991958,996873,996001,996368,990811,996298,998593,991685,995923,991200,999590,993637,996237,997920,999009,995899,990930,992408,996062\n", "Recall: 0.8667\n", "\n", + "Query ID: 479\n", + "Returned IDs: 998837,995588,991172,999784,991840,990142,999898,997135,999077,991999,993633,997238,990822,993282,994922,990366,997747,999613,990432,994964,996402,993708,992907,997275,990615,994789,990310,998582,993748,991485\n", + "Ground Truth: 998837,995588,991172,996352,999784,991840,990142,999898,997135,999077,991999,993633,997238,990822,993282,994922,990366,997747,999613,990432,994964,996402,993708,992907,997275,990615,994789,990310,998582,993748\n", + "Recall: 0.9667\n", + "\n", "Query ID: 480\n", - "Returned IDs: 994964,997913,992546,994100,995981,995942,997320,997470,999444,994778,990231,992330,999109,995560,990588,996219,991721,999374,999740,997301,993168,998540,993484,999328,998495,998797,999900,990925,996075,996809\n", + "Returned IDs: 994964,997913,992546,994100,995981,995942,997320,997470,999444,994778,990231,992330,996219,991721,999374,999740,997301,993168,998540,993484,999328,998495,998797,990925,996075,996809,999710,990957,997614,992443\n", "Ground Truth: 994964,997913,992546,994100,995981,995942,997320,997470,999444,994778,990231,992330,999109,995560,990588,996219,991721,999374,999740,997301,993168,998540,993484,999328,998495,998797,999900,990925,998163,996075\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 481\n", - "Returned IDs: 999860,992558,996609,996196,996523,993839,997002,997436,991334,992411,992161,993583,996606,995618,994418,996389,997003,991977,990701,997864,999240,999715,996429,996127,999540,996882,997457,997914,999374,990249\n", + "Returned IDs: 999860,992558,996609,996196,996523,993839,997002,991334,992411,992161,993583,996606,995618,994418,996389,995482,991977,998679,997864,999715,996127,999540,996882,997914,990249,990061,995332,994280,999711,991426\n", "Ground Truth: 999860,992558,996609,996196,996523,993839,997002,997436,991334,992411,992161,993583,996606,995618,994418,996389,997003,995482,991977,990701,998679,997864,999240,999715,996429,996127,999540,996882,997457,997914\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 482\n", + "Returned IDs: 998559,990647,996291,998779,997192,990399,991630,996727,997918,992080,995886,992330,995401,995336,995373,998921,991743,994334,998536,997571,998697,998259,993696,992384,995453,994757,995701,993143,993443,998401\n", + "Ground Truth: 998559,990647,996291,998779,998407,995374,997192,990399,991630,996727,997918,992080,995886,992330,995401,995336,995373,998921,991743,994334,998536,997571,998697,998259,993696,992384,995453,994757,995701,999450\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 483\n", + "Returned IDs: 998950,993018,999019,994171,997320,999701,996626,990083,990532,992944,996210,990843,991951,995560,993686,993080,991142,994081,991503,993696,995372,996024,996075,996726,994280,991348,992521,991224,991372,990375\n", + "Ground Truth: 998950,993018,999019,994171,997320,999701,996626,990083,990532,992944,996210,990843,991951,995560,993686,993080,998567,992318,991142,994081,991503,993696,995372,996024,996075,996726,994280,991348,992521,991224\n", "Recall: 0.9333\n", "\n", + "Query ID: 484\n", + "Returned IDs: 995633,995289,991463,993725,991989,995401,990033,998106,999739,995766,990392,992160,997822,993813,992595,994419,996867,993762,997155,999665,993561,991585,994742,998536,997141,994057,998070,992289,996001,992558\n", + "Ground Truth: 995633,995289,991463,993725,997682,997319,991989,995401,990562,990033,998106,999739,995766,990392,998574,992160,997822,993813,992595,994419,996867,993762,997155,999665,999052,993412,990930,993561,991585,996254\n", + "Recall: 0.7333\n", + "\n", "Query ID: 485\n", - "Returned IDs: 991610,992778,997009,990293,999555,999994,992136,992721,990167,995257,993076,991667,990177,992862,994374,998383,996408,991513,991987,998586,999430,994796,997506,990437,997752,991304,991569,996633,997655,995536\n", + "Returned IDs: 991610,992778,997009,990293,999555,999994,992136,992721,990167,995257,993076,991667,990177,992862,994374,996408,991513,991987,998586,997506,997752,993993,999216,997655,995536,998790,997328,993948,997666,996328\n", "Ground Truth: 991610,992778,997009,990293,999555,999994,992136,992721,990167,995257,993076,991667,990177,992862,994374,998383,996408,991513,991987,998586,999430,994796,997506,990437,997752,991304,991569,993993,999216,996633\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 486\n", + "Returned IDs: 998913,993696,992469,998950,994742,992661,994163,998790,997117,992289,999235,996809,991157,990340,992124,993809,996178,996609,996196,997320,992558,999639,992423,998662,999676,993355,994964,990866,998196,998827\n", + "Ground Truth: 998913,993696,992469,993529,998950,994742,992661,994163,998790,997117,991445,992289,999235,996809,991157,990340,992124,993809,990930,996178,996609,995873,996196,997320,992558,994317,991559,999639,994529,992423\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 487\n", + "Returned IDs: 991858,998906,990033,999450,991603,999037,996048,998045,993144,990940,991463,992519,994409,992125,995444,996388,990984,991958,991747,993813,998165,994140,998999,996719,994057,995702,999665,995052,995289,996482\n", + "Ground Truth: 991858,998906,990033,999450,991603,999037,996048,998045,993144,990940,991463,992519,994409,992125,995444,996388,990984,991958,991747,993813,995864,998165,994140,998999,997187,996719,994057,995702,999665,995052\n", "Recall: 0.9333\n", "\n", "Query ID: 488\n", @@ -6484,1130 +7569,1880 @@ "Ground Truth: 997270,990039,996475,998794,996650,997047,992780,990377,992100,993479,995750,999430,999323,996013,996830,999659,997939,993890,999551,990718,991959,993391,992872,998599,992754,993721,991382,999722,994558,997331\n", "Recall: 0.9667\n", "\n", + "Query ID: 489\n", + "Returned IDs: 999946,997476,990043,996093,997135,992169,992907,995791,999204,993570,990483,992612,996438,994922,993237,994144,999453,992632,990580,991146,994703,995489,992994,998734,998523,991473,995004,998837,993355,992813\n", + "Ground Truth: 999946,997476,990043,996093,993675,997135,992169,992907,995791,999204,993570,990483,992612,996438,994922,993237,994144,999453,992632,990580,994827,993238,991146,994703,995489,992994,992342,998734,994629,998523\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 490\n", + "Returned IDs: 994384,996140,991550,991817,990279,994732,998697,997739,996380,995983,992558,991458,995552,992661,998950,994341,996926,994171,994163,993686,990068,991791,990383,996196,991713,998827,997258,990456,990343,993018\n", + "Ground Truth: 994384,996140,991550,991817,990279,994732,998697,997739,996380,995983,992558,991458,995552,992661,998950,994341,996926,994171,994163,993686,990068,996894,991791,990383,995372,996196,998567,991713,998827,990769\n", + "Recall: 0.8667\n", + "\n", "Query ID: 491\n", - "Returned IDs: 990096,998968,998419,993624,994636,992716,996386,991936,999103,991167,997884,999711,999119,996209,999282,998861,990072,990154,999287,995536,997803,991451,999319,993055,993744,999443,991724,996283,999384,996893\n", + "Returned IDs: 990096,998968,993624,994636,992716,996386,999103,997884,999711,999119,996209,999282,998861,990072,990154,999287,995536,997803,991451,999319,993055,993744,999443,991724,996283,999384,996893,999744,994834,992068\n", "Ground Truth: 990096,998968,998419,993624,994636,992716,996386,991936,999103,991167,997884,999711,999119,996209,999282,998861,990072,990154,999287,995536,997803,991451,999319,993055,999443,993744,991724,996283,996429,999384\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 492\n", + "Returned IDs: 996284,997182,996809,996830,994465,994382,991382,997318,990167,995895,991881,992551,999555,994927,990361,996706,993391,993890,998495,990145,992752,990226,994431,996580,992780,995978,998006,996494,991767,998895\n", + "Ground Truth: 996284,997182,996809,996830,994465,994382,991382,997318,990167,995895,991881,992551,999555,994927,990361,996706,993391,993890,998495,990145,992752,990226,994431,996580,992780,999886,994355,995978,998006,996494\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 493\n", + "Returned IDs: 993283,991485,992309,999639,994042,991157,995562,990399,995516,998232,995443,991015,999806,998180,992124,996744,994809,998697,993260,999041,996368,991984,993055,993289,996657,992862,992905,991002,996138,998758\n", + "Ground Truth: 993283,991485,992309,999639,994042,991157,995562,990399,995516,998232,995443,991015,999806,998180,992124,996744,994809,998697,993260,999041,996368,991984,993055,994922,993289,996657,992147,992862,992905,991002\n", + "Recall: 0.9333\n", "\n", "Query ID: 494\n", - "Returned IDs: 992169,996325,997852,992152,997476,992632,992813,992309,994922,993717,996093,990490,999676,990266,994318,999843,999453,994761,996438,992612,990399,997822,994148,999077,995210,997797,998171,994956,990268,991334\n", + "Returned IDs: 992169,996325,997852,992152,997476,992632,992813,994922,993717,996093,999676,990266,994318,999453,994761,996438,992612,997822,994148,999077,997797,994956,990268,991334,998999,998765,993080,998138,998523,990080\n", "Ground Truth: 992169,996325,997852,992152,997476,992632,992813,992309,994922,993717,996093,990490,999676,990266,994318,999843,999453,994761,996438,992612,997645,990399,997822,994148,994161,999077,995210,997797,998171,994956\n", - "Recall: 0.9333\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 495\n", + "Returned IDs: 994960,991383,992164,999138,996537,990089,996011,998812,992026,996037,996301,992827,996328,990730,992372,998348,993335,993715,991390,991885,999227,995191,998076,997357,998671,992832,993568,992237,999430,993413\n", + "Ground Truth: 994960,991383,992164,999138,996537,990089,996011,998812,992026,996037,996301,992827,996328,990730,992372,998348,993335,993715,991390,991885,999227,995191,998076,997357,998671,992832,993568,992237,999430,990040\n", + "Recall: 0.9667\n", "\n", "Query ID: 496\n", - "Returned IDs: 992124,991984,995516,998697,994742,999136,995701,998758,993186,993727,995672,995562,993470,992747,993834,996075,994374,993342,996657,999041,991530,994082,991052,994809,999783,994879,998586,998950,995203,995981\n", + "Returned IDs: 992124,991984,995516,998697,994742,999136,995701,998758,993186,993727,995672,995562,992747,993834,996075,993342,996657,991530,994082,994809,999783,994879,998950,995203,995981,995119,994280,994325,996884,992480\n", "Ground Truth: 992124,991984,995516,998697,994742,999136,995701,998758,993186,993727,995672,995562,993470,992747,993834,996075,994374,993342,996657,999041,991530,994082,991052,994809,999783,997377,994879,998586,998950,995203\n", - "Recall: 0.9667\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 497\n", + "Returned IDs: 997969,993765,993642,997732,999151,994964,990226,996216,993929,990091,994927,993080,998436,998107,998495,998416,997410,997004,993906,991633,997453,990690,996647,990632,997586,990845,992820,993831,996171,998950\n", + "Ground Truth: 997969,993765,993642,997732,999151,994964,997111,990226,996216,995920,993929,990091,994927,990441,993080,998436,998107,998495,998416,997410,997004,993906,991633,997453,997112,990690,996647,990632,991239,990580\n", + "Recall: 0.8000\n", "\n", "Query ID: 498\n", - "Returned IDs: 999899,994906,996759,992648,992651,999977,998083,990304,991175,998301,992979,993713,992126,993275,996368,997059,999820,996714,996425,993032,996075,997554,993280,994860,997910,999587,999453,994334,994946,997002\n", + "Returned IDs: \n", "Ground Truth: 999899,994906,996759,992648,992651,999977,998083,990304,990260,991175,998301,992979,993713,992126,993275,996368,997059,999820,996714,996425,993032,996075,997554,993280,994860,997910,999587,999453,994334,994946\n", - "Recall: 0.9667\n", + "Recall: 0.0000\n", "\n", "Query ID: 499\n", - "Returned IDs: 998080,991334,999479,998820,991025,995190,993473,993931,997395,994222,990126,998493,994204,995612,991449,991743,990733,996609,993929,997212,992685,995401,999098,990249,992604,997515,995332,995289,997723,999788\n", + "Returned IDs: 998080,991334,999479,998820,991025,995190,993931,997395,994222,990126,998493,994204,991449,991743,990733,996609,992685,995401,999098,990249,992604,997515,995289,999788,990131,994431,999961,999533,993566,997209\n", "Ground Truth: 998080,991334,999479,998820,991025,995190,993473,993931,997395,994222,990126,997030,998493,999586,994204,995612,991449,991743,990733,996609,993929,997212,992685,995401,999098,990249,992604,997515,995332,995289\n", - "Recall: 0.9333\n", + "Recall: 0.7667\n", "\n", "Query ID: 500\n", - "Returned IDs: 999009,991776,995895,990096,993397,997586,990930,990786,999430,995455,994742,996624,990866,990701,990271,997028,996830,997768,999599,991154,995701,996216,998495,996192,995435,999639,991572,996001,999647,997378\n", + "Returned IDs: 999009,991776,995895,990096,993397,997586,990930,990786,999430,995455,994742,996624,990866,990701,990271,997028,996830,997768,999599,991154,995701,996216,998495,996192,995435,999639,995899,996001,997378,999261\n", "Ground Truth: 999009,991776,995895,990096,993397,997586,990930,990786,999430,995455,994742,996624,990866,990701,990271,997028,996830,997768,999599,991154,995701,996216,998495,996192,995435,999639,995899,991572,996001,999647\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 501\n", - "Returned IDs: 998158,997453,994463,992488,996062,998874,992322,990930,996915,991021,994783,995289,999139,991154,993561,993673,996071,990076,990104,991685,990226,995113,999091,995783,997797,996028,991857,992703,995586,992182\n", + "Returned IDs: 998158,997453,994463,992488,996062,998874,992322,990930,996915,991021,994783,995289,999139,993561,990076,990104,991685,990226,999091,992326,995783,996028,991857,992703,992182,990899,995126,991633,998302,998593\n", "Ground Truth: 998158,997453,994463,992488,996062,998874,992322,990930,996915,991021,994783,995289,999139,991154,994777,993561,993673,996071,990076,990104,991685,990226,995113,999091,992326,995783,997797,996028,991857,992703\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", "\n", "Query ID: 503\n", - "Returned IDs: 994894,998353,999599,995981,994100,997913,995942,992228,997946,999243,999658,991045,990701,997301,994266,998090,997728,998246,999374,996219,996805,993186,992384,997320,999331,997614,997304,999444,992848,993529\n", + "Returned IDs: 994894,998353,995981,994100,997913,995942,992228,997946,999243,991045,997301,994266,998090,997728,998246,999374,996219,996805,993186,992384,997320,999331,997614,997304,999444,992848,993529,992124,998417,995701\n", "Ground Truth: 994894,998353,999599,995981,994100,997913,995942,992228,997946,999243,999658,991045,990701,997301,994266,998090,997728,998246,999374,996219,991468,996805,993186,992384,997320,999331,997614,997304,999444,992848\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 504\n", - "Returned IDs: 990532,999453,995852,994081,995130,994955,993053,995295,996147,992408,994042,995203,995456,990392,990771,991609,999052,998039,997002,999639,996918,997918,993809,998855,993906,994226,996368,992832,992562,996001\n", + "Returned IDs: 990532,999453,995852,994081,994955,993053,995295,996147,992408,994042,995203,995456,990392,990771,991609,999052,998039,997002,999639,996918,997918,993809,998855,993906,996368,996001,992944,998536,996879,993157\n", "Ground Truth: 990532,999453,995852,994081,995130,994955,993053,995295,996147,992408,994042,995203,995456,990771,990392,991609,999052,998039,997002,999639,996918,995401,997918,993809,998855,993906,994226,996368,992832,992562\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 505\n", - "Returned IDs: 991698,997395,992790,990215,994671,991404,993409,992832,995375,995080,991607,994855,997515,990889,995685,992576,991576,995648,996245,992749,999052,990930,997391,998536,993412,995968,998158,997192,992036,995994\n", + "Returned IDs: 991698,997395,993407,992790,990215,994671,991404,993409,992832,995375,995080,991607,997515,990889,995685,992576,995648,996245,992749,999052,990930,998536,995968,998158,997192,992036,994280,999967,996918,994271\n", "Ground Truth: 991698,997395,993407,992790,990215,994671,991404,993409,992832,995375,995080,991607,994855,997515,990889,995685,992576,991576,995648,996245,992749,999052,990930,997391,998536,993412,995968,998158,997192,992036\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 507\n", - "Returned IDs: 994472,999791,991921,990089,992322,994402,990091,995222,992820,995720,998107,997939,999876,998765,996355,990622,998355,990226,999642,998630,995535,998594,996761,993775,993584,997349,995638,993929,991325,991751\n", + "Returned IDs: 994472,999791,991921,990089,992322,994402,990091,992820,995720,998107,997939,999876,996355,998355,990226,998630,995535,996761,993775,993584,997349,995638,993929,991751,998169,993378,991382,993637,994927,999294\n", "Ground Truth: 994472,999791,991921,990089,992322,994402,990091,995222,992820,995720,998107,997939,999876,998765,996355,990622,996050,998355,990226,999642,998630,995535,998594,996761,993775,993584,997349,995638,993929,991325\n", - "Recall: 0.9667\n", + "Recall: 0.7667\n", "\n", "Query ID: 508\n", - "Returned IDs: 992582,991743,990949,991782,990958,995274,995374,999533,993961,997614,994850,990490,994266,995949,995981,997645,991479,996648,999331,990957,997913,990588,990359,990493,991889,993431,999343,990910,998294,993412\n", + "Returned IDs: 992582,991743,990949,991782,990958,995274,999533,997614,994850,994266,995949,995981,999331,990957,997913,990493,991889,999343,990910,998921,998779,998302,997301,994919,997320,991984,995515,994894,997571,995886\n", "Ground Truth: 992582,991743,990949,991782,990958,995274,995374,999533,993961,997614,994850,990490,994266,995949,995981,997645,991479,996648,999331,990957,997913,990588,990359,990493,991889,999328,993431,999343,990910,998294\n", - "Recall: 0.9667\n", + "Recall: 0.6333\n", "\n", "Query ID: 509\n", - "Returned IDs: 998982,990328,992226,990528,990701,992068,998671,996939,992230,991334,990468,990711,998719,994751,999500,994783,994689,999916,995295,991771,997379,997586,990126,996116,990011,990864,991075,991464,996645,994558\n", + "Returned IDs: 998982,990328,992226,990528,990701,998671,992230,991334,990711,998719,994751,999500,994783,994689,995990,995295,997586,990126,996116,990864,991075,991464,996645,994558,997485,999430,998543,995020,993656,996481\n", "Ground Truth: 998982,990328,992226,990528,990701,996110,992068,998671,996939,992230,991334,990468,990711,998719,994751,999500,994783,994689,991239,999916,995990,995295,991771,997379,997586,990126,996116,990011,990864,991075\n", - "Recall: 0.9000\n", + "Recall: 0.7000\n", "\n", "Query ID: 510\n", - "Returned IDs: 997913,990650,991593,998500,994964,997301,994831,997320,998170,999982,997192,992978,994618,990066,995942,999948,999167,992087,998899,995802,996219,991335,997470,999647,996504,998550,995961,996053,990210,995949\n", + "Returned IDs: 997913,990650,991593,998500,994964,997301,997320,998170,999982,997192,992978,990066,999948,999167,992087,998899,995802,996219,991335,997470,998808,999647,996504,998550,995961,990210,995949,998302,998921,991479\n", "Ground Truth: 997913,990650,991593,998500,994964,997301,994831,997320,998170,999982,997192,992978,994618,990066,995942,999948,999167,992087,998899,995802,996219,991335,997470,998808,999647,996504,998550,995961,996053,990210\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 511\n", - "Returned IDs: 993387,994192,998899,992464,992119,997913,996592,990957,991479,990890,997320,993662,995336,994978,999658,996296,996075,993853,992330,999559,998550,998090,991131,994432,999948,990545,995826,992969,992255,991379\n", + "Returned IDs: \n", "Ground Truth: 993387,994192,998899,996946,992464,992119,997913,996592,990957,991479,990890,997320,993662,992134,993174,995336,994978,999658,996296,996075,993853,992330,991260,999559,998550,998090,991131,994432,999948,996397\n", - "Recall: 0.8333\n", + "Recall: 0.0000\n", "\n", "Query ID: 512\n", - "Returned IDs: 997168,996277,999281,991245,993275,999526,992315,994743,993715,990409,991685,992636,996001,998623,999724,990956,991349,991037,998536,993560,994016,990951,998045,995701,993561,995562,995835,993738,996677,995408\n", + "Returned IDs: 997168,996277,999281,991245,993275,999526,992315,994743,993715,990409,991685,992636,996001,998623,999724,990956,991349,991037,998536,993560,994016,990951,998045,995701,993561,995562,993738,994409,993948,994353\n", "Ground Truth: 997168,996277,999281,991245,993275,999526,992315,994743,993715,990409,991685,992636,996001,998623,999724,990956,991349,991037,998536,993560,994016,990951,998045,995701,993561,995562,991755,995835,993738,994409\n", "Recall: 0.9333\n", "\n", "Query ID: 513\n", - "Returned IDs: 991375,993906,996216,996082,997790,995991,994925,996062,998950,996599,996178,995107,991769,995895,996001,995961,999827,990786,990364,999009,993561,990886,992780,995705,994280,993931,991142,995586,992847,990096\n", + "Returned IDs: 991375,993906,996216,996082,997790,995991,994925,996062,998950,996599,996178,995107,991769,995895,996001,999827,990364,993561,992780,995705,994280,993931,991142,992847,990096,994742,992322,992680,990930,996196\n", "Ground Truth: 991375,993906,996216,996082,997790,995991,994925,996062,998950,996599,996178,995107,991769,995895,992473,996001,995961,999827,990786,990364,999009,993561,990886,992780,995705,994280,993931,990887,991142,995586\n", - "Recall: 0.9333\n", + "Recall: 0.7667\n", "\n", "Query ID: 514\n", - "Returned IDs: 991396,994160,990925,998797,990595,997972,999262,996959,995884,994964,997102,993914,995588,996532,990146,998501,993168,997040,999600,994673,993389,997320,993761,997189,990863,991008,993083,991086,990797,992284\n", + "Returned IDs: 991396,990925,998797,990595,997972,999262,996959,995884,994964,997102,993914,996532,990146,998501,997040,999600,994673,993389,997320,993761,990863,991008,993083,991086,990797,992284,992147,996714,999444,993484\n", "Ground Truth: 991396,994160,990925,998797,990595,997972,999262,996959,995884,994964,997102,993914,995588,996532,990146,998501,993168,997040,997028,999600,994673,993389,997320,995909,993761,997189,990863,991008,993083,991086\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", "\n", "Query ID: 515\n", - "Returned IDs: 996379,991255,995268,990930,995963,993561,992367,991747,991603,996913,991459,997925,990984,993673,992703,996198,996277,992294,994589,992115,995701,990133,996867,990899,994113,995289,991011,998779,996328,996918\n", + "Returned IDs: 996379,991255,995268,990930,993561,992367,997925,990984,993673,992703,996198,996277,992294,994589,992115,995701,990133,996867,990899,994113,995289,991011,998779,996328,996918,999977,990940,990033,996930,996678\n", "Ground Truth: 996379,991255,995268,990930,995963,993561,992367,991747,991603,996913,991459,997925,990984,993673,992703,996198,996277,992294,992660,994589,990897,992115,995701,990133,996867,990899,994113,995289,991011,998779\n", - "Recall: 0.9333\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 516\n", + "Returned IDs: 996187,997868,996605,998941,991349,991795,998527,995814,998255,990732,999121,999926,994409,998205,991878,997998,998796,990035,999731,991163,992858,991969,996618,992311,998143,995813,994638,994773,994140,995856\n", + "Ground Truth: 996187,997868,996605,998941,991349,991795,998527,995814,998255,990732,999121,999926,994409,998205,991878,997998,998796,990035,999731,991163,992858,991969,996618,992311,998143,995813,994638,994773,990481,994140\n", + "Recall: 0.9667\n", "\n", "Query ID: 517\n", - "Returned IDs: 996038,998832,995507,997723,995139,997046,996323,995905,999619,997406,996867,992545,993433,999000,998165,993421,999530,995289,992130,997832,996769,997020,996112,991027,993390,996166,996815,995753,992605,995052\n", + "Returned IDs: 996038,998832,995507,997723,995139,997046,996323,995905,999619,997406,996867,992545,993433,999000,998165,993421,999530,995289,992130,997832,996769,997020,996112,991027,993390,995753,992605,995052,997607,996968\n", "Ground Truth: 996038,998832,995507,997723,995139,997046,996323,995905,999619,997406,996867,992545,993433,999000,998165,993421,999530,995289,992130,997832,996769,997020,996112,991027,993390,996166,992660,996815,995753,992605\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 518\n", + "Returned IDs: 993501,999179,997733,998671,992193,992652,996803,996211,999832,990784,993931,997068,992827,994929,992237,997967,992981,998870,996809,997126,991672,994402,990186,990852,994119,993335,996328,994541,993929,997331\n", + "Ground Truth: 993501,999179,997733,998671,992193,992652,996803,996211,999832,990784,993931,997068,992827,994929,993439,992237,997967,992981,994094,998870,996809,997126,995790,991672,994402,990186,990852,994119,993335,996328\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 519\n", + "Returned IDs: 992315,991037,990596,990834,996562,994302,995562,994123,995349,996286,991648,995500,990630,998358,998052,996908,999435,995740,994955,990203,995366,998855,992199,996884,998303,993504,991212,996001,990409,994743\n", + "Ground Truth: 992315,991037,990596,990834,996562,994302,995562,994123,995349,996286,991648,995500,990630,998358,998052,996908,996677,999435,995740,994955,990203,995366,998855,992199,996884,998303,991956,993504,991212,996001\n", + "Recall: 0.9333\n", "\n", "Query ID: 520\n", - "Returned IDs: 992692,997320,997918,993512,990171,992521,998260,995336,999731,993071,994964,993492,994360,999516,996397,994081,998365,998246,994290,995826,992124,996417,995274,997977,992330,992909,991883,991880,992047,998568\n", + "Returned IDs: 997320,997918,993512,992521,998260,995336,999731,993071,994964,993492,994360,999516,996397,994081,998365,994290,995826,992124,995274,997977,992330,992909,991880,998568,992423,995560,996085,994577,998950,998550\n", "Ground Truth: 992692,997320,997918,993512,990171,992521,998260,995336,999731,993071,994964,993492,994360,999516,996397,994661,994081,998365,998246,994290,995826,992124,996417,995274,997977,992330,992909,991883,991880,992047\n", - "Recall: 0.9667\n", + "Recall: 0.7667\n", "\n", "Query ID: 521\n", - "Returned IDs: 999453,994081,990661,990532,996075,997370,990652,998567,998492,998734,996309,990882,995515,997320,996626,992521,995372,991713,993280,995203,991869,994789,990083,999954,996368,995401,992031,995245,998607,996966\n", + "Returned IDs: 999453,994081,990661,990532,996075,997370,990652,998567,998492,998734,996309,990882,995515,997320,996626,992521,995372,993280,995203,991869,994789,990083,996368,995245,993241,994449,990043,996001,992617,991348\n", "Ground Truth: 999453,994081,990661,990532,996075,997370,990652,998567,998492,998734,996309,990882,995515,997320,996626,992521,996193,995372,991713,993280,995203,991869,999123,994789,990083,999954,996368,995401,992031,995245\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 522\n", + "Returned IDs: 997822,993433,995289,995633,996042,998106,996494,996867,998165,992160,995401,997046,993390,997766,995040,990252,995200,994742,992558,993053,996769,992521,994783,995052,990435,994529,992448,997961,991301,992576\n", + "Ground Truth: 997822,993433,995289,995633,996042,998106,997797,993857,996494,996867,998165,992160,995401,995873,997046,993390,997918,997766,995040,990252,995200,994742,992558,993053,996769,992521,994783,996028,995052,990435\n", + "Recall: 0.8333\n", "\n", "Query ID: 523\n", - "Returned IDs: 998246,992896,990066,997320,993484,994225,996560,998540,999102,998634,999128,995904,995224,999005,992986,995400,994542,992839,998384,997610,995841,998603,990269,996598,997178,994164,994214,991999,994964,992054\n", + "Returned IDs: 998246,992896,990066,997320,993484,994225,996560,998540,998634,999005,995400,992839,997102,998384,997610,995841,998603,996598,997178,994164,994214,991999,994964,991016,994183,991281,994611,999810,994873,994632\n", "Ground Truth: 998246,992896,990066,997320,993484,994225,996560,999757,998540,999102,998634,999128,995904,995224,999005,992986,995400,994542,992839,991567,997102,998384,997610,995841,998603,990269,996598,997178,994164,994214\n", - "Recall: 0.9000\n", + "Recall: 0.7000\n", "\n", "Query ID: 524\n", - "Returned IDs: 990882,994612,997465,999698,998937,997403,993845,998607,995515,999036,996309,992228,994922,995788,997164,995154,993002,992309,996840,993028,998921,999671,998244,992843,993143,991510,995893,996653,998401,997486\n", + "Returned IDs: \n", "Ground Truth: 990882,994612,997465,999698,998937,997403,993845,998607,995515,999036,996309,992228,994922,995788,997164,995154,993002,992309,996840,993028,998921,999671,998244,992843,993143,997591,991510,995893,996653,998401\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 525\n", + "Returned IDs: 998225,990022,996075,997305,991348,993809,993341,996626,990697,995203,996001,994416,998359,992635,990532,996538,993721,993696,990083,995250,990573,990786,999289,993637,995899,990866,992557,994233,994081,999453\n", + "Ground Truth: 998225,990022,996075,997305,991348,993809,993341,996626,990697,995203,996001,994416,998359,992635,990532,996538,993721,993696,995613,990083,995250,990573,990786,999289,993637,995899,990866,992557,994233,994081\n", "Recall: 0.9667\n", "\n", + "Query ID: 526\n", + "Returned IDs: 998536,995655,995385,996245,991332,995168,992832,991630,995589,994280,995231,996760,993901,991654,998565,995536,992110,996501,999443,993489,992793,997965,993194,997145,993037,994380,998697,997158,998303,999994\n", + "Ground Truth: 998536,995655,997702,995385,996245,991332,995461,995168,992832,991630,995811,995589,994280,993381,995231,996760,993901,991654,990429,998565,995536,992110,996501,999443,993489,992793,997965,993194,993580,997145\n", + "Recall: 0.8000\n", + "\n", "Query ID: 527\n", - "Returned IDs: 995596,993341,996741,990697,997305,996075,994886,992041,994985,993785,996178,990571,993119,994859,996755,997661,994440,994971,993904,992661,990022,993696,996210,997291,992635,993741,991218,991650,997320,998715\n", + "Returned IDs: 995596,993341,996741,990697,997305,996075,994886,992041,994985,993785,996178,993119,994859,996755,993154,997661,994440,994971,993904,992661,990022,993696,996210,997291,992635,993741,991650,997320,998715,996616\n", "Ground Truth: 995596,993341,996741,990697,997305,996075,994886,992041,994985,993785,996178,990571,993119,994859,996755,993154,997661,994440,994971,993904,992661,990022,997699,993696,996210,997291,992635,993741,991218,991650\n", - "Recall: 0.9333\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 528\n", + "Returned IDs: 991795,999800,997868,996187,994140,996867,993003,990341,998205,992448,991969,990845,992994,998633,994419,994409,996042,993561,993813,995289,995864,996169,998361,997089,995702,996719,991790,997909,994386,999250\n", + "Ground Truth: 991795,999800,997868,993412,996187,994140,996867,993003,990341,998205,992448,991969,990845,992994,998633,994419,994409,996042,993561,993813,995289,995864,996169,998361,997089,995702,996719,991790,997909,994386\n", + "Recall: 0.9667\n", "\n", "Query ID: 529\n", - "Returned IDs: 992636,997168,995837,997201,995295,993715,994621,999052,998076,995159,994488,997838,991451,990956,992832,992595,996505,994380,990336,990887,990889,996147,995836,993448,996277,992973,998174,995521,991007,997329\n", + "Returned IDs: 992636,997168,995837,997201,995295,993715,994621,999052,998076,995159,994488,997838,991451,990956,996505,994380,998021,990336,990887,990889,996147,995836,993448,996277,992973,998174,995521,991007,997329,992450\n", "Ground Truth: 992636,997168,995837,997201,995295,993715,994621,999052,998076,995159,994488,997838,991451,990956,992832,992595,996505,994380,998021,990336,990887,990889,996147,995836,993448,996277,992973,998174,995521,991007\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 530\n", + "Returned IDs: 990072,990096,990154,999663,992716,995536,994103,998861,994834,994756,993055,994636,999711,999677,999898,992862,996406,993139,996733,996623,997506,992293,995963,997480,994640,994771,997154,996386,996209,995701\n", + "Ground Truth: 991167,990072,990096,990154,999663,992716,995536,994103,998861,994834,994756,993055,998419,994636,999711,995562,999677,999898,992862,996406,993139,993624,996733,996623,995103,997506,992293,992110,995963,997480\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 531\n", + "Returned IDs: 994749,991673,992685,992165,990772,995886,999182,993840,992677,999453,994655,997433,999586,992052,991943,995274,995401,991787,990432,995932,991585,995968,999820,997370,997574,990957,993260,995515,990103,990214\n", + "Ground Truth: 994749,991673,992685,992165,990772,995886,999182,995374,996949,993840,992677,997498,999453,991110,994655,997433,999586,992087,991052,992052,991943,995274,993673,995401,991787,990432,995932,991585,993412,995968\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 532\n", + "Returned IDs: 996216,990930,999009,998950,994964,995961,999453,993484,997320,995895,999716,993906,999294,993080,999430,990096,997379,994925,991498,998593,997918,990560,992780,999849,996911,998455,994742,993389,995438,998495\n", + "Ground Truth: 996216,990930,999009,998950,994964,995961,999453,993484,997320,995895,998123,999716,993906,999294,993080,999430,992497,990096,997379,994925,990445,991498,998593,997918,990560,992780,999849,996911,998455,992847\n", + "Recall: 0.8667\n", "\n", "Query ID: 533\n", - "Returned IDs: 997506,992154,996775,996055,995226,994891,997944,997050,991633,998933,998177,999139,999079,993665,999576,993347,991712,998536,992793,990194,996817,992170,997666,997906,994955,994280,996733,995536,999091,993074\n", + "Returned IDs: 997506,992154,996775,995226,994891,997944,997050,991633,998933,998177,999139,999079,993665,999576,993347,991712,998536,992793,990194,996817,990959,992170,997666,997906,994955,994280,996733,995536,993821,990196\n", "Ground Truth: 997506,992154,996775,996055,995226,994891,997944,997050,991633,998933,998177,999139,999079,993665,999576,993347,991712,998536,992793,990194,996817,990959,992170,997666,997906,994955,994280,996733,995536,999091\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 534\n", - "Returned IDs: 999876,993484,993504,998302,990092,993803,997320,998294,996869,997370,997913,998029,999363,991498,993741,994826,998580,992384,993885,999535,999374,990146,991043,991593,998246,994089,994366,995949,995372,997661\n", + "Returned IDs: 999876,993484,993504,998302,990092,993803,997320,998294,996869,997370,997913,998029,991498,993741,998580,992384,993885,999535,999374,990146,991043,991593,999982,998246,994089,994366,995949,995372,997661,992733\n", "Ground Truth: 999876,993484,993504,998302,990092,993803,997320,998294,996869,997370,997913,998029,999363,991498,993741,994826,998580,992384,993885,999535,999374,990146,991043,991593,999982,998246,994089,994366,995949,995372\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", - "Query ID: 541\n", - "Returned IDs: 998965,996219,994452,992998,995339,990780,999683,995455,996111,994777,991370,996178,992070,997032,997004,995460,990886,996612,998364,990931,990366,993128,997813,999343,998390,999013,997706,998593,999876,991179\n", - "Ground Truth: 998965,996219,994452,992998,995339,990780,999683,995455,996111,994777,991370,996178,992070,997032,997004,995460,994981,990886,996612,998364,991373,990931,990366,993128,999343,997813,993690,998390,999013,997706\n", + "Query ID: 535\n", + "Returned IDs: 994685,994160,996343,990936,998634,990925,993484,998966,990146,992546,999262,998644,998797,995884,993885,990958,997239,997102,991558,996532,993389,992147,999374,991396,993083,994964,996295,998153,990224,996874\n", + "Ground Truth: 994685,994160,996343,990936,994758,998634,990925,997876,993484,998966,990146,992546,999262,999620,998644,998797,995884,993885,990958,997239,997102,991558,996532,993389,992147,999374,991396,993083,994964,996295\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 536\n", + "Returned IDs: \n", + "Ground Truth: 993497,996763,998293,990956,995058,994073,995783,995159,996276,990149,992997,990872,995184,997713,991850,992981,997897,990306,992526,997746,993727,996189,992052,994280,993478,997752,996911,994315,992784,999514\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 537\n", + "Returned IDs: 997958,993767,997185,992272,994742,993997,999235,999279,998733,994730,993209,999402,993856,997566,995268,995633,999805,998162,999160,998962,991585,994140,996867,998748,993466,997660,995640,992289,993751,998898\n", + "Ground Truth: 997958,993767,997185,992272,994742,993997,999235,999279,998733,994730,993488,993209,999402,993856,992063,997566,995268,995633,999805,998162,999160,998962,991255,991585,994140,996867,998748,993466,997660,995640\n", "Recall: 0.9000\n", "\n", + "Query ID: 538\n", + "Returned IDs: 990589,999555,990167,992778,992234,994210,992353,996209,995610,991610,990194,994756,997242,998006,998495,992628,999634,994355,997850,993224,995879,998122,999994,998861,996794,991484,995209,997415,990107,998435\n", + "Ground Truth: 990589,999555,990167,992778,992234,994210,992353,996209,995610,991610,990194,994756,997242,998006,998495,996342,992628,999634,993055,994355,997850,993224,995879,998122,999994,998861,999771,996794,992745,994927\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 539\n", + "Returned IDs: 990157,995482,995083,992859,992450,992434,999934,994428,996306,991769,993731,992449,999240,994185,991935,995784,999695,998671,991185,992595,990593,994998,990331,993777,990784,993437,993856,998256,990126,990887\n", + "Ground Truth: 990157,995482,994976,995083,992859,992450,992434,999934,994428,996306,991769,993731,992449,999240,994185,991935,995784,999695,998671,991185,992595,990593,994998,990331,993777,990784,993437,993856,998256,990126\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 540\n", + "Returned IDs: \n", + "Ground Truth: 995300,999981,994742,995633,999474,992468,999372,998913,995012,990905,998765,992415,999788,995494,998543,998450,995289,996302,998557,992289,996057,998895,991590,991185,990126,990392,992598,999235,992239,995030\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 541\n", + "Returned IDs: 998965,996219,994452,992998,995339,990780,999683,995455,996111,994777,991370,996178,997004,995460,990886,996612,998364,991373,990931,990366,993128,999343,998390,998593,999876,991179,998157,996564,995438,995332\n", + "Ground Truth: 998965,996219,994452,992998,995339,990780,999683,995455,996111,994777,991370,996178,992070,997032,997004,995460,994981,990886,996612,998364,991373,990931,990366,993128,999343,997813,993690,998390,999013,997706\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 542\n", + "Returned IDs: 997131,993185,994063,999155,993232,995461,996809,994643,990255,997462,990941,994122,998632,995963,998855,990930,995421,998695,992809,998786,996930,997723,998919,996797,992558,998593,995766,991767,990956,991532\n", + "Ground Truth: 997131,993185,994063,999155,993232,995461,996809,994643,990255,997462,990941,994122,998632,995963,998855,990361,990930,995421,998695,992809,998786,996930,997723,998919,996797,992558,998593,995766,991767,990956\n", + "Recall: 0.9667\n", + "\n", "Query ID: 543\n", - "Returned IDs: 992944,993018,995560,998950,991142,995438,996726,998160,996140,990375,990843,991650,993324,998364,997320,991951,995107,999210,994171,994964,995130,999647,997379,995332,996370,998697,998180,993499,996653,996810\n", + "Returned IDs: 992944,993018,995560,998950,991142,995438,996726,998160,996140,990375,990843,991650,993324,998364,997320,991951,995107,999210,994171,994964,995130,995332,996370,998697,996653,990585,991206,997002,992318,996805\n", "Ground Truth: 992944,993018,995560,998950,991142,995438,996726,998160,996140,990375,990843,991650,993324,998364,997320,991951,995107,999210,994171,994964,995130,999647,997379,995332,996370,999109,998697,998180,993499,996653\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 544\n", - "Returned IDs: 993755,992761,991850,998778,991779,995222,997155,992595,995536,994356,992832,999107,999256,993055,995385,996001,996216,998536,992110,997158,990127,995501,991474,991644,995655,996328,999798,992561,997586,997141\n", + "Returned IDs: 993755,992761,991850,998778,991779,992595,995536,994356,992832,999107,999256,993055,995385,996001,996216,998536,990127,995501,999384,991644,995655,996328,999798,992561,997586,997141,991667,994076,992517,990392\n", "Ground Truth: 993755,992761,991850,998778,991779,995222,997155,992595,995536,994356,992832,999107,999256,993055,995385,996001,996216,998536,992110,997158,990127,995501,991474,999384,991644,995655,996328,992224,999798,992561\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", "\n", "Query ID: 545\n", - "Returned IDs: 997391,992428,991698,996930,992933,990438,998080,998855,994193,991619,993255,991257,993288,997918,991337,992179,995357,991743,992850,996762,995511,995401,994062,992685,999144,998180,992516,990459,996903,992531\n", + "Returned IDs: 997391,992428,991698,996930,992933,998080,998855,994193,993255,991257,993288,997918,991337,992179,995357,991743,996762,995401,999144,992516,990459,996903,992531,998121,991110,992604,997045,991459,997942,993673\n", "Ground Truth: 997391,992428,991698,996930,992933,990438,998080,998855,994193,991619,993255,991257,993288,997918,991337,992179,999527,995357,991743,992850,996762,995511,995401,994062,999144,992685,998180,992516,990459,996903\n", - "Recall: 0.9667\n", + "Recall: 0.7333\n", "\n", "Query ID: 546\n", - "Returned IDs: 994419,996745,992894,990252,997776,996322,996148,998593,992660,995643,997046,998486,997940,995289,992125,995470,991301,999482,995923,996867,992048,997822,998045,993412,995702,991958,995063,998564,995401,996323\n", + "Returned IDs: 994419,996745,992894,990252,997776,996322,996148,998593,992660,995643,997046,998486,997940,995289,992125,991301,999482,995923,996867,992048,997822,998045,995702,991958,995063,998564,996323,997937,994173,998165\n", "Ground Truth: 994419,996745,992894,990252,997776,996322,996148,998593,992660,995643,997046,998486,997940,995289,992125,995470,991301,999482,995923,996867,992048,997822,998045,993412,995702,991958,995063,998564,998105,995401\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 547\n", + "Returned IDs: 991630,992862,990005,999450,998536,994409,996368,992521,991844,998779,997918,991987,999653,991332,995701,995373,993948,995536,991667,995401,996809,995200,996657,994955,996328,991795,993813,998495,994946,995702\n", + "Ground Truth: 991630,992862,990005,999450,998536,998999,994409,996368,992521,991844,998779,997918,991987,999653,991332,995701,993412,995373,993948,998858,998697,991667,995536,997630,995401,996809,995864,995200,996657,994955\n", + "Recall: 0.8000\n", "\n", "Query ID: 548\n", - "Returned IDs: 990092,995716,990315,999261,994922,999620,990701,990934,997371,990648,996916,991921,997770,991196,999613,991588,994990,995222,995401,995693,998641,992944,992384,997403,997855,996219,991999,995551,999343,996840\n", + "Returned IDs: 995716,999261,994922,999620,990701,990934,997371,990648,996916,997770,999613,991588,994990,995693,992944,992384,997403,996219,991999,995551,999343,996840,996990,997529,995387,993215,990366,998921,999610,998523\n", "Ground Truth: 990092,995716,998991,990315,999261,994922,999620,997920,990701,990934,997371,990648,996916,991921,997770,991196,999613,991588,994990,995222,995401,995693,998641,992944,992384,996419,997403,997855,996219,999151\n", - "Recall: 0.8667\n", + "Recall: 0.6000\n", "\n", "Query ID: 549\n", - "Returned IDs: 997897,998250,993483,999463,992437,995421,992727,998593,990255,990521,992367,998934,993175,995626,998628,992407,996181,991245,998162,993489,994280,995401,990844,998032,996001,994809,997918,993727,996657,993232\n", + "Returned IDs: 997897,998250,993483,999463,992437,995421,992727,998593,990255,990521,992367,998934,993175,995626,998628,996181,991245,998162,994280,995401,990844,998032,996001,994809,997918,993727,996657,993232,996043,995823\n", "Ground Truth: 997897,998250,993483,999463,992437,995421,992727,998593,990255,990521,992367,998934,993175,995626,998628,992407,996181,991245,998162,993489,994280,995182,995401,990844,998032,996001,994809,997918,993727,996657\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 550\n", + "Returned IDs: 997320,991498,995841,995372,990585,993484,992944,996959,993018,997305,994964,996001,990949,997301,998302,998950,997370,995203,990096,993885,998246,998495,998849,995438,997913,995375,992119,997796,996318,990957\n", + "Ground Truth: 997320,991498,995841,995372,990585,993484,992944,996959,993018,997305,998445,994964,996001,990949,991052,997301,998302,998950,997370,995203,990096,993885,991743,998246,998495,998849,995438,993412,990772,997913\n", + "Recall: 0.8333\n", "\n", "Query ID: 551\n", - "Returned IDs: 991335,998500,990650,997006,995985,997688,999647,995942,994776,998139,998329,990092,994699,997192,992357,996727,997371,995370,992546,990934,990366,999898,995906,990980,993034,994778,990866,992994,994889,990315\n", + "Returned IDs: 991335,998500,997688,999647,998139,998329,990092,994699,997192,992357,996727,997371,995370,992546,990934,990366,999898,994778,996576,992994,990315,990505,999343,995203,990231,992330,996646,999201,994831,990957\n", "Ground Truth: 991335,998500,990650,997006,995985,997688,993174,999647,995942,994776,998139,998329,990092,994699,997192,992357,996727,997371,995370,992546,990934,990366,999898,995906,990980,992822,993034,994778,990866,996576\n", - "Recall: 0.9000\n", + "Recall: 0.6333\n", "\n", "Query ID: 552\n", - "Returned IDs: 995655,991667,997973,992517,997145,993001,991685,999107,990330,990458,991093,995222,993280,998861,990091,998122,997732,997679,992561,998536,997265,991569,996733,996028,992862,995835,997332,999827,995780,995536\n", + "Returned IDs: 995655,991667,997973,992517,997145,993001,991685,999107,990330,990458,991093,998861,998122,997732,997679,992561,998536,997265,991569,996733,996028,995835,997332,999827,995780,995536,990096,997319,995167,997685\n", "Ground Truth: 995655,991667,997973,992517,997145,993001,991685,999107,990330,990458,991093,995222,993280,998861,990091,998122,997732,997679,992561,998536,997265,991569,996733,996028,998710,992862,995835,997332,999827,993006\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 553\n", + "Returned IDs: 997698,993715,998716,998030,990392,993917,991663,992664,996183,995705,992636,991685,995837,997278,996986,990364,999052,998623,991620,998903,990998,997944,992407,992026,993366,991920,998440,994510,995735,991451\n", + "Ground Truth: 997698,993715,998716,998030,990392,993917,991663,992664,996183,995705,992636,991685,995837,997278,996986,990364,999052,998623,991620,998903,998174,992547,990998,997097,997944,992407,992026,993366,991920,998440\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 554\n", + "Returned IDs: 993305,992350,992258,991152,996163,996669,995248,997540,996620,997395,998892,995542,991705,997689,999579,994301,995772,995986,990019,999685,993255,997538,999298,997547,992150,990569,997169,994072,991977,990093\n", + "Ground Truth: 993305,992350,992258,991152,996163,996669,995248,997540,996620,997395,998892,995542,991705,997689,999579,994301,995772,995986,990019,999685,993255,997538,999298,997547,992150,990569,997169,994072,997523,991977\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 556\n", + "Returned IDs: 990399,993834,999041,993055,994237,998697,995184,999087,995701,996075,996230,999639,998758,992225,991042,991530,995672,998720,993727,995418,999724,995203,994167,994809,991256,994280,991485,995516,993283,998514\n", + "Ground Truth: 990399,993834,999041,993055,995855,994237,998697,995184,999087,995701,992774,991300,996075,996230,999639,998758,992225,999651,991042,993902,991530,995672,998720,993727,995418,991557,997531,999724,995203,994167\n", + "Recall: 0.7667\n", "\n", "Query ID: 557\n", - "Returned IDs: 995336,999900,992277,998090,992384,996296,990949,994100,996657,996001,997918,997320,998951,995873,997571,993143,995374,993809,997554,997913,990772,992330,993512,990339,996592,992880,992052,993492,999144,997156\n", + "Returned IDs: 995336,998090,992384,996296,990949,994100,996657,996001,997918,997320,998951,990343,995873,997571,993143,993809,997554,997913,990772,992330,990339,996592,992880,992052,993492,999144,992692,990897,990910,993289\n", "Ground Truth: 995336,999900,992277,998090,992384,996296,990949,994100,996657,996001,997918,997320,998951,990343,995873,997571,993143,995374,993809,997554,997913,990772,992330,993512,990339,996592,992880,992052,993492,999144\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 558\n", - "Returned IDs: 998934,998720,991657,991459,994167,994280,991130,996001,998593,992429,999639,999173,999216,993834,999408,995103,996408,992437,992441,993993,995158,995701,993375,996328,997790,992225,996526,993848,998864,997847\n", + "Returned IDs: 998934,998720,991459,994167,994280,991130,996001,998593,992429,999639,999173,999216,993834,992437,992441,993993,997337,995158,995701,993375,996328,997790,992225,996526,993848,998864,997847,995895,994610,990096\n", "Ground Truth: 998934,998720,991657,991459,994167,994280,991130,996001,992429,998593,999639,999173,999216,993834,999408,995103,996408,992437,992441,993993,997337,995158,995701,993375,996328,997790,992225,996526,997166,993848\n", - "Recall: 0.9333\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 559\n", + "Returned IDs: 996809,998495,990340,995895,997442,994529,990435,999500,993391,999377,994783,998913,997182,994830,999430,994269,999352,992827,990126,994402,996411,995633,991382,995535,992551,991128,996819,992566,993219,995476\n", + "Ground Truth: 996809,998495,990340,995895,997442,994529,990435,999500,993391,999377,994783,998913,997182,994830,999430,996049,994269,999352,992827,990126,994402,996411,995633,991382,995535,992551,991128,996819,992566,993219\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 560\n", + "Returned IDs: 997247,997977,995203,996868,999111,992082,993504,995740,992241,993157,996075,992747,995372,993174,999735,994167,995855,991425,993609,994672,996196,998303,992825,999526,993489,997368,996217,990196,998715,992571\n", + "Ground Truth: 997247,997977,995203,996868,999111,992082,993504,995740,992241,993157,996075,992747,995372,993174,999735,994167,995855,991425,993609,994672,996196,998303,992825,999526,993489,997368,996217,990196,998715,994694\n", + "Recall: 0.9667\n", "\n", "Query ID: 561\n", - "Returned IDs: 991753,995066,995401,996607,993003,991680,994106,996813,992448,997975,996304,990319,999054,991639,995864,998910,997544,993642,997091,996695,999759,997176,999646,997075,991795,995702,995490,992653,993209,991592\n", + "Returned IDs: 991753,995066,995401,996607,993003,991680,994106,996813,992448,997975,996304,990319,999054,993597,991639,995864,998910,997544,993642,997091,996695,999759,997176,999646,997075,991795,995490,992653,993209,991592\n", "Ground Truth: 991753,995066,995401,996607,993003,991680,994106,996813,992448,997975,996304,990319,999054,993597,991639,995864,998910,997544,993642,997091,996695,999759,999646,997176,997075,991795,995702,995490,992653,993209\n", "Recall: 0.9667\n", "\n", + "Query ID: 562\n", + "Returned IDs: 996043,997723,995401,991630,998267,990033,991954,997380,990525,991403,999595,993275,991074,990325,995940,991666,998166,994178,999277,993412,999587,995200,994409,999977,991794,994474,999228,997918,996903,998045\n", + "Ground Truth: 997957,996043,997723,995401,991630,998267,990033,991954,997380,991603,990525,991403,999595,993275,991074,990325,995940,992582,991666,998166,994178,999277,993412,999587,995200,996181,994409,999977,991794,997630\n", + "Recall: 0.8333\n", + "\n", "Query ID: 563\n", - "Returned IDs: 990508,998359,999717,990031,995206,991323,991650,993603,997581,991348,996178,992001,999132,996075,999258,993686,996626,994233,993809,999806,999473,994341,996458,990022,992557,990866,997442,990912,997367,999480\n", + "Returned IDs: 990508,998359,999717,995206,991323,991650,993603,997581,991348,996178,992001,996075,999258,993686,996626,994233,993809,999806,999473,996458,990022,992557,990866,997367,999480,994331,995596,995108,999230,990456\n", "Ground Truth: 990508,998359,999717,990031,995206,991323,991650,993603,997581,991348,996178,992001,999132,996075,999258,993686,996803,996626,994233,993809,999806,999473,994341,996458,990022,992557,990866,997442,990912,997367\n", - "Recall: 0.9667\n", + "Recall: 0.8000\n", "\n", "Query ID: 564\n", - "Returned IDs: 993591,997036,993015,993553,994275,997170,991989,996906,990368,994021,995923,995345,992585,998039,994663,990955,992798,994386,993561,993164,994379,992014,990845,993725,997357,993813,991168,993697,997933,995076\n", + "Returned IDs: \n", "Ground Truth: 993591,997036,993015,993553,994275,997170,991989,996906,990368,994021,995923,995345,992585,998039,994663,990955,992798,994386,993561,993164,994379,992014,990845,993725,997357,993813,997920,991168,993697,997933\n", - "Recall: 0.9667\n", + "Recall: 0.0000\n", "\n", "Query ID: 565\n", - "Returned IDs: 992286,993030,995854,997634,997020,994301,998386,995723,997114,996273,998080,993263,992683,998650,990591,992531,994677,997723,991423,996139,997395,990103,993717,990189,991404,995062,993223,995625,993127,990178\n", + "Returned IDs: 992286,993030,995854,997634,997020,994301,998386,995723,996273,998080,993263,998650,990591,992531,994677,997723,991423,996139,997395,993717,990189,991404,995062,993223,995625,993127,990178,999961,999020,992558\n", "Ground Truth: 992286,993030,995854,997634,997020,994301,998386,995723,997114,996273,998080,993263,992683,998650,995282,990591,992531,994677,997723,994360,991423,996139,997395,990103,993717,990189,991404,995062,993223,995625\n", - "Recall: 0.9333\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 566\n", + "Returned IDs: 991384,998049,996019,997675,998536,996501,995521,998933,995080,996506,999448,996245,996505,996918,996062,996313,995589,990479,999091,990537,997395,999079,993931,995536,998874,992211,994955,994280,995655,995408\n", + "Ground Truth: 991384,998049,996019,997675,998536,996501,995521,998933,991904,995474,995080,996506,999448,996245,996505,996918,991052,993251,998331,996062,996313,995589,990479,999091,990537,997395,999079,993931,995536,991758\n", + "Recall: 0.8000\n", "\n", "Query ID: 567\n", - "Returned IDs: 990974,994541,994557,993637,999009,996996,992551,999898,994638,992553,993807,997320,990096,997004,996216,993695,995906,993928,998633,991179,992927,998006,995895,990786,997410,996564,996944,995222,997732,998302\n", + "Returned IDs: 990974,994541,993637,999009,996996,992551,999898,993807,997320,990096,997004,996216,993695,998633,996830,991179,992927,995895,990786,997410,996564,996944,998302,996809,997126,994782,995638,998950,993053,991154\n", "Ground Truth: 990974,994541,994557,993637,999009,996996,992551,999898,994638,992553,993807,997320,990096,997004,996216,993695,995906,993928,998633,996830,991179,992927,998006,997577,995895,990786,997410,995755,996564,996944\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 568\n", + "Returned IDs: 993883,992859,990416,991815,992272,996898,994013,993908,990903,997566,999402,996109,998162,997185,990462,995268,993209,999279,998980,995544,991585,999172,997384,997710,998898,991507,999672,997031,998060,994991\n", + "Ground Truth: 993883,992859,990416,991815,992272,996898,994013,993908,990903,997566,999402,996109,998162,997185,998609,990462,995268,993209,999279,998980,995544,991585,999172,997384,993166,997710,998898,991507,995633,999672\n", "Recall: 0.9000\n", "\n", "Query ID: 569\n", - "Returned IDs: 994145,993561,993993,993939,996328,992862,996001,990887,994853,998710,995326,995535,992450,997878,996219,996132,997529,994742,999472,990096,994682,996408,991759,999444,998382,993342,996042,995895,994830,990918\n", + "Returned IDs: 994145,993561,993993,993939,996328,992862,996001,990887,994853,998710,992450,997878,996219,996132,997529,994742,990096,994682,996408,991759,999444,998382,993342,996042,995895,994830,990918,998221,996884,998495\n", "Ground Truth: 994145,993561,993993,993939,996328,992862,996001,990887,994853,998710,995326,995535,992450,997878,996219,996132,997529,994742,999472,990133,990096,994682,996408,991759,999444,998382,993342,996042,995895,994830\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 570\n", + "Returned IDs: 992293,990662,990097,996001,996657,998549,999853,992521,996028,996147,995283,998878,997305,993848,996984,997554,995401,996368,999099,995873,993053,992862,998593,998017,992395,995158,990811,990211,993725,990573\n", + "Ground Truth: 992293,990662,990097,996001,996657,998549,999853,992473,992521,996028,996147,995283,998878,997305,993433,993848,996984,997554,995401,996368,999099,995873,993053,997466,992862,995002,994531,999587,998593,998017\n", + "Recall: 0.8000\n", "\n", "Query ID: 571\n", - "Returned IDs: 993997,998962,990663,999279,997031,996786,997844,990758,996131,995633,996867,998609,996302,993920,993767,995640,995268,990157,994120,999996,993209,993488,997566,995766,995289,996346,999415,996109,991214,997223\n", + "Returned IDs: 993997,998962,990663,999279,997031,996786,997844,990758,996131,995633,996867,996302,993920,993767,995640,995268,990157,994120,993209,997566,995766,995289,996346,999415,996254,996109,994409,999672,994730,995240\n", "Ground Truth: 993997,998962,990663,999279,997031,996786,997844,990758,996131,995633,996867,998609,996302,993920,993767,995640,995268,990157,994120,999996,993412,993209,993488,997566,995766,995289,996346,999415,996254,996109\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 572\n", + "Returned IDs: 996001,995823,998593,991789,997768,996028,990930,992862,997586,994307,995536,997790,995002,993053,996368,995289,990811,998495,996147,996062,993342,999444,999413,993906,998874,990096,990399,998039,990382,993168\n", + "Ground Truth: 996001,995823,998593,991789,997768,996028,990930,992862,997586,994307,995536,997790,997920,995002,993053,996368,995289,990811,998495,995097,996147,991747,996062,993342,999444,999413,997691,993906,998874,995096\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 573\n", + "Returned IDs: 998171,998866,995923,993390,993561,991996,997822,995547,992660,998045,996042,995052,995701,991958,994409,997289,990137,997058,996867,995289,994396,998165,991011,993433,994140,999868,992576,996125,991415,997191\n", + "Ground Truth: 998171,998866,995923,993390,993561,991996,997822,995547,992660,998045,996042,995052,995701,991958,998323,994409,997289,990137,997058,996867,995289,990706,994396,998165,991011,993433,994140,999868,992576,996125\n", "Recall: 0.9333\n", "\n", "Query ID: 574\n", - "Returned IDs: 991348,995206,995108,996626,992318,993603,993053,995438,990083,998950,996024,991763,993621,994233,992944,991142,993166,996178,994742,993154,992617,998567,990031,995107,998180,996001,993696,997448,999426,995669\n", + "Returned IDs: 991348,995206,996626,992318,993603,993053,995438,990083,998950,996024,991763,993621,994233,992944,991142,993166,993747,996178,994742,993154,995107,996001,993696,999426,995669,990883,996653,996210,991997,990508\n", "Ground Truth: 991348,995206,995108,996626,992318,993603,993053,995438,990083,998950,996024,991763,993621,994233,992944,991142,993166,993747,996178,994742,993154,992617,998567,990031,995107,998180,996001,993696,997448,999426\n", - "Recall: 0.9667\n", + "Recall: 0.8000\n", "\n", "Query ID: 575\n", - "Returned IDs: 990450,998383,999136,998570,998586,994105,991844,999306,996446,995695,995257,996001,995142,994643,998593,996043,993948,990991,994185,991987,999653,994187,995455,997191,999117,995701,999872,991038,992945,995755\n", + "Returned IDs: 999136,998570,998586,991844,999306,995695,995257,996001,995142,994643,998593,993948,990991,994185,991987,994187,995455,999117,995701,991038,992945,995755,996623,996147,993628,995200,993342,992760,997369,993289\n", "Ground Truth: 990450,998383,999136,998570,998586,994105,991844,999306,996446,995695,995257,996001,995142,994643,998593,996043,993948,990991,994185,991987,999653,994187,997766,995455,997191,998493,999117,995701,999872,991038\n", - "Recall: 0.9333\n", + "Recall: 0.6667\n", "\n", "Query ID: 576\n", - "Returned IDs: 998623,990998,998536,995408,992052,994743,990096,999287,990255,996809,995203,997618,995536,994682,992558,991630,999107,994742,995401,994510,991629,991515,992933,994955,993910,999724,996483,995198,994954,991977\n", + "Returned IDs: 998623,990998,998536,995408,992052,994743,990096,999287,990255,996809,995203,997618,995536,994682,992558,994742,994510,991629,991515,994955,993910,999724,996483,994954,991977,992636,993948,996930,995237,994906\n", "Ground Truth: 998623,990998,998536,995408,992052,994743,990096,999287,990255,996809,995203,997618,995536,994682,992558,991630,999107,995864,994742,995401,994510,991629,991515,992933,994955,993910,999724,996483,995198,994954\n", - "Recall: 0.9667\n", + "Recall: 0.8000\n", "\n", "Query ID: 577\n", - "Returned IDs: 997918,993223,996903,995373,993512,995453,994088,991887,995444,996282,991844,994334,992604,994092,991698,995807,993412,994804,997675,998318,995685,991110,997515,997637,992933,999144,997110,999478,994819,996867\n", + "Returned IDs: 997918,996903,995373,995453,994088,991887,995444,991844,995080,994334,992604,994092,991698,995807,994804,997675,998318,995685,991110,997515,997637,992933,999144,997110,999478,994819,996867,993754,994170,996658\n", "Ground Truth: 997918,993223,996903,995373,993512,995453,994088,991887,995444,996282,991844,995080,994334,992604,994092,991698,995807,993412,994804,997675,998318,995685,991110,997515,997637,992933,999144,997110,999478,994819\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 578\n", - "Returned IDs: 997110,991987,995268,999478,994948,990991,997723,994643,993122,995848,998078,990805,991630,999994,990361,993381,990930,994125,997191,996968,991052,999117,996867,991954,999530,995963,994253,995701,999555,998257\n", + "Returned IDs: 997110,991987,995268,999478,994948,990991,997723,994643,993122,998078,991630,999994,990361,990930,994125,997191,996968,991052,999117,996867,999530,995963,994253,999555,992604,993232,992124,992861,995373,997406\n", "Ground Truth: 997110,991987,995268,999478,994948,990991,997723,994643,993122,995848,998078,990805,991630,999994,990361,993381,990930,994125,997191,996968,991052,999117,996867,991954,999530,995963,994253,995701,992556,999555\n", - "Recall: 0.9667\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 579\n", + "Returned IDs: 990195,991585,995216,999302,998060,996147,996001,998549,990975,991958,995873,998200,996219,994925,994013,997554,990887,995701,996657,993813,993561,994076,995079,999343,995555,997377,992384,991125,995016,999402\n", + "Ground Truth: 990195,991585,995216,999302,998060,996147,996001,998549,993377,990975,991958,995873,998200,996219,994925,994013,997920,997554,990887,995701,996657,993813,993561,994076,995079,999343,995555,997377,992384,991125\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 580\n", + "Returned IDs: \n", + "Ground Truth: 997126,997415,990893,994467,994903,993501,993460,997084,991303,998588,997340,990998,993084,993931,992056,991383,999430,999323,997196,990173,990579,994510,998716,991672,996706,999139,995895,997565,996184,994098\n", + "Recall: 0.0000\n", "\n", "Query ID: 581\n", - "Returned IDs: 995899,991154,998697,992167,998720,995536,993139,994167,995562,996075,999806,999724,996811,997305,999639,996001,992225,994577,997586,992423,993560,991157,995963,996894,999107,993055,995203,998514,998568,994636\n", + "Returned IDs: 995899,991154,998697,992167,998720,995536,993139,994167,995562,996075,999806,999724,997305,999639,996001,992225,994577,997586,992423,993560,991157,996894,993055,995203,998514,998568,994636,990051,991332,995516\n", "Ground Truth: 995899,991154,998697,992167,998720,995536,993139,994167,995562,996075,999806,999724,996811,997305,999639,996001,992225,994577,997586,992423,993560,991157,995963,996894,999107,993055,998105,995203,998514,998568\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 582\n", + "Returned IDs: 994806,992598,999852,998256,999192,998903,990069,994175,996633,998716,994830,996960,997202,990806,996495,996537,996301,993335,999628,999126,990998,996328,999372,996153,998761,996011,994608,993502,990985,991769\n", + "Ground Truth: 994806,992598,999852,998256,999192,998903,990069,994175,996633,998716,994830,996960,999441,997202,990806,996495,996537,996301,993335,999628,999126,990998,996328,999372,996153,998761,996011,994608,993502,990985\n", "Recall: 0.9667\n", "\n", "Query ID: 583\n", - "Returned IDs: 992680,995756,996645,993725,991669,999740,999356,991075,997798,992090,993444,999302,990743,999922,995435,991464,996411,993645,990528,992024,999261,990701,999280,992783,996627,993205,996969,998940,993053,996384\n", + "Returned IDs: 992680,995756,996645,993725,991669,999740,999356,991075,992090,993444,999302,990743,995435,991464,996411,993645,990528,992024,990701,999280,996627,993205,996969,998940,993053,996384,994783,995126,992679,995906\n", "Ground Truth: 992680,995756,996645,993725,991669,999740,999356,991075,997798,992090,993444,999302,990743,999922,995435,991464,996411,991624,993645,990528,992024,999261,990701,999280,992783,996627,993205,996969,998940,993053\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 584\n", - "Returned IDs: 998182,994799,990328,998105,993857,999460,991075,990980,998017,995166,998427,998197,996689,995295,996028,997961,995823,990145,999843,993055,992480,999230,993475,992374,994119,998982,996494,990536,990097,990811\n", + "Returned IDs: 998182,994799,990328,998105,993857,999460,991075,990980,995166,998427,998197,996689,995295,996028,997961,995823,990145,999843,992480,999230,993475,994119,998982,996494,990536,990097,990811,993053,993221,997822\n", "Ground Truth: 998182,994799,990328,998105,993857,999460,991075,990980,998017,995166,998427,998197,996689,995295,996028,997961,995823,990145,999843,993055,992480,999230,993475,992374,994119,998982,996494,990536,990854,990097\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 585\n", + "Returned IDs: 995755,997301,993484,994742,990096,998246,995942,990958,991743,996001,997614,997371,995701,993853,994682,992862,993048,997320,998294,998921,990949,990515,996805,999801,993342,991498,996521,990146,995132,997913\n", + "Ground Truth: 995755,997301,999533,993484,994742,990096,998246,995942,990958,991743,996001,997614,997371,995701,993853,994682,993412,992862,993048,997320,998294,993948,998921,990949,990515,998002,996805,999801,997533,990545\n", + "Recall: 0.8000\n", "\n", "Query ID: 586\n", - "Returned IDs: 992183,991479,995802,998011,992946,994778,993853,998550,994699,995396,990231,997439,991207,990647,994450,993662,999243,993403,995465,996912,997913,999647,996735,991147,990478,999710,995793,997194,991502,998899\n", + "Returned IDs: 992183,991479,990109,995802,998011,994778,993853,998550,994699,995396,990231,990647,994450,993662,999243,993403,997913,999710,995793,991502,998899,998828,998951,991249,995336,992147,990146,995824,990910,992442\n", "Ground Truth: 992183,991479,990109,995802,998011,992946,994778,999561,993853,998550,994699,995396,990231,997439,991207,990647,994450,993662,999243,993403,995465,996912,997913,999647,996735,991147,990478,999710,995793,997194\n", - "Recall: 0.9333\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 587\n", + "Returned IDs: 993562,990535,994685,997015,995947,991008,998992,991068,993083,997306,993484,992681,990936,991444,990925,999644,994234,995129,999531,992274,997543,993050,991396,996343,996504,994997,990146,990585,996959,998797\n", + "Ground Truth: 991433,993562,990535,994685,997015,995947,991008,998992,991068,993083,997306,993484,992681,990936,991444,990925,999644,994234,995129,990539,999531,992274,997543,993050,991396,999012,996343,996504,994997,990146\n", + "Recall: 0.9000\n", "\n", "Query ID: 588\n", - "Returned IDs: 990845,994925,996906,998593,992703,997453,998039,993561,991958,995162,996062,996001,999729,995401,991789,993906,991989,992334,996237,997028,990899,991011,990096,990076,995289,996311,996277,999340,998207,994424\n", + "Returned IDs: 990845,994925,996906,998593,992703,997453,998039,993561,991958,996062,996001,999729,991789,993906,991989,992334,996237,990899,991011,990096,990076,995289,996311,996277,994424,996834,993064,991869,999491,992576\n", "Ground Truth: 990845,994925,996906,998593,992703,997453,998039,993561,991958,995162,996062,996001,999729,995401,991789,993906,991989,992334,996237,997028,990899,991011,990096,990076,995289,996311,996277,999340,998207,995560\n", - "Recall: 0.9667\n", + "Recall: 0.8000\n", "\n", "Query ID: 589\n", - "Returned IDs: 990271,997878,990179,995222,991776,995868,990621,994448,992193,996219,998269,995719,994145,990369,996580,994689,994373,990272,997349,995096,991075,998204,990077,999102,990315,995438,995690,994682,994742,993725\n", + "Returned IDs: 990271,997878,990179,991776,995868,994448,992193,996219,994145,990369,994689,994373,990272,997349,996593,995096,991075,990077,999102,990315,995438,995690,994682,994742,993725,991283,999331,999144,992337,995220\n", "Ground Truth: 990271,997878,990179,995222,991776,995868,990621,994448,992193,996219,998269,995719,994145,990369,996580,994689,994373,990272,997349,996593,995096,991075,998204,990077,999102,990315,995438,995690,994682,994742\n", - "Recall: 0.9667\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 590\n", + "Returned IDs: 998792,990690,992169,990226,996028,994964,997822,998039,993717,995526,993906,999898,992944,992408,993197,996494,993807,994925,995200,993561,990845,995861,997320,996328,993637,993869,998593,990210,994340,999613\n", + "Ground Truth: 998792,990690,995360,992169,990226,996028,994964,997822,998039,993717,995526,993906,997732,999898,992944,992408,993197,990580,996494,993807,994925,995200,993561,990845,995906,995861,997320,996328,991780,993637\n", + "Recall: 0.8333\n", "\n", "Query ID: 591\n", - "Returned IDs: 993433,996042,995289,996607,992448,998910,995401,995146,993561,995820,995018,995200,996769,995521,998162,995864,996112,996815,991671,999611,992576,991699,996867,997723,991958,994409,997822,995766,999054,997406\n", + "Returned IDs: 993433,996042,995289,996607,992448,998910,995146,993561,995820,995018,997023,995200,996769,995521,998162,995864,996112,996815,991671,999611,992576,991699,996867,991958,994409,997822,995766,999054,997406,998564\n", "Ground Truth: 993433,996042,995289,996607,992448,998910,995401,995146,993561,996809,995820,995018,997023,995200,996769,995521,998162,995864,996112,996815,991671,999611,992576,991699,996867,997723,991958,994409,997822,995766\n", - "Recall: 0.9333\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 592\n", + "Returned IDs: 994315,997059,990853,993991,995808,996763,995159,994354,990956,995184,992526,996189,997949,995873,992052,994175,992427,991111,994621,994377,996277,991981,997041,990611,992981,994488,995047,991769,994185,992285\n", + "Ground Truth: 994315,997059,990853,993991,995808,996763,995159,994354,990956,995184,992526,996189,991864,997949,995873,992052,994175,992427,991111,997233,994621,994377,996277,991981,997041,990611,992981,994488,995047,991950\n", + "Recall: 0.9000\n", "\n", "Query ID: 593\n", - "Returned IDs: 992558,993255,994280,992790,995401,990790,997395,995374,995886,995134,990214,991404,999715,995145,996762,993478,991977,997918,990538,992685,999305,995119,995154,995968,991052,990969,992933,996803,999179,999154\n", + "Returned IDs: 992558,993255,994280,992790,995401,997395,995886,995134,990214,991404,999715,996762,993478,991977,997918,990538,992685,999305,995119,995154,995968,990969,992933,993143,999179,999154,991382,998950,999961,997822\n", "Ground Truth: 992558,993255,994280,992790,995401,990790,997395,995374,995886,995134,990214,991404,999715,995145,996762,993478,991977,997918,990538,992685,999305,995119,996721,995154,995968,991052,998045,990969,992933,999461\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 594\n", + "Returned IDs: 998314,990096,999282,990583,993086,999079,996733,999194,996209,999384,993744,993965,997821,999287,991243,992407,996893,993595,990072,999831,990920,998207,998536,995736,999411,997442,998510,991093,996124,999320\n", + "Ground Truth: 998314,990096,999282,990583,993086,999079,996733,999194,996209,999384,993744,993965,997821,999287,991243,992407,996893,993595,990072,999831,990920,998207,998419,998536,995736,999411,992112,997442,998510,993609\n", "Recall: 0.9000\n", "\n", "Query ID: 595\n", - "Returned IDs: 991828,992186,990580,991840,990310,997539,994922,999076,996402,996996,993036,993577,994005,990138,995515,996449,996478,991609,991172,999846,996840,993143,992207,991519,998455,995156,995767,998711,991072,994917\n", + "Returned IDs: 991828,992186,990580,991840,990310,997539,994922,999076,996402,996996,993036,993577,994005,990138,995515,996449,996478,991609,991172,999846,996840,993143,998455,995156,998711,990615,990957,993598,992611,995525\n", "Ground Truth: 991828,992186,990580,991840,990310,997539,994922,999076,996402,996996,993036,993577,994005,990138,995515,996449,996478,994700,991609,991172,999846,993278,996840,993143,992207,991519,998455,995156,995767,998711\n", - "Recall: 0.9333\n", + "Recall: 0.8333\n", "\n", "Query ID: 596\n", - "Returned IDs: 995981,997620,999374,997304,997437,998246,995792,995596,992357,994266,999331,997913,998697,992748,994100,995132,990588,997429,993186,994916,997320,994894,995638,990949,997614,996075,992384,999815,998353,991045\n", + "Returned IDs: 995981,997620,999374,997304,997437,998246,995596,992357,994266,999331,997913,998697,992748,994100,995132,990588,997429,993186,997320,998974,994894,995638,990949,997614,996075,992384,992124,998353,991045,990866\n", "Ground Truth: 995981,997620,999374,997304,997437,998246,995792,995596,992357,994266,999331,997913,998697,992748,992047,999710,994100,995132,990588,997429,993412,993186,994916,992583,997320,998974,994894,995638,990949,997614\n", - "Recall: 0.8333\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 597\n", + "Returned IDs: 995144,990116,996878,990325,997420,991145,999144,993727,995130,995357,993805,998045,997766,996811,995289,995562,996877,994051,996918,999365,993121,993433,997822,992558,993275,992718,990956,991131,992052,998758\n", + "Ground Truth: 995144,990116,996878,990325,997420,991145,999144,997723,993727,995130,993504,995357,993137,992315,993805,998045,997766,996811,995289,995562,996877,999527,995374,994051,996918,990527,999365,993121,993433,992933\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 598\n", + "Returned IDs: 992636,996986,995467,997340,999334,990438,990257,993715,993917,990951,999724,992282,991144,998671,996594,996183,993139,993055,997546,999373,997168,993342,993448,995326,990353,996408,995746,997577,998348,993224\n", + "Ground Truth: 992636,996986,995467,997340,999334,994680,990438,990257,993715,993719,993917,990951,999724,992282,991144,998671,996594,996183,993139,993055,997546,999373,997168,993342,993448,995326,990353,996408,995746,997577\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 599\n", + "Returned IDs: 996298,991464,999922,992240,992408,993725,990145,994472,996627,990531,990091,995295,993444,993906,997453,994765,990226,991021,993053,995126,998055,999126,999139,997944,992322,994663,992703,992169,992910,995680\n", + "Ground Truth: 996298,991464,999922,992240,991663,992408,993725,990145,994472,996627,990531,990091,995295,993444,993906,997453,994765,990226,991021,993053,995126,998055,999126,999139,997944,992322,994663,992703,991168,990712\n", + "Recall: 0.9000\n", "\n", "Query ID: 600\n", "Returned IDs: 996610,997787,991603,998906,996047,996388,990033,993390,995289,991463,997406,990681,998045,990940,994161,998165,993561,996112,992125,991367,991954,996769,998999,995182,992160,997046,995497,997832,995318,995052\n", "Ground Truth: 996610,997787,991603,998906,996047,996388,990033,993390,995289,991463,997406,990681,998045,990940,994161,998165,995040,993561,996112,992125,991367,991954,996769,998999,995182,992160,997046,995497,997832,995318\n", "Recall: 0.9667\n", "\n", + "Query ID: 601\n", + "Returned IDs: 995819,994119,995401,991943,994148,992832,994817,990814,991151,993857,996760,996328,999099,991747,999711,992595,992582,992289,996064,995295,990127,998067,990531,991569,994280,998536,997319,999107,995476,996689\n", + "Ground Truth: 995819,994119,995401,991079,991943,994148,992832,994817,992318,990814,991151,993857,996760,996328,999099,990769,991747,999711,992595,992582,992289,996064,997452,995295,990127,990531,998067,991569,995536,994280\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 603\n", + "Returned IDs: 997878,991464,993561,996834,998705,994838,998382,990315,993725,993166,993053,991958,994730,995679,994145,995435,995283,999151,999173,995384,994338,990621,990441,999740,990614,999302,992261,997529,996411,999126\n", + "Ground Truth: 991024,997878,991464,993561,996834,998705,994838,998382,996752,990315,993725,999472,996565,993166,993053,991958,994730,995679,994145,995435,995283,999151,999173,996943,995384,999352,994338,990621,990441,999740\n", + "Recall: 0.8000\n", + "\n", "Query ID: 604\n", - "Returned IDs: 994297,995244,994964,993452,991438,994919,998500,999261,999821,998837,999444,992169,995135,996854,995591,998909,991991,999545,999740,994922,991999,990736,990432,997984,996351,991840,998553,995336,994898,997720\n", + "Returned IDs: 994297,995244,994964,993452,994919,998500,999821,998837,999444,992169,995135,998909,991991,999545,999740,994922,991999,990736,990432,997984,996351,991840,998553,995336,998523,997192,999676,997320,994318,998921\n", "Ground Truth: 994297,995244,994964,993452,991438,994919,998500,999261,999821,998837,999444,990064,992169,995135,996854,995591,998909,991991,999545,999740,994922,991999,990736,990432,997984,996351,991840,998553,995336,991751\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", "\n", "Query ID: 605\n", - "Returned IDs: 999375,997152,992468,991960,998196,994193,991758,994119,999788,998274,992056,991447,995438,991515,998543,994742,994541,995494,997452,990814,991334,999981,990127,993180,997967,992077,998913,990796,996302,999543\n", + "Returned IDs: 999375,997152,991960,998196,994193,991758,994119,999788,998274,992056,991515,998543,994742,994541,997452,990814,991334,999981,993180,997967,992077,998913,990796,996302,999543,990722,998855,994893,993744,995200\n", "Ground Truth: 999375,997152,992468,991960,998196,994193,991758,994119,999788,998274,992056,991447,998466,995438,991515,998543,994742,994541,995494,997452,990814,991334,999981,990127,993180,997967,992077,998913,990796,996302\n", - "Recall: 0.9667\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 606\n", + "Returned IDs: 992195,993708,992907,994922,997135,995588,995693,995365,999784,991157,993696,999676,992039,991015,992612,998837,997379,995203,990310,998123,993143,996840,993355,991485,999077,994964,996727,998950,991848,997305\n", + "Ground Truth: 992195,993708,992907,994922,997135,995588,995693,995365,999784,991157,993696,999676,992039,991015,992612,998837,997379,995203,990310,998123,993143,996840,993355,991485,999077,997037,994964,993633,996727,998950\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 607\n", + "Returned IDs: 990146,999600,990958,990863,993083,993562,994964,994685,992301,994217,994425,996959,996532,990925,992968,995947,995884,993238,997320,999262,998302,994673,995716,999374,994922,991743,993168,995372,991008,998170\n", + "Ground Truth: 990146,999600,990958,991086,990863,993083,993562,994964,994685,992301,994217,995909,994425,996959,996532,999444,990652,990925,992968,997876,995947,995884,993238,997320,999262,998302,993412,994673,995716,999374\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 608\n", + "Returned IDs: 991569,997009,992561,996328,990437,993931,990040,997925,997611,990293,993001,991667,995159,997513,998720,990330,997973,990956,994509,998593,997204,995655,990364,995701,992832,990951,996408,991679,998067,998156\n", + "Ground Truth: 991569,997009,992561,996328,990437,993931,990040,997925,997611,990293,993001,993778,996706,991667,995159,997513,998720,990330,994796,997973,990956,994509,998593,997204,995655,990364,995701,992832,992850,990133\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 609\n", + "Returned IDs: 993053,998153,993721,994751,992680,993444,999740,998435,995861,998495,996685,992679,994558,991382,991128,995735,998055,998454,993984,999356,999864,993725,999280,998761,997933,996645,992079,998221,991075,994742\n", + "Ground Truth: 993053,998153,993721,994751,992680,993444,999740,998435,995861,998495,996685,992679,994558,991382,991128,995735,999295,998055,998454,993984,999356,999864,993725,999280,998761,997933,996645,992079,998221,993123\n", + "Recall: 0.9333\n", "\n", "Query ID: 610\n", - "Returned IDs: 991638,997732,994956,993929,997112,993765,992655,998107,992120,995861,992813,999827,992169,998458,992408,990226,997852,990441,991334,994131,993877,990889,990961,997453,990266,996328,994557,998495,997822,993080\n", + "Returned IDs: 991638,997732,994956,993929,993765,992655,998107,992120,995861,992813,999827,992169,998458,992408,990226,997852,991334,994131,993877,990889,990961,997453,996328,994557,998495,997822,993080,997969,990382,998171\n", "Ground Truth: 991638,997732,994956,993929,997112,993765,992655,998107,992120,995861,992813,999985,999827,992169,998458,992408,990226,997852,990441,991334,994131,993877,990889,990961,997453,990266,996328,994557,998495,997822\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 611\n", - "Returned IDs: 999056,999426,996744,996500,992167,996075,991984,997777,992041,992927,993766,995291,991823,993283,993156,997661,995596,996761,995132,993237,999041,999132,996169,996001,995516,995898,994809,993834,998990,999219\n", + "Returned IDs: 999056,999426,996744,996500,992167,996075,991984,992041,992927,993766,991823,993283,997661,995596,996761,995132,993237,999041,996169,996001,995516,994809,993834,995016,991392,998232,995855,996657,996576,992351\n", "Ground Truth: 999056,999426,996744,996500,992167,996075,991984,997777,992041,992927,993766,995291,991823,993283,993156,997661,995596,996761,995132,993237,995375,999041,999132,996169,996001,995516,995898,994809,993834,998990\n", - "Recall: 0.9667\n", + "Recall: 0.7667\n", "\n", "Query ID: 612\n", - "Returned IDs: 994830,994742,998950,993588,996809,992807,998913,998823,994806,995552,994269,992318,996805,991154,996301,999627,990392,991390,993342,998495,995299,999430,992598,992551,994751,992427,993053,995899,995002,994682\n", + "Returned IDs: 994830,994742,998950,993588,996809,992807,998913,998823,994806,995552,994269,992318,996805,991154,999627,990392,993342,998495,999430,992598,992551,992427,993053,995899,995002,994682,992415,997073,995289,999377\n", "Ground Truth: 994830,994742,998950,993588,996809,992807,998913,998823,994806,995552,994269,992318,996805,991154,996301,999627,990392,991390,993342,998495,995299,999430,992598,992551,994751,992427,993053,995899,995401,995002\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 613\n", - "Returned IDs: 994051,990256,991860,995516,997645,992915,997811,991799,995755,990862,992596,992165,995132,997429,993396,993336,997305,996856,996811,999251,991052,994879,992357,995963,991690,995184,996406,990958,996942,991982\n", + "Returned IDs: 994051,990256,991860,995516,992915,991799,995755,990862,992596,992165,997437,993396,999725,997305,996811,999251,991052,994879,995963,992546,991690,995184,996406,990958,996942,991982,996918,990096,994266,997154\n", "Ground Truth: 994051,990256,991860,995516,997645,992915,997811,991799,995755,990862,992596,992165,997437,995132,997429,993396,999725,993336,992238,997305,996856,996811,999251,994335,991052,994879,992357,995963,990178,992546\n", - "Recall: 0.8000\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 614\n", + "Returned IDs: 995769,995850,999606,998837,997275,993237,995004,990043,996686,994612,991407,990514,997486,994785,992612,995557,998657,991307,994967,995515,990580,991510,999714,993091,993570,998244,990965,995788,996402,992907\n", + "Ground Truth: 995769,998035,995850,999606,998837,997275,993237,995004,990043,996686,994612,991407,999888,998101,990514,997486,994785,992612,995557,998657,991307,994967,996561,995515,990580,991510,999714,993091,993570,998244\n", + "Recall: 0.8667\n", "\n", "Query ID: 615\n", - "Returned IDs: 996116,990701,997374,994350,997207,998913,997209,992024,994120,990126,995704,991156,994168,995482,997628,992599,992251,992871,993240,993384,990614,994730,999402,992156,993857,994728,995633,998982,991943,996481\n", + "Returned IDs: 996116,990701,997374,997207,998913,997209,992024,994120,990126,995704,994168,995482,997628,992599,993240,993384,990614,994730,999402,992156,993857,994728,995633,998982,991943,996481,990012,998495,998156,998722\n", "Ground Truth: 996116,990701,997374,994350,997207,998913,997209,992024,994120,990126,995704,999586,991156,994168,995482,997628,992599,992251,992871,993240,993384,990614,994730,999402,992156,993857,994728,995633,998982,991943\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 616\n", - "Returned IDs: 998566,997193,995691,997126,990305,994064,993566,996194,990184,996451,991636,996421,993127,998466,995482,990189,996170,990776,993678,999025,992841,995883,995620,992944,990948,993920,999158,992067,993181,994351\n", + "Returned IDs: 998566,997193,995691,997126,990305,993566,996194,990184,991636,996421,993127,998466,995482,990189,991157,990776,993678,991938,995883,992944,990948,991783,993920,999158,993181,996222,997674,998104,991052,994835\n", "Ground Truth: 998566,997193,995691,997126,990305,994064,993566,996194,990184,996451,991636,996421,993127,998466,995482,990189,991157,996170,990776,993678,999025,999586,992704,991938,992841,995883,990827,990052,995620,992944\n", - "Recall: 0.8000\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 617\n", + "Returned IDs: 991128,999282,990096,992557,999045,999079,999831,994661,999411,993721,997442,998510,997766,993744,996209,998302,997821,993139,992209,997906,999806,996494,999384,996893,990584,995206,991650,993215,994416,998359\n", + "Ground Truth: 991128,999282,990096,992557,997418,999045,999079,999831,994661,999411,993721,997442,998510,997524,997766,993744,996209,998302,997821,993139,992209,997906,998419,999806,996494,999384,996893,990584,995206,991650\n", + "Recall: 0.9000\n", "\n", "Query ID: 618\n", - "Returned IDs: 998888,999555,991944,990991,999584,991100,992638,991188,997154,997850,996393,995655,990167,998006,993948,993749,990423,990558,990154,994727,994467,993381,997030,993956,990361,994374,994643,991385,991596,998257\n", + "Returned IDs: \n", "Ground Truth: 998888,999555,991944,990991,999584,991100,992638,991188,997154,997850,996393,995655,990167,998006,993948,993749,990423,990558,990154,994727,994467,993381,997030,993956,990361,994374,991836,994643,991385,991596\n", - "Recall: 0.9667\n", + "Recall: 0.0000\n", "\n", "Query ID: 619\n", - "Returned IDs: 997995,997455,995095,996609,992281,997993,998221,990432,993219,995401,992619,991836,990435,993209,999820,990126,994587,995354,999923,992979,997185,995289,993412,999160,990157,993656,995633,995374,999240,998297\n", + "Returned IDs: 997995,997455,995095,996609,997993,998221,990432,993219,995401,992619,991836,990435,996088,999820,990126,994587,995354,999923,992979,997185,995289,993412,999160,993656,995633,995374,999240,998297,994993,999021\n", "Ground Truth: 997995,997455,995095,996609,992281,997993,998221,992087,990432,993219,991445,995401,992619,991836,990435,996088,993209,999820,990126,994587,995354,999923,992979,997185,995289,993412,999160,990157,993656,995633\n", - "Recall: 0.9000\n", + "Recall: 0.8333\n", "\n", "Query ID: 620\n", - "Returned IDs: 997652,997897,998428,992052,993621,998522,993171,990956,999002,999179,998250,995421,992028,992747,990409,997368,990626,994280,994765,995213,994743,994742,997494,991605,996328,996461,999587,999927,990364,993175\n", + "Returned IDs: 997652,997897,998428,992052,993621,998522,993171,990956,999002,999179,998250,995421,992028,992747,990409,990626,994280,994765,995213,994742,991605,996328,996461,999927,990364,993175,991212,996001,992437,993347\n", "Ground Truth: 997652,997897,998428,992052,993621,998522,993171,990956,999002,999179,998250,995421,992028,992747,990409,997368,990626,994280,994765,995213,994743,997723,994742,997494,991605,996328,996461,999587,999927,990364\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 621\n", - "Returned IDs: 995941,997371,990645,993507,994757,993818,998246,991782,995367,995755,996890,997192,993005,994281,990994,997194,992053,990863,999374,991379,991059,993518,997984,995353,999247,990969,993739,999290,992978,991456\n", + "Returned IDs: 995941,997371,990645,993507,994757,993818,998246,991782,995367,995755,996890,997192,994281,990994,997194,992053,990863,999374,991379,993518,997984,995353,999290,992978,991498,993852,995090,990094,990917,990647\n", "Ground Truth: 995941,997371,990645,993507,994757,993818,998246,991782,995367,995755,996890,997192,993005,994281,990994,997194,994183,992053,990863,999374,991379,993326,991059,993518,997984,995353,999247,997022,990969,993739\n", - "Recall: 0.9000\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 622\n", + "Returned IDs: 993539,993564,992167,992225,998758,999216,995701,994167,999144,999219,991459,993834,997258,995562,992429,993766,999783,992384,996368,991256,999041,993342,994906,996744,995132,999899,998232,993139,998720,992069\n", + "Ground Truth: 993539,993564,992167,992225,998758,999216,995701,994167,999144,999219,991459,993834,997258,995562,992429,993766,999783,992384,996368,991256,990041,999041,993342,996283,993517,994906,996744,995132,999899,999677\n", + "Recall: 0.8667\n", "\n", "Query ID: 623\n", - "Returned IDs: 992558,999564,992738,995724,998827,994372,993140,997124,990273,993077,995983,991009,997158,999146,996918,990538,992170,992179,994163,991260,991905,991157,997775,995411,999639,994742,993706,992615,998142,997859\n", + "Returned IDs: 992558,999564,992738,995724,998827,994372,997124,990273,993077,995983,991009,997158,999146,996918,990538,992170,992179,994163,991260,991905,991157,997775,999639,995401,993706,992615,990479,998142,997859,999908\n", "Ground Truth: 992558,999564,992738,995724,998827,994372,993140,997124,990273,993077,995983,991009,995281,997158,991585,999146,996918,990538,992170,992179,997411,994163,991260,991905,991157,997775,995411,999639,994742,995401\n", - "Recall: 0.8667\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 624\n", + "Returned IDs: 992832,998543,992595,997319,992713,990889,995633,991850,996328,998750,994891,992239,997698,999788,992077,991981,994742,999833,990516,992561,990270,992318,999827,998525,995030,990814,991769,999767,991590,999924\n", + "Ground Truth: 992832,998543,992595,997319,992713,990889,995633,991850,996328,998750,994891,992239,997698,991731,999788,992077,991981,994742,999833,990516,992561,990270,995222,992318,999827,998525,995030,990814,991769,999767\n", + "Recall: 0.9333\n", "\n", "Query ID: 625\n", - "Returned IDs: 997977,992823,993504,995168,994468,991425,992082,997618,994672,996217,998303,997140,998007,995536,995231,993260,990196,997002,993071,997790,997918,995740,999111,996001,994955,995203,991863,996884,990350,997913\n", + "Returned IDs: 997977,992823,993504,995168,994468,991425,992082,997618,994672,996217,998303,995536,995231,993260,990196,997002,993071,997790,997918,995740,999111,996001,994955,995203,991863,995167,996884,990350,992862,993489\n", "Ground Truth: 997977,992823,993504,995168,994468,991425,992082,997618,994672,996217,998303,997140,998007,995536,995231,993260,990196,997002,993071,997790,997918,995740,999111,996001,994955,995203,991863,995167,996884,990350\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 626\n", - "Returned IDs: 994906,993289,993342,999587,992664,993820,991459,995401,994946,992636,998855,998536,993275,990399,998623,999977,997332,996764,997002,996001,999820,999724,992179,996147,997910,994743,997790,993715,998831,999453\n", + "Returned IDs: 994906,993289,993342,999587,992664,993820,991459,995401,994946,992636,998855,998536,993275,990399,998623,999977,996764,997002,996001,999820,999724,992179,996147,997910,994743,997790,993715,998831,999453,994955\n", "Ground Truth: 994906,993289,993342,999587,992664,993820,991459,995401,994946,992636,998855,998536,993275,990399,998623,999977,997332,996764,997002,996001,999820,999724,996371,992179,996147,997910,994743,997790,993715,998831\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 627\n", + "Returned IDs: 992636,990998,996183,996408,998379,993335,994682,996594,991919,992026,999334,996328,993715,990096,990392,994830,993342,990173,997340,991007,993448,992764,994742,995695,995299,998716,990027,990257,993001,992479\n", + "Ground Truth: 992636,990998,996183,996408,998379,993335,994682,996594,991919,992026,991620,999334,996328,993715,990096,990392,994830,992933,995326,993342,990173,997340,991007,993448,992764,994742,995695,995299,998716,990027\n", + "Recall: 0.9000\n", "\n", "Query ID: 628\n", - "Returned IDs: 999457,990194,998279,999555,999994,999205,993461,990167,999223,998674,999430,992628,998072,993076,994109,993224,998586,998570,991038,997415,991610,995234,995848,991025,999985,998086,998078,995907,997506,994731\n", + "Returned IDs: 999457,990194,998279,999555,999994,999205,993461,990167,999223,998674,999430,992628,993076,994109,993224,998586,998570,991610,995234,995848,991025,998086,998078,995907,997506,994731,990402,997205,991987,997084\n", "Ground Truth: 999457,990194,998279,999555,999994,999205,993461,990167,999223,998674,999430,992628,998072,993076,994109,993224,998586,998570,991038,997415,991610,995234,995848,991025,999985,998086,998078,995907,995082,997506\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 629\n", - "Returned IDs: 993250,995135,990898,998751,999649,998660,995865,991582,991172,991840,996854,994819,998011,998996,996352,995365,999676,998915,993640,997479,990965,990310,993253,995693,990979,991999,994415,998540,997192,994497\n", + "Returned IDs: 995135,990898,998751,999649,998660,991172,991840,996854,998011,995365,999676,998915,997291,993640,990965,993253,995693,993404,991999,994415,998540,997192,994497,996976,993143,993809,997403,991833,992087,995090\n", "Ground Truth: 993250,995135,990898,998751,999649,998660,995865,991582,991172,991840,996854,994819,998011,998996,996352,997641,995365,999676,998915,997291,993640,997479,990965,990310,993253,995693,993404,990979,991999,994415\n", - "Recall: 0.9000\n", + "Recall: 0.6667\n", "\n", "Query ID: 630\n", - "Returned IDs: 994852,992190,999451,997106,990884,999797,994622,996894,995138,998455,995752,992349,993789,997818,993355,999645,997076,990096,996884,995769,993706,998974,992668,996196,999807,999639,996262,990522,993222,991609\n", + "Returned IDs: 994852,992190,999451,997106,990884,999797,994622,996894,995138,995752,992349,993789,997818,993355,999645,997076,990096,996884,993706,998974,992668,996196,999807,999639,996262,990522,995210,993222,990957,995436\n", "Ground Truth: 994852,992190,999451,997106,990884,999797,994622,996894,995138,998455,995752,992349,993789,997818,993355,999645,997076,990096,996884,995769,993706,998974,992668,996196,999807,991637,999639,992612,996262,990351\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 631\n", + "Returned IDs: 999934,999225,994529,997837,999372,990593,993856,990226,990126,995083,990340,998338,995300,992859,995633,990435,994742,992450,996042,995861,993100,992434,993731,996328,999830,991746,993433,999672,992449,993561\n", + "Ground Truth: 999934,999225,994529,997837,999372,990593,993856,990226,990126,995083,996372,990340,998338,995300,992859,995633,990435,994742,992450,996042,996306,995861,993100,992434,997185,993731,996328,999830,991746,993433\n", "Recall: 0.9000\n", "\n", "Query ID: 632\n", - "Returned IDs: 995210,996808,997918,996062,998436,993512,990366,993289,990891,997002,991425,994821,992711,990885,994468,994042,993260,994959,996001,993504,990622,995826,994694,991833,993268,999091,991311,992420,993157,994538\n", + "Returned IDs: 995210,996808,997918,996062,998436,993512,990366,993289,990891,997002,991425,997332,994821,990885,994042,993260,994959,996001,993504,990622,995826,994694,991833,991311,993157,998303,990491,996551,992571,994200\n", "Ground Truth: 995210,996808,997918,996062,998436,993512,990366,993289,990891,997002,991425,997332,994821,992711,990885,994468,994042,993260,992612,994959,997531,996001,993504,990622,995826,998611,994694,991833,993268,999091\n", - "Recall: 0.8667\n", + "Recall: 0.7667\n", "\n", "Query ID: 633\n", - "Returned IDs: 999854,990584,998228,993637,998436,997790,999827,996062,990557,995678,994472,997918,990091,997002,998039,999516,996001,993906,993765,997586,997920,992576,991994,992161,995107,993484,997453,992395,990080,998734\n", + "Returned IDs: 999854,998228,993637,998436,997790,999827,996062,990557,995678,994472,997918,990091,997002,998039,999516,996001,993906,995502,997586,992576,991994,992161,995107,993484,997453,992395,990080,994964,998416,993875\n", "Ground Truth: 999854,990584,998228,993637,998436,997790,999827,996062,990557,995678,994472,997918,990091,997002,998039,993238,999516,996001,993906,993765,995502,997586,992576,997920,991994,992161,995107,993484,997453,992395\n", - "Recall: 0.9333\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 634\n", + "Returned IDs: 990545,995260,992119,995942,997320,997913,994964,994100,998550,996143,996351,999647,992546,993877,995886,998495,998500,990958,991743,990648,998302,996219,997371,996554,998458,997984,994164,990936,999331,999374\n", + "Ground Truth: 990545,995260,992119,995942,997320,997913,994964,993154,994100,998550,996143,996351,999647,992546,993877,995886,998495,991720,998500,990958,991743,990648,998302,996219,997371,993238,994898,996554,998458,997984\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 635\n", + "Returned IDs: 995702,991973,992521,992703,997822,993293,998162,994409,990930,995766,994140,994742,998779,996368,993738,993561,991185,997723,995648,993831,998633,998165,997320,990845,995289,992448,995216,995052,995679,994711\n", + "Ground Truth: 995702,991973,992521,992703,997822,993293,998162,994409,990930,995766,994140,994742,999134,998779,996368,993738,993561,991185,997723,995648,993831,991603,998633,998165,997320,996042,990845,998999,995289,992448\n", + "Recall: 0.8667\n", "\n", "Query ID: 636\n", - "Returned IDs: 995557,999604,996596,990310,993748,999592,998899,993237,995365,991593,993279,996748,995588,993091,992207,998917,995952,993675,992195,994856,998997,998057,999039,998654,994263,992085,995072,991519,994964,997275\n", + "Returned IDs: 995557,996596,990310,993748,998899,993237,995365,991593,993279,996748,995588,993091,998917,995794,993675,992195,994856,998997,998057,999039,994263,992085,995072,994964,997275,992059,996007,999784,993083,991157\n", "Ground Truth: 995557,999604,996596,990310,993748,999592,998899,993237,995365,991593,993279,996748,995588,993091,992207,998917,995952,995794,993675,992195,994856,998997,998057,999039,998654,994263,992085,995072,994005,991519\n", - "Recall: 0.9333\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 637\n", + "Returned IDs: 998357,995965,993500,991977,993975,993583,994614,992376,993077,994760,996663,999171,998406,998142,999763,995408,992535,991740,996158,991432,994165,990249,993786,990908,993609,993839,992494,998879,995459,993810\n", + "Ground Truth: 998357,995965,993500,991977,993975,993583,994614,992376,993077,994760,996663,999171,998406,998142,999763,995408,992535,991740,996158,991432,994165,990249,993786,999008,990908,993609,993839,992494,998879,995459\n", + "Recall: 0.9667\n", "\n", "Query ID: 638\n", - "Returned IDs: 999638,990926,992349,992718,997816,997859,994163,991540,996894,995138,996318,994010,991498,991977,998697,998417,997113,998118,998334,999387,994964,992124,994165,990908,990343,995536,994972,991637,999908,995986\n", + "Returned IDs: 999638,990926,992349,992718,997816,997859,994163,991540,996894,995138,996318,994010,991498,991977,998697,998417,997113,998118,998334,999387,994964,992124,994165,990908,990343,995536,994972,992558,990083,990772\n", "Ground Truth: 999638,990926,992349,992718,997816,997859,994163,991540,993948,996894,995138,996318,994010,991498,991977,998697,998417,997113,998118,998334,999387,994964,992124,994165,990908,990343,995536,994972,991637,999908\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", "\n", "Query ID: 639\n", - "Returned IDs: 994137,999358,995378,990560,990390,999363,991900,998462,995942,998966,992310,991396,993403,990146,993072,991444,997972,993484,999229,990742,994964,997880,994479,999110,997539,999710,990311,996629,998365,995949\n", + "Returned IDs: 994137,999358,995378,990560,990390,999363,991900,998462,995942,998966,992310,991396,993403,990146,991444,997972,993484,990742,994964,994479,999110,999390,997539,999710,990311,996629,998365,995949,996959,994978\n", "Ground Truth: 994137,999358,995378,990560,990390,999363,991900,998462,995942,998966,992310,991396,993403,990146,993072,991444,997972,993484,991178,999229,990742,994964,997880,994479,999110,993284,999390,997539,999710,990311\n", - "Recall: 0.9000\n", + "Recall: 0.8333\n", "\n", "Query ID: 640\n", - "Returned IDs: 996127,993839,998220,999860,999639,993561,991923,990550,990303,994117,995013,991958,996181,996196,998593,991332,993489,990167,996391,994372,999393,991350,996379,995562,994796,998186,995983,998514,995855,998697\n", + "Returned IDs: 996127,993839,999860,999639,993561,991923,990550,994117,995013,991958,992473,996196,998593,991332,993489,990167,996391,994372,991350,996379,995562,998186,995983,998697,995894,996001,998142,998303,990096,995701\n", "Ground Truth: 996127,993839,998220,999860,999639,993561,991923,990550,990303,994117,995013,991958,992473,996181,996196,998593,991332,993489,990167,996391,994372,999393,991350,991212,996379,995562,994796,998186,995983,998514\n", - "Recall: 0.9333\n", + "Recall: 0.7667\n", "\n", "Query ID: 641\n", - "Returned IDs: 991810,996809,990127,993817,999788,994682,994742,992827,998063,999183,992372,997349,997319,993744,995633,999852,990223,995222,995895,990173,995401,998536,998495,990986,990722,996408,997401,990392,991620,998525\n", + "Returned IDs: 991810,996809,990127,993817,999788,994682,994742,992827,998063,992372,997349,997319,993744,995633,999852,990223,995895,990173,998536,998495,990986,990722,996408,997401,990392,997529,991620,998525,992832,997630\n", "Ground Truth: 991810,996809,990127,993817,999788,994682,994742,992827,998063,999183,992372,997349,997319,993744,995633,999852,990223,995222,992318,995895,990173,995401,998536,998495,990986,990722,996408,997401,990392,997529\n", - "Recall: 0.9333\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 642\n", + "Returned IDs: \n", + "Ground Truth: 999338,998921,997292,997301,994942,995659,996516,997873,992451,997984,998721,998445,990512,997668,997320,995306,999271,990560,990405,995045,995961,994964,997168,997286,999904,995623,992176,999526,995874,999533\n", + "Recall: 0.0000\n", "\n", "Query ID: 643\n", - "Returned IDs: 997749,999411,994909,995421,994765,993727,992934,998123,990521,994280,991491,996276,990844,994541,993678,998934,995963,997897,990998,995110,994697,998510,994809,992112,990021,993448,999601,992457,999179,995401\n", + "Returned IDs: 997749,999411,994909,995421,994765,993727,992934,990521,994280,991491,996276,994541,998934,995963,997897,995110,994697,994809,993448,999601,992457,996657,995203,990205,992052,995456,991850,998522,992933,998779\n", "Ground Truth: 997749,999411,994909,995421,994765,993727,992934,998123,990521,994280,999453,991491,996276,990844,994541,993678,998934,995963,997897,990998,995110,994697,998510,994809,992112,990021,993448,999601,992457,999179\n", - "Recall: 0.9667\n", + "Recall: 0.7000\n", "\n", "Query ID: 644\n", - "Returned IDs: 996296,991456,990176,997022,994842,996075,990742,997699,992931,992880,997178,990957,997918,991131,990949,995417,998540,993253,996592,993696,998967,992442,991634,995336,992942,993529,994100,999710,992277,992978\n", + "Returned IDs: \n", "Ground Truth: 996296,991456,990176,997022,994842,996075,990742,997699,992931,992880,997178,990957,997918,991131,990949,995417,998540,993253,996592,993696,998967,992442,991634,995336,993529,992942,994100,999710,992277,995981\n", - "Recall: 0.9667\n", + "Recall: 0.0000\n", "\n", "Query ID: 645\n", - "Returned IDs: 997867,991740,993839,992376,998357,991977,993583,998879,995965,990908,993647,994614,991064,999008,993975,990894,991225,995491,997547,996663,992333,995459,993681,996585,999022,994254,993500,994398,999763,999387\n", + "Returned IDs: 997867,991740,993839,992376,998357,991977,993583,998879,995965,990908,993647,997407,994614,999008,993975,990894,991225,995491,997547,996663,992333,995459,993681,996585,999022,994254,993500,994398,999763,999387\n", "Ground Truth: 997867,991740,993839,992376,998357,991977,993583,998879,995965,990908,993647,997407,994614,991064,999008,993975,990894,991225,995491,997547,996663,992333,995459,993681,996585,999022,994254,993500,994398,999763\n", "Recall: 0.9667\n", "\n", "Query ID: 646\n", - "Returned IDs: 997643,991309,994660,992638,991797,998277,994393,992943,997683,993566,993749,994294,997568,993666,999156,997110,994656,997030,992926,995044,997429,996393,990554,995994,997114,998888,995253,990132,991937,998858\n", + "Returned IDs: 997643,991309,994660,992638,991797,998277,994393,992943,997683,999530,993566,993749,994294,997568,993666,999156,997110,994656,997030,995044,997429,996393,990554,997114,998888,995253,990132,991937,998858,992369\n", "Ground Truth: 997643,991309,994660,992638,991797,998277,994393,992943,997683,999530,993566,993749,994294,997568,993666,999156,997110,994656,997030,992926,995044,997429,996393,990554,995994,997114,998888,995253,990132,991937\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 647\n", - "Returned IDs: 990679,994552,998734,995855,995250,999473,993154,997370,994789,995203,999954,994212,999898,996075,996623,997918,993243,990352,993484,990652,994829,990470,993646,999876,999384,999132,997920,990957,992548,996001\n", + "Returned IDs: 990679,998734,995250,999473,993154,997370,994789,995203,999898,996075,996623,997918,990352,993484,990652,994829,990470,993646,996452,999876,999384,990957,992548,996001,990123,999653,999017,990448,997688,994268\n", "Ground Truth: 990679,994552,998734,995855,995250,999473,993154,997370,994789,995203,999954,994212,999898,996075,996623,997918,993243,990352,993484,990652,994829,990470,993646,990809,996452,999876,999807,999384,999132,997920\n", - "Recall: 0.9000\n", + "Recall: 0.7000\n", "\n", "Query ID: 648\n", - "Returned IDs: 994125,990641,996609,990655,992604,997918,995401,997403,991844,995289,998913,991996,996809,993412,994317,995718,993433,996411,996196,991179,996075,993725,995274,990126,997906,998999,994603,993561,998495,995881\n", + "Returned IDs: 994125,990641,996609,990655,992604,997918,995401,991844,995289,998913,991996,996809,995718,996411,996196,996075,993725,995274,990126,998999,994603,993561,998495,995881,995633,996918,995886,990096,995200,995455\n", "Ground Truth: 994125,990641,996609,990655,992604,997918,995401,998383,997403,991844,998913,995289,991996,996809,993529,993412,994317,995718,993433,996411,996196,991179,996075,993725,995274,990126,997906,998999,994603,993561\n", - "Recall: 0.9333\n", + "Recall: 0.7333\n", "\n", "Query ID: 649\n", - "Returned IDs: 996360,998045,993561,998545,991052,991958,993433,996042,995702,996322,994409,997822,991923,993412,996867,994419,996769,995679,995648,994549,994280,997868,997386,995873,998564,995536,997674,997675,997289,990137\n", + "Returned IDs: 996360,998045,993561,998545,991052,991958,993433,996042,995702,996322,994409,997822,991923,996867,994419,996769,995679,995648,994549,994280,997868,997386,998564,995536,997674,997675,997289,990137,999530,997838\n", "Ground Truth: 996360,998045,993561,998545,991052,991958,999809,993433,996042,995702,996322,994409,990882,997822,991923,993412,996867,994419,996769,995679,995648,994549,994280,997868,997386,995873,998564,995536,997674,997675\n", - "Recall: 0.9333\n", + "Recall: 0.8667\n", "\n", "Query ID: 650\n", - "Returned IDs: 993839,996127,996658,991432,998372,993718,995340,990061,990487,991028,998079,997864,991923,993976,998370,991743,991350,996391,992813,990638,995691,994514,995515,993583,998779,998879,997914,998686,992929,990602\n", + "Returned IDs: 993839,996127,996658,991432,998372,995340,990061,990487,991028,997864,991923,993976,998370,991743,991350,996391,990638,995691,994514,995515,993583,998779,998879,997914,995453,992929,990602,998853,996742,996379\n", "Ground Truth: 993839,996127,996658,991432,998372,993718,995340,990061,990487,991028,998079,997864,991923,993976,998370,991743,991350,996391,992813,990638,995691,995515,994514,993583,998779,998879,997914,998686,995453,992929\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 651\n", + "Returned IDs: 995562,992315,990596,998303,991037,999144,997420,991332,993489,996562,992819,990834,997926,996908,996286,992199,993275,999435,998331,995894,998495,996233,993609,995349,998855,990630,997506,991425,995536,993560\n", + "Ground Truth: 995562,992315,990596,998303,991037,999144,997420,991332,993489,997531,996562,992819,991212,990834,997926,996908,996286,995740,992199,993275,999435,998331,995894,998495,996233,994946,993609,995835,995349,998855\n", + "Recall: 0.8333\n", "\n", "Query ID: 652\n", - "Returned IDs: 999328,996646,997320,999647,997913,994450,993253,999109,997470,999948,991743,999324,998967,998550,998287,990182,994192,990650,996296,992969,999390,993387,998302,997826,999701,994266,998644,991593,993031,991616\n", + "Returned IDs: 996646,997320,999647,997913,994450,993253,999109,997470,999948,999324,998550,990650,992950,992969,993387,998302,997826,999701,994266,991593,993031,990066,992972,992147,994604,993579,999036,991479,993662,999535\n", "Ground Truth: 999328,996646,997320,999647,997913,994450,993253,999109,997470,999948,991743,999324,998967,998550,998287,990182,994192,990650,997921,996296,992950,992969,999390,999258,993387,998302,997826,999701,994266,998644\n", - "Recall: 0.9000\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 653\n", + "Returned IDs: 996690,992608,994384,996196,997775,992215,997902,991772,996140,993670,994163,992661,995983,993706,991550,998950,996726,995019,991009,996262,990538,998697,990824,999639,998364,991822,995552,995972,997368,993500\n", + "Ground Truth: 996690,992608,994384,996196,997775,992215,997902,991772,996140,993670,994163,992661,995983,993706,991550,999908,998950,996726,995019,991009,996262,990538,998697,990824,999639,998364,991822,995552,995972,997368\n", + "Recall: 0.9667\n", "\n", "Query ID: 654\n", - "Returned IDs: 995374,993636,997192,993852,997571,998844,994281,999374,996554,994850,999953,995833,990275,992147,992183,994218,990432,992309,998011,998263,995354,999144,993529,995336,999317,993443,999883,999804,992854,996075\n", + "Returned IDs: 997192,993852,997571,998844,994281,999374,996554,994850,999953,995833,990275,992147,992183,994218,990432,992309,998011,998263,995354,999144,993529,995336,999317,993443,999883,999804,992854,996075,992452,996721\n", "Ground Truth: 995374,993636,997192,993852,997571,998844,994281,999374,996554,994850,999953,995833,990275,992147,992183,994218,990432,992309,998011,998263,995354,999144,993529,995336,999317,993443,999883,999562,999804,992854\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", "\n", "Query ID: 655\n", - "Returned IDs: 996247,991459,999804,990294,999639,995894,997918,991979,994125,996407,991630,996500,997957,998022,992748,992473,993948,996918,992516,992429,995932,990897,992558,993188,999306,991657,992293,995701,995336,998758\n", + "Returned IDs: 996247,991459,999804,990294,999639,995894,997918,991979,994125,996407,991630,996500,994911,998022,992473,996918,992516,992429,995932,990897,992558,993188,999306,993906,992293,995701,995336,998758,998697,993848\n", "Ground Truth: 996247,991459,999804,990294,999639,995894,997918,991979,994125,996407,995132,991630,996500,997957,994911,998022,992748,992473,993948,996918,992516,992429,995932,990897,992927,992558,993188,999306,993906,991657\n", - "Recall: 0.8667\n", + "Recall: 0.8000\n", "\n", "Query ID: 656\n", - "Returned IDs: 999544,999676,992348,999453,994800,991787,999452,995968,992612,994195,993683,991673,996309,992907,992309,992739,990080,993781,991485,995372,999686,998607,992298,999254,994964,993633,991999,995365,999951,997498\n", + "Returned IDs: 999544,999676,992348,999453,994800,991787,999452,995968,992612,994195,993683,991673,996309,992907,992309,992739,990080,993781,991485,995372,999686,998607,999254,994964,991999,991510,999210,990334,990233,991427\n", "Ground Truth: 999544,999676,992348,999453,994800,991787,999452,995968,992612,994195,993683,991673,996309,992907,992309,992739,990080,993781,991485,995372,996352,999686,998607,992298,999254,994964,993633,991999,995365,999951\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 657\n", + "Returned IDs: 995861,994751,990091,998055,990133,993939,992780,999673,997103,999740,991334,993730,995191,999378,993877,992240,990145,997026,998610,995895,990712,990531,992652,996328,994472,991390,995735,999458,990705,995326\n", + "Ground Truth: 995861,994751,990091,998055,990133,993939,992780,999673,997103,999740,991334,993730,995191,999441,996752,999378,993877,992240,997077,997933,990145,997026,999436,998610,995895,996566,990712,990531,993118,992652\n", + "Recall: 0.7667\n", "\n", "Query ID: 658\n", - "Returned IDs: 994488,994416,992318,999060,993772,996809,999179,997332,990843,998950,991650,994541,997679,996001,993603,998543,994955,998218,993347,993721,991142,990930,995899,995678,991409,999320,996024,993293,997442,999009\n", + "Returned IDs: 994488,994416,992318,999060,993772,996809,997332,990843,998950,991650,994541,997679,996001,993603,994955,993347,993721,991142,990930,993079,995899,995678,991409,995167,999320,996024,993293,999643,990096,992944\n", "Ground Truth: 994488,994416,992318,999060,993772,996809,999179,997332,990843,998950,991650,994541,997679,996001,993603,998543,994955,998218,993347,993721,991142,990930,993079,995899,995678,991409,995167,999320,996503,996024\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 660\n", + "Returned IDs: 990516,995948,994893,995633,995030,998557,997319,996747,996867,994009,992289,999235,996672,994283,994742,995190,995289,991810,996302,999957,997401,995558,999875,997406,999160,992239,998196,998750,994204,995302\n", + "Ground Truth: 990516,995948,994893,995633,995030,998557,997319,996747,996867,994009,992289,999235,996672,994283,994742,995190,995289,991810,996302,993856,999957,997401,995558,999875,997406,999805,995348,999160,992239,998196\n", "Recall: 0.9000\n", "\n", "Query ID: 661\n", - "Returned IDs: 993461,990479,996918,991052,996535,990897,998267,993014,999646,997723,997586,990041,991758,998564,992933,998518,999980,995963,997910,994282,992979,997675,997216,991891,993412,992068,991666,992558,990942,998067\n", + "Returned IDs: 993461,990479,996918,991052,996535,990897,998267,993014,999646,997723,997586,990041,991758,998564,998518,999980,995963,997910,994282,992979,997675,997216,991891,992068,991666,992558,990942,998067,990256,997420\n", "Ground Truth: 993461,990479,996918,991052,996535,990897,998267,993014,999646,997723,997586,990041,991758,998564,992933,998518,999980,995963,997910,998924,994282,992979,997675,997216,991891,998105,993412,992068,991666,992558\n", - "Recall: 0.9333\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 663\n", + "Returned IDs: 990225,999555,990167,993076,997506,999373,994834,990361,993366,996393,994374,995257,996408,993001,998495,996797,993224,995218,998536,999430,998888,996209,991100,999994,997154,991610,992470,997182,990173,990589\n", + "Ground Truth: 990225,999555,990167,993076,997506,999373,994834,995082,990361,993381,991881,993366,996393,994374,995257,996408,993001,998495,996797,999841,997142,993224,995218,998536,995222,999430,998888,996209,991100,995326\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 664\n", + "Returned IDs: 994978,992228,993579,991137,995841,999109,993845,991615,997984,998477,990585,990512,991479,995372,997024,998445,995668,999036,999024,995641,995623,993387,997018,996653,999358,999982,996959,997320,991450,993066\n", + "Ground Truth: 994978,992228,993579,992789,991137,995841,999109,993845,991615,997984,998477,990585,990512,991479,995372,997024,991090,998445,995668,999036,999024,995641,991468,991209,995623,993387,997018,996653,999358,993853\n", + "Recall: 0.8333\n", "\n", "Query ID: 665\n", - "Returned IDs: 997777,990949,991593,997320,999290,993484,997370,992839,993253,999855,999535,990957,997539,995515,997372,995203,999453,990742,990890,994486,995942,999982,995400,999680,994826,992069,993389,999437,995596,993278\n", + "Returned IDs: 997777,990949,991593,997320,999290,993484,997370,992839,993253,999535,990957,997539,995515,997372,995203,999453,996002,994486,999982,995400,992069,995596,994964,998950,994366,999586,997913,994789,995826,991131\n", "Ground Truth: 997777,990949,991593,997320,999290,993484,997370,992839,993253,999855,999535,990957,997539,995515,997372,995203,999453,990742,990890,996002,994486,995942,999982,995400,999680,994826,992069,993389,999437,995596\n", - "Recall: 0.9667\n", + "Recall: 0.7333\n", "\n", "Query ID: 666\n", - "Returned IDs: 994954,992521,993438,994409,999866,992448,995640,995873,996368,997868,993561,996987,991969,995820,998255,996187,997091,990005,995701,998205,999250,997466,991349,998045,992426,991967,997723,997554,992937,997612\n", + "Returned IDs: 994954,992521,993438,994409,999866,992448,995640,996368,996769,997868,993561,996987,991969,995820,998255,996187,997091,990005,998205,999250,997466,991349,998045,992426,991967,997554,992937,997612,995536,998075\n", "Ground Truth: 994954,992521,993438,994409,999866,992448,995640,995873,996368,996769,997868,993561,996987,991969,995820,998255,996187,997091,990005,995701,998205,999250,997466,991349,998045,992426,991967,997723,997554,993517\n", - "Recall: 0.9333\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 667\n", + "Returned IDs: 997652,993324,993347,995332,991255,990646,991961,993180,993079,996591,997752,990776,997309,996370,999305,996777,998272,996455,994765,996210,997914,992052,994119,994092,995766,993621,998416,997698,996633,994435\n", + "Ground Truth: 997652,993324,993347,995332,991255,990646,999190,991961,993180,996810,993079,996591,997752,990776,997309,994300,996370,999305,996777,998272,996455,994765,996210,997914,991990,992052,994119,994092,995766,993621\n", + "Recall: 0.8667\n", "\n", "Query ID: 668\n", - "Returned IDs: 999117,993122,994467,995625,995044,991188,994643,997149,997154,992809,995207,999555,990991,999155,996395,992945,993795,999663,995257,993469,993995,993232,998861,996209,996393,992638,998536,998006,991385,997205\n", + "Returned IDs: \n", "Ground Truth: 999117,993122,994467,995625,995044,991188,994643,997149,997154,992809,995207,999555,990991,999155,996395,992945,993795,999663,995257,993469,993995,993232,998861,996209,996393,992638,998536,995401,998006,999431\n", - "Recall: 0.9333\n", + "Recall: 0.0000\n", "\n", "Query ID: 669\n", - "Returned IDs: 996489,990442,996981,996186,990249,990256,990209,992558,994334,992017,995332,997380,991426,996811,995552,990538,997155,995449,992179,999860,993473,995401,992170,999859,996196,993515,990620,995528,999715,999908\n", + "Returned IDs: 996489,990442,996981,990249,990256,990209,992558,994334,992017,995332,997380,991426,996811,995552,990538,997155,995449,992179,999860,993473,995401,992170,996196,993515,990620,999715,999908,994661,996429,991337\n", "Ground Truth: 999901,996489,990442,996981,996186,990249,990256,990209,992558,994334,992017,995332,997380,991426,996811,995552,999480,990538,997155,995449,992179,999860,993473,995401,992170,998213,999859,996196,993515,990620\n", - "Recall: 0.9000\n", + "Recall: 0.8333\n", "\n", "Query ID: 670\n", - "Returned IDs: 998289,998104,993324,992558,994732,990264,998950,996140,995289,996810,990769,994697,993079,997752,998697,996194,992988,995783,995281,996210,996370,994146,995359,994817,993018,996276,990956,997723,992052,997713\n", + "Returned IDs: 998289,998104,993324,992558,994732,990264,998950,996140,995289,994697,993079,997752,994514,997630,998697,996194,995963,995783,996210,996370,994817,993018,997723,992052,997713,991981,993280,993948,993478,998160\n", "Ground Truth: 998289,998104,993324,992558,994732,990264,998950,996140,995289,996810,990769,995928,994697,993079,997752,994514,997630,998697,996852,996194,992988,995963,995783,995281,996210,996370,994146,995359,994817,993018\n", - "Recall: 0.8333\n", + "Recall: 0.7333\n", "\n", "Query ID: 671\n", - "Returned IDs: 991925,996811,991743,998334,992702,993721,994119,990585,996918,993229,991977,999601,995375,996318,994662,998495,997481,999453,998849,992944,995873,999711,999707,995130,990640,996219,997436,996001,993121,991807\n", + "Returned IDs: 991925,991743,998334,992702,993721,994119,990585,993229,991977,999601,995375,996318,998495,997481,999453,998849,992944,999711,995130,996219,996001,993342,990489,997309,993241,996328,991994,994199,992169,998045\n", "Ground Truth: 991925,996811,991743,998334,992702,993721,994119,990585,996918,993229,991977,999601,995375,996318,994662,998495,997481,999453,998849,994081,992944,997732,995873,999711,999707,995130,990640,996219,997436,996001\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 672\n", + "Returned IDs: 991618,991163,991969,991349,999731,993813,992521,995820,995813,990603,994140,991117,993738,991795,997868,996277,990930,993561,995701,997928,990193,991922,998205,996169,995864,995640,992448,995395,992851,994409\n", + "Ground Truth: 991618,991163,991969,991349,999731,993813,992521,995820,995813,990603,994140,991117,993738,991795,997868,996277,990930,993561,995701,997928,990193,991922,994353,998205,996169,995864,995640,992448,995395,999866\n", "Recall: 0.9333\n", "\n", "Query ID: 673\n", - "Returned IDs: 999703,992640,993314,993895,991161,993580,992728,990479,999520,998463,992598,994672,994280,991731,998045,990942,990882,994844,993315,998812,993609,995815,999441,996867,999052,999373,997192,994920,992752,995196\n", + "Returned IDs: 999703,992640,993314,993895,991161,993580,992728,990479,998463,992598,994672,993433,994280,998045,990942,994844,993609,995815,996867,999373,994920,995196,996328,996861,993313,990069,997506,998331,995835,994838\n", "Ground Truth: 999703,992640,993314,993895,991161,993580,992728,990479,999520,998463,992598,994672,993433,994280,991731,998045,990942,990882,994844,993315,998812,993609,995815,999441,992617,996867,999052,999373,997192,994920\n", - "Recall: 0.9333\n", + "Recall: 0.7000\n", "\n", "Query ID: 674\n", - "Returned IDs: 996913,994136,996761,996067,997839,997790,990550,996089,998473,999363,993834,998934,993766,990897,998185,993478,994910,992645,992429,992750,996795,990139,999216,999041,996547,993809,999384,993143,994915,996611\n", + "Returned IDs: 996913,994136,996761,996067,997839,997790,990550,996089,998473,993834,998934,993766,990897,998185,993478,994910,992750,999216,999041,996547,993809,999384,994915,991795,993832,991692,991657,994280,998200,995855\n", "Ground Truth: 996913,994136,996761,996067,997839,996144,997790,990550,996089,998473,999363,993834,998399,998934,993766,990897,998185,993478,994910,992645,992429,992750,996795,990139,999216,999041,996547,993809,999384,996803\n", - "Recall: 0.9000\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 675\n", + "Returned IDs: 995467,997506,997168,992636,998536,993715,992282,998045,996769,990438,992521,998021,990951,995375,998697,999338,999526,990096,993637,998921,992825,997586,994016,990930,996408,996594,998097,991245,992576,993001\n", + "Ground Truth: 995467,997506,997168,992636,998536,993715,992282,998045,996769,990438,992521,993681,998021,990951,995375,998697,999338,999526,990096,993637,998921,992825,993719,997586,994016,996918,997723,992933,990930,995562\n", + "Recall: 0.8000\n", "\n", "Query ID: 676\n", - "Returned IDs: 999290,997370,996840,992944,993484,998734,996762,995886,993143,999367,996949,996438,993052,992558,993080,995203,991840,990334,999426,991348,990697,998455,997839,990580,995596,993504,996397,992169,999453,993630\n", + "Returned IDs: 999290,997370,996840,992944,993484,998734,995886,993143,999367,996949,996438,993052,992558,993080,995203,991840,990334,999426,991348,990697,998455,997839,990580,995596,993504,996397,992169,999453,990142,998244\n", "Ground Truth: 999290,997370,996840,992944,993484,998734,996762,995886,993143,999367,996949,996438,993052,992558,993080,995203,991840,990334,999426,991348,990697,998455,990351,997839,990580,995596,993504,996397,992169,999453\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 678\n", - "Returned IDs: 997348,999647,991582,994450,991428,996805,997913,993853,998426,991550,990893,999444,997918,993264,990231,994964,990797,997320,997880,996041,998921,998029,998550,990271,997301,993529,997586,994503,996809,994266\n", + "Returned IDs: 997348,999647,994450,991428,996805,997913,993853,998426,991550,990893,999444,997918,993264,990231,994964,990797,997320,996041,998921,998029,991721,997301,993529,997586,994503,996809,994266,990647,992862,999982\n", "Ground Truth: 997348,999647,996904,991582,994450,991428,996805,997913,993853,998426,993141,991550,990893,997918,999444,993264,990231,994964,993412,990797,997320,997880,996041,996296,998921,998029,997772,998550,991721,999683\n", - "Recall: 0.7667\n", - "\n", - "Query ID: 680\n", - "Returned IDs: 997920,996232,996219,993412,994894,998649,999144,995274,991029,992337,999343,994757,996052,999261,999311,996504,995881,996001,995502,994819,999026,993168,993529,990949,999647,997479,995200,990975,990934,996869\n", - "Ground Truth: 997920,996232,996219,993412,994894,998649,999144,995274,991029,992337,999343,994757,996052,999261,999311,996504,991445,995881,995401,996001,995502,994819,999026,993168,993529,990949,999647,997479,995200,990975\n", - "Recall: 0.9333\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 679\n", + "Returned IDs: 992604,997598,994187,998697,991844,995455,992128,999136,996645,995755,990540,995435,999306,992133,999867,995409,992051,993560,998022,999740,994125,999478,995780,995268,995527,993820,996649,999356,999795,993561\n", + "Ground Truth: 992604,997598,994187,997212,998383,998697,991844,995455,992128,999136,996645,995755,990540,995435,999306,992133,999867,995409,996129,992051,993560,998022,996915,999740,994125,999478,995780,995268,995527,993820\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 680\n", + "Returned IDs: 996232,996219,994894,998649,999144,995274,991029,992337,999343,994757,996504,996001,995502,999026,993168,990949,999647,997479,995200,990975,990934,991096,999740,995569,998106,996721,992446,997192,998572,998495\n", + "Ground Truth: 997920,996232,996219,993412,994894,998649,999144,995274,991029,992337,999343,994757,996052,999261,999311,996504,991445,995881,995401,996001,995502,994819,999026,993168,993529,990949,999647,997479,995200,990975\n", + "Recall: 0.6667\n", "\n", "Query ID: 681\n", - "Returned IDs: 997975,999137,999415,998188,993209,991671,993586,995778,994730,996302,991639,997723,999581,991507,996226,992289,999875,997544,993412,999054,996867,996852,995268,996187,990361,999957,998910,999980,992448,994742\n", + "Returned IDs: 997975,999137,999415,998188,993209,991671,993586,995778,994730,996302,991639,999581,991507,996226,992289,999875,997544,999054,996867,996852,995268,996607,996187,999957,998910,992448,994742,990631,995640,996633\n", "Ground Truth: 997975,999137,999415,998188,993209,991671,993586,995778,994730,996302,991639,997723,999581,991507,996226,992289,999875,997544,993412,999054,996867,996852,995268,996607,996187,990361,999957,998910,999980,995963\n", - "Recall: 0.9333\n", + "Recall: 0.8333\n", "\n", "Query ID: 682\n", - "Returned IDs: 998104,994300,993799,998536,993609,995963,990392,994119,990879,992617,994711,998568,992521,993324,996760,992131,997723,997752,995783,994409,999129,998162,996987,996328,997309,994280,997420,999886,996867,995401\n", + "Returned IDs: 998104,993799,998536,993609,995963,990392,994119,994711,992521,993324,996760,997752,995783,994409,999129,998162,996987,996328,997309,994280,997420,996867,995401,994541,991712,993180,992569,992832,997320,997176\n", "Ground Truth: 998104,994300,993799,998536,993609,995963,990392,994119,990879,992617,994711,998568,992521,993324,999840,996760,992131,997723,997752,992289,995783,994409,999129,998162,996987,996328,997309,994280,997420,999886\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 683\n", + "Returned IDs: 999466,998993,995027,995890,996960,996611,998335,995626,992864,998279,990712,996537,990806,992151,998935,992726,991098,996082,996232,991243,999880,994280,995214,990705,996328,994098,993219,996599,993224,994351\n", + "Ground Truth: 999466,998993,995027,995890,996960,996611,998335,995626,992864,998279,990712,996537,990806,992151,998935,992726,995167,991098,996082,991232,996232,991243,999880,994280,995214,996328,990705,994098,993219,996599\n", "Recall: 0.9333\n", "\n", + "Query ID: 684\n", + "Returned IDs: 994742,997319,999367,995107,995401,996001,996219,995633,990089,995289,990096,998495,996411,992827,994532,992558,995200,999791,995536,990585,999026,997301,998950,991968,994972,994783,991588,994743,999009,992944\n", + "Ground Truth: 994742,997319,999367,995222,999377,995107,995401,996001,996219,995633,990089,995289,995611,993080,999647,990096,998495,996411,992827,994532,992558,990077,993725,995200,999791,991921,990585,995536,999026,997301\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 685\n", + "Returned IDs: 998184,994964,990361,998779,995204,995289,993238,995274,995702,996867,997822,997320,998999,991588,995633,995216,990845,992703,993648,999160,995052,995766,997192,993831,994742,993848,999587,992994,998458,998165\n", + "Ground Truth: 998184,994964,990361,998779,995204,995289,993238,995274,995702,996867,993673,997822,996959,997320,998999,993209,992334,991588,995633,999026,993389,993412,992582,995216,990845,992703,993648,999160,995052,995766\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 686\n", + "Returned IDs: 994776,999982,999180,998302,992848,992852,992384,998246,997320,997661,994415,997053,990275,997913,992119,991545,993853,999343,998660,994266,996075,990949,998580,992994,990958,993484,992896,998458,991264,991335\n", + "Ground Truth: 994776,999982,990020,999180,998302,992848,992852,997620,992384,998246,997320,997661,994415,997053,990275,997913,992119,991545,993853,997895,999343,998660,994266,996075,990949,998580,992994,990958,996143,993154\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 687\n", + "Returned IDs: 997584,992635,994971,993785,992041,997550,992601,995596,992671,990914,996075,997661,993447,998227,998498,993125,992733,994886,995372,996755,999426,992617,996186,996631,992515,995515,990142,993129,996875,999317\n", + "Ground Truth: 997584,992635,994971,993785,992041,997550,992601,995596,992671,990914,996075,997661,993447,998227,998498,993125,991218,992733,994886,999439,995372,996755,999426,992617,996186,996631,992515,992277,995515,990142\n", + "Recall: 0.9000\n", + "\n", "Query ID: 688\n", - "Returned IDs: 999812,993504,993024,996429,994964,990818,999339,993260,999111,995895,998303,998715,998950,993391,996216,992747,997586,992571,993805,994672,995622,999735,990409,997977,991458,993489,996196,995887,991946,990494\n", + "Returned IDs: 999812,993504,993024,996429,994964,990818,999339,993260,999111,995895,998303,998715,998950,993391,996216,992747,997586,992571,994672,990409,997977,991458,993489,996196,991946,990494,991206,993906,999840,995560\n", "Ground Truth: 999812,993504,993024,996429,994964,990818,999339,993260,999111,995895,998303,998715,998950,993391,996216,992747,997586,992571,996677,993805,992979,994672,995622,999735,990409,997977,991458,993489,996196,995887\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", "\n", "Query ID: 689\n", - "Returned IDs: 992493,997634,995163,992677,998316,992150,992148,993223,997824,998650,995854,994620,993263,995723,992700,995060,991067,993530,995091,997750,996139,991190,996273,995062,991100,998080,999961,993160,991052,997723\n", + "Returned IDs: 992493,997634,995163,992677,998316,992150,992148,993223,997824,998650,995854,994620,993263,995723,992700,995060,991067,993530,995091,997750,996139,991190,996273,995062,991100,998080,999961,993160,997723,994576\n", "Ground Truth: 992493,997634,995163,992677,998316,992150,992148,993223,997824,998650,995854,994620,993263,995723,992700,995060,991067,995963,993530,995091,997750,996139,991190,996273,995062,991100,998080,999961,993160,991052\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 690\n", - "Returned IDs: 992933,998536,999759,994835,992166,999259,993014,995175,996877,994119,990103,999533,995783,991052,994742,999847,994522,999244,995963,997309,997184,991712,994280,998021,991891,995685,997723,990184,995110,991747\n", + "Returned IDs: 998536,999759,994835,999259,993014,995175,996877,994119,990103,999533,995783,991052,994742,999847,994522,999244,997309,994280,998021,995685,995110,991747,996328,995328,993180,997675,993256,999601,995017,996087\n", "Ground Truth: 992933,998536,999759,994835,992166,999259,993014,995175,996877,994119,990103,999533,995783,991052,994742,999847,994522,999244,995963,997309,999527,997184,991712,994280,998021,991891,995685,997723,990184,995110\n", - "Recall: 0.9667\n", + "Recall: 0.7000\n", "\n", "Query ID: 691\n", - "Returned IDs: 999045,993721,994718,992551,990096,991409,998495,994269,991650,993053,996001,999336,996075,990843,996936,995361,994355,990584,991382,996219,999577,996631,998803,997550,991451,990091,996101,999343,994472,996016\n", + "Returned IDs: 999045,993721,994718,992551,990096,991409,998495,994269,991650,993053,996001,996075,990843,996936,995361,990584,991382,996219,996631,998803,997550,991451,999302,990091,996101,994472,996016,996013,995476,990786\n", "Ground Truth: 999045,993721,994718,992551,990096,991409,998495,994269,991650,993053,996001,999336,996075,990843,996936,995361,994355,990584,991382,996219,999577,996631,998803,997550,991451,999302,990091,996101,999343,994472\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 692\n", + "Returned IDs: 991977,993839,993583,996158,994398,992718,996585,993077,994080,993681,994119,992558,990894,992104,994334,999079,999771,994254,998357,990273,995459,992211,996733,992052,992824,995503,999565,990908,990776,998061\n", + "Ground Truth: 991977,993839,993583,996158,994398,992718,996585,993077,994080,993681,994119,992095,992558,990894,992104,994334,999079,999771,994254,998357,990273,995459,992211,996733,992052,992824,995503,990279,999565,990908\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 693\n", + "Returned IDs: 991462,992724,997486,992739,992632,993395,997326,998359,999015,992612,997772,996693,995119,991485,999453,992309,997539,992152,998401,990580,998950,995515,990334,999077,991402,994964,997775,994922,993781,990043\n", + "Ground Truth: 997448,991462,992724,997486,992739,992632,993395,997326,998359,999015,992612,997772,996693,995119,991485,999453,992309,993369,996829,997539,992152,998401,995721,990580,998950,995515,990334,999077,995767,991402\n", + "Recall: 0.8333\n", "\n", "Query ID: 694\n", - "Returned IDs: 990010,991425,999794,999211,993504,990956,993665,991552,998615,995349,992457,998303,999111,992137,991648,994672,996884,995541,992110,995103,992351,990752,997506,991037,996012,990658,993609,990409,994681,991257\n", + "Returned IDs: 991425,999794,993504,990956,993665,991552,998615,995349,992457,998303,999111,992137,991648,994672,998109,996884,992351,997506,991037,996012,990658,993609,990409,991257,993429,990959,992640,991779,990885,992819\n", "Ground Truth: 990010,991425,999794,999211,993504,990956,993665,991552,998615,995349,992457,998303,999111,992137,991648,994672,998109,996884,995541,992110,995103,992351,990752,997506,991037,996012,990658,993609,990409,994681\n", - "Recall: 0.9667\n", + "Recall: 0.7667\n", "\n", "Query ID: 696\n", - "Returned IDs: 993003,992267,996256,992917,994409,997466,991357,999250,995401,992448,997909,990295,992763,995242,996815,994508,994865,998910,998278,994106,993537,992521,997299,996968,998756,991197,995268,992260,999530,995864\n", + "Returned IDs: 993003,992267,996256,992917,994409,997466,991357,999250,995401,992448,997909,990295,992763,995242,996815,994508,994865,998910,998278,994106,993537,992521,997299,996968,998756,991197,995268,999530,995864,998361\n", "Ground Truth: 993003,992267,996256,992917,994409,997466,999250,991357,995401,992448,997909,990295,992763,995242,996815,994508,994865,998910,998278,994106,993537,992521,996223,997299,996968,998756,991197,995268,992260,999530\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 697\n", - "Returned IDs: 993512,995336,997571,993096,999144,999331,998090,990647,992134,993853,992384,994197,990214,994810,992692,994334,993492,991131,993153,995357,995374,994819,993925,996882,998562,993785,998410,996296,992453,997876\n", + "Returned IDs: 993512,995336,997571,993096,999144,999331,998090,990647,992134,993853,992384,994197,992692,994334,993492,991131,993153,995357,998562,993785,996296,992453,994227,994100,999453,992748,992337,999639,997918,991094\n", "Ground Truth: 993512,995336,997571,993096,999144,999331,998090,992134,990647,993853,992384,994197,990214,994810,992692,995748,994334,993492,991131,993153,995357,995374,994819,993925,991923,996882,990654,998562,993785,998410\n", - "Recall: 0.9000\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 699\n", + "Returned IDs: 999984,995589,995408,999543,993936,991011,998855,993778,991977,994113,993433,991962,994191,997944,999643,992576,995582,992179,998623,991556,994254,996277,992052,991515,996733,995113,998536,993281,999091,993979\n", + "Ground Truth: 999984,995589,995408,999543,993936,991011,998855,993778,991977,999461,994113,993433,991962,994191,997944,999643,992576,995582,992179,998623,991556,995401,994254,996277,992052,991515,996733,995113,998536,993281\n", + "Recall: 0.9333\n", "\n", "Query ID: 700\n", - "Returned IDs: 990956,996016,991382,992652,992827,993501,994927,998543,994541,998495,996809,994378,995191,999179,994765,995895,993460,998167,996830,997270,991469,996706,999830,993219,996301,998206,993948,990712,993391,997506\n", + "Returned IDs: 990956,996016,991382,992652,992827,993501,994927,998543,994541,998495,996809,995191,999179,994765,995895,993460,998167,996830,997270,991469,996706,999830,993219,996301,993391,997506,992628,999192,997442,994307\n", "Ground Truth: 990956,996016,991382,992652,992827,993501,994927,998543,994541,998495,996809,994378,995191,999179,994765,995895,993460,998167,996830,997270,995401,991469,996706,999830,993219,996301,998206,993948,990712,993391\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 701\n", - "Returned IDs: 990930,991459,992036,992790,999639,998874,995536,995401,996001,994987,990412,993515,993673,996247,995932,991408,991530,997918,998990,995672,998593,991052,996918,998303,991392,996181,997238,991721,994280,992749\n", + "Returned IDs: 990930,991459,992036,992790,999639,998874,995536,995401,996001,994987,990412,993515,993673,996247,995932,996371,991530,997918,995672,998593,996918,998303,996181,991721,994280,992749,990794,994809,993504,997191\n", "Ground Truth: 990930,991459,992036,992790,999639,998874,995536,995401,996001,994987,990412,993515,993673,990337,996247,995932,991408,996371,991530,997918,998990,995672,998593,991052,996918,998303,991392,996181,997238,991721\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 702\n", + "Returned IDs: 998039,990146,994925,994964,990925,990845,997969,992334,998593,998302,990560,994419,996001,996028,994853,998921,995923,993831,993561,997453,997822,994307,997586,992944,997320,997046,996785,999374,990529,991685\n", + "Ground Truth: 998039,990146,994925,995162,994964,990925,990845,997969,992334,998593,998302,990560,994419,996001,996028,993648,994853,993412,997920,998921,995923,993831,998228,995906,993561,997453,997822,994307,995401,997586\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 703\n", + "Returned IDs: 998668,994557,991377,991179,990506,995547,995289,998416,999294,993561,991923,996042,995923,997969,997732,994782,998292,997822,995643,993378,996494,996298,998133,996216,996080,997386,998039,993929,998382,990137\n", + "Ground Truth: 998668,994557,991377,991179,999785,990506,995547,995289,995162,998416,999294,993561,991923,996042,995923,997969,991790,997732,994782,998292,997822,995643,993378,993412,996494,996298,994026,998133,996216,996080\n", + "Recall: 0.8333\n", "\n", "Query ID: 704\n", - "Returned IDs: 990003,997913,999647,990210,990545,996053,998344,999331,994873,998302,990958,993031,991479,990231,998540,991429,995438,990650,991852,991782,994450,993484,997320,993990,996296,998550,992789,999167,999263,999109\n", + "Returned IDs: 990003,997913,999647,990210,990545,996053,999331,994873,998302,990958,993031,990231,998540,995438,990650,994450,993484,997320,991370,998550,999167,999263,994102,992147,990957,998899,995802,996075,992546,990949\n", "Ground Truth: 990003,997913,999647,990210,990545,992946,996053,998344,999331,994873,998302,990958,993031,991479,997439,990231,998540,991429,995438,990650,991852,991782,994450,993484,997320,993990,991370,994898,996296,998550\n", - "Recall: 0.8667\n", + "Recall: 0.6667\n", "\n", "Query ID: 705\n", - "Returned IDs: 997897,994280,995421,991605,991447,991232,993489,992052,990627,997442,990844,992864,997926,998071,995626,995203,992457,991332,996817,997531,995630,998428,990885,997790,991218,991212,993175,999550,993391,994738\n", + "Returned IDs: 997897,994280,995421,991605,991447,991232,993489,992052,990627,997442,990844,992864,997926,995626,995203,992457,991332,996817,995630,998428,990885,997790,991218,991212,993175,993391,994738,994531,990630,994351\n", "Ground Truth: 997897,994280,995421,991605,991447,991232,993489,992052,990627,997442,990844,992864,997926,998071,995626,993087,995203,992457,991332,996817,997531,995630,998428,990885,997790,991218,991212,993175,999550,993391\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 706\n", - "Returned IDs: 993368,998798,995374,996703,990992,997015,991467,992854,996353,998951,990585,994197,991504,997609,990535,999026,996959,994595,997009,991173,991498,999089,996714,992189,996658,993562,995824,995034,998844,999024\n", + "Returned IDs: 998798,995374,996703,990992,997015,991467,992854,996353,998951,990585,994197,991504,997609,990535,999026,996959,994595,997009,991173,991498,999089,996714,992189,996658,993562,995824,995034,998844,999024,995274\n", "Ground Truth: 993368,998798,995374,996703,990992,997015,991467,992854,996353,998951,990585,994197,991504,997609,990535,993909,999026,996959,994595,997009,991173,991498,999089,996714,992189,996658,993562,995824,995034,998844\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 707\n", + "Returned IDs: 997485,997529,993939,998610,995861,996752,996328,990133,992450,993538,997103,997544,994830,991114,998710,993079,990359,994185,999372,995295,990889,991334,998484,994835,993127,997933,992434,998671,995211,995620\n", + "Ground Truth: 997485,997529,993939,998610,995861,996752,996328,990133,992450,993538,997103,996042,997544,994830,991114,998710,993079,994131,990359,994185,999372,995295,990889,991334,998484,994835,993127,997933,992434,998671\n", + "Recall: 0.9333\n", "\n", "Query ID: 708\n", - "Returned IDs: 996052,996219,998383,990448,999998,998464,991743,995716,994964,993412,991844,996232,995581,990538,999876,997010,992604,990063,991650,990366,996001,998593,999791,992944,992337,995107,992558,998495,990783,991052\n", + "Returned IDs: 996052,996219,990448,999998,998464,991999,994964,991844,995581,990538,999876,997010,992604,990063,990366,996001,998593,999791,995596,992944,992337,992558,998495,991052,997470,995693,996959,996871,998758,990886\n", "Ground Truth: 996052,996219,998383,990448,999998,998464,991743,995716,991999,994964,993412,991844,996232,995581,990538,999876,997010,992604,990063,991650,990366,996001,998593,999791,995596,992944,992337,995107,992558,990091\n", - "Recall: 0.9000\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 709\n", + "Returned IDs: 997103,999980,990537,999581,993939,999054,992448,991722,990220,990251,998910,994861,999279,990372,998733,997544,991389,993127,992449,993856,995268,995778,997176,993308,991507,994380,991664,992097,991560,996505\n", + "Ground Truth: 997103,999980,990537,998283,999581,993939,999054,992448,991722,996897,990220,990251,998910,994861,999279,990372,998733,997544,991389,993127,992449,993856,995268,995778,997176,993308,993046,993003,991507,994380\n", + "Recall: 0.8667\n", "\n", "Query ID: 710\n", - "Returned IDs: 994922,991840,997591,990958,999261,992169,996001,998593,997822,990076,996351,992994,999729,990648,990399,996867,999676,991087,990064,993143,992692,996854,997320,992944,990866,990455,994380,992622,996438,995401\n", + "Returned IDs: 994922,991840,990958,992169,996001,998593,997822,990076,996351,992994,999729,996867,999676,990064,993143,992692,996854,997320,992944,990455,994380,992622,999453,991392,993168,998458,991015,995245,995638,992448\n", "Ground Truth: 994922,991840,997591,990958,999261,992169,996001,998593,997822,990076,996351,992994,999729,990648,990399,996867,999676,991087,990064,993143,992692,996854,997320,992944,990866,990455,997620,994380,992622,996438\n", - "Recall: 0.9667\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 711\n", + "Returned IDs: 993698,991066,995007,992576,991011,997331,997168,995589,998723,996505,994191,996067,995245,994380,991857,992472,996277,998045,996716,995815,996062,990537,995907,994113,991747,996769,993561,993813,997675,991958\n", + "Ground Truth: 993698,991066,995007,992576,991011,997331,997168,995589,998723,996505,994191,996932,996067,995245,990951,994380,993778,991857,992472,996277,994419,995431,998045,996716,995815,996062,990537,995907,994113,991747\n", + "Recall: 0.8333\n", "\n", "Query ID: 712\n", - "Returned IDs: 991720,990545,996296,998170,993143,999437,999764,991999,993253,998562,994251,990432,999077,999851,994964,995588,995515,995927,997876,998294,997908,993484,999343,997320,998550,992384,993885,990504,993238,994100\n", + "Returned IDs: 991720,990545,998170,993143,991999,993253,990432,999851,994964,995588,995515,995927,998294,997908,993484,997320,998550,992384,993885,994100,998101,999453,994561,997002,996001,992942,995400,994346,993877,991743\n", "Ground Truth: 991720,990545,996296,998170,993143,999437,999764,991999,993253,991715,998562,994251,992979,990432,999077,999851,994964,995588,995515,995927,997876,998294,997908,993484,999343,997320,998550,992384,995748,993885\n", - "Recall: 0.9000\n", + "Recall: 0.6333\n", "\n", "Query ID: 713\n", - "Returned IDs: 999384,994972,997811,990096,995536,998697,991179,997320,999898,999647,996001,996062,999144,997913,991154,995701,996219,995130,993504,998401,993809,997775,993906,996370,992047,995596,993561,991459,995560,995401\n", + "Returned IDs: 999384,994972,990096,995536,998697,997320,999898,999647,996001,996062,999144,997913,991154,995701,996219,995130,993504,998401,993809,997775,993906,996370,995596,993561,991459,995560,999653,997918,994964,991157\n", "Ground Truth: 999384,994972,997811,990096,995536,998697,991179,997320,999898,999647,996001,996062,999144,997913,991154,995701,996219,995130,993504,998401,993809,992692,997775,993906,996370,992047,995596,993561,991459,995560\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 714\n", - "Returned IDs: 992669,990063,997526,998858,992604,999961,997110,992439,993115,991309,998707,994724,994774,995982,993122,999478,990111,998383,990657,994569,999231,997205,991987,994454,997107,998806,995455,992051,991052,998297\n", + "Returned IDs: \n", "Ground Truth: 992669,990063,997526,998858,992604,999961,997110,992439,993115,995949,991309,998707,994724,994774,995982,993122,999478,990111,998383,990657,994569,999231,997205,991987,994454,997107,993976,998806,995455,992051\n", - "Recall: 0.9333\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 715\n", + "Returned IDs: 996042,997046,997822,993642,996328,997386,990137,993433,994419,996322,998545,997453,998165,996494,995289,998039,995200,997969,993561,993831,996867,995401,996062,991301,995685,998633,997289,994783,994742,994409\n", + "Ground Truth: 996042,997046,997822,993642,996328,997386,993827,990137,993433,993412,994419,996322,998545,997453,998165,996494,995289,998039,995200,997969,993561,993831,996867,995526,997732,994964,995401,993400,996062,991301\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 717\n", + "Returned IDs: 993179,992635,990697,991260,994886,991372,996884,996755,990532,995596,996001,992515,997550,992041,993820,990083,993071,998097,995515,996075,996805,999426,997002,990834,994862,997584,995358,993129,992733,994972\n", + "Ground Truth: 993179,992635,990697,991260,994886,991372,996884,996755,990532,995596,995622,996001,992515,997550,992041,993820,990083,990640,993071,998097,995515,996075,991142,996805,999426,997002,990834,994862,997584,991048\n", + "Recall: 0.8667\n", "\n", "Query ID: 718\n", - "Returned IDs: 996536,993588,994897,994830,993342,997952,992664,998716,993335,998092,995097,998623,998903,998440,997296,998256,997073,998030,990334,996183,992427,998097,994513,993866,990392,999963,992458,994682,992174,993715\n", + "Returned IDs: 996536,993588,994897,994830,993342,997952,992664,998716,993335,998092,995097,998623,998903,998440,997296,998256,997073,998030,996183,992427,998097,994513,993866,990392,992458,994682,993715,994906,997002,996001\n", "Ground Truth: 996536,993588,994897,994830,993342,997952,992664,998716,997301,993335,998092,995097,998623,998903,998440,997296,998256,997073,998030,990334,996183,992427,998097,994513,993866,990392,999963,992458,994682,992174\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 719\n", - "Returned IDs: 992330,997371,997913,995985,993072,990397,990469,990560,993048,998500,991468,990092,992384,992443,997889,996193,990595,998921,995260,996840,994360,992546,994281,999122,995365,997320,999676,997984,997305,997831\n", + "Returned IDs: 992330,997371,997913,995985,990397,990469,990560,998500,992384,992443,996193,990595,998921,995260,996840,994360,992546,999122,997320,999676,997984,997305,997831,993492,991626,996504,999603,995949,999165,998445\n", "Ground Truth: 992330,997371,997913,995985,993072,990397,990469,990560,993048,998500,991468,990092,992384,992443,997889,996193,990595,998921,995260,996840,994360,992546,994281,999613,991631,999122,995365,997320,999676,997984\n", - "Recall: 0.9333\n", + "Recall: 0.7000\n", "\n", "Query ID: 720\n", - "Returned IDs: 998555,998169,999781,990271,990096,997529,998122,999343,995638,992551,993800,997732,990091,994145,992120,995535,999103,993637,996219,990089,999647,999319,990690,997918,994532,997920,998314,990918,996328,990072\n", + "Returned IDs: 998555,998169,999781,990271,990096,997529,998122,999343,995638,992551,993800,997732,994145,992120,995535,999103,993637,996219,990089,999647,999319,990690,994532,998934,998314,990918,996328,990072,990366,999791\n", "Ground Truth: 998555,998169,999781,990271,990096,997529,998122,999343,995638,992551,993800,997732,990091,994145,992120,995535,999103,993637,996219,995216,990089,999647,999319,990690,997918,994532,998934,997920,998314,990918\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 721\n", + "Returned IDs: 990242,998162,992937,990940,995444,993831,997046,996388,992160,997466,991163,991603,991973,993390,995702,998503,996867,997723,992521,998238,998267,998165,999595,997868,997822,993678,996769,990110,994954,991415\n", + "Ground Truth: 990242,998162,992937,990940,995444,993831,993799,997046,996388,992160,997466,991163,991603,991973,993390,996038,995702,998503,996867,997723,992521,998238,998267,998165,999595,992977,997868,998941,997822,993678\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 722\n", + "Returned IDs: 998536,993965,992407,999448,995461,998331,992052,999079,994119,995408,991707,997897,998051,996817,999212,996633,999433,996245,995110,996733,998177,992320,990551,998874,997881,991515,990169,994743,996134,991245\n", + "Ground Truth: 998536,993965,992407,999448,995461,998331,992052,999079,994119,995408,991707,997897,998051,996817,999212,991093,996633,999433,996245,995110,996733,998177,992320,990551,998874,997881,991515,990169,994743,999179\n", "Recall: 0.9333\n", "\n", + "Query ID: 723\n", + "Returned IDs: 992636,995328,994522,999767,994742,992832,991850,991998,995012,992764,991007,992973,996911,998174,999759,990336,994835,992239,997153,994891,993747,990270,991629,998536,993621,993448,993959,996183,994175,996050\n", + "Ground Truth: 992636,995328,992933,994522,999767,994742,992832,996557,991850,991998,995012,992764,991007,992973,996911,998174,992779,999759,990336,994835,990364,992239,997153,994891,993747,990270,991629,998536,993621,993448\n", + "Recall: 0.8667\n", + "\n", "Query ID: 724\n", - "Returned IDs: 993775,998765,999791,995477,991921,990089,994742,997529,999058,996633,997401,995057,997698,996328,996302,999788,997309,999875,991588,990179,999192,999261,995438,994806,993053,993265,995289,995551,995600,999886\n", + "Returned IDs: 993775,998765,999791,995477,991921,990089,994742,997529,999058,996633,997401,995057,997698,996328,996302,999788,997309,999875,991588,990179,999192,999261,995438,994806,993053,993265,995289,995551,999138,991315\n", "Ground Truth: 993775,998765,999791,995477,991921,990089,994742,997529,999058,996633,997401,995057,997698,996328,996302,999788,997309,999875,991588,990179,999192,999261,995438,994806,993053,993265,995289,995551,999138,995600\n", "Recall: 0.9667\n", "\n", + "Query ID: 725\n", + "Returned IDs: 998168,992517,995768,995501,994356,995476,993755,990851,991382,991667,995479,992953,999425,990335,997340,995879,993319,991826,999130,998794,996184,994205,990145,998625,999107,999256,995073,995191,991779,992868\n", + "Ground Truth: 998168,992517,995768,995501,994356,995476,993755,990851,991382,991667,995479,999634,992953,999425,990335,995222,997340,995879,993319,991826,999130,998794,996184,994205,990145,998625,999107,999256,995073,995191\n", + "Recall: 0.9333\n", + "\n", "Query ID: 726\n", - "Returned IDs: 993035,995791,995004,995365,993570,993633,995588,999039,999592,993282,998837,990043,999620,993028,991999,997496,995155,994545,995657,994666,991407,991593,993675,994543,994922,993708,993946,991196,990155,996402\n", + "Returned IDs: \n", "Ground Truth: 993035,995791,995004,995365,993570,993633,995588,999039,999592,993282,998837,990043,999620,993028,991999,997496,995155,994545,995657,994666,991407,991593,993505,993675,994543,994922,993708,993946,991196,990155\n", - "Recall: 0.9667\n", + "Recall: 0.0000\n", "\n", "Query ID: 727\n", - "Returned IDs: 995865,994611,996143,993745,996318,993154,991872,994214,997320,990210,997178,993579,994812,992909,996805,997022,993885,999351,998550,998287,997305,991498,990772,998344,994964,999374,994699,994249,996755,992978\n", + "Returned IDs: 995865,990612,994611,996143,996318,991872,994214,997320,990210,997178,994812,992909,996805,997022,993885,998550,997305,991498,994964,999374,994699,994249,996755,992978,992066,998540,999982,994164,994778,992993\n", "Ground Truth: 993990,995865,990612,994670,994611,996143,993745,996318,993154,991872,994214,997320,990210,997178,993579,994812,992909,996805,997022,993885,999351,998550,998287,997305,991498,990772,998344,994964,999374,994699\n", - "Recall: 0.9000\n", + "Recall: 0.7000\n", "\n", "Query ID: 728\n", - "Returned IDs: 995932,993504,999516,990196,993071,993906,991440,997758,991648,997555,991206,994081,997135,995622,993515,990682,992082,993821,990350,997918,992571,993783,997790,992747,993260,997002,992351,991435,997158,995375\n", + "Returned IDs: 995932,993504,999516,990196,993071,993906,991440,991648,997555,991206,993515,992082,990350,997918,992571,993783,997790,992747,993696,993260,997002,992351,991435,995375,991994,996429,998726,996949,997965,993637\n", "Ground Truth: 995932,993504,999516,990196,993071,993906,991440,997758,991648,997555,991206,994081,997135,995622,993515,990682,992082,993821,990350,997918,992571,993783,997790,992747,993696,993260,997002,994146,992351,991435\n", - "Recall: 0.9333\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 729\n", + "Returned IDs: 995655,998625,990154,993001,992110,992517,999555,990167,992752,992444,991667,998536,994465,999433,998006,993948,991620,990173,996408,998888,995879,999256,994423,995768,992595,993641,999319,991421,999994,995936\n", + "Ground Truth: 995655,998625,990154,993001,992110,992517,999555,990167,992752,992444,991667,998536,994465,999433,998006,993948,991620,990173,991188,990429,996408,998888,995879,994796,999256,994423,995768,993641,992595,999319\n", + "Recall: 0.9000\n", "\n", "Query ID: 730\n", - "Returned IDs: 997025,995521,995907,998703,999653,995303,991161,992448,997251,995891,995537,997909,993275,993313,993342,994380,990593,995778,990725,994125,991844,997723,991338,997675,993561,996042,993127,991459,991447,991794\n", + "Returned IDs: 997025,995521,995907,998703,999653,995303,991161,992448,997251,995891,995537,993275,993313,993342,994380,990593,995778,994125,991844,997675,993561,996042,993127,998855,991459,991447,991794,995815,992576,994419\n", "Ground Truth: 997025,995521,995907,998703,999653,995303,991161,992448,997251,995891,995537,997909,993275,993313,993342,994380,990593,995778,990725,994125,991844,997723,991338,997675,993561,996042,993127,998855,991459,991447\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 731\n", - "Returned IDs: 999374,999710,997320,997913,999701,996504,992748,991860,999331,991801,998294,998495,998540,993484,995942,991743,997301,996178,992789,992147,995886,994964,993809,995961,994850,991498,998246,990958,990957,998550\n", + "Returned IDs: 999374,999710,997320,997913,996504,992748,991860,999331,991801,998495,998540,993484,995942,991743,997301,996178,992147,995886,994964,993809,995961,991498,998246,990958,990957,998550,990210,996318,998029,991593\n", "Ground Truth: 999374,999710,997320,997913,999701,991052,996504,992748,991860,999331,991801,998294,998495,998540,993484,995942,991743,995632,997301,996178,992789,992147,995886,994964,993809,995961,994850,991498,998246,990958\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 732\n", + "Returned IDs: 993561,996783,996313,996733,997023,996062,991958,993931,994444,990537,990096,998207,992322,999673,990382,991093,998536,994380,997453,998024,990899,990531,992752,995648,995521,993289,996001,998314,998416,997790\n", + "Ground Truth: 993561,996783,996313,996733,997023,996062,991958,993931,999436,994444,990537,990096,998207,992322,999673,990382,990072,991093,998536,994380,997453,995401,998024,990899,990531,992752,995648,995521,993289,996001\n", + "Recall: 0.9000\n", "\n", "Query ID: 733\n", - "Returned IDs: 992037,998022,996277,998934,994906,995401,996181,990400,996001,998720,991459,994444,999020,992685,999026,999543,997204,994105,995091,993652,994589,994424,992648,996379,992576,999759,998990,990453,991392,991542\n", + "Returned IDs: 992037,998022,996277,998934,994906,996181,990400,996001,998720,998080,991459,994444,999020,992685,999026,999543,997204,993652,995780,994589,994424,992648,996379,992576,999759,998990,991248,991574,996861,992516\n", "Ground Truth: 992037,998022,996277,998934,994906,995401,993880,996181,990400,996001,998720,998080,991459,999259,994444,999020,992685,999026,998221,999543,997204,994105,995091,996408,993652,995780,994589,994424,992648,996379\n", - "Recall: 0.8000\n", + "Recall: 0.7667\n", "\n", "Query ID: 734\n", - "Returned IDs: 992712,995855,995356,995563,998622,992163,996105,994167,990472,994955,993283,999216,997368,993300,998513,991232,996144,992484,999041,993174,999219,998697,997977,990688,999750,995672,998934,997417,995203,993821\n", + "Returned IDs: 992712,995855,995356,998622,992163,996105,994167,990472,994955,993283,999216,997368,993300,998513,991232,992484,999041,993174,999219,998697,997977,999750,995672,998934,997417,995203,993918,998447,993260,998393\n", "Ground Truth: 992712,995855,995356,995563,998622,992163,996105,994167,990472,994955,993283,999216,997368,993300,998513,991232,996144,992484,999041,993174,999219,998697,997977,990688,999750,995672,998934,997417,995284,995203\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 735\n", - "Returned IDs: 996196,997775,998950,995873,994163,996140,994661,995596,997320,995560,990538,990532,998417,996178,991157,995983,990930,990096,992124,992277,990901,992558,991923,997913,997227,995552,997379,999564,996380,996721\n", + "Returned IDs: 996196,997775,998950,994163,996140,994661,995596,997320,995560,990538,990532,996178,995983,990096,992124,990901,992558,991923,997913,997227,995552,995974,999564,996380,998697,996498,995289,995536,999639,998779\n", "Ground Truth: 996196,997775,998950,995873,994163,996140,994661,995596,997320,995560,990538,992521,990532,996178,998417,991157,995983,990930,990096,992124,992277,990901,992558,991923,997913,997227,995552,997379,995974,999587\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 736\n", + "Returned IDs: 995507,999619,997832,996038,999530,993421,996323,996618,995905,995139,991027,995470,992448,996867,994409,997359,993400,995685,997046,996322,992130,997289,990836,992605,998045,992169,998520,997723,999482,997466\n", + "Ground Truth: 995507,999619,997832,996038,999530,993421,996323,996618,995905,995139,991027,995470,992448,996867,994409,997359,993400,995685,997046,996322,992130,997289,990836,992605,998045,993468,992169,998520,997723,999482\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 737\n", + "Returned IDs: 998682,999066,995372,996309,991787,995968,999544,993840,995250,999254,996452,993241,992944,991673,998849,995515,991248,997498,991650,992348,999453,997320,998921,997301,993447,998734,999686,991157,997453,996805\n", + "Ground Truth: 998682,999066,995372,996309,991787,995968,999544,993840,995250,999254,996452,993241,992944,991673,998849,995515,991248,997498,991650,992348,999453,997320,998921,994826,997301,997906,993447,998734,999686,995154\n", "Recall: 0.9000\n", "\n", + "Query ID: 738\n", + "Returned IDs: 990115,998315,998925,992193,996431,995191,998055,994472,995223,990712,998671,996357,991682,994299,996016,992237,999205,991723,990187,997573,995789,999752,990091,996153,995919,991153,999832,992652,990931,996537\n", + "Ground Truth: 990115,998315,998925,992193,996431,995191,998055,994472,995223,990712,998251,998671,997954,996357,991682,994299,996016,992237,999205,999441,991723,990187,997573,995789,999752,996942,990091,997602,998084,998122\n", + "Recall: 0.7667\n", + "\n", "Query ID: 739\n", - "Returned IDs: 992932,990535,997304,998246,995981,998119,999761,992296,991900,998353,992815,997320,995473,991045,995132,991534,992238,995792,990588,993392,993154,990949,997428,998225,997353,992074,998950,990057,992357,999357\n", + "Returned IDs: 992932,990535,997304,998246,995981,998119,999761,992296,991900,998353,992815,997320,995473,991045,991534,992238,990588,993392,990949,998225,997353,992074,998950,990057,992357,999357,993809,992443,997913,995336\n", "Ground Truth: 992932,990535,997304,996732,998246,995981,998119,999761,992296,991900,998353,992815,997320,995473,991045,995132,991534,992238,995792,990588,993392,993154,996950,992927,990949,997428,998225,998361,997353,992074\n", - "Recall: 0.8667\n", + "Recall: 0.7333\n", "\n", "Query ID: 740\n", - "Returned IDs: 992862,998156,993857,993256,994564,991428,997173,990328,997374,994742,996328,995819,997452,997309,999843,997918,995633,995295,998708,996752,997852,997914,990814,998073,992473,991189,996411,997409,995600,997208\n", + "Returned IDs: 992862,998156,993256,994564,991428,996232,997173,990328,997374,992979,994742,996328,995819,997452,997309,999843,997918,995633,995295,998708,997914,990814,991189,996411,997208,993909,994382,990127,995274,999099\n", "Ground Truth: 992862,998156,993857,993256,994564,994940,991428,996232,997173,990328,993440,997374,992979,994742,996328,995819,997452,997309,999843,997918,995633,995295,994850,998708,996752,997852,997914,990814,999453,998073\n", - "Recall: 0.8000\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 741\n", + "Returned IDs: 994672,992351,990885,993504,998732,996062,991425,998303,999811,990409,990196,997918,991037,997227,997977,995740,993289,994964,997790,990557,997002,995203,999093,990096,991212,999840,995932,994221,996196,992571\n", + "Ground Truth: 994672,992351,990885,993504,998732,996062,991425,998303,994468,999811,990409,990196,997918,991037,997227,997977,995740,993289,994964,997790,990557,993609,997002,995203,999093,990096,991212,999840,995932,993333\n", + "Recall: 0.9000\n", "\n", "Query ID: 742\n", - "Returned IDs: 995231,997369,995168,994105,997666,992275,995385,990096,999176,996702,994779,995257,992987,993866,997965,990429,990658,999576,992945,997847,998536,995393,997086,998177,992664,999129,996050,994834,990670,991654\n", + "Returned IDs: 995231,997369,995168,994105,997666,992275,995385,990096,999176,996702,994779,995257,993866,997965,990658,992945,997847,998536,995393,997086,998177,992664,999129,996050,994834,991654,997142,995695,997576,997673\n", "Ground Truth: 995231,997369,995168,994105,997666,992275,995385,990096,999176,996702,994779,995257,992987,993866,997965,990429,990658,999576,992945,991313,997847,998536,995393,997086,998177,992664,999129,996050,994834,990670\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 743\n", - "Returned IDs: 995121,991630,997191,991140,998297,992977,996734,993738,995401,990005,999530,992448,994319,998045,997868,998205,995864,991969,995395,994019,997957,991958,993381,998346,996277,999820,992521,999731,995940,997723\n", + "Returned IDs: 991630,997191,991140,998297,996734,993738,995401,990005,999530,992448,994319,998045,997868,998205,991969,995395,994019,991958,996277,997928,999820,992521,999731,995940,995813,995507,995815,995994,992919,999653\n", "Ground Truth: 995121,991630,997191,991140,998297,992977,996734,995268,993738,995401,990005,999530,992448,994319,998045,997868,998205,995864,991969,995395,994019,997957,991958,993381,998346,996277,997928,999820,992521,999731\n", - "Recall: 0.9333\n", + "Recall: 0.7667\n", "\n", "Query ID: 744\n", - "Returned IDs: 997006,993885,998294,997876,997320,990898,999374,996296,993364,990917,991059,991900,995266,992896,993048,993853,994964,994776,990760,994476,999328,996218,993484,990863,999647,999823,998246,992931,990560,990146\n", + "Returned IDs: 993885,998294,997320,990898,999374,993364,990917,991900,992896,993853,994964,990760,994476,996218,993484,990863,998899,998246,990296,992931,993662,990560,990146,991498,991593,990925,992546,999263,996279,999710\n", "Ground Truth: 997006,993885,998294,997876,997320,990898,999374,996296,993364,990917,991059,991900,995266,992896,993048,993853,994964,994776,990760,994476,999328,996218,993484,999247,990863,998899,999647,999823,998246,990296\n", - "Recall: 0.9000\n", + "Recall: 0.6333\n", "\n", "Query ID: 745\n", - "Returned IDs: 995793,991022,992518,995833,991782,990862,998822,990535,996713,995771,998951,997976,994266,997320,993735,994100,995336,998263,996714,998721,996961,992646,992183,992384,999089,995279,997571,991889,993711,992609\n", + "Returned IDs: 995793,991022,992518,995833,990862,998822,990535,996713,995771,998951,997976,994266,997320,994100,995336,998263,996714,998721,996961,992646,992183,992384,995438,999089,995279,997571,991889,993711,992609,992874\n", "Ground Truth: 995793,991022,992518,995833,991782,990862,998822,990535,996713,995771,998951,997976,994266,997320,993735,994100,995336,998263,996714,998721,996961,992646,992183,992384,995438,999089,995279,997571,991889,999533\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 746\n", + "Returned IDs: 995030,990516,991590,998557,991968,992827,994716,994742,995633,999957,997662,996302,995289,994258,998196,992239,997319,992973,994893,992415,998999,991810,995200,993078,995012,998525,998543,999875,992364,999981\n", + "Ground Truth: 995030,990516,991590,998557,991968,992827,994716,994742,995633,999957,997662,996302,994317,995289,995239,994258,998196,992239,997319,992973,994893,992415,998999,991810,995200,993078,995012,998525,998543,999875\n", "Recall: 0.9333\n", "\n", + "Query ID: 747\n", + "Returned IDs: 998132,992037,997320,996277,992294,996497,996247,991011,999820,990409,997918,998874,996368,991349,995130,993433,992979,996181,992759,999019,995873,992165,997027,996127,993652,997897,990899,990885,999026,992576\n", + "Ground Truth: 998132,992037,997320,993542,996277,992294,996497,995701,996247,991011,999820,990409,997918,994113,998874,996368,990442,991349,995130,993433,992979,996181,996294,999804,995876,991556,995047,992759,999019,990723\n", + "Recall: 0.6667\n", + "\n", "Query ID: 748\n", - "Returned IDs: 999453,991930,990772,996001,991248,991440,991609,993280,990314,995250,994484,992395,995968,997332,991787,992884,990399,991581,995372,990532,993032,994449,999575,991158,994478,991542,991994,992182,992469,998567\n", + "Returned IDs: 999453,991930,990772,996001,991248,991440,991609,993280,990314,995250,992395,995968,997332,991787,990399,991581,995372,990532,993032,994449,991158,994478,991542,991994,992182,992469,998567,999019,991537,994081\n", "Ground Truth: 999453,991930,990772,996001,991248,991440,991609,993280,990314,995250,994484,992395,995968,997332,991787,998127,992884,990399,991581,995372,990532,993032,994449,999575,991158,994478,991542,991994,992182,992469\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 749\n", - "Returned IDs: 996111,997032,995482,994925,996232,998221,996328,994742,994798,999647,990096,993053,990435,994998,992450,996196,996178,998649,990887,998495,996001,991743,997379,999898,998067,996884,993783,991769,997126,998710\n", + "Returned IDs: 997032,995482,994925,996232,998221,996328,994742,999647,990096,993053,990435,994998,992450,996196,996178,998649,990887,998495,996001,999898,998067,996884,993783,991769,997126,998710,996062,993168,998550,996216\n", "Ground Truth: 996111,997032,995482,994925,996232,998221,996328,994742,994798,999647,990096,993053,990435,994998,992450,996196,996178,998649,990887,998495,999586,996001,991743,997379,999898,998067,996884,993783,991769,997126\n", - "Recall: 0.9667\n", - "\n", - "Query ID: 751\n", - "Returned IDs: 997015,991568,993168,993050,998992,990535,992744,995336,993562,996714,998951,995252,992646,999531,997082,994964,999026,990936,995824,992911,995536,992331,993648,999144,994197,999437,991444,997709,990925,993484\n", - "Ground Truth: 997015,991568,993168,993050,998992,990535,992744,995336,993562,996714,998951,995252,992646,999531,997082,994964,999026,990936,995824,992911,995536,992331,993648,999144,994197,998495,999437,991444,997709,990925\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 750\n", + "Returned IDs: \n", + "Ground Truth: 992701,995626,996082,991098,996599,995991,991236,995167,993375,994259,998389,990802,992437,995047,991327,990382,991741,990212,992247,998593,996001,997790,995568,997860,997494,991212,996733,997897,992862,991308\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 751\n", + "Returned IDs: 997015,991568,993168,993050,998992,990535,992744,995336,993562,996714,998951,995252,992646,999531,997082,994964,999026,995824,992911,992331,999144,994197,998495,999437,991444,997709,990925,993484,990797,997918\n", + "Ground Truth: 997015,991568,993168,993050,998992,990535,992744,995336,993562,996714,998951,995252,992646,999531,997082,994964,999026,990936,995824,992911,995536,992331,993648,999144,994197,998495,999437,991444,997709,990925\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 752\n", + "Returned IDs: 994259,995626,998376,994280,997790,998639,999093,995167,993262,993024,990212,991212,995343,997897,997534,990877,992864,995047,992168,996767,996216,992701,993187,995991,996599,991994,998032,992780,991098,991661\n", + "Ground Truth: 994259,995626,998376,994280,997790,996424,998639,999093,995167,993262,993024,990212,991212,995343,997897,997534,990877,992864,994815,995047,992168,990494,996767,997494,996216,992701,995705,993187,995991,996599\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 753\n", + "Returned IDs: 994140,993390,993831,991958,997046,991415,996042,998776,996719,995289,993561,994409,997822,995052,994783,993813,999099,991790,990940,996322,996388,998633,997554,998165,996867,997630,996924,993738,995679,994743\n", + "Ground Truth: 994140,993390,993831,991958,997046,991415,996042,998776,996719,995289,993561,994409,997822,995052,994783,990193,993813,999099,991790,990940,995873,996322,996388,998633,993856,997554,998165,996867,997630,996924\n", + "Recall: 0.9000\n", "\n", "Query ID: 754\n", - "Returned IDs: 998950,996805,999647,992944,993696,992041,998180,997906,999210,999426,997470,991650,997371,995633,997403,997913,993504,994742,997291,995401,999676,998495,999639,993747,993529,998790,992520,996196,996219,990096\n", + "Returned IDs: 998950,996805,992944,993696,992041,998180,999210,999426,997470,991650,997371,995633,997403,997913,993504,994742,999676,998495,998855,999639,993747,993529,998790,992520,996196,996219,990096,996397,999367,998913\n", "Ground Truth: 998950,996805,999647,992944,993696,992041,998180,997906,999210,999426,994089,997470,991650,997371,995633,997403,997913,993504,994742,997291,995401,999676,998495,998855,999639,993747,993529,998790,992520,996196\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 755\n", + "Returned IDs: 993791,999247,997320,998302,998090,996296,997661,993260,999290,994360,991774,998246,997470,993845,998337,997371,996592,990949,991743,992950,990231,997913,993741,995949,995260,997586,995596,993484,992896,990957\n", + "Ground Truth: 993791,999247,997320,998302,998090,996296,997661,993260,999290,994360,991774,998246,997470,993845,998337,990467,997371,996592,990949,991743,992950,990231,997913,993741,995949,995260,997586,995596,993484,993143\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 756\n", + "Returned IDs: 991507,991591,999957,998162,995778,999054,995958,997723,994861,998910,993209,997700,997020,999137,992448,991401,990105,994013,997046,999415,991671,995665,995493,996079,996867,998267,999787,997544,992002,999646\n", + "Ground Truth: 991507,991591,999957,998162,995778,999054,995958,997723,994861,998910,993209,997700,997020,999137,990836,992448,991401,999412,993899,990105,994013,997046,999415,991671,995665,995493,998609,996079,996867,998267\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 757\n", + "Returned IDs: 992372,995313,997856,998063,993335,997340,991620,998379,994682,995299,992026,990712,999381,991382,992882,990851,990173,999798,999348,995768,998625,997527,990579,994356,995501,992953,991667,992832,990392,996301\n", + "Ground Truth: 992372,995313,997856,998063,993335,997340,991620,998379,994682,995299,992026,990712,999381,991382,992882,990851,990173,999798,999348,995768,998625,995222,996037,997527,990579,994356,995501,992953,991667,992832\n", "Recall: 0.9333\n", "\n", "Query ID: 758\n", - "Returned IDs: 996001,991119,995455,995755,999009,998861,991881,997337,990505,998122,990096,996809,996406,994879,995701,990866,995425,999480,996408,998514,999388,997586,995401,990091,995132,991459,999289,991154,995536,997305\n", + "Returned IDs: 996001,991119,995455,995755,999009,998861,997337,990862,998122,990096,996809,994879,995701,990866,995425,999480,998514,999388,997586,995401,990091,991459,999289,991154,991903,995536,997305,999663,999898,990974\n", "Ground Truth: 996001,991119,995455,995755,999009,998861,991881,999916,997337,990862,990505,998122,990096,996809,996406,994879,995701,990866,995425,999480,996408,998514,999388,997586,995401,990091,995132,991459,999289,991154\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 759\n", + "Returned IDs: 994833,998959,993560,994834,999373,995393,996797,998623,999136,993055,995237,998787,993366,998831,997210,993948,995936,998383,993014,990952,991844,998758,998565,995780,998861,999724,995695,990005,992903,992636\n", + "Ground Truth: 994833,998959,993560,994834,999373,995393,996797,998623,999136,993055,995237,998787,993366,998831,997210,993948,995936,998383,993014,990952,994906,991844,998758,998565,995780,998861,999724,995695,990005,992903\n", + "Recall: 0.9667\n", "\n", "Query ID: 760\n", - "Returned IDs: 990061,990273,995309,993583,990206,994254,993500,996380,991064,997547,992052,990042,994679,993077,995459,999022,993811,991977,991432,997866,995728,993742,990776,996328,992558,990894,993879,994280,997494,991447\n", + "Returned IDs: 990061,990273,995309,993583,990206,994254,993500,996380,997547,992052,990042,994679,993077,995459,999022,993811,991977,991432,997866,995728,993742,990776,996328,992558,990894,993879,994280,997494,999639,994891\n", "Ground Truth: 990061,990273,995309,993583,990206,994254,993500,996380,991064,997547,992052,990042,994679,993077,995459,999022,993811,991977,991432,997866,995728,993742,990776,996328,995600,992558,990894,993879,994280,997494\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 761\n", - "Returned IDs: 990127,995198,990331,993209,995873,995778,992862,994409,995268,995401,990887,999179,993313,994488,992260,990975,996328,994954,992293,990930,992052,994835,992636,992595,995122,999581,991958,995864,998623,993561\n", + "Returned IDs: 990127,990331,993209,995778,992862,994409,995268,990887,995536,999179,993313,994488,992260,990975,996328,994954,990930,992052,994835,992636,993809,992595,991958,993561,992917,996001,996232,994742,995907,990593\n", "Ground Truth: 990127,995198,990331,993209,995873,995778,992862,994409,995268,995401,990887,995536,999179,993313,994488,992260,990975,997184,996328,994954,992293,990930,992052,994835,992636,993809,992595,995122,999581,991958\n", - "Recall: 0.9000\n", + "Recall: 0.7667\n", "\n", "Query ID: 762\n", - "Returned IDs: 997550,995596,994618,998921,999338,999426,996075,997661,997113,991992,995033,992041,995515,997448,996318,993785,992944,997584,996516,994971,997984,991984,993125,994233,996741,996631,996387,999698,999588,995398\n", + "Returned IDs: 997550,995596,994618,998921,999338,999426,996075,997661,997113,991992,995033,992041,995515,996318,993785,992944,997584,996516,994971,997984,991984,994233,996741,996631,996387,999698,999588,995467,992636,997291\n", "Ground Truth: 997550,995596,994618,998921,999338,999426,996075,997661,997113,991992,995033,992041,995515,997448,996318,993785,992944,997584,996516,994971,997984,991984,993125,995401,994233,996741,996631,996387,999698,999588\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", "\n", "Query ID: 763\n", - "Returned IDs: 995317,993558,993678,996194,992992,997854,993256,994541,993044,990987,997251,990069,999287,997429,995159,996438,991863,994064,999386,997806,994428,993621,993014,992313,996232,991285,997770,994639,999179,990776\n", + "Returned IDs: 995317,993678,996194,992992,997854,993256,994541,993044,990987,997251,990069,999287,995159,996438,994064,997806,993621,993014,992313,996232,991285,994639,999179,990776,991172,993948,994169,993039,990953,991319\n", "Ground Truth: 995317,993558,993678,996194,992992,997854,993256,994541,991075,993044,990987,997251,990069,995401,999287,997429,995159,996438,991863,994064,999386,997806,994428,993621,993014,992313,996232,991285,993454,997770\n", - "Recall: 0.9000\n", + "Recall: 0.7000\n", "\n", "Query ID: 764\n", - "Returned IDs: 994955,998933,993194,999798,991685,993665,999052,998536,997944,992825,994280,995168,997155,992850,995932,992561,993715,995589,995366,997453,992823,998778,995945,993931,993037,994167,990387,999840,990364,999543\n", + "Returned IDs: 994955,998933,993194,999798,991685,993665,998536,997944,992825,994280,995168,995932,992561,993715,995589,997453,992823,995945,993931,993037,994167,990387,999840,990364,999543,991712,995834,997506,990409,990658\n", "Ground Truth: 994955,998933,993194,999798,991685,993665,999052,998536,997944,992825,994280,995168,997155,992850,995932,992561,993715,995589,995366,997453,992823,998778,995945,993931,997926,993037,994167,990387,999840,990364\n", - "Recall: 0.9667\n", + "Recall: 0.8000\n", "\n", "Query ID: 765\n", - "Returned IDs: 994557,991058,997638,997807,993929,994119,995332,990879,990345,992704,997822,997309,990987,993014,990678,999210,997693,996071,991107,999643,990226,992147,994280,997945,994336,997126,992962,994541,991177,991487\n", + "Returned IDs: 994557,991058,997638,993929,994119,995332,990345,992704,997822,997309,990987,990678,999210,997693,991107,995620,999643,990226,992147,994280,997945,994336,997126,992962,994541,991177,999612,991378,991848,994142\n", "Ground Truth: 994557,991058,997638,997807,993929,994119,995332,990879,990345,992704,997822,997309,990987,993014,990678,999210,997693,996071,991107,995620,999643,990226,992147,994280,997945,994336,997126,992962,994541,991177\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 766\n", + "Returned IDs: 992293,990662,990771,998878,998017,998549,996147,996657,995158,999302,993848,996001,996759,999853,999922,997554,990811,999342,990573,992862,994223,991588,993344,996368,990211,997079,995575,998710,991585,993168\n", + "Ground Truth: 992293,990662,990771,998878,998017,998549,996147,996657,995158,999302,993848,996001,996759,995002,999853,999922,997554,990811,999342,990573,992862,994223,991588,999587,993344,993374,996368,990211,997079,995575\n", + "Recall: 0.9000\n", "\n", "Query ID: 768\n", - "Returned IDs: 999117,994643,993122,993608,995625,992677,999155,993232,990361,991987,996395,998078,999231,991630,997149,995807,999834,991052,996809,993795,998257,997110,994092,992206,997752,997568,995373,998536,995105,992288\n", + "Returned IDs: 999117,994643,993122,993608,995625,992677,999155,993232,990361,991987,996395,998078,999231,991630,997149,995807,999834,996809,993795,998257,997110,994092,997752,997568,995373,998536,995105,992288,993076,992685\n", "Ground Truth: 999117,994643,993122,993608,995625,992677,999155,993232,990361,996395,991987,998078,999231,991630,997149,995807,995401,999834,991052,996809,993795,998257,997110,994092,992206,997752,997568,995373,998536,995105\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 769\n", + "Returned IDs: 999388,990154,998971,999663,994636,999555,992282,992403,998701,991596,996406,997803,992155,993132,991385,999153,994181,990394,992716,994335,999898,994834,990305,994638,993458,994155,993560,999677,991154,997154\n", + "Ground Truth: 999388,990154,998971,999663,994636,999555,992282,992403,998701,991596,996406,992445,997803,992155,993132,991385,999153,994181,990394,992716,994335,999898,994834,990305,994638,993458,994155,993560,999677,991154\n", "Recall: 0.9667\n", "\n", "Query ID: 770\n", - "Returned IDs: 991889,995949,998294,995260,996053,998692,999374,991705,991406,996554,993837,992309,995928,993143,997627,996334,992979,995693,994160,994919,990432,996854,990490,995354,996202,992169,991445,997429,997083,994850\n", + "Returned IDs: 991889,995949,998294,995260,996053,998692,999374,991705,991406,992309,995928,993143,997627,996334,995693,994919,990432,996854,995354,992169,997429,990760,999432,990146,990958,998170,992252,994195,990189,996351\n", "Ground Truth: 991889,995949,998294,995260,996053,998692,999374,991705,991406,996554,993837,992309,990026,995928,993143,997627,996334,992979,995693,994160,994919,990432,990545,990103,993412,996854,990490,995354,996202,992169\n", - "Recall: 0.8667\n", + "Recall: 0.6667\n", "\n", "Query ID: 771\n", - "Returned IDs: 993416,992684,994855,991958,998158,998382,990794,993561,994987,996040,996062,990254,993673,991052,996371,999236,998593,992017,995923,994946,995547,993738,998323,994419,997453,999868,992260,998297,995373,998891\n", + "Returned IDs: 993416,992684,994855,991958,998158,998382,990794,993561,994987,996040,996062,990254,996371,998593,995923,995547,993738,994076,996834,994419,997453,999868,992260,998297,995373,998891,997406,998416,995401,996198\n", "Ground Truth: 993416,992684,994855,991958,998158,998382,990794,993561,994987,996040,996062,990254,993673,991052,996371,999236,998593,992017,995923,994946,995547,993738,994076,996834,998323,994419,997453,999868,992260,998297\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", "\n", "Query ID: 772\n", - "Returned IDs: 997002,998902,994906,992790,994334,997301,996658,999442,998951,995560,991621,994661,992239,996001,995401,992165,995374,995836,992759,996220,994362,993820,995357,998045,997918,990140,999587,994088,992641,995536\n", + "Returned IDs: 997002,998902,994906,992790,994334,995894,997301,996658,999442,998951,995560,991621,994661,992239,996001,995401,992165,995836,992759,996220,995357,997918,999587,994088,992641,995536,990561,991485,997320,994946\n", "Ground Truth: 997002,998902,994906,992790,994334,995894,997301,996658,999442,998951,995560,991621,994661,992239,996001,995401,992165,995374,995836,992759,996220,994362,993820,995357,998045,997918,991989,990140,999587,994088\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", "\n", "Query ID: 773\n", - "Returned IDs: 992041,996075,990022,990366,993785,992167,990697,991372,995515,994440,999426,991984,996001,991109,999473,997836,998654,997661,993928,995596,996227,992872,998498,996219,993484,992733,992927,999053,992617,999806\n", + "Returned IDs: 992041,996075,990022,990366,993785,992167,990697,991372,995515,994440,999426,991984,996001,991109,999473,997661,995596,992872,998498,996219,993484,992733,992927,999053,990934,992617,996996,999806,990957,993807\n", "Ground Truth: 992041,996075,990022,990366,993785,992167,990697,991372,995515,994440,999426,991984,996001,991109,999473,997836,994776,998654,997661,993928,995596,996227,992872,998498,996219,993484,992733,992927,999053,990934\n", - "Recall: 0.9333\n", + "Recall: 0.8333\n", "\n", "Query ID: 774\n", - "Returned IDs: 992383,990383,991550,990884,999387,996445,999451,990376,991923,994964,996894,995983,992824,996196,998600,998401,990538,993781,995427,995372,999639,993706,997903,994979,994852,995459,991817,993492,992190,996085\n", + "Returned IDs: 992383,990383,991550,990884,999387,999451,990376,994964,996894,995983,992824,996196,998600,998401,990538,993781,995427,995372,999639,993706,997903,994979,994852,995459,991817,993492,992190,996085,998697,995841\n", "Ground Truth: 992383,990383,991550,990884,999387,996445,999451,990376,991923,994964,996894,995983,992824,996196,998600,998401,990538,993781,995427,995372,999639,993706,997903,994979,994852,995459,991817,993492,992190,998545\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", "\n", "Query ID: 775\n", - "Returned IDs: 990832,995100,990194,993435,997725,997331,996475,996809,996219,996830,992892,999435,991187,995880,991459,995131,999331,999384,996067,996216,993342,998122,995130,999430,999388,999647,993192,996913,992100,997047\n", + "Returned IDs: 990832,995100,990194,993435,997725,997331,996475,996809,996219,996830,999435,995880,995131,999331,996216,993342,998084,998122,999430,999388,999647,992100,997047,992237,999898,993148,993391,993931,998387,990957\n", "Ground Truth: 990832,995100,990194,993435,997725,997331,996475,996809,996219,996830,992892,999435,991187,995880,991459,995131,999331,999384,996067,996216,993342,998084,998122,995130,999430,999388,999647,993192,996913,998925\n", - "Recall: 0.9333\n", + "Recall: 0.7000\n", "\n", "Query ID: 776\n", - "Returned IDs: 999159,991582,995632,997284,997354,994225,991348,996296,997320,992691,993018,991763,991992,993454,999777,995194,991879,996199,996387,991131,992896,990456,991662,996075,995108,996626,999658,994415,990375,998950\n", + "Returned IDs: 999159,991582,995632,997284,997354,991348,996296,997320,992691,993018,991763,993454,999777,995194,991879,996199,996387,991131,992896,990456,996075,996626,999658,994415,990375,998950,997370,994341,997315,993253\n", "Ground Truth: 999159,991582,995632,997284,997354,994225,991348,995878,992022,996296,997320,992691,993018,991763,991992,993454,999777,995194,991879,996199,996387,991131,992896,990456,991662,996075,995108,996626,999658,994415\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", "\n", "Query ID: 777\n", - "Returned IDs: 999663,995536,992225,998514,998861,999009,999711,990096,998720,991459,992293,996809,996001,992553,993139,994434,995401,995701,992862,998200,996406,999216,998082,998568,997586,993948,995961,997506,991893,998495\n", + "Returned IDs: 999663,995536,992225,998514,998861,999009,999711,990096,998720,991459,992293,996809,996001,992553,993139,994434,995701,992862,998200,999216,998082,997586,993948,997506,991893,998495,996996,991154,990786,999898\n", "Ground Truth: 999663,995536,992225,998514,998861,999009,999711,990096,998720,991459,992293,996809,996001,992553,993139,994434,995401,993821,995701,992862,998200,996406,999216,998082,998568,997586,993948,995961,997506,991893\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 778\n", - "Returned IDs: 994401,993039,997077,998972,995065,992985,993931,995159,990184,998851,998566,996277,993799,994835,992576,997181,995760,995268,993256,996505,999759,990291,990942,995790,995815,993879,997126,990956,990391,995685\n", + "Returned IDs: 994401,993039,998972,995065,992985,993931,995159,990184,998851,998566,996277,993799,994835,992576,997181,995648,995760,996223,995268,993256,996505,999759,995790,995815,993879,997126,990956,990391,995685,996173\n", "Ground Truth: 994401,993039,997077,998972,995065,992985,993931,995159,990184,998851,998566,996277,993799,994835,992576,997181,995648,995760,996223,995268,993256,996505,999759,990291,990942,995790,995815,993879,997126,990956\n", - "Recall: 0.9333\n", + "Recall: 0.9000\n", "\n", "Query ID: 779\n", - "Returned IDs: 998496,995840,994879,996075,991166,992442,991984,993834,997258,992041,998708,993902,998200,997586,997305,993766,992330,996230,993809,996933,991476,995701,994558,996001,999701,996657,993696,999041,993283,991823\n", + "Returned IDs: \n", "Ground Truth: 998496,995840,994879,996075,991166,992442,991984,993834,997258,992041,998708,993902,998200,997586,997305,997770,993766,992330,996230,993174,993809,996933,991476,995701,994558,996001,999701,996657,993696,999041\n", - "Recall: 0.9333\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 780\n", + "Returned IDs: \n", + "Ground Truth: 998428,996270,991605,997151,990010,999139,994196,993715,993181,991633,996483,990705,990712,994101,995421,994280,995349,999435,992779,993460,990998,993347,994351,998250,992441,994510,994193,991212,999205,997733\n", + "Recall: 0.0000\n", "\n", "Query ID: 781\n", - "Returned IDs: 992226,994273,990815,994098,999430,998671,995704,999144,990711,993100,999435,996816,994558,997955,998416,993725,996222,994801,991672,999225,997331,996328,991938,999532,991378,998729,992951,993501,996856,993127\n", + "Returned IDs: 994273,994098,999430,998671,999144,990711,993100,996816,991840,998416,993161,996222,994801,991672,999225,997331,996328,999532,991378,993501,993127,990392,997193,993929,994351,990226,993805,997967,990408,990340\n", "Ground Truth: 992226,994273,990815,994098,999430,998671,995704,999144,990711,993100,999435,996816,994558,991840,997955,998416,993725,993161,996222,994801,991672,999225,997331,996328,999171,998142,991938,999532,991378,998729\n", - "Recall: 0.8667\n", + "Recall: 0.6333\n", "\n", "Query ID: 782\n", - "Returned IDs: 999530,998070,992448,995685,991301,994409,996323,991699,994002,996987,998714,998545,997046,997675,995507,992605,999578,999043,996322,993412,991671,996607,991247,999884,991583,993400,996042,995401,991052,995139\n", + "Returned IDs: 999530,998070,992448,995685,991301,994409,996323,991699,994002,996987,998714,998545,997046,997675,995507,992605,999578,999043,996322,991671,996607,991247,999884,991583,996042,995139,993561,998910,998045,996390\n", "Ground Truth: 999530,998070,992448,995685,991301,994409,996323,991699,994002,996987,998714,998545,997046,997675,995507,992605,999578,999043,996322,993412,991671,996607,998697,991247,990337,999884,991583,993400,996042,995401\n", - "Recall: 0.9333\n", + "Recall: 0.8333\n", "\n", "Query ID: 783\n", - "Returned IDs: 990089,995454,995720,997292,995659,999791,995045,993637,999858,992334,991537,999444,990918,996147,998076,998097,992636,998302,999302,995306,992415,996516,997490,994373,995133,991144,996001,995755,997349,997933\n", + "Returned IDs: 990089,995454,997292,995659,999791,995045,993637,992334,999444,990918,996147,998076,998097,992636,998302,999302,995306,992415,996516,997490,991144,996001,997349,997933,995295,995345,999099,999338,997533,998921\n", "Ground Truth: 990089,995454,995720,997292,995659,999791,995045,993637,999858,992334,991537,999444,990918,996147,998076,998097,992636,998302,999302,995306,992415,996516,997490,994373,995133,991144,996001,995755,993663,997349\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 784\n", + "Returned IDs: 998256,999852,990998,998279,996986,996147,993335,996189,999342,999334,998343,995326,999273,999924,998959,996960,997790,994830,990402,994185,992636,990771,990956,998335,994891,990434,992664,996328,990027,996594\n", + "Ground Truth: 998256,999852,990998,998279,996986,991347,996147,993335,996189,999342,999334,998343,995326,999273,999924,999555,998959,996960,997790,998122,994830,990402,998680,994185,997506,993545,992636,990771,990956,997494\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 786\n", + "Returned IDs: 990635,998143,998205,993672,996187,997868,995864,994954,998941,999926,992521,991969,998587,991892,993269,990341,995814,999250,998527,996256,990603,996987,994409,997909,997466,992448,996169,998278,992426,996162\n", + "Ground Truth: 990635,998143,998205,993672,996187,997868,995864,994954,998941,999926,992521,991969,998587,991892,993269,990341,995814,999250,998527,996256,990603,996987,994409,997909,997466,992448,996383,996169,998278,992426\n", "Recall: 0.9667\n", "\n", "Query ID: 787\n", - "Returned IDs: 994670,991511,996219,992630,995274,990679,994916,992931,996714,990271,991085,992277,993154,994778,999243,997976,990910,995161,999340,997404,992748,999596,991456,995981,997018,995502,995901,990448,994894,990612\n", + "Returned IDs: 994670,991511,996219,992630,995274,990679,992931,992124,996714,990271,992277,993154,994778,999243,997976,990910,997404,992748,999596,991456,995981,995502,990448,994894,992134,997571,997348,998337,999804,993342\n", "Ground Truth: 994670,991511,996219,993517,992630,995274,990679,994916,990909,992931,992124,999723,996714,990271,991085,992277,993154,994778,999243,997976,990910,995161,999340,997404,992748,999596,991456,995981,993481,997018\n", - "Recall: 0.8333\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 788\n", + "Returned IDs: 997411,994730,992293,997436,993725,997925,997529,990435,999922,994606,992605,992582,995701,993342,996368,999740,998000,997723,996028,990811,995435,992024,995016,998934,992473,994382,994636,993168,996001,990012\n", + "Ground Truth: 997411,994730,992293,997436,993725,990815,997925,997529,990435,999922,994606,992605,998105,992582,993856,995701,993342,996368,995906,999740,998000,997723,991301,994674,996028,994728,990811,998708,995435,998181\n", + "Recall: 0.7000\n", "\n", "Query ID: 789\n", - "Returned IDs: 999240,993473,997195,993986,992558,999860,993811,991550,993077,995552,991025,992121,998929,990078,995972,999479,999715,996918,991817,996750,993158,993400,995967,996196,990249,994168,997054,991426,994679,992375\n", + "Returned IDs: 999240,993473,997195,993986,992558,999860,993811,991595,991550,993077,995552,991025,992121,998929,990078,999479,999715,996750,993158,995967,996196,990249,997054,991426,994679,992375,999400,992161,999540,996523\n", "Ground Truth: 999240,993473,997195,993986,992558,999860,993811,991595,991550,993077,995552,991025,992121,998929,990078,995972,999479,999715,996918,991817,996750,993158,993400,995967,996196,990249,999631,994168,997054,991426\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 790\n", + "Returned IDs: 992147,994964,994850,995516,999210,995927,995181,997305,996219,999374,998302,995132,994778,994584,990843,991650,998123,994789,991999,995755,998950,996075,991801,996504,991348,996186,996805,991498,997831,995656\n", + "Ground Truth: 992147,994964,994850,995516,998464,998287,999210,995927,993154,995181,997305,996219,991582,999374,998302,995132,994778,994584,990843,991650,998123,994789,991999,995755,998950,996075,997906,991801,996504,991348\n", + "Recall: 0.8333\n", "\n", "Query ID: 791\n", - "Returned IDs: 996633,992598,994639,996155,999059,999192,994541,998761,995551,990392,998348,993825,997026,994169,996960,992992,996194,997838,994356,993793,993448,993314,994742,993691,992832,992540,994327,993725,992595,994861\n", + "Returned IDs: 996633,992598,994639,996155,999059,999192,994541,998761,995551,990392,998348,993825,997026,994169,996960,996194,994356,993448,993314,994742,993691,992832,994327,993725,992595,994861,995122,999774,999225,997340\n", "Ground Truth: 996633,992598,994639,996155,999059,999192,994541,998761,995551,990392,998348,993825,997026,994169,996960,992992,996194,997838,994356,993793,993448,993314,995401,994742,993691,992832,992540,994327,993725,992595\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 792\n", - "Returned IDs: 998902,998236,991131,998967,994604,991999,991359,990444,998001,993399,995515,991499,997918,998492,995357,998734,990176,999698,992839,991157,997831,992944,994329,995865,995988,995364,994789,994049,991609,991407\n", + "Returned IDs: 998902,998236,991131,998967,996030,994604,991999,998001,993399,995515,997918,998492,995357,998734,990176,999698,991157,992944,994329,995865,995988,994789,991609,995372,991801,993244,999453,990843,994216,992510\n", "Ground Truth: 998902,998236,991131,998967,996030,994604,991999,991359,990444,998001,993399,995515,991499,997918,998492,995357,998734,990176,999698,992839,994449,991157,997831,992944,994329,990219,995865,995988,995364,994789\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 793\n", + "Returned IDs: 995047,997790,996001,995184,998593,999827,990399,993289,993280,990364,992862,994743,996657,998697,991212,994280,998250,999453,997897,997002,990096,992395,996043,994742,996062,996368,999019,998303,993931,995536\n", + "Ground Truth: 995047,997790,996001,995184,998593,999827,990399,993289,993280,990364,992862,995401,996657,994743,998697,991212,994280,998250,996277,999453,997897,997002,990096,992395,996043,994742,997494,996062,996368,999019\n", "Recall: 0.9000\n", "\n", "Query ID: 794\n", - "Returned IDs: 998679,990930,991585,993293,993813,998779,995633,995289,991141,994742,994730,994140,994436,993997,999402,998980,990435,993856,994409,991664,993831,995702,995648,997723,995216,997554,999805,995052,996277,999163\n", + "Returned IDs: 998679,990930,991585,993293,993813,998779,995633,995289,991141,994742,994730,994140,994436,993997,999402,998980,990435,993856,994409,991664,993831,995702,995648,995216,997554,995052,999163,990940,995268,995016\n", "Ground Truth: 998679,990930,991585,993293,993813,998779,995633,995289,991141,994742,994730,994140,994436,993997,999402,998980,990435,993856,994409,991664,993831,995702,995648,997723,990361,995216,997554,999805,995052,996277\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 795\n", + "Returned IDs: 997586,995887,994832,990605,992068,996429,997379,996217,999711,993260,990866,993114,992112,997334,992983,990547,999284,992661,991880,999176,996105,996305,995430,999210,994558,999430,998513,994247,998359,995672\n", + "Ground Truth: 997586,995887,994832,990605,992068,999467,996429,997379,996217,996901,999711,993260,990866,993114,999216,992112,997334,992983,994200,990547,999284,992661,991880,999176,996424,996105,996305,995321,995430,998825\n", + "Recall: 0.7667\n", "\n", "Query ID: 796\n", - "Returned IDs: 995252,992932,997320,997483,992096,990231,990541,997771,992228,995336,992277,992692,991900,992330,998090,992069,992969,998951,990885,998246,998365,998653,996296,993696,991131,990172,992978,995802,996196,992942\n", + "Returned IDs: 995252,992932,997320,997483,990231,997771,992228,995336,992692,991900,992330,998090,992069,992969,998951,990885,998246,998365,996296,993696,991131,990172,992978,995802,996196,992942,995942,996646,994757,993529\n", "Ground Truth: 995252,992932,997320,997483,992096,990231,990541,997771,992228,995336,992277,992692,991900,992330,998090,992069,992969,998951,990885,998246,998365,998653,996296,993696,991131,990172,992978,995802,996196,996246\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 797\n", + "Returned IDs: 990050,996825,999206,993165,992358,996087,998507,993155,994051,998267,996918,997420,991319,997806,997438,996394,999296,990116,995516,997752,999881,998537,991799,993727,998778,999646,993453,991107,999871,997942\n", + "Ground Truth: 990050,996825,999206,993165,992358,996087,998507,993155,994051,998267,996918,997420,991319,997806,997438,999105,998021,996394,999296,990116,990386,994301,995516,997752,999881,998537,991799,997416,993727,998778\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 798\n", + "Returned IDs: 992832,995551,996633,994806,995326,992598,997814,999126,997026,993314,996155,994541,998348,993448,990127,993715,991850,997838,998536,991712,994908,994835,996328,993014,997340,998073,995159,995122,993046,997282\n", + "Ground Truth: 992832,995551,996633,994806,995326,992598,997814,999126,997026,993314,995401,996155,994541,998348,993448,990127,993715,991850,997838,998536,991712,994908,994835,996328,993014,997340,990956,998073,993560,995159\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 799\n", + "Returned IDs: 990941,996809,993727,990255,991471,999158,995963,998536,991890,996930,990930,999906,990194,998250,993232,992809,995144,994351,999706,996797,990116,994431,995701,990521,990956,993293,993980,996408,991245,996531\n", + "Ground Truth: 990941,996809,993727,990255,996043,991471,999158,990361,995963,998536,991890,996930,990930,997723,999906,990194,998250,993232,992809,995144,994351,999706,996797,993137,990116,994431,995701,990521,990956,993293\n", + "Recall: 0.8667\n", "\n", "Query ID: 800\n", - "Returned IDs: 993708,995357,992134,993633,992692,997918,998967,993145,998996,991015,996592,991131,998011,999077,995336,995693,993492,993781,992946,990957,993512,998245,999144,996075,992277,993683,997470,999784,995210,996840\n", + "Returned IDs: 993708,995357,992134,992692,997918,998967,993145,991015,996592,993696,991131,999077,995336,993492,993781,990957,993512,998245,999144,996075,993683,999784,995210,996840,996085,994552,993529,998734,990949,994046\n", "Ground Truth: 993708,995357,992134,993633,992692,997918,998967,993145,998996,991015,996592,993696,991131,998011,999077,995336,995693,993492,993781,992946,990957,993512,998245,999144,996075,992277,993683,997470,999784,995210\n", - "Recall: 0.9667\n", + "Recall: 0.7667\n", "\n", "Query ID: 801\n", - "Returned IDs: 995203,993053,990918,993561,996645,990096,998416,995283,995873,990271,991464,994558,997370,994145,999653,994742,995310,997906,992334,998382,991168,994853,991565,999806,992408,995756,997920,990827,997305,999384\n", + "Returned IDs: \n", "Ground Truth: 995203,993053,990918,993561,996645,990096,998416,995283,995873,990271,991464,994558,997370,994145,999653,994742,996147,993154,995310,997906,992334,998382,991168,994853,991565,999806,992408,995756,997920,990827\n", - "Recall: 0.9333\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 802\n", + "Returned IDs: 998500,991720,998921,998839,996218,994281,995181,996504,996890,992147,999263,995212,996785,993048,998029,993648,993443,991743,999374,995985,997470,995372,997371,994850,996805,997301,994497,992789,999483,995884\n", + "Ground Truth: 998500,991720,995621,998921,998839,996218,994281,995181,996504,996890,992147,999263,995212,996785,993048,990362,998029,993648,993443,991743,999374,991626,995985,997470,995372,994179,997371,994850,996805,997301\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 803\n", + "Returned IDs: 992321,995404,994163,990159,996196,993612,991009,999564,990548,996766,996690,994384,994950,996637,994957,992608,992215,995013,996127,992558,993670,992661,997775,994912,998220,996963,997902,990303,991550,994000\n", + "Ground Truth: 992321,995404,994163,990159,996196,993612,992458,991009,999564,990548,996766,996690,994384,994950,991601,996637,994957,992608,992215,995013,996127,992558,993670,999639,992661,997775,994912,998220,996963,992650\n", + "Recall: 0.8667\n", "\n", "Query ID: 804\n", - "Returned IDs: 997309,997914,990705,992169,991720,998123,990776,999273,993677,994119,998495,996328,994765,994280,992364,996001,999179,995372,998180,990328,993993,991882,992944,991334,994310,994835,996809,994587,994531,997480\n", + "Returned IDs: 997309,997914,990705,992169,998123,990776,999273,993677,994119,998495,996328,994765,994280,992364,996001,999179,998180,990328,993993,991882,994817,992944,991334,994310,994835,996809,994587,997480,995295,995326\n", "Ground Truth: 997309,997914,990705,992169,991720,998123,990776,999273,993677,994119,998495,996328,994765,994280,992364,996001,999179,995372,998180,990328,993993,991882,998466,994817,992944,991334,994310,994835,996809,994587\n", - "Recall: 0.9333\n", + "Recall: 0.9000\n", "\n", "Query ID: 805\n", - "Returned IDs: 991642,990011,993906,995295,991435,993725,994879,995852,997002,994689,991315,998848,997790,992161,996811,990954,994206,994964,995823,992226,998359,996328,995586,998105,990494,997918,999500,994307,994012,994742\n", + "Returned IDs: 991642,990011,993906,995295,993725,994879,995852,997002,994689,991315,997790,996811,995823,998359,996328,995586,997918,999500,994307,994012,994742,990532,998067,999025,997379,990600,996132,990174,994817,999891\n", "Ground Truth: 991642,990011,993906,995295,991435,993725,994879,995852,997002,994689,991315,998848,997790,992161,996811,990954,994206,994964,995823,992226,998359,991989,996328,995586,998105,990494,997918,999500,994307,994012\n", - "Recall: 0.9667\n", + "Recall: 0.6667\n", "\n", "Query ID: 806\n", - "Returned IDs: 998343,990976,992056,995185,990255,990449,993875,997088,991256,995695,992293,998831,999334,995672,993545,994510,990399,998834,999498,998589,999663,998959,991836,999087,999502,997790,994815,995074,991038,992862\n", + "Returned IDs: 998343,990976,992056,995185,990255,990449,993875,995695,992293,998831,999334,995672,994510,998453,990399,999498,998589,999663,991836,999087,999502,997790,996082,994815,991038,992862,990959,993515,994365,990292\n", "Ground Truth: 998343,990976,992056,995185,990255,990449,993875,997088,991256,995695,992293,998831,999334,995672,993545,994510,998453,990399,998834,999498,998589,999663,998959,991836,999087,999502,997790,996082,995074,994815\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 807\n", + "Returned IDs: 993052,999453,996333,999122,992571,993260,990196,998123,996193,997918,999210,991713,993504,993080,992944,993071,992169,994107,993143,990882,995203,993355,998180,997002,996981,994200,997370,998950,993280,993781\n", + "Ground Truth: 993052,999453,996333,999122,992571,993260,990196,998123,996193,997918,999210,991713,993504,993080,990098,992944,993071,992169,994107,993143,991052,990882,995203,993355,998180,996676,997002,994162,996981,994200\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 808\n", + "Returned IDs: 999707,991783,997331,993192,996475,990039,995298,991382,997481,993617,994558,999430,997436,996830,999435,994830,993435,992628,991595,997442,996495,994098,994113,999556,995342,998926,997270,994897,992479,990377\n", + "Ground Truth: 999707,991783,997331,993192,996475,990039,995298,991382,997481,993617,990640,994558,999430,997436,990527,996830,999435,994830,993435,992628,996219,993566,991595,997442,996495,993260,994098,994113,999556,995342\n", + "Recall: 0.8333\n", "\n", "Query ID: 809\n", - "Returned IDs: 994964,996219,991582,993168,993289,999827,995927,997370,993143,997920,997822,999104,991052,997732,993412,997918,997586,994922,990866,994280,995210,990146,999343,992169,992147,997028,996001,995536,990382,990096\n", + "Returned IDs: 994964,996219,991582,993168,993289,999827,995927,997370,993143,997822,999104,997732,997918,997586,994922,990866,994280,990862,990146,999343,992169,992147,996001,995536,990382,990096,990224,996062,997002,997284\n", "Ground Truth: 994964,996219,991582,993168,993289,999827,995927,997370,993143,997920,997822,999104,991052,997732,993412,997918,997586,994922,990866,994280,995210,990862,990146,999343,992169,992147,997028,999289,996001,995536\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", "\n", "Query ID: 810\n", "Returned IDs: 998314,999831,995385,993086,990096,998536,996918,992407,998177,998405,999287,993965,999079,994661,999194,992747,991052,996733,991093,998968,991724,991243,993239,999320,991571,993609,993715,997897,999384,990583\n", "Ground Truth: 998314,999831,995385,993086,990096,998536,996918,992407,998177,998405,999287,993965,999079,994661,999194,992747,991052,996733,991093,998968,991724,991243,993239,999320,991571,993609,997775,993715,997897,999384\n", "Recall: 0.9667\n", "\n", + "Query ID: 811\n", + "Returned IDs: 992215,998213,992615,998827,994287,997859,996196,990068,991550,994163,995886,992747,997124,995552,996442,992558,990343,995372,995983,999639,993693,992170,991817,993130,997258,999385,993260,996127,996404,990620\n", + "Ground Truth: 992215,998213,992615,998827,994287,997859,996196,990068,991550,994163,995886,992747,997124,990486,995552,996442,992558,990343,995372,995983,993693,999639,992170,990910,991817,993130,997258,999385,993260,996127\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 812\n", + "Returned IDs: 990925,994160,996343,990958,998797,999262,995884,992546,995947,990535,991396,995485,990461,992994,993083,990146,992284,997102,991444,998458,995771,993562,999600,990936,994922,994685,998927,999644,997192,996959\n", + "Ground Truth: 990925,994160,996343,990958,998797,999262,995884,992546,995947,990535,991396,995485,990461,992994,993083,990146,992284,997102,997876,991444,998458,995771,993562,999600,993914,997053,990936,994922,994685,998927\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 813\n", + "Returned IDs: 994334,999144,997012,994819,994088,996658,995374,999820,998011,996903,995279,990215,997571,997918,999089,996425,992087,997910,999374,997002,996282,998385,995453,991131,991977,991074,993407,992147,990275,999026\n", + "Ground Truth: 994334,999144,997012,994819,994088,996658,995374,999820,998011,996903,995279,990215,997571,993868,997918,999089,996425,992087,997910,999374,993412,997002,996282,998385,995453,999562,991131,991977,991074,993407\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 815\n", + "Returned IDs: 991269,990392,995283,992713,994853,999126,998811,991168,993678,991189,995295,993561,991464,994742,996411,996752,991769,993813,997141,991789,997452,993725,995633,999496,991981,996628,993209,994090,994119,992832\n", + "Ground Truth: 991269,990392,995283,997682,992713,994853,999126,998811,991168,993678,991189,995295,993561,991464,994742,997933,996411,996752,991769,994386,993813,997141,996055,991789,990614,997452,993725,999496,995633,991981\n", + "Recall: 0.8333\n", + "\n", "Query ID: 816\n", - "Returned IDs: 993392,992027,995771,993620,997984,996504,998921,996218,993518,997704,994281,994939,991782,998692,993457,995985,991873,994922,998822,992709,991889,999374,990595,997192,997006,992546,994775,992147,993161,994808\n", + "Returned IDs: 993392,992027,995771,993620,997984,996504,998921,996218,993518,997704,994281,994939,991782,998692,995985,994922,998822,992709,991889,999374,990595,997192,992546,994775,992147,993161,999327,994641,995554,994630\n", "Ground Truth: 993392,992027,995771,993620,997984,996504,998921,996218,993518,997704,994281,994939,991782,998692,993457,995985,991873,994922,998822,992709,991889,992932,999374,990595,997192,997006,992546,994775,992147,993161\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 817\n", - "Returned IDs: 994304,997868,997586,992521,995814,994636,994954,996918,992293,997723,993438,997466,998260,995701,990866,999731,991790,998255,993831,997305,998346,995873,990097,991967,990078,996360,996187,994419,998934,997216\n", + "Returned IDs: 994304,997868,997586,992521,995814,994636,994954,996918,992293,997723,993438,997466,998260,995701,999731,991790,996386,998255,993831,997305,998346,990097,991967,990078,996187,997216,995963,996368,993053,991163\n", "Ground Truth: 994304,997868,997586,992521,995814,994636,994954,996918,992293,997723,993438,997466,998260,995701,990866,999731,991790,996386,998255,993831,997305,998346,995873,990097,991967,990078,996360,996187,994419,998564\n", - "Recall: 0.9333\n", + "Recall: 0.8333\n", "\n", "Query ID: 818\n", - "Returned IDs: 992909,994214,997320,998808,998090,992978,994290,991479,999621,990588,994427,991883,995372,995802,998364,992692,995826,995961,995228,994775,993662,992069,991503,991873,992969,994280,999761,997913,990910,993512\n", + "Returned IDs: \n", "Ground Truth: 992909,994214,995090,997320,998808,998090,992978,994290,991479,999621,990588,994427,991883,995372,995802,998364,992692,995826,995961,995228,994775,993662,992069,991503,991873,992969,994280,999761,997913,998821\n", - "Recall: 0.9333\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 819\n", + "Returned IDs: 992293,995701,993848,993834,992225,998549,996657,992862,998593,992367,990811,995158,990255,998886,990211,998200,994809,994587,998878,992773,993937,998215,999853,990662,990573,990399,990097,997687,999977,994409\n", + "Ground Truth: 992293,995701,993848,993834,992225,998549,996657,992862,998593,992367,990811,995158,990255,998886,990211,998200,994809,994587,998878,992773,993937,991603,998215,999853,990662,992473,990573,990399,991630,990097\n", + "Recall: 0.9000\n", "\n", "Query ID: 820\n", - "Returned IDs: 997331,998147,994113,995382,992433,993281,998770,999643,991011,996494,996722,993778,993939,996067,996866,995620,992840,991148,998484,990527,996219,991179,994692,995401,997837,996080,991377,994589,998855,994557\n", + "Returned IDs: 997331,998147,994113,995382,992433,993281,998770,996702,999643,991011,996494,996722,993778,993939,996067,995620,991148,998484,996219,991179,994692,997837,996080,991377,994589,994382,993936,996371,996475,995032\n", "Ground Truth: 997331,998147,994113,995382,992433,993281,998770,996702,999643,991011,996494,996722,993778,993939,996067,996866,995620,992840,991148,998484,990527,996219,991179,994692,994984,995401,992987,997837,996080,991377\n", - "Recall: 0.9000\n", + "Recall: 0.8000\n", "\n", "Query ID: 821\n", - "Returned IDs: 999837,992752,991633,998594,991242,991552,991605,998279,997673,993611,998389,997151,995167,996082,995421,998108,992864,990950,993715,999139,997050,995705,995626,990017,994734,996216,995232,994196,993347,995103\n", + "Returned IDs: 999837,992752,991633,998594,991552,991605,998279,997673,993611,998389,997151,996082,995421,998108,992864,993715,999139,997050,995705,995626,994734,996216,995232,994196,993347,996270,991469,993001,997897,996328\n", "Ground Truth: 999837,992752,991633,998594,991242,991552,990133,991605,998279,997673,993611,998389,997151,995167,996082,995421,998108,992864,990950,993715,998716,999139,997050,995705,995626,990017,996216,994734,995232,994196\n", - "Recall: 0.9333\n", + "Recall: 0.8000\n", "\n", "Query ID: 822\n", - "Returned IDs: 991765,993319,999798,996914,997340,996535,995768,998525,990408,997714,998168,994356,999601,990183,991779,997925,995222,992250,995501,992517,994841,992131,993993,996918,995755,990711,999886,999052,999496,998224\n", + "Returned IDs: 991765,993319,999798,993396,996914,997340,996535,995768,998525,990408,997714,998168,994356,999601,991779,992250,995501,992517,994841,993993,996918,999886,999052,999496,998224,991712,991850,992561,999256,992868\n", "Ground Truth: 991765,993319,999798,993396,996914,997340,996535,995768,998525,990408,997714,998168,994356,999601,990183,991779,997925,995222,992250,993817,995501,992517,994841,992131,993993,996918,995755,990711,999886,999052\n", - "Recall: 0.9333\n", + "Recall: 0.7667\n", "\n", "Query ID: 823\n", - "Returned IDs: 991356,992558,991977,997547,993255,990602,990226,994560,999908,993583,990894,999305,993077,997859,992113,993742,998151,992827,993839,994472,998892,995374,996318,995886,995552,995542,999026,990279,997918,998495\n", + "Returned IDs: 991356,992558,991977,997547,993255,990602,990226,994560,999908,993583,990894,999305,993077,997859,992113,993742,992827,993839,994472,998892,996318,995886,995552,995542,999026,990279,997918,998495,992824,994254\n", "Ground Truth: 991356,992558,991977,997547,993255,990602,990226,994560,999908,993583,997806,990894,999305,993077,997859,992113,993742,998151,992827,993839,994472,990995,998892,995374,996318,995886,995552,995542,999026,990279\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 824\n", + "Returned IDs: 993324,991258,998272,994349,996600,993079,994765,994587,996810,995130,992223,993347,992178,996175,996777,995401,994092,994235,993548,998543,994743,995017,991255,996314,994119,996930,998950,997993,990930,990668\n", + "Ground Truth: 993324,991258,998272,994349,996600,993079,994765,994587,996810,995130,992223,995375,993347,992178,996175,996277,996777,995401,994092,994235,993548,998543,994743,995017,991255,996314,998355,994119,996930,996318\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 825\n", + "Returned IDs: 994682,998006,997666,995209,990712,990998,992209,995798,993877,997527,998122,998084,992652,992237,990091,996209,992892,998055,990194,993224,996960,999273,990226,994280,997618,995223,991382,998671,990806,997506\n", + "Ground Truth: 994682,998006,997666,995209,990712,990998,992209,995798,993877,997527,998122,998084,992652,992237,995859,990091,996209,992892,998055,990194,993224,996960,999273,990226,994280,997618,995223,991382,999441,998671\n", "Recall: 0.9333\n", "\n", + "Query ID: 826\n", + "Returned IDs: 996362,993478,997020,992169,997723,992161,997918,998758,997420,997395,993030,995516,997675,995203,997693,999075,995783,996305,998593,994248,990103,999179,994409,996001,996918,994280,992124,996867,993809,996075\n", + "Ground Truth: 996362,993478,997020,992169,997723,992161,997918,998758,997420,997531,997395,993030,995516,997675,991799,995203,997693,999075,995783,996305,998593,992933,994248,990103,990279,999179,994409,996001,996918,994280\n", + "Recall: 0.8667\n", + "\n", "Query ID: 827\n", - "Returned IDs: 994051,993396,996899,999601,991399,994801,998766,997714,991052,996042,995963,996918,995342,997723,990189,994280,998123,993578,993805,993727,992933,996315,999820,996127,998267,993014,998778,993165,998397,990103\n", + "Returned IDs: 994051,993396,996899,999601,991399,994801,991052,996042,995963,996918,995342,997723,990189,994280,998123,993805,993727,996315,996127,995592,998267,993014,998778,993165,994817,998397,990103,992719,991765,997693\n", "Ground Truth: 994051,993396,996899,999601,991399,994801,998766,997714,991052,996042,995963,996918,995342,997723,990189,994280,992569,998123,999596,993578,993805,993727,992933,996315,999820,996127,997803,995592,998267,993014\n", - "Recall: 0.8667\n", + "Recall: 0.7333\n", "\n", "Query ID: 828\n", - "Returned IDs: 997116,996495,994142,999273,997506,998456,993139,996328,990712,995571,996408,993993,994680,990408,994196,998168,999985,991667,990091,999139,996431,991633,994012,993560,990705,994296,991552,990479,990449,998594\n", + "Returned IDs: \n", "Ground Truth: 997116,996495,994142,999273,997506,998456,993139,996328,990712,995571,996408,993993,994680,990408,994196,990293,998168,999985,991667,990091,999139,996431,991633,994012,993560,990705,994296,991552,990479,990449\n", - "Recall: 0.9667\n", + "Recall: 0.0000\n", "\n", "Query ID: 829\n", - "Returned IDs: 990310,993143,998033,991157,996840,999257,991392,995584,999122,999144,991609,991015,991840,995515,990580,999676,996829,997593,992950,992734,997379,993355,997320,996007,999546,997037,997918,998607,996282,992612\n", + "Returned IDs: \n", "Ground Truth: 990310,993143,998033,991157,996840,999257,991392,995584,999122,999144,991609,991015,991840,995515,990580,999676,996829,997593,992950,992734,997379,998909,993355,997320,996007,994757,999546,992979,997037,997918\n", - "Recall: 0.9000\n", + "Recall: 0.0000\n", "\n", "Query ID: 830\n", - "Returned IDs: 993324,995375,997184,990930,995130,990646,994092,991712,992933,993533,998272,998349,997675,996277,996328,993561,996591,991052,994743,993293,994409,992790,995080,993931,993347,999533,997309,993738,995401,995408\n", + "Returned IDs: 993324,995375,990930,995130,990646,994092,991712,998272,998349,997675,996277,996328,993561,996591,994743,993293,996809,994409,992790,995080,993931,993347,999533,997309,993738,995408,995560,994488,999091,993280\n", "Ground Truth: 993324,995375,997184,990930,995130,990646,994092,991712,992933,993533,998272,998349,997522,996277,997675,996328,993561,996591,991052,994743,993293,996809,994409,992790,995080,993931,993347,999533,997309,993738\n", - "Recall: 0.9333\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 831\n", + "Returned IDs: 992809,998593,996001,990255,998103,993232,990314,997374,994467,997332,996657,995158,990364,993819,991212,999099,998006,998878,993931,991610,998536,999555,997009,994667,995167,990288,991453,996809,990167,990573\n", + "Ground Truth: 992809,998593,996001,990255,998103,993232,990314,997374,994467,997332,996657,995158,990364,993819,991062,991212,999099,998006,992966,998878,993931,991610,998536,999555,990786,997009,994667,995167,990288,991453\n", + "Recall: 0.9000\n", "\n", "Query ID: 832\n", - "Returned IDs: 990934,995471,997529,996219,996507,999876,990974,996328,999343,995384,991335,996075,995107,996727,991743,990052,998668,999898,995096,992147,990866,998500,999144,995455,999374,997381,999417,999261,993412,991565\n", + "Returned IDs: 990934,997529,996219,996507,999876,990974,996328,999343,995384,991335,996075,995107,996727,998668,999898,992147,990866,998500,999144,995455,995536,999374,997381,998950,990275,992384,992291,997918,997192,993168\n", "Ground Truth: 990934,995471,997529,996219,996507,999876,990974,996328,999343,995384,991335,996075,995107,996727,991743,990052,998668,999898,995096,992147,990866,999784,998500,999144,995455,995536,999374,991096,993335,997381\n", - "Recall: 0.8667\n", + "Recall: 0.7667\n", "\n", "Query ID: 833\n", - "Returned IDs: 996339,998410,992582,996310,998011,997645,996220,992989,990284,998521,992558,992134,994819,997942,999820,995811,999400,997745,991389,996903,993296,990054,990921,994804,992428,999144,991680,995968,995886,997209\n", + "Returned IDs: \n", "Ground Truth: 996339,998410,992582,996310,998011,997645,996220,992989,990284,992738,999240,998521,992558,992134,994819,997942,999820,995811,999400,997745,991389,996903,993296,990054,990921,994804,992428,999144,991680,992638\n", - "Recall: 0.9000\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 834\n", + "Returned IDs: 998002,990167,999273,992862,994640,995835,995780,993231,993560,995231,996104,990551,992752,990449,998536,995257,998861,995695,995168,998716,998959,996209,997506,997965,998787,993866,992721,998623,999555,996406\n", + "Ground Truth: 998002,990167,999273,995326,992862,994640,995835,995780,993231,993560,995231,996104,990551,992752,990449,998536,995257,998861,995695,995168,998716,998959,996209,997506,997965,998787,993866,992721,998623,999555\n", + "Recall: 0.9667\n", "\n", "Query ID: 835\n", - "Returned IDs: 997301,997320,995274,998302,990231,995961,997913,999338,999109,997470,998779,990585,990949,991498,991743,995045,993484,990957,999533,990096,999290,992509,999710,994831,999801,994964,999647,999026,993431,998950\n", + "Returned IDs: 997301,997320,995274,998302,990231,995961,997913,999338,997470,998779,990585,990949,991498,991743,995045,993484,990957,990096,999290,992509,999710,999801,994964,999026,993431,998950,993845,990560,997775,998921\n", "Ground Truth: 997301,997320,995274,998302,990231,995961,997913,999338,999109,997470,998779,990585,990949,991498,991743,995045,993484,990957,999533,990096,999290,992509,999710,994831,999801,999432,994964,999647,999026,993431\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 836\n", + "Returned IDs: 995515,996840,997192,998921,993143,998401,991157,999122,991848,991392,994216,991840,999613,998607,996504,999327,999053,997496,997465,991687,999257,999676,995154,994800,991015,990142,997486,998915,994922,997403\n", + "Ground Truth: 995515,996840,997192,998921,993143,993028,998401,991157,999122,991848,991392,994216,991840,999613,998607,996504,999327,999053,997496,997465,991687,998711,999257,999676,995154,994800,997591,991015,997889,999785\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 837\n", + "Returned IDs: 993544,992646,998465,999144,993941,992183,997605,998951,998170,997571,990647,996658,996961,995824,994757,990509,990478,999883,994778,990994,998011,994100,990934,995336,998210,991511,999092,995793,991379,990275\n", + "Ground Truth: 993544,992646,998465,999144,993941,992183,997605,998951,998170,997571,990647,995014,996658,996961,995824,994757,990509,990478,999883,994778,992040,990994,998011,994100,990934,995336,998210,991511,999092,995793\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 838\n", + "Returned IDs: 991459,990096,997906,998934,999384,999724,992211,999079,998623,997884,993564,990072,993342,999622,994434,998720,995402,991451,993275,999736,994906,997506,998022,994147,996164,997666,996817,993848,997685,994682\n", + "Ground Truth: 991459,990096,997906,998934,999384,992429,999724,992211,999079,998623,997884,993564,990072,993342,999622,994434,998720,993609,995402,991451,998419,993275,999736,994906,997506,998022,994147,996164,997666,996817\n", + "Recall: 0.9000\n", "\n", "Query ID: 839\n", - "Returned IDs: 999216,992068,994988,995560,990786,992423,994638,996262,999806,999639,999711,990167,998383,992617,994852,993504,990769,996809,996053,999210,998292,993617,991157,998708,997864,994964,992282,991995,993355,990675\n", + "Returned IDs: \n", "Ground Truth: 999216,992068,994988,995560,990786,994638,992423,996262,999806,999639,999711,990957,990167,998383,992617,994852,993504,994598,994660,990769,996809,996053,996857,999210,998292,993617,991157,998708,992979,997864\n", - "Recall: 0.8333\n", + "Recall: 0.0000\n", "\n", "Query ID: 841\n", - "Returned IDs: 995443,990648,994046,998097,998726,998550,995080,998623,992933,993143,996292,999784,992636,999122,991999,998950,990257,992318,990096,995806,993719,999676,991801,992944,999210,993917,999701,998180,993260,997309\n", + "Returned IDs: 995443,994046,998097,998726,995080,998623,993143,999784,992636,999122,991999,998950,993512,990096,995806,999676,991801,992944,999210,993917,999701,998180,993260,997309,997301,995515,993866,995097,994640,997550\n", "Ground Truth: 995443,990648,994046,998097,998726,998550,995080,998623,992933,993143,996292,999784,992636,999122,991999,998950,990257,993512,992318,990096,995806,993719,999676,994835,991801,992944,999210,993917,999701,998180\n", - "Recall: 0.9333\n", + "Recall: 0.7333\n", "\n", "Query ID: 842\n", - "Returned IDs: 990670,999334,993535,993715,998468,996594,996050,998536,995746,999373,991436,995655,999711,992458,997685,996989,990293,992664,992933,997617,995969,995385,994723,991654,992110,990622,993917,992157,990942,995393\n", + "Returned IDs: 999334,993535,993715,996594,996050,998536,995746,999373,991436,995655,999711,992458,997685,996989,990293,992664,997617,995969,995385,994723,991654,990622,993917,992157,995393,996501,998330,993001,994226,999724\n", "Ground Truth: 990670,999334,993535,993715,998468,996594,996050,998536,995746,999373,991436,995655,999711,992458,997685,996989,990293,992664,992933,997617,995969,995385,994723,991654,992110,990622,993917,992157,990942,997470\n", - "Recall: 0.9667\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 843\n", + "Returned IDs: 992664,994280,991000,996147,998934,998716,994682,998593,996830,997790,999852,999342,994382,993335,996153,990771,998030,990675,996809,994131,990091,991081,993715,996001,995200,995326,999922,990811,993931,993391\n", + "Ground Truth: 992664,994280,991000,996147,998934,998716,994682,996028,998593,996830,997790,999130,999852,999342,994382,993335,996153,990771,998030,990675,996809,994131,990091,991081,993715,992780,996001,995200,995326,999922\n", + "Recall: 0.9000\n", "\n", "Query ID: 844\n", - "Returned IDs: 991470,996760,991257,997506,997019,998536,993381,995168,995257,998281,992110,997265,993076,996594,996395,995998,992721,990976,992793,998468,994374,993194,998331,998214,990292,991945,993715,999373,996245,993866\n", + "Returned IDs: 991470,996760,991257,997506,997019,998536,995168,995257,998281,992110,997265,996594,996395,992721,992793,993194,998331,990292,993715,999373,996245,993866,993936,990942,997666,991610,999129,999994,995231,990194\n", "Ground Truth: 991470,996760,991257,997506,997019,998536,993381,995168,995257,998281,992110,997265,993076,996594,996395,995998,992721,990976,992793,998468,994374,993194,998331,998214,990292,991945,993715,999373,996245,993936\n", - "Recall: 0.9667\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 845\n", + "Returned IDs: 991389,993314,997103,995268,998733,998851,999279,999241,994991,993466,997185,991664,990132,991585,993751,990251,999759,994380,997411,995633,999085,992940,993467,998067,990537,993883,993209,993939,996505,992448\n", + "Ground Truth: 991389,993314,997103,995268,998733,998851,999279,999241,994991,999436,993466,997185,991664,990132,991585,992091,993751,990251,999759,994380,998536,997411,996752,995633,999085,992940,993467,998067,990537,993883\n", + "Recall: 0.8667\n", "\n", "Query ID: 846\n", - "Returned IDs: 990624,997251,992483,992992,999529,996042,995464,992844,999845,996687,997344,999244,998838,991334,997234,991161,993857,995550,997918,994928,997601,992487,990686,993948,993313,990487,995763,995537,992639,993335\n", + "Returned IDs: 990624,997251,992483,992992,999529,996042,995464,992844,999845,996687,997344,998838,991334,997234,991161,993857,997918,994928,997601,990686,993948,993313,995763,995537,992639,993335,999392,997746,997840,994139\n", "Ground Truth: 990624,997251,992483,992992,999529,996042,995464,992844,999845,996687,997344,999244,998838,991334,997234,991161,993857,995550,997918,994928,997601,992487,990686,993948,993313,990487,995763,995537,996220,992639\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 847\n", - "Returned IDs: 999682,991374,992740,993530,992493,997232,990733,995807,995105,997574,992677,997675,997515,993263,996579,997634,995521,994823,995625,998806,999961,998080,996695,997149,990243,994774,991061,991052,993149,994677\n", + "Returned IDs: 999682,991374,992740,993530,992493,997232,990733,995807,995105,997574,992677,997675,997515,993263,997634,995521,995625,998806,999961,998080,996695,997149,990243,996094,994774,993149,994677,997723,993160,994301\n", "Ground Truth: 999682,991374,992740,993530,992493,997232,990733,995807,995105,997574,992677,997675,997515,993263,996579,997634,995521,994823,995625,998806,999961,998080,996695,997149,990243,999327,996094,994774,991061,991052\n", - "Recall: 0.9333\n", + "Recall: 0.8333\n", "\n", "Query ID: 848\n", - "Returned IDs: 998960,992511,999374,991444,996959,998211,990469,992147,998644,999263,992416,996218,997704,991362,998500,993250,992854,997192,998921,992546,994850,997543,997301,993457,999968,993364,994673,994939,990515,990215\n", + "Returned IDs: 998960,992511,999374,991444,996959,998211,990469,992147,998644,999263,992416,996218,997704,998500,993250,992854,997192,998921,992546,994850,990760,997543,997301,993457,999968,993364,994673,994939,990515,990215\n", "Ground Truth: 998960,992511,999374,991444,996959,998211,990469,992147,998644,999263,992416,996218,997704,991362,998500,993250,992854,997192,998921,992546,994850,990760,997543,993457,997301,999968,993364,994673,994939,990515\n", "Recall: 0.9667\n", "\n", "Query ID: 849\n", - "Returned IDs: 990266,991720,993656,992161,992147,997476,991377,990483,992169,994761,994922,992994,997429,996494,991464,990328,999343,997852,992226,996721,990441,998500,999122,999374,992979,990957,990080,992068,995332,996609\n", + "Returned IDs: 990266,993656,992161,992147,990483,992169,994761,994922,992994,997429,996494,991464,990328,997852,998500,999122,999374,992979,990957,990080,996609,998458,992637,996438,991984,999453,996093,990678,995279,996871\n", "Ground Truth: 990266,991720,993656,992161,992147,997476,991377,990483,992169,994761,997053,994922,992994,997429,996494,991464,990328,999343,997852,992226,996721,990441,998500,995210,999122,999374,992979,990957,990080,992068\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 850\n", + "Returned IDs: 993319,999798,994465,995501,995476,992372,999886,992595,997331,991667,995768,992868,993001,997565,998006,995073,991779,996809,999430,994356,994682,996580,990194,999256,999425,990173,996328,998536,997340,992832\n", + "Ground Truth: 993319,999798,994465,995501,995476,992372,999886,992595,997331,992752,991667,995768,992868,993001,997565,998006,995073,991779,996809,992110,999430,994356,994682,996580,990194,999256,999425,990173,996328,998536\n", "Recall: 0.9333\n", "\n", + "Query ID: 851\n", + "Returned IDs: 996277,992115,997168,997522,994488,990364,994574,992576,990956,999091,998102,993931,998045,992182,991977,991011,995047,993561,991712,998536,992521,992052,995184,991747,999543,995589,991459,998623,994770,996001\n", + "Ground Truth: 996277,992115,997168,997522,994353,994488,990364,994574,992576,990956,995408,999091,998102,993931,998045,992182,991977,991011,995047,993561,995326,991712,998536,992521,992052,995184,993778,991747,999543,995589\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 852\n", + "Returned IDs: 992617,992692,999806,992228,997320,995820,991137,992330,998255,991015,992927,992521,991157,996903,997605,994954,996592,999109,992124,999639,999711,996262,998075,999731,999144,994757,994577,993845,995372,995640\n", + "Ground Truth: 992617,992692,999806,992228,997320,995820,991137,992330,998255,991015,992927,992521,991157,996903,997605,995536,994954,996592,999109,992124,999639,999711,992277,995374,996262,998075,999731,999144,994757,994577\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 853\n", + "Returned IDs: 996695,990319,996813,992801,997742,997075,999646,995493,995521,996607,995146,996472,997077,999475,995450,996762,997675,992952,995423,992605,994554,995702,993385,993899,994835,995891,991699,998910,997975,996112\n", + "Ground Truth: 996695,990319,996813,992801,997742,997075,999646,995493,995521,996607,995146,996472,994823,997077,999475,995450,996762,997675,992952,995423,992605,994554,995702,993385,993899,994835,995891,991699,998910,997975\n", + "Recall: 0.9667\n", + "\n", "Query ID: 854\n", - "Returned IDs: 997383,999915,996001,991175,998097,992395,995873,996685,995476,993053,994742,992792,995073,997320,990930,999099,992568,996809,999560,998779,999399,997814,992318,991007,996043,995216,999453,998725,991245,996328\n", + "Returned IDs: 997383,999915,996001,998097,992395,995873,996685,995476,993053,994742,992792,995073,997320,990930,999099,992568,996809,999560,998779,999399,997814,992318,991007,995216,999453,998725,991245,993342,996328,994793\n", "Ground Truth: 997383,999915,996001,991175,998097,992395,995873,996685,995476,993053,994742,992792,995073,997320,990930,999099,992568,996809,999560,998779,999399,997814,992318,991007,996043,995216,999453,998725,991245,993342\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 855\n", - "Returned IDs: 994080,994760,992558,991977,992170,994119,992718,990776,990487,994254,996633,996809,995179,992052,998686,998480,995503,994956,995159,998243,995491,990908,994334,998331,990345,998372,996328,996391,993839,998370\n", + "Returned IDs: 994080,994760,992558,991977,992170,994119,992718,990776,990487,994254,996633,996809,992052,995503,995159,998243,990908,994334,998331,990345,998372,999179,996328,996391,993839,998370,990998,992615,993500,997309\n", "Ground Truth: 994080,994760,992558,991977,992170,994119,992718,990776,990487,994254,996633,996809,995179,992052,998686,998480,995503,994956,995159,998243,995491,990908,994334,998331,990345,998372,999179,996328,996391,993839\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 856\n", - "Returned IDs: 993289,992933,993845,993637,990438,993301,999338,992825,993719,994081,998021,995401,990855,999698,992176,993715,998726,998536,996918,995080,997002,990289,997918,990096,998950,999533,999724,996328,999791,999516\n", + "Returned IDs: 993289,993845,993637,990438,993301,999338,992825,993719,994081,998021,995401,990855,992176,993715,998726,998536,996918,997002,990289,997918,990096,998950,999724,996328,999791,999516,995932,997970,999453,992318\n", "Ground Truth: 993289,992933,993845,993637,990438,993301,999338,992825,993719,994081,998021,995401,990855,999698,992176,993715,998726,998536,996918,995080,997002,990289,997918,990096,998950,991758,999533,999724,996328,999791\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 857\n", - "Returned IDs: 990275,997439,996053,998713,995260,996554,994618,991018,990231,992363,990650,997371,990366,997451,992228,991479,997432,999243,995755,996219,995210,998744,992931,997913,992453,993387,999938,992087,994266,992942\n", + "Returned IDs: 990275,996053,998713,995260,996554,991018,990231,992363,997371,990366,992228,991479,999243,995755,992931,997913,992453,993387,997508,999938,992087,994266,992942,999144,998090,992370,994100,991379,997880,997192\n", "Ground Truth: 990275,997439,996053,998713,995260,996554,994618,991018,990231,992363,990650,997371,990366,997451,992228,991479,997432,999243,995755,993048,996219,995210,998744,992931,997913,992453,993387,997508,999938,992087\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 858\n", + "Returned IDs: 995679,994140,992448,994409,990081,999099,993813,997868,991585,990751,995702,991790,996042,998633,993561,992858,997832,997289,995216,991125,996169,993831,997822,996719,990576,996368,996038,998941,994419,997046\n", + "Ground Truth: 995679,994140,992448,994409,990081,999099,993813,997868,991585,990751,995702,991790,991603,996042,998633,993412,991024,993561,992858,997832,997289,995216,991125,996169,993831,997822,996719,990576,996368,996038\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 859\n", + "Returned IDs: 991618,998075,996987,997998,992448,994954,991117,998403,999421,996187,991357,998979,996621,992521,997176,998255,998910,997478,999121,999800,994106,998162,995813,991253,996169,990341,997909,992763,993672,999731\n", + "Ground Truth: 991618,998075,996987,997998,992448,994954,991117,998403,994773,999421,996187,991357,998979,996621,992521,997176,998255,998910,997478,999121,999800,994106,998162,995813,991253,996169,990341,997909,992763,993738\n", "Recall: 0.9333\n", "\n", "Query ID: 860\n", - "Returned IDs: 990275,996336,993853,991260,992384,992452,999647,995014,997976,994776,991829,992969,998899,994992,994192,990724,990862,993168,998514,990925,996727,997571,996714,990356,990957,992994,999089,995034,992911,995279\n", + "Returned IDs: 990275,993853,991260,992384,992452,999647,997976,992969,998899,994992,991893,993168,998514,990925,996727,997571,996714,990957,992994,999089,995034,992911,990505,995279,995755,995336,992546,994917,998951,995793\n", "Ground Truth: 990275,996336,993853,991260,992384,992452,999647,995014,997976,994776,991829,992969,998899,994992,994192,990724,991893,990862,990893,993168,998514,990925,997022,996727,997571,992909,996714,990356,990957,992994\n", - "Recall: 0.8667\n", + "Recall: 0.6333\n", "\n", "Query ID: 861\n", - "Returned IDs: 998302,994382,997937,991790,995204,991923,994707,990560,995672,997320,993412,997301,995702,992862,995981,991995,998294,994964,995949,997913,998779,990252,998807,996391,998495,995895,995755,990096,995976,997822\n", + "Returned IDs: 998302,994382,997937,991790,995204,990560,997320,997301,992862,995981,998294,994964,995949,997913,998779,990252,998807,996391,998495,995895,997969,995755,990096,995976,991696,997822,990224,995886,995274,998776\n", "Ground Truth: 998302,994382,999755,997937,991790,995204,991923,994707,990560,995672,997320,993412,997301,995702,992862,995981,991995,998294,994964,995949,997913,998779,990252,998807,996391,998495,995895,997969,995755,990096\n", - "Recall: 0.9333\n", + "Recall: 0.7667\n", "\n", "Query ID: 862\n", - "Returned IDs: 996219,997913,990843,997301,991085,994266,993662,998495,995942,990958,999533,998380,999542,999374,998540,995927,990776,997420,995802,995949,997320,998921,994964,991650,999331,994850,997855,995260,996318,996811\n", + "Returned IDs: 996219,997913,990843,997301,994266,993662,998495,995942,990958,999542,999374,998540,995927,990776,997420,995802,995949,997320,998921,994964,991650,999331,994850,995260,997470,994277,998550,999908,998758,990366\n", "Ground Truth: 996219,997913,990843,997301,991085,994266,993049,993662,998495,992946,995942,990958,999533,998380,997741,999542,999374,998540,995927,990776,997420,995802,995949,997320,998921,994964,991650,999331,994850,997855\n", - "Recall: 0.9000\n", + "Recall: 0.7667\n", "\n", "Query ID: 863\n", - "Returned IDs: 999912,992337,998115,994972,990366,997453,999544,999876,999949,996052,990179,995107,998122,990585,995222,990089,992047,999791,992907,995716,993139,999711,990043,995818,998349,996001,999898,997301,997796,997028\n", + "Returned IDs: 999912,992337,998115,994972,990366,997453,999544,999876,999949,996052,990179,995107,998122,990585,990089,992047,999791,992907,995716,993139,999711,990043,995818,998349,996001,999898,997796,993484,995372,998849\n", "Ground Truth: 999912,992337,998115,994972,990366,997453,999544,999876,999949,996052,990179,995107,998122,990585,995222,990089,992047,999791,996947,992907,995716,993139,999711,990043,995818,998349,996001,999898,997301,997796\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 864\n", + "Returned IDs: 992069,998779,998950,997918,996262,995694,994360,990949,993781,993504,993071,994450,993260,997002,995602,990957,997320,993492,999639,997640,991677,998170,993484,996140,994964,995560,996001,996196,999453,994742\n", + "Ground Truth: 992069,998779,998950,997918,996262,995694,994360,990949,994334,993781,993504,993071,994449,994450,993260,997002,995602,990957,997320,990930,993492,997739,999639,997640,991677,998170,993484,995826,996140,992532\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 865\n", + "Returned IDs: 995550,996455,991870,992973,991990,994175,995239,992981,993357,999775,991769,996328,992239,992784,999399,990956,995699,991111,992450,995159,997244,995637,999660,997153,993991,999924,997041,992713,994315,995184\n", + "Ground Truth: 995550,996455,991870,992973,991990,994175,995239,993980,992981,993357,999775,991769,996328,992784,992239,999399,990956,995699,991111,992450,995159,997244,995637,999660,997153,993991,999924,997041,993006,992713\n", + "Recall: 0.9333\n", "\n", "Query ID: 866\n", - "Returned IDs: 998558,994679,994098,995691,996882,999435,998993,990796,998652,994193,990765,991541,999171,994605,992056,994080,995618,990852,993718,996428,993267,994689,997864,990487,996413,990133,999158,998855,996930,992435\n", + "Returned IDs: 998558,994679,994098,995691,996882,999435,998993,990796,998652,994193,990765,999171,994605,992056,994080,995618,990852,993718,996428,993267,997864,990487,996413,999158,998855,996930,993976,991377,990998,993500\n", "Ground Truth: 998558,994679,994098,995691,996882,999435,998993,990796,998652,994193,990765,991541,999171,994605,992056,994080,995618,990852,993718,996428,993267,994689,997864,990487,996413,990133,999158,998855,996930,990189\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 867\n", - "Returned IDs: 999805,996867,998162,994409,997844,998943,993767,997868,990361,994730,990631,995702,994060,995766,996786,992582,997031,997723,995052,994954,993209,992063,995355,998181,993649,999459,991603,991508,998962,993856\n", + "Returned IDs: 999805,996867,998162,994409,997844,993767,997868,994730,990631,995702,996786,992582,997031,997723,995052,994954,993209,995355,998181,993649,999459,991508,998962,993856,994742,990435,996038,997544,992448,999787\n", "Ground Truth: 999805,996867,998162,994409,997844,998943,993767,993466,997868,990361,994730,990631,995702,994060,995766,996633,996786,992582,997031,997723,995052,994954,993209,992063,995355,998181,993649,999459,991603,991508\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 868\n", + "Returned IDs: 997554,994793,998060,990845,995216,993422,990081,992792,991585,995575,999099,996147,993841,996657,996001,990662,990255,996368,999402,994106,993344,990314,995679,990211,998878,993804,993032,998593,994013,990751\n", + "Ground Truth: 997554,994793,998060,990845,995216,993422,990081,992792,991585,995575,999099,996147,993841,996657,996001,994764,990662,990255,995963,996368,999402,994106,993344,990314,995679,990211,998878,993804,993032,998593\n", "Recall: 0.9333\n", "\n", "Query ID: 869\n", - "Returned IDs: 996411,995482,997374,993466,995020,995002,990435,999627,998156,990293,999599,993209,999785,994730,992024,998221,991480,994802,999179,995274,995633,998067,991075,995401,990758,998913,995374,995289,994817,993747\n", + "Returned IDs: 996411,995482,997374,993466,995020,990435,999627,998156,990293,999599,994142,993209,994730,991439,992024,998221,994802,999179,995274,995633,998067,990758,998913,995289,994817,993747,996867,999833,993725,994541\n", "Ground Truth: 996411,995482,997374,993466,995020,995002,990435,999627,998156,990293,999599,994142,993209,999785,994730,991439,992024,998221,991480,994802,998586,999179,995274,995633,998067,991075,995401,990758,998913,995374\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 870\n", + "Returned IDs: 997730,994409,998941,996077,993561,998697,997868,997832,996368,994636,992858,992169,991958,997723,993717,996867,991530,993813,998186,995894,995702,999343,993143,996042,990634,995679,998545,995701,998382,993882\n", + "Ground Truth: 997730,994409,998941,996077,993561,998697,997868,997832,996368,997630,994636,992858,992169,991958,997723,993717,996867,991530,993813,998186,995894,992268,995702,999343,993143,996042,994107,990634,990399,995679\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 871\n", + "Returned IDs: 999851,995336,997320,999883,997371,994415,996143,993364,997571,999982,997842,990231,995372,991782,992027,996516,999263,995841,999810,994164,998921,998170,995865,991379,991662,997913,996504,995515,994854,990958\n", + "Ground Truth: 999851,995336,997320,999883,997371,994415,996143,993364,997571,998344,999982,997842,990231,995372,991782,992027,996516,999263,995841,999810,994164,998921,998170,995865,991379,991662,995422,992789,997913,996504\n", "Recall: 0.9000\n", "\n", "Query ID: 872\n", - "Returned IDs: 996362,995783,994313,996087,992997,993497,997746,995184,990872,990149,995464,991285,994280,990343,997244,990021,994196,999893,996911,991111,999759,996276,990103,997713,993256,993478,993621,991981,992784,994541\n", + "Returned IDs: 996362,995783,994313,996087,992997,993497,997746,995184,990872,997997,995464,991285,994280,997244,990021,994196,996911,991111,999759,996276,990103,997713,993256,993478,993621,991981,994541,995696,992526,998201\n", "Ground Truth: 996362,995783,994313,996087,992997,993497,997746,995184,990872,990149,997997,995464,991285,994280,990343,997244,990021,994196,999893,996911,991111,999759,996276,990103,997713,993256,993478,993621,991981,992784\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 873\n", - "Returned IDs: 998382,996371,993561,995289,997386,995401,995200,990137,996219,991179,996042,992068,993433,999367,995535,994307,994577,996769,997406,991650,991996,998045,998495,999598,995923,991921,996805,990435,990096,990628\n", + "Returned IDs: 998382,996371,993561,999068,995289,997386,995401,995200,990137,996219,996042,992068,993433,999367,995535,994307,994577,996769,997406,991650,991996,998045,998495,995923,996805,990435,990096,992944,999780,998039\n", "Ground Truth: 998382,996371,993561,999068,995289,997386,995401,995200,990137,996219,991179,996042,992068,993433,995535,999367,994307,994577,996769,997406,991650,991996,998045,998495,999598,995923,991921,996805,990435,990096\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", "\n", "Query ID: 874\n", - "Returned IDs: 991255,997914,997542,993738,997918,991948,990814,998697,995281,992979,994552,992012,998427,994280,993143,996328,994163,993256,996305,993079,997251,997806,997555,990776,998860,990634,998123,993976,995401,999820\n", + "Returned IDs: 991255,997914,997542,993738,997918,990814,998697,992979,994552,991447,994280,996328,994163,993256,996305,993079,997251,997555,990776,998860,996848,996721,998123,993976,995401,999820,994040,993062,995559,990930\n", "Ground Truth: 991255,997914,997542,993738,997918,991948,990814,998697,995281,998567,992979,994552,992012,998097,991447,998427,994280,994972,993143,996328,994163,993256,996305,993079,995600,997251,997806,997555,990776,998860\n", - "Recall: 0.8333\n", + "Recall: 0.6667\n", "\n", "Query ID: 875\n", - "Returned IDs: 995922,998586,998790,994537,996395,991624,993381,991610,996424,990429,992136,997009,990912,991667,990167,991571,993995,990194,999216,996209,992721,990450,994643,996809,993194,991513,991332,991975,990991,994451\n", + "Returned IDs: 998586,998790,994537,996395,991610,992136,997009,990912,991667,990167,991571,993995,990194,999216,996209,992721,994643,996809,991513,991332,991975,990991,994451,991608,996960,999555,990786,997145,991139,997328\n", "Ground Truth: 995922,998586,998790,994537,996395,991624,993381,991610,996424,990429,992136,997009,990912,991667,990167,991571,993995,990194,999216,994981,996209,992721,990450,994643,996809,993194,991513,991332,991975,990991\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 876\n", + "Returned IDs: 998921,995372,990585,997286,997320,998721,996318,991503,995841,999338,999526,997984,990083,997113,998445,991348,991498,995515,993018,994618,995659,995375,997301,999698,998950,996653,998302,991545,996516,997470\n", + "Ground Truth: 998921,995372,990585,997286,997320,998721,996318,991503,995841,999338,999526,997984,990083,997113,998445,991348,991498,995515,993018,994618,995659,995375,997301,999698,998950,996653,999019,998302,991545,996516\n", "Recall: 0.9667\n", "\n", "Query ID: 877\n", - "Returned IDs: 990269,997320,994964,992331,992744,999159,995357,994873,998470,994225,995274,993168,998764,999437,995336,991433,993066,998228,999340,995438,993909,998365,991776,990210,999740,997178,994584,990535,996219,996401\n", + "Returned IDs: 997320,994964,997920,992331,992744,995357,994873,995274,993168,998764,999437,995336,998228,995438,998365,991776,990210,999740,997178,990535,996219,997330,992692,993484,997028,999344,992942,994432,991883,996001\n", "Ground Truth: 990269,997320,994964,997920,992331,992744,999159,995357,994873,998470,994225,995274,993168,998764,999437,995336,991433,993066,998228,999340,995438,993909,998365,991776,990210,997178,999740,994584,990535,996219\n", - "Recall: 0.9667\n", + "Recall: 0.7000\n", "\n", "Query ID: 878\n", "Returned IDs: 995955,998242,997215,990125,994305,995503,992306,992824,991333,995804,990538,992170,990721,990192,991034,993140,991834,998260,999908,992490,992113,997849,997547,999430,999639,996742,996510,995724,991977,997055\n", @@ -7615,303 +9450,583 @@ "Recall: 0.9333\n", "\n", "Query ID: 879\n", - "Returned IDs: 997529,993725,994730,998710,994742,991185,995633,994830,999377,992424,993166,990887,998106,993561,993053,995289,998913,999740,990681,994409,996411,993920,996328,992260,993209,994013,994419,996867,992230,996132\n", + "Returned IDs: 997529,993725,994730,998710,994742,991185,995633,994830,999377,992424,993166,990887,998106,993561,993053,995289,998913,999740,997167,994409,996411,993920,996328,993209,994013,994419,996867,992230,996132,995401\n", "Ground Truth: 997529,993725,994730,998710,994742,991185,995633,994830,999377,992424,993166,990887,998106,993561,993053,995289,998913,999740,990681,997167,994409,996411,993920,996328,992260,993209,994013,994419,996867,992230\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 880\n", - "Returned IDs: 992411,995552,992558,993920,995309,991851,998652,999206,996953,990843,994418,993474,996878,993077,995618,999305,991337,990279,996140,990345,998950,999540,993742,991263,993456,999908,990989,993534,990308,992318\n", + "Returned IDs: 992411,995552,992558,993920,995309,991851,998652,999206,996953,990843,994418,993474,996878,993077,995618,999305,991337,990279,996140,990345,994817,998950,999540,993742,990657,993456,999908,990989,993534,990308\n", "Ground Truth: 992411,995552,992558,993920,995309,991851,998652,999206,996953,990843,994418,993474,996878,993077,995618,999305,991337,990279,996140,990345,994817,998950,999540,992966,993742,991263,990657,993456,999908,990989\n", - "Recall: 0.9000\n", + "Recall: 0.9333\n", "\n", "Query ID: 882\n", - "Returned IDs: 992032,992293,991076,991203,996225,991954,996028,990662,993561,990811,993725,991087,998476,993857,994835,998823,991603,997529,991958,999286,990399,990097,990033,992636,994677,998197,991301,999922,998593,997554\n", + "Returned IDs: 992032,992293,991076,991203,991954,996028,990662,993561,990811,993725,998476,993857,994835,998823,997529,991958,999286,990097,992636,998197,991301,999922,998593,997554,995701,997822,992605,998105,995216,997079\n", "Ground Truth: 992032,992293,991076,991203,996225,991954,996028,990662,993561,990811,993725,991087,998476,993857,994835,998823,991603,997529,991958,999286,990399,990097,990033,992636,994677,991771,998197,991301,991466,999922\n", - "Recall: 0.9333\n", + "Recall: 0.7333\n", "\n", "Query ID: 883\n", - "Returned IDs: 997362,990958,996504,993813,997192,990925,996316,999789,998692,995755,992204,999483,998294,990118,997984,997371,995090,997320,999327,993518,991958,995132,995985,995367,998649,997911,997889,992789,995884,997620\n", + "Returned IDs: 997362,990958,996504,993813,997192,990925,996316,999789,998692,995755,999483,998294,990118,997984,997371,995090,997320,999327,993518,991958,995132,995985,995367,998649,995884,997620,991332,998303,996785,992994\n", "Ground Truth: 997362,990958,996504,993813,997192,990925,996316,999789,998692,995755,992204,999483,998294,990118,997984,997371,995090,997320,999327,993518,991958,995132,995985,991009,995367,998649,997911,997889,993902,993982\n", - "Recall: 0.9000\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 884\n", + "Returned IDs: 992809,991767,993232,998565,998623,996797,994643,998536,997019,993995,994839,993727,994901,999555,998787,995695,992110,993366,990255,998006,996395,994844,993306,997502,991257,993715,991228,990741,994374,992926\n", + "Ground Truth: 992809,991767,993232,999498,998565,998623,994274,996797,994643,998536,997019,993995,994839,993727,994901,999663,999555,998787,995695,992110,993366,990255,998006,996395,994844,993306,997502,991257,999458,993715\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 885\n", + "Returned IDs: 999869,994449,993260,995261,992748,992041,995336,996296,990532,994955,998951,994100,997498,997113,999698,993512,998550,997320,993529,990772,997918,996592,996075,991951,994171,992880,998090,990949,998246,996210\n", + "Ground Truth: 990778,999869,994449,993260,995261,992748,992041,995336,996296,990532,992933,994955,998951,994100,997498,999019,999351,997113,990519,999698,993512,998550,997320,993529,990772,997918,996592,996075,991951,994171\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 886\n", + "Returned IDs: 994463,992703,991734,994444,994424,991685,995126,995678,995945,995421,990520,990899,990255,992322,996062,998593,999491,991857,998250,990382,991789,994510,998039,993931,997453,996001,993289,992182,991093,993715\n", + "Ground Truth: 994463,992703,991734,994444,994424,991685,995126,995678,995945,995421,990520,990899,990255,992322,996062,998593,999491,991857,990133,998250,990382,991789,994510,998039,993931,997453,996001,993289,992182,991093\n", + "Recall: 0.9667\n", "\n", "Query ID: 887\n", - "Returned IDs: 995156,991432,995955,995562,999639,997032,997775,992435,996065,998260,990721,991157,994305,998142,994163,996592,997739,992497,995618,995983,991801,996510,992824,997614,990413,996884,990538,997420,991817,992095\n", + "Returned IDs: 991432,995955,995562,999639,997775,992435,996065,998260,990721,991157,994305,998142,994163,997739,995618,995983,996510,992824,997614,990413,996884,990538,997420,991817,992095,991923,990192,991015,997055,992306\n", "Ground Truth: 995156,991432,995955,995562,999639,997032,997775,992435,996065,998260,990721,991157,994305,998142,996511,990210,994163,996592,997739,992497,995618,992350,995983,996283,991801,996510,992824,997614,990413,996884\n", - "Recall: 0.8667\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 888\n", + "Returned IDs: 996759,999977,997002,996810,999587,995401,999595,991666,997554,998593,998779,998530,998123,992790,992884,997910,994819,999026,993032,994474,997012,991968,996657,990399,996088,996001,993289,996864,994946,994409\n", + "Ground Truth: 990304,996759,999977,997002,996810,992637,999587,995401,999595,991666,997554,998593,994178,998779,998530,998123,992790,992884,997910,994819,999026,993032,994474,997012,991968,996657,993820,990399,996088,995374\n", + "Recall: 0.8333\n", "\n", "Query ID: 889\n", - "Returned IDs: 995159,990355,998507,994929,996211,994541,996413,997746,998755,997722,996344,996276,992489,990931,994094,997879,997290,992445,994336,992313,994913,996430,997814,998174,992052,997420,992832,999179,992981,992934\n", + "Returned IDs: 995159,990355,998507,994929,996211,994541,996413,997746,998755,997722,996344,996276,992489,990931,994094,997290,992445,994336,992313,994913,996430,993621,997814,998174,992052,997420,992832,999179,992981,992934\n", "Ground Truth: 995159,990355,998507,994929,996211,994541,996413,997746,998755,997722,996344,996276,992489,990931,994094,997879,997290,992445,994336,992313,999974,994913,996430,993621,997814,998174,992052,997420,992832,999179\n", "Recall: 0.9333\n", "\n", + "Query ID: 890\n", + "Returned IDs: 999547,997723,992796,996867,994730,996633,990631,993209,995702,997868,998162,994380,995958,998067,995907,999805,999402,993466,994409,996042,992448,997411,991301,992959,991664,992051,992599,999241,994945,996109\n", + "Ground Truth: 999547,997723,992796,996867,994730,996633,990631,993209,995702,995470,997868,998162,994380,995958,998067,995907,999805,999402,993466,994409,992091,996042,991943,992448,997411,992024,991301,992959,991664,992051\n", + "Recall: 0.8667\n", + "\n", "Query ID: 891\n", - "Returned IDs: 990957,993283,992668,995752,991677,999561,996894,994852,999806,995974,998600,995580,998455,990949,995562,999714,995443,991801,992309,993781,998495,999898,993222,994979,995519,997903,991923,992124,992423,997864\n", + "Returned IDs: 990957,993283,992668,995752,991677,999561,996894,994852,999806,995974,998600,995580,990949,995562,999714,995443,991801,992309,993781,998495,999898,993222,994979,995519,997903,991923,992124,992423,997864,990383\n", "Ground Truth: 990957,993283,992668,995752,991677,999561,996894,994852,999806,995974,998600,995580,998232,998455,990949,995562,999714,995443,991801,992309,993781,998495,999898,993222,997032,994979,995519,997903,991923,992124\n", - "Recall: 0.9333\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 892\n", + "Returned IDs: 994742,999377,990435,994730,990096,993053,999279,998495,999402,995633,991185,990930,993642,998980,993623,992680,994759,992024,995289,990758,996001,993920,993725,999302,994830,990012,993209,996809,995435,999009\n", + "Ground Truth: 994742,999377,990435,994730,990096,993053,999279,998495,999402,995633,991185,990930,993642,998980,993623,992680,994759,998188,995198,992024,995289,990758,996001,997723,993920,993725,999302,994830,990012,995873\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 893\n", + "Returned IDs: 992748,995203,994081,997320,990083,996075,993809,999453,990949,991984,991002,992880,998697,998741,998951,995596,995250,993781,990532,995016,999317,998090,999639,990697,999698,991609,998232,999701,990339,999076\n", + "Ground Truth: 992748,995203,994081,997320,992497,990083,996075,993809,990778,999453,990949,991984,991002,992880,998697,998741,999338,998951,995596,995250,993781,995398,990532,995016,999317,998090,999639,990697,999698,991609\n", + "Recall: 0.8667\n", "\n", "Query ID: 894\n", - "Returned IDs: 994336,997309,993877,994280,992933,995861,996328,991378,994119,994956,999338,991334,991720,998076,992147,990998,993335,998123,997822,997159,998950,996855,995438,999533,992789,995125,998369,993929,996050,991007\n", + "Returned IDs: 994336,997309,993877,994280,993775,995861,996328,991378,994119,994956,991334,992147,993335,998123,997822,998950,996855,995438,999533,995125,993929,996050,991007,997918,994964,992052,996521,998495,997002,998779\n", "Ground Truth: 994336,997309,993877,994280,993775,992933,995861,996328,991378,994119,994956,999338,991334,991720,998076,992147,990998,993335,998123,997822,997159,998950,996855,995438,999533,992789,995125,998369,993929,996050\n", - "Recall: 0.9667\n", - "\n", - "Query ID: 897\n", - "Returned IDs: 997475,997652,991153,991075,997480,994891,993475,993621,993611,990696,999496,990600,992480,994765,993055,994558,995600,994313,997714,999205,998405,993165,992449,996856,990776,996429,994051,994307,994280,991723\n", - "Ground Truth: 997475,997652,991153,991075,997480,994891,993475,993621,993611,990696,999496,990600,992480,994765,991426,993055,995145,994558,995600,994313,997714,999205,998405,993165,992449,996856,990776,996429,994051,994307\n", - "Recall: 0.9333\n", + "Recall: 0.7333\n", "\n", - "Query ID: 898\n", - "Returned IDs: 991593,995260,999374,991999,994922,990278,998734,997620,998899,996351,990545,994497,991352,991889,999676,997985,990958,991172,999810,993885,992203,994776,990760,990026,991406,991720,994266,999658,995443,993387\n", - "Ground Truth: 991593,995260,992432,999374,991999,994922,990278,998734,997620,998899,996351,990545,994497,991352,991889,999676,997985,990958,991172,999810,993885,992203,997591,994776,990760,990026,991406,991720,994266,999658\n", - "Recall: 0.9333\n", + "Query ID: 895\n", + "Returned IDs: 995964,991430,997495,998417,991923,994126,997864,994543,995138,996163,990776,997407,999716,990894,999826,995510,994165,993572,993828,998142,996882,994163,994904,995580,994213,990908,990930,999171,995986,998061\n", + "Ground Truth: 995964,991430,997495,998417,991923,994126,997864,994543,995138,996163,990776,997407,999716,990894,999826,995510,994165,993572,993828,998142,992280,996882,994163,994904,995580,994213,990908,996216,990930,998904\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 896\n", + "Returned IDs: 991924,996328,998279,996362,991111,994742,991885,994496,998067,990854,996816,996728,991981,995735,993621,993450,994806,996495,992450,998251,992713,998256,998671,991428,994428,996189,991769,999852,995831,999812\n", + "Ground Truth: 991924,996328,998279,996362,991111,994742,991885,994496,998067,990854,996816,996728,995735,991981,993621,993450,994806,996495,992450,998251,992713,998256,998671,991428,994428,996189,991769,995315,999852,995831\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 897\n", + "Returned IDs: 997475,997652,991153,991075,997480,994891,993621,993611,990696,999496,990600,992480,994765,993055,994558,995600,997714,999205,998405,993165,992449,990776,996429,994307,994280,991723,998072,990097,992327,993004\n", + "Ground Truth: 997475,997652,991153,991075,997480,994891,993475,993621,993611,990696,999496,990600,992480,994765,991426,993055,995145,994558,995600,994313,997714,999205,998405,993165,992449,996856,990776,996429,994051,994307\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 898\n", + "Returned IDs: \n", + "Ground Truth: 991593,995260,992432,999374,991999,994922,990278,998734,997620,998899,996351,990545,994497,991352,991889,999676,997985,990958,991172,999810,993885,992203,997591,994776,990760,990026,991406,991720,994266,999658\n", + "Recall: 0.0000\n", "\n", "Query ID: 899\n", - "Returned IDs: 997992,993504,995918,991595,997790,999735,996067,999811,992082,993609,996082,990494,992747,993737,995167,993024,991212,992665,994134,993071,990434,992864,991741,996196,996599,994651,999093,991425,998732,991596\n", + "Returned IDs: 997992,993504,995918,991595,997790,999735,996067,999811,992082,996082,990494,992747,993737,995167,993024,991212,994134,993071,990434,992864,993611,991741,996196,996599,999093,991425,998732,990196,992247,994537\n", "Ground Truth: 997992,993504,995918,991595,997790,999735,996067,999811,996986,992082,993609,996082,990494,992747,993737,995167,993024,991212,992665,994134,993071,990434,992864,993611,991741,994815,996196,996599,994651,999093\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 900\n", + "Returned IDs: 995193,996581,997998,998527,997176,998255,991769,999047,992311,998075,995397,992521,996277,995640,995661,991922,991981,995820,993379,992097,998670,994742,996728,994954,998910,997612,998637,997091,990712,999208\n", + "Ground Truth: 995193,996581,997998,998527,997176,998255,991769,999047,992311,998075,995397,994773,992521,996277,995640,995661,991922,991981,995820,993379,992097,998670,994742,994690,996728,991057,994954,998910,997612,998637\n", "Recall: 0.9000\n", "\n", "Query ID: 901\n", - "Returned IDs: 995986,992718,992258,994163,994072,997689,998694,999908,999359,996163,991637,996926,997395,995301,994165,994195,993463,992645,996669,992558,997864,998079,997692,996214,991430,999432,999385,993609,995436,990367\n", + "Returned IDs: 995986,992718,992258,994163,997689,998694,999359,996163,991637,996926,995301,994165,993463,992645,996669,992558,997864,998079,997692,996214,991430,999432,995436,990367,997558,997538,999826,991977,996620,996127\n", "Ground Truth: 995986,992718,992258,994163,994072,997689,998694,999908,999359,996163,991637,996926,997395,995301,994165,994195,993463,992645,991172,998758,996669,992558,997864,998079,997692,996214,991430,999432,999385,993609\n", - "Recall: 0.9333\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 902\n", + "Returned IDs: 992289,995948,995289,995030,995633,996867,995190,998196,993261,997319,997630,991810,996411,997947,999957,993433,999375,994110,990516,998913,995401,990814,992364,996302,998765,994204,992827,994283,994783,995078\n", + "Ground Truth: 992289,995948,995289,995030,995633,996867,995190,998196,993261,997319,998557,995494,997797,997630,991810,996411,997947,999957,991447,993433,999375,994110,990516,998913,995401,990814,992364,996302,998765,994204\n", + "Recall: 0.8667\n", "\n", "Query ID: 903\n", - "Returned IDs: 999426,991984,990142,999769,993257,991013,998753,994971,997551,993785,992328,994489,998498,994789,996075,993582,991218,992635,995515,998734,990648,997584,995033,996631,995108,993675,996829,996840,998227,999132\n", + "Returned IDs: \n", "Ground Truth: 999426,991984,990142,999769,993257,991013,998753,994971,997551,993785,992328,994489,998498,994789,996075,993582,991218,992635,995515,998734,990648,997584,995033,997255,996631,995108,993675,996829,996840,998227\n", - "Recall: 0.9667\n", + "Recall: 0.0000\n", "\n", "Query ID: 904\n", - "Returned IDs: 992827,998779,994382,993744,999788,995300,992748,995200,994595,990722,998543,997529,994682,991334,998765,990934,996219,993856,991590,995401,994742,991885,995374,995861,990814,997933,994783,990701,998557,998862\n", + "Returned IDs: 992827,998779,994382,993744,999788,995300,995200,994595,990722,998543,997529,994682,998765,990934,996219,991590,995401,994742,991885,995861,990814,994783,999138,990701,998862,995633,999144,997918,998055,998671\n", "Ground Truth: 992827,998779,994382,993744,999788,995300,992748,999916,995200,994595,990722,998543,997529,994682,991334,998765,997348,990934,996219,993856,993412,991590,995401,994742,991885,995374,995861,990814,997933,991986\n", - "Recall: 0.8667\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 905\n", + "Returned IDs: 995113,994191,991962,993931,992771,996062,998593,999543,996001,994280,997675,995168,997453,990407,996733,995536,992437,999984,990802,998723,997506,998250,994463,999091,995017,999994,996028,996505,995421,994424\n", + "Ground Truth: 995113,994191,991962,993931,992771,996062,998593,999543,995401,996001,994280,997675,992793,995168,997453,990407,996733,996071,995536,999461,992437,990998,999984,990802,998723,997506,998250,994463,997732,999091\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 906\n", + "Returned IDs: 996890,994641,993518,992709,992107,997709,990595,995353,994630,995949,994192,993734,997535,993005,994939,990863,995367,997371,990397,995090,997972,994281,997525,996961,994757,995910,998378,996785,992511,997880\n", + "Ground Truth: 996890,994641,993518,992709,992107,997709,990595,995353,994630,995949,994192,993734,997535,993005,999112,994939,990863,992004,995367,997371,990397,995090,997972,994281,997525,996961,994757,995910,994953,998378\n", + "Recall: 0.9000\n", "\n", "Query ID: 907\n", - "Returned IDs: 995802,994778,999710,993492,992442,997006,992443,999982,993504,997483,993852,995826,992309,997913,992119,994670,992931,992228,994831,990231,995161,997605,990957,996993,992066,990890,996785,997320,998170,991873\n", + "Returned IDs: \n", "Ground Truth: 995802,994778,999710,993492,992442,997006,991009,992443,999982,993504,997483,993852,993048,995826,992309,997913,992119,994670,992931,992228,994831,990231,995161,997605,990957,996993,997579,992066,990890,996785\n", - "Recall: 0.9000\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 908\n", + "Returned IDs: 992020,993570,998551,996570,998492,998936,994922,995791,991806,994777,998983,997789,998734,997474,991405,990334,992138,992186,999453,999972,994438,995004,992146,994318,990043,991198,994089,990652,992944,992360\n", + "Ground Truth: 992020,993570,998551,996570,998492,998936,994922,995791,991806,990155,994777,998983,997789,998734,997474,991405,990334,992138,992186,999892,999453,998232,999972,994438,995004,992146,994318,990043,990849,990490\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 909\n", + "Returned IDs: 998303,998377,997926,995025,991332,999032,992866,999265,995894,992241,993489,994906,993260,998697,991630,991691,991295,996217,999840,995740,995203,995672,991530,993504,996001,998401,999977,996725,992163,997653\n", + "Ground Truth: 998303,998377,997926,995025,991332,999032,992866,999265,995894,992241,993489,994906,993260,998697,991630,991691,991295,996217,999840,995740,995203,995672,991530,993504,997531,996001,998401,999977,996725,992163\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 911\n", + "Returned IDs: 994783,990340,990852,999126,990126,991330,992827,995633,992566,992598,998913,993725,994878,990985,990069,996809,996302,995861,995289,999957,997733,994742,998804,994541,991464,999372,990929,991418,999740,996411\n", + "Ground Truth: 994783,990340,990852,999126,990126,991330,992827,995633,992566,992598,998913,993725,994878,990985,990069,996809,996302,995861,995289,999957,997733,994742,994317,998804,994541,991464,994751,999372,990929,991418\n", + "Recall: 0.9333\n", "\n", "Query ID: 912\n", - "Returned IDs: 999079,990096,999831,996918,999771,997906,998849,997305,999367,994955,992052,992944,996001,995203,994718,994226,995932,995536,998495,996124,995332,994081,997320,994643,996915,995401,994661,991498,994901,993609\n", + "Returned IDs: 999079,990096,999831,996918,999771,997906,998849,997305,999367,994955,992052,992944,996001,995203,995737,994718,995932,995536,998495,996124,997320,994643,996915,994661,991498,994901,999711,992407,995372,998302\n", "Ground Truth: 999079,990096,999831,996918,999771,997906,998849,997305,999367,994955,992052,992944,996001,995203,995737,994718,994226,995932,995536,998495,996124,995332,994081,997320,994643,996915,995401,994661,991498,994901\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 913\n", - "Returned IDs: 996868,996001,993174,998934,992072,995701,997920,998593,995216,992610,994419,994610,995823,998186,998710,996147,998349,995855,996401,991000,997897,991958,999590,993738,994280,996075,997687,993559,998382,993280\n", + "Returned IDs: 996868,996001,993174,998934,992072,995701,998593,995216,992610,994419,994610,995823,998186,998710,996147,998349,995855,997897,991958,999590,994280,996075,997687,998382,993280,996368,998039,997320,997586,993293\n", "Ground Truth: 996868,996001,993174,998934,992072,995701,997920,998593,995216,992610,994419,994610,995823,998186,998710,996147,998349,995855,996401,991000,997897,991958,999590,993738,994280,996075,997687,993559,998382,990557\n", - "Recall: 0.9667\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 914\n", + "Returned IDs: 991527,992448,991969,998205,997868,991795,990845,996987,996368,998278,999250,991220,996187,997909,992521,995864,993537,996256,997176,993003,999453,994409,995813,991892,990603,994106,994140,993813,997727,998910\n", + "Ground Truth: 991527,992448,991969,998205,997868,991795,990845,999866,996987,996368,998278,997216,999250,991220,996187,997909,992521,995864,993537,996256,997176,993003,993412,990341,999453,994409,995813,991892,990603,994106\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 915\n", + "Returned IDs: 992515,997215,999426,993125,990697,993447,991503,995372,994886,996805,997914,996075,997550,997584,998498,991070,993721,990538,992843,993341,995459,991426,991409,997775,994233,991272,992661,990192,998950,998933\n", + "Ground Truth: 992515,997215,999426,993125,997448,994626,990697,993447,991503,995372,994886,996805,997914,995398,996075,997550,997584,998498,991070,993721,990538,992843,993341,995459,991426,991409,997775,995622,994233,991272\n", + "Recall: 0.8667\n", "\n", "Query ID: 916\n", - "Returned IDs: 994673,992833,994160,997192,990724,991782,997826,993648,993412,998458,991585,999328,996111,991361,990760,992284,991743,995702,996721,992994,999263,996532,991602,997880,999374,992147,991213,990432,991958,998899\n", + "Returned IDs: 994673,992833,997192,993648,998458,991585,999328,991361,992284,991743,991534,995702,996721,992994,999263,996532,997880,999374,992147,991213,990432,991958,997913,992384,992922,992087,994383,990958,994919,993402\n", "Ground Truth: 994673,992833,994160,991900,997192,990724,991782,997826,993648,993412,998458,991585,999328,996111,991361,990760,992284,991743,991534,990462,995702,996721,992994,999263,996532,991602,997880,999374,992147,998215\n", - "Recall: 0.8667\n", + "Recall: 0.6333\n", "\n", "Query ID: 917\n", - "Returned IDs: 994013,994203,995544,992098,996898,998060,993377,991057,999689,997059,998162,994412,991507,999023,995216,993217,996328,992000,994409,994488,996001,999402,998045,997554,991255,996765,995766,997466,997566,997020\n", + "Returned IDs: 994013,994203,995544,992098,996898,998060,993377,991057,999689,997059,998162,994412,991507,999023,995216,993217,996328,992000,994409,994488,996001,999402,998045,997554,996765,995766,997466,996322,997566,997020\n", "Ground Truth: 994013,994203,995544,992098,996898,998060,993377,991057,999689,997059,998162,994412,991507,999023,995216,993217,996328,992000,994409,994488,996001,999402,998045,997554,991255,996765,995766,997466,996322,997566\n", "Recall: 0.9667\n", "\n", "Query ID: 918\n", - "Returned IDs: 991075,999472,990315,992156,997798,995455,996275,995633,995613,998017,996411,995600,994783,994879,995289,994431,995131,999064,999500,994558,998105,998982,999949,997079,992449,991315,993725,992079,995002,995831\n", + "Returned IDs: 991075,999472,990315,992156,995455,995633,995613,998017,996411,994783,995289,994431,995131,999064,999500,994558,998105,998982,994130,997079,992449,991315,993725,992079,995002,995831,999302,994307,999922,999356\n", "Ground Truth: 991075,999472,990315,992156,997798,996275,995455,995633,995613,998017,996411,995600,994783,994879,995289,994431,995131,999064,999500,994558,998105,998982,994130,999949,997079,992449,991315,993725,992079,995002\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 919\n", - "Returned IDs: 994342,992780,998631,994577,998109,996475,995776,992068,992070,990271,998568,994984,999806,997331,999780,996219,993055,997990,994558,999430,998495,997909,995895,997554,997480,992628,992423,992655,997529,993053\n", + "Returned IDs: 994342,992780,998631,994577,998109,995776,992070,998568,994984,999806,997331,999780,996219,994558,999430,998495,997909,995895,997480,992628,992423,992655,997529,993053,996216,999294,993617,992617,990435,995476\n", "Ground Truth: 994342,992780,998631,994577,998109,996475,995776,992068,992070,990271,998568,994984,999806,997331,999780,996219,993055,997990,994558,999430,998495,997909,995895,997116,997554,997480,992628,992423,993678,992655\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 920\n", + "Returned IDs: 993727,990391,998175,997693,997942,992809,998507,994407,991052,993980,998537,999773,992295,996098,993453,990297,996918,993155,995048,991054,998481,990479,994051,995790,994280,999105,997806,998267,994187,996395\n", + "Ground Truth: 993727,990391,998175,997693,998430,997942,992809,998507,994407,991052,993980,998537,999773,992295,996098,993453,990297,996918,993155,995048,991054,998481,990479,994051,995790,994280,999105,997429,997806,998267\n", "Recall: 0.9333\n", "\n", + "Query ID: 921\n", + "Returned IDs: 997261,994389,993357,990610,996591,994435,993079,997168,998049,996245,999438,997393,991850,993280,993738,993715,990810,993931,992239,994040,991551,997494,990264,993293,992832,998891,995059,996277,996505,994076\n", + "Ground Truth: 997261,994389,993357,991904,990610,996591,994435,995361,993079,997168,998049,996245,999438,997393,991850,993280,993738,990769,995359,993715,990810,993931,992239,994040,991551,997494,990264,993293,992832,998891\n", + "Recall: 0.8667\n", + "\n", "Query ID: 922\n", - "Returned IDs: 996840,996686,995495,998582,998523,995515,993028,997486,991392,993091,998244,999524,994195,990026,998692,995004,991485,999453,993143,991462,990300,992309,999613,998996,996351,992328,990702,998837,991149,995588\n", + "Returned IDs: 996840,996686,995495,998582,998523,995515,997486,991392,993091,998244,999524,994195,990026,998692,995004,991485,999453,993143,991462,990300,992309,999613,998996,996351,992328,998837,991149,995588,991889,994922\n", "Ground Truth: 996840,996686,995495,998582,998523,995515,993028,997486,991392,993091,998244,999524,994195,990026,998692,995004,991485,999453,993143,991462,990300,992309,999613,998996,996351,995401,992328,990702,998837,991149\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", "\n", - "Query ID: 925\n", - "Returned IDs: 992449,990220,993856,994998,993437,997975,997103,993939,990531,990887,997544,998067,997529,998710,998733,999532,997459,999054,995778,994830,999496,994185,999279,998910,995083,993100,990435,991769,999118,995873\n", - "Ground Truth: 992449,990220,993856,994998,993437,997975,997103,993939,990531,990887,997544,998067,997529,998710,998733,999532,997459,999054,995778,994830,999496,994185,999279,998910,995083,993100,990435,991769,999118,997554\n", - "Recall: 0.9667\n", + "Query ID: 923\n", + "Returned IDs: 995289,996610,997787,995182,997406,991603,993561,996769,997630,996388,995497,992125,998165,999080,994783,997822,997386,996042,991954,997058,996080,994742,995200,996473,991923,994396,993725,991958,993844,990984\n", + "Ground Truth: 995289,996610,997787,995182,997406,991603,993561,996769,997630,995497,996388,992125,998165,999080,994783,997822,997386,996042,991954,997058,996080,994742,995200,998999,996473,991923,994396,993725,991958,993412\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 924\n", + "Returned IDs: 998996,999453,996485,990965,995850,996419,998001,990043,995515,991485,991407,998837,997486,993237,992835,991131,994439,990827,997219,990858,998244,991157,993282,994800,996840,992944,998937,998782,994216,995372\n", + "Ground Truth: 998996,999453,996485,990965,995850,996419,998001,990043,995515,991485,991407,998837,997486,993237,992835,991131,994439,990827,997219,998236,996652,990858,998244,991157,993282,994800,998116,996840,992944,999671\n", + "Recall: 0.8667\n", "\n", "Query ID: 926\n", - "Returned IDs: 994830,996132,994204,990340,990711,994742,994783,995633,998913,990012,998710,997319,993053,998196,996411,991330,999377,995289,994529,992289,996302,993433,996049,997529,995948,995200,996809,990435,993166,997797\n", + "Returned IDs: 994830,996132,994204,990340,990711,994742,994783,995633,998913,990012,998710,997319,993053,998196,996411,991330,999377,995289,994529,992289,996302,993433,996049,995401,997529,995200,996809,990435,993166,997797\n", "Ground Truth: 994830,996132,994204,990340,990711,994742,994783,995633,998913,990012,998710,997319,993053,998196,996411,991330,999377,995289,994529,992289,996302,993433,996049,995401,997529,995948,995200,996809,990435,993166\n", "Recall: 0.9667\n", "\n", + "Query ID: 927\n", + "Returned IDs: 997513,991667,998335,992778,991610,990330,997145,990177,994467,994996,995655,992517,992561,998781,992721,990437,997009,991470,990293,992209,996134,996501,990292,990194,998720,997242,997328,991569,993381,997666\n", + "Ground Truth: 997513,991667,998335,992778,991610,990330,997145,990177,994467,992136,994996,995655,992517,992561,998781,992721,990437,997009,991470,991038,990293,992573,992209,996134,996501,990292,990194,996395,998720,997242\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 928\n", + "Returned IDs: 991037,991212,991227,993855,999510,991956,994221,996213,992315,997897,993504,995562,995213,998250,994510,992819,992168,990596,995500,996884,995167,992176,994302,991648,991425,994743,999139,990409,994101,991245\n", + "Ground Truth: 991037,991212,991227,993855,999510,991956,994221,996213,992315,997897,993504,995562,995213,998250,994510,992819,992168,996677,994468,990596,995500,991596,996884,995167,992176,994302,991648,991425,995421,993981\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 929\n", + "Returned IDs: 993209,995289,995633,996815,994529,995766,995401,997262,998913,991507,991958,994742,995268,995778,992488,993831,998165,998476,991954,993997,996302,999957,990126,993219,996867,994783,998196,996388,994013,993586\n", + "Ground Truth: 993209,995289,995198,995633,996815,996930,994529,995766,995401,997262,998913,997797,991507,991958,994742,999587,995268,996411,995778,990157,992488,993831,998165,998476,991954,993997,996302,999957,990126,993219\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 930\n", + "Returned IDs: 993426,998921,990898,996516,997984,997320,998445,995515,998721,996135,994415,997292,997192,997301,994757,993253,998294,993419,992451,997470,993484,995306,997571,997291,998246,991720,995841,999327,995372,992942\n", + "Ground Truth: 993426,998921,990898,996516,997984,997320,997873,998445,995515,998721,996135,994415,999036,997292,997192,997301,994757,993253,995401,998294,993419,996560,992451,997470,993484,998741,995306,997571,997291,998246\n", + "Recall: 0.8333\n", + "\n", "Query ID: 931\n", "Returned IDs: 995299,990074,998803,993413,992456,996183,998949,997340,996986,990730,996408,992372,998903,993335,994682,990998,998716,990700,991919,997577,995393,996357,992628,995313,990173,992664,994269,993917,993101,991382\n", "Ground Truth: 995299,990074,998803,993413,992456,996183,998949,997340,996986,990730,996408,992372,998903,993335,994682,990998,998716,990700,991919,997577,995393,996357,992628,995313,990173,992664,994269,993917,997116,993101\n", "Recall: 0.9667\n", "\n", "Query ID: 932\n", - "Returned IDs: 997918,993289,997002,995826,993504,993492,993512,999900,996511,994449,992571,993781,992029,994959,999840,997790,994268,990096,998090,996062,999019,990399,997320,997977,990930,999355,994280,997586,991206,996592\n", + "Returned IDs: 997918,993289,997002,995826,993504,993492,993512,996511,992571,993781,994959,999840,997790,994268,990096,998090,996062,999019,990399,997320,997977,999355,991648,994280,997586,991206,996592,998401,994661,995672\n", "Ground Truth: 997918,993289,997002,995826,993504,993492,993512,999900,996511,994449,992571,993781,992029,994959,999840,997790,994268,990096,998090,996062,999019,990399,997320,997977,990930,999355,991648,994280,997586,991206\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", "Query ID: 933\n", - "Returned IDs: 999784,996001,996028,998302,997371,997002,997320,998921,999806,994266,995560,993820,995596,998720,997913,993834,994964,993342,996840,993902,997301,990532,993906,996096,995886,993260,994689,995852,992330,997506\n", + "Returned IDs: 999784,996001,996028,998302,997371,997002,997320,998921,999806,994266,995560,993820,995596,997913,994964,993342,996840,993902,997301,990532,993906,995886,993260,994689,992330,997506,994360,990771,999453,996075\n", "Ground Truth: 999784,996001,996028,998302,997371,997002,997320,998921,999806,994266,995560,993820,995596,998720,997913,993834,994964,993342,996840,993902,997301,990532,995375,993906,996096,995886,993260,994689,995852,992330\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 934\n", + "Returned IDs: 999867,994964,995435,999302,996645,992546,990957,997371,995942,997913,998163,997320,995949,999374,994783,993853,995755,993725,990925,993484,998500,994831,995928,998294,993885,994179,999740,990736,999356,998326\n", + "Ground Truth: 999867,994964,995435,999302,996645,992546,990957,997371,995942,997913,998163,997320,995949,999374,994783,993853,995755,998365,990504,993725,990925,993484,998500,992107,994831,995928,993412,998294,999444,993885\n", + "Recall: 0.8333\n", "\n", "Query ID: 935\n", - "Returned IDs: 995981,996609,990444,996648,991721,994069,999374,991593,993412,990990,996085,998302,998777,997320,998246,996805,997301,995266,998339,993186,998849,991422,990585,998974,991602,993049,990096,990146,992291,991809\n", + "Returned IDs: 995981,996609,990444,996648,991721,994069,999374,991593,990990,998302,999046,997320,998246,996805,997301,998339,998849,991422,990585,998974,990096,990146,992291,990957,999533,994409,997614,998156,998294,995884\n", "Ground Truth: 995981,996609,990444,996648,991721,994069,999374,991593,993412,990990,996085,998302,998777,999046,997320,998246,996805,997301,995266,998339,993186,998849,991422,990585,998974,991602,993049,990096,990146,992291\n", - "Recall: 0.9667\n", + "Recall: 0.7667\n", "\n", "Query ID: 936\n", - "Returned IDs: 997700,999581,999054,991507,990537,991722,995106,994861,995268,993209,992448,993313,995778,996607,990372,997738,995820,998162,998910,998283,999279,995518,996505,994380,990924,991424,996813,995401,991699,995521\n", + "Returned IDs: 997700,999581,999054,991507,990537,991722,995106,994861,995268,993209,992448,993313,995778,996607,990372,997738,995820,998162,998910,999279,995518,996505,994380,990924,991424,996813,995401,991699,995521,999047\n", "Ground Truth: 997700,999581,999054,991507,990537,991722,995106,994861,995268,993209,992448,993313,995778,996607,990372,997738,995820,998162,998910,998283,999279,995518,996505,994380,990924,991424,993046,996813,995401,991699\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 937\n", + "Returned IDs: 992751,995443,996609,992944,997918,991801,996075,999355,995203,997002,998232,995336,994552,996871,993781,996193,999620,990043,995515,998180,996001,993283,996735,997305,995536,992880,991498,991131,994042,995372\n", + "Ground Truth: 992751,995443,996609,992944,997918,991801,996075,999355,995203,997002,998232,995336,994552,996871,993781,996193,999620,994162,990043,995515,998180,996744,996001,993283,994449,995365,996735,997305,995536,992880\n", + "Recall: 0.8667\n", "\n", "Query ID: 938\n", - "Returned IDs: 998777,996219,999676,995401,999453,998530,998797,992309,998495,993355,994318,990843,996840,998302,992950,990974,992328,994850,995016,991585,994195,993412,994964,990432,995899,992216,995374,999302,997554,996033\n", + "Returned IDs: 998777,996219,999676,995401,999453,998530,998797,992309,998495,993355,994318,990843,996840,998302,992950,990974,995016,991585,994195,993431,994964,990432,992216,999302,997554,991743,992169,990856,991732,993717\n", "Ground Truth: 998777,996219,999676,995401,999453,998530,998797,992309,998495,993355,994318,990843,996840,998302,992950,990974,992328,994850,995016,991585,994195,993431,993412,994964,995899,990432,992216,995374,999302,997554\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", "\n", "Query ID: 940\n", - "Returned IDs: 994897,993342,999806,993617,997335,999430,992628,991271,993528,992807,995393,990173,990194,991846,999555,994155,998787,997126,994150,996809,999834,996660,995895,997196,994122,993076,994269,996184,996944,994834\n", + "Returned IDs: 994897,993342,999806,993617,999430,992628,993528,992807,995393,990173,990194,991846,999555,994155,998787,994150,996809,999834,995895,994122,993076,994269,996184,996944,994834,991148,997906,997084,997378,994682\n", "Ground Truth: 994897,993342,999806,993617,997335,999430,992628,995039,991271,993528,992807,995393,990173,990194,991846,999555,994155,998214,998787,997126,994150,996809,999834,996660,995895,997196,994122,993076,994269,996184\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 941\n", + "Returned IDs: 993143,997370,990580,994789,995515,999077,993237,991840,999898,998837,999784,998401,991999,998734,992612,996840,994964,993260,990096,997918,992944,998180,991172,990142,996693,999210,999676,994195,997586,998455\n", + "Ground Truth: 993143,997370,990580,994789,995515,999077,993237,991840,999898,998837,999784,998401,991999,997135,998734,992612,996840,994964,993260,990096,997918,992944,995401,998180,991172,990142,996693,999210,999676,994195\n", "Recall: 0.9333\n", "\n", "Query ID: 942\n", - "Returned IDs: 998793,999476,997215,992802,997744,990782,990150,995552,990755,994965,995724,990192,990908,992608,999639,990334,999164,998242,996196,990930,997548,991791,998495,990538,994661,999908,995979,995430,995372,999294\n", + "Returned IDs: 998793,999476,997215,992802,997744,990150,995552,990755,994965,995724,990192,990908,992608,999639,999164,998242,996196,990930,997548,991791,998495,990538,994661,999908,995979,995430,995372,995983,998754,994165\n", "Ground Truth: 998793,999476,997215,992802,997744,990782,990150,995552,990755,994965,994536,995724,990192,990908,992608,999639,990334,999164,998242,996196,990930,997548,991791,998495,990538,994661,999908,995979,995430,995372\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", "\n", "Query ID: 943\n", - "Returned IDs: 994879,998347,992582,997728,996673,992760,994265,997353,998225,992502,991431,992293,996948,992862,997304,995981,999761,992481,990862,998708,994304,998986,994894,998697,993072,999243,993809,997620,995927,992183\n", + "Returned IDs: 994879,992582,997728,996673,997353,998225,992502,991431,992293,992862,997304,995981,999761,992481,998708,994304,998986,994894,998697,999243,993809,995927,992183,997154,992852,998200,997913,996714,993970,995701\n", "Ground Truth: 994879,998347,993222,992582,997728,996673,992760,994265,997353,998225,992502,991431,992293,996948,992862,997304,991680,995981,999761,992481,990862,998708,994304,998986,994894,998697,993072,999243,993809,997620\n", - "Recall: 0.9333\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 944\n", + "Returned IDs: 992868,995768,991490,997340,999798,991850,991779,992761,999256,991667,995655,990173,992517,995385,995501,994356,995476,993055,990154,990127,995879,993560,992372,995299,992561,998495,991313,997506,996580,993391\n", + "Ground Truth: 992868,995768,991490,997340,999798,991850,991779,992761,999256,991667,995655,990173,992110,992517,995385,995501,994356,999601,995476,993055,990154,995222,990127,991494,995879,993560,998167,992372,995299,992561\n", + "Recall: 0.8333\n", "\n", "Query ID: 945\n", - "Returned IDs: 993209,993299,996867,994317,990638,993586,997544,990082,994742,997723,997319,992582,990126,997077,993549,995030,994587,991943,995211,996302,998913,995633,991664,990493,998910,992289,995268,993920,996411,991639\n", + "Returned IDs: 993209,993299,996867,990638,993586,997544,990082,994742,997319,992582,990126,993549,995078,995030,994587,991943,995211,996302,998913,995633,990493,998910,992289,995268,993920,996411,991639,995274,999091,997746\n", "Ground Truth: 993209,993299,996867,994317,990638,993586,997544,990082,994742,997723,997319,992582,990126,997077,993549,995078,995030,994587,991943,995211,996302,992667,998913,995633,996277,991664,990493,998910,992289,995268\n", - "Recall: 0.9000\n", + "Recall: 0.8000\n", "\n", "Query ID: 946\n", - "Returned IDs: 995547,999530,994409,999367,995994,998545,991061,997723,993717,993561,991923,997191,996042,994948,996769,994026,996038,996918,997406,995401,995507,997832,991459,994002,997675,993143,997289,991052,995923,993271\n", + "Returned IDs: 995547,999530,994409,999367,998545,997723,993717,993561,991923,997191,996042,994948,996769,996038,996918,997406,995401,995507,997832,991459,994002,997675,993143,997289,991052,995923,993271,996048,992130,998045\n", "Ground Truth: 995547,999530,994409,999367,995994,998545,991061,997723,993717,993561,991923,997191,996042,994948,996769,994026,996038,996918,997406,995401,995507,997832,991459,993122,994002,993143,997675,997289,998250,991052\n", - "Recall: 0.9333\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 947\n", + "Returned IDs: 991298,992680,992469,995126,993053,993164,994140,993561,990529,990096,990845,996411,994925,996001,996062,996028,999729,990076,994783,999302,991789,995289,995216,998039,995823,994964,999099,997386,990930,994293\n", + "Ground Truth: 991298,992680,992469,995126,998633,993053,992334,993164,994140,993561,990529,990096,990845,996411,994925,996001,996062,996028,999729,990076,994783,999302,991789,995401,995289,997822,998940,995216,993433,998039\n", + "Recall: 0.8000\n", "\n", "Query ID: 948\n", - "Returned IDs: 996001,994449,995536,990532,996661,997586,997320,996824,992862,994200,999444,996178,999639,991630,992635,999587,995672,990814,993504,992469,999974,995983,995250,993157,990399,991981,998827,999453,995401,994334\n", + "Returned IDs: 996001,995536,990532,996661,997586,997320,992862,996368,994200,999444,996178,999639,999587,995672,990814,993504,992469,999974,995983,993157,990399,991981,998827,999453,995401,994334,995535,991157,993289,996147\n", "Ground Truth: 996001,994449,995536,990532,996661,997586,997320,996824,996511,992862,996368,994200,999444,996178,999639,991630,992521,992635,999587,995672,990814,993504,992469,999974,994343,995983,995250,993157,990399,991981\n", - "Recall: 0.8667\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 949\n", + "Returned IDs: 993209,995268,990361,999581,995106,995289,991630,991664,993920,990593,994742,995766,996867,998748,993586,999047,995778,997411,995052,993751,998188,993053,999653,997915,992521,998045,994409,993997,999279,991507\n", + "Ground Truth: 993209,995268,990361,997723,999581,995198,995106,995289,991630,991664,993920,990593,994742,995766,996867,998748,993586,996277,999047,995778,997411,992293,995052,996042,998999,993751,998188,997612,993053,999653\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 950\n", + "Returned IDs: 995226,993609,992457,997506,999139,996915,990169,990203,992640,995459,992170,995349,998933,996233,992824,991245,994672,995839,994743,993595,996742,997926,997442,992736,995740,992052,991138,993715,993055,992429\n", + "Ground Truth: 995226,993609,992457,997506,999139,996915,990169,990203,992640,995459,992170,995349,998933,996233,992824,991245,994672,995839,994743,993595,993077,996742,997926,997442,992736,995740,992052,991138,995408,993715\n", + "Recall: 0.9333\n", "\n", "Query ID: 951\n", - "Returned IDs: 999076,993355,997539,994922,990300,998424,990809,990580,995210,995515,997747,999582,993688,998236,998751,993237,998837,999469,994789,992612,991609,991392,997291,996561,992979,999097,998401,997772,997818,992950\n", + "Returned IDs: 999076,993355,997539,994922,990300,998424,990809,990580,995210,995515,997747,999582,993688,998236,998751,993237,998837,994789,992612,991392,997291,998401,997772,997818,992950,990615,993091,993143,997852,991833\n", "Ground Truth: 999076,993355,997539,994922,990300,998424,990809,990580,995210,995515,997747,999582,995430,993688,998236,998751,993237,998837,999469,994789,992612,993222,991609,991392,997291,996561,992979,999097,996793,998401\n", - "Recall: 0.9000\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 952\n", + "Returned IDs: 995358,990075,998568,994897,996217,999963,997617,996429,993866,993260,995203,996764,993055,999079,997965,997906,994832,992082,996075,993696,993342,990697,998697,993504,993560,991517,994680,992617,995455,999653\n", + "Ground Truth: 995358,990075,998568,994897,996217,999963,997617,996429,993866,993260,995203,996764,995622,993055,999079,997965,997906,997577,994832,992082,997506,996075,991630,993696,993342,990697,998697,993504,993560,991517\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 953\n", + "Returned IDs: 997320,990560,997371,998950,996075,994964,997913,991743,990949,997301,991260,998090,996805,993529,990910,998550,992443,995826,990231,999374,997192,993264,998365,995942,996219,992041,998246,995336,993504,992635\n", + "Ground Truth: 997320,990560,997371,998950,996075,994964,997913,991743,990949,997301,991260,998090,996805,993529,990910,998550,992443,992277,995826,990231,999374,997192,993264,998365,995942,996219,992041,998246,995336,993504\n", + "Recall: 0.9667\n", "\n", "Query ID: 954\n", - "Returned IDs: 995840,997807,998414,994782,998668,997586,998496,997004,995927,999210,999941,998068,993114,995944,999931,993689,993878,997364,990866,999898,995971,993624,997305,991259,998814,998708,990373,993831,999218,997552\n", + "Returned IDs: 995840,997807,994782,998668,997586,998496,997004,995927,999210,999941,993114,995944,993689,993878,997364,990866,999898,993624,997305,998814,998708,990373,993831,999218,997552,994815,996219,991938,991373,998495\n", "Ground Truth: 995840,997807,998414,994782,998668,997586,998496,997004,994756,995927,999210,999941,998068,993114,995944,999931,993689,993878,997364,990866,999898,998861,991006,995971,993624,997305,991259,998814,998708,990373\n", - "Recall: 0.9000\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 955\n", + "Returned IDs: 998921,997913,995693,998302,998950,990432,994964,994318,996053,997320,996219,992277,992944,996915,997470,998170,993431,997371,991999,996075,999210,999374,991743,996178,992978,999343,999077,992309,993143,994919\n", + "Ground Truth: 998921,995826,997913,995693,998302,998950,990432,994964,994318,996053,997320,996219,992277,998294,992944,996915,997470,998170,993431,997371,991999,992328,992140,996075,999210,999374,997379,991743,996178,992978\n", + "Recall: 0.8333\n", "\n", "Query ID: 956\n", - "Returned IDs: 992630,999343,991335,992384,996219,990934,990171,996401,994100,990974,990448,996143,999331,998779,997571,998170,997371,997920,996790,999102,998550,997006,990275,999647,996777,991862,990315,996727,998500,990666\n", + "Returned IDs: 992630,999343,991335,992384,996219,990934,992931,994100,990974,990448,996143,999331,998779,997571,998170,997371,997920,996790,999102,990275,999647,991862,996727,998500,994831,994964,995274,990271,998263,999144\n", "Ground Truth: 992630,999343,991335,992384,996219,990934,990171,992931,996401,994100,990974,990448,996143,999331,998779,997571,998170,997371,997920,996790,999102,998550,997006,990275,999647,996777,991862,990315,996727,998500\n", - "Recall: 0.9667\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 957\n", + "Returned IDs: 993561,991958,997406,998382,996080,995126,992334,992680,998593,993738,995923,997753,992469,998039,995216,997386,991125,999729,998165,995289,994925,991857,993645,990096,993390,996001,993433,992322,998999,995182\n", + "Ground Truth: 993561,991958,997406,998382,996080,995126,992334,992680,998593,993738,995923,997753,992469,998039,995216,997386,991125,999729,998165,995289,994925,991857,993645,990930,990096,993390,993725,996001,993433,992322\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 958\n", + "Returned IDs: 998827,999122,992558,990620,996140,999806,991009,990538,997913,997124,997859,991157,995596,995737,998892,997227,996404,998950,995983,991272,999562,992423,999804,996918,997320,994577,996196,993492,995203,997305\n", + "Ground Truth: 998827,999122,992558,990620,996140,999806,991009,990538,997913,997124,997859,991157,995596,995737,998892,997227,996404,998950,995983,991272,999562,992423,992971,999804,996918,997320,992612,994577,996196,993492\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 959\n", + "Returned IDs: 993764,996472,991987,994365,990942,990593,997884,991610,995891,996971,990331,995537,992631,995268,994002,998586,992136,991038,997506,992123,999129,997675,996623,998067,993111,992721,993956,990005,994809,994125\n", + "Ground Truth: 993764,996472,991987,994365,990942,993381,990593,997884,999906,991610,995891,996971,990331,995537,992631,995864,990998,995268,994002,991709,993629,998586,992136,991038,997506,992123,999129,997675,998959,992589\n", + "Recall: 0.7333\n", "\n", "Query ID: 960\n", - "Returned IDs: 998335,990627,992862,991304,996456,996104,995991,996594,992270,994564,990802,991447,995231,994742,995168,990998,996408,999319,993076,999334,995780,996611,998279,992636,996728,991098,994498,997506,995626,993638\n", + "Returned IDs: 998335,990627,992862,991304,996456,996104,994796,995991,996594,992270,994564,990802,995231,994742,995168,990998,996408,999334,995780,996611,998279,992636,996728,991098,997506,995626,993638,995530,999216,990844\n", "Ground Truth: 998335,990627,992862,991304,996456,996104,994796,995991,996594,992270,994564,990802,991447,995231,994742,995168,990998,996408,999319,993076,999334,995780,996611,998279,992636,996728,991098,994498,997506,995626\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 961\n", + "Returned IDs: 995935,997221,991463,993433,999742,998045,992125,998906,996125,991367,992556,996994,995439,998520,992521,996769,996719,996112,996323,997359,993844,995052,996277,997675,997406,990984,996048,992115,992089,997046\n", + "Ground Truth: 995935,997221,991463,993433,999742,998045,997187,992125,998906,996125,991367,992556,996994,995439,998520,992521,996769,996719,996112,996323,997359,993844,995052,996277,997675,997406,990984,996048,992115,992089\n", "Recall: 0.9667\n", "\n", "Query ID: 962\n", - "Returned IDs: 995336,997301,995438,994964,995961,994757,998246,999144,994360,999331,999710,997320,990949,995942,993492,992384,997470,990231,998302,994100,998090,990958,996592,995886,993241,992228,994081,990957,990096,991883\n", + "Returned IDs: 995336,997301,995438,994964,995961,994757,998246,999144,994360,999331,999710,997320,990949,995942,993492,992384,997470,990231,998302,997775,999301,994100,998090,990958,996592,995886,993241,992228,994081,990957\n", "Ground Truth: 995336,997301,995438,994964,995961,994757,998246,999144,994360,999331,993154,999710,997320,990949,995942,993492,992384,997470,990231,998302,997775,999301,994100,998090,990958,996592,995886,993241,992228,994081\n", - "Recall: 0.9000\n", + "Recall: 0.9667\n", "\n", "Query ID: 963\n", - "Returned IDs: 992135,991459,998623,990537,998720,990682,996001,995968,990457,996060,990502,991154,996247,990005,993673,996812,995027,994809,997091,994380,999453,999543,996417,990616,997965,996759,999724,995932,998068,995937\n", + "Returned IDs: 992135,991459,998623,990537,990682,996001,995968,996060,990502,996247,990005,996812,994809,994380,999453,999543,996759,999724,995932,998068,994835,995268,991389,999653,996256,991175,990251,995815,990294,999977\n", "Ground Truth: 992135,991459,998623,990537,998720,990682,996001,995968,990457,997140,996060,990502,991154,996247,990005,993673,996812,995027,993131,994809,997091,994380,999453,992124,999543,996417,990616,997965,996759,998029\n", - "Recall: 0.8667\n", + "Recall: 0.5667\n", "\n", - "Query ID: 965\n", - "Returned IDs: 994964,997379,995791,993035,992944,994777,996397,999210,998950,996811,993696,993633,993215,992612,990310,998665,995365,994922,996915,991999,999676,990690,993355,992169,993809,996093,994318,993561,992334,995596\n", - "Ground Truth: 994964,997379,995791,993035,992944,994777,996397,999210,998950,996811,993831,993696,993633,994089,993215,992612,990310,998665,995365,994922,996915,991999,999676,990690,993355,992169,993809,996093,994318,993561\n", - "Recall: 0.9333\n", + "Query ID: 964\n", + "Returned IDs: 991168,991464,992547,995283,995295,999519,995861,994742,995456,992334,991789,999126,994817,994853,999302,998224,993725,997933,992408,993831,998779,994765,993717,994783,996028,990266,990392,990889,993561,999341\n", + "Ground Truth: 991168,991464,992547,995283,995295,999519,995861,994742,995456,992334,990928,991789,999126,994817,994853,999302,991334,998224,996906,997933,993725,996308,992408,993831,998779,994765,993717,996147,994783,996028\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 965\n", + "Returned IDs: 994964,997379,995791,993035,992944,994777,999210,998950,996811,993696,993215,992612,990310,998665,994922,996915,991999,999676,990690,993355,992169,993809,996093,994318,993561,995596,997320,993906,996867,999076\n", + "Ground Truth: 994964,997379,995791,993035,992944,994777,996397,999210,998950,996811,993831,993696,993633,994089,993215,992612,990310,998665,995365,994922,996915,991999,999676,990690,993355,992169,993809,996093,994318,993561\n", + "Recall: 0.8333\n", "\n", "Query ID: 966\n", - "Returned IDs: 998797,999995,997040,990146,999262,995884,997320,990925,994964,998302,992546,996959,990936,990958,991852,995947,997586,992896,996785,997911,993484,998246,997913,996075,995336,997822,999343,999242,999374,994366\n", + "Returned IDs: 998797,999995,997040,990146,999262,995884,997320,990925,994964,998302,995873,992546,996959,990936,990958,995947,997586,992896,997911,993484,998246,997913,996075,995336,999343,999242,999374,992330,993168,991498\n", "Ground Truth: 998797,999995,997040,990146,999262,995884,997320,990925,994964,998302,995873,992546,996959,990936,990958,991852,995947,997586,992896,996785,997911,993484,998246,997913,996075,995336,997822,999343,996296,999242\n", - "Recall: 0.9333\n", + "Recall: 0.8667\n", "\n", "Query ID: 967\n", - "Returned IDs: 995349,997685,999079,992664,997673,991148,993866,997906,998468,999443,999576,992209,995969,995231,992275,994661,997369,990096,997666,993965,995064,995895,996733,991477,995695,993445,996124,995735,995393,994751\n", + "Returned IDs: 995349,997685,999079,992664,997673,991148,993866,997906,999443,999576,992209,995969,995231,992275,994661,997369,990096,997666,993965,995064,995895,996733,991477,995695,993445,996124,995735,995393,995168,990622\n", "Ground Truth: 995349,997685,999079,992664,997673,991148,993866,997906,998468,999443,999576,992209,995969,995231,992275,994661,997369,990096,997666,993965,995064,995895,996733,991477,995695,993445,996124,995735,992752,995393\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 968\n", + "Returned IDs: 992664,994105,998959,995168,991477,990998,994834,995231,994906,998623,997666,996050,998565,992760,993948,997332,999373,990261,998536,990952,992862,995695,997506,995835,993866,996393,995969,998352,995536,990622\n", + "Ground Truth: 992664,994105,998959,995168,991477,990998,994834,995231,994906,998623,997666,996050,998383,998565,991654,992760,993948,997332,999373,990261,998536,990952,999555,992862,995695,997506,995835,991188,993866,996393\n", + "Recall: 0.8667\n", "\n", "Query ID: 969\n", - "Returned IDs: 991110,990729,999737,998022,991767,998318,994063,991887,994088,995373,999662,995449,997131,995480,992604,990256,999445,994122,992133,996236,990577,996730,994577,998629,993137,997942,995080,990345,994539,993185\n", + "Returned IDs: 991110,990729,999737,998022,991767,998318,994063,991887,994088,995373,999662,994956,995449,997131,995480,992604,990256,999445,994122,992133,996236,990577,996730,998629,993137,997942,995080,990345,994539,993185\n", "Ground Truth: 991110,990729,999737,998022,991767,998318,994063,991887,994088,995373,999662,994956,995449,997131,995480,992604,990256,999445,994122,992133,996236,990577,996730,994577,998629,993137,997942,995080,990345,994539\n", "Recall: 0.9667\n", "\n", "Query ID: 970\n", - "Returned IDs: 993484,994964,993648,993642,993238,997320,999374,990995,992047,990399,999453,995755,993389,996959,990936,990224,997737,996785,993143,993885,993412,995961,993663,998294,991743,994922,995886,990958,994673,998692\n", + "Returned IDs: 993484,994964,993648,993642,993238,997320,999374,990399,999453,995755,993389,990224,997737,996785,993143,993885,995961,998294,991743,994922,995886,990958,994673,998692,996854,997796,999827,997822,994266,997301\n", "Ground Truth: 993484,994964,993648,993642,993238,997320,999374,990995,992047,990399,999453,995755,993389,996959,990936,990224,997737,996785,993143,993885,993412,995961,993663,992334,993400,998294,991743,994922,995886,990958\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 971\n", + "Returned IDs: 995940,991478,992364,997319,997723,998999,995685,991140,991590,995200,995190,996747,990223,991810,999788,997832,995289,993412,992289,992917,990940,995994,990986,998750,993293,991968,994948,995269,996302,993831\n", + "Ground Truth: 995940,991478,992364,997319,997723,998999,995685,991140,991590,995200,995190,996747,990223,991810,999788,997832,995289,993412,992289,992917,990940,995994,990986,998750,993293,991403,991968,991603,994948,995269\n", "Recall: 0.9333\n", "\n", "Query ID: 972\n", - "Returned IDs: 996001,993053,994106,992182,999352,995200,996219,993883,995873,994382,997909,990435,998725,999653,995216,994689,990557,991464,990752,995823,995289,993438,996657,994529,994783,998982,991212,999862,995222,996277\n", + "Returned IDs: 996001,993053,994106,992182,995200,996219,995873,994382,997909,990435,998725,999653,995216,994689,990557,991464,990752,995823,995289,993438,996657,993984,994529,994783,998982,991212,997529,994558,999402,999028\n", "Ground Truth: 996001,993053,994300,999151,994106,992182,999352,995200,998568,996219,993883,995873,994382,997909,990435,995384,998725,999653,995216,994689,996411,997837,990557,991464,992384,995823,990752,995289,995198,993438\n", - "Recall: 0.7333\n", + "Recall: 0.6667\n", "\n", "Query ID: 973\n", - "Returned IDs: 990296,992161,998221,993412,998649,998495,990957,999898,990399,997192,996869,992107,995942,990435,993656,996001,990934,991335,997582,991311,992087,996219,999144,999710,995289,998039,993168,991096,995755,995536\n", + "Returned IDs: 990296,992161,998221,998649,998495,990957,999898,990399,997192,996869,992107,995942,990435,993656,996001,990934,991335,997582,992087,996219,999144,995289,992124,993168,991096,995755,995536,990096,994961,993648\n", "Ground Truth: 990296,992161,998221,993412,998649,998495,990957,999898,990399,997920,997192,996869,992107,995942,990435,993656,996001,990934,991335,997582,991311,992087,996219,999144,999710,995289,992124,995899,998039,993168\n", - "Recall: 0.9000\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 975\n", + "Returned IDs: 995332,995560,991994,996609,995107,998950,997002,994280,998697,994783,999653,995535,994964,990435,995536,996429,992558,996219,991212,991382,992446,993504,999210,993079,994972,994742,999740,997305,996871,991571\n", + "Ground Truth: 995332,995560,991994,996609,995107,998950,997002,994280,991921,998697,994783,999653,996706,995535,994964,995401,990435,995536,996429,992558,996219,991212,991382,992446,993504,999210,993079,994972,994742,995873\n", + "Recall: 0.8667\n", "\n", "Query ID: 976\n", - "Returned IDs: 995958,995778,997023,992473,997723,993209,996376,994453,995401,996429,996226,995701,999675,998067,997601,997204,997216,991507,991044,996232,998495,997732,997544,993948,991747,993313,996328,991958,996506,998045\n", + "Returned IDs: 995958,995778,997023,992473,993209,995401,996429,995701,999675,995536,998067,997204,997216,991507,991044,999977,996232,997732,997544,991747,993313,996328,991958,996506,998045,996220,990537,992293,993561,995702\n", "Ground Truth: 995958,995778,997023,992473,997723,993209,996376,994453,995401,996429,996226,995701,999675,995536,998067,997601,997204,997216,991507,991044,999977,996232,998495,997732,997544,993948,990814,991747,993313,996328\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 977\n", + "Returned IDs: 996379,998186,994589,999977,998045,997058,994946,996368,992197,997554,993561,995401,998779,996391,991630,995268,995873,993412,995701,993275,993390,996028,994409,996169,990525,992367,993813,999587,991415,992362\n", + "Ground Truth: 996379,998186,994589,999977,998045,997058,994946,996368,992197,997554,993561,995401,998779,996391,991630,995268,995873,993412,995701,993275,993390,996028,997723,994409,996169,990525,992367,993813,999587,991415\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 978\n", + "Returned IDs: 993504,993071,992823,997977,995203,992635,992082,999111,991408,991648,999840,991009,990358,991157,991206,997320,997965,992747,994694,997327,999750,992119,998401,993515,990885,992351,990196,998950,994134,992036\n", + "Ground Truth: 993504,993071,992823,997977,995203,992635,992082,999111,992336,991408,991648,999840,991009,990358,992750,991157,991206,997320,997965,992747,994694,997327,999750,992119,998401,993515,990885,992351,990196,991542\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 980\n", + "Returned IDs: 995643,996322,997776,990252,994419,995547,996818,992894,998545,996042,996254,996148,991301,994173,998486,998866,996569,992660,997046,997822,990137,996719,997477,996360,998564,996745,990975,997937,997289,991011\n", + "Ground Truth: 995643,996322,997776,990252,994419,995547,996818,992894,998545,996042,996254,996148,991301,994173,998486,998866,996569,992660,997046,997822,990137,996719,998973,997477,993416,996360,998416,998564,996745,990975\n", "Recall: 0.9000\n", "\n", + "Query ID: 981\n", + "Returned IDs: 993857,997309,994119,994742,995166,994564,995295,990328,993384,997914,991334,997452,995456,999635,998180,997822,996689,993180,997173,997374,990392,998779,999843,993256,992703,998224,992582,999533,995633,994199\n", + "Ground Truth: 993857,997309,994119,994742,995166,994564,995295,990328,993909,993384,997208,997914,991334,997452,995456,999635,998180,997822,996689,993180,997173,991447,997374,990392,990133,998779,999843,992473,993256,992703\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 982\n", + "Returned IDs: 997544,990082,992582,990298,996220,997723,994453,993471,996269,998910,999646,991269,996304,998776,991136,995702,991334,995958,995685,999054,991639,999759,991507,993586,996226,990372,997466,990804,993209,990638\n", + "Ground Truth: 997544,990082,992582,990298,996220,998310,997723,994453,993471,991603,996269,998910,999646,991269,996304,998776,991136,995702,990035,991334,996038,995958,995685,999054,991639,999759,996777,991507,993586,996226\n", + "Recall: 0.8333\n", + "\n", "Query ID: 983\n", - "Returned IDs: 995295,996580,995283,994730,990528,992226,991464,993725,990531,990930,993585,990315,996411,994742,994682,993623,990614,990076,990133,995289,994783,999798,990811,992680,996136,993053,997141,993705,997529,997340\n", + "Returned IDs: 995295,996580,995283,994730,990528,992226,991464,993725,990531,990930,993585,990315,996411,994742,994682,993623,990614,990076,990133,995289,994783,999798,990811,992680,993053,997529,999916,997340,990392,991330\n", "Ground Truth: 995295,996580,995283,994730,990528,992226,991464,993725,990531,990930,993585,990315,996411,994742,994682,993623,990614,990076,990133,995289,994783,999798,990811,992680,996136,993053,997141,991629,993705,997529\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 984\n", + "Returned IDs: 991471,998086,995701,997576,991459,998861,999555,999994,990194,998212,999384,996623,993561,991630,994884,990167,999663,991038,997906,999977,996649,999724,998022,993275,992384,993342,997154,997242,996406,996809\n", + "Ground Truth: 991471,998086,995701,997576,991459,998861,999555,999994,990194,995319,998212,999384,996623,993561,991630,990708,994884,999653,990167,999663,991038,997906,999977,999647,991154,996649,999958,999724,994906,998022\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 985\n", + "Returned IDs: 990096,999831,995638,993499,998934,992551,991154,998968,999647,990583,998314,999384,990584,999194,998697,996216,999045,998495,995899,996811,996564,997586,993809,999499,990083,994661,993686,993458,997685,993055\n", + "Ground Truth: 990096,999831,995638,993499,998934,992551,993928,997524,991154,998968,999647,990583,998314,999384,990584,999194,998697,996216,999045,998495,995899,996811,996564,997586,993809,999499,992661,999343,990780,990083\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 986\n", + "Returned IDs: 991348,994552,997918,993260,993241,997320,992041,994290,999453,995372,999210,999077,997113,999317,990843,998950,999784,991999,993484,996318,993696,992944,995108,996075,991743,993018,996592,998464,994886,991131\n", + "Ground Truth: 991348,994552,997918,993260,993241,995622,997320,992041,994290,999453,995372,999210,999077,997113,999317,990843,998950,999784,991999,993484,996318,993696,992944,993365,995108,996075,991743,993018,992942,997135\n", + "Recall: 0.8667\n", "\n", "Query ID: 987\n", - "Returned IDs: 997204,992169,993288,995289,995374,999587,995359,999099,992685,995685,991151,997918,998530,993909,993412,991810,994409,996328,999820,992979,992239,995401,997395,997002,997630,994906,990337,996903,997447,995994\n", + "Returned IDs: 997204,992169,993288,995289,999587,995359,999099,992685,995685,997918,998530,993412,991810,994409,996328,996220,999820,992979,992239,995401,997395,997002,997630,994906,996903,995994,990940,999977,995611,990223\n", "Ground Truth: 997204,992169,993288,995289,995374,999587,995359,999099,992685,995685,991151,997918,998530,993909,993412,991810,994409,996328,996220,999820,992979,992239,995401,997395,997002,997630,994906,990337,996903,997447\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 988\n", + "Returned IDs: 998992,997015,993562,990535,991068,993050,994685,999531,991568,994997,994234,992744,993484,994964,991008,997306,994217,995336,990673,995824,996785,997320,993648,999757,991467,992709,996959,997314,993168,994111\n", + "Ground Truth: 998992,997015,993562,990535,991068,993050,991433,994685,999531,991568,994997,994234,992744,993484,994964,991008,997306,994217,995336,990673,995824,996785,997320,993648,999757,991467,992709,996959,997314,993168\n", "Recall: 0.9667\n", "\n", "Query ID: 989\n", - "Returned IDs: 997267,990129,991544,995891,996472,992448,992801,996042,991958,995521,994409,997884,995864,998045,997909,998703,995200,995198,993433,994002,994742,991671,994954,994413,993127,993561,998623,992123,995702,997077\n", + "Returned IDs: 997267,990129,991544,995891,996472,992448,992801,996042,991958,995521,994409,997884,998045,998703,990681,995200,993433,994002,994742,991671,994954,994413,993127,993561,998623,995702,996695,996607,994365,998910\n", "Ground Truth: 997267,990129,991544,995891,996472,992448,992801,996042,991958,995521,994409,997884,995864,995873,998045,997909,998703,990681,995200,995198,993433,994002,994742,991671,994954,994413,993127,993561,998623,992123\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 990\n", + "Returned IDs: 999139,992457,993567,994280,995600,997480,992824,998109,994130,994196,997442,998405,993055,992480,992237,992736,990929,996016,998251,991382,996270,999985,992052,991723,998631,995823,991128,999205,997050,990705\n", + "Ground Truth: 999139,992457,993567,994280,995600,997480,992824,998109,994130,994196,997442,998405,993055,992480,992237,992736,990929,996016,998251,991382,996270,999985,995459,992052,991723,998631,995823,991128,999205,993621\n", "Recall: 0.9333\n", "\n", "Query ID: 991\n", - "Returned IDs: 996571,991975,996120,993381,997145,997965,994724,991987,993641,991513,991038,999653,997369,995780,996920,997906,991844,997265,993629,992945,998586,994374,998861,990450,995168,996209,990991,995231,992862,996326\n", + "Returned IDs: 996571,991975,997965,994724,991987,993641,991513,991038,999653,997369,995780,996920,997906,991844,997265,992945,998586,994374,998861,995168,992233,996209,990991,995231,992862,996326,994451,998858,990167,995536\n", "Ground Truth: 996571,991975,996120,993381,997145,997965,994724,991987,993641,991513,991038,999653,997369,995780,996920,997906,991844,997265,993629,992945,998586,994374,998861,990450,995168,992233,996209,990991,995231,992862\n", - "Recall: 0.9667\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 992\n", + "Returned IDs: 993562,997015,993050,998992,990535,999531,997314,991568,990673,999026,993168,991444,992911,992744,995034,991391,990797,994964,993648,996714,994595,992646,999596,990817,999024,991467,991086,998951,995077,994100\n", + "Ground Truth: 993562,997015,993050,998992,990535,999531,997314,991568,991433,990673,999026,993168,991444,992911,992744,994265,996336,995034,991391,990797,994425,994964,993648,996714,994595,992646,999596,990817,999024,991467\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 993\n", + "Returned IDs: 993018,998950,997320,991157,992423,992617,990843,996262,992521,992944,997370,993840,995560,999432,999806,995596,999453,995515,995552,999210,995203,998255,996140,998232,990083,991801,994746,991348,991224,994577\n", + "Ground Truth: 993018,998950,997320,991157,992423,992617,990843,996262,992521,992944,997370,993840,995560,999432,999806,995596,999453,995515,995552,999210,995203,996368,998255,995250,996140,998232,990083,991801,995899,994746\n", + "Recall: 0.9000\n", "\n", "Query ID: 994\n", - "Returned IDs: 994102,990092,991862,991335,996052,999647,990957,990231,996219,992134,994831,990545,997439,997913,996143,992114,995942,991743,999343,994192,993264,997320,990171,999144,993135,992931,996053,995865,998302,990949\n", + "Returned IDs: \n", "Ground Truth: 994102,990092,991862,991335,996052,999647,990957,990231,996219,992134,994831,990545,997439,997913,996143,992114,995942,991743,995844,999343,999993,994192,993264,997320,990171,999144,993135,992931,996053,994898\n", - "Recall: 0.9000\n", + "Recall: 0.0000\n", "\n", "Query ID: 995\n", - "Returned IDs: 996627,994751,996411,993070,997729,990011,990012,990528,990743,996906,994663,995295,993725,992230,992680,992408,999740,999922,994830,990392,993240,994730,995076,996011,992024,993705,993906,995435,998030,991464\n", + "Returned IDs: 996627,994751,996411,997729,990011,990012,990528,990743,996906,994663,995295,993725,992230,992680,992408,999740,994830,990392,993240,994730,995076,996011,992024,996147,993705,993906,995435,998030,991464,999302\n", "Ground Truth: 996627,994751,996411,993070,997729,990011,990012,990528,990743,996906,994663,995295,993725,992230,992680,992408,999740,999922,994830,990392,993240,994730,995076,996011,992024,996147,993705,993906,995435,998030\n", - "Recall: 0.9667\n", + "Recall: 0.9333\n", "\n", "Query ID: 996\n", - "Returned IDs: 995193,992521,998527,992046,992657,993021,995820,995640,997023,996581,999047,999394,991922,991102,992917,994508,990099,992311,999731,998075,993107,994185,994690,998255,990295,992595,997554,998910,993998,997320\n", + "Returned IDs: 995193,992521,998527,992046,992657,993021,995820,995640,997023,996581,999047,999394,991922,991102,992917,994508,992311,999731,998075,993107,994185,998255,999992,990295,992595,998910,993998,993379,998060,994954\n", "Ground Truth: 995193,992521,998527,992046,992657,993021,995820,995640,997023,996581,999047,999394,991922,991102,992917,994508,990099,992311,999731,998075,993107,994185,994690,998255,999992,990295,992595,997554,998910,993998\n", - "Recall: 0.9667\n", + "Recall: 0.9000\n", "\n", "Query ID: 997\n", - "Returned IDs: 991052,997675,996918,993805,996506,991061,997723,995401,991529,993014,996505,995521,996879,998397,999646,991747,993931,998593,994280,995080,993313,990537,994662,993600,996042,992521,995994,994095,992933,997411\n", + "Returned IDs: 991052,997675,996918,993805,996506,991061,991529,993014,996505,995521,996879,998397,999646,991747,993931,998593,994280,995080,993313,990537,996042,992521,995685,992558,998536,995110,999179,992457,992801,999139\n", "Ground Truth: 991052,997675,996918,993805,996506,991061,997723,995401,991529,993014,996505,995521,996879,998397,999646,991747,993931,998593,994280,995080,993313,990537,994662,993600,998021,996042,992521,995994,994095,992933\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 998\n", + "Returned IDs: 990011,996275,990484,998710,998108,994830,994951,991885,996906,991826,994858,996025,993064,993070,993335,996925,997529,996301,998435,991383,995002,995293,994351,998642,991382,991313,991390,995476,993568,990186\n", + "Ground Truth: 990011,996275,997733,990484,998710,998108,994830,994951,991885,996906,991826,994858,996025,993064,993070,993335,996925,997529,996301,998435,991383,995002,995293,994351,998642,991382,991313,991390,995476,993568\n", "Recall: 0.9667\n", "\n", "Query ID: 999\n", - "Returned IDs: 999162,992654,991977,999638,995560,998334,999387,994010,994672,998118,992718,991540,998142,995503,993018,998280,990930,997407,991432,991157,997320,990843,990926,999954,997816,993976,996262,992124,990487,992944\n", + "Returned IDs: 999162,992654,991977,999638,998334,999387,994010,994672,998118,992718,991540,998142,995503,993018,997407,991432,991157,997320,990843,990926,999954,997816,993976,996262,992124,990487,992944,999019,995019,990532\n", "Ground Truth: 999162,992654,991977,999638,995560,998334,999387,994010,994672,998118,992718,991540,998142,995503,993018,998567,998280,990930,997407,991432,991157,997320,990843,990926,999954,997816,993976,996262,992124,990487\n", - "Recall: 0.9667\n", + "Recall: 0.8667\n", "\n", - "Total query IDs with recall < 1.0: 542\n", - "IDs: [np.int64(1), np.int64(2), np.int64(4), np.int64(5), np.int64(6), np.int64(8), np.int64(10), np.int64(19), np.int64(21), np.int64(22), np.int64(23), np.int64(24), np.int64(26), np.int64(29), np.int64(31), np.int64(34), np.int64(35), np.int64(40), np.int64(41), np.int64(43), np.int64(45), np.int64(46), np.int64(48), np.int64(52), np.int64(53), np.int64(54), np.int64(56), np.int64(58), np.int64(59), np.int64(60), np.int64(61), np.int64(62), np.int64(63), np.int64(65), np.int64(66), np.int64(67), np.int64(69), np.int64(71), np.int64(74), np.int64(75), np.int64(79), np.int64(81), np.int64(84), np.int64(85), np.int64(88), np.int64(89), np.int64(91), np.int64(93), np.int64(100), np.int64(102), np.int64(104), np.int64(105), np.int64(107), np.int64(108), np.int64(109), np.int64(110), np.int64(112), np.int64(113), np.int64(116), np.int64(117), np.int64(119), np.int64(127), np.int64(128), np.int64(129), np.int64(131), np.int64(132), np.int64(133), np.int64(134), np.int64(135), np.int64(136), np.int64(138), np.int64(139), np.int64(144), np.int64(150), np.int64(151), np.int64(153), np.int64(154), np.int64(156), np.int64(158), np.int64(159), np.int64(160), np.int64(161), np.int64(162), np.int64(163), np.int64(170), np.int64(171), np.int64(172), np.int64(175), np.int64(178), np.int64(179), np.int64(181), np.int64(183), np.int64(184), np.int64(187), np.int64(188), np.int64(191), np.int64(192), np.int64(199), np.int64(200), np.int64(203), np.int64(204), np.int64(208), np.int64(211), np.int64(212), np.int64(213), np.int64(215), np.int64(216), np.int64(218), np.int64(220), np.int64(221), np.int64(223), np.int64(224), np.int64(227), np.int64(228), np.int64(229), np.int64(230), np.int64(231), np.int64(233), np.int64(236), np.int64(240), np.int64(242), np.int64(247), np.int64(249), np.int64(251), np.int64(258), np.int64(259), np.int64(261), np.int64(264), np.int64(266), np.int64(268), np.int64(269), np.int64(270), np.int64(272), np.int64(273), np.int64(275), np.int64(277), np.int64(278), np.int64(281), np.int64(282), np.int64(283), np.int64(284), np.int64(285), np.int64(286), np.int64(287), np.int64(289), np.int64(293), np.int64(294), np.int64(295), np.int64(296), np.int64(297), np.int64(302), np.int64(303), np.int64(304), np.int64(305), np.int64(306), np.int64(308), np.int64(309), np.int64(311), np.int64(312), np.int64(315), np.int64(318), np.int64(319), np.int64(320), np.int64(321), np.int64(322), np.int64(323), np.int64(324), np.int64(325), np.int64(326), np.int64(327), np.int64(328), np.int64(331), np.int64(334), np.int64(337), np.int64(342), np.int64(344), np.int64(345), np.int64(346), np.int64(347), np.int64(349), np.int64(352), np.int64(356), np.int64(357), np.int64(358), np.int64(359), np.int64(360), np.int64(361), np.int64(362), np.int64(366), np.int64(367), np.int64(368), np.int64(370), np.int64(371), np.int64(372), np.int64(376), np.int64(379), np.int64(381), np.int64(382), np.int64(383), np.int64(385), np.int64(389), np.int64(390), np.int64(393), np.int64(394), np.int64(395), np.int64(399), np.int64(402), np.int64(403), np.int64(406), np.int64(407), np.int64(409), np.int64(410), np.int64(411), np.int64(412), np.int64(413), np.int64(417), np.int64(420), np.int64(423), np.int64(424), np.int64(425), np.int64(427), np.int64(429), np.int64(430), np.int64(431), np.int64(432), np.int64(433), np.int64(434), np.int64(437), np.int64(439), np.int64(440), np.int64(441), np.int64(442), np.int64(447), np.int64(448), np.int64(449), np.int64(450), np.int64(451), np.int64(452), np.int64(453), np.int64(455), np.int64(456), np.int64(457), np.int64(458), np.int64(459), np.int64(460), np.int64(461), np.int64(462), np.int64(467), np.int64(469), np.int64(470), np.int64(473), np.int64(475), np.int64(477), np.int64(480), np.int64(481), np.int64(485), np.int64(488), np.int64(491), np.int64(494), np.int64(496), np.int64(498), np.int64(499), np.int64(500), np.int64(501), np.int64(503), np.int64(504), np.int64(505), np.int64(507), np.int64(508), np.int64(509), np.int64(510), np.int64(511), np.int64(512), np.int64(513), np.int64(514), np.int64(515), np.int64(517), np.int64(520), np.int64(521), np.int64(523), np.int64(524), np.int64(527), np.int64(529), np.int64(533), np.int64(534), np.int64(541), np.int64(543), np.int64(544), np.int64(545), np.int64(546), np.int64(548), np.int64(549), np.int64(551), np.int64(552), np.int64(557), np.int64(558), np.int64(561), np.int64(563), np.int64(564), np.int64(565), np.int64(567), np.int64(569), np.int64(571), np.int64(574), np.int64(575), np.int64(576), np.int64(577), np.int64(578), np.int64(581), np.int64(583), np.int64(584), np.int64(586), np.int64(588), np.int64(589), np.int64(591), np.int64(593), np.int64(595), np.int64(596), np.int64(600), np.int64(604), np.int64(605), np.int64(610), np.int64(611), np.int64(612), np.int64(613), np.int64(615), np.int64(616), np.int64(618), np.int64(619), np.int64(620), np.int64(621), np.int64(623), np.int64(625), np.int64(626), np.int64(628), np.int64(629), np.int64(630), np.int64(632), np.int64(633), np.int64(636), np.int64(638), np.int64(639), np.int64(640), np.int64(641), np.int64(643), np.int64(644), np.int64(645), np.int64(646), np.int64(647), np.int64(648), np.int64(649), np.int64(650), np.int64(652), np.int64(654), np.int64(655), np.int64(656), np.int64(658), np.int64(661), np.int64(665), np.int64(666), np.int64(668), np.int64(669), np.int64(670), np.int64(671), np.int64(673), np.int64(674), np.int64(676), np.int64(678), np.int64(680), np.int64(681), np.int64(682), np.int64(688), np.int64(689), np.int64(690), np.int64(691), np.int64(694), np.int64(696), np.int64(697), np.int64(700), np.int64(701), np.int64(704), np.int64(705), np.int64(706), np.int64(708), np.int64(710), np.int64(712), np.int64(713), np.int64(714), np.int64(718), np.int64(719), np.int64(720), np.int64(724), np.int64(726), np.int64(727), np.int64(728), np.int64(730), np.int64(731), np.int64(733), np.int64(734), np.int64(735), np.int64(739), np.int64(740), np.int64(742), np.int64(743), np.int64(744), np.int64(745), np.int64(748), np.int64(749), np.int64(751), np.int64(754), np.int64(758), np.int64(760), np.int64(761), np.int64(762), np.int64(763), np.int64(764), np.int64(765), np.int64(768), np.int64(770), np.int64(771), np.int64(772), np.int64(773), np.int64(774), np.int64(775), np.int64(776), np.int64(777), np.int64(778), np.int64(779), np.int64(781), np.int64(782), np.int64(783), np.int64(787), np.int64(789), np.int64(791), np.int64(792), np.int64(794), np.int64(796), np.int64(800), np.int64(801), np.int64(804), np.int64(805), np.int64(806), np.int64(809), np.int64(810), np.int64(816), np.int64(817), np.int64(818), np.int64(820), np.int64(821), np.int64(822), np.int64(823), np.int64(827), np.int64(828), np.int64(829), np.int64(830), np.int64(832), np.int64(833), np.int64(835), np.int64(839), np.int64(841), np.int64(842), np.int64(844), np.int64(846), np.int64(847), np.int64(848), np.int64(849), np.int64(854), np.int64(855), np.int64(856), np.int64(857), np.int64(860), np.int64(861), np.int64(862), np.int64(863), np.int64(866), np.int64(867), np.int64(869), np.int64(872), np.int64(873), np.int64(874), np.int64(875), np.int64(877), np.int64(878), np.int64(879), np.int64(880), np.int64(882), np.int64(883), np.int64(887), np.int64(889), np.int64(891), np.int64(894), np.int64(897), np.int64(898), np.int64(899), np.int64(901), np.int64(903), np.int64(904), np.int64(907), np.int64(912), np.int64(913), np.int64(916), np.int64(917), np.int64(918), np.int64(919), np.int64(922), np.int64(925), np.int64(926), np.int64(931), np.int64(932), np.int64(933), np.int64(935), np.int64(936), np.int64(938), np.int64(940), np.int64(942), np.int64(943), np.int64(945), np.int64(946), np.int64(948), np.int64(951), np.int64(954), np.int64(956), np.int64(960), np.int64(962), np.int64(963), np.int64(965), np.int64(966), np.int64(967), np.int64(969), np.int64(970), np.int64(972), np.int64(973), np.int64(976), np.int64(983), np.int64(987), np.int64(989), np.int64(991), np.int64(994), np.int64(995), np.int64(996), np.int64(997), np.int64(999)]\n", - "Average Recall across all 1000 queries: 0.9664\n" + "Total query IDs with recall < 1.0: 972\n", + "IDs: [np.int64(0), np.int64(1), np.int64(2), np.int64(3), np.int64(4), np.int64(5), np.int64(6), np.int64(7), np.int64(8), np.int64(9), np.int64(10), np.int64(11), np.int64(12), np.int64(13), np.int64(14), np.int64(15), np.int64(16), np.int64(17), np.int64(18), np.int64(19), np.int64(20), np.int64(21), np.int64(22), np.int64(23), np.int64(24), np.int64(25), np.int64(26), np.int64(27), np.int64(28), np.int64(29), np.int64(30), np.int64(31), np.int64(32), np.int64(33), np.int64(34), np.int64(35), np.int64(36), np.int64(37), np.int64(38), np.int64(39), np.int64(40), np.int64(41), np.int64(42), np.int64(43), np.int64(44), np.int64(45), np.int64(46), np.int64(47), np.int64(48), np.int64(49), np.int64(50), np.int64(51), np.int64(52), np.int64(53), np.int64(54), np.int64(55), np.int64(56), np.int64(57), np.int64(58), np.int64(59), np.int64(60), np.int64(61), np.int64(62), np.int64(63), np.int64(64), np.int64(65), np.int64(66), np.int64(67), np.int64(68), np.int64(69), np.int64(70), np.int64(71), np.int64(72), np.int64(73), np.int64(74), np.int64(75), np.int64(76), np.int64(77), np.int64(78), np.int64(79), np.int64(80), np.int64(81), np.int64(82), np.int64(83), np.int64(84), np.int64(85), np.int64(86), np.int64(87), np.int64(88), np.int64(89), np.int64(90), np.int64(91), np.int64(92), np.int64(93), np.int64(94), np.int64(95), np.int64(96), np.int64(97), np.int64(98), np.int64(99), np.int64(100), np.int64(101), np.int64(102), np.int64(103), np.int64(104), np.int64(105), np.int64(106), np.int64(107), np.int64(108), np.int64(109), np.int64(110), np.int64(111), np.int64(112), np.int64(113), np.int64(114), np.int64(115), np.int64(116), np.int64(117), np.int64(118), np.int64(119), np.int64(120), np.int64(121), np.int64(122), np.int64(123), np.int64(124), np.int64(125), np.int64(126), np.int64(127), np.int64(128), np.int64(129), np.int64(130), np.int64(131), np.int64(132), np.int64(133), np.int64(134), np.int64(135), np.int64(136), np.int64(137), np.int64(138), np.int64(139), np.int64(140), np.int64(141), np.int64(142), np.int64(143), np.int64(144), np.int64(145), np.int64(146), np.int64(147), np.int64(148), np.int64(149), np.int64(150), np.int64(151), np.int64(152), np.int64(153), np.int64(154), np.int64(155), np.int64(156), np.int64(157), np.int64(158), np.int64(159), np.int64(160), np.int64(161), np.int64(162), np.int64(163), np.int64(164), np.int64(165), np.int64(166), np.int64(167), np.int64(168), np.int64(169), np.int64(170), np.int64(171), np.int64(172), np.int64(173), np.int64(174), np.int64(175), np.int64(176), np.int64(177), np.int64(178), np.int64(179), np.int64(180), np.int64(181), np.int64(182), np.int64(183), np.int64(184), np.int64(185), np.int64(186), np.int64(187), np.int64(188), np.int64(189), np.int64(190), np.int64(192), np.int64(193), np.int64(194), np.int64(195), np.int64(196), np.int64(197), np.int64(198), np.int64(199), np.int64(200), np.int64(201), np.int64(202), np.int64(203), np.int64(204), np.int64(205), np.int64(206), np.int64(207), np.int64(208), np.int64(209), np.int64(210), np.int64(211), np.int64(212), np.int64(213), np.int64(214), np.int64(215), np.int64(216), np.int64(218), np.int64(219), np.int64(220), np.int64(221), np.int64(223), np.int64(224), np.int64(225), np.int64(226), np.int64(227), np.int64(228), np.int64(229), np.int64(230), np.int64(231), np.int64(232), np.int64(233), np.int64(234), np.int64(235), np.int64(236), np.int64(237), np.int64(238), np.int64(239), np.int64(240), np.int64(241), np.int64(242), np.int64(243), np.int64(244), np.int64(245), np.int64(246), np.int64(247), np.int64(248), np.int64(249), np.int64(250), np.int64(251), np.int64(252), np.int64(253), np.int64(254), np.int64(255), np.int64(256), np.int64(257), np.int64(258), np.int64(259), np.int64(260), np.int64(261), np.int64(262), np.int64(264), np.int64(266), np.int64(267), np.int64(268), np.int64(269), np.int64(270), np.int64(271), np.int64(272), np.int64(273), np.int64(274), np.int64(275), np.int64(277), np.int64(278), np.int64(279), np.int64(280), np.int64(281), np.int64(282), np.int64(283), np.int64(284), np.int64(285), np.int64(286), np.int64(287), np.int64(288), np.int64(289), np.int64(290), np.int64(291), np.int64(292), np.int64(293), np.int64(294), np.int64(295), np.int64(296), np.int64(297), np.int64(298), np.int64(299), np.int64(300), np.int64(301), np.int64(302), np.int64(303), np.int64(304), np.int64(305), np.int64(306), np.int64(307), np.int64(308), np.int64(309), np.int64(310), np.int64(311), np.int64(312), np.int64(313), np.int64(314), np.int64(315), np.int64(316), np.int64(317), np.int64(318), np.int64(319), np.int64(320), np.int64(321), np.int64(322), np.int64(323), np.int64(324), np.int64(325), np.int64(326), np.int64(327), np.int64(328), np.int64(329), np.int64(330), np.int64(331), np.int64(332), np.int64(333), np.int64(334), np.int64(335), np.int64(336), np.int64(337), np.int64(338), np.int64(339), np.int64(340), np.int64(341), np.int64(342), np.int64(343), np.int64(344), np.int64(345), np.int64(346), np.int64(347), np.int64(348), np.int64(349), np.int64(350), np.int64(351), np.int64(352), np.int64(353), np.int64(354), np.int64(355), np.int64(356), np.int64(357), np.int64(358), np.int64(359), np.int64(360), np.int64(361), np.int64(362), np.int64(363), np.int64(364), np.int64(365), np.int64(366), np.int64(367), np.int64(368), np.int64(369), np.int64(370), np.int64(371), np.int64(372), np.int64(373), np.int64(374), np.int64(375), np.int64(376), np.int64(377), np.int64(378), np.int64(379), np.int64(380), np.int64(381), np.int64(382), np.int64(383), np.int64(384), np.int64(385), np.int64(386), np.int64(387), np.int64(388), np.int64(389), np.int64(390), np.int64(391), np.int64(392), np.int64(393), np.int64(394), np.int64(395), np.int64(396), np.int64(397), np.int64(399), np.int64(400), np.int64(401), np.int64(402), np.int64(403), np.int64(404), np.int64(405), np.int64(406), np.int64(407), np.int64(408), np.int64(409), np.int64(410), np.int64(411), np.int64(412), np.int64(413), np.int64(415), np.int64(416), np.int64(417), np.int64(418), np.int64(419), np.int64(420), np.int64(421), np.int64(422), np.int64(423), np.int64(424), np.int64(425), np.int64(426), np.int64(427), np.int64(428), np.int64(429), np.int64(430), np.int64(431), np.int64(432), np.int64(433), np.int64(434), np.int64(435), np.int64(436), np.int64(437), np.int64(438), np.int64(439), np.int64(440), np.int64(441), np.int64(442), np.int64(443), np.int64(444), np.int64(445), np.int64(446), np.int64(447), np.int64(448), np.int64(449), np.int64(450), np.int64(451), np.int64(452), np.int64(453), np.int64(454), np.int64(455), np.int64(456), np.int64(457), np.int64(458), np.int64(459), np.int64(460), np.int64(461), np.int64(462), np.int64(463), np.int64(464), np.int64(465), np.int64(466), np.int64(467), np.int64(468), np.int64(469), np.int64(470), np.int64(471), np.int64(472), np.int64(473), np.int64(474), np.int64(475), np.int64(476), np.int64(477), np.int64(478), np.int64(479), np.int64(480), np.int64(481), np.int64(482), np.int64(483), np.int64(484), np.int64(485), np.int64(486), np.int64(487), np.int64(488), np.int64(489), np.int64(490), np.int64(491), np.int64(492), np.int64(493), np.int64(494), np.int64(495), np.int64(496), np.int64(497), np.int64(498), np.int64(499), np.int64(500), np.int64(501), np.int64(503), np.int64(504), np.int64(505), np.int64(507), np.int64(508), np.int64(509), np.int64(510), np.int64(511), np.int64(512), np.int64(513), np.int64(514), np.int64(515), np.int64(516), np.int64(517), np.int64(518), np.int64(519), np.int64(520), np.int64(521), np.int64(522), np.int64(523), np.int64(524), np.int64(525), np.int64(526), np.int64(527), np.int64(528), np.int64(529), np.int64(530), np.int64(531), np.int64(532), np.int64(533), np.int64(534), np.int64(535), np.int64(536), np.int64(537), np.int64(538), np.int64(539), np.int64(540), np.int64(541), np.int64(542), np.int64(543), np.int64(544), np.int64(545), np.int64(546), np.int64(547), np.int64(548), np.int64(549), np.int64(550), np.int64(551), np.int64(552), np.int64(553), np.int64(554), np.int64(556), np.int64(557), np.int64(558), np.int64(559), np.int64(560), np.int64(561), np.int64(562), np.int64(563), np.int64(564), np.int64(565), np.int64(566), np.int64(567), np.int64(568), np.int64(569), np.int64(570), np.int64(571), np.int64(572), np.int64(573), np.int64(574), np.int64(575), np.int64(576), np.int64(577), np.int64(578), np.int64(579), np.int64(580), np.int64(581), np.int64(582), np.int64(583), np.int64(584), np.int64(585), np.int64(586), np.int64(587), np.int64(588), np.int64(589), np.int64(590), np.int64(591), np.int64(592), np.int64(593), np.int64(594), np.int64(595), np.int64(596), np.int64(597), np.int64(598), np.int64(599), np.int64(600), np.int64(601), np.int64(603), np.int64(604), np.int64(605), np.int64(606), np.int64(607), np.int64(608), np.int64(609), np.int64(610), np.int64(611), np.int64(612), np.int64(613), np.int64(614), np.int64(615), np.int64(616), np.int64(617), np.int64(618), np.int64(619), np.int64(620), np.int64(621), np.int64(622), np.int64(623), np.int64(624), np.int64(625), np.int64(626), np.int64(627), np.int64(628), np.int64(629), np.int64(630), np.int64(631), np.int64(632), np.int64(633), np.int64(634), np.int64(635), np.int64(636), np.int64(637), np.int64(638), np.int64(639), np.int64(640), np.int64(641), np.int64(642), np.int64(643), np.int64(644), np.int64(645), np.int64(646), np.int64(647), np.int64(648), np.int64(649), np.int64(650), np.int64(651), np.int64(652), np.int64(653), np.int64(654), np.int64(655), np.int64(656), np.int64(657), np.int64(658), np.int64(660), np.int64(661), np.int64(663), np.int64(664), np.int64(665), np.int64(666), np.int64(667), np.int64(668), np.int64(669), np.int64(670), np.int64(671), np.int64(672), np.int64(673), np.int64(674), np.int64(675), np.int64(676), np.int64(678), np.int64(679), np.int64(680), np.int64(681), np.int64(682), np.int64(683), np.int64(684), np.int64(685), np.int64(686), np.int64(687), np.int64(688), np.int64(689), np.int64(690), np.int64(691), np.int64(692), np.int64(693), np.int64(694), np.int64(696), np.int64(697), np.int64(699), np.int64(700), np.int64(701), np.int64(702), np.int64(703), np.int64(704), np.int64(705), np.int64(706), np.int64(707), np.int64(708), np.int64(709), np.int64(710), np.int64(711), np.int64(712), np.int64(713), np.int64(714), np.int64(715), np.int64(717), np.int64(718), np.int64(719), np.int64(720), np.int64(721), np.int64(722), np.int64(723), np.int64(724), np.int64(725), np.int64(726), np.int64(727), np.int64(728), np.int64(729), np.int64(730), np.int64(731), np.int64(732), np.int64(733), np.int64(734), np.int64(735), np.int64(736), np.int64(737), np.int64(738), np.int64(739), np.int64(740), np.int64(741), np.int64(742), np.int64(743), np.int64(744), np.int64(745), np.int64(746), np.int64(747), np.int64(748), np.int64(749), np.int64(750), np.int64(751), np.int64(752), np.int64(753), np.int64(754), np.int64(755), np.int64(756), np.int64(757), np.int64(758), np.int64(759), np.int64(760), np.int64(761), np.int64(762), np.int64(763), np.int64(764), np.int64(765), np.int64(766), np.int64(768), np.int64(769), np.int64(770), np.int64(771), np.int64(772), np.int64(773), np.int64(774), np.int64(775), np.int64(776), np.int64(777), np.int64(778), np.int64(779), np.int64(780), np.int64(781), np.int64(782), np.int64(783), np.int64(784), np.int64(786), np.int64(787), np.int64(788), np.int64(789), np.int64(790), np.int64(791), np.int64(792), np.int64(793), np.int64(794), np.int64(795), np.int64(796), np.int64(797), np.int64(798), np.int64(799), np.int64(800), np.int64(801), np.int64(802), np.int64(803), np.int64(804), np.int64(805), np.int64(806), np.int64(807), np.int64(808), np.int64(809), np.int64(810), np.int64(811), np.int64(812), np.int64(813), np.int64(815), np.int64(816), np.int64(817), np.int64(818), np.int64(819), np.int64(820), np.int64(821), np.int64(822), np.int64(823), np.int64(824), np.int64(825), np.int64(826), np.int64(827), np.int64(828), np.int64(829), np.int64(830), np.int64(831), np.int64(832), np.int64(833), np.int64(834), np.int64(835), np.int64(836), np.int64(837), np.int64(838), np.int64(839), np.int64(841), np.int64(842), np.int64(843), np.int64(844), np.int64(845), np.int64(846), np.int64(847), np.int64(848), np.int64(849), np.int64(850), np.int64(851), np.int64(852), np.int64(853), np.int64(854), np.int64(855), np.int64(856), np.int64(857), np.int64(858), np.int64(859), np.int64(860), np.int64(861), np.int64(862), np.int64(863), np.int64(864), np.int64(865), np.int64(866), np.int64(867), np.int64(868), np.int64(869), np.int64(870), np.int64(871), np.int64(872), np.int64(873), np.int64(874), np.int64(875), np.int64(876), np.int64(877), np.int64(878), np.int64(879), np.int64(880), np.int64(882), np.int64(883), np.int64(884), np.int64(885), np.int64(886), np.int64(887), np.int64(888), np.int64(889), np.int64(890), np.int64(891), np.int64(892), np.int64(893), np.int64(894), np.int64(895), np.int64(896), np.int64(897), np.int64(898), np.int64(899), np.int64(900), np.int64(901), np.int64(902), np.int64(903), np.int64(904), np.int64(905), np.int64(906), np.int64(907), np.int64(908), np.int64(909), np.int64(911), np.int64(912), np.int64(913), np.int64(914), np.int64(915), np.int64(916), np.int64(917), np.int64(918), np.int64(919), np.int64(920), np.int64(921), np.int64(922), np.int64(923), np.int64(924), np.int64(926), np.int64(927), np.int64(928), np.int64(929), np.int64(930), np.int64(931), np.int64(932), np.int64(933), np.int64(934), np.int64(935), np.int64(936), np.int64(937), np.int64(938), np.int64(940), np.int64(941), np.int64(942), np.int64(943), np.int64(944), np.int64(945), np.int64(946), np.int64(947), np.int64(948), np.int64(949), np.int64(950), np.int64(951), np.int64(952), np.int64(953), np.int64(954), np.int64(955), np.int64(956), np.int64(957), np.int64(958), np.int64(959), np.int64(960), np.int64(961), np.int64(962), np.int64(963), np.int64(964), np.int64(965), np.int64(966), np.int64(967), np.int64(968), np.int64(969), np.int64(970), np.int64(971), np.int64(972), np.int64(973), np.int64(975), np.int64(976), np.int64(977), np.int64(978), np.int64(980), np.int64(981), np.int64(982), np.int64(983), np.int64(984), np.int64(985), np.int64(986), np.int64(987), np.int64(988), np.int64(989), np.int64(990), np.int64(991), np.int64(992), np.int64(993), np.int64(994), np.int64(995), np.int64(996), np.int64(997), np.int64(998), np.int64(999)]\n", + "Average Recall across all 1000 queries: 0.7946\n" ] } ], @@ -8009,23 +10124,23 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'name': 'test_1M_numfilter_latest_master_stability_vaib2',\n", + "{'name': 'test_1M_vaib_labelfilter_0503_1_adup5',\n", " 'space_type': 'cosine',\n", " 'dimension': 768,\n", " 'sparse_dim': 0,\n", " 'is_hybrid': False,\n", - " 'count': 200000,\n", + " 'count': 1000000,\n", " 'precision': 'int16',\n", " 'M': 16}" ] }, - "execution_count": 10, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -8093,23 +10208,23 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'name': 'test_1M_vaib_reupsertcheck',\n", + "{'name': 'test_1M_vaib_labelfilter_0503_1_adup1',\n", " 'space_type': 'cosine',\n", " 'dimension': 768,\n", " 'sparse_dim': 0,\n", " 'is_hybrid': False,\n", - " 'count': 300000,\n", + " 'count': 200000,\n", " 'precision': 'int16',\n", " 'M': 16}" ] }, - "execution_count": 8, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -8120,7 +10235,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -8207,49 +10322,928 @@ "Upserted 806 vectors, total so far: 62392\n", "Upserted 791 vectors, total so far: 63183\n", "Upserted 787 vectors, total so far: 63970\n", - "Upserted 794 vectors, total so far: 64764\n" - ] - }, - { - "ename": "ConnectionError", - "evalue": "HTTPConnectionPool(host='148.113.54.173', port=8080): Max retries exceeded with url: /api/v1/index/test_1M_vaib_reupsertcheck_new1/vector/insert (Caused by NewConnectionError(\"HTTPConnection(host='148.113.54.173', port=8080): Failed to establish a new connection: [Errno 111] Connection refused\"))", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mConnectionRefusedError\u001b[39m Traceback (most recent call last)", - "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connection.py:204\u001b[39m, in \u001b[36mHTTPConnection._new_conn\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 203\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m204\u001b[39m sock = \u001b[43mconnection\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcreate_connection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 205\u001b[39m \u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_dns_host\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mport\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 206\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 207\u001b[39m \u001b[43m \u001b[49m\u001b[43msource_address\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msource_address\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 208\u001b[39m \u001b[43m \u001b[49m\u001b[43msocket_options\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msocket_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 209\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 210\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m socket.gaierror \u001b[38;5;28;01mas\u001b[39;00m e:\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/util/connection.py:85\u001b[39m, in \u001b[36mcreate_connection\u001b[39m\u001b[34m(address, timeout, source_address, socket_options)\u001b[39m\n\u001b[32m 84\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m85\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[32m 86\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m 87\u001b[39m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/util/connection.py:73\u001b[39m, in \u001b[36mcreate_connection\u001b[39m\u001b[34m(address, timeout, source_address, socket_options)\u001b[39m\n\u001b[32m 72\u001b[39m sock.bind(source_address)\n\u001b[32m---> \u001b[39m\u001b[32m73\u001b[39m \u001b[43msock\u001b[49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43msa\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 74\u001b[39m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n", - "\u001b[31mConnectionRefusedError\u001b[39m: [Errno 111] Connection refused", - "\nThe above exception was the direct cause of the following exception:\n", - "\u001b[31mNewConnectionError\u001b[39m Traceback (most recent call last)", - "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connectionpool.py:787\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 786\u001b[39m \u001b[38;5;66;03m# Make the request on the HTTPConnection object\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m787\u001b[39m response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 788\u001b[39m \u001b[43m \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 789\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 790\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 791\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 792\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 793\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 794\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 795\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 796\u001b[39m \u001b[43m \u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 797\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 798\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 799\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 800\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 802\u001b[39m \u001b[38;5;66;03m# Everything went great!\u001b[39;00m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connectionpool.py:493\u001b[39m, in \u001b[36mHTTPConnectionPool._make_request\u001b[39m\u001b[34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 492\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m493\u001b[39m \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 494\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 495\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 496\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 497\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 498\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 499\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 500\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 501\u001b[39m \u001b[43m \u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[43m=\u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 502\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 504\u001b[39m \u001b[38;5;66;03m# We are swallowing BrokenPipeError (errno.EPIPE) since the server is\u001b[39;00m\n\u001b[32m 505\u001b[39m \u001b[38;5;66;03m# legitimately able to close the connection after sending a valid response.\u001b[39;00m\n\u001b[32m 506\u001b[39m \u001b[38;5;66;03m# With this behaviour, the received response is still readable.\u001b[39;00m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connection.py:500\u001b[39m, in \u001b[36mHTTPConnection.request\u001b[39m\u001b[34m(self, method, url, body, headers, chunked, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 499\u001b[39m \u001b[38;5;28mself\u001b[39m.putheader(header, value)\n\u001b[32m--> \u001b[39m\u001b[32m500\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mendheaders\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 502\u001b[39m \u001b[38;5;66;03m# If we're given a body we start sending that in chunks.\u001b[39;00m\n", - "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/http/client.py:1298\u001b[39m, in \u001b[36mHTTPConnection.endheaders\u001b[39m\u001b[34m(self, message_body, encode_chunked)\u001b[39m\n\u001b[32m 1297\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m CannotSendHeader()\n\u001b[32m-> \u001b[39m\u001b[32m1298\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_send_output\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/http/client.py:1058\u001b[39m, in \u001b[36mHTTPConnection._send_output\u001b[39m\u001b[34m(self, message_body, encode_chunked)\u001b[39m\n\u001b[32m 1057\u001b[39m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m._buffer[:]\n\u001b[32m-> \u001b[39m\u001b[32m1058\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1060\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m message_body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 1061\u001b[39m \n\u001b[32m 1062\u001b[39m \u001b[38;5;66;03m# create a consistent interface to message_body\u001b[39;00m\n", - "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/http/client.py:996\u001b[39m, in \u001b[36mHTTPConnection.send\u001b[39m\u001b[34m(self, data)\u001b[39m\n\u001b[32m 995\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.auto_open:\n\u001b[32m--> \u001b[39m\u001b[32m996\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 997\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connection.py:331\u001b[39m, in \u001b[36mHTTPConnection.connect\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 330\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mconnect\u001b[39m(\u001b[38;5;28mself\u001b[39m) -> \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m331\u001b[39m \u001b[38;5;28mself\u001b[39m.sock = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_new_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 332\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._tunnel_host:\n\u001b[32m 333\u001b[39m \u001b[38;5;66;03m# If we're tunneling it means we're connected to our proxy.\u001b[39;00m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connection.py:219\u001b[39m, in \u001b[36mHTTPConnection._new_conn\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 218\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m--> \u001b[39m\u001b[32m219\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m NewConnectionError(\n\u001b[32m 220\u001b[39m \u001b[38;5;28mself\u001b[39m, \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mFailed to establish a new connection: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m 221\u001b[39m ) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01me\u001b[39;00m\n\u001b[32m 223\u001b[39m sys.audit(\u001b[33m\"\u001b[39m\u001b[33mhttp.client.connect\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28mself\u001b[39m, \u001b[38;5;28mself\u001b[39m.host, \u001b[38;5;28mself\u001b[39m.port)\n", - "\u001b[31mNewConnectionError\u001b[39m: HTTPConnection(host='148.113.54.173', port=8080): Failed to establish a new connection: [Errno 111] Connection refused", - "\nThe above exception was the direct cause of the following exception:\n", - "\u001b[31mMaxRetryError\u001b[39m Traceback (most recent call last)", - "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/requests/adapters.py:644\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 643\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m644\u001b[39m resp = \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 645\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 646\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 647\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 648\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 649\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 650\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 651\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 652\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 653\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 654\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 655\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 656\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 658\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connectionpool.py:871\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 868\u001b[39m log.warning(\n\u001b[32m 869\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mRetrying (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m) after connection broken by \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m\"\u001b[39m, retries, err, url\n\u001b[32m 870\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m871\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 872\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 873\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 874\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 875\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 876\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 877\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 878\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 879\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 880\u001b[39m \u001b[43m \u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 881\u001b[39m \u001b[43m \u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 882\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 883\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 884\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 885\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 886\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 887\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 889\u001b[39m \u001b[38;5;66;03m# Handle redirect?\u001b[39;00m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connectionpool.py:871\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 868\u001b[39m log.warning(\n\u001b[32m 869\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mRetrying (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m) after connection broken by \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m\"\u001b[39m, retries, err, url\n\u001b[32m 870\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m871\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 872\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 873\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 874\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 875\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 876\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 877\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 878\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 879\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 880\u001b[39m \u001b[43m \u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 881\u001b[39m \u001b[43m \u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 882\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 883\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 884\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 885\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 886\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 887\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 889\u001b[39m \u001b[38;5;66;03m# Handle redirect?\u001b[39;00m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connectionpool.py:871\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 868\u001b[39m log.warning(\n\u001b[32m 869\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mRetrying (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m) after connection broken by \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m\"\u001b[39m, retries, err, url\n\u001b[32m 870\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m871\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 872\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 873\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 874\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 875\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 876\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 877\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 878\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 879\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 880\u001b[39m \u001b[43m \u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 881\u001b[39m \u001b[43m \u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 882\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 883\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 884\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 885\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 886\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 887\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 889\u001b[39m \u001b[38;5;66;03m# Handle redirect?\u001b[39;00m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connectionpool.py:841\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 839\u001b[39m new_e = ProtocolError(\u001b[33m\"\u001b[39m\u001b[33mConnection aborted.\u001b[39m\u001b[33m\"\u001b[39m, new_e)\n\u001b[32m--> \u001b[39m\u001b[32m841\u001b[39m retries = \u001b[43mretries\u001b[49m\u001b[43m.\u001b[49m\u001b[43mincrement\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 842\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnew_e\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_pool\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[43m=\u001b[49m\u001b[43msys\u001b[49m\u001b[43m.\u001b[49m\u001b[43mexc_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[32;43m2\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[32m 843\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 844\u001b[39m retries.sleep()\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/util/retry.py:535\u001b[39m, in \u001b[36mRetry.increment\u001b[39m\u001b[34m(self, method, url, response, error, _pool, _stacktrace)\u001b[39m\n\u001b[32m 534\u001b[39m reason = error \u001b[38;5;129;01mor\u001b[39;00m ResponseError(cause)\n\u001b[32m--> \u001b[39m\u001b[32m535\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m MaxRetryError(_pool, url, reason) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mreason\u001b[39;00m \u001b[38;5;66;03m# type: ignore[arg-type]\u001b[39;00m\n\u001b[32m 537\u001b[39m log.debug(\u001b[33m\"\u001b[39m\u001b[33mIncremented Retry for (url=\u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m): \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m\"\u001b[39m, url, new_retry)\n", - "\u001b[31mMaxRetryError\u001b[39m: HTTPConnectionPool(host='148.113.54.173', port=8080): Max retries exceeded with url: /api/v1/index/test_1M_vaib_reupsertcheck_new1/vector/insert (Caused by NewConnectionError(\"HTTPConnection(host='148.113.54.173', port=8080): Failed to establish a new connection: [Errno 111] Connection refused\"))", - "\nDuring handling of the above exception, another exception occurred:\n", - "\u001b[31mConnectionError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[8]\u001b[39m\u001b[32m, line 52\u001b[39m\n\u001b[32m 42\u001b[39m vector_data = {\n\u001b[32m 43\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mid\u001b[39m\u001b[33m\"\u001b[39m: \u001b[38;5;28mstr\u001b[39m(row[\u001b[33m\"\u001b[39m\u001b[33mid\u001b[39m\u001b[33m\"\u001b[39m]),\n\u001b[32m 44\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mvector\u001b[39m\u001b[33m\"\u001b[39m: row[\u001b[33m\"\u001b[39m\u001b[33memb\u001b[39m\u001b[33m\"\u001b[39m],\n\u001b[32m (...)\u001b[39m\u001b[32m 48\u001b[39m }\n\u001b[32m 49\u001b[39m }\n\u001b[32m 50\u001b[39m batch_vectors.append(vector_data)\n\u001b[32m---> \u001b[39m\u001b[32m52\u001b[39m \u001b[43mindex\u001b[49m\u001b[43m.\u001b[49m\u001b[43mupsert\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch_vectors\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 53\u001b[39m total_inserted += \u001b[38;5;28mlen\u001b[39m(batch_vectors)\n\u001b[32m 54\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mUpserted \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(batch_vectors)\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m vectors, total so far: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtotal_inserted\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/endee/index.py:338\u001b[39m, in \u001b[36mIndex.upsert\u001b[39m\u001b[34m(self, input_array)\u001b[39m\n\u001b[32m 335\u001b[39m http_client = \u001b[38;5;28mself\u001b[39m._get_session_client()\n\u001b[32m 337\u001b[39m \u001b[38;5;66;03m# Sending the batch to the server\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m338\u001b[39m response = \u001b[43mhttp_client\u001b[49m\u001b[43m.\u001b[49m\u001b[43mpost\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 339\u001b[39m \u001b[43m \u001b[49m\u001b[33;43mf\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[33;43m/index/\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[33;43m/vector/insert\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 340\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 341\u001b[39m \u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m=\u001b[49m\u001b[43mserialized_data\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 342\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 344\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m response.status_code != \u001b[32m200\u001b[39m:\n\u001b[32m 345\u001b[39m raise_exception(response.status_code, response.text)\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/requests/sessions.py:637\u001b[39m, in \u001b[36mSession.post\u001b[39m\u001b[34m(self, url, data, json, **kwargs)\u001b[39m\n\u001b[32m 626\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mpost\u001b[39m(\u001b[38;5;28mself\u001b[39m, url, data=\u001b[38;5;28;01mNone\u001b[39;00m, json=\u001b[38;5;28;01mNone\u001b[39;00m, **kwargs):\n\u001b[32m 627\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33mr\u001b[39m\u001b[33;03m\"\"\"Sends a POST request. Returns :class:`Response` object.\u001b[39;00m\n\u001b[32m 628\u001b[39m \n\u001b[32m 629\u001b[39m \u001b[33;03m :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 634\u001b[39m \u001b[33;03m :rtype: requests.Response\u001b[39;00m\n\u001b[32m 635\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m637\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mPOST\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[43m=\u001b[49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/requests/sessions.py:589\u001b[39m, in \u001b[36mSession.request\u001b[39m\u001b[34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[39m\n\u001b[32m 584\u001b[39m send_kwargs = {\n\u001b[32m 585\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mtimeout\u001b[39m\u001b[33m\"\u001b[39m: timeout,\n\u001b[32m 586\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mallow_redirects\u001b[39m\u001b[33m\"\u001b[39m: allow_redirects,\n\u001b[32m 587\u001b[39m }\n\u001b[32m 588\u001b[39m send_kwargs.update(settings)\n\u001b[32m--> \u001b[39m\u001b[32m589\u001b[39m resp = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 591\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/requests/sessions.py:703\u001b[39m, in \u001b[36mSession.send\u001b[39m\u001b[34m(self, request, **kwargs)\u001b[39m\n\u001b[32m 700\u001b[39m start = preferred_clock()\n\u001b[32m 702\u001b[39m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m703\u001b[39m r = \u001b[43madapter\u001b[49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 705\u001b[39m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[32m 706\u001b[39m elapsed = preferred_clock() - start\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/requests/adapters.py:677\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 673\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e.reason, _SSLError):\n\u001b[32m 674\u001b[39m \u001b[38;5;66;03m# This branch is for urllib3 v1.22 and later.\u001b[39;00m\n\u001b[32m 675\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m SSLError(e, request=request)\n\u001b[32m--> \u001b[39m\u001b[32m677\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request=request)\n\u001b[32m 679\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m ClosedPoolError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 680\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request=request)\n", - "\u001b[31mConnectionError\u001b[39m: HTTPConnectionPool(host='148.113.54.173', port=8080): Max retries exceeded with url: /api/v1/index/test_1M_vaib_reupsertcheck_new1/vector/insert (Caused by NewConnectionError(\"HTTPConnection(host='148.113.54.173', port=8080): Failed to establish a new connection: [Errno 111] Connection refused\"))" + "Upserted 794 vectors, total so far: 64764\n", + "Upserted 803 vectors, total so far: 65567\n", + "Upserted 798 vectors, total so far: 66365\n", + "Upserted 813 vectors, total so far: 67178\n", + "Upserted 794 vectors, total so far: 67972\n", + "Upserted 800 vectors, total so far: 68772\n", + "Upserted 802 vectors, total so far: 69574\n", + "Upserted 798 vectors, total so far: 70372\n", + "Upserted 791 vectors, total so far: 71163\n", + "Upserted 794 vectors, total so far: 71957\n", + "Upserted 819 vectors, total so far: 72776\n", + "Upserted 795 vectors, total so far: 73571\n", + "Upserted 829 vectors, total so far: 74400\n", + "Upserted 807 vectors, total so far: 75207\n", + "Upserted 810 vectors, total so far: 76017\n", + "Upserted 795 vectors, total so far: 76812\n", + "Upserted 789 vectors, total so far: 77601\n", + "Upserted 799 vectors, total so far: 78400\n", + "Upserted 793 vectors, total so far: 79193\n", + "Upserted 813 vectors, total so far: 80006\n", + "Upserted 780 vectors, total so far: 80786\n", + "Upserted 781 vectors, total so far: 81567\n", + "Upserted 775 vectors, total so far: 82342\n", + "Upserted 799 vectors, total so far: 83141\n", + "Upserted 813 vectors, total so far: 83954\n", + "Upserted 800 vectors, total so far: 84754\n", + "Upserted 803 vectors, total so far: 85557\n", + "Upserted 795 vectors, total so far: 86352\n", + "Upserted 778 vectors, total so far: 87130\n", + "Upserted 817 vectors, total so far: 87947\n", + "Upserted 800 vectors, total so far: 88747\n", + "Upserted 798 vectors, total so far: 89545\n", + "Upserted 797 vectors, total so far: 90342\n", + "Upserted 811 vectors, total so far: 91153\n", + "Upserted 798 vectors, total so far: 91951\n", + "Upserted 788 vectors, total so far: 92739\n", + "Upserted 805 vectors, total so far: 93544\n", + "Upserted 811 vectors, total so far: 94355\n", + "Upserted 810 vectors, total so far: 95165\n", + "Upserted 791 vectors, total so far: 95956\n", + "Upserted 809 vectors, total so far: 96765\n", + "Upserted 797 vectors, total so far: 97562\n", + "Upserted 812 vectors, total so far: 98374\n", + "Upserted 795 vectors, total so far: 99169\n", + "Upserted 774 vectors, total so far: 99943\n", + "Upserted 800 vectors, total so far: 100743\n", + "Upserted 800 vectors, total so far: 101543\n", + "Upserted 799 vectors, total so far: 102342\n", + "Upserted 771 vectors, total so far: 103113\n", + "Upserted 796 vectors, total so far: 103909\n", + "Upserted 810 vectors, total so far: 104719\n", + "Upserted 817 vectors, total so far: 105536\n", + "Upserted 819 vectors, total so far: 106355\n", + "Upserted 776 vectors, total so far: 107131\n", + "Upserted 809 vectors, total so far: 107940\n", + "Upserted 801 vectors, total so far: 108741\n", + "Upserted 806 vectors, total so far: 109547\n", + "Upserted 821 vectors, total so far: 110368\n", + "Upserted 810 vectors, total so far: 111178\n", + "Upserted 805 vectors, total so far: 111983\n", + "Upserted 789 vectors, total so far: 112772\n", + "Upserted 809 vectors, total so far: 113581\n", + "Upserted 806 vectors, total so far: 114387\n", + "Upserted 810 vectors, total so far: 115197\n", + "Upserted 818 vectors, total so far: 116015\n", + "Upserted 808 vectors, total so far: 116823\n", + "Upserted 796 vectors, total so far: 117619\n", + "Upserted 809 vectors, total so far: 118428\n", + "Upserted 808 vectors, total so far: 119236\n", + "Upserted 800 vectors, total so far: 120036\n", + "Upserted 802 vectors, total so far: 120838\n", + "Upserted 787 vectors, total so far: 121625\n", + "Upserted 794 vectors, total so far: 122419\n", + "Upserted 805 vectors, total so far: 123224\n", + "Upserted 815 vectors, total so far: 124039\n", + "Upserted 801 vectors, total so far: 124840\n", + "Upserted 789 vectors, total so far: 125629\n", + "Upserted 795 vectors, total so far: 126424\n", + "Upserted 788 vectors, total so far: 127212\n", + "Upserted 796 vectors, total so far: 128008\n", + "Upserted 819 vectors, total so far: 128827\n", + "Upserted 799 vectors, total so far: 129626\n", + "Upserted 797 vectors, total so far: 130423\n", + "Upserted 817 vectors, total so far: 131240\n", + "Upserted 796 vectors, total so far: 132036\n", + "Upserted 822 vectors, total so far: 132858\n", + "Upserted 794 vectors, total so far: 133652\n", + "Upserted 806 vectors, total so far: 134458\n", + "Upserted 785 vectors, total so far: 135243\n", + "Upserted 788 vectors, total so far: 136031\n", + "Upserted 801 vectors, total so far: 136832\n", + "Upserted 808 vectors, total so far: 137640\n", + "Upserted 814 vectors, total so far: 138454\n", + "Upserted 797 vectors, total so far: 139251\n", + "Upserted 778 vectors, total so far: 140029\n", + "Upserted 799 vectors, total so far: 140828\n", + "Upserted 794 vectors, total so far: 141622\n", + "Upserted 782 vectors, total so far: 142404\n", + "Upserted 816 vectors, total so far: 143220\n", + "Upserted 797 vectors, total so far: 144017\n", + "Upserted 785 vectors, total so far: 144802\n", + "Upserted 781 vectors, total so far: 145583\n", + "Upserted 810 vectors, total so far: 146393\n", + "Upserted 810 vectors, total so far: 147203\n", + "Upserted 796 vectors, total so far: 147999\n", + "Upserted 813 vectors, total so far: 148812\n", + "Upserted 822 vectors, total so far: 149634\n", + "Upserted 795 vectors, total so far: 150429\n", + "Upserted 782 vectors, total so far: 151211\n", + "Upserted 795 vectors, total so far: 152006\n", + "Upserted 802 vectors, total so far: 152808\n", + "Upserted 773 vectors, total so far: 153581\n", + "Upserted 785 vectors, total so far: 154366\n", + "Upserted 796 vectors, total so far: 155162\n", + "Upserted 797 vectors, total so far: 155959\n", + "Upserted 794 vectors, total so far: 156753\n", + "Upserted 793 vectors, total so far: 157546\n", + "Upserted 794 vectors, total so far: 158340\n", + "Upserted 814 vectors, total so far: 159154\n", + "Upserted 782 vectors, total so far: 159936\n", + "Upserted 818 vectors, total so far: 160754\n", + "Upserted 787 vectors, total so far: 161541\n", + "Upserted 789 vectors, total so far: 162330\n", + "Upserted 794 vectors, total so far: 163124\n", + "Upserted 822 vectors, total so far: 163946\n", + "Upserted 813 vectors, total so far: 164759\n", + "Upserted 784 vectors, total so far: 165543\n", + "Upserted 810 vectors, total so far: 166353\n", + "Upserted 792 vectors, total so far: 167145\n", + "Upserted 808 vectors, total so far: 167953\n", + "Upserted 809 vectors, total so far: 168762\n", + "Upserted 782 vectors, total so far: 169544\n", + "Upserted 831 vectors, total so far: 170375\n", + "Upserted 808 vectors, total so far: 171183\n", + "Upserted 808 vectors, total so far: 171991\n", + "Upserted 810 vectors, total so far: 172801\n", + "Upserted 824 vectors, total so far: 173625\n", + "Upserted 809 vectors, total so far: 174434\n", + "Upserted 795 vectors, total so far: 175229\n", + "Upserted 814 vectors, total so far: 176043\n", + "Upserted 796 vectors, total so far: 176839\n", + "Upserted 802 vectors, total so far: 177641\n", + "Upserted 801 vectors, total so far: 178442\n", + "Upserted 762 vectors, total so far: 179204\n", + "Upserted 802 vectors, total so far: 180006\n", + "Upserted 806 vectors, total so far: 180812\n", + "Upserted 801 vectors, total so far: 181613\n", + "Upserted 800 vectors, total so far: 182413\n", + "Upserted 812 vectors, total so far: 183225\n", + "Upserted 788 vectors, total so far: 184013\n", + "Upserted 799 vectors, total so far: 184812\n", + "Upserted 802 vectors, total so far: 185614\n", + "Upserted 792 vectors, total so far: 186406\n", + "Upserted 789 vectors, total so far: 187195\n", + "Upserted 807 vectors, total so far: 188002\n", + "Upserted 801 vectors, total so far: 188803\n", + "Upserted 792 vectors, total so far: 189595\n", + "Upserted 811 vectors, total so far: 190406\n", + "Upserted 801 vectors, total so far: 191207\n", + "Upserted 799 vectors, total so far: 192006\n", + "Upserted 782 vectors, total so far: 192788\n", + "Upserted 824 vectors, total so far: 193612\n", + "Upserted 797 vectors, total so far: 194409\n", + "Upserted 813 vectors, total so far: 195222\n", + "Upserted 789 vectors, total so far: 196011\n", + "Upserted 803 vectors, total so far: 196814\n", + "Upserted 806 vectors, total so far: 197620\n", + "Upserted 815 vectors, total so far: 198435\n", + "Upserted 781 vectors, total so far: 199216\n", + "Upserted 817 vectors, total so far: 200033\n", + "Upserted 812 vectors, total so far: 200845\n", + "Upserted 796 vectors, total so far: 201641\n", + "Upserted 795 vectors, total so far: 202436\n", + "Upserted 795 vectors, total so far: 203231\n", + "Upserted 771 vectors, total so far: 204002\n", + "Upserted 793 vectors, total so far: 204795\n", + "Upserted 811 vectors, total so far: 205606\n", + "Upserted 805 vectors, total so far: 206411\n", + "Upserted 776 vectors, total so far: 207187\n", + "Upserted 809 vectors, total so far: 207996\n", + "Upserted 790 vectors, total so far: 208786\n", + "Upserted 793 vectors, total so far: 209579\n", + "Upserted 818 vectors, total so far: 210397\n", + "Upserted 791 vectors, total so far: 211188\n", + "Upserted 790 vectors, total so far: 211978\n", + "Upserted 782 vectors, total so far: 212760\n", + "Upserted 821 vectors, total so far: 213581\n", + "Upserted 805 vectors, total so far: 214386\n", + "Upserted 815 vectors, total so far: 215201\n", + "Upserted 798 vectors, total so far: 215999\n", + "Upserted 775 vectors, total so far: 216774\n", + "Upserted 808 vectors, total so far: 217582\n", + "Upserted 810 vectors, total so far: 218392\n", + "Upserted 810 vectors, total so far: 219202\n", + "Upserted 805 vectors, total so far: 220007\n", + "Upserted 805 vectors, total so far: 220812\n", + "Upserted 781 vectors, total so far: 221593\n", + "Upserted 804 vectors, total so far: 222397\n", + "Upserted 802 vectors, total so far: 223199\n", + "Upserted 790 vectors, total so far: 223989\n", + "Upserted 824 vectors, total so far: 224813\n", + "Upserted 794 vectors, total so far: 225607\n", + "Upserted 791 vectors, total so far: 226398\n", + "Upserted 804 vectors, total so far: 227202\n", + "Upserted 814 vectors, total so far: 228016\n", + "Upserted 803 vectors, total so far: 228819\n", + "Upserted 798 vectors, total so far: 229617\n", + "Upserted 807 vectors, total so far: 230424\n", + "Upserted 822 vectors, total so far: 231246\n", + "Upserted 821 vectors, total so far: 232067\n", + "Upserted 806 vectors, total so far: 232873\n", + "Upserted 812 vectors, total so far: 233685\n", + "Upserted 803 vectors, total so far: 234488\n", + "Upserted 809 vectors, total so far: 235297\n", + "Upserted 800 vectors, total so far: 236097\n", + "Upserted 805 vectors, total so far: 236902\n", + "Upserted 805 vectors, total so far: 237707\n", + "Upserted 782 vectors, total so far: 238489\n", + "Upserted 809 vectors, total so far: 239298\n", + "Upserted 812 vectors, total so far: 240110\n", + "Upserted 809 vectors, total so far: 240919\n", + "Upserted 797 vectors, total so far: 241716\n", + "Upserted 813 vectors, total so far: 242529\n", + "Upserted 822 vectors, total so far: 243351\n", + "Upserted 788 vectors, total so far: 244139\n", + "Upserted 808 vectors, total so far: 244947\n", + "Upserted 799 vectors, total so far: 245746\n", + "Upserted 814 vectors, total so far: 246560\n", + "Upserted 813 vectors, total so far: 247373\n", + "Upserted 798 vectors, total so far: 248171\n", + "Upserted 793 vectors, total so far: 248964\n", + "Upserted 794 vectors, total so far: 249758\n", + "Upserted 807 vectors, total so far: 250565\n", + "Upserted 803 vectors, total so far: 251368\n", + "Upserted 812 vectors, total so far: 252180\n", + "Upserted 814 vectors, total so far: 252994\n", + "Upserted 810 vectors, total so far: 253804\n", + "Upserted 802 vectors, total so far: 254606\n", + "Upserted 774 vectors, total so far: 255380\n", + "Upserted 795 vectors, total so far: 256175\n", + "Upserted 800 vectors, total so far: 256975\n", + "Upserted 782 vectors, total so far: 257757\n", + "Upserted 800 vectors, total so far: 258557\n", + "Upserted 804 vectors, total so far: 259361\n", + "Upserted 810 vectors, total so far: 260171\n", + "Upserted 824 vectors, total so far: 260995\n", + "Upserted 793 vectors, total so far: 261788\n", + "Upserted 782 vectors, total so far: 262570\n", + "Upserted 784 vectors, total so far: 263354\n", + "Upserted 764 vectors, total so far: 264118\n", + "Upserted 779 vectors, total so far: 264897\n", + "Upserted 816 vectors, total so far: 265713\n", + "Upserted 807 vectors, total so far: 266520\n", + "Upserted 792 vectors, total so far: 267312\n", + "Upserted 790 vectors, total so far: 268102\n", + "Upserted 786 vectors, total so far: 268888\n", + "Upserted 809 vectors, total so far: 269697\n", + "Upserted 803 vectors, total so far: 270500\n", + "Upserted 826 vectors, total so far: 271326\n", + "Upserted 775 vectors, total so far: 272101\n", + "Upserted 793 vectors, total so far: 272894\n", + "Upserted 795 vectors, total so far: 273689\n", + "Upserted 798 vectors, total so far: 274487\n", + "Upserted 818 vectors, total so far: 275305\n", + "Upserted 809 vectors, total so far: 276114\n", + "Upserted 789 vectors, total so far: 276903\n", + "Upserted 795 vectors, total so far: 277698\n", + "Upserted 794 vectors, total so far: 278492\n", + "Upserted 795 vectors, total so far: 279287\n", + "Upserted 795 vectors, total so far: 280082\n", + "Upserted 804 vectors, total so far: 280886\n", + "Upserted 814 vectors, total so far: 281700\n", + "Upserted 810 vectors, total so far: 282510\n", + "Upserted 788 vectors, total so far: 283298\n", + "Upserted 810 vectors, total so far: 284108\n", + "Upserted 794 vectors, total so far: 284902\n", + "Upserted 802 vectors, total so far: 285704\n", + "Upserted 790 vectors, total so far: 286494\n", + "Upserted 794 vectors, total so far: 287288\n", + "Upserted 811 vectors, total so far: 288099\n", + "Upserted 794 vectors, total so far: 288893\n", + "Upserted 819 vectors, total so far: 289712\n", + "Upserted 785 vectors, total so far: 290497\n", + "Upserted 810 vectors, total so far: 291307\n", + "Upserted 786 vectors, total so far: 292093\n", + "Upserted 793 vectors, total so far: 292886\n", + "Upserted 796 vectors, total so far: 293682\n", + "Upserted 809 vectors, total so far: 294491\n", + "Upserted 801 vectors, total so far: 295292\n", + "Upserted 805 vectors, total so far: 296097\n", + "Upserted 797 vectors, total so far: 296894\n", + "Upserted 806 vectors, total so far: 297700\n", + "Upserted 803 vectors, total so far: 298503\n", + "Upserted 827 vectors, total so far: 299330\n", + "Upserted 820 vectors, total so far: 300150\n", + "Upserted 793 vectors, total so far: 300943\n", + "Upserted 795 vectors, total so far: 301738\n", + "Upserted 774 vectors, total so far: 302512\n", + "Upserted 790 vectors, total so far: 303302\n", + "Upserted 801 vectors, total so far: 304103\n", + "Upserted 793 vectors, total so far: 304896\n", + "Upserted 801 vectors, total so far: 305697\n", + "Upserted 793 vectors, total so far: 306490\n", + "Upserted 815 vectors, total so far: 307305\n", + "Upserted 821 vectors, total so far: 308126\n", + "Upserted 812 vectors, total so far: 308938\n", + "Upserted 786 vectors, total so far: 309724\n", + "Upserted 784 vectors, total so far: 310508\n", + "Upserted 803 vectors, total so far: 311311\n", + "Upserted 777 vectors, total so far: 312088\n", + "Upserted 824 vectors, total so far: 312912\n", + "Upserted 799 vectors, total so far: 313711\n", + "Upserted 790 vectors, total so far: 314501\n", + "Upserted 809 vectors, total so far: 315310\n", + "Upserted 797 vectors, total so far: 316107\n", + "Upserted 784 vectors, total so far: 316891\n", + "Upserted 803 vectors, total so far: 317694\n", + "Upserted 774 vectors, total so far: 318468\n", + "Upserted 798 vectors, total so far: 319266\n", + "Upserted 791 vectors, total so far: 320057\n", + "Upserted 791 vectors, total so far: 320848\n", + "Upserted 809 vectors, total so far: 321657\n", + "Upserted 800 vectors, total so far: 322457\n", + "Upserted 799 vectors, total so far: 323256\n", + "Upserted 825 vectors, total so far: 324081\n", + "Upserted 819 vectors, total so far: 324900\n", + "Upserted 778 vectors, total so far: 325678\n", + "Upserted 782 vectors, total so far: 326460\n", + "Upserted 796 vectors, total so far: 327256\n", + "Upserted 792 vectors, total so far: 328048\n", + "Upserted 817 vectors, total so far: 328865\n", + "Upserted 801 vectors, total so far: 329666\n", + "Upserted 779 vectors, total so far: 330445\n", + "Upserted 804 vectors, total so far: 331249\n", + "Upserted 786 vectors, total so far: 332035\n", + "Upserted 783 vectors, total so far: 332818\n", + "Upserted 819 vectors, total so far: 333637\n", + "Upserted 790 vectors, total so far: 334427\n", + "Upserted 793 vectors, total so far: 335220\n", + "Upserted 811 vectors, total so far: 336031\n", + "Upserted 795 vectors, total so far: 336826\n", + "Upserted 808 vectors, total so far: 337634\n", + "Upserted 802 vectors, total so far: 338436\n", + "Upserted 816 vectors, total so far: 339252\n", + "Upserted 815 vectors, total so far: 340067\n", + "Upserted 798 vectors, total so far: 340865\n", + "Upserted 788 vectors, total so far: 341653\n", + "Upserted 782 vectors, total so far: 342435\n", + "Upserted 792 vectors, total so far: 343227\n", + "Upserted 779 vectors, total so far: 344006\n", + "Upserted 803 vectors, total so far: 344809\n", + "Upserted 818 vectors, total so far: 345627\n", + "Upserted 809 vectors, total so far: 346436\n", + "Upserted 803 vectors, total so far: 347239\n", + "Upserted 798 vectors, total so far: 348037\n", + "Upserted 795 vectors, total so far: 348832\n", + "Upserted 809 vectors, total so far: 349641\n", + "Upserted 756 vectors, total so far: 350397\n", + "Upserted 797 vectors, total so far: 351194\n", + "Upserted 800 vectors, total so far: 351994\n", + "Upserted 797 vectors, total so far: 352791\n", + "Upserted 793 vectors, total so far: 353584\n", + "Upserted 779 vectors, total so far: 354363\n", + "Upserted 796 vectors, total so far: 355159\n", + "Upserted 788 vectors, total so far: 355947\n", + "Upserted 811 vectors, total so far: 356758\n", + "Upserted 798 vectors, total so far: 357556\n", + "Upserted 812 vectors, total so far: 358368\n", + "Upserted 789 vectors, total so far: 359157\n", + "Upserted 808 vectors, total so far: 359965\n", + "Upserted 812 vectors, total so far: 360777\n", + "Upserted 803 vectors, total so far: 361580\n", + "Upserted 801 vectors, total so far: 362381\n", + "Upserted 796 vectors, total so far: 363177\n", + "Upserted 803 vectors, total so far: 363980\n", + "Upserted 807 vectors, total so far: 364787\n", + "Upserted 804 vectors, total so far: 365591\n", + "Upserted 810 vectors, total so far: 366401\n", + "Upserted 808 vectors, total so far: 367209\n", + "Upserted 808 vectors, total so far: 368017\n", + "Upserted 786 vectors, total so far: 368803\n", + "Upserted 799 vectors, total so far: 369602\n", + "Upserted 811 vectors, total so far: 370413\n", + "Upserted 803 vectors, total so far: 371216\n", + "Upserted 789 vectors, total so far: 372005\n", + "Upserted 788 vectors, total so far: 372793\n", + "Upserted 775 vectors, total so far: 373568\n", + "Upserted 789 vectors, total so far: 374357\n", + "Upserted 803 vectors, total so far: 375160\n", + "Upserted 816 vectors, total so far: 375976\n", + "Upserted 826 vectors, total so far: 376802\n", + "Upserted 807 vectors, total so far: 377609\n", + "Upserted 798 vectors, total so far: 378407\n", + "Upserted 805 vectors, total so far: 379212\n", + "Upserted 788 vectors, total so far: 380000\n", + "Upserted 804 vectors, total so far: 380804\n", + "Upserted 809 vectors, total so far: 381613\n", + "Upserted 833 vectors, total so far: 382446\n", + "Upserted 809 vectors, total so far: 383255\n", + "Upserted 814 vectors, total so far: 384069\n", + "Upserted 819 vectors, total so far: 384888\n", + "Upserted 792 vectors, total so far: 385680\n", + "Upserted 812 vectors, total so far: 386492\n", + "Upserted 817 vectors, total so far: 387309\n", + "Upserted 801 vectors, total so far: 388110\n", + "Upserted 799 vectors, total so far: 388909\n", + "Upserted 807 vectors, total so far: 389716\n", + "Upserted 791 vectors, total so far: 390507\n", + "Upserted 820 vectors, total so far: 391327\n", + "Upserted 810 vectors, total so far: 392137\n", + "Upserted 798 vectors, total so far: 392935\n", + "Upserted 816 vectors, total so far: 393751\n", + "Upserted 791 vectors, total so far: 394542\n", + "Upserted 815 vectors, total so far: 395357\n", + "Upserted 789 vectors, total so far: 396146\n", + "Upserted 793 vectors, total so far: 396939\n", + "Upserted 794 vectors, total so far: 397733\n", + "Upserted 802 vectors, total so far: 398535\n", + "Upserted 801 vectors, total so far: 399336\n", + "Upserted 825 vectors, total so far: 400161\n", + "Upserted 813 vectors, total so far: 400974\n", + "Upserted 797 vectors, total so far: 401771\n", + "Upserted 810 vectors, total so far: 402581\n", + "Upserted 838 vectors, total so far: 403419\n", + "Upserted 812 vectors, total so far: 404231\n", + "Upserted 795 vectors, total so far: 405026\n", + "Upserted 806 vectors, total so far: 405832\n", + "Upserted 777 vectors, total so far: 406609\n", + "Upserted 766 vectors, total so far: 407375\n", + "Upserted 811 vectors, total so far: 408186\n", + "Upserted 775 vectors, total so far: 408961\n", + "Upserted 794 vectors, total so far: 409755\n", + "Upserted 811 vectors, total so far: 410566\n", + "Upserted 788 vectors, total so far: 411354\n", + "Upserted 795 vectors, total so far: 412149\n", + "Upserted 789 vectors, total so far: 412938\n", + "Upserted 807 vectors, total so far: 413745\n", + "Upserted 803 vectors, total so far: 414548\n", + "Upserted 811 vectors, total so far: 415359\n", + "Upserted 772 vectors, total so far: 416131\n", + "Upserted 805 vectors, total so far: 416936\n", + "Upserted 788 vectors, total so far: 417724\n", + "Upserted 825 vectors, total so far: 418549\n", + "Upserted 815 vectors, total so far: 419364\n", + "Upserted 791 vectors, total so far: 420155\n", + "Upserted 799 vectors, total so far: 420954\n", + "Upserted 819 vectors, total so far: 421773\n", + "Upserted 799 vectors, total so far: 422572\n", + "Upserted 801 vectors, total so far: 423373\n", + "Upserted 803 vectors, total so far: 424176\n", + "Upserted 782 vectors, total so far: 424958\n", + "Upserted 778 vectors, total so far: 425736\n", + "Upserted 817 vectors, total so far: 426553\n", + "Upserted 788 vectors, total so far: 427341\n", + "Upserted 785 vectors, total so far: 428126\n", + "Upserted 795 vectors, total so far: 428921\n", + "Upserted 799 vectors, total so far: 429720\n", + "Upserted 805 vectors, total so far: 430525\n", + "Upserted 809 vectors, total so far: 431334\n", + "Upserted 782 vectors, total so far: 432116\n", + "Upserted 803 vectors, total so far: 432919\n", + "Upserted 820 vectors, total so far: 433739\n", + "Upserted 826 vectors, total so far: 434565\n", + "Upserted 801 vectors, total so far: 435366\n", + "Upserted 798 vectors, total so far: 436164\n", + "Upserted 780 vectors, total so far: 436944\n", + "Upserted 805 vectors, total so far: 437749\n", + "Upserted 804 vectors, total so far: 438553\n", + "Upserted 799 vectors, total so far: 439352\n", + "Upserted 806 vectors, total so far: 440158\n", + "Upserted 808 vectors, total so far: 440966\n", + "Upserted 803 vectors, total so far: 441769\n", + "Upserted 798 vectors, total so far: 442567\n", + "Upserted 816 vectors, total so far: 443383\n", + "Upserted 816 vectors, total so far: 444199\n", + "Upserted 791 vectors, total so far: 444990\n", + "Upserted 818 vectors, total so far: 445808\n", + "Upserted 803 vectors, total so far: 446611\n", + "Upserted 808 vectors, total so far: 447419\n", + "Upserted 838 vectors, total so far: 448257\n", + "Upserted 797 vectors, total so far: 449054\n", + "Upserted 797 vectors, total so far: 449851\n", + "Upserted 785 vectors, total so far: 450636\n", + "Upserted 786 vectors, total so far: 451422\n", + "Upserted 796 vectors, total so far: 452218\n", + "Upserted 815 vectors, total so far: 453033\n", + "Upserted 788 vectors, total so far: 453821\n", + "Upserted 796 vectors, total so far: 454617\n", + "Upserted 798 vectors, total so far: 455415\n", + "Upserted 787 vectors, total so far: 456202\n", + "Upserted 804 vectors, total so far: 457006\n", + "Upserted 791 vectors, total so far: 457797\n", + "Upserted 798 vectors, total so far: 458595\n", + "Upserted 784 vectors, total so far: 459379\n", + "Upserted 774 vectors, total so far: 460153\n", + "Upserted 817 vectors, total so far: 460970\n", + "Upserted 780 vectors, total so far: 461750\n", + "Upserted 791 vectors, total so far: 462541\n", + "Upserted 810 vectors, total so far: 463351\n", + "Upserted 822 vectors, total so far: 464173\n", + "Upserted 783 vectors, total so far: 464956\n", + "Upserted 806 vectors, total so far: 465762\n", + "Upserted 798 vectors, total so far: 466560\n", + "Upserted 794 vectors, total so far: 467354\n", + "Upserted 812 vectors, total so far: 468166\n", + "Upserted 776 vectors, total so far: 468942\n", + "Upserted 784 vectors, total so far: 469726\n", + "Upserted 808 vectors, total so far: 470534\n", + "Upserted 794 vectors, total so far: 471328\n", + "Upserted 817 vectors, total so far: 472145\n", + "Upserted 800 vectors, total so far: 472945\n", + "Upserted 810 vectors, total so far: 473755\n", + "Upserted 770 vectors, total so far: 474525\n", + "Upserted 807 vectors, total so far: 475332\n", + "Upserted 782 vectors, total so far: 476114\n", + "Upserted 812 vectors, total so far: 476926\n", + "Upserted 806 vectors, total so far: 477732\n", + "Upserted 807 vectors, total so far: 478539\n", + "Upserted 788 vectors, total so far: 479327\n", + "Upserted 805 vectors, total so far: 480132\n", + "Upserted 788 vectors, total so far: 480920\n", + "Upserted 798 vectors, total so far: 481718\n", + "Upserted 789 vectors, total so far: 482507\n", + "Upserted 802 vectors, total so far: 483309\n", + "Upserted 810 vectors, total so far: 484119\n", + "Upserted 802 vectors, total so far: 484921\n", + "Upserted 818 vectors, total so far: 485739\n", + "Upserted 800 vectors, total so far: 486539\n", + "Upserted 799 vectors, total so far: 487338\n", + "Upserted 795 vectors, total so far: 488133\n", + "Upserted 798 vectors, total so far: 488931\n", + "Upserted 816 vectors, total so far: 489747\n", + "Upserted 790 vectors, total so far: 490537\n", + "Upserted 816 vectors, total so far: 491353\n", + "Upserted 795 vectors, total so far: 492148\n", + "Upserted 798 vectors, total so far: 492946\n", + "Upserted 801 vectors, total so far: 493747\n", + "Upserted 801 vectors, total so far: 494548\n", + "Upserted 808 vectors, total so far: 495356\n", + "Upserted 795 vectors, total so far: 496151\n", + "Upserted 817 vectors, total so far: 496968\n", + "Upserted 812 vectors, total so far: 497780\n", + "Upserted 760 vectors, total so far: 498540\n", + "Upserted 819 vectors, total so far: 499359\n", + "Upserted 798 vectors, total so far: 500157\n", + "Upserted 801 vectors, total so far: 500958\n", + "Upserted 810 vectors, total so far: 501768\n", + "Upserted 778 vectors, total so far: 502546\n", + "Upserted 790 vectors, total so far: 503336\n", + "Upserted 791 vectors, total so far: 504127\n", + "Upserted 787 vectors, total so far: 504914\n", + "Upserted 814 vectors, total so far: 505728\n", + "Upserted 804 vectors, total so far: 506532\n", + "Upserted 789 vectors, total so far: 507321\n", + "Upserted 800 vectors, total so far: 508121\n", + "Upserted 802 vectors, total so far: 508923\n", + "Upserted 817 vectors, total so far: 509740\n", + "Upserted 782 vectors, total so far: 510522\n", + "Upserted 799 vectors, total so far: 511321\n", + "Upserted 806 vectors, total so far: 512127\n", + "Upserted 804 vectors, total so far: 512931\n", + "Upserted 810 vectors, total so far: 513741\n", + "Upserted 808 vectors, total so far: 514549\n", + "Upserted 799 vectors, total so far: 515348\n", + "Upserted 800 vectors, total so far: 516148\n", + "Upserted 779 vectors, total so far: 516927\n", + "Upserted 802 vectors, total so far: 517729\n", + "Upserted 818 vectors, total so far: 518547\n", + "Upserted 789 vectors, total so far: 519336\n", + "Upserted 800 vectors, total so far: 520136\n", + "Upserted 792 vectors, total so far: 520928\n", + "Upserted 798 vectors, total so far: 521726\n", + "Upserted 782 vectors, total so far: 522508\n", + "Upserted 777 vectors, total so far: 523285\n", + "Upserted 803 vectors, total so far: 524088\n", + "Upserted 799 vectors, total so far: 524887\n", + "Upserted 807 vectors, total so far: 525694\n", + "Upserted 812 vectors, total so far: 526506\n", + "Upserted 813 vectors, total so far: 527319\n", + "Upserted 792 vectors, total so far: 528111\n", + "Upserted 807 vectors, total so far: 528918\n", + "Upserted 791 vectors, total so far: 529709\n", + "Upserted 795 vectors, total so far: 530504\n", + "Upserted 802 vectors, total so far: 531306\n", + "Upserted 797 vectors, total so far: 532103\n", + "Upserted 782 vectors, total so far: 532885\n", + "Upserted 794 vectors, total so far: 533679\n", + "Upserted 822 vectors, total so far: 534501\n", + "Upserted 810 vectors, total so far: 535311\n", + "Upserted 786 vectors, total so far: 536097\n", + "Upserted 798 vectors, total so far: 536895\n", + "Upserted 797 vectors, total so far: 537692\n", + "Upserted 789 vectors, total so far: 538481\n", + "Upserted 800 vectors, total so far: 539281\n", + "Upserted 783 vectors, total so far: 540064\n", + "Upserted 794 vectors, total so far: 540858\n", + "Upserted 790 vectors, total so far: 541648\n", + "Upserted 800 vectors, total so far: 542448\n", + "Upserted 795 vectors, total so far: 543243\n", + "Upserted 811 vectors, total so far: 544054\n", + "Upserted 817 vectors, total so far: 544871\n", + "Upserted 794 vectors, total so far: 545665\n", + "Upserted 788 vectors, total so far: 546453\n", + "Upserted 810 vectors, total so far: 547263\n", + "Upserted 808 vectors, total so far: 548071\n", + "Upserted 795 vectors, total so far: 548866\n", + "Upserted 791 vectors, total so far: 549657\n", + "Upserted 794 vectors, total so far: 550451\n", + "Upserted 791 vectors, total so far: 551242\n", + "Upserted 793 vectors, total so far: 552035\n", + "Upserted 785 vectors, total so far: 552820\n", + "Upserted 822 vectors, total so far: 553642\n", + "Upserted 796 vectors, total so far: 554438\n", + "Upserted 798 vectors, total so far: 555236\n", + "Upserted 810 vectors, total so far: 556046\n", + "Upserted 804 vectors, total so far: 556850\n", + "Upserted 810 vectors, total so far: 557660\n", + "Upserted 812 vectors, total so far: 558472\n", + "Upserted 808 vectors, total so far: 559280\n", + "Upserted 792 vectors, total so far: 560072\n", + "Upserted 776 vectors, total so far: 560848\n", + "Upserted 797 vectors, total so far: 561645\n", + "Upserted 787 vectors, total so far: 562432\n", + "Upserted 799 vectors, total so far: 563231\n", + "Upserted 794 vectors, total so far: 564025\n", + "Upserted 796 vectors, total so far: 564821\n", + "Upserted 778 vectors, total so far: 565599\n", + "Upserted 789 vectors, total so far: 566388\n", + "Upserted 792 vectors, total so far: 567180\n", + "Upserted 803 vectors, total so far: 567983\n", + "Upserted 814 vectors, total so far: 568797\n", + "Upserted 795 vectors, total so far: 569592\n", + "Upserted 803 vectors, total so far: 570395\n", + "Upserted 804 vectors, total so far: 571199\n", + "Upserted 810 vectors, total so far: 572009\n", + "Upserted 800 vectors, total so far: 572809\n", + "Upserted 785 vectors, total so far: 573594\n", + "Upserted 792 vectors, total so far: 574386\n", + "Upserted 805 vectors, total so far: 575191\n", + "Upserted 809 vectors, total so far: 576000\n", + "Upserted 796 vectors, total so far: 576796\n", + "Upserted 807 vectors, total so far: 577603\n", + "Upserted 810 vectors, total so far: 578413\n", + "Upserted 771 vectors, total so far: 579184\n", + "Upserted 801 vectors, total so far: 579985\n", + "Upserted 811 vectors, total so far: 580796\n", + "Upserted 798 vectors, total so far: 581594\n", + "Upserted 802 vectors, total so far: 582396\n", + "Upserted 803 vectors, total so far: 583199\n", + "Upserted 785 vectors, total so far: 583984\n", + "Upserted 811 vectors, total so far: 584795\n", + "Upserted 795 vectors, total so far: 585590\n", + "Upserted 816 vectors, total so far: 586406\n", + "Upserted 794 vectors, total so far: 587200\n", + "Upserted 801 vectors, total so far: 588001\n", + "Upserted 815 vectors, total so far: 588816\n", + "Upserted 832 vectors, total so far: 589648\n", + "Upserted 782 vectors, total so far: 590430\n", + "Upserted 804 vectors, total so far: 591234\n", + "Upserted 800 vectors, total so far: 592034\n", + "Upserted 822 vectors, total so far: 592856\n", + "Upserted 783 vectors, total so far: 593639\n", + "Upserted 821 vectors, total so far: 594460\n", + "Upserted 791 vectors, total so far: 595251\n", + "Upserted 798 vectors, total so far: 596049\n", + "Upserted 787 vectors, total so far: 596836\n", + "Upserted 792 vectors, total so far: 597628\n", + "Upserted 786 vectors, total so far: 598414\n", + "Upserted 817 vectors, total so far: 599231\n", + "Upserted 784 vectors, total so far: 600015\n", + "Upserted 803 vectors, total so far: 600818\n", + "Upserted 788 vectors, total so far: 601606\n", + "Upserted 793 vectors, total so far: 602399\n", + "Upserted 803 vectors, total so far: 603202\n", + "Upserted 778 vectors, total so far: 603980\n", + "Upserted 803 vectors, total so far: 604783\n", + "Upserted 789 vectors, total so far: 605572\n", + "Upserted 807 vectors, total so far: 606379\n", + "Upserted 816 vectors, total so far: 607195\n", + "Upserted 811 vectors, total so far: 608006\n", + "Upserted 803 vectors, total so far: 608809\n", + "Upserted 819 vectors, total so far: 609628\n", + "Upserted 801 vectors, total so far: 610429\n", + "Upserted 797 vectors, total so far: 611226\n", + "Upserted 806 vectors, total so far: 612032\n", + "Upserted 781 vectors, total so far: 612813\n", + "Upserted 796 vectors, total so far: 613609\n", + "Upserted 815 vectors, total so far: 614424\n", + "Upserted 814 vectors, total so far: 615238\n", + "Upserted 820 vectors, total so far: 616058\n", + "Upserted 799 vectors, total so far: 616857\n", + "Upserted 786 vectors, total so far: 617643\n", + "Upserted 808 vectors, total so far: 618451\n", + "Upserted 797 vectors, total so far: 619248\n", + "Upserted 816 vectors, total so far: 620064\n", + "Upserted 826 vectors, total so far: 620890\n", + "Upserted 806 vectors, total so far: 621696\n", + "Upserted 802 vectors, total so far: 622498\n", + "Upserted 813 vectors, total so far: 623311\n", + "Upserted 807 vectors, total so far: 624118\n", + "Upserted 798 vectors, total so far: 624916\n", + "Upserted 792 vectors, total so far: 625708\n", + "Upserted 799 vectors, total so far: 626507\n", + "Upserted 789 vectors, total so far: 627296\n", + "Upserted 797 vectors, total so far: 628093\n", + "Upserted 789 vectors, total so far: 628882\n", + "Upserted 807 vectors, total so far: 629689\n", + "Upserted 798 vectors, total so far: 630487\n", + "Upserted 808 vectors, total so far: 631295\n", + "Upserted 785 vectors, total so far: 632080\n", + "Upserted 804 vectors, total so far: 632884\n", + "Upserted 797 vectors, total so far: 633681\n", + "Upserted 811 vectors, total so far: 634492\n", + "Upserted 795 vectors, total so far: 635287\n", + "Upserted 815 vectors, total so far: 636102\n", + "Upserted 801 vectors, total so far: 636903\n", + "Upserted 806 vectors, total so far: 637709\n", + "Upserted 772 vectors, total so far: 638481\n", + "Upserted 811 vectors, total so far: 639292\n", + "Upserted 817 vectors, total so far: 640109\n", + "Upserted 772 vectors, total so far: 640881\n", + "Upserted 808 vectors, total so far: 641689\n", + "Upserted 787 vectors, total so far: 642476\n", + "Upserted 791 vectors, total so far: 643267\n", + "Upserted 809 vectors, total so far: 644076\n", + "Upserted 781 vectors, total so far: 644857\n", + "Upserted 776 vectors, total so far: 645633\n", + "Upserted 810 vectors, total so far: 646443\n", + "Upserted 811 vectors, total so far: 647254\n", + "Upserted 800 vectors, total so far: 648054\n", + "Upserted 806 vectors, total so far: 648860\n", + "Upserted 799 vectors, total so far: 649659\n", + "Upserted 799 vectors, total so far: 650458\n", + "Upserted 789 vectors, total so far: 651247\n", + "Upserted 789 vectors, total so far: 652036\n", + "Upserted 804 vectors, total so far: 652840\n", + "Upserted 799 vectors, total so far: 653639\n", + "Upserted 792 vectors, total so far: 654431\n", + "Upserted 793 vectors, total so far: 655224\n", + "Upserted 807 vectors, total so far: 656031\n", + "Upserted 819 vectors, total so far: 656850\n", + "Upserted 796 vectors, total so far: 657646\n", + "Upserted 820 vectors, total so far: 658466\n", + "Upserted 798 vectors, total so far: 659264\n", + "Upserted 806 vectors, total so far: 660070\n", + "Upserted 816 vectors, total so far: 660886\n", + "Upserted 788 vectors, total so far: 661674\n", + "Upserted 805 vectors, total so far: 662479\n", + "Upserted 800 vectors, total so far: 663279\n", + "Upserted 795 vectors, total so far: 664074\n", + "Upserted 798 vectors, total so far: 664872\n", + "Upserted 804 vectors, total so far: 665676\n", + "Upserted 802 vectors, total so far: 666478\n", + "Upserted 796 vectors, total so far: 667274\n", + "Upserted 796 vectors, total so far: 668070\n", + "Upserted 792 vectors, total so far: 668862\n", + "Upserted 786 vectors, total so far: 669648\n", + "Upserted 797 vectors, total so far: 670445\n", + "Upserted 818 vectors, total so far: 671263\n", + "Upserted 781 vectors, total so far: 672044\n", + "Upserted 801 vectors, total so far: 672845\n", + "Upserted 796 vectors, total so far: 673641\n", + "Upserted 797 vectors, total so far: 674438\n", + "Upserted 803 vectors, total so far: 675241\n", + "Upserted 793 vectors, total so far: 676034\n", + "Upserted 804 vectors, total so far: 676838\n", + "Upserted 823 vectors, total so far: 677661\n", + "Upserted 803 vectors, total so far: 678464\n", + "Upserted 819 vectors, total so far: 679283\n", + "Upserted 770 vectors, total so far: 680053\n", + "Upserted 810 vectors, total so far: 680863\n", + "Upserted 811 vectors, total so far: 681674\n", + "Upserted 799 vectors, total so far: 682473\n", + "Upserted 805 vectors, total so far: 683278\n", + "Upserted 811 vectors, total so far: 684089\n", + "Upserted 810 vectors, total so far: 684899\n", + "Upserted 810 vectors, total so far: 685709\n", + "Upserted 801 vectors, total so far: 686510\n", + "Upserted 800 vectors, total so far: 687310\n", + "Upserted 795 vectors, total so far: 688105\n", + "Upserted 795 vectors, total so far: 688900\n", + "Upserted 827 vectors, total so far: 689727\n", + "Upserted 775 vectors, total so far: 690502\n", + "Upserted 797 vectors, total so far: 691299\n", + "Upserted 792 vectors, total so far: 692091\n", + "Upserted 806 vectors, total so far: 692897\n", + "Upserted 819 vectors, total so far: 693716\n", + "Upserted 803 vectors, total so far: 694519\n", + "Upserted 783 vectors, total so far: 695302\n", + "Upserted 788 vectors, total so far: 696090\n", + "Upserted 801 vectors, total so far: 696891\n", + "Upserted 813 vectors, total so far: 697704\n", + "Upserted 788 vectors, total so far: 698492\n", + "Upserted 815 vectors, total so far: 699307\n", + "Upserted 785 vectors, total so far: 700092\n", + "Upserted 787 vectors, total so far: 700879\n", + "Upserted 802 vectors, total so far: 701681\n", + "Upserted 794 vectors, total so far: 702475\n", + "Upserted 799 vectors, total so far: 703274\n", + "Upserted 810 vectors, total so far: 704084\n", + "Upserted 826 vectors, total so far: 704910\n", + "Upserted 783 vectors, total so far: 705693\n", + "Upserted 793 vectors, total so far: 706486\n", + "Upserted 806 vectors, total so far: 707292\n", + "Upserted 788 vectors, total so far: 708080\n", + "Upserted 789 vectors, total so far: 708869\n", + "Upserted 778 vectors, total so far: 709647\n", + "Upserted 790 vectors, total so far: 710437\n", + "Upserted 801 vectors, total so far: 711238\n", + "Upserted 811 vectors, total so far: 712049\n", + "Upserted 806 vectors, total so far: 712855\n", + "Upserted 803 vectors, total so far: 713658\n", + "Upserted 810 vectors, total so far: 714468\n", + "Upserted 796 vectors, total so far: 715264\n", + "Upserted 819 vectors, total so far: 716083\n", + "Upserted 816 vectors, total so far: 716899\n", + "Upserted 800 vectors, total so far: 717699\n", + "Upserted 775 vectors, total so far: 718474\n", + "Upserted 782 vectors, total so far: 719256\n", + "Upserted 820 vectors, total so far: 720076\n", + "Upserted 809 vectors, total so far: 720885\n", + "Upserted 795 vectors, total so far: 721680\n", + "Upserted 796 vectors, total so far: 722476\n", + "Upserted 796 vectors, total so far: 723272\n", + "Upserted 796 vectors, total so far: 724068\n", + "Upserted 809 vectors, total so far: 724877\n", + "Upserted 799 vectors, total so far: 725676\n", + "Upserted 814 vectors, total so far: 726490\n", + "Upserted 794 vectors, total so far: 727284\n", + "Upserted 786 vectors, total so far: 728070\n", + "Upserted 800 vectors, total so far: 728870\n", + "Upserted 805 vectors, total so far: 729675\n", + "Upserted 809 vectors, total so far: 730484\n", + "Upserted 797 vectors, total so far: 731281\n", + "Upserted 808 vectors, total so far: 732089\n", + "Upserted 790 vectors, total so far: 732879\n", + "Upserted 818 vectors, total so far: 733697\n", + "Upserted 806 vectors, total so far: 734503\n", + "Upserted 798 vectors, total so far: 735301\n", + "Upserted 811 vectors, total so far: 736112\n", + "Upserted 792 vectors, total so far: 736904\n", + "Upserted 798 vectors, total so far: 737702\n", + "Upserted 791 vectors, total so far: 738493\n", + "Upserted 781 vectors, total so far: 739274\n", + "Upserted 781 vectors, total so far: 740055\n", + "Upserted 797 vectors, total so far: 740852\n", + "Upserted 818 vectors, total so far: 741670\n", + "Upserted 783 vectors, total so far: 742453\n", + "Upserted 788 vectors, total so far: 743241\n", + "Upserted 794 vectors, total so far: 744035\n", + "Upserted 797 vectors, total so far: 744832\n", + "Upserted 812 vectors, total so far: 745644\n", + "Upserted 820 vectors, total so far: 746464\n", + "Upserted 789 vectors, total so far: 747253\n", + "Upserted 795 vectors, total so far: 748048\n", + "Upserted 810 vectors, total so far: 748858\n", + "Upserted 796 vectors, total so far: 749654\n", + "Upserted 797 vectors, total so far: 750451\n", + "Upserted 781 vectors, total so far: 751232\n", + "Upserted 810 vectors, total so far: 752042\n", + "Upserted 802 vectors, total so far: 752844\n", + "Upserted 811 vectors, total so far: 753655\n", + "Upserted 804 vectors, total so far: 754459\n", + "Upserted 804 vectors, total so far: 755263\n", + "Upserted 800 vectors, total so far: 756063\n", + "Upserted 790 vectors, total so far: 756853\n", + "Upserted 822 vectors, total so far: 757675\n", + "Upserted 814 vectors, total so far: 758489\n", + "Upserted 789 vectors, total so far: 759278\n", + "Upserted 793 vectors, total so far: 760071\n", + "Upserted 804 vectors, total so far: 760875\n", + "Upserted 777 vectors, total so far: 761652\n", + "Upserted 791 vectors, total so far: 762443\n", + "Upserted 809 vectors, total so far: 763252\n", + "Upserted 781 vectors, total so far: 764033\n", + "Upserted 803 vectors, total so far: 764836\n", + "Upserted 805 vectors, total so far: 765641\n", + "Upserted 816 vectors, total so far: 766457\n", + "Upserted 787 vectors, total so far: 767244\n", + "Upserted 788 vectors, total so far: 768032\n", + "Upserted 818 vectors, total so far: 768850\n", + "Upserted 810 vectors, total so far: 769660\n", + "Upserted 803 vectors, total so far: 770463\n", + "Upserted 797 vectors, total so far: 771260\n", + "Upserted 809 vectors, total so far: 772069\n", + "Upserted 799 vectors, total so far: 772868\n", + "Upserted 805 vectors, total so far: 773673\n", + "Upserted 797 vectors, total so far: 774470\n", + "Upserted 810 vectors, total so far: 775280\n", + "Upserted 804 vectors, total so far: 776084\n", + "Upserted 797 vectors, total so far: 776881\n", + "Upserted 813 vectors, total so far: 777694\n", + "Upserted 798 vectors, total so far: 778492\n", + "Upserted 808 vectors, total so far: 779300\n", + "Upserted 794 vectors, total so far: 780094\n", + "Upserted 792 vectors, total so far: 780886\n", + "Upserted 804 vectors, total so far: 781690\n", + "Upserted 804 vectors, total so far: 782494\n", + "Upserted 816 vectors, total so far: 783310\n", + "Upserted 793 vectors, total so far: 784103\n", + "Upserted 811 vectors, total so far: 784914\n", + "Upserted 778 vectors, total so far: 785692\n", + "Upserted 808 vectors, total so far: 786500\n", + "Upserted 806 vectors, total so far: 787306\n", + "Upserted 776 vectors, total so far: 788082\n", + "Upserted 782 vectors, total so far: 788864\n", + "Upserted 796 vectors, total so far: 789660\n", + "Upserted 802 vectors, total so far: 790462\n", + "Upserted 806 vectors, total so far: 791268\n", + "Upserted 793 vectors, total so far: 792061\n", + "Upserted 795 vectors, total so far: 792856\n", + "Upserted 784 vectors, total so far: 793640\n", + "Upserted 782 vectors, total so far: 794422\n", + "Upserted 792 vectors, total so far: 795214\n", + "Upserted 797 vectors, total so far: 796011\n", + "Upserted 795 vectors, total so far: 796806\n", + "Upserted 811 vectors, total so far: 797617\n", + "Upserted 781 vectors, total so far: 798398\n", + "Upserted 802 vectors, total so far: 799200\n", + "Upserted 800 vectors, total so far: 800000\n", + "Done! Total inserted: 800000 vectors with id range [0, 799999]\n", + "Time taken: 789.1315255020745\n" ] } ], @@ -8315,12 +11309,1655 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_labelfilter_0503_1_adup1',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 200000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "index.describe()" ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Finding all query IDs with recall < 1.0 ...\n", + "\n", + "Query ID: 14\n", + "Returned IDs: 90505,782992,219095,249142,87223,952696,606232,407566,646730,764993,281286,82235,256972,20952,766350,186364,28749,836818,195642,721815,980205,589088,954423,54884,766421,81742,294536,163812,926812,737136\n", + "Ground Truth: 90505,782992,219095,249142,87223,952696,606232,407566,646730,764993,281286,82235,256972,449934,20952,766350,186364,28749,836818,195642,721815,980205,589088,954423,54884,766421,81742,294536,163812,926812\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 17\n", + "Returned IDs: 791622,801493,917338,162827,700429,464741,899824,420959,982094,637463,453086,142000,792498,873829,923407,526327,251459,175303,383751,977959,226536,887923,337863,883750,341193,796512,820586,248141,538285,840976\n", + "Ground Truth: 791622,801493,917338,162827,700429,464741,899824,820440,420959,982094,637463,453086,142000,792498,873829,923407,526327,251459,175303,383751,977959,226536,887923,337863,883750,341193,796512,401213,820586,248141\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 21\n", + "Returned IDs: 920526,160246,461378,490822,913686,730272,594365,547721,812518,464691,213097,698008,678163,344393,352398,906772,107460,855755,935976,227866,148209,203353,75520,440403,909532,348812,60933,99177,161420,223699\n", + "Ground Truth: 920526,160246,461378,490822,913686,730272,594365,547721,812518,464691,213097,698008,678163,344393,352398,906772,107460,855755,935976,227866,148209,203353,75520,440403,909532,348812,60933,889335,99177,48713\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 23\n", + "Returned IDs: 766971,268212,739092,583683,75196,594757,776150,368812,651554,79487,183691,88311,371655,629135,451687,712041,10095,970062,57581,805963,245985,421560,745040,753442,925988,232568,543533,802140,506026,323509\n", + "Ground Truth: 766971,268212,425738,739092,583683,75196,594757,775830,776150,368812,982531,651554,79487,342446,532188,183691,800983,88311,371655,629135,451687,712041,10095,970062,57581,805963,99475,245985,421560,745040\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 29\n", + "Returned IDs: 162615,54651,938517,885500,799345,493886,28066,102866,650420,176242,556491,958524,602001,904083,544495,523005,486300,799290,665496,211986,255467,371942,247363,82546,688670,861290,833010,663128,906563,259734\n", + "Ground Truth: 162615,54651,938517,885500,799345,493886,28066,102866,650420,176242,556491,958524,602001,904083,544495,523005,486300,799290,665496,211986,255467,371942,57667,247363,82546,950962,688670,861290,833010,663128\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 35\n", + "Returned IDs: 777339,692866,419843,628429,824750,287582,640237,313026,807268,212512,928250,101369,842823,298527,968208,914572,133480,602224,887079,834243,308055,760402,340872,534604,868610,881109,757006,977680,760524,693316\n", + "Ground Truth: 777339,692866,419843,628429,824750,287582,640237,313026,807268,212512,928250,101369,842823,701387,298527,968208,914572,133480,602224,887079,834243,308055,760402,340872,534604,868610,881109,757006,977680,760524\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 38\n", + "Returned IDs: 962399,592238,714880,834195,742540,48493,978874,674724,270183,354882,469702,337817,651192,245492,872772,917903,574157,464990,983351,802140,654252,797627,379870,685914,958018,878677,230193,896532,690408,339724\n", + "Ground Truth: 962399,592238,714880,834195,742540,48493,978874,674724,270183,354882,469702,337817,651192,216031,245492,597835,872772,917903,574157,464990,983351,802140,654252,797627,379870,685914,875307,958018,878677,230193\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 42\n", + "Returned IDs: 668090,441261,814884,468965,193959,740860,475001,332119,692711,88587,250895,311373,820531,351238,230065,755511,726983,376905,20017,875904,56675,214936,742592,508171,23862,648642,751433,808368,337739,561529\n", + "Ground Truth: 668090,441261,814884,468965,193959,740860,475001,332119,692711,88587,250895,820531,311373,351238,230065,755511,726983,376905,20017,875904,56675,214936,742592,508171,496486,23862,648642,751433,808368,337739\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 44\n", + "Returned IDs: 479614,466750,145450,950332,711870,30738,371018,24581,555850,874350,674724,377730,323939,457239,173920,538260,364262,694508,776458,363978,962399,307151,38514,456474,753389,721440,385990,732256,236323,11937\n", + "Ground Truth: 479614,466750,145450,950332,711870,30738,371018,24581,555850,874350,674724,377730,323939,457239,173920,538260,908072,364262,694508,776458,363978,962399,307151,38514,523192,456474,753389,721440,385990,732256\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 51\n", + "Returned IDs: 251008,867561,5969,583259,142664,588037,171228,219201,745114,457877,861530,315569,749214,477691,921426,788453,641362,927146,301769,158176,42504,571774,842285,449569,140136,276076,499478,370057,835846,95034\n", + "Ground Truth: 251008,867561,5969,583259,142664,588037,171228,894379,219201,745114,457877,861530,315569,749214,477691,921426,788453,641362,300182,927146,301769,880278,158176,42504,571774,842285,449569,140136,276076,499478\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 53\n", + "Returned IDs: 141737,179537,243817,469626,402474,176428,830770,363949,494339,240766,354644,155093,532367,242056,899752,946264,473001,899325,569724,8347,200516,739091,932086,875931,236361,32391,436228,92037,195939,627416\n", + "Ground Truth: 141737,179537,243817,469626,402474,176428,830770,363949,494339,240766,354644,155093,532367,242056,899752,946264,473001,899325,569724,559856,8347,200516,739091,932086,875931,236361,32391,436228,92037,195939\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 54\n", + "Returned IDs: 825543,888417,180811,946830,793531,971475,706498,239515,24833,850903,629409,571252,786749,702623,718898,934115,668715,640449,448645,48292,728995,363401,453548,52110,253536,983944,457781,29743,728896,950215\n", + "Ground Truth: 825543,888417,180811,946830,793531,971475,706498,239515,24833,850903,629409,571252,786749,702623,718898,934115,668715,640449,448645,48292,728995,363401,453548,52110,253536,182494,983944,724879,457781,29743\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 55\n", + "Returned IDs: 284177,566607,767119,804416,748611,518411,686975,601153,298977,584380,363375,258208,380761,821125,874746,129911,317104,662020,135584,988553,773690,851260,295420,743414,409769,349994,442826,441763,92120,281012\n", + "Ground Truth: 284177,566607,767119,804416,748611,518411,686975,601153,298977,584380,363375,258208,380761,821125,322747,874746,129911,317104,662020,135584,988553,773690,851260,295420,743414,409769,349994,442826,441763,92120\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 59\n", + "Returned IDs: 107150,876544,597660,235773,440598,688978,178789,782015,881207,109470,384022,779596,727946,225790,926354,943592,246806,674671,673741,455279,762354,125990,720916,938843,413617,64472,467943,406786,474578,868512\n", + "Ground Truth: 107150,876544,597660,235773,440598,688978,178789,782015,881207,109470,384022,779596,727946,252183,225790,926354,943592,246806,861604,674671,673741,455279,762354,125990,720916,619947,938843,413617,64472,467943\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 62\n", + "Returned IDs: 510997,537505,808224,752551,9053,512600,689609,93786,91339,217433,394952,836682,917245,87523,120127,28671,459689,93902,810857,735874,652889,354367,127485,938018,935195,534936,797391,55674,898792,644749\n", + "Ground Truth: 510997,537505,808224,752551,703580,9053,512600,689609,93786,91339,217433,394952,836682,917245,87523,120127,28671,459689,93902,810857,735874,652889,354367,127485,938018,935195,534936,797391,55674,898792\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 66\n", + "Returned IDs: 567766,980989,867428,668441,148454,170649,564636,865309,841690,836001,139328,786944,802260,154859,109721,727956,677574,577423,311870,200168,740973,716451,645621,469163,167881,976432,156573,348712,976304,977029\n", + "Ground Truth: 567766,980989,867428,668441,148454,170649,564636,865309,841690,836001,139328,786944,802260,154859,109721,431103,727956,677574,577423,311870,200168,740973,716451,645621,469163,167881,976432,156573,348712,976304\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 71\n", + "Returned IDs: 295659,781007,766370,43965,979529,147989,622646,250569,332345,752258,895180,938652,733060,337864,917380,884454,357003,768038,327683,783686,838961,109336,281827,577423,496945,674142,61920,252912,65500,651173\n", + "Ground Truth: 298079,139415,295659,781007,766370,43965,979529,147989,622646,250569,332345,752258,895180,938652,733060,337864,917380,884454,713905,357003,768038,327683,783686,838961,281827,109336,577423,496945,674142,61920\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 74\n", + "Returned IDs: 293365,566134,747631,408762,57112,485429,874144,613039,329984,134932,739786,574123,773101,694788,39635,155378,630526,856657,545466,142005,805639,36971,869349,633551,481281,410686,899851,72689,443311,424779\n", + "Ground Truth: 293365,566134,747631,408762,57112,485429,874144,613039,329984,134932,739786,574123,773101,694788,39635,155378,630526,856657,545466,142005,776930,805639,36971,869349,633551,481281,410686,899851,72689,443311\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 75\n", + "Returned IDs: 270457,861440,632605,204160,497340,587475,202880,681223,178678,742203,375557,606800,936440,648433,726689,476593,653294,481462,516854,465745,265912,963513,982957,145450,425116,179305,875171,373928,740304,345127\n", + "Ground Truth: 270457,861440,632605,204160,497340,587475,202880,681223,178678,742203,375557,606800,936440,648433,726689,476593,653294,481462,516854,551466,465745,225250,485216,981056,265912,963513,982957,945993,145450,112929\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 77\n", + "Returned IDs: 250345,544808,486280,591053,74326,824011,975451,858898,204407,166067,904834,771553,515935,703439,897870,593024,699845,69423,54911,636335,864877,545787,890133,592947,368898,428688,432883,437649,139574,327748\n", + "Ground Truth: 250345,544808,486280,591053,74326,824011,160506,975451,858898,204407,166067,904834,771553,515935,703439,897870,593024,699845,69423,54911,636335,864877,545787,890133,592947,368898,432883,428688,437649,139574\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 79\n", + "Returned IDs: 209093,774492,640350,510809,115750,472848,803982,943480,441347,681266,1584,590786,24366,562163,960001,517488,168668,135491,676808,191053,373425,348118,488717,712943,463703,4009,13489,471163,967408,884397\n", + "Ground Truth: 209093,774492,640350,510809,115750,472848,228077,803982,943480,14945,441347,681266,1584,590786,24366,562163,960001,517488,477516,168668,135491,571953,7314,676808,191053,373425,348118,488717,712943,463703\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 81\n", + "Returned IDs: 879203,204905,958198,800603,210082,710583,569036,535309,670134,255197,875726,269020,764797,750116,665935,28976,770599,81809,901921,815496,890326,799424,541892,736540,362633,50422,863466,212258,423031,425385\n", + "Ground Truth: 879203,204905,958198,800603,210082,710583,569036,535309,670134,255197,875726,269020,764797,750116,501423,665935,28976,723751,770599,81809,901921,897652,152057,815496,181128,890326,799424,541892,462083,736540\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 84\n", + "Returned IDs: 700578,170730,910449,262180,765030,301351,309342,969655,111568,144380,908306,546145,888998,544148,246788,658561,290,303550,399085,120918,984431,429415,328818,443271,289261,36053,241251,617923,377430,560682\n", + "Ground Truth: 700578,170730,910449,262180,765030,301351,309342,969655,111568,144380,908306,546145,888998,544148,246788,658561,290,303550,941410,399085,357651,413742,120918,984431,429415,328818,443271,289261,36053,241251\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 85\n", + "Returned IDs: 724168,586281,877937,41578,78824,479608,691618,643079,318203,957263,32871,759205,746933,298679,341233,692310,341520,127241,571259,365886,819801,522376,709650,339050,530275,572875,594865,705241,29977,500760\n", + "Ground Truth: 724168,586281,877937,41578,78824,479608,691618,643079,318203,957263,32871,759205,746933,298679,341233,692310,341520,127241,571259,365886,819801,522376,709650,339050,379429,530275,572875,594865,705241,733502\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 91\n", + "Returned IDs: 19310,87228,310741,462905,286995,708069,539923,53855,411732,445148,577778,896003,220444,745699,226754,932691,239553,594009,556136,872572,580428,989691,953289,328014,129858,483406,153000,752153,928224,918027\n", + "Ground Truth: 19310,87228,310741,462905,286995,708069,539923,411732,53855,445148,577778,896003,220444,745699,226754,932691,239553,594009,556136,872572,580428,568264,989691,953289,909080,328014,129858,483406,153000,752153\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 93\n", + "Returned IDs: 480881,819734,682918,122495,250268,471117,517516,646565,549230,96638,784730,916167,459865,314101,92519,275201,308222,207509,497959,23811,355975,24421,965809,748100,968847,396868,772745,650805,803631,659229\n", + "Ground Truth: 480881,819734,682918,122495,791296,250268,471117,517516,646565,549230,96638,784730,916167,459865,314101,92519,275201,308222,207509,497959,23811,355975,24421,965809,748100,968847,396868,772745,650805,803631\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 95\n", + "Returned IDs: 986827,594271,176925,104444,883366,683471,931364,860983,73656,872989,843514,332990,651039,742142,898192,138918,382330,636316,93491,578177,430029,258234,22649,610379,768980,858837,323100,142801,917837,320815\n", + "Ground Truth: 782138,986827,594271,176925,749208,104444,883366,113441,580695,683471,931364,860983,73656,642231,872989,498042,843514,332990,651039,742142,898192,138918,117084,382330,636316,93491,722872,578177,410920,191865\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 97\n", + "Returned IDs: 319647,909281,227241,184788,448801,842885,372737,264335,876468,91576,334249,593760,362219,344550,631322,561529,930256,13115,575858,451349,215504,288237,381496,328405,612262,211018,815779,702026,381373,935306\n", + "Ground Truth: 319647,909281,227241,184788,448801,842885,372737,264335,876468,91576,334249,593760,362219,344550,631322,561529,930256,13115,575858,451349,215504,288237,381496,328405,612262,211018,885514,815779,702026,331664\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 99\n", + "Returned IDs: 574897,812478,294173,820038,549565,720663,748624,732980,149898,745733,739720,768280,141523,412944,498809,349562,166812,510019,28236,146061,608505,874668,179037,598222,474120,652532,250509,201124,891095,556454\n", + "Ground Truth: 574897,812478,294173,820038,549565,720663,748624,732980,149898,745733,739720,768280,141523,412944,498809,349562,166812,510019,849147,28236,146061,608505,874668,179037,974176,598222,474120,652532,250509,201124\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 100\n", + "Returned IDs: 599706,324713,47558,100309,327436,359908,788657,600079,324606,474,659285,168977,516541,513205,31515,137882,24529,964714,374157,911583,728183,525813,669455,427889,420317,494333,503269,787272,232726,249800\n", + "Ground Truth: 599706,324713,47558,100309,327436,359908,788657,600079,324606,474,659285,168977,516541,513205,31515,137882,24529,606190,964714,374157,911583,728183,525813,669455,427889,420317,494333,503269,787272,232726\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 105\n", + "Returned IDs: 104399,85039,233287,622802,930544,259545,860666,435437,827484,696234,724181,932849,277388,416062,100336,626521,51306,449120,490903,482919,911846,366157,771228,210682,512627,831032,85616,444779,809190,181900\n", + "Ground Truth: 104399,85039,233287,622802,930544,259545,860666,396706,435437,827484,163760,696234,724181,422532,932849,277388,416062,100336,626521,51306,449120,245328,490903,482919,274693,522024,911846,366157,771228,210682\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 106\n", + "Returned IDs: 870236,670134,26105,36909,905617,580357,753090,294991,73954,145450,603607,57581,268367,985759,237355,800528,775487,815856,954615,920107,962399,670922,556160,953485,204160,607535,148995,563108,688286,87493\n", + "Ground Truth: 870236,670134,26105,36909,905617,580357,753090,294991,73954,145450,603607,57581,268367,985759,864361,325852,237355,800528,775487,815856,954615,920107,962399,670922,556160,953485,204160,607535,148995,563108\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 107\n", + "Returned IDs: 758628,74785,461888,108492,534419,172052,321983,207667,798103,197798,430556,733636,561529,635018,30734,323100,583278,477731,920014,821246,907476,589155,594910,684006,125322,964621,434250,475185,330838,410378\n", + "Ground Truth: 758628,74785,461888,604867,108492,534419,172052,321983,207667,798103,197798,430556,733636,561529,635018,30734,323100,583278,477731,920014,821246,907476,589155,594910,684006,125322,964621,434250,475185,330838\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 108\n", + "Returned IDs: 683268,173502,166325,479728,52008,606695,246486,542432,626602,798814,724017,25375,926618,615906,962062,171920,880974,721929,760577,153471,258016,597061,48711,139629,386810,848622,224101,971463,148816,324007\n", + "Ground Truth: 683268,173502,166325,479728,52008,606695,246486,542432,626602,798814,724017,25375,926618,615906,962062,171920,880974,721929,760577,153471,496563,258016,597061,48711,139629,386810,848622,224101,971463,148816\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 109\n", + "Returned IDs: 983521,750571,256883,270788,552390,693683,358867,1678,202646,584189,926972,241578,238185,520667,685632,726748,884267,202333,373054,910872,236952,304311,149941,630096,768158,174012,192032,736462,839579,307955\n", + "Ground Truth: 983521,750571,256883,270788,552390,693683,358867,1678,202646,584189,926972,241578,238185,520667,756150,685632,726748,782900,884267,202333,373054,910872,236952,304311,149941,630096,768158,174012,192032,736462\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 114\n", + "Returned IDs: 828553,302817,771183,545372,707968,519418,174068,617228,893919,60666,83940,950722,349034,13212,46733,472685,281009,652050,173810,239608,909932,725121,967343,372374,442863,443577,287141,597169,650426,440627\n", + "Ground Truth: 828553,302817,771183,545372,707968,519418,174068,617228,893919,60666,83940,950722,349034,13212,46733,472685,281009,652050,173810,239608,909932,725121,967343,727361,635958,372374,442863,443577,287141,967917\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 116\n", + "Returned IDs: 421028,655436,379088,129237,278852,431284,303228,385802,650692,218503,231162,406104,256395,773252,364187,884706,635781,362233,597456,693711,235436,846242,830810,131836,894398,388722,50088,274829,388354,238265\n", + "Ground Truth: 421028,655436,379088,129237,278852,431284,303228,385802,650692,218503,231162,406104,256395,773252,364187,884706,635781,362233,597456,693711,235436,846242,830810,131836,894398,388722,50088,900465,274829,388354\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 117\n", + "Returned IDs: 913461,186350,705927,772115,149431,327061,149035,495713,691320,683792,808999,289480,27650,733818,629828,982574,522958,772747,549321,591264,558865,588446,368811,49354,236933,597081,828549,876623,442442,135842\n", + "Ground Truth: 913461,186350,705927,772115,149431,327061,149035,495713,691320,683792,808999,289480,27650,733818,629828,62726,982574,522958,772747,549321,591264,558865,588446,368811,49354,236933,597081,828549,876623,442442\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 118\n", + "Returned IDs: 309885,611973,204016,111578,211608,259929,365666,480679,948740,218105,445693,617693,128482,193632,537470,504970,556479,28524,278317,688900,643412,773990,758362,940780,625469,841244,102060,229254,732333,123589\n", + "Ground Truth: 309885,611973,204016,111578,211608,259929,365666,480679,11276,948740,218105,445693,617693,128482,193632,537470,504970,556479,28524,278317,688900,643412,773990,758362,940780,625469,841244,102060,229254,732333\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 120\n", + "Returned IDs: 657539,213079,951739,290875,397766,976040,52115,236793,562699,413665,557749,865968,453590,574718,533820,237792,678710,943178,481305,290104,606294,555566,729273,889779,397945,199967,47162,912908,934771,257049\n", + "Ground Truth: 657539,213079,951739,290875,397766,976040,52115,236793,562699,413665,557749,865968,453590,574718,533820,237792,678710,943178,173173,481305,290104,606294,555566,729273,889779,397945,199967,47162,912908,934771\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 121\n", + "Returned IDs: 137066,350568,446055,476614,699082,92519,169137,690404,790149,79291,520449,517735,672059,62256,243121,865547,726975,55444,626057,216923,468421,338365,659229,221351,194794,435330,812481,194376,545132,229519\n", + "Ground Truth: 137066,350568,446055,476614,699082,92519,169137,690404,790149,79291,520449,517735,672059,62256,243121,865547,726975,517730,55444,626057,216923,468421,338365,659229,221351,194794,435330,812481,217433,194376\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 129\n", + "Returned IDs: 734143,389648,122180,337716,239671,95517,649281,743681,613289,214882,862606,411755,204130,210160,760684,749090,895164,92812,436389,733239,77990,385333,258752,962283,556727,480027,942978,195759,822698,966991\n", + "Ground Truth: 734143,389648,122180,337716,239671,95517,649281,743681,613289,214882,862606,411755,204130,210160,760684,749090,895164,92812,436389,733239,77990,385333,258752,962283,556727,480027,179435,942978,195759,822698\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 130\n", + "Returned IDs: 397751,725506,378789,152546,601972,823982,301636,760534,732364,633368,712408,297283,845346,143584,215945,531498,984638,919426,805180,239967,624165,956308,890188,189734,766140,67881,21085,822001,561113,454851\n", + "Ground Truth: 397751,725506,378789,152546,601972,823982,301636,760534,732364,633368,729582,712408,297283,845346,143584,215945,531498,984638,919426,805180,239967,624165,956308,890188,189734,766140,67881,21085,822001,561113\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 132\n", + "Returned IDs: 130164,761418,952950,785887,4480,629413,332797,552220,253914,783795,98665,670815,538823,786797,967870,907090,614194,457691,770151,741106,28551,841800,656808,303162,108999,449935,674347,223326,478978,308890\n", + "Ground Truth: 130164,761418,952950,785887,4480,629413,332797,552220,253914,783795,98665,670815,538823,786797,967870,907090,614194,457691,770151,741106,28551,841800,392363,656808,303162,108999,449935,674347,223326,478978\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 135\n", + "Returned IDs: 703645,497731,706705,883660,330113,634696,375663,405927,842712,963845,460431,199408,263796,496199,393816,228083,43311,554660,866635,748100,504871,250885,765279,12812,761657,931112,606031,83807,537531,305330\n", + "Ground Truth: 703645,497731,706705,883660,330113,634696,375663,405927,424540,842712,963845,460431,199408,263796,496199,393816,228083,43311,554660,866635,748100,504871,250885,765279,239595,12812,761657,931112,346310,606031\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 139\n", + "Returned IDs: 6017,14964,776041,242625,641024,697795,458466,532383,640954,290578,977123,799113,192825,78583,777538,363759,697593,317838,193212,508879,804800,460171,225335,925009,611096,57485,886625,121875,957713,238929\n", + "Ground Truth: 6017,14964,776041,242625,641024,697795,458466,352087,532383,640954,290578,977123,799113,192825,78583,777538,363759,697593,567560,317838,193212,508879,804800,460171,225335,925009,787560,611096,40133,257097\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 140\n", + "Returned IDs: 402047,91289,396789,15335,509841,417002,695508,659239,38466,450159,7753,599688,818329,556866,700431,915421,664172,55791,161439,448200,497928,60040,389022,251996,95512,550208,183479,916915,241812,914584\n", + "Ground Truth: 402047,91289,396789,15335,509841,417002,695508,659239,38466,450159,7753,599688,818329,556866,700431,915421,664172,55791,161439,448200,497928,60040,389022,251996,95512,924963,550208,183479,916915,241812\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 142\n", + "Returned IDs: 77562,720452,791456,49275,668921,218728,398795,176860,121394,499795,223898,535434,623024,569337,840231,903396,735759,790172,67045,187350,149290,101125,433571,210457,447986,683512,344110,50647,468581,796908\n", + "Ground Truth: 853280,833976,77562,720452,791456,49275,668921,218728,398795,176860,121394,499795,223898,535434,623024,569337,840231,903396,735759,790172,67045,187350,149290,101125,433571,210457,447986,683512,344110,50647\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 144\n", + "Returned IDs: 366790,940327,786112,326662,973368,151928,725916,978557,950787,181844,397853,827444,628589,143835,150218,752213,748518,495406,110725,750340,309386,214953,333678,888928,174594,42516,809775,943874,478515,817895\n", + "Ground Truth: 366790,447010,940327,786112,326662,973368,151928,725916,978557,950787,181844,397853,827444,328199,628589,143835,754044,150218,752213,748518,495406,110725,750340,309386,214953,333678,888928,174594,42516,809775\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 146\n", + "Returned IDs: 605123,406077,597811,415054,46353,103754,69518,371932,501704,737834,933325,805788,929208,340431,807434,858534,269965,925239,726282,568081,666014,691313,726353,299153,950101,357003,525529,20570,908444,911043\n", + "Ground Truth: 605123,406077,597811,415054,46353,103754,69518,371932,501704,737834,933325,805788,929208,644085,340431,807434,858534,269965,925239,726282,568081,666014,691313,726353,299153,950101,357003,525529,20570,908444\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 148\n", + "Returned IDs: 963455,275273,166103,698345,587330,812632,126328,75520,513421,794504,679648,628275,337632,260088,745205,92466,224343,271503,179018,566449,988298,731704,119867,82790,545254,99748,203610,399531,41924,796886\n", + "Ground Truth: 963455,275273,166103,698345,587330,812632,126328,75520,513421,794504,679648,628275,337632,260088,745205,6647,92466,600943,224343,271503,179018,566449,988298,731704,119867,82790,545254,99748,203610,399531\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 149\n", + "Returned IDs: 735418,232010,571369,223050,787330,902483,427109,954505,854287,388704,366371,240447,155649,678234,313108,380488,369547,60814,635128,972728,599642,880036,546937,398275,564338,424644,702210,334303,297908,116847\n", + "Ground Truth: 735418,232010,571369,223050,787330,902483,427109,954505,854287,388704,366371,240447,155649,678234,313108,380488,369547,60814,635128,573141,972728,599642,880036,546937,398275,564338,424644,702210,334303,297908\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 154\n", + "Returned IDs: 297265,954748,620655,362217,568883,949219,850436,436588,796304,439867,912731,185399,733743,570630,530241,809357,345226,162763,255978,868463,172557,212127,845754,178887,376708,769769,523487,259723,716460,136518\n", + "Ground Truth: 297265,954748,620655,362217,568883,949219,850436,436588,796304,439867,912731,185399,733743,570630,530241,809357,345226,162763,255978,868463,172557,212127,845754,178887,376708,769769,523487,259723,716460,62864\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 158\n", + "Returned IDs: 437900,948497,725048,940583,654292,61229,473966,12651,931509,929848,503322,245492,127194,45723,899557,564041,906651,194654,433918,649461,713082,921954,836300,788310,82943,846658,951330,884714,464990,811313\n", + "Ground Truth: 437900,948497,725048,940583,654292,61229,473966,12651,931509,929848,503322,245492,127194,45723,899557,564041,906651,194654,433918,649461,713082,921954,836300,758241,332127,788310,82943,846658,951330,884714\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 170\n", + "Returned IDs: 117394,446515,1885,662652,987719,655867,19428,906150,300162,445078,570747,566803,933010,900824,880809,607597,74642,940177,568647,220216,113232,39117,637024,689703,738723,567057,661709,961044,165367,604068\n", + "Ground Truth: 117394,446515,1885,662652,987719,655867,19428,906150,300162,445078,570747,566803,933010,900824,880809,607597,74642,940177,568647,220216,113232,39117,769342,637024,689703,738723,567057,661709,961044,165367\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 173\n", + "Returned IDs: 245559,197229,306640,571521,902348,29498,353845,315315,291864,955056,258695,798606,232088,802605,741077,89843,336323,984433,647552,687225,550012,162412,131898,336361,272868,941810,831097,510405,160619,882850\n", + "Ground Truth: 245559,197229,306640,556701,571521,902348,29498,353845,315315,291864,955056,912243,529633,289726,896354,258695,798606,676632,232088,802605,741077,89843,336323,984433,647552,687225,550012,553036,162412,907353\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 176\n", + "Returned IDs: 271059,643685,479004,147714,785012,103714,746206,456565,123437,196690,246788,256821,553067,898405,871467,869729,542022,662659,148278,761156,544861,192792,766391,472281,577392,299395,858313,421380,572253,73544\n", + "Ground Truth: 211672,271059,316929,643685,479004,110544,147714,785012,103714,746206,456565,123437,196690,246788,256821,553067,898405,871467,869729,542022,662659,148278,761156,544861,192792,215597,766391,472281,74339,577392\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 178\n", + "Returned IDs: 445444,173059,559089,129629,5575,399183,984199,209275,497265,533604,906834,694445,6672,831758,250409,416996,819827,630806,850785,775618,530955,192418,469184,877511,367737,795403,629468,253280,715679,201767\n", + "Ground Truth: 445444,173059,559089,129629,219925,5575,399183,984199,209275,497265,533604,906834,694445,6672,373832,831758,250409,416996,819827,630806,850785,775618,8663,530955,192418,469184,877511,367737,795403,629468\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 181\n", + "Returned IDs: 98079,785599,2850,524471,217631,240254,254800,440838,54618,89057,539213,677304,873157,260160,37303,719376,478951,578343,897497,584079,396674,487818,623730,892465,897467,307715,684886,358280,987303,382890\n", + "Ground Truth: 98079,785599,2850,524471,217631,240254,254800,440838,54618,89057,539213,677304,873157,260160,37303,719376,27134,478951,578343,897497,584079,396674,487818,623730,892465,897467,307715,813626,684886,358280\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 184\n", + "Returned IDs: 823189,358770,509049,582180,459539,574930,506509,464501,635128,154214,299515,485709,734561,168649,590909,898863,263207,583901,194650,406192,602252,644042,679367,146135,708731,647345,68530,775223,482130,550396\n", + "Ground Truth: 823189,358770,509049,582180,459539,574930,506509,464501,635128,154214,299515,485709,734561,168649,590909,898863,263207,583901,194650,406192,602252,644042,679367,146135,708731,647345,68530,188854,775223,224180\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 197\n", + "Returned IDs: 623527,768980,631216,172052,785639,461888,321983,683471,288678,204901,74785,533987,748631,841117,477731,56347,550876,857163,566851,508513,834243,328481,589155,177681,756075,207667,68720,258234,686901,61754\n", + "Ground Truth: 623527,768980,631216,515404,172052,785639,461888,321983,683471,288678,204901,74785,533987,748631,841117,477731,56347,945948,550876,857163,566851,508513,834243,328481,589155,177681,756075,207667,68720,258234\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 199\n", + "Returned IDs: 909484,682475,604420,194351,904328,552724,531103,542345,599414,377380,715545,476603,261093,397525,36078,649285,201076,518806,15971,723983,728128,870046,470101,345157,873582,419876,104968,116152,205661,957478\n", + "Ground Truth: 909484,386821,682475,604420,194351,904328,552724,531103,542345,599414,377380,715545,476603,261093,397525,36078,649285,201076,518806,15971,517954,723983,728128,870046,470101,345157,873582,419876,104968,116152\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 200\n", + "Returned IDs: 674724,538768,193458,212925,851462,529826,381653,674122,555850,629123,670644,168255,174995,509207,620065,314182,111390,74806,950332,986919,25243,67133,855156,245492,960916,724617,30738,228870,732256,892694\n", + "Ground Truth: 674724,538768,193458,212925,851462,529826,381653,674122,555850,629123,670644,168255,174995,509207,620065,314182,111390,74806,289231,950332,986919,25243,67133,855156,245492,960916,724617,30738,228870,732256\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 203\n", + "Returned IDs: 805758,759303,902471,745320,274591,606321,863380,526343,519909,786906,823800,444099,4497,277435,410898,596254,701303,352643,272576,944271,208369,838345,396944,653285,674309,886443,954654,555834,881902,867562\n", + "Ground Truth: 805758,759303,902471,745320,274591,606321,863380,526343,519909,786906,823800,444099,4497,277435,410898,596254,701303,352643,272576,944271,208369,838345,396944,653285,674309,678123,886443,954654,100321,555834\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 211\n", + "Returned IDs: 729850,121585,895295,167500,699428,293455,454496,153115,650553,988387,675915,324814,57796,3570,494746,184373,127140,958051,151447,202514,769247,383248,499090,982678,376115,76640,900646,35558,709843,226570\n", + "Ground Truth: 729850,121585,895295,167500,699428,293455,454496,153115,650553,988387,675915,324814,57796,3570,494746,184373,127140,958051,151447,712788,847638,202514,769247,383248,499090,982678,376115,76640,900646,35558\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 214\n", + "Returned IDs: 913787,824306,22317,11040,404308,905082,736813,412251,465362,369246,721600,233162,977275,104209,266553,965199,931608,914701,664818,563048,79974,864877,185438,562084,629581,505744,486158,342175,29743,751146\n", + "Ground Truth: 913787,824306,22317,11040,404308,905082,736813,412251,465362,369246,721600,233162,977275,104209,31162,266553,965199,931608,914701,664818,563048,79974,185438,864877,562084,629581,505744,486158,342175,29743\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 215\n", + "Returned IDs: 474855,281914,813635,329072,358481,492076,946802,710308,185795,3569,888785,288716,35656,742150,109037,436268,397889,920689,18443,532960,535533,158530,544132,1963,829372,154951,770223,393182,291322,446905\n", + "Ground Truth: 474855,281914,813635,329072,358481,492076,946802,710308,249244,185795,3569,888785,288716,35656,742150,109037,436268,397889,920689,18443,532960,535533,158530,544132,1963,829372,154951,770223,393182,291322\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 216\n", + "Returned IDs: 523129,166510,263219,17945,19391,398467,125980,178134,647298,891102,251448,27138,90339,451291,467762,558878,961447,655615,711513,434544,929962,900993,566228,379767,681717,199239,625474,723161,380431,57392\n", + "Ground Truth: 523129,166510,263219,17945,19391,398467,125980,178134,647298,891102,251448,27138,90339,451291,467762,558878,961447,655615,711513,434544,929962,900993,961798,566228,379767,681717,199239,625474,723161,380431\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 221\n", + "Returned IDs: 278166,261319,350660,745077,125401,590776,111887,420444,238347,816509,818019,153569,454118,352639,378647,127568,797652,236692,449181,526913,104258,255692,960545,674152,628275,146289,120127,525709,967788,247445\n", + "Ground Truth: 278166,261319,350660,745077,125401,590776,111887,420444,238347,816509,818019,153569,454118,352639,378647,127568,797652,236692,449181,526913,104258,255692,960545,674152,628275,146289,120127,525709,537230,967788\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 225\n", + "Returned IDs: 34397,466849,852160,17048,521614,496387,114018,814780,873966,451658,440849,120137,191649,218774,81044,674921,320365,753936,878214,931125,142644,759450,760163,979520,975820,813801,965496,366079,642111,705726\n", + "Ground Truth: 34397,466849,852160,17048,521614,496387,114018,814780,873966,451658,440849,120137,191649,218774,81044,674921,320365,753936,878214,931125,142644,759450,133763,760163,979520,975820,813801,965496,366079,642111\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 226\n", + "Returned IDs: 836206,23960,668489,373812,420078,689569,233278,623236,559422,891056,691442,910854,781639,341264,270504,105073,205068,59780,32904,988193,398514,785332,272646,389800,324381,84941,239000,418997,814230,248822\n", + "Ground Truth: 836206,23960,668489,373812,420078,689569,233278,623236,559422,891056,691442,910854,781639,341264,270504,105073,205068,59780,32904,988193,398514,785332,272646,389800,951010,324381,84941,239000,418997,814230\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 245\n", + "Returned IDs: 238675,601033,849540,711345,936699,859908,646367,385805,173197,557338,688729,780433,67528,275883,661377,766421,797477,883425,638650,381681,608915,546076,301460,438598,780789,667180,547877,453554,859282,818951\n", + "Ground Truth: 238675,601033,849540,711345,936699,859908,646367,385805,173197,557338,688729,780433,67528,275883,661377,766421,797477,883425,638650,381681,608915,546076,301460,438598,780789,205313,667180,547877,453554,859282\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 248\n", + "Returned IDs: 370350,273458,433624,222387,106220,83940,567888,7898,578716,24911,52728,17906,610440,809130,929415,75276,89672,610794,564613,57311,94618,174068,523672,665109,516946,274012,418826,582644,979890,734178\n", + "Ground Truth: 370350,273458,433624,222387,106220,83940,567888,7898,578716,24911,52728,17906,610440,445747,809130,929415,75276,89672,610794,564613,57311,94618,174068,523672,665109,516946,274012,418826,582644,979890\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 249\n", + "Returned IDs: 359927,734269,407089,8211,276596,504312,678597,639694,866661,615923,433540,952149,956452,812522,603224,505268,580188,395363,624693,592359,479138,888028,162318,56767,440797,724246,761966,247314,919150,779952\n", + "Ground Truth: 359927,734269,712188,816880,407089,8211,276596,504312,678597,639694,866661,615923,433540,952149,956452,812522,603224,505268,580188,395363,624693,592359,457178,479138,888028,162318,56767,440797,724246,761966\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 254\n", + "Returned IDs: 971127,112696,467682,929224,784103,399371,42069,187311,247056,183715,705075,157398,903943,669772,247351,526657,440729,822874,206353,75584,902213,651122,821015,109770,706917,500028,260090,250118,201578,718946\n", + "Ground Truth: 971127,112696,467682,929224,784103,399371,42069,187311,247056,183715,705075,157398,903943,669772,247351,526657,440729,822874,206353,75584,733093,902213,651122,821015,109770,706917,500028,260090,250118,201578\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 256\n", + "Returned IDs: 662729,810204,651983,466095,231116,720605,327912,833916,695476,692089,850848,14508,373752,285783,286816,189073,312927,255293,721065,406505,578187,917206,660095,32134,428259,368301,341397,294012,164692,810403\n", + "Ground Truth: 662729,810204,651983,466095,231116,506349,720605,327912,833916,695476,692089,850848,14508,373752,285783,286816,189073,312927,255293,721065,406505,578187,917206,660095,32134,428259,368301,391481,341397,294012\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 257\n", + "Returned IDs: 520530,182575,656485,561016,739687,685199,801581,235842,413767,605712,126908,415211,529902,441079,792657,560139,301035,69005,116959,194654,561617,373723,197787,960762,580734,816375,144089,245492,780008,865618\n", + "Ground Truth: 520530,182575,656485,561016,739687,685199,801581,235842,413767,605712,126908,415211,529902,441079,792657,560139,301035,69005,116959,194654,561617,373723,197787,108046,960762,309942,580734,655851,816375,144089\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 261\n", + "Returned IDs: 538874,622232,968140,633499,843757,919475,666794,976944,5989,135538,631178,557546,549322,933329,530074,205724,797740,262864,892477,589194,59732,989983,900342,160924,259570,657979,258880,615710,560775,486677\n", + "Ground Truth: 538874,622232,968140,633499,843757,919475,666794,102636,976944,5989,135538,631178,557546,549322,933329,530074,226093,205724,797740,610528,262864,892477,589194,754176,59732,989983,900342,165563,160924,259570\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 266\n", + "Returned IDs: 278496,408425,256281,768978,543165,533511,350257,570329,342446,792275,485913,378098,670032,857889,891953,501260,978501,115595,144246,560644,618852,574231,844961,395736,782423,15457,824529,200188,904999,7380\n", + "Ground Truth: 278496,408425,256281,768978,543165,533511,350257,570329,342446,792275,485913,378098,670032,857889,891953,501260,978501,115595,144246,560644,653771,618852,574231,844961,395736,782423,15457,824529,200188,904999\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 268\n", + "Returned IDs: 557064,375571,952311,670178,295002,478185,350338,421707,488108,758266,565159,324048,424876,506978,850293,844653,542536,549070,367195,195722,845135,248094,706816,298515,832928,920969,356897,119086,286033,222095\n", + "Ground Truth: 557064,375571,952311,670178,295002,478185,350338,421707,488108,758266,565159,324048,424876,506978,850293,844653,542536,549070,367195,619332,195722,845135,248094,706816,298515,832928,920969,569992,356897,119086\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 272\n", + "Returned IDs: 523630,303153,616372,68676,911212,810475,518132,713073,568826,108100,187449,292580,785332,873159,59989,713333,522224,722063,897236,92,864588,326828,178364,600470,560668,988193,747308,272646,841694,481276\n", + "Ground Truth: 523630,303153,616372,68676,911212,810475,518132,713073,568826,108100,187449,292580,785332,873159,59989,713333,522224,579958,722063,897236,92,864588,326828,178364,600470,44584,560668,988193,747308,272646\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 278\n", + "Returned IDs: 982206,759798,142184,868252,653417,667437,859895,171195,724552,789679,129180,204436,779080,662393,502914,346865,96591,813373,120813,972676,71153,802014,318216,117401,455711,367987,505637,661994,533502,733966\n", + "Ground Truth: 982206,759798,142184,868252,653417,667437,859895,171195,724552,789679,129180,204436,779080,662393,502914,346865,96591,433572,813373,932967,120813,972676,71153,802014,318216,117401,455711,539006,367987,397736\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 282\n", + "Returned IDs: 394634,837983,144549,334654,158644,702348,876659,109932,749497,759446,670539,690370,662708,661301,800057,354764,241926,360335,587013,288537,961329,58526,714113,781641,761728,1290,460974,530325,910991,185661\n", + "Ground Truth: 394634,837983,144549,334654,158644,702348,876659,308000,270788,109932,749497,759446,670539,690370,662708,661301,800057,516432,316993,354764,241926,360335,587013,614127,288537,961329,58526,714113,781641,761728\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 283\n", + "Returned IDs: 53143,820863,980583,30807,921317,150131,34909,341924,19066,182920,717365,485724,759630,472075,961073,314545,251267,528507,933895,151479,5243,532772,446422,988019,420715,15415,621800,352636,915758,438831\n", + "Ground Truth: 53143,820863,980583,30807,921317,150131,34909,341924,19066,182920,717365,485724,759630,472075,961073,957794,314545,251267,528507,933895,151479,5243,532772,446422,829726,988019,420715,15415,621800,352636\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 284\n", + "Returned IDs: 324202,953996,510452,443900,797619,693962,355920,564006,737779,683263,472466,217520,329675,61405,929750,77971,637761,978748,931803,262938,260517,900117,302033,390114,302523,660890,823316,13656,216950,677307\n", + "Ground Truth: 324202,953996,510452,443900,797619,693962,355920,564006,737779,683263,472466,217520,329675,61405,929750,77971,637761,19403,978748,931803,262938,260517,900117,302033,390114,302523,660890,705159,426971,823316\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 285\n", + "Returned IDs: 189396,273979,715785,137419,846796,219595,551574,650907,576985,247409,142197,45230,745903,283183,915427,50878,660483,345528,612299,480741,983345,198707,383532,185593,37251,847199,55605,759159,728087,55633\n", + "Ground Truth: 189396,273979,715785,137419,846796,219595,551574,650907,576985,247409,142197,45230,745903,283183,915427,50878,660483,345528,612299,480741,983345,198707,383532,185593,37251,847199,55605,759159,757481,728087\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 289\n", + "Returned IDs: 147089,141162,386074,90785,32036,870594,41792,36853,608232,779273,598179,124720,441503,80061,676652,905330,109605,669208,292300,594432,23221,143759,800930,347089,598222,187744,371264,193383,453554,358248\n", + "Ground Truth: 147089,141162,386074,90785,32036,870594,41792,36853,608232,779273,598179,124720,441503,80061,676652,905330,109605,669208,292300,594432,57892,23221,4721,143759,800930,347089,598222,187744,371264,193383\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 290\n", + "Returned IDs: 933685,486977,904036,127539,302624,779714,359939,947243,539914,158559,230850,325887,341379,569269,575869,630846,209185,305931,933286,261292,523900,465685,571698,727546,140592,963215,440463,65679,890730,509924\n", + "Ground Truth: 933685,486977,904036,127539,302624,779714,359939,947243,539914,158559,230850,325887,341379,569269,575869,630846,209185,305931,933286,97096,261292,523900,465685,571698,727546,140592,963215,440463,65679,890730\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 295\n", + "Returned IDs: 77494,269632,670970,463595,250226,639660,867603,419994,580715,137327,377682,107025,469811,612536,629409,353820,751866,96869,494421,233645,314635,197107,949820,84430,108584,787240,418971,596932,944672,205625\n", + "Ground Truth: 77494,269632,670970,463595,250226,639660,867603,419994,580715,137327,377682,107025,469811,612536,629409,353820,751866,96869,494421,233645,314635,197107,949820,84430,979337,108584,787240,618214,418971,596932\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 299\n", + "Returned IDs: 395722,549247,731363,511381,491411,854362,289376,291036,741451,924599,430139,411675,59072,611023,725312,923163,27533,344332,304070,934016,137564,703467,168510,846981,567131,310026,960794,770271,732515,356391\n", + "Ground Truth: 395722,549247,731363,511381,491411,854362,289376,291036,741451,924599,430139,411675,59072,611023,725312,923163,27533,344332,304070,934016,137564,703467,168510,846981,567131,401819,310026,960794,770271,732515\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 306\n", + "Returned IDs: 147061,862080,706527,91071,397248,903113,327389,514520,943315,768629,582667,165802,11100,865812,376444,857894,551268,453427,244317,962363,131797,514380,199211,231592,822093,391972,692675,901644,439526,240791\n", + "Ground Truth: 147061,862080,706527,91071,397248,903113,327389,743603,514520,943315,768629,582667,165802,11100,865812,376444,857894,551268,453427,244317,962363,131797,514380,199211,231592,822093,391972,692675,901644,439526\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 308\n", + "Returned IDs: 384245,226961,402411,410779,144827,14996,387031,303843,900283,48961,233879,610787,775231,409606,162005,15636,722203,34147,929245,488766,980380,875600,629956,604832,880707,951560,870429,684523,823352,464663\n", + "Ground Truth: 384245,226961,402411,410779,144827,14996,387031,303843,900283,48961,102347,233879,610787,775231,409606,162005,15636,722203,34147,929245,488766,980380,921461,875600,629956,604832,880707,951560,870429,648896\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 310\n", + "Returned IDs: 933621,931437,348737,239316,336011,707335,287777,198148,75931,843265,700695,79414,503548,665192,907865,852663,826799,149143,448773,382486,574751,315670,218454,781113,344651,360510,963141,185588,387442,213581\n", + "Ground Truth: 933621,931437,348737,239316,336011,707335,287777,198148,75931,843265,700695,79414,503548,665192,907865,852663,826799,149143,448773,382486,574751,315670,218454,781113,650984,344651,360510,963141,185588,387442\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 311\n", + "Returned IDs: 337380,723106,203320,254905,856454,364943,121531,162155,119653,654240,96320,317929,474935,699852,221594,268643,916442,831213,471060,182716,294976,705253,918469,494746,640585,98406,178050,452384,239576,649065\n", + "Ground Truth: 337380,723106,203320,254905,856454,364943,121531,162155,119653,654240,96320,835331,317929,474935,699852,221594,268643,343527,916442,831213,471060,182716,294976,797214,705253,918469,494746,640585,98406,178050\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 315\n", + "Returned IDs: 239383,128883,773096,206657,111287,495544,641352,117884,227884,736311,497630,354691,837580,830496,547909,126822,413976,388284,723416,394112,126059,960,496151,832471,857891,941080,329291,696563,584206,557927\n", + "Ground Truth: 239383,128883,773096,206657,111287,495544,641352,117884,227884,736311,497630,354691,837580,830496,547909,126822,413976,388284,723416,394112,126059,960,496151,832471,857891,941080,329291,170538,696563,584206\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 317\n", + "Returned IDs: 450504,933123,388663,680748,886219,85130,838432,654311,456432,782638,327622,661821,478672,725747,136708,474549,368169,742624,917719,677021,708103,15337,667349,215375,813392,608014,975451,932773,837564,858011\n", + "Ground Truth: 450504,933123,388663,680748,886219,85130,838432,654311,456432,782638,327622,661821,478672,725747,136708,474549,368169,742624,917719,677021,708103,15337,667349,215375,813392,608014,975451,932773,837564,220444\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 319\n", + "Returned IDs: 837169,573129,250702,71625,967674,808442,962685,177548,191597,980565,294658,11218,608505,935366,770725,811589,217776,929284,344873,891095,698037,189859,334081,977414,923678,73944,205276,442698,750801,236498\n", + "Ground Truth: 837169,573129,250702,71625,967674,808442,962685,177548,191597,980565,294658,11218,608505,935366,770725,811589,217776,929284,344873,891095,698037,189859,334081,977414,923678,73944,205276,100596,442698,750801\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 320\n", + "Returned IDs: 98916,201781,301322,521266,514650,139651,403441,133884,115181,984001,648139,139578,376272,136233,646583,825901,289024,786394,584672,507224,864656,643267,984610,652491,738058,236447,343810,889773,607405,152478\n", + "Ground Truth: 98916,201781,301322,521266,514650,139651,403441,133884,115181,984001,648139,139578,376272,136233,646583,883918,576985,825901,289024,372780,786394,584672,507224,663116,864656,790432,643267,984610,652491,738058\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 322\n", + "Returned IDs: 822232,591645,969081,661340,209494,36092,412174,395810,873331,298127,592744,757357,607364,839315,39453,260557,746393,95314,738636,35554,24106,661193,24942,522712,433197,329342,592699,481783,435988,590175\n", + "Ground Truth: 822232,591645,969081,661340,209494,36092,412174,395810,873331,298127,592744,757357,607364,839315,39453,536807,260557,746393,95314,738636,35554,24106,661193,24942,522712,433197,329342,592699,481783,435988\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 327\n", + "Returned IDs: 767449,885140,828699,389380,437908,215330,455659,965241,518808,478494,254863,425749,852216,108367,916453,71814,284658,881621,905726,103307,142435,523912,652541,78224,957855,564508,874820,534158,709387,167766\n", + "Ground Truth: 767449,885140,828699,389380,437908,215330,455659,965241,518808,478494,254863,425749,296333,852216,108367,916453,71814,284658,881621,905726,103307,142435,523912,652541,78224,957855,822116,564508,874820,534158\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 331\n", + "Returned IDs: 632237,673093,949214,124645,42373,237807,270242,799351,67689,634295,187622,82549,334753,941753,651182,702277,894218,390093,98838,205071,388557,742232,485534,260461,981534,330172,91051,943057,724454,637145\n", + "Ground Truth: 632237,673093,949214,124645,42373,237807,270242,799351,67689,634295,187622,82549,334753,941753,651182,702277,894218,390093,98838,205071,388557,742232,485534,138878,260461,981534,330172,91051,943057,724454\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 332\n", + "Returned IDs: 282885,693868,667260,234787,968249,721493,758137,200123,485448,892013,826868,830470,158917,53855,727113,322073,651625,585707,663656,466174,937045,297344,94760,564341,160741,432146,230459,361116,448425,557225\n", + "Ground Truth: 282885,693868,667260,234787,968249,721493,758137,200123,485448,892013,826868,830470,158917,53855,727113,322073,651625,585707,663656,466174,937045,297344,94760,564341,160741,432146,230459,539992,361116,557225\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 341\n", + "Returned IDs: 730945,410821,647584,493725,794387,399586,386701,751012,399191,456044,662268,520429,946367,79167,966329,152703,346235,272675,411241,656090,483212,345910,314834,323552,380085,528136,871179,262431,975141,512847\n", + "Ground Truth: 730945,410821,647584,493725,794387,399586,386701,751012,399191,340297,456044,662268,520429,946367,606680,79167,966329,152703,346235,272675,411241,656090,483212,345910,314834,323552,380085,528136,871179,262431\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 343\n", + "Returned IDs: 58022,657480,701042,214283,105394,903341,352301,850477,542891,616295,676403,462139,868323,872676,690543,858456,102823,99022,708820,872119,539566,964769,325591,527888,829442,589810,5359,471673,186124,208635\n", + "Ground Truth: 58022,657480,701042,214283,105394,903341,352301,850477,542891,616295,676403,462139,868323,872676,690543,711810,858456,102823,99022,395359,708820,467462,872119,539566,964769,325591,527888,829442,589810,5359\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 344\n", + "Returned IDs: 445226,898932,85208,811563,739013,628984,375231,129826,842382,66257,509207,458501,822009,502850,928533,140713,817593,466432,114942,407280,274818,220033,725794,604263,290429,455279,973315,686285,375583,159841\n", + "Ground Truth: 445226,898932,85208,811563,739013,628984,375231,129826,842382,66257,509207,458501,822009,502850,928533,140713,817593,466432,114942,407280,274818,220033,332536,725794,604263,290429,455279,973315,686285,375583\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 345\n", + "Returned IDs: 739324,508813,483222,435482,826827,575673,790711,272952,268164,830086,168200,512346,475336,540275,277690,760494,911158,656653,73815,258966,793111,188545,953485,927482,74123,336137,694136,65854,374227,231749\n", + "Ground Truth: 739324,508813,483222,435482,826827,575673,790711,272952,268164,830086,168200,512346,475336,512136,540275,277690,760494,911158,656653,73815,258966,793111,188545,953485,927482,74123,336137,694136,459113,65854\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 350\n", + "Returned IDs: 811900,131135,246951,294867,949825,766421,332331,276691,942332,380959,344967,753892,218269,777569,235905,572825,844589,105928,456120,803892,731392,324643,577541,790373,261180,868595,179643,496058,378222,195168\n", + "Ground Truth: 811900,131135,246951,294867,949825,766421,332331,276691,942332,380959,344967,753892,218269,777569,235905,572825,765379,844589,105928,456120,900781,803892,731392,324643,577541,790373,261180,868595,179643,684195\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 351\n", + "Returned IDs: 531722,891443,743980,874484,101125,451502,835874,182699,901872,937538,139824,960228,693437,847469,660042,583566,740327,756075,61170,314182,195239,706903,714981,699627,625125,780540,342588,198364,792108,706040\n", + "Ground Truth: 531722,891443,743980,874484,101125,451502,835874,182699,901872,937538,139824,960228,693437,847469,660042,583566,740327,756075,61170,314182,195239,706903,714981,699627,625125,780540,342588,198364,792108,826276\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 353\n", + "Returned IDs: 343757,126830,987029,782305,378307,960649,762415,569317,539257,902508,559917,591964,766292,759732,727936,470722,49849,796785,847070,498090,641317,129500,599221,797139,773040,58603,495926,96142,546285,632573\n", + "Ground Truth: 343757,126830,987029,782305,378307,960649,762415,569317,539257,902508,559917,591964,766292,759732,727936,470722,49849,796785,603380,847070,498090,641317,129500,599221,797139,773040,58603,495926,486069,96142\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 355\n", + "Returned IDs: 454992,971228,752469,565543,648403,338525,839896,911182,957785,738229,127219,831117,430329,745771,850739,586415,807708,74519,139785,309299,684832,955092,384048,904218,444903,198307,147972,454331,665806,919632\n", + "Ground Truth: 454992,971228,752469,565543,648403,338525,839896,911182,957785,738229,127219,831117,430329,745771,850739,586415,807708,74519,139785,309299,684832,955092,384048,904218,349877,444903,198307,147972,454331,665806\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 365\n", + "Returned IDs: 786809,354410,459688,165495,740379,185847,687377,657423,19087,836530,230577,306994,321735,240033,163705,582071,917860,923123,290993,740599,832383,113206,327913,329701,474709,948969,519107,844822,766089,103798\n", + "Ground Truth: 786809,354410,107983,459688,165495,740379,185847,687377,657423,19087,836530,230577,306994,321735,240033,163705,582071,917860,923123,290993,740599,832383,113206,327913,329701,474709,948969,519107,844822,766089\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 366\n", + "Returned IDs: 960797,284658,89677,569346,282336,228656,372230,221141,83471,201723,337086,176155,901146,427971,646154,90210,192320,235787,695974,709822,84932,939629,820232,428677,8504,98246,659932,61093,787285,389746\n", + "Ground Truth: 960797,284658,43676,89677,569346,282336,228656,372230,221141,958462,768780,83471,201723,337086,157504,176155,901146,427971,646154,90210,192320,235787,713082,695974,709822,84932,939629,820232,428677,8504\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 369\n", + "Returned IDs: 432173,937696,950163,493752,819629,758369,467250,29001,21687,887631,775428,136949,301294,717008,263454,275909,846630,256701,218181,479178,291917,908022,934422,309679,359257,536649,305332,312575,835909,183180\n", + "Ground Truth: 432173,937696,950163,493752,819629,758369,467250,29001,21687,887631,775428,136949,301294,717008,263454,275909,846630,256701,218181,479178,291917,908022,934422,463780,309679,359257,536649,305332,312575,835909\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 370\n", + "Returned IDs: 88867,193458,491598,265912,879271,714343,609179,529826,911983,900954,145450,946452,751142,573207,935955,706584,75596,411221,581466,314395,752019,122630,384630,71535,524843,523192,742853,363519,631108,501135\n", + "Ground Truth: 88867,193458,491598,265912,879271,714343,215513,609179,529826,911983,900954,145450,946452,751142,573207,935955,706584,75596,411221,581466,314395,622745,752019,122630,384630,71535,524843,523192,742853,363519\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 379\n", + "Returned IDs: 86504,514427,631493,784430,163109,405390,552180,874891,472710,678867,457983,955007,556378,474544,270853,443261,99964,174808,980355,357003,761932,425585,467508,823980,503161,898306,697029,917942,366780,414037\n", + "Ground Truth: 86504,514427,631493,784430,163109,405390,552180,874891,472710,678867,457983,955007,556378,474544,270853,443261,99964,174808,980355,357003,761932,425585,309502,467508,823980,503161,898306,697029,917942,366780\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 389\n", + "Returned IDs: 922015,498708,319078,732758,377089,778248,818139,406838,566911,687756,957171,633939,284956,662740,368346,740082,425224,916421,423115,861193,589668,178035,580217,931456,715129,645397,389898,104219,401014,488040\n", + "Ground Truth: 197825,570279,922015,370284,689631,498708,319078,732758,778248,377089,818139,406838,52587,566911,286320,687756,957171,633939,284956,662740,368346,169404,740082,158897,425224,104856,916421,906323,423115,861193\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 394\n", + "Returned IDs: 363668,730141,933051,407367,841512,473875,652974,749220,595851,458879,18498,180680,354709,207667,774217,208195,857574,204901,394385,477731,469208,915210,74785,444970,468539,288678,243506,863685,431091,540275\n", + "Ground Truth: 363668,730141,933051,574031,407367,841512,473875,652974,749220,595851,458879,18498,447467,180680,354709,207667,774217,208195,857574,204901,394385,477731,469208,915210,74785,444970,468539,288678,243506,863685\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 395\n", + "Returned IDs: 860195,409212,545052,127016,912266,42313,506672,221215,430877,515449,715529,399786,115024,385484,324710,682639,889792,204538,242172,934778,734589,491424,390137,409731,344930,595238,383075,131932,519334,386668\n", + "Ground Truth: 860195,409212,545052,127016,912266,42313,573515,506672,221215,430877,515449,715529,399786,115024,784018,385484,324710,682639,889792,204538,242172,934778,734589,491424,390137,409731,344930,595238,383075,131932\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 399\n", + "Returned IDs: 5433,925753,675254,442603,202088,529510,183529,739956,13197,317454,226134,396668,407855,890743,949078,800982,279107,208039,333395,894377,55838,160409,733265,34677,666011,235328,357615,414019,785779,159136\n", + "Ground Truth: 5433,925753,675254,442603,202088,529510,183529,183264,739956,13197,317454,226134,266996,396668,407855,890743,949078,800982,279107,208039,333395,894377,55838,160409,733265,34677,666011,235328,357615,414019\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 400\n", + "Returned IDs: 886180,59136,194431,413437,588346,264061,190727,208572,573230,911038,913419,851238,269486,730936,11226,303693,338860,133127,60355,459283,74269,299128,247874,473100,521762,406357,621517,739471,839866,816348\n", + "Ground Truth: 886180,59136,194431,413437,588346,264061,190727,208572,573230,911038,913419,851238,269486,730936,11226,303693,338860,133127,60355,459283,74269,940405,299128,791545,247874,761924,657392,473100,521762,406357\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 401\n", + "Returned IDs: 421107,678267,365194,967592,428004,61808,916328,648958,695456,484345,798397,334990,726567,264360,174820,809112,944672,426440,598833,492725,480828,861695,641629,884539,704564,602017,722548,928454,143774,816097\n", + "Ground Truth: 421107,678267,365194,967592,428004,61808,916328,648958,695456,484345,798397,334990,726567,264360,174820,809112,944672,426440,598833,492725,480828,861695,641629,850247,884539,704564,602017,722548,928454,143774\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 403\n", + "Returned IDs: 762862,837824,552902,466958,856734,204428,144490,55691,489577,687441,359434,690558,958204,231864,514006,846245,631865,972340,678489,293552,561495,568085,780305,611218,186114,224279,858166,715255,824028,633280\n", + "Ground Truth: 762862,837824,552902,466958,856734,204428,144490,55691,489577,687441,359434,690558,958204,231864,514006,846245,631865,761710,972340,678489,600493,293552,561495,568085,780305,955680,611218,186114,77563,224279\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 406\n", + "Returned IDs: 241058,559486,292259,590965,219094,359462,416190,71364,417943,943728,534418,467477,456689,794858,196334,9921,910943,24806,955567,635338,667942,470417,563084,885567,771015,927053,420376,490518,455911,404311\n", + "Ground Truth: 241058,559486,292259,590965,219094,359462,416190,71364,417943,943728,534418,467477,456689,196334,794858,9921,910943,24806,955567,689630,635338,667942,470417,563084,885567,771015,927053,420376,490518,455911\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 410\n", + "Returned IDs: 483056,939858,468302,597887,248125,870144,391452,841741,225472,869078,490155,720764,949852,722707,645834,444305,933004,731060,834945,30492,272542,618518,96336,841244,178160,414620,391429,400541,832605,45733\n", + "Ground Truth: 483056,939858,468302,597887,248125,870144,391452,841741,225472,869078,490155,195173,942723,746432,720764,949852,445843,722707,645834,444305,933004,731060,834945,30492,370780,969033,668189,170527,272542,618518\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 411\n", + "Returned IDs: 144057,465671,238107,195972,844216,317121,77634,870760,634951,707024,103457,428956,276134,547996,174000,989668,962092,367941,144408,760920,658963,429535,810744,137884,534999,649842,137675,661994,863934,824104\n", + "Ground Truth: 144057,465671,238107,195972,844216,317121,77634,870760,634951,707024,103457,428956,276134,547996,174000,989668,962092,367941,144408,405450,760920,526197,658963,429535,810744,382870,137884,534999,892420,649842\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 413\n", + "Returned IDs: 616271,163161,481282,632657,875772,652623,40221,861880,377879,895645,256932,285361,354056,525142,473040,365551,973013,169869,206122,101951,766421,405408,844941,887547,755896,523387,453554,336898,234816,775309\n", + "Ground Truth: 616271,163161,481282,632657,875772,652623,40221,659479,861880,377879,895645,256932,285361,354056,525142,473040,365551,973013,169869,206122,101951,766421,405408,844941,887547,755896,523387,453554,336898,234816\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 419\n", + "Returned IDs: 54669,752279,465775,677556,613525,174186,872820,433279,482135,855417,945054,806221,322664,49561,955291,279855,884755,133853,189597,581740,285383,149650,339721,475508,311167,536633,47301,785589,479031,236359\n", + "Ground Truth: 54669,752279,465775,677556,613525,174186,872820,433279,482135,855417,945054,806221,322664,49561,955291,279855,884755,133853,189597,581740,285383,149650,339721,475508,311167,536633,47301,785589,580339,479031\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 421\n", + "Returned IDs: 658064,637506,816973,787257,901531,766843,121216,906072,366590,632814,285981,286657,513674,127308,98677,348542,385803,789255,415473,823952,575335,890033,716930,365394,919632,986442,516784,32703,483721,603224\n", + "Ground Truth: 658064,637506,816973,950233,787257,284813,901531,882665,766843,121216,906072,155737,366590,632814,285981,286657,889792,513674,127308,98677,179514,348542,330205,385803,789255,415473,823952,575335,579196,890033\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 424\n", + "Returned IDs: 955411,420247,693377,803830,687984,923431,573536,870473,495976,948663,970663,13243,891670,866914,40317,247950,128979,557577,154584,181216,542011,513492,234113,277261,112988,13971,638922,381927,749257,746112\n", + "Ground Truth: 955411,420247,693377,803830,687984,923431,573536,870473,495976,948663,970663,13243,891670,866914,40317,247950,128979,557577,154584,181216,542011,513492,234113,277261,112988,13971,638922,381927,10697,749257\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 430\n", + "Returned IDs: 532151,485140,540992,689703,529529,911792,48597,19807,568714,101501,317406,741193,432677,574335,928590,719345,686873,495938,767175,529999,106638,621503,914907,274736,884640,664005,514158,589352,280356,727308\n", + "Ground Truth: 532151,485140,540992,689703,529529,911792,48597,19807,568714,101501,317406,741193,432677,816606,574335,928590,719345,686873,495938,659238,767175,529999,106638,248184,621503,914907,274736,884640,664005,862488\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 434\n", + "Returned IDs: 747036,194838,321839,932337,695383,838546,656755,595044,847225,772969,82790,75596,107025,577259,192493,515498,830398,242214,967441,813043,977001,750583,245492,508563,841089,643148,51304,184068,882711,761144\n", + "Ground Truth: 747036,194838,321839,932337,695383,838546,656755,595044,847225,772969,82790,75596,107025,577259,192493,515498,830398,242214,967441,813043,977001,750583,245492,508563,841089,643148,207396,51304,184068,882711\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 435\n", + "Returned IDs: 491826,272115,331523,350298,859683,594477,941185,43705,520108,637098,898170,508516,351337,428272,102566,424263,166103,235595,468302,905768,615755,31998,618236,579714,626472,174556,66299,90671,189156,455332\n", + "Ground Truth: 491826,272115,331523,350298,859683,594477,941185,43705,520108,637098,898170,497921,508516,351337,428272,102566,424263,166103,235595,468302,905768,615755,31998,618236,579714,626472,174556,66299,549890,90671\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 438\n", + "Returned IDs: 486553,804703,418755,226022,340972,561653,232975,353165,184985,980428,183477,897794,48386,240024,177299,949159,947579,941252,870852,310766,233432,304965,67883,970615,713581,708519,207411,191103,679145,12772\n", + "Ground Truth: 486553,804703,418755,226022,340972,561653,232975,353165,184985,980428,183477,897794,48386,240024,177299,949159,288275,947579,941252,870852,310766,233432,304965,67883,970615,713581,708519,207411,191103,679145\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 442\n", + "Returned IDs: 585591,431277,430591,295895,601605,163752,155919,842884,976510,783587,262345,558928,421003,372119,51639,290601,726392,755820,864165,649883,534504,595848,385191,393041,934717,940629,119244,967391,516037,498106\n", + "Ground Truth: 585591,431277,430591,295895,601605,163752,155919,842884,976510,783587,715501,262345,652994,558928,421003,372119,51639,290601,726392,755820,864165,649883,801225,550041,447274,534504,595848,385191,393041,934717\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 448\n", + "Returned IDs: 696869,717101,623071,436802,514263,424373,237517,858705,736303,843961,42363,577497,256469,940547,13608,288948,966115,651192,269251,805270,739396,785739,193458,76693,263805,867819,295949,702496,65358,365325\n", + "Ground Truth: 696869,425252,502940,717101,623071,436802,514263,424373,237517,858705,736303,843961,42363,577497,256469,940547,13608,209901,681720,269874,288948,79921,966115,280611,148673,415761,651192,269251,492725,805270\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 450\n", + "Returned IDs: 983071,724424,456983,332855,506429,887374,643408,284708,738400,984087,534604,47459,612881,337691,758671,350257,553530,665167,984480,733527,255792,871574,193833,72256,839198,454898,364833,511221,475846,298337\n", + "Ground Truth: 983071,724424,456983,332855,506429,887374,643408,284708,738400,984087,534604,47459,612881,351862,934,337691,758671,350257,553530,92797,665167,984480,733527,255792,871574,590157,587825,422869,193833,598771\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 453\n", + "Returned IDs: 408183,193282,131850,764157,927555,383063,804375,543818,666074,774870,904547,554042,223829,773383,913535,486000,815866,274290,775550,512802,954208,733669,939770,429278,158316,665432,550036,151512,794003,168153\n", + "Ground Truth: 408183,193282,131850,764157,927555,383063,804375,543818,666074,774870,387184,904547,923016,554042,223829,773383,913535,486000,815866,274290,775550,512802,954208,733669,939770,429278,158316,665432,550036,151512\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 456\n", + "Returned IDs: 451511,555893,781448,311841,744649,319487,115855,571802,972715,591217,226787,153363,432720,768455,147940,656709,211230,398784,350553,118977,635652,597573,530337,384564,772540,329236,739700,570238,821675,905710\n", + "Ground Truth: 778797,677610,84869,278349,621721,799908,960847,802397,380073,325833,739222,857179,521841,544409,131463,75624,451511,729469,170284,6087,919726,555893,393220,887053,838932,746517,312661,781448,589923,505417\n", + "Recall: 0.1000\n", + "\n", + "Query ID: 460\n", + "Returned IDs: 636525,800512,669760,37859,819450,574083,653517,150455,957732,543553,404192,35066,193189,180683,413452,625836,148011,611797,404975,904343,354887,835865,384142,53732,80295,648520,26508,345578,514664,906435\n", + "Ground Truth: 636525,800512,669760,37859,819450,574083,653517,150455,957732,543553,404192,119668,287607,841757,35066,180683,193189,413452,625836,148011,550876,611797,404975,904343,354887,835865,441079,384142,710896,53732\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 463\n", + "Returned IDs: 247433,194491,885321,432883,741791,95329,870074,11112,888697,394921,383317,840004,44927,139201,138924,870190,861923,233816,368880,340238,584011,60154,975451,37948,740446,543250,484486,823996,881703,987087\n", + "Ground Truth: 247433,194491,885321,432883,741791,95329,870074,11112,888697,394921,383317,840004,44927,139201,138924,870190,861923,233816,657085,368880,340238,584011,60154,975451,37948,740446,543250,484486,823996,881703\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 464\n", + "Returned IDs: 960386,16389,563081,779910,34834,426672,438112,295434,262278,243409,829636,506138,291781,337352,342495,766281,711488,77752,22949,808248,218848,766675,693174,776598,135939,696761,963235,829972,662405,956937\n", + "Ground Truth: 960386,16389,563081,779910,34834,426672,438112,295434,262278,243409,829636,506138,291781,337352,342495,766281,711488,77752,22949,808248,218848,766675,693174,776598,135939,709980,696761,963235,829972,662405\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 468\n", + "Returned IDs: 363078,961264,286931,706139,915893,661806,782680,914927,698393,940106,762035,879753,21732,91216,360049,272152,2147,430821,187578,695222,540261,588434,558914,661258,629739,228061,509966,574086,690576,611103\n", + "Ground Truth: 363078,961264,286931,706139,915893,661806,782680,914927,698393,940106,762035,879753,21732,91216,360049,272152,2147,430821,187578,695222,540261,588434,558914,661258,629739,228061,509966,832969,574086,690576\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 472\n", + "Returned IDs: 985727,435690,435503,418392,440155,358487,240287,660519,495054,630829,970563,4571,735953,57204,62785,178798,586993,943743,827684,70875,422139,924809,146733,103691,13467,575626,258446,898405,803312,593914\n", + "Ground Truth: 985727,435690,435503,418392,440155,358487,240287,660519,495054,630829,970563,4571,735953,57204,38043,62785,178798,586993,943743,827684,917538,848547,70875,373270,422139,924809,258304,146733,103691,13467\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 475\n", + "Returned IDs: 199879,919728,619933,146806,678732,776150,280351,974077,576293,312561,970525,972684,51739,866307,694508,577806,73954,469300,926388,624998,405672,520376,548531,89671,313283,33587,128364,955968,510601,431788\n", + "Ground Truth: 199879,919728,619933,146806,257585,678732,776150,280351,974077,576293,312561,970525,972684,51739,866307,694508,577806,73954,469300,926388,624998,405672,520376,548531,66519,89671,313283,33587,729396,128364\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 480\n", + "Returned IDs: 108401,43078,147360,267874,195613,689511,732309,950310,78336,853452,479142,237462,908265,396511,109188,84835,345161,298527,745040,198364,456562,169418,75196,887079,862516,476506,337626,457797,107459,793520\n", + "Ground Truth: 108401,43078,147360,267874,195613,689511,732309,950310,78336,853452,479142,237462,908265,496815,396511,109188,84835,345161,298527,157466,745040,198364,456562,169418,75196,887079,548903,862516,476506,337626\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 481\n", + "Returned IDs: 834350,784817,569607,269487,440334,375702,531315,224642,202811,32946,484680,735628,810928,371225,980818,325885,345184,656961,662167,198441,651001,869392,396984,404308,733171,598132,736813,370104,909039,728512\n", + "Ground Truth: 834350,784817,569607,269487,440334,375702,531315,224642,202811,32946,484680,735628,810928,371225,980818,325885,345184,656961,97915,662167,198441,651001,869392,396984,404308,733171,598132,504160,736813,370104\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 483\n", + "Returned IDs: 972280,939115,633871,438132,409773,852429,324981,688260,586996,258949,135032,143381,339939,860815,550673,470126,454895,509624,227005,360595,904815,982369,109958,625955,372846,306836,416562,331705,245254,766773\n", + "Ground Truth: 972280,613928,939115,633871,438132,409773,852429,324981,688260,586996,258949,135032,143381,701374,339939,860815,550673,470126,454895,509624,227005,360595,904815,982369,109958,625955,372846,306836,416562,331705\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 486\n", + "Returned IDs: 501858,20844,922554,468968,974209,749225,654393,752272,884286,425949,845428,584197,761053,640539,176249,458007,791268,635576,594765,474016,363770,624244,726908,834808,245584,342673,578227,654438,199611,755797\n", + "Ground Truth: 501858,20844,922554,468968,974209,749225,654393,752272,884286,425949,845428,584197,761053,640539,176249,458007,791268,635576,594765,474016,624244,363770,726908,834808,245584,342673,578227,654438,199611,7570\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 496\n", + "Returned IDs: 748940,133107,612315,781246,876393,967452,121517,359569,665954,448757,327784,117150,802913,902050,869123,364432,699852,988464,416115,913858,97135,734292,278713,450347,439861,758258,983841,713718,409254,838551\n", + "Ground Truth: 748940,133107,612315,781246,876393,967452,121517,359569,665954,448757,327784,117150,802913,902050,869123,364432,699852,988464,416115,913858,737680,97135,734292,278713,450347,52520,439861,758258,983841,713718\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 501\n", + "Returned IDs: 456810,626192,563428,66488,969174,448830,670037,501009,232151,368156,535738,891542,462234,973591,448781,302030,923799,797607,74289,405619,877723,123250,485276,256112,307421,117836,205661,774241,397000,850963\n", + "Ground Truth: 334723,20527,99603,633867,655909,456810,57807,626192,563428,66488,969174,448830,670037,501009,232151,368156,535738,576724,891542,462234,973591,448781,433869,302030,923799,797607,224580,74289,405619,877723\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 503\n", + "Returned IDs: 412524,67663,278552,381627,634217,230071,679173,231080,6964,107191,792275,546728,862403,496084,661975,41904,48551,419728,193151,223503,106462,965718,248199,339063,904999,210682,573918,618852,472496,157138\n", + "Ground Truth: 412524,67663,278552,381627,634217,230071,679173,231080,6964,107191,792275,546728,862403,496084,661975,41904,77710,48551,419728,193151,223503,106462,53544,965718,248199,339063,904999,210682,582514,573918\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 508\n", + "Returned IDs: 173670,320036,276625,391244,114940,201839,297111,121190,634951,147221,647839,424505,28762,773965,186907,458279,799727,416538,778260,240051,37503,760829,841319,593142,690969,827551,573558,323205,989990,318399\n", + "Ground Truth: 173670,320036,276625,391244,114940,201839,297111,121190,634951,147221,647839,424505,28762,773965,186907,458279,799727,416538,778260,240051,37503,760829,841319,593142,690969,827551,476277,573558,323205,989990\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 509\n", + "Returned IDs: 73536,866616,485614,867308,683563,153341,948521,75822,192236,287701,461543,776700,361581,836905,930367,504471,533193,145385,60325,967611,440220,317661,895493,75182,546017,340596,830952,890032,464501,160859\n", + "Ground Truth: 73536,866616,485614,867308,683563,153341,948521,75822,192236,287701,461543,776700,361581,836905,930367,504471,505268,533193,145385,357436,389885,684188,60325,967611,183785,440220,317661,895493,75182,546017\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 510\n", + "Returned IDs: 912927,367074,709678,676675,826156,726540,479300,210682,417557,42850,289491,675660,648943,913194,629135,360457,208352,484373,449120,892420,650354,747870,435437,14538,609179,831032,148616,507491,539938,263771\n", + "Ground Truth: 912927,367074,709678,676675,826156,726540,479300,210682,417557,42850,289491,675660,648943,913194,629135,360457,208352,484373,449120,892420,650354,747870,435437,14538,609179,831032,148616,507491,539938,319014\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 511\n", + "Returned IDs: 243024,533494,258122,273207,636547,493509,12598,284323,448843,83798,159073,649263,871889,160963,793703,210682,448353,898936,238933,607731,90023,333234,419428,210476,802330,894597,506870,400660,429646,612219\n", + "Ground Truth: 243024,498493,533494,258122,273207,636547,493509,12598,284323,709690,448843,69844,83798,719321,159073,649263,871889,160963,793703,210682,65385,448353,690130,898936,238933,607731,90023,333234,419428,578772\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 513\n", + "Returned IDs: 896608,481574,668494,549799,185253,554539,84335,418857,363577,277253,325134,739916,18356,302278,179564,494430,909938,917331,189130,66077,753860,953292,837623,667982,498674,318803,559107,310207,345646,528500\n", + "Ground Truth: 896608,481574,668494,549799,185253,554539,84335,418857,363577,277253,325134,739916,18356,302278,179564,477641,494430,909938,917331,189130,66077,753860,953292,837623,667982,498674,318803,310207,559107,345646\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 520\n", + "Returned IDs: 705676,777630,440891,48190,847248,808761,349052,462218,201606,841244,775050,858287,713208,305809,481667,350991,142650,591431,682020,836711,41301,553067,920202,516596,712166,198553,484373,365325,798000,67647\n", + "Ground Truth: 705676,777630,440891,48190,847248,808761,349052,984112,462218,201606,841244,775050,858287,713208,305809,481667,350991,142650,591431,682020,836711,41301,553067,920202,759612,516596,712166,198553,484373,365325\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 521\n", + "Returned IDs: 648033,206367,840722,705308,492651,412515,699560,958647,401430,345406,382299,879707,203481,812738,423893,666952,241203,782920,28660,1848,514917,103811,686286,755584,284504,622367,670049,733905,231238,622124\n", + "Ground Truth: 648033,206367,840722,705308,492651,412515,699560,958647,401430,960037,345406,382299,879707,203481,812738,423893,666952,241203,782920,629849,28660,420344,1848,514917,103811,686286,755584,284504,622367,670049\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 522\n", + "Returned IDs: 822799,824222,610176,970778,679001,700511,418833,961788,762089,491154,501427,947642,369042,38981,946853,69596,493544,304515,639024,405271,734957,821921,730864,766421,755629,761945,444960,687975,438433,692066\n", + "Ground Truth: 822799,824222,610176,970778,679001,700511,418833,961788,762089,491154,501427,947642,369042,38981,946853,69596,493544,304515,639024,405271,734957,821921,730864,766421,755629,761945,409092,444960,687975,438433\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 523\n", + "Returned IDs: 181015,907155,289532,847333,185961,852762,608498,447812,390357,625813,597058,500930,930951,338619,8867,283016,425116,335016,256333,237504,62037,769722,37706,670768,922973,396407,935398,342954,194277,312628\n", + "Ground Truth: 181015,907155,289532,847333,185961,852762,608498,447812,390357,625813,597058,500930,930951,338619,8867,283016,425116,335016,256333,237504,281220,62037,769722,37706,670768,922973,396407,935398,342954,194277\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 527\n", + "Returned IDs: 802668,297384,73158,84228,584965,716356,578242,27601,390522,506914,665525,864668,962441,230866,680802,702104,378115,631377,335433,114942,195997,33995,158176,53622,861098,376749,586381,369898,652719,265323\n", + "Ground Truth: 802668,297384,73158,84228,584965,716356,578242,523551,27601,390522,506914,665525,864668,962441,230866,680802,702104,378115,631377,335433,114942,521214,178610,195997,33995,158176,53622,861098,376749,586381\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 531\n", + "Returned IDs: 90136,139246,845174,270355,300299,971765,124284,772792,80218,684477,480698,868150,492630,220195,534228,17180,650297,937050,808529,606804,26174,1875,304293,250471,416126,370565,721920,539176,322283,461510\n", + "Ground Truth: 90136,139246,845174,270355,300299,971765,124284,772792,80218,684477,480698,868150,492630,220195,534228,17180,650297,224071,937050,808529,606804,26174,1875,304293,416126,250471,370565,721920,539176,322283\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 534\n", + "Returned IDs: 924168,546303,870764,930648,93829,401194,10327,868436,645034,589927,828481,572435,549748,79803,10647,205168,245262,242887,882335,630829,209667,876204,418118,982879,614226,345422,70236,16829,654575,185159\n", + "Ground Truth: 924168,546303,870764,930648,93829,401194,10327,868436,645034,589927,828481,572435,418458,549748,190882,457987,79803,10647,205168,245262,242887,882335,630829,612226,209667,899328,793469,876204,418118,982879\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 543\n", + "Returned IDs: 649308,897930,688672,462870,232769,916039,733663,147884,137043,525841,280116,7358,48613,25518,8974,51689,796185,747782,374092,256889,962202,204536,373824,6307,544151,11937,725629,94850,192877,225759\n", + "Ground Truth: 649308,897930,688672,462870,232769,916039,733663,147884,137043,525841,280116,7358,48613,809691,25518,8974,51689,796185,747782,374092,256889,962202,204536,373824,6307,544151,11937,725629,94850,192877\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 547\n", + "Returned IDs: 597568,172305,223593,173816,450584,806011,940850,291898,449776,83140,166812,496063,631685,410917,972340,157103,297182,61729,704453,552459,953751,208586,307780,854181,968035,211781,883469,628051,242365,720887\n", + "Ground Truth: 597568,172305,223593,173816,450584,806011,940850,291898,449776,167742,83140,166812,496063,631685,410917,972340,868428,157103,297182,809524,61729,704453,552459,953751,208586,307780,854181,968035,211781,883469\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 548\n", + "Returned IDs: 458034,970939,354509,335297,365410,932664,830921,932821,853247,835981,739311,155114,957963,292177,363876,470722,545156,221999,29333,523323,939159,811965,878032,548636,967989,464510,931531,502111,74714,730665\n", + "Ground Truth: 303704,458034,970939,390991,354509,660821,335297,365410,824640,932664,830921,932821,853247,835981,739311,155114,957963,394353,292177,363876,470722,545156,221999,29333,523323,939159,54537,811965,712083,632260\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 551\n", + "Returned IDs: 44387,959746,559984,201607,697817,966254,460054,775061,976042,171151,475559,881865,244718,58297,261833,279683,848242,597228,73160,661561,288546,320815,109022,878634,959478,604881,109453,733538,25479,680991\n", + "Ground Truth: 44387,959746,559984,201607,697817,966254,460054,775061,976042,171151,343629,475559,881865,244718,58297,261833,279683,848242,597228,73160,661561,507593,288546,320815,109022,878634,959478,604881,908965,109453\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 556\n", + "Returned IDs: 379441,622460,299135,130421,794894,908920,686436,408219,734292,533438,950374,888352,906185,649910,699418,313878,87087,808940,97135,967952,656261,886207,239468,161049,70939,925179,280994,970615,613642,357575\n", + "Ground Truth: 379441,622460,299135,130421,794894,908920,686436,408219,734292,533438,950374,888352,906185,649910,919112,699418,313878,87087,808940,97135,967952,656261,886207,239468,161049,70939,925179,280994,970615,613642\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 557\n", + "Returned IDs: 327566,695930,614288,797325,361971,270209,603322,311610,387306,484847,406732,601699,661183,817653,99202,521147,619824,239457,866151,267786,130673,756957,892464,139856,841337,152449,460333,577666,816282,25031\n", + "Ground Truth: 327566,695930,614288,797325,361971,270209,603322,311610,387306,484847,406732,601699,171937,661183,817653,889335,99202,521147,619824,239457,267786,866151,130673,756957,892464,139856,841337,152449,460333,577666\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 562\n", + "Returned IDs: 984988,603416,261457,454735,39522,460875,247502,886983,978631,935038,921853,119978,357252,540143,321744,87323,24380,838549,305954,59068,397874,767866,518055,605351,469087,634008,593672,604611,896427,14108\n", + "Ground Truth: 984988,603416,261457,454735,39522,460875,247502,886983,978631,935038,921853,119978,781367,357252,540143,321744,87323,24380,838549,305954,59068,397874,767866,518055,605351,469087,634008,593672,604611,896427\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 563\n", + "Returned IDs: 989539,39836,724706,675225,563768,726187,486052,381482,304682,422566,27124,518049,589447,796515,746926,475342,847881,644301,858286,49953,60180,468397,784464,177723,94869,591138,546749,567897,346557,838517\n", + "Ground Truth: 989539,39836,724706,675225,563768,726187,173419,486052,381482,304682,422566,27124,518049,589447,796515,746926,475342,847881,644301,68147,858286,49953,417692,60180,351826,468397,784464,244126,177723,827990\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 569\n", + "Returned IDs: 954700,386722,889551,712807,155209,728397,392651,597962,634929,971050,640624,476457,619276,314560,559107,15206,988518,730935,205770,597366,965815,284628,934115,288373,843095,556870,296460,20043,745703,82471\n", + "Ground Truth: 954700,386722,889551,712807,155209,728397,392651,597962,634929,971050,640624,476457,558832,619276,314560,559107,15206,988518,225204,730935,205770,597366,965815,284628,934115,288373,843095,556870,296460,20043\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 574\n", + "Returned IDs: 910249,26540,28611,463070,884945,854530,146515,321172,573544,568015,27124,945323,78851,595362,661244,159421,719664,181709,880189,65464,486818,591539,974762,718513,423943,796185,851794,853373,64583,929713\n", + "Ground Truth: 910249,26540,28611,463070,884945,854530,146515,321172,368959,573544,568015,27124,945323,78851,595362,661244,159421,719664,181709,880189,65464,486818,591539,974762,718513,423943,796185,851794,853373,64583\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 577\n", + "Returned IDs: 418395,705163,168708,920655,315884,211695,404356,135559,4278,752683,673789,298318,146667,231884,978023,427508,152247,649519,842252,439073,479772,869113,476666,440521,747052,522006,506341,148880,501604,33383\n", + "Ground Truth: 418395,705163,168708,920655,315884,211695,404356,135559,4278,752683,673789,298318,146667,231884,978023,427508,152247,649519,842252,439073,479772,869113,476666,440521,747052,774523,522006,506341,148880,501604\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 583\n", + "Returned IDs: 132611,938757,107976,553778,679696,792762,910853,660779,533775,54299,762642,662170,227903,173575,741485,778568,519986,284662,356947,173102,659090,769378,957822,291640,523455,258208,881388,284177,519700,795026\n", + "Ground Truth: 132611,938757,107976,553778,679696,792762,910853,660779,533775,54299,762642,662170,227903,173575,741485,778568,519986,284662,356947,173102,659090,769378,957822,291640,523455,258208,881388,284177,519700,396259\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 585\n", + "Returned IDs: 764654,421754,162750,784327,272909,174661,671188,51285,770707,443840,887103,740409,683471,410935,343032,600421,296510,590738,805963,730735,684905,419368,554537,200172,235900,236837,811483,377245,354495,6854\n", + "Ground Truth: 764654,421754,162750,784327,272909,174661,671188,51285,770707,443840,887103,740409,683471,410935,343032,600421,296510,590738,805963,730735,684905,419368,554537,200172,235900,236837,260217,811483,377245,354495\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 586\n", + "Returned IDs: 655542,12598,318216,732255,621496,758738,866810,350991,730411,206021,844380,955248,926758,267786,545306,957647,884510,99357,661339,210316,581009,84052,67484,508376,97390,936210,221021,679260,62721,882915\n", + "Ground Truth: 655542,295823,12598,318216,732255,621496,758738,941410,866810,350991,730411,780119,206021,844380,955248,926758,267786,545306,339997,385678,957647,183012,884510,99357,661339,749672,210316,581009,84052,67484\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 587\n", + "Returned IDs: 762292,229264,565018,756183,894435,495249,896484,278925,142517,207640,405685,276989,20068,653182,586641,159045,550402,472776,934201,589499,210535,616481,368906,761487,121117,779659,703204,389850,394027,285723\n", + "Ground Truth: 912589,206628,762292,229264,565018,316867,756183,894435,790414,495249,825913,279458,896484,278925,142517,207640,405685,276989,20068,236574,653182,586641,121806,159045,550402,325429,313442,472776,934201,589499\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 596\n", + "Returned IDs: 714092,302591,891305,752708,723197,977488,483024,874171,879813,818058,12747,698243,80386,162341,367516,792275,167915,987644,478695,63813,323205,29666,739095,185931,298527,884847,904999,426054,957009,729643\n", + "Ground Truth: 714092,302591,891305,752708,723197,977488,483024,874171,879813,818058,12747,698243,80386,162341,367516,792275,167915,987644,478695,63813,323205,29666,739095,185931,298527,884847,142117,862403,904999,426054\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 599\n", + "Returned IDs: 453593,865552,138455,503494,734432,895442,654954,626768,257980,251847,420305,732121,336766,436268,770223,629797,19146,763129,571374,859257,592599,142607,278670,285525,787023,924556,232151,315381,287779,100083\n", + "Ground Truth: 453593,865552,138455,503494,734432,895442,654954,626768,405933,257980,251847,420305,732121,336766,436268,770223,629797,19146,7477,139314,763129,571374,859257,592599,676136,142607,278670,230624,285525,787023\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 607\n", + "Returned IDs: 288368,529056,869615,326141,502422,525646,348804,484236,148125,538959,683460,399653,101017,699388,613895,625670,596410,94503,685541,884706,505999,175963,511275,634206,715639,69459,654421,431732,262275,277165\n", + "Ground Truth: 288368,529056,869615,326141,502422,525646,348804,484236,148125,538959,683460,399653,101017,699388,613895,625670,596410,94503,685541,884706,505999,175963,511275,634206,715639,69459,825913,744798,654421,431732\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 615\n", + "Returned IDs: 232787,967342,218747,870068,914492,756720,96109,293334,740027,118868,733840,234787,760623,711725,181772,17058,169936,910647,980751,542774,913306,69773,83621,235716,354600,695856,318319,771088,498903,432146\n", + "Ground Truth: 232787,967342,218747,870068,914492,756720,96109,293334,740027,118868,733840,234787,760623,711725,181772,17058,169936,910647,980751,542774,753259,913306,69773,83621,235716,354600,695856,318319,771088,498903\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 616\n", + "Returned IDs: 352318,360133,115155,816373,451701,154191,686199,895391,202607,744117,85164,706509,495389,52402,99066,800609,109364,466299,285676,139505,237064,188796,526704,338450,691609,627749,801050,911432,324534,351247\n", + "Ground Truth: 352318,360133,115155,816373,451701,154191,686199,895391,202607,744117,85164,190865,706509,495389,52402,99066,800609,109364,466299,285676,139505,237064,188796,526704,338450,691609,627749,801050,911432,477373\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 625\n", + "Returned IDs: 719764,113053,608505,560999,68351,304168,491662,3859,984828,344873,770725,513672,265990,894142,560827,81574,877483,894655,507587,971222,732770,147122,35116,513965,794485,539786,850754,822002,79898,902303\n", + "Ground Truth: 719764,113053,608505,560999,68351,304168,491662,3859,984828,344873,770725,513672,265990,560827,894142,81574,894655,877483,507587,971222,264176,732770,147122,878987,35116,109183,513965,794485,539786,850754\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 626\n", + "Returned IDs: 332676,2431,303519,721561,96207,663677,873768,559273,706033,16903,731633,133528,548289,196690,806111,92420,446863,245480,860236,24749,128291,556123,564534,415640,466782,202015,190651,48759,770331,832804\n", + "Ground Truth: 332676,2431,303519,721561,96207,663677,873768,559273,706033,16903,731633,133528,548289,196690,806111,92420,446863,245480,860236,24749,128291,556123,564534,415640,739381,466782,202015,190651,48759,770331\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 629\n", + "Returned IDs: 877897,512461,686626,750728,434781,301466,846906,824309,904854,79699,796333,887731,872754,677229,327795,455810,520463,698968,900861,596456,669588,712306,241132,973674,456597,605220,187205,380458,16023,807866\n", + "Ground Truth: 17555,877897,512461,686626,750728,434781,301466,227517,846906,824309,904854,79699,796333,887731,872754,677229,327795,455810,520463,698968,231348,900861,596456,669588,712306,241132,973674,456597,605220,187205\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 632\n", + "Returned IDs: 777234,127043,536288,429603,201032,330579,492187,259734,507300,21340,545592,896450,292583,348303,35830,868300,385033,558027,330235,223233,113986,221832,803487,20546,812620,490249,211930,407619,405417,261193\n", + "Ground Truth: 777234,127043,536288,429603,386436,201032,330579,274402,492187,259734,507300,21340,545592,896450,292583,348303,35830,868300,385033,875478,558027,330235,223233,226235,113986,221832,803487,20546,812620,490249\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 634\n", + "Returned IDs: 473915,442150,458317,953670,965457,656372,536726,844751,910960,44107,285036,438661,657368,944499,695456,482695,840458,687724,404079,856526,750702,377137,668705,478559,910898,983474,674724,363676,683168,800528\n", + "Ground Truth: 473915,442150,458317,953670,965457,656372,536726,844751,910960,44107,285036,438661,657368,944499,695456,482695,840458,687724,404079,856526,750702,377137,668705,745052,478559,910898,983474,674724,363676,683168\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 636\n", + "Returned IDs: 521509,361907,439384,182274,956453,256252,710654,351357,914340,905975,348527,195164,217139,284252,574461,139272,958728,798599,909776,511085,267258,845177,814733,659929,319701,358356,823034,629529,866606,618079\n", + "Ground Truth: 521509,361907,439384,182274,956453,256252,710654,351357,914340,905975,348527,195164,217139,284252,574461,139272,933327,958728,798599,909776,511085,267258,845177,823885,814733,86068,659929,319701,358356,835145\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 640\n", + "Returned IDs: 36173,459141,78620,692869,615507,961536,655539,170153,146529,221057,883605,403957,753018,919972,362126,72539,488980,736023,165641,459591,385139,425929,706871,228061,374587,46534,311774,667373,106689,246967\n", + "Ground Truth: 36173,459141,78620,692869,615507,961536,679703,655539,170153,146529,221057,883605,403957,753018,919972,362126,72539,488980,736023,165641,459591,385139,425929,706871,228061,374587,46534,311774,667373,4521\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 647\n", + "Returned IDs: 702662,116638,474928,331942,918064,181466,143364,931832,220498,288579,655077,343314,115615,85956,742221,768712,462420,936312,315542,172823,549884,975217,823587,60933,228976,146733,953477,776671,257168,446452\n", + "Ground Truth: 702662,116638,474928,331942,918064,970675,181466,143364,931832,220498,288579,655077,343314,115615,85956,742221,768712,462420,936312,315542,172823,549884,975217,823587,60933,228976,146733,953477,776671,257168\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 649\n", + "Returned IDs: 471190,571998,865255,989546,69217,418466,98485,758425,179185,873119,518501,815324,620868,721351,326010,775258,959411,686956,613801,512099,489305,776851,753732,447715,205476,342208,579269,774722,435365,209503\n", + "Ground Truth: 471190,571998,865255,989546,69217,418466,98485,758425,127880,179185,873119,518501,815324,620868,721351,326010,775258,959411,686956,613801,512099,489305,381726,776851,753732,447715,205476,342208,579269,774722\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 650\n", + "Returned IDs: 701358,708905,650405,428059,814549,471308,567431,327052,178442,867242,341762,298457,91142,967747,129080,670196,804580,717173,483911,934486,692240,300091,497965,149097,881989,525135,329096,930583,409959,319249\n", + "Ground Truth: 701358,708905,650405,428059,814549,471308,567431,327052,178442,867242,341762,298457,91142,967747,129080,670196,804580,717173,483911,934486,843337,692240,300091,497965,149097,881989,525135,329096,930583,409959\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 652\n", + "Returned IDs: 765060,554017,332654,449120,576907,23358,169493,768517,926990,592652,725673,650354,771865,406838,73954,12598,298118,454204,702922,810382,573673,412330,928945,364039,841089,829356,652407,676675,913105,916625\n", + "Ground Truth: 276864,765060,554017,323348,662344,332654,449120,576907,938322,455898,23358,169493,768517,926990,664042,80791,592652,251821,725673,650354,771865,406838,73954,12598,298118,454204,702922,810382,915898,573673\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 654\n", + "Returned IDs: 222553,763418,472430,853894,75835,160556,575071,536561,512811,269606,248207,499147,372645,776526,927885,598259,500615,418743,904577,544679,495593,236517,866525,375574,638695,275917,159993,236846,873087,384982\n", + "Ground Truth: 222553,763418,472430,853894,75835,160556,575071,536561,512811,269606,248207,499147,372645,776526,927885,598259,500615,418743,904577,544679,495593,236517,866525,375574,638695,275917,159993,236846,873087,887298\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 655\n", + "Returned IDs: 691676,615209,636370,773533,192871,68721,273620,896801,263251,974235,769420,787961,779927,931364,916675,655309,191645,468572,769880,107685,891220,329449,782398,159371,629622,34288,558042,967712,981634,86007\n", + "Ground Truth: 830427,691676,615209,636370,773533,279936,192871,68721,273620,896801,581949,263251,974235,769420,787961,779927,269412,931364,298645,916675,564914,655309,117810,191645,800581,468572,769880,62712,107685,891220\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 656\n", + "Returned IDs: 65162,925920,10979,703539,451952,494027,206871,204958,389412,549368,15695,79697,317270,311572,182204,833875,684222,632680,340559,147802,853740,884332,422655,709017,293808,438153,905619,639640,5376,120638\n", + "Ground Truth: 65162,925920,10979,703539,451952,494027,206871,204958,389412,549368,15695,79697,317270,311572,182204,597866,833875,684222,632680,915601,340559,147802,853740,884332,422655,709017,427347,293808,438153,905619\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 668\n", + "Returned IDs: 36769,238323,389242,699085,703698,807649,741897,300098,159765,610851,205480,440209,215148,394260,851847,730324,865829,715818,22324,448620,984635,374421,610666,544576,309956,647330,735843,580025,143698,195799\n", + "Ground Truth: 36769,238323,389242,699085,703698,807649,741897,300098,159765,610851,205480,440209,215148,394260,851847,730324,865829,715818,22324,448620,984635,374421,610666,544576,309956,647330,735843,580025,143698,976048\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 669\n", + "Returned IDs: 627383,288484,556554,285851,166815,795228,325378,577649,893268,230411,785551,687681,260412,211877,679117,316276,959074,9552,562736,65145,741404,393,134059,981209,972742,786212,824435,649453,642381,506883\n", + "Ground Truth: 627383,288484,556554,285851,166815,795228,325378,577649,893268,230411,785551,687681,260412,211877,679117,316276,959074,9552,562736,65145,741404,393,134059,445806,981209,972742,786212,824435,649453,642381\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 674\n", + "Returned IDs: 700893,556541,570343,12307,499481,50660,726824,72821,577070,387473,966890,930825,460352,100034,176934,318773,116006,867147,937728,444696,656315,757525,756037,449405,651793,341494,225852,137204,294218,154863\n", + "Ground Truth: 700893,556541,570343,12307,499481,50660,726824,72821,577070,387473,966890,930825,460352,100034,176934,318773,116006,867147,937728,444696,656315,551641,757525,756037,449405,651793,341494,225852,137204,294218\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 675\n", + "Returned IDs: 613555,335776,786944,774094,796002,166245,444369,963343,622514,167419,315013,134824,708075,118233,42904,499771,977029,819893,352976,796173,498373,184242,888650,171993,909919,554308,73510,88209,226700,464749\n", + "Ground Truth: 613555,335776,786944,774094,796002,166245,444369,963343,622514,167419,315013,134824,708075,118233,42904,81152,499771,977029,819893,352976,796173,498373,184242,888650,171993,909919,554308,73510,88209,226700\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 676\n", + "Returned IDs: 89910,323244,112854,794176,524533,524117,377137,763094,675203,928219,35066,437969,529687,587330,777278,445426,945557,268512,4206,697828,463424,219094,314367,883115,197649,147562,888781,487822,953365,774332\n", + "Ground Truth: 89910,323244,112854,794176,524533,524117,377137,763094,675203,928219,35066,437969,529687,587330,777278,445426,945557,268512,4206,697828,463424,219094,314367,883115,197649,895451,147562,888781,739431,953365\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 677\n", + "Returned IDs: 293284,301772,47500,361959,860667,796908,452981,780887,615490,379974,965630,617817,636959,610279,896647,708573,544250,603654,465605,648440,605250,267373,159662,440252,54013,818408,232960,280124,828370,899824\n", + "Ground Truth: 293284,301772,47500,361959,860667,796908,452981,780887,33759,615490,379974,965630,617817,636959,610279,896647,708573,544250,603654,465605,648440,605250,267373,159662,440252,54013,818408,232960,280124,828370\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 682\n", + "Returned IDs: 224591,696796,341028,769726,965319,459120,857994,48097,659738,398188,48974,49097,718335,242705,468811,945662,754662,754254,9893,717104,807453,513406,175264,608857,236593,303659,592825,472461,338058,793387\n", + "Ground Truth: 224591,696796,341028,769726,965319,459120,857994,48097,659738,398188,48974,49097,718335,242705,468811,945662,754662,754254,9893,717104,807453,513406,175264,608857,236593,760494,303659,592825,472461,338058\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 683\n", + "Returned IDs: 209231,600973,223287,631231,610241,74037,15876,520706,429214,434647,888851,29264,618495,469865,712974,112274,110620,915523,803117,277402,16959,882699,293939,343140,408330,830565,885712,804087,384323,913679\n", + "Ground Truth: 209231,600973,223287,631231,610241,74037,15876,520706,429214,434647,888851,29264,618495,469865,712974,112274,110620,915523,803117,277402,16959,882699,293939,483995,343140,408330,830565,885712,804087,384323\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 694\n", + "Returned IDs: 725577,818087,822346,367991,266148,3722,918980,32300,817771,310965,840283,610786,873162,861392,374860,687441,97491,672437,479624,155772,967099,579618,693980,148328,531954,44478,378148,14967,504655,187501\n", + "Ground Truth: 725577,818087,822346,367991,266148,3722,918980,32300,817771,310965,840283,610786,873162,861392,374860,687441,97491,672437,479624,155772,22194,967099,579618,693980,148328,531954,44478,378148,14967,504655\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 697\n", + "Returned IDs: 575638,825378,299769,785326,826416,46586,851631,78792,822720,351865,652301,792881,223031,916871,223192,470934,591324,494359,379812,434575,152614,207667,375043,255839,529824,162654,973467,562902,249710,461715\n", + "Ground Truth: 575638,825378,299769,785326,826416,46586,851631,78792,822720,351865,652301,792881,223031,916871,223192,470934,591324,494359,379812,434575,152614,207667,375043,255839,529824,162654,620864,973467,562902,249710\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 702\n", + "Returned IDs: 175509,939242,354296,795126,19330,754458,298766,289376,662268,879630,530206,706040,750373,989691,715670,726578,304112,957337,444672,153735,923163,231399,623756,841307,624981,500352,558878,528445,255247,376480\n", + "Ground Truth: 677492,175509,390929,939242,481000,737464,109604,889256,354296,795126,19330,754458,888933,298766,289376,766918,662268,858528,879630,530206,735023,670382,706040,750373,989691,972814,715670,726578,304112,957337\n", + "Recall: 0.6000\n", + "\n", + "Query ID: 703\n", + "Returned IDs: 636314,940641,533504,965375,266999,649063,30646,32280,294667,552225,95939,319719,307750,827319,826315,131374,36393,902121,889385,987792,968905,582069,267825,228729,290247,103604,438894,92807,479606,503115\n", + "Ground Truth: 636314,940641,533504,965375,266999,649063,30646,32280,294667,552225,95939,319719,307750,827319,826315,131374,36393,902121,889385,987792,968905,582069,267825,76413,228729,823507,3476,290247,103604,438894\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 704\n", + "Returned IDs: 301193,984,108353,784406,228331,897008,596596,500036,816097,978234,88702,128177,581195,820777,296678,330740,126521,245492,399372,211312,845681,210682,449120,515498,31899,12598,238933,737227,865827,73123\n", + "Ground Truth: 301193,984,108353,784406,228331,897008,596596,500036,816097,978234,88702,128177,65853,581195,820777,296678,330740,126521,245492,399372,211312,845681,210682,449120,515498,31899,12598,238933,737227,865827\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 708\n", + "Returned IDs: 597881,361507,136949,443568,60682,988016,899186,538897,499396,709738,403311,804689,721697,128967,147981,981043,805840,471919,375722,726748,674133,328044,893662,347241,827328,102935,939159,60315,413816,230772\n", + "Ground Truth: 597881,361507,136949,443568,60682,988016,899186,538897,499396,709738,403311,804689,721697,128967,147981,981043,805840,471919,375722,399816,726748,674133,328044,893662,347241,827328,102935,939159,60315,413816\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 709\n", + "Returned IDs: 742278,467067,738627,168219,488406,578858,403026,281311,243670,302478,298411,67129,859699,870334,523618,511267,210,408339,384481,814833,581945,952618,822040,473263,183618,984594,11562,627089,518825,424696\n", + "Ground Truth: 742278,467067,738627,168219,488406,578858,403026,281311,243670,302478,298411,67129,859699,870334,523618,511267,210,408339,384481,814833,581945,952618,822040,473263,183618,498319,99645,984594,11562,627089\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 710\n", + "Returned IDs: 686971,195239,779394,881194,738079,273085,184135,153018,174068,411675,121722,854362,162559,565851,189815,828591,930057,170602,432286,56891,698111,692622,831597,909783,144205,105119,344839,649883,651512,545484\n", + "Ground Truth: 686971,195239,779394,881194,738079,273085,184135,153018,174068,411675,121722,854362,162559,565851,189815,828591,930057,170602,432286,56891,698111,564018,692622,831597,909783,144205,105119,344839,649883,651512\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 713\n", + "Returned IDs: 676059,545592,492187,549626,381396,766391,464843,26181,384188,294588,25377,919793,685447,149797,103791,970525,677600,663314,952353,115085,142665,654964,871476,603754,802643,291056,472281,20546,627873,898405\n", + "Ground Truth: 676059,545592,492187,549626,381396,766391,464843,26181,384188,294588,25377,919793,685447,149797,103791,970525,677600,663314,952353,115085,142665,654964,871476,603754,802643,240392,291056,472281,20546,627873\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 718\n", + "Returned IDs: 790282,814687,315140,464103,21514,170631,568936,706033,898715,613177,550866,408364,304660,271179,940528,920234,374740,406974,746969,200589,154567,72059,767663,22807,342918,366912,869141,29668,353012,854130\n", + "Ground Truth: 790282,814687,315140,464103,21514,170631,568936,706033,898715,613177,550866,794944,408364,304660,271179,940528,920234,374740,406974,746969,200589,154567,72059,767663,22807,342918,366912,869141,29668,353012\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 719\n", + "Returned IDs: 830162,979459,712306,389233,72342,585104,621678,413354,897289,747036,415762,631020,781808,861752,553067,73922,691555,517,50336,92921,560682,950759,582524,487080,322629,542022,159701,472768,335658,309342\n", + "Ground Truth: 830162,979459,53339,712306,389233,72342,585104,621678,413354,897289,869371,747036,415762,631020,781808,861752,553067,73922,691555,517,50336,92921,560682,950759,582524,487080,322629,542022,159701,472768\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 724\n", + "Returned IDs: 736959,828468,322656,94276,392750,20817,254353,953696,839079,954538,774524,515426,265671,118529,595157,866402,141923,368299,612063,814032,103536,713715,716189,283906,961247,190583,352849,433794,437760,861095\n", + "Ground Truth: 736959,828468,322656,94276,392750,20817,254353,953696,839079,954538,774524,515426,265671,118529,595157,866402,141923,368299,780956,612063,814032,103536,713715,716189,283906,961247,190583,352849,433794,437760\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 726\n", + "Returned IDs: 278297,912839,418744,772798,841934,110169,860568,359811,295939,18712,42569,804528,876551,93720,949221,104666,338906,573573,160060,569938,925035,655436,759313,840924,519673,960815,801692,252820,282028,653156\n", + "Ground Truth: 278297,912839,418744,772798,841934,26444,194724,110169,860568,359811,295939,18712,42569,804528,876551,93720,949221,104666,338906,573573,160060,569938,925035,655436,759313,837145,840924,519673,960815,305936\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 727\n", + "Returned IDs: 601774,919221,701625,224708,377205,603351,560588,701270,498240,380057,176743,931092,708095,824681,184615,566485,304274,253247,186079,979201,729526,73156,615473,750693,539379,982531,462519,250323,744732,716700\n", + "Ground Truth: 601774,543047,6458,767951,919221,701625,513333,828351,224708,377205,603351,395045,560588,701270,498240,364080,929775,774847,380057,682285,151959,692258,360352,176743,931092,708095,824681,184615,566485,304274\n", + "Recall: 0.5667\n", + "\n", + "Query ID: 728\n", + "Returned IDs: 211374,243658,362896,184385,796186,947541,703121,739700,715073,524003,221832,251280,665789,722430,253578,535822,733030,210238,342000,642,479116,601278,14190,443660,74806,238387,3470,730875,282892,689925\n", + "Ground Truth: 211374,243658,362896,184385,796186,947541,703121,739700,715073,524003,221832,251280,665789,722430,253578,535822,733030,210238,342000,642,479116,601278,14190,443660,74806,238387,3470,730875,282892,767332\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 731\n", + "Returned IDs: 902199,909639,132657,389154,466750,927245,954856,585781,379870,496815,23358,716890,625443,484722,806183,702922,328017,377137,662928,770707,700848,216560,231038,40824,130662,35066,525844,229519,73954,351337\n", + "Ground Truth: 902199,909639,132657,389154,466750,927245,954856,585781,379870,496815,131975,23358,716890,625443,484722,806183,702922,328017,377137,662928,770707,700848,216560,231038,40824,796801,130662,35066,525844,229519\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 733\n", + "Returned IDs: 630397,896650,273814,776952,278338,755697,456812,34635,327310,263372,184742,175060,443576,887994,499493,359243,974206,213340,451926,484597,770244,440607,81274,315178,344342,689283,911154,579178,980279,154600\n", + "Ground Truth: 630397,896650,273814,776952,278338,755697,456812,34635,327310,263372,184742,175060,443576,887994,499493,359243,974206,213340,451926,484597,770244,440607,81274,315178,840707,344342,689283,911154,579178,980279\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 735\n", + "Returned IDs: 480693,402811,648165,872301,564992,618076,31416,978420,84884,32240,880207,151245,805104,410882,540261,624883,183142,483973,445080,165641,904884,747337,456805,781453,970573,578011,717309,171861,473385,923245\n", + "Ground Truth: 480693,402811,648165,872301,350437,564992,618076,31416,978420,84884,32240,880207,151245,380320,805104,410882,540261,985164,624883,183142,483973,445080,165641,904884,747337,456805,781453,970573,578011,717309\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 737\n", + "Returned IDs: 619645,978813,849107,363318,373296,578366,526266,397535,316557,320524,32830,773290,164276,941783,876608,488006,314510,915139,278987,278268,95808,716348,38383,526494,41549,11227,491354,207375,832248,591719\n", + "Ground Truth: 619645,978813,849107,363318,373296,578366,526266,397535,316557,320524,32830,773290,164276,941783,876608,488006,314510,915139,507339,278987,278268,95808,716348,38383,526494,41549,11227,491354,207375,832248\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 740\n", + "Returned IDs: 644677,891474,349561,774893,794841,64191,733259,829783,763405,953325,677775,705630,418109,479989,209740,217537,564518,607057,486623,747314,721929,981577,111631,919042,587366,498821,51013,42015,278645,748676\n", + "Ground Truth: 644677,891474,532672,349561,774893,794841,502470,64191,733259,829783,763405,953325,514869,677775,705630,418109,479989,209740,480196,217537,564518,607057,937959,486623,611245,44242,747314,721929,889407,755251\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 741\n", + "Returned IDs: 271387,968410,306063,589936,339210,410226,12606,243823,601108,935175,971042,210059,885939,390173,85216,984828,539544,770725,668418,647269,131865,90375,472469,831453,239553,309695,267368,954125,265258,973599\n", + "Ground Truth: 271387,968410,306063,589936,339210,410226,12606,243823,601108,935175,281495,971042,210059,885939,390173,85216,984828,539544,770725,668418,647269,131865,90375,472469,831453,239553,309695,267368,954125,265258\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 744\n", + "Returned IDs: 851004,857095,892694,97241,40212,932870,510047,791959,193388,499906,921294,903712,556947,360738,619047,916571,733853,566261,528791,772820,3673,82794,636316,383259,342322,184811,928945,655764,615678,801878\n", + "Ground Truth: 851004,857095,892694,124754,97241,40212,932870,758010,510047,791959,691462,193388,499906,921294,903712,556947,222908,360738,619047,916571,280588,733853,566261,528791,772820,808467,13797,3673,82794,287544\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 747\n", + "Returned IDs: 613943,339708,786275,982988,428450,483553,280929,166823,830267,945999,530979,182212,503642,236554,22639,115602,744769,628546,313876,130203,104243,782695,191682,103607,121382,543216,935072,700089,463626,636683\n", + "Ground Truth: 613943,339708,786275,982988,428450,483553,280929,166823,830267,945999,530979,17394,182212,503642,236554,22639,115602,744769,628546,313876,130203,104243,782695,191682,103607,121382,543216,935072,700089,463626\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 749\n", + "Returned IDs: 570139,225335,78583,652024,111444,80259,70291,59793,921997,891344,605901,228529,906731,844582,251964,330063,499660,347906,779669,101505,138604,444003,361722,798355,528017,964169,302405,898868,727316,685447\n", + "Ground Truth: 570139,225335,78583,652024,111444,80259,70291,59793,921997,891344,605901,228529,906731,844582,251964,330063,499660,347906,779669,101505,138604,444003,361722,798355,528017,964169,302405,898868,672389,727316\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 754\n", + "Returned IDs: 474090,679777,568149,510076,271497,503519,469435,616568,717309,811349,165917,440388,938757,46156,236773,47625,914513,341037,273453,529687,871971,925668,545059,516390,794504,763094,133008,164462,166103,54841\n", + "Ground Truth: 474090,679777,568149,510076,271497,503519,469435,616568,717309,811349,165917,440388,938757,46156,236773,47625,653985,914513,857089,341037,273453,529687,378707,871971,925668,545059,299934,516390,794504,763094\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 755\n", + "Returned IDs: 307679,470060,615497,106018,687262,818104,927223,286791,960201,326060,834065,954929,4506,69809,871996,61080,228250,745388,813043,835919,382461,706276,921482,9781,140020,3186,694472,225107,734889,208066\n", + "Ground Truth: 307679,470060,615497,106018,687262,818104,927223,286791,960201,326060,834065,954929,4506,69809,871996,61080,228250,745388,813043,835919,382461,706276,921482,9781,140020,3186,694472,225107,734889,654575\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 761\n", + "Returned IDs: 627166,531403,258376,532356,98284,885327,56572,537506,626871,558981,346235,398253,207492,58129,137077,252624,489659,111158,588359,924066,775978,946358,340748,187448,721973,659530,301732,875753,197245,670856\n", + "Ground Truth: 627166,531403,258376,569533,532356,98284,942384,885327,56572,657497,537506,626871,808429,541292,558981,346235,398253,21698,207492,58129,137077,252624,489659,111158,588359,924066,370948,775978,946358,340748\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 762\n", + "Returned IDs: 784939,461074,351498,625820,24863,863971,43064,213446,239933,176805,175762,705205,253645,888814,964626,938520,876957,100857,952010,27601,84404,948969,86096,550389,897937,43080,828262,750632,695039,670909\n", + "Ground Truth: 784939,461074,351498,625820,24863,863971,43064,213446,239933,176805,175762,705205,253645,888814,964626,938520,876957,100857,952010,27601,84404,948969,86096,550389,897937,43080,867096,828262,750632,695039\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 763\n", + "Returned IDs: 728291,170403,107366,807709,601595,882271,616910,165224,437651,641012,759375,718836,383566,451966,272660,313221,367539,576729,956084,186113,641310,352325,577985,871709,267329,696373,146447,769725,705677,934632\n", + "Ground Truth: 728291,170403,107366,807709,601595,882271,616910,165224,437651,641012,718836,759375,383566,451966,272660,313221,367539,576729,956084,186113,641310,352325,577985,871709,267329,696373,146447,769725,705677,93476\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 770\n", + "Returned IDs: 818526,881410,749004,612005,480015,275570,282418,961282,916592,151899,854592,148074,241932,298210,679039,437268,371719,110627,568129,217858,209770,375362,109399,827256,498657,303177,719210,118962,664865,692679\n", + "Ground Truth: 818526,881410,749004,612005,480015,275570,282418,961282,916592,151899,854592,148074,241932,298210,679039,718744,437268,371719,110627,568129,217858,209770,375362,783711,109399,827256,498657,303177,719210,118962\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 771\n", + "Returned IDs: 612125,898912,716094,383459,157433,127418,50112,620959,571637,120409,830429,189432,478435,943426,754533,651381,692932,271381,843940,956172,141189,603949,259754,534494,281421,986047,671041,32659,457158,929703\n", + "Ground Truth: 612125,898912,716094,383459,157433,127418,50112,620959,571637,120409,830429,189432,478435,943426,754533,651381,692932,271381,843940,956172,141189,603949,259754,825694,534494,281421,986047,671041,32659,457158\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 773\n", + "Returned IDs: 135196,679058,578737,393271,709277,75402,759984,582920,364950,442311,788828,448179,526128,131127,623519,725627,942813,401402,431595,332810,772035,654528,609612,441622,162588,430176,67815,67136,30211,130172\n", + "Ground Truth: 135196,679058,578737,831148,393271,709277,75402,759984,582920,364950,442311,788828,448179,526128,131127,623519,725627,700863,942813,401402,431595,332810,772035,654528,609612,441622,162588,430176,67815,67136\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 776\n", + "Returned IDs: 110655,667266,289780,220173,720740,450538,960809,577277,320061,605939,22762,47736,409437,749867,724789,344823,341596,2617,288348,859918,525662,633080,741541,565220,975584,385827,46680,811782,561974,187087\n", + "Ground Truth: 110655,667266,289780,220173,720740,543580,450538,960809,577277,320061,605939,22762,47736,409437,749867,724789,344823,341596,2617,288348,859918,525662,633080,792682,741541,565220,975584,385827,46680,945123\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 783\n", + "Returned IDs: 922832,228863,28218,904559,595252,309804,373874,938233,606036,915445,504696,529811,704840,119798,331859,514098,281209,810927,239343,815065,49107,878754,468154,7571,171231,627067,388301,904238,57289,371657\n", + "Ground Truth: 922832,228863,28218,904559,595252,309804,373874,938233,606036,915445,504696,529811,704840,119798,331859,514098,281209,810927,239343,815065,49107,878754,468154,7571,171231,627067,388301,904238,57289,249244\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 788\n", + "Returned IDs: 620050,169755,82839,66991,374712,54680,250177,16579,461321,166695,973973,253332,973705,683807,357392,36527,350669,976687,893704,541044,145167,136032,248904,368717,909390,896505,327192,339786,948812,651429\n", + "Ground Truth: 620050,169755,82839,66991,374712,54680,250177,16579,461321,166695,973973,253332,973705,683807,357392,36527,350669,976687,893704,541044,145167,136032,248904,368717,909390,62168,896505,327192,720442,172640\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 790\n", + "Returned IDs: 494590,697069,476593,887583,68191,872474,147991,588273,446442,507095,189013,157715,662847,80487,892532,772796,152760,529135,846761,229519,498210,314279,728954,415574,545622,130604,155325,103827,726975,85881\n", + "Ground Truth: 494590,697069,476593,887583,68191,872474,147991,588273,446442,507095,189013,157715,472984,662847,80487,892532,307474,772796,152760,371361,529135,846761,250721,229519,498210,314279,527617,728954,415574,486364\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 791\n", + "Returned IDs: 643044,188503,950183,577547,233840,31025,85110,712573,263885,542409,94912,264481,808395,388584,647249,236796,257063,618839,11267,136482,470696,614936,973190,731983,950404,357520,107926,907974,138633,696958\n", + "Ground Truth: 643044,188503,950183,577547,233840,31025,85110,712573,263885,542409,94912,264481,808395,388584,647249,236796,257063,618839,1657,11267,136482,470696,614936,973190,731983,950404,357520,107926,907974,138633\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 792\n", + "Returned IDs: 103643,982841,76876,131614,304402,277688,233237,491898,472234,30128,791723,42844,818761,941100,8634,167210,676258,429624,944061,31446,811673,902018,64760,904442,824056,251243,237202,755251,190147,170414\n", + "Ground Truth: 103643,982841,76876,131614,304402,277688,233237,491898,472234,30128,791723,42844,532680,818761,941100,8634,167210,676258,429624,944061,31446,811673,902018,64760,904442,943067,353268,824056,517818,251243\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 800\n", + "Returned IDs: 443449,620473,127834,41146,343110,471608,71956,490155,820327,821507,19367,185266,359362,281247,933588,355884,182247,445843,901468,181412,879503,276481,965996,305809,394351,639169,385763,350991,486245,889335\n", + "Ground Truth: 443449,949216,740228,620473,918058,127834,323173,556530,41146,643603,234730,343110,471608,71956,727643,490155,669870,240726,820327,821507,19367,185266,359362,281247,933588,355884,182247,445843,901468,181412\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 802\n", + "Returned IDs: 269238,839967,412865,131067,822382,950900,585314,579302,554825,191077,310058,932573,465453,384598,163712,1915,629445,746003,847419,280836,825647,131531,592089,416435,66962,436819,466040,866614,747061,963898\n", + "Ground Truth: 269238,839967,412865,131067,358477,822382,950900,585314,579302,554825,191077,310058,932573,465453,384598,163712,1915,629445,746003,847419,280836,825647,131531,592089,416435,66962,436819,466040,866614,747061\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 807\n", + "Returned IDs: 210412,673161,344470,308984,799197,718452,539406,179708,802206,289930,839736,802277,649766,467441,823345,988298,55655,114623,317686,258591,501445,835892,464190,126328,620445,871971,883115,239245,975496,182270\n", + "Ground Truth: 210412,673161,344470,308984,799197,718452,539406,179708,802206,289930,839736,802277,649766,467441,823345,988298,417489,55655,114623,317686,258591,501445,835892,464190,126328,620445,871971,883115,239245,975496\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 812\n", + "Returned IDs: 420249,798562,125873,643276,430038,842812,471074,154843,874840,215560,469558,225999,607434,739129,481972,670727,11887,593669,528622,243340,473317,71885,486258,550160,735749,536763,691518,763904,774817,265727\n", + "Ground Truth: 420249,798562,125873,643276,430038,842812,471074,154843,874840,215560,469558,225999,607434,739129,481972,670727,11887,593669,894709,528622,243340,473317,71885,486258,550160,735749,536763,691518,763904,774817\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 814\n", + "Returned IDs: 892069,177321,140068,857569,864412,606617,253225,957570,184938,922172,811345,561074,831712,267584,539386,103717,595721,654658,530470,528534,712709,215663,730549,690319,703326,815253,432827,531675,520174,296938\n", + "Ground Truth: 892069,177321,140068,857569,864412,606617,253225,957570,184938,922172,811345,561074,831712,267584,539386,517770,103717,595721,654658,530470,528534,712709,215663,730549,690319,703326,815253,432827,535277,50556\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 815\n", + "Returned IDs: 219636,796436,78945,187548,915690,128205,302140,926385,680887,325681,450132,652363,954356,168702,67813,736204,348106,916780,246489,114519,125604,247594,865052,458563,526409,212182,479989,608771,382840,144283\n", + "Ground Truth: 219636,796436,78945,187548,915690,128205,302140,926385,680887,325681,450132,652363,954356,168702,67813,736204,348106,916780,246489,114519,125604,247594,865052,458563,526409,116390,212182,479989,608771,382840\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 818\n", + "Returned IDs: 222540,33088,296686,64424,29927,315322,829911,191867,5408,114079,954172,515763,586692,211249,467644,318000,714661,141461,232220,454204,343807,928945,23358,561633,677469,176925,354452,429522,759195,800775\n", + "Ground Truth: 222540,33088,296686,64424,29927,315322,829911,191867,5408,99589,114079,448769,954172,515763,586692,734864,211249,845749,467644,318000,714661,141461,298732,185773,232220,647478,454204,343807,928945,23358\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 832\n", + "Returned IDs: 369470,329052,948873,268162,223905,444438,523049,824460,418108,269367,150812,537834,725366,777799,181325,132671,641287,210095,724728,866770,237932,123832,229519,78224,61754,498376,898787,635318,160372,494693\n", + "Ground Truth: 369470,329052,948873,268162,223905,444438,523049,824460,418108,269367,150812,537834,725366,777799,181325,132671,641287,210095,724728,866770,237932,123832,229519,78224,61754,498376,898787,635318,182737,160372\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 837\n", + "Returned IDs: 450812,130298,102438,835119,751927,395640,620999,940878,763690,2345,527174,651936,836956,875904,834742,820352,285382,635290,533415,469340,822949,811872,544545,599988,449898,713410,238236,443785,260760,50609\n", + "Ground Truth: 450812,130298,102438,835119,751927,395640,620999,940878,763690,2345,527174,651936,536651,836956,875904,834742,820352,285382,708115,635290,533415,469340,822949,811872,544545,599988,449898,74019,713410,238236\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 849\n", + "Returned IDs: 9293,914011,255786,118357,222750,984794,566228,125512,115486,102584,509468,910639,519952,33227,658160,810463,389195,931801,359040,775261,821526,336013,378165,816470,300902,468424,90349,730306,67680,486997\n", + "Ground Truth: 9293,914011,255786,118357,222750,984794,566228,125512,115486,102584,509468,910639,519952,649943,832134,348170,33227,658160,810463,389195,931801,359040,775261,821526,659257,987729,336013,378165,816470,300902\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 855\n", + "Returned IDs: 727768,38546,291074,824962,217368,667688,843153,792223,511581,879825,546513,452628,930813,540288,119630,744117,231332,315913,445527,5575,574591,735598,845648,47536,671877,466719,169049,813947,744482,344072\n", + "Ground Truth: 727768,38546,291074,824962,217368,667688,843153,792223,511581,879825,546513,289972,452628,930813,540288,119630,744117,231332,315913,445527,5575,574591,735598,845648,47536,671877,466719,169049,813947,744482\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 856\n", + "Returned IDs: 907401,132176,295647,439880,606751,253645,162089,597394,159951,444369,786944,803865,61450,962635,175223,892285,70875,148756,931325,929698,819893,684317,759121,613491,220032,705991,304135,114363,832079,305575\n", + "Ground Truth: 907401,132176,295647,439880,606751,253645,162089,597394,159951,444369,786944,803865,61450,962635,175223,892285,70875,148756,931325,929698,819893,563865,615450,684317,759121,613491,220032,705991,304135,114363\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 861\n", + "Returned IDs: 869826,474900,848283,618192,658666,659128,834327,559216,734225,628304,17384,851425,623076,788745,962803,737465,79528,25982,342870,837641,421516,364319,263244,836487,688286,85458,94527,437455,799112,306268\n", + "Ground Truth: 869826,474900,848283,618192,658666,659128,834327,559216,734225,628304,17384,851425,623076,788745,962803,737465,79528,25982,342870,837641,421516,364319,263244,836487,637338,688286,85458,94527,437455,799112\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 862\n", + "Returned IDs: 112384,713861,459285,15666,94949,933618,882115,696932,979927,193458,805963,552361,556479,106462,143774,504182,944672,775181,790742,688726,786459,789100,295659,35132,651167,171418,473307,620860,755560,570790\n", + "Ground Truth: 733007,366573,112384,713861,459285,15666,94949,962134,619884,933618,882115,455657,696932,358398,246616,979927,333155,193458,805963,562510,552361,556479,134250,106462,143774,504182,944672,263749,775181,790742\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 864\n", + "Returned IDs: 507385,357775,756700,773062,558848,347884,77788,573479,838660,305083,943482,7853,650687,485809,84884,602759,823654,503411,497168,840305,311608,749429,29497,962118,437293,54512,896338,170922,448907,928428\n", + "Ground Truth: 507385,357775,756700,773062,558848,347884,77788,573479,838660,305083,943482,7853,650687,485809,84884,602759,823654,503411,497168,840305,311608,749429,29497,962118,437293,54512,896338,767632,170922,448907\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 870\n", + "Returned IDs: 823215,354345,187553,486944,282325,440122,499261,941006,707218,606988,766728,955785,918106,950285,984194,809763,957443,156336,870409,402964,367079,856449,722063,805290,918560,431744,818574,151167,289811,283616\n", + "Ground Truth: 823215,354345,187553,486944,282325,440122,499261,941006,707218,606988,766728,955785,918106,950285,984194,809763,957443,156336,870409,402964,367079,583107,856449,722063,805290,918560,431744,818574,151167,289811\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 871\n", + "Returned IDs: 244934,804071,160305,306125,497677,733053,850316,368551,111568,955585,349793,29618,706362,313207,98518,17190,200458,615023,715565,281070,469308,757543,875443,265956,586420,686531,448971,301536,897559,66043\n", + "Ground Truth: 244934,804071,160305,306125,497677,733053,850316,368551,111568,955585,349793,29618,706362,313207,98518,17190,153120,200458,615023,715565,281070,469308,757543,875443,913080,265956,586420,686531,448971,301536\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 874\n", + "Returned IDs: 144273,17885,816961,438367,977378,188338,672356,252379,92322,865856,522130,446162,710709,569951,422187,306829,667281,971729,680755,671449,211801,98623,605305,197364,305854,84040,881297,900470,778759,921305\n", + "Ground Truth: 144273,17885,816961,438367,977378,188338,672356,252379,92322,865856,522130,446162,710709,569951,422187,306829,667281,971729,680755,671449,211801,98623,605305,197364,305854,84040,881297,307139,900470,778759\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 875\n", + "Returned IDs: 377928,780788,325285,679329,415016,144490,257622,170538,278959,349444,808595,575433,48585,401386,37541,572088,747586,429263,98656,557400,32360,440482,967160,226652,29700,956999,592174,512815,496751,448378\n", + "Ground Truth: 377928,780788,325285,679329,415016,144490,257622,170538,278959,349444,808595,575433,48585,401386,37541,572088,747586,429263,98656,557400,32360,440482,967160,226652,29700,956999,592174,389824,512815,448378\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 877\n", + "Returned IDs: 407647,249140,459302,850162,469630,194461,790570,482938,284634,555877,249284,24771,865657,599676,614534,311462,716601,797290,611597,848134,467919,836540,621475,512998,339185,209362,341330,285864,739503,621195\n", + "Ground Truth: 407647,249140,459302,850162,469630,194461,790570,482938,284634,555877,249284,24771,865657,599676,614534,311462,716601,797290,611597,848134,467919,836540,621475,512998,339185,209362,497750,659824,571697,341330\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 883\n", + "Returned IDs: 444691,49388,601127,48679,430106,296236,870285,684431,636192,820180,228001,277333,319640,104341,736962,962007,980923,739095,2987,867529,336061,484857,136705,818456,81881,859033,180709,933777,962246,290435\n", + "Ground Truth: 444691,49388,601127,48679,430106,296236,870285,684431,636192,820180,228001,277333,319640,104341,736962,962007,980923,739095,2987,867529,336061,484857,136705,435183,818456,957369,81881,859033,180709,933777\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 885\n", + "Returned IDs: 513527,680643,126652,854583,870236,144177,118873,641713,987862,688076,667176,761486,738195,13941,613227,681062,137605,638786,594397,599817,183667,914605,212925,807621,794223,544626,111263,751106,186388,206926\n", + "Ground Truth: 513527,680643,126652,854583,870236,144177,118873,641713,987862,688076,667176,761486,738195,13941,613227,681062,258966,137605,638786,382421,594397,599817,183667,914605,212925,807621,481246,794223,464320,544626\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 894\n", + "Returned IDs: 872922,723485,810343,308304,635570,431480,339438,271857,334997,594990,71335,673035,795893,953349,760211,568152,49725,482827,516060,690408,391643,929962,669591,363620,418983,18362,978874,821331,253215,713388\n", + "Ground Truth: 872922,723485,810343,308304,635570,431480,339438,271857,334997,594990,71335,673035,795893,953349,760211,568152,49725,482827,516060,690408,391643,929962,669591,363620,418983,18362,978874,821331,10232,253215\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 898\n", + "Returned IDs: 692011,34657,467206,382450,361534,753431,517697,608214,727954,473726,649883,970534,929793,799647,989054,602764,136828,398228,606854,954673,271506,476614,91378,969987,705913,341158,373102,137710,428750,364147\n", + "Ground Truth: 692011,322724,34657,467206,382450,361534,753431,339520,445611,517697,608214,985784,692541,952510,727954,473726,649883,970534,277351,929793,635157,283825,799647,918989,989054,602764,136828,230283,319529,398228\n", + "Recall: 0.6000\n", + "\n", + "Query ID: 899\n", + "Returned IDs: 710239,892746,855179,154514,879398,967996,257239,802881,659905,302208,653036,425280,768387,785473,156108,274156,822313,818486,887453,787737,288799,949626,506185,545521,986783,746516,586866,944478,989545,557187\n", + "Ground Truth: 710239,892746,855179,154514,879398,967996,257239,802881,659905,302208,653036,425280,768387,785473,156108,274156,822313,818486,887453,787737,288799,800420,949626,506185,545521,986783,746516,586866,944478,989545\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 903\n", + "Returned IDs: 413461,771804,456130,168394,820157,164746,776820,71150,898674,176747,321616,337017,862417,482816,685244,145121,305353,15299,897486,132320,842511,28958,854427,217827,468916,347688,630921,827532,289085,187509\n", + "Ground Truth: 413461,771804,456130,168394,820157,164746,776820,71150,898674,176747,321616,337017,862417,482816,685244,145121,305353,15299,897486,304211,132320,842511,28958,854427,217827,468916,589659,347688,630921,827532\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 907\n", + "Returned IDs: 385032,608558,237047,463588,473156,459782,620768,82017,215475,519494,415195,950931,240901,467763,958906,766488,775718,965309,46392,417916,594597,736420,531217,320552,771894,371451,413722,882541,937845,513327\n", + "Ground Truth: 385032,608558,237047,463588,821306,473156,459782,620768,82017,215475,519494,415195,950931,240901,467763,958906,766488,775718,965309,46392,417916,594597,736420,531217,320552,771894,371451,413722,882541,937845\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 916\n", + "Returned IDs: 700942,20888,152088,490764,238020,479859,101340,236706,937984,850129,651208,250391,44450,90600,569789,852054,300599,560029,620909,634994,413813,536432,948071,914915,359538,590518,298080,98827,110531,70700\n", + "Ground Truth: 700942,20888,152088,490764,238020,479859,101340,236706,937984,850129,651208,250391,44450,90600,569789,852054,300599,560029,620909,634994,413813,536432,948071,914915,359538,590518,875632,298080,98827,110531\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 918\n", + "Returned IDs: 815042,369098,772750,342323,327675,860925,962004,865892,210006,461788,76678,331371,840048,382061,525860,894794,221248,250669,589625,429673,974497,780007,396259,924008,604008,964973,594563,798140,863351,304245\n", + "Ground Truth: 815042,369098,772750,342323,787157,327675,860925,962004,865892,210006,461788,76678,331371,840048,382061,525860,894794,221248,250669,589625,429673,974497,533667,780007,396259,924008,604008,964973,594563,798140\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 919\n", + "Returned IDs: 122855,612592,887462,197625,781313,82128,590056,690167,192123,878786,875282,150346,258435,38528,47911,436835,255900,831002,809784,94332,406077,870507,697122,395809,467753,628848,689403,933931,572216,857545\n", + "Ground Truth: 122855,612592,887462,197625,781313,82128,590056,690167,192123,878786,875282,150346,258435,38528,47911,436835,636637,255900,831002,809784,94332,406077,870507,697122,395809,467753,628848,689403,933931,572216\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 922\n", + "Returned IDs: 829763,835250,777446,556045,727132,23036,491145,531636,931799,726905,382118,906283,797257,917693,855897,760092,288465,305850,144662,727737,290395,106945,856940,906696,111659,770165,809236,576871,73303,576590\n", + "Ground Truth: 829763,835250,777446,556045,727132,23036,491145,531636,931799,726905,382118,218117,906283,797257,917693,855897,760092,288465,305850,144662,727737,290395,106945,856940,906696,111659,770165,809236,576871,73303\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 925\n", + "Returned IDs: 700318,544946,211038,874007,46612,766495,716014,454851,292768,191748,537941,41107,158353,515097,132356,258799,987635,989633,678087,875612,330388,368929,886259,969850,692066,384481,566844,42872,531498,566592\n", + "Ground Truth: 700318,544946,211038,874007,46612,766495,716014,454851,292768,191748,537941,41107,158353,515097,132356,258799,987635,989633,678087,875612,330388,368929,886259,969850,860256,692066,384481,566844,42872,531498\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 930\n", + "Returned IDs: 395142,59494,893656,857478,536944,955134,904245,693560,213628,104693,753849,945585,690746,153522,987883,472561,684999,399198,377176,966123,723045,741611,241140,837836,162005,253099,26906,435640,141890,750016\n", + "Ground Truth: 395142,59494,893656,857478,536944,955134,904245,693560,213628,104693,753849,945585,690746,153522,987883,472561,684999,399198,377176,966123,723045,741611,241140,837836,96382,162005,253099,26906,435640,141890\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 933\n", + "Returned IDs: 563108,921184,495323,153932,35077,202011,120918,889285,135420,503897,815359,560682,38653,817224,140661,300231,699768,418507,555826,662631,779023,112439,184158,20150,509207,782647,860043,986919,745699,94404\n", + "Ground Truth: 821800,563108,921184,267397,495323,153932,35077,96460,202011,120918,889285,135420,503897,809558,815359,560682,38653,817224,863864,140661,300231,699768,418507,555826,662631,757269,779023,112439,122778,184158\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 938\n", + "Returned IDs: 592965,595692,258407,484881,197407,282605,910314,897999,242735,966037,826765,384667,898290,580641,654292,643685,823144,650444,643917,774077,333481,608399,33113,90671,777933,245162,350669,586608,590540,475319\n", + "Ground Truth: 592965,595692,258407,484881,197407,282605,910314,897999,242735,966037,805438,852076,826765,255364,384667,898290,580641,654292,643685,823144,650444,643917,154301,689649,774077,333481,608399,33113,90671,777933\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 940\n", + "Returned IDs: 506417,801230,100599,492364,130786,107968,814245,397852,558055,96994,332370,91660,585233,703286,944443,287360,62852,707519,434284,42570,137476,933138,677396,953232,623534,520271,572516,621090,978697,239697\n", + "Ground Truth: 506417,801230,100599,492364,130786,607,107968,814245,397852,558055,96994,332370,91660,585233,703286,944443,287360,62852,707519,434284,42570,137476,933138,677396,953232,623534,520271,572516,621090,978697\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 941\n", + "Returned IDs: 443660,232450,868623,18712,900814,440597,381587,237988,616568,491147,892667,76270,551274,493103,671796,679648,414684,886837,549070,236773,888781,914017,344199,235851,680541,868705,221548,84188,781264,969112\n", + "Ground Truth: 443660,232450,868623,18712,900814,440597,381587,811664,237988,616568,131684,491147,892667,76270,551274,493103,671796,679648,507158,414684,886837,549070,236773,888781,914017,344199,235851,680541,868705,221548\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 943\n", + "Returned IDs: 703610,350244,247387,829745,683520,410310,652679,675093,95579,925906,155109,364172,84416,755666,479585,474779,75513,118988,731672,237779,843502,155401,310938,843237,644647,202937,857734,851692,462668,826825\n", + "Ground Truth: 703610,350244,247387,829745,683520,410310,652679,675093,95579,925906,155109,364172,84416,755666,479585,474779,75513,623374,118988,237779,731672,843502,155401,310938,843237,644647,202937,857734,851692,462668\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 946\n", + "Returned IDs: 201334,765788,69826,232031,533515,183831,460193,940438,707381,724000,249728,379962,139651,558838,340694,365523,168771,597680,520782,494764,841883,192434,508291,313055,587390,140814,525439,410622,307220,326828\n", + "Ground Truth: 201334,765788,526094,69826,232031,533515,183831,460193,940438,707381,724000,249728,379962,139651,558838,340694,365523,168771,597680,520782,494764,841883,192434,508291,313055,587390,140814,525439,410622,307220\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 947\n", + "Returned IDs: 1868,400255,210207,616906,335246,304112,671576,637270,717240,267526,323948,459637,929162,476106,369559,133153,255527,918027,757869,95504,312972,622516,629811,768890,179101,662268,309960,230183,116196,107550\n", + "Ground Truth: 1868,400255,210207,616906,335246,304112,671576,637270,267526,717240,323948,459637,929162,476106,369559,582181,133153,255527,918027,757869,95504,312972,622516,629811,768890,179101,662268,569829,309960,230183\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 948\n", + "Returned IDs: 691595,935020,584691,250702,965130,903411,149745,192379,617781,144058,638285,847946,756181,351865,912735,624883,663820,395574,454980,958030,947854,349742,447258,239576,408799,617490,108240,147009,209340,916740\n", + "Ground Truth: 691595,935020,584691,250702,965130,903411,149745,192379,617781,144058,638285,847946,756181,351865,912735,624883,318162,663820,395574,454980,958030,947854,349742,447258,239576,408799,617490,108240,147009,209340\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 953\n", + "Returned IDs: 216560,761249,351337,164273,299934,629135,53872,776150,821819,806183,623756,73954,123589,23358,151034,352956,608674,841978,328017,732256,65235,684679,6854,747036,683168,279488,150524,163105,345315,676675\n", + "Ground Truth: 127658,216560,761249,351337,164273,299934,629135,53872,776150,821819,806183,623756,73954,123589,23358,151034,352956,608674,841978,328017,839518,732256,65235,684679,6854,747036,683168,279488,150524,163105\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 955\n", + "Returned IDs: 12048,419284,352643,122230,884714,940827,245492,155893,45723,508567,671796,747036,130558,440388,834945,37314,444231,202141,467769,86040,124395,988460,546578,535506,164640,662631,21831,53872,12747,332716\n", + "Ground Truth: 12048,419284,352643,122230,884714,940827,245492,155893,45723,508567,671796,747036,130558,440388,838747,834945,752098,37314,444231,202141,467769,86040,124395,988460,546578,902128,535506,164640,432541,662631\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 956\n", + "Returned IDs: 696201,262413,939159,150524,523419,475218,739311,686965,629383,430212,123648,577301,651192,988016,339626,827328,36887,58297,99189,831938,216866,683471,25948,417267,401150,971517,887965,130790,102348,720964\n", + "Ground Truth: 696201,262413,939159,150524,523419,176916,475218,755742,739311,686965,629383,430212,123648,577301,651192,988016,339626,827328,36887,58297,99189,831938,216866,710290,683471,25948,417267,401150,971517,887965\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 958\n", + "Returned IDs: 626945,457332,84520,190255,165099,789942,489058,341221,363224,130875,156302,300649,683560,746390,314158,896013,187189,420899,763282,618920,415162,836061,287824,102823,153237,273759,120909,806568,260681,27410\n", + "Ground Truth: 626945,457332,84520,190255,165099,789942,489058,341221,363224,130875,156302,985774,300649,683560,746390,314158,896013,187189,420899,763282,618920,415162,836061,287824,689477,102823,153237,273759,120909,806568\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 962\n", + "Returned IDs: 17440,840691,802919,293805,856401,632549,384718,866909,96285,632640,264423,598582,806000,706040,648943,44308,738315,930286,823654,413754,92708,151352,866151,892464,323100,799071,834243,672869,150524,841244\n", + "Ground Truth: 17440,840691,802919,293805,856401,632549,384718,866909,96285,632640,264423,598582,806000,706040,648943,44308,738315,930286,823654,413754,600672,92708,151352,866151,892464,323100,799071,834243,672869,150524\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 963\n", + "Returned IDs: 123325,836793,423137,39458,687383,854250,434731,735170,601354,139227,678012,496388,983459,377043,742603,113191,156322,501105,177710,94812,541382,287291,981676,850700,693414,900655,278788,18246,39422,627339\n", + "Ground Truth: 123325,836793,423137,39458,687383,854250,434731,735170,601354,139227,678012,496388,983459,377043,742603,113191,834529,156322,501105,177710,94812,541382,287291,468552,981676,850700,693414,900655,278788,18246\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 966\n", + "Returned IDs: 784849,714981,853715,264740,187164,241313,37034,215538,326956,208021,825715,156392,656185,194713,625125,792108,395304,739129,182699,685541,199300,26759,601616,30040,593669,734837,848154,205065,183529,723312\n", + "Ground Truth: 784849,633289,714981,853715,264740,187164,241313,37034,215538,326956,208021,825715,156392,656185,194713,625125,792108,395304,739129,182699,685541,199300,26759,601616,30040,593669,734837,848154,205065,183529\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 969\n", + "Returned IDs: 90346,235446,105271,331662,492758,442586,414388,133351,869150,549859,363458,338026,863092,21950,399341,163951,830120,936531,525888,659479,678246,411815,879219,866289,790617,489862,909430,693221,875473,382699\n", + "Ground Truth: 90346,735842,235446,105271,331662,492758,442586,414388,133351,869150,549859,363458,338026,863092,21950,399341,163951,830120,936531,525888,659479,678246,411815,879219,740871,866289,790617,353614,489862,909430\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 970\n", + "Returned IDs: 163180,636316,306131,273286,674989,249244,380133,546618,801291,111253,683471,465950,323100,612459,937984,941292,756075,943151,184731,400310,474250,767107,348549,85881,950907,705053,184413,314182,268244,705794\n", + "Ground Truth: 163180,636316,306131,273286,674989,249244,380133,546618,801291,111253,683471,465950,323100,612459,937984,941292,756075,943151,184731,400310,474250,767107,348549,85881,950907,705053,184413,314182,268244,333080\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 976\n", + "Returned IDs: 546433,857681,571271,647596,430600,970256,230118,742698,979605,117816,858354,96593,940666,6313,687642,306908,771408,172485,408787,576533,581945,900664,615207,859621,568921,852373,905114,417978,632053,801447\n", + "Ground Truth: 546433,857681,571271,647596,430600,970256,230118,742698,979605,117816,858354,609565,96593,940666,6313,687642,306908,771408,172485,408787,827077,576533,581945,900664,615207,859621,568921,852373,905114,417978\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 978\n", + "Returned IDs: 211806,616373,763436,567108,431809,954125,520563,756669,238139,151112,594313,975638,98872,324480,390983,592169,140078,56570,834736,722764,971222,675527,453036,947625,535689,701455,958974,939976,6558,491662\n", + "Ground Truth: 211806,616373,763436,567108,431809,954125,520563,756669,238139,151112,594313,975638,98872,324480,390983,592169,140078,56570,834736,54467,722764,971222,675527,453036,947625,535689,701455,958974,939976,6558\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 983\n", + "Returned IDs: 700004,764404,915726,561695,178864,18263,411113,860968,128205,360074,400296,652803,925947,240905,353436,267526,959202,774926,704752,946795,60288,461277,485448,377476,240363,641338,380114,757421,142830,237857\n", + "Ground Truth: 700004,764404,915726,561695,178864,18263,411113,860968,128205,360074,400296,652803,925947,240905,353436,267526,959202,774926,704752,946795,60288,830952,461277,485448,377476,240363,641338,380114,757421,142830\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 985\n", + "Returned IDs: 363349,768075,746179,405796,768874,204791,344697,894155,376820,786770,175737,977374,861239,779679,816858,88671,614393,532188,327414,307039,587790,680799,880411,159864,868512,242729,410353,695506,166192,127698\n", + "Ground Truth: 363349,768075,746179,405796,768874,204791,344697,894155,376820,786770,175737,977374,861239,779679,816858,88671,614393,532188,327414,307039,587790,795565,680799,880411,159864,868512,242729,410353,695506,166192\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 986\n", + "Returned IDs: 278846,527042,590023,684601,397345,13131,942561,509579,909953,266750,261233,278236,464392,646624,455595,516390,955495,75175,406548,212925,963455,550051,420890,29259,10525,401067,18357,475467,751098,915514\n", + "Ground Truth: 278846,527042,590023,684601,397345,13131,942561,509579,909953,266750,261233,278236,464392,646624,455595,516390,955495,443021,220253,75175,406548,212925,963455,550051,420890,394322,29259,10525,401067,18357\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 988\n", + "Returned IDs: 667354,531719,140470,413076,546268,506263,170914,26304,612261,705870,252625,851098,210535,909567,10273,471423,857942,40746,704539,348514,235135,275911,126103,854057,709426,429948,552570,122372,691885,17653\n", + "Ground Truth: 667354,531719,140470,413076,546268,506263,170914,26304,612261,705870,252625,851098,210535,909567,10273,232250,471423,857942,40746,704539,348514,235135,275911,126103,854057,709426,429948,552570,122372,691885\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 990\n", + "Returned IDs: 538269,364698,834994,507882,519268,247634,501894,833836,684899,836413,765623,664547,929787,182790,256689,617911,247002,618315,781966,165373,787943,496058,988844,178898,21471,166846,264756,802190,560801,916166\n", + "Ground Truth: 538269,364698,836273,834994,507882,519268,247634,501894,833836,684899,836413,765623,664547,929787,182790,256689,617911,247002,618315,781966,165373,787943,496058,988844,178898,21471,166846,264756,802190,560801\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 992\n", + "Returned IDs: 770591,709426,217294,947430,209588,92787,831607,767668,975064,328481,669373,579782,794684,162963,104914,723717,920918,793343,798103,800090,956108,824637,755352,146771,707383,667354,166528,506263,444029,346073\n", + "Ground Truth: 770591,509690,212231,709426,217294,947430,262773,971408,209588,92787,831607,767668,975064,328481,669373,579782,794684,795431,162963,104914,723717,920918,28744,793343,798103,800090,956108,824637,755352,146771\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 993\n", + "Returned IDs: 321389,220988,572167,370532,238363,620065,360595,284590,713328,938683,223699,145879,389713,624300,827647,70169,878531,446292,401216,364518,156097,67133,690891,470126,739115,415924,255069,458373,145450,587007\n", + "Ground Truth: 321389,220988,572167,370532,238363,620065,360595,284590,249865,713328,938229,938683,223699,145879,389713,624300,827647,70169,878531,108742,446292,401216,364518,536744,156097,67133,690891,628930,470126,739115\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 994\n", + "Returned IDs: 283419,741794,251978,231246,430212,784473,253699,98655,549649,898405,202333,956045,25948,651192,174306,150524,841072,672965,939159,563721,73954,341890,722774,696201,298337,579235,868235,506026,148216,245492\n", + "Ground Truth: 240932,855988,283419,741794,251978,231246,430212,784473,253699,98655,549649,898405,327970,202333,956045,25948,354397,651192,781365,838308,174306,150524,841072,672965,939159,563721,73954,57112,341890,722774\n", + "Recall: 0.7667\n", + "\n", + "Total query IDs with recall < 1.0: 303\n", + "IDs: [np.int64(14), np.int64(17), np.int64(21), np.int64(23), np.int64(29), np.int64(35), np.int64(38), np.int64(42), np.int64(44), np.int64(51), np.int64(53), np.int64(54), np.int64(55), np.int64(59), np.int64(62), np.int64(66), np.int64(71), np.int64(74), np.int64(75), np.int64(77), np.int64(79), np.int64(81), np.int64(84), np.int64(85), np.int64(91), np.int64(93), np.int64(95), np.int64(97), np.int64(99), np.int64(100), np.int64(105), np.int64(106), np.int64(107), np.int64(108), np.int64(109), np.int64(114), np.int64(116), np.int64(117), np.int64(118), np.int64(120), np.int64(121), np.int64(129), np.int64(130), np.int64(132), np.int64(135), np.int64(139), np.int64(140), np.int64(142), np.int64(144), np.int64(146), np.int64(148), np.int64(149), np.int64(154), np.int64(158), np.int64(170), np.int64(173), np.int64(176), np.int64(178), np.int64(181), np.int64(184), np.int64(197), np.int64(199), np.int64(200), np.int64(203), np.int64(211), np.int64(214), np.int64(215), np.int64(216), np.int64(221), np.int64(225), np.int64(226), np.int64(245), np.int64(248), np.int64(249), np.int64(254), np.int64(256), np.int64(257), np.int64(261), np.int64(266), np.int64(268), np.int64(272), np.int64(278), np.int64(282), np.int64(283), np.int64(284), np.int64(285), np.int64(289), np.int64(290), np.int64(295), np.int64(299), np.int64(306), np.int64(308), np.int64(310), np.int64(311), np.int64(315), np.int64(317), np.int64(319), np.int64(320), np.int64(322), np.int64(327), np.int64(331), np.int64(332), np.int64(341), np.int64(343), np.int64(344), np.int64(345), np.int64(350), np.int64(351), np.int64(353), np.int64(355), np.int64(365), np.int64(366), np.int64(369), np.int64(370), np.int64(379), np.int64(389), np.int64(394), np.int64(395), np.int64(399), np.int64(400), np.int64(401), np.int64(403), np.int64(406), np.int64(410), np.int64(411), np.int64(413), np.int64(419), np.int64(421), np.int64(424), np.int64(430), np.int64(434), np.int64(435), np.int64(438), np.int64(442), np.int64(448), np.int64(450), np.int64(453), np.int64(456), np.int64(460), np.int64(463), np.int64(464), np.int64(468), np.int64(472), np.int64(475), np.int64(480), np.int64(481), np.int64(483), np.int64(486), np.int64(496), np.int64(501), np.int64(503), np.int64(508), np.int64(509), np.int64(510), np.int64(511), np.int64(513), np.int64(520), np.int64(521), np.int64(522), np.int64(523), np.int64(527), np.int64(531), np.int64(534), np.int64(543), np.int64(547), np.int64(548), np.int64(551), np.int64(556), np.int64(557), np.int64(562), np.int64(563), np.int64(569), np.int64(574), np.int64(577), np.int64(583), np.int64(585), np.int64(586), np.int64(587), np.int64(596), np.int64(599), np.int64(607), np.int64(615), np.int64(616), np.int64(625), np.int64(626), np.int64(629), np.int64(632), np.int64(634), np.int64(636), np.int64(640), np.int64(647), np.int64(649), np.int64(650), np.int64(652), np.int64(654), np.int64(655), np.int64(656), np.int64(668), np.int64(669), np.int64(674), np.int64(675), np.int64(676), np.int64(677), np.int64(682), np.int64(683), np.int64(694), np.int64(697), np.int64(702), np.int64(703), np.int64(704), np.int64(708), np.int64(709), np.int64(710), np.int64(713), np.int64(718), np.int64(719), np.int64(724), np.int64(726), np.int64(727), np.int64(728), np.int64(731), np.int64(733), np.int64(735), np.int64(737), np.int64(740), np.int64(741), np.int64(744), np.int64(747), np.int64(749), np.int64(754), np.int64(755), np.int64(761), np.int64(762), np.int64(763), np.int64(770), np.int64(771), np.int64(773), np.int64(776), np.int64(783), np.int64(788), np.int64(790), np.int64(791), np.int64(792), np.int64(800), np.int64(802), np.int64(807), np.int64(812), np.int64(814), np.int64(815), np.int64(818), np.int64(832), np.int64(837), np.int64(849), np.int64(855), np.int64(856), np.int64(861), np.int64(862), np.int64(864), np.int64(870), np.int64(871), np.int64(874), np.int64(875), np.int64(877), np.int64(883), np.int64(885), np.int64(894), np.int64(898), np.int64(899), np.int64(903), np.int64(907), np.int64(916), np.int64(918), np.int64(919), np.int64(922), np.int64(925), np.int64(930), np.int64(933), np.int64(938), np.int64(940), np.int64(941), np.int64(943), np.int64(946), np.int64(947), np.int64(948), np.int64(953), np.int64(955), np.int64(956), np.int64(958), np.int64(962), np.int64(963), np.int64(966), np.int64(969), np.int64(970), np.int64(976), np.int64(978), np.int64(983), np.int64(985), np.int64(986), np.int64(988), np.int64(990), np.int64(992), np.int64(993), np.int64(994)]\n", + "Average Recall across all 1000 queries: 0.9732\n" + ] + } + ], + "source": [ + "#For querying (int filter)- gte filter\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "base_path2 = \"/home/debian/latest_VDB/VectorDBBench/vectordataset_lte\"\n", + "#inputs\n", + "int_rate_query = \"99p\" # change to 1p, 50p, 80p, 99p\n", + "top_k = 30\n", + "\n", + "# Mode 1: Single query id\n", + "# Mode 2: Find all query ids with recall < 1\n", + "mode = 2\n", + "query_id = 457 # only used in mode 1\n", + "\n", + "# Map int_rate_query to filter range\n", + "range_map = {\n", + " \"1p\": [9999, 1000000],\n", + " \"50p\": [499999, 1000000],\n", + " \"80p\": [799999, 1000000],\n", + " \"99p\": [989999, 1000000],\n", + "}\n", + "filter_range = range_map[int_rate_query]\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path2, dataset_folder)\n", + "test_parquet_path = os.path.join(dataset_path, \"test.parquet\")\n", + "gt_parquet_path = os.path.join(dataset_path, f\"neighbors_int_{int_rate_query}.parquet\")\n", + "\n", + "# Read parquet files\n", + "test_df = pq.ParquetFile(test_parquet_path).read().to_pandas()\n", + "gt_df = pq.ParquetFile(gt_parquet_path).read().to_pandas()\n", + "\n", + "def run_query(query_id):\n", + " query_row = test_df[test_df[\"id\"] == query_id]\n", + " if query_row.empty:\n", + " print(f\"Query ID {query_id} not found in test.parquet\")\n", + " return None\n", + "\n", + " query_vector = query_row[\"emb\"].values[0]\n", + "\n", + " gt_row = gt_df[gt_df[\"id\"] == query_id]\n", + " if gt_row.empty:\n", + " print(f\"Query ID {query_id} not found in ground truth file\")\n", + " return None\n", + "\n", + " ground_truth = gt_row[\"neighbors_id\"].values[0][:top_k]\n", + "\n", + " results = index.query(\n", + " vector=query_vector,\n", + " top_k=top_k,\n", + " filter=[{\"id\": {\"$lte\": filter_range[0]}}],\n", + " include_vectors=False,\n", + " )\n", + " returned_ids = [int(r[\"id\"]) for r in results]\n", + "\n", + " intersection = len(set(returned_ids) & set(ground_truth))\n", + " recall = intersection / len(ground_truth)\n", + "\n", + " return returned_ids, ground_truth, recall\n", + "\n", + "\n", + "if mode == 1:\n", + " result = run_query(query_id)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " print(f\"Query ID: {query_id}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\")\n", + "\n", + "elif mode == 2:\n", + " print(f\"Finding all query IDs with recall < 1.0 ...\\n\")\n", + " low_recall_ids = []\n", + " all_recalls = []\n", + "\n", + " for qid in test_df[\"id\"].values:\n", + " result = run_query(qid)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " all_recalls.append(recall)\n", + " if recall < 1.0:\n", + " low_recall_ids.append(qid)\n", + " print(f\"Query ID: {qid}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\\n\")\n", + "\n", + " print(f\"Total query IDs with recall < 1.0: {len(low_recall_ids)}\")\n", + " print(f\"IDs: {low_recall_ids}\")\n", + " print(f\"Average Recall across all {len(all_recalls)} queries: {sum(all_recalls)/len(all_recalls):.4f}\")" + ] } ], "metadata": { diff --git a/intfilterscript_final.ipynb b/intfilterscript_final.ipynb index cb399efe4..5bf437f2f 100644 --- a/intfilterscript_final.ipynb +++ b/intfilterscript_final.ipynb @@ -20,16 +20,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Name: endee\n", + "Version: 0.1.13\n", + "Summary: Endee is the Next-Generation Vector Database for Scalable, High-Performance AI\n", + "Home-page: https://endee.io\n", + "Author: Endee Labs\n", + "Author-email: dev@endee.io\n", + "License: \n", + "Location: /home/debian/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages\n", + "Requires: httpx, msgpack, numpy, orjson, pydantic, requests\n", + "Required-by: \n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], "source": [ "pip show endee" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -39,9 +57,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " id emb\n", + "0 322406 [0.19600096, -0.5270862, -0.29519123, 0.429556...\n", + "1 337610 [0.32829463, -0.055560444, -0.06252914, -0.101...\n", + "2 714728 [-0.054155815, 0.009554057, 0.24696879, -0.142...\n", + "3 354737 [0.2501778, 0.017853737, -0.43795395, 0.522256...\n", + "4 290979 [0.3444257, -0.62243223, -0.20940691, -0.08620...\n" + ] + } + ], "source": [ "#For checking parquet file contents\n", "import pyarrow.parquet as pq\n", @@ -68,7 +99,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -79,9 +110,31 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "hybridbv_f5\n", + "1000\n", + "\t\n", + "test_1M_vaib_filter_1503\n", + "1000000\n", + "\t\n", + "test_1M_vaib_filter_1503_adup1\n", + "995001\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1\n", + "1000000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup_original\n", + "1000000\n", + "\t\n" + ] + } + ], "source": [ "#For checking indexes\n", "for obj in client.list_indexes().get('indexes'):\n", @@ -92,20 +145,38 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "#give the index name\n", - "index_name = \"test_1M_vaib_reupsertcheck_new1\"\n", + "index_name = \"test_1M_vaib_filter_1503_adup1\"\n", "index = client.get_index(index_name)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_filter_1503_adup1',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 1000000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "index.describe()" ] @@ -323,13 +394,21 @@ ], "metadata": { "kernelspec": { - "display_name": "venv", + "display_name": "venv2", "language": "python", "name": "python3" }, "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", "name": "python", - "version": "3.12.0" + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" } }, "nbformat": 4, diff --git a/intfilterscript_final_gt.ipynb b/intfilterscript_final_gt.ipynb new file mode 100644 index 000000000..be8b09a3f --- /dev/null +++ b/intfilterscript_final_gt.ipynb @@ -0,0 +1,1539 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: pandas in ./venv2/lib/python3.11/site-packages (3.0.1)\n", + "Requirement already satisfied: pyarrow in ./venv2/lib/python3.11/site-packages (23.0.1)\n", + "Requirement already satisfied: numpy>=1.26.0 in ./venv2/lib/python3.11/site-packages (from pandas) (2.4.2)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in ./venv2/lib/python3.11/site-packages (from pandas) (2.9.0.post0)\n", + "Requirement already satisfied: six>=1.5 in ./venv2/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m26.0.1\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "pip install pandas pyarrow" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: endee in ./venv2/lib/python3.11/site-packages (0.1.13)\n", + "Requirement already satisfied: requests>=2.28.0 in ./venv2/lib/python3.11/site-packages (from endee) (2.32.5)\n", + "Requirement already satisfied: httpx>=0.28.1 in ./venv2/lib/python3.11/site-packages (from httpx[http2]>=0.28.1->endee) (0.28.1)\n", + "Requirement already satisfied: numpy>=2.2.4 in ./venv2/lib/python3.11/site-packages (from endee) (2.4.2)\n", + "Requirement already satisfied: msgpack>=1.1.0 in ./venv2/lib/python3.11/site-packages (from endee) (1.1.2)\n", + "Requirement already satisfied: orjson>=3.11.5 in ./venv2/lib/python3.11/site-packages (from endee) (3.11.7)\n", + "Requirement already satisfied: pydantic>=2.0.0 in ./venv2/lib/python3.11/site-packages (from endee) (2.12.5)\n", + "Requirement already satisfied: anyio in ./venv2/lib/python3.11/site-packages (from httpx>=0.28.1->httpx[http2]>=0.28.1->endee) (4.12.1)\n", + "Requirement already satisfied: certifi in ./venv2/lib/python3.11/site-packages (from httpx>=0.28.1->httpx[http2]>=0.28.1->endee) (2026.1.4)\n", + "Requirement already satisfied: httpcore==1.* in ./venv2/lib/python3.11/site-packages (from httpx>=0.28.1->httpx[http2]>=0.28.1->endee) (1.0.9)\n", + "Requirement already satisfied: idna in ./venv2/lib/python3.11/site-packages (from httpx>=0.28.1->httpx[http2]>=0.28.1->endee) (3.11)\n", + "Requirement already satisfied: h11>=0.16 in ./venv2/lib/python3.11/site-packages (from httpcore==1.*->httpx>=0.28.1->httpx[http2]>=0.28.1->endee) (0.16.0)\n", + "Requirement already satisfied: h2<5,>=3 in ./venv2/lib/python3.11/site-packages (from httpx[http2]>=0.28.1->endee) (4.3.0)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in ./venv2/lib/python3.11/site-packages (from pydantic>=2.0.0->endee) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.41.5 in ./venv2/lib/python3.11/site-packages (from pydantic>=2.0.0->endee) (2.41.5)\n", + "Requirement already satisfied: typing-extensions>=4.14.1 in ./venv2/lib/python3.11/site-packages (from pydantic>=2.0.0->endee) (4.15.0)\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in ./venv2/lib/python3.11/site-packages (from pydantic>=2.0.0->endee) (0.4.2)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in ./venv2/lib/python3.11/site-packages (from requests>=2.28.0->endee) (3.4.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in ./venv2/lib/python3.11/site-packages (from requests>=2.28.0->endee) (2.6.3)\n", + "Requirement already satisfied: hyperframe<7,>=6.1 in ./venv2/lib/python3.11/site-packages (from h2<5,>=3->httpx[http2]>=0.28.1->endee) (6.1.0)\n", + "Requirement already satisfied: hpack<5,>=4.1 in ./venv2/lib/python3.11/site-packages (from h2<5,>=3->httpx[http2]>=0.28.1->endee) (4.1.0)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m26.0.1\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "pip install endee" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Name: endee\n", + "Version: 0.1.13\n", + "Summary: Endee is the Next-Generation Vector Database for Scalable, High-Performance AI\n", + "Home-page: https://endee.io\n", + "Author: Endee Labs\n", + "Author-email: dev@endee.io\n", + "License: \n", + "Location: /home/debian/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages\n", + "Requires: httpx, msgpack, numpy, orjson, pydantic, requests\n", + "Required-by: \n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "pip show endee" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "base_path = \"/home/debian/latest_VDB/VectorDBBench/vectordataset_gt\"\n", + "dataset_folder = \"cohere/cohere_medium_1m\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " id emb\n", + "0 322406 [0.19600096, -0.5270862, -0.29519123, 0.429556...\n", + "1 337610 [0.32829463, -0.055560444, -0.06252914, -0.101...\n", + "2 714728 [-0.054155815, 0.009554057, 0.24696879, -0.142...\n", + "3 354737 [0.2501778, 0.017853737, -0.43795395, 0.522256...\n", + "4 290979 [0.3444257, -0.62243223, -0.20940691, -0.08620...\n" + ] + } + ], + "source": [ + "#For checking parquet file contents\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "file_name = \"shuffle_train.parquet\" #input\n", + "\n", + "# Build full path\n", + "file_path = os.path.join(base_path, dataset_folder, file_name)\n", + "\n", + "\n", + "parquet_file = pq.ParquetFile(file_path)\n", + "\n", + "# Read only first batch of rows\n", + "first_batch = next(parquet_file.iter_batches(batch_size=5))\n", + "preview = first_batch.to_pandas()\n", + "\n", + "for col in preview.columns:\n", + " if preview[col].dtype == object and isinstance(preview[col].iloc[0], list):\n", + " preview[col] = preview[col].apply(lambda x: x[:5] if x is not None else x)\n", + "\n", + "print(preview)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from endee import Endee\n", + "client = Endee(token=\"localtest\")\n", + "client.set_base_url(\"http://148.113.54.173:8080/api/v1\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "test_1M_vaib_labelfilter_0503_1\n", + "1000000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup1\n", + "1000000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup4\n", + "1000000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup5\n", + "200000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup6\n", + "1000000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup7\n", + "995214\n", + "\t\n" + ] + } + ], + "source": [ + "#For checking indexes\n", + "for obj in client.list_indexes().get('indexes'):\n", + " print(obj['name'])\n", + " print(obj['total_elements'])\n", + " print('\\t')" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "#give the index name\n", + "index_name = \"test_1M_vaib_labelfilter_0503_1_adup7\"\n", + "index = client.get_index(index_name)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_labelfilter_0503_1_adup7',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 1000000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For querying (int filter)- gt filter\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "#inputs\n", + "int_rate_query = \"99p\" # change to 1p, 50p, 80p, 99p\n", + "top_k = 30\n", + "\n", + "# Mode 1: Single query id\n", + "# Mode 2: Find all query ids with recall < 1\n", + "mode = 2\n", + "query_id = 457 # only used in mode 1\n", + "\n", + "# Map int_rate_query to filter range\n", + "range_map = {\n", + " \"1p\": [10000, 1000000],\n", + " \"50p\": [500000, 1000000],\n", + " \"80p\": [800000, 1000000],\n", + " \"99p\": [990000, 1000000],\n", + "}\n", + "filter_range = range_map[int_rate_query]\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "test_parquet_path = os.path.join(dataset_path, \"test.parquet\")\n", + "gt_parquet_path = os.path.join(dataset_path, f\"neighbors_int_{int_rate_query}.parquet\")\n", + "\n", + "# Read parquet files\n", + "test_df = pq.ParquetFile(test_parquet_path).read().to_pandas()\n", + "gt_df = pq.ParquetFile(gt_parquet_path).read().to_pandas()\n", + "\n", + "def run_query(query_id):\n", + " query_row = test_df[test_df[\"id\"] == query_id]\n", + " if query_row.empty:\n", + " print(f\"Query ID {query_id} not found in test.parquet\")\n", + " return None\n", + "\n", + " query_vector = query_row[\"emb\"].values[0]\n", + "\n", + " gt_row = gt_df[gt_df[\"id\"] == query_id]\n", + " if gt_row.empty:\n", + " print(f\"Query ID {query_id} not found in ground truth file\")\n", + " return None\n", + "\n", + " ground_truth = gt_row[\"neighbors_id\"].values[0][:top_k]\n", + "\n", + " results = index.query(\n", + " vector=query_vector,\n", + " top_k=top_k,\n", + " filter=[{\"id\": {\"$gt\": filter_range[0]}}],\n", + " include_vectors=False,\n", + " )\n", + " returned_ids = [int(r[\"id\"]) for r in results]\n", + "\n", + " intersection = len(set(returned_ids) & set(ground_truth))\n", + " recall = intersection / len(ground_truth)\n", + "\n", + " return returned_ids, ground_truth, recall\n", + "\n", + "\n", + "if mode == 1:\n", + " result = run_query(query_id)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " print(f\"Query ID: {query_id}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\")\n", + "\n", + "elif mode == 2:\n", + " print(f\"Finding all query IDs with recall < 1.0 ...\\n\")\n", + " low_recall_ids = []\n", + " all_recalls = []\n", + "\n", + " for qid in test_df[\"id\"].values:\n", + " result = run_query(qid)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " all_recalls.append(recall)\n", + " if recall < 1.0:\n", + " low_recall_ids.append(qid)\n", + " print(f\"Query ID: {qid}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\\n\")\n", + "\n", + " print(f\"Total query IDs with recall < 1.0: {len(low_recall_ids)}\")\n", + " print(f\"IDs: {low_recall_ids}\")\n", + " print(f\"Average Recall across all {len(all_recalls)} queries: {sum(all_recalls)/len(all_recalls):.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_labelfilter_0503_1_adup7',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 1000000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Deleted vectors with id <= 800000\n", + "800001 vectors deleted\n" + ] + } + ], + "source": [ + "#For deleting (int filter) - lt\n", + "int_rate_delete = \"80p\" # change to 1p, 50p, 80p, 99p\n", + "\n", + "# Map int_rate to lower bound (gt value)\n", + "range_map = {\n", + " \"1p\": 10000,\n", + " \"50p\": 500000,\n", + " \"80p\": 800000,\n", + " \"99p\": 990000,\n", + "}\n", + "gt_value = range_map[int_rate_delete]\n", + "\n", + "result = index.delete_with_filter([{\"id\": {\"$lte\": gt_value}}])\n", + "print(f\"Deleted vectors with id <= {gt_value}\")\n", + "print(result)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_labelfilter_0503_1_adup7',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 1000000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Upserted 798 vectors, total so far: 798\n", + "Upserted 812 vectors, total so far: 1610\n", + "Upserted 785 vectors, total so far: 2395\n", + "Upserted 813 vectors, total so far: 3208\n", + "Upserted 789 vectors, total so far: 3997\n", + "Upserted 827 vectors, total so far: 4824\n", + "Upserted 796 vectors, total so far: 5620\n", + "Upserted 804 vectors, total so far: 6424\n", + "Upserted 793 vectors, total so far: 7217\n", + "Upserted 787 vectors, total so far: 8004\n", + "Upserted 794 vectors, total so far: 8798\n", + "Upserted 797 vectors, total so far: 9595\n", + "Upserted 797 vectors, total so far: 10392\n", + "Upserted 817 vectors, total so far: 11209\n", + "Upserted 787 vectors, total so far: 11996\n", + "Upserted 802 vectors, total so far: 12798\n", + "Upserted 800 vectors, total so far: 13598\n", + "Upserted 786 vectors, total so far: 14384\n", + "Upserted 811 vectors, total so far: 15195\n", + "Upserted 772 vectors, total so far: 15967\n", + "Upserted 816 vectors, total so far: 16783\n", + "Upserted 796 vectors, total so far: 17579\n", + "Upserted 811 vectors, total so far: 18390\n", + "Upserted 767 vectors, total so far: 19157\n", + "Upserted 816 vectors, total so far: 19973\n", + "Upserted 800 vectors, total so far: 20773\n", + "Upserted 792 vectors, total so far: 21565\n", + "Upserted 800 vectors, total so far: 22365\n", + "Upserted 806 vectors, total so far: 23171\n", + "Upserted 792 vectors, total so far: 23963\n", + "Upserted 799 vectors, total so far: 24762\n", + "Upserted 798 vectors, total so far: 25560\n", + "Upserted 817 vectors, total so far: 26377\n", + "Upserted 813 vectors, total so far: 27190\n", + "Upserted 809 vectors, total so far: 27999\n", + "Upserted 810 vectors, total so far: 28809\n", + "Upserted 805 vectors, total so far: 29614\n", + "Upserted 803 vectors, total so far: 30417\n", + "Upserted 790 vectors, total so far: 31207\n", + "Upserted 818 vectors, total so far: 32025\n", + "Upserted 805 vectors, total so far: 32830\n", + "Upserted 790 vectors, total so far: 33620\n", + "Upserted 795 vectors, total so far: 34415\n", + "Upserted 801 vectors, total so far: 35216\n", + "Upserted 789 vectors, total so far: 36005\n", + "Upserted 810 vectors, total so far: 36815\n", + "Upserted 796 vectors, total so far: 37611\n", + "Upserted 791 vectors, total so far: 38402\n", + "Upserted 785 vectors, total so far: 39187\n", + "Upserted 820 vectors, total so far: 40007\n", + "Upserted 812 vectors, total so far: 40819\n", + "Upserted 789 vectors, total so far: 41608\n", + "Upserted 796 vectors, total so far: 42404\n", + "Upserted 807 vectors, total so far: 43211\n", + "Upserted 805 vectors, total so far: 44016\n", + "Upserted 792 vectors, total so far: 44808\n", + "Upserted 788 vectors, total so far: 45596\n", + "Upserted 785 vectors, total so far: 46381\n", + "Upserted 818 vectors, total so far: 47199\n", + "Upserted 807 vectors, total so far: 48006\n", + "Upserted 818 vectors, total so far: 48824\n", + "Upserted 790 vectors, total so far: 49614\n", + "Upserted 797 vectors, total so far: 50411\n", + "Upserted 816 vectors, total so far: 51227\n", + "Upserted 795 vectors, total so far: 52022\n", + "Upserted 807 vectors, total so far: 52829\n", + "Upserted 778 vectors, total so far: 53607\n", + "Upserted 785 vectors, total so far: 54392\n", + "Upserted 807 vectors, total so far: 55199\n", + "Upserted 792 vectors, total so far: 55991\n", + "Upserted 822 vectors, total so far: 56813\n", + "Upserted 785 vectors, total so far: 57598\n", + "Upserted 789 vectors, total so far: 58387\n", + "Upserted 807 vectors, total so far: 59194\n", + "Upserted 811 vectors, total so far: 60005\n", + "Upserted 801 vectors, total so far: 60806\n", + "Upserted 780 vectors, total so far: 61586\n", + "Upserted 806 vectors, total so far: 62392\n", + "Upserted 791 vectors, total so far: 63183\n", + "Upserted 787 vectors, total so far: 63970\n", + "Upserted 794 vectors, total so far: 64764\n", + "Upserted 803 vectors, total so far: 65567\n", + "Upserted 798 vectors, total so far: 66365\n", + "Upserted 813 vectors, total so far: 67178\n", + "Upserted 794 vectors, total so far: 67972\n", + "Upserted 800 vectors, total so far: 68772\n", + "Upserted 802 vectors, total so far: 69574\n", + "Upserted 799 vectors, total so far: 70373\n", + "Upserted 791 vectors, total so far: 71164\n", + "Upserted 794 vectors, total so far: 71958\n", + "Upserted 819 vectors, total so far: 72777\n", + "Upserted 795 vectors, total so far: 73572\n", + "Upserted 829 vectors, total so far: 74401\n", + "Upserted 807 vectors, total so far: 75208\n", + "Upserted 810 vectors, total so far: 76018\n", + "Upserted 795 vectors, total so far: 76813\n", + "Upserted 789 vectors, total so far: 77602\n", + "Upserted 799 vectors, total so far: 78401\n", + "Upserted 793 vectors, total so far: 79194\n", + "Upserted 813 vectors, total so far: 80007\n", + "Upserted 780 vectors, total so far: 80787\n", + "Upserted 781 vectors, total so far: 81568\n", + "Upserted 775 vectors, total so far: 82343\n", + "Upserted 799 vectors, total so far: 83142\n", + "Upserted 813 vectors, total so far: 83955\n", + "Upserted 800 vectors, total so far: 84755\n", + "Upserted 803 vectors, total so far: 85558\n", + "Upserted 795 vectors, total so far: 86353\n", + "Upserted 778 vectors, total so far: 87131\n", + "Upserted 817 vectors, total so far: 87948\n", + "Upserted 800 vectors, total so far: 88748\n", + "Upserted 798 vectors, total so far: 89546\n", + "Upserted 797 vectors, total so far: 90343\n", + "Upserted 811 vectors, total so far: 91154\n", + "Upserted 798 vectors, total so far: 91952\n", + "Upserted 788 vectors, total so far: 92740\n", + "Upserted 805 vectors, total so far: 93545\n", + "Upserted 811 vectors, total so far: 94356\n", + "Upserted 810 vectors, total so far: 95166\n", + "Upserted 791 vectors, total so far: 95957\n", + "Upserted 809 vectors, total so far: 96766\n", + "Upserted 797 vectors, total so far: 97563\n", + "Upserted 812 vectors, total so far: 98375\n", + "Upserted 795 vectors, total so far: 99170\n", + "Upserted 774 vectors, total so far: 99944\n", + "Upserted 800 vectors, total so far: 100744\n", + "Upserted 800 vectors, total so far: 101544\n", + "Upserted 799 vectors, total so far: 102343\n", + "Upserted 771 vectors, total so far: 103114\n", + "Upserted 796 vectors, total so far: 103910\n", + "Upserted 810 vectors, total so far: 104720\n", + "Upserted 817 vectors, total so far: 105537\n", + "Upserted 819 vectors, total so far: 106356\n", + "Upserted 776 vectors, total so far: 107132\n", + "Upserted 809 vectors, total so far: 107941\n", + "Upserted 801 vectors, total so far: 108742\n", + "Upserted 806 vectors, total so far: 109548\n", + "Upserted 821 vectors, total so far: 110369\n", + "Upserted 810 vectors, total so far: 111179\n", + "Upserted 805 vectors, total so far: 111984\n", + "Upserted 789 vectors, total so far: 112773\n", + "Upserted 809 vectors, total so far: 113582\n", + "Upserted 806 vectors, total so far: 114388\n", + "Upserted 810 vectors, total so far: 115198\n", + "Upserted 818 vectors, total so far: 116016\n", + "Upserted 808 vectors, total so far: 116824\n", + "Upserted 796 vectors, total so far: 117620\n", + "Upserted 809 vectors, total so far: 118429\n", + "Upserted 808 vectors, total so far: 119237\n", + "Upserted 800 vectors, total so far: 120037\n", + "Upserted 802 vectors, total so far: 120839\n", + "Upserted 787 vectors, total so far: 121626\n", + "Upserted 794 vectors, total so far: 122420\n", + "Upserted 805 vectors, total so far: 123225\n", + "Upserted 815 vectors, total so far: 124040\n", + "Upserted 801 vectors, total so far: 124841\n", + "Upserted 789 vectors, total so far: 125630\n", + "Upserted 795 vectors, total so far: 126425\n", + "Upserted 788 vectors, total so far: 127213\n", + "Upserted 796 vectors, total so far: 128009\n", + "Upserted 819 vectors, total so far: 128828\n", + "Upserted 799 vectors, total so far: 129627\n", + "Upserted 797 vectors, total so far: 130424\n", + "Upserted 817 vectors, total so far: 131241\n", + "Upserted 796 vectors, total so far: 132037\n", + "Upserted 822 vectors, total so far: 132859\n", + "Upserted 794 vectors, total so far: 133653\n", + "Upserted 806 vectors, total so far: 134459\n", + "Upserted 785 vectors, total so far: 135244\n", + "Upserted 788 vectors, total so far: 136032\n", + "Upserted 801 vectors, total so far: 136833\n", + "Upserted 808 vectors, total so far: 137641\n", + "Upserted 814 vectors, total so far: 138455\n", + "Upserted 797 vectors, total so far: 139252\n", + "Upserted 778 vectors, total so far: 140030\n", + "Upserted 799 vectors, total so far: 140829\n", + "Upserted 794 vectors, total so far: 141623\n", + "Upserted 782 vectors, total so far: 142405\n", + "Upserted 816 vectors, total so far: 143221\n", + "Upserted 797 vectors, total so far: 144018\n", + "Upserted 785 vectors, total so far: 144803\n", + "Upserted 781 vectors, total so far: 145584\n", + "Upserted 810 vectors, total so far: 146394\n", + "Upserted 810 vectors, total so far: 147204\n", + "Upserted 796 vectors, total so far: 148000\n", + "Upserted 813 vectors, total so far: 148813\n", + "Upserted 822 vectors, total so far: 149635\n", + "Upserted 795 vectors, total so far: 150430\n", + "Upserted 782 vectors, total so far: 151212\n", + "Upserted 795 vectors, total so far: 152007\n", + "Upserted 802 vectors, total so far: 152809\n", + "Upserted 773 vectors, total so far: 153582\n", + "Upserted 785 vectors, total so far: 154367\n", + "Upserted 796 vectors, total so far: 155163\n", + "Upserted 797 vectors, total so far: 155960\n", + "Upserted 794 vectors, total so far: 156754\n", + "Upserted 793 vectors, total so far: 157547\n", + "Upserted 794 vectors, total so far: 158341\n", + "Upserted 814 vectors, total so far: 159155\n", + "Upserted 782 vectors, total so far: 159937\n", + "Upserted 818 vectors, total so far: 160755\n", + "Upserted 787 vectors, total so far: 161542\n", + "Upserted 789 vectors, total so far: 162331\n", + "Upserted 794 vectors, total so far: 163125\n", + "Upserted 822 vectors, total so far: 163947\n", + "Upserted 813 vectors, total so far: 164760\n", + "Upserted 784 vectors, total so far: 165544\n", + "Upserted 810 vectors, total so far: 166354\n", + "Upserted 792 vectors, total so far: 167146\n", + "Upserted 808 vectors, total so far: 167954\n", + "Upserted 809 vectors, total so far: 168763\n", + "Upserted 782 vectors, total so far: 169545\n", + "Upserted 831 vectors, total so far: 170376\n", + "Upserted 808 vectors, total so far: 171184\n", + "Upserted 808 vectors, total so far: 171992\n", + "Upserted 810 vectors, total so far: 172802\n", + "Upserted 824 vectors, total so far: 173626\n", + "Upserted 809 vectors, total so far: 174435\n", + "Upserted 795 vectors, total so far: 175230\n", + "Upserted 814 vectors, total so far: 176044\n", + "Upserted 796 vectors, total so far: 176840\n", + "Upserted 802 vectors, total so far: 177642\n", + "Upserted 801 vectors, total so far: 178443\n", + "Upserted 762 vectors, total so far: 179205\n", + "Upserted 802 vectors, total so far: 180007\n", + "Upserted 806 vectors, total so far: 180813\n", + "Upserted 801 vectors, total so far: 181614\n", + "Upserted 800 vectors, total so far: 182414\n", + "Upserted 812 vectors, total so far: 183226\n", + "Upserted 788 vectors, total so far: 184014\n", + "Upserted 799 vectors, total so far: 184813\n", + "Upserted 802 vectors, total so far: 185615\n", + "Upserted 792 vectors, total so far: 186407\n", + "Upserted 789 vectors, total so far: 187196\n", + "Upserted 807 vectors, total so far: 188003\n", + "Upserted 801 vectors, total so far: 188804\n", + "Upserted 792 vectors, total so far: 189596\n", + "Upserted 811 vectors, total so far: 190407\n", + "Upserted 801 vectors, total so far: 191208\n", + "Upserted 799 vectors, total so far: 192007\n", + "Upserted 782 vectors, total so far: 192789\n", + "Upserted 824 vectors, total so far: 193613\n", + "Upserted 797 vectors, total so far: 194410\n", + "Upserted 813 vectors, total so far: 195223\n", + "Upserted 789 vectors, total so far: 196012\n", + "Upserted 803 vectors, total so far: 196815\n", + "Upserted 806 vectors, total so far: 197621\n", + "Upserted 815 vectors, total so far: 198436\n", + "Upserted 781 vectors, total so far: 199217\n", + "Upserted 817 vectors, total so far: 200034\n", + "Upserted 812 vectors, total so far: 200846\n", + "Upserted 796 vectors, total so far: 201642\n", + "Upserted 795 vectors, total so far: 202437\n", + "Upserted 795 vectors, total so far: 203232\n", + "Upserted 771 vectors, total so far: 204003\n", + "Upserted 793 vectors, total so far: 204796\n", + "Upserted 811 vectors, total so far: 205607\n", + "Upserted 805 vectors, total so far: 206412\n", + "Upserted 776 vectors, total so far: 207188\n", + "Upserted 809 vectors, total so far: 207997\n", + "Upserted 790 vectors, total so far: 208787\n", + "Upserted 793 vectors, total so far: 209580\n", + "Upserted 818 vectors, total so far: 210398\n", + "Upserted 791 vectors, total so far: 211189\n", + "Upserted 790 vectors, total so far: 211979\n", + "Upserted 782 vectors, total so far: 212761\n", + "Upserted 821 vectors, total so far: 213582\n", + "Upserted 805 vectors, total so far: 214387\n", + "Upserted 815 vectors, total so far: 215202\n", + "Upserted 798 vectors, total so far: 216000\n", + "Upserted 775 vectors, total so far: 216775\n", + "Upserted 808 vectors, total so far: 217583\n", + "Upserted 810 vectors, total so far: 218393\n", + "Upserted 810 vectors, total so far: 219203\n", + "Upserted 805 vectors, total so far: 220008\n", + "Upserted 805 vectors, total so far: 220813\n", + "Upserted 781 vectors, total so far: 221594\n", + "Upserted 804 vectors, total so far: 222398\n", + "Upserted 802 vectors, total so far: 223200\n", + "Upserted 790 vectors, total so far: 223990\n", + "Upserted 824 vectors, total so far: 224814\n", + "Upserted 794 vectors, total so far: 225608\n", + "Upserted 791 vectors, total so far: 226399\n", + "Upserted 804 vectors, total so far: 227203\n", + "Upserted 814 vectors, total so far: 228017\n", + "Upserted 803 vectors, total so far: 228820\n", + "Upserted 798 vectors, total so far: 229618\n", + "Upserted 807 vectors, total so far: 230425\n", + "Upserted 822 vectors, total so far: 231247\n", + "Upserted 821 vectors, total so far: 232068\n", + "Upserted 806 vectors, total so far: 232874\n", + "Upserted 812 vectors, total so far: 233686\n", + "Upserted 803 vectors, total so far: 234489\n", + "Upserted 809 vectors, total so far: 235298\n", + "Upserted 800 vectors, total so far: 236098\n", + "Upserted 805 vectors, total so far: 236903\n", + "Upserted 805 vectors, total so far: 237708\n", + "Upserted 782 vectors, total so far: 238490\n", + "Upserted 809 vectors, total so far: 239299\n", + "Upserted 812 vectors, total so far: 240111\n", + "Upserted 809 vectors, total so far: 240920\n", + "Upserted 797 vectors, total so far: 241717\n", + "Upserted 813 vectors, total so far: 242530\n", + "Upserted 822 vectors, total so far: 243352\n", + "Upserted 788 vectors, total so far: 244140\n", + "Upserted 808 vectors, total so far: 244948\n", + "Upserted 799 vectors, total so far: 245747\n", + "Upserted 814 vectors, total so far: 246561\n", + "Upserted 813 vectors, total so far: 247374\n", + "Upserted 798 vectors, total so far: 248172\n", + "Upserted 793 vectors, total so far: 248965\n", + "Upserted 794 vectors, total so far: 249759\n", + "Upserted 807 vectors, total so far: 250566\n", + "Upserted 803 vectors, total so far: 251369\n", + "Upserted 812 vectors, total so far: 252181\n", + "Upserted 814 vectors, total so far: 252995\n", + "Upserted 810 vectors, total so far: 253805\n", + "Upserted 802 vectors, total so far: 254607\n", + "Upserted 774 vectors, total so far: 255381\n", + "Upserted 795 vectors, total so far: 256176\n", + "Upserted 800 vectors, total so far: 256976\n", + "Upserted 782 vectors, total so far: 257758\n", + "Upserted 800 vectors, total so far: 258558\n", + "Upserted 804 vectors, total so far: 259362\n", + "Upserted 810 vectors, total so far: 260172\n", + "Upserted 824 vectors, total so far: 260996\n", + "Upserted 793 vectors, total so far: 261789\n", + "Upserted 782 vectors, total so far: 262571\n", + "Upserted 784 vectors, total so far: 263355\n", + "Upserted 764 vectors, total so far: 264119\n", + "Upserted 779 vectors, total so far: 264898\n", + "Upserted 816 vectors, total so far: 265714\n", + "Upserted 807 vectors, total so far: 266521\n", + "Upserted 792 vectors, total so far: 267313\n", + "Upserted 790 vectors, total so far: 268103\n", + "Upserted 786 vectors, total so far: 268889\n", + "Upserted 809 vectors, total so far: 269698\n", + "Upserted 803 vectors, total so far: 270501\n", + "Upserted 826 vectors, total so far: 271327\n", + "Upserted 775 vectors, total so far: 272102\n", + "Upserted 793 vectors, total so far: 272895\n", + "Upserted 795 vectors, total so far: 273690\n", + "Upserted 798 vectors, total so far: 274488\n", + "Upserted 818 vectors, total so far: 275306\n", + "Upserted 809 vectors, total so far: 276115\n", + "Upserted 789 vectors, total so far: 276904\n", + "Upserted 795 vectors, total so far: 277699\n", + "Upserted 794 vectors, total so far: 278493\n", + "Upserted 795 vectors, total so far: 279288\n", + "Upserted 795 vectors, total so far: 280083\n", + "Upserted 804 vectors, total so far: 280887\n", + "Upserted 814 vectors, total so far: 281701\n", + "Upserted 810 vectors, total so far: 282511\n", + "Upserted 788 vectors, total so far: 283299\n", + "Upserted 810 vectors, total so far: 284109\n", + "Upserted 794 vectors, total so far: 284903\n", + "Upserted 802 vectors, total so far: 285705\n", + "Upserted 790 vectors, total so far: 286495\n", + "Upserted 794 vectors, total so far: 287289\n", + "Upserted 811 vectors, total so far: 288100\n", + "Upserted 794 vectors, total so far: 288894\n", + "Upserted 819 vectors, total so far: 289713\n", + "Upserted 785 vectors, total so far: 290498\n", + "Upserted 810 vectors, total so far: 291308\n", + "Upserted 786 vectors, total so far: 292094\n", + "Upserted 793 vectors, total so far: 292887\n", + "Upserted 796 vectors, total so far: 293683\n", + "Upserted 809 vectors, total so far: 294492\n", + "Upserted 801 vectors, total so far: 295293\n", + "Upserted 805 vectors, total so far: 296098\n", + "Upserted 797 vectors, total so far: 296895\n", + "Upserted 806 vectors, total so far: 297701\n", + "Upserted 803 vectors, total so far: 298504\n", + "Upserted 827 vectors, total so far: 299331\n", + "Upserted 820 vectors, total so far: 300151\n", + "Upserted 793 vectors, total so far: 300944\n", + "Upserted 795 vectors, total so far: 301739\n", + "Upserted 774 vectors, total so far: 302513\n", + "Upserted 790 vectors, total so far: 303303\n", + "Upserted 801 vectors, total so far: 304104\n", + "Upserted 793 vectors, total so far: 304897\n", + "Upserted 801 vectors, total so far: 305698\n", + "Upserted 793 vectors, total so far: 306491\n", + "Upserted 815 vectors, total so far: 307306\n", + "Upserted 821 vectors, total so far: 308127\n", + "Upserted 812 vectors, total so far: 308939\n", + "Upserted 786 vectors, total so far: 309725\n", + "Upserted 784 vectors, total so far: 310509\n", + "Upserted 803 vectors, total so far: 311312\n", + "Upserted 777 vectors, total so far: 312089\n", + "Upserted 824 vectors, total so far: 312913\n", + "Upserted 799 vectors, total so far: 313712\n", + "Upserted 790 vectors, total so far: 314502\n", + "Upserted 809 vectors, total so far: 315311\n", + "Upserted 797 vectors, total so far: 316108\n", + "Upserted 784 vectors, total so far: 316892\n", + "Upserted 803 vectors, total so far: 317695\n", + "Upserted 774 vectors, total so far: 318469\n", + "Upserted 798 vectors, total so far: 319267\n", + "Upserted 791 vectors, total so far: 320058\n", + "Upserted 791 vectors, total so far: 320849\n", + "Upserted 809 vectors, total so far: 321658\n", + "Upserted 800 vectors, total so far: 322458\n", + "Upserted 799 vectors, total so far: 323257\n", + "Upserted 825 vectors, total so far: 324082\n", + "Upserted 819 vectors, total so far: 324901\n", + "Upserted 778 vectors, total so far: 325679\n", + "Upserted 782 vectors, total so far: 326461\n", + "Upserted 796 vectors, total so far: 327257\n", + "Upserted 792 vectors, total so far: 328049\n", + "Upserted 817 vectors, total so far: 328866\n", + "Upserted 801 vectors, total so far: 329667\n", + "Upserted 779 vectors, total so far: 330446\n", + "Upserted 804 vectors, total so far: 331250\n", + "Upserted 786 vectors, total so far: 332036\n", + "Upserted 783 vectors, total so far: 332819\n", + "Upserted 819 vectors, total so far: 333638\n", + "Upserted 790 vectors, total so far: 334428\n", + "Upserted 793 vectors, total so far: 335221\n", + "Upserted 811 vectors, total so far: 336032\n", + "Upserted 795 vectors, total so far: 336827\n", + "Upserted 808 vectors, total so far: 337635\n", + "Upserted 802 vectors, total so far: 338437\n", + "Upserted 816 vectors, total so far: 339253\n", + "Upserted 815 vectors, total so far: 340068\n", + "Upserted 798 vectors, total so far: 340866\n", + "Upserted 788 vectors, total so far: 341654\n", + "Upserted 782 vectors, total so far: 342436\n", + "Upserted 792 vectors, total so far: 343228\n", + "Upserted 779 vectors, total so far: 344007\n", + "Upserted 803 vectors, total so far: 344810\n", + "Upserted 818 vectors, total so far: 345628\n", + "Upserted 809 vectors, total so far: 346437\n", + "Upserted 803 vectors, total so far: 347240\n", + "Upserted 798 vectors, total so far: 348038\n", + "Upserted 795 vectors, total so far: 348833\n", + "Upserted 809 vectors, total so far: 349642\n", + "Upserted 756 vectors, total so far: 350398\n", + "Upserted 797 vectors, total so far: 351195\n", + "Upserted 800 vectors, total so far: 351995\n", + "Upserted 797 vectors, total so far: 352792\n", + "Upserted 793 vectors, total so far: 353585\n", + "Upserted 779 vectors, total so far: 354364\n", + "Upserted 796 vectors, total so far: 355160\n", + "Upserted 788 vectors, total so far: 355948\n", + "Upserted 811 vectors, total so far: 356759\n", + "Upserted 798 vectors, total so far: 357557\n", + "Upserted 812 vectors, total so far: 358369\n", + "Upserted 789 vectors, total so far: 359158\n", + "Upserted 808 vectors, total so far: 359966\n", + "Upserted 812 vectors, total so far: 360778\n", + "Upserted 803 vectors, total so far: 361581\n", + "Upserted 801 vectors, total so far: 362382\n", + "Upserted 796 vectors, total so far: 363178\n", + "Upserted 803 vectors, total so far: 363981\n", + "Upserted 807 vectors, total so far: 364788\n", + "Upserted 804 vectors, total so far: 365592\n", + "Upserted 810 vectors, total so far: 366402\n", + "Upserted 808 vectors, total so far: 367210\n", + "Upserted 808 vectors, total so far: 368018\n", + "Upserted 786 vectors, total so far: 368804\n", + "Upserted 799 vectors, total so far: 369603\n", + "Upserted 811 vectors, total so far: 370414\n", + "Upserted 803 vectors, total so far: 371217\n", + "Upserted 789 vectors, total so far: 372006\n", + "Upserted 788 vectors, total so far: 372794\n", + "Upserted 775 vectors, total so far: 373569\n", + "Upserted 789 vectors, total so far: 374358\n", + "Upserted 803 vectors, total so far: 375161\n", + "Upserted 816 vectors, total so far: 375977\n", + "Upserted 826 vectors, total so far: 376803\n", + "Upserted 807 vectors, total so far: 377610\n", + "Upserted 798 vectors, total so far: 378408\n", + "Upserted 805 vectors, total so far: 379213\n", + "Upserted 788 vectors, total so far: 380001\n", + "Upserted 804 vectors, total so far: 380805\n", + "Upserted 809 vectors, total so far: 381614\n", + "Upserted 833 vectors, total so far: 382447\n", + "Upserted 809 vectors, total so far: 383256\n", + "Upserted 814 vectors, total so far: 384070\n", + "Upserted 819 vectors, total so far: 384889\n", + "Upserted 792 vectors, total so far: 385681\n", + "Upserted 812 vectors, total so far: 386493\n", + "Upserted 817 vectors, total so far: 387310\n", + "Upserted 801 vectors, total so far: 388111\n", + "Upserted 799 vectors, total so far: 388910\n", + "Upserted 807 vectors, total so far: 389717\n", + "Upserted 791 vectors, total so far: 390508\n", + "Upserted 820 vectors, total so far: 391328\n", + "Upserted 810 vectors, total so far: 392138\n", + "Upserted 798 vectors, total so far: 392936\n", + "Upserted 816 vectors, total so far: 393752\n", + "Upserted 791 vectors, total so far: 394543\n", + "Upserted 815 vectors, total so far: 395358\n", + "Upserted 789 vectors, total so far: 396147\n", + "Upserted 793 vectors, total so far: 396940\n", + "Upserted 794 vectors, total so far: 397734\n", + "Upserted 802 vectors, total so far: 398536\n", + "Upserted 801 vectors, total so far: 399337\n", + "Upserted 825 vectors, total so far: 400162\n", + "Upserted 813 vectors, total so far: 400975\n", + "Upserted 797 vectors, total so far: 401772\n", + "Upserted 810 vectors, total so far: 402582\n", + "Upserted 838 vectors, total so far: 403420\n", + "Upserted 812 vectors, total so far: 404232\n", + "Upserted 795 vectors, total so far: 405027\n", + "Upserted 806 vectors, total so far: 405833\n", + "Upserted 777 vectors, total so far: 406610\n", + "Upserted 766 vectors, total so far: 407376\n", + "Upserted 811 vectors, total so far: 408187\n", + "Upserted 775 vectors, total so far: 408962\n", + "Upserted 794 vectors, total so far: 409756\n", + "Upserted 811 vectors, total so far: 410567\n", + "Upserted 788 vectors, total so far: 411355\n", + "Upserted 795 vectors, total so far: 412150\n", + "Upserted 789 vectors, total so far: 412939\n", + "Upserted 807 vectors, total so far: 413746\n", + "Upserted 803 vectors, total so far: 414549\n", + "Upserted 811 vectors, total so far: 415360\n", + "Upserted 772 vectors, total so far: 416132\n", + "Upserted 805 vectors, total so far: 416937\n", + "Upserted 788 vectors, total so far: 417725\n", + "Upserted 825 vectors, total so far: 418550\n", + "Upserted 815 vectors, total so far: 419365\n", + "Upserted 791 vectors, total so far: 420156\n", + "Upserted 799 vectors, total so far: 420955\n", + "Upserted 819 vectors, total so far: 421774\n", + "Upserted 799 vectors, total so far: 422573\n", + "Upserted 801 vectors, total so far: 423374\n", + "Upserted 803 vectors, total so far: 424177\n", + "Upserted 782 vectors, total so far: 424959\n", + "Upserted 778 vectors, total so far: 425737\n", + "Upserted 817 vectors, total so far: 426554\n", + "Upserted 788 vectors, total so far: 427342\n", + "Upserted 785 vectors, total so far: 428127\n", + "Upserted 795 vectors, total so far: 428922\n", + "Upserted 799 vectors, total so far: 429721\n", + "Upserted 805 vectors, total so far: 430526\n", + "Upserted 809 vectors, total so far: 431335\n", + "Upserted 782 vectors, total so far: 432117\n", + "Upserted 803 vectors, total so far: 432920\n", + "Upserted 820 vectors, total so far: 433740\n", + "Upserted 826 vectors, total so far: 434566\n", + "Upserted 801 vectors, total so far: 435367\n", + "Upserted 798 vectors, total so far: 436165\n", + "Upserted 780 vectors, total so far: 436945\n", + "Upserted 805 vectors, total so far: 437750\n", + "Upserted 804 vectors, total so far: 438554\n", + "Upserted 799 vectors, total so far: 439353\n", + "Upserted 806 vectors, total so far: 440159\n", + "Upserted 808 vectors, total so far: 440967\n", + "Upserted 803 vectors, total so far: 441770\n", + "Upserted 798 vectors, total so far: 442568\n", + "Upserted 816 vectors, total so far: 443384\n", + "Upserted 816 vectors, total so far: 444200\n", + "Upserted 791 vectors, total so far: 444991\n", + "Upserted 818 vectors, total so far: 445809\n", + "Upserted 803 vectors, total so far: 446612\n", + "Upserted 808 vectors, total so far: 447420\n", + "Upserted 838 vectors, total so far: 448258\n", + "Upserted 797 vectors, total so far: 449055\n", + "Upserted 797 vectors, total so far: 449852\n", + "Upserted 785 vectors, total so far: 450637\n", + "Upserted 786 vectors, total so far: 451423\n", + "Upserted 796 vectors, total so far: 452219\n", + "Upserted 815 vectors, total so far: 453034\n", + "Upserted 788 vectors, total so far: 453822\n", + "Upserted 796 vectors, total so far: 454618\n", + "Upserted 798 vectors, total so far: 455416\n", + "Upserted 787 vectors, total so far: 456203\n", + "Upserted 804 vectors, total so far: 457007\n", + "Upserted 791 vectors, total so far: 457798\n", + "Upserted 798 vectors, total so far: 458596\n", + "Upserted 784 vectors, total so far: 459380\n", + "Upserted 774 vectors, total so far: 460154\n", + "Upserted 817 vectors, total so far: 460971\n", + "Upserted 780 vectors, total so far: 461751\n", + "Upserted 791 vectors, total so far: 462542\n", + "Upserted 810 vectors, total so far: 463352\n", + "Upserted 822 vectors, total so far: 464174\n", + "Upserted 783 vectors, total so far: 464957\n", + "Upserted 806 vectors, total so far: 465763\n", + "Upserted 798 vectors, total so far: 466561\n", + "Upserted 794 vectors, total so far: 467355\n", + "Upserted 812 vectors, total so far: 468167\n", + "Upserted 776 vectors, total so far: 468943\n", + "Upserted 784 vectors, total so far: 469727\n", + "Upserted 808 vectors, total so far: 470535\n", + "Upserted 794 vectors, total so far: 471329\n", + "Upserted 817 vectors, total so far: 472146\n", + "Upserted 800 vectors, total so far: 472946\n", + "Upserted 810 vectors, total so far: 473756\n", + "Upserted 770 vectors, total so far: 474526\n", + "Upserted 807 vectors, total so far: 475333\n", + "Upserted 782 vectors, total so far: 476115\n", + "Upserted 812 vectors, total so far: 476927\n", + "Upserted 806 vectors, total so far: 477733\n", + "Upserted 807 vectors, total so far: 478540\n", + "Upserted 788 vectors, total so far: 479328\n", + "Upserted 805 vectors, total so far: 480133\n", + "Upserted 788 vectors, total so far: 480921\n", + "Upserted 798 vectors, total so far: 481719\n", + "Upserted 789 vectors, total so far: 482508\n", + "Upserted 802 vectors, total so far: 483310\n", + "Upserted 810 vectors, total so far: 484120\n", + "Upserted 802 vectors, total so far: 484922\n", + "Upserted 818 vectors, total so far: 485740\n", + "Upserted 800 vectors, total so far: 486540\n", + "Upserted 799 vectors, total so far: 487339\n", + "Upserted 795 vectors, total so far: 488134\n", + "Upserted 798 vectors, total so far: 488932\n", + "Upserted 816 vectors, total so far: 489748\n", + "Upserted 790 vectors, total so far: 490538\n", + "Upserted 816 vectors, total so far: 491354\n", + "Upserted 795 vectors, total so far: 492149\n", + "Upserted 798 vectors, total so far: 492947\n", + "Upserted 801 vectors, total so far: 493748\n", + "Upserted 801 vectors, total so far: 494549\n", + "Upserted 808 vectors, total so far: 495357\n", + "Upserted 795 vectors, total so far: 496152\n", + "Upserted 817 vectors, total so far: 496969\n", + "Upserted 812 vectors, total so far: 497781\n", + "Upserted 760 vectors, total so far: 498541\n", + "Upserted 819 vectors, total so far: 499360\n", + "Upserted 798 vectors, total so far: 500158\n", + "Upserted 801 vectors, total so far: 500959\n", + "Upserted 810 vectors, total so far: 501769\n", + "Upserted 778 vectors, total so far: 502547\n", + "Upserted 790 vectors, total so far: 503337\n", + "Upserted 791 vectors, total so far: 504128\n", + "Upserted 787 vectors, total so far: 504915\n", + "Upserted 814 vectors, total so far: 505729\n", + "Upserted 804 vectors, total so far: 506533\n", + "Upserted 789 vectors, total so far: 507322\n", + "Upserted 800 vectors, total so far: 508122\n", + "Upserted 802 vectors, total so far: 508924\n", + "Upserted 817 vectors, total so far: 509741\n", + "Upserted 782 vectors, total so far: 510523\n", + "Upserted 799 vectors, total so far: 511322\n", + "Upserted 806 vectors, total so far: 512128\n", + "Upserted 804 vectors, total so far: 512932\n", + "Upserted 810 vectors, total so far: 513742\n", + "Upserted 808 vectors, total so far: 514550\n", + "Upserted 799 vectors, total so far: 515349\n", + "Upserted 800 vectors, total so far: 516149\n", + "Upserted 779 vectors, total so far: 516928\n", + "Upserted 802 vectors, total so far: 517730\n", + "Upserted 818 vectors, total so far: 518548\n", + "Upserted 789 vectors, total so far: 519337\n", + "Upserted 800 vectors, total so far: 520137\n", + "Upserted 792 vectors, total so far: 520929\n", + "Upserted 798 vectors, total so far: 521727\n", + "Upserted 782 vectors, total so far: 522509\n", + "Upserted 777 vectors, total so far: 523286\n", + "Upserted 803 vectors, total so far: 524089\n", + "Upserted 799 vectors, total so far: 524888\n", + "Upserted 807 vectors, total so far: 525695\n", + "Upserted 812 vectors, total so far: 526507\n", + "Upserted 813 vectors, total so far: 527320\n", + "Upserted 792 vectors, total so far: 528112\n", + "Upserted 807 vectors, total so far: 528919\n", + "Upserted 791 vectors, total so far: 529710\n", + "Upserted 795 vectors, total so far: 530505\n", + "Upserted 802 vectors, total so far: 531307\n", + "Upserted 797 vectors, total so far: 532104\n", + "Upserted 782 vectors, total so far: 532886\n", + "Upserted 794 vectors, total so far: 533680\n", + "Upserted 822 vectors, total so far: 534502\n", + "Upserted 810 vectors, total so far: 535312\n", + "Upserted 786 vectors, total so far: 536098\n", + "Upserted 798 vectors, total so far: 536896\n", + "Upserted 797 vectors, total so far: 537693\n", + "Upserted 789 vectors, total so far: 538482\n", + "Upserted 800 vectors, total so far: 539282\n", + "Upserted 783 vectors, total so far: 540065\n", + "Upserted 794 vectors, total so far: 540859\n", + "Upserted 790 vectors, total so far: 541649\n", + "Upserted 800 vectors, total so far: 542449\n", + "Upserted 795 vectors, total so far: 543244\n", + "Upserted 811 vectors, total so far: 544055\n", + "Upserted 817 vectors, total so far: 544872\n", + "Upserted 794 vectors, total so far: 545666\n", + "Upserted 788 vectors, total so far: 546454\n", + "Upserted 810 vectors, total so far: 547264\n", + "Upserted 808 vectors, total so far: 548072\n", + "Upserted 795 vectors, total so far: 548867\n", + "Upserted 791 vectors, total so far: 549658\n", + "Upserted 794 vectors, total so far: 550452\n", + "Upserted 791 vectors, total so far: 551243\n", + "Upserted 793 vectors, total so far: 552036\n", + "Upserted 785 vectors, total so far: 552821\n", + "Upserted 822 vectors, total so far: 553643\n", + "Upserted 796 vectors, total so far: 554439\n", + "Upserted 798 vectors, total so far: 555237\n", + "Upserted 810 vectors, total so far: 556047\n", + "Upserted 804 vectors, total so far: 556851\n", + "Upserted 810 vectors, total so far: 557661\n", + "Upserted 812 vectors, total so far: 558473\n", + "Upserted 808 vectors, total so far: 559281\n", + "Upserted 792 vectors, total so far: 560073\n", + "Upserted 776 vectors, total so far: 560849\n", + "Upserted 797 vectors, total so far: 561646\n", + "Upserted 787 vectors, total so far: 562433\n", + "Upserted 799 vectors, total so far: 563232\n", + "Upserted 794 vectors, total so far: 564026\n", + "Upserted 796 vectors, total so far: 564822\n", + "Upserted 778 vectors, total so far: 565600\n", + "Upserted 789 vectors, total so far: 566389\n", + "Upserted 792 vectors, total so far: 567181\n", + "Upserted 803 vectors, total so far: 567984\n", + "Upserted 814 vectors, total so far: 568798\n", + "Upserted 795 vectors, total so far: 569593\n", + "Upserted 803 vectors, total so far: 570396\n", + "Upserted 804 vectors, total so far: 571200\n", + "Upserted 810 vectors, total so far: 572010\n", + "Upserted 800 vectors, total so far: 572810\n", + "Upserted 785 vectors, total so far: 573595\n", + "Upserted 792 vectors, total so far: 574387\n", + "Upserted 805 vectors, total so far: 575192\n", + "Upserted 809 vectors, total so far: 576001\n", + "Upserted 796 vectors, total so far: 576797\n", + "Upserted 807 vectors, total so far: 577604\n", + "Upserted 810 vectors, total so far: 578414\n", + "Upserted 771 vectors, total so far: 579185\n", + "Upserted 801 vectors, total so far: 579986\n", + "Upserted 811 vectors, total so far: 580797\n", + "Upserted 798 vectors, total so far: 581595\n", + "Upserted 802 vectors, total so far: 582397\n", + "Upserted 803 vectors, total so far: 583200\n", + "Upserted 785 vectors, total so far: 583985\n", + "Upserted 811 vectors, total so far: 584796\n", + "Upserted 795 vectors, total so far: 585591\n", + "Upserted 816 vectors, total so far: 586407\n", + "Upserted 794 vectors, total so far: 587201\n", + "Upserted 801 vectors, total so far: 588002\n", + "Upserted 815 vectors, total so far: 588817\n", + "Upserted 832 vectors, total so far: 589649\n", + "Upserted 782 vectors, total so far: 590431\n", + "Upserted 804 vectors, total so far: 591235\n", + "Upserted 800 vectors, total so far: 592035\n", + "Upserted 822 vectors, total so far: 592857\n", + "Upserted 783 vectors, total so far: 593640\n", + "Upserted 821 vectors, total so far: 594461\n", + "Upserted 791 vectors, total so far: 595252\n", + "Upserted 798 vectors, total so far: 596050\n", + "Upserted 787 vectors, total so far: 596837\n", + "Upserted 792 vectors, total so far: 597629\n", + "Upserted 786 vectors, total so far: 598415\n", + "Upserted 817 vectors, total so far: 599232\n", + "Upserted 784 vectors, total so far: 600016\n", + "Upserted 803 vectors, total so far: 600819\n", + "Upserted 788 vectors, total so far: 601607\n", + "Upserted 793 vectors, total so far: 602400\n", + "Upserted 803 vectors, total so far: 603203\n", + "Upserted 778 vectors, total so far: 603981\n", + "Upserted 803 vectors, total so far: 604784\n", + "Upserted 789 vectors, total so far: 605573\n", + "Upserted 807 vectors, total so far: 606380\n", + "Upserted 816 vectors, total so far: 607196\n", + "Upserted 811 vectors, total so far: 608007\n", + "Upserted 803 vectors, total so far: 608810\n", + "Upserted 819 vectors, total so far: 609629\n", + "Upserted 801 vectors, total so far: 610430\n", + "Upserted 797 vectors, total so far: 611227\n", + "Upserted 806 vectors, total so far: 612033\n", + "Upserted 781 vectors, total so far: 612814\n", + "Upserted 796 vectors, total so far: 613610\n", + "Upserted 815 vectors, total so far: 614425\n", + "Upserted 814 vectors, total so far: 615239\n", + "Upserted 820 vectors, total so far: 616059\n", + "Upserted 799 vectors, total so far: 616858\n", + "Upserted 786 vectors, total so far: 617644\n", + "Upserted 808 vectors, total so far: 618452\n", + "Upserted 797 vectors, total so far: 619249\n", + "Upserted 816 vectors, total so far: 620065\n", + "Upserted 826 vectors, total so far: 620891\n", + "Upserted 806 vectors, total so far: 621697\n", + "Upserted 802 vectors, total so far: 622499\n", + "Upserted 813 vectors, total so far: 623312\n", + "Upserted 807 vectors, total so far: 624119\n", + "Upserted 798 vectors, total so far: 624917\n", + "Upserted 792 vectors, total so far: 625709\n", + "Upserted 799 vectors, total so far: 626508\n", + "Upserted 789 vectors, total so far: 627297\n", + "Upserted 797 vectors, total so far: 628094\n", + "Upserted 789 vectors, total so far: 628883\n", + "Upserted 807 vectors, total so far: 629690\n", + "Upserted 798 vectors, total so far: 630488\n", + "Upserted 808 vectors, total so far: 631296\n", + "Upserted 785 vectors, total so far: 632081\n", + "Upserted 804 vectors, total so far: 632885\n", + "Upserted 797 vectors, total so far: 633682\n", + "Upserted 811 vectors, total so far: 634493\n", + "Upserted 795 vectors, total so far: 635288\n", + "Upserted 815 vectors, total so far: 636103\n", + "Upserted 801 vectors, total so far: 636904\n", + "Upserted 806 vectors, total so far: 637710\n", + "Upserted 772 vectors, total so far: 638482\n", + "Upserted 811 vectors, total so far: 639293\n", + "Upserted 817 vectors, total so far: 640110\n", + "Upserted 772 vectors, total so far: 640882\n", + "Upserted 808 vectors, total so far: 641690\n", + "Upserted 787 vectors, total so far: 642477\n", + "Upserted 791 vectors, total so far: 643268\n", + "Upserted 809 vectors, total so far: 644077\n", + "Upserted 781 vectors, total so far: 644858\n", + "Upserted 776 vectors, total so far: 645634\n", + "Upserted 810 vectors, total so far: 646444\n", + "Upserted 811 vectors, total so far: 647255\n", + "Upserted 800 vectors, total so far: 648055\n", + "Upserted 806 vectors, total so far: 648861\n", + "Upserted 799 vectors, total so far: 649660\n", + "Upserted 799 vectors, total so far: 650459\n", + "Upserted 789 vectors, total so far: 651248\n", + "Upserted 789 vectors, total so far: 652037\n", + "Upserted 804 vectors, total so far: 652841\n", + "Upserted 799 vectors, total so far: 653640\n", + "Upserted 792 vectors, total so far: 654432\n", + "Upserted 793 vectors, total so far: 655225\n", + "Upserted 807 vectors, total so far: 656032\n", + "Upserted 819 vectors, total so far: 656851\n", + "Upserted 796 vectors, total so far: 657647\n", + "Upserted 820 vectors, total so far: 658467\n", + "Upserted 798 vectors, total so far: 659265\n", + "Upserted 806 vectors, total so far: 660071\n", + "Upserted 816 vectors, total so far: 660887\n", + "Upserted 788 vectors, total so far: 661675\n", + "Upserted 805 vectors, total so far: 662480\n", + "Upserted 800 vectors, total so far: 663280\n", + "Upserted 795 vectors, total so far: 664075\n", + "Upserted 798 vectors, total so far: 664873\n", + "Upserted 804 vectors, total so far: 665677\n", + "Upserted 802 vectors, total so far: 666479\n", + "Upserted 796 vectors, total so far: 667275\n", + "Upserted 796 vectors, total so far: 668071\n", + "Upserted 792 vectors, total so far: 668863\n", + "Upserted 786 vectors, total so far: 669649\n", + "Upserted 797 vectors, total so far: 670446\n", + "Upserted 818 vectors, total so far: 671264\n", + "Upserted 781 vectors, total so far: 672045\n", + "Upserted 801 vectors, total so far: 672846\n", + "Upserted 796 vectors, total so far: 673642\n", + "Upserted 797 vectors, total so far: 674439\n", + "Upserted 803 vectors, total so far: 675242\n", + "Upserted 793 vectors, total so far: 676035\n", + "Upserted 804 vectors, total so far: 676839\n", + "Upserted 823 vectors, total so far: 677662\n", + "Upserted 803 vectors, total so far: 678465\n", + "Upserted 819 vectors, total so far: 679284\n", + "Upserted 770 vectors, total so far: 680054\n", + "Upserted 810 vectors, total so far: 680864\n", + "Upserted 811 vectors, total so far: 681675\n", + "Upserted 799 vectors, total so far: 682474\n", + "Upserted 805 vectors, total so far: 683279\n", + "Upserted 811 vectors, total so far: 684090\n", + "Upserted 810 vectors, total so far: 684900\n", + "Upserted 810 vectors, total so far: 685710\n", + "Upserted 801 vectors, total so far: 686511\n", + "Upserted 800 vectors, total so far: 687311\n", + "Upserted 795 vectors, total so far: 688106\n", + "Upserted 795 vectors, total so far: 688901\n", + "Upserted 827 vectors, total so far: 689728\n", + "Upserted 775 vectors, total so far: 690503\n", + "Upserted 797 vectors, total so far: 691300\n", + "Upserted 792 vectors, total so far: 692092\n", + "Upserted 806 vectors, total so far: 692898\n", + "Upserted 819 vectors, total so far: 693717\n", + "Upserted 803 vectors, total so far: 694520\n", + "Upserted 783 vectors, total so far: 695303\n", + "Upserted 788 vectors, total so far: 696091\n", + "Upserted 801 vectors, total so far: 696892\n", + "Upserted 813 vectors, total so far: 697705\n", + "Upserted 788 vectors, total so far: 698493\n", + "Upserted 815 vectors, total so far: 699308\n", + "Upserted 785 vectors, total so far: 700093\n", + "Upserted 787 vectors, total so far: 700880\n", + "Upserted 802 vectors, total so far: 701682\n", + "Upserted 794 vectors, total so far: 702476\n", + "Upserted 799 vectors, total so far: 703275\n", + "Upserted 810 vectors, total so far: 704085\n", + "Upserted 826 vectors, total so far: 704911\n", + "Upserted 783 vectors, total so far: 705694\n", + "Upserted 793 vectors, total so far: 706487\n", + "Upserted 806 vectors, total so far: 707293\n", + "Upserted 788 vectors, total so far: 708081\n", + "Upserted 789 vectors, total so far: 708870\n", + "Upserted 778 vectors, total so far: 709648\n", + "Upserted 790 vectors, total so far: 710438\n", + "Upserted 801 vectors, total so far: 711239\n", + "Upserted 811 vectors, total so far: 712050\n", + "Upserted 806 vectors, total so far: 712856\n", + "Upserted 803 vectors, total so far: 713659\n", + "Upserted 810 vectors, total so far: 714469\n", + "Upserted 796 vectors, total so far: 715265\n", + "Upserted 819 vectors, total so far: 716084\n", + "Upserted 816 vectors, total so far: 716900\n", + "Upserted 800 vectors, total so far: 717700\n", + "Upserted 775 vectors, total so far: 718475\n", + "Upserted 782 vectors, total so far: 719257\n", + "Upserted 820 vectors, total so far: 720077\n", + "Upserted 809 vectors, total so far: 720886\n", + "Upserted 795 vectors, total so far: 721681\n", + "Upserted 796 vectors, total so far: 722477\n", + "Upserted 796 vectors, total so far: 723273\n", + "Upserted 796 vectors, total so far: 724069\n", + "Upserted 809 vectors, total so far: 724878\n", + "Upserted 799 vectors, total so far: 725677\n", + "Upserted 814 vectors, total so far: 726491\n", + "Upserted 794 vectors, total so far: 727285\n", + "Upserted 786 vectors, total so far: 728071\n", + "Upserted 800 vectors, total so far: 728871\n", + "Upserted 805 vectors, total so far: 729676\n", + "Upserted 809 vectors, total so far: 730485\n", + "Upserted 797 vectors, total so far: 731282\n", + "Upserted 808 vectors, total so far: 732090\n", + "Upserted 790 vectors, total so far: 732880\n", + "Upserted 818 vectors, total so far: 733698\n", + "Upserted 806 vectors, total so far: 734504\n", + "Upserted 798 vectors, total so far: 735302\n", + "Upserted 811 vectors, total so far: 736113\n", + "Upserted 792 vectors, total so far: 736905\n", + "Upserted 798 vectors, total so far: 737703\n", + "Upserted 791 vectors, total so far: 738494\n", + "Upserted 781 vectors, total so far: 739275\n", + "Upserted 781 vectors, total so far: 740056\n", + "Upserted 797 vectors, total so far: 740853\n", + "Upserted 818 vectors, total so far: 741671\n", + "Upserted 783 vectors, total so far: 742454\n", + "Upserted 788 vectors, total so far: 743242\n", + "Upserted 794 vectors, total so far: 744036\n", + "Upserted 797 vectors, total so far: 744833\n", + "Upserted 812 vectors, total so far: 745645\n", + "Upserted 820 vectors, total so far: 746465\n", + "Upserted 789 vectors, total so far: 747254\n", + "Upserted 795 vectors, total so far: 748049\n", + "Upserted 810 vectors, total so far: 748859\n", + "Upserted 796 vectors, total so far: 749655\n", + "Upserted 797 vectors, total so far: 750452\n", + "Upserted 781 vectors, total so far: 751233\n", + "Upserted 810 vectors, total so far: 752043\n", + "Upserted 802 vectors, total so far: 752845\n", + "Upserted 811 vectors, total so far: 753656\n", + "Upserted 804 vectors, total so far: 754460\n", + "Upserted 804 vectors, total so far: 755264\n", + "Upserted 800 vectors, total so far: 756064\n", + "Upserted 790 vectors, total so far: 756854\n", + "Upserted 822 vectors, total so far: 757676\n", + "Upserted 814 vectors, total so far: 758490\n", + "Upserted 789 vectors, total so far: 759279\n", + "Upserted 793 vectors, total so far: 760072\n", + "Upserted 804 vectors, total so far: 760876\n", + "Upserted 777 vectors, total so far: 761653\n", + "Upserted 791 vectors, total so far: 762444\n", + "Upserted 809 vectors, total so far: 763253\n", + "Upserted 781 vectors, total so far: 764034\n", + "Upserted 803 vectors, total so far: 764837\n", + "Upserted 805 vectors, total so far: 765642\n", + "Upserted 816 vectors, total so far: 766458\n", + "Upserted 787 vectors, total so far: 767245\n", + "Upserted 788 vectors, total so far: 768033\n", + "Upserted 818 vectors, total so far: 768851\n", + "Upserted 810 vectors, total so far: 769661\n", + "Upserted 803 vectors, total so far: 770464\n", + "Upserted 797 vectors, total so far: 771261\n", + "Upserted 809 vectors, total so far: 772070\n", + "Upserted 799 vectors, total so far: 772869\n", + "Upserted 805 vectors, total so far: 773674\n", + "Upserted 797 vectors, total so far: 774471\n", + "Upserted 810 vectors, total so far: 775281\n", + "Upserted 804 vectors, total so far: 776085\n", + "Upserted 797 vectors, total so far: 776882\n", + "Upserted 813 vectors, total so far: 777695\n", + "Upserted 798 vectors, total so far: 778493\n", + "Upserted 808 vectors, total so far: 779301\n", + "Upserted 794 vectors, total so far: 780095\n", + "Upserted 792 vectors, total so far: 780887\n", + "Upserted 804 vectors, total so far: 781691\n", + "Upserted 804 vectors, total so far: 782495\n", + "Upserted 816 vectors, total so far: 783311\n", + "Upserted 793 vectors, total so far: 784104\n", + "Upserted 811 vectors, total so far: 784915\n", + "Upserted 778 vectors, total so far: 785693\n", + "Upserted 808 vectors, total so far: 786501\n", + "Upserted 806 vectors, total so far: 787307\n", + "Upserted 776 vectors, total so far: 788083\n", + "Upserted 782 vectors, total so far: 788865\n", + "Upserted 796 vectors, total so far: 789661\n", + "Upserted 802 vectors, total so far: 790463\n", + "Upserted 806 vectors, total so far: 791269\n", + "Upserted 793 vectors, total so far: 792062\n", + "Upserted 795 vectors, total so far: 792857\n", + "Upserted 784 vectors, total so far: 793641\n", + "Upserted 782 vectors, total so far: 794423\n", + "Upserted 792 vectors, total so far: 795215\n", + "Upserted 797 vectors, total so far: 796012\n", + "Upserted 795 vectors, total so far: 796807\n", + "Upserted 811 vectors, total so far: 797618\n", + "Upserted 781 vectors, total so far: 798399\n", + "Upserted 802 vectors, total so far: 799201\n", + "Upserted 800 vectors, total so far: 800001\n", + "Done! Total inserted: 800001 vectors with id range [0, 800000]\n", + "Time taken: 753.6780933178961\n" + ] + } + ], + "source": [ + "#For reinserting deleted vectors (int filter)\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "import time\n", + "\n", + "# User inputs\n", + "int_rate_insert = \"80p\" # change to 1p, 50p, 80p, 99p\n", + "batch_size = 1000\n", + "\n", + "# Map int_rate to range\n", + "range_map = {\n", + " \"1p\": [0, 10000],\n", + " \"50p\": [0, 500000],\n", + " \"80p\": [0, 800000],\n", + " \"99p\": [0, 990000],\n", + "}\n", + "filter_range = range_map[int_rate_insert]\n", + "low, high = filter_range\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "emb_path = os.path.join(dataset_path, \"shuffle_train.parquet\")\n", + "\n", + "# Process shuffle_train in batches to avoid RAM issues\n", + "emb_file = pq.ParquetFile(emb_path)\n", + "total_inserted = 0\n", + "\n", + "start = time.perf_counter()\n", + "for batch in emb_file.iter_batches(batch_size=batch_size):\n", + " batch_df = batch.to_pandas()\n", + "\n", + " # Filter by id range\n", + " batch_df = batch_df[(batch_df[\"id\"] >= low) & (batch_df[\"id\"] <= high)]\n", + "\n", + " if batch_df.empty:\n", + " continue\n", + "\n", + " batch_vectors = []\n", + " for _, row in batch_df.iterrows():\n", + " vector_data = {\n", + " \"id\": str(row[\"id\"]),\n", + " \"vector\": row[\"emb\"],\n", + " \"meta\": {\"id\": row[\"id\"]},\n", + " \"filter\": {\n", + " \"id\": row[\"id\"],\n", + " }\n", + " }\n", + " batch_vectors.append(vector_data)\n", + "\n", + " index.upsert(batch_vectors)\n", + " total_inserted += len(batch_vectors)\n", + " print(f\"Upserted {len(batch_vectors)} vectors, total so far: {total_inserted}\")\n", + "end = time.perf_counter()\n", + "print(f\"Done! Total inserted: {total_inserted} vectors with id range {filter_range}\")\n", + "print(\"Time taken:\", end-start)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_labelfilter_0503_1_adup7',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 1000000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "index.describe()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv2", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/intfilterscript_final_lt.ipynb b/intfilterscript_final_lt.ipynb new file mode 100644 index 000000000..e9cd09ab7 --- /dev/null +++ b/intfilterscript_final_lt.ipynb @@ -0,0 +1,3075 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: pandas in ./venv2/lib/python3.11/site-packages (3.0.1)\n", + "Requirement already satisfied: pyarrow in ./venv2/lib/python3.11/site-packages (23.0.1)\n", + "Requirement already satisfied: numpy>=1.26.0 in ./venv2/lib/python3.11/site-packages (from pandas) (2.4.2)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in ./venv2/lib/python3.11/site-packages (from pandas) (2.9.0.post0)\n", + "Requirement already satisfied: six>=1.5 in ./venv2/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m26.0.1\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "pip install pandas pyarrow" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: endee in ./venv2/lib/python3.11/site-packages (0.1.13)\n", + "Requirement already satisfied: requests>=2.28.0 in ./venv2/lib/python3.11/site-packages (from endee) (2.32.5)\n", + "Requirement already satisfied: httpx>=0.28.1 in ./venv2/lib/python3.11/site-packages (from httpx[http2]>=0.28.1->endee) (0.28.1)\n", + "Requirement already satisfied: numpy>=2.2.4 in ./venv2/lib/python3.11/site-packages (from endee) (2.4.2)\n", + "Requirement already satisfied: msgpack>=1.1.0 in ./venv2/lib/python3.11/site-packages (from endee) (1.1.2)\n", + "Requirement already satisfied: orjson>=3.11.5 in ./venv2/lib/python3.11/site-packages (from endee) (3.11.7)\n", + "Requirement already satisfied: pydantic>=2.0.0 in ./venv2/lib/python3.11/site-packages (from endee) (2.12.5)\n", + "Requirement already satisfied: anyio in ./venv2/lib/python3.11/site-packages (from httpx>=0.28.1->httpx[http2]>=0.28.1->endee) (4.12.1)\n", + "Requirement already satisfied: certifi in ./venv2/lib/python3.11/site-packages (from httpx>=0.28.1->httpx[http2]>=0.28.1->endee) (2026.1.4)\n", + "Requirement already satisfied: httpcore==1.* in ./venv2/lib/python3.11/site-packages (from httpx>=0.28.1->httpx[http2]>=0.28.1->endee) (1.0.9)\n", + "Requirement already satisfied: idna in ./venv2/lib/python3.11/site-packages (from httpx>=0.28.1->httpx[http2]>=0.28.1->endee) (3.11)\n", + "Requirement already satisfied: h11>=0.16 in ./venv2/lib/python3.11/site-packages (from httpcore==1.*->httpx>=0.28.1->httpx[http2]>=0.28.1->endee) (0.16.0)\n", + "Requirement already satisfied: h2<5,>=3 in ./venv2/lib/python3.11/site-packages (from httpx[http2]>=0.28.1->endee) (4.3.0)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in ./venv2/lib/python3.11/site-packages (from pydantic>=2.0.0->endee) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.41.5 in ./venv2/lib/python3.11/site-packages (from pydantic>=2.0.0->endee) (2.41.5)\n", + "Requirement already satisfied: typing-extensions>=4.14.1 in ./venv2/lib/python3.11/site-packages (from pydantic>=2.0.0->endee) (4.15.0)\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in ./venv2/lib/python3.11/site-packages (from pydantic>=2.0.0->endee) (0.4.2)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in ./venv2/lib/python3.11/site-packages (from requests>=2.28.0->endee) (3.4.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in ./venv2/lib/python3.11/site-packages (from requests>=2.28.0->endee) (2.6.3)\n", + "Requirement already satisfied: hyperframe<7,>=6.1 in ./venv2/lib/python3.11/site-packages (from h2<5,>=3->httpx[http2]>=0.28.1->endee) (6.1.0)\n", + "Requirement already satisfied: hpack<5,>=4.1 in ./venv2/lib/python3.11/site-packages (from h2<5,>=3->httpx[http2]>=0.28.1->endee) (4.1.0)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m26.0.1\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "pip install endee" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Name: endee\n", + "Version: 0.1.13\n", + "Summary: Endee is the Next-Generation Vector Database for Scalable, High-Performance AI\n", + "Home-page: https://endee.io\n", + "Author: Endee Labs\n", + "Author-email: dev@endee.io\n", + "License: \n", + "Location: /home/debian/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages\n", + "Requires: httpx, msgpack, numpy, orjson, pydantic, requests\n", + "Required-by: \n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "pip show endee" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "base_path = \"/home/debian/latest_VDB/VectorDBBench/vectordataset_lt\"\n", + "dataset_folder = \"cohere/cohere_medium_1m\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " id emb\n", + "0 322406 [0.19600096, -0.5270862, -0.29519123, 0.429556...\n", + "1 337610 [0.32829463, -0.055560444, -0.06252914, -0.101...\n", + "2 714728 [-0.054155815, 0.009554057, 0.24696879, -0.142...\n", + "3 354737 [0.2501778, 0.017853737, -0.43795395, 0.522256...\n", + "4 290979 [0.3444257, -0.62243223, -0.20940691, -0.08620...\n" + ] + } + ], + "source": [ + "#For checking parquet file contents\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "file_name = \"shuffle_train.parquet\" #input\n", + "\n", + "# Build full path\n", + "file_path = os.path.join(base_path, dataset_folder, file_name)\n", + "\n", + "\n", + "parquet_file = pq.ParquetFile(file_path)\n", + "\n", + "# Read only first batch of rows\n", + "first_batch = next(parquet_file.iter_batches(batch_size=5))\n", + "preview = first_batch.to_pandas()\n", + "\n", + "for col in preview.columns:\n", + " if preview[col].dtype == object and isinstance(preview[col].iloc[0], list):\n", + " preview[col] = preview[col].apply(lambda x: x[:5] if x is not None else x)\n", + "\n", + "print(preview)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from endee import Endee\n", + "client = Endee(token=\"localtest\")\n", + "client.set_base_url(\"http://148.113.54.173:8080/api/v1\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "hybridbv_f5\n", + "1000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1\n", + "1000000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup1\n", + "1000000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup4\n", + "1000000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup5\n", + "200000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup6\n", + "1000000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup7\n", + "1000000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup8\n", + "996035\n", + "\t\n", + "test_filter_in3\n", + "10\n", + "\t\n" + ] + } + ], + "source": [ + "#For checking indexes\n", + "for obj in client.list_indexes().get('indexes'):\n", + " print(obj['name'])\n", + " print(obj['total_elements'])\n", + " print('\\t')" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "#give the index name\n", + "index_name = \"test_1M_vaib_labelfilter_0503_1_adup8\"\n", + "index = client.get_index(index_name)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_labelfilter_0503_1_adup8',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 1000000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Finding all query IDs with recall < 1.0 ...\n", + "\n", + "Query ID: 14\n", + "Returned IDs: 90505,782992,219095,249142,87223,952696,606232,407566,646730,764993,281286,82235,256972,20952,766350,186364,28749,836818,195642,721815,980205,589088,954423,54884,766421,81742,294536,163812,926812,737136\n", + "Ground Truth: 90505,782992,219095,249142,87223,952696,606232,407566,646730,764993,281286,82235,256972,449934,20952,766350,186364,28749,836818,195642,721815,980205,589088,954423,54884,766421,81742,294536,163812,926812\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 17\n", + "Returned IDs: 791622,801493,917338,162827,700429,464741,899824,420959,982094,637463,453086,142000,792498,873829,923407,526327,251459,175303,383751,977959,226536,887923,337863,883750,341193,796512,820586,248141,538285,840976\n", + "Ground Truth: 791622,801493,917338,162827,700429,464741,899824,820440,420959,982094,637463,453086,142000,792498,873829,923407,526327,251459,175303,383751,977959,226536,887923,337863,883750,341193,796512,401213,820586,248141\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 21\n", + "Returned IDs: 920526,160246,461378,490822,913686,730272,594365,547721,812518,464691,213097,698008,678163,344393,352398,906772,107460,855755,935976,227866,148209,203353,75520,440403,909532,348812,60933,99177,161420,223699\n", + "Ground Truth: 920526,160246,461378,490822,913686,730272,594365,547721,812518,464691,213097,698008,678163,344393,352398,906772,107460,855755,935976,227866,148209,203353,75520,440403,909532,348812,60933,889335,99177,48713\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 23\n", + "Returned IDs: 766971,268212,739092,583683,75196,594757,776150,368812,651554,79487,183691,88311,371655,629135,451687,712041,10095,970062,57581,805963,245985,421560,745040,753442,925988,232568,543533,802140,506026,323509\n", + "Ground Truth: 766971,268212,425738,739092,583683,75196,594757,775830,776150,368812,982531,651554,79487,342446,532188,183691,800983,88311,371655,629135,451687,712041,10095,970062,57581,805963,99475,245985,421560,745040\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 29\n", + "Returned IDs: 162615,54651,938517,885500,799345,493886,28066,102866,650420,176242,556491,958524,602001,904083,544495,523005,486300,799290,665496,211986,255467,371942,247363,82546,688670,861290,833010,663128,906563,259734\n", + "Ground Truth: 162615,54651,938517,885500,799345,493886,28066,102866,650420,176242,556491,958524,602001,904083,544495,523005,486300,799290,665496,211986,255467,371942,57667,247363,82546,950962,688670,861290,833010,663128\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 35\n", + "Returned IDs: 777339,692866,419843,628429,824750,287582,640237,313026,807268,212512,928250,101369,842823,298527,968208,914572,133480,602224,887079,834243,308055,760402,340872,534604,868610,881109,757006,977680,760524,693316\n", + "Ground Truth: 777339,692866,419843,628429,824750,287582,640237,313026,807268,212512,928250,101369,842823,701387,298527,968208,914572,133480,602224,887079,834243,308055,760402,340872,534604,868610,881109,757006,977680,760524\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 38\n", + "Returned IDs: 962399,592238,714880,834195,742540,48493,978874,674724,270183,354882,469702,337817,651192,245492,872772,917903,574157,464990,983351,802140,654252,797627,379870,685914,958018,878677,230193,896532,690408,339724\n", + "Ground Truth: 962399,592238,714880,834195,742540,48493,978874,674724,270183,354882,469702,337817,651192,216031,245492,597835,872772,917903,574157,464990,983351,802140,654252,797627,379870,685914,875307,958018,878677,230193\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 42\n", + "Returned IDs: 668090,441261,814884,468965,193959,740860,475001,332119,692711,88587,250895,311373,820531,351238,230065,755511,726983,376905,20017,875904,56675,214936,742592,508171,23862,648642,751433,808368,337739,561529\n", + "Ground Truth: 668090,441261,814884,468965,193959,740860,475001,332119,692711,88587,250895,820531,311373,351238,230065,755511,726983,376905,20017,875904,56675,214936,742592,508171,496486,23862,648642,751433,808368,337739\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 44\n", + "Returned IDs: 479614,466750,145450,950332,711870,30738,371018,24581,555850,874350,674724,377730,323939,457239,173920,538260,364262,694508,776458,363978,962399,307151,38514,456474,753389,721440,385990,732256,236323,11937\n", + "Ground Truth: 479614,466750,145450,950332,711870,30738,371018,24581,555850,874350,674724,377730,323939,457239,173920,538260,908072,364262,694508,776458,363978,962399,307151,38514,523192,456474,753389,721440,385990,732256\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 51\n", + "Returned IDs: 251008,867561,5969,583259,142664,588037,171228,219201,745114,457877,861530,315569,749214,477691,921426,788453,641362,927146,301769,158176,42504,571774,842285,449569,140136,276076,499478,370057,835846,95034\n", + "Ground Truth: 251008,867561,5969,583259,142664,588037,171228,894379,219201,745114,457877,861530,315569,749214,477691,921426,788453,641362,300182,927146,301769,880278,158176,42504,571774,842285,449569,140136,276076,499478\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 53\n", + "Returned IDs: 141737,179537,243817,469626,402474,176428,830770,363949,494339,240766,354644,155093,532367,242056,899752,946264,473001,899325,569724,8347,200516,739091,932086,875931,236361,32391,436228,92037,195939,627416\n", + "Ground Truth: 141737,179537,243817,469626,402474,176428,830770,363949,494339,240766,354644,155093,532367,242056,899752,946264,473001,899325,569724,559856,8347,200516,739091,932086,875931,236361,32391,436228,92037,195939\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 54\n", + "Returned IDs: 825543,888417,180811,946830,793531,971475,706498,239515,24833,850903,629409,571252,786749,702623,718898,934115,668715,640449,448645,48292,728995,363401,453548,52110,253536,983944,457781,29743,728896,950215\n", + "Ground Truth: 825543,888417,180811,946830,793531,971475,706498,239515,24833,850903,629409,571252,786749,702623,718898,934115,668715,640449,448645,48292,728995,363401,453548,52110,253536,182494,983944,724879,457781,29743\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 55\n", + "Returned IDs: 284177,566607,767119,804416,748611,518411,686975,601153,298977,584380,363375,258208,380761,821125,874746,129911,317104,662020,135584,988553,773690,851260,295420,743414,409769,349994,442826,441763,92120,281012\n", + "Ground Truth: 284177,566607,767119,804416,748611,518411,686975,601153,298977,584380,363375,258208,380761,821125,322747,874746,129911,317104,662020,135584,988553,773690,851260,295420,743414,409769,349994,442826,441763,92120\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 59\n", + "Returned IDs: 107150,876544,597660,235773,440598,688978,178789,782015,881207,109470,384022,779596,727946,225790,926354,943592,246806,674671,673741,455279,762354,125990,720916,938843,413617,64472,467943,406786,474578,868512\n", + "Ground Truth: 107150,876544,597660,235773,440598,688978,178789,782015,881207,109470,384022,779596,727946,252183,225790,926354,943592,246806,861604,674671,673741,455279,762354,125990,720916,619947,938843,413617,64472,467943\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 62\n", + "Returned IDs: 510997,537505,808224,752551,9053,512600,689609,93786,91339,217433,394952,836682,917245,87523,120127,28671,459689,93902,810857,735874,652889,354367,127485,938018,935195,534936,797391,55674,898792,644749\n", + "Ground Truth: 510997,537505,808224,752551,703580,9053,512600,689609,93786,91339,217433,394952,836682,917245,87523,120127,28671,459689,93902,810857,735874,652889,354367,127485,938018,935195,534936,797391,55674,898792\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 66\n", + "Returned IDs: 567766,980989,867428,668441,148454,170649,564636,865309,841690,836001,139328,786944,802260,154859,109721,727956,677574,577423,311870,200168,740973,716451,645621,469163,167881,976432,156573,348712,976304,977029\n", + "Ground Truth: 567766,980989,867428,668441,148454,170649,564636,865309,841690,836001,139328,786944,802260,154859,109721,431103,727956,677574,577423,311870,200168,740973,716451,645621,469163,167881,976432,156573,348712,976304\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 71\n", + "Returned IDs: 295659,781007,766370,43965,979529,147989,622646,250569,332345,752258,895180,938652,733060,337864,917380,884454,357003,768038,327683,783686,838961,109336,281827,577423,496945,674142,61920,252912,65500,651173\n", + "Ground Truth: 298079,139415,295659,781007,766370,43965,979529,147989,622646,250569,332345,752258,895180,938652,733060,337864,917380,884454,713905,357003,768038,327683,783686,838961,281827,109336,577423,496945,674142,61920\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 74\n", + "Returned IDs: 293365,566134,747631,408762,57112,485429,874144,613039,329984,134932,739786,574123,773101,694788,39635,155378,630526,856657,545466,142005,805639,36971,869349,633551,481281,410686,899851,72689,443311,424779\n", + "Ground Truth: 293365,566134,747631,408762,57112,485429,874144,613039,329984,134932,739786,574123,773101,694788,39635,155378,630526,856657,545466,142005,776930,805639,36971,869349,633551,481281,410686,899851,72689,443311\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 75\n", + "Returned IDs: 270457,861440,632605,204160,497340,587475,202880,681223,178678,742203,375557,606800,936440,648433,726689,476593,653294,481462,516854,465745,265912,963513,982957,145450,425116,179305,875171,373928,740304,345127\n", + "Ground Truth: 270457,861440,632605,204160,497340,587475,202880,681223,178678,742203,375557,606800,936440,648433,726689,476593,653294,481462,516854,551466,465745,225250,485216,981056,265912,963513,982957,945993,145450,112929\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 77\n", + "Returned IDs: 250345,544808,486280,591053,74326,824011,975451,858898,204407,166067,904834,771553,515935,703439,897870,593024,699845,69423,54911,636335,864877,545787,890133,592947,368898,428688,432883,437649,139574,327748\n", + "Ground Truth: 250345,544808,486280,591053,74326,824011,160506,975451,858898,204407,166067,904834,771553,515935,703439,897870,593024,699845,69423,54911,636335,864877,545787,890133,592947,368898,432883,428688,437649,139574\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 79\n", + "Returned IDs: 209093,774492,640350,510809,115750,472848,803982,943480,441347,681266,1584,590786,24366,562163,960001,517488,168668,135491,676808,191053,373425,348118,488717,712943,463703,4009,13489,471163,967408,884397\n", + "Ground Truth: 209093,774492,640350,510809,115750,472848,228077,803982,943480,14945,441347,681266,1584,590786,24366,562163,960001,517488,477516,168668,135491,571953,7314,676808,191053,373425,348118,488717,712943,463703\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 81\n", + "Returned IDs: 879203,204905,958198,800603,210082,710583,569036,535309,670134,255197,875726,269020,764797,750116,665935,28976,770599,81809,901921,815496,890326,799424,541892,736540,362633,50422,863466,212258,423031,425385\n", + "Ground Truth: 879203,204905,958198,800603,210082,710583,569036,535309,670134,255197,875726,269020,764797,750116,501423,665935,28976,723751,770599,81809,901921,897652,152057,815496,181128,890326,799424,541892,462083,736540\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 84\n", + "Returned IDs: 700578,170730,910449,262180,765030,301351,309342,969655,111568,144380,908306,546145,888998,544148,246788,658561,290,303550,399085,120918,984431,429415,328818,443271,289261,36053,241251,617923,377430,560682\n", + "Ground Truth: 700578,170730,910449,262180,765030,301351,309342,969655,111568,144380,908306,546145,888998,544148,246788,658561,290,303550,941410,399085,357651,413742,120918,984431,429415,328818,443271,289261,36053,241251\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 85\n", + "Returned IDs: 724168,586281,877937,41578,78824,479608,691618,643079,318203,957263,32871,759205,746933,298679,341233,692310,341520,127241,571259,365886,819801,522376,709650,339050,530275,572875,594865,705241,29977,500760\n", + "Ground Truth: 724168,586281,877937,41578,78824,479608,691618,643079,318203,957263,32871,759205,746933,298679,341233,692310,341520,127241,571259,365886,819801,522376,709650,339050,379429,530275,572875,594865,705241,733502\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 91\n", + "Returned IDs: 19310,87228,310741,462905,286995,708069,539923,53855,411732,445148,577778,896003,220444,745699,226754,932691,239553,594009,556136,872572,580428,989691,953289,328014,129858,483406,153000,752153,928224,918027\n", + "Ground Truth: 19310,87228,310741,462905,286995,708069,539923,411732,53855,445148,577778,896003,220444,745699,226754,932691,239553,594009,556136,872572,580428,568264,989691,953289,909080,328014,129858,483406,153000,752153\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 93\n", + "Returned IDs: 480881,819734,682918,122495,250268,471117,517516,646565,549230,96638,784730,916167,459865,314101,92519,275201,308222,207509,497959,23811,355975,24421,965809,748100,968847,396868,772745,650805,803631,659229\n", + "Ground Truth: 480881,819734,682918,122495,791296,250268,471117,517516,646565,549230,96638,784730,916167,459865,314101,92519,275201,308222,207509,497959,23811,355975,24421,965809,748100,968847,396868,772745,650805,803631\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 95\n", + "Returned IDs: 986827,594271,176925,104444,883366,683471,931364,860983,73656,872989,843514,332990,651039,742142,898192,138918,382330,636316,93491,578177,430029,258234,22649,610379,768980,858837,323100,142801,917837,320815\n", + "Ground Truth: 782138,986827,594271,176925,749208,104444,883366,113441,580695,683471,931364,860983,73656,642231,872989,498042,843514,332990,651039,742142,898192,138918,117084,382330,636316,93491,722872,578177,410920,191865\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 97\n", + "Returned IDs: 319647,909281,227241,184788,448801,842885,372737,264335,876468,91576,334249,593760,362219,344550,631322,561529,930256,13115,575858,451349,215504,288237,381496,328405,612262,211018,815779,702026,381373,935306\n", + "Ground Truth: 319647,909281,227241,184788,448801,842885,372737,264335,876468,91576,334249,593760,362219,344550,631322,561529,930256,13115,575858,451349,215504,288237,381496,328405,612262,211018,885514,815779,702026,331664\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 99\n", + "Returned IDs: 574897,812478,294173,820038,549565,720663,748624,732980,149898,745733,739720,768280,141523,412944,498809,349562,166812,510019,28236,146061,608505,874668,179037,598222,474120,652532,250509,201124,891095,556454\n", + "Ground Truth: 574897,812478,294173,820038,549565,720663,748624,732980,149898,745733,739720,768280,141523,412944,498809,349562,166812,510019,849147,28236,146061,608505,874668,179037,974176,598222,474120,652532,250509,201124\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 100\n", + "Returned IDs: 599706,324713,47558,100309,327436,359908,788657,600079,324606,474,659285,168977,516541,513205,31515,137882,24529,964714,374157,911583,728183,525813,669455,427889,420317,494333,503269,787272,232726,249800\n", + "Ground Truth: 599706,324713,47558,100309,327436,359908,788657,600079,324606,474,659285,168977,516541,513205,31515,137882,24529,606190,964714,374157,911583,728183,525813,669455,427889,420317,494333,503269,787272,232726\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 105\n", + "Returned IDs: 104399,85039,233287,622802,930544,259545,860666,435437,827484,696234,724181,932849,277388,416062,100336,626521,51306,449120,490903,482919,911846,366157,771228,210682,512627,831032,85616,444779,809190,181900\n", + "Ground Truth: 104399,85039,233287,622802,930544,259545,860666,396706,435437,827484,163760,696234,724181,422532,932849,277388,416062,100336,626521,51306,449120,245328,490903,482919,274693,522024,911846,366157,771228,210682\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 106\n", + "Returned IDs: 870236,670134,26105,36909,905617,580357,753090,294991,73954,145450,603607,57581,268367,985759,237355,800528,775487,815856,954615,920107,962399,670922,556160,953485,204160,607535,148995,563108,688286,87493\n", + "Ground Truth: 870236,670134,26105,36909,905617,580357,753090,294991,73954,145450,603607,57581,268367,985759,864361,325852,237355,800528,775487,815856,954615,920107,962399,670922,556160,953485,204160,607535,148995,563108\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 107\n", + "Returned IDs: 758628,74785,461888,108492,534419,172052,321983,207667,798103,197798,430556,733636,561529,635018,30734,323100,583278,477731,920014,821246,907476,589155,594910,684006,125322,964621,434250,475185,330838,410378\n", + "Ground Truth: 758628,74785,461888,604867,108492,534419,172052,321983,207667,798103,197798,430556,733636,561529,635018,30734,323100,583278,477731,920014,821246,907476,589155,594910,684006,125322,964621,434250,475185,330838\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 108\n", + "Returned IDs: 683268,173502,166325,479728,52008,606695,246486,542432,626602,798814,724017,25375,926618,615906,962062,171920,880974,721929,760577,153471,258016,597061,48711,139629,386810,848622,224101,971463,148816,324007\n", + "Ground Truth: 683268,173502,166325,479728,52008,606695,246486,542432,626602,798814,724017,25375,926618,615906,962062,171920,880974,721929,760577,496563,153471,258016,597061,48711,139629,386810,848622,224101,971463,148816\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 109\n", + "Returned IDs: 983521,750571,256883,270788,552390,693683,358867,1678,202646,584189,926972,241578,238185,520667,685632,726748,884267,202333,373054,910872,236952,304311,149941,630096,768158,174012,192032,736462,839579,307955\n", + "Ground Truth: 983521,750571,256883,270788,552390,693683,358867,1678,202646,584189,926972,241578,238185,520667,756150,685632,726748,782900,884267,202333,373054,910872,236952,304311,149941,630096,768158,174012,192032,736462\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 114\n", + "Returned IDs: 828553,302817,771183,545372,707968,519418,174068,617228,893919,60666,83940,950722,349034,13212,46733,472685,281009,652050,173810,239608,909932,725121,967343,372374,442863,443577,287141,597169,650426,440627\n", + "Ground Truth: 828553,302817,771183,545372,707968,519418,174068,617228,893919,60666,83940,950722,349034,13212,46733,472685,281009,652050,173810,239608,909932,725121,967343,727361,635958,372374,442863,443577,287141,967917\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 116\n", + "Returned IDs: 421028,655436,379088,129237,278852,431284,303228,385802,650692,218503,231162,406104,256395,773252,364187,884706,635781,362233,597456,693711,235436,846242,830810,131836,894398,388722,50088,274829,388354,238265\n", + "Ground Truth: 421028,655436,379088,129237,278852,431284,303228,385802,650692,218503,231162,406104,256395,773252,364187,884706,635781,362233,597456,693711,235436,846242,830810,131836,894398,388722,50088,900465,274829,388354\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 117\n", + "Returned IDs: 913461,186350,705927,772115,149431,327061,149035,495713,691320,683792,808999,289480,27650,733818,629828,982574,522958,772747,549321,591264,558865,588446,368811,49354,236933,597081,828549,876623,442442,135842\n", + "Ground Truth: 913461,186350,705927,772115,149431,327061,149035,495713,691320,683792,808999,289480,27650,733818,629828,62726,982574,522958,772747,549321,591264,558865,588446,368811,49354,236933,597081,828549,876623,442442\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 118\n", + "Returned IDs: 309885,611973,204016,111578,211608,259929,365666,480679,948740,218105,445693,617693,128482,193632,537470,504970,556479,28524,278317,688900,643412,773990,758362,940780,625469,841244,102060,229254,732333,123589\n", + "Ground Truth: 309885,611973,204016,111578,211608,259929,365666,480679,11276,948740,218105,445693,617693,128482,193632,537470,504970,556479,28524,278317,688900,643412,773990,758362,940780,625469,841244,102060,229254,732333\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 120\n", + "Returned IDs: 657539,213079,951739,290875,397766,976040,52115,236793,562699,413665,557749,865968,453590,574718,533820,237792,678710,943178,481305,290104,606294,555566,729273,889779,397945,199967,47162,912908,934771,257049\n", + "Ground Truth: 657539,213079,951739,290875,397766,976040,52115,236793,562699,413665,557749,865968,453590,574718,533820,237792,678710,943178,173173,481305,290104,606294,555566,729273,889779,397945,199967,47162,912908,934771\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 121\n", + "Returned IDs: 137066,350568,446055,476614,699082,92519,169137,690404,790149,79291,520449,517735,672059,62256,243121,865547,726975,55444,626057,216923,468421,338365,659229,221351,194794,435330,812481,194376,545132,229519\n", + "Ground Truth: 137066,350568,446055,476614,699082,92519,169137,690404,790149,79291,520449,517735,672059,62256,243121,865547,726975,517730,55444,626057,216923,468421,338365,659229,221351,194794,435330,812481,217433,194376\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 129\n", + "Returned IDs: 734143,389648,122180,337716,239671,95517,649281,743681,613289,214882,862606,411755,204130,210160,760684,749090,895164,92812,436389,733239,77990,385333,258752,962283,556727,480027,942978,195759,822698,966991\n", + "Ground Truth: 734143,389648,122180,337716,239671,95517,649281,743681,613289,214882,862606,411755,204130,210160,760684,749090,895164,92812,436389,733239,77990,385333,258752,962283,556727,480027,179435,942978,195759,822698\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 130\n", + "Returned IDs: 397751,725506,378789,152546,601972,823982,301636,760534,732364,633368,712408,297283,845346,143584,215945,531498,984638,919426,805180,239967,624165,956308,890188,189734,766140,67881,21085,822001,561113,454851\n", + "Ground Truth: 397751,725506,378789,152546,601972,823982,301636,760534,732364,633368,729582,712408,297283,845346,143584,215945,531498,984638,919426,805180,239967,624165,956308,890188,189734,766140,67881,21085,822001,561113\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 132\n", + "Returned IDs: 130164,761418,952950,785887,4480,629413,332797,552220,253914,783795,98665,670815,538823,786797,967870,907090,614194,457691,770151,741106,28551,841800,656808,303162,108999,449935,674347,223326,478978,308890\n", + "Ground Truth: 130164,761418,952950,785887,4480,629413,332797,552220,253914,783795,98665,670815,538823,786797,967870,907090,614194,457691,770151,741106,28551,841800,392363,656808,303162,108999,449935,674347,223326,478978\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 135\n", + "Returned IDs: 703645,497731,706705,883660,330113,634696,375663,405927,842712,963845,460431,199408,263796,496199,393816,228083,43311,554660,866635,748100,504871,250885,765279,12812,761657,931112,606031,83807,537531,305330\n", + "Ground Truth: 703645,497731,706705,883660,330113,634696,375663,405927,424540,842712,963845,460431,199408,263796,496199,393816,228083,43311,554660,866635,748100,504871,250885,765279,239595,12812,761657,931112,346310,606031\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 139\n", + "Returned IDs: 6017,14964,776041,242625,641024,697795,458466,532383,640954,290578,977123,799113,192825,78583,777538,363759,697593,317838,193212,508879,804800,460171,225335,925009,611096,57485,886625,121875,957713,238929\n", + "Ground Truth: 6017,14964,776041,242625,641024,697795,458466,352087,532383,640954,290578,977123,799113,192825,78583,777538,363759,697593,567560,317838,193212,508879,804800,460171,225335,925009,787560,611096,40133,257097\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 140\n", + "Returned IDs: 402047,91289,396789,15335,509841,417002,695508,659239,38466,450159,7753,599688,818329,556866,700431,915421,664172,55791,161439,448200,497928,60040,389022,251996,95512,550208,183479,916915,241812,914584\n", + "Ground Truth: 402047,91289,396789,15335,509841,417002,695508,659239,38466,450159,7753,599688,818329,556866,700431,915421,664172,55791,161439,448200,497928,60040,389022,251996,95512,924963,550208,183479,916915,241812\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 142\n", + "Returned IDs: 77562,720452,791456,49275,668921,218728,398795,176860,121394,499795,223898,535434,623024,569337,840231,903396,735759,790172,67045,187350,149290,101125,433571,210457,447986,683512,344110,50647,468581,796908\n", + "Ground Truth: 853280,833976,77562,720452,791456,49275,668921,218728,398795,176860,121394,499795,223898,535434,623024,569337,840231,903396,735759,790172,67045,187350,149290,101125,433571,210457,447986,683512,344110,50647\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 144\n", + "Returned IDs: 366790,940327,786112,326662,973368,151928,725916,978557,950787,181844,397853,827444,628589,143835,150218,752213,748518,495406,110725,750340,309386,214953,333678,888928,174594,42516,809775,943874,478515,817895\n", + "Ground Truth: 366790,447010,940327,786112,326662,973368,151928,725916,978557,950787,181844,397853,827444,328199,628589,143835,754044,150218,752213,748518,495406,110725,750340,309386,214953,333678,888928,174594,42516,809775\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 146\n", + "Returned IDs: 605123,406077,597811,415054,46353,103754,69518,371932,501704,737834,933325,805788,929208,340431,807434,858534,269965,925239,726282,568081,666014,691313,726353,299153,950101,357003,525529,20570,908444,911043\n", + "Ground Truth: 605123,406077,597811,415054,46353,103754,69518,371932,501704,737834,933325,805788,929208,644085,340431,807434,858534,269965,925239,726282,568081,666014,691313,726353,299153,950101,357003,525529,20570,908444\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 148\n", + "Returned IDs: 963455,275273,166103,698345,587330,812632,126328,75520,513421,794504,679648,628275,337632,260088,745205,92466,224343,271503,179018,566449,988298,731704,119867,82790,545254,99748,203610,399531,41924,796886\n", + "Ground Truth: 963455,275273,166103,698345,587330,812632,126328,75520,513421,794504,679648,628275,337632,260088,745205,6647,92466,600943,224343,271503,179018,566449,988298,731704,119867,82790,545254,99748,203610,399531\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 149\n", + "Returned IDs: 735418,232010,571369,223050,787330,902483,427109,954505,854287,388704,366371,240447,155649,678234,313108,380488,369547,60814,635128,972728,599642,880036,546937,398275,564338,424644,702210,334303,297908,116847\n", + "Ground Truth: 735418,232010,571369,223050,787330,902483,427109,954505,854287,388704,366371,240447,155649,678234,313108,380488,369547,60814,635128,573141,972728,599642,880036,546937,398275,564338,424644,702210,334303,297908\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 154\n", + "Returned IDs: 297265,954748,620655,362217,568883,949219,850436,436588,796304,439867,912731,185399,733743,570630,530241,809357,345226,162763,255978,868463,172557,212127,845754,178887,376708,769769,523487,259723,716460,136518\n", + "Ground Truth: 297265,954748,620655,362217,568883,949219,850436,436588,796304,439867,912731,185399,733743,570630,530241,809357,345226,162763,255978,868463,172557,212127,845754,178887,376708,769769,523487,259723,716460,62864\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 158\n", + "Returned IDs: 437900,948497,725048,940583,654292,61229,473966,12651,931509,929848,503322,245492,127194,45723,899557,564041,906651,194654,433918,649461,713082,921954,836300,788310,82943,846658,951330,884714,464990,811313\n", + "Ground Truth: 437900,948497,725048,940583,654292,61229,473966,12651,931509,929848,503322,245492,127194,45723,899557,564041,906651,194654,433918,649461,713082,921954,836300,758241,332127,788310,82943,846658,951330,884714\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 170\n", + "Returned IDs: 117394,446515,1885,662652,987719,655867,19428,906150,300162,445078,570747,566803,933010,900824,880809,607597,74642,940177,568647,220216,113232,39117,637024,689703,738723,567057,661709,961044,165367,604068\n", + "Ground Truth: 117394,446515,1885,662652,987719,655867,19428,906150,300162,445078,570747,566803,933010,900824,880809,607597,74642,940177,568647,220216,113232,39117,769342,637024,689703,738723,567057,661709,961044,165367\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 173\n", + "Returned IDs: 245559,197229,306640,571521,902348,29498,353845,315315,291864,955056,258695,798606,232088,802605,741077,89843,336323,984433,647552,687225,550012,162412,131898,336361,272868,941810,831097,510405,160619,882850\n", + "Ground Truth: 245559,197229,306640,556701,571521,902348,29498,353845,315315,291864,955056,912243,529633,289726,896354,258695,798606,676632,232088,802605,741077,89843,336323,984433,647552,687225,550012,553036,162412,907353\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 176\n", + "Returned IDs: 271059,643685,479004,147714,785012,103714,746206,456565,123437,196690,246788,256821,553067,898405,871467,869729,542022,662659,148278,761156,544861,192792,766391,472281,577392,299395,858313,421380,572253,73544\n", + "Ground Truth: 211672,271059,316929,643685,479004,110544,147714,785012,103714,746206,456565,123437,196690,246788,256821,553067,898405,871467,869729,542022,662659,148278,761156,544861,192792,215597,766391,472281,74339,577392\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 178\n", + "Returned IDs: 445444,173059,559089,129629,5575,399183,984199,209275,497265,533604,906834,694445,6672,831758,250409,416996,819827,630806,850785,775618,530955,192418,469184,877511,367737,795403,629468,253280,715679,201767\n", + "Ground Truth: 445444,173059,559089,129629,219925,5575,399183,984199,209275,497265,533604,906834,694445,6672,373832,831758,250409,416996,819827,630806,850785,775618,8663,530955,192418,469184,877511,367737,795403,629468\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 181\n", + "Returned IDs: 98079,785599,2850,524471,217631,240254,254800,440838,54618,89057,539213,677304,873157,260160,37303,719376,478951,578343,897497,584079,396674,487818,623730,892465,897467,307715,684886,358280,987303,382890\n", + "Ground Truth: 98079,785599,2850,524471,217631,240254,254800,440838,54618,89057,539213,677304,873157,260160,37303,719376,27134,478951,578343,897497,584079,396674,487818,623730,892465,897467,307715,813626,684886,358280\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 184\n", + "Returned IDs: 823189,358770,509049,582180,459539,574930,506509,464501,635128,154214,299515,485709,734561,168649,590909,898863,263207,583901,194650,406192,602252,644042,679367,146135,708731,647345,68530,775223,482130,550396\n", + "Ground Truth: 823189,358770,509049,582180,459539,574930,506509,464501,635128,154214,299515,485709,734561,168649,590909,898863,263207,583901,194650,406192,602252,644042,679367,146135,708731,647345,68530,188854,775223,224180\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 197\n", + "Returned IDs: 623527,768980,631216,172052,785639,461888,321983,683471,288678,204901,74785,533987,748631,841117,477731,56347,550876,857163,566851,508513,834243,328481,589155,177681,756075,207667,68720,258234,686901,61754\n", + "Ground Truth: 623527,768980,631216,515404,172052,785639,461888,321983,683471,288678,204901,74785,533987,748631,841117,477731,56347,945948,550876,857163,566851,508513,834243,328481,589155,177681,756075,207667,68720,258234\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 199\n", + "Returned IDs: 909484,682475,604420,194351,904328,552724,531103,542345,599414,377380,715545,476603,261093,397525,36078,649285,201076,518806,15971,723983,728128,870046,470101,345157,873582,419876,104968,116152,205661,957478\n", + "Ground Truth: 909484,386821,682475,604420,194351,904328,552724,531103,542345,599414,377380,715545,476603,261093,397525,36078,649285,201076,518806,15971,517954,723983,728128,870046,470101,345157,873582,419876,104968,116152\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 200\n", + "Returned IDs: 674724,538768,193458,212925,851462,529826,381653,674122,555850,629123,670644,168255,174995,509207,620065,314182,111390,74806,950332,986919,25243,67133,855156,245492,960916,724617,30738,228870,732256,892694\n", + "Ground Truth: 674724,538768,193458,212925,851462,529826,381653,674122,555850,629123,670644,168255,174995,509207,620065,314182,111390,74806,289231,950332,986919,25243,67133,855156,245492,960916,724617,30738,228870,732256\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 203\n", + "Returned IDs: 805758,759303,902471,745320,274591,606321,863380,526343,519909,786906,823800,444099,4497,277435,410898,596254,701303,352643,272576,944271,208369,838345,396944,653285,674309,886443,954654,555834,881902,867562\n", + "Ground Truth: 805758,759303,902471,745320,274591,606321,863380,526343,519909,786906,823800,444099,4497,277435,410898,596254,701303,352643,272576,944271,208369,838345,396944,653285,674309,678123,886443,954654,100321,555834\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 211\n", + "Returned IDs: 729850,121585,895295,167500,699428,293455,454496,153115,650553,988387,675915,324814,57796,3570,494746,184373,127140,958051,151447,202514,769247,383248,499090,982678,376115,76640,900646,35558,709843,226570\n", + "Ground Truth: 729850,121585,895295,167500,699428,293455,454496,153115,650553,988387,675915,324814,57796,3570,494746,184373,127140,958051,151447,712788,847638,202514,769247,383248,499090,982678,376115,76640,900646,35558\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 214\n", + "Returned IDs: 913787,824306,22317,11040,404308,905082,736813,412251,465362,369246,721600,233162,977275,104209,266553,965199,931608,914701,664818,563048,79974,864877,185438,562084,629581,505744,486158,342175,29743,751146\n", + "Ground Truth: 913787,824306,22317,11040,404308,905082,736813,412251,465362,369246,721600,233162,977275,104209,31162,266553,965199,931608,914701,664818,563048,79974,864877,185438,562084,629581,505744,486158,342175,29743\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 215\n", + "Returned IDs: 474855,281914,813635,329072,358481,492076,946802,710308,185795,3569,888785,288716,35656,742150,109037,436268,397889,920689,18443,532960,535533,158530,544132,1963,829372,154951,770223,393182,291322,446905\n", + "Ground Truth: 474855,281914,813635,329072,358481,492076,946802,710308,249244,185795,3569,888785,288716,35656,742150,109037,436268,397889,920689,18443,532960,535533,158530,544132,1963,829372,154951,770223,393182,291322\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 216\n", + "Returned IDs: 523129,166510,263219,17945,19391,398467,125980,178134,647298,891102,251448,27138,90339,451291,467762,558878,961447,655615,711513,434544,929962,900993,566228,379767,681717,199239,625474,723161,380431,57392\n", + "Ground Truth: 523129,166510,263219,17945,19391,398467,125980,178134,647298,891102,251448,27138,90339,451291,467762,558878,961447,655615,711513,434544,929962,900993,961798,566228,379767,681717,199239,625474,723161,380431\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 221\n", + "Returned IDs: 278166,261319,350660,745077,125401,590776,111887,420444,238347,816509,818019,153569,454118,352639,378647,127568,797652,236692,449181,526913,104258,255692,960545,674152,628275,146289,120127,525709,967788,247445\n", + "Ground Truth: 278166,261319,350660,745077,125401,590776,111887,420444,238347,816509,818019,153569,454118,352639,378647,127568,797652,236692,449181,526913,104258,255692,960545,674152,628275,146289,120127,525709,537230,967788\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 225\n", + "Returned IDs: 34397,466849,852160,17048,521614,496387,114018,814780,873966,451658,440849,120137,191649,218774,81044,674921,320365,753936,878214,931125,142644,759450,760163,979520,975820,813801,965496,366079,642111,705726\n", + "Ground Truth: 34397,466849,852160,17048,521614,496387,114018,814780,873966,451658,440849,120137,191649,218774,81044,674921,320365,753936,878214,931125,142644,759450,133763,760163,979520,975820,813801,965496,366079,642111\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 226\n", + "Returned IDs: 836206,23960,668489,373812,420078,689569,233278,623236,559422,891056,691442,910854,781639,341264,270504,105073,205068,59780,32904,988193,398514,785332,272646,389800,324381,84941,239000,418997,814230,248822\n", + "Ground Truth: 836206,23960,668489,373812,420078,689569,233278,623236,559422,891056,691442,910854,781639,341264,270504,105073,205068,59780,32904,988193,398514,785332,272646,389800,951010,324381,84941,239000,418997,814230\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 245\n", + "Returned IDs: 238675,601033,849540,711345,936699,859908,646367,385805,173197,557338,688729,780433,67528,275883,661377,766421,797477,883425,638650,381681,608915,546076,301460,438598,780789,667180,547877,453554,859282,818951\n", + "Ground Truth: 238675,601033,849540,711345,936699,859908,646367,385805,173197,557338,688729,780433,67528,275883,661377,766421,797477,883425,638650,381681,608915,546076,301460,438598,780789,205313,667180,547877,453554,859282\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 248\n", + "Returned IDs: 370350,273458,433624,222387,106220,83940,567888,7898,578716,24911,52728,17906,610440,809130,929415,75276,89672,610794,564613,57311,94618,174068,523672,665109,516946,274012,418826,582644,979890,734178\n", + "Ground Truth: 370350,273458,433624,222387,106220,83940,567888,7898,578716,24911,52728,17906,610440,445747,809130,929415,75276,89672,610794,564613,57311,94618,174068,523672,665109,516946,274012,418826,582644,979890\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 249\n", + "Returned IDs: 359927,734269,407089,8211,276596,504312,678597,639694,866661,615923,433540,952149,956452,812522,603224,505268,580188,395363,624693,592359,479138,888028,162318,56767,440797,724246,761966,247314,919150,779952\n", + "Ground Truth: 359927,734269,712188,816880,407089,8211,276596,504312,678597,639694,866661,615923,433540,952149,956452,812522,603224,505268,580188,395363,624693,592359,457178,479138,888028,162318,56767,440797,724246,761966\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 254\n", + "Returned IDs: 971127,112696,467682,929224,784103,399371,42069,187311,247056,183715,705075,157398,903943,669772,247351,526657,440729,822874,206353,75584,902213,651122,821015,109770,706917,500028,260090,250118,201578,718946\n", + "Ground Truth: 971127,112696,467682,929224,784103,399371,42069,187311,247056,183715,705075,157398,903943,669772,247351,526657,440729,822874,206353,75584,733093,902213,651122,821015,109770,706917,500028,260090,250118,201578\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 256\n", + "Returned IDs: 662729,810204,651983,466095,231116,720605,327912,833916,695476,692089,850848,14508,373752,285783,286816,189073,312927,255293,721065,406505,578187,917206,660095,32134,428259,368301,341397,294012,164692,810403\n", + "Ground Truth: 662729,810204,651983,466095,231116,506349,720605,327912,833916,695476,692089,850848,14508,373752,285783,286816,189073,312927,255293,721065,406505,578187,917206,660095,32134,428259,368301,391481,341397,294012\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 257\n", + "Returned IDs: 520530,182575,656485,561016,739687,685199,801581,235842,413767,605712,126908,415211,529902,441079,792657,560139,301035,69005,116959,194654,561617,373723,197787,960762,580734,816375,144089,245492,780008,865618\n", + "Ground Truth: 520530,182575,656485,561016,739687,685199,801581,235842,413767,605712,126908,415211,529902,441079,792657,560139,301035,69005,116959,194654,561617,373723,197787,108046,960762,309942,580734,655851,816375,144089\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 261\n", + "Returned IDs: 538874,622232,968140,633499,843757,919475,666794,976944,5989,135538,631178,557546,549322,933329,530074,205724,797740,262864,892477,589194,59732,989983,900342,160924,259570,657979,258880,615710,560775,486677\n", + "Ground Truth: 538874,622232,968140,633499,843757,919475,666794,102636,976944,5989,135538,631178,557546,549322,933329,530074,226093,205724,797740,610528,262864,892477,589194,754176,59732,989983,900342,165563,160924,259570\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 266\n", + "Returned IDs: 278496,408425,256281,768978,543165,533511,350257,570329,342446,792275,485913,378098,670032,857889,891953,501260,978501,115595,144246,560644,618852,574231,844961,395736,782423,15457,824529,200188,904999,7380\n", + "Ground Truth: 278496,408425,256281,768978,543165,533511,350257,570329,342446,792275,485913,378098,670032,857889,891953,501260,978501,115595,144246,560644,653771,618852,574231,844961,395736,782423,15457,824529,200188,904999\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 268\n", + "Returned IDs: 557064,375571,952311,670178,295002,478185,350338,421707,488108,758266,565159,324048,424876,506978,850293,844653,542536,549070,367195,195722,845135,248094,706816,298515,832928,920969,356897,119086,286033,222095\n", + "Ground Truth: 557064,375571,952311,670178,295002,478185,350338,421707,488108,758266,565159,324048,424876,506978,850293,844653,542536,549070,367195,619332,195722,845135,248094,706816,298515,832928,920969,569992,356897,119086\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 272\n", + "Returned IDs: 523630,303153,616372,68676,911212,810475,518132,713073,568826,108100,187449,292580,785332,873159,59989,713333,522224,722063,897236,92,864588,326828,178364,600470,560668,988193,747308,272646,841694,481276\n", + "Ground Truth: 523630,303153,616372,68676,911212,810475,518132,713073,568826,108100,187449,292580,785332,873159,59989,713333,522224,579958,722063,897236,92,864588,326828,178364,600470,44584,560668,988193,747308,272646\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 278\n", + "Returned IDs: 982206,759798,142184,868252,653417,667437,859895,171195,724552,789679,129180,204436,779080,662393,502914,346865,96591,813373,120813,972676,71153,802014,318216,117401,455711,367987,505637,661994,533502,733966\n", + "Ground Truth: 982206,759798,142184,868252,653417,667437,859895,171195,724552,789679,129180,204436,779080,662393,502914,346865,96591,433572,813373,932967,120813,972676,71153,802014,318216,117401,455711,539006,367987,397736\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 282\n", + "Returned IDs: 394634,837983,144549,334654,158644,702348,876659,109932,749497,759446,670539,690370,662708,661301,800057,354764,241926,360335,587013,288537,961329,58526,714113,781641,761728,1290,460974,530325,910991,185661\n", + "Ground Truth: 394634,837983,144549,334654,158644,702348,876659,308000,270788,109932,749497,759446,670539,690370,662708,661301,800057,516432,316993,354764,241926,360335,587013,614127,288537,961329,58526,714113,781641,761728\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 283\n", + "Returned IDs: 53143,820863,980583,30807,921317,150131,34909,341924,19066,182920,717365,485724,759630,472075,961073,314545,251267,528507,933895,151479,5243,532772,446422,988019,420715,15415,621800,352636,915758,438831\n", + "Ground Truth: 53143,820863,980583,30807,921317,150131,34909,341924,19066,182920,717365,485724,759630,472075,961073,957794,314545,251267,528507,933895,151479,5243,532772,446422,829726,988019,420715,15415,621800,352636\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 284\n", + "Returned IDs: 324202,953996,510452,443900,797619,693962,355920,564006,737779,683263,472466,217520,329675,61405,929750,77971,637761,978748,931803,262938,260517,900117,302033,390114,302523,660890,823316,13656,216950,677307\n", + "Ground Truth: 324202,953996,510452,443900,797619,693962,355920,564006,737779,683263,472466,217520,329675,61405,929750,77971,637761,19403,978748,931803,262938,260517,900117,302033,390114,302523,660890,705159,426971,823316\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 285\n", + "Returned IDs: 189396,273979,715785,137419,846796,219595,551574,650907,576985,247409,142197,45230,745903,283183,915427,50878,660483,345528,612299,480741,983345,198707,383532,185593,37251,847199,55605,759159,728087,55633\n", + "Ground Truth: 189396,273979,715785,137419,846796,219595,551574,650907,576985,247409,142197,45230,745903,283183,915427,50878,660483,345528,612299,480741,983345,198707,383532,185593,37251,847199,55605,759159,757481,728087\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 289\n", + "Returned IDs: 147089,141162,386074,90785,32036,870594,41792,36853,608232,779273,598179,124720,441503,80061,676652,905330,109605,669208,292300,594432,23221,143759,800930,347089,598222,187744,371264,193383,453554,358248\n", + "Ground Truth: 147089,141162,386074,90785,32036,870594,41792,36853,608232,779273,598179,124720,441503,80061,676652,905330,109605,669208,292300,594432,57892,23221,4721,143759,800930,347089,598222,187744,371264,193383\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 290\n", + "Returned IDs: 933685,486977,904036,127539,302624,779714,359939,947243,539914,158559,230850,325887,341379,569269,575869,630846,209185,305931,933286,261292,523900,465685,571698,727546,140592,963215,440463,65679,890730,509924\n", + "Ground Truth: 933685,486977,904036,127539,302624,779714,359939,947243,539914,158559,230850,325887,341379,569269,575869,630846,209185,305931,933286,97096,261292,523900,465685,571698,727546,140592,963215,440463,65679,890730\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 295\n", + "Returned IDs: 77494,269632,670970,463595,250226,639660,867603,419994,580715,137327,377682,107025,469811,612536,629409,353820,751866,96869,494421,233645,314635,197107,949820,84430,108584,787240,418971,596932,944672,205625\n", + "Ground Truth: 77494,269632,670970,463595,250226,639660,867603,419994,580715,137327,377682,107025,469811,612536,629409,353820,751866,96869,494421,233645,314635,197107,949820,84430,979337,108584,787240,618214,418971,596932\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 299\n", + "Returned IDs: 395722,549247,731363,511381,491411,854362,289376,291036,741451,924599,430139,411675,59072,611023,725312,923163,27533,344332,304070,934016,137564,703467,168510,846981,567131,310026,960794,770271,732515,356391\n", + "Ground Truth: 395722,549247,731363,511381,491411,854362,289376,291036,741451,924599,430139,411675,59072,611023,725312,923163,27533,344332,304070,934016,137564,703467,168510,846981,567131,401819,310026,960794,770271,732515\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 306\n", + "Returned IDs: 147061,862080,706527,91071,397248,903113,327389,514520,943315,768629,582667,165802,11100,865812,376444,857894,551268,453427,244317,962363,131797,514380,199211,231592,822093,391972,692675,901644,439526,240791\n", + "Ground Truth: 147061,862080,706527,91071,397248,903113,327389,743603,514520,943315,768629,582667,165802,11100,865812,376444,857894,551268,453427,244317,962363,131797,514380,199211,231592,822093,391972,692675,901644,439526\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 308\n", + "Returned IDs: 384245,226961,402411,410779,144827,14996,387031,303843,900283,48961,233879,610787,775231,409606,162005,15636,722203,34147,929245,488766,980380,875600,629956,604832,880707,951560,870429,684523,823352,464663\n", + "Ground Truth: 384245,226961,402411,410779,144827,14996,387031,303843,900283,48961,102347,233879,610787,775231,409606,162005,15636,722203,34147,929245,488766,980380,921461,875600,629956,604832,880707,951560,870429,648896\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 310\n", + "Returned IDs: 933621,931437,348737,239316,336011,707335,287777,198148,75931,843265,700695,79414,503548,665192,907865,852663,826799,149143,448773,382486,574751,315670,218454,781113,344651,360510,963141,185588,387442,213581\n", + "Ground Truth: 933621,931437,348737,239316,336011,707335,287777,198148,75931,843265,700695,79414,503548,665192,907865,852663,826799,149143,448773,382486,574751,315670,218454,781113,650984,344651,360510,963141,185588,387442\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 311\n", + "Returned IDs: 337380,723106,203320,254905,856454,364943,121531,162155,119653,654240,96320,317929,474935,699852,221594,268643,916442,831213,471060,182716,294976,705253,918469,494746,640585,98406,178050,452384,239576,649065\n", + "Ground Truth: 337380,723106,203320,254905,856454,364943,121531,162155,119653,654240,96320,835331,317929,474935,699852,221594,268643,343527,916442,831213,471060,182716,294976,797214,705253,918469,494746,640585,98406,178050\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 315\n", + "Returned IDs: 239383,128883,773096,206657,111287,495544,641352,117884,227884,736311,497630,354691,837580,830496,547909,126822,413976,388284,723416,394112,126059,960,496151,832471,857891,941080,329291,696563,584206,557927\n", + "Ground Truth: 239383,128883,773096,206657,111287,495544,641352,117884,227884,736311,497630,354691,837580,830496,547909,126822,413976,388284,723416,394112,126059,960,496151,832471,857891,941080,329291,170538,696563,584206\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 317\n", + "Returned IDs: 450504,933123,388663,680748,886219,85130,838432,654311,456432,782638,327622,661821,478672,725747,136708,474549,368169,742624,917719,677021,708103,15337,667349,215375,813392,608014,975451,932773,837564,858011\n", + "Ground Truth: 450504,933123,388663,680748,886219,85130,838432,654311,456432,782638,327622,661821,478672,725747,136708,474549,368169,742624,917719,677021,708103,15337,667349,215375,813392,608014,975451,932773,837564,220444\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 319\n", + "Returned IDs: 837169,573129,250702,71625,967674,808442,962685,177548,191597,980565,294658,11218,608505,935366,770725,811589,217776,929284,344873,891095,698037,189859,334081,977414,923678,73944,205276,442698,750801,236498\n", + "Ground Truth: 837169,573129,250702,71625,967674,808442,962685,177548,191597,980565,294658,11218,608505,935366,770725,811589,217776,929284,344873,891095,698037,189859,334081,977414,923678,73944,205276,100596,442698,750801\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 320\n", + "Returned IDs: 98916,201781,301322,521266,514650,139651,403441,133884,115181,984001,648139,139578,376272,136233,646583,825901,289024,786394,584672,507224,864656,643267,984610,652491,738058,236447,343810,889773,607405,152478\n", + "Ground Truth: 98916,201781,301322,521266,514650,139651,403441,133884,115181,984001,648139,139578,376272,136233,646583,883918,576985,825901,289024,372780,786394,584672,507224,663116,864656,790432,643267,984610,652491,738058\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 322\n", + "Returned IDs: 822232,591645,969081,661340,209494,36092,412174,395810,873331,298127,592744,757357,607364,839315,39453,260557,746393,95314,738636,35554,24106,661193,24942,522712,433197,329342,592699,481783,435988,590175\n", + "Ground Truth: 822232,591645,969081,661340,209494,36092,412174,395810,873331,298127,592744,757357,607364,839315,39453,536807,260557,746393,95314,738636,35554,24106,661193,24942,522712,433197,329342,592699,481783,435988\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 327\n", + "Returned IDs: 767449,885140,828699,389380,437908,215330,455659,965241,518808,478494,254863,425749,852216,108367,916453,71814,284658,881621,905726,103307,142435,523912,652541,78224,957855,564508,874820,534158,709387,167766\n", + "Ground Truth: 767449,885140,828699,389380,437908,215330,455659,965241,518808,478494,254863,425749,296333,852216,108367,916453,71814,284658,881621,905726,103307,142435,523912,652541,78224,957855,822116,564508,874820,534158\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 331\n", + "Returned IDs: 632237,673093,949214,124645,42373,237807,270242,799351,67689,634295,187622,82549,334753,941753,651182,702277,894218,390093,98838,205071,388557,742232,485534,260461,981534,330172,91051,943057,724454,637145\n", + "Ground Truth: 632237,673093,949214,124645,42373,237807,270242,799351,67689,634295,187622,82549,334753,941753,651182,702277,894218,390093,98838,205071,388557,742232,485534,138878,260461,981534,330172,91051,943057,724454\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 332\n", + "Returned IDs: 282885,693868,667260,234787,968249,721493,758137,200123,485448,892013,826868,830470,158917,53855,727113,322073,651625,585707,663656,466174,937045,297344,94760,564341,160741,432146,230459,361116,448425,557225\n", + "Ground Truth: 282885,693868,667260,234787,968249,721493,758137,200123,485448,892013,826868,830470,158917,53855,727113,322073,651625,585707,663656,466174,937045,297344,94760,564341,160741,432146,230459,539992,361116,557225\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 341\n", + "Returned IDs: 730945,410821,647584,493725,794387,399586,386701,751012,399191,456044,662268,520429,946367,79167,966329,152703,346235,272675,411241,656090,483212,345910,314834,323552,380085,528136,871179,262431,975141,512847\n", + "Ground Truth: 730945,410821,647584,493725,794387,399586,386701,751012,399191,340297,456044,662268,520429,946367,606680,79167,966329,152703,346235,272675,411241,656090,483212,345910,314834,323552,380085,528136,871179,262431\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 343\n", + "Returned IDs: 58022,657480,701042,214283,105394,903341,352301,850477,542891,616295,676403,462139,868323,872676,690543,858456,102823,99022,708820,872119,539566,964769,325591,527888,829442,589810,5359,471673,186124,208635\n", + "Ground Truth: 58022,657480,701042,214283,105394,903341,352301,850477,542891,616295,676403,462139,868323,872676,690543,711810,858456,102823,99022,395359,708820,467462,872119,539566,964769,325591,527888,829442,589810,5359\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 344\n", + "Returned IDs: 445226,898932,85208,811563,739013,628984,375231,129826,842382,66257,509207,458501,822009,502850,928533,140713,817593,466432,114942,407280,274818,220033,725794,604263,290429,455279,973315,686285,375583,159841\n", + "Ground Truth: 445226,898932,85208,811563,739013,628984,375231,129826,842382,66257,509207,458501,822009,502850,928533,140713,817593,466432,114942,407280,274818,220033,332536,725794,604263,290429,455279,973315,686285,375583\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 345\n", + "Returned IDs: 739324,508813,483222,435482,826827,575673,790711,272952,268164,830086,168200,512346,475336,540275,277690,760494,911158,656653,73815,258966,793111,188545,953485,927482,74123,336137,694136,65854,374227,231749\n", + "Ground Truth: 739324,508813,483222,435482,826827,575673,790711,272952,268164,830086,168200,512346,475336,512136,540275,277690,760494,911158,656653,73815,258966,793111,188545,953485,927482,74123,336137,694136,459113,65854\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 350\n", + "Returned IDs: 811900,131135,246951,294867,949825,766421,332331,276691,942332,380959,344967,753892,218269,777569,235905,572825,844589,105928,456120,803892,731392,324643,577541,790373,261180,868595,179643,496058,378222,195168\n", + "Ground Truth: 811900,131135,246951,294867,949825,766421,332331,276691,942332,380959,344967,753892,218269,777569,235905,572825,765379,844589,105928,456120,900781,803892,731392,324643,577541,790373,261180,868595,179643,684195\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 351\n", + "Returned IDs: 531722,891443,743980,874484,101125,451502,835874,182699,901872,937538,139824,960228,693437,847469,660042,583566,740327,756075,61170,314182,195239,706903,714981,699627,625125,780540,342588,198364,792108,706040\n", + "Ground Truth: 531722,891443,743980,874484,101125,451502,835874,182699,901872,937538,139824,960228,693437,847469,660042,583566,740327,756075,61170,314182,195239,706903,714981,699627,625125,780540,342588,198364,792108,826276\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 353\n", + "Returned IDs: 343757,126830,987029,782305,378307,960649,762415,569317,539257,902508,559917,591964,766292,759732,727936,470722,49849,796785,847070,498090,641317,129500,599221,797139,773040,58603,495926,96142,546285,632573\n", + "Ground Truth: 343757,126830,987029,782305,378307,960649,762415,569317,539257,902508,559917,591964,766292,759732,727936,470722,49849,796785,603380,847070,498090,641317,129500,599221,797139,773040,58603,495926,486069,96142\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 355\n", + "Returned IDs: 454992,971228,752469,565543,648403,338525,839896,911182,957785,738229,127219,831117,430329,745771,850739,586415,807708,74519,139785,309299,684832,955092,384048,904218,444903,198307,147972,454331,665806,919632\n", + "Ground Truth: 454992,971228,752469,565543,648403,338525,839896,911182,957785,738229,127219,831117,430329,745771,850739,586415,807708,74519,139785,309299,684832,955092,384048,904218,349877,444903,198307,147972,454331,665806\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 365\n", + "Returned IDs: 786809,354410,459688,165495,740379,185847,687377,657423,19087,836530,230577,306994,321735,240033,163705,582071,917860,923123,290993,740599,832383,113206,327913,329701,474709,948969,519107,844822,766089,103798\n", + "Ground Truth: 786809,354410,107983,459688,165495,740379,185847,687377,657423,19087,836530,230577,306994,321735,240033,163705,582071,917860,923123,290993,740599,832383,113206,327913,329701,474709,948969,519107,844822,766089\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 366\n", + "Returned IDs: 960797,284658,89677,569346,282336,228656,372230,221141,83471,201723,337086,176155,901146,427971,646154,90210,192320,235787,695974,709822,84932,939629,820232,428677,8504,98246,659932,61093,787285,389746\n", + "Ground Truth: 960797,284658,43676,89677,569346,282336,228656,372230,221141,958462,768780,83471,201723,337086,157504,176155,901146,427971,646154,90210,192320,235787,713082,695974,709822,84932,939629,820232,428677,8504\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 369\n", + "Returned IDs: 432173,937696,950163,493752,819629,758369,467250,29001,21687,887631,775428,136949,301294,717008,263454,275909,846630,256701,218181,479178,291917,908022,934422,309679,359257,536649,305332,312575,835909,183180\n", + "Ground Truth: 432173,937696,950163,493752,819629,758369,467250,29001,21687,887631,775428,136949,301294,717008,263454,275909,846630,256701,218181,479178,291917,908022,934422,463780,309679,359257,536649,305332,312575,835909\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 370\n", + "Returned IDs: 88867,193458,491598,265912,879271,714343,609179,529826,911983,900954,145450,946452,751142,573207,935955,706584,75596,411221,581466,314395,752019,122630,384630,71535,524843,523192,742853,363519,631108,501135\n", + "Ground Truth: 88867,193458,491598,265912,879271,714343,215513,609179,529826,911983,900954,145450,946452,751142,573207,935955,706584,75596,411221,581466,314395,622745,752019,122630,384630,71535,524843,523192,742853,363519\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 379\n", + "Returned IDs: 86504,514427,631493,784430,163109,405390,552180,874891,472710,678867,457983,955007,556378,474544,270853,443261,99964,174808,980355,357003,761932,425585,467508,823980,503161,898306,697029,917942,366780,414037\n", + "Ground Truth: 86504,514427,631493,784430,163109,405390,552180,874891,472710,678867,457983,955007,556378,474544,270853,443261,99964,174808,980355,357003,761932,425585,309502,467508,823980,503161,898306,697029,917942,366780\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 389\n", + "Returned IDs: 922015,498708,319078,732758,377089,778248,818139,406838,566911,687756,957171,633939,284956,662740,368346,740082,425224,916421,423115,861193,589668,178035,580217,931456,715129,645397,389898,104219,401014,488040\n", + "Ground Truth: 197825,570279,922015,370284,689631,498708,319078,732758,778248,377089,818139,406838,52587,566911,286320,687756,957171,633939,284956,662740,368346,169404,740082,158897,425224,104856,916421,906323,423115,861193\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 394\n", + "Returned IDs: 363668,730141,933051,407367,841512,473875,652974,749220,595851,458879,18498,180680,354709,207667,774217,208195,857574,204901,394385,477731,469208,915210,74785,444970,468539,288678,243506,863685,431091,540275\n", + "Ground Truth: 363668,730141,933051,574031,407367,841512,473875,652974,749220,595851,458879,18498,447467,180680,354709,207667,774217,208195,857574,204901,394385,477731,469208,915210,74785,444970,468539,288678,243506,863685\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 395\n", + "Returned IDs: 860195,409212,545052,127016,912266,42313,506672,221215,430877,515449,715529,399786,115024,385484,324710,682639,889792,204538,242172,934778,734589,491424,390137,409731,344930,595238,383075,131932,519334,386668\n", + "Ground Truth: 860195,409212,545052,127016,912266,42313,573515,506672,221215,430877,515449,715529,399786,115024,784018,385484,324710,682639,889792,204538,242172,934778,734589,491424,390137,409731,344930,595238,383075,131932\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 399\n", + "Returned IDs: 5433,925753,675254,442603,202088,529510,183529,739956,13197,317454,226134,396668,407855,890743,949078,800982,279107,208039,333395,894377,55838,160409,733265,34677,666011,235328,357615,414019,785779,159136\n", + "Ground Truth: 5433,925753,675254,442603,202088,529510,183529,183264,739956,13197,317454,226134,266996,396668,407855,890743,949078,800982,279107,208039,333395,894377,55838,160409,733265,34677,666011,235328,357615,414019\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 400\n", + "Returned IDs: 886180,59136,194431,413437,588346,264061,190727,208572,573230,911038,913419,851238,269486,730936,11226,303693,338860,133127,60355,459283,74269,299128,247874,473100,521762,406357,621517,739471,839866,816348\n", + "Ground Truth: 886180,59136,194431,413437,588346,264061,190727,208572,573230,911038,913419,851238,269486,730936,11226,303693,338860,133127,60355,459283,74269,940405,299128,791545,247874,761924,657392,473100,521762,406357\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 401\n", + "Returned IDs: 421107,678267,365194,967592,428004,61808,916328,648958,695456,484345,798397,334990,726567,264360,174820,809112,944672,426440,598833,492725,480828,861695,641629,884539,704564,602017,722548,928454,143774,816097\n", + "Ground Truth: 421107,678267,365194,967592,428004,61808,916328,648958,695456,484345,798397,334990,726567,264360,174820,809112,944672,426440,598833,492725,480828,861695,641629,850247,884539,704564,602017,722548,928454,143774\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 403\n", + "Returned IDs: 762862,837824,552902,466958,856734,204428,144490,55691,489577,687441,359434,690558,958204,231864,514006,846245,631865,972340,678489,293552,561495,568085,780305,611218,186114,224279,858166,715255,824028,633280\n", + "Ground Truth: 762862,837824,552902,466958,856734,204428,144490,55691,489577,687441,359434,690558,958204,231864,514006,846245,631865,761710,972340,678489,600493,293552,561495,568085,780305,955680,611218,186114,77563,224279\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 406\n", + "Returned IDs: 241058,559486,292259,590965,219094,359462,416190,71364,417943,943728,534418,467477,456689,794858,196334,9921,910943,24806,955567,635338,667942,470417,563084,885567,771015,927053,420376,490518,455911,404311\n", + "Ground Truth: 241058,559486,292259,590965,219094,359462,416190,71364,417943,943728,534418,467477,456689,196334,794858,9921,910943,24806,955567,689630,635338,667942,470417,563084,885567,771015,927053,420376,490518,455911\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 410\n", + "Returned IDs: 483056,939858,468302,597887,248125,870144,391452,841741,225472,869078,490155,720764,949852,722707,645834,444305,933004,731060,834945,30492,272542,618518,96336,841244,178160,414620,391429,400541,832605,45733\n", + "Ground Truth: 483056,939858,468302,597887,248125,870144,391452,841741,225472,869078,490155,195173,942723,746432,720764,949852,445843,722707,645834,444305,933004,731060,834945,30492,370780,969033,668189,170527,272542,618518\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 411\n", + "Returned IDs: 144057,465671,238107,195972,844216,317121,77634,870760,634951,707024,103457,428956,276134,547996,174000,989668,962092,367941,144408,760920,658963,429535,810744,137884,534999,649842,137675,661994,863934,824104\n", + "Ground Truth: 144057,465671,238107,195972,844216,317121,77634,870760,634951,707024,103457,428956,276134,547996,174000,989668,962092,367941,144408,405450,760920,526197,658963,429535,810744,382870,137884,534999,892420,649842\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 413\n", + "Returned IDs: 616271,163161,481282,632657,875772,652623,40221,861880,377879,895645,256932,285361,354056,525142,473040,365551,973013,169869,206122,101951,766421,405408,844941,887547,755896,523387,453554,336898,234816,775309\n", + "Ground Truth: 616271,163161,481282,632657,875772,652623,40221,659479,861880,377879,895645,256932,285361,354056,525142,473040,365551,973013,169869,206122,101951,766421,405408,844941,887547,755896,523387,453554,336898,234816\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 419\n", + "Returned IDs: 54669,752279,465775,677556,613525,174186,872820,433279,482135,855417,945054,806221,322664,49561,955291,279855,884755,133853,189597,581740,285383,149650,339721,475508,311167,536633,47301,785589,479031,236359\n", + "Ground Truth: 54669,752279,465775,677556,613525,174186,872820,433279,482135,855417,945054,806221,322664,49561,955291,279855,884755,133853,189597,581740,285383,149650,339721,475508,311167,536633,47301,785589,580339,479031\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 421\n", + "Returned IDs: 658064,637506,816973,787257,901531,766843,121216,906072,366590,632814,285981,286657,513674,127308,98677,348542,385803,789255,415473,823952,575335,890033,716930,365394,919632,986442,516784,32703,483721,603224\n", + "Ground Truth: 658064,637506,816973,950233,787257,284813,901531,882665,766843,121216,906072,155737,366590,632814,285981,286657,889792,513674,127308,98677,179514,348542,330205,385803,789255,415473,823952,575335,579196,890033\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 424\n", + "Returned IDs: 955411,420247,693377,803830,687984,923431,573536,870473,495976,948663,970663,13243,891670,866914,40317,247950,128979,557577,154584,181216,542011,513492,234113,277261,112988,13971,638922,381927,749257,746112\n", + "Ground Truth: 955411,420247,693377,803830,687984,923431,573536,870473,495976,948663,970663,13243,891670,866914,40317,247950,128979,557577,154584,181216,542011,513492,234113,277261,112988,13971,638922,381927,10697,749257\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 430\n", + "Returned IDs: 532151,485140,540992,689703,529529,911792,48597,19807,568714,101501,317406,741193,432677,574335,928590,719345,686873,495938,767175,529999,106638,621503,914907,274736,884640,664005,514158,589352,280356,727308\n", + "Ground Truth: 532151,485140,540992,689703,529529,911792,48597,19807,568714,101501,317406,741193,432677,816606,574335,928590,719345,686873,495938,659238,767175,529999,106638,248184,621503,914907,274736,884640,664005,862488\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 434\n", + "Returned IDs: 747036,194838,321839,932337,695383,838546,656755,595044,847225,772969,82790,75596,107025,577259,192493,515498,830398,242214,967441,813043,977001,750583,245492,508563,841089,643148,51304,184068,882711,761144\n", + "Ground Truth: 747036,194838,321839,932337,695383,838546,656755,595044,847225,772969,82790,75596,107025,577259,192493,515498,830398,242214,967441,813043,977001,750583,245492,508563,841089,643148,207396,51304,184068,882711\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 435\n", + "Returned IDs: 491826,272115,331523,350298,859683,594477,941185,43705,520108,637098,898170,508516,351337,428272,102566,424263,166103,235595,468302,905768,615755,31998,618236,579714,626472,174556,66299,90671,189156,455332\n", + "Ground Truth: 491826,272115,331523,350298,859683,594477,941185,43705,520108,637098,898170,497921,508516,351337,428272,102566,424263,166103,235595,468302,905768,615755,31998,618236,579714,626472,174556,66299,549890,90671\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 438\n", + "Returned IDs: 486553,804703,418755,226022,340972,561653,232975,353165,184985,980428,183477,897794,48386,240024,177299,949159,947579,941252,870852,310766,233432,304965,67883,970615,713581,708519,207411,191103,679145,12772\n", + "Ground Truth: 486553,804703,418755,226022,340972,561653,232975,353165,184985,980428,183477,897794,48386,240024,177299,949159,288275,947579,941252,870852,310766,233432,304965,67883,970615,713581,708519,207411,191103,679145\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 442\n", + "Returned IDs: 585591,431277,430591,295895,601605,163752,155919,842884,976510,783587,262345,558928,421003,372119,51639,290601,726392,755820,864165,649883,534504,595848,385191,393041,934717,940629,119244,967391,516037,498106\n", + "Ground Truth: 585591,431277,430591,295895,601605,163752,155919,842884,976510,783587,715501,262345,652994,558928,421003,372119,51639,290601,726392,755820,864165,649883,801225,550041,447274,534504,595848,385191,393041,934717\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 448\n", + "Returned IDs: 696869,717101,623071,436802,514263,424373,237517,858705,736303,843961,42363,577497,256469,940547,13608,288948,966115,651192,269251,805270,739396,785739,193458,76693,263805,867819,295949,702496,65358,365325\n", + "Ground Truth: 696869,425252,502940,717101,623071,436802,514263,424373,237517,858705,736303,843961,42363,577497,256469,940547,13608,209901,681720,269874,288948,79921,966115,280611,148673,415761,651192,269251,492725,805270\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 450\n", + "Returned IDs: 983071,724424,456983,332855,506429,887374,643408,284708,738400,984087,534604,47459,612881,337691,758671,350257,553530,665167,984480,733527,255792,871574,193833,72256,839198,454898,364833,511221,475846,298337\n", + "Ground Truth: 983071,724424,456983,332855,506429,887374,643408,284708,738400,984087,534604,47459,612881,351862,934,337691,758671,350257,553530,92797,665167,984480,733527,255792,871574,590157,587825,422869,193833,598771\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 453\n", + "Returned IDs: 408183,193282,131850,764157,927555,383063,804375,543818,666074,774870,904547,554042,223829,773383,913535,486000,815866,274290,775550,512802,954208,733669,939770,429278,158316,665432,550036,151512,794003,168153\n", + "Ground Truth: 408183,193282,131850,764157,927555,383063,804375,543818,666074,774870,387184,904547,923016,554042,223829,773383,913535,486000,815866,274290,775550,512802,954208,733669,939770,429278,158316,665432,550036,151512\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 456\n", + "Returned IDs: 451511,555893,781448,311841,744649,319487,115855,571802,972715,591217,226787,153363,432720,768455,147940,656709,211230,398784,350553,118977,635652,597573,530337,384564,772540,329236,739700,570238,821675,905710\n", + "Ground Truth: 778797,677610,84869,278349,621721,799908,960847,802397,380073,325833,739222,857179,521841,544409,131463,75624,451511,729469,170284,6087,919726,555893,393220,887053,838932,746517,312661,781448,589923,505417\n", + "Recall: 0.1000\n", + "\n", + "Query ID: 460\n", + "Returned IDs: 636525,800512,669760,37859,819450,574083,653517,150455,957732,543553,404192,35066,193189,180683,413452,625836,148011,611797,404975,904343,354887,835865,384142,53732,80295,648520,26508,345578,514664,906435\n", + "Ground Truth: 636525,800512,669760,37859,819450,574083,653517,150455,957732,543553,404192,119668,287607,841757,35066,180683,193189,413452,625836,148011,550876,611797,404975,904343,354887,835865,441079,384142,710896,53732\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 463\n", + "Returned IDs: 247433,194491,885321,432883,741791,95329,870074,11112,888697,394921,383317,840004,44927,139201,138924,870190,861923,233816,368880,340238,584011,60154,975451,37948,740446,543250,484486,823996,881703,987087\n", + "Ground Truth: 247433,194491,885321,432883,741791,95329,870074,11112,888697,394921,383317,840004,44927,139201,138924,870190,861923,233816,657085,368880,340238,584011,60154,975451,37948,740446,543250,484486,823996,881703\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 464\n", + "Returned IDs: 960386,16389,563081,779910,34834,426672,438112,295434,262278,243409,829636,506138,291781,337352,342495,766281,711488,77752,22949,808248,218848,766675,693174,776598,135939,696761,963235,829972,662405,956937\n", + "Ground Truth: 960386,16389,563081,779910,34834,426672,438112,295434,262278,243409,829636,506138,291781,337352,342495,766281,711488,77752,22949,808248,218848,766675,693174,776598,135939,709980,696761,963235,829972,662405\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 468\n", + "Returned IDs: 363078,961264,286931,706139,915893,661806,782680,914927,698393,940106,762035,879753,21732,91216,360049,272152,2147,430821,187578,695222,540261,588434,558914,661258,629739,228061,509966,574086,690576,611103\n", + "Ground Truth: 363078,961264,286931,706139,915893,661806,782680,914927,698393,940106,762035,879753,21732,91216,360049,272152,2147,430821,187578,695222,540261,588434,558914,661258,629739,228061,509966,832969,574086,690576\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 472\n", + "Returned IDs: 985727,435690,435503,418392,440155,358487,240287,660519,495054,630829,970563,4571,735953,57204,62785,178798,586993,943743,827684,70875,422139,924809,146733,103691,13467,575626,258446,898405,803312,593914\n", + "Ground Truth: 985727,435690,435503,418392,440155,358487,240287,660519,495054,630829,970563,4571,735953,57204,38043,62785,178798,586993,943743,827684,917538,848547,70875,373270,422139,924809,258304,146733,103691,13467\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 475\n", + "Returned IDs: 199879,919728,619933,146806,678732,776150,280351,974077,576293,312561,970525,972684,51739,866307,694508,577806,73954,469300,926388,624998,405672,520376,548531,89671,313283,33587,128364,955968,510601,431788\n", + "Ground Truth: 199879,919728,619933,146806,257585,678732,776150,280351,974077,576293,312561,970525,972684,51739,866307,694508,577806,73954,469300,926388,624998,405672,520376,548531,66519,89671,313283,33587,729396,128364\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 480\n", + "Returned IDs: 108401,43078,147360,267874,195613,689511,732309,950310,78336,853452,479142,237462,908265,396511,109188,84835,345161,298527,745040,198364,456562,169418,75196,887079,862516,476506,337626,457797,107459,793520\n", + "Ground Truth: 108401,43078,147360,267874,195613,689511,732309,950310,78336,853452,479142,237462,908265,496815,396511,109188,84835,345161,298527,157466,745040,198364,456562,169418,75196,887079,548903,862516,476506,337626\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 481\n", + "Returned IDs: 834350,784817,569607,269487,440334,375702,531315,224642,202811,32946,484680,735628,810928,371225,980818,325885,345184,656961,662167,198441,651001,869392,396984,404308,733171,598132,736813,370104,909039,728512\n", + "Ground Truth: 834350,784817,569607,269487,440334,375702,531315,224642,202811,32946,484680,735628,810928,371225,980818,325885,345184,656961,97915,662167,198441,651001,869392,396984,404308,733171,598132,504160,736813,370104\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 483\n", + "Returned IDs: 972280,939115,633871,438132,409773,852429,324981,688260,586996,258949,135032,143381,339939,860815,550673,470126,454895,509624,227005,360595,904815,982369,109958,625955,372846,306836,416562,331705,245254,766773\n", + "Ground Truth: 972280,613928,939115,633871,438132,409773,852429,324981,688260,586996,258949,135032,143381,701374,339939,860815,550673,470126,454895,509624,227005,360595,904815,982369,109958,625955,372846,306836,416562,331705\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 486\n", + "Returned IDs: 501858,20844,922554,468968,974209,749225,654393,752272,884286,425949,845428,584197,761053,640539,176249,458007,791268,635576,594765,474016,363770,624244,726908,834808,245584,342673,578227,654438,199611,755797\n", + "Ground Truth: 501858,20844,922554,468968,974209,749225,654393,752272,884286,425949,845428,584197,761053,640539,176249,458007,791268,635576,594765,474016,624244,363770,726908,834808,245584,342673,578227,654438,199611,7570\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 496\n", + "Returned IDs: 748940,133107,612315,781246,876393,967452,121517,359569,665954,448757,327784,117150,802913,902050,869123,364432,699852,988464,416115,913858,97135,734292,278713,450347,439861,758258,983841,713718,409254,838551\n", + "Ground Truth: 748940,133107,612315,781246,876393,967452,121517,359569,665954,448757,327784,117150,802913,902050,869123,364432,699852,988464,416115,913858,737680,97135,734292,278713,450347,52520,439861,758258,983841,713718\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 501\n", + "Returned IDs: 456810,626192,563428,66488,969174,448830,670037,501009,232151,368156,535738,891542,462234,973591,448781,302030,923799,797607,74289,405619,877723,123250,485276,256112,307421,117836,205661,774241,397000,850963\n", + "Ground Truth: 334723,20527,99603,633867,655909,456810,57807,626192,563428,66488,969174,448830,670037,501009,232151,368156,535738,576724,891542,462234,973591,448781,433869,302030,923799,797607,224580,74289,405619,877723\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 503\n", + "Returned IDs: 412524,67663,278552,381627,634217,230071,679173,231080,6964,107191,792275,546728,862403,496084,661975,41904,48551,419728,193151,223503,106462,965718,248199,339063,904999,210682,573918,618852,472496,157138\n", + "Ground Truth: 412524,67663,278552,381627,634217,230071,679173,231080,6964,107191,792275,546728,862403,496084,661975,41904,77710,48551,419728,193151,223503,106462,53544,965718,248199,339063,904999,210682,582514,573918\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 508\n", + "Returned IDs: 173670,320036,276625,391244,114940,201839,297111,121190,634951,147221,647839,424505,28762,773965,186907,458279,799727,416538,778260,240051,37503,760829,841319,593142,690969,827551,573558,323205,989990,318399\n", + "Ground Truth: 173670,320036,276625,391244,114940,201839,297111,121190,634951,147221,647839,424505,28762,773965,186907,458279,799727,416538,778260,240051,37503,760829,841319,593142,690969,827551,476277,573558,323205,989990\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 509\n", + "Returned IDs: 73536,866616,485614,867308,683563,153341,948521,75822,192236,287701,461543,776700,361581,836905,930367,504471,533193,145385,60325,967611,440220,317661,895493,75182,546017,340596,830952,890032,464501,160859\n", + "Ground Truth: 73536,866616,485614,867308,683563,153341,948521,75822,192236,287701,461543,776700,361581,836905,930367,504471,505268,533193,145385,357436,389885,684188,60325,967611,183785,440220,317661,895493,75182,546017\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 510\n", + "Returned IDs: 912927,367074,709678,676675,826156,726540,479300,210682,417557,42850,289491,675660,648943,913194,629135,360457,208352,484373,449120,892420,650354,747870,435437,14538,609179,831032,148616,507491,539938,263771\n", + "Ground Truth: 912927,367074,709678,676675,826156,726540,479300,210682,417557,42850,289491,675660,648943,913194,629135,360457,208352,484373,449120,892420,650354,747870,435437,14538,609179,831032,148616,507491,539938,319014\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 511\n", + "Returned IDs: 243024,533494,258122,273207,636547,493509,12598,284323,448843,83798,159073,649263,871889,160963,793703,210682,448353,898936,238933,607731,90023,333234,419428,210476,802330,894597,506870,400660,429646,612219\n", + "Ground Truth: 243024,498493,533494,258122,273207,636547,493509,12598,284323,709690,448843,69844,83798,719321,159073,649263,871889,160963,793703,210682,65385,448353,690130,898936,238933,607731,90023,333234,419428,578772\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 513\n", + "Returned IDs: 896608,481574,668494,549799,185253,554539,84335,418857,363577,277253,325134,739916,18356,302278,179564,494430,909938,917331,189130,66077,753860,953292,837623,667982,498674,318803,559107,310207,345646,528500\n", + "Ground Truth: 896608,481574,668494,549799,185253,554539,84335,418857,363577,277253,325134,739916,18356,302278,179564,477641,494430,909938,917331,189130,66077,753860,953292,837623,667982,498674,318803,310207,559107,345646\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 520\n", + "Returned IDs: 705676,777630,440891,48190,847248,808761,349052,462218,201606,841244,775050,858287,713208,305809,481667,350991,142650,591431,682020,836711,41301,553067,920202,516596,712166,198553,484373,365325,798000,67647\n", + "Ground Truth: 705676,777630,440891,48190,847248,808761,349052,984112,462218,201606,841244,775050,858287,713208,305809,481667,350991,142650,591431,682020,836711,41301,553067,920202,759612,516596,712166,198553,484373,365325\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 521\n", + "Returned IDs: 648033,206367,840722,705308,492651,412515,699560,958647,401430,345406,382299,879707,203481,812738,423893,666952,241203,782920,28660,1848,514917,103811,686286,755584,284504,622367,670049,733905,231238,622124\n", + "Ground Truth: 648033,206367,840722,705308,492651,412515,699560,958647,401430,960037,345406,382299,879707,203481,812738,423893,666952,241203,782920,629849,28660,420344,1848,514917,103811,686286,755584,284504,622367,670049\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 522\n", + "Returned IDs: 822799,824222,610176,970778,679001,700511,418833,961788,762089,491154,501427,947642,369042,38981,946853,69596,493544,304515,639024,405271,734957,821921,730864,766421,755629,761945,444960,687975,438433,692066\n", + "Ground Truth: 822799,824222,610176,970778,679001,700511,418833,961788,762089,491154,501427,947642,369042,38981,946853,69596,493544,304515,639024,405271,734957,821921,730864,766421,755629,761945,409092,444960,687975,438433\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 523\n", + "Returned IDs: 181015,907155,289532,847333,185961,852762,608498,447812,390357,625813,597058,500930,930951,338619,8867,283016,425116,335016,256333,237504,62037,769722,37706,670768,922973,396407,935398,342954,194277,312628\n", + "Ground Truth: 181015,907155,289532,847333,185961,852762,608498,447812,390357,625813,597058,500930,930951,338619,8867,283016,425116,335016,256333,237504,281220,62037,769722,37706,670768,922973,396407,935398,342954,194277\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 527\n", + "Returned IDs: 802668,297384,73158,84228,584965,716356,578242,27601,390522,506914,665525,864668,962441,230866,680802,702104,378115,631377,335433,114942,195997,33995,158176,53622,861098,376749,586381,369898,652719,265323\n", + "Ground Truth: 802668,297384,73158,84228,584965,716356,578242,523551,27601,390522,506914,665525,864668,962441,230866,680802,702104,378115,631377,335433,114942,521214,178610,195997,33995,158176,53622,861098,376749,586381\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 531\n", + "Returned IDs: 90136,139246,845174,270355,300299,971765,124284,772792,80218,684477,480698,868150,492630,220195,534228,17180,650297,937050,808529,606804,26174,1875,304293,250471,416126,370565,721920,539176,322283,461510\n", + "Ground Truth: 90136,139246,845174,270355,300299,971765,124284,772792,80218,684477,480698,868150,492630,220195,534228,17180,650297,224071,937050,808529,606804,26174,1875,304293,416126,250471,370565,721920,539176,322283\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 534\n", + "Returned IDs: 924168,546303,870764,930648,93829,401194,10327,868436,645034,589927,828481,572435,549748,79803,10647,205168,245262,242887,882335,630829,209667,876204,418118,982879,614226,345422,70236,16829,654575,185159\n", + "Ground Truth: 924168,546303,870764,930648,93829,401194,10327,868436,645034,589927,828481,572435,418458,549748,190882,457987,79803,10647,205168,245262,242887,882335,630829,612226,209667,899328,793469,876204,418118,982879\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 543\n", + "Returned IDs: 649308,897930,688672,462870,232769,916039,733663,147884,137043,525841,280116,7358,48613,25518,8974,51689,796185,747782,374092,256889,962202,204536,373824,6307,544151,11937,725629,94850,192877,225759\n", + "Ground Truth: 649308,897930,688672,462870,232769,916039,733663,147884,137043,525841,280116,7358,48613,809691,25518,8974,51689,796185,747782,374092,256889,962202,204536,373824,6307,544151,11937,725629,94850,192877\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 547\n", + "Returned IDs: 597568,172305,223593,173816,450584,806011,940850,291898,449776,83140,166812,496063,631685,410917,972340,157103,297182,61729,704453,552459,953751,208586,307780,854181,968035,211781,883469,628051,242365,720887\n", + "Ground Truth: 597568,172305,223593,173816,450584,806011,940850,291898,449776,167742,83140,166812,496063,631685,410917,972340,868428,157103,297182,809524,61729,704453,552459,953751,208586,307780,854181,968035,211781,883469\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 548\n", + "Returned IDs: 458034,970939,354509,335297,365410,932664,830921,932821,853247,835981,739311,155114,957963,292177,363876,470722,545156,221999,29333,523323,939159,811965,878032,548636,967989,464510,931531,502111,74714,730665\n", + "Ground Truth: 303704,458034,970939,390991,354509,660821,335297,365410,824640,932664,830921,932821,853247,835981,739311,155114,957963,394353,292177,363876,470722,545156,221999,29333,523323,939159,54537,811965,712083,632260\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 551\n", + "Returned IDs: 44387,959746,559984,201607,697817,966254,460054,775061,976042,171151,475559,881865,244718,58297,261833,279683,848242,597228,73160,661561,288546,320815,109022,878634,959478,604881,109453,733538,25479,680991\n", + "Ground Truth: 44387,959746,559984,201607,697817,966254,460054,775061,976042,171151,343629,475559,881865,244718,58297,261833,279683,848242,597228,73160,661561,507593,288546,320815,109022,878634,959478,604881,908965,109453\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 556\n", + "Returned IDs: 379441,622460,299135,130421,794894,908920,686436,408219,734292,533438,950374,888352,906185,649910,699418,313878,87087,808940,97135,967952,656261,886207,239468,161049,70939,925179,280994,970615,613642,357575\n", + "Ground Truth: 379441,622460,299135,130421,794894,908920,686436,408219,734292,533438,950374,888352,906185,649910,919112,699418,313878,87087,808940,97135,967952,656261,886207,239468,161049,70939,925179,280994,970615,613642\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 557\n", + "Returned IDs: 327566,695930,614288,797325,361971,270209,603322,311610,387306,484847,406732,601699,661183,817653,99202,521147,619824,239457,866151,267786,130673,756957,892464,139856,841337,152449,460333,577666,816282,25031\n", + "Ground Truth: 327566,695930,614288,797325,361971,270209,603322,311610,387306,484847,406732,601699,171937,661183,817653,889335,99202,521147,619824,239457,267786,866151,130673,756957,892464,139856,841337,152449,460333,577666\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 562\n", + "Returned IDs: 984988,603416,261457,454735,39522,460875,247502,886983,978631,935038,921853,119978,357252,540143,321744,87323,24380,838549,305954,59068,397874,767866,518055,605351,469087,634008,593672,604611,896427,14108\n", + "Ground Truth: 984988,603416,261457,454735,39522,460875,247502,886983,978631,935038,921853,119978,781367,357252,540143,321744,87323,24380,838549,305954,59068,397874,767866,518055,605351,469087,634008,593672,604611,896427\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 563\n", + "Returned IDs: 989539,39836,724706,675225,563768,726187,486052,381482,304682,422566,27124,518049,589447,796515,746926,475342,847881,644301,858286,49953,60180,468397,784464,177723,94869,591138,546749,567897,346557,838517\n", + "Ground Truth: 989539,39836,724706,675225,563768,726187,173419,486052,381482,304682,422566,27124,518049,589447,796515,746926,475342,847881,644301,68147,858286,49953,417692,60180,351826,468397,784464,244126,177723,827990\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 569\n", + "Returned IDs: 954700,386722,889551,712807,155209,728397,392651,597962,634929,971050,640624,476457,619276,314560,559107,15206,988518,730935,205770,597366,965815,284628,934115,288373,843095,556870,296460,20043,745703,82471\n", + "Ground Truth: 954700,386722,889551,712807,155209,728397,392651,597962,634929,971050,640624,476457,558832,619276,314560,559107,15206,988518,225204,730935,205770,597366,965815,284628,934115,288373,843095,556870,296460,20043\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 574\n", + "Returned IDs: 910249,26540,28611,463070,884945,854530,146515,321172,573544,568015,27124,945323,78851,595362,661244,159421,719664,181709,880189,65464,486818,591539,974762,718513,423943,796185,851794,853373,64583,929713\n", + "Ground Truth: 910249,26540,28611,463070,884945,854530,146515,321172,368959,573544,568015,27124,945323,78851,595362,661244,159421,719664,181709,880189,65464,486818,591539,974762,718513,423943,796185,851794,853373,64583\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 577\n", + "Returned IDs: 418395,705163,168708,920655,315884,211695,404356,135559,4278,752683,673789,298318,146667,231884,978023,427508,152247,649519,842252,439073,479772,869113,476666,440521,747052,522006,506341,148880,501604,33383\n", + "Ground Truth: 418395,705163,168708,920655,315884,211695,404356,135559,4278,752683,673789,298318,146667,231884,978023,427508,152247,649519,842252,439073,479772,869113,476666,440521,747052,774523,522006,506341,148880,501604\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 583\n", + "Returned IDs: 132611,938757,107976,553778,679696,792762,910853,660779,533775,54299,762642,662170,227903,173575,741485,778568,519986,284662,356947,173102,659090,769378,957822,291640,523455,258208,881388,284177,519700,795026\n", + "Ground Truth: 132611,938757,107976,553778,679696,792762,910853,660779,533775,54299,762642,662170,227903,173575,741485,778568,519986,284662,356947,173102,659090,769378,957822,291640,523455,258208,881388,284177,519700,396259\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 585\n", + "Returned IDs: 764654,421754,162750,784327,272909,174661,671188,51285,770707,443840,887103,740409,683471,410935,343032,600421,296510,590738,805963,730735,684905,419368,554537,200172,235900,236837,811483,377245,354495,6854\n", + "Ground Truth: 764654,421754,162750,784327,272909,174661,671188,51285,770707,443840,887103,740409,683471,410935,343032,600421,296510,590738,805963,730735,684905,419368,554537,200172,235900,236837,260217,811483,377245,354495\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 586\n", + "Returned IDs: 655542,12598,318216,732255,621496,758738,866810,350991,730411,206021,844380,955248,926758,267786,545306,957647,884510,99357,661339,210316,581009,84052,67484,508376,97390,936210,221021,679260,62721,882915\n", + "Ground Truth: 655542,295823,12598,318216,732255,621496,758738,941410,866810,350991,730411,780119,206021,844380,955248,926758,267786,545306,339997,385678,957647,183012,884510,99357,661339,749672,210316,581009,84052,67484\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 587\n", + "Returned IDs: 762292,229264,565018,756183,894435,495249,896484,278925,142517,207640,405685,276989,20068,653182,586641,159045,550402,472776,934201,589499,210535,616481,368906,761487,121117,779659,703204,389850,394027,285723\n", + "Ground Truth: 912589,206628,762292,229264,565018,316867,756183,894435,790414,495249,825913,279458,896484,278925,142517,207640,405685,276989,20068,236574,653182,586641,121806,159045,550402,325429,313442,472776,934201,589499\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 596\n", + "Returned IDs: 714092,302591,891305,752708,723197,977488,483024,874171,879813,818058,12747,698243,80386,162341,367516,792275,167915,987644,478695,63813,323205,29666,739095,185931,298527,884847,904999,426054,957009,729643\n", + "Ground Truth: 714092,302591,891305,752708,723197,977488,483024,874171,879813,818058,12747,698243,80386,162341,367516,792275,167915,987644,478695,63813,323205,29666,739095,185931,298527,884847,142117,862403,904999,426054\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 599\n", + "Returned IDs: 453593,865552,138455,503494,734432,895442,654954,626768,257980,251847,420305,732121,336766,436268,770223,629797,19146,763129,571374,859257,592599,142607,278670,285525,787023,924556,232151,315381,287779,100083\n", + "Ground Truth: 453593,865552,138455,503494,734432,895442,654954,626768,405933,257980,251847,420305,732121,336766,436268,770223,629797,19146,7477,139314,763129,571374,859257,592599,676136,142607,278670,230624,285525,787023\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 607\n", + "Returned IDs: 288368,529056,869615,326141,502422,525646,348804,484236,148125,538959,683460,399653,101017,699388,613895,625670,596410,94503,685541,884706,505999,175963,511275,634206,715639,69459,654421,431732,262275,277165\n", + "Ground Truth: 288368,529056,869615,326141,502422,525646,348804,484236,148125,538959,683460,399653,101017,699388,613895,625670,596410,94503,685541,884706,505999,175963,511275,634206,715639,69459,825913,744798,654421,431732\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 615\n", + "Returned IDs: 232787,967342,218747,870068,914492,756720,96109,293334,740027,118868,733840,234787,760623,711725,181772,17058,169936,910647,980751,542774,913306,69773,83621,235716,354600,695856,318319,771088,498903,432146\n", + "Ground Truth: 232787,967342,218747,870068,914492,756720,96109,293334,740027,118868,733840,234787,760623,711725,181772,17058,169936,910647,980751,542774,753259,913306,69773,83621,235716,354600,695856,318319,771088,498903\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 616\n", + "Returned IDs: 352318,360133,115155,816373,451701,154191,686199,895391,202607,744117,85164,706509,495389,52402,99066,800609,109364,466299,285676,139505,237064,188796,526704,338450,691609,627749,801050,911432,324534,351247\n", + "Ground Truth: 352318,360133,115155,816373,451701,154191,686199,895391,202607,744117,85164,190865,706509,495389,52402,99066,800609,109364,466299,285676,139505,237064,188796,526704,338450,691609,627749,801050,911432,477373\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 625\n", + "Returned IDs: 719764,113053,608505,560999,68351,304168,491662,3859,984828,344873,770725,513672,265990,894142,560827,81574,877483,894655,507587,971222,732770,147122,35116,513965,794485,539786,850754,822002,79898,902303\n", + "Ground Truth: 719764,113053,608505,560999,68351,304168,491662,3859,984828,344873,770725,513672,265990,560827,894142,81574,894655,877483,507587,971222,264176,732770,147122,878987,35116,109183,513965,794485,539786,850754\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 626\n", + "Returned IDs: 332676,2431,303519,721561,96207,663677,873768,559273,706033,16903,731633,133528,548289,196690,806111,92420,446863,245480,860236,24749,128291,556123,564534,415640,466782,202015,190651,48759,770331,832804\n", + "Ground Truth: 332676,2431,303519,721561,96207,663677,873768,559273,706033,16903,731633,133528,548289,196690,806111,92420,446863,245480,860236,24749,128291,556123,564534,415640,739381,466782,202015,190651,48759,770331\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 629\n", + "Returned IDs: 877897,512461,686626,750728,434781,301466,846906,824309,904854,79699,796333,887731,872754,677229,327795,455810,520463,698968,900861,596456,669588,712306,241132,973674,456597,605220,187205,380458,16023,807866\n", + "Ground Truth: 17555,877897,512461,686626,750728,434781,301466,227517,846906,824309,904854,79699,796333,887731,872754,677229,327795,455810,520463,698968,231348,900861,596456,669588,712306,241132,973674,456597,605220,187205\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 632\n", + "Returned IDs: 777234,127043,536288,429603,201032,330579,492187,259734,507300,21340,545592,896450,292583,348303,35830,868300,385033,558027,330235,223233,113986,221832,803487,20546,812620,490249,211930,407619,405417,261193\n", + "Ground Truth: 777234,127043,536288,429603,386436,201032,330579,274402,492187,259734,507300,21340,545592,896450,292583,348303,35830,868300,385033,875478,558027,330235,223233,226235,113986,221832,803487,20546,812620,490249\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 634\n", + "Returned IDs: 473915,442150,458317,953670,965457,656372,536726,844751,910960,44107,285036,438661,657368,944499,695456,482695,840458,687724,404079,856526,750702,377137,668705,478559,910898,983474,674724,363676,683168,800528\n", + "Ground Truth: 473915,442150,458317,953670,965457,656372,536726,844751,910960,44107,285036,438661,657368,944499,695456,482695,840458,687724,404079,856526,750702,377137,668705,745052,478559,910898,983474,674724,363676,683168\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 636\n", + "Returned IDs: 521509,361907,439384,182274,956453,256252,710654,351357,914340,905975,348527,195164,217139,284252,574461,139272,958728,798599,909776,511085,267258,845177,814733,659929,319701,358356,823034,629529,866606,618079\n", + "Ground Truth: 521509,361907,439384,182274,956453,256252,710654,351357,914340,905975,348527,195164,217139,284252,574461,139272,933327,958728,798599,909776,511085,267258,845177,823885,814733,86068,659929,319701,358356,835145\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 640\n", + "Returned IDs: 36173,459141,78620,692869,615507,961536,655539,170153,146529,221057,883605,403957,753018,919972,362126,72539,488980,736023,165641,459591,385139,425929,706871,228061,374587,46534,311774,667373,106689,246967\n", + "Ground Truth: 36173,459141,78620,692869,615507,961536,679703,655539,170153,146529,221057,883605,403957,753018,919972,362126,72539,488980,736023,165641,459591,385139,425929,706871,228061,374587,46534,311774,667373,4521\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 647\n", + "Returned IDs: 702662,116638,474928,331942,918064,181466,143364,931832,220498,288579,655077,343314,115615,85956,742221,768712,462420,936312,315542,172823,549884,975217,823587,60933,228976,146733,953477,776671,257168,446452\n", + "Ground Truth: 702662,116638,474928,331942,918064,970675,181466,143364,931832,220498,288579,655077,343314,115615,85956,742221,768712,462420,936312,315542,172823,549884,975217,823587,60933,228976,146733,953477,776671,257168\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 649\n", + "Returned IDs: 471190,571998,865255,989546,69217,418466,98485,758425,179185,873119,518501,815324,620868,721351,326010,775258,959411,686956,613801,512099,489305,776851,753732,447715,205476,342208,579269,774722,435365,209503\n", + "Ground Truth: 471190,571998,865255,989546,69217,418466,98485,758425,127880,179185,873119,518501,815324,620868,721351,326010,775258,959411,686956,613801,512099,489305,381726,776851,753732,447715,205476,342208,579269,774722\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 650\n", + "Returned IDs: 701358,708905,650405,428059,814549,471308,567431,327052,178442,867242,341762,298457,91142,967747,129080,670196,804580,717173,483911,934486,692240,300091,497965,149097,881989,525135,329096,930583,409959,319249\n", + "Ground Truth: 701358,708905,650405,428059,814549,471308,567431,327052,178442,867242,341762,298457,91142,967747,129080,670196,804580,717173,483911,934486,843337,692240,300091,497965,149097,881989,525135,329096,930583,409959\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 652\n", + "Returned IDs: 765060,554017,332654,449120,576907,23358,169493,768517,926990,592652,725673,650354,771865,406838,73954,12598,298118,454204,702922,810382,573673,412330,928945,364039,841089,829356,652407,676675,913105,916625\n", + "Ground Truth: 276864,765060,554017,323348,662344,332654,449120,576907,938322,455898,23358,169493,768517,926990,664042,80791,592652,251821,725673,650354,771865,406838,73954,12598,298118,454204,702922,810382,915898,573673\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 654\n", + "Returned IDs: 222553,763418,472430,853894,75835,160556,575071,536561,512811,269606,248207,499147,372645,776526,927885,598259,500615,418743,904577,544679,495593,236517,866525,375574,638695,275917,159993,236846,873087,384982\n", + "Ground Truth: 222553,763418,472430,853894,75835,160556,575071,536561,512811,269606,248207,499147,372645,776526,927885,598259,500615,418743,904577,544679,495593,236517,866525,375574,638695,275917,159993,236846,873087,887298\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 655\n", + "Returned IDs: 691676,615209,636370,773533,192871,68721,273620,896801,263251,974235,769420,787961,779927,931364,916675,655309,191645,468572,769880,107685,891220,329449,782398,159371,629622,34288,558042,967712,981634,86007\n", + "Ground Truth: 830427,691676,615209,636370,773533,279936,192871,68721,273620,896801,581949,263251,974235,769420,787961,779927,269412,931364,298645,916675,564914,655309,117810,191645,800581,468572,769880,62712,107685,891220\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 656\n", + "Returned IDs: 65162,925920,10979,703539,451952,494027,206871,204958,389412,549368,15695,79697,317270,311572,182204,833875,684222,632680,340559,147802,853740,884332,422655,709017,293808,438153,905619,639640,5376,120638\n", + "Ground Truth: 65162,925920,10979,703539,451952,494027,206871,204958,389412,549368,15695,79697,317270,311572,182204,597866,833875,684222,632680,915601,340559,147802,853740,884332,422655,709017,427347,293808,438153,905619\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 668\n", + "Returned IDs: 36769,238323,389242,699085,703698,807649,741897,300098,159765,610851,205480,440209,215148,394260,851847,730324,865829,715818,22324,448620,984635,374421,610666,544576,309956,647330,735843,580025,143698,195799\n", + "Ground Truth: 36769,238323,389242,699085,703698,807649,741897,300098,159765,610851,205480,440209,215148,394260,851847,730324,865829,715818,22324,448620,984635,374421,610666,544576,309956,647330,735843,580025,143698,976048\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 669\n", + "Returned IDs: 627383,288484,556554,285851,166815,795228,325378,577649,893268,230411,785551,687681,260412,211877,679117,316276,959074,9552,562736,65145,741404,393,134059,981209,972742,786212,824435,649453,642381,506883\n", + "Ground Truth: 627383,288484,556554,285851,166815,795228,325378,577649,893268,230411,785551,687681,260412,211877,679117,316276,959074,9552,562736,65145,741404,393,134059,445806,981209,972742,786212,824435,649453,642381\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 674\n", + "Returned IDs: 700893,556541,570343,12307,499481,50660,726824,72821,577070,387473,966890,930825,460352,100034,176934,318773,116006,867147,937728,444696,656315,757525,756037,449405,651793,341494,225852,137204,294218,154863\n", + "Ground Truth: 700893,556541,570343,12307,499481,50660,726824,72821,577070,387473,966890,930825,460352,100034,176934,318773,116006,867147,937728,444696,656315,551641,757525,756037,449405,651793,341494,225852,137204,294218\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 675\n", + "Returned IDs: 613555,335776,786944,774094,796002,166245,444369,963343,622514,167419,315013,134824,708075,118233,42904,499771,977029,819893,352976,796173,498373,184242,888650,171993,909919,554308,73510,88209,226700,464749\n", + "Ground Truth: 613555,335776,786944,774094,796002,166245,444369,963343,622514,167419,315013,134824,708075,118233,42904,81152,499771,977029,819893,352976,796173,498373,184242,888650,171993,909919,554308,73510,88209,226700\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 676\n", + "Returned IDs: 89910,323244,112854,794176,524533,524117,377137,763094,675203,928219,35066,437969,529687,587330,777278,445426,945557,268512,4206,697828,463424,219094,314367,883115,197649,147562,888781,487822,953365,774332\n", + "Ground Truth: 89910,323244,112854,794176,524533,524117,377137,763094,675203,928219,35066,437969,529687,587330,777278,445426,945557,268512,4206,697828,463424,219094,314367,883115,197649,895451,147562,888781,739431,953365\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 677\n", + "Returned IDs: 293284,301772,47500,361959,860667,796908,452981,780887,615490,379974,965630,617817,636959,610279,896647,708573,544250,603654,465605,648440,605250,267373,159662,440252,54013,818408,232960,280124,828370,899824\n", + "Ground Truth: 293284,301772,47500,361959,860667,796908,452981,780887,33759,615490,379974,965630,617817,636959,610279,896647,708573,544250,603654,465605,648440,605250,267373,159662,440252,54013,818408,232960,280124,828370\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 682\n", + "Returned IDs: 224591,696796,341028,769726,965319,459120,857994,48097,659738,398188,48974,49097,718335,242705,468811,945662,754662,754254,9893,717104,807453,513406,175264,608857,236593,303659,592825,472461,338058,793387\n", + "Ground Truth: 224591,696796,341028,769726,965319,459120,857994,48097,659738,398188,48974,49097,718335,242705,468811,945662,754662,754254,9893,717104,807453,513406,175264,608857,236593,760494,303659,592825,472461,338058\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 683\n", + "Returned IDs: 209231,600973,223287,631231,610241,74037,15876,520706,429214,434647,888851,29264,618495,469865,712974,112274,110620,915523,803117,277402,16959,882699,293939,343140,408330,830565,885712,804087,384323,913679\n", + "Ground Truth: 209231,600973,223287,631231,610241,74037,15876,520706,429214,434647,888851,29264,618495,469865,712974,112274,110620,915523,803117,277402,16959,882699,293939,483995,343140,408330,830565,885712,804087,384323\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 694\n", + "Returned IDs: 725577,818087,822346,367991,266148,3722,918980,32300,817771,310965,840283,610786,873162,861392,374860,687441,97491,672437,479624,155772,967099,579618,693980,148328,531954,44478,378148,14967,504655,187501\n", + "Ground Truth: 725577,818087,822346,367991,266148,3722,918980,32300,817771,310965,840283,610786,873162,861392,374860,687441,97491,672437,479624,155772,22194,967099,579618,693980,148328,531954,44478,378148,14967,504655\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 697\n", + "Returned IDs: 575638,825378,299769,785326,826416,46586,851631,78792,822720,351865,652301,792881,223031,916871,223192,470934,591324,494359,379812,434575,152614,207667,375043,255839,529824,162654,973467,562902,249710,461715\n", + "Ground Truth: 575638,825378,299769,785326,826416,46586,851631,78792,822720,351865,652301,792881,223031,916871,223192,470934,591324,494359,379812,434575,152614,207667,375043,255839,529824,162654,620864,973467,562902,249710\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 702\n", + "Returned IDs: 175509,939242,354296,795126,19330,754458,298766,289376,662268,879630,530206,706040,750373,989691,715670,726578,304112,957337,444672,153735,923163,231399,623756,841307,624981,500352,558878,528445,255247,376480\n", + "Ground Truth: 677492,175509,390929,939242,481000,737464,109604,889256,354296,795126,19330,754458,888933,298766,289376,766918,662268,858528,879630,530206,735023,670382,706040,750373,989691,972814,715670,726578,304112,957337\n", + "Recall: 0.6000\n", + "\n", + "Query ID: 703\n", + "Returned IDs: 636314,940641,533504,965375,266999,649063,30646,32280,294667,552225,95939,319719,307750,827319,826315,131374,36393,902121,889385,987792,968905,582069,267825,228729,290247,103604,438894,92807,479606,503115\n", + "Ground Truth: 636314,940641,533504,965375,266999,649063,30646,32280,294667,552225,95939,319719,307750,827319,826315,131374,36393,902121,889385,987792,968905,582069,267825,76413,228729,823507,3476,290247,103604,438894\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 704\n", + "Returned IDs: 301193,984,108353,784406,228331,897008,596596,500036,816097,978234,88702,128177,581195,820777,296678,330740,126521,245492,399372,211312,845681,210682,449120,515498,31899,12598,238933,737227,865827,73123\n", + "Ground Truth: 301193,984,108353,784406,228331,897008,596596,500036,816097,978234,88702,128177,65853,581195,820777,296678,330740,126521,245492,399372,211312,845681,210682,449120,515498,31899,12598,238933,737227,865827\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 708\n", + "Returned IDs: 597881,361507,136949,443568,60682,988016,899186,538897,499396,709738,403311,804689,721697,128967,147981,981043,805840,471919,375722,726748,674133,328044,893662,347241,827328,102935,939159,60315,413816,230772\n", + "Ground Truth: 597881,361507,136949,443568,60682,988016,899186,538897,499396,709738,403311,804689,721697,128967,147981,981043,805840,471919,375722,399816,726748,674133,328044,893662,347241,827328,102935,939159,60315,413816\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 709\n", + "Returned IDs: 742278,467067,738627,168219,488406,578858,403026,281311,243670,302478,298411,67129,859699,870334,523618,511267,210,408339,384481,814833,581945,952618,822040,473263,183618,984594,11562,627089,518825,424696\n", + "Ground Truth: 742278,467067,738627,168219,488406,578858,403026,281311,243670,302478,298411,67129,859699,870334,523618,511267,210,408339,384481,814833,581945,952618,822040,473263,183618,498319,99645,984594,11562,627089\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 710\n", + "Returned IDs: 686971,195239,779394,881194,738079,273085,184135,153018,174068,411675,121722,854362,162559,565851,189815,828591,930057,170602,432286,56891,698111,692622,831597,909783,144205,105119,344839,649883,651512,545484\n", + "Ground Truth: 686971,195239,779394,881194,738079,273085,184135,153018,174068,411675,121722,854362,162559,565851,189815,828591,930057,170602,432286,56891,698111,564018,692622,831597,909783,144205,105119,344839,649883,651512\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 713\n", + "Returned IDs: 676059,545592,492187,549626,381396,766391,464843,26181,384188,294588,25377,919793,685447,149797,103791,970525,677600,663314,952353,115085,142665,654964,871476,603754,802643,291056,472281,20546,627873,898405\n", + "Ground Truth: 676059,545592,492187,549626,381396,766391,464843,26181,384188,294588,25377,919793,685447,149797,103791,970525,677600,663314,952353,115085,142665,654964,871476,603754,802643,240392,291056,472281,20546,627873\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 718\n", + "Returned IDs: 790282,814687,315140,464103,21514,170631,568936,706033,898715,613177,550866,408364,304660,271179,940528,920234,374740,406974,746969,200589,154567,72059,767663,22807,342918,366912,869141,29668,353012,854130\n", + "Ground Truth: 790282,814687,315140,464103,21514,170631,568936,706033,898715,613177,550866,794944,408364,304660,271179,940528,920234,374740,406974,746969,200589,154567,72059,767663,22807,342918,366912,869141,29668,353012\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 719\n", + "Returned IDs: 830162,979459,712306,389233,72342,585104,621678,413354,897289,747036,415762,631020,781808,861752,553067,73922,691555,517,50336,92921,560682,950759,582524,487080,322629,542022,159701,472768,335658,309342\n", + "Ground Truth: 830162,979459,53339,712306,389233,72342,585104,621678,413354,897289,869371,747036,415762,631020,781808,861752,553067,73922,691555,517,50336,92921,560682,950759,582524,487080,322629,542022,159701,472768\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 724\n", + "Returned IDs: 736959,828468,322656,94276,392750,20817,254353,953696,839079,954538,774524,515426,265671,118529,595157,866402,141923,368299,612063,814032,103536,713715,716189,283906,961247,190583,352849,433794,437760,861095\n", + "Ground Truth: 736959,828468,322656,94276,392750,20817,254353,953696,839079,954538,774524,515426,265671,118529,595157,866402,141923,368299,780956,612063,814032,103536,713715,716189,283906,961247,190583,352849,433794,437760\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 726\n", + "Returned IDs: 278297,912839,418744,772798,841934,110169,860568,359811,295939,18712,42569,804528,876551,93720,949221,104666,338906,573573,160060,569938,925035,655436,759313,840924,519673,960815,801692,252820,282028,653156\n", + "Ground Truth: 278297,912839,418744,772798,841934,26444,194724,110169,860568,359811,295939,18712,42569,804528,876551,93720,949221,104666,338906,573573,160060,569938,925035,655436,759313,837145,840924,519673,960815,305936\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 727\n", + "Returned IDs: 601774,919221,701625,224708,377205,603351,560588,701270,498240,380057,176743,931092,708095,824681,184615,566485,304274,253247,186079,979201,729526,73156,615473,750693,539379,982531,462519,250323,744732,716700\n", + "Ground Truth: 601774,543047,6458,767951,919221,701625,513333,828351,224708,377205,603351,395045,560588,701270,498240,364080,929775,774847,380057,682285,151959,692258,360352,176743,931092,708095,824681,184615,566485,304274\n", + "Recall: 0.5667\n", + "\n", + "Query ID: 728\n", + "Returned IDs: 211374,243658,362896,184385,796186,947541,703121,739700,715073,524003,221832,251280,665789,722430,253578,535822,733030,210238,342000,642,479116,601278,14190,443660,74806,238387,3470,730875,282892,689925\n", + "Ground Truth: 211374,243658,362896,184385,796186,947541,703121,739700,715073,524003,221832,251280,665789,722430,253578,535822,733030,210238,342000,642,479116,601278,14190,443660,74806,238387,3470,730875,282892,767332\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 731\n", + "Returned IDs: 902199,909639,132657,389154,466750,927245,954856,585781,379870,496815,23358,716890,625443,484722,806183,702922,328017,377137,662928,770707,700848,216560,231038,40824,130662,35066,525844,229519,73954,351337\n", + "Ground Truth: 902199,909639,132657,389154,466750,927245,954856,585781,379870,496815,131975,23358,716890,625443,484722,806183,702922,328017,377137,662928,770707,700848,216560,231038,40824,796801,130662,35066,525844,229519\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 733\n", + "Returned IDs: 630397,896650,273814,776952,278338,755697,456812,34635,327310,263372,184742,175060,443576,887994,499493,359243,974206,213340,451926,484597,770244,440607,81274,315178,344342,689283,911154,579178,980279,154600\n", + "Ground Truth: 630397,896650,273814,776952,278338,755697,456812,34635,327310,263372,184742,175060,443576,887994,499493,359243,974206,213340,451926,484597,770244,440607,81274,315178,840707,344342,689283,911154,579178,980279\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 735\n", + "Returned IDs: 480693,402811,648165,872301,564992,618076,31416,978420,84884,32240,880207,151245,805104,410882,540261,624883,183142,483973,445080,165641,904884,747337,456805,781453,970573,578011,717309,171861,473385,923245\n", + "Ground Truth: 480693,402811,648165,872301,350437,564992,618076,31416,978420,84884,32240,880207,151245,380320,805104,410882,540261,985164,624883,183142,483973,445080,165641,904884,747337,456805,781453,970573,578011,717309\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 737\n", + "Returned IDs: 619645,978813,849107,363318,373296,578366,526266,397535,316557,320524,32830,773290,164276,941783,876608,488006,314510,915139,278987,278268,95808,716348,38383,526494,41549,11227,491354,207375,832248,591719\n", + "Ground Truth: 619645,978813,849107,363318,373296,578366,526266,397535,316557,320524,32830,773290,164276,941783,876608,488006,314510,915139,507339,278987,278268,95808,716348,38383,526494,41549,11227,491354,207375,832248\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 740\n", + "Returned IDs: 644677,891474,349561,774893,794841,64191,733259,829783,763405,953325,677775,705630,418109,479989,209740,217537,564518,607057,486623,747314,721929,981577,111631,919042,587366,498821,51013,42015,278645,748676\n", + "Ground Truth: 644677,891474,532672,349561,774893,794841,502470,64191,733259,829783,763405,953325,514869,677775,705630,418109,479989,209740,480196,217537,564518,607057,937959,486623,611245,44242,747314,721929,889407,755251\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 741\n", + "Returned IDs: 271387,968410,306063,589936,339210,410226,12606,243823,601108,935175,971042,210059,885939,390173,85216,984828,539544,770725,668418,647269,131865,90375,472469,831453,239553,309695,267368,954125,265258,973599\n", + "Ground Truth: 271387,968410,306063,589936,339210,410226,12606,243823,601108,935175,281495,971042,210059,885939,390173,85216,984828,539544,770725,668418,647269,131865,90375,472469,831453,239553,309695,267368,954125,265258\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 744\n", + "Returned IDs: 851004,857095,892694,97241,40212,932870,510047,791959,193388,499906,921294,903712,556947,360738,619047,916571,733853,566261,528791,772820,3673,82794,636316,383259,342322,184811,928945,655764,615678,801878\n", + "Ground Truth: 851004,857095,892694,124754,97241,40212,932870,758010,510047,791959,691462,193388,499906,921294,903712,556947,222908,360738,619047,916571,280588,733853,566261,528791,772820,808467,13797,3673,82794,287544\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 747\n", + "Returned IDs: 613943,339708,786275,982988,428450,483553,280929,166823,830267,945999,530979,182212,503642,236554,22639,115602,744769,628546,313876,130203,104243,782695,191682,103607,121382,543216,935072,700089,463626,636683\n", + "Ground Truth: 613943,339708,786275,982988,428450,483553,280929,166823,830267,945999,530979,17394,182212,503642,236554,22639,115602,744769,628546,313876,130203,104243,782695,191682,103607,121382,543216,935072,700089,463626\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 749\n", + "Returned IDs: 570139,225335,78583,652024,111444,80259,70291,59793,921997,891344,605901,228529,906731,844582,251964,330063,499660,347906,779669,101505,138604,444003,361722,798355,528017,964169,302405,898868,727316,685447\n", + "Ground Truth: 570139,225335,78583,652024,111444,80259,70291,59793,921997,891344,605901,228529,906731,844582,251964,330063,499660,347906,779669,101505,138604,444003,361722,798355,528017,964169,302405,898868,672389,727316\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 754\n", + "Returned IDs: 474090,679777,568149,510076,271497,503519,469435,616568,717309,811349,165917,440388,938757,46156,236773,47625,914513,341037,273453,529687,871971,925668,545059,516390,794504,763094,133008,164462,166103,54841\n", + "Ground Truth: 474090,679777,568149,510076,271497,503519,469435,616568,717309,811349,165917,440388,938757,46156,236773,47625,653985,914513,857089,341037,273453,529687,378707,871971,925668,545059,299934,516390,794504,763094\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 755\n", + "Returned IDs: 307679,470060,615497,106018,687262,818104,927223,286791,960201,326060,834065,954929,4506,69809,871996,61080,228250,745388,813043,835919,382461,706276,921482,9781,140020,3186,694472,225107,734889,208066\n", + "Ground Truth: 307679,470060,615497,106018,687262,818104,927223,286791,960201,326060,834065,954929,4506,69809,871996,61080,228250,745388,813043,835919,382461,706276,921482,9781,140020,3186,694472,225107,734889,654575\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 761\n", + "Returned IDs: 627166,531403,258376,532356,98284,885327,56572,537506,626871,558981,346235,398253,207492,58129,137077,252624,489659,111158,588359,924066,775978,946358,340748,187448,721973,659530,301732,875753,197245,670856\n", + "Ground Truth: 627166,531403,258376,569533,532356,98284,942384,885327,56572,657497,537506,626871,808429,541292,558981,346235,398253,21698,207492,58129,137077,252624,489659,111158,588359,924066,370948,775978,946358,340748\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 762\n", + "Returned IDs: 784939,461074,351498,625820,24863,863971,43064,213446,239933,176805,175762,705205,253645,888814,964626,938520,876957,100857,952010,27601,84404,948969,86096,550389,897937,43080,828262,750632,695039,670909\n", + "Ground Truth: 784939,461074,351498,625820,24863,863971,43064,213446,239933,176805,175762,705205,253645,888814,964626,938520,876957,100857,952010,27601,84404,948969,86096,550389,897937,43080,867096,828262,750632,695039\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 763\n", + "Returned IDs: 728291,170403,107366,807709,601595,882271,616910,165224,437651,641012,759375,718836,383566,451966,272660,313221,367539,576729,956084,186113,641310,352325,577985,871709,267329,696373,146447,769725,705677,934632\n", + "Ground Truth: 728291,170403,107366,807709,601595,882271,616910,165224,437651,641012,718836,759375,383566,451966,272660,313221,367539,576729,956084,186113,641310,352325,577985,871709,267329,696373,146447,769725,705677,93476\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 770\n", + "Returned IDs: 818526,881410,749004,612005,480015,275570,282418,961282,916592,151899,854592,148074,241932,298210,679039,437268,371719,110627,568129,217858,209770,375362,109399,827256,498657,303177,719210,118962,664865,692679\n", + "Ground Truth: 818526,881410,749004,612005,480015,275570,282418,961282,916592,151899,854592,148074,241932,298210,679039,718744,437268,371719,110627,568129,217858,209770,375362,783711,109399,827256,498657,303177,719210,118962\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 771\n", + "Returned IDs: 612125,898912,716094,383459,157433,127418,50112,620959,571637,120409,830429,189432,478435,943426,754533,651381,692932,271381,843940,956172,141189,603949,259754,534494,281421,986047,671041,32659,457158,929703\n", + "Ground Truth: 612125,898912,716094,383459,157433,127418,50112,620959,571637,120409,830429,189432,478435,943426,754533,651381,692932,271381,843940,956172,141189,603949,259754,825694,534494,281421,986047,671041,32659,457158\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 773\n", + "Returned IDs: 135196,679058,578737,393271,709277,75402,759984,582920,364950,442311,788828,448179,526128,131127,623519,725627,942813,401402,431595,332810,772035,654528,609612,441622,162588,430176,67815,67136,30211,130172\n", + "Ground Truth: 135196,679058,578737,831148,393271,709277,75402,759984,582920,364950,442311,788828,448179,526128,131127,623519,725627,700863,942813,401402,431595,332810,772035,654528,609612,441622,162588,430176,67815,67136\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 776\n", + "Returned IDs: 110655,667266,289780,220173,720740,450538,960809,577277,320061,605939,22762,47736,409437,749867,724789,344823,341596,2617,288348,859918,525662,633080,741541,565220,975584,385827,46680,811782,561974,187087\n", + "Ground Truth: 110655,667266,289780,220173,720740,543580,450538,960809,577277,320061,605939,22762,47736,409437,749867,724789,344823,341596,2617,288348,859918,525662,633080,792682,741541,565220,975584,385827,46680,945123\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 783\n", + "Returned IDs: 922832,228863,28218,904559,595252,309804,373874,938233,606036,915445,504696,529811,704840,119798,331859,514098,281209,810927,239343,815065,49107,878754,468154,7571,171231,627067,388301,904238,57289,371657\n", + "Ground Truth: 922832,228863,28218,904559,595252,309804,373874,938233,606036,915445,504696,529811,704840,119798,331859,514098,281209,810927,239343,815065,49107,878754,468154,7571,171231,627067,388301,904238,57289,249244\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 788\n", + "Returned IDs: 620050,169755,82839,66991,374712,54680,250177,16579,461321,166695,973973,253332,973705,683807,357392,36527,350669,976687,893704,541044,145167,136032,248904,368717,909390,896505,327192,339786,948812,651429\n", + "Ground Truth: 620050,169755,82839,66991,374712,54680,250177,16579,461321,166695,973973,253332,973705,683807,357392,36527,350669,976687,893704,541044,145167,136032,248904,368717,909390,62168,896505,327192,720442,172640\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 790\n", + "Returned IDs: 494590,697069,476593,887583,68191,872474,147991,588273,446442,507095,189013,157715,662847,80487,892532,772796,152760,529135,846761,229519,498210,314279,728954,415574,545622,130604,155325,103827,726975,85881\n", + "Ground Truth: 494590,697069,476593,887583,68191,872474,147991,588273,446442,507095,189013,157715,472984,662847,80487,892532,307474,772796,152760,371361,529135,846761,250721,229519,498210,314279,527617,728954,415574,486364\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 791\n", + "Returned IDs: 643044,188503,950183,577547,233840,31025,85110,712573,263885,542409,94912,264481,808395,388584,647249,236796,257063,618839,11267,136482,470696,614936,973190,731983,950404,357520,107926,907974,138633,696958\n", + "Ground Truth: 643044,188503,950183,577547,233840,31025,85110,712573,263885,542409,94912,264481,808395,388584,647249,236796,257063,618839,1657,11267,136482,470696,614936,973190,731983,950404,357520,107926,907974,138633\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 792\n", + "Returned IDs: 103643,982841,76876,131614,304402,277688,233237,491898,472234,30128,791723,42844,818761,941100,8634,167210,676258,429624,944061,31446,811673,902018,64760,904442,824056,251243,237202,755251,190147,170414\n", + "Ground Truth: 103643,982841,76876,131614,304402,277688,233237,491898,472234,30128,791723,42844,532680,818761,941100,8634,167210,676258,429624,944061,31446,811673,902018,64760,904442,943067,353268,824056,517818,251243\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 800\n", + "Returned IDs: 443449,620473,127834,41146,343110,471608,71956,490155,820327,821507,19367,185266,359362,281247,933588,355884,182247,445843,901468,181412,879503,276481,965996,305809,394351,639169,385763,350991,486245,889335\n", + "Ground Truth: 443449,949216,740228,620473,918058,127834,323173,556530,41146,643603,234730,343110,471608,71956,727643,490155,669870,240726,820327,821507,19367,185266,359362,281247,933588,355884,182247,445843,901468,181412\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 802\n", + "Returned IDs: 269238,839967,412865,131067,822382,950900,585314,579302,554825,191077,310058,932573,465453,384598,163712,1915,629445,746003,847419,280836,825647,131531,592089,416435,66962,436819,466040,866614,747061,963898\n", + "Ground Truth: 269238,839967,412865,131067,358477,822382,950900,585314,579302,554825,191077,310058,932573,465453,384598,163712,1915,629445,746003,847419,280836,825647,131531,592089,416435,66962,436819,466040,866614,747061\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 807\n", + "Returned IDs: 210412,673161,344470,308984,799197,718452,539406,179708,802206,289930,839736,802277,649766,467441,823345,988298,55655,114623,317686,258591,501445,835892,464190,126328,620445,871971,883115,239245,975496,182270\n", + "Ground Truth: 210412,673161,344470,308984,799197,718452,539406,179708,802206,289930,839736,802277,649766,467441,823345,988298,417489,55655,114623,317686,258591,501445,835892,464190,126328,620445,871971,883115,239245,975496\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 812\n", + "Returned IDs: 420249,798562,125873,643276,430038,842812,471074,154843,874840,215560,469558,225999,607434,739129,481972,670727,11887,593669,528622,243340,473317,71885,486258,550160,735749,536763,691518,763904,774817,265727\n", + "Ground Truth: 420249,798562,125873,643276,430038,842812,471074,154843,874840,215560,469558,225999,607434,739129,481972,670727,11887,593669,894709,528622,243340,473317,71885,486258,550160,735749,536763,691518,763904,774817\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 814\n", + "Returned IDs: 892069,177321,140068,857569,864412,606617,253225,957570,184938,922172,811345,561074,831712,267584,539386,103717,595721,654658,530470,528534,712709,215663,730549,690319,703326,815253,432827,531675,520174,296938\n", + "Ground Truth: 892069,177321,140068,857569,864412,606617,253225,957570,184938,922172,811345,561074,831712,267584,539386,517770,103717,595721,654658,530470,528534,712709,215663,730549,690319,703326,815253,432827,535277,50556\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 815\n", + "Returned IDs: 219636,796436,78945,187548,915690,128205,302140,926385,680887,325681,450132,652363,954356,168702,67813,736204,348106,916780,246489,114519,125604,247594,865052,458563,526409,212182,479989,608771,382840,144283\n", + "Ground Truth: 219636,796436,78945,187548,915690,128205,302140,926385,680887,325681,450132,652363,954356,168702,67813,736204,348106,916780,246489,114519,125604,247594,865052,458563,526409,116390,212182,479989,608771,382840\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 818\n", + "Returned IDs: 222540,33088,296686,64424,29927,315322,829911,191867,5408,114079,954172,515763,586692,211249,467644,318000,714661,141461,232220,454204,343807,928945,23358,561633,677469,176925,354452,429522,759195,800775\n", + "Ground Truth: 222540,33088,296686,64424,29927,315322,829911,191867,5408,99589,114079,448769,954172,515763,586692,734864,211249,845749,467644,318000,714661,141461,298732,185773,232220,647478,454204,343807,928945,23358\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 832\n", + "Returned IDs: 369470,329052,948873,268162,223905,444438,523049,824460,418108,269367,150812,537834,725366,777799,181325,132671,641287,210095,724728,866770,237932,123832,229519,78224,61754,498376,898787,635318,160372,494693\n", + "Ground Truth: 369470,329052,948873,268162,223905,444438,523049,824460,418108,269367,150812,537834,725366,777799,181325,132671,641287,210095,724728,866770,237932,123832,229519,78224,61754,498376,898787,635318,182737,160372\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 837\n", + "Returned IDs: 450812,130298,102438,835119,751927,395640,620999,940878,763690,2345,527174,651936,836956,875904,834742,820352,285382,635290,533415,469340,822949,811872,544545,599988,449898,713410,238236,443785,260760,50609\n", + "Ground Truth: 450812,130298,102438,835119,751927,395640,620999,940878,763690,2345,527174,651936,536651,836956,875904,834742,820352,285382,708115,635290,533415,469340,822949,811872,544545,599988,449898,74019,713410,238236\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 849\n", + "Returned IDs: 9293,914011,255786,118357,222750,984794,566228,125512,115486,102584,509468,910639,519952,33227,658160,810463,389195,931801,359040,775261,821526,336013,378165,816470,300902,468424,90349,730306,67680,486997\n", + "Ground Truth: 9293,914011,255786,118357,222750,984794,566228,125512,115486,102584,509468,910639,519952,649943,832134,348170,33227,658160,810463,389195,931801,359040,775261,821526,659257,987729,336013,378165,816470,300902\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 855\n", + "Returned IDs: 727768,38546,291074,824962,217368,667688,843153,792223,511581,879825,546513,452628,930813,540288,119630,744117,231332,315913,445527,5575,574591,735598,845648,47536,671877,466719,169049,813947,744482,344072\n", + "Ground Truth: 727768,38546,291074,824962,217368,667688,843153,792223,511581,879825,546513,289972,452628,930813,540288,119630,744117,231332,315913,445527,5575,574591,735598,845648,47536,671877,466719,169049,813947,744482\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 856\n", + "Returned IDs: 907401,132176,295647,439880,606751,253645,162089,597394,159951,444369,786944,803865,61450,962635,175223,892285,70875,148756,931325,929698,819893,684317,759121,613491,220032,705991,304135,114363,832079,305575\n", + "Ground Truth: 907401,132176,295647,439880,606751,253645,162089,597394,159951,444369,786944,803865,61450,962635,175223,892285,70875,148756,931325,929698,819893,563865,615450,684317,759121,613491,220032,705991,304135,114363\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 861\n", + "Returned IDs: 869826,474900,848283,618192,658666,659128,834327,559216,734225,628304,17384,851425,623076,788745,962803,737465,79528,25982,342870,837641,421516,364319,263244,836487,688286,85458,94527,437455,799112,306268\n", + "Ground Truth: 869826,474900,848283,618192,658666,659128,834327,559216,734225,628304,17384,851425,623076,788745,962803,737465,79528,25982,342870,837641,421516,364319,263244,836487,637338,688286,85458,94527,437455,799112\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 862\n", + "Returned IDs: 112384,713861,459285,15666,94949,933618,882115,696932,979927,193458,805963,552361,556479,106462,143774,504182,944672,775181,790742,688726,786459,789100,295659,35132,651167,171418,473307,620860,755560,570790\n", + "Ground Truth: 733007,366573,112384,713861,459285,15666,94949,962134,619884,933618,882115,455657,696932,358398,246616,979927,333155,193458,805963,562510,552361,556479,134250,106462,143774,504182,944672,263749,775181,790742\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 864\n", + "Returned IDs: 507385,357775,756700,773062,558848,347884,77788,573479,838660,305083,943482,7853,650687,485809,84884,602759,823654,503411,497168,840305,311608,749429,29497,962118,437293,54512,896338,170922,448907,928428\n", + "Ground Truth: 507385,357775,756700,773062,558848,347884,77788,573479,838660,305083,943482,7853,650687,485809,84884,602759,823654,503411,497168,840305,311608,749429,29497,962118,437293,54512,896338,767632,170922,448907\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 870\n", + "Returned IDs: 823215,354345,187553,486944,282325,440122,499261,941006,707218,606988,766728,955785,918106,950285,984194,809763,957443,156336,870409,402964,367079,856449,722063,805290,918560,431744,818574,151167,289811,283616\n", + "Ground Truth: 823215,354345,187553,486944,282325,440122,499261,941006,707218,606988,766728,955785,918106,950285,984194,809763,957443,156336,870409,402964,367079,583107,856449,722063,805290,918560,431744,818574,151167,289811\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 871\n", + "Returned IDs: 244934,804071,160305,306125,497677,733053,850316,368551,111568,955585,349793,29618,706362,313207,98518,17190,200458,615023,715565,281070,469308,757543,875443,265956,586420,686531,448971,301536,897559,66043\n", + "Ground Truth: 244934,804071,160305,306125,497677,733053,850316,368551,111568,955585,349793,29618,706362,313207,98518,17190,153120,200458,615023,715565,281070,469308,757543,875443,913080,265956,586420,686531,448971,301536\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 874\n", + "Returned IDs: 144273,17885,816961,438367,977378,188338,672356,252379,92322,865856,522130,446162,710709,569951,422187,306829,667281,971729,680755,671449,211801,98623,605305,197364,305854,84040,881297,900470,778759,921305\n", + "Ground Truth: 144273,17885,816961,438367,977378,188338,672356,252379,92322,865856,522130,446162,710709,569951,422187,306829,667281,971729,680755,671449,211801,98623,605305,197364,305854,84040,881297,307139,900470,778759\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 875\n", + "Returned IDs: 377928,780788,325285,679329,415016,144490,257622,170538,278959,349444,808595,575433,48585,401386,37541,572088,747586,429263,98656,557400,32360,440482,967160,226652,29700,956999,592174,512815,496751,448378\n", + "Ground Truth: 377928,780788,325285,679329,415016,144490,257622,170538,278959,349444,808595,575433,48585,401386,37541,572088,747586,429263,98656,557400,32360,440482,967160,226652,29700,956999,592174,389824,512815,448378\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 877\n", + "Returned IDs: 407647,249140,459302,850162,469630,194461,790570,482938,284634,555877,249284,24771,865657,599676,614534,311462,716601,797290,611597,848134,467919,836540,621475,512998,339185,209362,341330,285864,739503,621195\n", + "Ground Truth: 407647,249140,459302,850162,469630,194461,790570,482938,284634,555877,249284,24771,865657,599676,614534,311462,716601,797290,611597,848134,467919,836540,621475,512998,339185,209362,497750,659824,571697,341330\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 883\n", + "Returned IDs: 444691,49388,601127,48679,430106,296236,870285,684431,636192,820180,228001,277333,319640,104341,736962,962007,980923,739095,2987,867529,336061,484857,136705,818456,81881,859033,180709,933777,962246,290435\n", + "Ground Truth: 444691,49388,601127,48679,430106,296236,870285,684431,636192,820180,228001,277333,319640,104341,736962,962007,980923,739095,2987,867529,336061,484857,136705,435183,818456,957369,81881,859033,180709,933777\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 885\n", + "Returned IDs: 513527,680643,126652,854583,870236,144177,118873,641713,987862,688076,667176,761486,738195,13941,613227,681062,137605,638786,594397,599817,183667,914605,212925,807621,794223,544626,111263,751106,186388,206926\n", + "Ground Truth: 513527,680643,126652,854583,870236,144177,118873,641713,987862,688076,667176,761486,738195,13941,613227,681062,258966,137605,638786,382421,594397,599817,183667,914605,212925,807621,481246,794223,464320,544626\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 894\n", + "Returned IDs: 872922,723485,810343,308304,635570,431480,339438,271857,334997,594990,71335,673035,795893,953349,760211,568152,49725,482827,516060,690408,391643,929962,669591,363620,418983,18362,978874,821331,253215,713388\n", + "Ground Truth: 872922,723485,810343,308304,635570,431480,339438,271857,334997,594990,71335,673035,795893,953349,760211,568152,49725,482827,516060,690408,391643,929962,669591,363620,418983,18362,978874,821331,10232,253215\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 898\n", + "Returned IDs: 692011,34657,467206,382450,361534,753431,517697,608214,727954,473726,649883,970534,929793,799647,989054,602764,136828,398228,606854,954673,271506,476614,91378,969987,705913,341158,373102,137710,428750,364147\n", + "Ground Truth: 692011,322724,34657,467206,382450,361534,753431,339520,445611,517697,608214,985784,692541,952510,727954,473726,649883,970534,277351,929793,635157,283825,799647,918989,989054,602764,136828,230283,319529,398228\n", + "Recall: 0.6000\n", + "\n", + "Query ID: 899\n", + "Returned IDs: 710239,892746,855179,154514,879398,967996,257239,802881,659905,302208,653036,425280,768387,785473,156108,274156,822313,818486,887453,787737,288799,949626,506185,545521,986783,746516,586866,944478,989545,557187\n", + "Ground Truth: 710239,892746,855179,154514,879398,967996,257239,802881,659905,302208,653036,425280,768387,785473,156108,274156,822313,818486,887453,787737,288799,800420,949626,506185,545521,986783,746516,586866,944478,989545\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 903\n", + "Returned IDs: 413461,771804,456130,168394,820157,164746,776820,71150,898674,176747,321616,337017,862417,482816,685244,145121,305353,15299,897486,132320,842511,28958,854427,217827,468916,347688,630921,827532,289085,187509\n", + "Ground Truth: 413461,771804,456130,168394,820157,164746,776820,71150,898674,176747,321616,337017,862417,482816,685244,145121,305353,15299,897486,304211,132320,842511,28958,854427,217827,468916,589659,347688,630921,827532\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 907\n", + "Returned IDs: 385032,608558,237047,463588,473156,459782,620768,82017,215475,519494,415195,950931,240901,467763,958906,766488,775718,965309,46392,417916,594597,736420,531217,320552,771894,371451,413722,882541,937845,513327\n", + "Ground Truth: 385032,608558,237047,463588,821306,473156,459782,620768,82017,215475,519494,415195,950931,240901,467763,958906,766488,775718,965309,46392,417916,594597,736420,531217,320552,771894,371451,413722,882541,937845\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 916\n", + "Returned IDs: 700942,20888,152088,490764,238020,479859,101340,236706,937984,850129,651208,250391,44450,90600,569789,852054,300599,560029,620909,634994,413813,536432,948071,914915,359538,590518,298080,98827,110531,70700\n", + "Ground Truth: 700942,20888,152088,490764,238020,479859,101340,236706,937984,850129,651208,250391,44450,90600,569789,852054,300599,560029,620909,634994,413813,536432,948071,914915,359538,590518,875632,298080,98827,110531\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 918\n", + "Returned IDs: 815042,369098,772750,342323,327675,860925,962004,865892,210006,461788,76678,331371,840048,382061,525860,894794,221248,250669,589625,429673,974497,780007,396259,924008,604008,964973,594563,798140,863351,304245\n", + "Ground Truth: 815042,369098,772750,342323,787157,327675,860925,962004,865892,210006,461788,76678,331371,840048,382061,525860,894794,221248,250669,589625,429673,974497,533667,780007,396259,924008,604008,964973,594563,798140\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 919\n", + "Returned IDs: 122855,612592,887462,197625,781313,82128,590056,690167,192123,878786,875282,150346,258435,38528,47911,436835,255900,831002,809784,94332,406077,870507,697122,395809,467753,628848,689403,933931,572216,857545\n", + "Ground Truth: 122855,612592,887462,197625,781313,82128,590056,690167,192123,878786,875282,150346,258435,38528,47911,436835,636637,255900,831002,809784,94332,406077,870507,697122,395809,467753,628848,689403,933931,572216\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 922\n", + "Returned IDs: 829763,835250,777446,556045,727132,23036,491145,531636,931799,726905,382118,906283,797257,917693,855897,760092,288465,305850,144662,727737,290395,106945,856940,906696,111659,770165,809236,576871,73303,576590\n", + "Ground Truth: 829763,835250,777446,556045,727132,23036,491145,531636,931799,726905,382118,218117,906283,797257,917693,855897,760092,288465,305850,144662,727737,290395,106945,856940,906696,111659,770165,809236,576871,73303\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 925\n", + "Returned IDs: 700318,544946,211038,874007,46612,766495,716014,454851,292768,191748,537941,41107,158353,515097,132356,258799,987635,989633,678087,875612,330388,368929,886259,969850,692066,384481,566844,42872,531498,566592\n", + "Ground Truth: 700318,544946,211038,874007,46612,766495,716014,454851,292768,191748,537941,41107,158353,515097,132356,258799,987635,989633,678087,875612,330388,368929,886259,969850,860256,692066,384481,566844,42872,531498\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 930\n", + "Returned IDs: 395142,59494,893656,857478,536944,955134,904245,693560,213628,104693,753849,945585,690746,153522,987883,472561,684999,399198,377176,966123,723045,741611,241140,837836,162005,253099,26906,435640,141890,750016\n", + "Ground Truth: 395142,59494,893656,857478,536944,955134,904245,693560,213628,104693,753849,945585,690746,153522,987883,472561,684999,399198,377176,966123,723045,741611,241140,837836,96382,162005,253099,26906,435640,141890\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 933\n", + "Returned IDs: 563108,921184,495323,153932,35077,202011,120918,889285,135420,503897,815359,560682,38653,817224,140661,300231,699768,418507,555826,662631,779023,112439,184158,20150,509207,782647,860043,986919,745699,94404\n", + "Ground Truth: 821800,563108,921184,267397,495323,153932,35077,96460,202011,120918,889285,135420,503897,809558,815359,560682,38653,817224,863864,140661,300231,699768,418507,555826,662631,757269,779023,112439,122778,184158\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 938\n", + "Returned IDs: 592965,595692,258407,484881,197407,282605,910314,897999,242735,966037,826765,384667,898290,580641,654292,643685,823144,650444,643917,774077,333481,608399,33113,90671,777933,245162,350669,586608,590540,475319\n", + "Ground Truth: 592965,595692,258407,484881,197407,282605,910314,897999,242735,966037,805438,852076,826765,255364,384667,898290,580641,654292,643685,823144,650444,643917,154301,689649,774077,333481,608399,33113,90671,777933\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 940\n", + "Returned IDs: 506417,801230,100599,492364,130786,107968,814245,397852,558055,96994,332370,91660,585233,703286,944443,287360,62852,707519,434284,42570,137476,933138,677396,953232,623534,520271,572516,621090,978697,239697\n", + "Ground Truth: 506417,801230,100599,492364,130786,607,107968,814245,397852,558055,96994,332370,91660,585233,703286,944443,287360,62852,707519,434284,42570,137476,933138,677396,953232,623534,520271,572516,621090,978697\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 941\n", + "Returned IDs: 443660,232450,868623,18712,900814,440597,381587,237988,616568,491147,892667,76270,551274,493103,671796,679648,414684,886837,549070,236773,888781,914017,344199,235851,680541,868705,221548,84188,781264,969112\n", + "Ground Truth: 443660,232450,868623,18712,900814,440597,381587,811664,237988,616568,131684,491147,892667,76270,551274,493103,671796,679648,507158,414684,886837,549070,236773,888781,914017,344199,235851,680541,868705,221548\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 943\n", + "Returned IDs: 703610,350244,247387,829745,683520,410310,652679,675093,95579,925906,155109,364172,84416,755666,479585,474779,75513,118988,731672,237779,843502,155401,310938,843237,644647,202937,857734,851692,462668,826825\n", + "Ground Truth: 703610,350244,247387,829745,683520,410310,652679,675093,95579,925906,155109,364172,84416,755666,479585,474779,75513,623374,118988,237779,731672,843502,155401,310938,843237,644647,202937,857734,851692,462668\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 946\n", + "Returned IDs: 201334,765788,69826,232031,533515,183831,460193,940438,707381,724000,249728,379962,139651,558838,340694,365523,168771,597680,520782,494764,841883,192434,508291,313055,587390,140814,525439,410622,307220,326828\n", + "Ground Truth: 201334,765788,526094,69826,232031,533515,183831,460193,940438,707381,724000,249728,379962,139651,558838,340694,365523,168771,597680,520782,494764,841883,192434,508291,313055,587390,140814,525439,410622,307220\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 947\n", + "Returned IDs: 1868,400255,210207,616906,335246,304112,671576,637270,717240,267526,323948,459637,929162,476106,369559,133153,255527,918027,757869,95504,312972,622516,629811,768890,179101,662268,309960,230183,116196,107550\n", + "Ground Truth: 1868,400255,210207,616906,335246,304112,671576,637270,267526,717240,323948,459637,929162,476106,369559,582181,133153,255527,918027,757869,95504,312972,622516,629811,768890,179101,662268,569829,309960,230183\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 948\n", + "Returned IDs: 691595,935020,584691,250702,965130,903411,149745,192379,617781,144058,638285,847946,756181,351865,912735,624883,663820,395574,454980,958030,947854,349742,447258,239576,408799,617490,108240,147009,209340,916740\n", + "Ground Truth: 691595,935020,584691,250702,965130,903411,149745,192379,617781,144058,638285,847946,756181,351865,912735,624883,318162,663820,395574,454980,958030,947854,349742,447258,239576,408799,617490,108240,147009,209340\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 953\n", + "Returned IDs: 216560,761249,351337,164273,299934,629135,53872,776150,821819,806183,623756,73954,123589,23358,151034,352956,608674,841978,328017,732256,65235,684679,6854,747036,683168,279488,150524,163105,345315,676675\n", + "Ground Truth: 127658,216560,761249,351337,164273,299934,629135,53872,776150,821819,806183,623756,73954,123589,23358,151034,352956,608674,841978,328017,839518,732256,65235,684679,6854,747036,683168,279488,150524,163105\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 955\n", + "Returned IDs: 12048,419284,352643,122230,884714,940827,245492,155893,45723,508567,671796,747036,130558,440388,834945,37314,444231,202141,467769,86040,124395,988460,546578,535506,164640,662631,21831,53872,12747,332716\n", + "Ground Truth: 12048,419284,352643,122230,884714,940827,245492,155893,45723,508567,671796,747036,130558,440388,838747,834945,752098,37314,444231,202141,467769,86040,124395,988460,546578,902128,535506,164640,432541,662631\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 956\n", + "Returned IDs: 696201,262413,939159,150524,523419,475218,739311,686965,629383,430212,123648,577301,651192,988016,339626,827328,36887,58297,99189,831938,216866,683471,25948,417267,401150,971517,887965,130790,102348,720964\n", + "Ground Truth: 696201,262413,939159,150524,523419,176916,475218,755742,739311,686965,629383,430212,123648,577301,651192,988016,339626,827328,36887,58297,99189,831938,216866,710290,683471,25948,417267,401150,971517,887965\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 958\n", + "Returned IDs: 626945,457332,84520,190255,165099,789942,489058,341221,363224,130875,156302,300649,683560,746390,314158,896013,187189,420899,763282,618920,415162,836061,287824,102823,153237,273759,120909,806568,260681,27410\n", + "Ground Truth: 626945,457332,84520,190255,165099,789942,489058,341221,363224,130875,156302,985774,300649,683560,746390,314158,896013,187189,420899,763282,618920,415162,836061,287824,689477,102823,153237,273759,120909,806568\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 962\n", + "Returned IDs: 17440,840691,802919,293805,856401,632549,384718,866909,96285,632640,264423,598582,806000,706040,648943,44308,738315,930286,823654,413754,92708,151352,866151,892464,323100,799071,834243,672869,150524,841244\n", + "Ground Truth: 17440,840691,802919,293805,856401,632549,384718,866909,96285,632640,264423,598582,806000,706040,648943,44308,738315,930286,823654,413754,600672,92708,151352,866151,892464,323100,799071,834243,672869,150524\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 963\n", + "Returned IDs: 123325,836793,423137,39458,687383,854250,434731,735170,601354,139227,678012,496388,983459,377043,742603,113191,156322,501105,177710,94812,541382,287291,981676,850700,693414,900655,278788,18246,39422,627339\n", + "Ground Truth: 123325,836793,423137,39458,687383,854250,434731,735170,601354,139227,678012,496388,983459,377043,742603,113191,834529,156322,501105,177710,94812,541382,287291,468552,981676,850700,693414,900655,278788,18246\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 966\n", + "Returned IDs: 784849,714981,853715,264740,187164,241313,37034,215538,326956,208021,825715,156392,656185,194713,625125,792108,395304,739129,182699,685541,199300,26759,601616,30040,593669,734837,848154,205065,183529,723312\n", + "Ground Truth: 784849,633289,714981,853715,264740,187164,241313,37034,215538,326956,208021,825715,156392,656185,194713,625125,792108,395304,739129,182699,685541,199300,26759,601616,30040,593669,734837,848154,205065,183529\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 969\n", + "Returned IDs: 90346,235446,105271,331662,492758,442586,414388,133351,869150,549859,363458,338026,863092,21950,399341,163951,830120,936531,525888,659479,678246,411815,879219,866289,790617,489862,909430,693221,875473,382699\n", + "Ground Truth: 90346,735842,235446,105271,331662,492758,442586,414388,133351,869150,549859,363458,338026,863092,21950,399341,163951,830120,936531,525888,659479,678246,411815,879219,740871,866289,790617,353614,489862,909430\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 970\n", + "Returned IDs: 163180,636316,306131,273286,674989,249244,380133,546618,801291,111253,683471,465950,323100,612459,937984,941292,756075,943151,184731,400310,474250,767107,348549,85881,950907,705053,184413,314182,268244,705794\n", + "Ground Truth: 163180,636316,306131,273286,674989,249244,380133,546618,801291,111253,683471,465950,323100,612459,937984,941292,756075,943151,184731,400310,474250,767107,348549,85881,950907,705053,184413,314182,268244,333080\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 976\n", + "Returned IDs: 546433,857681,571271,647596,430600,970256,230118,742698,979605,117816,858354,96593,940666,6313,687642,306908,771408,172485,408787,576533,581945,900664,615207,859621,568921,852373,905114,417978,632053,801447\n", + "Ground Truth: 546433,857681,571271,647596,430600,970256,230118,742698,979605,117816,858354,609565,96593,940666,6313,687642,306908,771408,172485,408787,827077,576533,581945,900664,615207,859621,568921,852373,905114,417978\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 978\n", + "Returned IDs: 211806,616373,763436,567108,431809,954125,520563,756669,238139,151112,594313,975638,98872,324480,390983,592169,140078,56570,834736,722764,971222,675527,453036,947625,535689,701455,958974,939976,6558,491662\n", + "Ground Truth: 211806,616373,763436,567108,431809,954125,520563,756669,238139,151112,594313,975638,98872,324480,390983,592169,140078,56570,834736,54467,722764,971222,675527,453036,947625,535689,701455,958974,939976,6558\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 983\n", + "Returned IDs: 700004,764404,915726,561695,178864,18263,411113,860968,128205,360074,400296,652803,925947,240905,353436,267526,959202,774926,704752,946795,60288,461277,485448,377476,240363,641338,380114,757421,142830,237857\n", + "Ground Truth: 700004,764404,915726,561695,178864,18263,411113,860968,128205,360074,400296,652803,925947,240905,353436,267526,959202,774926,704752,946795,60288,830952,461277,485448,377476,240363,641338,380114,757421,142830\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 985\n", + "Returned IDs: 363349,768075,746179,405796,768874,204791,344697,894155,376820,786770,175737,977374,861239,779679,816858,88671,614393,532188,327414,307039,587790,680799,880411,159864,868512,242729,410353,695506,166192,127698\n", + "Ground Truth: 363349,768075,746179,405796,768874,204791,344697,894155,376820,786770,175737,977374,861239,779679,816858,88671,614393,532188,327414,307039,587790,795565,680799,880411,159864,868512,242729,410353,695506,166192\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 986\n", + "Returned IDs: 278846,527042,590023,684601,397345,13131,942561,509579,909953,266750,261233,278236,464392,646624,455595,516390,955495,75175,406548,212925,963455,550051,420890,29259,10525,401067,18357,475467,751098,915514\n", + "Ground Truth: 278846,527042,590023,684601,397345,13131,942561,509579,909953,266750,261233,278236,464392,646624,455595,516390,955495,443021,220253,75175,406548,212925,963455,550051,420890,394322,29259,10525,401067,18357\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 988\n", + "Returned IDs: 667354,531719,140470,413076,546268,506263,170914,26304,612261,705870,252625,851098,210535,909567,10273,471423,857942,40746,704539,348514,235135,275911,126103,854057,709426,429948,552570,122372,691885,17653\n", + "Ground Truth: 667354,531719,140470,413076,546268,506263,170914,26304,612261,705870,252625,851098,210535,909567,10273,232250,471423,857942,40746,704539,348514,235135,275911,126103,854057,709426,429948,552570,122372,691885\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 990\n", + "Returned IDs: 538269,364698,834994,507882,519268,247634,501894,833836,684899,836413,765623,664547,929787,182790,256689,617911,247002,618315,781966,165373,787943,496058,988844,178898,21471,166846,264756,802190,560801,916166\n", + "Ground Truth: 538269,364698,836273,834994,507882,519268,247634,501894,833836,684899,836413,765623,664547,929787,182790,256689,617911,247002,618315,781966,165373,787943,496058,988844,178898,21471,166846,264756,802190,560801\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 992\n", + "Returned IDs: 770591,709426,217294,947430,209588,92787,831607,767668,975064,328481,669373,579782,794684,162963,104914,723717,920918,793343,798103,800090,956108,824637,755352,146771,707383,667354,166528,506263,444029,346073\n", + "Ground Truth: 770591,509690,212231,709426,217294,947430,262773,971408,209588,92787,831607,767668,975064,328481,669373,579782,794684,795431,162963,104914,723717,920918,28744,793343,798103,800090,956108,824637,755352,146771\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 993\n", + "Returned IDs: 321389,220988,572167,370532,238363,620065,360595,284590,713328,938683,223699,145879,389713,624300,827647,70169,878531,446292,401216,364518,156097,67133,690891,470126,739115,415924,255069,458373,145450,587007\n", + "Ground Truth: 321389,220988,572167,370532,238363,620065,360595,284590,249865,713328,938229,938683,223699,145879,389713,624300,827647,70169,878531,108742,446292,401216,364518,536744,156097,67133,690891,628930,470126,739115\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 994\n", + "Returned IDs: 283419,741794,251978,231246,430212,784473,253699,98655,549649,898405,202333,956045,25948,651192,174306,150524,841072,672965,939159,563721,73954,341890,722774,696201,298337,579235,868235,506026,148216,245492\n", + "Ground Truth: 240932,855988,283419,741794,251978,231246,430212,784473,253699,98655,549649,898405,327970,202333,956045,25948,354397,651192,781365,838308,174306,150524,841072,672965,939159,563721,73954,57112,341890,722774\n", + "Recall: 0.7667\n", + "\n", + "Total query IDs with recall < 1.0: 303\n", + "IDs: [np.int64(14), np.int64(17), np.int64(21), np.int64(23), np.int64(29), np.int64(35), np.int64(38), np.int64(42), np.int64(44), np.int64(51), np.int64(53), np.int64(54), np.int64(55), np.int64(59), np.int64(62), np.int64(66), np.int64(71), np.int64(74), np.int64(75), np.int64(77), np.int64(79), np.int64(81), np.int64(84), np.int64(85), np.int64(91), np.int64(93), np.int64(95), np.int64(97), np.int64(99), np.int64(100), np.int64(105), np.int64(106), np.int64(107), np.int64(108), np.int64(109), np.int64(114), np.int64(116), np.int64(117), np.int64(118), np.int64(120), np.int64(121), np.int64(129), np.int64(130), np.int64(132), np.int64(135), np.int64(139), np.int64(140), np.int64(142), np.int64(144), np.int64(146), np.int64(148), np.int64(149), np.int64(154), np.int64(158), np.int64(170), np.int64(173), np.int64(176), np.int64(178), np.int64(181), np.int64(184), np.int64(197), np.int64(199), np.int64(200), np.int64(203), np.int64(211), np.int64(214), np.int64(215), np.int64(216), np.int64(221), np.int64(225), np.int64(226), np.int64(245), np.int64(248), np.int64(249), np.int64(254), np.int64(256), np.int64(257), np.int64(261), np.int64(266), np.int64(268), np.int64(272), np.int64(278), np.int64(282), np.int64(283), np.int64(284), np.int64(285), np.int64(289), np.int64(290), np.int64(295), np.int64(299), np.int64(306), np.int64(308), np.int64(310), np.int64(311), np.int64(315), np.int64(317), np.int64(319), np.int64(320), np.int64(322), np.int64(327), np.int64(331), np.int64(332), np.int64(341), np.int64(343), np.int64(344), np.int64(345), np.int64(350), np.int64(351), np.int64(353), np.int64(355), np.int64(365), np.int64(366), np.int64(369), np.int64(370), np.int64(379), np.int64(389), np.int64(394), np.int64(395), np.int64(399), np.int64(400), np.int64(401), np.int64(403), np.int64(406), np.int64(410), np.int64(411), np.int64(413), np.int64(419), np.int64(421), np.int64(424), np.int64(430), np.int64(434), np.int64(435), np.int64(438), np.int64(442), np.int64(448), np.int64(450), np.int64(453), np.int64(456), np.int64(460), np.int64(463), np.int64(464), np.int64(468), np.int64(472), np.int64(475), np.int64(480), np.int64(481), np.int64(483), np.int64(486), np.int64(496), np.int64(501), np.int64(503), np.int64(508), np.int64(509), np.int64(510), np.int64(511), np.int64(513), np.int64(520), np.int64(521), np.int64(522), np.int64(523), np.int64(527), np.int64(531), np.int64(534), np.int64(543), np.int64(547), np.int64(548), np.int64(551), np.int64(556), np.int64(557), np.int64(562), np.int64(563), np.int64(569), np.int64(574), np.int64(577), np.int64(583), np.int64(585), np.int64(586), np.int64(587), np.int64(596), np.int64(599), np.int64(607), np.int64(615), np.int64(616), np.int64(625), np.int64(626), np.int64(629), np.int64(632), np.int64(634), np.int64(636), np.int64(640), np.int64(647), np.int64(649), np.int64(650), np.int64(652), np.int64(654), np.int64(655), np.int64(656), np.int64(668), np.int64(669), np.int64(674), np.int64(675), np.int64(676), np.int64(677), np.int64(682), np.int64(683), np.int64(694), np.int64(697), np.int64(702), np.int64(703), np.int64(704), np.int64(708), np.int64(709), np.int64(710), np.int64(713), np.int64(718), np.int64(719), np.int64(724), np.int64(726), np.int64(727), np.int64(728), np.int64(731), np.int64(733), np.int64(735), np.int64(737), np.int64(740), np.int64(741), np.int64(744), np.int64(747), np.int64(749), np.int64(754), np.int64(755), np.int64(761), np.int64(762), np.int64(763), np.int64(770), np.int64(771), np.int64(773), np.int64(776), np.int64(783), np.int64(788), np.int64(790), np.int64(791), np.int64(792), np.int64(800), np.int64(802), np.int64(807), np.int64(812), np.int64(814), np.int64(815), np.int64(818), np.int64(832), np.int64(837), np.int64(849), np.int64(855), np.int64(856), np.int64(861), np.int64(862), np.int64(864), np.int64(870), np.int64(871), np.int64(874), np.int64(875), np.int64(877), np.int64(883), np.int64(885), np.int64(894), np.int64(898), np.int64(899), np.int64(903), np.int64(907), np.int64(916), np.int64(918), np.int64(919), np.int64(922), np.int64(925), np.int64(930), np.int64(933), np.int64(938), np.int64(940), np.int64(941), np.int64(943), np.int64(946), np.int64(947), np.int64(948), np.int64(953), np.int64(955), np.int64(956), np.int64(958), np.int64(962), np.int64(963), np.int64(966), np.int64(969), np.int64(970), np.int64(976), np.int64(978), np.int64(983), np.int64(985), np.int64(986), np.int64(988), np.int64(990), np.int64(992), np.int64(993), np.int64(994)]\n", + "Average Recall across all 1000 queries: 0.9732\n" + ] + } + ], + "source": [ + "#For querying (int filter)- lt filter\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "#inputs\n", + "int_rate_query = \"99p\" # change to 1p, 50p, 80p, 99p\n", + "top_k = 30\n", + "\n", + "# Mode 1: Single query id\n", + "# Mode 2: Find all query ids with recall < 1\n", + "mode = 2\n", + "query_id = 457 # only used in mode 1\n", + "\n", + "# Map int_rate_query to lt threshold\n", + "lt_map = {\n", + " \"1p\": 9999,\n", + " \"50p\": 499999,\n", + " \"80p\": 799999,\n", + " \"99p\": 989999,\n", + "}\n", + "lt_threshold = lt_map[int_rate_query]\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "lte_dataset_path = os.path.join(\"/home/debian/latest_VDB/VectorDBBench/vectordataset_lt\", dataset_folder)\n", + "test_parquet_path = os.path.join(dataset_path, \"test.parquet\")\n", + "gt_parquet_path = os.path.join(lte_dataset_path, f\"neighbors_int_{int_rate_query}.parquet\")\n", + "\n", + "# Read parquet files\n", + "test_df = pq.ParquetFile(test_parquet_path).read().to_pandas()\n", + "gt_df = pq.ParquetFile(gt_parquet_path).read().to_pandas()\n", + "\n", + "def run_query(query_id):\n", + " query_row = test_df[test_df[\"id\"] == query_id]\n", + " if query_row.empty:\n", + " print(f\"Query ID {query_id} not found in test.parquet\")\n", + " return None\n", + "\n", + " query_vector = query_row[\"emb\"].values[0]\n", + "\n", + " gt_row = gt_df[gt_df[\"id\"] == query_id]\n", + " if gt_row.empty:\n", + " print(f\"Query ID {query_id} not found in ground truth file\")\n", + " return None\n", + "\n", + " ground_truth = gt_row[\"neighbors_id\"].values[0][:top_k]\n", + "\n", + " results = index.query(\n", + " vector=query_vector,\n", + " top_k=top_k,\n", + " filter=[{\"id\": {\"$lt\": lt_threshold}}],\n", + " include_vectors=False,\n", + " )\n", + " returned_ids = [int(r[\"id\"]) for r in results]\n", + "\n", + " intersection = len(set(returned_ids) & set(ground_truth))\n", + " recall = intersection / len(ground_truth)\n", + "\n", + " return returned_ids, ground_truth, recall\n", + "\n", + "\n", + "if mode == 1:\n", + " result = run_query(query_id)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " print(f\"Query ID: {query_id}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\")\n", + "\n", + "elif mode == 2:\n", + " print(f\"Finding all query IDs with recall < 1.0 ...\\n\")\n", + " low_recall_ids = []\n", + " all_recalls = []\n", + "\n", + " for qid in test_df[\"id\"].values:\n", + " result = run_query(qid)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " all_recalls.append(recall)\n", + " if recall < 1.0:\n", + " low_recall_ids.append(qid)\n", + " print(f\"Query ID: {qid}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\\n\")\n", + "\n", + " print(f\"Total query IDs with recall < 1.0: {len(low_recall_ids)}\")\n", + " print(f\"IDs: {low_recall_ids}\")\n", + " print(f\"Average Recall across all {len(all_recalls)} queries: {sum(all_recalls)/len(all_recalls):.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_labelfilter_0503_1_adup8',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 1000000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Deleted vectors with id >= 9999\n", + "990001 vectors deleted\n" + ] + } + ], + "source": [ + "#For deleting (int filter) - lt\n", + "int_rate_delete = \"99p\" # change to 1p, 20p, 50p, 99p\n", + "\n", + "lt_map = {\n", + " \"1p\": 989999,\n", + " \"20p\": 799999,\n", + " \"50p\": 499999,\n", + " \"99p\": 9999,\n", + "}\n", + "lt_threshold = lt_map[int_rate_delete]\n", + "\n", + "result = index.delete_with_filter([{\"id\": {\"$gte\": lt_threshold}}])\n", + "print(f\"Deleted vectors with id >= {lt_threshold}\")\n", + "print(result)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_labelfilter_0503_1_adup8',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 1000000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Upserted 991 vectors, total so far: 991\n", + "Upserted 993 vectors, total so far: 1984\n", + "Upserted 989 vectors, total so far: 2973\n", + "Upserted 991 vectors, total so far: 3964\n", + "Upserted 992 vectors, total so far: 4956\n", + "Upserted 991 vectors, total so far: 5947\n", + "Upserted 990 vectors, total so far: 6937\n", + "Upserted 986 vectors, total so far: 7923\n", + "Upserted 996 vectors, total so far: 8919\n", + "Upserted 991 vectors, total so far: 9910\n", + "Upserted 993 vectors, total so far: 10903\n", + "Upserted 992 vectors, total so far: 11895\n", + "Upserted 989 vectors, total so far: 12884\n", + "Upserted 995 vectors, total so far: 13879\n", + "Upserted 988 vectors, total so far: 14867\n", + "Upserted 992 vectors, total so far: 15859\n", + "Upserted 988 vectors, total so far: 16847\n", + "Upserted 988 vectors, total so far: 17835\n", + "Upserted 996 vectors, total so far: 18831\n", + "Upserted 993 vectors, total so far: 19824\n", + "Upserted 993 vectors, total so far: 20817\n", + "Upserted 989 vectors, total so far: 21806\n", + "Upserted 993 vectors, total so far: 22799\n", + "Upserted 982 vectors, total so far: 23781\n", + "Upserted 991 vectors, total so far: 24772\n", + "Upserted 996 vectors, total so far: 25768\n", + "Upserted 993 vectors, total so far: 26761\n", + "Upserted 994 vectors, total so far: 27755\n", + "Upserted 993 vectors, total so far: 28748\n", + "Upserted 990 vectors, total so far: 29738\n", + "Upserted 989 vectors, total so far: 30727\n", + "Upserted 993 vectors, total so far: 31720\n", + "Upserted 987 vectors, total so far: 32707\n", + "Upserted 989 vectors, total so far: 33696\n", + "Upserted 987 vectors, total so far: 34683\n", + "Upserted 992 vectors, total so far: 35675\n", + "Upserted 988 vectors, total so far: 36663\n", + "Upserted 991 vectors, total so far: 37654\n", + "Upserted 990 vectors, total so far: 38644\n", + "Upserted 990 vectors, total so far: 39634\n", + "Upserted 993 vectors, total so far: 40627\n", + "Upserted 986 vectors, total so far: 41613\n", + "Upserted 991 vectors, total so far: 42604\n", + "Upserted 991 vectors, total so far: 43595\n", + "Upserted 993 vectors, total so far: 44588\n", + "Upserted 989 vectors, total so far: 45577\n", + "Upserted 990 vectors, total so far: 46567\n", + "Upserted 990 vectors, total so far: 47557\n", + "Upserted 993 vectors, total so far: 48550\n", + "Upserted 989 vectors, total so far: 49539\n", + "Upserted 987 vectors, total so far: 50526\n", + "Upserted 985 vectors, total so far: 51511\n", + "Upserted 995 vectors, total so far: 52506\n", + "Upserted 995 vectors, total so far: 53501\n", + "Upserted 992 vectors, total so far: 54493\n", + "Upserted 989 vectors, total so far: 55482\n", + "Upserted 989 vectors, total so far: 56471\n", + "Upserted 989 vectors, total so far: 57460\n", + "Upserted 988 vectors, total so far: 58448\n", + "Upserted 990 vectors, total so far: 59438\n", + "Upserted 988 vectors, total so far: 60426\n", + "Upserted 995 vectors, total so far: 61421\n", + "Upserted 989 vectors, total so far: 62410\n", + "Upserted 991 vectors, total so far: 63401\n", + "Upserted 990 vectors, total so far: 64391\n", + "Upserted 995 vectors, total so far: 65386\n", + "Upserted 990 vectors, total so far: 66376\n", + "Upserted 989 vectors, total so far: 67365\n", + "Upserted 990 vectors, total so far: 68355\n", + "Upserted 993 vectors, total so far: 69348\n", + "Upserted 995 vectors, total so far: 70343\n", + "Upserted 990 vectors, total so far: 71333\n", + "Upserted 989 vectors, total so far: 72322\n", + "Upserted 989 vectors, total so far: 73311\n", + "Upserted 994 vectors, total so far: 74305\n", + "Upserted 989 vectors, total so far: 75294\n", + "Upserted 989 vectors, total so far: 76283\n", + "Upserted 994 vectors, total so far: 77277\n", + "Upserted 991 vectors, total so far: 78268\n", + "Upserted 993 vectors, total so far: 79261\n", + "Upserted 996 vectors, total so far: 80257\n", + "Upserted 987 vectors, total so far: 81244\n", + "Upserted 990 vectors, total so far: 82234\n", + "Upserted 988 vectors, total so far: 83222\n", + "Upserted 988 vectors, total so far: 84210\n", + "Upserted 989 vectors, total so far: 85199\n", + "Upserted 987 vectors, total so far: 86186\n", + "Upserted 982 vectors, total so far: 87168\n", + "Upserted 990 vectors, total so far: 88158\n", + "Upserted 993 vectors, total so far: 89151\n", + "Upserted 982 vectors, total so far: 90133\n", + "Upserted 989 vectors, total so far: 91122\n", + "Upserted 988 vectors, total so far: 92110\n", + "Upserted 994 vectors, total so far: 93104\n", + "Upserted 985 vectors, total so far: 94089\n", + "Upserted 991 vectors, total so far: 95080\n", + "Upserted 986 vectors, total so far: 96066\n", + "Upserted 989 vectors, total so far: 97055\n", + "Upserted 988 vectors, total so far: 98043\n", + "Upserted 984 vectors, total so far: 99027\n", + "Upserted 990 vectors, total so far: 100017\n", + "Upserted 990 vectors, total so far: 101007\n", + "Upserted 995 vectors, total so far: 102002\n", + "Upserted 992 vectors, total so far: 102994\n", + "Upserted 993 vectors, total so far: 103987\n", + "Upserted 991 vectors, total so far: 104978\n", + "Upserted 991 vectors, total so far: 105969\n", + "Upserted 990 vectors, total so far: 106959\n", + "Upserted 990 vectors, total so far: 107949\n", + "Upserted 993 vectors, total so far: 108942\n", + "Upserted 992 vectors, total so far: 109934\n", + "Upserted 990 vectors, total so far: 110924\n", + "Upserted 988 vectors, total so far: 111912\n", + "Upserted 993 vectors, total so far: 112905\n", + "Upserted 994 vectors, total so far: 113899\n", + "Upserted 991 vectors, total so far: 114890\n", + "Upserted 993 vectors, total so far: 115883\n", + "Upserted 991 vectors, total so far: 116874\n", + "Upserted 990 vectors, total so far: 117864\n", + "Upserted 986 vectors, total so far: 118850\n", + "Upserted 988 vectors, total so far: 119838\n", + "Upserted 991 vectors, total so far: 120829\n", + "Upserted 989 vectors, total so far: 121818\n", + "Upserted 987 vectors, total so far: 122805\n", + "Upserted 996 vectors, total so far: 123801\n", + "Upserted 991 vectors, total so far: 124792\n", + "Upserted 994 vectors, total so far: 125786\n", + "Upserted 988 vectors, total so far: 126774\n", + "Upserted 988 vectors, total so far: 127762\n", + "Upserted 989 vectors, total so far: 128751\n", + "Upserted 994 vectors, total so far: 129745\n", + "Upserted 992 vectors, total so far: 130737\n", + "Upserted 992 vectors, total so far: 131729\n", + "Upserted 993 vectors, total so far: 132722\n", + "Upserted 993 vectors, total so far: 133715\n", + "Upserted 987 vectors, total so far: 134702\n", + "Upserted 989 vectors, total so far: 135691\n", + "Upserted 990 vectors, total so far: 136681\n", + "Upserted 990 vectors, total so far: 137671\n", + "Upserted 993 vectors, total so far: 138664\n", + "Upserted 991 vectors, total so far: 139655\n", + "Upserted 993 vectors, total so far: 140648\n", + "Upserted 990 vectors, total so far: 141638\n", + "Upserted 993 vectors, total so far: 142631\n", + "Upserted 983 vectors, total so far: 143614\n", + "Upserted 984 vectors, total so far: 144598\n", + "Upserted 986 vectors, total so far: 145584\n", + "Upserted 997 vectors, total so far: 146581\n", + "Upserted 987 vectors, total so far: 147568\n", + "Upserted 993 vectors, total so far: 148561\n", + "Upserted 990 vectors, total so far: 149551\n", + "Upserted 989 vectors, total so far: 150540\n", + "Upserted 989 vectors, total so far: 151529\n", + "Upserted 994 vectors, total so far: 152523\n", + "Upserted 981 vectors, total so far: 153504\n", + "Upserted 993 vectors, total so far: 154497\n", + "Upserted 994 vectors, total so far: 155491\n", + "Upserted 991 vectors, total so far: 156482\n", + "Upserted 990 vectors, total so far: 157472\n", + "Upserted 982 vectors, total so far: 158454\n", + "Upserted 989 vectors, total so far: 159443\n", + "Upserted 992 vectors, total so far: 160435\n", + "Upserted 996 vectors, total so far: 161431\n", + "Upserted 986 vectors, total so far: 162417\n", + "Upserted 991 vectors, total so far: 163408\n", + "Upserted 987 vectors, total so far: 164395\n", + "Upserted 990 vectors, total so far: 165385\n", + "Upserted 991 vectors, total so far: 166376\n", + "Upserted 989 vectors, total so far: 167365\n", + "Upserted 992 vectors, total so far: 168357\n", + "Upserted 988 vectors, total so far: 169345\n", + "Upserted 988 vectors, total so far: 170333\n", + "Upserted 988 vectors, total so far: 171321\n", + "Upserted 996 vectors, total so far: 172317\n", + "Upserted 992 vectors, total so far: 173309\n", + "Upserted 992 vectors, total so far: 174301\n", + "Upserted 989 vectors, total so far: 175290\n", + "Upserted 993 vectors, total so far: 176283\n", + "Upserted 992 vectors, total so far: 177275\n", + "Upserted 993 vectors, total so far: 178268\n", + "Upserted 991 vectors, total so far: 179259\n", + "Upserted 990 vectors, total so far: 180249\n", + "Upserted 992 vectors, total so far: 181241\n", + "Upserted 988 vectors, total so far: 182229\n", + "Upserted 986 vectors, total so far: 183215\n", + "Upserted 986 vectors, total so far: 184201\n", + "Upserted 993 vectors, total so far: 185194\n", + "Upserted 990 vectors, total so far: 186184\n", + "Upserted 989 vectors, total so far: 187173\n", + "Upserted 994 vectors, total so far: 188167\n", + "Upserted 985 vectors, total so far: 189152\n", + "Upserted 987 vectors, total so far: 190139\n", + "Upserted 993 vectors, total so far: 191132\n", + "Upserted 991 vectors, total so far: 192123\n", + "Upserted 992 vectors, total so far: 193115\n", + "Upserted 987 vectors, total so far: 194102\n", + "Upserted 987 vectors, total so far: 195089\n", + "Upserted 989 vectors, total so far: 196078\n", + "Upserted 988 vectors, total so far: 197066\n", + "Upserted 991 vectors, total so far: 198057\n", + "Upserted 988 vectors, total so far: 199045\n", + "Upserted 989 vectors, total so far: 200034\n", + "Upserted 993 vectors, total so far: 201027\n", + "Upserted 991 vectors, total so far: 202018\n", + "Upserted 984 vectors, total so far: 203002\n", + "Upserted 987 vectors, total so far: 203989\n", + "Upserted 998 vectors, total so far: 204987\n", + "Upserted 990 vectors, total so far: 205977\n", + "Upserted 992 vectors, total so far: 206969\n", + "Upserted 988 vectors, total so far: 207957\n", + "Upserted 990 vectors, total so far: 208947\n", + "Upserted 993 vectors, total so far: 209940\n", + "Upserted 986 vectors, total so far: 210926\n", + "Upserted 995 vectors, total so far: 211921\n", + "Upserted 989 vectors, total so far: 212910\n", + "Upserted 993 vectors, total so far: 213903\n", + "Upserted 993 vectors, total so far: 214896\n", + "Upserted 989 vectors, total so far: 215885\n", + "Upserted 993 vectors, total so far: 216878\n", + "Upserted 994 vectors, total so far: 217872\n", + "Upserted 992 vectors, total so far: 218864\n", + "Upserted 992 vectors, total so far: 219856\n", + "Upserted 987 vectors, total so far: 220843\n", + "Upserted 987 vectors, total so far: 221830\n", + "Upserted 990 vectors, total so far: 222820\n", + "Upserted 996 vectors, total so far: 223816\n", + "Upserted 987 vectors, total so far: 224803\n", + "Upserted 990 vectors, total so far: 225793\n", + "Upserted 990 vectors, total so far: 226783\n", + "Upserted 992 vectors, total so far: 227775\n", + "Upserted 991 vectors, total so far: 228766\n", + "Upserted 993 vectors, total so far: 229759\n", + "Upserted 991 vectors, total so far: 230750\n", + "Upserted 983 vectors, total so far: 231733\n", + "Upserted 984 vectors, total so far: 232717\n", + "Upserted 988 vectors, total so far: 233705\n", + "Upserted 992 vectors, total so far: 234697\n", + "Upserted 990 vectors, total so far: 235687\n", + "Upserted 992 vectors, total so far: 236679\n", + "Upserted 992 vectors, total so far: 237671\n", + "Upserted 989 vectors, total so far: 238660\n", + "Upserted 990 vectors, total so far: 239650\n", + "Upserted 991 vectors, total so far: 240641\n", + "Upserted 989 vectors, total so far: 241630\n", + "Upserted 989 vectors, total so far: 242619\n", + "Upserted 990 vectors, total so far: 243609\n", + "Upserted 992 vectors, total so far: 244601\n", + "Upserted 992 vectors, total so far: 245593\n", + "Upserted 988 vectors, total so far: 246581\n", + "Upserted 999 vectors, total so far: 247580\n", + "Upserted 993 vectors, total so far: 248573\n", + "Upserted 994 vectors, total so far: 249567\n", + "Upserted 990 vectors, total so far: 250557\n", + "Upserted 994 vectors, total so far: 251551\n", + "Upserted 997 vectors, total so far: 252548\n", + "Upserted 989 vectors, total so far: 253537\n", + "Upserted 986 vectors, total so far: 254523\n", + "Upserted 988 vectors, total so far: 255511\n", + "Upserted 990 vectors, total so far: 256501\n", + "Upserted 989 vectors, total so far: 257490\n", + "Upserted 991 vectors, total so far: 258481\n", + "Upserted 991 vectors, total so far: 259472\n", + "Upserted 986 vectors, total so far: 260458\n", + "Upserted 990 vectors, total so far: 261448\n", + "Upserted 992 vectors, total so far: 262440\n", + "Upserted 994 vectors, total so far: 263434\n", + "Upserted 989 vectors, total so far: 264423\n", + "Upserted 995 vectors, total so far: 265418\n", + "Upserted 990 vectors, total so far: 266408\n", + "Upserted 988 vectors, total so far: 267396\n", + "Upserted 992 vectors, total so far: 268388\n", + "Upserted 990 vectors, total so far: 269378\n", + "Upserted 991 vectors, total so far: 270369\n", + "Upserted 993 vectors, total so far: 271362\n", + "Upserted 981 vectors, total so far: 272343\n", + "Upserted 994 vectors, total so far: 273337\n", + "Upserted 988 vectors, total so far: 274325\n", + "Upserted 991 vectors, total so far: 275316\n", + "Upserted 987 vectors, total so far: 276303\n", + "Upserted 985 vectors, total so far: 277288\n", + "Upserted 992 vectors, total so far: 278280\n", + "Upserted 990 vectors, total so far: 279270\n", + "Upserted 994 vectors, total so far: 280264\n", + "Upserted 992 vectors, total so far: 281256\n", + "Upserted 985 vectors, total so far: 282241\n", + "Upserted 992 vectors, total so far: 283233\n", + "Upserted 994 vectors, total so far: 284227\n", + "Upserted 996 vectors, total so far: 285223\n", + "Upserted 992 vectors, total so far: 286215\n", + "Upserted 991 vectors, total so far: 287206\n", + "Upserted 990 vectors, total so far: 288196\n", + "Upserted 994 vectors, total so far: 289190\n", + "Upserted 983 vectors, total so far: 290173\n", + "Upserted 989 vectors, total so far: 291162\n", + "Upserted 986 vectors, total so far: 292148\n", + "Upserted 989 vectors, total so far: 293137\n", + "Upserted 989 vectors, total so far: 294126\n", + "Upserted 988 vectors, total so far: 295114\n", + "Upserted 992 vectors, total so far: 296106\n", + "Upserted 987 vectors, total so far: 297093\n", + "Upserted 989 vectors, total so far: 298082\n", + "Upserted 990 vectors, total so far: 299072\n", + "Upserted 988 vectors, total so far: 300060\n", + "Upserted 987 vectors, total so far: 301047\n", + "Upserted 993 vectors, total so far: 302040\n", + "Upserted 989 vectors, total so far: 303029\n", + "Upserted 990 vectors, total so far: 304019\n", + "Upserted 991 vectors, total so far: 305010\n", + "Upserted 987 vectors, total so far: 305997\n", + "Upserted 993 vectors, total so far: 306990\n", + "Upserted 993 vectors, total so far: 307983\n", + "Upserted 987 vectors, total so far: 308970\n", + "Upserted 994 vectors, total so far: 309964\n", + "Upserted 986 vectors, total so far: 310950\n", + "Upserted 989 vectors, total so far: 311939\n", + "Upserted 982 vectors, total so far: 312921\n", + "Upserted 988 vectors, total so far: 313909\n", + "Upserted 993 vectors, total so far: 314902\n", + "Upserted 991 vectors, total so far: 315893\n", + "Upserted 992 vectors, total so far: 316885\n", + "Upserted 987 vectors, total so far: 317872\n", + "Upserted 991 vectors, total so far: 318863\n", + "Upserted 987 vectors, total so far: 319850\n", + "Upserted 992 vectors, total so far: 320842\n", + "Upserted 996 vectors, total so far: 321838\n", + "Upserted 992 vectors, total so far: 322830\n", + "Upserted 991 vectors, total so far: 323821\n", + "Upserted 988 vectors, total so far: 324809\n", + "Upserted 993 vectors, total so far: 325802\n", + "Upserted 995 vectors, total so far: 326797\n", + "Upserted 994 vectors, total so far: 327791\n", + "Upserted 989 vectors, total so far: 328780\n", + "Upserted 990 vectors, total so far: 329770\n", + "Upserted 988 vectors, total so far: 330758\n", + "Upserted 993 vectors, total so far: 331751\n", + "Upserted 986 vectors, total so far: 332737\n", + "Upserted 986 vectors, total so far: 333723\n", + "Upserted 993 vectors, total so far: 334716\n", + "Upserted 983 vectors, total so far: 335699\n", + "Upserted 995 vectors, total so far: 336694\n", + "Upserted 986 vectors, total so far: 337680\n", + "Upserted 990 vectors, total so far: 338670\n", + "Upserted 990 vectors, total so far: 339660\n", + "Upserted 994 vectors, total so far: 340654\n", + "Upserted 991 vectors, total so far: 341645\n", + "Upserted 990 vectors, total so far: 342635\n", + "Upserted 991 vectors, total so far: 343626\n", + "Upserted 991 vectors, total so far: 344617\n", + "Upserted 992 vectors, total so far: 345609\n", + "Upserted 993 vectors, total so far: 346602\n", + "Upserted 986 vectors, total so far: 347588\n", + "Upserted 991 vectors, total so far: 348579\n", + "Upserted 990 vectors, total so far: 349569\n", + "Upserted 996 vectors, total so far: 350565\n", + "Upserted 992 vectors, total so far: 351557\n", + "Upserted 988 vectors, total so far: 352545\n", + "Upserted 994 vectors, total so far: 353539\n", + "Upserted 992 vectors, total so far: 354531\n", + "Upserted 995 vectors, total so far: 355526\n", + "Upserted 989 vectors, total so far: 356515\n", + "Upserted 988 vectors, total so far: 357503\n", + "Upserted 990 vectors, total so far: 358493\n", + "Upserted 994 vectors, total so far: 359487\n", + "Upserted 989 vectors, total so far: 360476\n", + "Upserted 984 vectors, total so far: 361460\n", + "Upserted 992 vectors, total so far: 362452\n", + "Upserted 985 vectors, total so far: 363437\n", + "Upserted 991 vectors, total so far: 364428\n", + "Upserted 990 vectors, total so far: 365418\n", + "Upserted 991 vectors, total so far: 366409\n", + "Upserted 994 vectors, total so far: 367403\n", + "Upserted 992 vectors, total so far: 368395\n", + "Upserted 990 vectors, total so far: 369385\n", + "Upserted 987 vectors, total so far: 370372\n", + "Upserted 990 vectors, total so far: 371362\n", + "Upserted 992 vectors, total so far: 372354\n", + "Upserted 989 vectors, total so far: 373343\n", + "Upserted 995 vectors, total so far: 374338\n", + "Upserted 987 vectors, total so far: 375325\n", + "Upserted 983 vectors, total so far: 376308\n", + "Upserted 988 vectors, total so far: 377296\n", + "Upserted 989 vectors, total so far: 378285\n", + "Upserted 992 vectors, total so far: 379277\n", + "Upserted 990 vectors, total so far: 380267\n", + "Upserted 993 vectors, total so far: 381260\n", + "Upserted 986 vectors, total so far: 382246\n", + "Upserted 986 vectors, total so far: 383232\n", + "Upserted 993 vectors, total so far: 384225\n", + "Upserted 985 vectors, total so far: 385210\n", + "Upserted 989 vectors, total so far: 386199\n", + "Upserted 988 vectors, total so far: 387187\n", + "Upserted 990 vectors, total so far: 388177\n", + "Upserted 994 vectors, total so far: 389171\n", + "Upserted 985 vectors, total so far: 390156\n", + "Upserted 986 vectors, total so far: 391142\n", + "Upserted 987 vectors, total so far: 392129\n", + "Upserted 985 vectors, total so far: 393114\n", + "Upserted 986 vectors, total so far: 394100\n", + "Upserted 990 vectors, total so far: 395090\n", + "Upserted 986 vectors, total so far: 396076\n", + "Upserted 988 vectors, total so far: 397064\n", + "Upserted 985 vectors, total so far: 398049\n", + "Upserted 987 vectors, total so far: 399036\n", + "Upserted 992 vectors, total so far: 400028\n", + "Upserted 990 vectors, total so far: 401018\n", + "Upserted 984 vectors, total so far: 402002\n", + "Upserted 991 vectors, total so far: 402993\n", + "Upserted 991 vectors, total so far: 403984\n", + "Upserted 990 vectors, total so far: 404974\n", + "Upserted 994 vectors, total so far: 405968\n", + "Upserted 991 vectors, total so far: 406959\n", + "Upserted 984 vectors, total so far: 407943\n", + "Upserted 993 vectors, total so far: 408936\n", + "Upserted 991 vectors, total so far: 409927\n", + "Upserted 986 vectors, total so far: 410913\n", + "Upserted 996 vectors, total so far: 411909\n", + "Upserted 990 vectors, total so far: 412899\n", + "Upserted 990 vectors, total so far: 413889\n", + "Upserted 985 vectors, total so far: 414874\n", + "Upserted 990 vectors, total so far: 415864\n", + "Upserted 988 vectors, total so far: 416852\n", + "Upserted 990 vectors, total so far: 417842\n", + "Upserted 992 vectors, total so far: 418834\n", + "Upserted 992 vectors, total so far: 419826\n", + "Upserted 990 vectors, total so far: 420816\n", + "Upserted 990 vectors, total so far: 421806\n", + "Upserted 992 vectors, total so far: 422798\n", + "Upserted 992 vectors, total so far: 423790\n", + "Upserted 989 vectors, total so far: 424779\n", + "Upserted 985 vectors, total so far: 425764\n", + "Upserted 991 vectors, total so far: 426755\n", + "Upserted 987 vectors, total so far: 427742\n", + "Upserted 986 vectors, total so far: 428728\n", + "Upserted 988 vectors, total so far: 429716\n", + "Upserted 990 vectors, total so far: 430706\n", + "Upserted 986 vectors, total so far: 431692\n", + "Upserted 991 vectors, total so far: 432683\n", + "Upserted 993 vectors, total so far: 433676\n", + "Upserted 989 vectors, total so far: 434665\n", + "Upserted 983 vectors, total so far: 435648\n", + "Upserted 992 vectors, total so far: 436640\n", + "Upserted 992 vectors, total so far: 437632\n", + "Upserted 993 vectors, total so far: 438625\n", + "Upserted 987 vectors, total so far: 439612\n", + "Upserted 986 vectors, total so far: 440598\n", + "Upserted 993 vectors, total so far: 441591\n", + "Upserted 989 vectors, total so far: 442580\n", + "Upserted 993 vectors, total so far: 443573\n", + "Upserted 993 vectors, total so far: 444566\n", + "Upserted 994 vectors, total so far: 445560\n", + "Upserted 992 vectors, total so far: 446552\n", + "Upserted 992 vectors, total so far: 447544\n", + "Upserted 986 vectors, total so far: 448530\n", + "Upserted 992 vectors, total so far: 449522\n", + "Upserted 990 vectors, total so far: 450512\n", + "Upserted 985 vectors, total so far: 451497\n", + "Upserted 996 vectors, total so far: 452493\n", + "Upserted 988 vectors, total so far: 453481\n", + "Upserted 986 vectors, total so far: 454467\n", + "Upserted 988 vectors, total so far: 455455\n", + "Upserted 990 vectors, total so far: 456445\n", + "Upserted 992 vectors, total so far: 457437\n", + "Upserted 989 vectors, total so far: 458426\n", + "Upserted 994 vectors, total so far: 459420\n", + "Upserted 995 vectors, total so far: 460415\n", + "Upserted 990 vectors, total so far: 461405\n", + "Upserted 989 vectors, total so far: 462394\n", + "Upserted 991 vectors, total so far: 463385\n", + "Upserted 988 vectors, total so far: 464373\n", + "Upserted 991 vectors, total so far: 465364\n", + "Upserted 988 vectors, total so far: 466352\n", + "Upserted 992 vectors, total so far: 467344\n", + "Upserted 993 vectors, total so far: 468337\n", + "Upserted 994 vectors, total so far: 469331\n", + "Upserted 988 vectors, total so far: 470319\n", + "Upserted 989 vectors, total so far: 471308\n", + "Upserted 993 vectors, total so far: 472301\n", + "Upserted 994 vectors, total so far: 473295\n", + "Upserted 994 vectors, total so far: 474289\n", + "Upserted 990 vectors, total so far: 475279\n", + "Upserted 983 vectors, total so far: 476262\n", + "Upserted 996 vectors, total so far: 477258\n", + "Upserted 987 vectors, total so far: 478245\n", + "Upserted 986 vectors, total so far: 479231\n", + "Upserted 992 vectors, total so far: 480223\n", + "Upserted 997 vectors, total so far: 481220\n", + "Upserted 989 vectors, total so far: 482209\n", + "Upserted 991 vectors, total so far: 483200\n", + "Upserted 992 vectors, total so far: 484192\n", + "Upserted 987 vectors, total so far: 485179\n", + "Upserted 991 vectors, total so far: 486170\n", + "Upserted 985 vectors, total so far: 487155\n", + "Upserted 990 vectors, total so far: 488145\n", + "Upserted 991 vectors, total so far: 489136\n", + "Upserted 992 vectors, total so far: 490128\n", + "Upserted 994 vectors, total so far: 491122\n", + "Upserted 993 vectors, total so far: 492115\n", + "Upserted 991 vectors, total so far: 493106\n", + "Upserted 989 vectors, total so far: 494095\n", + "Upserted 987 vectors, total so far: 495082\n", + "Upserted 985 vectors, total so far: 496067\n", + "Upserted 993 vectors, total so far: 497060\n", + "Upserted 989 vectors, total so far: 498049\n", + "Upserted 985 vectors, total so far: 499034\n", + "Upserted 991 vectors, total so far: 500025\n", + "Upserted 992 vectors, total so far: 501017\n", + "Upserted 991 vectors, total so far: 502008\n", + "Upserted 991 vectors, total so far: 502999\n", + "Upserted 990 vectors, total so far: 503989\n", + "Upserted 989 vectors, total so far: 504978\n", + "Upserted 994 vectors, total so far: 505972\n", + "Upserted 986 vectors, total so far: 506958\n", + "Upserted 995 vectors, total so far: 507953\n", + "Upserted 990 vectors, total so far: 508943\n", + "Upserted 991 vectors, total so far: 509934\n", + "Upserted 988 vectors, total so far: 510922\n", + "Upserted 993 vectors, total so far: 511915\n", + "Upserted 982 vectors, total so far: 512897\n", + "Upserted 989 vectors, total so far: 513886\n", + "Upserted 989 vectors, total so far: 514875\n", + "Upserted 988 vectors, total so far: 515863\n", + "Upserted 993 vectors, total so far: 516856\n", + "Upserted 991 vectors, total so far: 517847\n", + "Upserted 990 vectors, total so far: 518837\n", + "Upserted 989 vectors, total so far: 519826\n", + "Upserted 993 vectors, total so far: 520819\n", + "Upserted 991 vectors, total so far: 521810\n", + "Upserted 991 vectors, total so far: 522801\n", + "Upserted 991 vectors, total so far: 523792\n", + "Upserted 988 vectors, total so far: 524780\n", + "Upserted 991 vectors, total so far: 525771\n", + "Upserted 994 vectors, total so far: 526765\n", + "Upserted 990 vectors, total so far: 527755\n", + "Upserted 987 vectors, total so far: 528742\n", + "Upserted 994 vectors, total so far: 529736\n", + "Upserted 990 vectors, total so far: 530726\n", + "Upserted 989 vectors, total so far: 531715\n", + "Upserted 988 vectors, total so far: 532703\n", + "Upserted 988 vectors, total so far: 533691\n", + "Upserted 989 vectors, total so far: 534680\n", + "Upserted 988 vectors, total so far: 535668\n", + "Upserted 991 vectors, total so far: 536659\n", + "Upserted 992 vectors, total so far: 537651\n", + "Upserted 992 vectors, total so far: 538643\n", + "Upserted 984 vectors, total so far: 539627\n", + "Upserted 989 vectors, total so far: 540616\n", + "Upserted 993 vectors, total so far: 541609\n", + "Upserted 994 vectors, total so far: 542603\n", + "Upserted 988 vectors, total so far: 543591\n", + "Upserted 983 vectors, total so far: 544574\n", + "Upserted 985 vectors, total so far: 545559\n", + "Upserted 992 vectors, total so far: 546551\n", + "Upserted 989 vectors, total so far: 547540\n", + "Upserted 990 vectors, total so far: 548530\n", + "Upserted 988 vectors, total so far: 549518\n", + "Upserted 992 vectors, total so far: 550510\n", + "Upserted 984 vectors, total so far: 551494\n", + "Upserted 988 vectors, total so far: 552482\n", + "Upserted 986 vectors, total so far: 553468\n", + "Upserted 990 vectors, total so far: 554458\n", + "Upserted 994 vectors, total so far: 555452\n", + "Upserted 997 vectors, total so far: 556449\n", + "Upserted 988 vectors, total so far: 557437\n", + "Upserted 994 vectors, total so far: 558431\n", + "Upserted 982 vectors, total so far: 559413\n", + "Upserted 993 vectors, total so far: 560406\n", + "Upserted 988 vectors, total so far: 561394\n", + "Upserted 989 vectors, total so far: 562383\n", + "Upserted 987 vectors, total so far: 563370\n", + "Upserted 986 vectors, total so far: 564356\n", + "Upserted 990 vectors, total so far: 565346\n", + "Upserted 993 vectors, total so far: 566339\n", + "Upserted 992 vectors, total so far: 567331\n", + "Upserted 991 vectors, total so far: 568322\n", + "Upserted 985 vectors, total so far: 569307\n", + "Upserted 990 vectors, total so far: 570297\n", + "Upserted 989 vectors, total so far: 571286\n", + "Upserted 990 vectors, total so far: 572276\n", + "Upserted 995 vectors, total so far: 573271\n", + "Upserted 992 vectors, total so far: 574263\n", + "Upserted 995 vectors, total so far: 575258\n", + "Upserted 993 vectors, total so far: 576251\n", + "Upserted 986 vectors, total so far: 577237\n", + "Upserted 987 vectors, total so far: 578224\n", + "Upserted 988 vectors, total so far: 579212\n", + "Upserted 987 vectors, total so far: 580199\n", + "Upserted 990 vectors, total so far: 581189\n", + "Upserted 987 vectors, total so far: 582176\n", + "Upserted 994 vectors, total so far: 583170\n", + "Upserted 983 vectors, total so far: 584153\n", + "Upserted 989 vectors, total so far: 585142\n", + "Upserted 996 vectors, total so far: 586138\n", + "Upserted 993 vectors, total so far: 587131\n", + "Upserted 991 vectors, total so far: 588122\n", + "Upserted 988 vectors, total so far: 589110\n", + "Upserted 989 vectors, total so far: 590099\n", + "Upserted 992 vectors, total so far: 591091\n", + "Upserted 994 vectors, total so far: 592085\n", + "Upserted 991 vectors, total so far: 593076\n", + "Upserted 993 vectors, total so far: 594069\n", + "Upserted 988 vectors, total so far: 595057\n", + "Upserted 992 vectors, total so far: 596049\n", + "Upserted 985 vectors, total so far: 597034\n", + "Upserted 994 vectors, total so far: 598028\n", + "Upserted 988 vectors, total so far: 599016\n", + "Upserted 985 vectors, total so far: 600001\n", + "Upserted 984 vectors, total so far: 600985\n", + "Upserted 993 vectors, total so far: 601978\n", + "Upserted 991 vectors, total so far: 602969\n", + "Upserted 984 vectors, total so far: 603953\n", + "Upserted 994 vectors, total so far: 604947\n", + "Upserted 985 vectors, total so far: 605932\n", + "Upserted 991 vectors, total so far: 606923\n", + "Upserted 987 vectors, total so far: 607910\n", + "Upserted 989 vectors, total so far: 608899\n", + "Upserted 995 vectors, total so far: 609894\n", + "Upserted 992 vectors, total so far: 610886\n", + "Upserted 991 vectors, total so far: 611877\n", + "Upserted 991 vectors, total so far: 612868\n", + "Upserted 991 vectors, total so far: 613859\n", + "Upserted 996 vectors, total so far: 614855\n", + "Upserted 994 vectors, total so far: 615849\n", + "Upserted 987 vectors, total so far: 616836\n", + "Upserted 992 vectors, total so far: 617828\n", + "Upserted 988 vectors, total so far: 618816\n", + "Upserted 991 vectors, total so far: 619807\n", + "Upserted 994 vectors, total so far: 620801\n", + "Upserted 988 vectors, total so far: 621789\n", + "Upserted 992 vectors, total so far: 622781\n", + "Upserted 991 vectors, total so far: 623772\n", + "Upserted 984 vectors, total so far: 624756\n", + "Upserted 987 vectors, total so far: 625743\n", + "Upserted 992 vectors, total so far: 626735\n", + "Upserted 992 vectors, total so far: 627727\n", + "Upserted 995 vectors, total so far: 628722\n", + "Upserted 986 vectors, total so far: 629708\n", + "Upserted 990 vectors, total so far: 630698\n", + "Upserted 992 vectors, total so far: 631690\n", + "Upserted 991 vectors, total so far: 632681\n", + "Upserted 990 vectors, total so far: 633671\n", + "Upserted 993 vectors, total so far: 634664\n", + "Upserted 992 vectors, total so far: 635656\n", + "Upserted 984 vectors, total so far: 636640\n", + "Upserted 989 vectors, total so far: 637629\n", + "Upserted 992 vectors, total so far: 638621\n", + "Upserted 989 vectors, total so far: 639610\n", + "Upserted 990 vectors, total so far: 640600\n", + "Upserted 991 vectors, total so far: 641591\n", + "Upserted 987 vectors, total so far: 642578\n", + "Upserted 992 vectors, total so far: 643570\n", + "Upserted 993 vectors, total so far: 644563\n", + "Upserted 994 vectors, total so far: 645557\n", + "Upserted 993 vectors, total so far: 646550\n", + "Upserted 989 vectors, total so far: 647539\n", + "Upserted 986 vectors, total so far: 648525\n", + "Upserted 995 vectors, total so far: 649520\n", + "Upserted 990 vectors, total so far: 650510\n", + "Upserted 989 vectors, total so far: 651499\n", + "Upserted 988 vectors, total so far: 652487\n", + "Upserted 991 vectors, total so far: 653478\n", + "Upserted 988 vectors, total so far: 654466\n", + "Upserted 988 vectors, total so far: 655454\n", + "Upserted 992 vectors, total so far: 656446\n", + "Upserted 991 vectors, total so far: 657437\n", + "Upserted 992 vectors, total so far: 658429\n", + "Upserted 989 vectors, total so far: 659418\n", + "Upserted 993 vectors, total so far: 660411\n", + "Upserted 992 vectors, total so far: 661403\n", + "Upserted 992 vectors, total so far: 662395\n", + "Upserted 990 vectors, total so far: 663385\n", + "Upserted 988 vectors, total so far: 664373\n", + "Upserted 990 vectors, total so far: 665363\n", + "Upserted 989 vectors, total so far: 666352\n", + "Upserted 987 vectors, total so far: 667339\n", + "Upserted 989 vectors, total so far: 668328\n", + "Upserted 990 vectors, total so far: 669318\n", + "Upserted 991 vectors, total so far: 670309\n", + "Upserted 992 vectors, total so far: 671301\n", + "Upserted 992 vectors, total so far: 672293\n", + "Upserted 990 vectors, total so far: 673283\n", + "Upserted 990 vectors, total so far: 674273\n", + "Upserted 989 vectors, total so far: 675262\n", + "Upserted 992 vectors, total so far: 676254\n", + "Upserted 988 vectors, total so far: 677242\n", + "Upserted 989 vectors, total so far: 678231\n", + "Upserted 987 vectors, total so far: 679218\n", + "Upserted 988 vectors, total so far: 680206\n", + "Upserted 993 vectors, total so far: 681199\n", + "Upserted 992 vectors, total so far: 682191\n", + "Upserted 989 vectors, total so far: 683180\n", + "Upserted 991 vectors, total so far: 684171\n", + "Upserted 996 vectors, total so far: 685167\n", + "Upserted 988 vectors, total so far: 686155\n", + "Upserted 989 vectors, total so far: 687144\n", + "Upserted 988 vectors, total so far: 688132\n", + "Upserted 989 vectors, total so far: 689121\n", + "Upserted 988 vectors, total so far: 690109\n", + "Upserted 996 vectors, total so far: 691105\n", + "Upserted 986 vectors, total so far: 692091\n", + "Upserted 994 vectors, total so far: 693085\n", + "Upserted 994 vectors, total so far: 694079\n", + "Upserted 988 vectors, total so far: 695067\n", + "Upserted 990 vectors, total so far: 696057\n", + "Upserted 986 vectors, total so far: 697043\n", + "Upserted 991 vectors, total so far: 698034\n", + "Upserted 992 vectors, total so far: 699026\n", + "Upserted 988 vectors, total so far: 700014\n", + "Upserted 988 vectors, total so far: 701002\n", + "Upserted 993 vectors, total so far: 701995\n", + "Upserted 989 vectors, total so far: 702984\n", + "Upserted 986 vectors, total so far: 703970\n", + "Upserted 986 vectors, total so far: 704956\n", + "Upserted 997 vectors, total so far: 705953\n", + "Upserted 996 vectors, total so far: 706949\n", + "Upserted 994 vectors, total so far: 707943\n", + "Upserted 979 vectors, total so far: 708922\n", + "Upserted 988 vectors, total so far: 709910\n", + "Upserted 989 vectors, total so far: 710899\n", + "Upserted 992 vectors, total so far: 711891\n", + "Upserted 990 vectors, total so far: 712881\n", + "Upserted 988 vectors, total so far: 713869\n", + "Upserted 987 vectors, total so far: 714856\n", + "Upserted 991 vectors, total so far: 715847\n", + "Upserted 989 vectors, total so far: 716836\n", + "Upserted 989 vectors, total so far: 717825\n", + "Upserted 993 vectors, total so far: 718818\n", + "Upserted 981 vectors, total so far: 719799\n", + "Upserted 990 vectors, total so far: 720789\n", + "Upserted 993 vectors, total so far: 721782\n", + "Upserted 994 vectors, total so far: 722776\n", + "Upserted 995 vectors, total so far: 723771\n", + "Upserted 992 vectors, total so far: 724763\n", + "Upserted 982 vectors, total so far: 725745\n", + "Upserted 989 vectors, total so far: 726734\n", + "Upserted 984 vectors, total so far: 727718\n", + "Upserted 992 vectors, total so far: 728710\n", + "Upserted 988 vectors, total so far: 729698\n", + "Upserted 986 vectors, total so far: 730684\n", + "Upserted 993 vectors, total so far: 731677\n", + "Upserted 993 vectors, total so far: 732670\n", + "Upserted 988 vectors, total so far: 733658\n", + "Upserted 988 vectors, total so far: 734646\n", + "Upserted 991 vectors, total so far: 735637\n", + "Upserted 993 vectors, total so far: 736630\n", + "Upserted 992 vectors, total so far: 737622\n", + "Upserted 986 vectors, total so far: 738608\n", + "Upserted 991 vectors, total so far: 739599\n", + "Upserted 989 vectors, total so far: 740588\n", + "Upserted 981 vectors, total so far: 741569\n", + "Upserted 995 vectors, total so far: 742564\n", + "Upserted 995 vectors, total so far: 743559\n", + "Upserted 985 vectors, total so far: 744544\n", + "Upserted 991 vectors, total so far: 745535\n", + "Upserted 985 vectors, total so far: 746520\n", + "Upserted 987 vectors, total so far: 747507\n", + "Upserted 991 vectors, total so far: 748498\n", + "Upserted 994 vectors, total so far: 749492\n", + "Upserted 987 vectors, total so far: 750479\n", + "Upserted 991 vectors, total so far: 751470\n", + "Upserted 993 vectors, total so far: 752463\n", + "Upserted 992 vectors, total so far: 753455\n", + "Upserted 986 vectors, total so far: 754441\n", + "Upserted 987 vectors, total so far: 755428\n", + "Upserted 989 vectors, total so far: 756417\n", + "Upserted 992 vectors, total so far: 757409\n", + "Upserted 991 vectors, total so far: 758400\n", + "Upserted 985 vectors, total so far: 759385\n", + "Upserted 985 vectors, total so far: 760370\n", + "Upserted 988 vectors, total so far: 761358\n", + "Upserted 991 vectors, total so far: 762349\n", + "Upserted 989 vectors, total so far: 763338\n", + "Upserted 996 vectors, total so far: 764334\n", + "Upserted 992 vectors, total so far: 765326\n", + "Upserted 987 vectors, total so far: 766313\n", + "Upserted 986 vectors, total so far: 767299\n", + "Upserted 988 vectors, total so far: 768287\n", + "Upserted 992 vectors, total so far: 769279\n", + "Upserted 990 vectors, total so far: 770269\n", + "Upserted 987 vectors, total so far: 771256\n", + "Upserted 987 vectors, total so far: 772243\n", + "Upserted 988 vectors, total so far: 773231\n", + "Upserted 993 vectors, total so far: 774224\n", + "Upserted 992 vectors, total so far: 775216\n", + "Upserted 990 vectors, total so far: 776206\n", + "Upserted 991 vectors, total so far: 777197\n", + "Upserted 986 vectors, total so far: 778183\n", + "Upserted 990 vectors, total so far: 779173\n", + "Upserted 993 vectors, total so far: 780166\n", + "Upserted 990 vectors, total so far: 781156\n", + "Upserted 993 vectors, total so far: 782149\n", + "Upserted 987 vectors, total so far: 783136\n", + "Upserted 995 vectors, total so far: 784131\n", + "Upserted 991 vectors, total so far: 785122\n", + "Upserted 989 vectors, total so far: 786111\n", + "Upserted 991 vectors, total so far: 787102\n", + "Upserted 991 vectors, total so far: 788093\n", + "Upserted 992 vectors, total so far: 789085\n", + "Upserted 988 vectors, total so far: 790073\n", + "Upserted 992 vectors, total so far: 791065\n", + "Upserted 986 vectors, total so far: 792051\n", + "Upserted 987 vectors, total so far: 793038\n", + "Upserted 990 vectors, total so far: 794028\n", + "Upserted 990 vectors, total so far: 795018\n", + "Upserted 993 vectors, total so far: 796011\n", + "Upserted 983 vectors, total so far: 796994\n", + "Upserted 988 vectors, total so far: 797982\n", + "Upserted 985 vectors, total so far: 798967\n", + "Upserted 993 vectors, total so far: 799960\n", + "Upserted 987 vectors, total so far: 800947\n", + "Upserted 991 vectors, total so far: 801938\n", + "Upserted 991 vectors, total so far: 802929\n", + "Upserted 989 vectors, total so far: 803918\n", + "Upserted 988 vectors, total so far: 804906\n", + "Upserted 990 vectors, total so far: 805896\n", + "Upserted 992 vectors, total so far: 806888\n", + "Upserted 988 vectors, total so far: 807876\n", + "Upserted 991 vectors, total so far: 808867\n", + "Upserted 992 vectors, total so far: 809859\n", + "Upserted 988 vectors, total so far: 810847\n", + "Upserted 987 vectors, total so far: 811834\n", + "Upserted 993 vectors, total so far: 812827\n", + "Upserted 994 vectors, total so far: 813821\n", + "Upserted 992 vectors, total so far: 814813\n", + "Upserted 995 vectors, total so far: 815808\n", + "Upserted 989 vectors, total so far: 816797\n", + "Upserted 987 vectors, total so far: 817784\n", + "Upserted 986 vectors, total so far: 818770\n", + "Upserted 993 vectors, total so far: 819763\n", + "Upserted 993 vectors, total so far: 820756\n", + "Upserted 989 vectors, total so far: 821745\n", + "Upserted 990 vectors, total so far: 822735\n", + "Upserted 991 vectors, total so far: 823726\n", + "Upserted 988 vectors, total so far: 824714\n", + "Upserted 986 vectors, total so far: 825700\n", + "Upserted 985 vectors, total so far: 826685\n", + "Upserted 992 vectors, total so far: 827677\n", + "Upserted 987 vectors, total so far: 828664\n", + "Upserted 989 vectors, total so far: 829653\n", + "Upserted 990 vectors, total so far: 830643\n", + "Upserted 984 vectors, total so far: 831627\n", + "Upserted 986 vectors, total so far: 832613\n", + "Upserted 982 vectors, total so far: 833595\n", + "Upserted 992 vectors, total so far: 834587\n", + "Upserted 992 vectors, total so far: 835579\n", + "Upserted 990 vectors, total so far: 836569\n", + "Upserted 987 vectors, total so far: 837556\n", + "Upserted 989 vectors, total so far: 838545\n", + "Upserted 987 vectors, total so far: 839532\n", + "Upserted 991 vectors, total so far: 840523\n", + "Upserted 993 vectors, total so far: 841516\n", + "Upserted 996 vectors, total so far: 842512\n", + "Upserted 992 vectors, total so far: 843504\n", + "Upserted 993 vectors, total so far: 844497\n", + "Upserted 988 vectors, total so far: 845485\n", + "Upserted 987 vectors, total so far: 846472\n", + "Upserted 994 vectors, total so far: 847466\n", + "Upserted 990 vectors, total so far: 848456\n", + "Upserted 988 vectors, total so far: 849444\n", + "Upserted 985 vectors, total so far: 850429\n", + "Upserted 986 vectors, total so far: 851415\n", + "Upserted 989 vectors, total so far: 852404\n", + "Upserted 989 vectors, total so far: 853393\n", + "Upserted 994 vectors, total so far: 854387\n", + "Upserted 994 vectors, total so far: 855381\n", + "Upserted 992 vectors, total so far: 856373\n", + "Upserted 991 vectors, total so far: 857364\n", + "Upserted 993 vectors, total so far: 858357\n", + "Upserted 991 vectors, total so far: 859348\n", + "Upserted 986 vectors, total so far: 860334\n", + "Upserted 994 vectors, total so far: 861328\n", + "Upserted 992 vectors, total so far: 862320\n", + "Upserted 989 vectors, total so far: 863309\n", + "Upserted 991 vectors, total so far: 864300\n", + "Upserted 993 vectors, total so far: 865293\n", + "Upserted 985 vectors, total so far: 866278\n", + "Upserted 991 vectors, total so far: 867269\n", + "Upserted 990 vectors, total so far: 868259\n", + "Upserted 991 vectors, total so far: 869250\n", + "Upserted 993 vectors, total so far: 870243\n", + "Upserted 985 vectors, total so far: 871228\n", + "Upserted 985 vectors, total so far: 872213\n", + "Upserted 989 vectors, total so far: 873202\n", + "Upserted 990 vectors, total so far: 874192\n", + "Upserted 988 vectors, total so far: 875180\n", + "Upserted 987 vectors, total so far: 876167\n", + "Upserted 989 vectors, total so far: 877156\n", + "Upserted 993 vectors, total so far: 878149\n", + "Upserted 992 vectors, total so far: 879141\n", + "Upserted 990 vectors, total so far: 880131\n", + "Upserted 994 vectors, total so far: 881125\n", + "Upserted 984 vectors, total so far: 882109\n", + "Upserted 991 vectors, total so far: 883100\n", + "Upserted 987 vectors, total so far: 884087\n", + "Upserted 993 vectors, total so far: 885080\n", + "Upserted 988 vectors, total so far: 886068\n", + "Upserted 992 vectors, total so far: 887060\n", + "Upserted 991 vectors, total so far: 888051\n", + "Upserted 992 vectors, total so far: 889043\n", + "Upserted 989 vectors, total so far: 890032\n", + "Upserted 994 vectors, total so far: 891026\n", + "Upserted 992 vectors, total so far: 892018\n", + "Upserted 992 vectors, total so far: 893010\n", + "Upserted 989 vectors, total so far: 893999\n", + "Upserted 986 vectors, total so far: 894985\n", + "Upserted 992 vectors, total so far: 895977\n", + "Upserted 988 vectors, total so far: 896965\n", + "Upserted 992 vectors, total so far: 897957\n", + "Upserted 989 vectors, total so far: 898946\n", + "Upserted 994 vectors, total so far: 899940\n", + "Upserted 984 vectors, total so far: 900924\n", + "Upserted 985 vectors, total so far: 901909\n", + "Upserted 987 vectors, total so far: 902896\n", + "Upserted 989 vectors, total so far: 903885\n", + "Upserted 995 vectors, total so far: 904880\n", + "Upserted 990 vectors, total so far: 905870\n", + "Upserted 988 vectors, total so far: 906858\n", + "Upserted 995 vectors, total so far: 907853\n", + "Upserted 989 vectors, total so far: 908842\n", + "Upserted 987 vectors, total so far: 909829\n", + "Upserted 992 vectors, total so far: 910821\n", + "Upserted 988 vectors, total so far: 911809\n", + "Upserted 985 vectors, total so far: 912794\n", + "Upserted 986 vectors, total so far: 913780\n", + "Upserted 989 vectors, total so far: 914769\n", + "Upserted 990 vectors, total so far: 915759\n", + "Upserted 990 vectors, total so far: 916749\n", + "Upserted 992 vectors, total so far: 917741\n", + "Upserted 991 vectors, total so far: 918732\n", + "Upserted 991 vectors, total so far: 919723\n", + "Upserted 992 vectors, total so far: 920715\n", + "Upserted 990 vectors, total so far: 921705\n", + "Upserted 991 vectors, total so far: 922696\n", + "Upserted 986 vectors, total so far: 923682\n", + "Upserted 985 vectors, total so far: 924667\n", + "Upserted 990 vectors, total so far: 925657\n", + "Upserted 991 vectors, total so far: 926648\n", + "Upserted 990 vectors, total so far: 927638\n", + "Upserted 985 vectors, total so far: 928623\n", + "Upserted 993 vectors, total so far: 929616\n", + "Upserted 985 vectors, total so far: 930601\n", + "Upserted 982 vectors, total so far: 931583\n", + "Upserted 991 vectors, total so far: 932574\n", + "Upserted 991 vectors, total so far: 933565\n", + "Upserted 990 vectors, total so far: 934555\n", + "Upserted 994 vectors, total so far: 935549\n", + "Upserted 989 vectors, total so far: 936538\n", + "Upserted 989 vectors, total so far: 937527\n", + "Upserted 987 vectors, total so far: 938514\n", + "Upserted 991 vectors, total so far: 939505\n", + "Upserted 992 vectors, total so far: 940497\n", + "Upserted 984 vectors, total so far: 941481\n", + "Upserted 992 vectors, total so far: 942473\n", + "Upserted 990 vectors, total so far: 943463\n", + "Upserted 990 vectors, total so far: 944453\n", + "Upserted 993 vectors, total so far: 945446\n", + "Upserted 990 vectors, total so far: 946436\n", + "Upserted 978 vectors, total so far: 947414\n", + "Upserted 988 vectors, total so far: 948402\n", + "Upserted 986 vectors, total so far: 949388\n", + "Upserted 990 vectors, total so far: 950378\n", + "Upserted 994 vectors, total so far: 951372\n", + "Upserted 988 vectors, total so far: 952360\n", + "Upserted 989 vectors, total so far: 953349\n", + "Upserted 987 vectors, total so far: 954336\n", + "Upserted 991 vectors, total so far: 955327\n", + "Upserted 991 vectors, total so far: 956318\n", + "Upserted 994 vectors, total so far: 957312\n", + "Upserted 992 vectors, total so far: 958304\n", + "Upserted 988 vectors, total so far: 959292\n", + "Upserted 989 vectors, total so far: 960281\n", + "Upserted 982 vectors, total so far: 961263\n", + "Upserted 992 vectors, total so far: 962255\n", + "Upserted 986 vectors, total so far: 963241\n", + "Upserted 995 vectors, total so far: 964236\n", + "Upserted 992 vectors, total so far: 965228\n", + "Upserted 990 vectors, total so far: 966218\n", + "Upserted 990 vectors, total so far: 967208\n", + "Upserted 993 vectors, total so far: 968201\n", + "Upserted 993 vectors, total so far: 969194\n", + "Upserted 990 vectors, total so far: 970184\n", + "Upserted 994 vectors, total so far: 971178\n", + "Upserted 995 vectors, total so far: 972173\n", + "Upserted 996 vectors, total so far: 973169\n", + "Upserted 993 vectors, total so far: 974162\n", + "Upserted 992 vectors, total so far: 975154\n", + "Upserted 986 vectors, total so far: 976140\n", + "Upserted 986 vectors, total so far: 977126\n", + "Upserted 983 vectors, total so far: 978109\n", + "Upserted 992 vectors, total so far: 979101\n", + "Upserted 991 vectors, total so far: 980092\n", + "Upserted 992 vectors, total so far: 981084\n", + "Upserted 991 vectors, total so far: 982075\n", + "Upserted 992 vectors, total so far: 983067\n", + "Upserted 989 vectors, total so far: 984056\n", + "Upserted 989 vectors, total so far: 985045\n", + "Upserted 991 vectors, total so far: 986036\n", + "Upserted 989 vectors, total so far: 987025\n", + "Upserted 991 vectors, total so far: 988016\n", + "Upserted 993 vectors, total so far: 989009\n", + "Upserted 992 vectors, total so far: 990001\n", + "Done! Total inserted: 990001 vectors with id >= 9999\n", + "Time taken: 956.1230444270186\n" + ] + } + ], + "source": [ + "#For reinserting deleted vectors (int filter - lte)\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "import time\n", + "\n", + "# User inputs\n", + "int_rate_insert = \"99p\" # change to 1p, 20p, 50p, 99p\n", + "batch_size = 1000\n", + "\n", + "# Map int_rate to gte threshold (same as delete)\n", + "lt_map = {\n", + " \"1p\": 989999,\n", + " \"20p\": 799999,\n", + " \"50p\": 499999,\n", + " \"99p\": 9999,\n", + "}\n", + "lt_threshold = lt_map[int_rate_insert]\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "emb_path = os.path.join(dataset_path, \"shuffle_train.parquet\")\n", + "\n", + "# Process shuffle_train in batches to avoid RAM issues\n", + "emb_file = pq.ParquetFile(emb_path)\n", + "total_inserted = 0\n", + "\n", + "start = time.perf_counter()\n", + "for batch in emb_file.iter_batches(batch_size=batch_size):\n", + " batch_df = batch.to_pandas()\n", + "\n", + " # Filter by id >= lte_threshold\n", + " batch_df = batch_df[batch_df[\"id\"] >= lt_threshold]\n", + "\n", + " if batch_df.empty:\n", + " continue\n", + "\n", + " batch_vectors = []\n", + " for _, row in batch_df.iterrows():\n", + " vector_data = {\n", + " \"id\": str(row[\"id\"]),\n", + " \"vector\": row[\"emb\"],\n", + " \"meta\": {\"id\": row[\"id\"]},\n", + " \"filter\": {\n", + " \"id\": row[\"id\"],\n", + " }\n", + " }\n", + " batch_vectors.append(vector_data)\n", + "\n", + " index.upsert(batch_vectors)\n", + " total_inserted += len(batch_vectors)\n", + " print(f\"Upserted {len(batch_vectors)} vectors, total so far: {total_inserted}\")\n", + "\n", + "end = time.perf_counter()\n", + "print(f\"Done! Total inserted: {total_inserted} vectors with id >= {lt_threshold}\")\n", + "print(\"Time taken:\", end - start)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_labelfilter_0503_1_adup8',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 1000000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "index.describe()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv2", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/intfilterscript_final_lte.ipynb b/intfilterscript_final_lte.ipynb new file mode 100644 index 000000000..d675e33d8 --- /dev/null +++ b/intfilterscript_final_lte.ipynb @@ -0,0 +1,6551 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: pandas in ./venv2/lib/python3.11/site-packages (3.0.1)\n", + "Requirement already satisfied: pyarrow in ./venv2/lib/python3.11/site-packages (23.0.1)\n", + "Requirement already satisfied: numpy>=1.26.0 in ./venv2/lib/python3.11/site-packages (from pandas) (2.4.2)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in ./venv2/lib/python3.11/site-packages (from pandas) (2.9.0.post0)\n", + "Requirement already satisfied: six>=1.5 in ./venv2/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m26.0.1\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "pip install pandas pyarrow" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: endee in ./venv2/lib/python3.11/site-packages (0.1.13)\n", + "Requirement already satisfied: requests>=2.28.0 in ./venv2/lib/python3.11/site-packages (from endee) (2.32.5)\n", + "Requirement already satisfied: httpx>=0.28.1 in ./venv2/lib/python3.11/site-packages (from httpx[http2]>=0.28.1->endee) (0.28.1)\n", + "Requirement already satisfied: numpy>=2.2.4 in ./venv2/lib/python3.11/site-packages (from endee) (2.4.2)\n", + "Requirement already satisfied: msgpack>=1.1.0 in ./venv2/lib/python3.11/site-packages (from endee) (1.1.2)\n", + "Requirement already satisfied: orjson>=3.11.5 in ./venv2/lib/python3.11/site-packages (from endee) (3.11.7)\n", + "Requirement already satisfied: pydantic>=2.0.0 in ./venv2/lib/python3.11/site-packages (from endee) (2.12.5)\n", + "Requirement already satisfied: anyio in ./venv2/lib/python3.11/site-packages (from httpx>=0.28.1->httpx[http2]>=0.28.1->endee) (4.12.1)\n", + "Requirement already satisfied: certifi in ./venv2/lib/python3.11/site-packages (from httpx>=0.28.1->httpx[http2]>=0.28.1->endee) (2026.1.4)\n", + "Requirement already satisfied: httpcore==1.* in ./venv2/lib/python3.11/site-packages (from httpx>=0.28.1->httpx[http2]>=0.28.1->endee) (1.0.9)\n", + "Requirement already satisfied: idna in ./venv2/lib/python3.11/site-packages (from httpx>=0.28.1->httpx[http2]>=0.28.1->endee) (3.11)\n", + "Requirement already satisfied: h11>=0.16 in ./venv2/lib/python3.11/site-packages (from httpcore==1.*->httpx>=0.28.1->httpx[http2]>=0.28.1->endee) (0.16.0)\n", + "Requirement already satisfied: h2<5,>=3 in ./venv2/lib/python3.11/site-packages (from httpx[http2]>=0.28.1->endee) (4.3.0)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in ./venv2/lib/python3.11/site-packages (from pydantic>=2.0.0->endee) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.41.5 in ./venv2/lib/python3.11/site-packages (from pydantic>=2.0.0->endee) (2.41.5)\n", + "Requirement already satisfied: typing-extensions>=4.14.1 in ./venv2/lib/python3.11/site-packages (from pydantic>=2.0.0->endee) (4.15.0)\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in ./venv2/lib/python3.11/site-packages (from pydantic>=2.0.0->endee) (0.4.2)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in ./venv2/lib/python3.11/site-packages (from requests>=2.28.0->endee) (3.4.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in ./venv2/lib/python3.11/site-packages (from requests>=2.28.0->endee) (2.6.3)\n", + "Requirement already satisfied: hyperframe<7,>=6.1 in ./venv2/lib/python3.11/site-packages (from h2<5,>=3->httpx[http2]>=0.28.1->endee) (6.1.0)\n", + "Requirement already satisfied: hpack<5,>=4.1 in ./venv2/lib/python3.11/site-packages (from h2<5,>=3->httpx[http2]>=0.28.1->endee) (4.1.0)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m26.0.1\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "pip install endee" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Name: endee\n", + "Version: 0.1.13\n", + "Summary: Endee is the Next-Generation Vector Database for Scalable, High-Performance AI\n", + "Home-page: https://endee.io\n", + "Author: Endee Labs\n", + "Author-email: dev@endee.io\n", + "License: \n", + "Location: /home/debian/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages\n", + "Requires: httpx, msgpack, numpy, orjson, pydantic, requests\n", + "Required-by: \n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "pip show endee" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "base_path = \"/home/debian/latest_VDB/VectorDBBench/vectordataset_lte\"\n", + "dataset_folder = \"cohere/cohere_medium_1m\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " id emb\n", + "0 322406 [0.19600096, -0.5270862, -0.29519123, 0.429556...\n", + "1 337610 [0.32829463, -0.055560444, -0.06252914, -0.101...\n", + "2 714728 [-0.054155815, 0.009554057, 0.24696879, -0.142...\n", + "3 354737 [0.2501778, 0.017853737, -0.43795395, 0.522256...\n", + "4 290979 [0.3444257, -0.62243223, -0.20940691, -0.08620...\n" + ] + } + ], + "source": [ + "#For checking parquet file contents\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "file_name = \"shuffle_train.parquet\" #input\n", + "\n", + "# Build full path\n", + "file_path = os.path.join(base_path, dataset_folder, file_name)\n", + "\n", + "\n", + "parquet_file = pq.ParquetFile(file_path)\n", + "\n", + "# Read only first batch of rows\n", + "first_batch = next(parquet_file.iter_batches(batch_size=5))\n", + "preview = first_batch.to_pandas()\n", + "\n", + "for col in preview.columns:\n", + " if preview[col].dtype == object and isinstance(preview[col].iloc[0], list):\n", + " preview[col] = preview[col].apply(lambda x: x[:5] if x is not None else x)\n", + "\n", + "print(preview)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from endee import Endee\n", + "client = Endee(token=\"localtest\")\n", + "client.set_base_url(\"http://148.113.54.173:8080/api/v1\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "test_1M_vaib_labelfilter_0503_1\n", + "1000000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup1\n", + "1000000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup3\n", + "1000000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup4\n", + "1000000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup5\n", + "200000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup6\n", + "10000\n", + "\t\n" + ] + } + ], + "source": [ + "#For checking indexes\n", + "for obj in client.list_indexes().get('indexes'):\n", + " print(obj['name'])\n", + " print(obj['total_elements'])\n", + " print('\\t')" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "#give the index name\n", + "index_name = \"test_1M_vaib_labelfilter_0503_1_adup6\"\n", + "index = client.get_index(index_name)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_labelfilter_0503_1_adup6',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 10000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Finding all query IDs with recall < 1.0 ...\n", + "\n", + "Query ID: 0\n", + "Returned IDs: \n", + "Ground Truth: 940,707,7629,476,9354,8339,9711,7862,3526,3651,5672,1569,4213,8423,7248,7091,536,6181,5945,7663,7193,5093,3532,5840,6714,9207,4581,5040,4688,9511\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 1\n", + "Returned IDs: \n", + "Ground Truth: 7793,2368,525,646,4998,181,9359,7788,987,7515,5872,4948,6100,9093,3315,6643,8754,7594,40,1885,4400,7870,312,179,6219,4729,4132,8620,4586,285\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 2\n", + "Returned IDs: 2927,91,6170\n", + "Ground Truth: 4031,6525,1838,608,1490,2174,5885,8649,4497,9138,8227,4707,2927,2838,290,3707,3474,3738,2848,6854,3622,3013,5110,3755,4974,1683,7348,2012,6511,934\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 3\n", + "Returned IDs: 9796\n", + "Ground Truth: 3745,3136,208,8661,9742,9191,7532,129,3890,9038,3477,2241,8561,9796,9246,1416,7621,4083,6538,7052,2527,7476,5862,7923,112,5852,3831,2187,8995,7554\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 4\n", + "Returned IDs: \n", + "Ground Truth: 7968,2957,9161,8580,5436,7899,5870,4256,2141,4904,2942,85,2078,9288,6915,1963,5662,1548,7800,2356,5074,6992,2837,8900,2437,2315,679,2239,3258,1141\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 5\n", + "Returned IDs: 6324,427\n", + "Ground Truth: 5885,608,5851,1861,2012,290,9711,8423,2243,6854,2927,1518,7966,3474,2174,8892,1595,192,8649,9770,2196,9777,6752,2998,7267,7816,5404,6220,7069,9875\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 6\n", + "Returned IDs: \n", + "Ground Truth: 9711,751,6630,1301,9116,7544,5885,4914,7193,6794,608,3749,5972,9024,7466,6854,3707,536,3627,6153,517,2217,2524,1922,5568,2618,3734,2293,3622,3526\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 7\n", + "Returned IDs: \n", + "Ground Truth: 8366,7807,3757,8879,5465,2434,9371,8233,7951,3954,8056,214,2696,9935,5217,3033,6551,8462,3319,4400,7078,8026,1159,9788,4293,8858,7924,8914,3751,8112\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 8\n", + "Returned IDs: \n", + "Ground Truth: 2174,6854,5110,6220,4634,8625,9345,290,7217,6307,1518,7542,536,3738,4308,9593,7966,6324,649,8569,2196,9738,3800,7943,2646,1861,2468,7570,3759,7027\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 9\n", + "Returned IDs: \n", + "Ground Truth: 608,3622,5885,2012,536,3477,6854,828,2998,2196,4887,2927,3734,370,6662,9770,7544,9875,1817,3987,6307,3474,8594,9219,9287,2404,8302,3755,5278,8879\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 10\n", + "Returned IDs: \n", + "Ground Truth: 3184,399,8376,1435,5929,463,5106,4337,9784,1562,6842,5898,352,9250,4683,1344,4293,6235,5649,5597,2027,698,3935,6100,1004,7629,4256,2506,883,4231\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 11\n", + "Returned IDs: \n", + "Ground Truth: 350,7568,1178,4269,7750,36,3993,8140,8312,4308,5961,6029,4205,6883,1223,3491,3775,9991,2862,9035,6789,8433,9430,7869,8167,9356,8038,9335,825,1837\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 12\n", + "Returned IDs: 2367,5708,5302\n", + "Ground Truth: 3588,8375,3781,2277,5500,9093,9497,1473,9879,2915,2367,8559,2434,3873,8097,2039,6252,7788,525,7566,9942,4061,4113,8099,9243,3606,4903,5784,5496,6741\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 13\n", + "Returned IDs: \n", + "Ground Truth: 1907,5940,664,5787,750,8796,2694,6603,8223,6589,4053,1531,7455,5989,6459,7413,9562,600,1805,8545,2571,3201,632,6688,8932,856,7673,3042,5514,7634\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 14\n", + "Returned IDs: \n", + "Ground Truth: 4061,4689,3445,1278,4113,1619,6422,7511,8953,8643,4693,5470,7284,3216,2300,233,9459,7466,6624,9047,5489,2735,2608,5858,3116,8080,4574,654,8014,4345\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 15\n", + "Returned IDs: \n", + "Ground Truth: 5796,1663,4252,8342,6268,5671,6693,1866,1528,4701,4256,5294,7687,3110,2374,3717,9745,5776,5971,9685,2530,615,6451,5675,5288,976,9824,5806,1817,9371\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 16\n", + "Returned IDs: \n", + "Ground Truth: 3235,6574,9365,5887,4796,4896,8119,566,6235,6815,1685,7089,5617,4015,7983,8903,7824,7789,1090,4721,5658,3191,7432,9354,7076,7949,3501,8149,7557,3056\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 17\n", + "Returned IDs: \n", + "Ground Truth: 1687,7352,1117,309,1562,2146,4844,6839,5509,8864,2556,506,9543,9055,5700,8583,9024,9750,9875,1253,1944,3578,8988,3273,5561,2493,7412,6041,8914,536\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 18\n", + "Returned IDs: \n", + "Ground Truth: 9563,7554,8589,655,6867,8587,7900,9900,6372,4000,2194,1132,8436,1663,1866,2527,1886,9994,828,7059,6960,1963,792,8161,6884,9481,5007,4730,5443,4141\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 19\n", + "Returned IDs: \n", + "Ground Truth: 7576,4215,4889,706,5819,6073,4557,9622,7217,9641,8892,9875,4039,5278,9489,3116,6610,2019,5472,5034,4911,7721,6814,8854,3610,5464,5509,6321,9770,643\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 20\n", + "Returned IDs: \n", + "Ground Truth: 86,1371,4948,1963,9093,2696,233,3258,8736,9359,5572,7594,7660,8279,4574,6722,6408,9038,6608,4562,2850,7733,8379,7780,1885,3825,3477,2473,314,4943\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 21\n", + "Returned IDs: \n", + "Ground Truth: 1956,2352,4634,8558,5089,1497,1702,6073,8625,3347,4497,7358,1861,7809,6635,7225,6000,3199,7029,3819,2174,6683,2712,5751,9676,4521,5110,8155,8302,3436\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 22\n", + "Returned IDs: \n", + "Ground Truth: 8720,1140,9198,159,1764,7338,2,339,3999,654,669,9426,9748,3681,2728,1816,3056,240,323,4485,8112,8238,4061,9743,3593,860,5711,7377,2139,9858\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 23\n", + "Returned IDs: \n", + "Ground Truth: 690,6662,7452,3185,9345,7125,5675,1602,8411,5359,1518,1861,761,5458,3086,7625,1935,8625,3596,4398,3477,7542,9324,1584,9071,6106,36,6557,3800,7259\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 24\n", + "Returned IDs: \n", + "Ground Truth: 2930,8090,8800,6689,5240,5543,7389,4884,7859,309,137,438,3377,2916,1944,1151,5940,3292,6764,4075,6168,2147,4688,9468,4548,6037,6513,6151,9207,7193\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 25\n", + "Returned IDs: \n", + "Ground Truth: 5934,2595,7146,3391,420,9595,7798,9371,1457,6247,5444,798,8093,498,6532,4293,909,6898,7402,463,3629,5649,1965,9555,7089,926,1818,908,8856,1273\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 26\n", + "Returned IDs: \n", + "Ground Truth: 3731,4831,776,2309,3518,3033,9214,5575,3220,8413,8312,7345,9912,2332,8871,2899,3163,8342,539,5776,4701,1944,4157,4134,5860,4582,4256,7051,6179,8087\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 27\n", + "Returned IDs: \n", + "Ground Truth: 4535,9939,6219,6613,628,4514,898,1885,1105,7594,4821,3633,2736,2761,5034,3280,6825,1670,5925,3477,2122,9751,2642,2359,7404,1985,5540,4308,4991,4008\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 28\n", + "Returned IDs: \n", + "Ground Truth: 9512,7925,8687,9733,9853,5805,1157,1399,1531,3094,9755,2596,5514,8753,9984,664,7352,2937,386,6615,744,254,4174,3867,3436,7604,7702,309,2477,8169\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 29\n", + "Returned IDs: \n", + "Ground Truth: 7836,9097,9878,1954,5432,2753,1134,4044,1996,7285,9296,8025,9576,3695,736,9827,3037,5403,5573,7376,1920,9101,4157,624,6418,292,2927,9864,3973,3166\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 30\n", + "Returned IDs: \n", + "Ground Truth: 4779,7426,7905,5195,3907,8342,1663,1963,5892,5067,9069,4157,3110,1970,9639,9563,5175,3424,615,4701,5848,7554,828,2519,1903,6297,3477,6771,6933,1150\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 31\n", + "Returned IDs: \n", + "Ground Truth: 8892,1894,3947,5363,3175,7669,1881,8844,8653,7353,1068,7204,3687,5330,5243,6288,8164,3772,9582,6097,2520,6563,4991,3484,467,1691,8288,1483,3913,3207\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 32\n", + "Returned IDs: \n", + "Ground Truth: 7966,5205,6491,6165,1248,7346,4699,5789,290,9593,8223,8794,5098,5787,23,5404,7497,2196,6425,3800,6418,5005,2111,1187,3859,7029,536,3867,5885,4308\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 33\n", + "Returned IDs: \n", + "Ground Truth: 4061,1278,8643,3445,1619,7466,1800,1792,3723,2608,9987,2827,1395,8953,6033,9093,5774,7637,1462,4796,5972,3681,7511,5504,7432,6747,4721,5480,2295,5221\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 34\n", + "Returned IDs: \n", + "Ground Truth: 6189,2849,7938,6858,2632,346,9588,4132,5074,7041,4094,1885,1007,6505,1017,4400,679,7599,7660,1963,5072,2191,1383,5626,9903,2507,5840,1224,1551,6115\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 35\n", + "Returned IDs: \n", + "Ground Truth: 3987,7966,5146,608,2865,934,2032,8549,7764,9462,9529,3863,1533,6495,9613,2196,3937,5569,6340,6418,1403,3008,2998,8318,8627,447,4991,1103,5404,9134\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 36\n", + "Returned IDs: \n", + "Ground Truth: 2187,9061,9038,981,2943,4587,7699,3832,9790,2438,3700,853,132,6481,6032,6759,3686,4416,2104,8379,4794,8244,7767,3460,6794,4562,3477,1327,9725,9346\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 37\n", + "Returned IDs: \n", + "Ground Truth: 7375,4783,2865,6387,2449,6283,7629,9371,2112,3871,4991,7690,3629,3624,2462,9483,5287,788,5972,3649,549,3764,3430,2505,1243,4289,4293,5146,4400,4497\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 38\n", + "Returned IDs: 5213\n", + "Ground Truth: 3008,7790,254,1861,2986,3610,7217,8889,3863,7642,3347,6119,6854,4634,1497,6046,6662,1687,6307,155,3826,3474,836,6721,1362,6073,9055,4548,6615,8247\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 39\n", + "Returned IDs: \n", + "Ground Truth: 7089,2506,463,4293,4061,549,2580,1273,909,1344,2264,1457,698,1642,2541,2271,6474,9365,7557,3191,5553,5721,4406,6532,3310,319,8119,5716,4846,3411\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 40\n", + "Returned IDs: \n", + "Ground Truth: 6635,1861,2168,7427,8625,797,4634,1972,1920,6557,706,2753,3320,5452,3610,9237,7217,3929,6647,292,5403,3563,659,6721,2501,6084,3008,9395,536,3322\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 41\n", + "Returned IDs: \n", + "Ground Truth: 3832,7554,6821,893,8589,9061,2527,7621,8674,9994,9346,6759,655,1963,8587,3686,6884,7767,4416,8342,2194,8379,4897,9065,208,2241,2438,8064,3661,132\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 42\n", + "Returned IDs: \n", + "Ground Truth: 7069,5885,8028,5748,91,608,7042,3426,5773,2998,1217,9956,9615,4490,3448,1403,2196,6662,5404,6418,1503,3707,4034,9593,1944,3738,6211,1098,370,2927\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 43\n", + "Returned IDs: 6764\n", + "Ground Truth: 6764,6643,6803,1893,2219,683,4343,6314,8103,5401,2231,4909,8307,7893,4318,5290,9879,5065,4548,8442,5829,5470,2784,5572,4953,4859,699,9408,7410,8067\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 44\n", + "Returned IDs: \n", + "Ground Truth: 2174,9345,3749,9770,1861,2974,984,4634,7217,6337,7031,8625,706,3347,3474,4744,2412,6557,8247,1757,290,8933,9395,3320,6124,4255,352,7016,5738,5994\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 45\n", + "Returned IDs: 4497\n", + "Ground Truth: 9318,5896,4621,9576,5675,1409,5964,2850,624,9405,8653,9371,7887,2356,2078,828,6871,3867,15,5288,3432,3695,5671,4843,2370,8025,2554,7560,7129,2385\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 46\n", + "Returned IDs: \n", + "Ground Truth: 8222,7558,1687,2153,513,7947,3548,155,1415,9988,7093,2162,3086,1362,5361,6904,2165,1179,6285,2514,5487,8323,8583,2450,7709,4856,9055,8988,1692,7753\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 47\n", + "Returned IDs: \n", + "Ground Truth: 7076,7036,3235,8119,5174,437,6815,469,4896,7091,7629,6235,2124,566,7949,9365,4307,5617,3310,1206,1236,4218,8518,1654,7663,5945,9711,8346,6574,1685\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 48\n", + "Returned IDs: \n", + "Ground Truth: 2831,3551,2404,8406,4658,3871,5649,9219,5925,1105,1885,6683,1881,6776,8123,920,3477,5363,8653,5480,4318,8388,4991,3432,828,6854,3282,3484,6561,6219\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 49\n", + "Returned IDs: \n", + "Ground Truth: 705,1281,8672,3878,9190,6768,8499,8576,243,4144,5838,2688,4676,5311,9269,7431,3628,5591,2147,2862,955,2332,5218,5930,3638,5398,2642,2459,4724,5041\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 50\n", + "Returned IDs: \n", + "Ground Truth: 7381,7696,5139,943,1817,4824,3908,4226,5861,8263,6573,5679,9087,3246,5694,277,4688,1700,9213,7724,3112,2799,2655,7270,8542,2621,7628,8577,5078,9181\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 51\n", + "Returned IDs: \n", + "Ground Truth: 5969,4514,8271,2061,4535,1885,1622,6219,1312,4821,5241,2736,8973,8030,3961,8946,3705,6134,1519,6583,9090,7990,7037,4136,2051,3610,5239,5283,3834,496\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 52\n", + "Returned IDs: \n", + "Ground Truth: 2631,4061,974,9800,8403,7452,1003,5675,4831,5671,8615,5359,4357,6918,6733,5422,2374,4296,9324,5806,1593,8042,6323,9865,5363,2859,9405,624,1700,6693\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 53\n", + "Returned IDs: 652,2595\n", + "Ground Truth: 8347,6951,7010,8892,8717,258,671,3426,5885,1074,9529,5034,5965,6468,3087,4553,4973,9496,6849,4069,9301,1789,9134,3556,7916,416,7125,5480,1562,2032\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 54\n", + "Returned IDs: \n", + "Ground Truth: 2201,1156,9158,7629,162,1379,3320,9237,4400,2642,1344,7248,2174,9095,7267,2418,8609,4734,2078,4364,7966,6775,3828,5840,8569,3195,671,2112,5945,6850\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 55\n", + "Returned IDs: \n", + "Ground Truth: 5917,6642,8093,5016,2521,4859,5701,7511,4557,1792,9287,7522,3633,8103,9013,2779,3648,2716,2160,2575,9662,3424,6765,7477,7187,6886,5736,783,8105,5470\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 56\n", + "Returned IDs: 2472\n", + "Ground Truth: 8093,7153,2258,6945,3282,7798,8123,8406,9054,3985,679,6753,9851,7226,5649,828,3391,3932,4318,2850,9473,4830,3865,3477,8667,3633,4389,2472,1885,420\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 57\n", + "Returned IDs: \n", + "Ground Truth: 311,3700,2824,7189,9232,4157,1575,9051,1024,7815,2763,5152,8407,7097,8342,3847,9401,7561,1188,499,4389,5112,2187,7963,2840,5892,2012,3945,3591,6645\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 58\n", + "Returned IDs: \n", + "Ground Truth: 9190,4144,9167,4094,5332,2151,4852,8499,5600,841,8292,168,2841,9430,6598,5961,5157,5041,5204,5218,8488,41,3009,9269,5840,9273,3544,3276,1281,5631\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 59\n", + "Returned IDs: \n", + "Ground Truth: 4602,2572,7522,5464,6369,4296,3999,7542,8366,860,7069,6074,2631,7865,6047,9999,7560,3707,9907,6073,1597,5278,3033,7822,7031,4099,404,8180,8247,3987\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 60\n", + "Returned IDs: \n", + "Ground Truth: 8127,910,7472,6419,153,8393,2247,5848,9133,4562,8765,3385,8105,3809,9213,4992,6741,4061,9093,9914,5470,2603,4859,3873,4460,2231,1562,1501,414,6004\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 61\n", + "Returned IDs: 4649\n", + "Ground Truth: 3624,7690,3757,3319,4078,2388,4116,9824,2201,2418,1032,3954,4140,7248,8235,9371,7012,7629,1569,7033,8879,7898,992,124,8914,8205,3253,2907,816,3234\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 62\n", + "Returned IDs: \n", + "Ground Truth: 9053,7373,2450,9293,6714,3199,2111,4667,8574,3094,1518,6615,5523,309,7125,5940,352,6817,7243,8340,3317,5181,7193,6106,9269,8000,4176,3243,4497,1310\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 63\n", + "Returned IDs: \n", + "Ground Truth: 7771,2247,3870,3757,3977,9938,7718,3145,6115,3593,2225,5825,4690,4886,4167,2728,2603,7632,4218,9485,3839,5457,7364,3999,4118,8238,4515,6452,1764,5084\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 64\n", + "Returned IDs: \n", + "Ground Truth: 2329,7532,5148,9425,3281,6712,1885,8970,3883,3670,3002,7594,5061,7037,2066,4588,8098,5190,6354,2872,6110,1125,5816,9287,3074,7115,7056,5731,6933,5852\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 65\n", + "Returned IDs: \n", + "Ground Truth: 728,4924,6608,3256,9042,6040,3980,1548,1551,1525,1988,1996,1837,1920,8590,8482,5711,2911,3111,1827,5447,7539,8569,2853,461,2425,529,2975,9296,652\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 66\n", + "Returned IDs: \n", + "Ground Truth: 839,9518,1827,8247,2646,5598,8117,1452,4410,6558,7016,4400,5472,7677,3973,6124,7865,4308,7560,1593,6594,9123,2741,4390,8134,3530,3970,468,8312,2745\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 67\n", + "Returned IDs: \n", + "Ground Truth: 943,1963,7381,5074,7696,1817,5139,1700,7628,4824,4688,3246,6489,1523,2131,5662,3973,154,5679,3112,2132,839,1964,2081,4400,615,4562,3908,6573,5615\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 68\n", + "Returned IDs: \n", + "Ground Truth: 8825,36,4831,7600,4308,4269,2111,8893,8038,443,7634,2017,2023,3993,1075,4936,8401,2896,8312,2084,2862,8569,1003,3491,9236,825,1071,1105,9958,7685\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 69\n", + "Returned IDs: \n", + "Ground Truth: 3474,6106,2592,2931,8649,4497,2111,6838,2011,1584,659,9117,608,290,7943,2838,6133,91,3898,7032,5303,6647,9738,7865,8184,6580,3477,3738,9907,3856\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 70\n", + "Returned IDs: \n", + "Ground Truth: 2569,6808,3729,8363,7143,2032,7880,7432,3265,2640,192,3520,7919,4697,6650,4081,4985,850,67,3385,654,2774,8463,9134,3270,5192,4934,263,3277,324\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 71\n", + "Returned IDs: \n", + "Ground Truth: 6557,2596,8893,7865,3604,539,6608,6307,1920,6500,8569,1988,5940,4357,9543,4831,1996,8445,782,724,4308,8405,1253,7432,4924,36,8340,5480,5575,8275\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 72\n", + "Returned IDs: 5486\n", + "Ground Truth: 2268,162,2237,1573,4400,6775,2564,3630,5589,1569,2744,6670,5945,7629,707,6100,2785,7060,2428,3195,6041,101,6714,4982,5276,2859,6238,8346,5280,4256\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 73\n", + "Returned IDs: \n", + "Ground Truth: 3870,442,9684,8026,5949,9861,7041,7632,5662,6115,3757,2655,9506,7381,7771,8283,8238,9876,2247,483,5284,9213,9898,277,2524,6452,5472,9145,2717,5021\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 74\n", + "Returned IDs: \n", + "Ground Truth: 9505,3800,1920,6046,6285,6418,7093,2590,6220,4856,6971,6084,2596,6557,1403,3495,8522,5329,1584,5110,4726,1518,3417,9098,1687,4009,1265,8405,7043,6495\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 75\n", + "Returned IDs: \n", + "Ground Truth: 2110,6261,1543,7597,3863,4708,2747,4470,7009,5459,6321,3563,2000,8050,1593,9641,9720,2196,3338,3800,4678,4250,2174,6046,3063,7747,8318,8021,994,3397\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 76\n", + "Returned IDs: \n", + "Ground Truth: 929,3299,2395,8284,293,1963,154,2696,9982,8089,8900,9525,7660,4012,1383,322,7425,8279,9161,556,1444,7303,4633,9635,4778,8379,3518,3655,3722,8014\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 77\n", + "Returned IDs: 6803\n", + "Ground Truth: 6764,8442,9408,8283,7566,683,4562,50,5079,6803,4168,1866,3450,6718,6643,3445,98,2696,6765,8858,5483,6489,5925,1840,3050,7951,8643,7377,4548,9359\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 78\n", + "Returned IDs: \n", + "Ground Truth: 3407,9664,3220,2899,6500,3601,8413,1433,3305,5860,3962,3984,8891,4724,4642,2314,6770,5986,6724,9912,8087,3562,7715,9532,8859,3950,1175,3491,6672,7125\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 79\n", + "Returned IDs: \n", + "Ground Truth: 1584,7314,4009,3474,7703,9374,2173,3008,8549,3592,5459,1518,6542,290,3671,3863,4185,7865,8634,2325,2000,2730,8394,7126,4255,8791,4285,9197,6680,934\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 80\n", + "Returned IDs: \n", + "Ground Truth: 8405,9553,6307,6557,3086,1066,1362,9345,6686,8583,9055,6153,6285,3417,1298,8715,2196,7790,258,2596,536,1996,8369,836,6662,9237,8787,9711,4932,1518\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 81\n", + "Returned IDs: \n", + "Ground Truth: 9372,241,6213,8171,1826,7727,3377,6106,6353,6459,7946,6337,6689,4272,7389,8583,3849,6615,622,2941,4327,8420,357,8796,2011,6037,2477,2629,2596,6647\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 82\n", + "Returned IDs: \n", + "Ground Truth: 2943,3262,4400,7022,7143,6253,5589,4817,6594,8346,2785,4460,5992,548,9095,4293,6670,3629,1593,6197,2112,3501,5525,9371,828,6159,8653,1144,154,7248\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 83\n", + "Returned IDs: \n", + "Ground Truth: 6326,9777,9301,8563,7922,4293,3629,5649,406,2032,7089,4425,9008,9158,7501,7929,7207,215,5807,3262,4991,7213,8328,7629,8717,68,3710,9095,4069,7557\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 84\n", + "Returned IDs: 2282\n", + "Ground Truth: 290,5940,8223,6854,8782,658,7946,6165,6615,3063,8533,6508,1151,6871,9371,1862,8889,9477,1687,4514,2712,7217,3987,3800,3320,3377,1849,6647,2443,5157\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 85\n", + "Returned IDs: 7375\n", + "Ground Truth: 1327,7108,9061,1722,5446,7097,5122,6817,4385,3665,2700,9051,2045,981,8407,5520,1575,4196,6032,3591,1357,8342,5378,6759,2104,9107,3978,6107,3686,8524\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 86\n", + "Returned IDs: 8233\n", + "Ground Truth: 9181,9005,5074,9133,391,4141,154,1748,9682,839,4562,4400,4688,9283,4824,2785,8275,2507,1700,3166,2356,679,6594,8914,1016,2023,9288,2943,9903,4512\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 87\n", + "Returned IDs: \n", + "Ground Truth: 8583,9055,1687,9024,1664,8435,9693,6557,6854,6285,868,3417,7790,6312,6459,4923,2834,2153,3273,3086,1944,9378,7093,9237,2660,9423,1298,2998,8340,7947\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 88\n", + "Returned IDs: \n", + "Ground Truth: 4556,290,3395,7225,6497,9100,3155,4785,357,8830,5223,3801,26,6287,6025,9355,9781,425,4764,6662,934,3312,9077,5778,8165,2174,5681,5595,6332,3193\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 89\n", + "Returned IDs: 6996\n", + "Ground Truth: 6996,9180,8385,123,1302,4671,1451,9692,1250,4891,8969,7790,7016,9578,3987,6752,5034,5314,5560,3345,4700,9570,7348,1637,4655,9770,7380,1796,5754,3474\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 90\n", + "Returned IDs: \n", + "Ground Truth: 9238,1318,8883,6598,1706,2403,7003,7872,9562,4308,9721,5295,7049,164,9325,7442,8799,1944,5364,2862,2017,782,3796,3276,3818,1536,5809,5278,8771,2642\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 91\n", + "Returned IDs: \n", + "Ground Truth: 8026,4688,4887,6489,7628,2404,3687,1817,920,9861,8653,5377,228,828,5480,2831,943,9354,3356,548,7956,3432,5589,1290,3551,7501,6639,6871,5671,60\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 92\n", + "Returned IDs: \n", + "Ground Truth: 4319,6411,5074,298,1885,7900,795,1476,7899,1469,7745,3633,7734,5190,8379,8098,1903,5034,1105,5925,2527,8334,2850,1985,7554,7594,4514,9870,1677,9359\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 93\n", + "Returned IDs: \n", + "Ground Truth: 123,3523,6849,3426,1944,5885,9237,4973,1831,2147,3317,6833,872,4490,3476,5565,4431,9263,1789,1514,309,4069,1999,6324,3721,4061,3833,416,9448,6681\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 94\n", + "Returned IDs: \n", + "Ground Truth: 3671,1985,2168,66,4648,4410,182,8688,7029,1783,4390,5278,9770,6145,3286,8247,690,3041,5219,8883,8981,3116,3970,2712,3610,5181,4134,6521,2997,5994\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 95\n", + "Returned IDs: \n", + "Ground Truth: 1134,8594,1687,3078,5363,3320,4497,5480,9617,3426,9055,6693,8369,4887,428,1817,3695,9371,7248,1379,2642,5806,7125,8340,3622,7790,9781,920,6615,5219\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 96\n", + "Returned IDs: 7910\n", + "Ground Truth: 1985,6212,3659,5994,1312,8688,8991,9391,7910,178,9424,5752,2168,5219,1446,9472,8243,5181,8981,182,1783,5809,3671,5464,8946,7017,7175,6483,6145,797\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 97\n", + "Returned IDs: \n", + "Ground Truth: 2998,3707,6493,91,3757,5885,5723,7902,1944,9532,3461,9988,3755,3426,7042,7100,9707,7069,608,7816,5613,1094,6211,8423,9406,2578,6170,4400,5934,9777\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 98\n", + "Returned IDs: \n", + "Ground Truth: 8379,1764,6722,1963,7660,154,7594,2696,5776,1677,9682,233,7905,4225,3376,4572,9288,1501,1903,7519,2759,6449,1728,4614,5436,8279,4574,2239,4778,9359\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 99\n", + "Returned IDs: \n", + "Ground Truth: 728,2975,2554,9824,5675,5671,2642,624,9745,2403,3867,9181,4308,4497,2907,9405,6817,4256,3880,7801,6584,5278,1809,8313,6594,132,2952,9314,1663,2370\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 100\n", + "Returned IDs: \n", + "Ground Truth: 474,3078,6089,5940,2011,2866,5376,6508,3063,2694,8949,6892,5056,6213,1851,5181,8090,438,5624,664,3094,4327,1151,3201,8340,5718,386,2820,7095,4405\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 101\n", + "Returned IDs: \n", + "Ground Truth: 9715,6563,9780,3484,7629,8844,5330,7432,1204,1378,6885,612,3629,5965,8892,7743,8346,1144,1376,3561,9777,2541,8717,1983,4293,3310,4460,3710,7494,7602\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 102\n", + "Returned IDs: \n", + "Ground Truth: 8023,3242,2084,3321,4609,5317,7268,2403,3684,4308,1071,4269,7832,2646,5390,1944,9356,7991,247,8038,7336,576,4852,7750,89,3239,3491,7471,905,5364\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 103\n", + "Returned IDs: \n", + "Ground Truth: 5377,9290,1544,1427,3551,4765,4887,9215,9455,3356,2575,3938,4444,2831,920,8844,2404,8892,5330,1871,8026,3865,1817,4687,4688,3687,3310,9792,7475,3985\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 104\n", + "Returned IDs: 1818,8164,536,6735\n", + "Ground Truth: 4293,3310,7248,3444,6221,963,1344,4557,1562,8976,1751,2774,1890,2929,2032,7949,688,2603,460,9773,3319,9242,8406,7432,5480,1861,2976,5566,6266,9492\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 105\n", + "Returned IDs: \n", + "Ground Truth: 7280,2783,1972,3495,3417,6662,3800,8162,3563,9233,7225,5383,8411,2174,984,6418,6665,290,7200,9355,7203,2416,8787,4592,9536,9360,26,6511,2380,8420\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 106\n", + "Returned IDs: \n", + "Ground Truth: 8104,9770,2000,6307,2174,290,2011,6106,1518,6557,7016,1584,9345,3749,309,8247,690,9148,536,7029,6647,1265,6337,2997,3317,5403,1362,2848,5278,3320\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 107\n", + "Returned IDs: \n", + "Ground Truth: 5885,2998,7966,9615,2927,2196,258,1944,5033,9750,6850,7069,3426,5404,3320,3058,3987,3622,6418,1595,1403,542,9237,4293,1562,828,608,8423,4932,6468\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 108\n", + "Returned IDs: 9567\n", + "Ground Truth: 5343,6395,2507,8319,5568,9660,1389,1394,5863,2247,5721,772,4689,1823,4061,6102,9684,1164,6172,8211,6989,9074,1619,1840,4859,2621,1457,1944,2499,2266\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 109\n", + "Returned IDs: \n", + "Ground Truth: 1678,2419,3205,6292,5925,7217,7444,3865,4875,3426,2739,4318,8406,6165,1137,154,8889,3963,9120,4039,8307,9986,1151,3754,7374,3282,2691,2917,9870,7313\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 110\n", + "Returned IDs: \n", + "Ground Truth: 1137,7799,9717,9582,9196,6814,3378,3116,3469,5243,1881,2419,2999,8653,2596,9120,1650,7545,1710,3271,3054,3888,7406,7374,643,2492,7721,6488,1068,3218\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 111\n", + "Returned IDs: \n", + "Ground Truth: 4614,2696,4343,8089,910,7742,5245,2759,8914,4887,1817,5662,2957,1728,3112,8442,6764,2231,9935,2507,1519,3751,4039,2655,9133,9242,8736,2131,2524,7893\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 112\n", + "Returned IDs: 652,2595\n", + "Ground Truth: 9405,3987,8247,1518,362,6511,6794,7380,1958,3704,923,3937,9117,9536,1403,8969,7217,2980,7248,746,3615,9770,7764,6580,3926,8649,5294,608,6495,4891\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 113\n", + "Returned IDs: \n", + "Ground Truth: 8983,3709,7297,2572,8216,9711,9457,6235,9777,6342,4767,3235,2670,9410,4810,5617,103,3707,9785,8149,5203,1555,4543,8609,4830,2535,1687,3033,9028,3526\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 114\n", + "Returned IDs: \n", + "Ground Truth: 4688,9219,1602,7966,3356,2785,4069,8627,7898,5330,7348,1838,4444,8892,5432,3165,2404,9865,2642,3569,5062,7374,7217,9616,1861,8430,3629,4497,1544,3426\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 115\n", + "Returned IDs: \n", + "Ground Truth: 1431,1390,5955,4034,9593,9956,91,8712,3317,7151,1944,8353,6752,9785,155,6153,5026,4490,4411,608,1584,2927,1503,6511,5752,8021,2012,9024,5748,6375\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 116\n", + "Returned IDs: \n", + "Ground Truth: 9059,664,1944,7946,6615,1727,5940,8223,2998,241,7865,4493,137,5514,8545,4932,2554,8569,7193,2179,8090,3086,9371,5885,8583,5480,3078,3292,3340,8753\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 117\n", + "Returned IDs: \n", + "Ground Truth: 8487,9914,6451,1866,8754,458,4562,389,3171,8858,7603,5949,3669,9893,5879,1827,3289,7472,1728,2374,7831,1764,299,646,3468,525,3120,9040,777,7078\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 118\n", + "Returned IDs: \n", + "Ground Truth: 3738,7542,6750,2578,9593,5885,9367,8522,6854,3826,6220,3800,6497,8787,2174,3474,8155,9406,3755,3320,7280,4699,3225,9694,1849,6331,386,6192,4180,9237\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 119\n", + "Returned IDs: \n", + "Ground Truth: 8247,146,5179,5303,1781,7543,2283,63,5472,2468,4634,901,9037,540,1518,8782,8558,1403,8104,8507,2970,6728,590,8360,6680,8549,2450,2927,4775,5754\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 120\n", + "Returned IDs: 860,3999\n", + "Ground Truth: 8508,4213,5084,3156,4702,2262,3873,7868,8238,312,7078,3770,2039,6815,1694,9989,525,9354,6197,3492,4686,9087,860,3325,414,3081,4346,4413,4873,8097\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 121\n", + "Returned IDs: \n", + "Ground Truth: 872,3707,602,1687,9371,7790,6794,1944,2117,3317,1202,7093,9423,4655,3721,9781,5885,3734,4535,2245,3448,6521,920,416,4891,6495,7432,1403,5314,7031\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 122\n", + "Returned IDs: \n", + "Ground Truth: 7016,9614,6487,4235,7471,7425,2646,1827,4647,5403,9518,3299,6124,7217,3587,9770,2303,9234,228,8398,1593,3530,101,4400,4044,9033,7515,2011,8501,7943\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 123\n", + "Returned IDs: \n", + "Ground Truth: 248,2194,7968,7259,7554,3358,2438,2975,4141,4292,9992,5792,5671,6372,4330,227,6970,9324,5202,8284,7303,112,3641,952,523,2683,9227,6007,5,2707\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 124\n", + "Returned IDs: 7977,8729,3329,4239\n", + "Ground Truth: 6153,3596,5764,6662,727,1958,1687,7093,1553,3202,2450,6658,2778,608,5844,91,517,2468,8411,5039,5033,5130,9553,6170,1692,2153,9310,1202,5773,4844\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 125\n", + "Returned IDs: 3088,2207,6114\n", + "Ground Truth: 9028,3088,6235,9711,9365,1090,7391,2670,8119,3310,1206,8609,7070,1457,7089,9777,4905,8000,3235,9875,7432,1376,5553,9354,4460,7494,7743,9410,7297,7929\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 126\n", + "Returned IDs: \n", + "Ground Truth: 2283,3286,8247,2712,8991,6728,1446,5994,7865,7570,7560,7542,1720,6145,8604,1985,146,2511,3222,5181,8533,5464,1743,2647,5303,6103,7677,7016,7093,5262\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 127\n", + "Returned IDs: \n", + "Ground Truth: 9770,9717,3469,3008,2243,2567,4889,6261,5034,3474,5213,5452,3987,7217,6219,6814,3116,4726,5403,3322,8096,8639,3818,1861,3523,6854,1298,4009,1593,8533\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 128\n", + "Returned IDs: \n", + "Ground Truth: 3060,3219,539,5575,2899,4831,2017,2309,7442,7003,9912,8413,6500,3220,8354,782,9358,8445,9810,36,2672,1075,3090,8275,9664,2062,3731,3068,7194,4269\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 129\n", + "Returned IDs: \n", + "Ground Truth: 9794,2834,8830,3943,5607,3568,4250,106,3863,968,9169,1173,4978,1930,9498,5960,491,5913,3037,806,5164,3025,8867,4041,7348,4932,5412,2267,4195,4068\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 130\n", + "Returned IDs: \n", + "Ground Truth: 3576,7893,3120,8086,6419,6932,1817,5217,5309,6314,1893,9125,4887,5377,9290,4557,1732,9879,4953,5679,6764,8773,5664,9678,7628,1852,8074,6451,5848,2219\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 131\n", + "Returned IDs: \n", + "Ground Truth: 1682,2952,5946,6372,2931,5367,9181,6031,6970,328,5202,4008,2519,5338,6739,2975,4532,1548,4649,569,9796,7123,9234,2438,9226,9400,2188,4157,6316,8342\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 132\n", + "Returned IDs: \n", + "Ground Truth: 4480,707,5280,6103,7956,4493,2744,760,2428,8653,1809,8233,6714,7998,6187,8498,5589,292,4400,2419,6758,7129,3717,2564,13,8244,8042,1041,6292,7754\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 133\n", + "Returned IDs: \n", + "Ground Truth: 1439,9388,1477,9371,4351,2428,7356,828,3383,9283,5589,9824,5776,5278,6693,2706,7085,5946,8233,4293,4400,1663,7854,8340,228,4467,6387,1383,4256,6136\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 134\n", + "Returned IDs: \n", + "Ground Truth: 962,3468,3833,525,4132,9365,5818,3757,3731,793,6540,6825,4460,8267,8275,4061,9992,149,3873,3610,9093,5200,807,2915,2269,2262,3317,312,7865,6648\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 135\n", + "Returned IDs: \n", + "Ground Truth: 651,8696,3461,3564,6322,8988,4773,68,6690,5093,5521,4389,5761,9707,5386,9229,8609,9777,1944,7853,3014,4767,6681,1156,9793,8663,4801,3426,8511,823\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 136\n", + "Returned IDs: \n", + "Ground Truth: 6004,9287,9914,5848,4887,5469,6886,2716,4373,5309,2743,6056,8777,7522,6712,7594,6844,5423,4599,8156,3282,1168,1885,2338,6504,6765,963,5917,2082,4859\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 137\n", + "Returned IDs: 9024,1503,9693,8938\n", + "Ground Truth: 3800,2627,1687,9392,5539,2679,9024,5341,696,6312,6459,7093,2165,1211,9055,1202,1403,2153,3273,7947,7215,6662,6341,2162,4041,1503,8215,5430,6285,9693\n", + "Recall: 0.1000\n", + "\n", + "Query ID: 138\n", + "Returned IDs: \n", + "Ground Truth: 8442,6057,8284,683,711,7554,9900,9371,6129,8180,5925,98,5369,8334,2404,4141,5483,3977,6489,9071,5776,1866,7951,8089,2454,8421,1894,5972,1639,8726\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 139\n", + "Returned IDs: 323\n", + "Ground Truth: 6017,847,9093,4475,483,628,3116,6647,4497,608,8879,4953,7217,5654,1278,4215,5480,880,6747,7364,4310,6854,4039,9269,654,2147,3917,7935,6529,4414\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 140\n", + "Returned IDs: \n", + "Ground Truth: 7753,6813,848,2174,8369,7790,2927,8411,996,1298,779,3426,4935,5564,8405,5875,2012,6854,3417,1753,9378,517,1861,1687,3563,7491,5867,1692,6153,1518\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 141\n", + "Returned IDs: 3081\n", + "Ground Truth: 7415,9059,1727,1766,7934,9255,664,7455,9984,9512,8090,1805,6037,600,5576,9387,9663,9323,114,7546,1876,1704,1167,6599,8783,4053,844,750,8753,6513\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 142\n", + "Returned IDs: 536,1818,8164,6735\n", + "Ground Truth: 9024,3824,5341,5539,3086,2679,9392,7877,3749,9711,1687,9693,19,1503,1362,3273,5885,8583,3717,8340,536,6519,696,9875,6046,9237,3548,290,3293,9553\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 143\n", + "Returned IDs: \n", + "Ground Truth: 2388,2276,1027,1622,1817,5679,3234,483,6079,3154,9684,5426,281,2374,1681,7901,8089,8879,1827,9213,9371,2454,7213,2621,1619,7465,9006,2404,5742,7570\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 144\n", + "Returned IDs: \n", + "Ground Truth: 5181,8949,1310,3340,2836,9853,7702,4174,7095,6046,7158,6457,7946,3094,4053,6688,2443,4769,3607,6087,3610,1446,1727,939,8182,2647,750,6131,1729,5875\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 145\n", + "Returned IDs: \n", + "Ground Truth: 2359,2188,4400,8741,6741,2943,798,7801,8914,828,1208,1324,2746,3637,9093,7143,4460,7951,3033,3686,414,5675,1888,7123,5939,8340,9939,2524,767,3118\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 146\n", + "Returned IDs: \n", + "Ground Truth: 3973,2545,4497,725,8893,5314,5359,8445,2111,3880,4256,7865,7032,1075,8061,6557,7016,6615,7003,8883,9468,2554,6041,3078,2017,3033,2642,3094,6725,4308\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 147\n", + "Returned IDs: \n", + "Ground Truth: 7690,5152,1795,7942,9540,4198,8363,6673,7935,9824,4116,3573,4586,1389,4625,9115,992,4595,9524,5506,8700,9780,7432,2183,9777,2223,7629,3253,4200,4343\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 148\n", + "Returned IDs: \n", + "Ground Truth: 6647,352,3199,4497,6721,7016,4634,3347,5666,7032,2011,6307,8223,7529,3859,290,7595,1861,5005,6615,2009,5447,2753,1920,654,1551,2567,7031,7754,6073\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 149\n", + "Returned IDs: \n", + "Ground Truth: 7432,4586,9684,6640,5230,6378,7466,9426,5622,6747,4061,5972,6376,1802,3270,4301,8026,7874,5360,1389,5192,6650,4212,8363,3999,5917,1443,2300,483,8953\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 150\n", + "Returned IDs: \n", + "Ground Truth: 2943,4293,6162,3385,7143,7213,2112,5663,1540,3493,3957,8630,6197,5525,1562,6684,5533,6326,4926,7064,1872,7022,3501,312,210,3710,5149,414,8498,16\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 151\n", + "Returned IDs: \n", + "Ground Truth: 3715,8748,4236,1642,2506,1344,2595,7929,148,3920,1206,4406,3501,2264,4846,8884,8336,6898,4722,2107,3551,3369,229,459,6148,5992,3840,8466,8419,5884\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 152\n", + "Returned IDs: \n", + "Ground Truth: 4404,5806,6521,1950,5359,5546,2393,9673,7285,3072,1290,1465,9546,2642,3003,624,7801,9371,4821,9456,702,798,4497,3880,3623,9576,9369,4059,7137,9827\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 153\n", + "Returned IDs: \n", + "Ground Truth: 8112,9426,3995,531,9935,3692,3999,3757,9485,339,3145,1323,4218,214,32,8187,3546,2728,7033,3187,4462,8879,4732,1424,4485,4886,669,8026,170,1140\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 154\n", + "Returned IDs: \n", + "Ground Truth: 483,4215,7410,8879,3004,2201,5568,2717,7737,3165,3291,4039,3153,192,9134,8319,9684,688,4934,3566,7267,6610,2524,5146,6529,654,6551,4991,1619,2036\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 155\n", + "Returned IDs: \n", + "Ground Truth: 5885,8743,3495,8715,836,9553,2671,1066,2927,7280,6418,8405,9101,542,1861,8924,2174,3861,8787,5033,6046,6220,6307,4885,8024,9956,8569,3245,9532,1650\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 156\n", + "Returned IDs: \n", + "Ground Truth: 2655,278,1619,701,5330,1388,1983,5343,1428,6541,8604,9405,9959,4845,9744,775,9288,5146,8638,3527,4887,6683,3112,8247,1695,4497,920,2295,2986,1243\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 157\n", + "Returned IDs: \n", + "Ground Truth: 3348,9844,9946,8027,8773,3424,4343,6643,6013,3223,8837,6764,9492,6451,9433,1630,257,5866,4562,8442,4914,6741,5217,6723,2943,8212,2331,2231,7554,8066\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 158\n", + "Returned IDs: \n", + "Ground Truth: 2018,2439,3540,6459,5787,9495,2730,8120,1248,1956,241,290,8411,2998,3347,8545,6647,3863,1403,6220,6615,6374,7280,2405,3754,5940,7221,1162,1374,1497\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 159\n", + "Returned IDs: \n", + "Ground Truth: 3431,7861,8588,8029,7909,5792,2944,7767,414,7271,3954,1012,8462,9605,548,4292,3808,8622,9568,6191,8900,3304,2115,8935,3289,2199,2512,1855,9227,9898\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 160\n", + "Returned IDs: \n", + "Ground Truth: 6258,7535,7362,9927,5873,6398,2045,1573,3591,2769,3645,2028,5358,3015,5184,807,8033,9622,8407,3757,5612,5205,1012,6100,1247,5560,2700,7167,6716,3270\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 161\n", + "Returned IDs: \n", + "Ground Truth: 1687,3086,8583,7753,2466,1497,8411,3273,290,2153,9155,1518,2778,2848,1362,6854,5487,1298,7627,6337,608,7162,6278,3979,3798,6658,2012,3548,2963,2334\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 162\n", + "Returned IDs: \n", + "Ground Truth: 1649,5046,1438,533,6225,9269,4256,2112,8275,8498,2943,6707,6253,7671,7603,9371,7439,4531,7143,9654,6197,1866,9263,9095,4903,4460,6103,4817,7078,920\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 163\n", + "Returned IDs: 6293\n", + "Ground Truth: 136,2115,9724,500,2177,6500,6313,8393,6448,6557,2943,8498,5714,1767,8588,6004,5774,9759,4562,414,8138,176,7868,7522,1164,4998,1368,8136,1619,920\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 164\n", + "Returned IDs: \n", + "Ground Truth: 782,1438,2309,2298,9214,8312,6500,3731,6829,6208,8087,3977,9992,2092,4238,23,2009,9025,7051,2017,4831,6737,7194,3546,5465,8264,1470,7140,7271,5084\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 165\n", + "Returned IDs: \n", + "Ground Truth: 1593,8879,4044,7560,9770,8304,2427,8398,7085,9371,9123,7129,5278,7356,9296,6693,3167,5381,6124,2404,154,5403,4734,3871,5806,3078,6571,7016,3973,2078\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 166\n", + "Returned IDs: \n", + "Ground Truth: 1840,7143,8892,6764,2831,2759,534,2262,3627,7737,6563,3484,6885,3687,8430,9219,8638,536,5243,3216,1483,5483,4343,615,5289,8442,7522,2717,2,7033\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 167\n", + "Returned IDs: \n", + "Ground Truth: 9994,5776,7858,7554,8587,1866,2714,6884,8064,7968,2878,851,683,6411,4330,2068,5923,9283,9073,7635,8425,2527,8754,761,1963,2385,879,7136,9587,6449\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 168\n", + "Returned IDs: \n", + "Ground Truth: 4293,5480,4460,1480,1344,420,5589,3551,3310,6898,8406,7629,1818,4400,828,8343,7929,3484,8653,2505,5278,4887,7143,6580,5649,2929,6532,8879,6563,5945\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 169\n", + "Returned IDs: \n", + "Ground Truth: 9114,415,1268,8393,4542,7497,575,2603,7051,245,8052,5030,7472,4528,8302,6637,3931,3059,8283,9093,5470,6648,6825,9176,7211,4994,7793,3628,4389,3238\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 170\n", + "Returned IDs: \n", + "Ground Truth: 1885,1677,7745,7068,5074,3757,5910,2132,8933,6608,2333,5034,2249,4357,1551,7594,6505,628,1963,5840,841,8653,9903,5662,6452,5925,8939,2850,2642,6219\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 171\n", + "Returned IDs: \n", + "Ground Truth: 3711,3865,4726,5363,2831,6219,3205,3477,8594,4913,6580,7361,3629,1650,6951,6990,5034,3963,4549,1881,628,3469,1068,6776,1885,8653,828,9117,5466,6801\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 172\n", + "Returned IDs: \n", + "Ground Truth: 9006,9290,5377,1871,2621,9914,2521,4887,5679,2831,6219,6580,172,1817,9017,913,8892,1827,8026,1619,3657,2753,5330,9509,6747,920,9744,7432,5230,4190\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 173\n", + "Returned IDs: 7337,7838\n", + "Ground Truth: 2499,5340,5378,5388,4417,9952,5055,2249,1027,7990,4939,6136,7594,2696,2605,1885,9543,5873,74,5042,5510,807,2078,1184,5079,1385,706,1383,883,8230\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 174\n", + "Returned IDs: \n", + "Ground Truth: 29,506,1117,2556,1253,6839,295,341,7245,5761,6350,3939,8194,2179,1094,2708,5093,2744,1660,2766,9757,681,2146,8795,707,6815,8057,4517,3490,6670\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 175\n", + "Returned IDs: \n", + "Ground Truth: 5565,3548,7578,5885,1302,1861,3990,4599,8484,608,8340,4431,7162,8129,9219,1562,4490,1753,547,1944,3965,4400,7542,7125,7641,3454,3345,1451,536,6854\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 176\n", + "Returned IDs: \n", + "Ground Truth: 7570,5445,9907,9345,7129,2021,4658,536,309,3759,1818,9738,2539,6557,2712,6854,4099,495,3310,9711,2011,4256,2310,5677,3987,3150,7452,7032,1920,6788\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 177\n", + "Returned IDs: 287\n", + "Ground Truth: 5093,2269,882,8194,309,793,1998,2708,1094,1117,5386,2146,4643,8585,4132,807,869,681,1253,4781,4745,4380,3163,2744,6735,5945,6670,8230,4460,8346\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 178\n", + "Returned IDs: \n", + "Ground Truth: 5575,6672,8663,4060,3962,9301,9912,9358,4604,8198,6696,8275,4703,4521,1547,539,9860,9756,3628,2899,4039,2309,7962,1118,9892,82,3068,4724,3691,4831\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 179\n", + "Returned IDs: \n", + "Ground Truth: 4108,7721,3610,9582,7529,8867,7636,1968,3340,5480,7946,7095,3296,2999,5045,3757,3063,6669,134,7068,4860,3449,5662,8939,7980,2759,5363,9637,3078,6615\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 180\n", + "Returned IDs: \n", + "Ground Truth: 4132,2603,7868,4460,4400,1470,8953,4061,3757,962,6825,525,40,9938,5905,149,7771,3873,7831,9759,3317,5840,4167,312,9365,8366,8393,9459,4113,2368\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 181\n", + "Returned IDs: \n", + "Ground Truth: 2850,4600,802,1885,5447,5910,9561,3966,5711,3530,1827,4117,7594,4012,2500,5034,9914,3432,5363,9922,1817,8590,3860,6825,3043,7990,7660,628,6038,2853\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 182\n", + "Returned IDs: \n", + "Ground Truth: 2,5480,5146,1273,8844,4293,4460,6235,3484,7432,1619,3310,698,7213,2032,8953,7629,2499,5716,2585,4721,920,1462,1225,4218,4796,9838,9684,7399,2506\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 183\n", + "Returned IDs: \n", + "Ground Truth: 4991,4455,3430,397,3752,8130,4110,2409,7213,4713,2377,8962,8557,7782,8336,2390,1055,8406,1270,5925,3957,6546,1225,4406,3005,7257,1096,7143,9354,4181\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 184\n", + "Returned IDs: \n", + "Ground Truth: 6927,50,6756,3033,6648,5954,348,5948,4937,920,5230,1208,4524,5671,7012,5586,4252,4649,4400,7387,3530,2710,4046,8138,3374,4702,6594,1477,2696,3199\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 185\n", + "Returned IDs: \n", + "Ground Truth: 4059,1822,5313,214,1462,8879,7629,8205,2063,3319,7064,5434,9134,2618,2505,9935,4293,2418,3757,1044,5550,3277,2388,1344,3624,8630,2696,192,688,1596\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 186\n", + "Returned IDs: \n", + "Ground Truth: 3908,8528,3445,3246,778,7381,6794,1619,5472,5021,2375,9302,6573,7270,6140,6938,4061,9922,1105,5283,7817,2132,5303,5915,314,2198,3633,8616,3848,5861\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 187\n", + "Returned IDs: \n", + "Ground Truth: 2744,2275,3180,1094,5715,5093,3939,3707,5386,1618,416,5404,1253,1694,8511,7152,8194,4213,2179,7682,7543,292,4256,5311,5058,892,6100,2556,8038,8057\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 188\n", + "Returned IDs: \n", + "Ground Truth: 1386,2753,9470,2403,4497,5364,3414,6647,9779,4521,5077,1700,8129,6361,9405,5187,7003,1064,3818,591,5432,1944,8883,2078,1654,6598,2642,814,1747,1706\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 189\n", + "Returned IDs: \n", + "Ground Truth: 2419,5589,828,4400,4293,679,228,5480,1593,3262,8653,1041,8530,1342,7956,3530,1650,5278,2850,974,1344,7217,2404,6871,9903,3551,2078,1383,2831,3687\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 190\n", + "Returned IDs: \n", + "Ground Truth: 4157,2707,2859,1088,8342,1061,5338,7381,6430,1663,9371,9069,8900,8498,2931,3110,7341,2519,1963,5615,3770,9431,5671,3432,6615,1886,6255,2975,2359,2706\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 191\n", + "Returned IDs: \n", + "Ground Truth: 4991,688,4557,276,9371,456,5217,9669,7399,57,2929,8089,8892,5540,8803,2423,7465,7213,5434,7560,6454,1025,8976,1622,2404,1817,1800,9509,2282,4110\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 192\n", + "Returned IDs: \n", + "Ground Truth: 5995,4672,1141,1920,1738,1762,9427,852,8090,8443,8415,999,3247,2876,2989,2427,8727,2567,3695,9717,2773,8339,1686,6801,8141,8170,1954,7845,2712,8071\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 193\n", + "Returned IDs: \n", + "Ground Truth: 3421,5945,7271,1681,3651,5176,8879,5792,1379,7078,7432,1569,7267,2628,5485,3501,4012,4171,5525,437,8119,5683,3506,4512,2992,5992,2943,586,4892,5588\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 194\n", + "Returned IDs: \n", + "Ground Truth: 9074,5171,774,6102,8504,8319,3723,654,9660,7098,3294,2609,6529,4796,6051,7364,6395,1394,7874,5979,1619,7511,4061,339,772,7231,4689,1273,549,5711\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 195\n", + "Returned IDs: \n", + "Ground Truth: 5711,7364,1827,2853,3040,529,2785,8590,3111,9095,1278,4992,3966,728,8535,9042,6886,2201,1885,8319,8593,3260,5363,5152,4198,5949,7769,15,8707,8482\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 196\n", + "Returned IDs: \n", + "Ground Truth: 7690,9824,2418,4116,3624,1032,9371,4400,4078,2907,8412,5675,6673,992,3757,8177,1935,3629,2763,3234,4157,7123,7540,6817,7290,4649,7248,624,2188,820\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 197\n", + "Returned IDs: \n", + "Ground Truth: 5885,7069,2927,5404,2998,3622,2196,91,5480,1298,3551,6211,608,9750,7816,7966,3707,9711,4293,8423,542,6854,2012,3987,9875,9237,9303,1861,5033,1503\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 198\n", + "Returned IDs: \n", + "Ground Truth: 9294,6614,9039,309,6021,5181,3078,9671,7772,5893,9795,3377,5674,5219,2859,2011,6508,3347,6131,6213,651,7389,3199,8949,3320,6615,673,241,5433,352\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 199\n", + "Returned IDs: \n", + "Ground Truth: 3004,4688,4075,228,556,1383,154,6470,156,6435,6017,1963,1913,8232,1544,4638,7529,4377,1079,2875,1150,7501,4018,236,6489,9133,5240,5662,7599,5589\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 200\n", + "Returned IDs: \n", + "Ground Truth: 3610,939,6046,2174,5666,3317,2567,6307,4255,6662,137,6557,4497,3749,3008,8443,3863,1849,3800,6854,7560,516,5181,706,9345,352,9770,3474,2896,5311\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 201\n", + "Returned IDs: \n", + "Ground Truth: 6049,3325,3840,3089,6661,1438,6241,2131,4256,122,4573,8584,5992,3118,5792,317,7131,3552,5675,5481,9263,4460,7801,6779,3501,3715,1872,4982,4681,7793\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 202\n", + "Returned IDs: \n", + "Ground Truth: 2655,5139,930,1811,1369,7053,6794,920,1619,8653,701,4824,5021,2745,7632,6580,6333,3759,4887,9307,3870,3445,8134,4547,3551,1817,4132,2247,2575,8537\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 203\n", + "Returned IDs: \n", + "Ground Truth: 4497,4924,2078,8276,8490,4416,290,2141,597,132,7772,5706,9885,1886,8796,728,2437,6853,536,1614,5422,3859,5359,9644,5940,1809,3199,1003,9405,8889\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 204\n", + "Returned IDs: \n", + "Ground Truth: 5438,3214,3266,7051,1379,2201,2309,9777,6179,483,3501,1341,1344,2756,5575,8879,3310,643,776,2256,4512,3947,463,7207,3444,8177,8319,9095,6673,1890\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 205\n", + "Returned IDs: \n", + "Ground Truth: 9684,954,2831,1964,6097,1619,2655,8706,9861,9213,8026,483,1172,8134,548,4547,8711,2659,5679,9455,8528,218,8653,194,3112,1817,3687,4887,9872,2524\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 206\n", + "Returned IDs: \n", + "Ground Truth: 9237,9345,9711,6119,8442,6854,3293,2696,7918,6660,6350,3757,5662,5324,9875,4562,8609,3477,2618,8490,7217,5773,6337,3317,154,9935,9528,1861,7009,4343\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 207\n", + "Returned IDs: \n", + "Ground Truth: 909,7557,8123,1090,7089,7929,8466,463,6474,4960,3597,4293,1457,4704,9586,6532,1255,1216,3313,698,5363,9595,3282,3391,2506,8563,3561,9365,2027,1273\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 208\n", + "Returned IDs: \n", + "Ground Truth: 3310,67,6419,3328,5848,2746,7432,263,3576,210,1642,2967,2785,7213,6747,8852,688,8892,8052,1540,5934,8557,4557,4722,420,460,3794,7143,9433,8210\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 209\n", + "Returned IDs: \n", + "Ground Truth: 8254,1963,2437,1677,8900,2078,2500,8589,1088,8866,7425,8815,7337,6372,3842,563,4622,3636,828,9638,6693,8284,4448,4075,802,9227,1005,4000,6992,792\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 210\n", + "Returned IDs: \n", + "Ground Truth: 5806,428,7772,3880,2642,780,3471,2370,431,4404,4497,2045,2402,7442,7570,7285,7801,2447,9878,6788,624,6646,3452,8879,8893,4621,1003,6827,5294,3717\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 211\n", + "Returned IDs: 4497\n", + "Ground Truth: 3570,7442,8025,2865,8907,3624,2201,7560,4745,1379,2943,4939,2045,5725,8914,654,1518,2554,3880,9616,6126,2370,5806,4137,7573,4039,486,2763,6646,4891\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 212\n", + "Returned IDs: \n", + "Ground Truth: 36,9356,2727,4269,7003,825,8038,6071,1003,6644,7600,5715,8859,3321,4308,833,9453,1071,7691,7750,6214,2403,1075,3993,1062,8312,3092,8569,7268,7954\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 213\n", + "Returned IDs: \n", + "Ground Truth: 5611,4493,2037,5683,3688,5945,3840,5992,4400,533,8455,4982,5965,7131,5589,1206,9354,512,4573,3089,1273,3717,7143,7629,8233,8498,1253,2588,3654,9875\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 214\n", + "Returned IDs: \n", + "Ground Truth: 5046,154,5815,1866,5662,5840,7951,3757,707,8283,4400,8653,1723,7381,8233,2428,312,1963,2548,9824,3977,9093,683,8580,6452,2382,1809,5784,7566,5045\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 215\n", + "Returned IDs: \n", + "Ground Truth: 3569,1963,3477,4688,154,792,5688,228,828,1213,3530,9287,7951,1885,9903,2618,5447,2356,8442,9900,3432,8726,5034,7594,8674,4400,8042,3661,5776,3627\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 216\n", + "Returned IDs: \n", + "Ground Truth: 2201,3317,9711,6662,1301,7544,7248,8609,5885,6091,9237,4041,7918,3477,1861,6660,3755,8498,8879,1179,525,828,7603,2890,9875,4059,1344,1872,3385,1497\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 217\n", + "Returned IDs: \n", + "Ground Truth: 1027,8089,1622,6079,6610,1504,7465,9509,1728,4887,9492,4182,7901,2131,2404,107,9982,2454,1894,688,546,483,5480,1817,4772,3234,2494,1827,5917,2785\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 218\n", + "Returned IDs: 2521\n", + "Ground Truth: 1619,5808,6747,2521,4887,4548,2779,4061,8892,6642,9492,772,8156,5016,9542,7466,5917,4460,2831,4557,1817,8614,1871,9287,4953,5736,483,3963,9219,3987\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 219\n", + "Returned IDs: \n", + "Ground Truth: 3426,608,9223,6813,7069,9711,2261,8830,8594,3474,7217,9750,7966,4392,290,8972,3202,6854,2196,7151,9117,3008,8169,6662,536,7033,9965,5404,3554,8369\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 220\n", + "Returned IDs: \n", + "Ground Truth: 132,2911,2425,807,6966,8407,8342,4992,5367,9042,5650,8707,6794,461,2975,6594,1869,1334,728,1663,5238,4729,2943,1575,2045,9145,5742,499,9838,1988\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 221\n", + "Returned IDs: \n", + "Ground Truth: 2011,3078,5940,6508,6213,254,9617,3320,8340,5181,4497,2545,7389,8038,8796,9269,7125,2937,6165,352,1687,247,309,7217,2111,9795,386,7594,920,7032\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 222\n", + "Returned IDs: \n", + "Ground Truth: 1344,2897,2506,7432,263,1270,6851,4759,5830,6723,3549,4406,5649,4776,7143,8210,5533,3715,4236,8852,3501,6235,2746,1824,1642,3794,5553,8546,3328,698\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 223\n", + "Returned IDs: \n", + "Ground Truth: 400,4196,4595,7266,2197,816,9712,5112,7086,2904,8407,8191,7942,4236,2281,5892,569,4256,1795,9149,5577,678,6349,8700,9796,9073,8850,9051,8342,711\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 224\n", + "Returned IDs: \n", + "Ground Truth: 2145,6208,1289,3794,23,2503,38,8122,5840,2835,6354,5719,8322,7842,761,1978,9062,6578,9797,4571,1441,727,3501,1438,3757,8796,2438,2332,9623,2840\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 225\n", + "Returned IDs: \n", + "Ground Truth: 8212,3686,1724,3258,525,4256,4400,9093,3552,2943,9851,208,2532,4366,7974,6159,7793,2131,8580,1324,4571,8340,312,8080,4343,2360,4467,3118,7143,6451\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 226\n", + "Returned IDs: \n", + "Ground Truth: 9777,7070,9783,476,7364,8518,1376,4992,8654,3464,1562,3260,7391,5716,2939,92,8346,3488,6235,6326,3501,5934,7929,7078,6660,2027,3081,985,5253,204\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 227\n", + "Returned IDs: \n", + "Ground Truth: 8845,2709,8484,3345,3970,1944,3990,9621,9692,7578,8660,7380,6324,9044,7641,1293,8628,380,7350,3998,2403,6153,3796,2980,6693,7173,7816,4431,92,2023\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 228\n", + "Returned IDs: \n", + "Ground Truth: 5389,8089,5434,688,1751,2929,9371,6454,1622,4092,3792,4393,2423,7560,456,7952,2801,1027,3543,7629,6693,276,6613,2865,6079,333,2717,8803,5525,5313\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 229\n", + "Returned IDs: \n", + "Ground Truth: 7125,8312,5311,2893,2862,2147,3731,1187,5157,7431,8569,8576,3601,2459,5930,7243,8413,8859,9269,9145,8868,5278,2332,2356,4308,2642,258,3868,9093,3407\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 230\n", + "Returned IDs: \n", + "Ground Truth: 659,3800,9081,1403,3320,608,8024,2174,7225,3462,3338,2416,3738,8349,6511,6046,7816,1518,3563,1620,3863,934,3031,2012,2927,7249,6337,7248,6647,4655\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 231\n", + "Returned IDs: \n", + "Ground Truth: 2231,5504,2735,4061,8112,5499,7807,1395,772,654,988,8643,1278,7377,5937,2746,1619,4345,1424,3723,7231,5192,7487,4039,6493,4721,7284,6051,8301,240\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 232\n", + "Returned IDs: \n", + "Ground Truth: 8247,4436,7865,8583,1993,5514,1790,6557,3222,3704,7093,5032,9292,1196,3542,2970,9024,8711,9037,6680,590,5179,9378,6285,4923,5303,2313,5281,8445,2243\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 233\n", + "Returned IDs: 5910\n", + "Ground Truth: 2359,4400,7951,7566,8340,3477,8371,962,2197,50,9796,828,7259,8393,920,4824,9939,2188,2840,7452,4562,9093,2603,7569,5671,3118,414,4649,7635,8342\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 234\n", + "Returned IDs: \n", + "Ground Truth: 2174,5289,8972,2660,1518,6165,7753,8120,5940,4921,6307,6767,8770,7017,1134,2927,9593,3516,6106,3610,2450,3317,5371,8889,9148,290,9958,8349,9733,1451\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 235\n", + "Returned IDs: \n", + "Ground Truth: 2966,6747,4061,4953,7789,2743,9492,8953,5354,9215,1619,1817,5034,4796,4887,4548,1881,7217,4721,5480,920,2952,1278,5200,3445,4417,6538,7499,9914,5146\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 236\n", + "Returned IDs: \n", + "Ground Truth: 9900,9384,3569,4688,3661,5024,6109,2812,501,9587,392,8726,2239,15,5067,8244,8358,1659,7241,4141,7554,1213,2527,7569,7981,2954,9682,1383,8442,8681\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 237\n", + "Returned IDs: \n", + "Ground Truth: 2437,2078,6992,4622,2497,2804,5574,846,8866,6118,1430,4075,2837,3859,2573,6103,2859,4256,8284,6232,5538,1809,679,1886,9120,9069,3871,5657,5918,7956\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 238\n", + "Returned IDs: \n", + "Ground Truth: 7432,7364,4796,8346,7193,3526,4061,9711,3651,4460,3622,2785,8844,6235,7089,463,7494,9777,4218,3310,9095,8892,2943,309,7929,1685,1379,5716,1344,1090\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 239\n", + "Returned IDs: \n", + "Ground Truth: 309,386,7542,6615,352,2554,3867,7389,2111,7822,5505,5181,1151,1310,7946,7865,4606,841,2011,2642,8796,5303,1849,728,9733,5680,6493,3517,9254,7570\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 240\n", + "Returned IDs: 5303\n", + "Ground Truth: 3222,5179,5303,8360,8247,6512,9794,1790,8937,9024,8108,5607,6854,3704,63,2243,9417,6802,3780,4381,1861,8711,1993,9155,8196,2753,7790,2970,590,6680\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 241\n", + "Returned IDs: \n", + "Ground Truth: 2642,8117,5806,2356,4109,4457,9824,5294,9296,1861,9405,8340,3194,8842,1003,9796,8141,4497,7016,7125,2554,7603,7243,6594,4256,6693,2974,9617,3078,2431\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 242\n", + "Returned IDs: \n", + "Ground Truth: 3222,8247,5472,4091,239,146,8787,8711,836,8360,4634,6307,6369,3542,9037,5281,6980,8604,4436,590,3185,493,7358,3604,2861,3704,8743,6557,8894,7790\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 243\n", + "Returned IDs: \n", + "Ground Truth: 2119,6854,4431,3822,6324,9744,6638,5565,6683,9219,2147,3937,1602,6219,6925,1613,5927,6647,3871,309,7043,2404,1871,7030,7246,286,7578,3963,608,3197\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 244\n", + "Returned IDs: \n", + "Ground Truth: 6514,9665,4129,1643,2147,8150,4144,6967,3878,5117,7431,8499,8267,8672,3158,2495,5311,6936,1696,9269,7125,4666,4755,5900,2308,7482,9964,164,6080,936\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 245\n", + "Returned IDs: \n", + "Ground Truth: 4061,1619,8953,3445,654,9561,6747,7466,4113,7594,1278,7410,5572,5480,5937,3520,5340,5774,7511,3501,7432,9093,1540,5504,772,9492,5972,4586,2745,2223\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 246\n", + "Returned IDs: \n", + "Ground Truth: 487,1595,91,6575,7485,4029,7042,1403,3904,6211,5844,6525,7543,7638,9968,9303,4490,5130,5242,1217,3596,7709,8147,3898,608,2165,2927,1503,2948,5404\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 247\n", + "Returned IDs: \n", + "Ground Truth: 3585,2174,3317,6495,541,8318,5878,2980,1634,4497,7380,4918,3734,6647,5885,1451,7248,3320,6511,7816,2533,6981,2927,6459,1403,7348,1518,7966,4214,4891\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 248\n", + "Returned IDs: \n", + "Ground Truth: 7898,2542,5330,8412,1602,154,69,5806,2865,5468,192,4688,2961,3764,7248,3486,6492,7267,992,608,2642,6947,9134,1144,8305,4625,2876,5154,2907,3629\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 249\n", + "Returned IDs: \n", + "Ground Truth: 8211,4548,2918,5808,8156,6764,6747,7522,3963,3576,4909,9492,5736,2521,5917,4953,9287,8892,6803,4859,6314,4463,7168,8103,5290,5146,3477,9408,8393,5340\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 250\n", + "Returned IDs: \n", + "Ground Truth: 4877,1206,7949,3310,9365,4905,7391,7798,4460,1398,3501,981,1457,377,6235,5716,5721,909,1376,3024,7070,4896,1344,7089,8346,1090,5553,4293,7629,9938\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 251\n", + "Returned IDs: 1905\n", + "Ground Truth: 6843,2146,2269,4573,3635,8033,7674,9024,4683,1830,9283,3015,506,5549,6913,7033,4400,9911,476,8455,6706,5898,8233,657,5106,9521,4141,1253,8936,5988\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 252\n", + "Returned IDs: \n", + "Ground Truth: 4742,7869,9035,1627,8038,2084,3491,2397,1223,5961,36,7832,5390,825,7600,5837,5531,1062,9453,2057,8087,4269,6649,3211,2862,6557,8019,4308,833,9430\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 253\n", + "Returned IDs: \n", + "Ground Truth: 4157,4532,5671,624,4649,992,2370,6146,2859,6646,5278,1132,4622,2931,2975,3770,952,4400,7381,9371,8042,9639,5615,7963,1809,3110,4256,7123,5338,3652\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 254\n", + "Returned IDs: 4388\n", + "Ground Truth: 6004,6538,9822,8393,233,5848,4562,4799,1619,5401,5664,7522,9542,2247,9287,1935,4460,5034,2952,2231,8066,4953,6794,3246,5029,2082,828,450,1289,2603\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 255\n", + "Returned IDs: \n", + "Ground Truth: 4308,7634,2084,9035,7691,9453,825,36,6644,539,3876,2017,8893,7003,756,4936,3211,7442,5157,9214,5390,4831,2169,7568,782,2938,4269,6859,1075,5675\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 256\n", + "Returned IDs: \n", + "Ground Truth: 8340,4649,312,2943,4400,2045,9371,7603,7793,1724,7047,4256,7997,7123,2840,2763,7143,6646,6594,3717,798,5992,8914,6100,2952,9824,8061,1663,1156,5278\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 257\n", + "Returned IDs: 1497,2123,2138\n", + "Ground Truth: 1003,1415,9781,6119,6615,155,8932,1687,536,8889,5773,3826,6904,8625,1861,5885,6662,1497,7217,6338,290,8753,6794,6165,1162,6046,3707,6106,7093,3086\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 258\n", + "Returned IDs: \n", + "Ground Truth: 9059,6901,3042,2596,1727,9984,6615,5940,4148,4053,3296,9663,5787,2086,5133,5514,6843,1545,9186,3094,2937,8090,5181,3749,3902,7193,6459,1805,6089,1531\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 259\n", + "Returned IDs: \n", + "Ground Truth: 679,6374,7951,1885,8442,4100,7891,1239,4658,5254,9903,1677,1070,7406,3377,3963,9709,1881,1963,7217,2850,7040,9117,4400,2078,8653,7202,5104,7061,4921\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 260\n", + "Returned IDs: \n", + "Ground Truth: 1963,7900,1885,7951,7013,5416,679,792,5067,4157,2078,2356,893,8900,8342,8141,3432,3477,9234,8407,1715,2972,4688,154,828,6551,2975,5447,9742,8706\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 261\n", + "Returned IDs: \n", + "Ground Truth: 5989,3377,9663,2086,664,9105,7547,6353,8753,1545,9668,9255,600,3296,1907,8223,8447,6459,5968,6282,4103,9266,9186,4264,8090,9378,6087,750,553,8350\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 262\n", + "Returned IDs: \n", + "Ground Truth: 9872,2322,3112,5979,6152,412,7111,1619,3527,3551,775,9625,699,2802,2295,1964,920,8892,194,9684,9690,7499,218,2717,1823,483,2655,5934,5679,5363\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 263\n", + "Returned IDs: \n", + "Ground Truth: 2524,8238,8879,978,214,179,2759,5953,8442,2637,1501,4614,6871,467,2696,4400,3546,9134,9684,5662,5784,4343,7951,8858,4059,9935,2618,7410,7143,414\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 264\n", + "Returned IDs: \n", + "Ground Truth: 8933,1103,7636,5729,8411,1242,8360,3202,8247,1747,2293,6854,2472,3008,9893,3627,3937,1861,9711,7427,803,7217,6557,5472,8711,4357,1960,9345,2266,3185\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 265\n", + "Returned IDs: \n", + "Ground Truth: 5525,1822,4181,7064,5992,7856,4293,2423,4855,8235,2943,7432,9217,3792,7248,3586,5047,3430,9371,4400,2112,1439,4512,1540,5589,1096,7629,7417,3960,2771\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 266\n", + "Returned IDs: \n", + "Ground Truth: 7380,4214,8597,2743,4246,4891,7350,3553,1518,8969,608,7778,7816,704,8717,8569,3474,6964,6094,3798,7966,3987,4492,8625,6324,2161,5958,9134,7348,6742\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 267\n", + "Returned IDs: \n", + "Ground Truth: 2406,7501,1079,8623,4075,1383,154,4865,5394,7022,3376,9605,228,3432,7767,7887,8232,8026,9903,1342,556,2790,4892,3687,8653,3262,1016,4688,5363,6489\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 268\n", + "Returned IDs: \n", + "Ground Truth: 773,8787,6307,3377,6369,9468,836,8443,6040,4634,7017,4959,4635,3604,352,4206,2596,9556,3264,6615,6608,1745,390,1749,6980,4421,9490,386,7141,1738\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 269\n", + "Returned IDs: \n", + "Ground Truth: 3458,9308,2530,8313,8849,2599,2837,2785,4826,203,4924,6103,8141,8398,8042,2641,642,846,1548,1827,7384,2804,4830,624,6430,6647,2078,6124,359,6721\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 270\n", + "Returned IDs: \n", + "Ground Truth: 1352,7217,4497,5885,9536,7772,2356,8498,2428,7027,7570,5946,7816,2927,3717,1861,4745,8183,7966,5750,6647,9294,243,1003,5385,2785,6321,9296,3034,9016\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 271\n", + "Returned IDs: \n", + "Ground Truth: 9138,3798,3448,9024,5706,3273,2679,9693,2778,2589,6459,3417,4727,3548,9423,8152,3528,7753,2380,7542,4932,7282,4398,8405,3185,1990,2162,2174,3644,8411\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 272\n", + "Returned IDs: 3935\n", + "Ground Truth: 92,1562,811,3081,1376,6153,3935,4767,6405,9711,5965,5700,5553,4896,7629,1944,4460,5112,476,9777,8654,9145,4887,7949,6842,8988,8000,309,3578,1273\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 273\n", + "Returned IDs: \n", + "Ground Truth: 5776,7625,1222,2530,6372,1150,6847,160,8060,7554,2714,132,9796,5650,9415,7699,8587,7767,392,7059,9448,8057,712,8338,4734,15,3652,3717,879,2278\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 274\n", + "Returned IDs: 7143\n", + "Ground Truth: 2746,2178,6723,7143,7131,1273,5945,8837,9062,7798,3484,3159,1344,6851,4460,4466,2541,698,4281,5649,7922,1457,3840,7629,2,5992,8143,1658,986,420\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 275\n", + "Returned IDs: \n", + "Ground Truth: 8237,5154,1290,4506,878,2040,3320,2346,6106,6683,536,9308,7029,8042,1861,351,6580,6639,6220,4655,5278,9448,5306,7966,2778,7093,66,4658,8184,920\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 276\n", + "Returned IDs: 1844\n", + "Ground Truth: 7634,1844,4308,8982,5420,1177,919,2084,5390,756,9557,833,6649,4609,4271,4788,8433,7691,6644,7568,1349,7600,576,825,9130,7750,7619,1837,3242,2169\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 277\n", + "Returned IDs: \n", + "Ground Truth: 3474,3979,3086,517,3386,994,1683,7380,6981,2117,5110,7282,3987,6495,3317,2778,9016,1687,7093,5717,1403,4769,5985,2729,608,6046,3800,4027,7816,2450\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 278\n", + "Returned IDs: \n", + "Ground Truth: 4935,5596,2730,3176,2468,8522,2174,8155,2325,608,9495,2012,4250,3474,8173,6854,5110,7542,7282,4634,6220,2838,6046,2927,8227,5607,2834,241,1362,3185\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 279\n", + "Returned IDs: \n", + "Ground Truth: 2112,3710,4896,7929,5276,3651,8119,7629,5945,1344,7091,4293,7432,9780,2036,9095,4477,8346,5525,2505,2746,3829,8498,7375,2383,4400,1206,5992,3262,3310\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 280\n", + "Returned IDs: 8098,7734,298,6411\n", + "Ground Truth: 2872,7241,7976,1440,1476,2066,6983,3281,7899,2329,3161,9425,6110,8098,5074,6062,1885,451,8970,9947,3633,6502,9191,7532,6246,8454,1105,7900,620,2825\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 281\n", + "Returned IDs: \n", + "Ground Truth: 8557,7637,751,4586,9341,1800,6345,2108,3520,3148,1462,2745,5480,414,134,8481,4334,3310,8100,9446,1225,1802,7466,1540,7089,7591,7432,3793,6378,210\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 282\n", + "Returned IDs: \n", + "Ground Truth: 1290,9448,6121,226,8430,4688,2759,2524,8550,8498,3717,4310,3320,6451,1818,3636,4268,9969,548,8096,69,6871,6939,2961,2404,74,2831,3687,4734,3367\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 283\n", + "Returned IDs: \n", + "Ground Truth: 5243,5868,7891,7956,1359,7374,4658,3282,9723,8892,3913,5031,1881,2405,6887,8737,1861,9117,1068,8307,3515,8823,1544,3694,186,4548,3551,3963,5656,9287\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 284\n", + "Returned IDs: \n", + "Ground Truth: 1465,8061,2393,9456,3880,725,5728,1064,9325,5359,2228,804,7801,7125,9411,9405,2545,6521,2306,3980,767,5047,2567,1105,2403,3623,5797,7570,2427,3868\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 285\n", + "Returned IDs: \n", + "Ground Truth: 8052,1866,5671,8340,4683,6103,2931,8915,6257,6374,9351,8198,5324,2045,64,3476,181,5840,9503,5803,2744,2530,679,4256,6817,2986,828,9263,203,2099\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 286\n", + "Returned IDs: \n", + "Ground Truth: 9496,9777,8609,4069,1562,68,9301,2032,1944,8988,3564,416,3014,5575,3477,7949,9418,3426,4460,4215,8892,9711,3629,7629,6326,9095,2012,727,4293,8423\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 287\n", + "Returned IDs: \n", + "Ground Truth: 6623,7740,2603,3385,372,6422,525,7472,2943,8099,414,4562,6006,3686,4460,6540,5905,7591,8858,3931,7868,9459,5022,7279,6741,6759,2863,5496,8663,3995\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 288\n", + "Returned IDs: \n", + "Ground Truth: 3518,818,1742,8087,8900,539,4951,5575,3117,869,125,5084,3731,8312,2531,776,6113,4831,1648,9928,2899,7292,5986,825,6307,7469,8413,3220,8284,5311\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 289\n", + "Returned IDs: \n", + "Ground Truth: 4721,1619,3723,7364,654,1278,309,7557,5349,4061,4132,7193,5489,7089,7822,3867,8953,9365,2291,3757,6493,9063,1273,6090,7432,7499,9992,6825,4796,8879\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 290\n", + "Returned IDs: \n", + "Ground Truth: 992,7085,3651,7432,8498,8879,2374,920,5480,7267,5330,5958,7143,2201,7089,1344,2404,1379,9095,1827,3629,5588,2454,4796,5363,6266,548,5716,734,9777\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 291\n", + "Returned IDs: \n", + "Ground Truth: 5287,7375,2095,3454,4289,4783,5002,9483,2462,3486,3649,278,4940,8153,1614,6387,6853,2449,5313,1096,3020,7359,3430,7629,5330,4991,6283,2306,191,2929\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 292\n", + "Returned IDs: \n", + "Ground Truth: 3715,6851,1167,6898,7131,1642,4236,4722,4466,420,3840,1785,263,398,3689,4455,3328,1292,1191,1160,2107,2624,1183,7253,1750,4386,904,7807,1540,1206\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 293\n", + "Returned IDs: \n", + "Ground Truth: 2023,2188,8893,9288,1827,2837,5574,4831,2646,3859,4296,9682,9181,8398,6992,36,4400,3722,6103,4497,2078,3511,9614,642,2804,2437,4308,210,3973,2074\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 294\n", + "Returned IDs: 754\n", + "Ground Truth: 3689,5884,4406,1642,3715,2107,3501,5649,5552,8884,7070,4386,1270,754,1596,2103,1480,8466,1435,2027,2624,5553,1344,3104,7089,9354,1813,8419,8450,7929\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 295\n", + "Returned IDs: 2282\n", + "Ground Truth: 7816,1861,2174,9875,9396,7217,9781,2488,155,536,8369,1415,6854,1753,4457,6871,5750,2927,1451,2778,836,3078,9593,7966,6000,8353,2642,8972,8340,1930\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 296\n", + "Returned IDs: \n", + "Ground Truth: 6615,2450,254,1518,727,2869,7558,5404,1687,3707,737,1497,6794,6488,6491,7322,2514,493,7925,6119,9750,3340,2596,8411,2162,2769,9071,3826,8104,3931\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 297\n", + "Returned IDs: \n", + "Ground Truth: 532,8961,8389,3386,4699,4246,7225,5885,9405,9093,2450,2533,300,2356,2773,6580,4891,3987,1518,5878,994,3615,3474,2937,1134,4470,9117,7865,8318,1753\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 298\n", + "Returned IDs: 9735\n", + "Ground Truth: 3438,9735,7270,5148,8393,3246,6004,3908,5848,1539,8105,3857,4824,7115,3112,14,3120,8263,6419,2131,6573,7569,7381,1523,6298,9076,176,3074,4562,3809\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 299\n", + "Returned IDs: \n", + "Ground Truth: 2831,9969,2961,5330,278,3629,9625,1964,3551,4444,3687,3759,3764,2322,8653,2078,5468,2404,3484,8892,8856,3356,8430,8205,6097,7267,4991,6563,7628,7669\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 300\n", + "Returned IDs: \n", + "Ground Truth: 5069,912,4742,3962,4638,2177,500,3491,8588,1062,3477,6811,8576,2724,3539,7241,1213,3514,5837,1392,3407,1175,136,7390,4642,8087,6724,9269,8859,2332\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 301\n", + "Returned IDs: \n", + "Ground Truth: 5048,4288,622,4176,6459,3896,6615,8949,5787,5543,3604,2596,1727,8583,3377,1068,5940,5452,8171,8090,4195,4009,8340,4327,7193,2836,7946,2998,8223,699\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 302\n", + "Returned IDs: \n", + "Ground Truth: 9981,3291,3984,2324,3407,6834,6691,4638,7917,789,4094,2314,6936,2332,7336,6029,9569,8413,6592,1254,3214,2664,2531,147,742,4407,5311,7750,1035,4550\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 303\n", + "Returned IDs: \n", + "Ground Truth: 36,1837,4308,3276,7634,1178,6883,7568,825,8332,2938,350,9430,7750,5961,9622,905,539,5043,3131,6598,9234,3514,3859,8799,3199,6165,3130,7542,1648\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 304\n", + "Returned IDs: \n", + "Ground Truth: 713,4354,7117,3564,5860,6849,8230,1944,9707,3461,416,3426,9024,1481,9777,1562,9406,68,6322,1503,7916,1073,6681,3755,4553,9988,258,1267,7152,2744\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 305\n", + "Returned IDs: \n", + "Ground Truth: 5158,8780,6874,9617,8025,5294,309,3078,7129,8216,3033,6493,2370,8278,3570,7865,3473,8879,5288,4781,7965,1253,5340,2554,2568,7916,2572,9254,845,4867\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 306\n", + "Returned IDs: \n", + "Ground Truth: 1373,9467,500,9287,4168,8765,3406,6741,6873,932,4562,3686,2810,2231,7487,9682,5972,5784,7951,6825,9851,6151,7262,129,7161,5592,40,7423,3118,7149\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 307\n", + "Returned IDs: \n", + "Ground Truth: 5452,3734,4887,9291,7248,9770,2012,517,7790,2404,6854,9877,828,8583,3477,1213,9237,4655,3202,3317,9345,6662,6106,9371,6285,4688,8896,3474,8405,1593\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 308\n", + "Returned IDs: \n", + "Ground Truth: 8247,146,3185,1790,2970,7706,7790,7314,2283,9821,5281,8411,5303,6854,63,1196,1993,1152,4436,8108,3222,6046,9037,6686,540,6153,4634,590,1400,9476\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 309\n", + "Returned IDs: \n", + "Ground Truth: 688,8089,9242,5476,483,2717,5146,2494,4984,4557,1622,920,7465,5230,276,1025,460,1504,2454,5714,9133,7144,7742,1619,6129,2263,6610,3112,4343,910\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 310\n", + "Returned IDs: 7360,5296\n", + "Ground Truth: 5949,8914,6540,7603,7078,9184,2559,1905,7587,8647,3873,3146,1276,1163,6387,8204,6594,349,5496,6252,962,1573,3171,6138,3492,4512,2785,5525,5879,3669\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 311\n", + "Returned IDs: 4497\n", + "Ground Truth: 4475,7442,3880,7217,428,5294,8025,3717,4497,8061,8162,9405,5509,536,7578,6862,5885,3320,5789,8340,2907,5994,4467,7032,3695,2712,7801,6693,2370,1613\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 312\n", + "Returned IDs: \n", + "Ground Truth: 804,9093,9795,1105,3078,5940,5480,628,1885,309,8002,7529,6004,352,6615,1619,2567,8796,2078,6324,2850,1861,9843,7966,4973,8879,8014,4497,3517,7095\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 313\n", + "Returned IDs: \n", + "Ground Truth: 4197,5018,4535,898,795,3961,2736,7037,1885,6583,1541,3007,4319,6095,5034,8317,6134,5983,4647,8926,9832,7990,7056,3977,4821,4563,3706,5925,4136,1476\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 314\n", + "Returned IDs: \n", + "Ground Truth: 7,6863,2358,2712,5219,8688,733,6106,5994,2642,4708,8422,4497,2997,5666,9011,318,66,6788,5464,3479,3659,9016,6976,9294,8558,2168,1985,9371,3707\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 315\n", + "Returned IDs: 6759,7020\n", + "Ground Truth: 960,1963,8379,8265,5385,893,1903,8407,821,4683,6103,7040,4400,7554,8342,2131,4242,7013,9149,7767,5338,7594,6257,3432,4992,5972,1397,3965,2975,1677\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 316\n", + "Returned IDs: 7297\n", + "Ground Truth: 7297,9028,1555,3578,3088,9410,2207,9711,3081,9206,6235,9875,8983,2316,8519,3310,103,7078,5965,8654,3145,3526,3033,476,1051,9777,2094,6405,2271,5716\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 317\n", + "Returned IDs: \n", + "Ground Truth: 1105,9682,7594,1501,6765,2575,3633,628,1885,1168,7742,8334,5034,6825,4562,1278,4887,4831,7381,9302,4439,9861,6110,8970,9408,6639,4688,1700,9914,6038\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 318\n", + "Returned IDs: \n", + "Ground Truth: 8853,8706,1369,1963,277,6449,2745,3445,7381,9359,5021,5321,2655,4324,8537,3908,3112,4638,7951,2696,5662,5067,7487,6594,839,8442,2131,828,4824,8283\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 319\n", + "Returned IDs: 7337,7838\n", + "Ground Truth: 7129,5988,6758,6103,5538,8313,2837,8653,7956,8042,4400,9843,8233,127,2078,3695,2370,2023,228,3859,4256,1809,1524,2859,2573,5432,7471,2706,679,6647\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 320\n", + "Returned IDs: \n", + "Ground Truth: 8933,9914,9907,6073,5898,3610,642,3757,3347,1830,2310,1593,9843,495,2642,920,5900,8892,9506,2831,5385,7027,1920,9510,9287,1290,3828,5278,3551,3695\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 321\n", + "Returned IDs: \n", + "Ground Truth: 5761,7091,6874,4745,3721,8025,7442,29,3947,1944,834,8591,3426,1253,45,5980,4240,3642,5186,2383,7125,8498,9671,6028,8609,7482,3564,940,9777,1447\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 322\n", + "Returned IDs: \n", + "Ground Truth: 2545,2228,9562,4655,7137,5806,2249,8061,5940,658,8406,1871,8489,3623,1105,7950,2078,804,6219,9415,2127,2219,4206,8276,2919,243,8340,2693,1029,2147\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 323\n", + "Returned IDs: 9764,2384\n", + "Ground Truth: 9754,5898,8774,7491,883,3184,4683,1412,1830,5388,2605,6843,9269,8316,401,5873,6269,5372,1961,8801,1229,5932,6258,5071,3488,6059,4472,1330,3927,6706\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 324\n", + "Returned IDs: 287\n", + "Ground Truth: 9792,5319,2831,9724,920,5377,2122,6016,570,9021,9307,1440,1164,1885,6115,1709,9861,4858,7401,9455,4619,9781,1083,6683,2743,4953,4765,9934,3940,5031\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 325\n", + "Returned IDs: \n", + "Ground Truth: 2960,771,7753,8571,1518,8969,9148,6324,1047,3800,5110,2920,6813,2205,5082,7538,8876,1692,6653,1921,3548,90,6964,3185,2174,8174,7816,146,320,6220\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 326\n", + "Returned IDs: \n", + "Ground Truth: 2671,3761,1098,8194,7816,3622,1683,608,6752,309,2174,5885,1503,8423,659,6046,9711,4492,9367,1403,1518,2117,5110,9451,5242,101,8712,7943,6350,7638\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 327\n", + "Returned IDs: 7356\n", + "Ground Truth: 2507,8319,3033,7248,654,6266,1168,1771,7217,6580,5034,5480,9310,5711,3116,2472,4293,3244,8504,7424,4557,3624,8594,4497,1818,3629,1379,9692,7252,3477\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 328\n", + "Returned IDs: \n", + "Ground Truth: 8850,2045,3978,4389,9051,6620,3700,7097,8407,4385,9643,5202,582,678,3847,8244,9921,1398,3551,4244,1911,7310,499,1562,8342,9824,7123,8177,4256,4400\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 329\n", + "Returned IDs: \n", + "Ground Truth: 4400,5589,154,3629,8198,8233,6594,7033,3262,3757,4460,7248,769,5477,2865,9371,1243,4293,7754,6714,4512,7267,8205,1439,4140,6057,3624,6253,7078,2943\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 330\n", + "Returned IDs: 7356\n", + "Ground Truth: 3040,3366,1525,3111,7898,2291,3665,6571,6249,4992,6810,6551,8707,7078,9742,7085,5381,7800,4552,4012,154,5949,9042,6266,2732,1827,6969,6420,2785,2911\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 331\n", + "Returned IDs: \n", + "Ground Truth: 7754,1963,3832,1663,8442,7554,8244,9912,5892,2359,3661,1540,4534,7259,9796,8342,4157,7951,2097,2683,4330,6238,9072,132,8674,893,9065,4649,6821,5216\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 332\n", + "Returned IDs: \n", + "Ground Truth: 7377,9861,8026,9992,9213,8103,1619,9529,6764,6803,5679,9684,1840,7807,4061,2524,483,4887,3484,988,8892,7555,1817,5079,1235,4795,2,5917,323,2247\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 333\n", + "Returned IDs: \n", + "Ground Truth: 1526,4904,9972,2736,1105,6142,3281,1885,9947,5983,4569,4197,628,6983,7745,1125,9751,6219,4563,605,1670,7099,100,7594,898,7539,3633,3043,3477,4136\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 334\n", + "Returned IDs: \n", + "Ground Truth: 1965,1457,8123,4704,5363,8703,2271,3551,6096,4386,1813,6247,8856,463,3282,5649,3056,4846,1216,7089,3477,6532,8653,4960,4293,420,7146,8093,4722,7823\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 335\n", + "Returned IDs: \n", + "Ground Truth: 415,4950,3757,3595,4400,76,4600,4311,9788,6115,8238,40,4515,6452,9506,9176,3999,3977,9992,7497,6825,5949,8312,3629,414,4061,7957,312,2664,7410\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 336\n", + "Returned IDs: 2194,2438\n", + "Ground Truth: 5737,454,1905,4998,9133,6643,6091,1544,9359,179,3477,8442,1963,154,3351,828,7788,8653,3199,312,3050,7793,5784,8754,3963,3825,9093,6313,7040,285\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 337\n", + "Returned IDs: \n", + "Ground Truth: 2039,414,6516,2052,3873,525,9133,9093,4998,962,2112,8858,1815,7271,5784,8559,2921,389,3371,7089,5496,3526,4460,5046,3468,9898,312,8953,7078,1457\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 338\n", + "Returned IDs: \n", + "Ground Truth: 1996,145,2437,7990,2427,642,6103,5385,2235,142,8537,846,3934,3050,1738,1105,8117,4821,5918,3966,8304,4014,9050,9296,679,1076,5314,5590,4263,9682\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 339\n", + "Returned IDs: \n", + "Ground Truth: 5480,1168,3102,2409,3752,8557,9063,9341,7432,9492,4334,1800,210,4460,8953,4293,2,6378,8563,7591,204,2503,7399,2745,8852,7929,9198,8614,3520,8119\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 340\n", + "Returned IDs: \n", + "Ground Truth: 1148,3873,389,859,525,3371,4600,6252,312,1655,2921,962,8099,3530,9989,5881,5141,647,9093,768,6825,4346,2041,3669,4400,5666,415,6268,7078,348\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 341\n", + "Returned IDs: \n", + "Ground Truth: 3551,1216,5649,8093,5480,1168,1457,1779,3282,7798,8123,8856,4687,5363,7089,6887,8892,3484,4460,8328,5934,2831,463,8653,377,1813,3985,4293,8844,4846\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 342\n", + "Returned IDs: 3033\n", + "Ground Truth: 4063,7248,9821,3342,1451,9024,4185,4557,7862,3523,3797,4551,2146,6018,5311,5667,3709,5282,2012,7093,3750,7700,9188,2179,3528,6223,341,1137,3570,8216\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 343\n", + "Returned IDs: \n", + "Ground Truth: 5359,2545,8061,9964,6584,725,4497,1837,4655,2228,5241,3862,8569,2642,3320,3859,6967,5043,7003,9094,31,7125,2427,7032,5525,3871,3807,1096,4523,94\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 344\n", + "Returned IDs: \n", + "Ground Truth: 8533,5464,6829,7822,5925,7570,4991,9907,6073,5752,1339,7031,590,7009,4602,7560,7486,254,9770,3987,5162,41,3749,8175,1467,1583,3610,516,5818,3150\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 345\n", + "Returned IDs: 6814\n", + "Ground Truth: 2405,7248,2012,2001,9750,6814,706,8096,7720,2717,7966,643,8443,1134,3202,3033,2243,4140,700,8302,3320,3795,7267,7402,9763,4557,2201,2655,8879,3627\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 346\n", + "Returned IDs: \n", + "Ground Truth: 2842,7389,5562,2951,250,8943,8796,600,9059,5805,1944,7549,6513,3377,8753,9010,1151,3564,216,664,6508,2778,2554,5940,7373,1531,4497,6106,2111,7586\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 347\n", + "Returned IDs: 652,2595\n", + "Ground Truth: 660,7257,8130,2098,6911,4497,6387,2390,4256,6615,5972,5287,278,3430,4783,4400,9483,8406,6853,7389,7072,8153,3649,5940,341,628,654,8879,4991,2556\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 348\n", + "Returned IDs: 5710,7731\n", + "Ground Truth: 586,5792,1827,1866,8622,3492,8900,4986,1963,6253,3048,2728,3260,2683,1005,7425,8815,9745,2500,1060,2628,2785,3220,7831,3343,8198,5776,9227,1681,3463\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 349\n", + "Returned IDs: \n", + "Ground Truth: 7535,9927,3591,6258,2769,3015,7362,5873,1577,3140,8033,3645,2028,6398,4898,2045,1573,477,9622,6620,5122,9212,1674,7167,9373,1247,1722,5560,807,1830\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 350\n", + "Returned IDs: \n", + "Ground Truth: 5238,2603,9007,414,5592,6073,4778,4562,312,4168,6825,40,4460,98,3118,2915,5972,9359,7271,7660,2895,3140,5200,6623,3635,7466,4400,7905,6741,3811\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 351\n", + "Returned IDs: \n", + "Ground Truth: 9378,1298,3417,6261,8247,3704,3222,8583,3987,2293,2963,6307,8405,6285,8369,700,8711,2763,7865,5867,6337,9237,9576,4932,6634,2245,6615,5913,7544,258\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 352\n", + "Returned IDs: \n", + "Ground Truth: 6690,9024,8196,3177,1202,2012,2243,5262,258,6084,7790,2414,4009,4932,1861,6153,751,4775,5452,1298,3604,8787,9423,6341,7093,7246,5032,3312,6261,8583\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 353\n", + "Returned IDs: \n", + "Ground Truth: 3094,5240,1399,664,7373,3340,347,6689,9222,744,5361,3560,254,6615,5023,3243,5403,8438,2937,6508,9186,5733,8753,7073,6131,1045,5787,2111,7038,8794\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 354\n", + "Returned IDs: \n", + "Ground Truth: 9006,5861,172,5679,5085,9215,913,6580,3145,1817,2798,2753,7223,7401,2831,3246,4887,9017,1827,1700,5694,2621,3973,7499,1650,7381,943,9914,8026,2252\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 355\n", + "Returned IDs: \n", + "Ground Truth: 309,4400,6615,7754,5561,8340,6557,312,9875,5509,6165,6313,828,8134,4278,5671,5341,1562,4256,1944,506,2943,6103,1593,9093,5713,8057,9750,4460,6714\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 356\n", + "Returned IDs: \n", + "Ground Truth: 3033,7248,9371,7560,828,3750,2404,4887,5278,9354,2554,9770,920,5742,1593,6925,3526,4293,7356,4557,6387,1379,8879,8096,2431,4497,7629,2706,5403,3319\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 357\n", + "Returned IDs: \n", + "Ground Truth: 1290,2209,5169,3987,1861,9772,9472,7340,733,1213,6794,290,9703,2174,3320,5343,7452,1624,2021,3759,7381,66,6854,7027,1784,9770,1683,2974,2876,8389\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 358\n", + "Returned IDs: \n", + "Ground Truth: 2949,9860,5927,2174,3090,9103,1635,7816,3779,6544,3690,2616,8139,3386,1293,31,7134,5077,5440,9721,75,2625,1554,4398,7977,5298,8131,8018,6361,5885\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 359\n", + "Returned IDs: \n", + "Ground Truth: 792,1886,8589,9870,9900,1963,3007,9971,5024,4665,9314,2812,2804,679,9994,4833,9563,9073,1866,9346,1663,9069,7858,8358,2859,8265,3661,1639,3569,2510\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 360\n", + "Returned IDs: \n", + "Ground Truth: 6307,5925,8653,4132,6598,6608,652,4924,1920,642,7745,1885,8403,4357,6071,6182,4887,7570,1164,1809,6776,8406,3477,1551,4318,8524,4953,9120,6647,5363\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 361\n", + "Returned IDs: \n", + "Ground Truth: 4722,4236,1998,1292,5649,2078,3551,8123,6255,1604,1167,5615,904,8177,6633,4196,6889,4386,1327,7131,7253,4256,7213,148,3477,9120,761,9975,3840,6374\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 362\n", + "Returned IDs: \n", + "Ground Truth: 476,6235,4380,220,8149,3688,8498,5945,4442,5093,8346,9365,7432,6815,7091,5683,1156,1206,9711,5617,3526,3235,3651,9777,68,5386,8390,9354,2943,437\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 363\n", + "Returned IDs: \n", + "Ground Truth: 3757,9576,1593,9684,8313,1988,8879,5472,8711,9448,8247,3828,516,4460,9935,4400,483,6551,1003,2174,203,706,642,536,1827,8953,6608,6329,2356,1881\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 364\n", + "Returned IDs: \n", + "Ground Truth: 9796,2527,7059,7554,8064,2194,8191,683,4330,6884,2438,6007,5776,8082,6821,8436,2954,6372,1476,8012,2197,9481,1132,7900,1663,9346,3477,7136,9900,9203\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 365\n", + "Returned IDs: \n", + "Ground Truth: 3286,2896,8782,1720,1518,2712,3041,1783,5994,8981,7017,1985,8787,8688,7677,2283,7031,7093,9472,496,2168,7280,8533,590,8247,9893,8558,2511,797,7282\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 366\n", + "Returned IDs: \n", + "Ground Truth: 8504,3734,1619,6747,1394,654,6493,9838,4061,9764,4113,6615,608,517,3723,7643,5499,7432,727,9492,1389,1168,7231,9690,9095,3265,6179,3520,5504,772\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 367\n", + "Returned IDs: \n", + "Ground Truth: 6275,5947,1014,2869,6750,4357,7526,5466,5223,6488,1881,9694,1219,8411,1861,1968,737,3426,6910,7790,3963,1497,6557,3949,4180,3205,5940,7217,7956,9495\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 368\n", + "Returned IDs: \n", + "Ground Truth: 5586,2831,920,2122,4178,5230,6129,9838,6959,2575,6391,9462,5146,4953,5377,4298,308,8340,6448,6165,9751,8421,1792,9792,8035,7432,8026,6219,7594,9093\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 369\n", + "Returned IDs: \n", + "Ground Truth: 6289,1920,652,4924,6307,1996,8653,6557,7865,9345,2785,706,9552,1881,2642,3545,9451,8933,2596,3759,4744,6615,7141,1809,6608,2862,728,1650,2646,6584\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 370\n", + "Returned IDs: \n", + "Ground Truth: 4250,3863,8155,8247,2747,425,8830,2000,7790,700,5867,5445,6046,2174,8411,3474,1298,5403,1437,3008,2012,4009,3338,3563,7225,9495,6980,6680,2416,2312\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 371\n", + "Returned IDs: \n", + "Ground Truth: 8244,7123,3569,132,4416,4483,9346,2188,1032,9132,4330,5299,2700,7561,5615,2527,6255,6372,5454,9051,4532,3110,4568,7554,3388,2762,5202,9897,4621,8674\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 372\n", + "Returned IDs: \n", + "Ground Truth: 742,5654,3291,2308,9809,9269,4084,2531,4571,3948,9857,1643,4132,9145,8588,5311,7243,4094,2944,6936,2155,8521,8312,3431,7482,7390,3878,8026,8878,5344\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 373\n", + "Returned IDs: \n", + "Ground Truth: 2720,7772,2006,5806,624,4459,2402,3072,780,5294,702,5169,5380,1064,6621,1409,2703,4122,9827,8025,2412,4044,1003,3695,3880,2392,4497,9179,9153,428\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 374\n", + "Returned IDs: \n", + "Ground Truth: 5394,154,4688,3160,564,7501,7078,4012,8879,6226,9371,1593,6329,4546,1407,1827,1016,3432,7085,4293,3366,6115,5485,7629,4400,1079,4778,1569,9288,3040\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 375\n", + "Returned IDs: \n", + "Ground Truth: 7790,517,8411,7093,3863,6176,2162,8247,9055,3063,1518,5242,2174,6337,1687,1009,3734,5867,2514,6285,8583,2986,690,2000,6046,4931,659,6662,9423,3610\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 376\n", + "Returned IDs: 7218\n", + "Ground Truth: 6557,2596,6608,1265,8588,6307,6615,2937,6647,724,7865,9818,1920,1996,9451,2753,1060,2567,1551,5464,247,1593,2944,8443,8458,4357,9101,706,8445,3604\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 377\n", + "Returned IDs: \n", + "Ground Truth: 9215,1871,9017,4887,9006,2404,1817,2575,9509,7511,6747,9914,2521,9744,9200,5377,9290,8483,1619,1885,7570,8858,7628,5085,9738,8953,4765,3633,9844,514\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 378\n", + "Returned IDs: \n", + "Ground Truth: 6855,7699,4599,4649,1122,9722,2001,8177,828,2385,6057,9796,4157,5288,712,5007,992,4293,8379,549,1935,3477,9287,2952,132,3473,6722,9283,5589,1663\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 379\n", + "Returned IDs: \n", + "Ground Truth: 5759,8883,9562,8451,2062,9958,8329,2017,6598,5295,6737,36,9079,1947,6363,9601,7003,4703,7886,7962,4521,9779,9325,3796,5364,3970,2403,164,5399,4497\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 380\n", + "Returned IDs: \n", + "Ground Truth: 2698,2430,6810,5919,5071,6792,678,9349,8850,1188,7561,3700,499,5209,4992,6957,2045,582,8654,816,8633,2904,8033,6032,8342,5454,5577,1104,8407,1758\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 381\n", + "Returned IDs: \n", + "Ground Truth: 3270,5675,5200,8116,7880,6650,2840,8366,9063,7410,6998,7801,6427,7279,7591,6491,1438,414,9628,7271,6422,2197,2116,1159,3729,958,8953,4460,2710,4061\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 382\n", + "Returned IDs: \n", + "Ground Truth: 8302,9371,5321,4571,6856,8096,1379,7248,8879,8340,4307,3320,5337,536,2957,3033,3078,643,5776,6693,2201,828,548,9561,228,4830,4039,8653,6374,4843\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 383\n", + "Returned IDs: \n", + "Ground Truth: 3698,9145,2147,2009,8312,5157,8859,5311,2862,539,841,7431,684,9843,7125,1837,982,2642,8413,1187,3256,789,3878,8569,744,6615,7243,9269,4638,8576\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 384\n", + "Returned IDs: \n", + "Ground Truth: 7410,8363,4985,7399,958,1159,4814,5217,5340,7559,7353,7466,1542,2619,9492,1619,414,3216,2223,1583,5146,1840,4110,4343,4460,8643,8588,3751,9198,6764\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 385\n", + "Returned IDs: \n", + "Ground Truth: 5543,3004,5789,5940,6615,1858,7364,7193,5738,7529,9468,352,6425,309,772,4061,654,8879,137,386,9207,3347,9617,9750,250,4021,9059,7389,2654,2443\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 386\n", + "Returned IDs: \n", + "Ground Truth: 8915,4649,7123,181,7815,2045,798,7136,1663,8412,7091,992,2840,8342,4078,4587,707,2865,1208,5671,3118,9685,5992,828,2952,7131,6594,4400,7629,7801\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 387\n", + "Returned IDs: \n", + "Ground Truth: 1895,7522,1093,6593,8093,8614,4234,5917,1695,2779,3359,8638,720,420,4466,772,1619,398,8156,3928,4190,483,4887,1316,6451,8608,7748,5808,9744,7985\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 388\n", + "Returned IDs: \n", + "Ground Truth: 612,2,1462,4218,688,4991,5354,3593,5958,5313,4886,3310,9715,339,1344,2494,9743,8844,7662,8112,8130,8319,4796,8879,6283,4293,2404,3501,7213,5550\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 389\n", + "Returned IDs: \n", + "Ground Truth: 140,2834,6497,5095,2110,26,3863,8652,7642,6153,1211,1362,90,6662,5352,5900,4844,3581,2532,8796,290,3317,9592,1712,8084,9169,155,9378,8830,6331\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 390\n", + "Returned IDs: \n", + "Ground Truth: 6257,4968,7482,8825,9269,9764,4132,8871,2053,5819,3291,1719,8588,5117,4960,7089,7537,9809,4380,1619,4638,1830,1090,8346,981,9410,5654,2887,309,8857\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 391\n", + "Returned IDs: \n", + "Ground Truth: 348,920,1477,4400,7807,2696,3319,50,8914,5525,8653,7033,3757,6594,9371,9499,7432,1827,8366,548,3545,4649,6648,6551,1439,5256,7012,6283,2785,228\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 392\n", + "Returned IDs: 3908,7381,7696,1700\n", + "Ground Truth: 5321,314,6359,8706,7951,1903,3908,277,7040,9684,8355,8528,5021,1677,6449,7311,9587,1885,615,3477,3246,7905,9075,9359,1963,7381,3825,7594,5891,6004\n", + "Recall: 0.0667\n", + "\n", + "Query ID: 393\n", + "Returned IDs: 8198\n", + "Ground Truth: 8086,7051,9263,2450,3476,2309,8198,7432,6702,8663,9866,4132,6723,9731,5200,9578,9992,4061,9797,5890,7554,9408,3258,3757,5671,4831,9212,4830,1924,2332\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 394\n", + "Returned IDs: \n", + "Ground Truth: 8594,2196,6261,5885,1189,2998,8649,5913,3863,5404,9717,7943,6951,4189,8423,3008,1066,9750,7966,7248,2012,5472,4726,643,5480,6814,9615,5033,1134,1861\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 395\n", + "Returned IDs: 3112,9550,5423,7742,2745\n", + "Ground Truth: 7027,7466,2516,9446,9315,3707,7956,5278,7257,9217,6557,9893,495,8501,8799,3430,5069,1003,2017,2592,2744,2608,5972,2556,6647,7432,9536,7892,3112,1857\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 396\n", + "Returned IDs: \n", + "Ground Truth: 2554,2642,1809,9371,8096,5675,3624,4887,3033,3530,3871,828,3320,3880,4256,5278,4400,8653,228,7560,7248,2201,1593,9770,1003,2837,2975,2404,6214,2431\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 397\n", + "Returned IDs: 536,1818,8164,6735\n", + "Ground Truth: 6896,3150,6854,3717,8313,7570,9345,6369,2370,3759,2642,1817,228,2078,3474,2174,4390,536,2161,8933,1861,8879,1290,6220,1003,624,642,7217,3973,9738\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 398\n", + "Returned IDs: \n", + "Ground Truth: 1424,1591,4689,2728,3811,6445,8112,9198,240,7377,4061,7034,7909,5865,4839,2,1395,6764,783,8133,7555,32,1619,4260,3995,3999,531,8643,2298,9426\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 399\n", + "Returned IDs: \n", + "Ground Truth: 5433,9055,5341,3128,9378,4590,8025,1934,1687,7442,9237,239,146,8369,7753,834,1738,3444,1117,341,3273,2174,3397,2556,9024,1838,725,2753,3236,4490\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 400\n", + "Returned IDs: \n", + "Ground Truth: 7432,9939,1944,5873,5898,5716,9269,3426,3501,8654,4061,4582,2744,807,3591,6100,4400,3477,9354,5480,6620,6817,8033,2269,4586,6398,7091,5649,6509,3578\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 401\n", + "Returned IDs: \n", + "Ground Truth: 1861,7282,6307,6854,8625,290,8165,8787,5303,536,8042,984,4634,2078,3863,6369,6165,9674,8411,6980,7542,292,9118,8937,4726,9781,9355,6634,8889,8830\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 402\n", + "Returned IDs: \n", + "Ground Truth: 7561,816,678,6792,5577,3700,9349,8850,1911,4595,2104,3573,2824,585,582,8407,3646,1231,3178,6739,499,2904,7097,6794,7189,4992,7963,9151,9501,6772\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 403\n", + "Returned IDs: \n", + "Ground Truth: 4157,5671,2963,4137,2385,163,7613,2943,2188,828,9346,4519,6817,6960,1663,7047,2045,8244,8342,6255,3259,761,5650,8124,4854,5615,5964,7376,8042,9283\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 404\n", + "Returned IDs: \n", + "Ground Truth: 7491,4719,5873,4683,6999,7674,2426,5992,5898,7297,2037,1674,401,2269,4337,8033,48,6843,7067,8993,4293,1004,8654,5965,4400,3184,1986,9875,3015,3103\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 405\n", + "Returned IDs: \n", + "Ground Truth: 2,4796,1685,8157,7629,7949,4218,9267,4460,566,7743,6235,4293,612,7078,7089,1376,3501,9777,3484,9095,2417,3651,5064,371,9464,4830,6155,3551,5480\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 406\n", + "Returned IDs: \n", + "Ground Truth: 9921,2045,1601,6398,6794,5359,4389,5089,9473,6363,2258,4497,2393,3591,6521,36,6817,2545,352,3980,3078,2642,6615,7097,4244,9733,5112,3448,2370,798\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 407\n", + "Returned IDs: \n", + "Ground Truth: 4198,3040,8053,7078,3366,7531,9824,6387,8879,5949,4256,8498,3935,3160,992,8609,1827,6673,7085,4293,7690,4116,1343,807,4012,2201,3651,2975,9371,4400\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 408\n", + "Returned IDs: \n", + "Ground Truth: 9777,1376,1562,7089,9711,4767,7391,9418,9875,1944,1074,371,6235,3935,8609,1206,8419,4810,9158,7091,9028,7070,92,437,9239,8390,2103,9095,7097,598\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 409\n", + "Returned IDs: \n", + "Ground Truth: 5555,168,2274,5600,9273,8150,2841,2775,7750,3130,9190,2529,841,5157,2399,2481,4770,2927,243,4852,4144,5041,1230,8038,7003,7431,2862,3544,8292,2147\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 410\n", + "Returned IDs: \n", + "Ground Truth: 5623,6814,1403,2578,290,8549,5787,3800,2174,1613,3397,7515,654,1451,8024,2414,3347,2592,7009,9641,5447,2567,1593,6337,3008,1509,649,1835,8782,4911\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 411\n", + "Returned IDs: \n", + "Ground Truth: 8175,2730,7642,26,6046,1861,8827,2174,9355,3816,5049,6497,1930,1134,3320,2989,3037,3397,3867,8522,7810,9370,2405,3863,7902,90,4209,6220,8155,2567\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 412\n", + "Returned IDs: \n", + "Ground Truth: 4181,1576,2063,5525,5047,2306,767,3430,6283,1344,7997,3285,8235,3586,2390,4400,2112,6387,6594,5550,8647,9483,3444,2423,1439,9938,1113,7707,2418,7064\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 413\n", + "Returned IDs: \n", + "Ground Truth: 312,1815,2863,2045,807,5238,3999,7594,3270,170,3757,8407,8643,2028,3873,9989,506,5949,1133,654,1562,8312,4061,7807,2696,132,4460,7466,7078,8953\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 414\n", + "Returned IDs: \n", + "Ground Truth: 2017,4641,3017,5012,1328,4831,7685,7194,8445,9248,3290,782,6617,209,4521,9135,8893,4938,4308,9199,1252,8540,5575,36,3060,9236,2642,7140,8879,7125\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 415\n", + "Returned IDs: \n", + "Ground Truth: 2553,6334,4576,5501,2281,4330,9481,5776,5202,5923,4296,4252,7968,6733,9796,9073,6675,4831,2581,9497,9324,8199,5675,3679,879,6227,7770,8425,2239,7136\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 416\n", + "Returned IDs: \n", + "Ground Truth: 1905,8858,5784,6566,4934,8283,4562,5046,7951,8442,3650,4998,3546,4343,683,8051,2696,414,5302,1303,500,7566,8490,9133,8580,2655,8219,1501,5387,3450\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 417\n", + "Returned IDs: \n", + "Ground Truth: 1545,600,5514,8796,852,5940,5787,9059,1727,3094,8223,3078,4053,6615,8090,8949,3296,1151,6037,6282,3610,7413,103,2596,9048,1531,664,8753,7038,773\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 418\n", + "Returned IDs: \n", + "Ground Truth: 5434,6404,7064,3792,2929,1044,2801,1822,2423,5313,688,1344,3837,3020,9974,1614,6853,871,7629,333,1462,4293,3319,6188,8205,7248,460,162,7395,1540\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 419\n", + "Returned IDs: \n", + "Ground Truth: 3117,9040,312,5496,5592,7924,7603,5949,7078,6444,2374,6209,3873,1012,7254,2465,4643,8487,9280,8580,2603,1649,4460,525,807,414,8375,2120,5238,1199\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 420\n", + "Returned IDs: \n", + "Ground Truth: 9242,7935,3253,483,1817,9509,1885,9561,6673,1064,7569,4625,5984,15,1343,656,5034,1602,4887,8879,2190,992,688,9371,2403,8800,5711,920,7707,1344\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 421\n", + "Returned IDs: \n", + "Ground Truth: 3426,836,9095,7432,1902,358,8275,7816,9954,463,4060,6100,2032,9777,1187,2196,6647,4497,371,3707,5716,2147,164,574,5885,5789,1619,292,319,1988\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 422\n", + "Returned IDs: \n", + "Ground Truth: 1797,1963,6871,9288,978,1383,2759,4440,9682,3190,8681,4562,4688,6538,8442,154,6197,228,4779,4638,4141,4614,1593,839,8537,4225,1866,5176,6336,5645\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 423\n", + "Returned IDs: \n", + "Ground Truth: 1973,2696,699,1068,3477,5853,6764,8430,7033,1815,1881,7594,3872,1885,283,9117,1948,4463,2,339,8594,4562,2850,1903,3999,6551,828,1840,2266,2603\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 424\n", + "Returned IDs: \n", + "Ground Truth: 1817,5958,7085,5363,4887,7669,6288,6693,2404,992,8758,5330,1827,3206,8703,3214,1344,9914,2756,1681,2027,6339,463,1168,7223,1663,4012,5649,4796,828\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 425\n", + "Returned IDs: 5688,8141,4924,2754,9765\n", + "Ground Truth: 5589,154,4688,2428,8442,2078,3477,4141,679,2356,5688,1963,9283,4948,6336,9903,3190,1593,4400,7767,3725,2688,9133,1079,9288,9900,2785,6103,9072,828\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 426\n", + "Returned IDs: \n", + "Ground Truth: 2869,8251,5480,1881,3477,309,8458,6425,7529,4846,6615,8614,3860,5031,4206,3814,1168,6721,186,3551,1968,1457,493,2831,7389,2596,654,7946,5363,737\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 427\n", + "Returned IDs: \n", + "Ground Truth: 2763,9746,1064,624,2876,9576,9673,6794,828,2815,1602,2840,8862,6551,1409,2850,7801,4649,9109,1861,7285,1935,2385,2907,7570,5675,9254,2696,798,3972\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 428\n", + "Returned IDs: \n", + "Ground Truth: 4586,1802,6378,134,6184,4334,3148,4127,7432,2643,7466,4212,5972,6377,6962,8557,1443,3520,7637,8100,3675,6919,1101,3051,2745,3018,3457,1225,5535,1800\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 429\n", + "Returned IDs: \n", + "Ground Truth: 1376,7629,8654,5965,4293,476,8346,1562,7091,4896,5945,3935,707,9783,7949,1206,2769,4460,8518,9777,3488,4400,1344,6159,4493,5480,9711,3651,9354,3262\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 430\n", + "Returned IDs: \n", + "Ground Truth: 5283,4939,1385,5504,1885,2082,6219,1551,2122,4373,6648,4061,6054,7594,4953,9093,1944,8275,8393,9914,3445,1817,5034,2249,8014,628,5340,5031,5654,1728\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 431\n", + "Returned IDs: \n", + "Ground Truth: 6387,6112,5972,312,1133,9483,2556,807,4400,3717,3757,5278,1439,5949,8498,7868,2975,7956,2785,7432,5589,341,4887,652,2771,3430,9448,9145,278,7466\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 432\n", + "Returned IDs: \n", + "Ground Truth: 9406,4002,2616,7485,2442,1613,4029,4490,6019,5764,3755,1831,1308,91,7543,5110,1415,3622,6575,6681,9422,7902,2153,9873,9988,1584,8272,2196,1572,542\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 433\n", + "Returned IDs: \n", + "Ground Truth: 1117,2179,8194,1569,8216,2556,7248,9757,4571,2275,5612,3033,6187,3709,4820,7862,3319,6271,164,707,1853,3734,2774,9247,9918,5093,2431,506,7605,3373\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 434\n", + "Returned IDs: \n", + "Ground Truth: 1861,7358,4497,2174,9355,3800,8625,4634,1972,5383,1849,5043,6021,984,8223,9781,3859,7790,3312,1687,3528,7542,6980,6854,7282,6106,3186,3395,3338,3130\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 435\n", + "Returned IDs: 241\n", + "Ground Truth: 1861,241,6854,5783,4634,2831,2578,2174,7217,3320,2405,3292,8711,8827,6021,3738,7280,6307,3474,352,8155,1948,7966,4497,2243,6340,2567,6552,9223,6647\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 436\n", + "Returned IDs: \n", + "Ground Truth: 6701,9551,8012,2082,6354,7452,2410,3348,1159,6765,683,8635,9497,879,2683,6825,7817,6007,5925,962,233,768,3633,1105,2538,3666,5776,5675,3376,1963\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 437\n", + "Returned IDs: \n", + "Ground Truth: 9694,706,9098,6124,2596,2414,3749,6557,384,3859,8243,7772,2370,2836,7910,6338,9033,4718,649,6635,4476,5809,8331,5422,9564,911,8825,516,6776,7218\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 438\n", + "Returned IDs: \n", + "Ground Truth: 2038,160,8269,624,9576,5806,4622,6871,2642,2876,3426,1064,828,3610,2675,658,9824,6039,9371,1650,3432,3320,2545,2706,5675,7444,6547,292,9456,2530\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 439\n", + "Returned IDs: \n", + "Ground Truth: 3085,7699,9283,2385,6794,615,2472,3825,1963,829,4731,6594,1935,5650,2850,5776,2763,8706,5954,3751,9897,15,7085,2952,5670,2356,481,2262,4562,1747\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 440\n", + "Returned IDs: \n", + "Ground Truth: 8333,2071,3519,8247,9405,8649,3386,1613,7380,4560,3704,3303,9143,3474,590,9582,9684,6580,7441,4966,1993,7522,8022,6794,1443,4170,3973,7790,4891,1915\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 441\n", + "Returned IDs: \n", + "Ground Truth: 5621,4041,7790,6176,8382,5867,8896,9367,1497,6662,3395,3185,3734,8411,5242,9378,155,4844,6285,517,8583,895,6615,6153,2778,3086,9423,1009,9376,3863\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 442\n", + "Returned IDs: 4222\n", + "Ground Truth: 9449,6037,8753,6353,2162,7504,3498,3734,5361,3548,4339,6663,1687,9423,2477,3086,517,9988,593,3105,7263,673,9553,6275,3063,6519,3397,3377,7947,8042\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 443\n", + "Returned IDs: \n", + "Ground Truth: 2011,9395,7016,6073,6980,4634,7217,4400,6307,9770,3929,819,2907,3987,4206,5403,7560,7141,9345,4308,5445,3757,5278,5282,8443,9145,7570,3474,352,8653\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 444\n", + "Returned IDs: \n", + "Ground Truth: 4406,2506,463,3689,4293,1344,698,4687,7557,2027,5649,7089,909,3391,7143,1273,3501,8466,9595,8856,1642,5363,1457,1216,6247,2580,5895,8703,6702,5444\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 445\n", + "Returned IDs: \n", + "Ground Truth: 1588,3526,7193,9898,5859,3652,6703,734,3627,536,2201,15,5278,7248,9448,4581,6119,7364,7862,6630,2618,2785,4688,4213,8026,1809,8423,1569,9455,7432\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 446\n", + "Returned IDs: \n", + "Ground Truth: 3004,7529,6119,5940,8442,2011,8340,2831,7193,3627,6764,9207,4400,7951,154,9455,5776,8653,6425,1973,5543,6703,9295,4206,1016,5403,1963,3477,2472,4421\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 447\n", + "Returned IDs: 6740\n", + "Ground Truth: 7643,3070,6740,4490,5171,3771,658,4100,7677,4921,6990,1131,6813,7100,1134,9562,4146,1881,1885,9738,600,8892,2567,1930,949,8002,7444,5940,8681,1350\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 448\n", + "Returned IDs: \n", + "Ground Truth: 4009,7790,5213,3863,3800,2747,3474,3563,9495,6662,2243,8925,1518,8050,9385,8830,2000,8196,7597,3378,3397,8120,3540,9077,5564,6813,2834,5681,2590,4250\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 449\n", + "Returned IDs: \n", + "Ground Truth: 3426,5031,727,6153,6662,290,7217,7790,1134,3515,6374,6580,5346,3477,7069,2937,7033,7444,5940,416,1518,6488,828,9750,654,4658,2450,658,1068,4844\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 450\n", + "Returned IDs: \n", + "Ground Truth: 934,8024,2490,3286,8649,1518,8030,7,7606,5885,3937,3987,7225,5969,7778,9472,3479,7349,6340,4708,7703,2000,9579,1403,1861,8549,6525,9241,4099,8389\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 451\n", + "Returned IDs: 2282\n", + "Ground Truth: 8025,7442,5278,3880,9770,878,7016,2370,3490,7772,3610,7865,6646,5806,4497,5154,3078,4843,9667,2208,5725,8498,2545,5311,4655,3717,2530,2431,3515,3008\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 452\n", + "Returned IDs: \n", + "Ground Truth: 614,3222,6662,8711,9711,239,8247,2427,4091,6854,2165,6153,2405,5940,9345,1673,309,8369,9292,7352,5773,5346,7946,6331,8537,2660,6307,8583,590,3516\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 453\n", + "Returned IDs: \n", + "Ground Truth: 9737,1389,4521,3501,1134,4200,2201,5146,8177,4414,3749,8363,4039,7442,2568,7508,7143,1379,1861,8155,725,8522,9095,6179,4985,6747,2087,5963,539,8879\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 454\n", + "Returned IDs: 652,2595\n", + "Ground Truth: 646,2557,1289,3794,8143,5848,9799,6422,7949,5470,4080,5678,9524,7529,3717,8692,6998,2643,8097,6419,4200,7279,4460,7022,1344,6615,2479,7842,210,1214\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 455\n", + "Returned IDs: \n", + "Ground Truth: 6313,6095,9093,6197,2915,8340,9524,1133,6756,6615,8198,5896,4225,1988,5480,1540,4831,2943,6594,8312,962,4343,9459,1501,646,9628,2116,7594,8138,6608\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 456\n", + "Returned IDs: \n", + "Ground Truth: 6087,3350,6584,267,5749,6647,9552,9490,9294,9795,1509,706,5181,4978,4615,4497,7032,5624,8799,6871,5464,7865,6165,8443,3707,3759,9477,3150,766,5869\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 457\n", + "Returned IDs: \n", + "Ground Truth: 8247,2998,1584,4699,6338,2196,2325,7865,5695,2896,1066,3827,5404,9101,9451,1790,9553,2596,5110,1362,3937,2997,3185,9407,7790,3987,819,797,8333,1993\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 458\n", + "Returned IDs: \n", + "Ground Truth: 3216,7848,9946,8895,2552,1840,3424,4343,8490,4188,1940,1630,8066,1159,1893,3546,8080,9459,5489,2655,6765,3751,3722,9408,7807,8283,5423,9361,8837,4574\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 459\n", + "Returned IDs: \n", + "Ground Truth: 3591,8407,2045,2824,3140,7167,1247,9927,6398,5973,4317,5873,807,353,4683,9269,9921,6620,436,4256,4595,7362,499,2769,8033,3310,582,4389,1562,4244\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 460\n", + "Returned IDs: \n", + "Ground Truth: 6459,9378,5564,9024,5773,1362,9392,6337,9956,3824,6285,2488,5352,3826,1298,696,7642,7544,7947,6854,8830,5341,491,8411,2243,9720,6868,2998,6307,7794\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 461\n", + "Returned IDs: \n", + "Ground Truth: 7466,9217,5972,4855,8481,3622,6646,2943,7560,1940,1134,7213,6650,5946,5363,5278,6615,1540,2404,3167,1225,2608,2706,2070,7432,9684,1477,428,9371,1379\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 462\n", + "Returned IDs: \n", + "Ground Truth: 278,3759,5434,1762,5525,635,3764,4293,1344,6853,3444,1822,1854,2404,4289,4181,2831,7064,3319,2505,3310,7629,6580,1614,9245,2418,5589,4512,9371,9483\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 463\n", + "Returned IDs: \n", + "Ground Truth: 9359,4439,1866,9287,3371,1611,4903,6803,9994,4948,1963,1325,3348,3661,7554,6298,6765,683,5079,7519,98,3477,3376,3445,3977,233,893,153,7569,1105\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 464\n", + "Returned IDs: \n", + "Ground Truth: 3633,729,1885,5034,2575,1105,4514,9215,6219,7900,8089,5925,2712,2850,4535,3043,628,7511,2404,4197,5969,8098,6134,3977,4953,5074,9770,5949,4887,1622\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 465\n", + "Returned IDs: \n", + "Ground Truth: 8002,1064,9576,846,5466,9276,1524,3871,2641,8653,4474,4098,6272,2078,654,1881,2567,5105,2404,9179,1219,6615,949,2427,7847,2876,4790,4450,2437,7452\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 466\n", + "Returned IDs: \n", + "Ground Truth: 2776,9305,8954,8340,4094,3140,7123,8198,5940,4132,8893,1944,3564,761,9257,533,5840,2045,3582,9062,7310,9866,3591,8264,6817,9939,4524,7497,4256,3340\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 467\n", + "Returned IDs: 8819\n", + "Ground Truth: 665,8819,1748,4440,9682,2188,4562,8012,9796,7019,7185,912,7487,9005,8051,828,6115,50,6666,7569,1501,9792,2131,9419,9075,8914,1663,2454,7887,5034\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 468\n", + "Returned IDs: \n", + "Ground Truth: 2147,3310,1988,8312,5157,7125,8612,4308,2862,9269,841,8569,7243,3530,1214,8023,9738,9145,5447,7089,309,3130,6647,2853,5654,7436,7929,8038,3117,9843\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 469\n", + "Returned IDs: \n", + "Ground Truth: 6584,9552,4400,1809,3757,9824,5278,4256,9371,536,4308,2642,228,2356,6871,6647,2592,3629,5282,7125,7754,9345,2975,1593,8653,3624,6594,9145,5662,3977\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 470\n", + "Returned IDs: 7356\n", + "Ground Truth: 6693,4725,7356,9371,7560,3871,4843,2404,8141,4688,7687,6814,2427,8653,2078,8042,5278,2706,5363,828,2505,9576,4887,1809,1885,3320,2500,4075,8002,3207\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 471\n", + "Returned IDs: \n", + "Ground Truth: 6344,5330,1344,5851,671,5859,3484,7326,7248,9969,5468,162,6563,6329,2542,5374,3486,4991,2831,7605,2078,3004,5059,6404,8096,9371,6326,215,9591,4364\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 472\n", + "Returned IDs: 2653,6183\n", + "Ground Truth: 4571,2653,3610,6608,2530,8501,1988,6103,7990,5311,6990,4655,8269,2744,2556,2804,203,8194,2837,6814,7217,6714,7369,9097,5324,5289,2179,4830,2564,3043\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 473\n", + "Returned IDs: \n", + "Ground Truth: 1517,6852,1612,6816,1150,4688,4840,261,9288,8464,6645,15,3787,5943,9536,7887,4377,6664,5670,2054,1970,3900,355,9181,8261,5067,5252,5615,8681,5501\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 474\n", + "Returned IDs: \n", + "Ground Truth: 1946,8683,9308,8398,9818,3610,5403,9371,3034,4655,4263,2078,9576,7956,5181,2404,7836,6814,9770,5241,4647,8233,583,2567,4506,8688,3929,6615,706,7990\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 475\n", + "Returned IDs: \n", + "Ground Truth: 3999,8889,7217,5447,290,4634,649,536,597,1861,8625,6854,8313,7699,860,3008,6103,8042,4357,6307,6608,3859,6047,2174,9907,9451,6073,6369,7570,3474\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 476\n", + "Returned IDs: \n", + "Ground Truth: 4053,6457,8090,7209,1727,8949,8634,8331,9059,8443,1805,2273,7193,9584,7455,7980,9642,7413,1876,223,516,8753,3888,6399,7946,8223,5940,5181,4165,6607\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 477\n", + "Returned IDs: \n", + "Ground Truth: 2882,3913,5940,2778,1710,2130,4535,4658,8269,1881,6488,2866,5737,6508,5458,2696,6374,3949,6467,7140,8340,3477,4868,8753,4245,186,4815,120,8442,3377\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 478\n", + "Returned IDs: \n", + "Ground Truth: 3632,7669,2404,8627,8653,3356,7313,1144,3207,4887,6384,6097,3871,5062,5330,6563,2078,3262,1983,3629,4858,5403,74,1650,7830,6814,2759,2505,7501,4688\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 479\n", + "Returned IDs: \n", + "Ground Truth: 8223,6353,600,5940,8090,7546,9059,8796,6037,1862,834,309,6508,7946,744,90,6513,664,658,3377,1944,5054,8753,7389,4497,2147,1907,9863,1687,5718\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 480\n", + "Returned IDs: \n", + "Ground Truth: 8155,8411,258,7966,1362,2174,3474,517,8867,1861,6854,5110,7816,1518,3673,5447,6307,7348,8625,9378,7282,8889,4417,4932,8830,9576,7544,1584,7642,2012\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 481\n", + "Returned IDs: \n", + "Ground Truth: 8087,6834,3117,4094,3628,2147,8576,1433,8859,4676,2862,4831,3357,6768,4007,5340,8312,7424,7431,2167,8672,4582,1923,7272,4265,8545,8262,5920,4417,3407\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 482\n", + "Returned IDs: 8850\n", + "Ground Truth: 9225,3564,416,3707,1944,6342,8988,3329,1156,2450,8553,7542,3235,793,7682,2778,9671,810,68,8609,4483,3461,3578,3998,9256,5278,3340,4389,9371,8057\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 483\n", + "Returned IDs: \n", + "Ground Truth: 7358,5472,9451,2902,7486,8933,8352,2642,8974,706,2161,8664,6073,7141,7031,5464,3008,4715,2003,3987,1003,7068,4634,7570,3757,3347,1863,8649,4390,2465\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 484\n", + "Returned IDs: \n", + "Ground Truth: 7402,7823,2271,7798,2771,9365,4460,7146,7089,1457,1168,3551,9074,1619,5480,8856,9838,9213,8892,4846,6747,8895,4293,3359,2850,1273,3391,2785,1813,7364\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 485\n", + "Returned IDs: \n", + "Ground Truth: 5425,2187,4587,8617,792,2763,1663,6538,6316,6042,8080,2356,5034,8995,7162,9561,6794,8342,2840,9722,8379,5577,3477,6481,2188,3460,1771,2403,2519,5200\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 486\n", + "Returned IDs: \n", + "Ground Truth: 7570,2862,7966,2554,4256,4255,4852,1467,8569,841,8269,654,5278,8893,8879,8319,4172,2743,2642,7125,7364,309,2779,7511,4796,4655,1809,6558,4308,1619\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 487\n", + "Returned IDs: \n", + "Ground Truth: 9158,4293,9064,7629,5965,7624,7929,4314,7501,1376,7213,6474,3629,5480,3501,909,463,698,7743,5835,4460,5248,1818,3597,1457,7146,992,6329,6247,5716\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 488\n", + "Returned IDs: \n", + "Ground Truth: 4136,4197,4514,4995,1885,4373,1125,387,9751,2736,5190,1526,9972,3281,3122,7594,4319,6219,3633,1677,7615,6142,628,1105,8973,1611,9947,3043,7539,7745\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 489\n", + "Returned IDs: \n", + "Ground Truth: 5240,5543,3094,1151,4316,9582,5787,864,1745,2869,664,4288,9059,3165,4421,1805,438,3199,3598,137,8438,3292,8949,773,7980,3377,5640,2472,8443,4176\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 490\n", + "Returned IDs: 652,2595\n", + "Ground Truth: 2642,1568,2169,5157,7795,2862,7068,6248,841,7540,8499,4308,9145,3276,5311,1616,8939,7570,6598,9273,4094,7031,9411,7125,9238,8569,7750,8312,9453,7744\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 491\n", + "Returned IDs: \n", + "Ground Truth: 2197,6733,8615,2840,6918,9796,3582,5675,851,3118,3999,920,8425,4252,2581,5202,4649,8736,9324,7452,6227,1935,7770,9732,7059,6756,7625,624,7801,2696\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 492\n", + "Returned IDs: \n", + "Ground Truth: 4859,5074,2088,4439,1611,3477,7569,2850,3111,9149,9492,5034,7742,6765,130,1325,8671,8138,9419,879,628,1885,5447,4992,1105,7631,4097,7452,8301,2876\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 493\n", + "Returned IDs: \n", + "Ground Truth: 7016,4414,2554,8302,7772,3880,3078,5403,3034,8340,5278,1680,4745,2201,7373,2370,1477,428,8498,4044,9296,8042,4497,5806,309,6551,4400,3695,700,4109\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 494\n", + "Returned IDs: \n", + "Ground Truth: 3004,5543,2930,7529,250,2916,7193,4021,7505,6703,8800,8339,9207,6119,699,9059,5410,137,9295,6615,4039,5240,7389,9690,5098,9777,3517,4176,2289,6711\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 495\n", + "Returned IDs: \n", + "Ground Truth: 314,8635,5536,7742,3908,6938,3445,7817,442,7311,9302,9287,8853,8528,1903,1963,4368,893,6722,2655,1768,657,9359,2507,1935,5816,5423,7900,4168,5021\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 496\n", + "Returned IDs: \n", + "Ground Truth: 2045,5949,6103,2952,4256,4497,624,9824,8879,6647,9921,399,3695,2642,6794,9576,1003,2431,8342,8893,4781,2837,8313,2370,7801,2530,5789,9654,4887,3973\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 497\n", + "Returned IDs: \n", + "Ground Truth: 5737,2560,3207,4521,7891,4889,7487,7551,7092,4440,448,828,228,3477,2310,4688,8844,1885,3913,2472,2869,5092,7700,5625,5343,8442,4731,5062,1881,1861\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 498\n", + "Returned IDs: \n", + "Ground Truth: 3624,7248,2388,3570,1569,9554,2530,3033,2865,2418,8914,559,7862,8216,2201,9332,4140,9354,4557,1477,2431,3473,7690,8567,6872,1379,6756,2748,8498,9371\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 499\n", + "Returned IDs: \n", + "Ground Truth: 2887,5654,9932,136,7390,9269,3591,981,8468,4683,4061,1214,2183,6529,2499,656,996,2937,1397,2053,8700,1961,2944,8095,9124,4930,5774,3723,4548,7466\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 500\n", + "Returned IDs: 3963\n", + "Ground Truth: 9738,3987,1885,1602,3963,1948,547,3871,9717,4726,5447,2743,2831,7217,6647,4535,9219,4887,1881,2521,1988,2850,7966,9117,9134,2404,7745,7110,1964,6340\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 501\n", + "Returned IDs: \n", + "Ground Truth: 6727,6670,870,7022,5324,1079,8232,1593,5363,1278,8275,7364,9903,4493,2937,3262,7501,2406,154,8653,679,4500,3432,2404,7956,2078,1213,9682,4688,1144\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 502\n", + "Returned IDs: \n", + "Ground Truth: 8856,5541,1813,5649,4846,4687,6474,4293,549,3551,2271,3910,1457,7146,8123,3597,5658,6096,3056,652,3985,7823,8626,3228,1609,3369,6385,9354,1216,698\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 503\n", + "Returned IDs: \n", + "Ground Truth: 6964,9781,8969,7380,3119,4497,5885,290,8625,7816,1518,746,6662,547,539,7217,4622,9219,7578,3717,8349,7542,7125,3386,9593,6324,9117,3395,8484,8569\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 504\n", + "Returned IDs: 4372\n", + "Ground Truth: 5403,2044,7208,2831,7016,7966,5278,7865,8096,8026,706,2646,9371,2567,9459,1597,2201,5297,2370,4744,9898,3759,137,548,4400,5282,9724,1817,7560,5954\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 505\n", + "Returned IDs: \n", + "Ground Truth: 3788,6103,6714,6670,3081,5761,9521,2708,2146,707,4493,3212,6509,2226,6159,2744,666,495,7121,9314,2507,5106,2179,5561,5981,4242,8230,4683,5324,3138\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 506\n", + "Returned IDs: \n", + "Ground Truth: 6617,9135,4938,209,2017,8893,8068,8540,7685,7674,3017,3491,8445,3290,4283,9810,9236,9248,36,8372,3876,2940,9238,8793,3276,782,9199,9035,7750,7568\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 507\n", + "Returned IDs: \n", + "Ground Truth: 6139,5868,6765,2937,3477,1881,1213,9883,4461,828,9307,2575,8643,7381,9724,3099,6580,4569,5363,1857,9117,6557,8588,1210,4061,6643,1673,240,579,6776\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 508\n", + "Returned IDs: \n", + "Ground Truth: 4891,6153,4844,6220,2293,7432,3386,6854,3317,290,5885,7790,6647,1687,608,2012,9055,1753,4887,7248,7466,9838,2927,3417,748,4497,2134,3800,6100,1930\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 509\n", + "Returned IDs: \n", + "Ground Truth: 9861,1792,7196,9987,9408,7511,2716,4909,6523,5289,4178,9176,7168,3963,2521,4953,2247,8340,6764,3670,8322,2575,628,3860,2300,4542,4117,8786,415,3445\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 510\n", + "Returned IDs: \n", + "Ground Truth: 2174,8830,7542,6337,1990,8247,7642,4398,9355,1518,3826,425,155,9378,1415,9204,8625,146,3417,1861,4297,3800,8533,4634,8411,9367,3863,9593,7017,7110\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 511\n", + "Returned IDs: \n", + "Ground Truth: 3563,4247,290,1956,4974,3707,9781,8024,6662,1403,4414,3436,7982,8155,1134,6106,2245,2927,5581,155,7282,1960,5404,2174,4708,3320,3417,425,4497,1620\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 512\n", + "Returned IDs: \n", + "Ground Truth: 679,769,4400,9611,7629,2262,5589,2432,6594,9371,839,8205,3190,8462,8233,3530,8835,1015,4293,3624,7951,9288,4008,2237,1593,8366,7471,156,1244,5278\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 513\n", + "Returned IDs: \n", + "Ground Truth: 548,5662,1388,2618,8026,2078,3207,8381,1710,1650,2831,2524,8879,8653,1519,3871,8340,2236,1619,649,1602,4887,9903,2696,3757,2759,3987,679,2252,2356\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 514\n", + "Returned IDs: \n", + "Ground Truth: 1176,6301,4932,1189,1455,4147,7835,1861,7186,9291,1298,915,6459,9237,3314,8125,700,106,8583,1930,8867,751,7248,9024,5913,2950,6261,4664,1284,7629\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 515\n", + "Returned IDs: \n", + "Ground Truth: 6783,3195,1133,9448,2785,7629,3710,8498,6815,9095,6326,3651,6670,9898,3772,4293,7163,1344,8346,8129,5480,2383,9301,7091,7364,5176,7432,5566,5589,9354\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 516\n", + "Returned IDs: \n", + "Ground Truth: 3454,2036,5250,3020,1378,5313,2929,4991,8205,1344,5287,2740,8762,5330,5003,7432,3752,2726,9974,9939,5434,4253,2095,557,8305,6283,5484,1614,7449,627\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 517\n", + "Returned IDs: \n", + "Ground Truth: 5553,2027,3501,1642,1344,7070,3689,5884,8466,7089,5552,2506,5649,3104,7502,4406,1824,459,1273,7929,4759,1206,698,9354,3310,754,1435,9777,549,1376\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 518\n", + "Returned IDs: \n", + "Ground Truth: 5079,683,6803,9408,4903,3348,4948,2231,6765,252,1611,6023,9359,8514,5818,6643,1371,7252,7631,3424,5956,3963,2810,8970,9492,8086,7951,414,7788,4343\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 519\n", + "Returned IDs: 6487,7303\n", + "Ground Truth: 3587,2432,1015,6487,2251,4400,7425,8250,7303,3530,7016,1593,507,8462,1383,1963,1827,156,679,7501,1744,9371,2646,7951,4504,228,9234,7471,8144,5574\n", + "Recall: 0.0667\n", + "\n", + "Query ID: 520\n", + "Returned IDs: 2282\n", + "Ground Truth: 6418,290,8787,4497,9615,1098,3759,309,7966,5404,2545,8569,5205,5303,3871,7816,5885,7324,292,5278,3918,5385,1996,7282,1960,7064,4924,3474,1762,2196\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 521\n", + "Returned IDs: \n", + "Ground Truth: 1848,706,9668,1103,1360,6129,4749,5869,3699,3094,9770,9853,2567,1509,6080,2865,3610,5314,5386,7411,7865,2908,7560,3987,8443,7236,1310,6814,3038,5311\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 522\n", + "Returned IDs: \n", + "Ground Truth: 2541,1168,5480,1619,8343,5540,5716,9095,3561,463,204,7432,1273,4460,8177,920,7267,654,2,6714,1945,1457,9492,8014,9365,8953,7089,4061,8659,6615\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 523\n", + "Returned IDs: \n", + "Ground Truth: 8867,1347,3861,5867,5352,190,8411,6686,3863,667,9720,1864,3063,5164,5213,9717,7093,8050,7726,1162,9763,8318,9578,2565,6261,8196,7790,7658,8203,7171\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 524\n", + "Returned IDs: \n", + "Ground Truth: 9733,3867,5251,5805,9294,2450,5181,7822,5729,8988,4174,7702,6584,3436,309,7373,4624,1075,5898,2111,2554,5489,673,5562,3560,7586,3517,7977,4166,2443\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 525\n", + "Returned IDs: 2263,589\n", + "Ground Truth: 5162,3479,4602,7560,5403,7031,5982,6521,7141,9371,5278,9579,2061,4150,6814,1337,4991,2907,5314,3610,3987,145,5925,2404,8340,2865,2127,8208,5464,1985\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 526\n", + "Returned IDs: \n", + "Ground Truth: 1886,5988,7652,9069,2278,9314,8060,5007,5981,3473,2366,2179,6329,6430,5338,8057,9308,4259,5615,9283,6103,2319,9749,1426,1150,4416,2981,3717,8498,7057\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 527\n", + "Returned IDs: \n", + "Ground Truth: 1312,4821,7141,1985,8533,1743,2712,2168,1710,6827,9579,3610,4166,5994,2974,8981,6047,8946,779,8688,41,8135,7595,7486,3671,6212,2654,3007,2427,3320\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 528\n", + "Returned IDs: 3602\n", + "Ground Truth: 9245,278,3454,8892,5287,2462,4783,8844,1818,4940,7375,5330,3484,9483,2449,3310,6955,1378,1462,7432,3486,6387,2961,5313,3551,1614,8130,1168,652,192\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 529\n", + "Returned IDs: \n", + "Ground Truth: 9684,2655,9213,1311,769,1172,657,8355,8366,277,2652,7951,3757,4400,4140,483,5021,5954,2524,7012,4343,5948,3938,50,8026,3033,7033,6082,5679,8858\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 530\n", + "Returned IDs: \n", + "Ground Truth: 9796,2197,5650,920,7625,8615,2840,6918,9673,792,6733,2706,9612,2385,8244,4688,8177,1935,2581,4078,9732,9073,3653,1150,6372,4649,5288,7801,9900,5675\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 531\n", + "Returned IDs: 9764,2384,1719\n", + "Ground Truth: 1875,3610,9269,2788,706,3184,1944,3564,5386,6080,5282,309,9145,3153,5093,4185,5278,2348,9764,6677,7793,5311,8878,6100,8981,9489,9093,6584,5523,1387\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 532\n", + "Returned IDs: \n", + "Ground Truth: 1861,706,2753,2567,2831,6647,1650,536,7217,6399,8879,9576,1885,5403,649,2472,2078,3292,2642,6721,9795,3749,1677,7966,223,7700,290,516,352,6219\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 533\n", + "Returned IDs: \n", + "Ground Truth: 8081,1150,9682,1003,1866,7956,7900,4831,9181,3641,3463,1764,2023,5776,6372,6825,1663,8199,36,4562,1005,9745,2199,6030,2356,6214,7625,9994,4269,6329\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 534\n", + "Returned IDs: 8797\n", + "Ground Truth: 733,7790,9582,7217,1920,7248,7956,6557,1861,6421,292,9703,6686,66,4506,3884,9237,9100,6350,6647,6106,6635,2753,8498,7029,4222,6307,3610,2012,9024\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 535\n", + "Returned IDs: \n", + "Ground Truth: 594,5913,7790,9291,9055,1455,9024,8938,4195,1298,5452,8583,3287,6301,5539,9693,1664,5430,6312,8196,9423,9620,1176,2679,9237,9895,3177,751,6421,8867\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 536\n", + "Returned IDs: \n", + "Ground Truth: 9243,372,3873,6540,6252,3894,4903,8283,8858,414,5046,962,1905,3781,50,5500,2039,7335,5496,312,8559,7078,1358,9695,525,1892,3757,7548,5267,5582\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 537\n", + "Returned IDs: \n", + "Ground Truth: 204,2774,6579,3520,2585,9646,5480,2656,9838,2223,9492,6286,6747,2032,2929,2725,4991,6650,4110,2976,7399,3333,5146,5357,185,688,7216,8557,1583,963\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 538\n", + "Returned IDs: \n", + "Ground Truth: 9281,2246,3477,1476,3582,9796,879,6316,7900,3890,2197,4897,8970,4644,9203,8736,7594,7259,3281,4563,9742,8379,6722,7241,4330,9324,7020,6821,4572,7569\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 539\n", + "Returned IDs: \n", + "Ground Truth: 8695,5340,958,9550,2552,2604,4763,5264,5217,1630,5636,4343,9844,2619,3216,1583,4340,3277,7353,3223,5146,2007,9063,5963,9433,7410,6535,8040,6998,2608\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 540\n", + "Returned IDs: \n", + "Ground Truth: 9198,6764,6445,4260,2,7662,8643,7377,1323,4485,612,2298,5354,5865,8133,4796,4061,7789,772,8642,7909,2219,7410,9992,339,4460,8112,5290,1424,3681\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 541\n", + "Returned IDs: \n", + "Ground Truth: 4658,1948,1416,1885,2043,1190,2030,8653,8823,9883,9091,6158,3205,3581,9090,6488,536,8271,4191,3218,6054,5134,7217,4939,4100,4538,4098,186,7427,4921\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 542\n", + "Returned IDs: \n", + "Ground Truth: 9042,5238,8958,1334,8707,2785,7364,7603,2425,3111,2291,4992,9499,6249,1827,9095,7078,8482,4582,807,3545,5711,1024,5485,8342,529,9838,132,9898,8535\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 543\n", + "Returned IDs: 7358,4634\n", + "Ground Truth: 7358,8974,6307,4634,9395,2642,6248,8625,5472,5666,1702,9451,819,6369,1861,3949,9345,3757,9676,2646,2831,6854,339,7938,6557,3759,1650,6505,2896,8653\n", + "Recall: 0.0667\n", + "\n", + "Query ID: 544\n", + "Returned IDs: \n", + "Ground Truth: 8310,2472,9283,2782,481,2263,6127,2385,9314,4599,9271,9174,2266,71,3473,6233,9897,670,4497,2001,2903,5171,828,615,9562,3550,6481,5981,2370,4540\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 545\n", + "Returned IDs: 287\n", + "Ground Truth: 506,3939,9212,807,5058,2744,2028,5612,1957,5873,5238,2269,1051,8057,4278,1253,4517,312,8648,1618,8807,9206,3163,1444,1551,2179,7594,4218,8194,6398\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 546\n", + "Returned IDs: \n", + "Ground Truth: 1126,338,2271,4687,1457,5649,1592,4466,1642,4293,7798,8856,420,6898,3391,3985,3551,8618,6474,3689,463,1273,377,7089,1965,3501,2746,9354,1167,4846\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 547\n", + "Returned IDs: \n", + "Ground Truth: 2952,3564,2778,1944,4580,9095,8177,2975,2438,5716,9824,8498,6794,5946,9269,9371,3629,4400,7091,2943,132,3426,1156,3477,1809,8342,6817,7542,3501,608\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 548\n", + "Returned IDs: 2230\n", + "Ground Truth: 3826,4100,4463,2869,3426,3377,8458,4470,6901,2739,1518,5254,6951,3094,6275,6289,5289,7891,2596,744,3395,8093,6750,6508,309,9117,1881,3936,4461,7389\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 549\n", + "Returned IDs: \n", + "Ground Truth: 5485,7078,3366,3040,2374,4986,2706,4012,3289,7894,4654,1827,5949,7085,2785,8535,3160,3260,7319,4892,6693,8815,4546,8026,3432,3757,893,1963,6267,3501\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 550\n", + "Returned IDs: \n", + "Ground Truth: 6307,3749,6557,6647,1861,6854,9770,8405,1738,9345,6289,8247,7790,536,4634,8924,9451,7217,9237,6073,4009,2567,836,1920,5818,1988,652,5445,9395,2243\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 551\n", + "Returned IDs: \n", + "Ground Truth: 7444,425,2443,9219,6211,9667,3863,9097,9148,3426,659,3247,6338,3610,4869,5289,9781,2739,6813,4821,517,6508,7282,1687,3474,727,2743,7911,7258,4398\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 552\n", + "Returned IDs: \n", + "Ground Truth: 4599,4030,6855,7699,1122,3477,4688,2507,8379,5343,9287,6722,8726,2266,7202,2,1963,5007,828,6057,8681,3629,9283,5863,4887,71,9722,154,2001,8177\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 553\n", + "Returned IDs: \n", + "Ground Truth: 9087,2524,5139,3246,615,5694,9213,2655,5679,4865,6573,8681,1970,548,228,5954,60,6157,8026,277,3606,1166,2710,5021,3450,4688,7501,2652,5214,2759\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 554\n", + "Returned IDs: \n", + "Ground Truth: 4938,9248,9236,9810,3017,209,2017,4820,4683,9199,8801,782,6617,8540,8893,3103,9135,6843,1412,8859,1401,9741,1397,1187,1414,7125,7147,136,3876,8789\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 555\n", + "Returned IDs: 5982\n", + "Ground Truth: 3286,2712,8030,5994,5262,6863,2127,8946,7677,7570,4821,310,8281,1946,7031,7141,9308,5666,2511,1446,6145,7486,195,107,9962,1985,3150,6521,583,7\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 556\n", + "Returned IDs: \n", + "Ground Truth: 624,528,2907,5806,7285,9824,3194,7801,3695,428,1950,820,9576,9296,1032,9796,7016,7560,4964,9405,5294,8560,7690,9371,9288,4497,2706,4794,3880,5675\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 557\n", + "Returned IDs: \n", + "Ground Truth: 8649,1518,3800,290,3322,7943,3317,91,2174,1403,4414,7966,3319,5789,7069,4491,8024,1595,7031,798,2197,4887,2592,9738,2196,875,1593,89,9750,9371\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 558\n", + "Returned IDs: \n", + "Ground Truth: 4734,2374,4044,1388,548,5580,2370,5662,1700,6492,2618,9288,2078,2785,1501,6756,2696,9287,4854,7887,2524,1003,5343,7573,2507,15,5806,7217,8879,1983\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 559\n", + "Returned IDs: 2230\n", + "Ground Truth: 1105,1278,7631,6219,9914,7511,7594,1885,9287,2850,3477,628,3633,3963,1611,5910,6795,988,8953,9093,8014,4061,7635,1619,4953,6142,9492,1463,4439,676\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 560\n", + "Returned IDs: \n", + "Ground Truth: 3859,3434,6232,7711,7687,8117,5574,1430,5538,1145,2023,1003,2370,2427,5219,109,9296,8398,6558,3854,9827,6103,7425,702,2837,8569,7039,9371,7129,7016\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 561\n", + "Returned IDs: 2745,3112,9550,7742,5423\n", + "Ground Truth: 6919,3196,1800,7432,4586,2668,263,6378,3793,5830,2897,5533,1443,7874,5544,210,983,5192,4289,3942,5819,9684,3005,3454,1160,8546,3062,2991,3602,2745\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 562\n", + "Returned IDs: \n", + "Ground Truth: 2535,4830,7432,5093,9538,1456,9354,845,5945,6235,3917,4460,793,7629,3750,6815,7036,49,2179,9064,3561,5320,4442,9527,1731,3501,7862,2895,3033,8346\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 563\n", + "Returned IDs: \n", + "Ground Truth: 5283,7570,2712,5162,6782,189,6054,9751,5994,5464,7031,6827,2127,3732,7795,3590,6219,5373,729,3347,3127,4150,8933,5666,6466,2161,3987,66,3150,4655\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 564\n", + "Returned IDs: \n", + "Ground Truth: 412,218,194,6383,1823,9625,6101,9872,2250,2322,2295,8999,9021,4444,2961,6097,8430,543,8892,5330,954,3551,8844,1964,775,6563,3556,483,1172,4688\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 565\n", + "Returned IDs: \n", + "Ground Truth: 4683,2350,2605,4337,3297,229,1830,1397,821,4472,401,8801,3184,5597,883,6876,7464,9183,1185,6842,4739,1674,960,8571,5372,8063,8721,5898,4231,4729\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 566\n", + "Returned IDs: \n", + "Ground Truth: 8584,8730,4256,6100,4573,5988,4467,6253,4817,7652,9283,9033,666,6197,9181,8057,4400,2975,5589,2859,1540,793,3840,9945,8537,8676,7129,5776,6594,2472\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 567\n", + "Returned IDs: \n", + "Ground Truth: 1678,5925,2917,9369,4519,5483,4821,4875,9870,5447,9840,4318,1861,4300,5327,8177,4658,8307,2831,9906,3157,8012,2472,3977,5006,3477,5074,4824,1885,5737\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 568\n", + "Returned IDs: \n", + "Ground Truth: 9458,257,2585,2774,7612,688,276,9838,9669,3332,1751,4064,1583,4991,4199,2409,963,7937,2725,3223,204,2032,7213,6579,3333,5146,3265,4110,5480,3493\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 569\n", + "Returned IDs: \n", + "Ground Truth: 5146,4887,2618,6871,5034,3477,1817,2696,1388,2850,2759,7594,761,4039,828,1134,6794,7985,6538,8238,8653,1935,1493,9529,1728,5480,5289,5892,8177,1168\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 570\n", + "Returned IDs: \n", + "Ground Truth: 920,1652,2404,1622,4887,483,7465,9242,4234,6561,1027,9371,5958,2454,828,8269,3234,9057,1105,5330,3033,9787,1256,6387,1817,4625,6756,5701,2706,7935\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 571\n", + "Returned IDs: \n", + "Ground Truth: 7216,3520,2774,9838,963,2976,1583,9646,2125,2223,8494,7643,9492,6747,2585,7403,2547,4684,8557,1619,2725,7611,8148,9844,7612,3493,652,7883,5480,1818\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 572\n", + "Returned IDs: 8914\n", + "Ground Truth: 2404,154,4887,9914,6647,828,8653,2850,9738,5363,2427,7217,2575,8627,1650,1388,2078,2831,5447,1817,920,6136,5377,8002,1533,7966,548,9133,7956,8141\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 573\n", + "Returned IDs: \n", + "Ground Truth: 7824,2858,2541,8123,8494,3551,1902,7629,3432,6474,7089,377,5649,7468,3484,4425,1216,4687,9354,420,8856,9555,4293,7091,4991,6292,4386,549,2561,2036\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 574\n", + "Returned IDs: \n", + "Ground Truth: 2161,5472,404,2631,1223,6552,3008,1321,8933,7031,7141,1519,3987,7560,1406,1881,6073,189,3150,3757,5925,729,7570,7511,9160,6219,8664,7217,2902,9395\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 575\n", + "Returned IDs: \n", + "Ground Truth: 8342,3110,1663,6817,5650,5892,5238,9308,2840,4256,7789,8407,8714,3040,2356,4157,8056,1817,2859,7801,2530,6794,6297,6847,3450,9354,7125,3310,981,2952\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 576\n", + "Returned IDs: \n", + "Ground Truth: 6594,5675,4061,4157,5202,5200,807,6551,4649,2952,828,5288,3033,2374,5671,9213,4854,1827,8953,7432,8498,8026,7868,8914,5716,7267,4400,2785,5949,5078\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 577\n", + "Returned IDs: \n", + "Ground Truth: 4278,6398,8033,477,4317,807,6601,9373,1562,6211,9622,224,1156,5205,2769,3978,6817,1555,2028,8529,2856,506,5873,7297,3707,3015,4337,7638,6342,3578\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 578\n", + "Returned IDs: \n", + "Ground Truth: 1188,5122,8407,371,9725,5480,4992,1344,2856,1988,202,7629,4587,499,3501,4610,4896,7929,807,5152,2045,2404,7091,1247,1398,9927,4293,2698,7125,4524\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 579\n", + "Returned IDs: \n", + "Ground Truth: 4991,9575,2418,2404,5442,9371,3234,9134,1602,2423,3764,3253,4887,2388,3624,4110,1894,1817,192,626,5217,4293,6814,8205,7607,1480,8653,5480,3320,8892\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 580\n", + "Returned IDs: \n", + "Ground Truth: 4402,5852,7383,1885,3387,8616,5079,9162,8341,8514,7554,4319,6354,1987,6038,1105,7381,9578,4012,4514,4921,4562,1155,2716,5671,7594,2507,5074,6573,5239\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 581\n", + "Returned IDs: \n", + "Ground Truth: 5671,9827,4497,9958,2642,5806,624,3973,5675,2370,9405,3880,1003,5288,9254,4414,7801,4044,7016,3695,828,7560,9673,4256,3078,7442,9371,1680,4400,7123\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 582\n", + "Returned IDs: \n", + "Ground Truth: 5029,3246,3450,5200,1817,6451,233,1815,3074,3156,3848,2277,7817,3348,5866,6741,9844,5848,8342,277,9213,9825,6478,1970,4562,9596,8635,3424,3908,879\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 583\n", + "Returned IDs: 2521\n", + "Ground Truth: 7522,2779,7477,2457,2521,5016,5808,2716,4887,6642,1001,4435,2743,5917,3928,6056,8483,3175,5701,2934,5377,8471,6747,8093,5105,6639,1619,3282,2943,9049\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 584\n", + "Returned IDs: 3112,2745,9550,7742,5423\n", + "Ground Truth: 8421,4178,5230,1610,920,5377,9290,1427,4651,3206,5586,3112,4343,9187,1871,8014,7729,9215,9455,7511,8026,9724,9987,308,4953,3938,9093,2575,3477,1893\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 585\n", + "Returned IDs: \n", + "Ground Truth: 6854,8879,2927,7069,6106,8583,3317,547,2012,7753,3734,2174,6630,2778,5303,5675,2952,9875,6794,4492,8972,1817,6153,7790,5679,2753,9711,7629,3800,9237\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 586\n", + "Returned IDs: \n", + "Ground Truth: 5607,8549,901,3221,8152,2205,2927,8889,3707,5564,5464,5383,7816,8155,5110,9374,3474,6160,3738,2468,624,7543,2747,2848,1220,2000,4634,1403,602,9862\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 587\n", + "Returned IDs: 3994,5096,3417,984,9697\n", + "Ground Truth: 9915,3058,5913,9620,1520,3994,5033,3317,258,1298,5539,3346,1741,915,6301,9854,751,5180,1189,9553,6458,7046,1066,9055,3086,8761,1284,184,979,1301\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 588\n", + "Returned IDs: 7337,7838\n", + "Ground Truth: 8653,2790,2078,6097,3356,548,7669,6736,3262,154,1894,4444,5330,2404,3432,1964,9625,1093,3808,1016,8627,4190,1650,4658,4889,6885,8346,1388,3207,5062\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 589\n", + "Returned IDs: \n", + "Ground Truth: 3205,2301,8102,3515,949,920,1359,930,6275,4887,6580,4304,4921,7985,7594,2016,5816,8717,1068,9287,2696,157,9529,5858,6158,9117,6764,7100,3610,8393\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 590\n", + "Returned IDs: 4357\n", + "Ground Truth: 6417,2869,5925,6275,3551,7374,1483,7353,7628,9117,4318,1351,6508,8307,3477,5737,8823,7529,5940,3282,2986,7389,4206,2249,9207,1964,679,9625,3929,1861\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 591\n", + "Returned IDs: \n", + "Ground Truth: 2264,2506,5830,4061,2390,1160,210,420,1270,1096,8820,7064,7131,8406,4455,5480,2409,6546,2580,3723,1206,4386,5992,2112,5649,2451,7629,3501,3715,549\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 592\n", + "Returned IDs: \n", + "Ground Truth: 9695,7639,3954,1473,4462,2921,3969,2701,1725,7951,7924,1633,3757,7940,1348,4140,7807,8233,3894,2434,3873,4346,8044,8366,5046,5285,3624,3820,7405,2696\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 593\n", + "Returned IDs: \n", + "Ground Truth: 2556,8859,2269,6789,1403,2862,5340,6100,6765,7031,1719,7466,5900,6350,7594,5069,1253,8528,2115,5181,5117,5311,7027,6647,807,9269,5972,7125,5093,4252\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 594\n", + "Returned IDs: \n", + "Ground Truth: 2239,6047,1016,5074,879,4274,8425,7202,5776,8144,391,761,1593,3118,7319,6701,6227,4141,1079,2509,4018,9865,3256,5599,6420,3977,5925,4296,9345,7625\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 595\n", + "Returned IDs: \n", + "Ground Truth: 5328,5940,6508,7595,2464,2525,6037,6689,8350,1678,1151,9313,8090,7139,9516,2349,4884,1981,3377,2930,3849,758,8782,69,7753,6513,196,4949,8411,7233\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 596\n", + "Returned IDs: \n", + "Ground Truth: 700,3519,2245,5885,798,3386,7217,7380,2927,8717,6981,2117,4417,3320,9148,8969,2876,9055,2554,425,5949,7285,3987,3426,8269,7790,4497,7027,2642,9274\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 597\n", + "Returned IDs: 7044\n", + "Ground Truth: 9365,7515,3530,7509,2124,7756,4308,9561,3501,9263,6165,4400,1837,6540,2850,9750,4460,6487,2493,4517,6815,1988,9354,9206,1562,7089,4729,1944,6313,807\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 598\n", + "Returned IDs: \n", + "Ground Truth: 6449,7844,8706,2080,1935,9405,314,1963,8528,5021,3908,9181,5321,9971,6794,7951,3973,1903,7381,5425,277,6031,2952,7900,5288,9288,828,7452,1677,8853\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 599\n", + "Returned IDs: \n", + "Ground Truth: 7477,7522,4688,2790,3004,9448,1213,548,95,9213,4865,8726,9662,5917,6573,2759,9861,7501,7487,4279,5979,1602,6643,3376,211,8442,6639,9310,2239,6727\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 600\n", + "Returned IDs: \n", + "Ground Truth: 7557,7146,6096,909,4314,3710,9064,7629,7929,5658,9996,1902,9158,7089,371,9777,3191,1206,4293,319,4960,5965,8466,9977,8563,8417,8517,420,1457,2541\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 601\n", + "Returned IDs: 652,2595\n", + "Ground Truth: 9893,9660,8892,7874,4110,6233,2266,3153,2494,1389,6378,7662,467,9684,483,7143,6395,7466,1027,4215,1540,4343,2032,7432,1840,1323,8133,5426,6197,6610\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 602\n", + "Returned IDs: \n", + "Ground Truth: 559,5945,1569,3570,5683,7432,9770,6671,8648,5386,4400,7629,5278,7143,1058,6187,5093,8879,2275,7248,4830,3033,3717,1647,1253,4460,3526,7862,7016,536\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 603\n", + "Returned IDs: \n", + "Ground Truth: 9529,5762,8892,2521,6642,3747,5466,5016,3484,9049,6563,5146,1661,9715,2677,6747,6523,2743,8093,3175,5330,6580,1983,9276,5540,1619,9448,2618,5105,2787\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 604\n", + "Returned IDs: \n", + "Ground Truth: 8245,7504,5393,7592,1451,9570,8411,4227,8353,1302,6615,9180,6213,6750,693,123,1382,6459,6275,1878,3571,3185,8458,7582,3477,5361,8340,9372,7414,1497\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 605\n", + "Returned IDs: \n", + "Ground Truth: 6445,6764,9743,9198,4489,5290,1424,7985,7497,8720,1840,3999,5865,4562,7267,7377,2863,2,3145,8900,4690,4886,339,6038,5653,8653,8953,7909,566,32\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 606\n", + "Returned IDs: \n", + "Ground Truth: 9468,804,309,999,658,352,7389,386,9016,5181,1745,6689,3078,6508,6615,5940,9059,664,4683,852,1849,8061,6213,3347,8177,2866,5056,8893,864,8223\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 607\n", + "Returned IDs: \n", + "Ground Truth: 3874,8938,6951,1176,1922,2343,8867,9578,6519,1298,8937,751,3058,2012,6459,6476,1861,2217,6285,5452,712,1066,3863,5342,7246,5361,6331,8411,2950,4147\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 608\n", + "Returned IDs: \n", + "Ground Truth: 1771,2507,6410,8080,5425,4778,1663,2200,6722,5863,4562,9870,6471,7699,1122,4734,8379,9346,9005,1184,3216,9436,1052,1389,2099,4037,154,1628,7767,2472\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 609\n", + "Returned IDs: \n", + "Ground Truth: 8454,3633,6712,5016,2575,4535,3610,8093,8014,6110,7187,2716,4514,1105,3280,3477,6142,9287,3424,5034,5540,6073,5731,4887,3472,9492,6219,6765,7985,9093\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 610\n", + "Returned IDs: \n", + "Ground Truth: 1823,5737,1544,4824,3445,5437,2310,9336,5092,9708,3614,1973,7389,8726,8483,8700,7487,9550,7529,3477,9221,98,4815,454,218,6119,7660,5927,9679,7424\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 611\n", + "Returned IDs: \n", + "Ground Truth: 3570,8422,5806,4497,8688,7248,7442,1003,2654,5789,5219,428,4390,2201,8389,2418,5725,2370,1379,2358,66,2865,2859,6103,5509,7865,8057,7560,8498,8415\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 612\n", + "Returned IDs: \n", + "Ground Truth: 5504,4061,5861,8879,1619,4953,5187,9914,2831,5447,5666,772,3445,2798,7555,4796,5694,3145,1817,5679,8504,3973,6747,7570,3450,1278,1827,2966,2521,6529\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 613\n", + "Returned IDs: \n", + "Ground Truth: 1419,8598,4383,4004,181,6392,8107,8620,7137,8580,5872,7681,1892,3512,3317,6646,312,9734,5900,6116,1757,2919,5803,8915,1988,5272,2079,6004,3816,7793\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 614\n", + "Returned IDs: 7970\n", + "Ground Truth: 9011,9959,5874,750,600,6603,7038,9294,8949,7546,664,8943,1862,4981,6131,6463,5514,8796,9853,8223,6513,2571,2891,6037,1522,4458,7946,2111,4103,3292\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 615\n", + "Returned IDs: \n", + "Ground Truth: 9013,4007,977,1619,2887,4953,4215,7985,8156,4815,4795,8892,6747,6764,6642,9861,483,272,7410,4548,5808,8319,8301,9269,4859,5146,5963,741,9622,4061\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 616\n", + "Returned IDs: \n", + "Ground Truth: 618,3731,7051,3062,3261,8086,5290,6208,2009,2435,9667,2131,6419,539,8771,5572,646,8831,36,4252,4256,2696,7387,6764,2231,7432,1289,8343,1935,9958\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 617\n", + "Returned IDs: \n", + "Ground Truth: 2631,6047,711,5925,7031,4061,4296,590,1985,7770,8604,4821,1159,4206,5359,761,9288,8933,9879,4831,7822,9411,7570,5256,7452,6829,9751,3999,8366,879\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 618\n", + "Returned IDs: \n", + "Ground Truth: 7645,4353,6032,3832,6810,5209,4992,6147,1188,8191,9796,2527,8407,3665,5919,9203,2187,2438,4284,4649,132,6316,6257,7554,5071,8342,9149,1714,569,1226\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 619\n", + "Returned IDs: \n", + "Ground Truth: 4266,7238,3844,9063,3564,7682,495,7509,4345,7399,6579,4061,6010,8504,3056,6324,4113,8524,3707,7949,4522,6998,1944,8643,1562,3935,3681,4460,2600,4317\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 620\n", + "Returned IDs: \n", + "Ground Truth: 5141,8444,6825,3463,40,3492,8312,7360,2603,414,3366,3722,8914,343,4562,233,5084,7078,8117,1827,6741,8393,564,5296,1593,2331,9145,1424,2510,2728\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 621\n", + "Returned IDs: \n", + "Ground Truth: 1390,1431,9593,9902,9781,9237,9024,9180,5955,1584,2100,6752,2488,3581,2504,6375,9148,8385,3128,9693,9378,7896,3826,5750,2995,3548,5242,6153,1838,5026\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 622\n", + "Returned IDs: \n", + "Ground Truth: 5806,1935,9958,3880,5294,9405,2370,834,8600,4497,4044,1279,3078,7801,4954,1003,536,8340,1253,5671,8914,7772,624,1885,5891,608,2971,2530,4621,9174\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 623\n", + "Returned IDs: \n", + "Ground Truth: 9798,1075,2147,9453,8038,7243,7344,1071,3638,2275,5117,8296,3130,793,3428,8023,309,8569,2642,6967,2174,6871,2862,7140,5961,36,4256,4936,9296,7750\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 624\n", + "Returned IDs: \n", + "Ground Truth: 8112,9743,3999,9198,4689,1424,531,1815,2,5340,1840,2266,5079,4485,666,2655,1323,8895,3650,9426,8490,654,3995,6445,3546,7807,8056,1140,1943,9213\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 625\n", + "Returned IDs: \n", + "Ground Truth: 3859,6103,2837,8398,5538,7129,6232,4622,6124,5385,5574,2804,3871,2370,5988,8866,6558,2497,9371,9576,4400,2023,4649,6329,5432,2646,4924,3432,3757,7956\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 626\n", + "Returned IDs: \n", + "Ground Truth: 2431,6187,3033,3234,3973,6594,4400,1817,6146,5278,536,9770,8136,7016,3717,1970,9354,1253,228,1700,1569,3526,8498,2370,7381,6671,4581,2859,615,828\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 627\n", + "Returned IDs: \n", + "Ground Truth: 8528,2952,3908,9922,1728,4816,483,7381,5776,6594,5021,1935,3973,1817,879,839,4824,5202,7594,1663,4845,8056,7742,7817,6334,1700,277,6268,6825,8105\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 628\n", + "Returned IDs: \n", + "Ground Truth: 1257,6759,311,6408,853,7020,4562,7476,3136,8617,5866,5177,3686,1885,4587,3348,5034,2136,2241,8802,3424,1129,6741,7569,8348,628,6810,7594,130,1188\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 629\n", + "Returned IDs: \n", + "Ground Truth: 6545,3734,5867,1687,3548,6176,1692,6720,744,866,2334,8411,3063,2404,4227,4297,727,3201,6813,366,654,2165,895,4423,7542,7504,9374,7947,4431,8753\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 630\n", + "Returned IDs: \n", + "Ground Truth: 8320,1889,8131,1944,9860,5927,31,7003,5759,8883,8771,3090,7194,6598,1999,8340,3871,3818,2324,6361,7872,2017,8451,8569,841,2403,9721,547,5295,7345\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 631\n", + "Returned IDs: \n", + "Ground Truth: 9433,4343,8773,6013,4340,9844,9492,9062,1583,1159,8643,2,8837,5830,8343,5480,6357,1893,7511,4311,9970,5264,2409,6764,1168,3216,3223,3424,5470,9198\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 632\n", + "Returned IDs: \n", + "Ground Truth: 5574,9097,4571,5403,292,6103,5278,7956,1809,228,5432,4830,3610,8398,2419,7966,8042,7369,7357,6992,2592,7444,5280,2023,2318,1145,3008,1383,1677,3859\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 633\n", + "Returned IDs: \n", + "Ground Truth: 7830,9761,570,228,9033,2419,3687,9455,7381,2472,5278,4688,8653,2831,1747,1544,5363,2243,6097,5480,3551,8823,5254,3629,9582,8134,4824,3477,5483,7887\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 634\n", + "Returned IDs: \n", + "Ground Truth: 1497,8411,6662,6337,2174,984,3563,3800,9781,1861,3395,3826,3185,1014,425,7790,2986,1518,7322,6263,3717,4206,155,7125,9055,3347,4003,3610,7282,1162\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 635\n", + "Returned IDs: \n", + "Ground Truth: 5006,5480,2785,4293,3484,1156,7267,7929,5330,920,7111,1827,3444,8879,2961,9483,7364,5716,2201,8844,3629,7078,5550,7629,7432,652,6561,192,7213,8205\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 636\n", + "Returned IDs: 5940,6508\n", + "Ground Truth: 8796,7171,7760,8753,5514,3548,3668,8660,806,2525,5023,9055,3201,3377,4564,1704,7753,2460,9039,1907,5940,6459,4405,8182,7455,8196,2851,5361,1838,1380\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 637\n", + "Returned IDs: \n", + "Ground Truth: 3539,3720,3962,3514,36,2899,2167,8312,8087,8167,8891,5986,9823,539,4831,3318,9756,4269,7715,2531,1175,9686,5489,7336,2017,7194,7503,1062,4638,8329\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 638\n", + "Returned IDs: 9145\n", + "Ground Truth: 8445,4479,2017,7125,9145,3970,782,5246,5311,8036,6363,5077,8569,6557,5318,8859,5809,2169,3017,9358,3818,1988,6590,1228,6598,3828,2642,4889,4836,7147\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 639\n", + "Returned IDs: \n", + "Ground Truth: 4093,2660,6459,8196,9024,1861,5995,5821,1738,5564,836,9839,8405,491,6337,9055,3495,2414,5607,8411,9169,2848,9032,8583,9693,8761,3395,3086,6868,4698\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 640\n", + "Returned IDs: 2756\n", + "Ground Truth: 4521,1281,4622,6437,5438,7243,4256,2078,7431,4638,8296,3233,5157,6406,1643,6834,1880,5330,9269,5218,2642,1030,8042,3214,2399,1466,5363,3370,4831,8653\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 641\n", + "Returned IDs: \n", + "Ground Truth: 7034,5711,4934,2266,4061,9922,9561,778,7635,1035,3477,2850,3648,339,8528,4992,7461,391,1278,5915,4562,5074,7817,3999,4012,5340,879,4548,2785,8953\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 642\n", + "Returned IDs: \n", + "Ground Truth: 8247,4436,3222,146,8711,239,9292,4634,4091,5303,4620,7217,590,6153,9055,536,8360,3876,2970,9405,1152,6331,9037,6854,4410,1993,3185,9345,1861,2517\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 643\n", + "Returned IDs: \n", + "Ground Truth: 7078,5949,7573,525,5200,3019,8487,7603,2374,5465,3873,2728,6516,5879,9499,312,7740,9922,200,1424,6693,458,8117,4717,2427,3117,8134,9989,7497,4061\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 644\n", + "Returned IDs: 2927,4516,7289\n", + "Ground Truth: 5404,290,3738,6418,901,8549,90,9750,8155,1403,5303,7225,5051,247,7280,4680,4974,7077,2773,5955,1712,89,8830,8423,1136,1584,8545,429,9760,5717\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 645\n", + "Returned IDs: \n", + "Ground Truth: 3318,7715,3962,9686,9912,2899,5837,36,8891,1062,4604,5854,4831,3572,825,3720,3514,1174,4638,8087,2531,5344,3539,8167,1175,539,8495,7503,7600,2594\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 646\n", + "Returned IDs: \n", + "Ground Truth: 9349,5919,6032,4595,7645,816,289,9269,1795,562,5071,585,5929,3184,678,1623,4683,4256,6792,486,2430,2698,9149,5873,9540,499,5577,5368,5649,4317\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 647\n", + "Returned IDs: \n", + "Ground Truth: 5241,7545,7023,3610,2805,6863,5403,583,6272,131,4491,6646,7029,8398,6557,658,3008,6321,3834,247,9670,4185,9582,6688,931,7120,7560,2530,5787,7956\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 648\n", + "Returned IDs: \n", + "Ground Truth: 2258,798,4385,5016,8093,1988,4389,6794,4061,8953,2753,8569,3591,5716,7220,1619,4460,9054,3551,4139,3391,6615,9095,9473,9371,6647,7798,2769,2862,807\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 649\n", + "Returned IDs: \n", + "Ground Truth: 5649,3501,3391,4256,1344,8837,3578,2131,8343,5363,498,2943,1809,671,1902,3551,4400,549,8406,6532,2505,9844,1206,9595,4343,7798,7629,8419,8653,9371\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 650\n", + "Returned IDs: \n", + "Ground Truth: 8663,2566,406,6834,5575,7466,6672,5438,5084,9145,8609,2330,5715,1733,7751,5311,8891,8413,9269,4638,4531,742,9928,9301,7051,3513,8087,2531,9992,3564\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 651\n", + "Returned IDs: 6487,7303\n", + "Ground Truth: 6487,2251,2432,4400,7425,7303,5671,7123,3587,3530,1963,4256,2975,8462,7016,8340,828,1015,9234,6165,9745,9181,8342,1827,4831,4497,3973,9371,8900,4504\n", + "Recall: 0.0667\n", + "\n", + "Query ID: 652\n", + "Returned IDs: \n", + "Ground Truth: 7282,5867,6662,1298,26,9155,2834,425,290,3863,3436,3395,8558,8155,8830,3734,3221,1861,120,4634,9378,3222,3474,3738,9750,3819,3320,6176,155,5773\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 653\n", + "Returned IDs: \n", + "Ground Truth: 2642,841,2151,3130,6598,5218,4770,4144,2862,9273,3211,5900,5157,3828,8292,7744,8569,9798,4852,9552,3151,8038,5043,684,5311,7750,5555,3370,7068,2938\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 654\n", + "Returned IDs: \n", + "Ground Truth: 2111,1403,5404,4490,164,6106,8234,5613,3707,4389,659,7543,4029,5885,608,3461,2769,91,6220,2000,9263,7069,2450,9294,6153,8988,8600,3474,8353,8972\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 655\n", + "Returned IDs: \n", + "Ground Truth: 2370,4467,1432,2530,6871,2599,4256,624,5769,2428,2556,7131,4213,5803,8649,7091,5278,798,6874,5755,5011,3622,359,5885,7442,8498,7208,4390,2859,9864\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 656\n", + "Returned IDs: \n", + "Ground Truth: 5376,8331,2554,5523,2891,7772,8690,7585,120,8443,2753,2414,3165,386,8825,3034,9098,706,7910,8949,7549,2596,4630,5805,3292,5505,6557,5940,7586,2602\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 657\n", + "Returned IDs: \n", + "Ground Truth: 5216,3648,5731,9133,5848,4343,3424,1523,2131,414,6110,8093,6712,7569,4208,8442,7594,4562,683,1728,4265,6764,910,6643,9337,1943,4487,7905,3477,893\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 658\n", + "Returned IDs: 1639\n", + "Ground Truth: 1639,5925,1866,1235,8281,4141,3157,3757,9451,5034,7031,7677,7560,1079,4514,4300,4197,5483,3977,7767,4012,8398,9071,2837,5656,5074,4318,9994,8754,7486\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 659\n", + "Returned IDs: \n", + "Ground Truth: 6110,2784,6803,3424,683,5816,2872,4735,9287,6765,6354,8334,2527,8970,1325,6246,8270,3348,5829,7569,3281,2082,3666,7742,8595,1161,3472,6764,7554,9425\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 660\n", + "Returned IDs: \n", + "Ground Truth: 7789,3593,6713,159,3453,4260,3723,790,4796,4839,1112,5354,633,8112,1619,4218,4061,8319,9858,8895,9426,3310,5958,2609,6529,235,2,9365,4781,4548\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 661\n", + "Returned IDs: 287\n", + "Ground Truth: 4256,3501,3613,3310,6100,8539,9182,2622,6197,7743,8343,4400,3444,5377,8419,3140,5671,4467,9875,3118,2856,2943,9709,4293,8580,828,9093,3328,6419,7929\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 662\n", + "Returned IDs: \n", + "Ground Truth: 2664,2531,7634,6936,3276,825,9686,7309,8581,5344,3514,36,3131,7568,8891,1178,4311,7071,8312,5961,8332,5043,8087,4950,9958,6598,9206,350,9788,1837\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 663\n", + "Returned IDs: \n", + "Ground Truth: 7907,1535,7311,7972,1903,8342,3477,7594,9400,5202,6792,2952,130,6372,6810,4483,4992,9181,6794,5946,2430,8407,9287,5971,828,4859,5112,569,9349,5892\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 664\n", + "Returned IDs: \n", + "Ground Truth: 2466,6584,5729,3436,5532,1066,9024,3086,8411,5740,3861,9553,3185,5867,8247,6046,4761,6557,3222,8583,5224,6686,6750,1362,8405,4247,5646,3495,1200,8715\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 665\n", + "Returned IDs: \n", + "Ground Truth: 4443,2730,3749,2753,2401,1861,1134,7314,51,3008,6321,8025,8155,2086,8028,994,290,3474,6046,7203,5940,3107,2412,2243,7810,5049,3320,1162,9579,6338\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 666\n", + "Returned IDs: \n", + "Ground Truth: 3444,5550,652,4181,767,2306,4293,6387,278,5525,5330,8406,1576,3430,3649,3586,8177,8686,2423,5480,5589,5047,2505,4289,671,9371,4783,1344,3960,7304\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 667\n", + "Returned IDs: \n", + "Ground Truth: 2603,4132,4167,7868,1453,7620,8238,3757,5905,9145,9506,8953,5592,6825,5840,4460,4562,7771,7831,5238,5103,40,8312,6138,9989,3828,935,6452,4400,2331\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 668\n", + "Returned IDs: \n", + "Ground Truth: 3832,9061,7645,8407,8191,400,6107,8342,1008,3307,6257,9501,678,912,5971,3700,1327,1663,5892,4992,1357,9919,1188,3665,1575,9203,5796,5152,8587,8244\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 669\n", + "Returned IDs: \n", + "Ground Truth: 9552,393,9269,8579,4582,2489,5900,2744,5889,2146,4975,1447,1837,1757,8567,982,7243,5117,9306,742,9345,125,8871,8640,5311,3342,8498,3130,7568,2642\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 670\n", + "Returned IDs: \n", + "Ground Truth: 9506,3757,9211,2642,3977,4582,7868,9395,1453,368,525,9145,6540,6452,7938,8238,7620,4717,7603,8099,3117,4702,3873,7548,4515,312,7793,8974,1168,8933\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 671\n", + "Returned IDs: \n", + "Ground Truth: 6608,6095,9895,8014,3116,9093,4197,5230,8754,7016,2850,5034,6959,2567,3722,8590,7594,7125,3610,539,6615,2450,9561,9622,1827,5065,1562,9734,3717,6500\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 672\n", + "Returned IDs: \n", + "Ground Truth: 635,8305,1462,7629,5330,4293,3486,8205,1614,7267,3444,5550,652,3020,4364,3629,1096,3710,4400,5965,2505,2418,162,1822,767,1378,4783,5525,5477,5480\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 673\n", + "Returned IDs: \n", + "Ground Truth: 3393,3944,8477,3302,9459,4281,5939,2943,6225,2833,5489,9063,4771,4460,2952,2093,546,4998,2655,1344,4343,3868,9033,256,450,3601,5202,2178,8116,2112\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 674\n", + "Returned IDs: \n", + "Ground Truth: 4899,2720,9648,5806,1417,2412,7970,3695,2370,6348,2599,1141,3778,4459,9016,8789,7956,3347,3072,1932,2642,1003,714,8392,312,7660,2472,4013,8795,5022\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 675\n", + "Returned IDs: \n", + "Ground Truth: 6594,4400,839,9405,4497,3876,1619,4293,4308,828,3687,1963,4824,5278,8134,2083,6165,8247,7865,3629,6794,1663,9900,6159,9938,5525,9021,7089,228,9684\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 676\n", + "Returned IDs: \n", + "Ground Truth: 4206,352,309,6689,7217,9372,5403,3347,6854,386,1968,2174,5181,9059,7193,999,6615,254,7946,1849,4634,2712,516,6721,5940,2427,9562,2986,7031,3749\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 677\n", + "Returned IDs: \n", + "Ground Truth: 19,9024,4636,696,9693,2679,6459,5341,3241,6183,3086,4664,2369,5539,7709,9055,6730,1687,8583,6879,6312,7947,3824,9423,1503,4567,6341,9148,6584,2162\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 678\n", + "Returned IDs: 290\n", + "Ground Truth: 8625,1415,6904,7542,4932,258,4634,8362,3564,9576,536,8120,9935,6925,2468,6369,8876,8889,7282,2696,6192,7217,7988,949,6021,841,727,247,9289,5439\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 679\n", + "Returned IDs: 3259,9500\n", + "Ground Truth: 3591,9927,3259,353,2045,2840,6258,9500,8407,2769,5520,3941,7067,6817,6620,1637,5685,5560,2297,2699,2779,3310,7362,7535,562,9921,924,8617,1851,7491\n", + "Recall: 0.0667\n", + "\n", + "Query ID: 680\n", + "Returned IDs: \n", + "Ground Truth: 4431,4490,4385,4658,6794,3426,9054,6813,2261,654,9263,6580,1302,7151,7069,9423,7578,5885,2743,1988,4061,3551,3548,4411,8953,3476,823,5565,7757,6468\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 681\n", + "Returned IDs: \n", + "Ground Truth: 8557,3520,4334,7637,1800,3310,2656,4586,6747,7432,2108,2223,210,5480,2643,3457,7929,7591,7466,2897,204,6427,2929,5354,1462,284,9838,3148,6114,57\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 682\n", + "Returned IDs: \n", + "Ground Truth: 9893,7793,3081,3277,312,40,2465,4717,6233,4061,4761,8858,7267,9093,3033,1866,1156,4400,7466,525,5480,3649,4293,7570,3873,2975,4008,2603,2943,414\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 683\n", + "Returned IDs: \n", + "Ground Truth: 8561,7621,208,2277,9093,112,6915,9638,8379,4816,7059,1235,1963,5153,879,4168,8635,9246,3111,4778,6722,7899,434,5816,5949,2781,2696,5572,6701,3934\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 684\n", + "Returned IDs: \n", + "Ground Truth: 1881,2798,4061,1168,7033,7217,2831,6580,6901,170,9223,2266,6776,1619,3759,5289,6854,5480,3610,8953,5445,1068,4830,309,2631,5679,7868,6233,4548,3987\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 685\n", + "Returned IDs: \n", + "Ground Truth: 6476,751,5480,9711,6630,1068,7432,9777,7193,3627,3752,3484,7862,3526,150,6854,2201,4932,9237,7364,1302,7929,9875,1894,8844,612,3651,9492,5064,3987\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 686\n", + "Returned IDs: \n", + "Ground Truth: 2119,1518,7790,3800,1687,8247,3937,6662,6106,6557,3338,1403,7911,190,7380,6153,3185,8583,2217,9578,6046,5644,7093,2778,3474,4655,608,8411,3386,6338\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 687\n", + "Returned IDs: \n", + "Ground Truth: 3286,2712,1446,7031,2896,1720,5994,2490,7677,2283,1743,8946,182,5262,9141,404,8981,496,6731,9781,9770,5464,4707,5181,8533,2263,6558,7093,7324,1518\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 688\n", + "Returned IDs: \n", + "Ground Truth: 7068,5385,2837,2573,2141,2642,6608,2437,9050,1650,9914,5657,8653,3616,1885,9843,792,203,6103,1861,1003,1677,2078,292,2023,3934,6647,2974,982,9738\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 689\n", + "Returned IDs: \n", + "Ground Truth: 5929,399,4719,3184,6059,4472,3927,9784,8063,2698,9250,7306,7491,9754,9269,4683,8721,2147,1004,1831,2426,1830,1674,7297,1581,7148,6999,1414,5898,4317\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 690\n", + "Returned IDs: \n", + "Ground Truth: 4998,7793,6386,7591,40,2465,5230,9093,312,414,2603,6159,5784,7143,5972,525,2131,8346,8105,4460,5200,7472,3385,7230,2943,6564,7432,181,9254,7279\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 691\n", + "Returned IDs: 7724\n", + "Ground Truth: 1476,4319,6219,5034,3633,8218,4535,6134,5074,7452,451,2631,7890,4831,4197,5925,7404,1985,9947,3477,3977,3610,8366,9578,4514,9093,7734,5262,9287,4061\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 692\n", + "Returned IDs: \n", + "Ground Truth: 3220,4831,8891,8087,36,7125,8859,3514,6770,6500,9664,5837,539,2899,8312,5575,8264,7271,4400,5715,3962,3601,7908,9912,5675,7259,7968,3318,5860,8445\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 693\n", + "Returned IDs: 5738\n", + "Ground Truth: 7822,7549,5562,9959,3517,4193,9294,953,6513,7772,8331,250,8634,5940,7946,4006,6,2951,1467,9016,3743,3292,6463,1151,3189,7389,841,3774,6689,2554\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 694\n", + "Returned IDs: \n", + "Ground Truth: 3722,1015,2023,8444,1005,2497,828,9459,6722,4986,8144,9635,5776,6214,1764,391,8141,3530,9682,5574,8477,9181,3300,2432,9033,4622,9745,5645,5670,170\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 695\n", + "Returned IDs: \n", + "Ground Truth: 6746,3520,7353,1583,3270,958,5217,5480,7559,7399,7919,6650,7403,9063,1619,7612,2223,1885,9852,6535,5340,1168,9047,3223,6747,8892,6286,9134,4110,204\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 696\n", + "Returned IDs: \n", + "Ground Truth: 2390,3430,7375,5533,4783,6853,4181,4999,7213,6911,5781,7257,5047,5064,2098,7432,3919,1671,5716,5287,8092,3649,1809,6387,4497,8130,5553,2027,2306,8833\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 697\n", + "Returned IDs: \n", + "Ground Truth: 8708,7515,5205,6165,4019,9721,370,5885,6211,5773,8353,9391,8184,2998,1944,9750,2261,155,6418,2196,3409,6106,2997,7943,5613,3738,2459,2592,6021,7017\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 698\n", + "Returned IDs: \n", + "Ground Truth: 2066,2795,9287,1105,1943,1442,4995,176,6004,2329,2716,5018,7270,7511,3633,4953,9419,2082,3963,7635,3687,8595,2613,7027,6354,3670,6765,4439,9729,1885\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 699\n", + "Returned IDs: 7317\n", + "Ground Truth: 7767,2199,7761,8498,5176,4750,7317,8900,1963,7754,3304,2428,4157,8342,1586,6191,7956,8198,4493,4400,3717,5575,8379,9227,828,8233,5776,8622,2419,1827\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 700\n", + "Returned IDs: \n", + "Ground Truth: 1670,1105,6765,6825,2850,7594,8970,414,3963,1866,4514,1885,9287,8334,7754,683,6764,4778,7569,5925,4562,1027,1060,9989,3480,6268,2853,1235,8590,9914\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 701\n", + "Returned IDs: \n", + "Ground Truth: 2785,5208,8653,8042,9576,6647,3432,2370,5324,4745,7364,9363,2428,5988,2975,1144,9371,1593,3262,8057,1569,9354,359,6103,8346,8025,7442,7956,3717,7129\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 702\n", + "Returned IDs: \n", + "Ground Truth: 6854,8844,3629,3687,6097,3432,1861,3551,8653,2831,6307,3484,9345,9237,5363,4444,7204,6563,2404,4887,1483,5330,4932,2961,8896,6647,536,1818,9219,7966\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 703\n", + "Returned IDs: 9840,8307,2310\n", + "Ground Truth: 3476,3282,8123,9840,3551,8667,8856,6636,5737,1902,6887,5125,3877,8307,6829,6300,4953,7374,6509,5934,2310,1965,5925,4687,1544,1871,1204,9293,4824,3687\n", + "Recall: 0.1000\n", + "\n", + "Query ID: 704\n", + "Returned IDs: \n", + "Ground Truth: 984,4634,6025,290,425,6337,6662,3800,3312,8830,6658,9750,7217,7790,8625,8709,7031,3436,1861,3338,4206,6369,370,496,373,9781,8558,3395,9077,7093\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 705\n", + "Returned IDs: 7356\n", + "Ground Truth: 9161,7740,8900,3722,7078,4256,6693,8284,5574,8815,7854,6376,9525,1444,2867,7687,9371,6707,4012,1764,8588,3492,6318,8080,5200,5113,4416,2957,2863,3463\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 706\n", + "Returned IDs: \n", + "Ground Truth: 8446,4010,888,518,7350,6360,7127,1503,9357,4699,9024,5039,5695,4627,4801,6536,2998,91,9863,5637,8256,6850,5033,9961,542,9615,3138,3058,4062,608\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 707\n", + "Returned IDs: \n", + "Ground Truth: 9550,3112,7466,5848,2516,5470,7432,6419,3648,4343,5545,9446,7742,5540,5245,2131,6764,9684,1840,3751,1817,6119,4557,8773,5230,5830,2608,2231,5146,6451\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 708\n", + "Returned IDs: \n", + "Ground Truth: 6813,3426,2069,3380,3941,5560,7946,6620,3610,3477,2824,1673,3110,2045,9562,6307,6776,1068,353,4206,9120,1948,4658,9582,5146,7069,7982,7033,6990,3591\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 709\n", + "Returned IDs: \n", + "Ground Truth: 210,1540,6578,7143,4460,5192,2585,67,2411,4855,8852,6378,6197,3123,7466,3493,6447,5848,8363,8481,2223,5876,4343,767,4200,8692,7432,3333,3576,4586\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 710\n", + "Returned IDs: \n", + "Ground Truth: 3094,4190,3551,6580,2011,7193,654,2961,1204,4176,420,6213,5330,2404,7946,6508,192,9770,8340,1378,5403,278,8653,9293,744,5940,828,120,8892,7389\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 711\n", + "Returned IDs: \n", + "Ground Truth: 2428,4047,4400,1041,2199,6238,4293,5589,679,7956,7317,7767,2419,8498,1344,6103,8233,4527,3262,8530,7951,3958,6441,1540,6100,9283,7022,3190,1963,671\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 712\n", + "Returned IDs: 1422\n", + "Ground Truth: 3467,9750,5867,1497,7790,1632,2937,1956,3540,6153,1422,5136,155,6119,1162,7093,654,3086,7217,5361,8091,6662,4003,7446,8753,3929,8423,9988,1687,9553\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 713\n", + "Returned IDs: 2282\n", + "Ground Truth: 4400,290,3757,9345,4497,7217,6165,292,2642,6103,6871,8155,228,5187,1593,2078,6714,8042,652,2712,3973,3320,1988,8313,5278,5925,5447,536,642,2927\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 714\n", + "Returned IDs: \n", + "Ground Truth: 5560,3591,5873,1944,436,1247,3832,6620,2824,3941,6258,6153,2769,9269,3140,6957,2045,5112,9250,4389,9725,7491,7362,9473,1024,7535,5388,2699,6842,7067\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 715\n", + "Returned IDs: 1813\n", + "Ground Truth: 7226,8844,3551,1483,5363,5470,3477,6097,7374,2831,3985,5895,8856,420,4704,9884,3865,3112,8653,3282,5480,4386,8093,6854,8008,2575,1168,8406,7253,4824\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 716\n", + "Returned IDs: \n", + "Ground Truth: 9190,5384,982,5555,677,2481,936,3544,8499,2862,2529,1349,5600,4770,4308,8672,2084,5631,5157,9430,4852,2443,2938,8793,2151,5041,5930,5118,993,7527\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 717\n", + "Returned IDs: \n", + "Ground Truth: 9123,4723,4390,1279,3973,2606,9543,8239,2777,9914,2646,6487,4647,6558,2712,3530,5472,496,3234,5666,5385,7016,7217,2654,5969,3610,3671,1743,5282,1593\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 718\n", + "Returned IDs: \n", + "Ground Truth: 2753,2646,3246,5679,5694,6573,8136,1597,1384,1650,3757,3145,5187,4615,9306,3973,1817,9006,3022,615,3234,1700,5861,2831,1208,3033,7457,5931,4400,3011\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 719\n", + "Returned IDs: \n", + "Ground Truth: 517,9367,8896,3185,4041,9376,658,3395,3505,2778,5773,90,1687,9602,7389,1861,2514,309,6307,8411,8405,9345,7456,2174,4027,2153,659,6338,6106,5940\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 720\n", + "Returned IDs: \n", + "Ground Truth: 9244,7069,9223,6801,6139,6580,9117,3477,8076,7966,4548,737,2631,5031,1885,2021,1673,2294,4688,9630,7381,6630,2581,8717,7452,1881,8615,1948,7625,1068\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 721\n", + "Returned IDs: \n", + "Ground Truth: 3310,6993,1206,8390,7449,7494,1273,2256,3444,557,7743,5693,7432,5895,6454,7070,7213,9028,7193,1445,1818,9711,1090,7629,7929,3235,5934,7146,2036,1962\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 722\n", + "Returned IDs: \n", + "Ground Truth: 9344,1827,2785,7101,2074,9314,8482,8198,4831,4654,9095,5489,8313,8413,7319,653,5007,4448,3833,2374,5575,6253,4061,5485,6334,6329,3316,3220,2262,8663\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 723\n", + "Returned IDs: \n", + "Ground Truth: 1323,3419,9018,3999,2465,2798,4839,5853,6556,9093,4061,5679,1102,1311,3450,1817,5284,170,9876,2681,277,3546,3995,462,8355,9426,3781,7591,2717,214\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 724\n", + "Returned IDs: \n", + "Ground Truth: 1881,6580,4906,4887,1817,2798,5679,654,3648,2596,9690,1068,5958,3477,4061,1619,4460,4548,4721,5466,5410,4176,5021,8319,1932,1582,2247,9426,775,4007\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 725\n", + "Returned IDs: \n", + "Ground Truth: 3550,670,6127,8744,9722,629,2263,9174,6222,8334,9283,8080,6869,492,1611,5536,9897,7594,2507,683,7452,6481,2538,2082,2784,7206,3348,6722,3281,4210\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 726\n", + "Returned IDs: \n", + "Ground Truth: 6213,8359,7843,4085,1434,8796,1766,5940,7673,1727,6459,5718,8090,9059,6615,7946,6149,1704,1098,4979,5712,8949,8783,5242,6607,5181,826,6513,3181,3201\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 727\n", + "Returned IDs: \n", + "Ground Truth: 6458,7581,3008,9513,6557,2960,4019,8558,5303,3800,2357,2000,4026,1518,9821,4470,4963,8411,7314,4361,9117,6338,4678,3863,9367,7542,290,4009,2955,8766\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 728\n", + "Returned IDs: \n", + "Ground Truth: 642,3470,3854,3859,846,7956,5385,5422,5278,173,6124,2898,9644,2472,2370,1385,6822,3717,834,228,2785,5205,6558,4141,2805,8042,3511,2753,1613,5789\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 729\n", + "Returned IDs: 9796\n", + "Ground Truth: 9897,7699,6316,9796,132,9346,2366,1747,6722,3111,9133,2782,9283,5536,130,6127,9314,2472,4599,3832,2187,15,6855,2706,6372,1748,1663,2281,8191,6481\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 730\n", + "Returned IDs: \n", + "Ground Truth: 9062,2178,6723,761,7131,4722,420,4466,5992,3477,4400,7798,8837,3892,5649,3501,2007,1041,3840,8212,7143,6945,5945,1344,2975,6646,4846,7064,4181,7629\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 731\n", + "Returned IDs: \n", + "Ground Truth: 8924,6337,290,7217,4634,3317,7031,3749,6854,9345,7570,7642,5447,7790,1861,2974,1362,9093,1687,7816,7542,312,6307,3199,8104,7943,8625,4497,2174,9711\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 732\n", + "Returned IDs: \n", + "Ground Truth: 1016,8442,2472,1540,679,4731,9271,1747,1239,9283,5556,2714,5074,4688,4293,228,7629,1593,5776,8232,6736,4531,1748,3865,5589,9133,3501,3262,5649,9903\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 733\n", + "Returned IDs: \n", + "Ground Truth: 4745,6775,4157,309,5953,2568,5819,5946,6257,5386,5175,2952,2262,5589,3570,8914,162,2201,9078,2115,1379,8498,2785,2383,8879,3776,9764,101,2268,1253\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 734\n", + "Returned IDs: \n", + "Ground Truth: 6780,8294,1003,3510,4826,8344,8398,9179,2437,9097,9279,2876,6321,2374,6272,5692,3871,6406,5219,2006,9800,2370,3499,3718,428,1954,7687,1946,9644,8526\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 735\n", + "Returned IDs: \n", + "Ground Truth: 2147,6647,7125,5157,2567,4497,2642,7140,8893,8799,2862,8569,4609,652,7089,4308,1944,2174,8933,4852,1593,8653,352,9269,7865,1988,6854,7570,2753,4521\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 736\n", + "Returned IDs: 754\n", + "Ground Truth: 5884,2107,3689,4406,8466,2809,8884,459,3081,7070,9330,5552,1642,8450,2027,3501,3715,1206,1344,1596,5649,5553,8419,3104,1479,7391,2103,1435,8092,1376\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 737\n", + "Returned IDs: \n", + "Ground Truth: 9694,6557,2596,2896,6635,2788,8782,3671,2005,1584,5445,5181,706,4476,4647,136,8981,3610,6688,8893,1920,9098,404,7910,6212,4912,2414,3116,9378,5262\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 738\n", + "Returned IDs: \n", + "Ground Truth: 3348,5009,9359,457,6643,9767,8911,4574,9911,7976,4948,5216,4168,646,5731,3424,7905,5556,5858,692,1611,9724,2231,9005,4141,8442,6091,2368,683,4799\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 739\n", + "Returned IDs: \n", + "Ground Truth: 541,7380,6981,7966,2174,3615,5885,3553,1298,9692,5408,1938,7249,1313,4027,8969,7816,1518,2980,4214,5728,2729,6742,7348,3320,6220,6151,3454,1098,6324\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 740\n", + "Returned IDs: 652,2595\n", + "Ground Truth: 654,7497,483,6794,3004,6615,4485,8490,6125,8661,6119,4416,4061,5568,6921,1840,8340,4983,4470,9866,9838,6488,4113,9093,1619,2887,9242,6551,4844,9935\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 741\n", + "Returned IDs: \n", + "Ground Truth: 6124,2837,292,2497,1809,6103,2804,4622,3859,5574,8398,228,2023,7956,1996,203,7501,2078,2642,2437,2573,8141,6992,6487,9050,2419,9371,679,4256,3530\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 742\n", + "Returned IDs: \n", + "Ground Truth: 9069,5195,7426,1886,5615,9639,8060,9870,1663,8342,6372,6430,3907,2707,4157,8944,1061,9132,6255,5892,2859,2078,1018,7348,4779,2135,3900,7013,7554,9346\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 743\n", + "Returned IDs: \n", + "Ground Truth: 7091,1376,5965,7629,4896,3935,5186,202,162,707,4293,4400,8654,947,1206,9783,3501,7248,7400,1562,6853,9354,5276,5945,5617,6723,8368,6159,5512,5486\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 744\n", + "Returned IDs: \n", + "Ground Truth: 3673,9155,7753,2174,9055,1838,7093,8583,3644,9345,2012,815,5196,3581,7280,6662,779,8203,6338,3548,7879,4031,1918,8411,6634,8558,3320,9148,696,6584\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 745\n", + "Returned IDs: 2927\n", + "Ground Truth: 608,2927,5404,7543,3317,4844,7485,370,5130,3800,3755,6662,3474,9848,3987,7380,8649,1403,2012,5764,2153,5033,7538,7966,6511,7642,5885,2000,9824,659\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 746\n", + "Returned IDs: 4839\n", + "Ground Truth: 4690,1591,9198,4061,1619,3033,7034,4689,4839,2728,170,7868,8112,3593,3999,7909,5679,1424,6445,4460,9213,2,8238,1815,772,3145,9684,8953,4485,240\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 747\n", + "Returned IDs: \n", + "Ground Truth: 9388,6100,8233,6325,6126,5817,1379,2268,5093,2785,2383,4293,1573,4307,101,162,6028,679,6609,6815,2201,707,6137,7217,3262,5654,5589,6736,1512,9806\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 748\n", + "Returned IDs: \n", + "Ground Truth: 706,8096,5311,2596,4476,4039,643,9770,6814,5403,7324,7363,2629,7248,5278,7411,2946,3292,3008,4602,1875,9582,7604,2788,7016,2454,5789,3775,7721,3859\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 749\n", + "Returned IDs: \n", + "Ground Truth: 3233,6854,5146,8040,8430,8653,5940,5217,943,3871,2078,4497,4343,7217,1871,9738,3445,2575,1840,5447,9219,7669,2831,6165,9667,2696,3477,277,5636,335\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 750\n", + "Returned IDs: 7356\n", + "Ground Truth: 2957,1982,5662,2618,1388,6484,5381,8089,4614,154,3040,9371,6136,2706,6251,7356,6571,6547,9736,1963,9133,6420,8442,9075,7085,9455,7161,7621,978,2696\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 751\n", + "Returned IDs: 6850\n", + "Ground Truth: 3622,6525,5033,6850,8948,258,4189,3058,2998,2927,5885,9371,9645,542,91,5165,1520,5764,8649,8423,915,6418,6536,2196,7248,7966,6468,2012,370,1298\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 752\n", + "Returned IDs: \n", + "Ground Truth: 85,6547,5662,7610,4454,9576,9614,5074,2141,5436,2078,2957,548,2249,5815,8879,2356,2437,1963,2837,2618,5522,5657,9903,1501,7517,5363,2641,1548,1996\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 753\n", + "Returned IDs: \n", + "Ground Truth: 788,549,8844,3310,3764,8892,8653,9064,7267,4991,2505,2541,9780,1462,7089,4061,5330,2,4293,7432,2112,8879,377,1818,3629,162,4687,9715,240,6474\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 754\n", + "Returned IDs: \n", + "Ground Truth: 309,4061,2174,654,5445,7570,6382,9738,1988,2753,6776,4410,6854,2596,352,2712,8443,1920,6307,3530,3759,4308,5403,1861,9616,3871,2831,386,2567,1827\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 755\n", + "Returned IDs: 4497,3973\n", + "Ground Truth: 4506,9781,3186,4497,290,1861,2174,8625,6854,9703,2567,7280,2427,4634,6647,3320,6165,536,1451,8990,3185,3578,7217,7542,1972,3707,292,8406,3871,1551\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 756\n", + "Returned IDs: \n", + "Ground Truth: 8557,7637,8852,210,5876,7591,2445,5397,1273,2108,4127,5480,1540,4334,7974,4455,8466,7432,5253,7368,3457,8833,6876,7937,3501,7143,1800,7364,2745,6378\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 757\n", + "Returned IDs: 287\n", + "Ground Truth: 8616,7817,6140,3656,2538,670,5536,4805,3908,4368,8635,2850,8528,4588,5776,6722,391,879,2375,2507,3550,6127,6573,4934,9922,7635,3250,9283,2472,2482\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 758\n", + "Returned IDs: \n", + "Ground Truth: 4821,9369,7214,3987,8550,2494,9870,9117,5443,7217,2696,8977,9900,9546,5483,828,4734,4772,8379,5806,3501,5925,4655,8442,7745,7554,3977,1963,2262,6610\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 759\n", + "Returned IDs: \n", + "Ground Truth: 5202,2952,328,6970,1682,4649,4008,5946,9181,7123,2824,6372,9400,3973,2045,6594,7997,2965,9685,6031,4335,6817,3178,6739,3847,1208,2931,4244,2975,7651\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 760\n", + "Returned IDs: \n", + "Ground Truth: 8312,8167,2531,4582,2489,8663,8610,5084,8891,9171,3305,9205,8087,3220,6113,3117,7309,8859,3258,4094,1035,4831,4951,3995,3514,6954,6936,3999,8588,36\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 761\n", + "Returned IDs: 652,2595\n", + "Ground Truth: 7560,4400,3871,5278,210,6454,4744,1602,2404,9134,7213,7974,3310,5217,2771,547,9738,9838,5248,1096,6468,7559,1480,6794,7629,7217,5480,8056,7143,7807\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 762\n", + "Returned IDs: \n", + "Ground Truth: 8247,3286,4410,2712,5472,8981,9405,1985,6145,7560,7570,4400,2642,4134,4390,2283,8604,8443,66,404,5994,2092,7017,7031,6615,2554,4436,254,3671,2011\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 763\n", + "Returned IDs: \n", + "Ground Truth: 346,200,4242,3258,1447,8122,6006,6423,2503,6741,5765,3729,4956,23,9127,20,5259,38,2277,7476,1453,6794,5065,2943,6313,8212,56,9851,4256,9093\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 764\n", + "Returned IDs: \n", + "Ground Truth: 9283,2472,1866,9181,4824,666,8398,4075,481,2837,6103,5403,6329,5589,1827,8537,3725,2395,5988,4998,1886,5278,154,7501,5981,7459,4400,7129,9897,6739\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 765\n", + "Returned IDs: \n", + "Ground Truth: 4042,9879,3931,8754,2310,6643,4953,8275,7831,1601,9293,8393,2269,3468,2512,1866,6764,2421,7374,3809,3870,5470,6191,6091,2067,7788,787,3949,5489,8697\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 766\n", + "Returned IDs: \n", + "Ground Truth: 4887,2388,7465,2454,4234,3234,1379,4557,2404,9242,1622,2639,9371,6756,3206,1817,920,5434,2912,8914,2201,5806,5701,1105,5476,2418,8089,483,1027,688\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 767\n", + "Returned IDs: \n", + "Ground Truth: 9693,6879,4280,7463,9024,3241,9055,6375,1687,6183,2679,7947,3273,9423,7835,6312,7877,7558,4664,4636,2311,2162,1415,8583,3010,4923,6285,2165,5539,7709\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 768\n", + "Returned IDs: \n", + "Ground Truth: 6107,8407,582,1188,3832,5152,8342,4610,5122,4992,9061,8850,5873,5112,4859,499,202,3700,1247,1997,9725,9269,1525,2699,8244,1575,6620,2430,5071,9256\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 769\n", + "Returned IDs: 8284\n", + "Ground Truth: 1659,9369,4519,7728,8550,6884,9994,1866,8587,7214,183,2244,3653,7510,9796,7566,2762,6316,2335,2197,7479,3661,3851,9897,7180,8442,9732,7885,5923,8012\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 770\n", + "Returned IDs: \n", + "Ground Truth: 8932,6459,9376,1692,6647,1930,3284,3317,3734,3695,4423,1944,9055,536,2165,9711,5106,4683,3434,1933,1851,7294,8362,1362,1826,8763,7790,5940,9954,2589\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 771\n", + "Returned IDs: 7337,7838\n", + "Ground Truth: 707,9354,5363,1512,7629,8653,495,2713,6292,1965,359,5324,6670,3687,154,3262,3432,2258,3877,3865,4445,1126,4466,2708,7091,8641,1093,6828,5649,7956\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 772\n", + "Returned IDs: 6548\n", + "Ground Truth: 6074,3709,536,2572,7865,844,3707,7248,3033,4308,292,2712,8858,9371,3757,1809,482,4699,3532,4493,3017,6854,348,5278,2831,4400,7570,4256,3749,1569\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 773\n", + "Returned IDs: \n", + "Ground Truth: 8688,5207,5806,3234,4647,3880,1985,7471,1459,9667,4655,1541,5219,2111,1312,3610,5297,9123,1783,2418,3515,6374,4100,2865,6832,8025,4390,4400,2402,3286\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 774\n", + "Returned IDs: \n", + "Ground Truth: 7210,7003,1536,5809,1318,4261,5759,7872,6598,8883,3796,539,8893,7049,1680,8799,1944,5157,7988,9779,4497,36,3690,3970,825,9430,2403,9163,841,1837\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 775\n", + "Returned IDs: \n", + "Ground Truth: 9667,1885,3670,7532,1963,7780,9674,6093,5055,9522,3705,8349,1385,7788,3205,8348,3367,1677,679,9234,7786,653,6608,5239,8590,9117,3281,2043,6643,5816\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 776\n", + "Returned IDs: \n", + "Ground Truth: 2617,6337,3826,4634,2000,2110,7225,290,7280,7358,8558,3347,1571,3286,6248,5464,8190,5869,7017,4743,6814,6119,8625,1403,6946,4264,7031,8880,3863,8239\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 777\n", + "Returned IDs: \n", + "Ground Truth: 5806,4821,9673,2244,4157,9732,4649,9576,624,1602,1935,5675,9796,792,9870,5671,1988,9369,4924,9958,2370,1663,5447,7452,9612,5288,6608,7745,2078,3695\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 778\n", + "Returned IDs: \n", + "Ground Truth: 8741,312,7882,4242,18,3293,9007,8805,40,7040,4400,9684,1101,5753,769,6779,8219,6952,7161,6202,8233,5972,6238,7951,6229,9573,5589,4573,2967,3270\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 779\n", + "Returned IDs: 3347,9603\n", + "Ground Truth: 1458,5789,1205,7995,1345,624,5806,4887,9279,2064,3234,9415,6389,5239,7349,4172,2712,5886,2989,6217,3003,6521,602,1078,2840,7217,8168,5100,948,7285\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 780\n", + "Returned IDs: \n", + "Ground Truth: 3463,4440,6825,8900,6376,197,4012,4562,5285,7360,1963,9087,4188,9005,8080,1764,9161,8284,3722,9288,4986,7078,4546,1407,8681,3492,3737,40,7740,5489\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 781\n", + "Returned IDs: \n", + "Ground Truth: 6208,6419,7569,6764,8393,5480,60,4562,1963,6643,1885,3860,2967,9408,5034,8093,1214,5664,9054,4756,7594,7634,3501,339,1168,6038,8777,9724,1562,3310\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 782\n", + "Returned IDs: \n", + "Ground Truth: 4406,148,2103,7391,1160,652,4181,9875,1206,420,5533,2390,8008,1642,2809,8419,2506,5649,1596,3551,8546,2943,3430,3920,7070,2595,8844,167,7375,1435\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 783\n", + "Returned IDs: \n", + "Ground Truth: 7571,6580,8247,6639,2655,6794,4190,9213,9684,3704,3222,1861,5472,8360,9143,2717,5820,3987,66,4887,543,954,9405,7574,7144,9801,5954,1619,8711,5883\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 784\n", + "Returned IDs: \n", + "Ground Truth: 5522,2696,8342,1663,7660,50,7487,6268,4157,5321,8379,7594,6825,5288,2581,1122,1903,4252,1764,5436,9346,6722,615,1817,4614,1127,1728,9563,3477,5815\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 785\n", + "Returned IDs: \n", + "Ground Truth: 1677,1885,387,5190,1125,4514,4136,174,2831,4197,4850,5034,7096,7037,1963,3124,4373,5910,8590,457,2850,6643,3670,3963,7569,605,9724,1105,5074,2716\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 786\n", + "Returned IDs: 3310,1206\n", + "Ground Truth: 2726,2095,5434,557,2306,1344,3454,5525,9483,6387,46,5777,3531,1113,8785,6853,5287,4181,8153,3310,4940,6283,5781,5047,3649,2462,5250,3444,545,5550\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 787\n", + "Returned IDs: \n", + "Ground Truth: 7943,7966,608,8649,6951,3904,659,4691,5885,8717,8349,2012,9877,3863,3474,290,9117,2627,5803,3008,1595,2578,7480,1429,6662,3622,6658,6418,4726,6854\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 788\n", + "Returned IDs: \n", + "Ground Truth: 9673,6747,9016,4388,3809,5480,3551,286,2181,2585,1619,4953,6673,1389,5146,5330,9787,3024,920,828,1861,5701,6819,4887,5275,9914,2785,2876,5885,5244\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 789\n", + "Returned IDs: 3878,7140\n", + "Ground Truth: 705,9269,3878,9809,5654,260,2308,8672,2893,3291,7376,243,8087,6811,7390,8499,8063,5117,2053,1745,6768,9540,7768,4676,575,7243,2167,136,8521,3184\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 790\n", + "Returned IDs: \n", + "Ground Truth: 8889,3610,7093,8580,3008,2000,8649,6557,2773,3063,2011,66,155,4255,517,6776,6285,6307,9093,2592,3347,6046,6521,6073,9345,658,5900,9750,2896,920\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 791\n", + "Returned IDs: \n", + "Ground Truth: 1657,5678,7174,4460,9844,2686,6894,5021,9459,5029,8066,7591,3794,6451,5200,2503,210,8143,9213,5302,618,8343,288,8692,6741,1289,666,5708,7635,3829\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 792\n", + "Returned IDs: \n", + "Ground Truth: 8634,9391,4009,889,9164,3863,2836,3008,6513,911,864,2596,7392,3474,7865,3340,7946,7029,6615,593,6457,7988,4895,5089,3738,1518,6688,5098,7358,5439\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 793\n", + "Returned IDs: \n", + "Ground Truth: 9371,228,1593,2356,8879,2706,9145,564,6871,3757,4688,2404,8653,9824,7501,7356,7560,548,9576,6136,6669,5278,9865,828,3432,323,2268,1016,6551,3624\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 794\n", + "Returned IDs: \n", + "Ground Truth: 2032,7432,192,6808,7629,9134,5146,7143,6747,1818,4293,5480,2774,1344,8363,9838,7919,1462,9777,6563,1619,9492,8844,3484,2087,4061,2,6219,5340,4991\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 795\n", + "Returned IDs: \n", + "Ground Truth: 4474,1064,1732,846,9099,6272,5909,7219,8808,7220,8679,2437,3871,7285,1996,5325,4098,7452,2837,2228,6066,6521,2427,3623,5783,6219,160,2840,2953,975\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 796\n", + "Returned IDs: \n", + "Ground Truth: 3245,541,5885,7280,4885,3495,2174,8787,4019,7966,5383,6418,3436,4678,1518,4708,1066,9118,8649,3800,3320,1356,6005,2416,8028,7816,1738,8549,8024,370\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 797\n", + "Returned IDs: \n", + "Ground Truth: 3889,2723,3238,9280,525,2877,9124,6540,6444,7603,4998,5200,372,8580,807,2603,859,8859,5496,2434,5238,3873,7472,9093,3347,962,153,7078,312,5949\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 798\n", + "Returned IDs: \n", + "Ground Truth: 5678,4061,4460,5029,9213,793,5021,9844,2655,4778,5200,483,210,7591,8366,4934,7874,4887,3145,2717,9459,414,7635,4286,2952,6825,2608,8858,615,9675\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 799\n", + "Returned IDs: \n", + "Ground Truth: 5949,2262,7078,5711,1827,4012,529,8535,4198,15,7603,5238,3160,5485,2425,2374,6268,9042,4992,2785,653,3669,1444,8879,4365,9561,9234,9938,8707,5084\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 800\n", + "Returned IDs: 290\n", + "Ground Truth: 9750,3738,5404,6165,309,7943,3707,5050,8339,6689,7389,7816,4497,9615,658,7282,7966,5205,2011,4667,5723,2927,2997,386,3826,6418,3560,7455,4411,4634\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 801\n", + "Returned IDs: \n", + "Ground Truth: 3610,534,4554,8649,9021,9770,2831,4658,5480,7217,157,9861,4655,4506,1710,7765,1817,4887,8093,3515,5925,5620,8014,761,920,6468,1881,1134,9049,1885\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 802\n", + "Returned IDs: \n", + "Ground Truth: 1915,4560,6177,19,4297,155,2778,445,1415,7093,6338,2165,3548,7790,8729,3826,140,2450,1202,8411,9781,9270,5875,7558,6153,6686,3185,517,7456,9406\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 803\n", + "Returned IDs: \n", + "Ground Truth: 5555,2775,9190,2862,5157,5527,5838,684,7243,677,8499,3544,4852,8292,2147,7723,2151,5384,2642,5600,8672,8150,3980,8569,2437,841,8298,8576,936,2841\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 804\n", + "Returned IDs: \n", + "Ground Truth: 4614,5436,6615,654,2603,2696,7729,2937,5070,3385,9108,2863,2188,6756,4485,3722,1497,8900,5972,2111,828,3477,8914,8340,6179,5387,6566,8284,6764,9295\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 805\n", + "Returned IDs: \n", + "Ground Truth: 2898,189,7511,9861,7401,4037,9455,570,8026,548,9093,5377,9914,9408,2122,4457,2716,920,5278,6004,2249,2831,4562,546,9063,7477,8653,3445,278,277\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 806\n", + "Returned IDs: \n", + "Ground Truth: 1032,4854,4116,4688,2878,9525,7085,2763,4075,992,5954,2972,9824,967,9115,4621,3160,9371,5381,4078,3432,2454,7569,9576,828,154,8706,3040,9814,60\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 807\n", + "Returned IDs: \n", + "Ground Truth: 1103,352,5282,5005,6491,2427,6647,9552,6615,6584,7497,7966,3859,3977,6175,1809,7016,3629,7754,2642,4308,9345,7700,6721,1451,846,9371,4497,3034,5403\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 808\n", + "Returned IDs: \n", + "Ground Truth: 5008,6095,9895,1885,4197,4647,8535,6608,7532,8317,9561,628,6219,5034,8014,7594,86,3480,4948,4535,1963,6643,7539,7634,1827,9799,1288,5848,8590,646\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 809\n", + "Returned IDs: \n", + "Ground Truth: 6119,154,1134,6615,137,8889,6337,1861,8340,8653,7217,290,6647,9223,3477,2618,3008,2937,7946,1944,3627,2405,3426,3004,2147,5940,9576,1544,828,7966\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 810\n", + "Returned IDs: \n", + "Ground Truth: 6047,1505,2074,1866,2785,4357,2782,3118,9288,4061,1016,4296,4831,9345,1003,9914,5840,7319,1079,5776,8144,761,1827,2507,728,8425,1593,597,2631,228\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 811\n", + "Returned IDs: \n", + "Ground Truth: 4770,6514,7125,6558,2862,5677,2169,8569,539,2933,8206,1616,5311,4880,5453,6080,1430,1880,7795,5157,2147,9964,7031,9145,3564,2775,9190,2427,5527,3211\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 812\n", + "Returned IDs: \n", + "Ground Truth: 2153,9291,6301,2950,4932,5913,8896,1176,8583,1189,4195,1455,1298,8867,7544,4041,3086,7246,4022,6312,1490,4147,3181,7186,1780,135,8938,517,1687,5452\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 813\n", + "Returned IDs: \n", + "Ground Truth: 8988,6170,861,3707,2179,2275,6575,8216,5058,1217,7865,7543,6342,3461,3564,9410,7297,1687,487,1415,2572,9303,5039,3413,7682,8585,727,1562,5117,91\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 814\n", + "Returned IDs: \n", + "Ground Truth: 9788,5113,7566,7033,1323,2039,5608,1725,7951,1,3999,3669,669,2921,7335,3781,6556,4311,2728,8187,1159,9426,4950,4600,3894,2219,1424,8366,4732,343\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 815\n", + "Returned IDs: \n", + "Ground Truth: 9684,278,1840,2608,2524,7785,2655,2831,7591,9213,3112,3053,7466,1501,483,194,5972,1389,9935,7432,5568,7143,1964,8892,3187,467,6378,4845,1619,2499\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 816\n", + "Returned IDs: 536,8164,1818,6735\n", + "Ground Truth: 5137,7558,9848,9896,9781,1415,513,2450,6904,3273,8125,7709,1687,2679,9423,2162,9055,7947,7093,4423,9693,3548,7982,8411,5242,8583,1066,5885,8247,3516\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 817\n", + "Returned IDs: 3531,3507\n", + "Ground Truth: 7935,3310,8235,7368,7707,4655,2036,1344,1064,5434,828,7929,3586,2871,767,6454,5525,1462,3501,483,545,2403,4181,2306,1751,8965,1465,9371,6283,6166\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 818\n", + "Returned IDs: \n", + "Ground Truth: 5408,8411,1313,6230,90,8830,290,3127,6369,8787,2174,7282,6331,7570,3880,3738,2848,7544,2730,2000,370,4019,3563,2110,7003,3221,5773,5740,3436,9399\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 819\n", + "Returned IDs: \n", + "Ground Truth: 7085,992,1602,1343,8498,9824,2387,4069,2374,5330,4625,8879,5588,921,6387,3253,734,2201,2454,5806,5949,7078,7267,1622,3651,2785,3160,1935,3987,9115\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 820\n", + "Returned IDs: \n", + "Ground Truth: 1288,13,4750,7767,2496,8285,9069,2078,8653,1965,8498,7317,4948,5363,1963,1827,8406,1116,8123,2850,646,1061,5848,6643,6336,263,2419,1885,9194,1338\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 821\n", + "Returned IDs: \n", + "Ground Truth: 3337,3463,2188,7262,9005,9682,613,5436,3134,5879,4168,5943,5387,9288,9635,3737,4562,3406,6376,8819,4904,1748,4688,9161,3762,7487,9110,7740,9989,7019\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 822\n", + "Returned IDs: 8552,1463,7635\n", + "Ground Truth: 3420,8552,7015,6125,2507,6869,9828,4599,1463,4245,5536,2655,2247,8393,6504,3825,7780,8733,2622,4210,3534,6127,7635,5456,1501,4217,6722,4689,8950,4998\n", + "Recall: 0.1000\n", + "\n", + "Query ID: 823\n", + "Returned IDs: \n", + "Ground Truth: 3220,4742,2017,8413,7125,8859,4513,5715,5069,8588,8445,1944,5311,6825,1213,7027,9269,6557,579,9430,9145,2937,147,3117,2155,5860,8038,9236,7431,8883\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 824\n", + "Returned IDs: 8346\n", + "Ground Truth: 2302,7041,3839,1017,4132,7036,2269,9938,3757,2785,7868,8346,807,4582,312,3068,1012,6023,1164,9568,7364,4836,3117,8238,6670,7771,7938,6159,5716,4061\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 825\n", + "Returned IDs: 2581,8736\n", + "Ground Truth: 2166,3594,7787,2410,5067,3477,9359,4330,879,7817,5776,7554,4572,2082,5688,6765,2527,8379,9796,3250,3424,7968,8334,7742,6701,1903,8064,8342,6372,7271\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 826\n", + "Returned IDs: \n", + "Ground Truth: 1830,2037,3501,7031,8283,1199,8275,6348,2269,1187,6647,4256,9455,7603,9371,3340,5885,7801,2895,2837,2045,3591,2603,154,6706,3104,4891,7123,828,3613\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 827\n", + "Returned IDs: 652,2595\n", + "Ground Truth: 7472,3254,2373,181,6006,574,4717,2603,1562,5841,23,8393,3833,6540,4607,395,3526,4998,1800,7102,406,1477,962,6313,5294,6491,1268,2833,4158,5480\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 828\n", + "Returned IDs: \n", + "Ground Truth: 2356,2262,5776,8379,5910,5954,6030,4562,9455,2188,829,1501,233,4934,4440,1963,4614,6594,6794,6538,5377,1935,60,6871,4168,2696,5489,7487,9792,2603\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 829\n", + "Returned IDs: \n", + "Ground Truth: 2694,6021,8886,4174,7389,8574,1157,2011,3078,6213,9039,1861,3377,247,6615,1862,6871,9864,7282,1518,2443,7038,309,4497,4272,7384,6037,4256,6356,679\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 830\n", + "Returned IDs: \n", + "Ground Truth: 3757,7771,4167,4400,3138,6670,8238,7629,2262,8312,5840,707,6179,2785,4460,1017,40,7620,9938,7143,1827,6041,5965,3629,6775,6386,8697,3033,3870,8346\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 831\n", + "Returned IDs: \n", + "Ground Truth: 523,2438,7800,5742,3040,8089,2425,227,9042,2454,2943,9503,6970,9242,4734,6115,2559,2975,2262,7085,132,9866,9509,5187,2785,1987,1827,3033,8707,8080\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 832\n", + "Returned IDs: \n", + "Ground Truth: 5031,2778,6374,9819,4535,4490,5458,6488,3426,5146,4658,5289,8269,1663,6292,9223,7946,1105,5480,9562,5737,779,1134,8198,7129,4655,1861,9781,5885,7069\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 833\n", + "Returned IDs: \n", + "Ground Truth: 5872,489,6100,1726,2179,9757,6640,4755,4973,4129,2275,7445,8522,4615,8481,9543,3413,7874,1403,5622,3602,4586,6350,6165,3707,5946,312,7417,4683,4830\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 834\n", + "Returned IDs: 9796\n", + "Ground Truth: 8888,5367,6372,9635,5338,2188,1888,355,6666,4854,2952,9796,5892,4649,4157,665,6794,2081,1935,7123,2763,2963,15,8342,5187,5615,5067,6551,7136,2683\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 835\n", + "Returned IDs: \n", + "Ground Truth: 2468,8247,8711,6854,8625,1861,2174,239,7542,536,8522,4634,3185,5303,6153,3436,3987,9223,146,290,493,2243,5568,7217,9781,3222,2753,6307,9593,6046\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 836\n", + "Returned IDs: \n", + "Ground Truth: 2111,7095,673,3340,2450,3094,309,7373,8988,9468,6904,1415,7946,254,1687,2836,6089,8949,7389,5181,5805,6615,2147,9016,9269,5940,4405,6689,9733,7558\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 837\n", + "Returned IDs: \n", + "Ground Truth: 2345,2927,7042,7543,1503,7485,1217,91,7703,4490,659,8353,3898,8660,2862,704,3170,7240,4029,5613,926,6917,608,8147,9374,6211,3904,5764,4213,3008\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 838\n", + "Returned IDs: 5175,4958,2695,6744,7998\n", + "Ground Truth: 6733,5675,4252,8498,1003,1505,5176,3654,5806,3717,8953,7452,5175,9324,9431,6756,2599,974,4296,8676,2370,5776,2319,712,5288,8057,6334,4831,1286,5668\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 839\n", + "Returned IDs: \n", + "Ground Truth: 8569,8177,1819,1418,9798,5241,4924,725,1122,2642,1837,4761,2403,9796,7750,9473,2753,2974,3980,8883,7754,5940,132,5447,728,539,7003,1849,5187,1187\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 840\n", + "Returned IDs: \n", + "Ground Truth: 7146,5721,698,6532,1273,8450,4314,1457,2771,3910,909,8466,463,5658,6474,6247,4960,1090,319,7089,9365,3228,5958,6385,7823,4293,3597,7929,9595,8856\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 841\n", + "Returned IDs: \n", + "Ground Truth: 9417,5303,3973,6794,2011,4206,5615,6165,5124,1757,2753,9254,5230,9750,5614,309,4605,5139,7865,1549,1963,6615,6513,6871,6739,9745,6508,7552,483,8971\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 842\n", + "Returned IDs: \n", + "Ground Truth: 5615,8944,4410,7724,9069,7381,6794,904,4157,4824,5303,3973,6329,5422,7426,3110,1886,5425,2472,1963,8681,1700,6970,1663,2135,2782,8342,4192,8967,6255\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 843\n", + "Returned IDs: \n", + "Ground Truth: 1817,4824,1700,7381,5363,5074,2507,2131,1885,7569,5679,6115,7742,2454,9922,7628,2628,3112,1970,615,1523,4400,3594,5711,8105,9287,228,2374,3206,1164\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 844\n", + "Returned IDs: \n", + "Ground Truth: 3682,5615,4416,1886,132,2943,7426,6169,1663,6329,712,8342,3665,2366,6794,3700,2975,2187,6430,9069,9038,4330,9283,1912,6372,4400,8398,8060,839,1150\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 845\n", + "Returned IDs: \n", + "Ground Truth: 1540,8363,4202,2093,4343,7399,8116,9063,6378,2585,6747,7629,7880,5963,2655,7466,9838,4586,8481,2223,3493,9628,7143,5972,263,916,6197,9524,6650,3253\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 846\n", + "Returned IDs: \n", + "Ground Truth: 2776,4623,4536,9257,9305,8954,2674,3794,282,4055,9062,4281,4771,6615,8212,7682,533,7997,6723,4157,7123,1852,1344,9939,6128,6820,8340,3729,2453,3715\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 847\n", + "Returned IDs: \n", + "Ground Truth: 7334,4472,5932,9368,4337,5898,2426,8721,5873,5372,4643,5273,1674,1397,4683,1330,2269,8774,1581,4810,8936,3103,1199,1004,9754,8265,6398,1830,1191,7645\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 848\n", + "Returned IDs: \n", + "Ground Truth: 7463,9024,9693,696,4636,6879,1202,6285,19,3010,9956,7947,2369,9392,5341,4664,8405,1687,3827,1922,1664,4280,7093,5539,7633,888,7215,6519,3476,4699\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 849\n", + "Returned IDs: \n", + "Ground Truth: 9293,7859,6312,3153,8302,8393,3004,8188,9376,8340,758,4490,4039,1680,3258,4463,2332,1403,1248,137,8663,7291,4414,8498,1544,9295,5714,9448,9139,6151\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 850\n", + "Returned IDs: \n", + "Ground Truth: 5536,670,3550,6316,2507,9174,9283,4217,2472,8334,3111,2850,132,8694,6410,7594,9828,1122,6610,9682,481,5340,6127,4934,2696,6481,8310,6471,7569,2266\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 851\n", + "Returned IDs: \n", + "Ground Truth: 8233,769,5589,4400,4293,9611,4817,2262,7629,2199,5176,8498,4140,6329,1586,8346,7064,978,7767,8366,679,671,2418,4460,414,5525,7267,1540,5313,7022\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 852\n", + "Returned IDs: 2282\n", + "Ground Truth: 2545,3867,4497,3444,6584,5729,1960,4761,725,9893,9303,1200,2370,7032,728,8799,7267,7865,5330,6100,1861,1403,8025,4634,5404,2862,9770,8649,9641,7677\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 853\n", + "Returned IDs: 983\n", + "Ground Truth: 4482,983,5197,6148,7131,5992,3005,7997,7067,6146,1270,2112,3840,8922,8546,8748,4332,1292,3715,2967,3089,5972,2390,4236,5533,2506,7432,6090,3285,8376\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 854\n", + "Returned IDs: \n", + "Ground Truth: 483,6610,1987,8089,6129,1827,9093,2717,2696,2745,6815,7217,2850,9242,7364,7432,4891,3234,3033,2785,8014,3374,5917,1622,7252,5711,7677,179,2621,2567\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 855\n", + "Returned IDs: \n", + "Ground Truth: 5575,3518,8238,4060,3731,4582,3220,3562,8312,2899,9214,9145,2167,4831,5311,8663,9912,539,4615,4638,3870,776,8891,807,9468,3117,8859,1589,8087,125\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 856\n", + "Returned IDs: \n", + "Ground Truth: 8134,8537,3081,5472,6386,6871,442,7865,3867,2493,3871,6103,3865,1619,536,2370,8528,9095,476,6594,5422,2831,6854,4460,2837,839,4562,7369,590,9405\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 857\n", + "Returned IDs: \n", + "Ground Truth: 155,4247,1990,6658,4016,2174,3943,4490,8382,1859,4974,6662,5773,6794,2468,7542,7069,7093,7816,8660,9289,3548,9781,7753,2960,8972,4785,8411,6196,7661\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 858\n", + "Returned IDs: \n", + "Ground Truth: 3484,5330,9777,6563,4991,8844,7143,3310,5649,1206,671,9715,1462,8892,2505,1344,3764,1818,652,3551,4293,278,4460,3486,192,4687,1378,7605,7629,9591\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 859\n", + "Returned IDs: \n", + "Ground Truth: 635,7299,4364,7782,8934,5227,8675,6897,2801,5313,3444,3531,4289,4940,1962,4181,7304,3752,7064,767,871,2871,2929,5525,1300,4293,6454,3285,7629,8235\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 860\n", + "Returned IDs: \n", + "Ground Truth: 1490,7614,1687,608,8227,577,8583,2012,5963,1403,7543,8442,9968,2130,6106,5613,3498,370,4821,3462,6211,5120,9546,6849,2927,1602,4135,432,7661,4022\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 861\n", + "Returned IDs: \n", + "Ground Truth: 6153,9223,6854,536,9711,1692,1944,693,2403,7003,9965,1267,7544,2012,9875,3965,6918,3235,5700,7790,1861,1457,8609,5226,5364,7263,9095,15,7217,1753\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 862\n", + "Returned IDs: \n", + "Ground Truth: 9367,3317,8893,6662,9093,3417,6557,836,539,8275,2000,155,6307,5713,2174,1920,2963,7790,4297,5447,7542,4308,9355,3610,8558,4634,7219,792,8155,5750\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 863\n", + "Returned IDs: \n", + "Ground Truth: 9120,8076,1068,6557,6289,4972,8240,6307,7033,9498,3477,2596,6801,6488,9117,7799,4569,6580,1673,699,1881,6615,4192,6776,9936,9223,4830,1920,4357,6951\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 864\n", + "Returned IDs: \n", + "Ground Truth: 7853,7344,9238,7542,2174,9721,164,3690,1944,290,2572,1562,649,7016,536,3008,7966,7865,841,6074,8569,6647,9258,352,5573,3867,2567,3866,7217,2862\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 865\n", + "Returned IDs: \n", + "Ground Truth: 2637,7566,5113,8858,1,3546,5996,1501,8238,5582,8187,2728,9063,6252,2710,5784,3679,6556,3999,170,531,5462,214,1323,6527,3606,1905,2039,6566,4485\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 866\n", + "Returned IDs: \n", + "Ground Truth: 3518,818,7292,5692,9129,9928,1329,6672,5986,5575,539,8535,7875,7634,8900,1963,6113,3966,5084,1288,9499,1742,2531,8815,9201,8087,3731,1444,776,7951\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 867\n", + "Returned IDs: 68\n", + "Ground Truth: 3821,2725,7432,2445,8628,185,57,8148,7368,9208,3805,6224,7255,3501,1479,2967,1800,6357,204,3310,192,7611,7193,688,9202,2585,1273,3453,3270,9783\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 868\n", + "Returned IDs: \n", + "Ground Truth: 5149,643,8089,5434,7248,5566,3033,1890,5160,7395,8096,1379,4557,4754,688,9242,6853,3319,9591,3361,9371,4983,4039,6339,1894,8976,483,7465,9773,7432\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 869\n", + "Returned IDs: \n", + "Ground Truth: 8504,1840,1389,4061,8080,2717,9093,6313,2145,772,414,9838,6747,2774,3317,6422,3873,3233,2300,1394,1619,3520,9293,5016,9893,8319,4113,6998,1168,7466\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 870\n", + "Returned IDs: \n", + "Ground Truth: 5649,3977,3501,8177,5330,3104,549,3551,9555,7629,8653,4991,1809,8466,4116,3629,1409,761,9777,6563,2618,2139,7375,2078,8718,7193,671,3689,8129,4293\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 871\n", + "Returned IDs: \n", + "Ground Truth: 7093,8782,6686,1518,6046,6658,8522,6285,1403,7485,9367,5464,291,1687,659,8247,3094,2109,9821,3086,3202,6106,3863,3712,7282,7943,9148,7712,8104,2450\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 872\n", + "Returned IDs: \n", + "Ground Truth: 5267,6006,8099,4346,6540,414,3873,8580,525,3588,372,1453,2925,9506,1892,6623,4998,9211,9243,8283,5302,8858,5465,5905,312,3894,7548,3995,4917,8565\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 873\n", + "Returned IDs: \n", + "Ground Truth: 5656,6887,3282,2831,8123,6205,4704,3551,1871,5480,5363,4953,8653,6776,5925,920,5979,3477,8406,3432,1749,6829,8933,1105,6254,1650,3112,5031,4318,9307\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 874\n", + "Returned IDs: \n", + "Ground Truth: 3757,8653,3729,5662,5311,7957,841,3629,2642,4132,7721,707,9095,9552,3199,7213,5093,2151,920,348,5840,7068,8079,9345,2412,3980,5517,3551,8879,1477\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 875\n", + "Returned IDs: \n", + "Ground Truth: 2187,132,1886,7699,6794,4924,624,2943,1548,7059,3980,1963,846,9796,2078,5425,9061,904,4256,8490,1663,2837,728,9346,8141,642,8866,4157,6759,1122\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 876\n", + "Returned IDs: \n", + "Ground Truth: 8247,4436,5464,5472,590,7865,2896,8711,404,3222,690,7093,4134,6557,7560,4410,3749,6046,8933,9395,146,6307,3008,4911,4602,6794,7217,239,7031,3604\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 877\n", + "Returned IDs: 6850\n", + "Ground Truth: 4189,6850,2243,6261,9717,8649,9750,9770,6814,5033,3987,7515,6951,2416,8022,3008,4003,2196,3800,7966,3474,8549,258,2743,9581,654,8867,2986,5213,6580\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 878\n", + "Returned IDs: \n", + "Ground Truth: 4550,576,4308,4269,7813,5961,36,1071,4831,7832,9236,1075,361,7568,1837,7685,8019,8982,3511,825,8893,7600,2280,6725,8332,7634,2084,1885,7750,6871\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 879\n", + "Returned IDs: 652,2595\n", + "Ground Truth: 4772,2404,7499,1619,1168,5217,4991,5480,4887,8892,2743,9492,9200,772,8093,6747,2779,7807,3551,4110,5146,2521,1840,7432,2831,1817,3751,4460,7213,2409\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 880\n", + "Returned IDs: \n", + "Ground Truth: 9189,3270,4582,8612,3117,8267,5117,7044,1035,9145,6320,3731,1433,3407,2459,6493,3220,3628,6113,41,9499,8163,5311,2531,8038,3601,8859,2147,7431,8576\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 881\n", + "Returned IDs: 3715,2897\n", + "Ground Truth: 3715,2897,8210,8820,2390,2264,4236,5871,238,5843,7375,5830,1936,148,210,2506,7391,263,4466,5533,4406,8336,397,1344,1206,8852,1270,229,9960,4722\n", + "Recall: 0.0667\n", + "\n", + "Query ID: 882\n", + "Returned IDs: 2595,652\n", + "Ground Truth: 920,3561,5230,3206,7432,2745,4887,420,1817,9787,9509,2404,2912,6756,2494,1168,263,1389,5958,3315,7465,4466,4234,3112,1663,3016,9242,1812,5480,688\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 883\n", + "Returned IDs: \n", + "Ground Truth: 2987,7348,9781,6987,6620,7263,3086,2404,9237,4832,2245,6953,7162,2450,1692,1298,2078,3386,3474,4147,9423,7558,7790,6817,2162,4844,7816,7982,3734,7285\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 884\n", + "Returned IDs: \n", + "Ground Truth: 523,829,2438,6970,2559,2430,5793,4621,3492,6249,15,5489,7376,7898,2943,8407,1334,2188,728,9459,6624,2425,4992,6739,6130,5946,202,5891,4335,3546\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 885\n", + "Returned IDs: \n", + "Ground Truth: 5049,8184,2592,2997,8057,5459,8649,8549,3008,3610,6124,1956,447,89,4708,2370,5311,9613,7,3738,9770,9101,6106,2000,2747,4263,1437,7282,7943,155\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 886\n", + "Returned IDs: \n", + "Ground Truth: 8232,4865,7501,3160,6420,4688,1079,154,4012,3289,1016,1407,2790,6885,3432,5394,1963,6097,1747,1342,548,564,8442,8653,6736,7022,3625,1593,2406,1388\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 887\n", + "Returned IDs: \n", + "Ground Truth: 6029,9430,2412,4308,7750,6165,2084,1837,6871,7634,8793,4521,7568,4060,2147,4497,8340,3078,2646,825,9248,1688,7832,9272,2403,7600,4638,9864,5447,3684\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 888\n", + "Returned IDs: \n", + "Ground Truth: 7248,6018,7862,3033,3526,2535,4158,3750,1456,3651,4557,7432,1379,2201,2572,4830,9371,68,7629,9898,6775,6493,6670,751,9711,4442,5672,9538,9777,5093\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 889\n", + "Returned IDs: \n", + "Ground Truth: 9359,1287,4060,6078,6252,4903,3635,4207,8086,3317,4615,8858,8720,7566,2637,1235,8348,3468,1473,7793,3873,8580,1,1633,1611,2052,2701,9869,414,6803\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 890\n", + "Returned IDs: 6114,3088,2207\n", + "Ground Truth: 7143,7216,3501,7368,3310,1344,6224,2585,483,3270,7432,5484,9838,4343,6673,7743,5819,5253,963,2445,8363,7949,4985,7399,5963,7064,1642,4684,4293,6090\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 891\n", + "Returned IDs: \n", + "Ground Truth: 5759,7003,8320,7872,9325,8883,8139,164,6361,2403,9958,9163,3090,6598,1688,2017,3796,5809,2143,1318,2174,2062,8131,1680,6165,6106,1999,5399,1889,4703\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 892\n", + "Returned IDs: \n", + "Ground Truth: 6747,5480,4334,1792,9134,3116,2521,9492,2774,3987,6219,9738,3520,7432,8614,192,1881,4215,1619,9838,6324,9861,5016,920,1168,4846,1650,9914,5146,2404\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 893\n", + "Returned IDs: \n", + "Ground Truth: 7865,9399,4497,4491,5278,3078,5464,9391,9770,2712,5885,7966,2997,7943,8443,4390,2642,3008,2592,2837,2859,5509,5314,516,5403,2554,2000,7016,2998,1518\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 894\n", + "Returned IDs: \n", + "Ground Truth: 3870,4824,6119,5303,4634,2603,3317,5340,4607,4060,9711,7381,1817,5230,6073,3033,2952,4831,483,8393,6764,1497,1619,6794,7660,2247,1973,2531,5575,2450\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 895\n", + "Returned IDs: 8926,7726,3043\n", + "Ground Truth: 5077,8309,539,3162,8036,4703,6672,4601,5246,7667,2129,7634,1819,9079,2017,2084,1885,6598,4529,611,8087,1228,5575,8569,1837,4308,2938,9823,8445,2472\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 896\n", + "Returned IDs: 967\n", + "Ground Truth: 1473,2277,2696,7052,7476,3216,4188,5177,4562,2434,1728,3450,3686,2231,4168,9946,6741,8765,233,9935,1519,5387,3406,1501,7927,3751,6004,1205,6566,2131\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 897\n", + "Returned IDs: \n", + "Ground Truth: 8393,920,6314,8668,9287,7716,6741,3238,8765,5818,2608,7472,6623,2603,8914,525,1732,4562,2231,7729,2915,2499,6825,4953,8138,9467,3406,3959,2310,1331\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 898\n", + "Returned IDs: \n", + "Ground Truth: 3563,474,8867,7456,155,3010,3448,9423,9251,8932,3734,6662,3476,1497,4895,5803,7790,6353,1968,3221,8090,2162,9376,3094,5940,8339,3317,254,103,9116\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 899\n", + "Returned IDs: \n", + "Ground Truth: 2573,8141,2675,9843,2804,5657,344,5574,94,8945,3859,228,9894,8653,9191,6232,9296,292,2060,2497,1885,2581,9903,8866,8049,6992,4014,6190,8398,5538\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 900\n", + "Returned IDs: \n", + "Ground Truth: 9094,3609,7064,1086,1746,2423,5247,7267,1462,6447,3792,8630,7952,5313,7782,2929,9699,1344,7432,1822,9684,3649,7466,57,5484,5434,8205,5525,8879,7495\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 901\n", + "Returned IDs: \n", + "Ground Truth: 684,3017,9358,7194,8499,9135,8063,8576,782,3876,1401,8087,4724,6590,9269,7431,9009,9199,6617,8789,7685,8893,91,7125,1187,7147,539,1944,6989,5069\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 902\n", + "Returned IDs: \n", + "Ground Truth: 3723,4796,654,323,3453,4061,2621,3681,8319,2300,4690,7789,159,1619,4689,860,9858,339,4260,7710,2,8112,7098,5711,1840,6615,7807,1815,1090,7499\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 903\n", + "Returned IDs: \n", + "Ground Truth: 1743,5325,7570,4857,8443,2712,6145,1446,2111,1875,6615,3340,9579,3286,1207,6863,6513,8135,600,8431,155,7175,3560,7946,1720,9652,4206,8946,5940,5181\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 904\n", + "Returned IDs: \n", + "Ground Truth: 4295,3087,3999,9743,9426,9198,339,9162,4061,1943,5079,9816,8105,9302,6764,2,9907,6119,240,4548,3648,3477,3426,4562,7742,4460,783,7377,8103,1159\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 905\n", + "Returned IDs: \n", + "Ground Truth: 7767,5363,4400,1079,893,154,1963,8653,4416,7956,5776,8622,5589,9903,5972,6329,5988,6670,6430,2078,1016,3551,2226,5688,6253,2428,7022,3262,3865,8342\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 906\n", + "Returned IDs: \n", + "Ground Truth: 1692,7753,1047,771,1415,1142,2589,2920,9180,7295,3548,9656,1934,2778,5110,7558,2488,2995,1859,9148,9376,8660,4405,1831,3186,5750,5082,1921,7300,2851\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 907\n", + "Returned IDs: 2690,8937\n", + "Ground Truth: 4019,7280,1749,7816,1849,1518,8028,8549,2848,1098,2000,1584,3395,8184,9148,5383,1374,9003,5196,6338,7414,1003,4009,2380,2174,7110,8969,5408,6854,290\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 908\n", + "Returned IDs: \n", + "Ground Truth: 2005,516,7065,2567,5403,9186,8331,7843,7193,3517,8443,1531,4053,4288,250,6457,9584,3610,8659,5514,9461,6020,3094,137,4752,9059,600,9853,3340,1593\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 909\n", + "Returned IDs: \n", + "Ground Truth: 5091,3921,2319,2599,7273,5538,3859,1809,8313,9131,9824,8025,7129,8398,8117,8491,8042,2370,4256,2023,9371,5806,6232,9115,15,2859,9181,5278,6693,9448\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 910\n", + "Returned IDs: 9013\n", + "Ground Truth: 2247,6152,8156,5917,5448,4557,1619,5220,3657,574,1121,483,9684,2717,1792,4887,5679,6747,6524,2655,7522,4507,272,7563,5736,3556,1214,6391,2831,2524\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 911\n", + "Returned IDs: 3424\n", + "Ground Truth: 3424,988,772,2735,1278,8953,2521,5504,4061,7279,7511,1619,9992,8892,7364,2943,6886,8643,4201,3648,3723,5866,8093,6825,4721,1424,6933,4460,4796,3021\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 912\n", + "Returned IDs: \n", + "Ground Truth: 6557,7016,5675,4357,5181,4744,9865,2074,761,1061,7956,1593,2084,4296,2859,5491,6487,4831,5776,4061,7452,9345,3610,5278,7560,1881,5175,7425,4252,8445\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 913\n", + "Returned IDs: \n", + "Ground Truth: 7687,6693,2427,1817,2404,8653,5363,7669,2505,5692,2831,3871,5330,9576,8117,920,4887,1827,9354,1650,4688,5176,3432,8892,5278,1779,992,4444,2628,2567\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 914\n", + "Returned IDs: \n", + "Ground Truth: 278,6853,1614,3444,4293,3430,5287,2449,635,1344,2095,4181,8130,767,4783,7064,5550,5781,9245,7629,5047,4289,2462,7248,5330,3764,4497,3531,9371,9483\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 915\n", + "Returned IDs: \n", + "Ground Truth: 1446,8825,8312,8326,496,1223,4166,1985,36,4390,1743,164,2712,4269,2736,1075,7822,7568,4308,4132,5426,6369,3604,7141,8401,2092,5262,1837,8038,6483\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 916\n", + "Returned IDs: \n", + "Ground Truth: 9185,2073,3749,1967,9063,2778,8266,9032,1753,4726,4658,2488,3236,9024,6153,7790,8169,7225,8892,6008,6119,5146,8006,1838,2444,7835,4146,1930,1530,9693\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 917\n", + "Returned IDs: \n", + "Ground Truth: 276,688,9669,6807,1751,7730,4991,7368,7213,5540,5434,963,1800,1273,1622,4466,2725,6693,920,2929,2801,3805,5563,8089,3455,4893,3752,9371,9509,9134\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 918\n", + "Returned IDs: \n", + "Ground Truth: 5736,4677,4953,5377,1105,4887,1619,8471,8765,9287,7452,9492,4234,2457,5283,7511,4562,9290,2575,8643,8892,4373,2231,9215,6219,2521,5289,3348,4463,9830\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 919\n", + "Returned IDs: \n", + "Ground Truth: 9673,725,2736,5983,426,2850,2403,1526,9939,1885,4904,3477,7594,665,1105,256,1164,7099,828,94,6219,1428,7037,1106,8297,8177,4845,2696,9522,761\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 920\n", + "Returned IDs: \n", + "Ground Truth: 5949,8732,2448,6171,7254,6444,5496,5238,8378,5891,4004,7603,807,2530,8440,9980,6676,181,5413,2911,5872,9729,4369,8375,5866,2498,7376,7423,4528,3889\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 921\n", + "Returned IDs: \n", + "Ground Truth: 3977,3757,4023,6115,6452,4573,6138,4400,5103,6860,657,8228,4515,3138,9702,9283,8233,3629,6940,3624,1017,2632,2472,9271,8697,7779,4167,7957,5074,7690\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 922\n", + "Returned IDs: \n", + "Ground Truth: 5056,2165,7586,8943,6904,6170,7946,864,7373,664,3094,9016,744,5893,5376,9733,5523,8262,7925,1849,1415,8949,9140,6508,3078,4327,1151,1399,5874,8753\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 923\n", + "Returned IDs: \n", + "Ground Truth: 8123,909,7557,8563,1817,1619,7089,1216,8517,6096,2831,3112,8856,7432,1813,7629,5146,3484,3056,9684,9696,2,9804,5679,8328,2826,8105,4460,3282,9922\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 924\n", + "Returned IDs: \n", + "Ground Truth: 7865,8949,5056,8545,6589,7038,5514,1766,3078,1522,3012,864,9853,7934,5787,9140,6089,600,7634,6457,3094,8223,8796,9750,8443,2694,9059,9663,7547,6282\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 925\n", + "Returned IDs: \n", + "Ground Truth: 5848,5217,6932,4343,6764,2231,2608,7432,1540,4557,8692,8481,2585,210,4110,2223,5495,6747,4460,2976,7466,3576,1168,2219,2619,5701,2247,4887,7807,6419\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 926\n", + "Returned IDs: 9492\n", + "Ground Truth: 5504,1510,1619,6747,1278,4061,9492,7499,2521,4887,3381,7789,1817,988,7277,9914,772,3723,2798,8953,6529,2404,4796,9542,9287,1168,3648,4113,4721,5354\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 927\n", + "Returned IDs: \n", + "Ground Truth: 5425,9868,8080,9346,5299,9722,9790,2187,1963,1663,4416,7554,4000,208,1122,7699,6265,4587,8379,6481,2194,8862,9612,5456,2507,6866,6722,8342,5863,2200\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 928\n", + "Returned IDs: \n", + "Ground Truth: 2432,6226,1744,8284,8425,1015,7687,3530,9327,7303,586,2623,154,228,5176,4546,5590,3299,2804,1016,2041,3160,3655,9371,8835,3977,5574,1593,1866,6992\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 929\n", + "Returned IDs: \n", + "Ground Truth: 6546,5480,9047,6911,7364,8953,2409,5064,9492,204,6650,4460,4796,3520,7403,7432,7559,7213,1685,2223,1619,2535,7022,3102,3651,8557,772,9063,4061,4110\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 930\n", + "Returned IDs: \n", + "Ground Truth: 8247,1687,7790,4891,517,8369,1518,3749,7093,9733,2596,9770,3063,2450,9750,6285,8583,3086,5867,9405,3474,3395,4436,8753,3987,6338,6901,7217,1362,6106\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 931\n", + "Returned IDs: \n", + "Ground Truth: 8528,3973,7452,2132,6794,7270,1885,3908,2850,367,2952,1155,4792,7381,6140,3323,615,8012,6538,9288,5615,2148,9254,5021,1700,3246,4331,3000,4824,5910\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 932\n", + "Returned IDs: \n", + "Ground Truth: 292,5278,7966,6103,5205,9097,1809,536,5403,8042,228,2592,290,6647,7457,6925,5282,6814,5154,7956,2011,8184,7570,6871,9770,608,6124,8313,2975,3629\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 933\n", + "Returned IDs: \n", + "Ground Truth: 3516,6608,6307,5773,6647,5806,4497,3320,6615,1849,2831,920,2601,4887,2530,9405,4187,3734,5154,9097,9536,6338,1134,1597,9821,658,9781,6331,1861,9576\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 934\n", + "Returned IDs: 536,8164,1818,6735\n", + "Ground Truth: 848,7790,6854,7753,9055,3202,1861,779,9237,8405,2521,4932,1134,4398,3320,4557,2716,4887,5110,7125,1298,2174,3963,6813,3426,9287,5016,8093,6642,6647\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 935\n", + "Returned IDs: \n", + "Ground Truth: 6220,4301,9462,4093,6647,6854,6046,7816,1753,3673,1738,7414,8625,3863,1602,9378,4263,9578,593,3987,3738,547,2293,1403,1400,1861,6495,8155,2236,3101\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 936\n", + "Returned IDs: \n", + "Ground Truth: 210,7143,1540,8599,67,280,2360,5357,8557,8852,2223,4460,980,7550,2643,8336,7974,2409,2943,2585,8481,8630,5397,7637,4855,6851,7239,5480,3493,4400\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 937\n", + "Returned IDs: \n", + "Ground Truth: 659,5403,8443,4263,8340,9371,2592,2011,7016,5325,5181,5538,4655,6119,2596,2859,2773,7029,9101,6615,8796,5278,4737,5311,2997,6106,1920,6046,5205,3929\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 938\n", + "Returned IDs: \n", + "Ground Truth: 4889,4414,654,3320,2554,6493,1881,4991,7946,5466,3517,6854,1805,4497,120,3738,7217,3279,6846,864,1602,386,7966,6776,7389,5505,5146,2831,7248,5445\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 939\n", + "Returned IDs: \n", + "Ground Truth: 2554,386,3292,600,5505,3517,1151,7586,6615,3078,7946,4497,7822,2111,8223,250,6493,309,1944,6324,9059,4308,2567,7966,5940,1849,654,137,352,2646\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 940\n", + "Returned IDs: \n", + "Ground Truth: 607,7037,7013,1384,616,7745,4605,4519,3973,4647,7539,4532,9181,9234,1682,2975,5615,3022,5946,6372,298,2952,8012,6077,5447,2931,9547,1885,7900,898\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 941\n", + "Returned IDs: \n", + "Ground Truth: 7389,3377,7217,309,5940,352,3347,654,386,6615,4497,254,6689,658,5181,600,2011,3320,9294,4206,9059,7946,4256,3340,9562,6165,2443,2986,6647,6021\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 942\n", + "Returned IDs: \n", + "Ground Truth: 9130,1833,1177,8331,7991,5317,8401,5390,4788,2084,4609,4500,4058,223,7268,226,4271,4308,7600,825,4936,1706,9764,6725,5420,8326,7634,6260,5117,872\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 943\n", + "Returned IDs: \n", + "Ground Truth: 7263,5033,2217,7397,2533,6094,7380,5130,7778,8349,5878,3553,7543,3386,8389,2987,6918,2980,746,9482,4246,1595,7340,779,7801,3800,9602,2065,1888,4625\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 944\n", + "Returned IDs: \n", + "Ground Truth: 9469,670,6127,4599,9897,9283,2385,6481,2815,5536,9093,3550,8862,6855,8080,5288,7635,2782,2001,314,629,1476,2507,2472,4030,481,7452,4210,9174,828\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 945\n", + "Returned IDs: \n", + "Ground Truth: 7466,7432,9992,1802,4113,4061,5972,2608,6378,9838,2300,5480,4586,4334,9459,3457,3520,1815,4690,3270,7591,772,6422,5774,654,6179,983,8319,9217,8504\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 946\n", + "Returned IDs: \n", + "Ground Truth: 3501,2027,4722,5649,1902,3977,3104,8654,5925,7629,8653,8419,1435,273,1206,3551,8524,828,1344,3488,2939,904,8536,2595,4896,7635,5480,4236,2706,3689\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 947\n", + "Returned IDs: 3356\n", + "Ground Truth: 1868,3551,8653,3625,7574,6097,2831,5480,3484,5330,6647,1650,3764,2521,8494,7956,8319,5363,9861,5468,7267,5016,5917,2404,3632,1881,292,2759,5568,3772\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 948\n", + "Returned IDs: \n", + "Ground Truth: 2554,6430,3234,2642,7243,8653,6647,3880,7016,5742,1003,6558,8858,6646,5157,4390,7125,536,9770,2850,6871,8319,5505,9354,5278,8025,4887,4308,1663,5677\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 949\n", + "Returned IDs: \n", + "Ground Truth: 5480,3493,7091,9838,7929,7432,4334,3520,9063,3649,16,2223,8363,7949,3270,7143,2201,3501,6650,2943,1800,3651,1168,4293,2771,3957,6747,204,7591,2774\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 950\n", + "Returned IDs: \n", + "Ground Truth: 6214,4269,2023,1827,9614,36,9314,3530,4831,2837,839,2510,4052,3641,9288,8117,9518,1061,1003,8569,2646,2370,9616,2074,2642,3316,1866,4400,9345,9745\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 951\n", + "Returned IDs: \n", + "Ground Truth: 3096,8350,9253,5940,8090,1151,6037,7389,6862,5680,6607,216,8339,266,654,5634,103,5023,9760,4497,7455,8362,3721,748,1805,1944,6131,386,8223,2627\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 952\n", + "Returned IDs: \n", + "Ground Truth: 2837,8398,3022,6608,9234,4647,1205,3610,3234,3859,9745,4263,4474,7016,4532,2712,9308,3973,2418,6103,7123,1985,3766,7754,2975,7615,8313,2859,6124,8183\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 953\n", + "Returned IDs: \n", + "Ground Truth: 6854,9386,2174,290,8625,6106,8533,7816,1098,9345,1518,6307,1861,4634,6324,2000,8889,8522,309,3800,649,536,3474,9378,4707,7282,2196,7542,9593,7217\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 954\n", + "Returned IDs: \n", + "Ground Truth: 6217,1131,1458,1559,5239,792,8168,5437,6647,2653,6801,1732,8307,1541,1345,5125,3705,5196,5443,4451,3007,7219,2840,6314,3605,8771,4599,2828,3423,4539\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 955\n", + "Returned IDs: \n", + "Ground Truth: 536,2937,6615,654,6647,6584,6854,290,4497,1849,7790,9750,8653,3707,3222,6307,386,1362,3749,7093,4634,1687,5940,2567,6213,2011,658,8753,5505,6662\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 956\n", + "Returned IDs: \n", + "Ground Truth: 3426,7069,6951,2261,737,7444,8972,1661,7033,7790,3205,8594,6580,9929,6662,8727,5289,7043,3551,3515,5363,3395,4149,4490,658,4921,4154,6630,6658,290\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 957\n", + "Returned IDs: \n", + "Ground Truth: 1983,5958,8653,3632,4887,7089,8844,6563,9049,5649,5256,7313,2404,6474,1083,1093,8319,4796,5062,7574,3484,1817,2521,3629,6885,7501,7929,1344,1144,5147\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 958\n", + "Returned IDs: 6967,9798,4880\n", + "Ground Truth: 8038,2545,1075,7125,2642,2862,6967,8883,7243,8569,892,9269,2275,4256,1616,9798,804,2147,2169,1648,3868,3052,4770,982,9248,5154,3130,3671,2841,4308\n", + "Recall: 0.0667\n", + "\n", + "Query ID: 959\n", + "Returned IDs: \n", + "Ground Truth: 2364,3216,4236,1208,1872,3637,8934,8080,2943,9371,4497,6063,8617,4157,5299,9838,8342,5202,1663,1324,1327,2356,1441,8340,4722,1935,2188,9975,5022,5520\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 960\n", + "Returned IDs: \n", + "Ground Truth: 9246,129,2078,9639,8661,8080,879,9069,8056,3923,5662,7807,208,6547,8879,8451,4330,1663,8490,9796,5776,7554,4622,5338,2356,5425,8340,4531,2618,4075\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 961\n", + "Returned IDs: \n", + "Ground Truth: 4293,5950,2771,4314,7114,1344,5248,5658,6474,549,5589,6532,5525,9371,2027,3310,1206,9555,3228,1593,909,5992,371,7629,3235,3530,1273,2404,7089,5480\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 962\n", + "Returned IDs: \n", + "Ground Truth: 2174,290,1518,5885,3395,6854,3008,8569,1584,6046,6813,6418,9593,608,3749,8830,7943,8649,536,2196,309,3800,3738,5031,8625,7217,6220,2927,1861,7816\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 963\n", + "Returned IDs: 8914\n", + "Ground Truth: 2568,9075,2865,4401,6594,2262,5972,634,9813,8914,4305,6146,4615,1432,6646,4467,4157,101,4366,8340,4512,9543,4745,4649,820,6197,4855,8498,6736,7432\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 964\n", + "Returned IDs: 9213\n", + "Ground Truth: 2831,194,1964,3112,1881,5679,9684,5714,9861,9872,483,9213,1661,9529,548,6764,278,3465,4215,3484,2524,8892,6639,3527,4557,1817,1840,5480,218,3501\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 965\n", + "Returned IDs: \n", + "Ground Truth: 9795,2831,8443,8653,7570,4272,8223,658,6776,9272,2078,2419,6213,8581,2405,292,5363,6647,2011,1650,2869,5403,7529,2554,6854,7966,3347,2044,4497,309\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 966\n", + "Returned IDs: \n", + "Ground Truth: 9750,1298,1189,9291,5913,8423,3800,2196,93,9711,2174,2950,9553,6557,7790,6714,4003,2012,828,1753,5452,1455,2998,8583,2404,9237,1861,4932,241,6647\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 967\n", + "Returned IDs: \n", + "Ground Truth: 9069,1061,5252,355,7900,4831,9583,1970,1886,5615,5074,8676,6372,8342,5848,5501,2135,7013,879,9639,7426,2837,5067,3900,712,5195,7319,9071,5338,5029\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 968\n", + "Returned IDs: \n", + "Ground Truth: 4157,2519,8342,4649,6826,7381,5615,6372,1663,7963,1088,5892,2359,3770,5338,2859,5688,2707,5202,3110,9796,1970,4854,1817,1963,6430,2824,615,7123,9975\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 969\n", + "Returned IDs: 1222,7101\n", + "Ground Truth: 506,9212,3259,7224,3591,4582,6398,169,2993,8714,6935,9500,807,9431,3140,7167,5889,9919,1939,3163,8033,1850,3682,395,8469,2045,4291,3178,9373,8407\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 970\n", + "Returned IDs: \n", + "Ground Truth: 9875,9055,5885,9711,1861,7193,9237,4316,7629,5480,8729,3749,8879,536,3734,751,8844,7790,4222,3086,1298,1301,1687,2162,7248,6338,9024,309,8609,6285\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 971\n", + "Returned IDs: \n", + "Ground Truth: 4173,7949,1112,578,9074,4218,9412,9365,6235,7213,4796,4460,1462,8844,3501,371,612,1376,7929,165,4721,8149,1562,4260,7789,7070,1685,5865,2,3593\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 972\n", + "Returned IDs: \n", + "Ground Truth: 5434,5540,920,9939,2404,9492,8089,5363,4983,7560,8653,1622,2696,1519,1751,3112,3610,8014,8269,9242,3501,6057,654,7187,628,2454,3477,5480,8035,9371\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 973\n", + "Returned IDs: \n", + "Ground Truth: 7701,547,4490,6813,6925,9863,4431,6647,9223,9063,8892,6468,5031,8879,3556,6324,1967,3477,2927,3426,1944,4549,320,3548,6046,9055,4417,1831,9576,3963\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 974\n", + "Returned IDs: 5187\n", + "Ground Truth: 1175,8087,8891,3407,3514,1640,5986,6644,8644,36,8312,3775,825,2899,7125,1062,3318,7634,9686,8167,7715,355,5848,6110,5390,6004,539,7309,9335,1970\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 975\n", + "Returned IDs: \n", + "Ground Truth: 1988,8653,1996,3757,203,1551,1168,4357,9576,5711,4924,2753,2850,4132,1548,5324,1650,6608,949,8953,5069,5840,5662,9345,7742,2785,5480,652,8275,1881\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 976\n", + "Returned IDs: \n", + "Ground Truth: 6313,6427,5972,6197,1802,5920,3501,7196,4334,8557,3148,227,4734,5192,7213,67,4215,7637,2943,920,4586,3123,7929,9935,483,1540,7091,2608,7143,2696\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 977\n", + "Returned IDs: \n", + "Ground Truth: 3764,7248,8498,7629,1569,5575,4380,4293,788,7091,9777,5683,7432,2943,9354,1379,1562,3688,68,845,6670,3033,2785,5589,8346,1344,5330,1586,5278,1818\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 978\n", + "Returned IDs: \n", + "Ground Truth: 6558,3859,6103,642,8025,5385,2090,2837,2370,4430,3671,6124,9033,6967,6232,8057,2599,2712,4390,4109,2862,1003,8038,6487,1996,9671,8313,8117,3717,5219\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 979\n", + "Returned IDs: \n", + "Ground Truth: 9089,833,9356,9335,3092,8137,441,7600,6029,1227,6539,3993,6697,1177,4269,5390,4271,8401,6649,6644,1174,6725,5478,7691,7309,9130,36,7199,7634,825\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 980\n", + "Returned IDs: 7757,2541,6475\n", + "Ground Truth: 377,5649,4687,3932,9595,6702,3985,338,4466,989,1642,2435,1126,1167,7226,1965,7757,7146,1273,420,9172,1484,2541,3391,6507,8618,398,2505,5444,1592\n", + "Recall: 0.0667\n", + "\n", + "Query ID: 981\n", + "Returned IDs: \n", + "Ground Truth: 7497,9684,9499,1840,5230,8340,483,5070,6129,1619,7432,5200,4061,5846,3385,6764,4690,7143,2603,5586,9935,2524,8056,8319,548,1389,2,9095,6551,3154\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 982\n", + "Returned IDs: \n", + "Ground Truth: 4586,7432,7076,3148,5622,2516,6919,9711,6378,4217,4855,4216,1802,9446,7466,5972,437,7193,8428,9684,7070,8912,3501,8518,1443,7213,3651,8481,4127,2967\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 983\n", + "Returned IDs: \n", + "Ground Truth: 9872,5808,8892,1698,278,8156,8614,8638,1359,9063,7522,9492,7985,2521,5679,2655,1619,920,5917,9499,9861,7364,954,1792,9724,7499,3465,547,483,720\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 984\n", + "Returned IDs: \n", + "Ground Truth: 8129,798,1222,3388,9796,8060,6794,1286,9448,3900,2530,9900,624,2785,4905,4033,4924,8042,4854,8498,5577,6817,5443,1654,2078,3665,5711,4141,9415,5650\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 985\n", + "Returned IDs: \n", + "Ground Truth: 5925,7452,5359,2642,5074,1861,2974,4357,4821,2631,6334,2472,4061,3118,2078,9369,1700,7625,4383,7217,8144,3999,3320,2021,5776,3717,1885,1593,9907,4400\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 986\n", + "Returned IDs: \n", + "Ground Truth: 9579,2773,5787,5404,7865,2896,6635,7141,6557,9093,6615,5464,2567,7217,155,5994,254,3008,3610,2753,7,7031,1863,6608,290,4263,5403,3474,7943,8175\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 987\n", + "Returned IDs: \n", + "Ground Truth: 8504,858,3153,8319,476,9410,1647,8648,3526,7682,1389,4061,3033,4218,3578,1830,1122,707,1253,2507,8455,9365,4690,4643,4400,6563,1619,9660,7949,7605\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 988\n", + "Returned IDs: \n", + "Ground Truth: 5033,3346,3058,9645,876,9915,7186,7227,8948,258,9854,915,1520,5096,1741,6536,5165,1298,1189,4699,4596,6850,3994,9957,1284,6261,751,8411,2217,8405\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 989\n", + "Returned IDs: 2409\n", + "Ground Truth: 1096,2451,7131,8210,4236,2897,6063,5992,5830,4455,7432,3501,7213,2409,1292,1167,3840,397,4991,2036,420,8040,263,1270,5525,3715,4722,9838,4293,8892\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 990\n", + "Returned IDs: \n", + "Ground Truth: 4562,9093,7020,7027,6741,5069,6825,3463,665,1278,1105,6648,7452,1905,2837,5363,9005,8859,692,2575,8765,233,1827,5956,8393,4269,4061,7511,6933,9133\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 991\n", + "Returned IDs: \n", + "Ground Truth: 4137,4532,4483,6817,311,5671,5946,6372,8060,8342,7097,8244,9308,6646,761,2824,4157,3665,4587,3682,3832,6255,6794,8177,624,6759,2188,1024,4539,4416\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 992\n", + "Returned IDs: \n", + "Ground Truth: 8948,5033,3058,1520,9854,3994,9645,915,7227,2998,6418,258,751,1284,4596,9915,2196,1298,542,3346,9615,6850,9374,7046,6468,5096,370,1490,5165,453\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 993\n", + "Returned IDs: \n", + "Ground Truth: 2642,6363,9451,8974,5666,7358,6073,7217,2567,1861,2907,4497,7016,9770,247,8649,3444,9395,6307,3474,3929,4920,3008,5885,5311,6608,7560,1003,6521,3749\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 994\n", + "Returned IDs: \n", + "Ground Truth: 6165,1518,290,7069,3426,3826,7542,7217,2174,7816,8716,9148,6511,6307,8184,8709,7033,658,5303,1584,6658,7017,5289,2196,3963,8120,6662,9781,493,9816\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 995\n", + "Returned IDs: \n", + "Ground Truth: 7522,2260,5917,2521,9914,2831,5679,5016,1964,7477,4688,5085,1792,1619,1278,6004,4315,3246,8093,5808,2404,9822,1817,4887,3648,7381,8156,7742,2716,9662\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 996\n", + "Returned IDs: 9094\n", + "Ground Truth: 9094,1096,8235,2036,5525,7213,1225,1671,1462,4400,2282,7495,6546,7005,4110,2929,7952,4991,5064,5047,7342,7782,3586,3752,2771,5247,9699,4855,397,2219\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 997\n", + "Returned IDs: \n", + "Ground Truth: 9945,1866,5992,1438,6197,3501,4169,533,4256,7022,2943,6253,5945,7143,312,6313,3840,5589,8275,5480,3262,1959,4460,7831,8858,4824,7929,8198,2090,4400\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 998\n", + "Returned IDs: 3120\n", + "Ground Truth: 3120,1105,9914,308,6712,6061,770,6004,1728,4887,2200,1885,7628,9297,8156,9744,1771,4234,3857,9006,3817,310,1871,6219,6354,2507,6573,8226,9215,2329\n", + "Recall: 0.0333\n", + "\n", + "Query ID: 999\n", + "Returned IDs: \n", + "Ground Truth: 6363,8445,36,6737,9145,7125,2017,4479,7194,5986,3970,5311,8569,7667,9451,1340,8087,3757,3818,4009,6557,404,1062,4521,2009,8398,1433,6608,825,2896\n", + "Recall: 0.0000\n", + "\n", + "Total query IDs with recall < 1.0: 1000\n", + "IDs: [np.int64(0), np.int64(1), np.int64(2), np.int64(3), np.int64(4), np.int64(5), np.int64(6), np.int64(7), np.int64(8), np.int64(9), np.int64(10), np.int64(11), np.int64(12), np.int64(13), np.int64(14), np.int64(15), np.int64(16), np.int64(17), np.int64(18), np.int64(19), np.int64(20), np.int64(21), np.int64(22), np.int64(23), np.int64(24), np.int64(25), np.int64(26), np.int64(27), np.int64(28), np.int64(29), np.int64(30), np.int64(31), np.int64(32), np.int64(33), np.int64(34), np.int64(35), np.int64(36), np.int64(37), np.int64(38), np.int64(39), np.int64(40), np.int64(41), np.int64(42), np.int64(43), np.int64(44), np.int64(45), np.int64(46), np.int64(47), np.int64(48), np.int64(49), np.int64(50), np.int64(51), np.int64(52), np.int64(53), np.int64(54), np.int64(55), np.int64(56), np.int64(57), np.int64(58), np.int64(59), np.int64(60), np.int64(61), np.int64(62), np.int64(63), np.int64(64), np.int64(65), np.int64(66), np.int64(67), np.int64(68), np.int64(69), np.int64(70), np.int64(71), np.int64(72), np.int64(73), np.int64(74), np.int64(75), np.int64(76), np.int64(77), np.int64(78), np.int64(79), np.int64(80), np.int64(81), np.int64(82), np.int64(83), np.int64(84), np.int64(85), np.int64(86), np.int64(87), np.int64(88), np.int64(89), np.int64(90), np.int64(91), np.int64(92), np.int64(93), np.int64(94), np.int64(95), np.int64(96), np.int64(97), np.int64(98), np.int64(99), np.int64(100), np.int64(101), np.int64(102), np.int64(103), np.int64(104), np.int64(105), np.int64(106), np.int64(107), np.int64(108), np.int64(109), np.int64(110), np.int64(111), np.int64(112), np.int64(113), np.int64(114), np.int64(115), np.int64(116), np.int64(117), np.int64(118), np.int64(119), np.int64(120), np.int64(121), np.int64(122), np.int64(123), np.int64(124), np.int64(125), np.int64(126), np.int64(127), np.int64(128), np.int64(129), np.int64(130), np.int64(131), np.int64(132), np.int64(133), np.int64(134), np.int64(135), np.int64(136), np.int64(137), np.int64(138), np.int64(139), np.int64(140), np.int64(141), np.int64(142), np.int64(143), np.int64(144), np.int64(145), np.int64(146), np.int64(147), np.int64(148), np.int64(149), np.int64(150), np.int64(151), np.int64(152), np.int64(153), np.int64(154), np.int64(155), np.int64(156), np.int64(157), np.int64(158), np.int64(159), np.int64(160), np.int64(161), np.int64(162), np.int64(163), np.int64(164), np.int64(165), np.int64(166), np.int64(167), np.int64(168), np.int64(169), np.int64(170), np.int64(171), np.int64(172), np.int64(173), np.int64(174), np.int64(175), np.int64(176), np.int64(177), np.int64(178), np.int64(179), np.int64(180), np.int64(181), np.int64(182), np.int64(183), np.int64(184), np.int64(185), np.int64(186), np.int64(187), np.int64(188), np.int64(189), np.int64(190), np.int64(191), np.int64(192), np.int64(193), np.int64(194), np.int64(195), np.int64(196), np.int64(197), np.int64(198), np.int64(199), np.int64(200), np.int64(201), np.int64(202), np.int64(203), np.int64(204), np.int64(205), np.int64(206), np.int64(207), np.int64(208), np.int64(209), np.int64(210), np.int64(211), np.int64(212), np.int64(213), np.int64(214), np.int64(215), np.int64(216), np.int64(217), np.int64(218), np.int64(219), np.int64(220), np.int64(221), np.int64(222), np.int64(223), np.int64(224), np.int64(225), np.int64(226), np.int64(227), np.int64(228), np.int64(229), np.int64(230), np.int64(231), np.int64(232), np.int64(233), np.int64(234), np.int64(235), np.int64(236), np.int64(237), np.int64(238), np.int64(239), np.int64(240), np.int64(241), np.int64(242), np.int64(243), np.int64(244), np.int64(245), np.int64(246), np.int64(247), np.int64(248), np.int64(249), np.int64(250), np.int64(251), np.int64(252), np.int64(253), np.int64(254), np.int64(255), np.int64(256), np.int64(257), np.int64(258), np.int64(259), np.int64(260), np.int64(261), np.int64(262), np.int64(263), np.int64(264), np.int64(265), np.int64(266), np.int64(267), np.int64(268), np.int64(269), np.int64(270), np.int64(271), np.int64(272), np.int64(273), np.int64(274), np.int64(275), np.int64(276), np.int64(277), np.int64(278), np.int64(279), np.int64(280), np.int64(281), np.int64(282), np.int64(283), np.int64(284), np.int64(285), np.int64(286), np.int64(287), np.int64(288), np.int64(289), np.int64(290), np.int64(291), np.int64(292), np.int64(293), np.int64(294), np.int64(295), np.int64(296), np.int64(297), np.int64(298), np.int64(299), np.int64(300), np.int64(301), np.int64(302), np.int64(303), np.int64(304), np.int64(305), np.int64(306), np.int64(307), np.int64(308), np.int64(309), np.int64(310), np.int64(311), np.int64(312), np.int64(313), np.int64(314), np.int64(315), np.int64(316), np.int64(317), np.int64(318), np.int64(319), np.int64(320), np.int64(321), np.int64(322), np.int64(323), np.int64(324), np.int64(325), np.int64(326), np.int64(327), np.int64(328), np.int64(329), np.int64(330), np.int64(331), np.int64(332), np.int64(333), np.int64(334), np.int64(335), np.int64(336), np.int64(337), np.int64(338), np.int64(339), np.int64(340), np.int64(341), np.int64(342), np.int64(343), np.int64(344), np.int64(345), np.int64(346), np.int64(347), np.int64(348), np.int64(349), np.int64(350), np.int64(351), np.int64(352), np.int64(353), np.int64(354), np.int64(355), np.int64(356), np.int64(357), np.int64(358), np.int64(359), np.int64(360), np.int64(361), np.int64(362), np.int64(363), np.int64(364), np.int64(365), np.int64(366), np.int64(367), np.int64(368), np.int64(369), np.int64(370), np.int64(371), np.int64(372), np.int64(373), np.int64(374), np.int64(375), np.int64(376), np.int64(377), np.int64(378), np.int64(379), np.int64(380), np.int64(381), np.int64(382), np.int64(383), np.int64(384), np.int64(385), np.int64(386), np.int64(387), np.int64(388), np.int64(389), np.int64(390), np.int64(391), np.int64(392), np.int64(393), np.int64(394), np.int64(395), np.int64(396), np.int64(397), np.int64(398), np.int64(399), np.int64(400), np.int64(401), np.int64(402), np.int64(403), np.int64(404), np.int64(405), np.int64(406), np.int64(407), np.int64(408), np.int64(409), np.int64(410), np.int64(411), np.int64(412), np.int64(413), np.int64(414), np.int64(415), np.int64(416), np.int64(417), np.int64(418), np.int64(419), np.int64(420), np.int64(421), np.int64(422), np.int64(423), np.int64(424), np.int64(425), np.int64(426), np.int64(427), np.int64(428), np.int64(429), np.int64(430), np.int64(431), np.int64(432), np.int64(433), np.int64(434), np.int64(435), np.int64(436), np.int64(437), np.int64(438), np.int64(439), np.int64(440), np.int64(441), np.int64(442), np.int64(443), np.int64(444), np.int64(445), np.int64(446), np.int64(447), np.int64(448), np.int64(449), np.int64(450), np.int64(451), np.int64(452), np.int64(453), np.int64(454), np.int64(455), np.int64(456), np.int64(457), np.int64(458), np.int64(459), np.int64(460), np.int64(461), np.int64(462), np.int64(463), np.int64(464), np.int64(465), np.int64(466), np.int64(467), np.int64(468), np.int64(469), np.int64(470), np.int64(471), np.int64(472), np.int64(473), np.int64(474), np.int64(475), np.int64(476), np.int64(477), np.int64(478), np.int64(479), np.int64(480), np.int64(481), np.int64(482), np.int64(483), np.int64(484), np.int64(485), np.int64(486), np.int64(487), np.int64(488), np.int64(489), np.int64(490), np.int64(491), np.int64(492), np.int64(493), np.int64(494), np.int64(495), np.int64(496), np.int64(497), np.int64(498), np.int64(499), np.int64(500), np.int64(501), np.int64(502), np.int64(503), np.int64(504), np.int64(505), np.int64(506), np.int64(507), np.int64(508), np.int64(509), np.int64(510), np.int64(511), np.int64(512), np.int64(513), np.int64(514), np.int64(515), np.int64(516), np.int64(517), np.int64(518), np.int64(519), np.int64(520), np.int64(521), np.int64(522), np.int64(523), np.int64(524), np.int64(525), np.int64(526), np.int64(527), np.int64(528), np.int64(529), np.int64(530), np.int64(531), np.int64(532), np.int64(533), np.int64(534), np.int64(535), np.int64(536), np.int64(537), np.int64(538), np.int64(539), np.int64(540), np.int64(541), np.int64(542), np.int64(543), np.int64(544), np.int64(545), np.int64(546), np.int64(547), np.int64(548), np.int64(549), np.int64(550), np.int64(551), np.int64(552), np.int64(553), np.int64(554), np.int64(555), np.int64(556), np.int64(557), np.int64(558), np.int64(559), np.int64(560), np.int64(561), np.int64(562), np.int64(563), np.int64(564), np.int64(565), np.int64(566), np.int64(567), np.int64(568), np.int64(569), np.int64(570), np.int64(571), np.int64(572), np.int64(573), np.int64(574), np.int64(575), np.int64(576), np.int64(577), np.int64(578), np.int64(579), np.int64(580), np.int64(581), np.int64(582), np.int64(583), np.int64(584), np.int64(585), np.int64(586), np.int64(587), np.int64(588), np.int64(589), np.int64(590), np.int64(591), np.int64(592), np.int64(593), np.int64(594), np.int64(595), np.int64(596), np.int64(597), np.int64(598), np.int64(599), np.int64(600), np.int64(601), np.int64(602), np.int64(603), np.int64(604), np.int64(605), np.int64(606), np.int64(607), np.int64(608), np.int64(609), np.int64(610), np.int64(611), np.int64(612), np.int64(613), np.int64(614), np.int64(615), np.int64(616), np.int64(617), np.int64(618), np.int64(619), np.int64(620), np.int64(621), np.int64(622), np.int64(623), np.int64(624), np.int64(625), np.int64(626), np.int64(627), np.int64(628), np.int64(629), np.int64(630), np.int64(631), np.int64(632), np.int64(633), np.int64(634), np.int64(635), np.int64(636), np.int64(637), np.int64(638), np.int64(639), np.int64(640), np.int64(641), np.int64(642), np.int64(643), np.int64(644), np.int64(645), np.int64(646), np.int64(647), np.int64(648), np.int64(649), np.int64(650), np.int64(651), np.int64(652), np.int64(653), np.int64(654), np.int64(655), np.int64(656), np.int64(657), np.int64(658), np.int64(659), np.int64(660), np.int64(661), np.int64(662), np.int64(663), np.int64(664), np.int64(665), np.int64(666), np.int64(667), np.int64(668), np.int64(669), np.int64(670), np.int64(671), np.int64(672), np.int64(673), np.int64(674), np.int64(675), np.int64(676), np.int64(677), np.int64(678), np.int64(679), np.int64(680), np.int64(681), np.int64(682), np.int64(683), np.int64(684), np.int64(685), np.int64(686), np.int64(687), np.int64(688), np.int64(689), np.int64(690), np.int64(691), np.int64(692), np.int64(693), np.int64(694), np.int64(695), np.int64(696), np.int64(697), np.int64(698), np.int64(699), np.int64(700), np.int64(701), np.int64(702), np.int64(703), np.int64(704), np.int64(705), np.int64(706), np.int64(707), np.int64(708), np.int64(709), np.int64(710), np.int64(711), np.int64(712), np.int64(713), np.int64(714), np.int64(715), np.int64(716), np.int64(717), np.int64(718), np.int64(719), np.int64(720), np.int64(721), np.int64(722), np.int64(723), np.int64(724), np.int64(725), np.int64(726), np.int64(727), np.int64(728), np.int64(729), np.int64(730), np.int64(731), np.int64(732), np.int64(733), np.int64(734), np.int64(735), np.int64(736), np.int64(737), np.int64(738), np.int64(739), np.int64(740), np.int64(741), np.int64(742), np.int64(743), np.int64(744), np.int64(745), np.int64(746), np.int64(747), np.int64(748), np.int64(749), np.int64(750), np.int64(751), np.int64(752), np.int64(753), np.int64(754), np.int64(755), np.int64(756), np.int64(757), np.int64(758), np.int64(759), np.int64(760), np.int64(761), np.int64(762), np.int64(763), np.int64(764), np.int64(765), np.int64(766), np.int64(767), np.int64(768), np.int64(769), np.int64(770), np.int64(771), np.int64(772), np.int64(773), np.int64(774), np.int64(775), np.int64(776), np.int64(777), np.int64(778), np.int64(779), np.int64(780), np.int64(781), np.int64(782), np.int64(783), np.int64(784), np.int64(785), np.int64(786), np.int64(787), np.int64(788), np.int64(789), np.int64(790), np.int64(791), np.int64(792), np.int64(793), np.int64(794), np.int64(795), np.int64(796), np.int64(797), np.int64(798), np.int64(799), np.int64(800), np.int64(801), np.int64(802), np.int64(803), np.int64(804), np.int64(805), np.int64(806), np.int64(807), np.int64(808), np.int64(809), np.int64(810), np.int64(811), np.int64(812), np.int64(813), np.int64(814), np.int64(815), np.int64(816), np.int64(817), np.int64(818), np.int64(819), np.int64(820), np.int64(821), np.int64(822), np.int64(823), np.int64(824), np.int64(825), np.int64(826), np.int64(827), np.int64(828), np.int64(829), np.int64(830), np.int64(831), np.int64(832), np.int64(833), np.int64(834), np.int64(835), np.int64(836), np.int64(837), np.int64(838), np.int64(839), np.int64(840), np.int64(841), np.int64(842), np.int64(843), np.int64(844), np.int64(845), np.int64(846), np.int64(847), np.int64(848), np.int64(849), np.int64(850), np.int64(851), np.int64(852), np.int64(853), np.int64(854), np.int64(855), np.int64(856), np.int64(857), np.int64(858), np.int64(859), np.int64(860), np.int64(861), np.int64(862), np.int64(863), np.int64(864), np.int64(865), np.int64(866), np.int64(867), np.int64(868), np.int64(869), np.int64(870), np.int64(871), np.int64(872), np.int64(873), np.int64(874), np.int64(875), np.int64(876), np.int64(877), np.int64(878), np.int64(879), np.int64(880), np.int64(881), np.int64(882), np.int64(883), np.int64(884), np.int64(885), np.int64(886), np.int64(887), np.int64(888), np.int64(889), np.int64(890), np.int64(891), np.int64(892), np.int64(893), np.int64(894), np.int64(895), np.int64(896), np.int64(897), np.int64(898), np.int64(899), np.int64(900), np.int64(901), np.int64(902), np.int64(903), np.int64(904), np.int64(905), np.int64(906), np.int64(907), np.int64(908), np.int64(909), np.int64(910), np.int64(911), np.int64(912), np.int64(913), np.int64(914), np.int64(915), np.int64(916), np.int64(917), np.int64(918), np.int64(919), np.int64(920), np.int64(921), np.int64(922), np.int64(923), np.int64(924), np.int64(925), np.int64(926), np.int64(927), np.int64(928), np.int64(929), np.int64(930), np.int64(931), np.int64(932), np.int64(933), np.int64(934), np.int64(935), np.int64(936), np.int64(937), np.int64(938), np.int64(939), np.int64(940), np.int64(941), np.int64(942), np.int64(943), np.int64(944), np.int64(945), np.int64(946), np.int64(947), np.int64(948), np.int64(949), np.int64(950), np.int64(951), np.int64(952), np.int64(953), np.int64(954), np.int64(955), np.int64(956), np.int64(957), np.int64(958), np.int64(959), np.int64(960), np.int64(961), np.int64(962), np.int64(963), np.int64(964), np.int64(965), np.int64(966), np.int64(967), np.int64(968), np.int64(969), np.int64(970), np.int64(971), np.int64(972), np.int64(973), np.int64(974), np.int64(975), np.int64(976), np.int64(977), np.int64(978), np.int64(979), np.int64(980), np.int64(981), np.int64(982), np.int64(983), np.int64(984), np.int64(985), np.int64(986), np.int64(987), np.int64(988), np.int64(989), np.int64(990), np.int64(991), np.int64(992), np.int64(993), np.int64(994), np.int64(995), np.int64(996), np.int64(997), np.int64(998), np.int64(999)]\n", + "Average Recall across all 1000 queries: 0.0029\n" + ] + } + ], + "source": [ + "#For querying (int filter)- lte filter\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "#inputs\n", + "int_rate_query = \"1p\" # change to 1p, 50p, 80p, 99p\n", + "top_k = 30\n", + "\n", + "# Mode 1: Single query id\n", + "# Mode 2: Find all query ids with recall < 1\n", + "mode = 2\n", + "query_id = 457 # only used in mode 1\n", + "\n", + "# Map int_rate_query to lte threshold\n", + "lte_map = {\n", + " \"1p\": 9999,\n", + " \"50p\": 499999,\n", + " \"80p\": 799999,\n", + " \"99p\": 989999,\n", + "}\n", + "lte_threshold = lte_map[int_rate_query]\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "lte_dataset_path = os.path.join(\"/home/debian/latest_VDB/VectorDBBench/vectordataset_lte\", dataset_folder)\n", + "test_parquet_path = os.path.join(dataset_path, \"test.parquet\")\n", + "gt_parquet_path = os.path.join(lte_dataset_path, f\"neighbors_int_{int_rate_query}.parquet\")\n", + "\n", + "# Read parquet files\n", + "test_df = pq.ParquetFile(test_parquet_path).read().to_pandas()\n", + "gt_df = pq.ParquetFile(gt_parquet_path).read().to_pandas()\n", + "\n", + "def run_query(query_id):\n", + " query_row = test_df[test_df[\"id\"] == query_id]\n", + " if query_row.empty:\n", + " print(f\"Query ID {query_id} not found in test.parquet\")\n", + " return None\n", + "\n", + " query_vector = query_row[\"emb\"].values[0]\n", + "\n", + " gt_row = gt_df[gt_df[\"id\"] == query_id]\n", + " if gt_row.empty:\n", + " print(f\"Query ID {query_id} not found in ground truth file\")\n", + " return None\n", + "\n", + " ground_truth = gt_row[\"neighbors_id\"].values[0][:top_k]\n", + "\n", + " results = index.query(\n", + " vector=query_vector,\n", + " top_k=top_k,\n", + " filter=[{\"id\": {\"$lte\": lte_threshold}}],\n", + " include_vectors=False,\n", + " )\n", + " returned_ids = [int(r[\"id\"]) for r in results]\n", + "\n", + " intersection = len(set(returned_ids) & set(ground_truth))\n", + " recall = intersection / len(ground_truth)\n", + "\n", + " return returned_ids, ground_truth, recall\n", + "\n", + "\n", + "if mode == 1:\n", + " result = run_query(query_id)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " print(f\"Query ID: {query_id}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\")\n", + "\n", + "elif mode == 2:\n", + " print(f\"Finding all query IDs with recall < 1.0 ...\\n\")\n", + " low_recall_ids = []\n", + " all_recalls = []\n", + "\n", + " for qid in test_df[\"id\"].values:\n", + " result = run_query(qid)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " all_recalls.append(recall)\n", + " if recall < 1.0:\n", + " low_recall_ids.append(qid)\n", + " print(f\"Query ID: {qid}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\\n\")\n", + "\n", + " print(f\"Total query IDs with recall < 1.0: {len(low_recall_ids)}\")\n", + " print(f\"IDs: {low_recall_ids}\")\n", + " print(f\"Average Recall across all {len(all_recalls)} queries: {sum(all_recalls)/len(all_recalls):.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_labelfilter_0503_1_adup6',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 10000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Deleted vectors with id >= 10000\n", + "990000 vectors deleted\n" + ] + } + ], + "source": [ + "#For deleting (int filter) - lte\n", + "int_rate_delete = \"99p\" # change to 1p, 20p, 50p, 99p\n", + "\n", + "lte_map = {\n", + " \"1p\": 990000,\n", + " \"20p\": 800000,\n", + " \"50p\": 500000,\n", + " \"99p\": 10000,\n", + "}\n", + "lte_threshold = lte_map[int_rate_delete]\n", + "\n", + "result = index.delete_with_filter([{\"id\": {\"$gte\": lte_threshold}}])\n", + "print(f\"Deleted vectors with id >= {lte_threshold}\")\n", + "print(result)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_labelfilter_0503_1_adup6',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 10000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Upserted 991 vectors, total so far: 991\n", + "Upserted 993 vectors, total so far: 1984\n", + "Upserted 989 vectors, total so far: 2973\n", + "Upserted 991 vectors, total so far: 3964\n", + "Upserted 992 vectors, total so far: 4956\n", + "Upserted 991 vectors, total so far: 5947\n", + "Upserted 990 vectors, total so far: 6937\n", + "Upserted 986 vectors, total so far: 7923\n", + "Upserted 996 vectors, total so far: 8919\n", + "Upserted 991 vectors, total so far: 9910\n", + "Upserted 993 vectors, total so far: 10903\n", + "Upserted 992 vectors, total so far: 11895\n", + "Upserted 989 vectors, total so far: 12884\n", + "Upserted 995 vectors, total so far: 13879\n", + "Upserted 988 vectors, total so far: 14867\n", + "Upserted 992 vectors, total so far: 15859\n", + "Upserted 988 vectors, total so far: 16847\n", + "Upserted 988 vectors, total so far: 17835\n", + "Upserted 996 vectors, total so far: 18831\n", + "Upserted 993 vectors, total so far: 19824\n", + "Upserted 993 vectors, total so far: 20817\n", + "Upserted 989 vectors, total so far: 21806\n", + "Upserted 993 vectors, total so far: 22799\n", + "Upserted 982 vectors, total so far: 23781\n", + "Upserted 991 vectors, total so far: 24772\n", + "Upserted 996 vectors, total so far: 25768\n", + "Upserted 993 vectors, total so far: 26761\n", + "Upserted 994 vectors, total so far: 27755\n", + "Upserted 993 vectors, total so far: 28748\n", + "Upserted 990 vectors, total so far: 29738\n", + "Upserted 989 vectors, total so far: 30727\n", + "Upserted 993 vectors, total so far: 31720\n", + "Upserted 987 vectors, total so far: 32707\n", + "Upserted 989 vectors, total so far: 33696\n", + "Upserted 987 vectors, total so far: 34683\n", + "Upserted 992 vectors, total so far: 35675\n", + "Upserted 988 vectors, total so far: 36663\n", + "Upserted 991 vectors, total so far: 37654\n", + "Upserted 990 vectors, total so far: 38644\n", + "Upserted 990 vectors, total so far: 39634\n", + "Upserted 993 vectors, total so far: 40627\n", + "Upserted 986 vectors, total so far: 41613\n", + "Upserted 991 vectors, total so far: 42604\n", + "Upserted 991 vectors, total so far: 43595\n", + "Upserted 993 vectors, total so far: 44588\n", + "Upserted 989 vectors, total so far: 45577\n", + "Upserted 990 vectors, total so far: 46567\n", + "Upserted 990 vectors, total so far: 47557\n", + "Upserted 993 vectors, total so far: 48550\n", + "Upserted 989 vectors, total so far: 49539\n", + "Upserted 987 vectors, total so far: 50526\n", + "Upserted 985 vectors, total so far: 51511\n", + "Upserted 995 vectors, total so far: 52506\n", + "Upserted 995 vectors, total so far: 53501\n", + "Upserted 992 vectors, total so far: 54493\n", + "Upserted 989 vectors, total so far: 55482\n", + "Upserted 989 vectors, total so far: 56471\n", + "Upserted 989 vectors, total so far: 57460\n", + "Upserted 988 vectors, total so far: 58448\n", + "Upserted 990 vectors, total so far: 59438\n", + "Upserted 988 vectors, total so far: 60426\n", + "Upserted 995 vectors, total so far: 61421\n", + "Upserted 989 vectors, total so far: 62410\n", + "Upserted 991 vectors, total so far: 63401\n", + "Upserted 990 vectors, total so far: 64391\n", + "Upserted 995 vectors, total so far: 65386\n", + "Upserted 990 vectors, total so far: 66376\n", + "Upserted 989 vectors, total so far: 67365\n", + "Upserted 990 vectors, total so far: 68355\n", + "Upserted 993 vectors, total so far: 69348\n", + "Upserted 995 vectors, total so far: 70343\n", + "Upserted 990 vectors, total so far: 71333\n", + "Upserted 989 vectors, total so far: 72322\n", + "Upserted 989 vectors, total so far: 73311\n", + "Upserted 994 vectors, total so far: 74305\n", + "Upserted 989 vectors, total so far: 75294\n", + "Upserted 989 vectors, total so far: 76283\n", + "Upserted 994 vectors, total so far: 77277\n", + "Upserted 991 vectors, total so far: 78268\n", + "Upserted 993 vectors, total so far: 79261\n", + "Upserted 996 vectors, total so far: 80257\n", + "Upserted 987 vectors, total so far: 81244\n", + "Upserted 990 vectors, total so far: 82234\n", + "Upserted 988 vectors, total so far: 83222\n", + "Upserted 988 vectors, total so far: 84210\n", + "Upserted 989 vectors, total so far: 85199\n", + "Upserted 987 vectors, total so far: 86186\n", + "Upserted 982 vectors, total so far: 87168\n", + "Upserted 990 vectors, total so far: 88158\n", + "Upserted 993 vectors, total so far: 89151\n", + "Upserted 982 vectors, total so far: 90133\n", + "Upserted 989 vectors, total so far: 91122\n", + "Upserted 988 vectors, total so far: 92110\n", + "Upserted 994 vectors, total so far: 93104\n", + "Upserted 985 vectors, total so far: 94089\n", + "Upserted 991 vectors, total so far: 95080\n", + "Upserted 986 vectors, total so far: 96066\n", + "Upserted 989 vectors, total so far: 97055\n", + "Upserted 988 vectors, total so far: 98043\n", + "Upserted 984 vectors, total so far: 99027\n", + "Upserted 990 vectors, total so far: 100017\n", + "Upserted 990 vectors, total so far: 101007\n", + "Upserted 995 vectors, total so far: 102002\n", + "Upserted 992 vectors, total so far: 102994\n", + "Upserted 993 vectors, total so far: 103987\n", + "Upserted 991 vectors, total so far: 104978\n", + "Upserted 991 vectors, total so far: 105969\n", + "Upserted 990 vectors, total so far: 106959\n", + "Upserted 990 vectors, total so far: 107949\n", + "Upserted 993 vectors, total so far: 108942\n", + "Upserted 992 vectors, total so far: 109934\n", + "Upserted 990 vectors, total so far: 110924\n", + "Upserted 988 vectors, total so far: 111912\n", + "Upserted 993 vectors, total so far: 112905\n", + "Upserted 994 vectors, total so far: 113899\n", + "Upserted 991 vectors, total so far: 114890\n", + "Upserted 993 vectors, total so far: 115883\n", + "Upserted 991 vectors, total so far: 116874\n", + "Upserted 990 vectors, total so far: 117864\n", + "Upserted 986 vectors, total so far: 118850\n", + "Upserted 988 vectors, total so far: 119838\n", + "Upserted 991 vectors, total so far: 120829\n", + "Upserted 989 vectors, total so far: 121818\n", + "Upserted 987 vectors, total so far: 122805\n", + "Upserted 996 vectors, total so far: 123801\n", + "Upserted 991 vectors, total so far: 124792\n", + "Upserted 994 vectors, total so far: 125786\n", + "Upserted 988 vectors, total so far: 126774\n", + "Upserted 988 vectors, total so far: 127762\n", + "Upserted 989 vectors, total so far: 128751\n", + "Upserted 994 vectors, total so far: 129745\n", + "Upserted 992 vectors, total so far: 130737\n", + "Upserted 992 vectors, total so far: 131729\n", + "Upserted 993 vectors, total so far: 132722\n", + "Upserted 993 vectors, total so far: 133715\n", + "Upserted 987 vectors, total so far: 134702\n", + "Upserted 989 vectors, total so far: 135691\n", + "Upserted 990 vectors, total so far: 136681\n", + "Upserted 990 vectors, total so far: 137671\n", + "Upserted 993 vectors, total so far: 138664\n", + "Upserted 991 vectors, total so far: 139655\n", + "Upserted 993 vectors, total so far: 140648\n", + "Upserted 990 vectors, total so far: 141638\n", + "Upserted 993 vectors, total so far: 142631\n", + "Upserted 983 vectors, total so far: 143614\n", + "Upserted 984 vectors, total so far: 144598\n", + "Upserted 986 vectors, total so far: 145584\n", + "Upserted 997 vectors, total so far: 146581\n", + "Upserted 987 vectors, total so far: 147568\n", + "Upserted 993 vectors, total so far: 148561\n", + "Upserted 990 vectors, total so far: 149551\n", + "Upserted 989 vectors, total so far: 150540\n", + "Upserted 989 vectors, total so far: 151529\n", + "Upserted 994 vectors, total so far: 152523\n", + "Upserted 981 vectors, total so far: 153504\n", + "Upserted 993 vectors, total so far: 154497\n", + "Upserted 994 vectors, total so far: 155491\n", + "Upserted 991 vectors, total so far: 156482\n", + "Upserted 990 vectors, total so far: 157472\n", + "Upserted 982 vectors, total so far: 158454\n", + "Upserted 989 vectors, total so far: 159443\n", + "Upserted 992 vectors, total so far: 160435\n", + "Upserted 996 vectors, total so far: 161431\n", + "Upserted 986 vectors, total so far: 162417\n", + "Upserted 991 vectors, total so far: 163408\n", + "Upserted 987 vectors, total so far: 164395\n", + "Upserted 990 vectors, total so far: 165385\n", + "Upserted 991 vectors, total so far: 166376\n", + "Upserted 989 vectors, total so far: 167365\n", + "Upserted 992 vectors, total so far: 168357\n", + "Upserted 988 vectors, total so far: 169345\n", + "Upserted 988 vectors, total so far: 170333\n", + "Upserted 988 vectors, total so far: 171321\n", + "Upserted 996 vectors, total so far: 172317\n", + "Upserted 992 vectors, total so far: 173309\n", + "Upserted 992 vectors, total so far: 174301\n", + "Upserted 989 vectors, total so far: 175290\n", + "Upserted 993 vectors, total so far: 176283\n", + "Upserted 992 vectors, total so far: 177275\n", + "Upserted 993 vectors, total so far: 178268\n", + "Upserted 991 vectors, total so far: 179259\n", + "Upserted 990 vectors, total so far: 180249\n", + "Upserted 992 vectors, total so far: 181241\n", + "Upserted 988 vectors, total so far: 182229\n", + "Upserted 986 vectors, total so far: 183215\n", + "Upserted 986 vectors, total so far: 184201\n", + "Upserted 993 vectors, total so far: 185194\n", + "Upserted 990 vectors, total so far: 186184\n", + "Upserted 989 vectors, total so far: 187173\n", + "Upserted 994 vectors, total so far: 188167\n", + "Upserted 985 vectors, total so far: 189152\n", + "Upserted 987 vectors, total so far: 190139\n", + "Upserted 993 vectors, total so far: 191132\n", + "Upserted 991 vectors, total so far: 192123\n", + "Upserted 992 vectors, total so far: 193115\n", + "Upserted 987 vectors, total so far: 194102\n", + "Upserted 987 vectors, total so far: 195089\n", + "Upserted 989 vectors, total so far: 196078\n", + "Upserted 988 vectors, total so far: 197066\n", + "Upserted 991 vectors, total so far: 198057\n", + "Upserted 988 vectors, total so far: 199045\n", + "Upserted 989 vectors, total so far: 200034\n", + "Upserted 993 vectors, total so far: 201027\n", + "Upserted 991 vectors, total so far: 202018\n", + "Upserted 984 vectors, total so far: 203002\n", + "Upserted 987 vectors, total so far: 203989\n", + "Upserted 998 vectors, total so far: 204987\n", + "Upserted 990 vectors, total so far: 205977\n", + "Upserted 992 vectors, total so far: 206969\n", + "Upserted 988 vectors, total so far: 207957\n", + "Upserted 990 vectors, total so far: 208947\n", + "Upserted 993 vectors, total so far: 209940\n", + "Upserted 986 vectors, total so far: 210926\n", + "Upserted 995 vectors, total so far: 211921\n", + "Upserted 989 vectors, total so far: 212910\n", + "Upserted 993 vectors, total so far: 213903\n", + "Upserted 993 vectors, total so far: 214896\n", + "Upserted 989 vectors, total so far: 215885\n", + "Upserted 993 vectors, total so far: 216878\n", + "Upserted 994 vectors, total so far: 217872\n", + "Upserted 992 vectors, total so far: 218864\n", + "Upserted 992 vectors, total so far: 219856\n", + "Upserted 987 vectors, total so far: 220843\n", + "Upserted 987 vectors, total so far: 221830\n", + "Upserted 990 vectors, total so far: 222820\n", + "Upserted 996 vectors, total so far: 223816\n", + "Upserted 987 vectors, total so far: 224803\n", + "Upserted 990 vectors, total so far: 225793\n", + "Upserted 990 vectors, total so far: 226783\n", + "Upserted 992 vectors, total so far: 227775\n", + "Upserted 991 vectors, total so far: 228766\n", + "Upserted 993 vectors, total so far: 229759\n", + "Upserted 991 vectors, total so far: 230750\n", + "Upserted 983 vectors, total so far: 231733\n", + "Upserted 984 vectors, total so far: 232717\n", + "Upserted 988 vectors, total so far: 233705\n", + "Upserted 992 vectors, total so far: 234697\n", + "Upserted 990 vectors, total so far: 235687\n", + "Upserted 992 vectors, total so far: 236679\n", + "Upserted 992 vectors, total so far: 237671\n", + "Upserted 989 vectors, total so far: 238660\n", + "Upserted 990 vectors, total so far: 239650\n", + "Upserted 991 vectors, total so far: 240641\n", + "Upserted 989 vectors, total so far: 241630\n", + "Upserted 989 vectors, total so far: 242619\n", + "Upserted 990 vectors, total so far: 243609\n", + "Upserted 992 vectors, total so far: 244601\n", + "Upserted 992 vectors, total so far: 245593\n", + "Upserted 988 vectors, total so far: 246581\n", + "Upserted 999 vectors, total so far: 247580\n", + "Upserted 993 vectors, total so far: 248573\n", + "Upserted 994 vectors, total so far: 249567\n", + "Upserted 990 vectors, total so far: 250557\n", + "Upserted 994 vectors, total so far: 251551\n", + "Upserted 997 vectors, total so far: 252548\n", + "Upserted 989 vectors, total so far: 253537\n", + "Upserted 986 vectors, total so far: 254523\n", + "Upserted 988 vectors, total so far: 255511\n", + "Upserted 990 vectors, total so far: 256501\n", + "Upserted 989 vectors, total so far: 257490\n", + "Upserted 991 vectors, total so far: 258481\n", + "Upserted 991 vectors, total so far: 259472\n", + "Upserted 986 vectors, total so far: 260458\n", + "Upserted 990 vectors, total so far: 261448\n", + "Upserted 992 vectors, total so far: 262440\n", + "Upserted 994 vectors, total so far: 263434\n", + "Upserted 989 vectors, total so far: 264423\n", + "Upserted 995 vectors, total so far: 265418\n", + "Upserted 990 vectors, total so far: 266408\n", + "Upserted 988 vectors, total so far: 267396\n", + "Upserted 992 vectors, total so far: 268388\n", + "Upserted 990 vectors, total so far: 269378\n", + "Upserted 991 vectors, total so far: 270369\n", + "Upserted 993 vectors, total so far: 271362\n", + "Upserted 981 vectors, total so far: 272343\n", + "Upserted 994 vectors, total so far: 273337\n", + "Upserted 988 vectors, total so far: 274325\n", + "Upserted 991 vectors, total so far: 275316\n", + "Upserted 987 vectors, total so far: 276303\n", + "Upserted 985 vectors, total so far: 277288\n", + "Upserted 992 vectors, total so far: 278280\n", + "Upserted 990 vectors, total so far: 279270\n", + "Upserted 994 vectors, total so far: 280264\n", + "Upserted 992 vectors, total so far: 281256\n", + "Upserted 985 vectors, total so far: 282241\n", + "Upserted 992 vectors, total so far: 283233\n", + "Upserted 994 vectors, total so far: 284227\n", + "Upserted 996 vectors, total so far: 285223\n", + "Upserted 992 vectors, total so far: 286215\n", + "Upserted 991 vectors, total so far: 287206\n", + "Upserted 990 vectors, total so far: 288196\n", + "Upserted 994 vectors, total so far: 289190\n", + "Upserted 983 vectors, total so far: 290173\n", + "Upserted 989 vectors, total so far: 291162\n", + "Upserted 986 vectors, total so far: 292148\n", + "Upserted 989 vectors, total so far: 293137\n", + "Upserted 989 vectors, total so far: 294126\n", + "Upserted 988 vectors, total so far: 295114\n", + "Upserted 992 vectors, total so far: 296106\n", + "Upserted 987 vectors, total so far: 297093\n", + "Upserted 989 vectors, total so far: 298082\n", + "Upserted 990 vectors, total so far: 299072\n", + "Upserted 988 vectors, total so far: 300060\n", + "Upserted 987 vectors, total so far: 301047\n", + "Upserted 993 vectors, total so far: 302040\n", + "Upserted 989 vectors, total so far: 303029\n", + "Upserted 990 vectors, total so far: 304019\n", + "Upserted 991 vectors, total so far: 305010\n", + "Upserted 987 vectors, total so far: 305997\n", + "Upserted 993 vectors, total so far: 306990\n", + "Upserted 993 vectors, total so far: 307983\n", + "Upserted 987 vectors, total so far: 308970\n", + "Upserted 994 vectors, total so far: 309964\n", + "Upserted 986 vectors, total so far: 310950\n", + "Upserted 989 vectors, total so far: 311939\n", + "Upserted 982 vectors, total so far: 312921\n", + "Upserted 988 vectors, total so far: 313909\n", + "Upserted 993 vectors, total so far: 314902\n", + "Upserted 991 vectors, total so far: 315893\n", + "Upserted 992 vectors, total so far: 316885\n", + "Upserted 987 vectors, total so far: 317872\n", + "Upserted 991 vectors, total so far: 318863\n", + "Upserted 987 vectors, total so far: 319850\n", + "Upserted 992 vectors, total so far: 320842\n", + "Upserted 996 vectors, total so far: 321838\n", + "Upserted 992 vectors, total so far: 322830\n", + "Upserted 991 vectors, total so far: 323821\n", + "Upserted 988 vectors, total so far: 324809\n", + "Upserted 993 vectors, total so far: 325802\n", + "Upserted 995 vectors, total so far: 326797\n", + "Upserted 994 vectors, total so far: 327791\n", + "Upserted 989 vectors, total so far: 328780\n", + "Upserted 990 vectors, total so far: 329770\n", + "Upserted 988 vectors, total so far: 330758\n", + "Upserted 993 vectors, total so far: 331751\n", + "Upserted 986 vectors, total so far: 332737\n", + "Upserted 986 vectors, total so far: 333723\n", + "Upserted 993 vectors, total so far: 334716\n", + "Upserted 983 vectors, total so far: 335699\n", + "Upserted 995 vectors, total so far: 336694\n", + "Upserted 986 vectors, total so far: 337680\n", + "Upserted 990 vectors, total so far: 338670\n", + "Upserted 990 vectors, total so far: 339660\n", + "Upserted 994 vectors, total so far: 340654\n", + "Upserted 991 vectors, total so far: 341645\n", + "Upserted 990 vectors, total so far: 342635\n", + "Upserted 991 vectors, total so far: 343626\n", + "Upserted 991 vectors, total so far: 344617\n", + "Upserted 992 vectors, total so far: 345609\n", + "Upserted 993 vectors, total so far: 346602\n", + "Upserted 986 vectors, total so far: 347588\n", + "Upserted 991 vectors, total so far: 348579\n", + "Upserted 990 vectors, total so far: 349569\n", + "Upserted 996 vectors, total so far: 350565\n", + "Upserted 992 vectors, total so far: 351557\n", + "Upserted 988 vectors, total so far: 352545\n", + "Upserted 994 vectors, total so far: 353539\n", + "Upserted 992 vectors, total so far: 354531\n", + "Upserted 995 vectors, total so far: 355526\n", + "Upserted 989 vectors, total so far: 356515\n", + "Upserted 988 vectors, total so far: 357503\n", + "Upserted 990 vectors, total so far: 358493\n", + "Upserted 994 vectors, total so far: 359487\n", + "Upserted 989 vectors, total so far: 360476\n", + "Upserted 984 vectors, total so far: 361460\n", + "Upserted 992 vectors, total so far: 362452\n", + "Upserted 985 vectors, total so far: 363437\n", + "Upserted 991 vectors, total so far: 364428\n", + "Upserted 990 vectors, total so far: 365418\n", + "Upserted 991 vectors, total so far: 366409\n", + "Upserted 994 vectors, total so far: 367403\n", + "Upserted 992 vectors, total so far: 368395\n", + "Upserted 990 vectors, total so far: 369385\n", + "Upserted 987 vectors, total so far: 370372\n", + "Upserted 990 vectors, total so far: 371362\n", + "Upserted 992 vectors, total so far: 372354\n", + "Upserted 989 vectors, total so far: 373343\n", + "Upserted 995 vectors, total so far: 374338\n", + "Upserted 987 vectors, total so far: 375325\n", + "Upserted 983 vectors, total so far: 376308\n", + "Upserted 988 vectors, total so far: 377296\n", + "Upserted 989 vectors, total so far: 378285\n", + "Upserted 992 vectors, total so far: 379277\n", + "Upserted 990 vectors, total so far: 380267\n", + "Upserted 993 vectors, total so far: 381260\n", + "Upserted 986 vectors, total so far: 382246\n", + "Upserted 986 vectors, total so far: 383232\n", + "Upserted 993 vectors, total so far: 384225\n", + "Upserted 985 vectors, total so far: 385210\n", + "Upserted 989 vectors, total so far: 386199\n", + "Upserted 988 vectors, total so far: 387187\n", + "Upserted 990 vectors, total so far: 388177\n", + "Upserted 994 vectors, total so far: 389171\n", + "Upserted 985 vectors, total so far: 390156\n", + "Upserted 986 vectors, total so far: 391142\n", + "Upserted 987 vectors, total so far: 392129\n", + "Upserted 985 vectors, total so far: 393114\n", + "Upserted 986 vectors, total so far: 394100\n", + "Upserted 990 vectors, total so far: 395090\n", + "Upserted 986 vectors, total so far: 396076\n", + "Upserted 988 vectors, total so far: 397064\n", + "Upserted 985 vectors, total so far: 398049\n", + "Upserted 987 vectors, total so far: 399036\n", + "Upserted 992 vectors, total so far: 400028\n", + "Upserted 990 vectors, total so far: 401018\n", + "Upserted 984 vectors, total so far: 402002\n", + "Upserted 991 vectors, total so far: 402993\n", + "Upserted 991 vectors, total so far: 403984\n", + "Upserted 990 vectors, total so far: 404974\n", + "Upserted 994 vectors, total so far: 405968\n", + "Upserted 991 vectors, total so far: 406959\n", + "Upserted 984 vectors, total so far: 407943\n", + "Upserted 993 vectors, total so far: 408936\n", + "Upserted 991 vectors, total so far: 409927\n", + "Upserted 986 vectors, total so far: 410913\n", + "Upserted 996 vectors, total so far: 411909\n", + "Upserted 990 vectors, total so far: 412899\n", + "Upserted 990 vectors, total so far: 413889\n", + "Upserted 985 vectors, total so far: 414874\n", + "Upserted 990 vectors, total so far: 415864\n", + "Upserted 988 vectors, total so far: 416852\n", + "Upserted 990 vectors, total so far: 417842\n", + "Upserted 992 vectors, total so far: 418834\n", + "Upserted 992 vectors, total so far: 419826\n", + "Upserted 990 vectors, total so far: 420816\n", + "Upserted 990 vectors, total so far: 421806\n", + "Upserted 992 vectors, total so far: 422798\n", + "Upserted 992 vectors, total so far: 423790\n", + "Upserted 989 vectors, total so far: 424779\n", + "Upserted 985 vectors, total so far: 425764\n", + "Upserted 991 vectors, total so far: 426755\n", + "Upserted 987 vectors, total so far: 427742\n", + "Upserted 986 vectors, total so far: 428728\n", + "Upserted 988 vectors, total so far: 429716\n", + "Upserted 990 vectors, total so far: 430706\n", + "Upserted 986 vectors, total so far: 431692\n", + "Upserted 991 vectors, total so far: 432683\n", + "Upserted 993 vectors, total so far: 433676\n", + "Upserted 989 vectors, total so far: 434665\n", + "Upserted 983 vectors, total so far: 435648\n", + "Upserted 992 vectors, total so far: 436640\n", + "Upserted 992 vectors, total so far: 437632\n", + "Upserted 993 vectors, total so far: 438625\n", + "Upserted 987 vectors, total so far: 439612\n", + "Upserted 986 vectors, total so far: 440598\n", + "Upserted 993 vectors, total so far: 441591\n", + "Upserted 989 vectors, total so far: 442580\n", + "Upserted 993 vectors, total so far: 443573\n", + "Upserted 993 vectors, total so far: 444566\n", + "Upserted 994 vectors, total so far: 445560\n", + "Upserted 992 vectors, total so far: 446552\n", + "Upserted 992 vectors, total so far: 447544\n", + "Upserted 986 vectors, total so far: 448530\n", + "Upserted 992 vectors, total so far: 449522\n", + "Upserted 990 vectors, total so far: 450512\n", + "Upserted 985 vectors, total so far: 451497\n", + "Upserted 996 vectors, total so far: 452493\n", + "Upserted 988 vectors, total so far: 453481\n", + "Upserted 986 vectors, total so far: 454467\n", + "Upserted 988 vectors, total so far: 455455\n", + "Upserted 990 vectors, total so far: 456445\n", + "Upserted 992 vectors, total so far: 457437\n", + "Upserted 989 vectors, total so far: 458426\n", + "Upserted 994 vectors, total so far: 459420\n", + "Upserted 995 vectors, total so far: 460415\n", + "Upserted 990 vectors, total so far: 461405\n", + "Upserted 989 vectors, total so far: 462394\n", + "Upserted 991 vectors, total so far: 463385\n", + "Upserted 988 vectors, total so far: 464373\n", + "Upserted 991 vectors, total so far: 465364\n", + "Upserted 988 vectors, total so far: 466352\n", + "Upserted 992 vectors, total so far: 467344\n", + "Upserted 993 vectors, total so far: 468337\n", + "Upserted 994 vectors, total so far: 469331\n", + "Upserted 988 vectors, total so far: 470319\n", + "Upserted 989 vectors, total so far: 471308\n", + "Upserted 993 vectors, total so far: 472301\n", + "Upserted 994 vectors, total so far: 473295\n", + "Upserted 994 vectors, total so far: 474289\n", + "Upserted 990 vectors, total so far: 475279\n", + "Upserted 983 vectors, total so far: 476262\n", + "Upserted 996 vectors, total so far: 477258\n", + "Upserted 987 vectors, total so far: 478245\n", + "Upserted 986 vectors, total so far: 479231\n", + "Upserted 992 vectors, total so far: 480223\n", + "Upserted 997 vectors, total so far: 481220\n", + "Upserted 989 vectors, total so far: 482209\n", + "Upserted 991 vectors, total so far: 483200\n", + "Upserted 992 vectors, total so far: 484192\n", + "Upserted 987 vectors, total so far: 485179\n", + "Upserted 991 vectors, total so far: 486170\n", + "Upserted 985 vectors, total so far: 487155\n", + "Upserted 990 vectors, total so far: 488145\n", + "Upserted 991 vectors, total so far: 489136\n", + "Upserted 992 vectors, total so far: 490128\n", + "Upserted 994 vectors, total so far: 491122\n", + "Upserted 993 vectors, total so far: 492115\n", + "Upserted 991 vectors, total so far: 493106\n", + "Upserted 989 vectors, total so far: 494095\n", + "Upserted 987 vectors, total so far: 495082\n", + "Upserted 985 vectors, total so far: 496067\n", + "Upserted 993 vectors, total so far: 497060\n", + "Upserted 989 vectors, total so far: 498049\n", + "Upserted 985 vectors, total so far: 499034\n", + "Upserted 991 vectors, total so far: 500025\n", + "Upserted 992 vectors, total so far: 501017\n", + "Upserted 991 vectors, total so far: 502008\n", + "Upserted 991 vectors, total so far: 502999\n", + "Upserted 990 vectors, total so far: 503989\n", + "Upserted 989 vectors, total so far: 504978\n", + "Upserted 994 vectors, total so far: 505972\n", + "Upserted 986 vectors, total so far: 506958\n", + "Upserted 995 vectors, total so far: 507953\n", + "Upserted 990 vectors, total so far: 508943\n", + "Upserted 991 vectors, total so far: 509934\n", + "Upserted 988 vectors, total so far: 510922\n", + "Upserted 993 vectors, total so far: 511915\n", + "Upserted 982 vectors, total so far: 512897\n", + "Upserted 989 vectors, total so far: 513886\n", + "Upserted 989 vectors, total so far: 514875\n", + "Upserted 988 vectors, total so far: 515863\n", + "Upserted 993 vectors, total so far: 516856\n", + "Upserted 991 vectors, total so far: 517847\n", + "Upserted 990 vectors, total so far: 518837\n", + "Upserted 989 vectors, total so far: 519826\n", + "Upserted 993 vectors, total so far: 520819\n", + "Upserted 991 vectors, total so far: 521810\n", + "Upserted 991 vectors, total so far: 522801\n", + "Upserted 991 vectors, total so far: 523792\n", + "Upserted 988 vectors, total so far: 524780\n", + "Upserted 991 vectors, total so far: 525771\n", + "Upserted 994 vectors, total so far: 526765\n", + "Upserted 990 vectors, total so far: 527755\n", + "Upserted 987 vectors, total so far: 528742\n", + "Upserted 994 vectors, total so far: 529736\n", + "Upserted 990 vectors, total so far: 530726\n", + "Upserted 989 vectors, total so far: 531715\n", + "Upserted 988 vectors, total so far: 532703\n", + "Upserted 988 vectors, total so far: 533691\n", + "Upserted 989 vectors, total so far: 534680\n", + "Upserted 988 vectors, total so far: 535668\n", + "Upserted 991 vectors, total so far: 536659\n", + "Upserted 992 vectors, total so far: 537651\n", + "Upserted 992 vectors, total so far: 538643\n", + "Upserted 984 vectors, total so far: 539627\n", + "Upserted 989 vectors, total so far: 540616\n", + "Upserted 993 vectors, total so far: 541609\n", + "Upserted 994 vectors, total so far: 542603\n", + "Upserted 988 vectors, total so far: 543591\n", + "Upserted 983 vectors, total so far: 544574\n", + "Upserted 985 vectors, total so far: 545559\n", + "Upserted 992 vectors, total so far: 546551\n", + "Upserted 989 vectors, total so far: 547540\n", + "Upserted 990 vectors, total so far: 548530\n", + "Upserted 988 vectors, total so far: 549518\n", + "Upserted 992 vectors, total so far: 550510\n", + "Upserted 984 vectors, total so far: 551494\n", + "Upserted 988 vectors, total so far: 552482\n", + "Upserted 986 vectors, total so far: 553468\n", + "Upserted 990 vectors, total so far: 554458\n", + "Upserted 994 vectors, total so far: 555452\n", + "Upserted 997 vectors, total so far: 556449\n", + "Upserted 988 vectors, total so far: 557437\n", + "Upserted 994 vectors, total so far: 558431\n", + "Upserted 982 vectors, total so far: 559413\n", + "Upserted 993 vectors, total so far: 560406\n", + "Upserted 988 vectors, total so far: 561394\n", + "Upserted 989 vectors, total so far: 562383\n", + "Upserted 987 vectors, total so far: 563370\n", + "Upserted 986 vectors, total so far: 564356\n", + "Upserted 990 vectors, total so far: 565346\n", + "Upserted 993 vectors, total so far: 566339\n", + "Upserted 992 vectors, total so far: 567331\n", + "Upserted 991 vectors, total so far: 568322\n", + "Upserted 985 vectors, total so far: 569307\n", + "Upserted 990 vectors, total so far: 570297\n", + "Upserted 989 vectors, total so far: 571286\n", + "Upserted 990 vectors, total so far: 572276\n", + "Upserted 995 vectors, total so far: 573271\n", + "Upserted 992 vectors, total so far: 574263\n", + "Upserted 995 vectors, total so far: 575258\n", + "Upserted 993 vectors, total so far: 576251\n", + "Upserted 986 vectors, total so far: 577237\n", + "Upserted 987 vectors, total so far: 578224\n", + "Upserted 988 vectors, total so far: 579212\n", + "Upserted 987 vectors, total so far: 580199\n", + "Upserted 990 vectors, total so far: 581189\n", + "Upserted 987 vectors, total so far: 582176\n", + "Upserted 994 vectors, total so far: 583170\n", + "Upserted 983 vectors, total so far: 584153\n", + "Upserted 989 vectors, total so far: 585142\n", + "Upserted 996 vectors, total so far: 586138\n", + "Upserted 993 vectors, total so far: 587131\n", + "Upserted 991 vectors, total so far: 588122\n", + "Upserted 988 vectors, total so far: 589110\n", + "Upserted 989 vectors, total so far: 590099\n", + "Upserted 992 vectors, total so far: 591091\n", + "Upserted 994 vectors, total so far: 592085\n", + "Upserted 991 vectors, total so far: 593076\n", + "Upserted 993 vectors, total so far: 594069\n", + "Upserted 988 vectors, total so far: 595057\n", + "Upserted 992 vectors, total so far: 596049\n", + "Upserted 985 vectors, total so far: 597034\n", + "Upserted 994 vectors, total so far: 598028\n", + "Upserted 988 vectors, total so far: 599016\n", + "Upserted 985 vectors, total so far: 600001\n", + "Upserted 984 vectors, total so far: 600985\n", + "Upserted 993 vectors, total so far: 601978\n", + "Upserted 991 vectors, total so far: 602969\n", + "Upserted 984 vectors, total so far: 603953\n", + "Upserted 994 vectors, total so far: 604947\n", + "Upserted 985 vectors, total so far: 605932\n", + "Upserted 991 vectors, total so far: 606923\n", + "Upserted 987 vectors, total so far: 607910\n", + "Upserted 989 vectors, total so far: 608899\n", + "Upserted 995 vectors, total so far: 609894\n", + "Upserted 992 vectors, total so far: 610886\n", + "Upserted 991 vectors, total so far: 611877\n", + "Upserted 991 vectors, total so far: 612868\n", + "Upserted 991 vectors, total so far: 613859\n", + "Upserted 996 vectors, total so far: 614855\n", + "Upserted 994 vectors, total so far: 615849\n", + "Upserted 987 vectors, total so far: 616836\n", + "Upserted 992 vectors, total so far: 617828\n", + "Upserted 988 vectors, total so far: 618816\n", + "Upserted 991 vectors, total so far: 619807\n", + "Upserted 994 vectors, total so far: 620801\n", + "Upserted 988 vectors, total so far: 621789\n", + "Upserted 992 vectors, total so far: 622781\n", + "Upserted 991 vectors, total so far: 623772\n", + "Upserted 984 vectors, total so far: 624756\n", + "Upserted 987 vectors, total so far: 625743\n", + "Upserted 992 vectors, total so far: 626735\n", + "Upserted 992 vectors, total so far: 627727\n", + "Upserted 995 vectors, total so far: 628722\n", + "Upserted 986 vectors, total so far: 629708\n", + "Upserted 990 vectors, total so far: 630698\n", + "Upserted 992 vectors, total so far: 631690\n", + "Upserted 991 vectors, total so far: 632681\n", + "Upserted 990 vectors, total so far: 633671\n", + "Upserted 993 vectors, total so far: 634664\n", + "Upserted 992 vectors, total so far: 635656\n", + "Upserted 984 vectors, total so far: 636640\n", + "Upserted 989 vectors, total so far: 637629\n", + "Upserted 992 vectors, total so far: 638621\n", + "Upserted 989 vectors, total so far: 639610\n", + "Upserted 990 vectors, total so far: 640600\n", + "Upserted 991 vectors, total so far: 641591\n", + "Upserted 987 vectors, total so far: 642578\n", + "Upserted 992 vectors, total so far: 643570\n", + "Upserted 993 vectors, total so far: 644563\n", + "Upserted 994 vectors, total so far: 645557\n", + "Upserted 993 vectors, total so far: 646550\n", + "Upserted 989 vectors, total so far: 647539\n", + "Upserted 986 vectors, total so far: 648525\n", + "Upserted 995 vectors, total so far: 649520\n", + "Upserted 990 vectors, total so far: 650510\n", + "Upserted 989 vectors, total so far: 651499\n", + "Upserted 988 vectors, total so far: 652487\n", + "Upserted 991 vectors, total so far: 653478\n", + "Upserted 988 vectors, total so far: 654466\n", + "Upserted 988 vectors, total so far: 655454\n", + "Upserted 992 vectors, total so far: 656446\n", + "Upserted 991 vectors, total so far: 657437\n", + "Upserted 992 vectors, total so far: 658429\n", + "Upserted 989 vectors, total so far: 659418\n", + "Upserted 993 vectors, total so far: 660411\n", + "Upserted 992 vectors, total so far: 661403\n", + "Upserted 992 vectors, total so far: 662395\n", + "Upserted 990 vectors, total so far: 663385\n", + "Upserted 988 vectors, total so far: 664373\n", + "Upserted 990 vectors, total so far: 665363\n", + "Upserted 989 vectors, total so far: 666352\n", + "Upserted 987 vectors, total so far: 667339\n", + "Upserted 989 vectors, total so far: 668328\n", + "Upserted 990 vectors, total so far: 669318\n", + "Upserted 991 vectors, total so far: 670309\n", + "Upserted 992 vectors, total so far: 671301\n", + "Upserted 992 vectors, total so far: 672293\n", + "Upserted 990 vectors, total so far: 673283\n", + "Upserted 990 vectors, total so far: 674273\n", + "Upserted 989 vectors, total so far: 675262\n", + "Upserted 992 vectors, total so far: 676254\n", + "Upserted 988 vectors, total so far: 677242\n", + "Upserted 989 vectors, total so far: 678231\n", + "Upserted 987 vectors, total so far: 679218\n", + "Upserted 988 vectors, total so far: 680206\n", + "Upserted 993 vectors, total so far: 681199\n", + "Upserted 992 vectors, total so far: 682191\n", + "Upserted 989 vectors, total so far: 683180\n", + "Upserted 991 vectors, total so far: 684171\n", + "Upserted 996 vectors, total so far: 685167\n", + "Upserted 988 vectors, total so far: 686155\n", + "Upserted 989 vectors, total so far: 687144\n", + "Upserted 988 vectors, total so far: 688132\n", + "Upserted 989 vectors, total so far: 689121\n", + "Upserted 988 vectors, total so far: 690109\n", + "Upserted 996 vectors, total so far: 691105\n", + "Upserted 986 vectors, total so far: 692091\n", + "Upserted 994 vectors, total so far: 693085\n", + "Upserted 994 vectors, total so far: 694079\n", + "Upserted 988 vectors, total so far: 695067\n", + "Upserted 990 vectors, total so far: 696057\n", + "Upserted 986 vectors, total so far: 697043\n", + "Upserted 991 vectors, total so far: 698034\n", + "Upserted 992 vectors, total so far: 699026\n", + "Upserted 988 vectors, total so far: 700014\n", + "Upserted 988 vectors, total so far: 701002\n", + "Upserted 993 vectors, total so far: 701995\n", + "Upserted 989 vectors, total so far: 702984\n", + "Upserted 986 vectors, total so far: 703970\n", + "Upserted 986 vectors, total so far: 704956\n", + "Upserted 997 vectors, total so far: 705953\n", + "Upserted 996 vectors, total so far: 706949\n", + "Upserted 994 vectors, total so far: 707943\n", + "Upserted 979 vectors, total so far: 708922\n", + "Upserted 988 vectors, total so far: 709910\n", + "Upserted 989 vectors, total so far: 710899\n", + "Upserted 992 vectors, total so far: 711891\n", + "Upserted 990 vectors, total so far: 712881\n", + "Upserted 988 vectors, total so far: 713869\n", + "Upserted 987 vectors, total so far: 714856\n", + "Upserted 990 vectors, total so far: 715846\n", + "Upserted 989 vectors, total so far: 716835\n", + "Upserted 989 vectors, total so far: 717824\n", + "Upserted 993 vectors, total so far: 718817\n", + "Upserted 981 vectors, total so far: 719798\n", + "Upserted 990 vectors, total so far: 720788\n", + "Upserted 993 vectors, total so far: 721781\n", + "Upserted 994 vectors, total so far: 722775\n", + "Upserted 995 vectors, total so far: 723770\n", + "Upserted 992 vectors, total so far: 724762\n", + "Upserted 982 vectors, total so far: 725744\n", + "Upserted 989 vectors, total so far: 726733\n", + "Upserted 984 vectors, total so far: 727717\n", + "Upserted 992 vectors, total so far: 728709\n", + "Upserted 988 vectors, total so far: 729697\n", + "Upserted 986 vectors, total so far: 730683\n", + "Upserted 993 vectors, total so far: 731676\n", + "Upserted 993 vectors, total so far: 732669\n", + "Upserted 988 vectors, total so far: 733657\n", + "Upserted 988 vectors, total so far: 734645\n", + "Upserted 991 vectors, total so far: 735636\n", + "Upserted 993 vectors, total so far: 736629\n", + "Upserted 992 vectors, total so far: 737621\n", + "Upserted 986 vectors, total so far: 738607\n", + "Upserted 991 vectors, total so far: 739598\n", + "Upserted 989 vectors, total so far: 740587\n", + "Upserted 981 vectors, total so far: 741568\n", + "Upserted 995 vectors, total so far: 742563\n", + "Upserted 995 vectors, total so far: 743558\n", + "Upserted 985 vectors, total so far: 744543\n", + "Upserted 991 vectors, total so far: 745534\n", + "Upserted 985 vectors, total so far: 746519\n", + "Upserted 987 vectors, total so far: 747506\n", + "Upserted 991 vectors, total so far: 748497\n", + "Upserted 994 vectors, total so far: 749491\n", + "Upserted 987 vectors, total so far: 750478\n", + "Upserted 991 vectors, total so far: 751469\n", + "Upserted 993 vectors, total so far: 752462\n", + "Upserted 992 vectors, total so far: 753454\n", + "Upserted 986 vectors, total so far: 754440\n", + "Upserted 987 vectors, total so far: 755427\n", + "Upserted 989 vectors, total so far: 756416\n", + "Upserted 992 vectors, total so far: 757408\n", + "Upserted 991 vectors, total so far: 758399\n", + "Upserted 985 vectors, total so far: 759384\n", + "Upserted 985 vectors, total so far: 760369\n", + "Upserted 988 vectors, total so far: 761357\n", + "Upserted 991 vectors, total so far: 762348\n", + "Upserted 989 vectors, total so far: 763337\n", + "Upserted 996 vectors, total so far: 764333\n", + "Upserted 992 vectors, total so far: 765325\n", + "Upserted 987 vectors, total so far: 766312\n", + "Upserted 986 vectors, total so far: 767298\n", + "Upserted 988 vectors, total so far: 768286\n", + "Upserted 992 vectors, total so far: 769278\n", + "Upserted 990 vectors, total so far: 770268\n", + "Upserted 987 vectors, total so far: 771255\n", + "Upserted 987 vectors, total so far: 772242\n", + "Upserted 988 vectors, total so far: 773230\n", + "Upserted 993 vectors, total so far: 774223\n", + "Upserted 992 vectors, total so far: 775215\n", + "Upserted 990 vectors, total so far: 776205\n", + "Upserted 991 vectors, total so far: 777196\n", + "Upserted 986 vectors, total so far: 778182\n", + "Upserted 990 vectors, total so far: 779172\n", + "Upserted 993 vectors, total so far: 780165\n", + "Upserted 990 vectors, total so far: 781155\n", + "Upserted 993 vectors, total so far: 782148\n", + "Upserted 987 vectors, total so far: 783135\n", + "Upserted 995 vectors, total so far: 784130\n", + "Upserted 991 vectors, total so far: 785121\n", + "Upserted 989 vectors, total so far: 786110\n", + "Upserted 991 vectors, total so far: 787101\n", + "Upserted 991 vectors, total so far: 788092\n", + "Upserted 992 vectors, total so far: 789084\n", + "Upserted 988 vectors, total so far: 790072\n", + "Upserted 992 vectors, total so far: 791064\n", + "Upserted 986 vectors, total so far: 792050\n", + "Upserted 987 vectors, total so far: 793037\n", + "Upserted 990 vectors, total so far: 794027\n", + "Upserted 990 vectors, total so far: 795017\n", + "Upserted 993 vectors, total so far: 796010\n", + "Upserted 983 vectors, total so far: 796993\n", + "Upserted 988 vectors, total so far: 797981\n", + "Upserted 985 vectors, total so far: 798966\n", + "Upserted 993 vectors, total so far: 799959\n", + "Upserted 987 vectors, total so far: 800946\n", + "Upserted 991 vectors, total so far: 801937\n", + "Upserted 991 vectors, total so far: 802928\n", + "Upserted 989 vectors, total so far: 803917\n", + "Upserted 988 vectors, total so far: 804905\n", + "Upserted 990 vectors, total so far: 805895\n", + "Upserted 992 vectors, total so far: 806887\n", + "Upserted 988 vectors, total so far: 807875\n", + "Upserted 991 vectors, total so far: 808866\n", + "Upserted 992 vectors, total so far: 809858\n", + "Upserted 988 vectors, total so far: 810846\n", + "Upserted 987 vectors, total so far: 811833\n", + "Upserted 993 vectors, total so far: 812826\n", + "Upserted 994 vectors, total so far: 813820\n", + "Upserted 992 vectors, total so far: 814812\n", + "Upserted 995 vectors, total so far: 815807\n", + "Upserted 989 vectors, total so far: 816796\n", + "Upserted 987 vectors, total so far: 817783\n", + "Upserted 986 vectors, total so far: 818769\n", + "Upserted 993 vectors, total so far: 819762\n", + "Upserted 993 vectors, total so far: 820755\n", + "Upserted 989 vectors, total so far: 821744\n", + "Upserted 990 vectors, total so far: 822734\n", + "Upserted 991 vectors, total so far: 823725\n", + "Upserted 988 vectors, total so far: 824713\n", + "Upserted 986 vectors, total so far: 825699\n", + "Upserted 985 vectors, total so far: 826684\n", + "Upserted 992 vectors, total so far: 827676\n", + "Upserted 987 vectors, total so far: 828663\n", + "Upserted 989 vectors, total so far: 829652\n", + "Upserted 990 vectors, total so far: 830642\n", + "Upserted 984 vectors, total so far: 831626\n", + "Upserted 986 vectors, total so far: 832612\n", + "Upserted 982 vectors, total so far: 833594\n", + "Upserted 992 vectors, total so far: 834586\n", + "Upserted 992 vectors, total so far: 835578\n", + "Upserted 990 vectors, total so far: 836568\n", + "Upserted 987 vectors, total so far: 837555\n", + "Upserted 989 vectors, total so far: 838544\n", + "Upserted 987 vectors, total so far: 839531\n", + "Upserted 991 vectors, total so far: 840522\n", + "Upserted 993 vectors, total so far: 841515\n", + "Upserted 996 vectors, total so far: 842511\n", + "Upserted 992 vectors, total so far: 843503\n", + "Upserted 993 vectors, total so far: 844496\n", + "Upserted 988 vectors, total so far: 845484\n", + "Upserted 987 vectors, total so far: 846471\n", + "Upserted 994 vectors, total so far: 847465\n", + "Upserted 990 vectors, total so far: 848455\n", + "Upserted 988 vectors, total so far: 849443\n", + "Upserted 985 vectors, total so far: 850428\n", + "Upserted 986 vectors, total so far: 851414\n", + "Upserted 989 vectors, total so far: 852403\n", + "Upserted 989 vectors, total so far: 853392\n", + "Upserted 994 vectors, total so far: 854386\n", + "Upserted 994 vectors, total so far: 855380\n", + "Upserted 992 vectors, total so far: 856372\n", + "Upserted 991 vectors, total so far: 857363\n", + "Upserted 993 vectors, total so far: 858356\n", + "Upserted 991 vectors, total so far: 859347\n", + "Upserted 986 vectors, total so far: 860333\n", + "Upserted 994 vectors, total so far: 861327\n", + "Upserted 992 vectors, total so far: 862319\n", + "Upserted 989 vectors, total so far: 863308\n", + "Upserted 991 vectors, total so far: 864299\n", + "Upserted 993 vectors, total so far: 865292\n", + "Upserted 985 vectors, total so far: 866277\n", + "Upserted 991 vectors, total so far: 867268\n", + "Upserted 990 vectors, total so far: 868258\n", + "Upserted 991 vectors, total so far: 869249\n", + "Upserted 993 vectors, total so far: 870242\n", + "Upserted 985 vectors, total so far: 871227\n", + "Upserted 985 vectors, total so far: 872212\n", + "Upserted 989 vectors, total so far: 873201\n", + "Upserted 990 vectors, total so far: 874191\n", + "Upserted 988 vectors, total so far: 875179\n", + "Upserted 987 vectors, total so far: 876166\n", + "Upserted 989 vectors, total so far: 877155\n", + "Upserted 993 vectors, total so far: 878148\n", + "Upserted 992 vectors, total so far: 879140\n", + "Upserted 990 vectors, total so far: 880130\n", + "Upserted 994 vectors, total so far: 881124\n", + "Upserted 984 vectors, total so far: 882108\n", + "Upserted 991 vectors, total so far: 883099\n", + "Upserted 987 vectors, total so far: 884086\n", + "Upserted 993 vectors, total so far: 885079\n", + "Upserted 988 vectors, total so far: 886067\n", + "Upserted 992 vectors, total so far: 887059\n", + "Upserted 991 vectors, total so far: 888050\n", + "Upserted 992 vectors, total so far: 889042\n", + "Upserted 989 vectors, total so far: 890031\n", + "Upserted 994 vectors, total so far: 891025\n", + "Upserted 992 vectors, total so far: 892017\n", + "Upserted 992 vectors, total so far: 893009\n", + "Upserted 989 vectors, total so far: 893998\n", + "Upserted 986 vectors, total so far: 894984\n", + "Upserted 992 vectors, total so far: 895976\n", + "Upserted 988 vectors, total so far: 896964\n", + "Upserted 992 vectors, total so far: 897956\n", + "Upserted 989 vectors, total so far: 898945\n", + "Upserted 994 vectors, total so far: 899939\n", + "Upserted 984 vectors, total so far: 900923\n", + "Upserted 985 vectors, total so far: 901908\n", + "Upserted 987 vectors, total so far: 902895\n", + "Upserted 989 vectors, total so far: 903884\n", + "Upserted 995 vectors, total so far: 904879\n", + "Upserted 990 vectors, total so far: 905869\n", + "Upserted 988 vectors, total so far: 906857\n", + "Upserted 995 vectors, total so far: 907852\n", + "Upserted 989 vectors, total so far: 908841\n", + "Upserted 987 vectors, total so far: 909828\n", + "Upserted 992 vectors, total so far: 910820\n", + "Upserted 988 vectors, total so far: 911808\n", + "Upserted 985 vectors, total so far: 912793\n", + "Upserted 986 vectors, total so far: 913779\n", + "Upserted 989 vectors, total so far: 914768\n", + "Upserted 990 vectors, total so far: 915758\n", + "Upserted 990 vectors, total so far: 916748\n", + "Upserted 992 vectors, total so far: 917740\n", + "Upserted 991 vectors, total so far: 918731\n", + "Upserted 991 vectors, total so far: 919722\n", + "Upserted 992 vectors, total so far: 920714\n", + "Upserted 990 vectors, total so far: 921704\n", + "Upserted 991 vectors, total so far: 922695\n", + "Upserted 986 vectors, total so far: 923681\n", + "Upserted 985 vectors, total so far: 924666\n", + "Upserted 990 vectors, total so far: 925656\n", + "Upserted 991 vectors, total so far: 926647\n", + "Upserted 990 vectors, total so far: 927637\n", + "Upserted 985 vectors, total so far: 928622\n", + "Upserted 993 vectors, total so far: 929615\n", + "Upserted 985 vectors, total so far: 930600\n", + "Upserted 982 vectors, total so far: 931582\n", + "Upserted 991 vectors, total so far: 932573\n", + "Upserted 991 vectors, total so far: 933564\n", + "Upserted 990 vectors, total so far: 934554\n", + "Upserted 994 vectors, total so far: 935548\n", + "Upserted 989 vectors, total so far: 936537\n", + "Upserted 989 vectors, total so far: 937526\n", + "Upserted 987 vectors, total so far: 938513\n", + "Upserted 991 vectors, total so far: 939504\n", + "Upserted 992 vectors, total so far: 940496\n", + "Upserted 984 vectors, total so far: 941480\n", + "Upserted 992 vectors, total so far: 942472\n", + "Upserted 990 vectors, total so far: 943462\n", + "Upserted 990 vectors, total so far: 944452\n", + "Upserted 993 vectors, total so far: 945445\n", + "Upserted 990 vectors, total so far: 946435\n", + "Upserted 978 vectors, total so far: 947413\n", + "Upserted 988 vectors, total so far: 948401\n", + "Upserted 986 vectors, total so far: 949387\n", + "Upserted 990 vectors, total so far: 950377\n", + "Upserted 994 vectors, total so far: 951371\n", + "Upserted 988 vectors, total so far: 952359\n", + "Upserted 989 vectors, total so far: 953348\n", + "Upserted 987 vectors, total so far: 954335\n", + "Upserted 991 vectors, total so far: 955326\n", + "Upserted 991 vectors, total so far: 956317\n", + "Upserted 994 vectors, total so far: 957311\n", + "Upserted 992 vectors, total so far: 958303\n", + "Upserted 988 vectors, total so far: 959291\n", + "Upserted 989 vectors, total so far: 960280\n", + "Upserted 982 vectors, total so far: 961262\n", + "Upserted 992 vectors, total so far: 962254\n", + "Upserted 986 vectors, total so far: 963240\n", + "Upserted 995 vectors, total so far: 964235\n", + "Upserted 992 vectors, total so far: 965227\n", + "Upserted 990 vectors, total so far: 966217\n", + "Upserted 990 vectors, total so far: 967207\n", + "Upserted 993 vectors, total so far: 968200\n", + "Upserted 993 vectors, total so far: 969193\n", + "Upserted 990 vectors, total so far: 970183\n", + "Upserted 994 vectors, total so far: 971177\n", + "Upserted 995 vectors, total so far: 972172\n", + "Upserted 996 vectors, total so far: 973168\n", + "Upserted 993 vectors, total so far: 974161\n", + "Upserted 992 vectors, total so far: 975153\n", + "Upserted 986 vectors, total so far: 976139\n", + "Upserted 986 vectors, total so far: 977125\n", + "Upserted 983 vectors, total so far: 978108\n", + "Upserted 992 vectors, total so far: 979100\n", + "Upserted 991 vectors, total so far: 980091\n", + "Upserted 992 vectors, total so far: 981083\n", + "Upserted 991 vectors, total so far: 982074\n", + "Upserted 992 vectors, total so far: 983066\n", + "Upserted 989 vectors, total so far: 984055\n", + "Upserted 989 vectors, total so far: 985044\n", + "Upserted 991 vectors, total so far: 986035\n", + "Upserted 989 vectors, total so far: 987024\n", + "Upserted 991 vectors, total so far: 988015\n", + "Upserted 993 vectors, total so far: 989008\n", + "Upserted 992 vectors, total so far: 990000\n", + "Done! Total inserted: 990000 vectors with id >= 10000\n", + "Time taken: 905.4688215106726\n" + ] + } + ], + "source": [ + "#For reinserting deleted vectors (int filter - lte)\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "import time\n", + "\n", + "# User inputs\n", + "int_rate_insert = \"99p\" # change to 1p, 20p, 50p, 99p\n", + "batch_size = 1000\n", + "\n", + "# Map int_rate to gte threshold (same as delete)\n", + "lte_map = {\n", + " \"1p\": 990000,\n", + " \"20p\": 800000,\n", + " \"50p\": 500000,\n", + " \"99p\": 10000,\n", + "}\n", + "lte_threshold = lte_map[int_rate_insert]\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "emb_path = os.path.join(dataset_path, \"shuffle_train.parquet\")\n", + "\n", + "# Process shuffle_train in batches to avoid RAM issues\n", + "emb_file = pq.ParquetFile(emb_path)\n", + "total_inserted = 0\n", + "\n", + "start = time.perf_counter()\n", + "for batch in emb_file.iter_batches(batch_size=batch_size):\n", + " batch_df = batch.to_pandas()\n", + "\n", + " # Filter by id >= lte_threshold\n", + " batch_df = batch_df[batch_df[\"id\"] >= lte_threshold]\n", + "\n", + " if batch_df.empty:\n", + " continue\n", + "\n", + " batch_vectors = []\n", + " for _, row in batch_df.iterrows():\n", + " vector_data = {\n", + " \"id\": str(row[\"id\"]),\n", + " \"vector\": row[\"emb\"],\n", + " \"meta\": {\"id\": row[\"id\"]},\n", + " \"filter\": {\n", + " \"id\": row[\"id\"],\n", + " }\n", + " }\n", + " batch_vectors.append(vector_data)\n", + "\n", + " index.upsert(batch_vectors)\n", + " total_inserted += len(batch_vectors)\n", + " print(f\"Upserted {len(batch_vectors)} vectors, total so far: {total_inserted}\")\n", + "\n", + "end = time.perf_counter()\n", + "print(f\"Done! Total inserted: {total_inserted} vectors with id >= {lte_threshold}\")\n", + "print(\"Time taken:\", end - start)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_labelfilter_0503_1_adup6',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 10000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "index.describe()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv2", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/labelfilter_bench.py b/labelfilter_bench.py new file mode 100644 index 000000000..8d103481e --- /dev/null +++ b/labelfilter_bench.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +""" +Label Filter Benchmark Script - LabelFilterPerformanceCase +Index: (set INDEX_NAME below — 1M vectors pre-loaded, no load step) + +Workflow: + For each filter_boost_percentage in [0, 25, 50, 75, 100]: + Run vectordbbench for label percentages: 0.001, 0.002, 0.01, 0.02, 0.05, 0.10, 0.20, 0.50 + For label_pct in [0.01, 0.02, 0.05]: also run with prefilter_cardinality_threshold=100000 + 10-second gap between every run. + +Output: Excel file with one table per boost percentage. +""" + +import subprocess +import json +import os +import time +import glob as glob_module +import openpyxl +from openpyxl.styles import Font, PatternFill, Alignment, Border, Side +from datetime import datetime + +# ============================================================ +# CONFIGURATION +# ============================================================ +TOKEN = "localtest" +BASE_URL = "http://148.113.54.173:8080/api/v1" +INDEX_NAME = "test_latest_master_1803_2" # <-- set before running +DATASET_LOCAL_DIR = "/home/debian/latest_VDB/VectorDBBench/vectordataset_label" +RESULTS_DIR = "/home/debian/latest_VDB/VectorDBBench/vectordb_bench/results/Endee" +REGION = "india-west-1" +TASK_LABEL = "20260107" + +M = 16 +EF_SEARCH = 128 +EF_CON = 128 +TOP_K = 100 +CONCURRENCY = 16 +CONCURRENCY_DUR = 30 +PRECISION = "int16" + +LABEL_PERCENTAGES = [0.50, 0.20, 0.10, 0.05, 0.02, 0.01, 0.002, 0.001] +PRECARDINALITY = 100000 # extra run for label_pct in [0.01, 0.02, 0.05] +PREFILTER_LPCTS = {0.01, 0.02, 0.05} +BOOST_PERCENTAGES = [0, 25, 50, 75, 100] + +OUTPUT_EXCEL = os.path.join( + "/home/debian/latest_VDB/VectorDBBench", + f"labelfilter_bench_{datetime.now().strftime('%Y%m%d_%H%M%S')}.xlsx" +) + +# ============================================================ +# BENCHMARK RUNNER +# ============================================================ + +def run_vectordbbench(label_pct: float, boost_pct: int, prefilter: int = None) -> dict: + before = set(glob_module.glob(os.path.join(RESULTS_DIR, "*.json"))) + + prefilter_part = f"--prefilter-cardinality-threshold {prefilter} " if prefilter else "" + boost_part = f"--filter-boost-percentage {boost_pct} " if boost_pct is not None else "" + + cmd = ( + f'DATASET_LOCAL_DIR="{DATASET_LOCAL_DIR}" vectordbbench endee ' + f'--token "{TOKEN}" ' + f'--region {REGION} ' + f'--base-url "{BASE_URL}" ' + f'--index-name {INDEX_NAME} ' + f'--task-label "{TASK_LABEL}" ' + f'--m {M} ' + f'--ef-con {EF_CON} ' + f'--ef-search {EF_SEARCH} ' + f'--space-type cosine ' + f'{prefilter_part}' + f'{boost_part}' + f'--precision {PRECISION} ' + f'--version 1 ' + f'--case-type LabelFilterPerformanceCase ' + f'--dataset-with-size-type "Medium Cohere (768dim, 1M)" ' + f'--label-percentage {label_pct} ' + f'--k {TOP_K} ' + f'--num-concurrency "{CONCURRENCY}" ' + f'--concurrency-duration {CONCURRENCY_DUR} ' + f'--concurrency-timeout 3600 ' + f'--skip-drop-old ' + f'--skip-load ' + f'--search-concurrent ' + f'--search-serial' + ) + + label = f"label_pct={label_pct}, boost={boost_pct}%" + (f", prefilter={prefilter}" if prefilter else "") + print(f"\n [RUN] {label}") + proc = subprocess.run(cmd, shell=True, text=True) + if proc.returncode != 0: + print(f" [WARN] vectordbbench exited with code {proc.returncode}") + + time.sleep(5) + after = set(glob_module.glob(os.path.join(RESULTS_DIR, "*.json"))) + new_files = after - before + + if not new_files: + print(f" [ERROR] No new result file for {label}") + return {"recall": None, "qps": None, "p99_latency": None, "load_duration": None} + + result_file = max(new_files, key=os.path.getmtime) + print(f" [FILE] {os.path.basename(result_file)}") + + with open(result_file) as f: + data = json.load(f) + + metrics = data["results"][0]["metrics"] + recall = metrics.get("recall") + qps = metrics.get("qps") + p99 = metrics.get("serial_latency_p99") + load_dur = metrics.get("load_duration") + + print(f" [METRICS] recall={recall}, qps={qps}, p99={p99}, load_duration={load_dur}") + return {"recall": recall, "qps": qps, "p99_latency": p99, "load_duration": load_dur} + + +# ============================================================ +# EXCEL WRITER +# ============================================================ + +def write_excel(all_data: dict, output_path: str): + """ + all_data: {boost_pct: [row_dict, ...]} + Each row_dict has: label_pct, prefilter, recall, qps, p99_latency, load_duration + """ + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "Label Filter Bench" + + thin = Side(style="thin", color="000000") + border = Border(left=thin, right=thin, top=thin, bottom=thin) + center = Alignment(horizontal="center", vertical="center", wrap_text=True) + left = Alignment(horizontal="left", vertical="center") + + HDR_BG = "2D6A4F" + HDR_FONT = Font(bold=True, color="FFFFFF") + ROW_ODD = "F0F7F4" + ROW_EVEN = "FFFFFF" + + DATASET_NAME = "Cohere 1M (768D)" + FILTER_CASE = "LabelFilterPerformanceCase(StrEqual)" + + columns = [ + ("Dataset", 22), + ("Precision", 12), + ("Filter Case Type", 38), + ("Label Fraction", 16), + ("m", 7), + ("ef_search", 11), + ("ef_con", 10), + ("topK", 8), + ("Concurrency", 14), + ("Recall", 10), + ("QPS", 14), + ("Latency (p99)(in sec)", 20), + ("Load Duration(in sec)", 20), + ] + + NUM_COLS = len(columns) + + # Set column widths once + for col_idx, (_, width) in enumerate(columns, start=1): + ws.column_dimensions[openpyxl.utils.get_column_letter(col_idx)].width = width + + current_row = 1 + + for boost_pct in BOOST_PERCENTAGES: + rows = all_data.get(boost_pct, []) + + # --- Boost label row (outside table) --- + label_cell = ws.cell(row=current_row, column=1, + value=f"Filter Boost Percentage: {boost_pct}%") + label_cell.font = Font(bold=True, size=12) + label_cell.alignment = left + ws.merge_cells( + start_row=current_row, start_column=1, + end_row=current_row, end_column=NUM_COLS + ) + ws.row_dimensions[current_row].height = 22 + current_row += 1 + + # --- Header row --- + ws.row_dimensions[current_row].height = 28 + for col_idx, (header, _) in enumerate(columns, start=1): + c = ws.cell(row=current_row, column=col_idx, value=header) + c.font = HDR_FONT + c.fill = PatternFill("solid", fgColor=HDR_BG) + c.alignment = center + c.border = border + current_row += 1 + + # --- Data rows --- + for row_local_idx, r in enumerate(rows): + ws.row_dimensions[current_row].height = 22 + bg = ROW_ODD if row_local_idx % 2 == 0 else ROW_EVEN + rf = PatternFill("solid", fgColor=bg) + + case_label = FILTER_CASE + if r.get("prefilter"): + case_label += f"\n(prefilter={r['prefilter']})" + + recall = r.get("recall") + qps = r.get("qps") + p99 = r.get("p99_latency") + load_dur = r.get("load_duration") + + values = [ + DATASET_NAME, + PRECISION, + case_label, + r["label_pct"], + M, + EF_SEARCH, + EF_CON, + TOP_K, + CONCURRENCY, + round(recall * 100, 2) if recall is not None else "N/A", + round(qps, 4) if qps is not None else "N/A", + round(p99, 6) if p99 is not None else "N/A", + round(load_dur, 4) if load_dur is not None else "N/A", + ] + + for col_idx, val in enumerate(values, start=1): + c = ws.cell(row=current_row, column=col_idx, value=val) + c.fill = rf + c.alignment = center if col_idx != 1 else left + c.border = border + + current_row += 1 + + # 2 blank rows before next section + current_row += 2 + + wb.save(output_path) + print(f"\n[EXCEL] Saved → {output_path}") + + +# ============================================================ +# MAIN +# ============================================================ + +def main(): + print("=" * 60) + print("Label Filter Benchmark") + print(f"Index : {INDEX_NAME}") + print(f"Output: {OUTPUT_EXCEL}") + print("=" * 60) + + all_data = {} + + for boost_pct in BOOST_PERCENTAGES: + print(f"\n{'='*60}") + print(f"BOOST PERCENTAGE: {boost_pct}%") + print(f"{'='*60}") + rows = [] + + for lp in LABEL_PERCENTAGES: + metrics = run_vectordbbench(lp, boost_pct) + rows.append({ + "label_pct": lp, + "prefilter": None, + **metrics, + }) + + if lp in PREFILTER_LPCTS: + print(f"\n [WAIT] 10s before precardinality run ...") + time.sleep(10) + metrics_pre = run_vectordbbench(lp, boost_pct, prefilter=PRECARDINALITY) + rows.append({ + "label_pct": lp, + "prefilter": PRECARDINALITY, + **metrics_pre, + }) + + print(f"\n [WAIT] 10s before next run ...") + time.sleep(10) + + all_data[boost_pct] = rows + + write_excel(all_data, OUTPUT_EXCEL) + + +if __name__ == "__main__": + main() diff --git a/prepare_groundtruth.py b/prepare_groundtruth.py new file mode 100644 index 000000000..5332be050 --- /dev/null +++ b/prepare_groundtruth.py @@ -0,0 +1,88 @@ +import pyarrow.parquet as pq +import pyarrow as pa +import numpy as np +import os + +base_path = "/home/debian/latest_VDB/VectorDBBench/vectordataset" +dataset_folder = "cohere/cohere_medium_1m" +dataset_path = os.path.join(base_path, dataset_folder) + +# Output path - new folder for LTE ground truth +lte_base_path = "/home/debian/latest_VDB/VectorDBBench/vectordataset_lte" +lte_dataset_path = os.path.join(lte_base_path, dataset_folder) +os.makedirs(lte_dataset_path, exist_ok=True) + +# ---- INPUTS ---- +int_rate = "99p" # change to 1p, 50p, 80p, 99p +top_k = 1000 + +# LTE threshold map (opposite of GTE) +# GTE "1p" means id >= 10000 (99% pass) +# LTE "1p" means id <= 9999 (1% pass) +lte_map = { + "1p": 9999, + "50p": 499999, + "80p": 799999, + "99p": 989999, +} +lte_threshold = lte_map[int_rate] +print(f"Filter: id <= {lte_threshold} ({int_rate})") + +# ---- STEP 1: Load filtered train vectors ---- +print("Loading filtered train vectors...") +train_file = pq.ParquetFile(os.path.join(dataset_path, "shuffle_train.parquet")) + +filtered_ids = [] +filtered_vecs = [] + +for batch in train_file.iter_batches(batch_size=10000): + df = batch.to_pandas() + df = df[df["id"] <= lte_threshold] # Apply LTE filter + if not df.empty: + filtered_ids.extend(df["id"].tolist()) + filtered_vecs.extend(df["emb"].tolist()) + +filtered_ids = np.array(filtered_ids) +filtered_vecs = np.array(filtered_vecs, dtype=np.float32) + +# Normalize for cosine similarity +norms = np.linalg.norm(filtered_vecs, axis=1, keepdims=True) +filtered_vecs = filtered_vecs / norms + +print(f"Filtered vectors: {len(filtered_ids)} (expected ~{lte_threshold + 1})") + +# ---- STEP 2: Load test queries ---- +print("Loading test queries...") +test_df = pq.ParquetFile(os.path.join(dataset_path, "test.parquet")).read().to_pandas() +test_vecs = np.array(test_df["emb"].tolist(), dtype=np.float32) + +# Normalize test queries too +norms = np.linalg.norm(test_vecs, axis=1, keepdims=True) +test_vecs = test_vecs / norms + +print(f"Test queries: {len(test_df)}") + +# ---- STEP 3: Brute force exact nearest neighbors ---- +print("Computing exact nearest neighbors (brute force)...") +results = [] + +CHUNK = 100 # Process test queries in chunks to save memory +for i in range(0, len(test_vecs), CHUNK): + chunk = test_vecs[i:i+CHUNK] + # Cosine similarity = dot product (since both are normalized) + sims = chunk @ filtered_vecs.T # (CHUNK, num_filtered) + top_k_idx = np.argsort(-sims, axis=1)[:, :top_k] + top_k_ids = filtered_ids[top_k_idx] + results.extend(top_k_ids.tolist()) + print(f" Processed {min(i+CHUNK, len(test_vecs))}/{len(test_vecs)}") + +# ---- STEP 4: Save as parquet ---- +output_df = test_df[["id"]].copy() +output_df["neighbors_id"] = results + +table = pa.Table.from_pandas(output_df, preserve_index=False) +output_path = os.path.join(lte_dataset_path, f"neighbors_int_{int_rate}.parquet") +pa.parquet.write_table(table, output_path) +print(f"\n✅ Saved: {output_path}") +print(f"Shape: {output_df.shape}") +print(f"Sample (first row): {output_df['neighbors_id'].iloc[0][:5]}") diff --git a/prepare_groundtruth_gt.py b/prepare_groundtruth_gt.py new file mode 100644 index 000000000..9dfeed3e8 --- /dev/null +++ b/prepare_groundtruth_gt.py @@ -0,0 +1,88 @@ +import pyarrow.parquet as pq +import pyarrow as pa +import numpy as np +import os + +base_path = "/home/debian/latest_VDB/VectorDBBench/vectordataset" +dataset_folder = "cohere/cohere_medium_1m" +dataset_path = os.path.join(base_path, dataset_folder) + +# Output path - new folder for GT ground truth +gt_base_path = "/home/debian/latest_VDB/VectorDBBench/vectordataset_gt" +gt_dataset_path = os.path.join(gt_base_path, dataset_folder) +os.makedirs(gt_dataset_path, exist_ok=True) + +# ---- INPUTS ---- +int_rate = "99p" # change to 1p, 50p, 80p, 99p +top_k = 1000 + +# GT threshold map (strictly greater than) +# GT "1p": id > 10000 → ~989,999 vectors pass +# GT "99p": id > 990000 → ~9,999 vectors pass +gt_map = { + "1p": 10000, + "50p": 500000, + "80p": 800000, + "99p": 990000, +} +gt_threshold = gt_map[int_rate] +print(f"Filter: id > {gt_threshold} ({int_rate})") + +# ---- STEP 1: Load filtered train vectors ---- +print("Loading filtered train vectors...") +train_file = pq.ParquetFile(os.path.join(dataset_path, "shuffle_train.parquet")) + +filtered_ids = [] +filtered_vecs = [] + +for batch in train_file.iter_batches(batch_size=10000): + df = batch.to_pandas() + df = df[df["id"] > gt_threshold] # Apply GT filter (strictly greater than) + if not df.empty: + filtered_ids.extend(df["id"].tolist()) + filtered_vecs.extend(df["emb"].tolist()) + +filtered_ids = np.array(filtered_ids) +filtered_vecs = np.array(filtered_vecs, dtype=np.float32) + +# Normalize for cosine similarity +norms = np.linalg.norm(filtered_vecs, axis=1, keepdims=True) +filtered_vecs = filtered_vecs / norms + +print(f"Filtered vectors: {len(filtered_ids)} (expected ~{999999 - gt_threshold})") + +# ---- STEP 2: Load test queries ---- +print("Loading test queries...") +test_df = pq.ParquetFile(os.path.join(dataset_path, "test.parquet")).read().to_pandas() +test_vecs = np.array(test_df["emb"].tolist(), dtype=np.float32) + +# Normalize test queries too +norms = np.linalg.norm(test_vecs, axis=1, keepdims=True) +test_vecs = test_vecs / norms + +print(f"Test queries: {len(test_df)}") + +# ---- STEP 3: Brute force exact nearest neighbors ---- +print("Computing exact nearest neighbors (brute force)...") +results = [] + +CHUNK = 100 # Process test queries in chunks to save memory +for i in range(0, len(test_vecs), CHUNK): + chunk = test_vecs[i:i+CHUNK] + # Cosine similarity = dot product (since both are normalized) + sims = chunk @ filtered_vecs.T # (CHUNK, num_filtered) + top_k_idx = np.argsort(-sims, axis=1)[:, :top_k] + top_k_ids = filtered_ids[top_k_idx] + results.extend(top_k_ids.tolist()) + print(f" Processed {min(i+CHUNK, len(test_vecs))}/{len(test_vecs)}") + +# ---- STEP 4: Save as parquet ---- +output_df = test_df[["id"]].copy() +output_df["neighbors_id"] = results + +table = pa.Table.from_pandas(output_df, preserve_index=False) +output_path = os.path.join(gt_dataset_path, f"neighbors_int_{int_rate}.parquet") +pa.parquet.write_table(table, output_path) +print(f"\n✅ Saved: {output_path}") +print(f"Shape: {output_df.shape}") +print(f"Sample (first row): {output_df['neighbors_id'].iloc[0][:5]}") diff --git a/prepare_groundtruth_lt.py b/prepare_groundtruth_lt.py new file mode 100644 index 000000000..a346aa2f6 --- /dev/null +++ b/prepare_groundtruth_lt.py @@ -0,0 +1,88 @@ +import pyarrow.parquet as pq +import pyarrow as pa +import numpy as np +import os + +base_path = "/home/debian/latest_VDB/VectorDBBench/vectordataset" +dataset_folder = "cohere/cohere_medium_1m" +dataset_path = os.path.join(base_path, dataset_folder) + +# Output path - new folder for LT ground truth +lt_base_path = "/home/debian/latest_VDB/VectorDBBench/vectordataset_lt" +lt_dataset_path = os.path.join(lt_base_path, dataset_folder) +os.makedirs(lt_dataset_path, exist_ok=True) + +# ---- INPUTS ---- +int_rate = "99p" # change to 1p, 50p, 80p, 99p +top_k = 1000 + +# LT threshold map (strictly less than) +# LT "1p": id < 9999 → ~9,999 vectors pass (1 fewer than LTE id <= 9999) +# LT "99p": id < 989999 → ~989,999 vectors pass (1 fewer than LTE id <= 989999) +lt_map = { + "1p": 9999, + "50p": 499999, + "80p": 799999, + "99p": 989999, +} +lt_threshold = lt_map[int_rate] +print(f"Filter: id < {lt_threshold} ({int_rate})") + +# ---- STEP 1: Load filtered train vectors ---- +print("Loading filtered train vectors...") +train_file = pq.ParquetFile(os.path.join(dataset_path, "shuffle_train.parquet")) + +filtered_ids = [] +filtered_vecs = [] + +for batch in train_file.iter_batches(batch_size=10000): + df = batch.to_pandas() + df = df[df["id"] < lt_threshold] # Apply LT filter (strictly less than) + if not df.empty: + filtered_ids.extend(df["id"].tolist()) + filtered_vecs.extend(df["emb"].tolist()) + +filtered_ids = np.array(filtered_ids) +filtered_vecs = np.array(filtered_vecs, dtype=np.float32) + +# Normalize for cosine similarity +norms = np.linalg.norm(filtered_vecs, axis=1, keepdims=True) +filtered_vecs = filtered_vecs / norms + +print(f"Filtered vectors: {len(filtered_ids)} (expected ~{lt_threshold})") + +# ---- STEP 2: Load test queries ---- +print("Loading test queries...") +test_df = pq.ParquetFile(os.path.join(dataset_path, "test.parquet")).read().to_pandas() +test_vecs = np.array(test_df["emb"].tolist(), dtype=np.float32) + +# Normalize test queries too +norms = np.linalg.norm(test_vecs, axis=1, keepdims=True) +test_vecs = test_vecs / norms + +print(f"Test queries: {len(test_df)}") + +# ---- STEP 3: Brute force exact nearest neighbors ---- +print("Computing exact nearest neighbors (brute force)...") +results = [] + +CHUNK = 100 # Process test queries in chunks to save memory +for i in range(0, len(test_vecs), CHUNK): + chunk = test_vecs[i:i+CHUNK] + # Cosine similarity = dot product (since both are normalized) + sims = chunk @ filtered_vecs.T # (CHUNK, num_filtered) + top_k_idx = np.argsort(-sims, axis=1)[:, :top_k] + top_k_ids = filtered_ids[top_k_idx] + results.extend(top_k_ids.tolist()) + print(f" Processed {min(i+CHUNK, len(test_vecs))}/{len(test_vecs)}") + +# ---- STEP 4: Save as parquet ---- +output_df = test_df[["id"]].copy() +output_df["neighbors_id"] = results + +table = pa.Table.from_pandas(output_df, preserve_index=False) +output_path = os.path.join(lt_dataset_path, f"neighbors_int_{int_rate}.parquet") +pa.parquet.write_table(table, output_path) +print(f"\n✅ Saved: {output_path}") +print(f"Shape: {output_df.shape}") +print(f"Sample (first row): {output_df['neighbors_id'].iloc[0][:5]}") diff --git a/range_filter_bench.py b/range_filter_bench.py new file mode 100644 index 000000000..ced3e863e --- /dev/null +++ b/range_filter_bench.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +""" +Range Filter Benchmark Script - NewIntFilterPerformanceCase +Index: (set INDEX_NAME below — 1M vectors pre-loaded, no load step) + +Workflow: + For each filter_boost_percentage in [0, 25, 50, 75, 100]: + Run vectordbbench for filter rates: 0.01, 0.50, 0.80, 0.99 + For filter_rate=0.99: also run with prefilter_cardinality_threshold=50000 + 10-second gap between every run. + +Output: Excel file with one table per boost percentage. +""" + +import subprocess +import json +import os +import time +import glob as glob_module +import openpyxl +from openpyxl.styles import Font, PatternFill, Alignment, Border, Side +from datetime import datetime + +# ============================================================ +# CONFIGURATION +# ============================================================ +TOKEN = "localtest" +BASE_URL = "http://148.113.54.173:8080/api/v1" +INDEX_NAME = "test_latest_master_1803_2" # <-- set before running +DATASET_LOCAL_DIR = "/home/debian/latest_VDB/VectorDBBench/vectordataset" +RESULTS_DIR = "/home/debian/latest_VDB/VectorDBBench/vectordb_bench/results/Endee" +REGION = "india-west-1" +TASK_LABEL = "20260107" + +M = 16 +EF_SEARCH = 128 +EF_CON = 128 +TOP_K = 100 +CONCURRENCY = 16 +CONCURRENCY_DUR = 30 +PRECISION = "int16" + +FILTER_RATES = [0.01, 0.50, 0.80, 0.99] +PRECARDINALITY = 50000 # extra run for filter_rate=0.80 and 0.99 +BOOST_PERCENTAGES = [0, 25, 50, 75, 100] + +OUTPUT_EXCEL = os.path.join( + "/home/debian/latest_VDB/VectorDBBench", + f"range_filter_bench_{datetime.now().strftime('%Y%m%d_%H%M%S')}.xlsx" +) + +# ============================================================ +# BENCHMARK RUNNER +# ============================================================ + +def run_vectordbbench(filter_rate: float, boost_pct: int, prefilter: int = None) -> dict: + before = set(glob_module.glob(os.path.join(RESULTS_DIR, "*.json"))) + + prefilter_part = f"--prefilter-cardinality-threshold {prefilter} " if prefilter else "" + boost_part = f"--filter-boost-percentage {boost_pct} " if boost_pct is not None else "" + + cmd = ( + f'DATASET_LOCAL_DIR="{DATASET_LOCAL_DIR}" vectordbbench endee ' + f'--token "{TOKEN}" ' + f'--region {REGION} ' + f'--base-url "{BASE_URL}" ' + f'--index-name {INDEX_NAME} ' + f'--task-label "{TASK_LABEL}" ' + f'--m {M} ' + f'--ef-con {EF_CON} ' + f'--ef-search {EF_SEARCH} ' + f'--space-type cosine ' + f'{prefilter_part}' + f'{boost_part}' + f'--precision {PRECISION} ' + f'--version 1 ' + f'--case-type NewIntFilterPerformanceCase ' + f'--dataset-with-size-type "Medium Cohere (768dim, 1M)" ' + f'--filter-rate {filter_rate} ' + f'--k {TOP_K} ' + f'--num-concurrency "{CONCURRENCY}" ' + f'--concurrency-duration {CONCURRENCY_DUR} ' + f'--concurrency-timeout 3600 ' + f'--skip-drop-old ' + f'--skip-load ' + f'--search-concurrent ' + f'--search-serial' + ) + + label = f"filter_rate={filter_rate}, boost={boost_pct}%" + (f", prefilter={prefilter}" if prefilter else "") + print(f"\n [RUN] {label}") + proc = subprocess.run(cmd, shell=True, text=True) + if proc.returncode != 0: + print(f" [WARN] vectordbbench exited with code {proc.returncode}") + + time.sleep(5) + after = set(glob_module.glob(os.path.join(RESULTS_DIR, "*.json"))) + new_files = after - before + + if not new_files: + print(f" [ERROR] No new result file for {label}") + return {"recall": None, "qps": None, "p99_latency": None, "load_duration": None} + + result_file = max(new_files, key=os.path.getmtime) + print(f" [FILE] {os.path.basename(result_file)}") + + with open(result_file) as f: + data = json.load(f) + + metrics = data["results"][0]["metrics"] + recall = metrics.get("recall") + qps = metrics.get("qps") + p99 = metrics.get("serial_latency_p99") + load_dur = metrics.get("load_duration") + + print(f" [METRICS] recall={recall}, qps={qps}, p99={p99}, load_duration={load_dur}") + return {"recall": recall, "qps": qps, "p99_latency": p99, "load_duration": load_dur} + + +# ============================================================ +# EXCEL WRITER +# ============================================================ + +def write_excel(all_data: dict, output_path: str): + """ + all_data: {boost_pct: [row_dict, ...]} + Each row_dict has: filter_rate, prefilter, recall, qps, p99_latency, load_duration + """ + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "Range Filter Bench" + + thin = Side(style="thin", color="000000") + border = Border(left=thin, right=thin, top=thin, bottom=thin) + center = Alignment(horizontal="center", vertical="center", wrap_text=True) + left = Alignment(horizontal="left", vertical="center") + + HDR_BG = "2D6A4F" + HDR_FONT = Font(bold=True, color="FFFFFF") + ROW_ODD = "F0F7F4" + ROW_EVEN = "FFFFFF" + + DATASET_NAME = "Cohere 1M (768D)" + FILTER_CASE = "NewIntFilterPerf" + + columns = [ + ("Dataset", 22), + ("Precision", 12), + ("Filter Case", 22), + ("Filter Rate", 13), + ("m", 7), + ("ef_search", 11), + ("ef_con", 10), + ("topK", 8), + ("Concurrency", 14), + ("Recall", 10), + ("QPS", 12), + ("Latency (p99)(in sec)", 16), + ("Load Duration(in sec)", 16), + ] + + NUM_COLS = len(columns) + + # Set column widths once + for col_idx, (_, width) in enumerate(columns, start=1): + ws.column_dimensions[openpyxl.utils.get_column_letter(col_idx)].width = width + + current_row = 1 + + for boost_pct in BOOST_PERCENTAGES: + rows = all_data.get(boost_pct, []) + + # --- Boost label row (outside table) --- + label_cell = ws.cell(row=current_row, column=1, + value=f"Filter Boost Percentage: {boost_pct}%") + label_cell.font = Font(bold=True, size=12) + label_cell.alignment = left + ws.merge_cells( + start_row=current_row, start_column=1, + end_row=current_row, end_column=NUM_COLS + ) + ws.row_dimensions[current_row].height = 22 + current_row += 1 + + # --- Header row --- + ws.row_dimensions[current_row].height = 28 + for col_idx, (header, _) in enumerate(columns, start=1): + c = ws.cell(row=current_row, column=col_idx, value=header) + c.font = HDR_FONT + c.fill = PatternFill("solid", fgColor=HDR_BG) + c.alignment = center + c.border = border + current_row += 1 + + # --- Data rows --- + for row_local_idx, r in enumerate(rows): + ws.row_dimensions[current_row].height = 22 + bg = ROW_ODD if row_local_idx % 2 == 0 else ROW_EVEN + rf = PatternFill("solid", fgColor=bg) + + case_label = FILTER_CASE + if r.get("prefilter"): + case_label += f"\n(prefilter={r['prefilter']})" + + recall = r.get("recall") + qps = r.get("qps") + p99 = r.get("p99_latency") + load_dur = r.get("load_duration") + + values = [ + DATASET_NAME, + PRECISION, + case_label, + r["filter_rate"], + M, + EF_SEARCH, + EF_CON, + TOP_K, + CONCURRENCY, + round(recall * 100, 2) if recall is not None else "N/A", + round(qps, 4) if qps is not None else "N/A", + round(p99, 6) if p99 is not None else "N/A", + round(load_dur, 4) if load_dur is not None else "N/A", + ] + + for col_idx, val in enumerate(values, start=1): + c = ws.cell(row=current_row, column=col_idx, value=val) + c.fill = rf + c.alignment = center if col_idx != 1 else left + c.border = border + + current_row += 1 + + # 2 blank rows before next section + current_row += 2 + + wb.save(output_path) + print(f"\n[EXCEL] Saved → {output_path}") + + +# ============================================================ +# MAIN +# ============================================================ + +def main(): + print("=" * 60) + print("Range Filter Benchmark") + print(f"Index : {INDEX_NAME}") + print(f"Output: {OUTPUT_EXCEL}") + print("=" * 60) + + all_data = {} + + for boost_pct in BOOST_PERCENTAGES: + print(f"\n{'='*60}") + print(f"BOOST PERCENTAGE: {boost_pct}%") + print(f"{'='*60}") + rows = [] + + for fr in FILTER_RATES: + metrics = run_vectordbbench(fr, boost_pct) + rows.append({ + "filter_rate": fr, + "prefilter": None, + **metrics, + }) + + if fr == 0.99: + print(f"\n [WAIT] 10s before precardinality run ...") + time.sleep(10) + metrics_pre = run_vectordbbench(fr, boost_pct, prefilter=PRECARDINALITY) + rows.append({ + "filter_rate": fr, + "prefilter": PRECARDINALITY, + **metrics_pre, + }) + + print(f"\n [WAIT] 10s before next run ...") + time.sleep(10) + + all_data[boost_pct] = rows + + write_excel(all_data, OUTPUT_EXCEL) + + +if __name__ == "__main__": + main() diff --git a/script.ipynb b/script.ipynb index 520dc89c0..17a28b516 100644 --- a/script.ipynb +++ b/script.ipynb @@ -20,9 +20,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Name: endee\n", + "Version: 0.1.13\n", + "Summary: Endee is the Next-Generation Vector Database for Scalable, High-Performance AI\n", + "Home-page: https://endee.io\n", + "Author: Endee Labs\n", + "Author-email: dev@endee.io\n", + "License: \n", + "Location: /home/debian/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages\n", + "Requires: httpx, msgpack, numpy, orjson, pydantic, requests\n", + "Required-by: \n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], "source": [ "pip show endee" ] @@ -39,9 +57,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " id emb\n", + "0 322406 [0.19600096, -0.5270862, -0.29519123, 0.429556...\n", + "1 337610 [0.32829463, -0.055560444, -0.06252914, -0.101...\n", + "2 714728 [-0.054155815, 0.009554057, 0.24696879, -0.142...\n", + "3 354737 [0.2501778, 0.017853737, -0.43795395, 0.522256...\n", + "4 290979 [0.3444257, -0.62243223, -0.20940691, -0.08620...\n" + ] + } + ], "source": [ "#For checking parquet file contents\n", "import pyarrow.parquet as pq\n", @@ -79,9 +110,49 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "test_1M_normal_latest_master\n", + "1000000\n", + "\t\n", + "test_1M_numfilter_latest_master_stability_vaib\n", + "1000000\n", + "\t\n", + "test_1M_numfilter_latest_master_stability_vaib2\n", + "200000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1\n", + "1000000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup1\n", + "1000000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup3\n", + "1000000\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_adup4\n", + "998004\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_asyncbackup\n", + "0\n", + "\t\n", + "test_1M_vaib_labelfilter_0503_1_duplicate1\n", + "1000000\n", + "\t\n", + "test_1M_vaib_labelfilter_reupsertcheck\n", + "500000\n", + "\t\n", + "test_1M_vaib_reupsertcheck\n", + "300000\n", + "\t\n" + ] + } + ], "source": [ "#For checking indexes\n", "for obj in client.list_indexes().get('indexes'):\n", @@ -97,24 +168,3224 @@ "outputs": [], "source": [ "#give the index name\n", - "index_name = \"test_1M_labelfilter_int16_latest_master\"\n", + "index_name = \"test_1M_vaib_labelfilter_0503_1_adup4\"\n", "index = client.get_index(index_name)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_labelfilter_0503_1_adup4',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 1000000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "index.describe()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Finding all query IDs with recall < 1.0 ...\n", + "\n", + "Query ID: 2\n", + "Returned IDs: 916915,153590,759005,515871,151336,91663,375948,500463,225501,360986,265421,402047,294599,492005,632708,911260,561483,408751,959763,554056,930465,21239,124402,73355,770379,18830,401384,350257,782199,918389\n", + "Ground Truth: 916915,153590,759005,515871,151336,91663,375948,500463,225501,360986,209632,265421,402047,294599,492005,632708,911260,561483,408751,959763,554056,930465,21239,124402,73355,24196,770379,18830,401384,350257\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 6\n", + "Returned IDs: 235467,945780,366682,809535,755666,392618,18830,109016,912561,878069,9024,393202,612297,916915,87686,913268,652336,331608,734241,23934,653858,12387,982164,695965,381462,266404,882066,120980,335933,6153\n", + "Ground Truth: 235467,945780,366682,809535,755666,392618,18830,109016,912561,878069,9024,285884,393202,612297,916915,87686,913268,652336,331608,734241,23934,653858,12387,500463,982164,695965,381462,266404,882066,120980\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 9\n", + "Returned IDs: 612297,695965,982164,379767,734241,393202,299027,990925,945780,447505,24196,916915,98827,87686,109016,469341,91663,674899,185309,61856,417872,579250,670380,132941,311197,632640,245332,492260,632814,745185\n", + "Ground Truth: 612297,695965,982164,379767,734241,393202,299027,990925,945780,447505,24196,916915,98827,87686,109016,469341,755666,91663,674899,185309,61856,417872,579250,670380,132941,311197,632640,32213,245332,492260\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 10\n", + "Returned IDs: 77201,639776,579886,314296,90963,309668,576985,978112,20341,39655,12057,988834,940505,753890,777241,784785,712316,203699,336780,941526,502367,422648,845747,823159,938312,632080,424560,621662,231837,19390\n", + "Ground Truth: 77201,639776,579886,314296,90963,309668,576985,978112,20341,39655,12057,988834,940505,753890,777241,784785,712316,203699,336780,405009,941526,502367,422648,111684,845747,823159,938312,632080,424560,621662\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 13\n", + "Returned IDs: 112463,890776,775128,812245,869978,408819,892053,151308,115722,594577,612263,866926,467340,867184,763870,841229,647270,371698,226695,775449,943933,631399,569756,342624,964924,388990,381463,816457,194982,655988\n", + "Ground Truth: 112463,890776,775128,812245,869978,408819,892053,46417,151308,115722,594577,612263,866926,467340,867184,763870,841229,647270,371698,226695,775449,943933,631399,569756,342624,964924,97858,388990,381463,816457\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 14\n", + "Returned IDs: 606232,281286,673374,264793,365399,34721,199555,147610,857420,534157,379767,907280,443374,230441,861524,599977,698775,83341,232478,899260,311637,65773,756688,283960,839052,706695,309663,110705,894962,765792\n", + "Ground Truth: 606232,281286,673374,264793,365399,34721,199555,147610,857420,534157,379767,907280,443374,230441,861524,599977,698775,83341,232478,899260,311637,65773,756688,283960,839052,463070,706695,309663,110705,894962\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 15\n", + "Returned IDs: 769951,643363,550440,454216,475406,85649,703283,664911,693483,103210,548164,757816,588446,13631,297165,615,5675,447505,797438,638004,19091,872921,854483,38444,643793,379767,902631,469341,135017,560935\n", + "Ground Truth: 769951,643363,550440,454216,206564,475406,85649,703283,664911,693483,103210,369753,548164,757816,588446,13631,297165,615,5675,447505,797438,638004,19091,872921,854483,38444,643793,379767,312098,902631\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 17\n", + "Returned IDs: \n", + "Ground Truth: 652336,409731,166392,972594,335201,405252,63145,602442,288817,233361,9024,809535,339223,952711,355866,255425,409130,998445,693697,585246,169807,451846,456861,452997,912561,379767,315231,879890,614606,30526\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 18\n", + "Returned IDs: 7554,777369,305207,50469,84975,18914,302347,92914,476498,640956,68616,571874,791872,643028,79772,245332,232478,412873,287704,80935,638004,83986,676833,94599,300415,167544,453446,162001,568276,63041\n", + "Ground Truth: 7554,777369,305207,50469,84975,18914,302347,92914,476498,640956,68616,571874,791872,643028,79772,245332,232478,412873,287704,80935,638004,83986,676833,94599,300415,167544,453446,162001,568276,153485\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 19\n", + "Returned IDs: 5819,463070,826126,544437,12387,544861,602557,23934,658397,305663,579250,761947,514080,469341,917766,940047,166392,940505,595330,972926,311016,783900,976404,117859,813664,632814,757634,344779,976507,687287\n", + "Ground Truth: 5819,463070,826126,544437,12387,544861,602557,23934,658397,305663,579250,761947,514080,469341,917766,940047,166392,940505,595330,972926,311016,783900,976404,117859,940691,813664,632814,757634,344779,976507\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 21\n", + "Returned IDs: 203353,595330,904507,772035,344779,427455,236271,411253,216243,189442,814015,494082,939981,454192,853066,844571,155550,921446,543653,482176,316751,436016,445843,497305,663199,371655,435655,41301,670380,316940\n", + "Ground Truth: 203353,595330,904507,772035,344779,427455,236271,411253,216243,103104,189442,814015,494082,939981,454192,832291,853066,844571,155550,921446,543653,482176,316751,564509,436016,445843,497305,663199,371655,435655\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 22\n", + "Returned IDs: 212360,381162,647935,862705,99717,83341,239718,730077,347041,623711,251650,359649,65773,974601,378429,443374,967189,441323,18651,294724,8238,607059,663376,573452,412471,951923,462345,354435,4690,47260\n", + "Ground Truth: 212360,381162,647935,862705,99717,83341,239718,730077,347041,623711,251650,359649,65773,974601,378429,443374,967189,441323,18651,294724,8238,607059,663376,573452,412471,951923,462345,354435,4690,816270\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 23\n", + "Returned IDs: 739092,183691,371655,996053,695965,486447,473819,378429,504643,225501,18529,670380,916247,700519,554056,932030,5675,476498,207488,552386,445370,263260,974941,747373,541811,729998,414862,48844,333703,597805\n", + "Ground Truth: 739092,183691,371655,996053,695965,486447,473819,378429,504643,225501,18529,670380,916247,700519,554056,932030,5675,476498,656276,207488,552386,445370,263260,974941,747373,541811,729998,414862,48844,333703\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 24\n", + "Returned IDs: 585034,101705,855217,290395,650367,85322,237494,897186,685917,86869,827573,971100,857783,364798,413202,340429,649929,952626,664624,166392,155972,894834,631733,155550,435655,871262,18914,576985,447505,407127\n", + "Ground Truth: 585034,101705,855217,290395,650367,85322,237494,897186,685917,86869,827573,971100,857783,364798,413202,340429,649929,952626,664624,256011,166392,48754,155972,894834,631733,155550,435655,871262,18914,576985\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 25\n", + "Returned IDs: 79971,319829,102502,737354,816365,269551,636425,961677,537644,893429,28590,408339,147951,12057,216428,374483,314296,948640,166392,256720,43081,826126,832609,23755,923534,836247,217463,379767,202057,380365\n", + "Ground Truth: 79971,319829,102502,737354,816365,269551,87204,636425,961677,537644,893429,28590,408339,147951,12057,216428,374483,314296,948640,166392,256720,830239,43081,542881,826126,832609,23755,923534,836247,217463\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 26\n", + "Returned IDs: 11868,743119,889023,479630,379478,603588,618559,43097,616251,790358,643702,9912,393110,312103,623692,489145,19687,48754,770124,220615,551028,857420,476932,14079,416923,626132,531095,796629,818007,323691\n", + "Ground Truth: 11868,743119,889023,479630,379478,603588,618559,43097,616251,790358,643702,9912,393110,623692,312103,489145,19687,48754,770124,651878,220615,551028,857420,476932,896406,14079,416923,626132,531095,796629\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 27\n", + "Returned IDs: 73368,168038,126043,631399,324863,18529,687634,851546,311604,675467,632080,235379,543879,927595,845326,133770,632814,72631,328230,965362,779055,830510,772035,451846,826126,653213,224067,683736,544437,453995\n", + "Ground Truth: 73368,168038,126043,631399,324863,18529,687634,851546,41301,311604,675467,632080,235379,543879,927595,845326,133770,632814,72631,328230,965362,779055,405219,830510,534157,772035,451846,826126,653213,224067\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 28\n", + "Returned IDs: 96316,564835,828466,7925,973356,256011,964592,791291,947151,311016,619250,80061,878069,504944,105354,822414,997496,841229,766202,251296,393944,158406,871262,585034,238088,206153,674336,492133,481624,972594\n", + "Ground Truth: 96316,564835,828466,7925,973356,256011,964592,791291,947151,311016,619250,80061,878069,504944,105354,822414,997496,841229,766202,251296,393944,158406,871262,585034,238088,903090,206153,674336,492133,481624\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 29\n", + "Returned IDs: 582477,745484,276603,477305,837946,772146,92915,21239,300183,831415,792485,940384,902871,950257,48813,88046,221330,561483,759986,848945,489975,827778,918389,742412,992979,393202,850390,828137,703283,486393\n", + "Ground Truth: 582477,489257,745484,276603,477305,837946,772146,55426,92915,21239,300183,831415,792485,940384,902871,950257,48813,88046,221330,561483,759986,848945,489975,827778,918389,742412,992979,393202,534447,850390\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 30\n", + "Returned IDs: 68616,763712,874173,220366,428592,187247,476498,171146,92914,217311,65388,791872,50469,659249,615,643363,851293,535419,7554,232478,18914,631177,85649,571874,681267,798845,173393,64898,393869,934693\n", + "Ground Truth: 68616,763712,874173,220366,428592,187247,476498,171146,92914,217311,65388,791872,50469,659249,615,643363,851293,535419,7554,232478,18914,631177,85649,571874,681267,798845,640740,173393,64898,393869\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 32\n", + "Returned IDs: 445843,904507,579250,670380,69280,413275,828137,737825,632708,94264,706695,67033,507158,839858,88319,464286,586039,210958,950257,413276,379767,631399,987690,309885,21239,417872,166392,645340,343498,357908\n", + "Ground Truth: 445843,904507,579250,670380,69280,413275,828137,737825,632708,94264,706695,67033,390336,507158,839858,88319,464286,586039,210958,950257,413276,379767,631399,987690,309885,21239,417872,166392,645340,343498\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 33\n", + "Returned IDs: 34721,490901,577512,899260,403345,443374,397738,80169,982943,650808,614821,412471,912407,47260,945335,881397,684037,365399,861524,175988,826126,115218,632080,958864,347041,336167,83341,547111,648624,805947\n", + "Ground Truth: 34721,490901,577512,899260,403345,443374,397738,80169,982943,650808,614821,412471,912407,47260,945335,881397,901831,684037,365399,861524,175988,826126,115218,632080,958864,462345,347041,336167,83341,547111\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 34\n", + "Returned IDs: 249501,497776,448520,170464,470623,954884,379478,406519,655585,230204,719438,164034,88740,983345,94582,758831,704903,72631,994452,954074,970437,674606,921716,631399,77412,571942,176615,19215,501526,629171\n", + "Ground Truth: 249501,497776,448520,170464,470623,954884,379478,406519,655585,230204,719438,164034,88740,983345,94582,758831,704903,72631,994452,954074,970437,674606,798845,921716,631399,77412,571942,176615,747404,19215\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 38\n", + "Returned IDs: 850030,379767,464940,626727,771765,932395,38444,916915,467871,612297,497305,88319,552386,473819,94582,225501,752362,476743,504643,670380,14050,373723,206694,309885,802512,357908,299325,63145,6153,131244\n", + "Ground Truth: 850030,379767,464940,626727,771765,932395,38444,916915,467871,612297,497305,88319,552386,473819,94582,225501,752362,476743,504643,670380,14050,373723,206694,309885,802512,357908,354656,299325,63145,6153\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 40\n", + "Returned IDs: 772035,214269,672661,169283,357908,797399,843163,405014,371655,190861,926388,525546,904507,445843,194718,615485,677600,503574,352330,22440,802512,737135,64273,344779,282975,3563,203353,740082,6084,921446\n", + "Ground Truth: 772035,214269,672661,169283,357908,797399,843163,405014,371655,190861,926388,525546,904507,562452,445843,194718,677600,615485,503574,352330,22440,802512,737135,64273,344779,777881,282975,3563,203353,740082\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 41\n", + "Returned IDs: 7554,838315,221414,676833,50469,14205,777369,155226,476498,604939,535046,461875,535419,878762,63041,302347,574445,135017,65388,851968,428592,812051,84975,653253,68616,547437,554907,860068,643363,67449\n", + "Ground Truth: 7554,838315,221414,676833,50469,14205,777369,155226,476498,604939,535046,461875,535419,878762,63041,302347,574445,135017,65388,851968,428592,812051,84975,653253,68616,547437,554907,860068,935154,643363\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 42\n", + "Returned IDs: 250895,21239,67033,417872,757034,69280,632640,10387,832291,152655,579250,158212,210958,299412,421218,561483,777636,665024,950257,158345,660351,670380,374017,6211,754108,710026,127351,759354,904519,166392\n", + "Ground Truth: 250895,21239,67033,417872,757034,69280,632640,10387,832291,152655,579250,158212,210958,299412,421218,561483,777636,665024,950257,158345,660351,670380,374017,6211,754108,710026,507123,127351,759354,904519\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 45\n", + "Returned IDs: 735520,745484,582477,541811,169048,5675,135017,230543,41450,832866,92915,848945,517119,643363,408318,447505,97135,473718,746324,160315,932030,564509,560935,494155,729710,521734,534447,378429,765929,313043\n", + "Ground Truth: 735520,745484,582477,541811,169048,5675,135017,230543,41450,832866,92915,848945,517119,643363,408318,447505,97135,473718,746324,160315,932030,564509,560935,494155,729710,521734,534447,828119,378429,765929\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 48\n", + "Returned IDs: 632080,18529,826126,785700,664447,683736,507158,514080,569165,672435,632814,176958,548164,879652,142868,772183,982943,445370,788045,600302,491253,544861,96536,225260,797438,443374,897988,967574,577876,405219\n", + "Ground Truth: 632080,18529,826126,785700,664447,683736,507158,514080,569165,672435,632814,176958,548164,879652,142868,772183,982943,445370,788045,600302,491253,544861,96536,225260,797438,443374,897988,771909,967574,577876\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 49\n", + "Returned IDs: 770242,251853,595241,554395,339090,436016,866849,961536,308145,807139,522824,107476,985773,949777,819432,502245,699616,618559,230204,290832,441607,945882,422454,64898,800420,919652,835425,48754,819636,151127\n", + "Ground Truth: 770242,251853,595241,554395,339090,436016,866849,961536,308145,807139,522824,107476,985773,949777,819432,502245,699616,443235,618559,230204,290832,441607,945882,422454,64898,800420,919652,835425,48754,819636\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 51\n", + "Returned IDs: 14061,994452,73368,789560,774962,180138,930566,514235,298323,222936,585177,133770,72631,772035,451846,960624,708578,803520,812194,45059,544437,233252,912393,391459,900272,69915,552536,236271,79103,295742\n", + "Ground Truth: 14061,994452,73368,789560,753044,774962,180138,930566,356168,514235,298323,222936,585177,133770,72631,772035,451846,960624,708578,803520,812194,45059,544437,233252,912393,391459,900272,69915,552536,236271\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 52\n", + "Returned IDs: \n", + "Ground Truth: 183691,849922,13631,5675,915297,486447,523990,560935,206564,19091,385517,745484,450215,580578,207488,94599,888980,18529,451846,472963,388778,643363,643793,26867,274473,386589,230470,378429,544861,660556\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 53\n", + "Returned IDs: 185309,680438,87686,720964,632814,926270,541811,982164,100503,341783,132941,354173,180244,600302,264174,612297,379767,98827,645340,632640,93212,43740,757816,481922,815877,67332,474041,757034,140706,198289\n", + "Ground Truth: 185309,680438,87686,720964,632814,926270,541811,982164,100503,341783,132941,354173,180244,600302,264174,612297,379767,98827,645340,632640,93212,43740,757816,481922,142868,739555,815877,67332,474041,757034\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 54\n", + "Returned IDs: 207041,652336,379767,884104,488003,783900,691762,735531,84473,732515,464286,345023,893919,712316,543216,212576,612297,834937,579250,81619,674899,294741,296625,166392,90307,100503,845747,30077,37639,447505\n", + "Ground Truth: 207041,652336,379767,884104,488003,783900,691762,735531,84473,732515,464286,345023,893919,712316,212576,543216,612297,834937,579250,81619,674899,294741,296625,166392,90307,100503,845747,30077,37639,15628\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 56\n", + "Returned IDs: 636425,213196,288413,256720,224067,799943,637050,209361,908520,51876,600302,927668,827552,154045,832609,40011,998221,457556,142868,148565,637450,266784,354173,548164,799937,38596,339768,967574,26089,408339\n", + "Ground Truth: 636425,213196,288413,256720,224067,799943,637050,209361,908520,51876,600302,927668,827552,154045,832609,40011,998221,457556,142868,148565,637450,266784,354173,548164,799937,38596,339768,862897,967574,26089\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 57\n", + "Returned IDs: 176219,619050,829365,722841,806837,921652,25722,885971,443302,735252,622920,179653,993115,683321,848907,781529,16209,856202,740178,183691,68616,952810,171146,390830,67449,362227,411599,851293,714118,427961\n", + "Ground Truth: 176219,619050,829365,722841,806837,921652,865411,25722,885971,443302,735252,622920,179653,993115,683321,848907,781529,16209,856202,740178,430561,183691,68616,952810,171146,390830,67449,362227,411599,851293\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 58\n", + "Returned IDs: \n", + "Ground Truth: 41301,415795,561189,209514,232674,502245,107476,595241,951427,429735,945617,282676,753675,290832,945882,777136,687257,151127,119012,389961,961536,541759,796555,807139,648655,695625,819432,807662,251853,292970\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 59\n", + "Returned IDs: 451846,608314,28015,453789,236271,214017,309885,670380,117859,607269,479819,18529,84965,463070,645340,808161,825868,660351,65773,409130,189365,920761,126043,453995,26637,337458,547104,831378,886952,166392\n", + "Ground Truth: 451846,608314,28015,453789,236271,214017,309885,670380,117859,607269,479819,18529,84965,463070,645340,808161,825868,660351,65773,548164,409130,189365,920761,126043,453995,26637,337458,547104,831378,886952\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 60\n", + "Returned IDs: 155572,602043,587995,935499,650153,706695,103210,588446,94582,765561,733126,379767,876126,37002,381389,465190,548164,62820,121404,61856,799967,303352,908344,343268,464820,815877,644347,797438,382527,158639\n", + "Ground Truth: 155572,602043,587995,935499,650153,706695,103210,588446,94582,765561,733126,418369,379767,876126,37002,381389,465190,410267,548164,62820,121404,61856,799967,303352,230358,908344,343268,464820,815877,644347\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 61\n", + "Returned IDs: 316175,451709,581151,311068,447505,932030,468935,657935,230132,898791,764836,683549,543216,642021,463206,324937,735531,406380,318010,376992,364869,830239,585438,166392,604149,924661,2907,5675,700519,712636\n", + "Ground Truth: 316175,451709,581151,311068,447505,932030,468935,657935,230132,898791,764836,683549,543216,642021,463206,81020,324937,735531,406380,318010,842095,376992,364869,830239,585438,166392,604149,924661,2907,5675\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 62\n", + "Returned IDs: 180244,18529,413276,435655,626132,576985,379767,997496,14717,828137,541759,167515,445370,33942,522760,695965,479167,972594,702181,931780,600530,581151,24196,139298,343498,706695,735531,88319,552386,544861\n", + "Ground Truth: 180244,18529,413276,435655,626132,576985,379767,997496,14717,828137,541759,167515,445370,33942,522760,695965,479167,972594,702181,931780,600530,581151,24196,139298,343498,706695,735531,88319,552386,345023\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 65\n", + "Returned IDs: 834937,775428,706880,359649,802512,327683,364526,183537,770486,714118,112463,105355,470066,533939,357403,371655,177766,134420,498815,583350,619853,265408,357908,885971,677600,735531,447505,445370,544861,770401\n", + "Ground Truth: 834937,775428,706880,359649,802512,327683,364526,183537,770486,714118,112463,105355,470066,125135,533939,357403,371655,927148,177766,134420,498815,583350,619853,265408,357908,885971,677600,735531,447505,445370\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 66\n", + "Returned IDs: 467212,439147,708881,134420,729710,758338,918746,18529,976404,533939,13631,11461,583350,358859,670380,585177,775428,637465,530372,178358,495244,35443,469341,742370,677600,730997,858729,454216,194718,451846\n", + "Ground Truth: 467212,439147,708881,134420,729710,758338,918746,18529,976404,533939,13631,11461,583350,358859,670380,284349,585177,775428,637465,530372,178358,495244,35443,469341,742370,677600,730997,858729,454216,194718\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 69\n", + "Returned IDs: 917766,279054,21239,916915,918389,531095,612297,632814,804808,41301,26637,311016,417872,91663,893451,152655,670380,974941,373723,710026,940047,785700,265421,643793,600530,982993,813664,392449,879890,632080\n", + "Ground Truth: 917766,21239,279054,916915,918389,531095,612297,632814,804808,41301,26637,311016,417872,91663,893451,152655,670380,974941,373723,710026,940047,785700,265421,643793,600530,982993,813664,392449,486639,879890\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 70\n", + "Returned IDs: 146294,780650,369554,235267,874072,796405,27634,360005,649772,717859,428741,862039,221891,508562,591880,856773,534299,990435,953310,258029,275992,178400,712316,173843,109720,702553,357010,261414,408339,234369\n", + "Ground Truth: 146294,780650,369554,235267,874072,796405,27634,360005,649772,717859,428741,862039,221891,508562,591880,856773,534299,990435,826244,953310,258029,275992,178400,712316,173843,109720,702553,357010,261414,408339\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 71\n", + "Returned IDs: 327683,874438,848945,393202,735531,976404,245559,237494,938312,531095,788045,357403,95342,616251,998445,729998,967574,400877,112463,589289,621454,409731,554056,296053,687287,834937,910768,631399,18529,600530\n", + "Ground Truth: 327683,874438,848945,393202,735531,976404,245559,237494,938312,531095,788045,357403,95342,616251,998445,729998,967574,400877,181413,112463,589289,621454,409731,554056,296053,687287,834937,910768,631399,18529\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 75\n", + "Returned IDs: 587475,214269,744848,73156,362244,493436,739353,544589,371655,525546,3563,549510,572761,710026,373729,729998,561146,564700,445843,120980,333703,560714,554056,405014,269375,443127,944204,579250,396777,775428\n", + "Ground Truth: 587475,214269,744848,73156,362244,493436,739353,544589,371655,525546,3563,549510,572761,373729,710026,729998,561146,994542,564700,445843,120980,333703,560714,554056,405014,587721,269375,3397,443127,944204\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 76\n", + "Returned IDs: 942506,33361,946518,469341,364886,363525,379478,88740,393168,652155,693483,446499,857420,292065,907254,89030,64898,634400,258100,497776,343268,232478,976404,234369,924273,706968,180880,683549,749769,830270\n", + "Ground Truth: 942506,33361,946518,469341,364886,363525,379478,88740,393168,652155,693483,446499,857420,292065,907254,89030,64898,634400,258100,497776,343268,232478,976404,234369,924273,706968,180880,683549,749769,280671\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 77\n", + "Returned IDs: 343268,974601,871011,230358,239718,304643,327403,876126,402371,691063,232478,740741,698771,548164,254476,476932,378429,857420,903148,142868,459789,634000,381162,694245,826126,463070,83986,683413,839052,46900\n", + "Ground Truth: 343268,974601,871011,230358,239718,304643,327403,876126,402371,691063,232478,320419,740741,698771,548164,254476,476932,378429,857420,903148,142868,459789,901831,888976,634000,638004,381162,694245,826126,463070\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 79\n", + "Returned IDs: 322391,997843,316940,608244,681219,408751,886952,729998,375948,250007,349747,720943,333703,129088,683097,405014,976507,214269,43563,940691,686749,259018,701006,597161,63343,6680,473819,974941,445924,424069\n", + "Ground Truth: 322391,997843,316940,608244,681219,408751,886952,729998,375948,250007,349747,407250,720943,333703,129088,683097,405014,976507,214269,43563,940691,686749,259018,701006,597161,63343,6680,473819,974941,445924\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 80\n", + "Returned IDs: 612297,337185,393202,24196,98827,615339,998445,554056,63145,373723,612117,695965,450445,120980,729998,167515,6153,381462,910768,796968,225501,402047,582616,780978,990925,720943,579250,228302,552386,396777\n", + "Ground Truth: 612297,337185,393202,24196,98827,615339,998445,554056,63145,676869,373723,612117,695965,450445,120980,729998,167515,6153,381462,910768,796968,225501,402047,582616,780978,990925,720943,579250,228302,552386\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 81\n", + "Returned IDs: 128137,789826,241,163908,331075,529474,356119,778930,115722,342624,579250,105768,219321,102754,426496,478851,774724,536352,304921,855897,299325,343498,333703,890360,200560,12071,965132,815827,931681,63471\n", + "Ground Truth: 915301,128137,789826,241,163908,331075,529474,356119,778930,115722,342624,579250,701006,105768,219321,102754,426496,478851,774724,536352,304921,855897,299325,343498,333703,890360,200560,12071,965132,815827\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 83\n", + "Returned IDs: \n", + "Ground Truth: 100503,79971,137292,132941,795691,228850,187685,23755,712316,4425,702096,97402,817938,932033,645340,12057,881364,16878,294741,857887,43266,374483,167474,632080,845747,784568,724881,660886,433808,632814\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 84\n", + "Returned IDs: 916915,602442,507158,115722,41301,97858,473819,579250,695553,64898,73368,964592,524973,544861,544437,316940,687287,867184,18830,572761,7312,878069,804808,126043,492133,594577,342624,29560,903465,342834\n", + "Ground Truth: 916915,602442,507158,115722,41301,97858,473819,579250,695553,64898,73368,964592,524973,544861,544437,316940,834937,687287,867184,18830,572761,7312,878069,804808,126043,492133,594577,342624,29560,903465\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 86\n", + "Returned IDs: 893494,582477,408318,669722,186375,447505,837640,682356,755815,502590,355239,482569,847054,379478,10915,662929,315425,813664,135017,417861,4512,141284,654485,702096,236959,996028,541811,284436,325191,389553\n", + "Ground Truth: 893494,582477,408318,669722,186375,447505,837640,682356,755815,502590,355239,482569,92136,847054,379478,10915,662929,15628,315425,813664,135017,417861,4512,141284,654485,702096,236959,861057,765929,996028\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 87\n", + "Returned IDs: 855908,450445,916915,9024,94963,337185,288817,882066,441751,809535,797895,454064,633831,904507,393202,814886,102243,998445,63145,381462,990925,299325,575450,878069,6153,24196,396777,773701,169807,525546\n", + "Ground Truth: 855908,450445,916915,9024,94963,337185,288817,882066,441751,809535,797895,454064,633831,904507,393202,814886,102243,998445,63145,381462,990925,299325,575450,878069,6153,24196,396777,226695,773701,169807\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 88\n", + "Returned IDs: 150633,64273,507123,316940,404975,375318,165630,819223,333703,511963,770462,371655,54633,671162,184048,274079,124402,378362,209632,210167,944204,650478,872486,340206,880836,170539,445843,585438,558539,207937\n", + "Ground Truth: 150633,64273,507123,316940,404975,375318,165630,819223,333703,511963,770462,371655,54633,671162,184048,274079,124402,378362,209632,210167,944204,650478,850030,872486,340206,880836,170539,445843,585438,558539\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 89\n", + "Returned IDs: 225706,479254,65550,714134,42182,683097,28151,745185,885272,282531,371655,498312,677809,632814,926270,445370,686408,698050,264334,976915,779585,714118,758960,729998,166200,259054,916915,285666,350257,996053\n", + "Ground Truth: 225706,479254,65550,714134,42182,683097,28151,745185,885272,282531,371655,498312,677809,632814,926270,445370,686408,698050,264334,976915,779585,714118,758960,729998,166200,259054,458408,916915,285666,350257\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 91\n", + "Returned IDs: 502001,299027,691997,166392,996028,561835,728241,643793,664447,925914,142868,135017,469341,492260,548164,700352,797438,514643,106582,789801,18529,236959,757816,632080,64898,65773,643363,467212,514080,854483\n", + "Ground Truth: 502001,299027,691997,166392,996028,561835,728241,643793,664447,925914,142868,135017,469341,492260,847918,548164,700352,797438,514643,106582,789801,18529,236959,757816,632080,64898,65773,643363,467212,514080\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 92\n", + "Returned IDs: 476498,966463,84975,72631,600302,339187,135017,693174,311604,35812,7554,779055,266466,298323,597589,187105,275549,343268,631399,589282,249700,211556,750654,142868,577468,357314,967663,729163,880573,385517\n", + "Ground Truth: 476498,966463,84975,72631,600302,339187,135017,693174,311604,35812,7554,779055,266466,298323,597589,187105,275549,343268,631399,589282,249700,211556,750654,142868,577468,357314,967663,729163,880573,976239\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 94\n", + "Returned IDs: 139298,772035,130172,986603,837825,978689,236271,60713,213575,917766,544861,484173,686408,138910,950893,476683,264045,337224,710026,451846,920761,131053,775428,687287,199796,938584,166392,115722,585177,631399\n", + "Ground Truth: 139298,772035,130172,986603,837825,978689,236271,60713,213575,917766,544861,484173,686408,138910,950893,476683,264045,337224,710026,451846,920761,131053,775428,687287,199796,469341,938584,166392,115722,585177\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 95\n", + "Returned IDs: 18830,779902,745484,632814,477417,224067,612297,963711,393202,844571,166392,916915,447505,598044,41301,207041,672661,232600,759354,643363,996028,64898,659407,98827,492133,541811,703283,469341,814001,631399\n", + "Ground Truth: 18830,779902,745484,632814,61461,477417,224067,612297,963711,393202,844571,166392,916915,447505,598044,41301,207041,672661,232600,759354,903465,643363,996028,274079,64898,659407,98827,492133,541811,703283\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 97\n", + "Returned IDs: 100503,551028,250895,751087,759354,166392,904635,79971,707601,796968,579250,645340,198289,135278,754108,299412,665024,775370,417872,445843,228850,97858,6211,712316,21239,879890,688125,456861,47411,265421\n", + "Ground Truth: 100503,551028,250895,751087,759354,166392,904635,79971,707601,796968,579250,645340,198289,135278,754108,299412,665024,775370,417872,445843,228850,97858,158342,6211,712316,21239,879890,688125,456861,47411\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 101\n", + "Returned IDs: 984194,712316,418997,207041,384934,932033,900301,732515,433808,173843,408339,142273,142868,882914,187655,632080,23755,816481,945780,664447,379767,876110,940505,856773,927346,845747,65773,462345,100503,672125\n", + "Ground Truth: 984194,712316,418997,207041,384934,932033,900301,732515,433808,173843,408339,142273,142868,882914,29774,187655,386486,632080,23755,816481,945780,664447,379767,876110,940505,856773,927346,845747,65773,462345\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 102\n", + "Returned IDs: 721382,319288,906512,756114,818897,531095,457668,874438,800801,282213,473460,541811,760641,445483,626132,232795,748959,490297,505142,87981,623687,989157,265061,669722,342926,577280,96037,268670,544401,764838\n", + "Ground Truth: 721382,319288,906512,756114,818897,531095,457668,874438,800801,473460,282213,541811,760641,445483,626132,232795,748959,490297,505142,784961,87981,623687,989157,265061,669722,342926,577280,96037,268670,544401\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 103\n", + "Returned IDs: 760970,516493,479197,664447,449334,893919,142868,451708,900301,601998,925914,4765,237148,561835,483807,746633,826126,548164,393759,87686,82336,996028,577876,166392,18529,319829,319630,897240,502001,224067\n", + "Ground Truth: 760970,516493,479197,664447,449334,893919,142868,451708,900301,601998,925914,4765,237148,561835,483807,746633,826126,548164,393759,87686,82336,996028,577876,166392,676582,18529,319829,319630,897240,502001\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 105\n", + "Returned IDs: 507123,554056,306280,93883,773701,225501,150633,3563,320831,27280,371655,916915,393202,64273,185932,579250,646462,928872,614675,91728,944204,939774,304318,378362,227025,445843,333703,103442,120980,404975\n", + "Ground Truth: 507123,554056,306280,93883,773701,225501,150633,3563,320831,27280,371655,916915,393202,64273,185932,579250,646462,928872,7203,614675,91728,944204,939774,304318,378362,227025,445843,333703,103442,120980\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 106\n", + "Returned IDs: 602442,988176,38444,333703,554056,886952,363525,405252,371655,710026,143117,259018,917766,344295,488003,729998,612297,91663,579250,124402,41301,473819,670380,904507,417872,465498,445843,514080,802512,683097\n", + "Ground Truth: 602442,988176,38444,333703,554056,886952,363525,405252,371655,710026,143117,259018,917766,344295,488003,729998,612297,91663,579250,124402,41301,473819,670380,264045,904507,417872,465498,445843,514080,802512\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 107\n", + "Returned IDs: 759354,632640,632814,612297,284708,916915,379767,360986,104851,777636,982164,779902,488003,945780,258356,745484,785700,579250,844571,959763,561483,652336,33942,695965,341783,6211,734241,221330,955379,21239\n", + "Ground Truth: 759354,632640,632814,612297,832291,284708,916915,379767,360986,104851,777636,982164,779902,488003,945780,258356,745484,785700,579250,844571,959763,561483,652336,33942,512544,695965,341783,232002,6211,734241\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 108\n", + "Returned IDs: 407004,367145,263842,268761,145514,828009,947990,443374,819736,65773,79971,265408,185240,363784,712316,403345,845747,366544,973264,506971,617507,871262,299027,32213,348013,865901,83341,557130,623711,504324\n", + "Ground Truth: 407004,367145,263842,268761,145514,828009,947990,154098,443374,819736,65773,79971,265408,185240,363784,712316,403345,845747,366544,973264,506971,617507,311068,871262,299027,32213,348013,865901,83341,557130\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 109\n", + "Returned IDs: 801132,671854,785700,915692,266784,612297,803520,796968,320419,804808,739507,525334,253845,48965,571942,272687,683736,963711,166392,728837,142868,762693,600302,597589,631399,902366,69609,224067,4439,806445\n", + "Ground Truth: 801132,671854,785700,915692,266784,612297,803520,796968,320419,804808,739507,525334,253845,48965,571942,739555,272687,683736,342926,963711,166392,728837,142868,762693,600302,597589,631399,902366,69609,224067\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 110\n", + "Returned IDs: 388670,772998,801132,34523,613133,785700,313747,652155,756114,892610,204003,934124,763544,366055,916247,789801,906623,897988,972926,84233,541980,463206,656749,977282,924669,995102,510910,746633,944060,683736\n", + "Ground Truth: 388670,772998,801132,34523,613133,785700,313747,652155,756114,892610,204003,934124,763544,366055,916247,789801,906623,897988,458408,972926,439503,84233,541980,463206,656749,977282,924669,995102,510910,746633\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 111\n", + "Returned IDs: 64898,876126,913486,587995,548164,310435,469341,854483,303352,343268,82273,953345,758267,228850,764656,990435,353167,364886,272209,13747,42809,797438,275549,153836,632080,993163,894731,673519,502001,912407\n", + "Ground Truth: 64898,876126,913486,587995,548164,310435,469341,854483,303352,343268,82273,953345,758267,228850,764656,990435,353167,364886,272209,13747,42809,797438,826126,275549,153836,632080,993163,894731,673519,502001\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 112\n", + "Returned IDs: 414215,350257,974849,76673,282531,369956,956035,702096,193485,467212,612297,337596,998445,492005,656276,763544,715091,850390,692188,445370,759354,729998,18529,286691,937944,469341,486447,670380,447505,341783\n", + "Ground Truth: 414215,350257,974849,76673,282531,369956,956035,702096,193485,467212,185931,612297,408962,337596,998445,492005,772511,656276,763544,715091,850390,692188,205204,445370,759354,729998,18529,67332,286691,937944\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 113\n", + "Returned IDs: 233361,494866,895648,166392,209887,798058,734193,698668,303917,495445,4810,712316,468049,23755,382494,393420,198289,702283,150886,990791,785474,204354,879890,670380,962777,754108,474180,610912,79971,623200\n", + "Ground Truth: 233361,494866,895648,166392,209887,798058,734193,698668,303917,495445,4810,712316,468049,23755,382494,393420,198289,702283,150886,990791,785474,941576,204354,879890,670380,962777,754108,474180,610912,79971\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 115\n", + "Returned IDs: 370134,870371,91663,904519,6153,688125,683097,556108,598946,759354,10387,475187,904635,21239,690733,405252,307167,9024,492133,526403,496966,407127,891494,265421,955841,916915,745185,524973,446450,561483\n", + "Ground Truth: 370134,870371,91663,926388,904519,6153,688125,683097,556108,598946,97973,759354,360299,10387,475187,904635,21239,690733,405252,307167,9024,492133,526403,496966,407127,515938,891494,265421,955841,916915\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 118\n", + "Returned IDs: 309885,830735,453995,402047,579250,519324,392449,597805,554056,120980,500463,373723,627144,832291,417872,298415,29560,510927,445843,91663,371655,632708,214269,662701,443127,916915,835672,572761,242065,405252\n", + "Ground Truth: 309885,830735,453995,402047,579250,519324,392449,597805,554056,120980,500463,373723,627144,832291,417872,298415,29560,510927,445843,91663,371655,632708,214269,662701,443127,916915,835672,572761,152655,242065\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 119\n", + "Returned IDs: 988176,415578,369956,956035,971257,926388,172629,507840,417928,407250,689858,131045,976036,536226,841254,360299,359315,729998,604832,772035,660351,473819,998445,646462,221292,476683,959480,748087,873535,519324\n", + "Ground Truth: 988176,415578,369956,956035,971257,926388,172629,507840,417928,407250,689858,131045,976036,536226,841254,360299,359315,729998,604832,772035,660351,473819,998445,646462,221292,476683,959480,511895,748087,873535\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 121\n", + "Returned IDs: 695965,350257,282531,621301,878069,498312,479167,710026,600530,916915,371655,158345,88319,814886,166392,529324,850275,18529,454064,185309,632205,523569,324863,875052,311197,473819,927595,456861,396777,357908\n", + "Ground Truth: 695965,350257,282531,621301,878069,498312,479167,710026,600530,916915,371655,158345,88319,814886,166392,529324,850275,585177,18529,454064,185309,632205,523569,126043,324863,875052,311197,473819,927595,456861\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 122\n", + "Returned IDs: 35443,918746,495244,645299,410377,967181,858865,583564,637465,729998,753971,65199,710026,976404,357268,290966,513181,615485,813664,544861,134420,363525,319776,143117,677600,221330,388778,166392,259018,861541\n", + "Ground Truth: 35443,918746,495244,645299,410377,967181,858865,583564,637465,729998,753971,65199,710026,976404,357268,290966,513181,615485,813664,544861,134420,8501,363525,319776,143117,677600,221330,388778,166392,259018\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 125\n", + "Returned IDs: 754108,712316,624723,79971,173863,319829,114932,625806,166392,509073,392918,845747,752443,462345,464357,314296,773822,537644,382494,945780,932033,83341,474180,494866,207041,940505,286585,65773,100503,61856\n", + "Ground Truth: 754108,712316,624723,79971,173863,319829,114932,625806,166392,509073,392918,845747,752443,462345,368219,464357,314296,773822,537644,382494,945780,932033,83341,474180,494866,207041,940505,286585,65773,100503\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 127\n", + "Returned IDs: 549510,763544,292065,861541,729998,543859,469341,833707,756114,98827,602557,341783,376992,402987,379767,739725,269375,734241,612297,445370,686408,488003,802512,801132,350257,534044,613133,350669,789801,919506\n", + "Ground Truth: 549510,763544,292065,861541,729998,543859,458408,469341,833707,756114,98827,602557,29964,490286,341783,376992,402987,379767,739725,269375,734241,612297,445370,686408,488003,802512,801132,350257,534044,613133\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 128\n", + "Returned IDs: 425354,938312,599943,561318,726269,412721,9912,784448,591193,811153,272687,737809,623692,796629,616251,621622,950869,735531,281122,531095,600530,998026,474764,71455,754497,613045,230470,679202,440224,496225\n", + "Ground Truth: 425354,938312,599943,561318,726269,412721,9912,784448,591193,811153,272687,737809,623692,796629,616251,621622,950869,735531,281122,531095,600530,548164,998026,474764,71455,754497,613045,230470,679202,440224\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 129\n", + "Returned IDs: \n", + "Ground Truth: 743681,671196,370518,830889,73156,430477,505804,797286,822466,283734,760653,773701,164782,24851,498925,150633,188112,73673,3568,558062,392618,861074,707311,693371,647801,32755,404975,962207,894086,967720\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 130\n", + "Returned IDs: 597325,952965,304643,797438,435078,663219,520805,194080,949708,634000,548164,989555,13747,827552,393551,464820,434990,657493,155572,820124,741137,565810,237148,700519,627018,205839,813236,456640,600333,230441\n", + "Ground Truth: 597325,952965,304643,797438,435078,663219,520805,194080,949708,634000,548164,989555,13747,827552,603588,393551,464820,434990,578570,657493,155572,820124,741137,565810,237148,700519,627018,205839,813236,456640\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 131\n", + "Returned IDs: 777646,330823,700275,468935,778612,88916,724205,294057,778134,95826,748194,613195,790358,811835,11461,80104,348940,63041,742412,98935,516989,714118,83811,791872,631177,730420,362227,265408,523911,541811\n", + "Ground Truth: 777646,330823,700275,468935,778612,88916,724205,294057,778134,95826,748194,613195,790358,811835,11461,80104,348940,63041,742412,98935,516989,714118,83811,791872,631177,730420,362227,116922,265408,523911\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 132\n", + "Returned IDs: 28551,4493,764472,640584,166392,662929,522760,762693,242938,564509,331195,643363,50104,595698,632417,586293,266075,270641,543216,380901,828137,706880,683549,479774,918389,849922,786565,141367,211528,785700\n", + "Ground Truth: 28551,4493,764472,640584,166392,662929,522760,762693,242938,564509,331195,643363,50104,813520,595698,632417,586293,266075,270641,543216,380901,828137,706880,683549,479774,918389,849922,786565,141367,211528\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 133\n", + "Returned IDs: 479774,15628,755815,84043,64898,828137,643667,513181,88319,643363,447505,975590,757816,166392,221330,900720,673607,15988,544861,560935,211528,691762,345023,849922,256484,664911,120704,313043,5525,721836\n", + "Ground Truth: 479774,15628,755815,84043,64898,828137,643667,513181,992979,88319,643363,447505,975590,757816,166392,221330,900720,673607,15988,544861,560935,211528,691762,345023,849922,256484,92136,664911,120704,313043\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 135\n", + "Returned IDs: \n", + "Ground Truth: 234552,258029,825802,702283,879890,645340,785474,407127,142146,104181,612002,895648,112851,954554,185309,560044,801035,334703,826244,100503,151457,813817,228850,163610,166392,255425,589794,853894,693668,21761\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 136\n", + "Returned IDs: 121404,797438,744631,303352,103210,363784,815877,483807,275549,600333,963474,548164,465190,565810,794954,809597,357314,194080,563956,783367,230441,378429,650153,996028,18529,41699,454519,460371,303085,1011\n", + "Ground Truth: 121404,797438,744631,303352,103210,363784,815877,483807,275549,600333,963474,548164,465190,565810,794954,809597,357314,194080,563956,783367,230441,249700,378429,650153,996028,18529,41699,454519,460371,303085\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 137\n", + "Returned IDs: 582616,63145,203351,9024,288817,638708,197146,707311,272290,418915,24196,882066,839858,792691,94963,376815,832882,916915,955841,695965,185898,42567,612117,258356,469984,396777,610912,357908,399872,941576\n", + "Ground Truth: 582616,63145,203351,9024,288817,638708,197146,707311,272290,418915,24196,882066,839858,792691,94963,376815,832882,916915,955841,695965,185898,42567,612117,258356,469984,396777,610912,243546,357908,569477\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 138\n", + "Returned IDs: 313747,364886,463206,645316,763544,469341,135821,7554,33361,502001,24110,272930,470415,748833,378429,846063,289778,773964,598044,577445,561835,16508,643028,902521,459789,695553,571874,878762,142868,838315\n", + "Ground Truth: 313747,364886,463206,645316,763544,469341,135821,7554,320419,33361,502001,24110,272930,470415,748833,378429,846063,289778,773964,598044,577445,561835,16508,643028,902521,459789,695553,571874,878762,142868\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 139\n", + "Returned IDs: 957713,698775,899260,574793,664447,874438,447505,687287,565810,731160,22871,813664,523569,379767,982164,230291,336167,488003,585438,350669,507158,281286,300183,945780,516184,80169,365399,753890,706695,135278\n", + "Ground Truth: 957713,698775,899260,574793,664447,874438,447505,687287,565810,731160,22871,813664,523569,379767,982164,230291,336167,488003,585438,350669,507158,281286,763544,300183,945780,516184,80169,365399,753890,706695\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 140\n", + "Returned IDs: 402047,916915,172701,184048,18830,492133,529324,124402,672661,721575,804808,616554,265421,40011,882806,364319,695965,575389,371655,837339,612297,309885,152655,995884,725978,6153,259018,63145,360986,602442\n", + "Ground Truth: 402047,916915,172701,184048,18830,492133,529324,124402,672661,721575,804808,616554,265421,40011,882806,364319,695965,575389,371655,612297,837339,309885,3563,152655,995884,725978,6153,259018,63145,360986\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 141\n", + "Returned IDs: 650588,668176,666791,463657,828466,226695,307166,637049,504404,855897,128809,871262,753869,585246,238088,155550,564835,80061,96316,812245,417948,223552,196092,890509,710060,80107,502919,775128,505995,343498\n", + "Ground Truth: 650588,668176,666791,463657,828466,226695,307166,637049,504404,855897,128809,871262,753869,585246,238088,155550,564835,80061,96316,812245,756965,935749,417948,223552,196092,890509,710060,80107,502919,775128\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 142\n", + "Returned IDs: 418915,63145,9024,288817,792691,335201,376815,203703,169807,399872,652336,203351,582616,569477,615339,450445,362380,612117,102243,998445,695965,612297,393202,579250,638708,769082,645340,374244,109016,24196\n", + "Ground Truth: 418915,63145,9024,288817,792691,335201,376815,203703,169807,399872,652336,203351,582616,569477,615339,450445,362380,612117,102243,998445,695965,612297,393202,579250,638708,769082,645340,826126,374244,109016\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 144\n", + "Returned IDs: 115855,102754,572761,619009,947151,538547,890360,791684,206153,238088,105354,668817,810411,115722,855897,313302,746059,95761,947950,960591,60713,322603,463657,324863,150592,297731,344295,342624,731618,919726\n", + "Ground Truth: 115855,102754,572761,619009,947151,538547,890360,791684,206153,238088,105354,668817,810411,115722,855897,313302,746059,95761,947950,960591,60713,322603,463657,324863,650588,150592,297731,344295,731618,342624\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 148\n", + "Returned IDs: 619853,507158,904507,735994,88319,710026,564509,445843,336977,435655,357908,309885,482176,996693,41099,41301,259018,402790,76036,382030,94264,677600,166392,670380,931681,34947,85322,13656,435000,141715\n", + "Ground Truth: 619853,507158,904507,735994,88319,710026,564509,445843,336977,435655,357908,309885,482176,996693,41099,41301,259018,402790,76036,530372,382030,94264,677600,166392,658210,670380,931681,34947,85322,13656\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 149\n", + "Returned IDs: 170989,403345,466219,828589,175621,771091,49331,796405,897240,304164,912407,850368,831132,765561,379767,561835,987344,614935,65773,955379,73777,717859,368888,35617,839858,397738,778760,818903,962546,493831\n", + "Ground Truth: 170989,403345,466219,828589,175621,771091,49331,796405,897240,304164,912407,850368,831132,765561,379767,975967,561835,987344,614935,65773,955379,73777,717859,368888,35617,925919,839858,397738,778760,818903\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 154\n", + "Returned IDs: 85322,565810,311016,539994,340429,235768,504324,585034,971100,900720,523569,567033,871262,689181,649772,432785,106582,639913,238846,80169,21501,447505,331608,212360,101705,299027,212060,919506,310435,407004\n", + "Ground Truth: 85322,565810,311016,539994,340429,235768,504324,807333,585034,971100,900720,523569,567033,871262,689181,649772,432785,106582,639913,238846,80169,21501,447505,331608,212360,101705,634279,299027,212060,919506\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 155\n", + "Returned IDs: 35407,759354,554056,579250,393202,50827,306280,253712,237494,615339,621454,445924,670380,140903,729998,91663,20434,734241,176751,344295,773701,612297,561483,450445,632640,381462,67033,91728,163554,315364\n", + "Ground Truth: 35407,759354,554056,579250,393202,50827,306280,748087,253712,237494,615339,178358,621454,445924,670380,140903,729998,91663,20434,734241,176751,344295,773701,612297,561483,450445,632640,381462,67033,91728\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 156\n", + "Returned IDs: 755577,664447,350669,122120,920983,66984,557561,951496,820124,523569,116625,744935,729710,205204,232377,909390,18529,369097,492260,211058,477417,451708,467212,245332,443374,447546,548164,276798,799967,113563\n", + "Ground Truth: 755577,664447,350669,122120,920983,66984,557561,951496,820124,523569,116625,744935,147502,729710,205204,232377,909390,18529,369097,492260,211058,477417,660351,451708,467212,245332,726544,443374,447546,548164\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 158\n", + "Returned IDs: 396777,69280,988352,296478,316940,206694,170539,850030,203353,414862,150633,771765,597161,918389,299412,241,445843,735994,497305,50102,331075,701006,14050,853066,357908,785700,450445,73673,73048,148986\n", + "Ground Truth: 396777,69280,988352,296478,316940,206694,170539,850030,138910,203353,414862,150633,771765,597161,918389,299412,241,445843,616554,735994,497305,50102,331075,701006,14050,853066,357908,785700,450445,73673\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 161\n", + "Returned IDs: \n", + "Ground Truth: 551059,492133,916915,402047,330370,479167,855908,850030,366682,990925,158345,695965,454064,404975,225501,172701,773701,332080,344295,373723,529474,902031,135657,63145,266404,435655,575389,114822,206153,288817\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 162\n", + "Returned IDs: 37639,62121,664911,325191,534803,784568,408339,406380,119735,712316,560360,166392,736060,142273,581151,156060,769082,837640,353484,447505,420265,757816,193493,379767,542574,761593,266075,95342,781200,347177\n", + "Ground Truth: 37639,62121,664911,325191,534803,784568,408339,406380,119735,712316,750188,560360,166392,736060,142273,581151,156060,769082,837640,353484,447505,420265,757816,193493,394343,379767,542574,761593,266075,95342\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 163\n", + "Returned IDs: 996028,865474,948300,421446,780978,760970,183865,951211,166392,254476,576985,550602,590195,652336,152860,909278,327683,784568,94582,764472,938786,464820,726308,940443,155572,449334,548164,781200,12487,613195\n", + "Ground Truth: 996028,865474,948300,421446,780978,760970,183865,951211,166392,254476,576985,550602,590195,652336,152860,847918,909278,327683,784568,94582,764472,938786,464820,726308,940443,155572,449334,548164,781200,19091\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 164\n", + "Returned IDs: 969153,183883,26637,805036,418021,932797,527877,955878,599943,667450,229596,274281,293149,364404,941183,940047,300728,593512,592165,71455,902521,35312,270589,30398,189365,20341,834937,354173,311016,581559\n", + "Ground Truth: 969153,183883,26637,805036,418021,932797,527877,955878,599943,667450,229596,274281,293149,602557,364404,941183,940047,300728,593512,592165,71455,902521,38248,35312,270589,30398,189365,20341,834937,354173\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 165\n", + "Returned IDs: 535663,744404,120704,109135,469341,92915,677600,289778,384909,135017,284349,447505,619853,585438,735520,363525,64898,84043,386589,701447,15988,687287,311387,849922,583350,296625,643793,691762,834937,337126\n", + "Ground Truth: 535663,744404,120704,109135,469341,92915,677600,289778,384909,135017,284349,447505,619853,585438,735520,363525,64898,84043,386589,249528,701447,15988,687287,311387,849922,583350,296625,643793,691762,834937\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 166\n", + "Returned IDs: 87686,826126,376992,612297,142868,548164,343268,310435,632814,784568,632080,441323,445370,364886,46160,111147,408339,853903,602557,893919,876126,871011,135017,283397,358859,797438,299027,615,121403,789801\n", + "Ground Truth: 87686,826126,376992,648098,612297,142868,548164,343268,310435,632814,978956,784568,632080,441323,445370,364886,46160,111147,408339,853903,602557,893919,876126,871011,135017,283397,358859,797438,299027,615\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 168\n", + "Returned IDs: 408339,632080,900301,643363,932033,826126,784568,166392,712316,815388,757816,319829,64898,353622,830239,43081,207041,612297,425093,853903,893919,447505,174389,664447,232600,797438,358859,87686,604597,79971\n", + "Ground Truth: 408339,632080,900301,643363,932033,826126,784568,166392,712316,815388,757816,319829,64898,353622,830239,43081,207041,612297,425093,853903,893919,447505,174389,664447,797438,232600,799399,358859,87686,604597\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 169\n", + "Returned IDs: 15045,379767,115080,94582,908539,860454,839858,137310,61856,413275,706695,550440,576985,843427,592165,264793,483079,809728,536683,343310,989476,216367,234140,464820,281286,983345,884481,911007,450140,173078\n", + "Ground Truth: 15045,379767,115080,94582,908539,860454,839858,137310,61856,413275,706695,550440,576985,843427,592165,264793,483079,809728,536683,343310,989476,216367,234140,464820,281286,577035,983345,884481,544542,911007\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 171\n", + "Returned IDs: \n", + "Ground Truth: 632814,763544,845243,142868,833707,600302,350669,455789,785700,543859,631399,577876,844571,925914,342174,341783,801132,166392,319630,572761,354173,708492,751839,659598,984170,612297,225260,972926,236959,128123\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 173\n", + "Returned IDs: 245559,281286,304643,576985,637811,40011,785700,887624,851968,163106,631399,680128,181413,719438,108363,570591,64898,299027,339050,469341,94599,683736,804038,224067,507158,706695,956107,980242,232478,343268\n", + "Ground Truth: 245559,281286,304643,576985,637811,40011,785700,887624,851968,163106,631399,680128,181413,719438,108363,570591,512544,64898,299027,339050,469341,94599,683736,804038,224067,507177,507158,706695,956107,980242\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 174\n", + "Returned IDs: 361000,215179,261509,678945,215451,502403,341,233361,295553,166392,231564,113086,879890,18793,602657,166804,112851,301135,305307,35786,420265,236023,716947,162962,387488,867149,783900,260847,564509,655099\n", + "Ground Truth: 361000,215179,261509,678945,215451,502403,341,233361,295553,166392,231564,113086,879890,18793,602657,166804,112851,301135,305307,35786,420265,236023,716947,162962,387488,867149,783900,812389,260847,677600\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 175\n", + "Returned IDs: 500463,926270,431046,324851,12387,379767,94582,695965,706695,795002,350669,78624,541811,27876,235467,445370,87686,864983,849922,918943,612297,417872,269375,83341,665024,632814,350257,692188,561483,483807\n", + "Ground Truth: 500463,926270,431046,324851,12387,379767,94582,904507,695965,706695,795002,350669,78624,541811,27876,235467,445370,87686,864983,849922,918943,612297,417872,269375,83341,665024,632814,350257,331608,597589\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 176\n", + "Returned IDs: 544861,126043,18529,464286,631399,41301,904507,597589,608314,607269,687287,762693,445843,796968,397196,828137,166392,409731,242065,510927,96536,522760,507158,12387,849922,97858,632080,99285,670380,180244\n", + "Ground Truth: 544861,126043,18529,464286,631399,41301,904507,597589,608314,607269,687287,762693,445843,796968,397196,828137,166392,409731,242065,510927,96536,522760,507158,12387,849922,339223,97858,632080,99285,670380\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 177\n", + "Returned IDs: \n", + "Ground Truth: 306444,632585,530781,230650,940505,825802,301135,339223,872921,616251,632417,231837,738229,827334,879890,764472,332355,255425,37793,379478,212043,113086,602657,429758,813520,146431,36724,903064,452997,781200\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 179\n", + "Returned IDs: 802512,602557,337726,488161,376992,789801,615857,628922,95761,115855,470784,311016,801035,834937,533939,971928,457263,469341,413276,297731,632080,964592,919506,342624,441323,763544,405219,18689,826126,602442\n", + "Ground Truth: 802512,602557,337726,488161,376992,789801,615857,628922,95761,115855,470784,311016,24642,801035,703091,685915,834937,533939,971928,457263,469341,413276,297731,632080,964592,406519,919506,342624,441323,242065\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 180\n", + "Returned IDs: 735531,364967,788045,780978,675117,379767,94582,464481,26867,588446,706695,712387,970775,592165,376992,656132,712316,534803,802512,616251,37639,544861,62121,77750,261613,157745,775491,441577,834937,413275\n", + "Ground Truth: 735531,364967,788045,780978,675117,379767,94582,464481,26867,588446,706695,712387,970775,592165,376992,656132,463070,712316,534803,802512,616251,37639,544861,62121,77750,261613,157745,775491,441577,834937\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 181\n", + "Returned IDs: 873157,769905,343268,683022,147610,683736,686805,858729,354173,638804,638004,303352,693483,193914,818625,729163,45059,621129,576985,245332,657778,369753,170451,462345,600302,857420,72631,891903,783900,736060\n", + "Ground Truth: 873157,769905,343268,683022,147610,683736,686805,858729,354173,638804,638004,303352,693483,193914,818625,729163,45059,621129,576985,245332,657778,369753,170451,462345,534553,600302,857420,72631,891903,783900\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 184\n", + "Returned IDs: 613195,561835,514643,959174,359450,839858,996028,790421,752162,369753,390336,587977,413275,35435,708631,166392,151127,847322,179676,106582,367512,970607,470415,291665,65773,805195,879890,67272,953810,43958\n", + "Ground Truth: 613195,561835,514643,959174,359450,839858,996028,790421,752162,369753,390336,587977,413275,35435,708631,166392,151127,847322,179676,239920,106582,367512,618559,970607,470415,649707,291665,65773,805195,879890\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 185\n", + "Returned IDs: 90307,18350,345023,296625,384934,310435,830239,82993,316175,543216,353167,691762,800767,64898,74171,365662,234273,632080,276441,712316,974601,353622,84473,74360,463697,900301,408339,732515,634279,447505\n", + "Ground Truth: 90307,18350,345023,296625,384934,379644,310435,830239,82993,316175,543216,353167,691762,800767,64898,74171,365662,234273,632080,276441,712316,974601,353622,84473,74360,1596,463697,900301,408339,732515\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 186\n", + "Returned IDs: 797026,549876,48357,854414,717473,911639,499757,672435,35812,464839,230358,370516,908344,18529,935499,182045,758267,783367,548164,530372,755577,159239,399258,116625,574438,183565,797438,145306,854801,729710\n", + "Ground Truth: 797026,549876,48357,854414,717473,911639,499757,672435,35812,464839,230358,370516,908344,18529,935499,182045,758267,783367,548164,530372,755577,159239,399258,116625,574438,183565,797438,145306,854801,826126\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 187\n", + "Returned IDs: 233361,962777,752393,166392,610912,925667,992979,704746,337458,305307,261509,117966,687257,41301,293159,643363,602657,813520,339223,879890,595698,262101,940505,522760,658210,198289,710026,201945,702283,738229\n", + "Ground Truth: 233361,962777,752393,166392,610912,925667,992979,704746,337458,305307,261509,117966,687257,41301,293159,643363,602657,813520,339223,879890,595698,262101,940505,522760,658210,198289,413276,710026,201945,702283\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 188\n", + "Returned IDs: 555135,660587,268670,783367,640584,183565,621622,850390,708130,632618,945682,431659,79055,366544,174808,447505,58844,745484,96037,902631,502001,541811,100503,874438,820124,687287,331608,423033,618727,813664\n", + "Ground Truth: 555135,660587,268670,783367,640584,183565,621622,850390,708130,632618,945682,431659,79055,366544,174808,447505,58844,745484,96037,902631,502001,541811,100503,616554,874438,820124,687287,331608,893919,423033\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 189\n", + "Returned IDs: 643363,597551,462345,946437,135017,64898,207041,785700,350580,940505,660886,33923,783900,245332,757816,543216,632080,482569,612297,166392,488003,849922,331195,232600,283397,560935,813664,43081,507158,87686\n", + "Ground Truth: 643363,597551,462345,946437,135017,64898,207041,785700,350580,940505,660886,33923,783900,245332,757816,543216,632080,482569,612297,166392,488003,849922,331195,232600,933349,283397,560935,813664,43081,507158\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 191\n", + "Returned IDs: 982943,632080,234369,562190,133860,826126,504540,71204,598044,830239,469341,797438,805947,819024,851546,793020,493704,124522,857564,275992,984648,712558,548164,202580,408339,687634,451708,900301,490923,211318\n", + "Ground Truth: 982943,632080,234369,562190,133860,826126,504540,71204,598044,830239,469341,797438,805947,819024,851546,793020,493704,124522,857564,275992,984648,712558,548164,850029,202580,408339,687634,451708,900301,643793\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 192\n", + "Returned IDs: 849699,150047,64179,963987,194718,745484,531337,600302,631399,702020,468691,801132,553818,939188,967574,737135,826369,940816,522760,862264,950893,36989,48439,641683,687287,285877,544861,875380,88319,706695\n", + "Ground Truth: 849699,150047,64179,963987,194718,745484,531337,600302,631399,702020,468691,801132,553818,939188,967574,501985,737135,295742,597551,826369,940816,522760,72631,862264,950893,36989,890028,48439,892989,641683\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 195\n", + "Returned IDs: 265408,447505,583350,873157,375665,359649,768388,83811,550602,469341,714118,312154,584897,541811,858729,286585,783900,770401,303352,813664,712316,238846,874438,684956,333893,903337,724881,183537,403345,365399\n", + "Ground Truth: 265408,447505,583350,873157,375665,359649,768388,83811,550602,469341,714118,312154,584897,541811,858729,286585,783900,770401,303352,813664,712316,238846,696808,683736,874438,684956,333893,903337,724881,183537\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 198\n", + "Returned IDs: 454724,382030,316751,435655,528944,952626,122407,143117,997496,12846,976705,675195,950893,139298,878843,674336,541428,703283,745484,783900,313302,828466,115855,839439,409130,976507,435000,86869,978689,494082\n", + "Ground Truth: 454724,382030,316751,435655,528944,952626,122407,143117,997496,12846,976705,675195,950893,139298,878843,674336,541428,703283,745484,783900,313302,828466,115855,203353,839439,409130,976507,435000,86869,978689\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 199\n", + "Returned IDs: 101705,631733,334918,94297,721548,888976,924904,135017,299027,447505,652745,465152,633488,85322,650367,406987,541992,419480,236959,245332,426893,313747,971100,798177,354173,491594,571874,552299,495221,697555\n", + "Ground Truth: 101705,631733,334918,94297,721548,888976,924904,135017,299027,447505,652745,465152,633488,85322,650367,12713,406987,419480,541992,236959,245332,426893,313747,971100,798177,354173,340206,491594,571874,552299\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 201\n", + "Returned IDs: 835824,610505,664911,949404,142273,151832,836987,442162,851607,182165,314296,295855,955878,955838,469149,5675,735531,874189,952711,514540,827334,576985,502367,815388,429758,194920,166392,552650,484965,792696\n", + "Ground Truth: 835824,610505,664911,949404,142273,151832,836987,442162,851607,182165,314296,295855,955878,955838,469149,5675,951496,735531,874189,952711,514540,775491,827334,576985,40940,502367,815388,429758,550236,194920\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 202\n", + "Returned IDs: 755577,951496,548164,499757,761814,787972,358859,400877,544861,557561,18529,934083,66984,911639,195445,744935,576985,13631,152860,443374,90217,447546,827552,182045,664447,379767,908344,640111,166392,451708\n", + "Ground Truth: 755577,951496,548164,499757,761814,787972,358859,400877,544861,955838,557561,18529,934083,66984,911639,195445,744935,576985,701447,13631,152860,443374,90217,447546,827552,182045,664447,379767,908344,640111\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 203\n", + "Returned IDs: 602761,768892,371655,177766,585438,828137,238440,445843,345046,97858,66624,996053,706880,972926,677600,181092,41301,135017,212576,447505,357428,706695,691762,848945,502590,993783,579250,724205,799050,552386\n", + "Ground Truth: 602761,768892,371655,177766,585438,828137,238440,445843,345046,97858,66624,996053,706880,972926,677600,181092,41301,135017,212576,447505,357428,706695,691762,848945,502590,993783,579250,724205,635965,799050\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 204\n", + "Returned IDs: 783717,179536,634279,313013,79136,100503,984648,701460,857887,259720,331608,698668,729997,433808,402987,292970,952937,466784,4512,794260,598044,712316,619211,337126,874438,659407,137292,74171,818007,403345\n", + "Ground Truth: 783717,179536,634279,313013,202057,79136,100503,984648,701460,857887,259720,331608,698668,729997,433808,402987,292970,952937,466784,4512,794260,598044,712316,619211,393110,337126,874438,659407,137292,74171\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 206\n", + "Returned IDs: 652336,945780,364886,94582,754108,379767,64898,63145,612297,143668,446499,109016,459789,142868,166392,706695,691762,376992,783900,445370,978005,259343,700519,313747,570591,886952,469341,334918,38444,463724\n", + "Ground Truth: 652336,945780,364886,94582,754108,379767,64898,612297,63145,19053,143668,446499,109016,459789,142868,166392,706695,691762,376992,783900,680128,445370,978005,259343,700519,313747,570591,886952,469341,334918\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 208\n", + "Returned IDs: 408339,761593,368488,368683,247116,632080,234369,142273,34721,982943,434044,876126,945335,745339,456808,431100,560360,552650,940505,826126,364667,187060,71204,712316,815388,465859,319829,99498,950043,491812\n", + "Ground Truth: 408339,761593,368488,368683,247116,632080,234369,142273,34721,982943,434044,876126,945335,745339,456808,431100,560360,552650,940505,826126,364667,187060,71204,55517,712316,815388,465859,319829,99498,950043\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 210\n", + "Returned IDs: 666020,585438,559074,97135,961483,642021,950893,701447,454216,529848,522760,565055,745484,447505,641329,581151,722187,418519,800801,529957,589289,610987,896151,209632,772696,687287,544861,25702,726931,708576\n", + "Ground Truth: 666020,585438,559074,97135,237085,961483,642021,950893,938312,701447,454216,529848,522760,565055,745484,800649,447505,641329,581151,722187,418519,410462,800801,529957,589289,441928,610987,896151,209632,772696\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 213\n", + "Returned IDs: 154098,4493,931772,166392,586293,932033,317591,420265,30526,429758,949404,685915,805195,50104,664911,764836,37639,482522,874189,182165,440285,314296,385733,377442,38248,769082,632080,142273,632417,146418\n", + "Ground Truth: 154098,4493,931772,166392,586293,932033,317591,420265,468430,30526,429758,949404,685915,805195,50104,664911,764836,37639,482522,874189,182165,440285,314296,385733,377442,38248,769082,632080,142273,632417\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 214\n", + "Returned IDs: 81619,42809,394343,94582,459789,691762,37639,735531,983345,970775,847322,289778,64898,697555,706695,447505,740741,361863,166392,643667,677600,818496,178659,548164,600302,345023,238440,757816,463724,364886\n", + "Ground Truth: 81619,42809,394343,94582,459789,691762,37639,735531,983345,970775,847322,289778,64898,697555,706695,846063,447505,740741,361863,166392,643667,677600,818496,178659,548164,600302,345023,238440,757816,463724\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 215\n", + "Returned IDs: 378429,135017,600302,166392,571874,343268,354173,245332,24110,996028,64898,687287,878762,966463,453100,447505,84975,142868,87686,900720,849922,597589,683736,299027,706695,476498,754108,7554,50469,916247\n", + "Ground Truth: 378429,135017,600302,166392,571874,343268,354173,245332,24110,996028,64898,687287,878762,966463,453100,447505,84975,142868,87686,900720,849922,597589,683736,299027,631399,706695,476498,754108,7554,50469\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 216\n", + "Returned IDs: \n", + "Ground Truth: 379767,706695,235467,638708,94582,24196,945780,335933,646934,61856,366682,652336,695965,612297,158862,78624,545048,734241,615402,882806,447505,48257,87686,990925,982164,225760,995884,576985,878069,373923\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 220\n", + "Returned IDs: 714118,438609,85649,319689,265408,626132,917041,605682,230104,885971,835244,555552,693483,498815,768388,712316,706880,830349,304144,596858,227382,940478,177766,595382,195445,745185,83811,134420,576985,312098\n", + "Ground Truth: 714118,438609,85649,319689,265408,626132,364526,917041,605682,230104,885971,835244,555552,693483,498815,768388,712316,706880,830349,304144,596858,227382,940478,177766,595382,195445,745185,83811,134420,576985\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 221\n", + "Returned IDs: 507158,828466,454724,973356,115722,441607,382030,600302,992979,886727,180244,350669,663199,342624,34947,435655,150592,597507,931681,18529,916247,85322,706808,631399,751334,212360,971100,738229,281286,626132\n", + "Ground Truth: 507158,828466,454724,973356,115722,441607,382030,729738,600302,992979,886727,180244,350669,663199,342624,34947,435655,150592,597507,931681,18529,916247,375513,85322,706808,631399,751334,949777,814001,664447\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 224\n", + "Returned IDs: 952108,839619,539150,201582,462977,950043,994326,319473,264537,273768,815388,983345,355729,408339,581559,331997,283669,53080,67272,41699,427331,923303,360005,226364,838377,221035,689742,576985,237537,158639\n", + "Ground Truth: 952108,839619,539150,201582,462977,950043,994326,319473,264537,273768,815388,983345,355729,668176,405219,408339,581559,331997,283669,53080,67272,786565,194718,41699,427331,923303,360005,226364,838377,164870\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 225\n", + "Returned IDs: 648783,749584,761593,283101,576985,544542,94582,460806,983345,142273,394343,843427,581151,404618,991066,379767,64898,773964,706695,245293,287879,264793,408339,166392,230836,664911,148565,799399,14452,67901\n", + "Ground Truth: 648783,749584,761593,283101,576985,544542,94582,460806,983345,142273,394343,843427,581151,404618,991066,261613,379767,64898,773964,706695,245293,287879,264793,408339,166392,230836,664911,23755,148565,799399\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 227\n", + "Returned IDs: 793197,8845,319257,864983,797938,541811,910344,758960,91663,554056,938397,185931,479087,120528,225706,807441,500463,558880,483807,774644,318917,765044,506549,6153,899944,926270,194570,665024,275414,893451\n", + "Ground Truth: 793197,8845,319257,864983,797938,541811,910344,758960,91663,554056,938397,185931,479087,120528,807441,225706,500463,558880,483807,230470,60767,774644,318917,765044,506549,6153,899944,926270,194570,665024\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 230\n", + "Returned IDs: 959763,350257,930712,321739,656276,150633,225501,695965,69280,282531,248449,500463,737135,316175,214269,57142,384822,916915,242065,505357,944204,982993,668240,3563,443127,908541,127351,227025,371655,124402\n", + "Ground Truth: 959763,350257,930712,321739,656276,150633,225501,695965,69280,282531,500463,248449,737135,316175,214269,57142,384822,916915,242065,505357,944204,982993,55983,668240,3563,443127,908541,127351,227025,135657\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 233\n", + "Returned IDs: 469149,343268,757816,94582,664911,847322,706695,818496,581151,550440,406380,166392,379767,772183,548164,815388,600041,622977,484965,983345,791872,447505,289778,87686,275549,37639,206564,62121,729163,64898\n", + "Ground Truth: 469149,343268,757816,94582,390336,664911,847322,706695,818496,581151,550440,406380,166392,379767,992979,772183,548164,815388,600041,622977,484965,983345,791872,447505,289778,87686,275549,37639,206564,62121\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 234\n", + "Returned IDs: \n", + "Ground Truth: 804808,744015,333703,683097,415463,632814,435655,974941,153795,552386,729738,492133,916915,735994,507158,481565,402047,209632,18830,357908,336458,543653,163554,711982,29560,918389,138910,14717,934210,549024\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 237\n", + "Returned IDs: 940217,777881,614507,600302,657512,274156,779309,272151,400032,768892,66624,22789,787372,414278,563216,101983,677600,318050,521810,757816,238440,946518,35070,612108,296440,583564,662929,321321,963987,564509\n", + "Ground Truth: 940217,777881,614507,600302,657512,274156,779309,272151,400032,768892,66624,22789,787372,414278,563216,101983,677600,318050,521810,757816,238440,946518,35070,612108,687257,296440,583564,662929,321321,963987\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 241\n", + "Returned IDs: 924661,447505,427331,97135,813664,643363,701447,19091,529957,687287,454216,64898,544861,600302,85649,755815,961536,706880,750188,85530,212576,175218,178358,560935,903465,811799,643793,210437,824289,232674\n", + "Ground Truth: 924661,447505,427331,97135,813664,643363,701447,19091,529957,687287,454216,64898,544861,600302,85649,755815,961536,706880,750188,85530,263915,212576,175218,178358,613195,560935,903465,811799,643793,210437\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 242\n", + "Returned IDs: 339188,573332,579250,421017,58920,670380,253712,959480,131045,409130,750503,557561,570805,359315,303917,971257,400877,739507,450131,355866,672168,270851,998445,522543,625366,867864,730997,320831,309885,360299\n", + "Ground Truth: 339188,573332,579250,421017,58920,670380,253712,959480,131045,409130,750503,557561,570805,359315,303917,971257,400877,739507,450131,355866,672168,270851,998445,522543,625366,867864,730997,320831,848043,309885\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 244\n", + "Returned IDs: 6514,180244,429735,85445,89460,227440,727702,567353,695960,595241,911912,436651,233593,455501,847369,828761,92861,541759,648655,658210,414278,201945,53181,706695,436016,679498,490901,397230,68834,993147\n", + "Ground Truth: 6514,180244,429735,85445,89460,227440,727702,567353,695960,595241,911912,436651,233593,455501,847369,828761,92861,541759,648655,658210,414278,201945,117104,53181,706695,436016,61298,679498,490901,397230\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 245\n", + "Returned IDs: 336167,34721,408339,443374,966232,22871,899260,982943,343268,65773,212360,397738,80169,514540,194080,698775,632080,281286,379767,303828,147610,797438,912407,403345,717859,506149,175988,490901,365399,83341\n", + "Ground Truth: 336167,34721,408339,443374,966232,22871,899260,982943,343268,65773,212360,397738,80169,514540,194080,698775,632080,281286,379767,819736,303828,147610,797438,912407,403345,717859,506149,175988,490901,365399\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 246\n", + "Returned IDs: 21239,6211,421218,598946,619981,973196,668240,572442,503405,855553,799281,879890,632640,665024,982993,579250,847173,496966,408751,823543,671342,408442,695938,250895,537521,299412,272720,202373,167515,91663\n", + "Ground Truth: 21239,6211,421218,598946,619981,973196,668240,572442,503405,855553,799281,879890,632640,665024,982993,579250,847173,496966,408751,823543,671342,408442,249943,695938,250895,537521,299412,272720,202373,167515\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 247\n", + "Returned IDs: 938397,288817,217639,974849,409731,892658,287335,904507,73355,277299,692188,509555,725978,652336,959763,124402,602761,695965,656276,350257,804808,918389,828137,561483,660159,167515,264334,715091,795321,335778\n", + "Ground Truth: 938397,288817,217639,974849,409731,892658,287335,904507,73355,277299,692188,509555,725978,652336,959763,841556,124402,602761,695965,656276,350257,804808,918389,881149,500463,828137,510464,561483,150640,660159\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 249\n", + "Returned IDs: 5808,36743,457292,437403,317007,270067,232478,756688,254476,827552,304643,343268,894962,201294,194080,272703,797438,598144,336167,967574,637561,12387,990435,815877,465859,665744,548164,673519,83986,228850\n", + "Ground Truth: 5808,36743,457292,437403,317007,270067,232478,756688,254476,827552,304643,343268,894962,201294,194080,272703,797438,598144,336167,967574,637561,12387,990435,815877,465859,743915,665744,548164,673519,83986\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 250\n", + "Returned IDs: 712316,773822,79971,314296,952937,286585,319829,218542,114932,625806,173863,410267,353622,28590,565531,109630,740178,537644,940505,207041,886180,280745,32213,686429,297415,102502,66074,637450,604597,988834\n", + "Ground Truth: 712316,773822,79971,314296,952937,286585,319829,218542,114932,625806,173863,410267,353622,28590,565531,109630,740178,537644,383541,940505,207041,886180,280745,686429,32213,297415,102502,66074,637450,87204\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 251\n", + "Returned IDs: 952711,874189,420265,536427,568706,171311,495445,867149,424288,475148,851607,9024,737583,879890,347755,668747,489788,266075,621662,966580,581151,652336,101253,135821,656657,417132,63145,640584,35435,169807\n", + "Ground Truth: 952711,874189,420265,536427,568706,171311,495445,867149,424288,475148,851607,9024,737583,879890,347755,668747,489788,605639,266075,621662,966580,581151,652336,101253,135821,178659,615,656657,417132,63145\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 252\n", + "Returned IDs: \n", + "Ground Truth: 444315,315480,856864,361192,265061,282213,989157,10521,800801,658210,376197,596858,623687,531095,156608,825700,799378,389961,232795,173577,704746,58122,35312,490297,5531,174609,505142,818897,665291,948300\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 253\n", + "Returned IDs: 772146,447505,320696,742412,11461,848757,841556,530372,175218,643793,468935,308620,849922,718644,513181,813664,14079,583564,603588,997618,753971,790358,135017,664911,389553,757816,97135,385517,463206,902631\n", + "Ground Truth: 772146,447505,320696,742412,11461,848757,841556,530372,175218,643793,468935,308620,849922,718644,513181,813664,14079,583564,603588,997618,714436,753971,790358,135017,664911,389553,757816,740178,97135,385517\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 255\n", + "Returned IDs: 989157,531095,180244,156608,232795,229596,174609,623687,903520,658210,420224,984027,874438,818897,800801,618559,712538,272687,230470,547104,178358,626132,736737,791000,728471,453995,856746,750188,489015,5675\n", + "Ground Truth: 989157,531095,180244,156608,232795,229596,174609,623687,903520,658210,420224,984027,874438,818897,800801,618559,712538,272687,230470,547104,178358,626132,736737,712316,791000,728471,453995,856746,750188,489015\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 257\n", + "Returned IDs: 373723,602442,298415,64898,771765,309885,916915,18830,69280,670380,370134,332080,497305,311197,456861,779902,167515,427455,393202,672661,454064,507158,172701,510927,479167,112463,195445,6338,288817,996053\n", + "Ground Truth: 373723,850030,602442,298415,902031,64898,771765,309885,916915,18830,69280,670380,370134,332080,497305,311197,956610,456861,779902,167515,427455,393202,672661,454064,507158,172701,510927,479167,112463,195445\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 259\n", + "Returned IDs: 600302,906623,762693,749769,343268,683736,507158,65199,245332,364967,445370,304643,837204,791872,903148,35070,87686,72631,719438,799722,327573,222936,697555,897071,403464,283397,632814,900704,614897,218663\n", + "Ground Truth: 600302,906623,762693,749769,343268,683736,978956,637050,507158,65199,245332,364967,167544,742504,445370,304643,837204,791872,903148,35070,87686,72631,719438,799722,327573,862039,222936,697555,897071,403464\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 261\n", + "Returned IDs: 214181,552482,812245,720838,889426,339808,55886,573718,307420,816457,892053,371698,890084,524973,904507,502919,801369,505995,510469,867184,417948,809823,763870,677763,637049,87110,546584,789826,947950,131674\n", + "Ground Truth: 214181,552482,812245,720838,889426,339808,55886,573718,307420,816457,892053,371698,890084,524973,904507,502919,801369,505995,510469,242704,46417,867184,417948,809823,763870,677763,637049,87110,546584,789826\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 265\n", + "Returned IDs: 5525,958864,675467,345023,794380,4855,769082,794260,403345,632080,569165,177415,665325,166392,296625,647965,488219,491812,4512,691762,84473,773964,364667,712316,30526,90307,735531,581151,353622,216428\n", + "Ground Truth: 5525,958864,675467,345023,794380,4855,769082,794260,403345,632080,569165,177415,665325,166392,296625,647965,488219,491812,4512,691762,84473,773964,15628,364667,712316,30526,90307,735531,581151,353622\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 266\n", + "Returned IDs: 350257,282531,656276,392373,377063,483807,541811,844571,384822,498312,450472,686408,695965,972228,926270,601987,849623,371655,41301,135278,18529,959763,670380,303935,656698,779902,891494,714118,311197,248449\n", + "Ground Truth: 350257,282531,656276,392373,377063,483807,541811,844571,384822,498312,450472,686408,625806,695965,972228,926270,601987,849623,371655,41301,135278,18529,959763,670380,303935,656698,779902,891494,714118,311197\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 268\n", + "Returned IDs: 286033,112463,775449,663199,834937,435655,569756,322603,868204,482176,706808,621454,141715,592091,26572,237494,316101,80107,594577,965132,853066,552386,115855,413202,770486,342624,279057,298415,382030,687436\n", + "Ground Truth: 286033,112463,775449,663199,834937,435655,569756,322603,868204,482176,706808,621454,141715,592091,26572,237494,316101,951473,80107,594577,965132,853066,552386,115855,413202,770486,342624,279057,298415,382030\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 269\n", + "Returned IDs: 166392,94894,918389,583564,205310,911031,600302,285877,745484,221330,606478,64898,584897,716969,286585,274156,447505,482380,983345,582477,619853,850368,813664,963987,582982,754108,71214,35070,357434,729738\n", + "Ground Truth: 166392,94894,918389,583564,205310,911031,600302,285877,745484,221330,606478,64898,584897,716969,286585,274156,447505,482380,983345,582477,619853,794252,850368,813664,963987,582982,754108,71214,35070,357434\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 270\n", + "Returned IDs: 783900,177766,995005,19091,813664,560461,677600,542842,745484,851545,447505,64898,541811,435655,850390,242434,687257,318050,153485,544861,687287,643793,718644,537299,619853,454216,211528,585438,479774,679498\n", + "Ground Truth: 783900,177766,995005,19091,813664,560461,677600,542842,745484,574793,851545,447505,64898,541811,435655,850390,242434,687257,318050,153485,544861,687287,643793,718644,537299,619853,454216,211528,67332,585438\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 271\n", + "Returned IDs: 644073,421218,720943,472121,9024,398327,656220,940755,371655,916915,63145,378362,554056,784919,288817,225501,814886,333703,984225,418915,393202,392618,150640,536307,955841,402047,93883,775370,266404,575058\n", + "Ground Truth: 644073,421218,720943,472121,9024,398327,656220,940755,371655,916915,63145,378362,554056,784919,288817,225501,814886,333703,984225,418915,393202,392618,383793,150640,536307,955841,402047,93883,775370,266404\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 272\n", + "Returned IDs: 956689,314296,636425,6153,79971,502367,23755,449571,712316,624723,878069,218542,418997,100707,652336,576985,495445,114932,77201,632080,740178,978112,83341,166392,474180,369154,961677,198289,212360,18793\n", + "Ground Truth: 956689,314296,636425,6153,79971,502367,23755,449571,712316,624723,878069,218542,418997,100707,652336,576985,495445,114932,534803,77201,632080,740178,978112,83341,166392,383541,474180,369154,961677,198289\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 275\n", + "Returned IDs: 96536,109226,698426,641988,784698,142868,502001,166392,850390,135017,728241,483652,697522,18830,488003,859196,570591,417872,215599,783900,333703,29560,968295,482380,561835,300183,469341,602442,41301,643793\n", + "Ground Truth: 96536,109226,698426,641988,784698,142868,502001,166392,850390,135017,728241,720943,483652,697522,18830,488003,859196,570591,417872,215599,783900,333703,29560,968295,482380,561835,300183,469341,944204,602442\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 277\n", + "Returned IDs: 695965,288817,686408,214108,3386,492133,554056,510927,984655,18830,371655,779902,63145,529474,621717,916915,217639,795321,124402,938397,660159,247044,860213,656276,5985,454064,849623,612117,355866,344295\n", + "Ground Truth: 695965,288817,686408,214108,3386,492133,554056,510927,984655,18830,371655,779902,63145,529474,621717,916915,217639,795321,124402,938397,660159,247044,860213,83442,656276,5985,454064,849623,612117,355866\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 278\n", + "Returned IDs: 830735,3176,507123,930712,402047,309885,66967,497305,253712,214269,656220,355866,120980,920773,554056,546621,916915,672661,67549,392618,632708,485035,771765,901003,220971,214108,241,988352,805604,414862\n", + "Ground Truth: 830735,3176,507123,930712,402047,309885,66967,497305,253712,214269,656220,355866,120980,920773,554056,546621,916915,342537,672661,67549,392618,632708,485035,771765,901003,220971,214108,241,988352,805604\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 279\n", + "Returned IDs: 286585,207041,789685,783900,932033,940505,816481,712316,379644,5525,491812,408339,592278,565531,225057,216428,166392,187655,311941,795343,665325,167474,837526,23755,384482,140085,79971,581151,634279,43501\n", + "Ground Truth: 286585,207041,789685,783900,932033,940505,816481,712316,379644,5525,491812,408339,592278,565531,225057,16878,216428,166392,187655,795343,311941,167474,665325,837526,23755,665070,384482,140085,79971,581151\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 281\n", + "Returned IDs: 351364,648624,408339,958864,912407,805947,910402,737650,692693,239691,732715,397738,881397,293676,61435,73777,173863,584518,966232,795343,982943,412471,359266,852119,493704,79136,899260,778590,208930,899582\n", + "Ground Truth: 351364,648624,408339,958864,912407,805947,910402,737650,692693,495200,239691,732715,397738,881397,293676,61435,73777,173863,584518,966232,795343,982943,412471,359266,852119,493704,79136,273403,899260,778590\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 282\n", + "Returned IDs: 241926,915692,643793,320419,491594,980270,946747,483807,701447,106582,142868,187328,64898,902631,224067,311387,873778,541957,59522,634029,489257,565810,194080,687287,405219,543879,893919,603588,74067,506534\n", + "Ground Truth: 241926,915692,643793,320419,491594,980270,946747,483807,478600,701447,106582,142868,187328,64898,902631,224067,311387,873778,541957,59522,634029,489257,359266,854291,565810,194080,687287,859196,405219,543879\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 284\n", + "Returned IDs: 13656,529957,336977,888729,164422,561189,494089,450472,847062,264045,646000,565055,498312,287335,701447,88319,415054,466875,178358,732909,642021,889469,454216,521706,242434,648783,310119,486447,772696,208424\n", + "Ground Truth: 13656,529957,336977,888729,164422,561189,494089,450472,702589,847062,264045,646000,565055,498312,287335,701447,88319,415054,466875,178358,732909,642021,889469,454216,521706,242434,310119,648783,486447,772696\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 285\n", + "Returned IDs: 576985,983345,185593,931780,379767,94582,908539,798845,541428,264793,757816,304062,718291,796968,729738,61856,815877,643363,831934,563778,582982,600302,592681,992979,167544,12489,113086,491812,504223,289537\n", + "Ground Truth: 576985,983345,185593,931780,379767,94582,908539,798845,541428,264793,757816,304062,718291,796968,729738,61856,815877,643363,831934,563778,582982,600302,592681,992979,730418,167544,357403,12489,113086,491812\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 287\n", + "Returned IDs: 907990,326309,622292,646934,158639,379767,302843,514903,311637,765065,534803,577120,389553,541904,581559,413275,259033,854904,706695,976682,281286,318014,94582,259018,592165,125577,460806,441577,374244,61856\n", + "Ground Truth: 907990,326309,622292,646934,158639,379767,302843,514903,311637,765065,534803,577120,389553,541904,581559,765929,413275,259033,854904,706695,976682,281286,318014,94582,259018,592165,125577,41099,460806,441577\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 289\n", + "Returned IDs: 80061,871262,607059,614821,155550,504324,314296,379478,828466,899260,265408,311016,142142,462345,83341,632080,256011,443374,698775,834643,85649,412471,857783,824499,982771,528944,79971,34721,154098,695130\n", + "Ground Truth: 80061,871262,607059,614821,155550,504324,314296,379478,828466,261613,899260,265408,311016,142142,462345,83341,632080,256011,443374,698775,901831,834643,85649,412471,857783,824499,982771,528944,79971,34721\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 290\n", + "Returned IDs: 712316,702096,447505,745522,403345,79136,469341,982164,563923,207041,783900,620407,109135,463206,65773,982943,179536,376992,659407,64898,6266,407004,132941,299027,772146,523569,565531,874072,265408,176059\n", + "Ground Truth: 712316,702096,447505,745522,403345,79136,469341,982164,563923,207041,783900,620407,109135,463206,65773,982943,179536,376992,418997,659407,583350,64898,6266,407004,132941,299027,772146,523569,565531,874072\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 292\n", + "Returned IDs: 745339,211488,510938,918406,762181,788045,434044,867914,819036,138525,517191,604597,159489,53763,916450,552650,339768,18686,912991,632080,187060,289925,697883,664911,525665,680121,173576,17986,385733,343352\n", + "Ground Truth: 745339,211488,510938,918406,762181,788045,434044,867914,819036,986435,138525,517191,604597,159489,53763,916450,552650,339768,18686,912991,632080,187060,289925,697883,664911,525665,680121,173576,17986,385733\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 295\n", + "Returned IDs: 850390,652336,672661,982993,904507,364319,402047,968295,612297,579250,813664,680706,670380,29560,632814,285666,248449,916915,525546,287335,229125,63145,124402,350580,885272,350669,959181,492133,544861,150640\n", + "Ground Truth: 850390,652336,672661,982993,904507,364319,731330,402047,968295,612297,579250,813664,680706,670380,29560,632814,285666,248449,916915,525546,287335,229125,63145,124402,350580,885272,350669,959181,492133,464743\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 296\n", + "Returned IDs: 373723,23222,850030,298415,771765,691481,332080,24196,660351,766202,6488,295500,195445,929864,519324,309885,167515,7925,882066,463070,350669,479167,441323,878069,653858,670380,541980,564835,828466,804068\n", + "Ground Truth: 373723,23222,850030,298415,771765,691481,486639,332080,24196,660351,766202,6488,295500,195445,929864,519324,309885,167515,7925,882066,463070,350669,479167,441323,878069,653858,670380,541980,564835,828466\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 297\n", + "Returned IDs: 373723,282531,350257,938397,844571,656989,588446,3386,686408,672661,298415,498312,625858,475669,826493,33942,706695,379767,405252,608314,656276,612297,217639,67033,350669,759354,695965,891149,760173,445370\n", + "Ground Truth: 373723,282531,350257,938397,844571,656989,902031,588446,3386,686408,511821,672661,298415,498312,625858,475669,826493,33942,706695,379767,405252,608314,656276,369956,612297,217639,67033,350669,316751,759354\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 299\n", + "Returned IDs: 732515,893919,664447,785700,300026,848232,996028,32875,142868,492260,479197,474041,641988,183854,660886,225260,597589,708492,548164,307804,514080,601998,354173,376992,632080,76961,826126,87686,358859,920983\n", + "Ground Truth: 732515,893919,664447,785700,300026,848232,996028,32875,142868,492260,479197,474041,641988,183854,660886,225260,597589,708492,548164,307804,514080,601998,354173,376992,76961,632080,826126,87686,358859,918553\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 301\n", + "Returned IDs: 219321,847729,894696,321309,774724,478851,136912,24196,536352,815827,775449,759354,98827,735994,320557,238088,101653,814886,572761,878069,739725,931681,632640,342624,982164,105768,237494,450445,612297,250951\n", + "Ground Truth: 219321,847729,894696,321309,774724,478851,136912,24196,536352,815827,362278,63471,775449,759354,98827,735994,792419,320557,238088,101653,875745,514080,814886,572761,878069,739725,931681,632640,342624,982164\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 304\n", + "Returned IDs: 792818,142146,313219,371594,922747,506202,826244,29003,420265,785474,678945,680438,437989,645340,166392,9024,100503,104181,604752,376861,610912,899944,170950,575922,995369,228850,456861,9985,308018,539301\n", + "Ground Truth: 792818,142146,313219,371594,922747,506202,826244,29003,420265,785474,678945,680438,437989,645340,166392,9024,100503,104181,604752,376861,610912,899944,170950,575922,194570,995369,228850,456861,9985,791291\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 305\n", + "Returned IDs: 372132,678044,612002,703283,233361,343506,912561,331816,992979,832778,950893,605550,632417,872921,84917,855897,489975,311016,475163,780251,313013,19091,563473,329284,970607,698668,151457,113066,166392,517119\n", + "Ground Truth: 372132,678044,612002,703283,233361,343506,912561,331816,992979,832778,950893,605550,632417,872921,84917,855897,489975,311016,475163,97973,780251,313013,19091,563473,329284,970607,698668,151457,113066,517119\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 307\n", + "Returned IDs: 98827,861541,695965,612297,24196,615402,990925,982164,849922,844571,341783,916915,299027,492260,692188,245332,687287,350257,498312,379767,393202,938397,492133,804586,282531,64898,602442,87686,739482,109016\n", + "Ground Truth: 98827,861541,695965,612297,24196,615402,990925,982164,849922,844571,341783,23934,916915,299027,492260,692188,245332,687287,350257,498312,916247,379767,393202,938397,492133,804586,282531,64898,602442,375385\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 308\n", + "Returned IDs: 604832,971257,131045,214725,519324,554056,632778,959470,963057,988176,19706,476683,597805,398426,841254,956035,355866,360299,253776,492133,670380,729998,579578,57192,867864,167545,359315,299325,286691,6153\n", + "Ground Truth: 604832,971257,131045,214725,519324,554056,632778,959470,963057,988176,19706,476683,597805,841254,398426,956035,355866,360299,253776,492133,670380,729998,579578,755002,57192,867864,167545,359315,299325,286691\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 309\n", + "Returned IDs: 984648,598044,303352,664447,857564,826979,891795,469341,548164,24692,965362,205204,65773,799317,996028,955142,275549,443374,828009,276304,7144,525149,343268,997452,234369,155572,502922,133770,876126,6266\n", + "Ground Truth: 984648,598044,303352,664447,857564,826979,891795,469341,548164,24692,965362,205204,65773,763544,799317,996028,955142,275549,443374,828009,276304,7144,525149,343268,997452,234369,155572,502922,133770,876126\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 311\n", + "Returned IDs: 745484,97135,19091,92915,12387,13656,950893,139298,615673,529957,454216,931780,536591,710026,377161,447505,517119,445843,541811,904507,632708,2907,579250,762693,544861,576985,543879,211528,259018,529848\n", + "Ground Truth: 745484,97135,19091,92915,12387,13656,950893,139298,615673,529957,534447,454216,931780,536591,641683,710026,377161,447505,517119,445843,65199,541811,904507,632708,2907,579250,762693,544861,576985,543879\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 312\n", + "Returned IDs: 237494,600302,85322,435655,212360,502001,64898,663199,413276,828466,650367,476798,632080,141715,631399,971100,757816,897186,18529,967574,507158,580486,94582,643793,881397,483807,343268,343498,447505,382030\n", + "Ground Truth: 237494,600302,85322,435655,212360,502001,64898,663199,413276,454701,828466,134044,650367,476798,632080,72631,141715,631399,971100,873157,757816,897186,18529,967574,507158,580486,94582,643793,881397,483807\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 315\n", + "Returned IDs: 135017,486854,50469,600302,88740,7554,749769,64898,851968,576985,364886,683736,832866,570591,406987,400032,732233,186375,458815,265408,166392,314296,535419,838315,849922,34721,753890,476498,605682,405219\n", + "Ground Truth: 135017,486854,50469,600302,88740,7554,749769,64898,851968,576985,364886,683736,832866,618727,570591,406987,400032,732233,186375,458815,265408,166392,314296,535419,838315,849922,34721,753890,476498,514540\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 318\n", + "Returned IDs: 854414,760046,652155,426893,548164,911639,343268,325330,888976,630890,292605,858269,740741,908344,245006,245332,64898,715216,503617,549876,540542,12713,370516,294152,820124,142868,135017,378429,948817,419480\n", + "Ground Truth: 854414,760046,652155,426893,548164,911639,343268,325330,888976,630890,292605,858269,740741,908344,245006,245332,64898,715216,503617,549876,540542,770672,12713,370516,294152,820124,142868,135017,378429,320419\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 319\n", + "Returned IDs: 364869,662929,71214,619853,141367,135017,522760,706880,479774,729710,785700,564509,583564,849922,463724,834937,677600,221330,828137,211300,41253,582477,21791,64898,526925,272687,245332,357428,918389,850368\n", + "Ground Truth: 364869,662929,71214,619853,141367,135017,522760,706880,479774,729710,785700,564509,583564,849922,463724,834937,677600,221330,828137,211300,41253,582477,21791,64898,526925,272687,245332,357428,918389,824289\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 320\n", + "Returned IDs: 576985,544861,266784,480071,463070,775428,564509,379767,463724,464286,677600,544437,227382,177766,788045,735531,927595,141170,448520,805131,570591,522760,371557,541811,259401,445370,631399,533939,554056,357908\n", + "Ground Truth: 576985,544861,266784,182165,480071,463070,775428,564509,379767,927297,463724,464286,677600,544437,227382,177766,788045,954884,735531,927595,141170,448520,805131,300183,787734,570591,522760,371557,998221,541811\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 321\n", + "Returned IDs: \n", + "Ground Truth: 678945,29003,703283,729997,783900,211528,420265,316751,79971,207041,120482,103508,19091,592278,612002,163610,100503,645340,643793,738229,679498,961677,786565,166392,745484,369154,833986,706880,553818,878086\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 323\n", + "Returned IDs: 979439,309668,232228,936048,101253,978112,636081,825157,457242,973233,188311,537553,941526,750140,683177,449339,740178,119782,437964,825049,129632,823159,124345,489788,712538,879826,254515,77201,376693,429754\n", + "Ground Truth: 979439,309668,232228,936048,101253,978112,636081,825157,457242,973233,188311,537553,941526,750140,683177,449339,740178,119782,916915,437964,825049,129632,823159,124345,489788,712538,879826,254515,77201,376693\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 324\n", + "Returned IDs: 996028,142868,579162,860363,600302,516493,925914,41856,96536,548164,760970,801132,295742,500336,67272,402243,514643,631399,166392,225260,89997,577876,664447,561835,564509,835244,104952,929731,254476,63990\n", + "Ground Truth: 996028,142868,579162,860363,600302,516493,925914,41856,96536,548164,760970,801132,295742,500336,67272,402243,514643,631399,166392,225260,89997,577876,792419,664447,561835,564509,48813,835244,104952,929731\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 325\n", + "Returned IDs: 371655,529324,55167,305570,882917,665024,318917,849623,344295,402047,755666,793037,431046,506549,371360,507123,220971,214108,125536,225501,104394,970655,492133,548697,990925,385819,837410,725978,554056,124402\n", + "Ground Truth: 371655,529324,55167,305570,882917,665024,318917,849623,344295,402047,755666,793037,431046,506549,371360,507123,220971,214108,125536,311197,672168,225501,104394,970655,492133,548697,990925,385819,837410,725978\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 326\n", + "Returned IDs: 832291,214269,725978,379767,282531,695965,350257,710026,850390,982993,259018,311197,672661,670380,926270,374017,41301,227025,959763,166392,371655,778930,456861,195375,940691,331195,143117,61856,486639,729998\n", + "Ground Truth: 832291,214269,725978,379767,282531,695965,350257,710026,850390,982993,259018,311197,672661,670380,926270,355866,374017,41301,227025,959763,166392,371655,778930,456861,195375,940691,331195,143117,61856,793875\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 327\n", + "Returned IDs: 29964,982164,763544,18529,797938,899944,272209,331608,963711,6266,23755,893919,919506,708566,523569,443374,479197,830239,632814,664447,341783,447505,135278,445370,149032,659407,350669,276304,303352,65773\n", + "Ground Truth: 29964,982164,763544,18529,797938,899944,272209,331608,963711,6266,819736,669722,23755,893919,919506,708566,523569,443374,479197,830239,632814,664447,341783,462345,447505,135278,445370,149032,659407,564509\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 329\n", + "Returned IDs: 421628,937745\n", + "Ground Truth: 784568,166392,242161,142868,548164,732515,376992,655568,665325,474041,735531,804808,74171,345023,283397,837640,824289,801035,616251,659407,4512,632080,563923,463724,447505,781200,712316,408339,379478,940505\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 334\n", + "Returned IDs: 79971,380365,176958,462345,102502,817938,632080,961677,570591,660886,202057,339768,525665,319829,826126,862897,410489,420265,982771,894768,269551,712316,385733,166392,954884,686429,694177,554865,637050,142868\n", + "Ground Truth: 79971,380365,176958,462345,102502,817938,632080,961677,570591,660886,202057,339768,525665,319829,826126,862897,410489,420265,982771,87204,894768,269551,712316,385733,166392,954884,686429,694177,554865,637050\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 338\n", + "Returned IDs: 834937,619853,563216,722764,533939,677600,775428,345766,980270,274156,254125,612108,952288,318050,315425,541811,5918,84043,167622,345046,544861,463724,466875,970145,458745,976404,736060,363525,564509,988030\n", + "Ground Truth: 834937,619853,563216,722764,533939,677600,775428,345766,980270,274156,254125,612108,952288,318050,661690,227025,315425,541811,5918,84043,167622,345046,544861,463724,466875,970145,458745,976404,736060,542755\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 342\n", + "Returned IDs: 764472,683549,652336,233361,180441,9024,115855,872723,513955,331977,739725,813520,344351,393202,810307,166392,120482,915218,215750,313013,243546,733946,335201,949077,482932,413363,418915,364886,341,399872\n", + "Ground Truth: 764472,683549,652336,233361,952738,180441,9024,115855,872723,513955,331977,739725,813520,344351,393202,810307,166392,120482,915218,215750,313013,243546,733946,335201,949077,482932,413363,418915,364886,341\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 343\n", + "Returned IDs: 41301,466875,124402,409172,627144,455501,429735,712869,889469,92861,675467,623702,271111,745484,180244,178358,541759,411988,5525,517119,531095,829977,84473,687287,345023,876110,828137,976404,530372,894608\n", + "Ground Truth: 41301,466875,124402,409172,627144,455501,15628,429735,712869,889469,92861,675467,623702,271111,745484,180244,178358,541759,411988,5525,517119,531095,829977,84473,687287,345023,876110,828137,976404,530372\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 344\n", + "Returned IDs: 387460,397196,607269,494823,232588,309885,643793,796968,117859,402790,65199,463070,451846,672023,434018,479405,632080,785025,927595,282337,445370,672168,453789,320419,544861,742370,73368,988030,883744,453995\n", + "Ground Truth: 387460,397196,607269,494823,232588,309885,643793,796968,117859,402790,65199,463070,451846,672023,434018,479405,632080,785025,927595,282337,445370,672168,453789,320419,544861,826126,742370,73368,988030,883744\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 345\n", + "Returned IDs: 919506,602557,605639,763544,534044,841232,393202,691762,24196,331608,23934,331816,720943,612297,995102,376992,38444,313013,687287,64898,191784,341783,939774,463206,209207,379767,602442,402987,445843,660351\n", + "Ground Truth: 919506,602557,605639,308588,763544,534044,841232,393202,691762,24196,331608,23934,331816,680706,834643,720943,612297,995102,376992,38444,41699,313013,687287,64898,191784,341783,939774,463206,209207,379767\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 346\n", + "Returned IDs: 727333,501165,671404,855897,21239,695965,476798,256011,992979,463657,504858,768892,744351,371655,364798,426496,558975,54091,687436,567342,485948,42567,997496,841229,982375,128484,947950,973356,952626,890509\n", + "Ground Truth: 727333,501165,671404,855897,21239,695965,476798,256011,992979,463657,504858,768892,744351,371655,364798,426496,558975,54091,687436,567342,485948,42567,997496,841229,743654,982375,128484,947950,973356,805604\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 347\n", + "Returned IDs: 213007,795124,734574,502590,848708,679235,380616,299492,632080,799050,687352,448587,296718,802512,79136,768892,647965,301135,977757,331145,662929,698775,608899,963711,34721,576985,166392,868021,447505,850368\n", + "Ground Truth: 213007,795124,734574,502590,848708,679235,380616,299492,632080,799050,407127,687352,448587,296718,802512,79136,768892,647965,301135,977757,331145,662929,698775,608899,963711,663376,34721,576985,166392,868021\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 349\n", + "Returned IDs: 47062,965292,9927,840484,825157,669723,30315,532064,174683,254515,568706,909553,825446,492663,280800,424560,524377,232228,24410,710180,830349,712924,426423,195445,565227,378378,581151,942244,171311,992387\n", + "Ground Truth: 47062,965292,9927,840484,825157,669723,30315,532064,174683,254515,568706,909553,825446,492663,280800,249377,424560,524377,232228,24410,710180,830349,712924,426423,195445,565227,378378,581151,942244,171311\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 351\n", + "Returned IDs: 393202,612297,729998,450445,998445,98827,24196,878069,341783,804586,990925,544861,615402,421017,632640,340503,288817,368888,900639,579250,612117,763544,996053,97858,602442,525546,359315,209632,479167,695965\n", + "Ground Truth: 393202,612297,729998,450445,998445,98827,24196,878069,341783,804586,990925,544861,615402,421017,264334,632640,340503,288817,368888,900639,579250,612117,763544,680706,996053,97858,602442,525546,359315,209632\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 352\n", + "Returned IDs: 814886,9024,876434,804586,519324,288817,6084,680438,916915,90600,763431,645340,485739,214269,454027,6153,437092,729998,450282,63145,120980,417928,100503,337185,158345,998445,73156,293149,225501,378362\n", + "Ground Truth: 814886,9024,876434,804586,519324,288817,6084,680438,916915,90600,955841,763431,645340,485739,214269,454027,6153,437092,729998,450282,63145,120980,417928,100503,337185,158345,998445,73156,293149,225501\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 354\n", + "Returned IDs: 183565,431021,500438,328025,238846,785700,783367,403345,689840,198208,32213,516493,469341,902631,311387,445370,393759,530372,548164,683736,755577,230358,18529,728241,166392,22871,797438,145306,772146,751839\n", + "Ground Truth: 183565,431021,500438,328025,238846,785700,783367,403345,689840,198208,32213,516493,469341,902631,311387,445370,393759,530372,548164,683736,755577,230358,18529,728241,166392,22871,797438,145306,772146,658210\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 355\n", + "Returned IDs: 738229,379767,37639,429758,94582,706695,166392,335201,780978,643363,507158,413275,301135,879890,670380,230650,413276,61856,113086,664911,306444,754108,180244,940505,787734,576985,652336,886727,735531,612297\n", + "Ground Truth: 738229,379767,37639,429758,94582,929864,706695,166392,335201,780978,643363,507158,413275,301135,879890,670380,230650,413276,61856,113086,311068,664911,97858,306444,754108,180244,940505,787734,576985,652336\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 356\n", + "Returned IDs: 166392,469341,272209,912561,376992,331608,992979,643363,996028,826244,757816,652752,879890,64898,830239,872921,829977,245332,447505,801035,612297,379767,299027,735531,258029,313013,893919,311016,534553,613195\n", + "Ground Truth: 166392,469341,272209,155972,912561,376992,331608,992979,643363,996028,826244,757816,652752,879890,64898,830239,872921,829977,245332,447505,801035,612297,379767,299027,735531,258029,313013,893919,311016,534553\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 357\n", + "Returned IDs: 84471,959763,18529,950726,608314,445843,482380,55426,927595,194718,18830,512544,656276,849922,525546,341783,632708,797438,282531,270774,904507,631399,544861,710026,686408,772035,660159,813664,634029,120980\n", + "Ground Truth: 84471,959763,18529,950726,608314,445843,482380,55426,927595,194718,18830,512544,656276,849922,525546,503974,341783,632708,159627,797438,282531,270774,904507,631399,544861,710026,686408,772035,660159,813664\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 358\n", + "Returned IDs: \n", + "Ground Truth: 571857,850183,359082,908538,326805,359095,905854,234842,342309,66967,232002,543653,3386,861074,363218,407127,814001,412721,507123,662353,40163,881687,257854,129270,178358,276475,813664,510927,937944,571814\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 360\n", + "Returned IDs: 266784,775428,677600,834937,242750,419803,90217,544861,706880,788045,112463,770486,987690,400877,533939,622977,829092,371557,954884,990435,338559,369753,463724,342326,920983,917766,783900,597589,802512,631399\n", + "Ground Truth: 266784,775428,677600,834937,242750,419803,90217,544861,706880,788045,112463,770486,987690,400877,533939,622977,829092,371557,679498,954884,990435,548164,338559,369753,463724,310435,342326,920983,917766,783900\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 366\n", + "Returned IDs: 878069,350669,506149,235267,516184,982164,185309,175988,632080,826126,927148,336167,304164,998221,443374,230291,745185,155550,311016,523569,26089,29964,607059,77750,445370,951923,982943,299027,990435,212360\n", + "Ground Truth: 643678,878069,350669,506149,235267,516184,982164,185309,175988,632080,826126,927148,336167,304164,998221,443374,230291,828466,745185,155550,311016,523569,26089,29964,607059,77750,445370,951923,982943,299027\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 367\n", + "Returned IDs: 728837,626727,701821,850030,350669,6488,823406,206694,332080,352154,309885,771765,667220,134044,63990,552386,622977,967574,298415,245559,373723,794954,785700,691481,796968,857861,507158,14717,265421,6153\n", + "Ground Truth: 728837,626727,701821,850030,350669,6488,823406,823366,206694,332080,352154,309885,771765,667220,134044,63990,552386,622977,967574,298415,245559,373723,794954,785700,691481,796968,857861,507158,14717,265421\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 368\n", + "Returned IDs: 443831,996028,841823,826126,708631,827552,514080,613195,403345,561835,678338,469341,413275,272703,964324,439626,470415,534553,548164,664447,67272,760970,151127,170989,18529,965362,831268,16656,769082,612297\n", + "Ground Truth: 443831,996028,841823,826126,708631,827552,514080,613195,403345,561835,678338,469341,413275,272703,964324,439626,470415,534553,548164,664447,67272,760970,151127,170989,18529,965362,831268,16656,14717,769082\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 371\n", + "Returned IDs: 571874,840076,393060,14079,312098,201269,236116,547621,832465,845079,740653,430561,289537,181092,319689,541449,7554,755759,207488,574445,167544,772146,362227,18686,730420,276798,664911,192154,528978,151962\n", + "Ground Truth: 571874,840076,393060,112799,14079,312098,236116,201269,547621,832465,845079,740653,430561,289537,181092,319689,952810,541449,7554,755759,207488,574445,167544,772146,362227,155895,935154,18686,730420,276798\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 372\n", + "Returned IDs: 151127,894314,443235,467724,836659,649355,695625,696363,454347,766432,23788,347177,618559,983345,340110,774212,377104,417032,203782,479331,576985,987690,195888,685983,577035,89460,12487,830523,497342,547104\n", + "Ground Truth: 151127,894314,443235,467724,836659,649355,695625,696363,454347,766432,679498,23788,347177,618559,983345,340110,774212,377104,417032,203782,479331,576985,987690,195888,685983,577035,89460,12487,976297,830523\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 374\n", + "Returned IDs: 15988,33361,236959,135658,17107,563923,886932,447505,542755,337126,600302,415235,469341,662351,64898,495221,81619,74067,135017,401966,166392,311068,946518,267310,712316,311387,109135,826126,620407,79136\n", + "Ground Truth: 15988,33361,236959,135658,17107,563923,886932,447505,542755,337126,600302,415235,469341,662351,64898,495221,81619,74067,135017,401966,166392,311068,946518,267310,712316,311387,109135,826126,834937,620407\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 379\n", + "Returned IDs: 174808,479855,961747,58844,209103,204185,627084,26637,275414,621725,939371,589289,531095,327683,950869,129270,201245,71455,732233,431659,772945,263842,183883,245045,229882,401110,22142,621622,423033,517119\n", + "Ground Truth: 174808,479855,961747,58844,209103,204185,627084,26637,275414,621725,939371,589289,531095,327683,950869,129270,201245,71455,732233,431659,772945,263842,412721,183883,245045,229882,401110,22142,621622,423033\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 382\n", + "Returned IDs: 64898,210233,692552,447505,494155,166392,502001,155972,598044,85530,463206,313013,742370,191784,673607,84043,602557,695553,643363,970607,451709,410377,963711,469341,687287,560935,753971,568879,992979,917766\n", + "Ground Truth: 64898,210233,692552,447505,494155,166392,502001,155972,598044,85530,463206,313013,658397,742370,191784,673607,84043,602557,695553,643363,970607,451709,410377,963711,469341,687287,560935,753971,568879,992979\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 384\n", + "Returned IDs: 862039,310435,931425,927148,826126,632080,65773,898071,27634,215630,333969,717859,803864,594344,234369,989555,966232,645359,258784,212060,408339,343268,347041,239821,684037,990435,689181,235267,868021,998221\n", + "Ground Truth: 862039,310435,931425,927148,826126,632080,65773,898071,27634,333969,215630,717859,803864,594344,234369,989555,966232,645359,258784,212060,408339,343268,347041,239821,684037,990435,689181,235267,868021,451708\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 385\n", + "Returned IDs: 971100,85322,619468,288169,706695,664624,871262,180244,865901,744351,154098,504324,576985,94582,115722,441751,997496,507158,663199,897186,340429,814015,731160,623711,632708,379767,843427,476798,79136,65773\n", + "Ground Truth: 971100,85322,619468,288169,706695,664624,871262,180244,865901,744351,154098,504324,576985,94582,115722,441751,997496,507158,663199,897186,340429,814015,731160,299027,623711,632708,463206,232795,379767,843427\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 387\n", + "Returned IDs: 45393,598144,238816,906702,27081,403724,28151,303085,850029,938373,826126,632814,36743,407004,857738,156663,614821,232377,443374,548164,13747,565750,303352,982164,483807,636425,5808,110705,163564,235267\n", + "Ground Truth: 45393,598144,238816,906702,27081,403724,28151,303085,553824,850029,938373,826126,632814,36743,407004,857738,156663,614821,232377,443374,548164,13747,565750,303352,982164,483807,755666,636425,5808,110705\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 388\n", + "Returned IDs: 516184,773822,83341,234273,632080,65773,712316,212360,623711,856773,412471,207041,634279,408339,384934,259720,845747,982943,469341,617507,974601,32213,234369,801035,74171,463697,643131,951923,365662,443374\n", + "Ground Truth: 516184,773822,83341,234273,632080,65773,712316,212360,623711,856773,412471,207041,634279,408339,384934,259720,845747,982943,469341,617507,974601,32213,234369,801035,74171,463697,643131,951923,365662,37639\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 389\n", + "Returned IDs: 740082,693371,902031,524973,6153,69280,607382,93883,299325,841790,939774,63145,337185,37859,371655,830889,334309,150633,984225,288169,404975,755104,8084,944204,120980,467686,214269,445843,409731,36535\n", + "Ground Truth: 740082,693371,902031,524973,12071,6153,69280,850030,607382,93883,299325,841790,729062,939774,63145,421017,337185,37859,371655,830889,334309,150633,984225,288169,404975,755104,8084,24851,944204,120980\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 390\n", + "Returned IDs: 51333,443235,940505,576985,783900,409731,245559,5819,982771,819736,516184,652336,614606,286585,173863,618559,866595,151127,180244,379767,142273,712316,625806,140085,61856,392157,462345,155550,281286,712203\n", + "Ground Truth: 51333,443235,940505,576985,783900,409731,245559,5819,982771,819736,547189,42567,516184,652336,614606,286585,173863,618559,866595,19912,151127,180244,379767,195445,809535,142273,712316,625806,140085,826126\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 392\n", + "Returned IDs: 760046,860811,182045,549876,343268,783367,881792,854801,354435,553763,790358,941506,854414,820124,615,854483,64898,697555,416310,683736,88740,426893,908344,135729,579195,548164,797438,135017,935154,449334\n", + "Ground Truth: 760046,860811,182045,549876,343268,783367,881792,854801,354435,553763,790358,941506,854414,820124,615,854483,64898,697555,416310,683736,88740,426893,908344,135729,579195,548164,797438,135017,111147,935154\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 393\n", + "Returned IDs: 983345,706695,390200,649865,419803,563778,404740,544542,36724,94582,142273,457556,74771,469149,955838,237537,167544,664911,780978,843427,462034,539150,431100,588446,576985,735531,379767,992979,195445,313620\n", + "Ground Truth: 983345,706695,390200,649865,419803,563778,404740,544542,618559,36724,94582,142273,457556,74771,469149,955838,237537,167544,664911,780978,843427,462034,539150,431100,588446,576985,735531,379767,992979,195445\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 394\n", + "Returned IDs: 759354,632640,561483,597161,777636,832291,587475,757034,67033,632708,445843,450445,248449,962207,488003,341783,844571,272209,990925,632814,64898,360986,720964,687757,316940,831415,269986,379767,602557,772511\n", + "Ground Truth: 759354,632640,561483,597161,777636,832291,587475,757034,67033,632708,445843,450445,248449,962207,488003,341783,844571,272209,990925,632814,64898,360986,720964,274079,687757,316940,831415,269986,379767,602557\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 395\n", + "Returned IDs: 409731,684037,351364,401473,242065,955838,618727,155566,938786,719294,564509,632080,65773,602442,912407,255425,405252,608314,339223,940505,445843,8501,166392,541811,958864,927595,309885,18529,407127,849922\n", + "Ground Truth: 409731,684037,351364,401473,242065,955838,618727,155566,938786,719294,564509,632080,65773,602442,912407,255425,405252,608314,904507,339223,940505,445843,8501,166392,541811,958864,927595,309885,124402,18529\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 396\n", + "Returned IDs: 311016,5675,657235,669722,755577,834937,364869,64898,829977,772146,469341,522760,166392,742370,447505,494155,735531,135017,832778,18529,177766,643793,813664,802512,463070,712316,492260,632080,313013,126043\n", + "Ground Truth: 311016,5675,657235,669722,755577,834937,364869,64898,829977,772146,469341,522760,166392,742370,447505,494155,735531,135017,832778,18529,177766,643793,813664,15628,802512,463070,712316,492260,632080,313013\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 399\n", + "Returned IDs: 29003,316751,652336,916915,506549,678945,402047,887742,124402,112463,850390,672661,505357,779902,266404,102320,638247,202635,548697,73673,318181,481233,904507,324554,341,902031,32161,500463,513181,371360\n", + "Ground Truth: 29003,316751,652336,916915,355866,506549,678945,402047,887742,526107,124402,112463,850390,672661,505357,779902,266404,102320,638247,202635,548697,318181,73673,481233,904507,324554,341,3397,902031,213208\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 400\n", + "Returned IDs: 886180,712316,938786,166392,864975,181413,868021,940505,603521,576985,176958,632080,379767,502367,403345,738229,286585,314296,982943,301135,490901,420265,152655,899260,704998,572455,61856,65773,834057,632814\n", + "Ground Truth: 886180,712316,938786,166392,864975,181413,868021,940505,603521,576985,176958,632080,379767,502367,403345,738229,286585,314296,982943,301135,490901,420265,152655,899260,514540,704998,572455,61856,489788,65773\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 401\n", + "Returned IDs: 771765,373940,371655,918389,465498,333703,73156,64273,29560,139554,414862,66624,670380,773701,554056,14050,904507,607382,150640,626727,549510,362244,284708,850030,507123,398327,350580,281792,147428,600302\n", + "Ground Truth: 771765,373940,371655,918389,465498,333703,73156,64273,29560,422335,139554,414862,66624,670380,773701,554056,14050,904507,607382,150640,626727,549510,362244,284708,850030,507123,398327,350580,281792,147428\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 403\n", + "Returned IDs: 151962,175218,263260,547437,236116,541449,703283,931780,790062,430561,334437,687287,287704,473718,442774,848757,50469,14079,888729,183691,24110,65388,263915,386620,357185,99837,777369,289537,180880,535046\n", + "Ground Truth: 151962,175218,263260,547437,236116,541449,703283,931780,790062,430561,334437,687287,287704,796968,473718,442774,848757,169048,50469,14079,888729,183691,24110,65388,263915,325330,386620,357185,99837,777369\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 405\n", + "Returned IDs: \n", + "Ground Truth: 712316,845747,442922,12057,314296,152070,773822,355840,433808,418997,632080,905134,83341,231837,23755,32805,357628,816481,817938,607059,75502,258029,774027,657098,286585,379644,485011,29774,266334,951923\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 408\n", + "Returned IDs: 624723,754108,580501,878069,79971,369154,100503,740178,899260,997473,297542,446305,173863,494866,945780,712316,398522,698668,456861,914264,502367,623200,4810,749704,132941,284845,257854,528098,187685,418997\n", + "Ground Truth: 624723,754108,580501,878069,79971,369154,100503,740178,899260,997473,297542,446305,173863,494866,945780,712316,625806,398522,698668,456861,914264,502367,623200,4810,368219,749704,132941,284845,257854,528098\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 410\n", + "Returned IDs: 445843,69280,97858,425496,904507,334309,587475,754490,42037,371655,258433,94264,407127,333703,918389,405252,670380,299412,528809,259018,787734,392449,214269,3397,632708,940816,12387,405014,740082,707601\n", + "Ground Truth: 445843,69280,97858,425496,904507,334309,587475,754490,42037,138910,371655,258433,94264,407127,333703,918389,405252,670380,299412,528809,259018,787734,392449,214269,3397,320419,64179,632708,507123,940816\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 411\n", + "Returned IDs: 238107,830735,445843,937821,371655,554056,278540,740082,417384,771765,902031,184048,66967,42037,68785,904507,182722,259018,383793,214269,120980,524973,404649,497305,357908,331816,10387,597161,607382,910875\n", + "Ground Truth: 238107,830735,445843,937821,371655,554056,278540,740082,417384,771765,902031,184048,66967,42037,68785,904507,182722,259018,103104,469324,383793,214269,120980,524973,404649,700738,3397,497305,357908,331816\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 413\n", + "Returned IDs: 576985,179288,712316,175218,592165,180244,65773,232478,544437,37639,780978,626132,605639,738229,314296,828521,581151,510662,619050,735531,940478,379767,181413,941506,802512,195445,966232,857420,235267,475148\n", + "Ground Truth: 576985,179288,712316,175218,592165,40940,180244,65773,232478,544437,37639,780978,151127,626132,605639,738229,314296,828521,581151,534803,510662,619050,735531,940478,379767,181413,941506,802512,195445,966232\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 419\n", + "Returned IDs: 146431,26867,612593,347177,835244,318014,422220,994193,618515,223274,117355,534803,909092,310427,219244,675117,588446,244662,528574,576985,952711,458815,712387,377442,401565,279455,644596,37639,447505,43681\n", + "Ground Truth: 146431,26867,612593,347177,835244,318014,422220,994193,618515,223274,117355,534803,909092,310427,219244,675117,588446,988834,244662,528574,576985,952711,458815,712387,377442,401565,279455,644596,37639,447505\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 421\n", + "Returned IDs: 632814,100503,379767,955379,712316,65773,464286,79971,670380,12387,576985,886180,176958,230650,645340,826244,124402,331195,759354,166392,554697,445843,407127,618727,12057,579250,874438,286585,982943,804808\n", + "Ground Truth: 632814,100503,379767,955379,712316,65773,464286,79971,670380,12387,576985,886180,176958,230650,645340,826244,124402,331195,759354,166392,554697,445843,407127,618727,176751,12057,579250,874438,286585,982943\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 422\n", + "Returned IDs: 12713,227327,180880,775948,706968,224318,128191,103989,770672,681267,195445,283397,888976,243912,272687,426893,600302,758635,354173,135017,386620,64898,292605,296440,33361,399024,773068,650848,837640,548164\n", + "Ground Truth: 12713,227327,180880,775948,706968,224318,128191,103989,770672,681267,195445,283397,888976,243912,272687,426893,600302,758635,354173,135017,386620,64898,887727,292605,296440,33361,399024,773068,650848,837640\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 425\n", + "Returned IDs: 482569,783900,664911,757816,600302,64898,479774,283397,33923,837640,571874,354173,111684,135017,245332,859855,975590,447505,543216,269352,211300,34979,426893,643363,242938,319630,889753,166392,849922,275549\n", + "Ground Truth: 482569,783900,664911,757816,600302,64898,479774,283397,33923,837640,571874,354173,111684,135017,245332,859855,975590,447505,543216,269352,211300,34979,426893,643363,242938,319630,889753,166392,849922,155972\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 426\n", + "Returned IDs: 295500,995591,541428,728837,194080,85322,632814,245332,83341,683736,632080,785700,5031,712316,176958,445370,507158,462345,134044,314296,212360,242065,147428,476798,811826,735994,773822,447505,141367,612297\n", + "Ground Truth: 295500,995591,541428,728837,194080,85322,632814,245332,83341,683736,632080,785700,5031,712316,176958,445370,507158,868021,462345,134044,314296,212360,242065,147428,476798,811826,473460,735994,608899,773822\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 427\n", + "Returned IDs: 932030,745484,18529,55426,995005,668876,97135,69527,64898,642021,177766,87859,208424,153636,772146,600302,457756,849922,924661,881687,245332,544861,585438,687287,2907,791872,560935,534447,276798,5675\n", + "Ground Truth: 932030,745484,18529,55426,995005,668876,97135,69527,64898,642021,177766,87859,208424,153636,772146,600302,457756,849922,924661,881687,245332,169048,544861,585438,687287,2907,791872,560935,534447,276798\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 430\n", + "Returned IDs: 326309,631399,72631,375420,873157,643667,94582,176615,281286,379767,311637,14717,441607,996149,206564,343268,232478,911639,18529,369753,706695,599977,465190,194080,234140,185593,448520,482019,798845,193914\n", + "Ground Truth: 326309,631399,72631,375420,873157,643667,94582,176615,281286,379767,311637,14717,441607,996149,206564,343268,232478,911639,18529,369753,706695,599977,135017,498101,465190,194080,234140,185593,847404,683736\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 433\n", + "Returned IDs: 513955,764472,216480,233361,166392,879890,610912,700079,522760,992979,733946,378893,599163,151457,266075,687287,564509,167515,80420,170950,377842,912561,447505,595698,361000,563778,98827,4493,654926,41301\n", + "Ground Truth: 513955,764472,216480,233361,166392,879890,610912,700079,522760,992979,733946,378893,599163,151457,266075,687287,564509,167515,2774,80420,170950,377842,912561,447505,595698,361000,563778,98827,361863,4493\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 434\n", + "Returned IDs: 371655,115722,670380,579250,333703,357908,355866,904507,101653,916915,771765,29560,737135,441751,445843,236271,309885,976705,627144,672661,383793,414862,124402,454064,482176,112463,595330,203353,14717,184048\n", + "Ground Truth: 371655,115722,670380,579250,333703,357908,355866,904507,101653,916915,771765,29560,737135,441751,445843,236271,309885,976705,627144,672661,383793,414862,124402,454064,482176,112463,595330,203353,14717,612263\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 436\n", + "Returned IDs: 44594,275549,89030,713355,509338,770124,206564,18914,994101,343268,640363,88740,84975,476498,518738,165843,232478,857420,873157,861057,818625,572833,631398,682567,593512,779874,436535,499757,5675,729163\n", + "Ground Truth: 44594,275549,89030,713355,509338,770124,206564,18914,994101,343268,640363,88740,84975,476498,518738,165843,232478,182045,857420,873157,861057,818625,572833,631398,682567,593512,779874,436535,499757,5675\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 437\n", + "Returned IDs: 479405,949711,392788,917766,744351,88319,461836,421446,600530,683097,6338,22440,387460,742370,115855,972926,138910,10572,519324,579250,976404,892079,297731,862264,337726,115722,192701,14717,12387,126043\n", + "Ground Truth: 479405,949711,392788,917766,744351,88319,461836,421446,600530,683097,6338,22440,387460,742370,115855,972926,138910,10572,625617,519324,579250,976404,476683,892079,297731,862264,337726,115722,192701,14717\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 440\n", + "Returned IDs: 586441,76673,282531,409130,613209,3386,608244,476683,492005,90600,734241,332080,916915,760046,984432,341783,350257,486639,445843,467212,247044,956035,214269,608314,415899,850390,613417,925919,6153,182045\n", + "Ground Truth: 586441,76673,282531,409130,613209,3386,608244,476683,28015,492005,90600,734241,332080,374914,916915,760046,984432,341783,498312,350257,278540,486639,445843,467212,872921,247044,956035,214269,608314,415899\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 442\n", + "Returned IDs: 366682,218270,791291,890360,299325,735994,270634,301466,478400,878069,407577,270796,947950,126770,486639,541428,73673,479167,428106,373940,584838,324554,482807,209632,206153,916915,695965,782180,450833,344295\n", + "Ground Truth: 366682,467316,218270,791291,392618,890360,299325,735994,800150,270634,776912,301466,478400,878069,407577,270796,947950,126770,486639,541428,73673,479167,428106,373940,584838,324554,482807,209632,850030,206153\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 445\n", + "Returned IDs: 166392,702283,734241,59835,945682,61856,207041,65773,732515,814534,100503,87686,754108,724881,893919,940384,664447,523569,266075,379767,878086,652336,712316,996028,955379,447505,179536,299027,135017,79136\n", + "Ground Truth: 166392,702283,734241,59835,945682,61856,207041,65773,732515,516493,814534,100503,87686,754108,724881,893919,940384,664447,523569,266075,379767,878086,652336,712316,996028,955379,447505,179536,299027,135017\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 446\n", + "Returned IDs: 735994,664447,85322,64898,334918,85943,212360,354173,340429,142868,663199,991066,94582,600302,900720,141367,507158,115722,650367,46160,87686,166392,849922,447505,971100,876126,343268,548164,413276,488003\n", + "Ground Truth: 735994,664447,85322,64898,334918,85943,212360,354173,340429,142868,292605,663199,991066,94582,600302,900720,141367,507158,115722,650367,46160,87686,166392,849922,447505,971100,876126,343268,548164,413276\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 447\n", + "Returned IDs: 340206,425476,137022,787094,153795,631399,963711,388670,458408,426496,68785,455789,804808,994792,897071,469428,834643,978533,833707,477305,701821,632814,867184,845243,41169,772998,796968,965362,640250,499757\n", + "Ground Truth: 340206,425476,137022,787094,153795,631399,963711,388670,458408,426496,68785,12713,455789,804808,742504,994792,897071,469428,834643,978533,833707,477305,701821,632814,910256,867184,758635,845243,41169,772998\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 448\n", + "Returned IDs: \n", + "Ground Truth: 944204,771765,519324,886952,3563,214269,188747,275459,299325,445843,552386,956610,240932,729998,485035,238107,994542,693604,988352,680706,73156,396777,64273,507770,3397,184048,366272,994164,915301,597805\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 449\n", + "Returned IDs: 963711,492133,5031,804808,6153,445370,612297,544861,720964,806445,631399,695965,916915,683736,24196,87686,632814,931780,6488,579250,18529,379767,64898,945780,166392,469428,882806,154098,73368,507158\n", + "Ground Truth: 963711,492133,5031,804808,6153,445370,612297,544861,720964,806445,631399,695965,916915,683736,24196,87686,632814,931780,6488,579250,18529,69609,379767,64898,945780,166392,469428,882806,154098,73368\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 450\n", + "Returned IDs: 284708,350257,608244,686408,997843,656989,456450,294599,282531,959763,656698,261675,772035,247044,687757,85808,837825,656276,214269,298323,654082,41301,784919,176200,759354,328230,236271,209838,402427,676666\n", + "Ground Truth: 284708,350257,608244,686408,997843,656989,456450,294599,282531,959763,599813,656698,261675,772035,247044,687757,85808,837825,656276,940691,214269,298323,654082,41301,784919,176200,759354,328230,236271,209838\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 451\n", + "Returned IDs: 892079,139298,641329,517119,671964,88319,308620,316751,109226,405252,745484,363525,950893,115855,398242,772035,643793,357434,421446,513181,972926,411988,264045,917766,978689,466875,41301,18830,211528,729998\n", + "Ground Truth: 892079,139298,976507,641329,517119,671964,88319,308620,316751,109226,405252,745484,363525,950893,115855,398242,772035,643793,357434,421446,513181,972926,411988,264045,917766,978689,466875,41301,18830,211528\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 452\n", + "Returned IDs: 614,971257,739507,878069,867184,393202,407127,590921,809535,360299,251296,579250,259343,6153,998445,652336,722286,158345,339223,570805,959480,288817,441751,897937,931780,492133,112463,529474,544861,97858\n", + "Ground Truth: 614,971257,739507,878069,867184,393202,407127,577491,590921,809535,360299,251296,579250,259343,6153,998445,652336,722286,158345,339223,570805,959480,288817,441751,897937,931780,492133,112463,529474,544861\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 453\n", + "Returned IDs: 783900,887959,672614,285884,313929,796405,506149,796968,674899,781200,23934,639431,152655,100503,745484,410377,830239,712316,756642,362956,385733,350669,595687,403345,323734,576985,479774,517119,176059,313013\n", + "Ground Truth: 783900,887959,672614,285884,313929,796405,506149,796968,674899,781200,23934,639431,152655,827552,100503,745484,410377,382030,830239,402047,712316,756642,362956,948112,385733,350669,595687,403345,323734,576985\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 455\n", + "Returned IDs: 687004,323103,80622,18207,940443,378836,413275,587977,976404,802512,675798,830270,571572,706968,788045,784568,379767,9628,769082,108932,764838,780978,325191,295855,581331,283397,912407,706695,129512,400877\n", + "Ground Truth: 687004,323103,80622,18207,940443,378836,413275,587977,976404,802512,675798,830270,571572,706968,788045,784568,379767,9628,769082,108932,764838,780978,325191,295855,909092,581331,283397,912407,565531,706695\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 456\n", + "Returned IDs: 919726,3350,115855,466875,320927,112463,160315,393944,298415,464286,381463,640584,150090,670380,419803,232391,775449,286033,931681,9024,829977,560461,568773,920983,392449,342624,949077,80107,316030,335201\n", + "Ground Truth: 919726,3350,115855,466875,320927,112463,160315,393944,298415,464286,381463,640584,826126,150090,900717,670380,419803,232391,925919,775449,286033,931681,9024,126043,829977,560461,568773,26152,920983,392449\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 457\n", + "Returned IDs: 662261,409130,720943,612117,988176,729998,6338,582616,496966,253776,172629,632640,959480,597805,998445,554056,445924,473819,293149,886952,755002,369956,167545,214017,867864,608244,737135,600530,893451,258356\n", + "Ground Truth: 662261,409130,720943,612117,988176,729998,6338,582616,496966,253776,172629,632640,959480,597805,998445,554056,445924,473819,293149,886952,755002,369956,167545,214017,134346,867864,608244,737135,600530,893451\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 459\n", + "Returned IDs: 232228,181413,426423,581151,825157,619050,830349,571493,9927,848907,314296,424560,510662,305975,319689,521706,694921,992387,89042,784785,611032,576985,622920,715223,77201,740178,443302,840484,921652,712924\n", + "Ground Truth: 232228,181413,426423,581151,249377,825157,619050,830349,571493,9927,848907,314296,424560,510662,305975,319689,521706,694921,992387,89042,784785,611032,576985,622920,715223,77201,740178,443302,840484,921652\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 460\n", + "Returned IDs: 37859,404975,814886,27919,64273,63145,418915,288817,9024,759354,529474,955841,680706,773701,102243,711982,299325,904507,14717,335201,172701,441751,916915,735994,63471,796968,6338,652336,579250,450445\n", + "Ground Truth: 37859,404975,814886,27919,64273,63145,418915,288817,9024,759354,529474,955841,680706,773701,102243,711982,299325,904507,14717,335201,172701,441751,916915,735994,63471,796968,6338,652336,575058,579250\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 461\n", + "Returned IDs: 963866,323734,958864,429758,966401,296010,852119,632080,4855,962546,310435,207041,175621,899582,364667,912407,382634,544861,403345,881397,68181,331816,851415,420265,353622,186375,773831,709777,779902,351364\n", + "Ground Truth: 963866,323734,958864,429758,966401,296010,852119,632080,4855,962546,310435,207041,175621,899582,364667,912407,382634,544861,513624,403345,881397,327573,68181,331816,851415,420265,353622,186375,773831,156060\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 462\n", + "Returned IDs: 5525,353622,372134,664447,685079,49209,958864,296718,297601,910402,732515,691762,532792,331608,488219,272209,307804,632080,893919,605415,79136,331145,296625,166392,379644,407004,299492,4512,998739,675467\n", + "Ground Truth: 5525,353622,372134,664447,685079,49209,958864,296718,297601,910402,732515,691762,532792,331608,488219,272209,307804,632080,893919,605415,79136,331145,514540,296625,166392,379644,407004,299492,4512,998739\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 465\n", + "Returned IDs: 980270,582477,280671,626727,779309,35070,600302,727222,581180,777881,657512,92915,378429,616877,700519,88319,927346,662929,677600,687287,881687,73368,631733,13656,6488,363525,906623,354173,679479,785700\n", + "Ground Truth: 980270,582477,280671,626727,779309,35070,600302,727222,581180,777881,657512,668876,92915,378429,616877,700519,88319,927346,662929,516184,677600,687287,881687,73368,631733,13656,6488,363525,906623,354173\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 466\n", + "Returned IDs: 460806,413275,709497,581151,909092,706695,379767,618559,94582,9257,580578,101388,616251,791727,843427,839858,847322,864975,576985,780978,353622,998221,390336,735531,392918,301135,696808,521706,176615,626132\n", + "Ground Truth: 460806,413275,709497,581151,987690,909092,24196,706695,379767,618559,94582,9257,580578,101388,616251,791727,843427,839858,847322,864975,576985,780978,353622,998221,390336,735531,392918,301135,206564,696808\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 467\n", + "Returned IDs: 261893,296374,775948,325330,600302,755815,892989,240380,224067,735520,27283,263915,555721,750654,355239,230543,571137,924661,981517,426893,888976,249700,343268,491594,389553,799317,24110,791872,932030,64898\n", + "Ground Truth: 261893,296374,775948,325330,600302,755815,892989,240380,224067,735520,27283,263915,555721,750654,355239,230543,571137,924661,981517,426893,861057,888976,249700,343268,491594,389553,799317,24110,791872,932030\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 468\n", + "Returned IDs: 544861,357403,750188,541759,687287,180244,379767,576985,626132,151127,178358,466784,79971,259401,561189,445370,12387,292970,856746,900187,643363,977982,473460,745185,744631,749691,706695,103210,121404,925914\n", + "Ground Truth: 544861,357403,750188,541759,687287,180244,379767,576985,626132,151127,178358,466784,79971,259401,561189,445370,12387,292970,856746,900187,643363,977982,473460,745185,19912,744631,425093,749691,706695,103210\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 470\n", + "Returned IDs: 745484,549892,582477,64898,469341,598044,447753,789801,350580,502314,378429,173073,92915,447505,541811,643363,386589,583564,38444,757816,718644,581180,637811,494155,232600,166392,319630,245332,912561,358859\n", + "Ground Truth: 745484,549892,582477,64898,469341,598044,447753,789801,350580,502314,378429,173073,92915,447505,541811,643363,386589,583564,38444,757816,718644,581180,637811,494155,232600,166392,319630,245332,673607,912561\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 471\n", + "Returned IDs: 30077,100503,848600,870288,634279,893919,76961,818896,981131,781200,685079,307804,732515,946437,600302,236959,664447,46160,828843,353622,6404,479197,866595,372134,294741,372207,111684,137292,671985,652752\n", + "Ground Truth: 30077,100503,848600,870288,634279,893919,76961,818896,981131,781200,685079,307804,732515,946437,600302,236959,664447,46160,828843,353622,6404,479197,866595,372134,571413,294741,372207,111684,137292,671985\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 472\n", + "Returned IDs: 976404,729738,363525,544861,645299,564509,683549,677600,801132,65199,73368,166392,775428,8501,600302,533939,770486,967181,615485,652155,464286,643793,458408,834937,631399,813664,19091,143117,272687,194718\n", + "Ground Truth: 976404,729738,363525,544861,645299,564509,683549,677600,801132,65199,73368,166392,775428,8501,600302,533939,770486,967181,615485,652155,464286,643793,458408,834937,631399,813664,80031,19091,143117,235379\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 473\n", + "Returned IDs: 263260,255378,416923,126973,888121,88916,889023,813664,902631,516989,238846,854414,657256,129434,424363,553661,196926,322294,447505,663708,173393,882993,325330,313043,746324,227327,360551,910313,269352,240380\n", + "Ground Truth: 263260,255378,416923,126973,888121,88916,889023,813664,902631,516989,238846,854414,487737,657256,129434,424363,553661,196926,322294,447505,663708,173393,882993,325330,313043,746324,227327,360551,910313,343842\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 474\n", + "Returned IDs: 963987,533939,88319,583564,15628,862264,775428,445121,544861,802512,552386,582477,615485,531337,770486,469341,773964,591521,772035,828137,834937,64898,99285,44813,972926,729998,619853,345023,54025,742370\n", + "Ground Truth: 963987,533939,88319,583564,15628,862264,775428,445121,544861,802512,552386,582477,615485,531337,770486,469341,773964,525546,591521,772035,828137,834937,64898,99285,44813,972926,729998,619853,345023,54025\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 476\n", + "Returned IDs: 270134,22440,467340,719860,888255,55886,972926,878693,342624,647270,775449,201261,136912,166392,572761,687436,866926,238088,241,80107,855897,827499,102754,775128,112463,115722,334282,95761,931780,488003\n", + "Ground Truth: 270134,22440,467340,719860,888255,55886,972926,878693,342624,647270,775449,201261,337726,136912,166392,572761,687436,866926,238088,241,80107,855897,827499,102754,775128,112463,978594,115722,334282,95761\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 477\n", + "Returned IDs: 104889,347775,408339,701821,96536,787094,222936,823366,69609,6488,689227,232391,360401,931780,945864,879652,245332,603712,796968,343498,272533,519773,925914,142868,378429,687287,785700,158639,687102,73368\n", + "Ground Truth: 104889,347775,408339,701821,96536,787094,222936,823366,69609,6488,689227,232391,360401,931780,945864,879652,245332,603712,796968,343498,272533,519773,925914,965362,142868,378429,687287,775449,785700,158639\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 478\n", + "Returned IDs: 893919,232600,23755,708492,141367,783900,848232,660886,785700,980081,236959,299027,469341,534044,637811,789801,479197,784568,358859,225260,597589,30077,495221,664447,474041,763544,834937,601998,174389,441928\n", + "Ground Truth: 893919,232600,23755,708492,141367,783900,848232,660886,785700,980081,236959,299027,469341,534044,637811,789801,479197,784568,358859,225260,597589,30077,495221,664447,474041,763544,633425,834937,601998,174389\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 479\n", + "Returned IDs: 902031,952626,621454,343498,331075,867184,283989,904507,251296,435000,96536,949777,992979,446064,737135,101983,102754,371698,997496,206153,931780,502919,522760,569756,687436,759354,773701,932134,890360,467438\n", + "Ground Truth: 902031,952626,621454,343498,331075,867184,283989,904507,251296,435000,96536,949777,992979,607382,446064,737135,101983,102754,371698,997496,206153,931780,502919,48754,522760,569756,687436,759354,773701,932134\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 480\n", + "Returned IDs: 672661,916915,282531,371655,214269,402047,360986,29560,350257,392618,552386,670380,206694,405014,98827,124402,18830,725978,918389,475669,99082,612297,549510,656276,91663,695965,974941,83341,990925,120980\n", + "Ground Truth: 672661,916915,282531,371655,214269,402047,360986,29560,350257,392618,552386,670380,206694,404649,405014,819736,98827,124402,18830,725978,918389,475669,99082,612297,549510,656276,91663,784919,695965,543653\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 481\n", + "Returned IDs: 541759,393110,522824,595241,835425,441607,110400,251853,659147,466784,770242,455501,949777,422454,874438,934315,286854,739816,448518,357403,436016,796629,727702,900187,131487,618559,230204,339090,788045,961536\n", + "Ground Truth: 541759,393110,522824,595241,835425,441607,110400,251853,659147,466784,770242,455501,949777,422454,874438,934315,286854,739816,448518,357403,436016,796629,534299,727702,900187,131487,618559,230204,339090,788045\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 482\n", + "Returned IDs: 635965,123797,167515,680438,100503,826244,908724,166392,454064,645340,922747,684477,604752,79971,69280,23755,710026,541759,447505,132941,158345,712316,772146,300183,580501,698668,67033,742412,623200,899944\n", + "Ground Truth: 635965,123797,167515,680438,100503,826244,908724,166392,454064,645340,922747,684477,604752,79971,69280,23755,710026,541759,447505,132941,151457,158345,465145,712316,621725,772146,300183,580501,698668,67033\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 483\n", + "Returned IDs: 463070,117859,309885,608314,214269,802512,173422,628922,544437,722764,677600,691762,775428,533939,419803,883744,189442,805131,510927,544861,657402,800649,921446,595330,331816,397196,978005,494823,259018,181709\n", + "Ground Truth: 463070,117859,309885,608314,214269,802512,173422,628922,544437,722764,677600,691762,775428,533939,419803,960873,883744,189442,805131,510927,544861,657402,800649,921446,595330,331816,397196,978005,494823,259018\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 484\n", + "Returned IDs: 834643,163564,745185,407004,607059,185240,826126,79971,234369,319829,548164,773822,712316,376992,636425,314296,604597,632080,940505,632814,443374,637450,514643,299027,639932,83341,623711,445370,817938,408339\n", + "Ground Truth: 834643,163564,745185,407004,607059,185240,826126,79971,234369,319829,548164,773822,712316,376992,636425,314296,312098,604597,632080,940505,632814,443374,637450,514643,299027,639932,83341,623711,445370,817938\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 486\n", + "Returned IDs: 41301,663376,674336,523990,291163,193914,528944,83341,126043,951923,745484,80169,331816,950893,632080,627144,259401,687257,522760,476798,180244,178358,311016,643793,300183,710026,141170,967189,927595,317944\n", + "Ground Truth: 41301,663376,674336,523990,291163,193914,528944,83341,126043,951923,745484,80169,331816,950893,632080,292970,627144,259401,687257,522760,476798,180244,178358,745185,311016,643793,300183,710026,141170,967189\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 487\n", + "Returned IDs: 712316,882914,12057,43266,314296,232600,79971,817938,297415,923534,86716,319829,380365,433808,845747,495445,231837,137292,632080,23755,932033,284845,686429,420265,111684,217463,752443,384635,537644,357628\n", + "Ground Truth: 712316,882914,12057,43266,87204,314296,232600,79971,817938,297415,923534,86716,319829,380365,433808,845747,495445,231837,137292,632080,23755,463206,932033,284845,686429,420265,730876,111684,217463,752443\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 489\n", + "Returned IDs: 855217,888255,480557,115722,973356,585034,433497,85322,196555,650367,764427,290395,735994,761308,413202,398242,825868,642050,938409,814015,313302,34523,388736,449613,904507,631733,454678,612263,650588,276371\n", + "Ground Truth: 855217,888255,480557,115722,973356,585034,433497,190576,85322,196555,650367,764427,290395,735994,761308,413202,398242,825868,642050,938409,814015,313302,34523,388736,449613,904507,631733,454678,612263,650588\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 492\n", + "Returned IDs: 311604,4439,187105,466224,275549,325874,266466,577468,29472,265408,63990,319630,824160,272687,235379,90449,476498,359649,887727,582636,65388,502001,783367,854483,891795,18529,61845,548164,303352,550602\n", + "Ground Truth: 311604,4439,187105,466224,275549,325874,266466,577468,29472,265408,63990,319630,824160,272687,235379,90449,476498,359649,887727,582636,65388,502001,783367,286559,854483,891795,18529,61845,548164,303352\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 493\n", + "Returned IDs: 585438,447505,13656,745484,517119,924661,904507,210437,641329,522760,837895,828137,379767,972926,477417,896151,878693,687287,88319,589289,813664,41301,61856,630220,582477,931780,968295,772146,621622,97135\n", + "Ground Truth: 585438,447505,13656,745484,517119,924661,904507,210437,641329,522760,837895,828137,379767,972926,477417,169048,896151,878693,687287,88319,589289,813664,41301,61856,630220,582477,931780,968295,772146,86869\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 494\n", + "Returned IDs: 871262,971100,504324,897186,340429,85322,290395,467340,101705,509748,196555,650427,155550,743654,472415,179536,288169,731160,855217,814015,433497,855897,641164,447505,90803,476798,454678,571376,94582,553958\n", + "Ground Truth: 871262,971100,504324,897186,340429,85322,290395,467340,545048,101705,509748,196555,650427,155550,743654,472415,179536,288169,731160,855217,814015,433497,855897,641164,447505,90803,476798,407004,454678,180244\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 496\n", + "Returned IDs: 97135,447505,454216,687287,185593,259018,850368,811799,522760,677600,797438,544861,576985,701447,530372,600302,423033,753890,379767,849922,800649,710026,364869,835244,357908,850390,581151,523569,175218,477417\n", + "Ground Truth: 97135,447505,454216,687287,185593,259018,850368,811799,522760,677600,797438,544861,576985,701447,530372,600302,423033,753890,379767,849922,800649,710026,364869,835244,357908,850390,740178,581151,523569,175218\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 497\n", + "Returned IDs: 519773,925914,319630,687102,142868,491594,823366,224067,658397,236959,848232,650613,245332,354173,967574,796968,902366,676582,69609,502001,664447,261893,230358,600302,905384,906520,479197,482019,411149,64753\n", + "Ground Truth: 519773,925914,319630,687102,142868,491594,823366,224067,658397,236959,848232,650613,245332,354173,967574,796968,902366,676582,69609,502001,664447,454519,261893,230358,600302,905384,906520,479197,482019,411149\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 498\n", + "Returned IDs: 308417,120482,191784,166392,74171,23934,733946,992979,378893,801035,702283,313013,912561,879890,216480,388778,447505,563473,274612,700079,595698,513955,598044,266075,780251,645340,809053,410377,599163,349346\n", + "Ground Truth: 308417,120482,191784,166392,74171,23934,733946,992979,378893,801035,702283,313013,912561,879890,216480,388778,447505,563473,274612,84965,700079,595698,830239,513955,598044,266075,780251,645340,809053,410377\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 499\n", + "Returned IDs: 516184,34355,116739,687287,89460,159408,454347,848655,663376,943766,819736,195445,897240,147614,576985,570591,419480,283960,350669,625806,377104,304164,361590,434300,790176,577035,392957,379767,409731,940662\n", + "Ground Truth: 516184,34355,116739,687287,89460,159408,454347,848655,663376,943766,819736,195445,897240,147614,576985,961178,570591,419480,523990,283960,350669,625806,377104,547189,304164,361590,317812,434300,497305,790176\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 500\n", + "Returned IDs: 530151,445370,967574,541811,600302,686408,354173,632814,12387,242434,597589,637050,799317,458408,789801,18529,613133,851545,343268,62820,492260,483807,87686,745484,465190,720964,99285,498312,926270,378429\n", + "Ground Truth: 530151,445370,967574,541811,600302,686408,354173,632814,12387,242434,597589,637050,799317,458408,789801,18529,613133,851545,343268,62820,492260,483807,87686,745484,465190,720964,99285,498312,61845,926270\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 501\n", + "Returned IDs: 576724,893919,564509,783900,482569,91320,781200,601998,940505,4493,664447,154098,319630,784568,236959,924904,350669,358859,902521,142868,479197,488003,708492,135017,119066,166392,176958,140085,36724,996028\n", + "Ground Truth: 576724,893919,564509,783900,482569,91320,781200,601998,940505,4493,664447,154098,319630,784568,236959,924904,350669,358859,902521,142868,479197,488003,708492,135017,119066,166392,901831,176958,140085,36724\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 502\n", + "Returned IDs: 102502,147951,109630,319829,686429,216428,339768,923534,982771,537644,632080,932033,919913,462345,176958,893429,26053,565531,637450,61475,231837,838678,615795,53763,816365,817938,79971,952937,606966,211488\n", + "Ground Truth: 102502,147951,109630,319829,686429,216428,339768,923534,982771,537644,632080,932033,462345,919913,176958,893429,26053,565531,637450,61475,231837,838678,615795,87204,53763,816365,817938,79971,952937,606966\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 503\n", + "Returned IDs: 371655,63145,504643,541811,692188,445370,225501,437989,350257,544437,695965,916915,500463,232002,874438,311197,813664,456861,938397,64273,579250,978956,109016,796968,188374,612297,850390,686408,739092,344295\n", + "Ground Truth: 371655,63145,504643,541811,692188,445370,225501,437989,350257,544437,695965,916915,500463,232002,874438,311197,813664,456861,938397,64273,579250,978956,109016,796968,188374,612297,850390,266404,686408,739092\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 506\n", + "Returned IDs: 255136,712538,559942,737230,163762,954978,800801,938312,487810,927568,874268,367627,531095,916353,945052,563087,624402,582058,104439,754497,144348,800471,835425,150564,175852,863748,156608,410462,749777,259685\n", + "Ground Truth: 255136,712538,559942,737230,163762,954978,800801,938312,487810,927568,874268,985292,367627,531095,916353,945052,563087,624402,582058,104439,754497,144348,800471,835425,150564,175852,863748,156608,410462,749777\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 507\n", + "Returned IDs: 445294,352154,887727,319630,65773,599977,354173,69609,683736,347041,119066,503974,612297,664447,275549,560935,576724,910768,801132,245332,691997,888411,135729,64275,467212,242065,909278,18529,565068,376151\n", + "Ground Truth: 445294,352154,887727,319630,65773,599977,354173,69609,683736,347041,119066,503974,612297,664447,275549,560935,576724,910768,801132,245332,691997,888411,135729,582058,64275,467212,242065,909278,18529,565068\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 508\n", + "Returned IDs: 695965,6153,830735,755666,805604,916915,379767,409731,259018,850275,3386,403345,282531,24196,865901,500463,405252,945780,493831,490901,278540,701447,350669,311197,357908,54091,73777,368888,968295,393202\n", + "Ground Truth: 695965,6153,830735,755666,805604,916915,379767,409731,259018,850275,3386,403345,282531,24196,865901,500463,405252,945780,493831,490901,599813,278540,701447,350669,311197,357908,54091,73777,368888,185931\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 509\n", + "Returned IDs: 304643,543859,343268,743915,827552,514080,194080,804662,477417,828009,664447,83986,631399,708631,839858,254476,573939,728241,403394,964324,797438,151369,519135,512544,22871,131161,41699,336167,841327,815877\n", + "Ground Truth: 304643,543859,343268,743915,827552,514080,194080,804662,477417,43249,828009,664447,83986,631399,708631,839858,254476,573939,908344,728241,403394,964324,797438,151369,519135,512544,22871,131161,41699,336167\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 511\n", + "Returned IDs: 3563,299412,784919,701006,427455,69280,64273,371655,124402,552562,10387,333703,445843,288169,916915,237956,672661,94264,523190,370134,393202,331816,200855,172629,41301,279054,833949,561483,632708,417872\n", + "Ground Truth: 3563,299412,784919,701006,427455,69280,64273,371655,124402,552562,10387,333703,445843,288169,916915,237956,672661,94264,523190,370134,393202,331816,200855,172629,41301,279054,221330,833949,561483,632708\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 513\n", + "Returned IDs: 64898,405219,358859,141367,600302,469341,238440,637811,445370,927595,242434,135017,463070,785700,544437,755577,563923,541811,534553,176144,272533,310435,354173,980270,463724,664447,514080,173422,706968,683736\n", + "Ground Truth: 64898,405219,358859,141367,600302,469341,238440,637811,445370,927595,242434,135017,463070,785700,544437,755577,563923,541811,534553,176144,272533,310435,354173,464286,980270,463724,664447,514080,173422,706968\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 514\n", + "Returned IDs: 98827,481922,990925,847030,73673,927167,270634,614475,995884,633831,934006,632778,525546,804586,166473,434427,633425,340503,176751,228265,9024,904507,94963,612297,450445,780336,855908,225390,288817,758525\n", + "Ground Truth: 98827,481922,990925,847030,73673,927167,270634,614475,995884,633831,934006,676869,632778,525546,804586,166473,434427,633425,340503,176751,228265,9024,904507,94963,612297,450445,780336,855908,575058,225390\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 515\n", + "Returned IDs: 545183,272209,650153,543216,702096,783900,106582,294741,166392,335184,757816,137292,702283,111684,232600,23755,652336,408339,940505,884104,207041,482569,555552,712316,523569,286585,891930,718291,219344,698668\n", + "Ground Truth: 545183,272209,650153,543216,702096,783900,106582,294741,166392,335184,757816,137292,702283,111684,232600,23755,652336,408339,940505,884104,207041,482569,555552,712316,523569,11606,286585,891930,718291,104851\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 517\n", + "Returned IDs: 903268,297203,599683,314296,429080,319829,502367,102502,982771,171349,418997,565531,510938,176958,17986,280745,374483,622318,173863,712316,795343,142273,952937,232600,773822,784586,289925,217463,632080,112040\n", + "Ground Truth: 903268,297203,599683,314296,429080,319829,502367,102502,982771,171349,418997,565531,510938,383541,176958,17986,280745,374483,622318,173863,712316,795343,142273,952937,232600,773822,784586,289925,217463,632080\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 518\n", + "Returned IDs: 847322,232478,343268,490901,229019,655585,901831,599977,780978,254476,841327,638804,885555,402371,157745,98027,638004,894731,948817,673519,228238,694245,94582,873157,488434,706695,89030,857420,24625,459789\n", + "Ground Truth: 847322,232478,343268,490901,229019,655585,901831,599977,780978,254476,841327,638804,885555,402371,157745,98027,638004,894731,948817,673519,228238,848352,694245,94582,873157,488434,706695,89030,857420,24625\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 520\n", + "Returned IDs: 41301,632708,15628,445843,279054,579250,120980,417872,745185,124402,832291,759354,345023,958864,706880,677600,670380,976404,554056,918389,181784,632640,828137,275706,643793,327683,879652,623702,21239,12387\n", + "Ground Truth: 41301,632708,15628,445843,279054,579250,120980,417872,745185,124402,832291,759354,345023,958864,706880,677600,670380,976404,554056,918389,541811,181784,632640,828137,275706,643793,327683,879652,623702,21239\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 521\n", + "Returned IDs: 690058,115855,885272,166392,972926,737396,393944,231416,572761,331977,466875,405919,345023,773964,972594,126043,964592,848656,812245,293149,531337,770486,764427,900717,528917,269375,802512,494823,801035,311016\n", + "Ground Truth: 690058,115855,885272,972926,166392,737396,696215,393944,231416,572761,331977,466875,652155,792419,405919,345023,773964,972594,126043,964592,26152,848656,812245,293149,531337,770486,764427,900717,528917,269375\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 523\n", + "Returned IDs: 275459,180658,700738,549510,73156,973611,512910,492133,350257,572761,505804,63145,904300,728623,925312,114250,880836,830889,994542,814886,259054,525546,519870,680706,373940,291558,138910,473819,994164,519324\n", + "Ground Truth: 275459,180658,700738,549510,73156,973611,512910,492133,350257,572761,505804,63145,904300,728623,925312,552482,114250,880836,830889,994542,170539,814886,259054,525546,26152,519870,680706,373940,291558,138910\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 524\n", + "Returned IDs: 855897,664624,463657,871262,311016,304921,907602,658210,155550,331816,878843,972594,126043,195445,432017,409130,115855,973356,828466,80061,283989,150640,971100,446064,255425,447546,300183,223552,117859,355866\n", + "Ground Truth: 855897,664624,463657,871262,311016,304921,907602,658210,155550,331816,878843,358121,972594,126043,195445,432017,409130,115855,973356,828466,80061,283989,150640,971100,446064,255425,447546,300183,223552,117859\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 525\n", + "Returned IDs: 775428,695553,773964,602557,533939,457263,802512,447819,320419,660159,883744,572761,463070,73368,2907,469341,632080,117859,544861,181709,834937,434018,805131,126043,282337,861541,18529,927595,643793,64898\n", + "Ground Truth: 775428,695553,773964,602557,533939,457263,802512,447819,320419,660159,753044,883744,572761,463070,73368,2907,469341,632080,117859,544861,181709,834937,434018,805131,126043,282337,861541,18529,927595,643793\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 531\n", + "Returned IDs: 684477,83775,940047,658397,124345,409731,988834,475148,115855,908724,311016,947990,387620,126043,536427,576985,305975,242065,12387,398697,342624,632814,465601,151127,626132,665024,825802,940505,964592,753890\n", + "Ground Truth: 684477,83775,940047,658397,124345,409731,988834,475148,115855,908724,866595,413276,311016,947990,387620,126043,536427,576985,305975,242065,12387,398697,342624,632814,465601,151127,626132,665024,915153,825802\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 533\n", + "Returned IDs: 135017,236023,672650,675798,10915,832866,907990,476498,441577,283397,142219,691577,231182,37793,89030,296374,848043,227327,765929,849922,84975,850368,597589,887727,463724,347056,315425,619853,991066,124617\n", + "Ground Truth: 135017,236023,672650,675798,10915,832866,907990,476498,441577,283397,142219,691577,231182,37793,89030,296374,848043,227327,765929,849922,84975,850368,597589,887727,463724,347056,315425,835425,619853,25127\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 534\n", + "Returned IDs: 354656,737135,269375,525546,612297,166392,172701,243546,916915,850390,470685,19053,143668,341783,652336,331195,357908,615485,420052,271897,466875,9024,904507,801132,632708,579250,802512,371655,418993,529324\n", + "Ground Truth: 354656,737135,269375,525546,612297,949711,166392,172701,243546,823366,916915,850390,470685,19053,143668,341783,652336,209632,331195,357908,615485,420052,271897,466875,9024,26152,904507,801132,804808,632708\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 537\n", + "Returned IDs: 989555,125135,982943,146294,2774,793020,339516,88385,931626,452902,293082,689181,234369,632080,235267,888723,808519,258784,941924,328406,717859,261414,931425,301508,453216,408339,27634,780650,61435,927148\n", + "Ground Truth: 989555,125135,982943,146294,2774,793020,339516,88385,931626,452902,293082,689181,234369,632080,235267,888723,412471,808519,258784,941924,328406,717859,261414,931425,301508,453216,408339,27634,780650,61435\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 539\n", + "Returned IDs: 862039,27634,678390,810657,931425,435078,310435,822661,304643,645359,764656,343268,670106,482272,144977,803864,548164,876126,476932,208350,74771,632080,109720,291665,226364,688864,853903,347279,887959,990435\n", + "Ground Truth: 862039,27634,678390,810657,931425,435078,310435,822661,304643,645359,764656,343268,670106,482272,144977,803864,548164,876126,476932,208350,74771,632080,109720,291665,226364,688864,931369,787980,853903,347279\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 541\n", + "Returned IDs: 837204,253845,704903,994452,109502,754215,719438,696450,316624,904507,519773,631399,327573,464286,6488,544437,378429,350669,388670,453633,28623,689772,897071,806445,123467,906623,600302,823366,281286,275613\n", + "Ground Truth: 837204,253845,704903,994452,109502,754215,719438,696450,316624,904507,903465,360401,826042,519773,564509,631399,327573,464286,6488,544437,652670,378429,350669,388670,453633,372766,588577,28623,689772,897071\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 542\n", + "Returned IDs: 32666,595382,137310,347177,265408,375665,238846,714118,737031,230104,835244,555552,768388,83811,824289,447505,886180,712316,146431,917455,583350,886727,498815,745185,830523,592165,940478,359649,175218,600302\n", + "Ground Truth: 32666,595382,137310,347177,265408,375665,238846,714118,737031,230104,835244,555552,768388,83811,824289,447505,886180,712316,146431,917455,583350,259401,886727,498815,745185,19687,830523,826244,606478,592165\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 544\n", + "Returned IDs: 267323,661690,710995,514643,167622,819986,160315,976404,60063,828009,541811,544861,573801,313043,891795,740716,331816,729710,671964,166392,96536,937556,582636,542125,744618,217686,495722,631399,18529,451566\n", + "Ground Truth: 267323,661690,710995,514643,167622,819986,160315,976404,60063,828009,541811,544861,573801,313043,891795,740716,331816,729710,671964,166392,96536,937556,582636,542125,744618,217686,495722,902631,631399,18529\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 547\n", + "Returned IDs: 714118,712316,940505,502590,85649,23755,100503,447505,265408,232228,212576,772146,166392,312154,709777,826244,364869,632080,778134,463206,916915,79971,135017,768388,664911,630048,132941,581151,175218,523569\n", + "Ground Truth: 714118,712316,940505,502590,85649,23755,100503,447505,265408,232228,212576,772146,166392,312154,709777,826244,364869,632080,778134,463206,916915,79971,135017,768388,664911,645340,630048,132941,581151,175218\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 548\n", + "Returned IDs: \n", + "Ground Truth: 365410,813184,435736,264334,947950,492133,632814,720964,613133,691481,366682,966357,437403,906623,695965,175415,774724,151369,931780,867184,373940,340815,841327,279057,488821,612297,28151,759129,87686,435655\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 549\n", + "Returned IDs: 835244,15988,830523,99907,243912,975590,286702,447505,620407,542755,339271,946518,293049,415235,469341,897535,849922,696808,684956,521810,337126,265408,267310,584897,583350,479774,563923,768388,173073,454243\n", + "Ground Truth: 835244,15988,830523,99907,243912,975590,286702,447505,620407,542755,339271,946518,293049,415235,469341,897535,849922,696808,684956,521810,337126,265408,267310,374483,584897,5064,583350,479774,563923,768388\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 550\n", + "Returned IDs: 393202,802512,405252,998445,579250,379767,576985,729998,612297,554056,544861,533939,670380,775428,742370,916915,445370,371655,259018,772035,357908,514080,24196,333703,286033,112463,469341,225501,739725,14717\n", + "Ground Truth: 393202,802512,405252,998445,579250,379767,576985,729998,612297,554056,544861,533939,670380,775428,742370,916915,445370,371655,259018,772035,357908,514080,24196,333703,286033,112463,796968,376992,469341,225501\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 551\n", + "Returned IDs: \n", + "Ground Truth: 209632,916915,929277,575389,505357,486639,739555,547598,6211,492133,656276,980266,443127,768511,982164,975240,672661,350257,656220,725978,6338,944204,695965,18830,112256,150633,804808,360986,554056,615339\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 552\n", + "Returned IDs: 482020,498458,361557,135017,732233,363784,990458,212060,571874,623711,419480,236116,92914,589060,287704,541811,299027,548164,65773,426893,667761,84975,996028,354435,87686,849922,514643,312098,237148,712636\n", + "Ground Truth: 482020,498458,361557,135017,732233,363784,990458,212060,571874,623711,419480,236116,92914,589060,287704,541811,299027,548164,65773,426893,131243,667761,84975,996028,354435,87686,849922,514643,312098,237148\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 556\n", + "Returned IDs: 97135,447505,582477,924661,2907,92915,55426,642021,585438,745484,811799,454216,316175,932030,888980,325330,40240,902631,600302,529848,687287,11461,740178,441928,529957,544861,482380,610987,347323,567510\n", + "Ground Truth: 97135,447505,582477,924661,2907,169048,92915,55426,642021,585438,745484,811799,454216,316175,932030,888980,325330,40240,902631,600302,529848,687287,11461,740178,441928,529957,544861,482380,610987,347323\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 557\n", + "Returned IDs: 445843,259018,69280,67033,379767,579250,316940,670380,710026,632708,279054,542064,772511,632640,418369,759354,124402,971928,534803,166392,272209,333703,488003,464286,163554,507158,64898,94264,879890,413276\n", + "Ground Truth: 445843,259018,69280,67033,379767,579250,316940,670380,710026,632708,279054,542064,772511,632640,418369,759354,124402,659407,971928,534803,67332,166392,272209,333703,488003,464286,37639,163554,507158,64898\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 560\n", + "Returned IDs: 257445,745484,619853,363525,614507,975504,583564,662929,762471,318050,834937,742370,687366,710026,770486,829977,88319,398976,357434,82995,925504,862264,265653,196154,11461,479774,391478,212457,522760,13631\n", + "Ground Truth: 257445,745484,619853,363525,614507,975504,583564,662929,762471,318050,834937,742370,687366,710026,770486,296505,829977,88319,398976,357434,82995,925504,862264,265653,196154,11461,479774,391478,212457,522760\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 561\n", + "Returned IDs: 331808,584518,243077,765887,326505,587307,904005,73777,686173,5544,476673,723702,438048,403345,595886,968242,799050,839268,886180,552650,448587,99498,408339,5819,79136,683413,881397,781947,173576,643793\n", + "Ground Truth: 331808,584518,243077,765887,326505,587307,904005,73777,686173,5544,476673,723702,438048,403345,595886,968242,799050,839268,886180,552650,448587,99498,408339,5819,79136,683413,881397,781947,173576,177415\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 563\n", + "Returned IDs: 982675,600302,26572,463070,22998,342174,230204,631399,812194,30110,18529,402790,173422,927595,808579,357966,965362,397196,544437,883744,434018,272533,130172,320419,315776,608314,181709,205310,162050,785025\n", + "Ground Truth: 982675,600302,26572,463070,22998,342174,230204,631399,812194,30110,18529,402790,173422,927595,808579,405219,357966,965362,397196,544437,883744,434018,272533,130172,320419,300183,315776,351974,608314,270774\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 564\n", + "Returned IDs: 380506,866797,20900,297601,407004,994275,358859,479197,664447,943572,509748,232377,116625,99377,56109,238816,893919,443374,899121,299027,925919,66984,76518,300026,826126,32875,676582,467212,145707,660886\n", + "Ground Truth: 380506,866797,20900,297601,407004,994275,358859,479197,664447,943572,509748,232377,116625,99377,56109,238816,893919,263901,443374,899121,299027,925919,66984,76518,300026,826126,32875,676582,467212,145707\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 568\n", + "Returned IDs: 2774,463697,504540,235267,226364,234369,88385,18687,632080,604670,71204,981291,49919,982943,874072,702553,534299,258784,692131,643077,826126,146294,453216,612717,275992,989555,18529,357010,931626,865184\n", + "Ground Truth: 971846,2774,463697,504540,235267,226364,234369,88385,18687,632080,604670,71204,981291,49919,982943,874072,702553,534299,258784,692131,643077,826126,146294,453216,612717,275992,989555,18529,357010,931626\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 571\n", + "Returned IDs: 989555,2774,927148,235267,506149,452902,258784,125135,632080,914120,350669,717859,930237,830239,88385,239821,649772,293082,826126,870105,357010,833929,982943,881364,689181,900301,981291,941924,537644,287729\n", + "Ground Truth: 989555,2774,927148,235267,506149,452902,258784,125135,632080,914120,350669,717859,930237,830239,88385,239821,649772,293082,826126,870105,357010,833929,982943,881364,842911,336780,689181,900301,981291,941924\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 572\n", + "Returned IDs: 492260,469341,984170,378429,64898,245332,996028,474041,447505,358859,597551,772183,479197,664447,708492,783900,925919,632814,600302,445370,763544,67332,893919,236959,637811,601998,135017,597589,577876,755577\n", + "Ground Truth: 492260,469341,984170,378429,64898,245332,996028,474041,447505,358859,597551,772183,479197,664447,708492,783900,925919,632814,600302,445370,763544,67332,893919,236959,637811,819736,601998,135017,597589,577876\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 574\n", + "Returned IDs: 463070,181709,927595,544437,342174,173422,282337,453789,451846,883744,775428,901831,117859,802512,169118,434018,402790,672023,441323,456243,533939,494823,397196,608314,825606,805131,26572,469341,364628,371557\n", + "Ground Truth: 463070,181709,927595,544437,342174,173422,282337,453789,451846,883744,775428,901831,159239,117859,802512,169118,434018,402790,672023,441323,456243,405219,533939,494823,397196,608314,825606,805131,26572,469341\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 575\n", + "Returned IDs: 851293,705065,468677,85649,175218,740178,334437,851968,861934,428011,32213,605682,581151,64898,32805,687287,772146,824499,603588,166392,392918,820124,48844,354173,378429,643793,773822,714118,899227,797438\n", + "Ground Truth: 851293,705065,468677,85649,175218,740178,334437,851968,861934,428011,32213,605682,581151,64898,32805,687287,772146,824499,603588,166392,392918,820124,48844,354173,378429,643793,773822,494155,714118,857420\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 577\n", + "Returned IDs: 198289,475187,218542,611032,521706,581151,420265,895648,667974,474180,940505,6211,303917,166392,879890,891494,738229,798058,12436,233361,491812,670380,209887,510662,641969,47062,745185,840484,195445,962777\n", + "Ground Truth: 198289,475187,218542,611032,521706,581151,420265,895648,667974,474180,940505,6211,303917,166392,547189,879890,891494,738229,798058,12436,233361,491812,670380,209887,510662,641969,47062,745185,840484,195445\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 578\n", + "Returned IDs: 424560,245559,581151,885971,245642,712316,612846,757816,153485,286585,181413,769082,773822,664911,717859,357628,12057,940505,740178,491812,576985,979209,207041,180244,9927,23755,232228,314296,83341,405009\n", + "Ground Truth: 424560,245559,581151,885971,245642,712316,612846,757816,153485,286585,181413,769082,773822,664911,717859,383541,357628,12057,940505,740178,491812,576985,979209,207041,180244,87204,9927,23755,232228,314296\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 579\n", + "Returned IDs: 632080,24692,830239,331608,490286,818896,191784,272209,900301,664447,149032,477417,598044,447505,463206,41301,797438,469341,965362,534044,643793,695553,23755,699749,811799,234369,702096,12057,819024,652752\n", + "Ground Truth: 632080,24692,830239,331608,490286,818896,191784,272209,900301,664447,149032,477417,598044,447505,789801,463206,41301,797438,469341,965362,534044,643793,695553,23755,699749,811799,234369,702096,12057,819024\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 580\n", + "Returned IDs: 402371,955794,786055,360690,363784,13747,638804,725006,770124,818625,18939,541811,276335,7554,824160,600302,183967,243080,698771,72631,343268,84169,167544,415235,783367,897295,285553,488434,412873,873157\n", + "Ground Truth: 402371,955794,786055,360690,363784,13747,638804,725006,770124,818625,18939,541811,276335,7554,824160,600302,183967,243080,698771,72631,343268,84169,167544,389357,415235,783367,897295,285553,488434,412873\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 581\n", + "Returned IDs: 227209,151962,701447,848757,97135,687287,13631,745484,5675,703283,530372,641329,447505,902631,643793,522760,183691,813664,710026,837895,544861,175218,643363,297165,350580,364869,772146,418519,18529,670380\n", + "Ground Truth: 227209,151962,701447,848757,97135,687287,13631,745484,5675,703283,530372,641329,447505,902631,643793,658210,522760,183691,813664,710026,837895,544861,175218,643363,297165,350580,364869,772146,418519,18529\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 583\n", + "Returned IDs: 857738,498312,5808,776644,708492,598144,483807,449334,162277,217118,602272,596480,827552,121404,601987,797438,637017,81538,507209,613133,303085,3175,18529,548164,624016,622694,664447,82336,634000,746633\n", + "Ground Truth: 857738,498312,5808,776644,708492,598144,483807,449334,162277,217118,602272,596480,827552,121404,601987,797438,624862,637017,81538,507209,613133,303085,3175,18529,548164,624016,622694,664447,82336,634000\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 584\n", + "Returned IDs: 708631,841823,16656,212280,561835,548164,760970,828589,443831,664447,18529,470415,502001,142868,996028,827552,451708,413275,846025,797438,794669,152860,151369,179676,891795,813691,24784,897240,131161,67272\n", + "Ground Truth: 708631,841823,16656,212280,561835,548164,760970,828589,443831,664447,18529,470415,502001,142868,996028,827552,451708,413275,846025,797438,794669,152860,151369,179676,891795,289778,813691,24784,897240,131161\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 585\n", + "Returned IDs: 850390,916915,364319,612297,371655,579250,393202,841556,32213,5675,695965,978956,554056,402047,804808,6153,242065,652336,220971,632814,878069,445370,945780,971257,18529,712316,755666,857861,492133,541811\n", + "Ground Truth: 850390,916915,364319,612297,371655,468935,294158,579250,393202,85649,841556,32213,5675,695965,978956,554056,97858,402047,804808,6153,242065,652336,220971,632814,878069,445370,945780,971257,18529,712316\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 586\n", + "Returned IDs: 554056,445843,918389,427455,404975,371655,473819,333703,976915,632708,968887,507123,579250,259018,916915,357908,64273,299412,908541,33402,941314,791228,940691,660351,949366,492133,465498,384822,699487,587475\n", + "Ground Truth: 554056,445843,918389,427455,404975,371655,473819,333703,976915,632708,968887,507123,579250,259018,916915,357908,64273,299412,908541,33402,941314,791228,850030,940691,660351,949366,492133,465498,384822,699487\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 588\n", + "Returned IDs: 479197,474041,848232,236959,664447,785700,88046,141367,601998,708492,826126,67332,656749,676582,699047,299027,492260,469341,534044,183854,462345,893919,88999,64898,570591,637811,331608,408339,784568,395038\n", + "Ground Truth: 479197,474041,848232,236959,664447,785700,88046,141367,601998,708492,826126,67332,656749,676582,699047,299027,492260,469341,534044,183854,462345,893919,763544,88999,64898,570591,637811,331608,408339,784568\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 589\n", + "Returned IDs: 113908,815877,804808,667220,963711,977282,764694,103210,516493,409094,254476,854801,26892,492260,135729,632814,477417,796968,343268,657493,897988,365410,571942,221834,975798,541980,626727,304643,891903,69609\n", + "Ground Truth: 113908,815877,804808,667220,963711,977282,764694,103210,516493,409094,254476,854801,26892,492260,135729,122120,632814,477417,796968,343268,512544,657493,897988,365410,571942,929731,221834,413275,975798,973196\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 593\n", + "Returned IDs: 901831,934315,835425,879890,441607,113086,337596,65773,938786,20416,738229,124345,741427,989157,618559,576985,886727,706695,940505,166392,490901,436016,94582,900187,684037,729163,797438,379767,780978,927564\n", + "Ground Truth: 901831,934315,835425,879890,441607,113086,337596,65773,938786,20416,738229,124345,741427,989157,289778,618559,576985,886727,706695,940505,166392,490901,436016,94582,927595,687287,903465,900187,684037,729163\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 594\n", + "Returned IDs: \n", + "Ground Truth: 797382,475232,84975,830616,580578,542755,682356,300415,644852,94599,889753,453446,379478,245332,571874,459310,536815,385517,26867,415017,798019,896406,269352,747373,694327,416802,677600,153485,699092,893494\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 595\n", + "Returned IDs: 685917,546584,526107,435655,461880,876494,347031,128484,501165,727021,631127,342624,672661,343498,982375,102754,237494,290395,606870,567342,54764,594577,371698,916915,778472,650588,647270,775128,444157,3397\n", + "Ground Truth: 685917,546584,526107,60553,366682,435655,461880,876494,347031,128484,501165,727021,631127,342624,672661,751782,343498,982375,102754,237494,290395,606870,567342,391061,54764,594577,371698,916915,778472,650588\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 596\n", + "Returned IDs: 185931,916915,984432,938397,135278,3386,692188,500463,725978,770379,488003,734166,350257,612297,844571,371655,695965,850275,804808,554056,259018,431046,445370,498312,124402,793037,414215,882806,235467,331195\n", + "Ground Truth: 185931,916915,984432,938397,135278,3386,692188,500463,725978,770379,488003,734166,350257,612297,844571,371655,695965,850275,804808,554056,259018,431046,445370,498312,793037,124402,414215,541759,874438,882806\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 597\n", + "Returned IDs: 379767,137780,152655,592165,61856,94582,982771,185593,749804,148827,37639,600530,452997,683022,114932,507158,736060,631399,166392,886727,85649,26867,541759,342926,17986,134420,230650,693483,67033,588446\n", + "Ground Truth: 379767,137780,152655,592165,61856,94582,982771,185593,749804,148827,37639,600530,452997,683022,114932,507158,79971,736060,631399,166392,886727,85649,26867,541759,342926,392918,17986,134420,230650,693483\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 599\n", + "Returned IDs: 577426,444450,119066,887727,601998,142868,598144,548164,495221,236959,319630,491594,299027,893919,567033,238846,783367,334918,313140,946747,230358,275549,582616,860044,868934,608285,630195,664447,691997,502001\n", + "Ground Truth: 577426,444450,119066,887727,601998,142868,598144,548164,495221,236959,319630,491594,299027,893919,567033,238846,783367,334918,313140,946747,230358,892989,275549,582616,860044,868934,608285,630195,664447,691997\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 600\n", + "Returned IDs: 817938,712316,79971,384635,686429,23755,284845,606966,12057,429080,297415,932033,565531,537644,418997,314296,657098,502367,462345,16878,29774,319829,216428,882914,164021,355840,379644,485011,773822,816481\n", + "Ground Truth: 817938,712316,79971,384635,686429,23755,284845,606966,12057,429080,297415,932033,565531,537644,418997,314296,657098,502367,462345,16878,29774,319829,216428,882914,164021,355840,379644,485011,773822,87204\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 601\n", + "Returned IDs: 947990,902521,368888,828009,65773,541811,871262,58820,663219,784568,984648,965362,550249,623711,344253,581331,167634,403345,117859,135017,798019,997452,325191,504324,818903,632080,353622,83341,432017,46160\n", + "Ground Truth: 947990,902521,368888,828009,65773,185718,541811,871262,58820,663219,784568,984648,965362,550249,623711,344253,581331,167634,403345,117859,135017,798019,997452,325191,504324,818903,632080,353622,83341,432017\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 603\n", + "Returned IDs: 113908,610503,664447,708492,49328,996834,87686,504007,982164,826126,893919,76518,498312,612297,32875,751839,350669,797438,479197,3175,632080,900301,626727,927346,984194,740836,632814,622694,978956,905941\n", + "Ground Truth: 113908,610503,664447,708492,49328,996834,87686,504007,982164,826126,893919,76518,498312,612297,32875,751839,350669,797438,479197,3175,996943,632080,900301,626727,927346,984194,116312,740836,632814,622694\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 604\n", + "Returned IDs: 541428,101653,752362,366682,934210,479254,878069,450131,28346,475599,148986,590960,373940,771765,65550,492133,206153,355866,369233,882066,978594,698050,220971,702181,476798,94582,18830,225706,251296,332080\n", + "Ground Truth: 541428,101653,752362,366682,934210,479254,878069,450131,28346,995591,475599,148986,590960,976705,373940,65550,771765,492133,206153,497305,355866,369233,882066,978594,698050,563823,220971,702181,476798,94582\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 606\n", + "Returned IDs: 435655,286033,931780,702020,435000,569756,371655,382030,952197,631733,796968,445843,744351,112463,41301,952626,279057,141715,223552,347031,206153,567342,658210,237494,576985,343498,115722,787734,86869,552386\n", + "Ground Truth: 435655,286033,931780,302640,702020,435000,569756,371655,382030,952197,631733,796968,445843,744351,112463,41301,952626,279057,141715,223552,347031,206153,567342,658210,237494,576985,343498,115722,787734,86869\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 607\n", + "Returned IDs: 73673,98827,8938,434427,731172,847030,286569,612297,450445,676869,270634,269352,200560,990925,176751,288817,319630,301466,995884,228265,18830,90600,925370,166392,814001,633831,487383,614475,143668,863859\n", + "Ground Truth: 73673,98827,8938,575058,434427,731172,847030,286569,612297,450445,676869,270634,269352,200560,990925,176751,288817,319630,301466,995884,228265,18830,115832,90600,925370,166392,814001,633831,487383,614475\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 608\n", + "Returned IDs: 363784,310605,18529,541811,212060,226088,638004,92914,447505,135017,325874,263915,408339,390968,662180,732233,64898,600302,874218,50469,1628,361557,877724,557130,672435,180880,582636,548164,664447,946518\n", + "Ground Truth: 363784,310605,18529,541811,212060,226088,74067,638004,92914,447505,135017,325874,263915,408339,390968,662180,732233,64898,600302,874218,50469,1628,361557,877724,557130,672435,441168,180880,582636,419480\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 610\n", + "Returned IDs: 133235,925914,482019,908344,142868,865749,319630,166392,519773,963711,9679,224067,340429,796968,664447,85322,971100,743654,87686,545048,289778,871262,64753,69609,445370,94582,447505,658397,635589,622977\n", + "Ground Truth: 133235,925914,482019,908344,142868,865749,319630,166392,519773,963711,9679,224067,340429,796968,664447,85322,299325,971100,743654,87686,545048,289778,871262,64753,69609,445370,94582,447505,658397,635589\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 613\n", + "Returned IDs: 643082,94582,379767,983345,877652,61856,931780,576985,490901,932030,493831,917370,405252,588446,706695,937862,357908,700846,297165,906702,712387,225760,916915,24196,545048,450140,216367,849922,14717,19091\n", + "Ground Truth: 643082,94582,379767,983345,877652,61856,931780,576985,490901,932030,493831,917370,405252,588446,706695,15045,937862,357908,692188,700846,297165,906702,712387,225760,916915,24196,545048,450140,216367,849922\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 614\n", + "Returned IDs: 855897,637049,248801,585246,998101,911538,478851,105354,409130,321443,971100,219321,612263,248035,241,206153,331075,288169,744351,892967,853066,890360,890509,408819,832778,115722,726445,226695,113366,372132\n", + "Ground Truth: 855897,637049,216480,248801,585246,998101,911538,478851,105354,409130,321443,971100,219321,612263,248035,241,206153,331075,288169,744351,892967,853066,890360,890509,408819,832778,115722,726445,226695,113366\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 616\n", + "Returned IDs: 355729,949053,74771,501099,351974,962546,142273,34721,183883,364965,408339,815388,304643,802254,552650,742504,12960,311202,400368,293149,900704,404618,431100,476932,592380,243982,534157,947121,654515,342926\n", + "Ground Truth: 355729,949053,74771,501099,351974,962546,142273,34721,183883,364965,408339,815388,547104,304643,802254,552650,742504,12960,311202,647270,400368,293149,750188,900704,404618,431100,476932,837825,592380,243982\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 617\n", + "Returned IDs: 607269,580578,300415,451846,18529,624016,207488,911639,453995,126043,927668,848043,84975,830616,183691,211058,645316,470415,439147,30398,536815,622977,861381,94599,486447,282213,26867,523990,274473,644852\n", + "Ground Truth: 607269,580578,300415,451846,18529,624016,207488,911639,453995,126043,927668,848043,84975,830616,183691,211058,645316,470415,439147,30398,392918,536815,622977,861381,94599,486447,282213,26867,523990,274473\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 618\n", + "Returned IDs: 154699,571874,484250,360625,50901,476498,18914,85649,549805,27052,957471,664911,237224,790358,604939,571493,559932,740178,979209,730420,811835,149903,952810,7554,298911,818496,966580,468935,395466,885971\n", + "Ground Truth: 417599,154699,571874,484250,360625,50901,476498,18914,85649,549805,27052,957471,664911,237224,790358,604939,571493,559932,740178,979209,732233,730420,811835,149903,952810,7554,298911,818496,966580,468935\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 619\n", + "Returned IDs: 633498,711435,215465,899260,210179,975533,754070,332355,874072,336167,175988,998221,637561,635422,990435,119445,698775,80169,291163,992979,506149,462345,863464,235768,756342,868021,418997,606232,154098,44698\n", + "Ground Truth: 633498,711435,215465,899260,210179,975533,754070,332355,874072,336167,175988,998221,637561,635422,990435,119445,698775,80169,745185,291163,992979,506149,462345,863464,235768,756342,868021,418997,606232,154098\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 621\n", + "Returned IDs: \n", + "Ground Truth: 558539,397772,360299,194570,882066,525546,9024,305570,434066,584838,507123,417928,486739,333703,418321,355866,688472,526403,993326,407127,371655,558880,125536,327868,402998,6153,850837,77905,392373,503574\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 622\n", + "Returned IDs: 98529,308620,447505,477417,585438,13631,388778,811799,631399,643793,745484,641329,263260,671964,888980,191784,308417,726931,820124,210437,702096,357434,2971,576724,703283,848757,779902,837097,166392,701447\n", + "Ground Truth: 98529,308620,447505,477417,585438,13631,388778,811799,631399,643793,745484,641329,263260,671964,888980,191784,308417,726931,820124,210437,702096,357434,2971,576724,703283,848757,301135,779902,837097,166392\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 623\n", + "Returned IDs: 626132,886727,156608,787486,92861,712538,418519,180244,988325,531095,800801,874438,292970,19687,618559,750188,541759,259401,473460,342926,425985,415795,576985,668324,178358,856746,311016,768532,232795,687257\n", + "Ground Truth: 626132,886727,156608,787486,92861,712538,418519,180244,988325,531095,800801,874438,292970,19687,618559,750188,541759,259401,473460,342926,425985,415795,576985,668324,178358,856746,712316,311016,571572,768532\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 625\n", + "Returned IDs: 850368,135017,662929,177766,997618,357428,687287,88319,677600,11461,522760,363525,600302,976404,463724,141367,405219,427331,828137,468935,619853,221330,308620,772146,862264,834937,785700,280671,37793,706880\n", + "Ground Truth: 850368,135017,662929,177766,997618,357428,687287,88319,677600,11461,522760,363525,600302,976404,463724,141367,405219,427331,828137,468935,619853,221330,308620,772146,862264,834937,272151,785700,280671,37793\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 628\n", + "Returned IDs: \n", + "Ground Truth: 933655,815388,416518,812051,343268,571137,791872,886254,935459,155226,905169,579175,14205,605682,2136,212360,281286,72705,65388,851069,408339,88740,242688,851968,45275,887624,484250,636425,804038,615911\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 629\n", + "Returned IDs: 301466,529324,775370,572761,102754,492133,342834,479167,291163,528944,695965,307166,371655,373940,892053,525546,890360,529474,313302,544589,526107,941576,541428,350257,176514,897266,454064,773701,244717,393777\n", + "Ground Truth: 301466,529324,775370,572761,102754,492133,342834,761308,479167,291163,528944,695965,307166,371655,373940,514789,892053,525546,890360,529474,313302,544589,526107,941576,541428,350257,176514,897266,454064,773701\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 630\n", + "Returned IDs: 765458,126596,48754,950869,540931,452134,357403,784961,423033,455501,79055,230470,961747,431659,12387,904507,465924,613045,908538,259401,618257,58844,531095,636312,91007,40163,86869,129270,234842,14717\n", + "Ground Truth: 765458,126596,48754,950869,540931,452134,357403,784961,423033,455501,79055,230470,961747,626132,431659,12387,904507,465924,613045,758960,908538,259401,618257,58844,531095,636312,91007,40163,86869,129270\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 632\n", + "Returned IDs: 837946,662929,221330,166392,41301,282975,850020,615485,464286,363525,955379,987690,477305,88319,96536,135017,828137,216928,687366,522760,762693,447505,405219,808509,627144,911031,119066,800968,833986,680128\n", + "Ground Truth: 837946,662929,221330,166392,41301,282975,850020,615485,464286,363525,955379,987690,281792,477305,88319,96536,135017,828137,216928,687366,522760,762693,447505,405219,808509,627144,169048,911031,119066,800968\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 633\n", + "Returned IDs: 680706,142868,491594,299027,510910,358859,612297,94582,544861,775428,388670,166392,906623,447505,570591,378429,270250,541980,643793,861541,600302,354173,664447,502001,85322,652155,320419,801132,519773,893919\n", + "Ground Truth: 680706,142868,491594,299027,510910,358859,612297,94582,373923,544861,775428,388670,166392,906623,447505,570591,378429,270250,541980,643793,861541,600302,354173,664447,322603,502001,826126,85322,652155,320419\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 634\n", + "Returned IDs: 225501,332080,206694,916915,3563,371655,14050,373723,18830,497305,770379,151336,771765,850030,726307,452034,552386,744848,944204,69280,880836,333703,148986,882806,994164,124402,849922,772035,554056,14717\n", + "Ground Truth: 225501,332080,206694,916915,3563,371655,14050,373723,18830,497305,770379,151336,232002,771765,850030,726307,452034,552386,744848,944204,69280,880836,333703,916247,148986,882806,994164,124402,558539,849922\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 636\n", + "Returned IDs: 102754,151388,435655,356119,631733,226192,816457,124371,229125,66603,916915,993279,922032,105768,812245,223552,832405,502919,634500,525546,960673,128484,226695,445843,911538,73673,761308,337185,98827,608244\n", + "Ground Truth: 102754,151388,435655,356119,631733,226192,816457,124371,229125,66603,916915,993279,922032,105768,812245,223552,467686,832405,502919,634500,525546,426741,103104,740082,960673,307166,128484,226695,445843,911538\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 637\n", + "Returned IDs: 679202,596858,249837,43097,489145,312103,834993,410071,129313,1175,870304,258100,11868,531095,229596,779461,618559,631068,894834,186666,195445,476932,551028,349750,623692,201245,479630,9912,623687,938312\n", + "Ground Truth: 679202,596858,249837,43097,489145,312103,834993,410071,129313,1175,870304,258100,11868,531095,229596,779461,618559,631068,894834,186666,195445,476932,551028,349750,623692,142142,201245,479630,9912,623687\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 639\n", + "Returned IDs: 36535,633831,9024,579250,185933,654082,306280,662701,63145,333703,626382,445298,850030,393202,615339,992310,35727,225418,421017,814886,398327,24851,229543,288817,335911,434197,91728,744848,485035,337185\n", + "Ground Truth: 36535,633831,9024,579250,185933,654082,306280,662701,63145,333703,626382,445298,850030,393202,615339,992310,35727,54091,225418,421017,814886,398327,24851,229543,288817,335911,751782,434197,91728,744848\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 640\n", + "Returned IDs: 961536,232674,679498,600302,628830,71455,393110,745484,551160,68834,151127,951427,900187,477417,695625,541759,668324,946518,386589,408339,783900,100503,813664,245332,687257,819432,229723,323977,354173,64898\n", + "Ground Truth: 961536,232674,679498,600302,628830,71455,393110,745484,551160,68834,151127,951427,900187,477417,695625,541759,252995,153836,668324,135017,740178,946518,386589,408339,783900,100503,502001,813664,245332,687257\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 641\n", + "Returned IDs: 683736,623711,83341,443374,65773,32213,441323,343268,445370,3648,354435,894962,773822,491545,555552,135729,212360,109191,908344,758267,797438,145306,422940,265408,818625,541811,845747,161960,333068,516184\n", + "Ground Truth: 683736,623711,16319,83341,443374,65773,32213,441323,343268,445370,3648,354435,894962,773822,491545,555552,135729,212360,109191,908344,758267,797438,145306,422940,265408,818625,541811,845747,161960,333068\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 643\n", + "Returned IDs: 531771,753101,736060,213374,454216,592165,534803,353484,654515,26867,418369,300728,528574,835244,204311,715216,447505,769905,293049,77750,873157,976682,907990,294724,534553,897535,329191,705667,544861,244891\n", + "Ground Truth: 531771,753101,736060,213374,454216,592165,534803,353484,654515,26867,418369,300728,528574,835244,204311,522317,715216,447505,769905,293049,77750,873157,267043,765929,976682,907990,294724,534553,897535,329191\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 644\n", + "Returned IDs: 639913,778930,446450,316940,608244,350743,103442,941314,299412,597161,523190,349747,69280,94264,632708,97858,587475,67033,42037,918389,477305,554056,950257,561483,759354,265421,124402,75841,389798,371655\n", + "Ground Truth: 639913,778930,446450,316940,608244,350743,103442,941314,299412,597161,523190,349747,69280,127351,94264,632708,97858,587475,67033,42037,918389,477305,554056,950257,561483,759354,265421,124402,75841,389798\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 646\n", + "Returned IDs: 50901,298911,278741,606324,735252,740178,25567,543420,952810,166298,71855,220536,784785,791651,212993,984783,793973,241091,263248,77497,411599,237224,77201,232228,452995,23116,546849,89042,265408,200327\n", + "Ground Truth: 50901,298911,278741,606324,735252,740178,25567,543420,952810,166298,71855,220536,784785,791651,212993,417599,984783,793973,241091,263248,77497,411599,237224,77201,232228,452995,23116,546849,89042,265408\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 647\n", + "Returned IDs: 615485,632708,94264,602442,166392,375190,600203,640584,587475,269375,287544,849922,972926,754621,84233,656028,831415,259018,978689,745484,405252,22440,445121,541980,290966,525546,34523,65199,67033,139298\n", + "Ground Truth: 615485,632708,94264,602442,166392,375190,600203,640584,587475,269375,287544,900272,849922,972926,754621,84233,656028,831415,259018,978689,745484,405252,22440,445121,541980,290966,958244,525546,576724,34523\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 648\n", + "Returned IDs: 558153,498312,227382,18529,209361,176958,899260,772361,745185,714118,309663,636425,643793,148565,712316,51876,23755,627243,581151,195445,507158,695130,85649,245559,797438,241011,449345,64898,34721,194080\n", + "Ground Truth: 558153,498312,227382,18529,209361,176958,899260,772361,745185,714118,309663,643793,636425,148565,712316,172701,51876,23755,627243,581151,195445,507158,695130,85649,245559,797438,241011,449345,64898,34721\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 650\n", + "Returned IDs: 910064,100503,214921,948300,596858,263842,811153,971100,466784,393110,228850,809535,938584,599943,904930,835425,857887,613468,562648,621301,8238,541759,575922,796629,71455,623711,679202,628830,653858,493831\n", + "Ground Truth: 910064,100503,214921,948300,596858,805604,263842,811153,195445,971100,466784,393110,228850,809535,938584,599943,904930,835425,857887,613468,562648,621301,8238,541759,575922,796629,71455,623711,679202,628830\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 652\n", + "Returned IDs: 378362,29560,672661,579250,333703,93883,73156,941314,203353,124402,632708,472121,214269,575058,631399,916915,830735,587475,299325,309885,554056,597161,355866,975967,362244,259018,371655,850030,172629,117859\n", + "Ground Truth: 378362,94866,29560,672661,579250,333703,93883,73156,941314,203353,124402,632708,472121,214269,575058,631399,916915,830735,587475,299325,309885,554056,597161,355866,975967,362244,259018,371655,25421,850030\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 654\n", + "Returned IDs: 853894,544679,21239,167515,895648,67033,974941,710026,982993,695965,265421,211285,407127,370134,6153,808593,344295,688125,41301,135278,124402,206153,777809,180244,473819,916915,6211,33942,891494,908724\n", + "Ground Truth: 853894,544679,21239,167515,895648,67033,974941,710026,982993,695965,265421,211285,407127,370134,715023,6153,808593,344295,688125,41301,135278,124402,206153,777809,180244,473819,916915,6211,33942,891494\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 655\n", + "Returned IDs: 513181,421446,610987,936249,738229,211528,477417,765929,679498,835244,961536,266075,643793,215750,166392,120482,745484,754621,879890,180244,923610,745522,256391,67033,474782,773831,702096,454216,783900,386589\n", + "Ground Truth: 513181,421446,610987,936249,738229,211528,516493,477417,765929,679498,835244,961536,266075,315231,643793,215750,166392,120482,745484,754621,879890,180244,923610,745522,256391,67033,474782,773831,702096,454216\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 658\n", + "Returned IDs: 533939,272687,463070,445370,802512,976404,901557,135821,320419,194718,834937,775428,677600,451846,544437,826126,632080,169118,927595,18529,269352,371557,463724,173422,565909,490901,779055,628922,376992,773964\n", + "Ground Truth: 533939,272687,463070,445370,802512,976404,901557,135821,320419,194718,834937,775428,234369,677600,451846,954884,544437,826126,632080,169118,927595,18529,269352,371557,463724,173422,310435,565909,490901,779055\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 660\n", + "Returned IDs: 443374,607059,951923,212360,83341,811772,554970,412471,967189,756923,834643,251650,647935,317944,862705,142142,623711,983714,18651,925919,468872,32213,773822,516184,602549,872829,706510,819736,745185,347041\n", + "Ground Truth: 443374,607059,951923,212360,83341,811772,554970,506149,412471,967189,756923,834643,251650,647935,317944,862705,142142,623711,983714,18651,925919,468872,773822,32213,516184,602549,872829,706510,819736,745185\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 661\n", + "Returned IDs: 757816,659249,316537,336548,465859,103210,185593,372222,408339,718291,643363,761593,521706,791872,153485,424560,886727,9182,166392,37639,152655,612593,325191,815388,592681,979427,117355,313571,332907,530340\n", + "Ground Truth: 757816,659249,316537,336548,465859,103210,185593,372222,408339,718291,643363,761593,521706,791872,153485,424560,108643,886727,9182,166392,37639,152655,992979,612593,325191,815388,592681,979427,117355,313571\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 662\n", + "Returned IDs: 549448,174609,875436,900187,765043,818220,847720,903520,596858,151127,511931,547104,628830,779461,488434,618559,651376,342926,531095,156608,626132,870304,736737,772316,874438,934315,948300,350,379901,259401\n", + "Ground Truth: 549448,174609,875436,900187,765043,818220,847720,903520,596858,151127,511931,547104,628830,779461,488434,618559,651376,342926,531095,156608,626132,870304,736737,772316,874438,934315,948300,350,379901,79103\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 663\n", + "Returned IDs: 824043,65388,777646,265408,90449,319689,670852,730420,797438,750654,791872,88916,294152,149903,619050,212993,50901,343268,97402,85649,278741,790358,426893,90462,298911,571874,484250,237224,664457,605682\n", + "Ground Truth: 824043,65388,777646,265408,90449,319689,670852,730420,797438,750654,791872,88916,294152,149903,619050,212993,50901,343268,97402,85649,278741,63990,790358,426893,90462,298911,240090,571874,484250,237224\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 664\n", + "Returned IDs: 729998,337185,317773,378362,554056,381462,393202,723550,720943,93883,355866,9024,228302,998445,398870,120980,421017,697754,150640,612297,971257,541811,360299,887742,344295,335911,373940,306280,454064,63145\n", + "Ground Truth: 729998,337185,317773,378362,554056,381462,393202,723550,720943,93883,355866,9024,228302,998445,398870,120980,421017,697754,150640,612297,971257,541811,24851,360299,887742,344295,335911,373940,306280,454064\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 665\n", + "Returned IDs: 816023,837339,972926,214269,129088,473819,357908,740082,123566,745484,672661,853066,701006,988352,715023,576985,850390,519324,668240,7203,772035,565055,701447,737135,316751,69280,950893,650787,6338,120980\n", + "Ground Truth: 816023,837339,972926,214269,129088,473819,357908,976507,740082,123566,745484,672661,853066,701006,988352,715023,576985,850390,519324,668240,7203,407127,772035,565055,701447,737135,316751,69280,950893,650787\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 666\n", + "Returned IDs: 488219,675467,224503,259720,691762,345023,830239,5525,612429,708680,848708,900301,643363,958864,353622,623702,712316,331608,634279,794260,794380,802512,735531,166392,30077,207041,851546,742354,411988,664911\n", + "Ground Truth: 488219,675467,224503,259720,691762,345023,830239,5525,612429,708680,848708,900301,15628,643363,958864,353622,623702,712316,331608,634279,794260,794380,802512,735531,166392,30077,207041,851546,742354,411988\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 667\n", + "Returned IDs: 364967,656132,596181,517755,262546,8238,775491,322825,735531,780978,10915,678126,588446,81619,406519,259033,170464,802512,910768,404155,376992,79971,897535,788045,158639,629171,952711,463724,679041,448520\n", + "Ground Truth: 364967,656132,596181,517755,262546,8238,775491,322825,735531,780978,10915,678126,588446,81619,406519,259033,170464,802512,910768,404155,376992,79971,897535,788045,158639,629171,997698,952711,463724,679041\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 668\n", + "Returned IDs: 829365,516115,339050,85649,622920,18914,596949,154699,774748,147614,799050,972599,449911,427961,740178,287125,885971,93680,156624,297165,484099,777369,845079,468101,605682,166298,498815,265408,7554,825171\n", + "Ground Truth: 829365,516115,339050,85649,417599,622920,18914,596949,154699,774748,147614,799050,972599,449911,28015,427961,740178,287125,885971,93680,156624,297165,484099,777369,845079,468101,605682,947895,166298,220615\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 670\n", + "Returned IDs: 736060,884481,534803,81619,753101,628922,775428,802512,19687,613195,142087,735531,292970,213374,746155,65773,712316,8238,715216,544437,982670,180244,679041,834937,408339,592165,463070,506091,239981,26867\n", + "Ground Truth: 736060,884481,534803,81619,753101,628922,775428,802512,19687,613195,142087,735531,292970,213374,746155,65773,712316,8238,715216,544437,982670,877304,180244,679041,834937,408339,592165,463070,506091,239981\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 673\n", + "Returned IDs: 283101,622292,607912,961310,287866,27081,787972,606232,705667,312098,487634,820983,404112,408339,166392,23761,548164,218889,363784,408318,953810,773964,474764,790358,87686,388326,223741,541904,78598,691762\n", + "Ground Truth: 283101,622292,607912,961310,287866,27081,787972,606232,705667,312098,487634,820983,404112,408339,166392,2093,23761,548164,218889,363784,408318,261613,953810,773964,474764,790358,87686,388326,223741,541904\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 674\n", + "Returned IDs: 275613,109226,568301,477417,701447,529996,679498,569985,506534,902631,300183,600302,980242,643793,754429,811799,850390,10133,887626,264793,726931,629891,631399,274030,331816,696033,677600,917445,668324,877893\n", + "Ground Truth: 275613,109226,568301,477417,701447,529996,679498,569985,506534,902631,300183,600302,980242,643793,754429,811799,286436,850390,10133,887626,264793,726931,971100,629891,850030,631399,274030,331816,696033,677600\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 675\n", + "Returned IDs: 729710,670380,134420,90217,467212,691771,984170,149903,632080,358859,712316,426893,463724,507158,85649,530372,447505,773964,940505,18529,544861,652155,314296,735531,691762,834937,820124,263915,5525,166392\n", + "Ground Truth: 729710,670380,134420,90217,467212,691771,984170,149903,632080,358859,712316,426893,463724,507158,85649,530372,447505,773964,938312,940505,213575,18529,544861,652155,314296,735531,691762,834937,820124,263915\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 676\n", + "Returned IDs: 507158,141715,435655,92861,663199,112463,407127,357908,772035,972594,402790,544861,85322,706695,259018,80107,576985,744848,880836,564509,395038,664624,115855,482176,445843,371655,409731,904507,867184,155550\n", + "Ground Truth: 507158,141715,435655,92861,663199,112463,407127,357908,772035,972594,402790,544861,85322,706695,259018,80107,576985,744848,880836,564509,395038,664624,115855,482176,466875,445843,371655,409731,904507,867184\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 677\n", + "Returned IDs: 582616,9024,169807,63145,102243,612117,288817,418915,203703,849778,792691,271050,399872,376815,955841,335201,203351,381462,337185,511850,226695,695965,374244,481233,805604,402047,569477,396777,797895,645340\n", + "Ground Truth: 582616,9024,169807,63145,102243,612117,288817,418915,203703,849778,792691,271050,399872,376815,955841,335201,203351,381462,337185,511850,226695,956689,695965,374244,481233,805604,402047,569477,396777,797895\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 678\n", + "Returned IDs: 371594,912678,29560,371655,554056,225501,579250,329489,878069,672661,454064,83341,297282,497305,238440,974941,311197,971257,552386,463070,722188,656276,362244,300183,978956,612297,64898,316887,445370,333703\n", + "Ground Truth: 371594,912678,29560,371655,554056,225501,579250,329489,878069,672661,454064,83341,297282,497305,238440,177766,974941,311197,97135,971257,552386,463070,722188,739507,656276,362244,300183,978956,612297,64898\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 679\n", + "Returned IDs: 9927,470164,848907,830349,40011,825157,334437,596480,521706,510662,992128,175218,581151,426423,507209,714118,474911,532064,813236,740178,832465,232228,813664,956107,26089,840484,643793,847404,649566,965292\n", + "Ground Truth: 9927,470164,848907,830349,40011,825157,334437,596480,521706,510662,992128,175218,581151,426423,507209,714118,474911,532064,813236,740178,832465,232228,813664,956107,642021,283960,26089,840484,643793,847404\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 680\n", + "Returned IDs: 745185,309663,804808,926270,632814,445370,104733,492133,148565,245559,891494,379767,431046,857861,87686,612297,166392,417872,564509,643363,706695,87859,795002,570591,899944,893429,849922,232002,632080,897266\n", + "Ground Truth: 745185,309663,804808,926270,632814,445370,104733,90600,492133,148565,245559,819736,891494,379767,431046,354173,857861,87686,612297,166392,417872,564509,903465,643363,706695,87859,795002,126043,885971,570591\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 682\n", + "Returned IDs: 691762,117859,588446,379767,940505,36724,600530,434246,541811,18529,94582,195445,715216,134420,379478,544861,61856,952711,432017,982943,788045,706695,949077,675117,37639,665325,581559,732515,393944,376992\n", + "Ground Truth: 691762,117859,588446,379767,940505,36724,600530,434246,541811,18529,94582,195445,715216,134420,379478,544861,61856,952711,432017,982943,522317,788045,706695,949077,675117,37639,665325,581559,732515,393944\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 683\n", + "Returned IDs: 882699,946518,732120,924273,579195,84975,643363,793267,428228,941506,239718,830270,343268,749769,264793,476498,4816,89030,173073,271090,87065,63587,232478,163135,241838,83986,693483,434,64898,165843\n", + "Ground Truth: 882699,946518,732120,924273,579195,84975,643363,793267,428228,941506,239718,830270,343268,749769,264793,476498,4816,89030,173073,271090,267516,87065,63587,232478,163135,241838,83986,693483,434,64898\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 686\n", + "Returned IDs: 626382,280478,695965,344295,622961,844571,500463,225501,916915,185931,18830,978956,437403,772035,206694,832882,729325,214108,519324,6153,411988,972228,686408,375948,73355,498925,660351,3386,6338,544589\n", + "Ground Truth: 626382,280478,695965,344295,622961,844571,500463,225501,916915,185931,18830,978956,437403,772035,206694,832882,729325,214108,519324,6153,411988,972228,514080,686408,375948,73355,498925,660351,597589,3386\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 687\n", + "Returned IDs: 402427,920761,761170,772035,324863,60713,131053,837825,328230,121904,342174,552536,139298,729998,333990,585177,686408,321810,68367,959469,907602,232588,451846,710026,418993,88319,774367,264045,311016,41301\n", + "Ground Truth: 402427,920761,761170,772035,324863,60713,131053,837825,328230,121904,342174,552536,139298,729998,333990,585177,686408,321810,68367,959469,907602,232588,451846,710026,599813,447287,327683,418993,88319,774367\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 688\n", + "Returned IDs: 600302,677600,309728,374176,619853,345046,141367,318050,464286,94894,785700,274156,685395,946518,834937,544861,779309,648483,583564,831934,521810,813664,783367,405219,976404,459789,158223,729738,983345,657512\n", + "Ground Truth: 600302,677600,309728,374176,619853,345046,141367,318050,464286,94894,785700,274156,685395,946518,834937,544861,779309,648483,583564,831934,469428,521810,813664,783367,405219,976404,992979,459789,158223,729738\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 691\n", + "Returned IDs: 880573,339187,144213,211058,55022,451846,514235,680064,311604,631399,637905,249700,976239,126043,169118,45059,779055,476498,292065,313747,73368,61845,915297,354173,407911,937789,600302,133770,463070,187105\n", + "Ground Truth: 880573,339187,144213,211058,55022,451846,514235,680064,311604,631399,753044,637905,249700,976239,126043,169118,779055,45059,476498,292065,313747,73368,548164,61845,915297,354173,407911,937789,600302,133770\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 692\n", + "Returned IDs: 596858,811153,531095,545096,780978,393110,626132,12572,934315,249837,207488,361945,305016,466784,912073,955878,9912,5675,679202,704746,938312,337458,575922,564835,206564,796629,631068,180244,10915,422454\n", + "Ground Truth: 596858,811153,531095,545096,780978,393110,626132,12572,934315,249837,207488,361945,305016,466784,912073,955878,9912,5675,679202,704746,938312,337458,575922,564835,602549,206564,796629,495445,631068,180244\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 693\n", + "Returned IDs: 855897,126043,971100,333765,476798,202676,871262,115855,744351,33972,996693,151173,426496,274727,85322,417948,878843,417195,206153,982375,931681,86869,589292,167748,115722,433497,743654,673485,775449,48754\n", + "Ground Truth: 855897,126043,971100,333765,476798,202676,871262,115855,744351,33972,996693,151173,426496,274727,85322,417948,878843,417195,206153,982375,931681,86869,379901,589292,232674,167748,115722,433497,743654,673485\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 694\n", + "Returned IDs: \n", + "Ground Truth: 775948,600302,883700,886932,315425,750654,415235,952288,347056,360551,541811,976404,463724,602272,408318,600041,58714,907254,916247,672968,180480,832866,887727,503974,13964,318050,369753,637465,934693,212457\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 697\n", + "Returned IDs: 632708,417872,587475,67033,979332,238650,221330,379767,6211,507158,69280,124402,21239,491660,777636,405252,392449,955379,443127,759354,968295,632640,710026,327683,242065,445843,10387,627144,930465,166392\n", + "Ground Truth: 632708,417872,587475,48754,67033,979332,238650,221330,379767,6211,507158,69280,124402,21239,491660,777636,405252,392449,955379,443127,759354,968295,632640,710026,327683,242065,445843,10387,627144,930465\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 698\n", + "Returned IDs: \n", + "Ground Truth: 449334,62325,465859,187105,967574,275549,266466,44594,357314,311604,509338,82336,465190,693174,691997,488384,104952,760970,145306,548164,797438,146561,541811,454701,4439,319630,476498,563956,640363,783367\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 699\n", + "Returned IDs: 655583,975590,10915,379478,800767,484099,479774,399095,644318,176615,166392,4493,638004,63990,463206,854042,837640,643363,543216,283397,212043,813664,4047,19091,135017,7554,242938,637465,660251,878762\n", + "Ground Truth: 655583,975590,10915,379478,800767,484099,479774,399095,644318,176615,166392,4493,638004,63990,463206,854042,837640,643363,543216,283397,992979,212043,813664,4047,19091,135017,7554,242938,637465,660251\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 700\n", + "Returned IDs: 967574,412873,311604,104373,638804,18914,476498,887727,824160,45059,83986,927595,736060,873157,232478,275549,857420,818625,729163,266466,295742,592165,871011,378429,25127,897295,193914,369753,170451,204253\n", + "Ground Truth: 967574,412873,311604,104373,638804,18914,476498,887727,824160,45059,83986,927595,736060,873157,232478,275549,857420,818625,729163,266466,295742,592165,816270,871011,378429,25127,897295,288468,193914,369753\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 704\n", + "Returned IDs: 225501,357908,371655,554056,507123,253712,507158,14717,519324,587959,996053,916915,511850,607269,316940,968295,445843,333703,379767,182722,850030,670380,311197,94866,417928,497305,69280,443127,229125,417384\n", + "Ground Truth: 225501,357908,371655,554056,507123,253712,507158,14717,519324,587959,996053,916915,511850,607269,740716,316940,968295,445843,333703,379767,182722,850030,670380,311197,94866,417928,497305,69280,443127,229125\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 705\n", + "Returned IDs: 907990,946518,574945,286702,696363,264793,40022,293049,394343,764977,15988,770401,664911,300415,125577,729163,243912,339271,33361,521810,472963,643363,318050,987690,686805,812771,603588,563923,830616,64898\n", + "Ground Truth: 907990,946518,574945,286702,696363,264793,40022,293049,394343,764977,15988,770401,664911,300415,125577,729163,243912,339271,33361,521810,472963,643363,441607,318050,987690,686805,812771,603588,563923,830616\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 706\n", + "Returned IDs: 938335,899944,855553,496966,632640,273467,598885,351661,591286,9024,363784,612117,258356,195623,210958,645340,390979,337185,879890,759354,541811,5655,445924,67033,126840,751087,393202,379767,305038,6211\n", + "Ground Truth: 938335,899944,855553,496966,632640,273467,598885,351661,591286,9024,363784,612117,258356,195623,210958,645340,390979,337185,836668,879890,759354,541811,5655,445924,67033,126840,751087,640111,178358,393202\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 708\n", + "Returned IDs: 510662,26089,631399,683736,848907,632814,612297,804808,897071,833949,581151,460479,361964,469428,910768,259401,963711,97858,84233,926270,19215,331195,417872,903465,455789,600302,65199,173422,759354,720964\n", + "Ground Truth: 510662,26089,631399,683736,848907,632814,612297,804808,897071,833949,581151,460479,361964,469428,910768,963711,259401,97858,84233,926270,19215,331195,417872,903465,455789,90600,600302,65199,173422,759354\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 710\n", + "Returned IDs: 664447,87686,366682,166392,341783,115722,342624,893919,96536,735994,843989,407004,572761,541428,18529,770906,30077,307804,488003,826126,601998,331195,447505,632080,185309,24642,708492,379767,784568,695965\n", + "Ground Truth: 664447,87686,366682,166392,341783,115722,342624,893919,96536,735994,843989,407004,572761,541428,18529,770906,30077,307804,488003,826126,601998,331195,447505,632080,82993,903465,185309,24642,708492,379767\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 712\n", + "Returned IDs: 296478,332080,276127,892665,887742,203353,794932,333703,24196,63145,740082,6153,882806,69280,988176,528809,435655,916915,648511,878069,728954,670380,652336,379767,373723,124402,362244,290270,519324,129088\n", + "Ground Truth: 296478,332080,276127,892665,887742,203353,794932,333703,24196,63145,740082,6153,882806,992979,69280,988176,528809,435655,916915,497305,648511,284708,878069,728954,670380,652336,379767,373723,124402,362244\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 713\n", + "Returned IDs: 677600,463724,166392,735531,931780,331816,804808,616251,522760,579250,18529,645299,135017,785700,920983,631399,834937,379767,544861,800649,729738,464286,976404,41301,687287,73368,167622,762693,691762,507158\n", + "Ground Truth: 677600,463724,166392,735531,931780,331816,804808,616251,522760,579250,18529,645299,135017,142868,796968,785700,920983,631399,834937,379767,544861,800649,729738,464286,976404,41301,687287,73368,167622,762693\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 714\n", + "Returned IDs: 784785,571493,839568,460479,181413,183246,712924,443302,77201,163106,848907,232228,339050,591171,952810,993115,195445,825157,984783,753890,191619,671593,581151,50901,452995,47062,254515,40011,83172,979209\n", + "Ground Truth: 784785,571493,839568,460479,181413,183246,712924,443302,77201,163106,848907,232228,339050,591171,6153,952810,804808,993115,195445,825157,984783,753890,191619,671593,581151,50901,452995,47062,683177,254515\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 717\n", + "Returned IDs: 772035,333990,88319,931681,978689,774367,813664,837825,173422,469341,280671,247420,728241,918746,883358,608314,236271,311387,99039,451846,495221,615485,357268,680128,585177,421446,972926,834937,927595,342926\n", + "Ground Truth: 772035,333990,88319,931681,978689,774367,813664,837825,173422,469341,280671,247420,483079,728241,918746,883358,608314,236271,976507,311387,99039,451846,495221,615485,357268,680128,585177,421446,972926,834937\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 720\n", + "Returned IDs: 445370,577876,430778,242065,341783,87686,319630,165843,893919,5031,612297,664447,135017,18529,422940,142868,513245,493428,32213,899944,854483,849922,597589,683736,492260,502001,245332,138015,734241,269352\n", + "Ground Truth: 445370,577876,430778,242065,341783,87686,319630,165843,893919,206394,5031,612297,664447,135017,18529,422940,142868,513245,493428,32213,899944,105966,854483,849922,597589,683736,492260,502001,245332,138015\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 723\n", + "Returned IDs: 294724,635025,911639,480695,623711,831132,418369,297824,22636,235555,291665,62121,353167,142142,443374,872829,413766,828589,413276,951923,898071,37639,920852,588446,747460,797438,765561,613195,328908,845747\n", + "Ground Truth: 294724,635025,911639,480695,623711,831132,418369,297824,22636,235555,291665,62121,353167,142142,522317,443374,872829,311016,413766,828589,413276,951923,898071,37639,816270,920852,588446,747460,797438,765561\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 724\n", + "Returned IDs: 26892,32213,977282,261595,441323,422940,443374,997698,65773,3648,783619,854483,194080,238846,826126,378429,550249,831132,773822,516184,103210,785700,408339,430778,652670,701821,819736,965362,412471,852381\n", + "Ground Truth: 26892,32213,977282,261595,441323,422940,443374,997698,65773,3648,783619,854483,194080,238846,826126,378429,550249,831132,773822,516184,103210,785700,408339,430778,652670,701821,462345,965362,819736,412471\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 725\n", + "Returned IDs: 588012,557219,891795,404618,217686,155201,640740,636262,300999,24784,582636,363784,515535,267323,542125,187105,291934,476498,360625,789998,553763,711663,758144,729163,310605,133770,60063,653378,211556,661690\n", + "Ground Truth: 588012,557219,891795,404618,217686,155201,640740,636262,300999,24784,582636,363784,515535,267323,542125,187105,291934,476498,838315,360625,789998,553763,711663,758144,729163,310605,133770,60063,653378,211556\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 726\n", + "Returned IDs: 841229,882066,867184,812245,431942,735994,775449,697522,205211,997496,541428,775128,919726,438095,373940,396777,283989,151308,789826,943933,286033,971732,102754,24642,343498,60143,33942,774724,892053,710060\n", + "Ground Truth: 841229,882066,867184,812245,431942,575058,735994,775449,697522,205211,997496,541428,775128,919726,438095,373940,396777,283989,151308,789826,943933,286033,971732,102754,24642,343498,563778,60143,33942,774724\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 728\n", + "Returned IDs: 253578,99285,135017,619853,677600,643793,88319,703091,479774,510927,944060,680128,849922,363525,357268,583564,862264,139298,416802,109226,847918,245559,993783,335201,336977,632708,405009,400032,37793,18529\n", + "Ground Truth: 253578,99285,135017,619853,677600,643793,88319,703091,479774,510927,944060,680128,849922,363525,357268,583564,862264,139298,631733,416802,109226,847918,245559,993783,335201,336977,632708,405009,400032,37793\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 730\n", + "Returned IDs: 181290,815388,632080,187060,353622,448601,313476,867914,237537,594344,761593,387460,38596,245178,772146,664911,604597,932033,408339,603588,903148,175218,254370,184382,236589,637450,631399,643793,532718,517191\n", + "Ground Truth: 181290,815388,632080,187060,353622,448601,313476,867914,237537,594344,761593,387460,38596,245178,772146,664911,604597,932033,408339,603588,903148,175218,254370,184382,236589,637450,791872,631399,848708,643793\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 731\n", + "Returned IDs: 371655,333703,904507,259018,379767,405252,69280,916915,357908,507158,445843,14717,299325,576985,579250,124402,828137,710026,407127,525546,242065,670380,311197,63145,97858,206694,652336,88319,445370,357403\n", + "Ground Truth: 371655,333703,904507,259018,379767,405252,69280,916915,357908,507158,445843,14717,299325,902031,576985,579250,643667,124402,828137,710026,407127,525546,242065,670380,311197,63145,97858,206694,652336,88319\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 733\n", + "Returned IDs: 5819,813664,632585,766432,581331,783900,543216,253637,246885,313013,889753,854483,642295,772146,738229,245332,940505,85649,955838,872921,539486,295553,491545,61856,884104,403345,23934,902631,957713,465145\n", + "Ground Truth: 5819,813664,632585,766432,581331,783900,543216,253637,246885,313013,889753,854483,642295,772146,738229,245332,940505,85649,142142,955838,872921,15628,539486,998221,295553,565531,473460,491545,61856,884104\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 734\n", + "Returned IDs: 677600,82995,345046,672348,386589,777758,308620,657512,722764,290966,482380,152112,282531,372766,69207,881687,368101,374176,363525,583564,671964,584897,777881,489753,357434,975504,849922,656276,92915,923610\n", + "Ground Truth: 677600,82995,345046,672348,386589,777758,308620,657512,722764,290966,482380,152112,282531,372766,69207,881687,368101,374176,363525,881648,583564,671964,584897,777881,489753,357434,975504,849922,656276,923610\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 735\n", + "Returned IDs: 541759,874438,834937,750188,772035,701447,619853,178358,626132,40940,632814,576985,687287,342926,245559,785700,547104,706880,464286,259401,180244,628830,283218,350580,544861,100503,670380,853066,579250,466784\n", + "Ground Truth: 541759,874438,834937,750188,772035,336780,701447,619853,178358,626132,40940,632814,576985,687287,342926,245559,785700,547104,706880,464286,259401,180244,628830,283218,903465,350580,544861,100503,670380,853066\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 737\n", + "Returned IDs: 392788,949711,917766,401070,327683,139298,858945,976404,729998,293149,419447,245559,865749,958244,409130,328230,126043,658397,214017,142868,253776,138910,885272,674336,475148,315231,687287,780978,115855,514080\n", + "Ground Truth: 392788,949711,917766,401070,327683,139298,959469,858945,976404,729998,293149,419447,245559,865749,958244,409130,328230,126043,658397,214017,142868,253776,138910,885272,674336,475148,315231,687287,780978,115855\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 738\n", + "Returned IDs: \n", + "Ground Truth: 885555,887624,254476,142868,596683,712364,232478,672650,904447,491594,343268,548164,159633,275549,871011,691997,76350,640363,319630,857420,640740,697555,638804,476498,24625,94582,638004,673519,252874,412873\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 739\n", + "Returned IDs: 120980,350257,837410,431420,761313,725978,974849,608314,656276,282531,227025,450472,510927,21239,538890,93883,409731,431046,770379,309885,294599,634029,124402,242065,506450,656989,632708,869710,984655,248449\n", + "Ground Truth: 120980,350257,837410,431420,761313,725978,974849,608314,656276,282531,227025,450472,809373,510927,21239,538890,93883,409731,881149,431046,770379,309885,294599,634029,124402,242065,506450,218270,656989,632708\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 740\n", + "Returned IDs: 205023,839858,828009,368888,65773,509748,281286,151369,80169,413275,350669,623711,706695,142273,254384,664447,185718,997452,6488,672614,865901,407004,955142,443374,740082,801311,64898,617507,155572,567342\n", + "Ground Truth: 205023,839858,828009,368888,65773,509748,281286,151369,80169,413275,350669,623711,706695,992979,142273,254384,664447,185718,997452,6488,84043,988352,672614,865901,24196,407004,955142,443374,915169,389333\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 741\n", + "Returned IDs: 600302,619853,463724,662929,862264,834937,677600,687366,533939,64898,522760,363525,221330,585438,447505,829977,828137,614507,280671,135017,66624,757816,927297,364869,564509,94894,687287,318050,88319,583564\n", + "Ground Truth: 600302,619853,463724,662929,862264,834937,677600,687366,533939,64898,522760,363525,221330,585438,447505,829977,828137,614507,280671,135017,66624,757816,927297,364869,949777,564509,94894,687287,318050,88319\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 744\n", + "Returned IDs: \n", + "Ground Truth: 287544,916915,978956,525546,450445,197146,172701,602442,529324,335201,492133,69280,549024,102243,393202,612297,38444,343842,214032,73355,6338,855908,18830,253712,897071,104733,680706,364886,579578,288817\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 745\n", + "Returned IDs: 734241,523569,971928,695965,832882,959763,225501,91663,575799,466877,608244,265421,619981,195375,379767,632640,317494,863812,670380,598764,167515,902137,384822,67033,855553,451244,214269,690733,759354,158345\n", + "Ground Truth: 881149,734241,523569,971928,695965,832882,959763,225501,91663,575799,466877,608244,265421,619981,195375,379767,632640,317494,194329,863812,670380,598764,167515,902137,384822,67033,855553,451244,214269,690733\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 747\n", + "Returned IDs: 543216,479774,783900,377842,781200,856746,166392,286585,421446,331195,884104,313013,378598,245332,33923,936249,211528,889753,111684,452997,992979,940505,30077,469341,447505,659407,677600,64898,375513,652336\n", + "Ground Truth: 543216,479774,783900,377842,781200,856746,166392,286585,421446,331195,884104,313013,378598,245332,33923,936249,244662,211528,889753,111684,452997,992979,940505,30077,469341,447505,659407,677600,389798,628830\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 748\n", + "Returned IDs: 602557,402987,742370,919506,695553,311016,392788,763544,139298,115855,463206,469341,912561,572761,789801,598044,976507,813664,272209,802512,23934,409130,972926,33361,619294,363525,387460,558726,376992,674336\n", + "Ground Truth: 602557,402987,742370,919506,695553,311016,392788,29964,763544,139298,115855,463206,469341,912561,572761,789801,598044,976507,155972,813664,818220,272209,802512,23934,409130,972926,33361,619294,363525,387460\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 749\n", + "Returned IDs: 507158,637811,631399,445370,998221,141367,477417,744397,576985,785700,990435,632814,664447,643793,557954,797438,300183,447505,564509,512544,94599,343268,984170,508644,585438,18529,902631,242434,194080,529778\n", + "Ground Truth: 507158,637811,631399,445370,998221,141367,477417,744397,576985,785700,990435,632814,664447,643793,557954,797438,300183,447505,564509,512544,94599,343268,984170,508644,585438,18529,902631,242434,993783,194080\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 751\n", + "Returned IDs: 759354,632640,258356,24196,269835,41266,561483,598764,413918,360986,612297,64898,94963,421218,772511,98827,757816,680438,911260,67033,855553,568927,126840,69280,445924,804586,916915,401384,579250,990925\n", + "Ground Truth: 759354,632640,258356,24196,269835,41266,561483,598764,413918,360986,612297,64898,94963,421218,772511,98827,757816,680438,911260,67033,855553,568927,126840,69280,445924,804586,712316,916915,401384,579250\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 752\n", + "Returned IDs: 593355,238440,67272,600302,813039,980270,353100,64898,448729,52713,84043,581180,757816,42809,846063,770401,946518,793267,230204,732120,742654,274156,142868,447505,463724,582477,677600,956252,563923,405219\n", + "Ground Truth: 593355,238440,67272,600302,813039,980270,353100,64898,448729,52713,84043,581180,757816,42809,846063,770401,946518,793267,230204,732120,742654,274156,142868,447505,463724,582477,153836,677600,956252,563923\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 754\n", + "Returned IDs: 735994,18529,445370,927595,745484,510927,371655,523990,291163,476798,687287,785700,632814,242065,986603,534553,576985,686408,772035,126043,585438,643793,139298,12387,236271,528944,631399,300183,41301,834937\n", + "Ground Truth: 735994,18529,445370,927595,745484,510927,371655,523990,291163,476798,687287,785700,632814,242065,986603,534553,576985,686408,772035,126043,585438,643793,139298,12387,236271,528944,992520,631399,300183,41301\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 755\n", + "Returned IDs: 210167,64273,371655,311197,631399,507158,597161,918389,92861,632708,344295,554056,602442,333703,830735,69280,350580,677600,10387,904507,445843,214269,336977,309885,447287,14717,834937,643793,772035,259018\n", + "Ground Truth: 210167,64273,371655,311197,631399,507158,597161,918389,92861,632708,344295,554056,602442,333703,830735,69280,350580,677600,10387,904507,445843,214269,336977,407127,309885,447287,14717,635965,834937,643793\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 756\n", + "Returned IDs: 805947,293676,958864,142273,496838,408339,778590,940443,514540,338793,692131,522421,297203,364667,952937,761593,632080,982943,826126,75502,881397,371584,966232,813715,314296,493704,259720,358621,795343,60700\n", + "Ground Truth: 805947,293676,958864,142273,496838,408339,778590,940443,514540,338793,692131,522421,297203,364667,614606,952937,761593,632080,982943,826126,75502,881397,371584,966232,813715,314296,493704,259720,358621,795343\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 758\n", + "Returned IDs: 445370,84233,378429,320419,541980,469341,12291,73368,350086,600302,597589,700519,263915,458408,85649,568276,640397,364886,48844,27052,492260,634029,142868,313747,714322,245332,24110,50469,177766,813664\n", + "Ground Truth: 445370,84233,378429,320419,541980,469341,12291,73368,350086,600302,597589,700519,263915,458408,85649,568276,640397,408962,364886,48844,27052,32805,492260,634029,142868,313747,714322,245332,24110,50469\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 759\n", + "Returned IDs: 748194,468935,811835,778134,14079,613195,921652,777646,530372,790358,581151,330823,772146,319689,260443,183691,872921,95826,348940,820124,523911,88916,666870,778612,85649,11461,263260,389553,935154,740178\n", + "Ground Truth: 748194,468935,811835,778134,14079,613195,921652,777646,530372,790358,581151,330823,772146,319689,260443,230280,183691,872921,95826,348940,820124,523911,88916,666870,778612,85649,11461,263260,389553,935154\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 760\n", + "Returned IDs: 982565,934315,940384,357388,453995,948300,19687,305016,393110,901831,460806,587977,764838,835425,552835,626132,541759,74360,900187,390336,262546,596858,814221,489145,441607,441577,379767,564835,151127,43097\n", + "Ground Truth: 982565,934315,940384,357388,453995,948300,19687,305016,393110,901831,460806,587977,764838,835425,552835,626132,541759,74360,511931,900187,390336,262546,596858,814221,489145,441607,441577,379767,613195,564835\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 761\n", + "Returned IDs: 632080,815388,868021,364667,408339,769082,881397,83341,664911,445370,12387,18529,486447,853903,712316,353622,613195,37639,623702,368888,514540,311068,539150,932033,851546,803864,548164,587658,581331,709777\n", + "Ground Truth: 632080,815388,868021,364667,408339,441577,769082,881397,83341,664911,445370,12387,18529,669722,103210,486447,853903,712316,353622,613195,37639,623702,368888,514540,311068,539150,932033,851546,803864,548164\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 763\n", + "Returned IDs: 463874,843427,357403,360005,204311,723800,581559,379767,302843,408339,514903,706695,24196,729163,531771,544861,805036,576985,940443,390336,142273,457556,929864,325191,720758,689742,712387,67901,418369,252874\n", + "Ground Truth: 463874,843427,357403,360005,204311,723800,581559,632560,379767,302843,408339,706695,514903,24196,729163,531771,544861,805036,576985,940443,390336,142273,457556,929864,325191,720758,689742,712387,67901,418369\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 767\n", + "Returned IDs: 582616,225390,9024,638708,396777,944994,203351,271050,848230,418915,337185,481233,203703,642949,362380,27919,683097,882066,792691,573380,878069,797895,955841,849778,916915,407577,808593,791291,330370,55307\n", + "Ground Truth: 582616,225390,9024,638708,396777,944994,203351,271050,848230,418915,978594,337185,481233,203703,642949,362380,27919,683097,882066,792691,573380,878069,797895,955841,849778,916915,407577,808593,791291,330370\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 770\n", + "Returned IDs: 904507,996053,638247,537553,240996,867184,264381,54091,479167,890360,488003,437964,124892,379767,244717,850275,294158,878069,331195,992979,805604,364319,80905,480071,652336,124402,584838,97858,605547,206153\n", + "Ground Truth: 904507,996053,638247,537553,240996,867184,264381,54091,479167,890360,488003,437964,124892,379767,244717,339223,850275,294158,878069,331195,992979,72035,805604,364319,80905,480071,652336,124402,739092,584838\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 771\n", + "Returned IDs: 420265,570591,482522,961677,380308,166392,429758,730418,100503,525665,411149,632080,527507,482569,87686,554865,778091,183028,344422,142868,643793,23755,294741,517191,845747,447505,174389,210748,783900,996834\n", + "Ground Truth: 420265,570591,482522,961677,380308,166392,429758,730418,100503,525665,411149,632080,527507,482569,87686,554865,778091,183028,344422,142868,154098,643793,23755,294741,517191,845747,447505,174389,210748,783900\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 772\n", + "Returned IDs: 879890,645340,84965,166392,233361,904507,912561,30526,390336,409130,670380,541811,849658,522760,712316,735531,830239,488003,364869,4493,324863,564509,331816,735520,579250,245559,421446,855553,801035,335201\n", + "Ground Truth: 879890,645340,84965,166392,233361,904507,912561,30526,390336,409130,670380,541811,849658,775491,522760,712316,735531,830239,488003,364869,4493,324863,564509,331816,735520,579250,245559,769082,421446,855553\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 775\n", + "Returned IDs: \n", + "Ground Truth: 568276,755601,704903,698771,72631,943038,638804,135017,631399,18830,873157,265421,122463,587405,980242,890531,45275,150633,802768,701488,519135,482380,512544,183967,638004,74067,343268,88740,447505,495244\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 776\n", + "Returned IDs: 720740,445924,740082,402790,724130,97858,69280,258433,999777,926388,773701,672023,940691,902031,572761,915301,169283,366272,607269,445843,397196,794932,227407,115722,128137,163516,173422,904507,405252,434018\n", + "Ground Truth: 720740,445924,740082,402790,724130,97858,409288,69280,258433,999777,926388,73081,773701,454216,672023,940691,152252,902031,572761,915301,169283,366272,607269,445843,397196,794932,700738,227407,209838,115722\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 779\n", + "Returned IDs: 627123,950925,632447,508644,130172,710026,791872,686408,811799,772035,692188,213575,782393,92915,634079,482380,615485,853508,950726,14061,633806,270774,694660,677600,530372,408962,88319,634029,97135,541811\n", + "Ground Truth: 627123,950925,632447,508644,130172,710026,791872,686408,811799,772035,692188,213575,782393,92915,634079,482380,615485,853508,510464,94970,950726,14061,633806,270774,677600,530372,694660,408962,88319,404975\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 781\n", + "Returned IDs: 304643,815388,343268,873157,408339,465859,174701,212360,254476,749769,634000,638804,728241,815877,876126,88740,636425,927595,355729,631399,520805,757816,855217,741137,966232,454519,797438,637050,930237,632814\n", + "Ground Truth: 304643,815388,343268,873157,408339,465859,174701,212360,254476,749769,634000,638804,728241,815877,876126,88740,636425,927595,355729,631399,520805,298323,757816,62820,855217,741137,966232,454519,797438,637050\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 783\n", + "Returned IDs: 467212,755577,341783,205204,956035,557561,417928,18529,548164,476683,358859,443374,230358,450445,471044,353167,7144,820124,313929,844571,602557,976239,135729,76673,66984,613209,448994,729710,502001,550249\n", + "Ground Truth: 467212,755577,341783,205204,956035,557561,417928,18529,548164,476683,358859,443374,230358,450445,471044,353167,7144,820124,313929,844571,602557,976239,135729,76673,66984,613209,448994,729710,502001,263901\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 787\n", + "Returned IDs: 598764,782439,33942,265421,465498,759354,632708,417872,973196,982993,97858,632814,811826,445370,687757,350669,602442,360986,492133,67033,379767,612297,683736,541428,492260,770379,561483,185931,445843,918389\n", + "Ground Truth: 598764,782439,33942,265421,249943,823244,465498,759354,632708,417872,973196,982993,97858,209632,632814,811826,445370,687757,421628,600302,350669,602442,360986,492133,67033,379767,612297,683736,541428,492260\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 788\n", + "Returned IDs: 350669,909390,212993,303352,483807,815877,447505,632814,990435,672614,465859,403345,826126,565810,122120,740178,735252,622977,744935,201294,313929,607375,492260,744631,469341,797438,940505,900153,22871,600302\n", + "Ground Truth: 350669,909390,212993,303352,483807,815877,447505,632814,990435,672614,465859,403345,752268,826126,135278,816205,565810,122120,740178,735252,622977,744935,201294,313929,854483,607375,492260,744631,469341,797438\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 789\n", + "Returned IDs: 151127,195888,203782,467724,89460,23788,774212,766432,695625,703312,454347,836659,443235,649355,618559,457706,473460,753101,925063,699616,479331,436016,502245,687287,377104,441607,770242,687257,12487,818287\n", + "Ground Truth: 151127,195888,203782,467724,89460,23788,774212,766432,695625,703312,454347,836659,443235,649355,618559,457706,473460,753101,925063,699616,479331,436016,502245,687287,377104,441607,770242,687257,689181,12487\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 790\n", + "Returned IDs: 728954,850030,396777,41301,552386,737135,576985,602442,479167,49838,14717,711982,544861,357908,379767,392618,413276,477417,882066,908541,94264,126043,794244,337185,916915,828137,729998,891494,124402,473819\n", + "Ground Truth: 728954,850030,396777,41301,552386,737135,282531,576985,602442,479167,49838,14717,711982,544861,357908,379767,392618,413276,477417,882066,908541,94264,126043,794244,337185,916915,828137,729998,891494,124402\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 791\n", + "Returned IDs: 858173,87497,966232,761593,408339,950043,363117,14452,639552,204311,661315,815388,797438,653378,539150,898071,204253,157745,243049,623711,56022,838377,804038,834643,498181,355729,636425,954276,283397,565750\n", + "Ground Truth: 858173,87497,966232,761593,408339,950043,363117,14452,639552,204311,661315,815388,425079,797438,653378,539150,898071,204253,795691,157745,243049,623711,56022,838377,804038,834643,355729,498181,954276,283397\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 793\n", + "Returned IDs: 64898,469341,447505,15988,563923,135017,829977,245332,712316,37639,543216,495221,283397,299027,849922,830239,84043,735531,837640,912561,389553,560935,284349,166392,236959,33361,582477,757816,378429,534553\n", + "Ground Truth: 64898,469341,447505,15988,563923,135017,829977,245332,712316,37639,543216,495221,12057,283397,299027,292970,849922,830239,84043,735531,837640,912561,560935,389553,284349,166392,236959,33361,15628,582477\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 794\n", + "Returned IDs: 671985,712316,982943,234369,146294,773822,369554,632080,990435,927148,506149,874072,275992,132941,856773,859885,235267,2774,408339,534299,79971,433808,845747,376992,932033,881397,445370,93981,12057,125135\n", + "Ground Truth: 671985,712316,982943,234369,146294,773822,369554,632080,826244,990435,927148,506149,874072,275992,132941,856773,859885,235267,2774,408339,534299,83341,79971,433808,845747,376992,932033,534553,881397,445370\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 795\n", + "Returned IDs: 635965,280671,881687,631399,336977,35070,687287,779309,428914,464743,940217,932030,575467,787734,983345,699616,205310,276798,544861,14717,777881,405219,94894,642665,357403,92915,379767,97858,980270,853066\n", + "Ground Truth: 635965,280671,881687,631399,336977,35070,687287,779309,428914,464743,940217,932030,575467,787734,983345,699616,233972,205310,276798,544861,14717,777881,405219,94894,642665,357403,92915,379767,97858,980270\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 798\n", + "Returned IDs: 408339,548164,581331,62121,966232,607912,862039,831132,204253,33043,77750,312098,443374,876126,381348,858173,352341,758267,283397,368888,295855,661315,715216,761593,87497,623711,815388,495445,103210,363117\n", + "Ground Truth: 408339,548164,581331,62121,966232,607912,862039,831132,204253,33043,795691,77750,312098,443374,876126,381348,858173,352341,758267,283397,368888,295855,406845,661315,715216,761593,87497,623711,815388,495445\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 799\n", + "Returned IDs: 447505,693483,555552,835244,873157,583350,638004,940478,83811,375665,265408,714118,712316,590195,88740,175218,768388,343268,753890,724881,32666,408339,146431,815388,858729,584897,563683,544437,183537,29472\n", + "Ground Truth: 447505,693483,555552,835244,873157,583350,638004,940478,83811,375665,265408,714118,712316,590195,88740,175218,198289,768388,343268,753890,724881,32666,408339,146431,815388,696808,858729,584897,563683,544437\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 800\n", + "Returned IDs: 445843,507158,41301,65199,94264,670380,627144,417872,663199,904507,67033,229125,124402,435655,279054,554056,710026,631399,939981,69280,85322,343498,918389,322603,309885,480557,282975,143117,94866,91663\n", + "Ground Truth: 445843,507158,41301,65199,94264,670380,627144,780421,417872,663199,904507,67033,229125,124402,435655,279054,554056,710026,631399,939981,728695,69280,85322,343498,918389,322603,309885,480557,282975,143117\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 802\n", + "Returned IDs: 18830,115135,575389,337185,232002,916915,492133,167515,373723,355866,299325,768511,298415,814886,486639,6338,662701,402047,752362,437989,309885,158345,340815,456861,6153,112256,454064,120980,491461,554056\n", + "Ground Truth: 18830,115135,575389,337185,232002,916915,492133,167515,373723,355866,299325,768511,298415,814886,486639,6338,662701,402047,25421,752362,437989,309885,158345,340815,324863,456861,6153,112256,454064,120980\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 803\n", + "Returned IDs: 290832,910869,961536,209666,267886,928664,178358,107476,251853,699616,180244,232674,951427,465924,595241,259401,389961,753675,807139,886431,679498,132947,750188,914996,626132,600606,429735,52769,541759,668324\n", + "Ground Truth: 290832,910869,961536,209666,267886,928664,178358,107476,251853,699616,180244,232674,951427,465924,595241,259401,389961,753675,807139,886431,679498,132947,750188,914996,626132,600606,429735,52769,541759,600302\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 805\n", + "Returned IDs: 67272,983345,379767,94582,516493,827552,142868,843427,631399,724742,166392,29560,577254,706695,613195,405219,548164,728241,664447,477417,576985,151127,74067,502001,285105,978533,343268,194080,608759,980270\n", + "Ground Truth: 67272,983345,379767,94582,516493,827552,142868,843427,631399,724742,166392,29560,577254,706695,613195,405219,548164,728241,664447,828009,957713,477417,576985,151127,74067,502001,285105,978533,343268,194080\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 806\n", + "Returned IDs: 932835,240090,447505,325330,468935,258109,889023,347323,868934,819347,894834,577220,724881,311068,959902,313043,971538,643363,989316,476932,347056,92915,415235,236959,451709,924661,746324,603588,888976,503974\n", + "Ground Truth: 932835,240090,698033,447505,325330,468935,258109,889023,347323,868934,819347,894834,577220,65388,724881,311068,959902,313043,971538,643363,989316,476932,347056,92915,415235,236959,451709,924661,746324,603588\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 807\n", + "Returned IDs: 320927,476743,413276,377161,435655,141715,88319,85943,166392,619853,286033,112463,237494,507158,466875,735531,829977,544861,871262,74171,342624,435061,476798,85322,381463,115855,454216,904507,155550,316327\n", + "Ground Truth: 320927,476743,413276,377161,435655,141715,88319,85943,227382,166392,619853,286033,513181,112463,237494,507158,466875,735531,829977,544861,871262,74171,342624,435061,476798,85322,381463,115855,454216,904507\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 809\n", + "Returned IDs: 643363,94582,379767,166392,612297,221330,842578,87686,24196,670380,488003,69280,804808,64898,706695,245332,507158,931780,904507,476743,61856,762693,664447,754108,358859,541428,331195,354173,626727,115722\n", + "Ground Truth: 643363,94582,379767,166392,612297,221330,842578,87686,24196,670380,488003,69280,804808,64898,706695,245332,507158,931780,436016,904507,476743,61856,762693,664447,754108,358859,541428,331195,354173,626727\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 811\n", + "Returned IDs: 178358,497421,728471,6514,699616,389961,738348,92861,180244,141715,979774,429735,259018,596759,144665,292970,847369,479819,541759,357403,375971,626132,731780,319339,989157,874438,342926,753101,104733,659758\n", + "Ground Truth: 178358,497421,728471,6514,699616,389961,738348,92861,180244,141715,979774,429735,259018,596759,144665,292970,847369,479819,541759,357403,375971,626132,731780,319339,989157,874438,342926,753101,293149,104733\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 813\n", + "Returned IDs: 198289,458935,233361,610912,895648,209887,925667,166392,813520,581636,684477,261509,752393,813817,808161,337458,621301,152655,456861,293254,777809,825802,612002,688125,992979,180244,390979,167515,104181,698668\n", + "Ground Truth: 198289,458935,233361,610912,895648,209887,925667,166392,813520,684477,581636,261509,752393,813817,808161,337458,621301,230650,152655,456861,293254,777809,825802,612002,688125,992979,180244,390979,167515,104181\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 815\n", + "Returned IDs: 828009,11606,155572,955142,912407,407004,58820,368888,664447,79136,90002,175621,443374,514540,893919,962546,900301,987421,441577,797438,534553,514080,65773,438048,30526,353622,403345,441323,966401,783900\n", + "Ground Truth: 828009,11606,155572,955142,912407,407004,58820,368888,664447,79136,90002,175621,443374,268509,514540,893919,962546,900301,987421,441577,797438,925919,534553,514080,299027,65773,438048,30526,353622,403345\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 816\n", + "Returned IDs: 966257,579578,396777,695965,638708,355866,867864,102243,340815,456861,612117,481233,112256,337185,558880,586441,305570,344295,362380,492133,18830,849778,805604,147428,339223,725978,288817,167515,916915,582616\n", + "Ground Truth: 966257,579578,396777,695965,638708,355866,867864,102243,340815,456861,612117,481233,112256,337185,558880,947151,586441,305570,344295,362380,945780,492133,18830,849778,805604,147428,339223,725978,288817,167515\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 817\n", + "Returned IDs: 619211,488219,634279,109135,175855,259720,5525,952937,643363,794380,408339,358621,385378,544861,331608,523569,441928,84473,18529,876110,687287,724881,610749,818900,447505,166392,34245,92915,826126,79971\n", + "Ground Truth: 619211,488219,634279,109135,175855,259720,5525,952937,643363,794380,408339,358621,385378,544861,331608,523569,441928,84473,18529,876110,687287,724881,610749,818900,447505,166392,34245,385517,92915,826126\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 818\n", + "Returned IDs: 309885,439573,968887,670380,227025,120980,24321,850030,14717,27280,69280,210958,392618,150640,373723,886952,379767,272256,355866,29560,507123,331816,456861,918389,214269,974941,427455,3563,40196,729062\n", + "Ground Truth: 309885,439573,968887,670380,227025,120980,24321,850030,14717,27280,69280,210958,392618,150640,373723,886952,410117,379767,272256,355866,29560,507123,331816,927063,456861,918389,214269,974941,521840,427455\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 819\n", + "Returned IDs: 109135,441928,92915,447505,331608,702096,523569,311068,258109,724881,584897,179536,745522,281320,79136,336620,403345,64898,932030,176059,463206,384909,500438,712316,153836,132941,311387,207041,468935,772146\n", + "Ground Truth: 109135,441928,92915,447505,331608,702096,523569,311068,258109,724881,584897,179536,745522,281320,79136,336620,403345,64898,932030,176059,463206,384909,533939,500438,712316,153836,132941,311387,207041,468935\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 820\n", + "Returned IDs: 63990,719438,854042,34721,504223,683736,154098,555590,578050,304643,931772,631399,786565,749769,729738,967574,783900,778091,570591,839268,504702,743915,176615,759211,491594,33923,141367,331195,399095,501526\n", + "Ground Truth: 63990,719438,854042,34721,504223,795971,683736,154098,555590,578050,304643,72058,931772,631399,786565,749769,729738,951645,967574,783900,778091,570591,839268,504702,743915,176615,759211,491594,33923,141367\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 821\n", + "Returned IDs: 27283,107750,286436,793267,355239,765404,173393,275549,141284,946518,180880,764977,261893,732120,691577,243912,773068,662929,301983,840519,40161,892989,857420,286702,775948,854904,259234,126973,685318,63587\n", + "Ground Truth: 27283,107750,286436,793267,355239,765404,173393,275549,141284,946518,217605,180880,764977,261893,732120,691577,243912,773068,662929,301983,840519,40161,892989,857420,286702,775948,854904,259234,126973,685318\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 823\n", + "Returned IDs: 20416,949777,829563,835425,19687,180244,626132,883700,422454,934315,596858,531095,706695,916247,963673,206564,308145,780978,948300,151127,67033,901831,886727,900187,909278,119066,704746,242065,576985,938786\n", + "Ground Truth: 20416,949777,829563,835425,19687,180244,626132,883700,422454,934315,596858,531095,706695,916247,963673,206564,677600,308145,780978,948300,151127,67033,901831,886727,900187,909278,119066,704746,242065,576985\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 824\n", + "Returned IDs: 406519,134867,775491,940505,65773,679041,170464,77412,735531,327683,245559,952711,419803,802512,36724,783900,81619,642295,19687,8238,954884,880249,835244,712316,416802,955379,845747,207041,865474,977041\n", + "Ground Truth: 406519,134867,775491,940505,65773,679041,170464,77412,735531,327683,245559,952711,419803,802512,36724,783900,81619,642295,19687,8238,954884,880249,835244,712316,416802,955379,845747,207041,491545,865474\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 825\n", + "Returned IDs: 857420,476498,84975,428592,871011,589282,275549,237345,357314,7554,770124,207488,187105,18914,488384,571874,854483,779874,418218,135017,343268,88740,691997,267534,385517,729163,574445,885971,453446,640363\n", + "Ground Truth: 857420,476498,84975,428592,871011,589282,275549,237345,357314,7554,770124,207488,187105,18914,488384,571874,854483,779874,418218,135017,343268,88740,267534,691997,385517,729163,997618,574445,885971,453446\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 826\n", + "Returned IDs: 182165,385733,736060,152655,37639,740178,259018,544861,379767,76095,20341,570591,600302,646934,166392,599683,677600,314296,582477,804808,447505,754108,877652,581151,212576,61856,319689,454216,909092,670380\n", + "Ground Truth: 182165,385733,736060,152655,37639,740178,558153,259018,544861,379767,76095,20341,570591,600302,646934,166392,599683,677600,314296,582477,804808,447505,754108,877652,581151,212576,61856,319689,454216,909092\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 827\n", + "Returned IDs: 115080,94582,462977,379767,450140,864975,550440,857887,270579,940443,646934,815877,963474,577120,343310,979427,103210,61856,390200,292970,218542,316537,801035,995342,814526,560329,37639,180244,576985,821368\n", + "Ground Truth: 115080,94582,462977,379767,450140,864975,550440,857887,270579,940443,646934,815877,963474,577120,343310,979427,103210,61856,390200,292970,758582,218542,316537,801035,995342,814526,560329,37639,180244,576985\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 828\n", + "Returned IDs: 89030,770672,640740,441323,706968,888976,857420,775948,502001,613195,67272,393775,232478,135017,343268,685561,261893,292605,362286,837640,325330,426893,643363,84975,937619,956986,638004,470233,535046,746324\n", + "Ground Truth: 89030,770672,640740,441323,706968,888976,857420,775948,502001,613195,67272,393775,232478,135017,343268,685561,261893,292605,910313,362286,837640,325330,426893,643363,84975,937619,956986,638004,470233,535046\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 829\n", + "Returned IDs: 382030,435655,454724,880098,115722,100188,122407,672661,670380,476798,206153,360986,745484,643363,355866,290270,583999,674336,528944,930905,710026,857783,97135,212360,971732,448125,41301,403331,631733,507158\n", + "Ground Truth: 382030,435655,454724,880098,115722,100188,122407,756965,672661,670380,284708,476798,97244,992979,206153,360986,745484,643363,355866,290270,583999,674336,450331,528944,930905,710026,686408,857783,97135,212360\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 832\n", + "Returned IDs: 523049,882066,5031,967574,18529,492133,140706,963711,104733,857861,73368,720964,6488,67332,628922,564509,445370,469428,632814,96536,417872,796968,787094,142868,27081,903465,701821,643363,365410,147165\n", + "Ground Truth: 523049,882066,5031,771909,967574,18529,492133,140706,963711,104733,857861,73368,720964,6488,67332,628922,564509,445370,469428,632814,96536,417872,796968,787094,142868,27081,903465,701821,973196,643363\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 833\n", + "Returned IDs: 387488,230650,493831,800285,771091,490901,49331,73777,283265,114839,857078,403345,669415,377842,610912,351364,700079,889367,349772,916915,794932,323734,392518,488399,958864,108363,925667,796968,618727,955379\n", + "Ground Truth: 387488,230650,493831,800285,771091,490901,49331,73777,283265,114839,857078,403345,669415,377842,610912,351364,700079,889367,349772,916915,794932,48754,323734,392518,488399,958864,712924,362956,108363,925667\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 834\n", + "Returned IDs: 32170,724205,603588,296374,777646,468935,482797,755759,135017,932030,83811,399008,27283,11461,742158,791872,813664,502590,255378,849922,775948,463206,812051,65388,902631,541811,393060,604939,447505,263260\n", + "Ground Truth: 32170,724205,603588,296374,777646,468935,482797,755759,135017,932030,83811,399008,27283,11461,742158,791872,813664,502590,574445,255378,849922,775948,463206,812051,65388,902631,541811,393060,604939,447505\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 835\n", + "Returned IDs: 670380,672661,554056,18529,371655,805604,878069,120980,541811,6153,579250,369956,899944,379767,309885,476683,235467,83341,467212,612297,632708,12387,100503,904507,155766,974941,150640,916915,729710,749704\n", + "Ground Truth: 670380,672661,554056,18529,371655,805604,878069,120980,541811,6153,579250,369956,899944,379767,309885,476683,235467,83341,467212,612297,632708,12387,230470,100503,904507,155766,974941,150640,916915,729710\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 836\n", + "Returned IDs: 828466,855897,890360,435655,791291,311016,997496,973356,167515,33942,206153,947151,454064,492133,180244,972594,18830,105354,895648,638708,256011,36836,695965,585246,952626,324863,890509,223552,195445,301135\n", + "Ground Truth: 828466,855897,890360,435655,791291,311016,997496,973356,167515,33942,206153,947151,454064,492133,180244,972594,18830,105354,895648,638708,256011,36836,695965,585246,324863,952626,890509,223552,195445,886727\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 837\n", + "Returned IDs: \n", + "Ground Truth: 973196,619981,21239,855553,759354,651824,153946,265421,374017,417872,384822,6211,40196,779902,69280,408751,636912,322391,5655,315364,405819,143117,598946,284708,850837,152655,608244,879890,473819,529848\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 839\n", + "Returned IDs: 371655,181092,687257,18529,820441,292970,689772,232674,600302,271111,589289,904507,691762,536591,354173,745484,677600,996053,770401,124402,881687,87859,13656,394343,142996,345023,290832,149903,393202,640176\n", + "Ground Truth: 371655,181092,687257,18529,820441,292970,984783,869693,689772,232674,600302,271111,589289,904507,691762,536591,354173,745484,677600,996053,770401,124402,881687,87859,13656,394343,142996,652215,345023,290832\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 840\n", + "Returned IDs: 319829,297415,615795,582949,621187,28590,780909,43081,109630,314296,102502,565531,79971,817938,981852,537644,462345,604597,185240,232600,216428,737354,773822,982771,948640,686429,606966,86716,143721,392918\n", + "Ground Truth: 319829,297415,615795,582949,621187,28590,780909,43081,109630,314296,542881,102502,565531,79971,817938,981852,537644,462345,604597,185240,232600,216428,737354,773822,982771,87204,948640,686429,606966,86716\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 841\n", + "Returned IDs: 721592,911639,590960,854414,885793,728241,298415,331075,663199,507158,972594,343498,34947,847918,14079,530372,183565,381463,971100,692677,613195,322603,687436,106582,403345,18689,603588,820124,447546,893451\n", + "Ground Truth: 721592,911639,590960,854414,885793,728241,298415,331075,618559,663199,507158,972594,343498,34947,847918,422335,14079,530372,183565,381463,971100,692677,613195,322603,687436,106582,403345,18689,603588,820124\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 845\n", + "Returned IDs: 845548,717859,769082,466219,2093,803864,30526,400026,862039,982943,632080,403345,514540,381950,630890,975317,637561,391194,522421,325191,318010,595687,142273,826126,918830,328833,749437,9628,369554,621822\n", + "Ground Truth: 845548,717859,769082,466219,2093,803864,30526,400026,862039,982943,632080,403345,514540,381950,630890,975317,637561,391194,522421,325191,318010,226364,595687,142273,826126,918830,328833,749437,9628,369554\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 846\n", + "Returned IDs: 75102,733050,791727,516859,281286,9257,460806,498181,961310,142273,74360,353622,43764,490600,539150,237537,916369,413275,49945,404618,636425,283175,581151,581559,706695,298031,548164,912407,864975,544861\n", + "Ground Truth: 75102,733050,791727,516859,281286,9257,460806,498181,961310,142273,74360,353622,43764,490600,539150,237537,916369,413275,49945,404618,636425,283175,581151,581559,24196,706695,726544,298031,548164,912407\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 847\n", + "Returned IDs: 334694,489788,609144,750140,437964,949614,439747,314296,232488,753359,879826,863748,181413,19390,54533,188311,537553,825049,203699,403269,611032,940505,526000,874189,988834,641909,851607,495044,979439,644596\n", + "Ground Truth: 334694,489788,609144,750140,437964,949614,439747,314296,232488,753359,879826,863748,181413,19390,4810,54533,188311,537553,825049,203699,403269,611032,940505,526000,874189,988834,641909,851607,495044,979439\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 849\n", + "Returned IDs: 839858,904507,706695,42567,664447,24196,695965,917370,519135,929864,512544,124345,67033,390336,844571,794244,992979,730418,155972,643363,558153,971100,297165,916915,970607,692552,185309,804808,567342,932030\n", + "Ground Truth: 839858,904507,706695,42567,664447,24196,695965,917370,519135,929864,512544,124345,827573,67033,390336,844571,794244,992979,730418,155972,643363,558153,971100,931780,297165,916915,970607,902031,692552,185309\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 852\n", + "Returned IDs: 729998,670380,279054,333703,612429,13656,720943,84473,225501,264045,167515,405009,627144,224503,41301,124402,69280,706880,561483,623702,210958,393202,579250,691762,742354,675467,143117,150640,541811,554056\n", + "Ground Truth: 729998,670380,279054,333703,612429,13656,720943,84473,225501,264045,167515,405009,627144,224503,41301,124402,69280,706880,561483,623702,210958,748087,393202,579250,691762,500463,742354,675467,143117,150640\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 853\n", + "Returned IDs: 781086,587307,187060,434044,184382,361700,942331,740178,174596,314296,712316,326505,560360,111684,632080,827832,782221,408339,159863,570591,491812,112040,448587,142273,55517,4776,552650,456808,5544,5819\n", + "Ground Truth: 781086,587307,187060,434044,184382,361700,942331,740178,174596,314296,712316,326505,560360,111684,632080,827832,782221,408339,159863,570591,491812,112040,968242,448587,142273,55517,4776,552650,456808,5544\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 855\n", + "Returned IDs: 8238,552835,977051,764838,354435,484099,19687,214921,796629,393110,623692,195445,9912,643702,378429,379478,220615,599943,180244,596858,679202,37793,811153,735531,11868,831446,886727,541759,575922,212043\n", + "Ground Truth: 8238,552835,977051,764838,354435,484099,19687,214921,796629,393110,623692,195445,32666,9912,643702,378429,379478,220615,599943,180244,596858,679202,37793,811153,735531,11868,831446,886727,541759,970607\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 856\n", + "Returned IDs: 467212,670380,166392,358859,729710,755577,18529,541811,65199,195445,299027,522760,261509,956035,507158,665325,680128,544861,911639,463724,643793,849922,548164,751796,576985,379767,687287,783900,677600,405219\n", + "Ground Truth: 467212,670380,166392,358859,729710,755577,18529,541811,65199,195445,299027,522760,261509,956035,89824,507158,665325,680128,544861,37002,911639,463724,643793,849922,548164,751796,576985,379767,113086,687287\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 857\n", + "Returned IDs: \n", + "Ground Truth: 549024,739092,812990,150640,944204,916915,327868,18830,996053,215745,371655,311197,317494,492133,507123,529324,814001,974849,242065,362244,374017,32618,220971,454064,33402,402047,848230,850030,392618,899944\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 860\n", + "Returned IDs: 401384,269375,350086,454064,450445,916915,700519,266404,445370,158345,779902,759354,437403,6211,79772,820124,899944,505357,990925,375948,632640,139554,18830,554056,12291,91663,24196,902137,695965,831415\n", + "Ground Truth: 401384,269375,350086,454064,450445,916915,700519,266404,445370,158345,779902,759354,437403,6211,79772,820124,899944,505357,990925,375948,632640,139554,674899,18830,554056,105966,12291,91663,24196,902137\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 861\n", + "Returned IDs: 364319,6153,805604,755666,945780,612297,652336,945682,454064,541811,366544,235467,932030,706880,100503,855908,409731,83341,916915,26089,379767,908750,670380,65550,371655,232002,350669,714118,225501,342390\n", + "Ground Truth: 364319,6153,805604,755666,945780,612297,652336,945682,454064,541811,366544,235467,932030,706880,100503,855908,409731,83341,916915,26089,379767,908750,686429,109630,670380,65550,621622,371655,232002,350669\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 862\n", + "Returned IDs: 379767,511850,371655,64273,342926,333703,357908,373940,259018,796968,29560,712387,739725,37859,874438,968295,916915,392618,94582,618559,712538,627144,867233,739092,809535,579250,830735,886727,554056,14717\n", + "Ground Truth: 379767,511850,371655,342926,64273,333703,357908,373940,259018,796968,29560,712387,739725,37859,874438,968295,916915,392618,94582,618559,568658,712538,20341,541759,908539,627144,867233,739092,809535,579250\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 863\n", + "Returned IDs: 910768,372041,700640,612297,740241,445121,142868,897071,224067,657402,804808,112463,41169,787094,441323,6488,450445,48813,541980,735520,286033,848945,809535,683736,907217,570591,267447,593933,775449,963711\n", + "Ground Truth: 910768,372041,700640,612297,740241,445121,142868,897071,792419,224067,657402,804808,112463,41169,787094,441323,6488,450445,48813,541980,735520,286033,848945,809535,683736,907217,570591,267447,593933,775449\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 866\n", + "Returned IDs: 551028,740655,900187,186666,88740,214921,900704,796629,769932,116371,873157,91446,610357,426773,818220,796917,693483,994193,74771,453633,651878,651376,292065,23788,618559,407266,304643,228850,312103,977051\n", + "Ground Truth: 551028,740655,900187,186666,88740,214921,900704,796629,769932,116371,873157,91446,610357,426773,818220,796917,693483,994193,74771,453633,651878,651376,292065,23788,618559,407266,304643,894768,228850,312103\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 867\n", + "Returned IDs: 337480,339516,212360,516017,408339,672614,780650,631834,982943,62800,643077,88385,235267,125135,510938,692131,265408,712316,418997,989555,261414,584840,876126,874072,591880,607059,593617,34721,856773,369554\n", + "Ground Truth: 337480,339516,212360,516017,408339,672614,780650,631834,982943,62800,643077,88385,235267,125135,510938,692131,265408,712316,418997,989555,261414,584840,876126,874072,591880,607059,593617,34721,856773,99498\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 868\n", + "Returned IDs: 44896,984648,564621,857564,598044,272209,274079,444077,469341,872723,24692,712558,331608,313013,830239,234369,471044,758981,93981,602557,402987,887799,179536,947990,525149,652752,275992,613235,463697,443374\n", + "Ground Truth: 44896,984648,564621,857564,598044,272209,274079,444077,469341,872723,24692,465859,712558,331608,313013,830239,234369,471044,758981,93981,602557,402987,887799,179536,947990,525149,652752,275992,613235,463697\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 870\n", + "Returned IDs: 984194,350669,433808,100503,207041,952937,838678,712316,664447,502367,685915,212360,176958,632080,754108,364869,166392,843989,993882,234273,523569,740178,687287,419480,599683,179503,355340,642021,516184,507158\n", + "Ground Truth: 984194,350669,875745,433808,100503,207041,952937,838678,712316,664447,502367,685915,212360,176958,632080,951211,754108,364869,745484,166392,843989,993882,234273,523569,740178,687287,419480,599683,179503,355340\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 871\n", + "Returned IDs: 18830,602442,112256,402047,55307,396777,265421,916915,214269,695965,944204,697754,63145,672661,939774,982993,519324,288817,579578,492133,38444,848230,612117,129088,994164,939067,355697,554056,373940,778930\n", + "Ground Truth: 18830,602442,112256,402047,55307,396777,265421,916915,214269,695965,944204,697754,63145,672661,939774,982993,519324,288817,579578,492133,76701,38444,848230,612117,129088,994164,939067,355697,554056,373940\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 872\n", + "Returned IDs: 213374,318014,753101,736060,534803,884481,976682,301198,920852,808521,592165,646934,916369,989476,418369,712387,715216,723800,952711,953810,622292,514903,82917,345521,661315,759551,629171,259033,248355,587977\n", + "Ground Truth: 213374,318014,753101,736060,534803,884481,976682,301198,920852,808521,592165,646934,916369,989476,418369,712387,410815,715216,723800,10161,952711,953810,622292,454216,262546,514903,82917,345521,661315,759551\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 874\n", + "Returned IDs: 966033,81619,987690,735531,454216,830270,230204,691762,581151,18529,607665,364869,429735,394343,843427,345023,798284,839858,502001,757816,180244,37639,379767,447505,408339,616251,94582,963711,207041,382634\n", + "Ground Truth: 966033,81619,987690,735531,454216,441607,830270,230204,691762,581151,18529,607665,992979,364869,429735,394343,843427,345023,798284,195445,839858,502001,757816,180244,37639,106582,379767,179503,447505,408339\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 875\n", + "Returned IDs: 600302,534052,885971,714118,92914,589060,832866,458487,812051,568276,312098,604527,441999,535419,19134,205310,535046,180880,861934,135017,390830,421360,571874,105355,84975,494089,732233,336977,583350,498815\n", + "Ground Truth: 600302,534052,885971,714118,92914,589060,832866,458487,812051,568276,312098,604527,441999,212576,535419,19134,205310,535046,180880,861934,135017,390830,421360,571874,105355,84975,494089,732233,336977,583350\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 876\n", + "Returned IDs: 421017,18529,573332,476683,670380,214269,463070,533939,998445,976404,729998,117859,405252,479167,453789,672023,417928,775428,393202,90217,409130,544861,579250,882114,600530,729710,691762,652336,400877,730997\n", + "Ground Truth: 421017,18529,573332,476683,670380,214269,463070,533939,998445,976404,729998,117859,405252,479167,26152,453789,672023,417928,775428,393202,90217,409130,544861,579250,882114,600530,729710,691762,652336,400877\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 877\n", + "Returned IDs: 203153,612297,598764,341783,734241,98827,360986,602442,408442,730865,549510,757034,680706,904300,579250,413918,632640,652781,729998,487383,405014,572761,445843,759354,152655,264692,445924,409288,686408,541980\n", + "Ground Truth: 203153,612297,598764,341783,734241,98827,360986,29964,602442,408442,730865,549510,757034,680706,327573,904300,127351,579250,38444,413918,632640,652781,729998,826831,487383,405014,572761,445843,759354,152655\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 882\n", + "Returned IDs: 16656,407004,403345,996028,353622,708631,232600,492260,12057,712316,500438,852381,771351,439626,696808,664447,311068,502001,272209,740178,194080,949708,522421,469341,619331,797438,22871,109630,87686,159863\n", + "Ground Truth: 16656,407004,403345,996028,353622,708631,232600,492260,12057,712316,500438,852381,771351,99498,439626,696808,664447,311068,502001,272209,740178,194080,949708,469341,522421,619331,797438,22871,109630,87686\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 883\n", + "Returned IDs: 980923,311197,882806,486639,739482,990925,695965,961891,311481,98827,371655,558539,797895,910344,793875,869710,3386,944994,104851,525546,841556,454064,158345,400889,804312,341783,445370,479197,882066,150640\n", + "Ground Truth: 980923,311197,882806,486639,739482,990925,695965,961891,311481,98827,371655,635965,558539,797895,910344,793875,869710,3386,944994,104851,525546,841556,158345,454064,400889,804312,977672,341783,445370,479197\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 884\n", + "Returned IDs: 673374,555552,734533,550602,447505,312098,665929,768388,85649,685561,463206,857420,354583,468935,265408,835244,10915,415235,888411,535046,489145,375665,724205,155226,83811,319689,917455,700079,334437,905439\n", + "Ground Truth: 673374,555552,734533,550602,447505,312098,665929,768388,85649,685561,463206,857420,354583,468935,265408,835244,10915,590195,415235,888411,535046,489145,421360,375665,724205,155226,83811,319689,917455,700079\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 886\n", + "Returned IDs: 982114,234260,354173,924904,236959,17107,72749,495221,87686,601998,245332,157625,479197,415235,893919,482569,299027,411149,454243,784568,15988,135017,238846,64898,358859,597551,334918,394829,174389,337126\n", + "Ground Truth: 982114,234260,354173,924904,236959,17107,72749,495221,87686,601998,245332,157625,479197,415235,893919,482569,299027,411149,454243,784568,15988,135017,238846,64898,358859,74067,597551,334918,394829,174389\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 887\n", + "Returned IDs: 794285,319288,810989,626132,342926,904507,232795,242753,628830,772316,783900,589079,283218,596858,488434,856746,390259,900187,623687,544437,156608,727702,874438,214921,561189,658210,663199,395038,696450,219559\n", + "Ground Truth: 794285,319288,213129,810989,626132,342926,904507,232795,242753,628830,772316,783900,589079,283218,596858,488434,856746,390259,900187,623687,544437,156608,727702,874438,214921,899684,561189,344779,658210,663199\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 888\n", + "Returned IDs: 702283,826244,166392,198289,712316,801035,698668,912561,984648,313013,100503,879890,120482,65773,258029,708566,955379,75502,513955,872921,469341,758525,379767,940505,872723,79136,218542,652336,720918,272209\n", + "Ground Truth: 702283,826244,166392,198289,712316,801035,698668,912561,984648,313013,100503,879890,120482,65773,258029,708566,955379,75502,513955,872921,469341,758525,379767,940505,493831,872723,79136,379644,218542,652336\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 889\n", + "Returned IDs: \n", + "Ground Truth: 553824,547964,157745,367512,135821,378429,948817,715216,203351,847322,514903,62121,345521,519958,77750,588446,135930,418021,406380,545048,970899,381162,279763,550440,839052,14452,490901,775037,795968,765561\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 891\n", + "Returned IDs: 126596,129270,950869,40163,40011,357908,431659,916915,58844,14717,571857,488869,589289,621622,613045,124402,904507,576985,772035,851335,41301,234842,591193,174808,259018,423033,517119,455501,554056,626132\n", + "Ground Truth: 126596,129270,950869,40163,40011,357908,431659,916915,58844,14717,571857,488869,589289,621622,613045,124402,904507,576985,772035,851335,41301,234842,591193,174808,259018,423033,177766,517119,455501,554056\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 892\n", + "Returned IDs: 990435,483807,258784,691468,881397,445370,235267,927595,640141,2774,443374,797438,303085,982943,712316,888723,701821,234369,632080,142142,146294,452902,506149,746633,925914,534553,967574,632814,989555,217118\n", + "Ground Truth: 990435,483807,258784,691468,881397,445370,235267,927595,640141,2774,443374,797438,303085,982943,712316,888723,701821,234369,632080,142142,146294,452902,506149,746633,925914,534553,967574,632814,989555,637811\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 893\n", + "Returned IDs: 143117,710026,980434,885272,832778,670380,657235,511821,674899,166392,139298,447505,976404,701447,178358,41301,405252,950893,65199,878693,561483,445843,641329,463070,279054,522760,331816,587475,632708,608314\n", + "Ground Truth: 143117,710026,980434,885272,832778,670380,657235,511821,674899,166392,139298,447505,701447,976404,178358,41301,405252,950893,65199,878693,561483,381463,445843,641329,454724,463070,279054,522760,331816,587475\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 894\n", + "Returned IDs: 87204,542881\n", + "Ground Truth: 379767,94582,843427,311789,831132,670380,612297,548164,195445,973024,464940,828137,228850,706695,464820,441323,333703,588446,373723,65773,758267,357908,507158,378429,891494,576985,857420,780978,797438,166392\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 898\n", + "Returned IDs: 288169,172701,270634,3563,275459,875052,299325,486639,426496,687436,342834,373723,371655,398242,916915,479167,603669,421218,85322,473819,194570,102754,354656,882066,939774,402047,555467,541428,206153,151388\n", + "Ground Truth: 288169,172701,270634,3563,24642,275459,875052,299325,486639,426496,687436,342834,472415,373723,924820,371655,398242,916915,479167,603669,421218,85322,473819,194570,102754,354656,882066,939774,402047,555467\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 899\n", + "Returned IDs: 274156,800420,600302,64898,946518,793267,363525,238440,318050,619853,851545,987690,296440,900720,955873,677600,406987,221330,66624,673607,459789,740741,502001,84043,165554,643793,15988,405219,321321,685395\n", + "Ground Truth: 274156,800420,600302,64898,946518,793267,363525,238440,318050,619853,851545,987690,296440,900720,955873,677600,301983,406987,221330,66624,673607,459789,72631,292605,436016,740741,502001,84043,165554,643793\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 901\n", + "Returned IDs: 331328,903271,938312,686222,180244,255136,784448,422454,559942,13255,602478,618559,250895,712538,927568,577280,621662,949777,936501,750188,158769,796629,339090,979774,547189,737809,229680,410462,357403,203390\n", + "Ground Truth: 331328,903271,938312,412721,686222,180244,255136,784448,422454,559942,13255,602478,618559,250895,712538,927568,577280,621662,949777,936501,750188,158769,796629,339090,979774,547189,737809,229680,410462,357403\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 902\n", + "Returned IDs: 279492,317451\n", + "Ground Truth: 623711,83341,65773,212360,34721,283960,443374,412471,462345,516184,647935,819736,607059,834643,32213,698775,4690,565531,365399,154098,194080,381162,910183,951923,18651,408339,663376,347041,625806,997452\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 903\n", + "Returned IDs: \n", + "Ground Truth: 898674,737135,283989,907602,324863,188626,890360,126043,41301,131053,940816,402427,837825,121904,112463,920761,141715,24642,115722,60713,321810,597507,855897,572761,264045,236271,357262,772035,316030,978689\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 904\n", + "Returned IDs: 720964,163554,632814,65773,871011,683736,327573,759354,314104,34721,161960,333068,343268,212360,443374,3648,294724,228850,83341,764656,857420,303085,876126,548164,777636,931369,381162,623711,243070,607059\n", + "Ground Truth: 720964,163554,632814,65773,871011,683736,327573,759354,314104,34721,161960,333068,343268,212360,443374,3648,294724,228850,152655,978956,83341,764656,857420,303085,876126,548164,777636,931369,381162,623711\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 905\n", + "Returned IDs: 135017,64898,245332,564509,783900,826126,961677,933349,142868,732233,664911,283397,269352,548164,415235,940505,849922,447505,5031,111684,166392,325191,784568,563923,757816,925434,778308,706695,242938,638004\n", + "Ground Truth: 135017,64898,245332,655568,564509,783900,826126,961677,933349,142868,732233,664911,283397,269352,548164,415235,940505,849922,447505,5031,111684,166392,325191,784568,563923,757816,925434,778308,706695,242938\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 906\n", + "Returned IDs: 882917,305570,929362,318181,506549,548697,307013,721575,32618,454064,355866,194570,874199,481233,55167,916915,158345,244717,371655,199677,402047,333398,934006,805604,579578,672661,571814,725978,804586,492133\n", + "Ground Truth: 882917,305570,929362,318181,506549,548697,307013,721575,32618,454064,355866,194570,874199,481233,55167,916915,184048,158345,244717,371655,199677,402047,333398,934006,805604,579578,672661,571814,725978,804586\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 907\n", + "Returned IDs: 259018,729998,249943,918389,150640,597161,554056,698050,211285,988352,333703,975967,124402,163554,579250,940691,12387,29560,970655,974941,206694,91663,597805,409731,6338,949366,371655,150633,316940,301135\n", + "Ground Truth: 259018,729998,249943,918389,150640,597161,554056,209632,698050,211285,988352,333703,975967,124402,163554,734574,579250,421628,543653,940691,12387,29560,970655,974941,206694,91663,50104,597805,409731,6338\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 910\n", + "Returned IDs: 787972,908344,360701,925160,369097,744935,163564,897240,58820,232377,121404,624771,208163,443374,813236,201294,450196,194080,290902,507637,158639,397738,37002,155572,116625,45393,165610,925914,94817,12387\n", + "Ground Truth: 787972,908344,360701,925160,369097,744935,163564,897240,58820,232377,121404,624771,208163,443374,813236,201294,450196,194080,290902,507637,158639,397738,37002,155572,116625,45393,915297,165610,925914,94817\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 912\n", + "Returned IDs: 622977,976404,451846,183691,742370,207488,544861,486447,682356,802512,5675,265864,747373,26867,523990,643793,274473,848043,677600,849922,758338,476743,304144,18529,911031,194718,379478,813664,337458,885971\n", + "Ground Truth: 622977,976404,451846,183691,742370,207488,544861,486447,682356,802512,5675,265864,747373,26867,523990,643793,274473,848043,677600,849922,758338,476743,304144,18529,911031,931780,194718,379478,813664,337458\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 914\n", + "Returned IDs: 372134,685079,848708,634279,708680,49209,977757,203198,780481,353622,145644,307804,311941,488219,296718,732515,893919,958864,331608,726544,712316,632080,438048,224503,500945,830239,379644,79136,212576,149032\n", + "Ground Truth: 372134,685079,848708,634279,708680,49209,977757,203198,780481,353622,145644,307804,311941,488219,296718,732515,893919,958864,331608,726544,712316,632080,514540,438048,224503,500945,830239,379644,79136,12387\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 916\n", + "Returned IDs: 90600,98827,184048,225390,537521,891494,976915,584838,826783,350669,614475,9024,974941,975967,6153,544679,859196,916915,751635,266109,270634,707999,805604,260423,185309,836797,153795,176751,341783,705136\n", + "Ground Truth: 90600,98827,184048,225390,537521,891494,976915,584838,826783,350669,570591,614475,9024,974941,975967,6153,544679,859196,140706,103104,916915,751635,32875,266109,270634,707999,805604,260423,185309,109016\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 917\n", + "Returned IDs: 504540,805947,604670,712558,71204,692131,234369,133860,813715,687634,857564,984648,202580,888723,632080,982943,469341,49919,437473,444077,887799,534299,124522,851546,276410,451846,490923,598044,425079,18529\n", + "Ground Truth: 504540,805947,604670,712558,71204,692131,234369,133860,813715,687634,857564,984648,202580,888723,632080,982943,469341,49919,437473,444077,887799,534299,124522,851546,276410,451846,668761,490923,598044,425079\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 918\n", + "Returned IDs: 221248,255826,488821,55083,194080,439626,492260,325904,967574,700352,232478,65773,28151,516184,18529,839268,155572,281286,548164,283960,516493,357314,350669,437403,760970,687287,897240,498312,365399,877450\n", + "Ground Truth: 221248,255826,488821,55083,194080,439626,492260,325904,967574,700352,232478,65773,28151,516184,18529,839268,155572,281286,517185,548164,283960,516493,357314,350669,437403,760970,687287,897240,498312,365399\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 919\n", + "Returned IDs: 756575,829092,894608,701488,967574,851546,980270,631399,13656,466875,917766,887335,672968,854801,816205,133770,891795,672614,600302,600333,41699,18529,461631,350669,181092,622977,640141,976404,83986,916247\n", + "Ground Truth: 756575,829092,894608,701488,967574,851546,980270,631399,13656,466875,917766,887335,672968,854801,816205,133770,891795,672614,600302,600333,41699,18529,461631,350669,181092,622977,155291,640141,976404,796968\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 920\n", + "Returned IDs: 621005,146431,447505,251899,835244,181482,313571,550602,888411,612593,909092,627271,191212,187394,903064,923437,117355,319689,659249,608209,749584,618515,185593,919298,968216,802837,265408,487634,877652,232228\n", + "Ground Truth: 621005,146431,447505,251899,835244,181482,313571,550602,888411,612593,909092,627271,191212,108643,187394,903064,923437,354583,117355,319689,659249,608209,749584,618515,185593,919298,968216,802837,265408,487634\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 923\n", + "Returned IDs: 817938,286585,565531,462345,712316,79971,194080,632080,982771,694177,443374,314296,686429,124269,65773,29774,355840,664447,957670,845747,548164,34721,228850,317007,547111,83341,23755,319829,525665,425093\n", + "Ground Truth: 817938,286585,565531,462345,712316,79971,194080,632080,982771,694177,443374,314296,686429,124269,65773,29774,355840,664447,957670,845747,548164,34721,228850,317007,547111,83341,23755,819736,319829,525665\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 928\n", + "Returned IDs: 907254,204309,886932,394101,347020,946173,15988,600302,239430,883700,33361,553480,406987,900720,363525,637465,74426,918746,634965,389553,296440,758033,354173,415235,994101,495221,319013,483079,272687,989316\n", + "Ground Truth: 907254,204309,886932,394101,347020,946173,15988,600302,239430,883700,33361,553480,406987,900720,363525,311068,637465,74426,918746,634965,389553,296440,758033,354173,415235,111684,994101,495221,319013,483079\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 930\n", + "Returned IDs: 988176,479167,791291,492133,998445,476683,695965,344295,737135,417928,613209,882806,579578,612297,956035,916915,355866,339223,298415,369956,341783,511821,115855,105354,231611,612117,729998,311016,939774,115722\n", + "Ground Truth: 988176,479167,791291,492133,998445,476683,695965,344295,737135,525546,417928,613209,882806,579578,612297,956035,916915,355866,339223,298415,152655,369956,341783,511821,115855,105354,231611,612117,729998,311016\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 932\n", + "Returned IDs: 464286,166392,488003,522760,680128,670380,904507,912561,619853,632708,710026,849922,221330,13792,885793,282975,828137,579250,364869,643793,259401,643363,834937,467212,826244,141367,911031,379767,762693,508494\n", + "Ground Truth: 464286,166392,488003,522760,680128,670380,904507,912561,619853,632708,710026,849922,221330,13792,885793,282975,828137,579250,364869,643793,259401,643363,834937,467212,826244,534447,141367,911031,379767,762693\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 933\n", + "Returned IDs: 701447,29560,350580,544861,996028,64898,834937,643793,469341,602442,393202,18830,687287,225501,585438,802512,600302,631399,308620,405219,166392,621454,209632,6338,613195,363525,135017,450445,695965,770486\n", + "Ground Truth: 701447,29560,350580,544861,996028,64898,834937,643793,469341,602442,393202,18830,687287,225501,585438,802512,600302,631399,308620,405219,166392,621454,209632,6338,613195,980270,751782,363525,135017,450445\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 934\n", + "Returned IDs: 402047,18830,98827,916915,827552,726699,844571,371655,40011,483807,184048,632814,232002,990925,804808,725978,529324,560714,612297,124402,685694,575389,672661,695965,982164,745484,492133,622694,525546,78624\n", + "Ground Truth: 402047,18830,98827,916915,827552,726699,844571,371655,40011,483807,184048,632814,232002,990925,804808,616554,725978,529324,560714,612297,124402,685694,575389,672661,695965,982164,745484,492133,622694,525546\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 935\n", + "Returned IDs: 882066,805604,350669,147165,850275,318917,972319,523049,543859,705136,124402,554056,607992,220971,259018,674823,695965,120528,807441,934210,176751,504643,825043,15628,793197,12387,469498,918389,135278,974849\n", + "Ground Truth: 882066,805604,350669,147165,850275,318917,972319,523049,543859,705136,124402,554056,607992,220971,597589,259018,674823,695965,392449,597805,78624,120528,807441,934210,176751,504643,825043,15628,793197,924968\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 938\n", + "Returned IDs: 350669,971100,899260,476798,85322,865901,488003,585438,311016,23755,832291,350580,248449,309885,86869,565051,805131,589292,21941,874072,731160,290395,826244,299027,366055,232002,523569,464286,814015,749704\n", + "Ground Truth: 350669,453789,971100,899260,476798,85322,865901,488003,585438,978956,311016,23755,616554,832291,350580,805604,248449,309885,86869,565051,805131,589292,21941,623711,874072,731160,290395,826244,299027,366055\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 939\n", + "Returned IDs: 971100,85322,871262,643793,731160,311016,814015,507158,687436,744351,476798,371655,710026,290395,115722,865901,564835,585246,331075,86869,996693,180244,855897,242065,435655,41099,541759,567342,76036,670380\n", + "Ground Truth: 971100,85322,871262,643793,731160,311016,902031,814015,507158,687436,744351,476798,371655,710026,290395,115722,865901,564835,585246,331075,86869,996693,180244,855897,242065,435655,41099,541759,567342,76036\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 940\n", + "Returned IDs: 520271,406932,10387,330823,631399,38443,470066,175218,137310,358675,249700,29472,205310,72631,50469,940478,742412,740900,371565,116922,818625,603588,948673,32666,600302,922191,813664,851545,35812,167622\n", + "Ground Truth: 520271,406932,10387,330823,631399,38443,937745,470066,175218,137310,358675,249700,29472,205310,488434,72631,50469,940478,742412,740900,371565,116922,818625,603588,558224,948673,32666,600302,922191,813664\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 941\n", + "Returned IDs: 507158,115722,331075,631399,585438,126043,18529,579250,931681,867184,904507,476798,141715,996693,687287,564509,670380,522760,931780,289626,342624,745484,435655,602442,300183,80107,677600,166392,658210,445370\n", + "Ground Truth: 507158,115722,331075,631399,585438,126043,18529,579250,931681,867184,904507,476798,141715,996693,687287,564509,641329,670380,522760,931780,289626,342624,745484,435655,602442,300183,477417,80107,677600,166392\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 942\n", + "Returned IDs: 18108,760641,874438,748959,664680,544401,232795,320557,307020,531095,453995,257069,290284,436385,452585,445370,166392,157454,342926,699616,800801,100503,245559,861541,80935,702283,180392,180244,109852,259401\n", + "Ground Truth: 18108,760641,874438,748959,664680,544401,232795,320557,944060,307020,531095,453995,257069,290284,436385,452585,319630,445370,473718,712316,166392,157454,342926,699616,800801,100503,799317,245559,861541,80935\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 943\n", + "Returned IDs: 755666,882768,935911,692188,950726,194329,967548,686408,18633,938397,529957,619768,450472,3386,294599,795321,541811,350257,529848,55199,712888,984655,944387,891149,932030,284708,642021,619211,973196,959763\n", + "Ground Truth: 755666,882768,935911,692188,950726,194329,881149,967548,686408,18633,938397,529957,619768,450472,3386,294599,795321,541811,350257,529848,55199,712888,984655,40163,944387,891149,229125,599813,650367,932030\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 946\n", + "Returned IDs: 502367,675950,664911,954884,757816,232600,932033,894768,622977,771351,961677,176958,377442,166392,565531,23755,112040,429758,643363,194920,732233,314296,862897,12057,53763,266784,740178,622318,560935,599683\n", + "Ground Truth: 792696,502367,675950,664911,954884,757816,232600,932033,894768,622977,771351,486447,961677,176958,377442,166392,565531,23755,112040,429758,643363,602664,194920,732233,314296,862897,12057,53763,266784,740178\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 947\n", + "Returned IDs: 893919,785700,927297,664447,463724,905941,597589,708492,331608,217118,149032,996028,620393,176958,462345,784568,641988,32875,927346,272209,238846,632080,479197,469341,363525,828009,18529,236959,912561,24692\n", + "Ground Truth: 893919,785700,927297,664447,463724,905941,258784,597589,708492,331608,217118,149032,996028,620393,176958,462345,784568,641988,32875,927346,272209,238846,632080,479197,469341,363525,828009,18529,236959,912561\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 948\n", + "Returned IDs: 680128,772146,832778,829977,11461,38444,327683,194080,464286,65773,687287,706880,311016,19687,166392,19091,643793,245559,492260,135017,378429,488003,418519,643363,178358,259401,893919,502314,469341,608314\n", + "Ground Truth: 680128,772146,832778,903465,829977,11461,38444,327683,194080,464286,65773,687287,706880,311016,19687,166392,19091,643793,245559,492260,135017,378429,488003,418519,643363,15628,178358,259401,893919,502314\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 949\n", + "Returned IDs: 712316,235267,966232,868021,100798,632080,881397,169805,717859,927148,125135,400026,781947,845747,408339,239691,391194,982943,258784,146294,826126,940443,957764,888723,286585,930237,314296,931626,989555,772146\n", + "Ground Truth: 712316,235267,966232,868021,100798,632080,881397,169805,717859,927148,125135,400026,781947,845747,408339,239691,391194,982943,258784,146294,826126,813664,940443,957764,888723,286585,930237,314296,931626,989555\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 951\n", + "Returned IDs: 501165,838964,526107,69601,556794,329782,904507,964592,894185,812245,128484,726445,572761,327644,567342,619188,347031,992979,241,321443,947151,916915,594577,226695,931780,930905,86869,210958,565051,371360\n", + "Ground Truth: 501165,838964,526107,69601,556794,329782,904507,964592,894185,812245,800968,128484,726445,572761,569913,327644,567342,619188,347031,992979,562237,241,321443,947151,916915,307013,79055,594577,226695,234552\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 953\n", + "Returned IDs: 41301,445843,579250,710026,371655,445370,632814,916915,465498,507158,124402,504643,686408,670380,541811,29560,120980,554056,18830,602442,73368,38444,544861,608314,918389,510927,443127,488003,729998,643793\n", + "Ground Truth: 41301,445843,579250,710026,371655,445370,632814,916915,465498,507158,124402,209632,504643,686408,670380,541811,29560,120980,554056,18830,602442,73368,544861,38444,608314,918389,510927,443127,488003,729998\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 954\n", + "Returned IDs: 950925,634079,232391,508644,627123,104889,12291,856511,544861,687287,632447,791872,931780,694660,908539,629891,97984,685643,643363,145306,101604,719438,377161,588577,35070,687102,69527,478600,908344,465152\n", + "Ground Truth: 950925,634079,232391,508644,627123,104889,12291,856511,544861,687287,632447,791872,931780,491253,694660,908539,629891,97984,685643,643363,145306,101604,719438,377161,910325,588577,35070,687102,69527,166583\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 955\n", + "Returned IDs: 507158,97858,701821,124402,371655,904507,115722,445843,996053,916915,497305,184048,670380,687287,309885,602442,564509,182722,298415,105354,333703,735994,69280,964592,878069,101653,464286,126043,867184,850030\n", + "Ground Truth: 507158,97858,701821,124402,371655,904507,115722,445843,996053,916915,497305,184048,670380,687287,309885,602442,564509,182722,298415,105354,333703,735994,69280,964592,878069,101653,464286,126043,902031,867184\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 956\n", + "Returned IDs: 720964,857861,492133,632814,963711,331195,18830,514080,209632,598764,477417,465498,844571,523049,670380,899944,598946,437989,249943,417872,664447,579250,445370,612297,265421,978533,87686,467212,529324,6153\n", + "Ground Truth: 720964,857861,492133,632814,963711,331195,18830,514080,209632,598764,477417,465498,844571,523049,670380,899944,598946,437989,249943,417872,664447,579250,353622,867184,445370,612297,12057,265421,978533,87686\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 959\n", + "Returned IDs: 603567,664911,630048,458487,890698,85649,851293,604527,757816,8934,18686,581151,14079,824499,840519,787972,708284,494089,732233,534157,720691,318010,885971,952738,239920,430561,539150,835428,865411,257356\n", + "Ground Truth: 603567,664911,630048,458487,890698,85649,851293,604527,757816,8934,18686,581151,14079,824499,840519,787972,708284,952810,494089,732233,534157,720691,318010,902521,885971,952738,239920,430561,539150,835428\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 960\n", + "Returned IDs: 63587,300798,946518,777881,768892,135017,600302,943363,530092,84043,167544,643363,64898,385517,577468,791872,534553,757816,790358,7554,238440,426893,343268,180244,646552,793267,354173,849922,571874,888976\n", + "Ground Truth: 63587,300798,946518,777881,768892,135017,600302,943363,530092,84043,167544,643363,64898,385517,577468,791872,300816,714436,534553,757816,790358,7554,238440,426893,343268,180244,646552,793267,354173,289778\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 963\n", + "Returned IDs: 23934,468264,903064,918894,820124,581331,706968,211528,334437,555030,543216,643793,246885,889753,111684,783900,632080,230132,4512,405009,441577,41450,74171,30526,4855,120482,712316,738229,687287,757634\n", + "Ground Truth: 23934,468264,903064,918894,820124,581331,706968,211528,334437,555030,543216,643793,246885,889753,111684,783900,632080,230132,4512,405009,441577,41450,74171,30526,4855,934315,120482,712316,738229,687287\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 965\n", + "Returned IDs: 697522,85322,464286,488003,176958,687436,735994,320557,122407,814015,60143,141715,405219,480557,476798,347031,594577,643793,433497,350580,445843,319630,834643,785700,775449,632814,664447,510927,41301,413276\n", + "Ground Truth: 697522,85322,464286,488003,176958,687436,735994,320557,122407,814015,60143,141715,462345,405219,480557,476798,347031,594577,643793,433497,350580,445843,319630,834643,785700,775449,632814,664447,510927,41301\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 966\n", + "Returned IDs: 98827,995884,341783,990925,124402,477305,804586,579250,73673,612297,680438,248449,916915,632640,316940,228265,757816,780336,331195,633425,379767,670380,393202,67033,893919,371655,350580,652336,24196,668240\n", + "Ground Truth: 98827,995884,341783,990925,124402,477305,804586,579250,73673,612297,680438,659407,248449,708492,916915,632640,228265,316940,757816,780336,331195,633425,670380,379767,241,393202,67033,893919,371655,350580\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 967\n", + "Returned IDs: 509697,685670,881287,476498,94599,364623,40022,84975,68616,631177,173393,850368,433964,763712,386620,135017,870304,548164,830616,568276,203779,275549,848043,463724,849922,183565,428011,783367,902631,995102\n", + "Ground Truth: 509697,685670,881287,476498,94599,364623,40022,84975,68616,631177,173393,850368,433964,763712,386620,135017,870304,313747,548164,830616,568276,203779,275549,848043,463724,849922,183565,428011,783367,902631\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 968\n", + "Returned IDs: 41253,468935,313140,631177,11461,868934,854483,320696,548164,24785,790358,603588,14079,135017,615,183691,664911,605682,476498,166392,777646,48844,700275,851293,131919,85649,772146,311068,724881,10915\n", + "Ground Truth: 41253,468935,313140,631177,11461,868934,854483,714436,320696,548164,24785,790358,603588,14079,135017,615,183691,664911,605682,476498,166392,777646,48844,700275,851293,131919,85649,772146,311068,724881\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 969\n", + "Returned IDs: 521706,709497,891299,492970,568706,32666,254515,830349,510662,847404,942244,137310,847803,47062,195445,865749,181413,232228,840484,664970,334437,179288,665929,475187,533672,9927,755759,362005,347177,738229\n", + "Ground Truth: 521706,709497,891299,492970,568706,32666,254515,830349,510662,847404,942244,137310,847803,47062,949077,195445,865749,181413,232228,840484,664970,66074,23755,334437,422454,179288,665929,475187,533672,9927\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 970\n", + "Returned IDs: 652336,945780,344295,98827,612297,24196,63145,525546,288817,64898,488003,166392,560714,916915,379767,638708,882806,695965,389102,9024,237494,904507,754108,447505,612117,757816,844571,878069,104851,142868\n", + "Ground Truth: 652336,945780,344295,98827,612297,24196,63145,525546,288817,64898,488003,560714,166392,916915,379767,638708,882806,940505,695965,389102,6338,9024,237494,904507,754108,610912,763544,447505,676869,757816\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 971\n", + "Returned IDs: 218542,66074,32213,712316,845747,491545,83341,65773,516184,286585,504305,335184,418997,173863,412471,643131,623711,121385,311929,344253,940505,79971,607059,773822,625806,834643,967189,408339,3648,565531\n", + "Ground Truth: 218542,66074,32213,712316,845747,491545,83341,65773,516184,286585,504305,335184,418997,173863,412471,643131,623711,121385,311929,865555,344253,940505,79971,607059,773822,625806,834643,37639,967189,408339\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 973\n", + "Returned IDs: 12387,926270,447287,804808,998221,916915,897266,899944,431046,483807,5031,571814,632814,990435,826126,153795,857861,273467,830239,210748,469428,500463,104733,795002,379767,124402,445370,929277,305663,544861\n", + "Ground Truth: 12387,926270,447287,804808,998221,916915,897266,899944,431046,483807,5031,571814,632814,990435,826126,153795,857861,273467,830239,210748,469428,500463,104733,795002,379767,124402,729710,445370,929277,305663\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 974\n", + "Returned IDs: 870304,1175,679202,531095,11868,596858,249837,818220,955518,912520,511931,422454,623687,229596,545096,489015,949777,174609,43097,19687,393110,320932,129313,811153,779461,305016,797438,912073,558243,948300\n", + "Ground Truth: 870304,1175,679202,531095,11868,596858,249837,818220,533672,955518,912520,511931,422454,623687,229596,545096,489015,949777,174609,43097,19687,393110,320932,129313,811153,779461,305016,797438,912073,558243\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 975\n", + "Returned IDs: 980270,770401,775428,677600,834937,802512,785700,544861,687287,73368,112463,327683,405219,419803,147610,272687,616251,357908,735531,788045,982771,64898,990435,83341,18529,927595,394343,42809,812067,436016\n", + "Ground Truth: 980270,770401,775428,677600,834937,802512,785700,544861,687287,73368,112463,327683,405219,419803,147610,272687,616251,354435,357908,735531,788045,954884,982771,64898,990435,83341,18529,927595,394343,42809\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 976\n", + "Returned IDs: 940443,903148,408339,296010,142273,37639,732715,912407,712316,600302,522421,345023,664447,184904,864975,683413,353622,390968,431100,818903,958864,258486,826126,724881,289778,206137,400026,899582,185279,772183\n", + "Ground Truth: 940443,903148,408339,810278,296010,142273,37639,732715,912407,712316,600302,522421,345023,664447,184904,864975,683413,353622,390968,431100,818903,958864,258486,826126,724881,289778,206137,849922,400026,899582\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 978\n", + "Returned IDs: 722764,205310,259018,829977,139298,967181,363525,421446,266075,130172,643793,834937,677600,35443,88319,253578,813664,19091,357268,564509,828137,178358,37793,336977,637465,318050,236271,301135,392788,391479\n", + "Ground Truth: 722764,205310,259018,829977,139298,967181,363525,421446,266075,130172,960873,643793,834937,677600,35443,88319,253578,813664,19091,357268,564509,828137,178358,37793,336977,637465,318050,236271,301135,392788\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 981\n", + "Returned IDs: 205023,828589,413275,43764,550249,534553,155572,997452,443374,65773,706695,562648,854483,839858,955142,403345,4690,964324,567033,58820,170989,664447,876126,79136,612297,940443,784568,230441,142273,831132\n", + "Ground Truth: 205023,828589,413275,43764,550249,534553,155572,997452,443374,706695,65773,562648,624862,854483,839858,955142,403345,4690,964324,567033,58820,170989,664447,876126,79136,612297,940443,784568,230441,142273\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 983\n", + "Returned IDs: 894962,955142,828009,632814,938373,483807,5808,607059,258784,598144,797438,68811,826126,317007,45393,664447,613133,443374,982164,632080,548164,897240,194080,502001,142142,746633,363784,986333,541811,758267\n", + "Ground Truth: 894962,955142,828009,632814,938373,483807,5808,607059,258784,529778,598144,797438,68811,826126,317007,45393,664447,613133,443374,982164,632080,548164,897240,194080,502001,142142,452902,402371,746633,363784\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 984\n", + "Returned IDs: 244780,48844,683321,447505,85649,166392,568276,50469,356913,849922,754108,24110,820124,980493,745484,386589,132941,712316,529848,835244,482380,714118,87859,179779,935459,486447,19091,860068,420265,813664\n", + "Ground Truth: 244780,48844,683321,447505,85649,166392,568276,50469,356913,849922,754108,24110,820124,980493,311197,745484,386589,132941,712316,529848,835244,482380,714118,87859,179779,935459,486447,19091,860068,420265\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 985\n", + "Returned IDs: 253845,915297,134044,805131,40940,450215,475232,245559,849922,453446,94599,677600,695450,679336,269352,560935,544437,523990,902631,153485,385517,541149,408948,350669,416802,445370,138015,737583,155647,18529\n", + "Ground Truth: 253845,915297,134044,805131,40940,450215,475232,245559,849922,453446,94599,677600,695450,679336,269352,560935,544437,523990,902631,153485,385517,541149,408948,350669,416802,445370,138015,227025,737583,155647\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 986\n", + "Returned IDs: 587475,978689,519324,203353,670380,112463,94264,533939,737135,141715,978005,180138,507158,482176,632708,772035,927595,67033,445843,615485,510927,405252,88319,309885,227025,900717,402790,363525,853066,130172\n", + "Ground Truth: 587475,978689,519324,203353,670380,112463,94264,533939,737135,141715,978005,180138,507158,482176,902031,632708,772035,927595,67033,445843,615485,510927,405252,88319,309885,227025,900717,402790,363525,853066\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 988\n", + "Returned IDs: 413918,709106,212049,153960,551148,906097,434427,759354,612508,126840,270634,894696,632640,955841,102881,863859,258356,614475,772511,612297,351661,286569,891494,379767,69280,360986,381462,98827,549029,758525\n", + "Ground Truth: 413918,709106,212049,40196,153960,551148,906097,434427,759354,612508,126840,270634,894696,632640,955841,102881,863859,258356,614475,772511,612297,351661,286569,891494,379767,69280,360986,381462,98827,549029\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 989\n", + "Returned IDs: 38292,386063,664911,176444,632080,12138,85649,982943,125135,217463,468430,788045,927148,532718,712316,187060,636197,560360,552650,474180,824499,510938,49919,290400,408339,452465,781947,842911,5525,425093\n", + "Ground Truth: 38292,986435,386063,664911,176444,632080,12138,85649,982943,125135,217463,468430,788045,927148,532718,712316,187060,636197,560360,552650,474180,824499,684312,510938,49919,290400,408339,452465,781947,842911\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 990\n", + "Returned IDs: 369753,937619,272687,83986,206564,587977,604149,232478,706695,94582,606232,378429,815877,780978,441607,804662,775948,311637,18529,672650,599977,756688,996028,631399,729163,180880,449334,206216,355239,203390\n", + "Ground Truth: 369753,937619,272687,83986,206564,587977,604149,232478,706695,94582,606232,378429,815877,780978,441607,804662,775948,311637,18529,672650,599977,756688,996028,631399,729163,180880,449334,206216,104439,970607\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 991\n", + "Returned IDs: 861934,705065,263260,772146,14079,778134,589060,768892,163480,494089,364869,850368,468935,730420,742412,604939,911031,390830,65388,739092,550236,534052,135017,849922,832465,237345,747373,827628,790358,860068\n", + "Ground Truth: 861934,705065,263260,772146,14079,778134,589060,768892,163480,494089,364869,850368,468935,730420,742412,482797,604939,911031,390830,65388,739092,550236,534052,135017,849922,832465,237345,747373,952810,827628\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 992\n", + "Returned IDs: 126840,413918,759354,799572,709106,258356,20434,336941,549029,855553,445924,777636,153960,212049,325987,906097,863859,632640,434427,633831,401384,374017,24196,758525,98827,633425,575058,94963,773701,351661\n", + "Ground Truth: 126840,413918,759354,799572,709106,258356,20434,336941,549029,855553,445924,777636,153960,212049,325987,906097,863859,632640,434427,633831,401384,374017,24196,758525,98827,633425,115832,575058,94963,773701\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 993\n", + "Returned IDs: 351408,802512,628922,675467,411253,544437,2907,853066,344779,672023,576985,201261,843163,949549,756114,139298,940047,88319,345023,988352,214269,319339,813664,409130,572761,419803,436016,691762,311016,641329\n", + "Ground Truth: 351408,802512,628922,675467,411253,544437,2907,853066,344779,672023,576985,201261,843163,949549,756114,139298,940047,88319,345023,498312,988352,214269,319339,813664,256547,409130,572761,419803,436016,691762\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 994\n", + "Returned IDs: \n", + "Ground Truth: 240932,855988,772035,492133,857861,519324,209632,124402,797639,804808,720964,833707,579250,632814,18830,770379,886952,916915,554056,206694,371655,443127,771765,24851,670380,465498,996053,632708,867184,362244\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 997\n", + "Returned IDs: 369753,37639,784568,588446,62121,182165,450140,408339,940505,655568,618559,576985,949404,533672,961677,353622,780978,979427,95342,712387,548164,166392,185593,315231,379478,289778,379767,718291,632080,769082\n", + "Ground Truth: 369753,37639,784568,588446,62121,182165,450140,940505,408339,655568,618559,576985,949404,533672,961677,353622,780978,979427,95342,712387,548164,166392,185593,315231,798845,379478,289778,379767,718291,632080\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 999\n", + "Returned IDs: 71455,904930,201245,245045,818220,596858,531095,178358,412721,938312,802512,401110,938584,626132,988352,754497,558638,641329,26637,621622,961536,949777,420224,536828,229596,726269,35312,712316,544437,494823\n", + "Ground Truth: 71455,904930,201245,245045,818220,596858,531095,178358,412721,938312,802512,401110,938584,626132,988352,754497,558638,641329,26637,621622,961536,949777,420224,536828,229596,726269,35312,201261,712316,544437\n", + "Recall: 0.9667\n", + "\n", + "Total query IDs with recall < 1.0: 634\n", + "IDs: [np.int64(2), np.int64(6), np.int64(9), np.int64(10), np.int64(13), np.int64(14), np.int64(15), np.int64(17), np.int64(18), np.int64(19), np.int64(21), np.int64(22), np.int64(23), np.int64(24), np.int64(25), np.int64(26), np.int64(27), np.int64(28), np.int64(29), np.int64(30), np.int64(32), np.int64(33), np.int64(34), np.int64(38), np.int64(40), np.int64(41), np.int64(42), np.int64(45), np.int64(48), np.int64(49), np.int64(51), np.int64(52), np.int64(53), np.int64(54), np.int64(56), np.int64(57), np.int64(58), np.int64(59), np.int64(60), np.int64(61), np.int64(62), np.int64(65), np.int64(66), np.int64(69), np.int64(70), np.int64(71), np.int64(75), np.int64(76), np.int64(77), np.int64(79), np.int64(80), np.int64(81), np.int64(83), np.int64(84), np.int64(86), np.int64(87), np.int64(88), np.int64(89), np.int64(91), np.int64(92), np.int64(94), np.int64(95), np.int64(97), np.int64(101), np.int64(102), np.int64(103), np.int64(105), np.int64(106), np.int64(107), np.int64(108), np.int64(109), np.int64(110), np.int64(111), np.int64(112), np.int64(113), np.int64(115), np.int64(118), np.int64(119), np.int64(121), np.int64(122), np.int64(125), np.int64(127), np.int64(128), np.int64(129), np.int64(130), np.int64(131), np.int64(132), np.int64(133), np.int64(135), np.int64(136), np.int64(137), np.int64(138), np.int64(139), np.int64(140), np.int64(141), np.int64(142), np.int64(144), np.int64(148), np.int64(149), np.int64(154), np.int64(155), np.int64(156), np.int64(158), np.int64(161), np.int64(162), np.int64(163), np.int64(164), np.int64(165), np.int64(166), np.int64(168), np.int64(169), np.int64(171), np.int64(173), np.int64(174), np.int64(175), np.int64(176), np.int64(177), np.int64(179), np.int64(180), np.int64(181), np.int64(184), np.int64(185), np.int64(186), np.int64(187), np.int64(188), np.int64(189), np.int64(191), np.int64(192), np.int64(195), np.int64(198), np.int64(199), np.int64(201), np.int64(202), np.int64(203), np.int64(204), np.int64(206), np.int64(208), np.int64(210), np.int64(213), np.int64(214), np.int64(215), np.int64(216), np.int64(220), np.int64(221), np.int64(224), np.int64(225), np.int64(227), np.int64(230), np.int64(233), np.int64(234), np.int64(237), np.int64(241), np.int64(242), np.int64(244), np.int64(245), np.int64(246), np.int64(247), np.int64(249), np.int64(250), np.int64(251), np.int64(252), np.int64(253), np.int64(255), np.int64(257), np.int64(259), np.int64(261), np.int64(265), np.int64(266), np.int64(268), np.int64(269), np.int64(270), np.int64(271), np.int64(272), np.int64(275), np.int64(277), np.int64(278), np.int64(279), np.int64(281), np.int64(282), np.int64(284), np.int64(285), np.int64(287), np.int64(289), np.int64(290), np.int64(292), np.int64(295), np.int64(296), np.int64(297), np.int64(299), np.int64(301), np.int64(304), np.int64(305), np.int64(307), np.int64(308), np.int64(309), np.int64(311), np.int64(312), np.int64(315), np.int64(318), np.int64(319), np.int64(320), np.int64(321), np.int64(323), np.int64(324), np.int64(325), np.int64(326), np.int64(327), np.int64(329), np.int64(334), np.int64(338), np.int64(342), np.int64(343), np.int64(344), np.int64(345), np.int64(346), np.int64(347), np.int64(349), np.int64(351), np.int64(352), np.int64(354), np.int64(355), np.int64(356), np.int64(357), np.int64(358), np.int64(360), np.int64(366), np.int64(367), np.int64(368), np.int64(371), np.int64(372), np.int64(374), np.int64(379), np.int64(382), np.int64(384), np.int64(385), np.int64(387), np.int64(388), np.int64(389), np.int64(390), np.int64(392), np.int64(393), np.int64(394), np.int64(395), np.int64(396), np.int64(399), np.int64(400), np.int64(401), np.int64(403), np.int64(405), np.int64(408), np.int64(410), np.int64(411), np.int64(413), np.int64(419), np.int64(421), np.int64(422), np.int64(425), np.int64(426), np.int64(427), np.int64(430), np.int64(433), np.int64(434), np.int64(436), np.int64(437), np.int64(440), np.int64(442), np.int64(445), np.int64(446), np.int64(447), np.int64(448), np.int64(449), np.int64(450), np.int64(451), np.int64(452), np.int64(453), np.int64(455), np.int64(456), np.int64(457), np.int64(459), np.int64(460), np.int64(461), np.int64(462), np.int64(465), np.int64(466), np.int64(467), np.int64(468), np.int64(470), np.int64(471), np.int64(472), np.int64(473), np.int64(474), np.int64(476), np.int64(477), np.int64(478), np.int64(479), np.int64(480), np.int64(481), np.int64(482), np.int64(483), np.int64(484), np.int64(486), np.int64(487), np.int64(489), np.int64(492), np.int64(493), np.int64(494), np.int64(496), np.int64(497), np.int64(498), np.int64(499), np.int64(500), np.int64(501), np.int64(502), np.int64(503), np.int64(506), np.int64(507), np.int64(508), np.int64(509), np.int64(511), np.int64(513), np.int64(514), np.int64(515), np.int64(517), np.int64(518), np.int64(520), np.int64(521), np.int64(523), np.int64(524), np.int64(525), np.int64(531), np.int64(533), np.int64(534), np.int64(537), np.int64(539), np.int64(541), np.int64(542), np.int64(544), np.int64(547), np.int64(548), np.int64(549), np.int64(550), np.int64(551), np.int64(552), np.int64(556), np.int64(557), np.int64(560), np.int64(561), np.int64(563), np.int64(564), np.int64(568), np.int64(571), np.int64(572), np.int64(574), np.int64(575), np.int64(577), np.int64(578), np.int64(579), np.int64(580), np.int64(581), np.int64(583), np.int64(584), np.int64(585), np.int64(586), np.int64(588), np.int64(589), np.int64(593), np.int64(594), np.int64(595), np.int64(596), np.int64(597), np.int64(599), np.int64(600), np.int64(601), np.int64(603), np.int64(604), np.int64(606), np.int64(607), np.int64(608), np.int64(610), np.int64(613), np.int64(614), np.int64(616), np.int64(617), np.int64(618), np.int64(619), np.int64(621), np.int64(622), np.int64(623), np.int64(625), np.int64(628), np.int64(629), np.int64(630), np.int64(632), np.int64(633), np.int64(634), np.int64(636), np.int64(637), np.int64(639), np.int64(640), np.int64(641), np.int64(643), np.int64(644), np.int64(646), np.int64(647), np.int64(648), np.int64(650), np.int64(652), np.int64(654), np.int64(655), np.int64(658), np.int64(660), np.int64(661), np.int64(662), np.int64(663), np.int64(664), np.int64(665), np.int64(666), np.int64(667), np.int64(668), np.int64(670), np.int64(673), np.int64(674), np.int64(675), np.int64(676), np.int64(677), np.int64(678), np.int64(679), np.int64(680), np.int64(682), np.int64(683), np.int64(686), np.int64(687), np.int64(688), np.int64(691), np.int64(692), np.int64(693), np.int64(694), np.int64(697), np.int64(698), np.int64(699), np.int64(700), np.int64(704), np.int64(705), np.int64(706), np.int64(708), np.int64(710), np.int64(712), np.int64(713), np.int64(714), np.int64(717), np.int64(720), np.int64(723), np.int64(724), np.int64(725), np.int64(726), np.int64(728), np.int64(730), np.int64(731), np.int64(733), np.int64(734), np.int64(735), np.int64(737), np.int64(738), np.int64(739), np.int64(740), np.int64(741), np.int64(744), np.int64(745), np.int64(747), np.int64(748), np.int64(749), np.int64(751), np.int64(752), np.int64(754), np.int64(755), np.int64(756), np.int64(758), np.int64(759), np.int64(760), np.int64(761), np.int64(763), np.int64(767), np.int64(770), np.int64(771), np.int64(772), np.int64(775), np.int64(776), np.int64(779), np.int64(781), np.int64(783), np.int64(787), np.int64(788), np.int64(789), np.int64(790), np.int64(791), np.int64(793), np.int64(794), np.int64(795), np.int64(798), np.int64(799), np.int64(800), np.int64(802), np.int64(803), np.int64(805), np.int64(806), np.int64(807), np.int64(809), np.int64(811), np.int64(813), np.int64(815), np.int64(816), np.int64(817), np.int64(818), np.int64(819), np.int64(820), np.int64(821), np.int64(823), np.int64(824), np.int64(825), np.int64(826), np.int64(827), np.int64(828), np.int64(829), np.int64(832), np.int64(833), np.int64(834), np.int64(835), np.int64(836), np.int64(837), np.int64(839), np.int64(840), np.int64(841), np.int64(845), np.int64(846), np.int64(847), np.int64(849), np.int64(852), np.int64(853), np.int64(855), np.int64(856), np.int64(857), np.int64(860), np.int64(861), np.int64(862), np.int64(863), np.int64(866), np.int64(867), np.int64(868), np.int64(870), np.int64(871), np.int64(872), np.int64(874), np.int64(875), np.int64(876), np.int64(877), np.int64(882), np.int64(883), np.int64(884), np.int64(886), np.int64(887), np.int64(888), np.int64(889), np.int64(891), np.int64(892), np.int64(893), np.int64(894), np.int64(898), np.int64(899), np.int64(901), np.int64(902), np.int64(903), np.int64(904), np.int64(905), np.int64(906), np.int64(907), np.int64(910), np.int64(912), np.int64(914), np.int64(916), np.int64(917), np.int64(918), np.int64(919), np.int64(920), np.int64(923), np.int64(928), np.int64(930), np.int64(932), np.int64(933), np.int64(934), np.int64(935), np.int64(938), np.int64(939), np.int64(940), np.int64(941), np.int64(942), np.int64(943), np.int64(946), np.int64(947), np.int64(948), np.int64(949), np.int64(951), np.int64(953), np.int64(954), np.int64(955), np.int64(956), np.int64(959), np.int64(960), np.int64(963), np.int64(965), np.int64(966), np.int64(967), np.int64(968), np.int64(969), np.int64(970), np.int64(971), np.int64(973), np.int64(974), np.int64(975), np.int64(976), np.int64(978), np.int64(981), np.int64(983), np.int64(984), np.int64(985), np.int64(986), np.int64(988), np.int64(989), np.int64(990), np.int64(991), np.int64(992), np.int64(993), np.int64(994), np.int64(997), np.int64(999)]\n", + "Average Recall across all 1000 queries: 0.9268\n" + ] + } + ], "source": [ "#For querying\n", "import pyarrow.parquet as pq\n", @@ -199,22 +3470,49 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_labelfilter_0503_1_adup4',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 1000000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "index.describe()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Deleted vectors with label: label_50p\n", + "500000 vectors deleted\n" + ] + } + ], "source": [ "#For deleting\n", "# Delete by label\n", - "label_to_delete = \"label_1p\" #input\n", + "label_to_delete = \"label_50p\" #input\n", "\n", "result = index.delete_with_filter([{\"label\": {\"$eq\": label_to_delete}}])\n", "print(f\"Deleted vectors with label: {label_to_delete}\")\n", @@ -223,18 +3521,1045 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_labelfilter_reupsertcheck',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 1000000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "index.describe()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total vectors to insert: 500000\n", + "Upserted 501 vectors, total so far: 501\n", + "Upserted 513 vectors, total so far: 1014\n", + "Upserted 481 vectors, total so far: 1495\n", + "Upserted 513 vectors, total so far: 2008\n", + "Upserted 513 vectors, total so far: 2521\n", + "Upserted 520 vectors, total so far: 3041\n", + "Upserted 504 vectors, total so far: 3545\n", + "Upserted 510 vectors, total so far: 4055\n", + "Upserted 503 vectors, total so far: 4558\n", + "Upserted 480 vectors, total so far: 5038\n", + "Upserted 479 vectors, total so far: 5517\n", + "Upserted 478 vectors, total so far: 5995\n", + "Upserted 491 vectors, total so far: 6486\n", + "Upserted 535 vectors, total so far: 7021\n", + "Upserted 499 vectors, total so far: 7520\n", + "Upserted 477 vectors, total so far: 7997\n", + "Upserted 507 vectors, total so far: 8504\n", + "Upserted 486 vectors, total so far: 8990\n", + "Upserted 501 vectors, total so far: 9491\n", + "Upserted 488 vectors, total so far: 9979\n", + "Upserted 535 vectors, total so far: 10514\n", + "Upserted 514 vectors, total so far: 11028\n", + "Upserted 500 vectors, total so far: 11528\n", + "Upserted 488 vectors, total so far: 12016\n", + "Upserted 474 vectors, total so far: 12490\n", + "Upserted 486 vectors, total so far: 12976\n", + "Upserted 496 vectors, total so far: 13472\n", + "Upserted 489 vectors, total so far: 13961\n", + "Upserted 489 vectors, total so far: 14450\n", + "Upserted 482 vectors, total so far: 14932\n", + "Upserted 511 vectors, total so far: 15443\n", + "Upserted 494 vectors, total so far: 15937\n", + "Upserted 492 vectors, total so far: 16429\n", + "Upserted 507 vectors, total so far: 16936\n", + "Upserted 518 vectors, total so far: 17454\n", + "Upserted 492 vectors, total so far: 17946\n", + "Upserted 479 vectors, total so far: 18425\n", + "Upserted 505 vectors, total so far: 18930\n", + "Upserted 495 vectors, total so far: 19425\n", + "Upserted 509 vectors, total so far: 19934\n", + "Upserted 485 vectors, total so far: 20419\n", + "Upserted 517 vectors, total so far: 20936\n", + "Upserted 528 vectors, total so far: 21464\n", + "Upserted 499 vectors, total so far: 21963\n", + "Upserted 525 vectors, total so far: 22488\n", + "Upserted 494 vectors, total so far: 22982\n", + "Upserted 507 vectors, total so far: 23489\n", + "Upserted 521 vectors, total so far: 24010\n", + "Upserted 512 vectors, total so far: 24522\n", + "Upserted 508 vectors, total so far: 25030\n", + "Upserted 512 vectors, total so far: 25542\n", + "Upserted 497 vectors, total so far: 26039\n", + "Upserted 489 vectors, total so far: 26528\n", + "Upserted 481 vectors, total so far: 27009\n", + "Upserted 553 vectors, total so far: 27562\n", + "Upserted 512 vectors, total so far: 28074\n", + "Upserted 487 vectors, total so far: 28561\n", + "Upserted 495 vectors, total so far: 29056\n", + "Upserted 493 vectors, total so far: 29549\n", + "Upserted 502 vectors, total so far: 30051\n", + "Upserted 498 vectors, total so far: 30549\n", + "Upserted 515 vectors, total so far: 31064\n", + "Upserted 503 vectors, total so far: 31567\n", + "Upserted 491 vectors, total so far: 32058\n", + "Upserted 532 vectors, total so far: 32590\n", + "Upserted 477 vectors, total so far: 33067\n", + "Upserted 476 vectors, total so far: 33543\n", + "Upserted 501 vectors, total so far: 34044\n", + "Upserted 503 vectors, total so far: 34547\n", + "Upserted 475 vectors, total so far: 35022\n", + "Upserted 481 vectors, total so far: 35503\n", + "Upserted 522 vectors, total so far: 36025\n", + "Upserted 497 vectors, total so far: 36522\n", + "Upserted 496 vectors, total so far: 37018\n", + "Upserted 479 vectors, total so far: 37497\n", + "Upserted 510 vectors, total so far: 38007\n", + "Upserted 523 vectors, total so far: 38530\n", + "Upserted 488 vectors, total so far: 39018\n", + "Upserted 524 vectors, total so far: 39542\n", + "Upserted 506 vectors, total so far: 40048\n", + "Upserted 509 vectors, total so far: 40557\n", + "Upserted 506 vectors, total so far: 41063\n", + "Upserted 513 vectors, total so far: 41576\n", + "Upserted 480 vectors, total so far: 42056\n", + "Upserted 496 vectors, total so far: 42552\n", + "Upserted 474 vectors, total so far: 43026\n", + "Upserted 497 vectors, total so far: 43523\n", + "Upserted 488 vectors, total so far: 44011\n", + "Upserted 520 vectors, total so far: 44531\n", + "Upserted 485 vectors, total so far: 45016\n", + "Upserted 478 vectors, total so far: 45494\n", + "Upserted 489 vectors, total so far: 45983\n", + "Upserted 483 vectors, total so far: 46466\n", + "Upserted 489 vectors, total so far: 46955\n", + "Upserted 499 vectors, total so far: 47454\n", + "Upserted 494 vectors, total so far: 47948\n", + "Upserted 511 vectors, total so far: 48459\n", + "Upserted 507 vectors, total so far: 48966\n", + "Upserted 524 vectors, total so far: 49490\n", + "Upserted 472 vectors, total so far: 49962\n", + "Upserted 496 vectors, total so far: 50458\n", + "Upserted 477 vectors, total so far: 50935\n", + "Upserted 488 vectors, total so far: 51423\n", + "Upserted 502 vectors, total so far: 51925\n", + "Upserted 505 vectors, total so far: 52430\n", + "Upserted 486 vectors, total so far: 52916\n", + "Upserted 516 vectors, total so far: 53432\n", + "Upserted 517 vectors, total so far: 53949\n", + "Upserted 477 vectors, total so far: 54426\n", + "Upserted 500 vectors, total so far: 54926\n", + "Upserted 497 vectors, total so far: 55423\n", + "Upserted 493 vectors, total so far: 55916\n", + "Upserted 487 vectors, total so far: 56403\n", + "Upserted 501 vectors, total so far: 56904\n", + "Upserted 523 vectors, total so far: 57427\n", + "Upserted 511 vectors, total so far: 57938\n", + "Upserted 510 vectors, total so far: 58448\n", + "Upserted 484 vectors, total so far: 58932\n", + "Upserted 492 vectors, total so far: 59424\n", + "Upserted 475 vectors, total so far: 59899\n", + "Upserted 516 vectors, total so far: 60415\n", + "Upserted 511 vectors, total so far: 60926\n", + "Upserted 498 vectors, total so far: 61424\n", + "Upserted 503 vectors, total so far: 61927\n", + "Upserted 506 vectors, total so far: 62433\n", + "Upserted 501 vectors, total so far: 62934\n", + "Upserted 518 vectors, total so far: 63452\n", + "Upserted 505 vectors, total so far: 63957\n", + "Upserted 513 vectors, total so far: 64470\n", + "Upserted 502 vectors, total so far: 64972\n", + "Upserted 519 vectors, total so far: 65491\n", + "Upserted 482 vectors, total so far: 65973\n", + "Upserted 510 vectors, total so far: 66483\n", + "Upserted 484 vectors, total so far: 66967\n", + "Upserted 490 vectors, total so far: 67457\n", + "Upserted 490 vectors, total so far: 67947\n", + "Upserted 482 vectors, total so far: 68429\n", + "Upserted 496 vectors, total so far: 68925\n", + "Upserted 535 vectors, total so far: 69460\n", + "Upserted 494 vectors, total so far: 69954\n", + "Upserted 498 vectors, total so far: 70452\n", + "Upserted 493 vectors, total so far: 70945\n", + "Upserted 528 vectors, total so far: 71473\n", + "Upserted 519 vectors, total so far: 71992\n", + "Upserted 504 vectors, total so far: 72496\n", + "Upserted 507 vectors, total so far: 73003\n", + "Upserted 501 vectors, total so far: 73504\n", + "Upserted 490 vectors, total so far: 73994\n", + "Upserted 495 vectors, total so far: 74489\n", + "Upserted 503 vectors, total so far: 74992\n", + "Upserted 518 vectors, total so far: 75510\n", + "Upserted 536 vectors, total so far: 76046\n", + "Upserted 480 vectors, total so far: 76526\n", + "Upserted 454 vectors, total so far: 76980\n", + "Upserted 510 vectors, total so far: 77490\n", + "Upserted 496 vectors, total so far: 77986\n", + "Upserted 498 vectors, total so far: 78484\n", + "Upserted 496 vectors, total so far: 78980\n", + "Upserted 525 vectors, total so far: 79505\n", + "Upserted 506 vectors, total so far: 80011\n", + "Upserted 491 vectors, total so far: 80502\n", + "Upserted 511 vectors, total so far: 81013\n", + "Upserted 501 vectors, total so far: 81514\n", + "Upserted 516 vectors, total so far: 82030\n", + "Upserted 498 vectors, total so far: 82528\n", + "Upserted 512 vectors, total so far: 83040\n", + "Upserted 503 vectors, total so far: 83543\n", + "Upserted 471 vectors, total so far: 84014\n", + "Upserted 528 vectors, total so far: 84542\n", + "Upserted 480 vectors, total so far: 85022\n", + "Upserted 493 vectors, total so far: 85515\n", + "Upserted 478 vectors, total so far: 85993\n", + "Upserted 496 vectors, total so far: 86489\n", + "Upserted 465 vectors, total so far: 86954\n", + "Upserted 506 vectors, total so far: 87460\n", + "Upserted 513 vectors, total so far: 87973\n", + "Upserted 502 vectors, total so far: 88475\n", + "Upserted 534 vectors, total so far: 89009\n", + "Upserted 485 vectors, total so far: 89494\n", + "Upserted 494 vectors, total so far: 89988\n", + "Upserted 486 vectors, total so far: 90474\n", + "Upserted 473 vectors, total so far: 90947\n", + "Upserted 481 vectors, total so far: 91428\n", + "Upserted 512 vectors, total so far: 91940\n", + "Upserted 491 vectors, total so far: 92431\n", + "Upserted 509 vectors, total so far: 92940\n", + "Upserted 507 vectors, total so far: 93447\n", + "Upserted 511 vectors, total so far: 93958\n", + "Upserted 495 vectors, total so far: 94453\n", + "Upserted 463 vectors, total so far: 94916\n", + "Upserted 502 vectors, total so far: 95418\n", + "Upserted 518 vectors, total so far: 95936\n", + "Upserted 473 vectors, total so far: 96409\n", + "Upserted 498 vectors, total so far: 96907\n", + "Upserted 507 vectors, total so far: 97414\n", + "Upserted 497 vectors, total so far: 97911\n", + "Upserted 474 vectors, total so far: 98385\n", + "Upserted 471 vectors, total so far: 98856\n", + "Upserted 511 vectors, total so far: 99367\n", + "Upserted 522 vectors, total so far: 99889\n", + "Upserted 492 vectors, total so far: 100381\n", + "Upserted 514 vectors, total so far: 100895\n", + "Upserted 506 vectors, total so far: 101401\n", + "Upserted 508 vectors, total so far: 101909\n", + "Upserted 501 vectors, total so far: 102410\n", + "Upserted 488 vectors, total so far: 102898\n", + "Upserted 510 vectors, total so far: 103408\n", + "Upserted 502 vectors, total so far: 103910\n", + "Upserted 476 vectors, total so far: 104386\n", + "Upserted 472 vectors, total so far: 104858\n", + "Upserted 501 vectors, total so far: 105359\n", + "Upserted 487 vectors, total so far: 105846\n", + "Upserted 523 vectors, total so far: 106369\n", + "Upserted 496 vectors, total so far: 106865\n", + "Upserted 525 vectors, total so far: 107390\n", + "Upserted 508 vectors, total so far: 107898\n", + "Upserted 489 vectors, total so far: 108387\n", + "Upserted 498 vectors, total so far: 108885\n", + "Upserted 521 vectors, total so far: 109406\n", + "Upserted 492 vectors, total so far: 109898\n", + "Upserted 497 vectors, total so far: 110395\n", + "Upserted 486 vectors, total so far: 110881\n", + "Upserted 463 vectors, total so far: 111344\n", + "Upserted 499 vectors, total so far: 111843\n", + "Upserted 509 vectors, total so far: 112352\n", + "Upserted 502 vectors, total so far: 112854\n", + "Upserted 482 vectors, total so far: 113336\n", + "Upserted 490 vectors, total so far: 113826\n", + "Upserted 489 vectors, total so far: 114315\n", + "Upserted 519 vectors, total so far: 114834\n", + "Upserted 517 vectors, total so far: 115351\n", + "Upserted 502 vectors, total so far: 115853\n", + "Upserted 507 vectors, total so far: 116360\n", + "Upserted 480 vectors, total so far: 116840\n", + "Upserted 487 vectors, total so far: 117327\n", + "Upserted 502 vectors, total so far: 117829\n", + "Upserted 486 vectors, total so far: 118315\n", + "Upserted 510 vectors, total so far: 118825\n", + "Upserted 495 vectors, total so far: 119320\n", + "Upserted 546 vectors, total so far: 119866\n", + "Upserted 470 vectors, total so far: 120336\n", + "Upserted 483 vectors, total so far: 120819\n", + "Upserted 516 vectors, total so far: 121335\n", + "Upserted 518 vectors, total so far: 121853\n", + "Upserted 517 vectors, total so far: 122370\n", + "Upserted 499 vectors, total so far: 122869\n", + "Upserted 512 vectors, total so far: 123381\n", + "Upserted 476 vectors, total so far: 123857\n", + "Upserted 515 vectors, total so far: 124372\n", + "Upserted 518 vectors, total so far: 124890\n", + "Upserted 517 vectors, total so far: 125407\n", + "Upserted 527 vectors, total so far: 125934\n", + "Upserted 513 vectors, total so far: 126447\n", + "Upserted 506 vectors, total so far: 126953\n", + "Upserted 459 vectors, total so far: 127412\n", + "Upserted 506 vectors, total so far: 127918\n", + "Upserted 504 vectors, total so far: 128422\n", + "Upserted 492 vectors, total so far: 128914\n", + "Upserted 486 vectors, total so far: 129400\n", + "Upserted 486 vectors, total so far: 129886\n", + "Upserted 473 vectors, total so far: 130359\n", + "Upserted 468 vectors, total so far: 130827\n", + "Upserted 506 vectors, total so far: 131333\n", + "Upserted 489 vectors, total so far: 131822\n", + "Upserted 513 vectors, total so far: 132335\n", + "Upserted 500 vectors, total so far: 132835\n", + "Upserted 512 vectors, total so far: 133347\n", + "Upserted 525 vectors, total so far: 133872\n", + "Upserted 503 vectors, total so far: 134375\n", + "Upserted 465 vectors, total so far: 134840\n", + "Upserted 502 vectors, total so far: 135342\n", + "Upserted 511 vectors, total so far: 135853\n", + "Upserted 493 vectors, total so far: 136346\n", + "Upserted 520 vectors, total so far: 136866\n", + "Upserted 498 vectors, total so far: 137364\n", + "Upserted 485 vectors, total so far: 137849\n", + "Upserted 482 vectors, total so far: 138331\n", + "Upserted 501 vectors, total so far: 138832\n", + "Upserted 502 vectors, total so far: 139334\n", + "Upserted 504 vectors, total so far: 139838\n", + "Upserted 493 vectors, total so far: 140331\n", + "Upserted 516 vectors, total so far: 140847\n", + "Upserted 499 vectors, total so far: 141346\n", + "Upserted 515 vectors, total so far: 141861\n", + "Upserted 522 vectors, total so far: 142383\n", + "Upserted 497 vectors, total so far: 142880\n", + "Upserted 510 vectors, total so far: 143390\n", + "Upserted 512 vectors, total so far: 143902\n", + "Upserted 494 vectors, total so far: 144396\n", + "Upserted 514 vectors, total so far: 144910\n", + "Upserted 545 vectors, total so far: 145455\n", + "Upserted 486 vectors, total so far: 145941\n", + "Upserted 499 vectors, total so far: 146440\n", + "Upserted 479 vectors, total so far: 146919\n", + "Upserted 524 vectors, total so far: 147443\n", + "Upserted 517 vectors, total so far: 147960\n", + "Upserted 506 vectors, total so far: 148466\n", + "Upserted 525 vectors, total so far: 148991\n", + "Upserted 527 vectors, total so far: 149518\n", + "Upserted 511 vectors, total so far: 150029\n", + "Upserted 485 vectors, total so far: 150514\n", + "Upserted 524 vectors, total so far: 151038\n", + "Upserted 494 vectors, total so far: 151532\n", + "Upserted 480 vectors, total so far: 152012\n", + "Upserted 491 vectors, total so far: 152503\n", + "Upserted 506 vectors, total so far: 153009\n", + "Upserted 497 vectors, total so far: 153506\n", + "Upserted 486 vectors, total so far: 153992\n", + "Upserted 497 vectors, total so far: 154489\n", + "Upserted 516 vectors, total so far: 155005\n", + "Upserted 516 vectors, total so far: 155521\n", + "Upserted 490 vectors, total so far: 156011\n", + "Upserted 516 vectors, total so far: 156527\n", + "Upserted 480 vectors, total so far: 157007\n", + "Upserted 489 vectors, total so far: 157496\n", + "Upserted 489 vectors, total so far: 157985\n", + "Upserted 502 vectors, total so far: 158487\n", + "Upserted 505 vectors, total so far: 158992\n", + "Upserted 501 vectors, total so far: 159493\n", + "Upserted 531 vectors, total so far: 160024\n", + "Upserted 528 vectors, total so far: 160552\n", + "Upserted 510 vectors, total so far: 161062\n", + "Upserted 481 vectors, total so far: 161543\n", + "Upserted 521 vectors, total so far: 162064\n", + "Upserted 520 vectors, total so far: 162584\n", + "Upserted 538 vectors, total so far: 163122\n", + "Upserted 504 vectors, total so far: 163626\n", + "Upserted 509 vectors, total so far: 164135\n", + "Upserted 473 vectors, total so far: 164608\n", + "Upserted 511 vectors, total so far: 165119\n", + "Upserted 521 vectors, total so far: 165640\n", + "Upserted 522 vectors, total so far: 166162\n", + "Upserted 511 vectors, total so far: 166673\n", + "Upserted 497 vectors, total so far: 167170\n", + "Upserted 511 vectors, total so far: 167681\n", + "Upserted 512 vectors, total so far: 168193\n", + "Upserted 487 vectors, total so far: 168680\n", + "Upserted 509 vectors, total so far: 169189\n", + "Upserted 461 vectors, total so far: 169650\n", + "Upserted 515 vectors, total so far: 170165\n", + "Upserted 486 vectors, total so far: 170651\n", + "Upserted 490 vectors, total so far: 171141\n", + "Upserted 510 vectors, total so far: 171651\n", + "Upserted 487 vectors, total so far: 172138\n", + "Upserted 517 vectors, total so far: 172655\n", + "Upserted 486 vectors, total so far: 173141\n", + "Upserted 505 vectors, total so far: 173646\n", + "Upserted 498 vectors, total so far: 174144\n", + "Upserted 496 vectors, total so far: 174640\n", + "Upserted 514 vectors, total so far: 175154\n", + "Upserted 518 vectors, total so far: 175672\n", + "Upserted 523 vectors, total so far: 176195\n", + "Upserted 519 vectors, total so far: 176714\n", + "Upserted 511 vectors, total so far: 177225\n", + "Upserted 489 vectors, total so far: 177714\n", + "Upserted 502 vectors, total so far: 178216\n", + "Upserted 517 vectors, total so far: 178733\n", + "Upserted 498 vectors, total so far: 179231\n", + "Upserted 519 vectors, total so far: 179750\n", + "Upserted 509 vectors, total so far: 180259\n", + "Upserted 504 vectors, total so far: 180763\n", + "Upserted 487 vectors, total so far: 181250\n", + "Upserted 504 vectors, total so far: 181754\n", + "Upserted 509 vectors, total so far: 182263\n", + "Upserted 485 vectors, total so far: 182748\n", + "Upserted 504 vectors, total so far: 183252\n", + "Upserted 494 vectors, total so far: 183746\n", + "Upserted 511 vectors, total so far: 184257\n", + "Upserted 494 vectors, total so far: 184751\n", + "Upserted 482 vectors, total so far: 185233\n", + "Upserted 489 vectors, total so far: 185722\n", + "Upserted 472 vectors, total so far: 186194\n", + "Upserted 531 vectors, total so far: 186725\n", + "Upserted 487 vectors, total so far: 187212\n", + "Upserted 502 vectors, total so far: 187714\n", + "Upserted 489 vectors, total so far: 188203\n", + "Upserted 521 vectors, total so far: 188724\n", + "Upserted 503 vectors, total so far: 189227\n", + "Upserted 496 vectors, total so far: 189723\n", + "Upserted 505 vectors, total so far: 190228\n", + "Upserted 509 vectors, total so far: 190737\n", + "Upserted 485 vectors, total so far: 191222\n", + "Upserted 501 vectors, total so far: 191723\n", + "Upserted 482 vectors, total so far: 192205\n", + "Upserted 501 vectors, total so far: 192706\n", + "Upserted 485 vectors, total so far: 193191\n", + "Upserted 525 vectors, total so far: 193716\n", + "Upserted 482 vectors, total so far: 194198\n", + "Upserted 500 vectors, total so far: 194698\n", + "Upserted 515 vectors, total so far: 195213\n", + "Upserted 516 vectors, total so far: 195729\n", + "Upserted 517 vectors, total so far: 196246\n", + "Upserted 511 vectors, total so far: 196757\n", + "Upserted 492 vectors, total so far: 197249\n", + "Upserted 496 vectors, total so far: 197745\n", + "Upserted 501 vectors, total so far: 198246\n", + "Upserted 510 vectors, total so far: 198756\n", + "Upserted 516 vectors, total so far: 199272\n", + "Upserted 505 vectors, total so far: 199777\n", + "Upserted 479 vectors, total so far: 200256\n", + "Upserted 490 vectors, total so far: 200746\n", + "Upserted 492 vectors, total so far: 201238\n", + "Upserted 483 vectors, total so far: 201721\n", + "Upserted 485 vectors, total so far: 202206\n", + "Upserted 492 vectors, total so far: 202698\n", + "Upserted 491 vectors, total so far: 203189\n", + "Upserted 485 vectors, total so far: 203674\n", + "Upserted 499 vectors, total so far: 204173\n", + "Upserted 485 vectors, total so far: 204658\n", + "Upserted 498 vectors, total so far: 205156\n", + "Upserted 509 vectors, total so far: 205665\n", + "Upserted 508 vectors, total so far: 206173\n", + "Upserted 479 vectors, total so far: 206652\n", + "Upserted 515 vectors, total so far: 207167\n", + "Upserted 512 vectors, total so far: 207679\n", + "Upserted 506 vectors, total so far: 208185\n", + "Upserted 476 vectors, total so far: 208661\n", + "Upserted 477 vectors, total so far: 209138\n", + "Upserted 506 vectors, total so far: 209644\n", + "Upserted 500 vectors, total so far: 210144\n", + "Upserted 508 vectors, total so far: 210652\n", + "Upserted 511 vectors, total so far: 211163\n", + "Upserted 485 vectors, total so far: 211648\n", + "Upserted 520 vectors, total so far: 212168\n", + "Upserted 502 vectors, total so far: 212670\n", + "Upserted 498 vectors, total so far: 213168\n", + "Upserted 495 vectors, total so far: 213663\n", + "Upserted 511 vectors, total so far: 214174\n", + "Upserted 480 vectors, total so far: 214654\n", + "Upserted 522 vectors, total so far: 215176\n", + "Upserted 512 vectors, total so far: 215688\n", + "Upserted 492 vectors, total so far: 216180\n", + "Upserted 482 vectors, total so far: 216662\n", + "Upserted 511 vectors, total so far: 217173\n", + "Upserted 493 vectors, total so far: 217666\n", + "Upserted 512 vectors, total so far: 218178\n", + "Upserted 499 vectors, total so far: 218677\n", + "Upserted 531 vectors, total so far: 219208\n", + "Upserted 511 vectors, total so far: 219719\n", + "Upserted 503 vectors, total so far: 220222\n", + "Upserted 488 vectors, total so far: 220710\n", + "Upserted 497 vectors, total so far: 221207\n", + "Upserted 499 vectors, total so far: 221706\n", + "Upserted 504 vectors, total so far: 222210\n", + "Upserted 501 vectors, total so far: 222711\n", + "Upserted 500 vectors, total so far: 223211\n", + "Upserted 533 vectors, total so far: 223744\n", + "Upserted 486 vectors, total so far: 224230\n", + "Upserted 479 vectors, total so far: 224709\n", + "Upserted 518 vectors, total so far: 225227\n", + "Upserted 509 vectors, total so far: 225736\n", + "Upserted 492 vectors, total so far: 226228\n", + "Upserted 497 vectors, total so far: 226725\n", + "Upserted 499 vectors, total so far: 227224\n", + "Upserted 492 vectors, total so far: 227716\n", + "Upserted 487 vectors, total so far: 228203\n", + "Upserted 494 vectors, total so far: 228697\n", + "Upserted 503 vectors, total so far: 229200\n", + "Upserted 486 vectors, total so far: 229686\n", + "Upserted 485 vectors, total so far: 230171\n", + "Upserted 482 vectors, total so far: 230653\n", + "Upserted 508 vectors, total so far: 231161\n", + "Upserted 502 vectors, total so far: 231663\n", + "Upserted 471 vectors, total so far: 232134\n", + "Upserted 532 vectors, total so far: 232666\n", + "Upserted 510 vectors, total so far: 233176\n", + "Upserted 512 vectors, total so far: 233688\n", + "Upserted 508 vectors, total so far: 234196\n", + "Upserted 494 vectors, total so far: 234690\n", + "Upserted 463 vectors, total so far: 235153\n", + "Upserted 525 vectors, total so far: 235678\n", + "Upserted 516 vectors, total so far: 236194\n", + "Upserted 507 vectors, total so far: 236701\n", + "Upserted 486 vectors, total so far: 237187\n", + "Upserted 515 vectors, total so far: 237702\n", + "Upserted 499 vectors, total so far: 238201\n", + "Upserted 501 vectors, total so far: 238702\n", + "Upserted 497 vectors, total so far: 239199\n", + "Upserted 489 vectors, total so far: 239688\n", + "Upserted 499 vectors, total so far: 240187\n", + "Upserted 515 vectors, total so far: 240702\n", + "Upserted 484 vectors, total so far: 241186\n", + "Upserted 541 vectors, total so far: 241727\n", + "Upserted 492 vectors, total so far: 242219\n", + "Upserted 522 vectors, total so far: 242741\n", + "Upserted 497 vectors, total so far: 243238\n", + "Upserted 495 vectors, total so far: 243733\n", + "Upserted 508 vectors, total so far: 244241\n", + "Upserted 458 vectors, total so far: 244699\n", + "Upserted 473 vectors, total so far: 245172\n", + "Upserted 499 vectors, total so far: 245671\n", + "Upserted 515 vectors, total so far: 246186\n", + "Upserted 514 vectors, total so far: 246700\n", + "Upserted 493 vectors, total so far: 247193\n", + "Upserted 503 vectors, total so far: 247696\n", + "Upserted 512 vectors, total so far: 248208\n", + "Upserted 496 vectors, total so far: 248704\n", + "Upserted 493 vectors, total so far: 249197\n", + "Upserted 519 vectors, total so far: 249716\n", + "Upserted 481 vectors, total so far: 250197\n", + "Upserted 508 vectors, total so far: 250705\n", + "Upserted 503 vectors, total so far: 251208\n", + "Upserted 489 vectors, total so far: 251697\n", + "Upserted 499 vectors, total so far: 252196\n", + "Upserted 504 vectors, total so far: 252700\n", + "Upserted 509 vectors, total so far: 253209\n", + "Upserted 502 vectors, total so far: 253711\n", + "Upserted 489 vectors, total so far: 254200\n", + "Upserted 529 vectors, total so far: 254729\n", + "Upserted 480 vectors, total so far: 255209\n", + "Upserted 497 vectors, total so far: 255706\n", + "Upserted 504 vectors, total so far: 256210\n", + "Upserted 488 vectors, total so far: 256698\n", + "Upserted 476 vectors, total so far: 257174\n", + "Upserted 514 vectors, total so far: 257688\n", + "Upserted 493 vectors, total so far: 258181\n", + "Upserted 494 vectors, total so far: 258675\n", + "Upserted 513 vectors, total so far: 259188\n", + "Upserted 483 vectors, total so far: 259671\n", + "Upserted 524 vectors, total so far: 260195\n", + "Upserted 506 vectors, total so far: 260701\n", + "Upserted 502 vectors, total so far: 261203\n", + "Upserted 502 vectors, total so far: 261705\n", + "Upserted 507 vectors, total so far: 262212\n", + "Upserted 469 vectors, total so far: 262681\n", + "Upserted 495 vectors, total so far: 263176\n", + "Upserted 476 vectors, total so far: 263652\n", + "Upserted 504 vectors, total so far: 264156\n", + "Upserted 495 vectors, total so far: 264651\n", + "Upserted 481 vectors, total so far: 265132\n", + "Upserted 517 vectors, total so far: 265649\n", + "Upserted 485 vectors, total so far: 266134\n", + "Upserted 489 vectors, total so far: 266623\n", + "Upserted 484 vectors, total so far: 267107\n", + "Upserted 501 vectors, total so far: 267608\n", + "Upserted 478 vectors, total so far: 268086\n", + "Upserted 486 vectors, total so far: 268572\n", + "Upserted 495 vectors, total so far: 269067\n", + "Upserted 527 vectors, total so far: 269594\n", + "Upserted 512 vectors, total so far: 270106\n", + "Upserted 507 vectors, total so far: 270613\n", + "Upserted 471 vectors, total so far: 271084\n", + "Upserted 507 vectors, total so far: 271591\n", + "Upserted 529 vectors, total so far: 272120\n", + "Upserted 509 vectors, total so far: 272629\n", + "Upserted 486 vectors, total so far: 273115\n", + "Upserted 487 vectors, total so far: 273602\n", + "Upserted 511 vectors, total so far: 274113\n", + "Upserted 479 vectors, total so far: 274592\n", + "Upserted 504 vectors, total so far: 275096\n", + "Upserted 525 vectors, total so far: 275621\n", + "Upserted 504 vectors, total so far: 276125\n", + "Upserted 502 vectors, total so far: 276627\n", + "Upserted 495 vectors, total so far: 277122\n", + "Upserted 488 vectors, total so far: 277610\n", + "Upserted 512 vectors, total so far: 278122\n", + "Upserted 489 vectors, total so far: 278611\n", + "Upserted 519 vectors, total so far: 279130\n", + "Upserted 507 vectors, total so far: 279637\n", + "Upserted 499 vectors, total so far: 280136\n", + "Upserted 489 vectors, total so far: 280625\n", + "Upserted 494 vectors, total so far: 281119\n", + "Upserted 497 vectors, total so far: 281616\n", + "Upserted 501 vectors, total so far: 282117\n", + "Upserted 479 vectors, total so far: 282596\n", + "Upserted 525 vectors, total so far: 283121\n", + "Upserted 503 vectors, total so far: 283624\n", + "Upserted 494 vectors, total so far: 284118\n", + "Upserted 504 vectors, total so far: 284622\n", + "Upserted 520 vectors, total so far: 285142\n", + "Upserted 485 vectors, total so far: 285627\n", + "Upserted 497 vectors, total so far: 286124\n", + "Upserted 489 vectors, total so far: 286613\n", + "Upserted 515 vectors, total so far: 287128\n", + "Upserted 498 vectors, total so far: 287626\n", + "Upserted 499 vectors, total so far: 288125\n", + "Upserted 466 vectors, total so far: 288591\n", + "Upserted 491 vectors, total so far: 289082\n", + "Upserted 494 vectors, total so far: 289576\n", + "Upserted 512 vectors, total so far: 290088\n", + "Upserted 499 vectors, total so far: 290587\n", + "Upserted 497 vectors, total so far: 291084\n", + "Upserted 495 vectors, total so far: 291579\n", + "Upserted 516 vectors, total so far: 292095\n", + "Upserted 510 vectors, total so far: 292605\n", + "Upserted 504 vectors, total so far: 293109\n", + "Upserted 511 vectors, total so far: 293620\n", + "Upserted 484 vectors, total so far: 294104\n", + "Upserted 519 vectors, total so far: 294623\n", + "Upserted 511 vectors, total so far: 295134\n", + "Upserted 474 vectors, total so far: 295608\n", + "Upserted 509 vectors, total so far: 296117\n", + "Upserted 491 vectors, total so far: 296608\n", + "Upserted 477 vectors, total so far: 297085\n", + "Upserted 499 vectors, total so far: 297584\n", + "Upserted 533 vectors, total so far: 298117\n", + "Upserted 494 vectors, total so far: 298611\n", + "Upserted 516 vectors, total so far: 299127\n", + "Upserted 492 vectors, total so far: 299619\n", + "Upserted 487 vectors, total so far: 300106\n", + "Upserted 514 vectors, total so far: 300620\n", + "Upserted 506 vectors, total so far: 301126\n", + "Upserted 513 vectors, total so far: 301639\n", + "Upserted 494 vectors, total so far: 302133\n", + "Upserted 513 vectors, total so far: 302646\n", + "Upserted 533 vectors, total so far: 303179\n", + "Upserted 486 vectors, total so far: 303665\n", + "Upserted 500 vectors, total so far: 304165\n", + "Upserted 479 vectors, total so far: 304644\n", + "Upserted 503 vectors, total so far: 305147\n", + "Upserted 487 vectors, total so far: 305634\n", + "Upserted 488 vectors, total so far: 306122\n", + "Upserted 532 vectors, total so far: 306654\n", + "Upserted 521 vectors, total so far: 307175\n", + "Upserted 504 vectors, total so far: 307679\n", + "Upserted 467 vectors, total so far: 308146\n", + "Upserted 504 vectors, total so far: 308650\n", + "Upserted 503 vectors, total so far: 309153\n", + "Upserted 482 vectors, total so far: 309635\n", + "Upserted 500 vectors, total so far: 310135\n", + "Upserted 497 vectors, total so far: 310632\n", + "Upserted 494 vectors, total so far: 311126\n", + "Upserted 530 vectors, total so far: 311656\n", + "Upserted 508 vectors, total so far: 312164\n", + "Upserted 462 vectors, total so far: 312626\n", + "Upserted 483 vectors, total so far: 313109\n", + "Upserted 489 vectors, total so far: 313598\n", + "Upserted 525 vectors, total so far: 314123\n", + "Upserted 514 vectors, total so far: 314637\n", + "Upserted 471 vectors, total so far: 315108\n", + "Upserted 502 vectors, total so far: 315610\n", + "Upserted 462 vectors, total so far: 316072\n", + "Upserted 515 vectors, total so far: 316587\n", + "Upserted 510 vectors, total so far: 317097\n", + "Upserted 517 vectors, total so far: 317614\n", + "Upserted 509 vectors, total so far: 318123\n", + "Upserted 507 vectors, total so far: 318630\n", + "Upserted 510 vectors, total so far: 319140\n", + "Upserted 494 vectors, total so far: 319634\n", + "Upserted 478 vectors, total so far: 320112\n", + "Upserted 493 vectors, total so far: 320605\n", + "Upserted 495 vectors, total so far: 321100\n", + "Upserted 507 vectors, total so far: 321607\n", + "Upserted 500 vectors, total so far: 322107\n", + "Upserted 540 vectors, total so far: 322647\n", + "Upserted 505 vectors, total so far: 323152\n", + "Upserted 510 vectors, total so far: 323662\n", + "Upserted 473 vectors, total so far: 324135\n", + "Upserted 475 vectors, total so far: 324610\n", + "Upserted 494 vectors, total so far: 325104\n", + "Upserted 487 vectors, total so far: 325591\n", + "Upserted 488 vectors, total so far: 326079\n", + "Upserted 505 vectors, total so far: 326584\n", + "Upserted 515 vectors, total so far: 327099\n", + "Upserted 507 vectors, total so far: 327606\n", + "Upserted 478 vectors, total so far: 328084\n", + "Upserted 512 vectors, total so far: 328596\n", + "Upserted 505 vectors, total so far: 329101\n", + "Upserted 504 vectors, total so far: 329605\n", + "Upserted 534 vectors, total so far: 330139\n", + "Upserted 497 vectors, total so far: 330636\n", + "Upserted 487 vectors, total so far: 331123\n", + "Upserted 517 vectors, total so far: 331640\n", + "Upserted 504 vectors, total so far: 332144\n", + "Upserted 498 vectors, total so far: 332642\n", + "Upserted 478 vectors, total so far: 333120\n", + "Upserted 485 vectors, total so far: 333605\n", + "Upserted 498 vectors, total so far: 334103\n", + "Upserted 498 vectors, total so far: 334601\n", + "Upserted 525 vectors, total so far: 335126\n", + "Upserted 507 vectors, total so far: 335633\n", + "Upserted 506 vectors, total so far: 336139\n", + "Upserted 508 vectors, total so far: 336647\n", + "Upserted 520 vectors, total so far: 337167\n", + "Upserted 480 vectors, total so far: 337647\n", + "Upserted 506 vectors, total so far: 338153\n", + "Upserted 489 vectors, total so far: 338642\n", + "Upserted 489 vectors, total so far: 339131\n", + "Upserted 470 vectors, total so far: 339601\n", + "Upserted 482 vectors, total so far: 340083\n", + "Upserted 500 vectors, total so far: 340583\n", + "Upserted 534 vectors, total so far: 341117\n", + "Upserted 496 vectors, total so far: 341613\n", + "Upserted 495 vectors, total so far: 342108\n", + "Upserted 486 vectors, total so far: 342594\n", + "Upserted 506 vectors, total so far: 343100\n", + "Upserted 476 vectors, total so far: 343576\n", + "Upserted 492 vectors, total so far: 344068\n", + "Upserted 485 vectors, total so far: 344553\n", + "Upserted 506 vectors, total so far: 345059\n", + "Upserted 504 vectors, total so far: 345563\n", + "Upserted 502 vectors, total so far: 346065\n", + "Upserted 489 vectors, total so far: 346554\n", + "Upserted 487 vectors, total so far: 347041\n", + "Upserted 540 vectors, total so far: 347581\n", + "Upserted 476 vectors, total so far: 348057\n", + "Upserted 485 vectors, total so far: 348542\n", + "Upserted 497 vectors, total so far: 349039\n", + "Upserted 496 vectors, total so far: 349535\n", + "Upserted 494 vectors, total so far: 350029\n", + "Upserted 519 vectors, total so far: 350548\n", + "Upserted 510 vectors, total so far: 351058\n", + "Upserted 500 vectors, total so far: 351558\n", + "Upserted 488 vectors, total so far: 352046\n", + "Upserted 539 vectors, total so far: 352585\n", + "Upserted 467 vectors, total so far: 353052\n", + "Upserted 486 vectors, total so far: 353538\n", + "Upserted 498 vectors, total so far: 354036\n", + "Upserted 508 vectors, total so far: 354544\n", + "Upserted 514 vectors, total so far: 355058\n", + "Upserted 509 vectors, total so far: 355567\n", + "Upserted 508 vectors, total so far: 356075\n", + "Upserted 523 vectors, total so far: 356598\n", + "Upserted 509 vectors, total so far: 357107\n", + "Upserted 494 vectors, total so far: 357601\n", + "Upserted 485 vectors, total so far: 358086\n", + "Upserted 490 vectors, total so far: 358576\n", + "Upserted 473 vectors, total so far: 359049\n", + "Upserted 479 vectors, total so far: 359528\n", + "Upserted 521 vectors, total so far: 360049\n", + "Upserted 497 vectors, total so far: 360546\n", + "Upserted 500 vectors, total so far: 361046\n", + "Upserted 505 vectors, total so far: 361551\n", + "Upserted 493 vectors, total so far: 362044\n", + "Upserted 488 vectors, total so far: 362532\n", + "Upserted 512 vectors, total so far: 363044\n", + "Upserted 491 vectors, total so far: 363535\n", + "Upserted 513 vectors, total so far: 364048\n", + "Upserted 483 vectors, total so far: 364531\n", + "Upserted 478 vectors, total so far: 365009\n", + "Upserted 493 vectors, total so far: 365502\n", + "Upserted 486 vectors, total so far: 365988\n", + "Upserted 521 vectors, total so far: 366509\n", + "Upserted 497 vectors, total so far: 367006\n", + "Upserted 495 vectors, total so far: 367501\n", + "Upserted 511 vectors, total so far: 368012\n", + "Upserted 497 vectors, total so far: 368509\n", + "Upserted 483 vectors, total so far: 368992\n", + "Upserted 512 vectors, total so far: 369504\n", + "Upserted 514 vectors, total so far: 370018\n", + "Upserted 491 vectors, total so far: 370509\n", + "Upserted 511 vectors, total so far: 371020\n", + "Upserted 502 vectors, total so far: 371522\n", + "Upserted 511 vectors, total so far: 372033\n", + "Upserted 474 vectors, total so far: 372507\n", + "Upserted 507 vectors, total so far: 373014\n", + "Upserted 515 vectors, total so far: 373529\n", + "Upserted 514 vectors, total so far: 374043\n", + "Upserted 508 vectors, total so far: 374551\n", + "Upserted 487 vectors, total so far: 375038\n", + "Upserted 496 vectors, total so far: 375534\n", + "Upserted 505 vectors, total so far: 376039\n", + "Upserted 506 vectors, total so far: 376545\n", + "Upserted 523 vectors, total so far: 377068\n", + "Upserted 503 vectors, total so far: 377571\n", + "Upserted 483 vectors, total so far: 378054\n", + "Upserted 501 vectors, total so far: 378555\n", + "Upserted 493 vectors, total so far: 379048\n", + "Upserted 495 vectors, total so far: 379543\n", + "Upserted 471 vectors, total so far: 380014\n", + "Upserted 499 vectors, total so far: 380513\n", + "Upserted 514 vectors, total so far: 381027\n", + "Upserted 461 vectors, total so far: 381488\n", + "Upserted 509 vectors, total so far: 381997\n", + "Upserted 481 vectors, total so far: 382478\n", + "Upserted 510 vectors, total so far: 382988\n", + "Upserted 502 vectors, total so far: 383490\n", + "Upserted 503 vectors, total so far: 383993\n", + "Upserted 516 vectors, total so far: 384509\n", + "Upserted 492 vectors, total so far: 385001\n", + "Upserted 485 vectors, total so far: 385486\n", + "Upserted 478 vectors, total so far: 385964\n", + "Upserted 488 vectors, total so far: 386452\n", + "Upserted 485 vectors, total so far: 386937\n", + "Upserted 522 vectors, total so far: 387459\n", + "Upserted 509 vectors, total so far: 387968\n", + "Upserted 482 vectors, total so far: 388450\n", + "Upserted 490 vectors, total so far: 388940\n", + "Upserted 519 vectors, total so far: 389459\n", + "Upserted 507 vectors, total so far: 389966\n", + "Upserted 498 vectors, total so far: 390464\n", + "Upserted 496 vectors, total so far: 390960\n", + "Upserted 490 vectors, total so far: 391450\n", + "Upserted 496 vectors, total so far: 391946\n", + "Upserted 499 vectors, total so far: 392445\n", + "Upserted 481 vectors, total so far: 392926\n", + "Upserted 510 vectors, total so far: 393436\n", + "Upserted 535 vectors, total so far: 393971\n", + "Upserted 505 vectors, total so far: 394476\n", + "Upserted 506 vectors, total so far: 394982\n", + "Upserted 488 vectors, total so far: 395470\n", + "Upserted 528 vectors, total so far: 395998\n", + "Upserted 485 vectors, total so far: 396483\n", + "Upserted 479 vectors, total so far: 396962\n", + "Upserted 519 vectors, total so far: 397481\n", + "Upserted 515 vectors, total so far: 397996\n", + "Upserted 521 vectors, total so far: 398517\n", + "Upserted 496 vectors, total so far: 399013\n", + "Upserted 529 vectors, total so far: 399542\n", + "Upserted 500 vectors, total so far: 400042\n", + "Upserted 501 vectors, total so far: 400543\n", + "Upserted 481 vectors, total so far: 401024\n", + "Upserted 487 vectors, total so far: 401511\n", + "Upserted 493 vectors, total so far: 402004\n", + "Upserted 519 vectors, total so far: 402523\n", + "Upserted 490 vectors, total so far: 403013\n", + "Upserted 520 vectors, total so far: 403533\n", + "Upserted 493 vectors, total so far: 404026\n", + "Upserted 503 vectors, total so far: 404529\n", + "Upserted 482 vectors, total so far: 405011\n", + "Upserted 528 vectors, total so far: 405539\n", + "Upserted 480 vectors, total so far: 406019\n", + "Upserted 503 vectors, total so far: 406522\n", + "Upserted 512 vectors, total so far: 407034\n", + "Upserted 485 vectors, total so far: 407519\n", + "Upserted 497 vectors, total so far: 408016\n", + "Upserted 497 vectors, total so far: 408513\n", + "Upserted 497 vectors, total so far: 409010\n", + "Upserted 497 vectors, total so far: 409507\n", + "Upserted 480 vectors, total so far: 409987\n", + "Upserted 489 vectors, total so far: 410476\n", + "Upserted 486 vectors, total so far: 410962\n", + "Upserted 488 vectors, total so far: 411450\n", + "Upserted 501 vectors, total so far: 411951\n", + "Upserted 489 vectors, total so far: 412440\n", + "Upserted 506 vectors, total so far: 412946\n", + "Upserted 510 vectors, total so far: 413456\n", + "Upserted 514 vectors, total so far: 413970\n", + "Upserted 475 vectors, total so far: 414445\n", + "Upserted 500 vectors, total so far: 414945\n", + "Upserted 494 vectors, total so far: 415439\n", + "Upserted 495 vectors, total so far: 415934\n", + "Upserted 494 vectors, total so far: 416428\n", + "Upserted 490 vectors, total so far: 416918\n", + "Upserted 532 vectors, total so far: 417450\n", + "Upserted 484 vectors, total so far: 417934\n", + "Upserted 473 vectors, total so far: 418407\n", + "Upserted 502 vectors, total so far: 418909\n", + "Upserted 502 vectors, total so far: 419411\n", + "Upserted 492 vectors, total so far: 419903\n", + "Upserted 505 vectors, total so far: 420408\n", + "Upserted 494 vectors, total so far: 420902\n", + "Upserted 523 vectors, total so far: 421425\n", + "Upserted 511 vectors, total so far: 421936\n", + "Upserted 493 vectors, total so far: 422429\n", + "Upserted 514 vectors, total so far: 422943\n", + "Upserted 476 vectors, total so far: 423419\n", + "Upserted 526 vectors, total so far: 423945\n", + "Upserted 500 vectors, total so far: 424445\n", + "Upserted 517 vectors, total so far: 424962\n", + "Upserted 530 vectors, total so far: 425492\n", + "Upserted 506 vectors, total so far: 425998\n", + "Upserted 506 vectors, total so far: 426504\n", + "Upserted 505 vectors, total so far: 427009\n", + "Upserted 502 vectors, total so far: 427511\n", + "Upserted 503 vectors, total so far: 428014\n", + "Upserted 504 vectors, total so far: 428518\n", + "Upserted 525 vectors, total so far: 429043\n", + "Upserted 485 vectors, total so far: 429528\n", + "Upserted 488 vectors, total so far: 430016\n", + "Upserted 484 vectors, total so far: 430500\n", + "Upserted 506 vectors, total so far: 431006\n", + "Upserted 501 vectors, total so far: 431507\n", + "Upserted 518 vectors, total so far: 432025\n", + "Upserted 488 vectors, total so far: 432513\n", + "Upserted 494 vectors, total so far: 433007\n", + "Upserted 504 vectors, total so far: 433511\n", + "Upserted 491 vectors, total so far: 434002\n", + "Upserted 508 vectors, total so far: 434510\n", + "Upserted 501 vectors, total so far: 435011\n", + "Upserted 504 vectors, total so far: 435515\n", + "Upserted 490 vectors, total so far: 436005\n", + "Upserted 521 vectors, total so far: 436526\n", + "Upserted 528 vectors, total so far: 437054\n", + "Upserted 532 vectors, total so far: 437586\n", + "Upserted 489 vectors, total so far: 438075\n", + "Upserted 511 vectors, total so far: 438586\n", + "Upserted 515 vectors, total so far: 439101\n", + "Upserted 511 vectors, total so far: 439612\n", + "Upserted 493 vectors, total so far: 440105\n", + "Upserted 517 vectors, total so far: 440622\n", + "Upserted 496 vectors, total so far: 441118\n", + "Upserted 498 vectors, total so far: 441616\n", + "Upserted 471 vectors, total so far: 442087\n", + "Upserted 521 vectors, total so far: 442608\n", + "Upserted 495 vectors, total so far: 443103\n", + "Upserted 484 vectors, total so far: 443587\n", + "Upserted 496 vectors, total so far: 444083\n", + "Upserted 491 vectors, total so far: 444574\n", + "Upserted 466 vectors, total so far: 445040\n", + "Upserted 490 vectors, total so far: 445530\n", + "Upserted 507 vectors, total so far: 446037\n", + "Upserted 509 vectors, total so far: 446546\n", + "Upserted 496 vectors, total so far: 447042\n", + "Upserted 507 vectors, total so far: 447549\n", + "Upserted 474 vectors, total so far: 448023\n", + "Upserted 476 vectors, total so far: 448499\n", + "Upserted 506 vectors, total so far: 449005\n", + "Upserted 492 vectors, total so far: 449497\n", + "Upserted 510 vectors, total so far: 450007\n", + "Upserted 527 vectors, total so far: 450534\n", + "Upserted 477 vectors, total so far: 451011\n", + "Upserted 493 vectors, total so far: 451504\n", + "Upserted 514 vectors, total so far: 452018\n", + "Upserted 521 vectors, total so far: 452539\n", + "Upserted 512 vectors, total so far: 453051\n", + "Upserted 505 vectors, total so far: 453556\n", + "Upserted 535 vectors, total so far: 454091\n", + "Upserted 517 vectors, total so far: 454608\n", + "Upserted 500 vectors, total so far: 455108\n", + "Upserted 512 vectors, total so far: 455620\n", + "Upserted 491 vectors, total so far: 456111\n", + "Upserted 515 vectors, total so far: 456626\n", + "Upserted 506 vectors, total so far: 457132\n", + "Upserted 506 vectors, total so far: 457638\n", + "Upserted 504 vectors, total so far: 458142\n", + "Upserted 524 vectors, total so far: 458666\n", + "Upserted 479 vectors, total so far: 459145\n", + "Upserted 510 vectors, total so far: 459655\n", + "Upserted 508 vectors, total so far: 460163\n", + "Upserted 496 vectors, total so far: 460659\n", + "Upserted 512 vectors, total so far: 461171\n", + "Upserted 474 vectors, total so far: 461645\n", + "Upserted 527 vectors, total so far: 462172\n", + "Upserted 515 vectors, total so far: 462687\n", + "Upserted 512 vectors, total so far: 463199\n", + "Upserted 504 vectors, total so far: 463703\n", + "Upserted 489 vectors, total so far: 464192\n", + "Upserted 496 vectors, total so far: 464688\n", + "Upserted 501 vectors, total so far: 465189\n", + "Upserted 484 vectors, total so far: 465673\n", + "Upserted 496 vectors, total so far: 466169\n", + "Upserted 501 vectors, total so far: 466670\n", + "Upserted 489 vectors, total so far: 467159\n", + "Upserted 488 vectors, total so far: 467647\n", + "Upserted 506 vectors, total so far: 468153\n", + "Upserted 469 vectors, total so far: 468622\n", + "Upserted 495 vectors, total so far: 469117\n", + "Upserted 479 vectors, total so far: 469596\n", + "Upserted 495 vectors, total so far: 470091\n", + "Upserted 511 vectors, total so far: 470602\n", + "Upserted 491 vectors, total so far: 471093\n", + "Upserted 502 vectors, total so far: 471595\n", + "Upserted 487 vectors, total so far: 472082\n", + "Upserted 509 vectors, total so far: 472591\n", + "Upserted 483 vectors, total so far: 473074\n", + "Upserted 512 vectors, total so far: 473586\n", + "Upserted 527 vectors, total so far: 474113\n", + "Upserted 502 vectors, total so far: 474615\n", + "Upserted 500 vectors, total so far: 475115\n", + "Upserted 497 vectors, total so far: 475612\n", + "Upserted 496 vectors, total so far: 476108\n", + "Upserted 480 vectors, total so far: 476588\n", + "Upserted 510 vectors, total so far: 477098\n", + "Upserted 474 vectors, total so far: 477572\n", + "Upserted 490 vectors, total so far: 478062\n", + "Upserted 475 vectors, total so far: 478537\n", + "Upserted 493 vectors, total so far: 479030\n", + "Upserted 480 vectors, total so far: 479510\n", + "Upserted 498 vectors, total so far: 480008\n", + "Upserted 477 vectors, total so far: 480485\n", + "Upserted 496 vectors, total so far: 480981\n", + "Upserted 505 vectors, total so far: 481486\n", + "Upserted 506 vectors, total so far: 481992\n", + "Upserted 464 vectors, total so far: 482456\n", + "Upserted 527 vectors, total so far: 482983\n", + "Upserted 485 vectors, total so far: 483468\n", + "Upserted 509 vectors, total so far: 483977\n", + "Upserted 503 vectors, total so far: 484480\n", + "Upserted 511 vectors, total so far: 484991\n", + "Upserted 525 vectors, total so far: 485516\n", + "Upserted 502 vectors, total so far: 486018\n", + "Upserted 504 vectors, total so far: 486522\n", + "Upserted 501 vectors, total so far: 487023\n", + "Upserted 498 vectors, total so far: 487521\n", + "Upserted 498 vectors, total so far: 488019\n", + "Upserted 485 vectors, total so far: 488504\n", + "Upserted 479 vectors, total so far: 488983\n", + "Upserted 489 vectors, total so far: 489472\n", + "Upserted 501 vectors, total so far: 489973\n", + "Upserted 506 vectors, total so far: 490479\n", + "Upserted 504 vectors, total so far: 490983\n", + "Upserted 483 vectors, total so far: 491466\n", + "Upserted 509 vectors, total so far: 491975\n", + "Upserted 516 vectors, total so far: 492491\n", + "Upserted 484 vectors, total so far: 492975\n", + "Upserted 485 vectors, total so far: 493460\n", + "Upserted 505 vectors, total so far: 493965\n", + "Upserted 544 vectors, total so far: 494509\n", + "Upserted 475 vectors, total so far: 494984\n", + "Upserted 489 vectors, total so far: 495473\n", + "Upserted 488 vectors, total so far: 495961\n", + "Upserted 501 vectors, total so far: 496462\n", + "Upserted 525 vectors, total so far: 496987\n", + "Upserted 522 vectors, total so far: 497509\n", + "Upserted 495 vectors, total so far: 498004\n", + "Upserted 492 vectors, total so far: 498496\n", + "Upserted 490 vectors, total so far: 498986\n", + "Upserted 511 vectors, total so far: 499497\n", + "Upserted 503 vectors, total so far: 500000\n", + "Done! Total inserted: 500000 vectors with label 'label_50p'\n" + ] + } + ], "source": [ "#For reinserting delted vectors\n", "import pyarrow.parquet as pq\n", @@ -242,7 +4567,7 @@ "import os\n", "\n", "# User inputs\n", - "label_to_insert = \"label_1p\"\n", + "label_to_insert = \"label_50p\"\n", "batch_size = 1000\n", "\n", "# Build full paths\n", @@ -297,9 +4622,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_labelfilter_reupsertcheck',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 1000000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "index.describe()" ] diff --git a/stability_test_gt.py b/stability_test_gt.py new file mode 100644 index 000000000..0065450b0 --- /dev/null +++ b/stability_test_gt.py @@ -0,0 +1,472 @@ +#!/usr/bin/env python3 +""" +Stability Test Script - GT Filter Operator +Index: (set INDEX_NAME below — 1M vectors pre-loaded, no load step) + +Dataset: vectordataset_gt +Delete filter: $lte (e.g. delete id <= 10000 removes 10001 vectors for 1p) + +Workflow: + 1. Fresh run -> 4 filter rates (0.99, 0.80, 0.50, 0.01) + + 0.99 re-run with prefilter_cardinality_threshold=9999 + 2. Delete 1p -> verify count (989999) -> bench 4 rates + 0.99 prefilter + 3. Wait 10s -> Reinsert 1p -> verify count (1000000) -> bench 4 rates + 0.99 prefilter + 4. Wait 10s -> Delete 50p -> verify count (499999) -> bench 4 rates + 0.99 prefilter + 5. Wait 10s -> Reinsert 50p -> verify count (1000000) -> bench 4 rates + 0.99 prefilter + 6. Wait 10s -> Delete 80p -> verify count (199999) -> bench 4 rates + 0.99 prefilter + 7. Wait 10s -> Reinsert 80p -> verify count (1000000) -> bench 4 rates + 0.99 prefilter + +Delete counts note: + $lte 10000 -> ids 0..10000 inclusive = 10001 deleted -> 989999 remain + $lte 500000 -> 500001 deleted -> 499999 remain + $lte 800000 -> 800001 deleted -> 199999 remain + +Recorded per bench run : Recall (%), QPS, P99 Latency (s) +Recorded per operation : Delete duration (s), Insert duration (s) +Output : Excel file with all results +""" + +import subprocess +import json +import os +import time +import glob as glob_module +import pyarrow.parquet as pq +from endee import Endee as EndeeClient +import openpyxl +from openpyxl.styles import Font, PatternFill, Alignment, Border, Side +from datetime import datetime + +# ============================================================ +# CONFIGURATION +# ============================================================ +TOKEN = "localtest" +BASE_URL = "http://148.113.54.173:8080/api/v1" +INDEX_NAME = "test_1M_vaib_filter_1603_adup5" # <-- set before running +DATASET_LOCAL_DIR = "/home/debian/latest_VDB/VectorDBBench/vectordataset_gt" +BASE_PATH = "/home/debian/latest_VDB/VectorDBBench/vectordataset_gt" +DATASET_FOLDER = "cohere/cohere_medium_1m" +RESULTS_DIR = "/home/debian/latest_VDB/VectorDBBench/vectordb_bench/results/Endee" + +FILTER_RATES = [0.99, 0.80, 0.50, 0.01] +DELETE_RATES = ["1p", "50p", "80p"] + +# Delete: remove vectors with id <= threshold (includes threshold id) +RANGE_MAP_DELETE = { + "1p": 10000, # $lte 10000 -> 10001 deleted -> 989999 remain + "50p": 500000, # $lte 500000 -> 500001 deleted -> 499999 remain + "80p": 800000, # $lte 800000 -> 800001 deleted -> 199999 remain +} + +# Reinsert: re-upsert vectors in [low, high] inclusive +RANGE_MAP_INSERT = { + "1p": [0, 10000], + "50p": [0, 500000], + "80p": [0, 800000], +} + +# Expected count after delete +EXPECTED_COUNT_AFTER_DELETE = { + "1p": 989999, + "50p": 499999, + "80p": 199999, +} + +TOTAL_VECTORS = 1000000 +BATCH_SIZE = 1000 + +OUTPUT_EXCEL = os.path.join( + "/home/debian/latest_VDB/VectorDBBench", + f"stability_test_gt_{datetime.now().strftime('%Y%m%d_%H%M%S')}.xlsx" +) + +# ============================================================ +# VECTORDBBENCH RUNNER +# ============================================================ + +def run_vectordbbench(filter_rate: float, prefilter: int | None = None) -> dict: + """Run vectordbbench for one filter rate; return parsed metrics dict.""" + before = set(glob_module.glob(os.path.join(RESULTS_DIR, "*.json"))) + + prefilter_flag = f'--prefilter-cardinality-threshold {prefilter} ' if prefilter else '' + + cmd = ( + f'DATASET_LOCAL_DIR="{DATASET_LOCAL_DIR}" vectordbbench endee ' + f'--token "{TOKEN}" ' + f'--region india-west-1 ' + f'--base-url "{BASE_URL}" ' + f'--index-name {INDEX_NAME} ' + f'--task-label "20260107" ' + f'--m 16 ' + f'--ef-con 128 ' + f'--ef-search 128 ' + f'--space-type cosine ' + f'{prefilter_flag}' + f'--precision int16 ' + f'--version 1 ' + f'--case-type NewIntFilterPerformanceCase ' + f'--dataset-with-size-type "Medium Cohere (768dim, 1M)" ' + f'--filter-rate {filter_rate} ' + f'--k 30 ' + f'--num-concurrency "16" ' + f'--concurrency-duration 30 ' + f'--concurrency-timeout 3600 ' + f'--skip-drop-old ' + f'--skip-load ' + f'--search-concurrent ' + f'--search-serial' + ) + + prefilter_info = f", prefilter={prefilter}" if prefilter else "" + print(f"\n [RUN] filter_rate={filter_rate}{prefilter_info}") + proc = subprocess.run(cmd, shell=True, text=True) + if proc.returncode != 0: + print(f" [WARN] vectordbbench exited with code {proc.returncode}") + + time.sleep(3) + after = set(glob_module.glob(os.path.join(RESULTS_DIR, "*.json"))) + new_files = after - before + + if not new_files: + print(f" [ERROR] No new result file found for filter_rate={filter_rate}{prefilter_info}") + return {"recall": None, "qps": None, "p99_latency": None} + + result_file = max(new_files, key=os.path.getmtime) + print(f" [FILE] {os.path.basename(result_file)}") + + with open(result_file) as f: + data = json.load(f) + + metrics = data["results"][0]["metrics"] + recall = metrics.get("recall") + qps = metrics.get("qps") + p99 = metrics.get("serial_latency_p99") + + print(f" [METRICS] recall={recall}, qps={qps}, p99={p99}") + return {"recall": recall, "qps": qps, "p99_latency": p99} + + +def run_all_filter_rates(label: str) -> dict: + """ + Run vectordbbench for all 4 filter rates. + For filter_rate=0.99, also runs a second time with prefilter=9999. + Returns {filter_rate: metrics, "0.99_prefilter": metrics}. + """ + print(f"\n==> Running all filter rates [{label}]") + results = {} + for fr in FILTER_RATES: + results[fr] = run_vectordbbench(fr) + print(" [WAIT] 5s gap before next filter rate run ...") + time.sleep(5) + if fr == 0.99: + print(f" [RUN] filter_rate=0.99 (prefilter=9999) ...") + results["0.99_prefilter"] = run_vectordbbench(0.99, prefilter=9999) + print(" [WAIT] 5s gap before next filter rate run ...") + time.sleep(5) + return results + + +# ============================================================ +# DELETE / VERIFY / REINSERT +# ============================================================ + +def delete_vectors(index, rate_key: str) -> float: + """Delete vectors with id <= threshold ($lte). Returns elapsed seconds.""" + threshold = RANGE_MAP_DELETE[rate_key] + print(f"\n==> DELETE {rate_key}: removing vectors with id <= {threshold}") + start = time.perf_counter() + result = index.delete_with_filter([{"id": {"$lte": threshold}}]) + elapsed = time.perf_counter() - start + print(f" [DELETE] Completed in {elapsed:.2f}s | result: {result}") + print(" [WAIT] 5s pause before verify count ...") + time.sleep(5) + return elapsed + + +def verify_count(index, expected: int, context: str = "") -> int: + """Call index.describe() and print count vs expected.""" + desc = index.describe() + actual = desc.get("count", "UNKNOWN") + status = "OK" if actual == expected else "MISMATCH" + tag = f" [{context}]" if context else "" + print(f" [VERIFY{tag}] Expected={expected} Actual={actual} [{status}]") + return actual + + +def reinsert_vectors(index, rate_key: str) -> float: + """Reinsert vectors in [0, threshold]. Returns elapsed seconds.""" + low, high = RANGE_MAP_INSERT[rate_key] + emb_path = os.path.join(BASE_PATH, DATASET_FOLDER, "shuffle_train.parquet") + print(f"\n==> REINSERT {rate_key}: upserting vectors with id in [{low}, {high}]") + + emb_file = pq.ParquetFile(emb_path) + total = 0 + start = time.perf_counter() + + for batch in emb_file.iter_batches(batch_size=BATCH_SIZE): + df = batch.to_pandas() + df = df[(df["id"] >= low) & (df["id"] <= high)] + if df.empty: + continue + + vectors = [ + { + "id": str(row["id"]), + "vector": row["emb"].tolist() if hasattr(row["emb"], "tolist") else list(row["emb"]), + "meta": {"id": row["id"]}, + "filter": {"id": row["id"]}, + } + for _, row in df.iterrows() + ] + index.upsert(vectors) + total += len(vectors) + print(f" [INSERT] upserted {len(vectors)}, cumulative: {total}") + + elapsed = time.perf_counter() - start + print(f" [INSERT] Done. Total={total} vectors in {elapsed:.2f}s") + return elapsed + + +# ============================================================ +# EXCEL WRITER +# ============================================================ + +def write_excel(all_data: dict, output_path: str): + """Write all collected results to a formatted Excel file.""" + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "Stability Test GT" + + def hdr(bold=True, color="FFFFFF"): + return Font(bold=bold, color=color) + + def fill(hex_color): + return PatternFill("solid", fgColor=hex_color) + + center = Alignment(horizontal="center", vertical="center", wrap_text=True) + left = Alignment(horizontal="left", vertical="center") + thin = Side(style="thin", color="000000") + border = Border(left=thin, right=thin, top=thin, bottom=thin) + + COLOR_DARK = "2E4057" + COLOR_BLUE = "1565C0" + COLOR_PURPLE = "6A1B9A" # distinct colour for 0.99 prefilter group + COLOR_LBLUE = "BBDEFB" + COLOR_LPURPLE = "E1BEE7" + COLOR_GREEN = "1B5E20" + COLOR_LGREEN = "C8E6C9" + COLOR_ROW_ODD = "F5F5F5" + + ws.row_dimensions[1].height = 30 + ws.row_dimensions[2].height = 28 + + # --- Row 1: top-level headers --- + # Column layout: + # A = Phase + # B-D = Filter Rate 0.99 (normal) + # E-G = Filter Rate 0.99 (prefilter=9999) + # H-J = Filter Rate 0.80 + # K-M = Filter Rate 0.50 + # N-P = Filter Rate 0.01 + # Q-R = Operation Duration + + ws.merge_cells("A1:A2") + c = ws["A1"] + c.value = "Phase" + c.font = hdr(); c.fill = fill(COLOR_DARK) + c.alignment = center; c.border = border + + # 0.99 normal + ws.merge_cells("B1:D1") + c = ws["B1"] + c.value = "Filter Rate 0.99" + c.font = hdr(); c.fill = fill(COLOR_BLUE) + c.alignment = center; c.border = border + + # 0.99 prefilter + ws.merge_cells("E1:G1") + c = ws["E1"] + c.value = "Filter Rate 0.99\n(prefilter_cardinality=9999)" + c.font = hdr(); c.fill = fill(COLOR_PURPLE) + c.alignment = center; c.border = border + + # 0.80, 0.50, 0.01 + remaining_rates = [0.80, 0.50, 0.01] + fr_labels = {0.80: "Filter Rate 0.80", 0.50: "Filter Rate 0.50", 0.01: "Filter Rate 0.01"} + col = 8 # column H + for fr in remaining_rates: + ws.merge_cells(start_row=1, start_column=col, end_row=1, end_column=col + 2) + c = ws.cell(row=1, column=col, value=fr_labels[fr]) + c.font = hdr(); c.fill = fill(COLOR_BLUE) + c.alignment = center; c.border = border + col += 3 + + # Operation duration (cols Q-R = 17-18) + ws.merge_cells(start_row=1, start_column=col, end_row=1, end_column=col + 1) + c = ws.cell(row=1, column=col, value="Operation Duration") + c.font = hdr(); c.fill = fill(COLOR_GREEN) + c.alignment = center; c.border = border + + # --- Row 2: sub-headers --- + sub_cols = [ + ("B", COLOR_LBLUE), ("C", COLOR_LBLUE), ("D", COLOR_LBLUE), # 0.99 normal + ("E", COLOR_LPURPLE), ("F", COLOR_LPURPLE), ("G", COLOR_LPURPLE), # 0.99 prefilter + ("H", COLOR_LBLUE), ("I", COLOR_LBLUE), ("J", COLOR_LBLUE), # 0.80 + ("K", COLOR_LBLUE), ("L", COLOR_LBLUE), ("M", COLOR_LBLUE), # 0.50 + ("N", COLOR_LBLUE), ("O", COLOR_LBLUE), ("P", COLOR_LBLUE), # 0.01 + ] + for col_letter, bg in sub_cols: + c = ws[f"{col_letter}2"] + # cycle Recall/QPS/P99 by position + idx = "BCDEFGHIJKLMNOP".index(col_letter) + c.value = ("Recall (%)", "QPS", "P99 Latency (s)")[idx % 3] + c.font = Font(bold=True, color="000000") + c.fill = fill(bg) + c.alignment = center; c.border = border + + for col_letter, label in [("Q", "Delete Duration (s)"), ("R", "Insert Duration (s)")]: + c = ws[f"{col_letter}2"] + c.value = label + c.font = Font(bold=True, color="000000") + c.fill = fill(COLOR_LGREEN) + c.alignment = center; c.border = border + + # --- Phase definitions --- + phases = [ + ("Fresh Run\n(1M vectors, no delete)", "fresh", None, None), + ] + for rk in DELETE_RATES: + exp = EXPECTED_COUNT_AFTER_DELETE[rk] + phases.append(( + f"After Delete {rk}\n(verify count={exp:,})", + f"post_delete_{rk}", f"delete_{rk}", None + )) + phases.append(( + f"After Reinsert {rk}\n(verify count={TOTAL_VECTORS:,})", + f"post_insert_{rk}", None, f"insert_{rk}" + )) + + # --- Data rows --- + # Column order for metrics: 0.99, 0.99_prefilter, 0.80, 0.50, 0.01 + metric_keys = [0.99, "0.99_prefilter", 0.80, 0.50, 0.01] + + for i, (phase_name, bench_key, del_key, ins_key) in enumerate(phases): + row = 3 + i + ws.row_dimensions[row].height = 32 + row_fill = fill(COLOR_ROW_ODD) if i % 2 == 0 else fill("FFFFFF") + + c = ws.cell(row=row, column=1, value=phase_name) + c.font = Font(bold=True); c.fill = row_fill + c.alignment = left; c.border = border + + bench_results = all_data.get(bench_key) or {} + col = 2 + for mk in metric_keys: + m = bench_results.get(mk) or {} + recall = m.get("recall") + qps = m.get("qps") + p99 = m.get("p99_latency") + + vals = [ + round(recall * 100, 2) if recall is not None else "N/A", + round(qps, 4) if qps is not None else "N/A", + round(p99, 6) if p99 is not None else "N/A", + ] + for v in vals: + c = ws.cell(row=row, column=col, value=v) + c.fill = row_fill; c.alignment = center; c.border = border + col += 1 + + del_dur = all_data.get(del_key) if del_key else None + ins_dur = all_data.get(ins_key) if ins_key else None + + c = ws.cell(row=row, column=col, + value=round(del_dur, 2) if del_dur is not None else "") + c.fill = row_fill; c.alignment = center; c.border = border + + c = ws.cell(row=row, column=col + 1, + value=round(ins_dur, 2) if ins_dur is not None else "") + c.fill = row_fill; c.alignment = center; c.border = border + + # --- Column widths --- + ws.column_dimensions["A"].width = 30 + for col_letter in ["B","C","D","E","F","G","H","I","J","K","L","M","N","O","P"]: + ws.column_dimensions[col_letter].width = 15 + ws.column_dimensions["Q"].width = 22 + ws.column_dimensions["R"].width = 22 + + ws.freeze_panes = "B3" + + wb.save(output_path) + print(f"\n[EXCEL] Saved: {output_path}") + + +# ============================================================ +# MAIN +# ============================================================ + +def main(): + print("=" * 65) + print(" Stability Test — GT Filter Operator") + print(f" Index : {INDEX_NAME}") + print(f" Output: {OUTPUT_EXCEL}") + print("=" * 65) + + client = EndeeClient(token=TOKEN) + client.set_base_url(BASE_URL) + index = client.get_index(INDEX_NAME) + print(f"\n[INIT] Connected to index '{INDEX_NAME}'") + print(f"[INIT] Current state: {index.describe()}") + + all_data: dict = {} + + # ---- Phase 1: Fresh run ---- + all_data["fresh"] = run_all_filter_rates("Fresh — 1M vectors") + + for rate_key in DELETE_RATES: + # ---- Delete ---- + all_data[f"delete_{rate_key}"] = delete_vectors(index, rate_key) + + # Verify count after delete + verify_count(index, expected=EXPECTED_COUNT_AFTER_DELETE[rate_key], + context=f"after delete {rate_key}") + + # ---- Bench post-delete ---- + all_data[f"post_delete_{rate_key}"] = run_all_filter_rates( + f"Post-Delete {rate_key}" + ) + + # Wait before reinsert + print(f"\n[WAIT] 10s pause before reinsert {rate_key} ...") + time.sleep(10) + + # ---- Reinsert ---- + all_data[f"insert_{rate_key}"] = reinsert_vectors(index, rate_key) + + # Verify count after reinsert + print(" [WAIT] 5s pause before verify count after reinsert ...") + time.sleep(5) + verify_count(index, expected=TOTAL_VECTORS, + context=f"after reinsert {rate_key}") + + # ---- Bench post-insert ---- + all_data[f"post_insert_{rate_key}"] = run_all_filter_rates( + f"Post-Reinsert {rate_key}" + ) + + # Wait before next delete cycle (skip after last) + if rate_key != DELETE_RATES[-1]: + print(f"\n[WAIT] 10s pause before next delete cycle ...") + time.sleep(10) + + # ---- Save results ---- + write_excel(all_data, OUTPUT_EXCEL) + + print("\n" + "=" * 65) + print(" Stability test complete!") + print(f" Results: {OUTPUT_EXCEL}") + print("=" * 65) + + +if __name__ == "__main__": + main() diff --git a/stability_test_gte.py b/stability_test_gte.py new file mode 100644 index 000000000..631c58db8 --- /dev/null +++ b/stability_test_gte.py @@ -0,0 +1,434 @@ +#!/usr/bin/env python3 +""" +Stability Test Script - GTE Filter Operator +Index: test_1M_vaib_filter_1503_adup1 (1M vectors pre-loaded, no load step) + +Workflow: + 1. Fresh run -> 4 filter rates (0.99, 0.80, 0.50, 0.01) + 2. Delete 1p -> verify count (990000) -> bench 4 rates + 3. Wait 10s -> Reinsert 1p -> bench 4 rates + 4. Wait 10s -> Delete 50p -> verify count (500000) -> bench 4 rates + 5. Wait 10s -> Reinsert 50p -> bench 4 rates + 6. Wait 10s -> Delete 80p -> verify count (200000) -> bench 4 rates + 7. Wait 10s -> Reinsert 80p -> bench 4 rates + +Recorded per bench run: Recall (%), QPS, P99 Latency (s) +Recorded per operation: Delete duration (s), Insert duration (s) +Output: Excel file with all results +""" + +import subprocess +import json +import os +import time +import glob as glob_module +import pyarrow.parquet as pq +from endee import Endee as EndeeClient +import openpyxl +from openpyxl.styles import Font, PatternFill, Alignment, Border, Side +from datetime import datetime + +# ============================================================ +# CONFIGURATION +# ============================================================ +TOKEN = "localtest" +BASE_URL = "http://148.113.54.173:8080/api/v1" +INDEX_NAME = "test_1M_vaib_filter_1503_adup1" +DATASET_LOCAL_DIR = "/home/debian/latest_VDB/VectorDBBench/vectordataset" +BASE_PATH = "/home/debian/latest_VDB/VectorDBBench/vectordataset" +DATASET_FOLDER = "cohere/cohere_medium_1m" +RESULTS_DIR = "/home/debian/latest_VDB/VectorDBBench/vectordb_bench/results/Endee" + +FILTER_RATES = [0.99, 0.80, 0.50, 0.01] +DELETE_RATES = ["1p", "50p", "80p"] # 99p excluded per spec + +# For delete: delete vectors with id < gte_value +RANGE_MAP_DELETE = { + "1p": 10000, + "50p": 500000, + "80p": 800000, +} + +# For reinsert: reinsert vectors with id in [low, high] +RANGE_MAP_INSERT = { + "1p": [0, 9999], + "50p": [0, 499999], + "80p": [0, 799999], +} + +# Expected total_elements count after delete +EXPECTED_COUNT_AFTER_DELETE = { + "1p": 990000, + "50p": 500000, + "80p": 200000, +} + +BATCH_SIZE = 1000 + +OUTPUT_EXCEL = os.path.join( + "/home/debian/latest_VDB/VectorDBBench", + f"stability_test_gte_{datetime.now().strftime('%Y%m%d_%H%M%S')}.xlsx" +) + +# ============================================================ +# VECTORDBBENCH RUNNER +# ============================================================ + +def run_vectordbbench(filter_rate: float, prefilter: int | None = None) -> dict: + """Run vectordbbench for one filter rate; return parsed metrics dict.""" + # Snapshot files before run so we can identify the new result file + before = set(glob_module.glob(os.path.join(RESULTS_DIR, "*.json"))) + + prefilter_flag = f'--prefilter-cardinality-threshold {prefilter} ' if prefilter else '' + + cmd = ( + f'DATASET_LOCAL_DIR="{DATASET_LOCAL_DIR}" vectordbbench endee ' + f'--token "{TOKEN}" ' + f'--region india-west-1 ' + f'--base-url "{BASE_URL}" ' + f'--index-name {INDEX_NAME} ' + f'--task-label "20260107" ' + f'--m 16 ' + f'--ef-con 128 ' + f'--ef-search 128 ' + f'--space-type cosine ' + f'{prefilter_flag}' + f'--precision int16 ' + f'--version 1 ' + f'--case-type NewIntFilterPerformanceCase ' + f'--dataset-with-size-type "Medium Cohere (768dim, 1M)" ' + f'--filter-rate {filter_rate} ' + f'--k 30 ' + f'--num-concurrency "16" ' + f'--concurrency-duration 30 ' + f'--concurrency-timeout 3600 ' + f'--skip-drop-old ' + f'--skip-load ' + f'--search-concurrent ' + f'--search-serial' + ) + + prefilter_info = f", prefilter={prefilter}" if prefilter else "" + print(f"\n [RUN] filter_rate={filter_rate}{prefilter_info}") + proc = subprocess.run(cmd, shell=True, text=True) + if proc.returncode != 0: + print(f" [WARN] vectordbbench exited with code {proc.returncode}") + + # Give the framework a moment to flush the result file + time.sleep(3) + after = set(glob_module.glob(os.path.join(RESULTS_DIR, "*.json"))) + new_files = after - before + + if not new_files: + print(f" [ERROR] No new result file found for filter_rate={filter_rate}") + return {"recall": None, "qps": None, "p99_latency": None} + + result_file = max(new_files, key=os.path.getmtime) + print(f" [FILE] {os.path.basename(result_file)}") + + with open(result_file) as f: + data = json.load(f) + + metrics = data["results"][0]["metrics"] + recall = metrics.get("recall") + qps = metrics.get("qps") + p99 = metrics.get("serial_latency_p99") + + print(f" [METRICS] recall={recall}, qps={qps}, p99={p99}") + return {"recall": recall, "qps": qps, "p99_latency": p99} + + +def run_all_filter_rates(label: str, prefilter: int | None = None) -> dict: + """Run vectordbbench for all 4 filter rates. Returns {filter_rate: metrics}.""" + print(f"\n==> Running all filter rates [{label}]") + results = {} + for fr in FILTER_RATES: + results[fr] = run_vectordbbench(fr, prefilter=prefilter) + print(" [WAIT] 5s gap before next filter rate run ...") + time.sleep(5) + return results + + +# ============================================================ +# DELETE / VERIFY / REINSERT +# ============================================================ + +def delete_vectors(index, rate_key: str) -> float: + """Delete vectors with id < threshold. Returns elapsed seconds.""" + threshold = RANGE_MAP_DELETE[rate_key] + print(f"\n==> DELETE {rate_key}: removing vectors with id < {threshold}") + start = time.perf_counter() + result = index.delete_with_filter([{"id": {"$lt": threshold}}]) + elapsed = time.perf_counter() - start + print(f" [DELETE] Completed in {elapsed:.2f}s | result: {result}") + print(" [WAIT] 5s pause before verify count ...") + time.sleep(5) + return elapsed + + +def verify_count(index, rate_key: str) -> int: + """Call index.describe() and print count vs expected.""" + desc = index.describe() + # Support both key names used by different endee SDK versions + actual = desc.get("count", "UNKNOWN") + expected = EXPECTED_COUNT_AFTER_DELETE[rate_key] + status = "OK" if actual == expected else "MISMATCH" + print(f" [VERIFY] Expected={expected} Actual={actual} [{status}]") + return actual + + +def reinsert_vectors(index, rate_key: str) -> float: + """Reinsert vectors in the deleted id range. Returns elapsed seconds.""" + low, high = RANGE_MAP_INSERT[rate_key] + emb_path = os.path.join(BASE_PATH, DATASET_FOLDER, "shuffle_train.parquet") + print(f"\n==> REINSERT {rate_key}: upserting vectors with id in [{low}, {high}]") + + emb_file = pq.ParquetFile(emb_path) + total = 0 + start = time.perf_counter() + + for batch in emb_file.iter_batches(batch_size=BATCH_SIZE): + df = batch.to_pandas() + df = df[(df["id"] >= low) & (df["id"] <= high)] + if df.empty: + continue + + vectors = [ + { + "id": str(row["id"]), + "vector": row["emb"].tolist() if hasattr(row["emb"], "tolist") else list(row["emb"]), + "meta": {"id": row["id"]}, + "filter": {"id": row["id"]}, + } + for _, row in df.iterrows() + ] + index.upsert(vectors) + total += len(vectors) + print(f" [INSERT] upserted {len(vectors)}, cumulative: {total}") + + elapsed = time.perf_counter() - start + print(f" [INSERT] Done. Total={total} vectors in {elapsed:.2f}s") + return elapsed + + +# ============================================================ +# EXCEL WRITER +# ============================================================ + +def write_excel(all_data: dict, output_path: str): + """Write all collected results to a formatted Excel file.""" + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "Stability Test GTE" + + # --- Style helpers --- + def hdr(bold=True, color="FFFFFF"): + return Font(bold=bold, color=color) + + def fill(hex_color): + return PatternFill("solid", fgColor=hex_color) + + center = Alignment(horizontal="center", vertical="center", wrap_text=True) + left = Alignment(horizontal="left", vertical="center") + thin = Side(style="thin", color="000000") + border = Border(left=thin, right=thin, top=thin, bottom=thin) + + COLOR_DARK = "2E4057" # dark navy for phase header + COLOR_BLUE = "1565C0" # blue for filter rate group header + COLOR_LBLUE = "BBDEFB" # light blue for sub-headers + COLOR_GREEN = "1B5E20" # dark green for operation header + COLOR_LGREEN = "C8E6C9" # light green for operation sub-header + COLOR_ROW_ODD = "F5F5F5" # alternating row shade + + # --- Row 1: Top-level headers --- + # Col 1 = Phase + # Cols 2-13 = 4 filter-rate groups x 3 metrics + # Cols 14-15 = Delete Duration, Insert Duration + + ws.row_dimensions[1].height = 30 + ws.row_dimensions[2].height = 28 + + # Phase cell (spans rows 1-2) + ws.merge_cells("A1:A2") + c = ws["A1"] + c.value = "Phase" + c.font = hdr(); c.fill = fill(COLOR_DARK) + c.alignment = center; c.border = border + + # Filter rate group headers + fr_labels = {0.99: "Filter Rate 0.99", 0.80: "Filter Rate 0.80", + 0.50: "Filter Rate 0.50", 0.01: "Filter Rate 0.01"} + col = 2 + for fr in FILTER_RATES: + ws.merge_cells( + start_row=1, start_column=col, + end_row=1, end_column=col + 2 + ) + c = ws.cell(row=1, column=col, value=fr_labels[fr]) + c.font = hdr(); c.fill = fill(COLOR_BLUE) + c.alignment = center; c.border = border + col += 3 + + # Operation headers (spans rows 1-2 not needed; keep row2 for sub-labels) + ws.merge_cells( + start_row=1, start_column=col, + end_row=1, end_column=col + 1 + ) + c = ws.cell(row=1, column=col, value="Operation Duration") + c.font = hdr(); c.fill = fill(COLOR_GREEN) + c.alignment = center; c.border = border + + # --- Row 2: Sub-headers --- + ws.cell(row=2, column=1) # already merged above + + col = 2 + for _ in FILTER_RATES: + for sub in ("Recall (%)", "QPS", "P99 Latency (s)"): + c = ws.cell(row=2, column=col, value=sub) + c.font = Font(bold=True, color="000000") + c.fill = fill(COLOR_LBLUE) + c.alignment = center; c.border = border + col += 1 + + for sub in ("Delete Duration (s)", "Insert Duration (s)"): + c = ws.cell(row=2, column=col, value=sub) + c.font = Font(bold=True, color="000000") + c.fill = fill(COLOR_LGREEN) + c.alignment = center; c.border = border + col += 1 + + # --- Data rows --- + # Order of phases + phases = [ + ("Fresh Insertion\n(1M vectors, no delete)", "fresh", None, None), + ] + for rk in DELETE_RATES: + phases.append((f"After Delete {rk}\n(verify count)", f"post_delete_{rk}", f"delete_{rk}", None)) + if rk == "80p": + phases.append(("After Delete 80p\n(prefilter_cardinality_threshold=20000)", "post_delete_80p_prefilter", None, None)) + phases.append((f"After Reinsert {rk}", f"post_insert_{rk}", None, f"insert_{rk}")) + + for i, (phase_name, bench_key, del_key, ins_key) in enumerate(phases): + row = 3 + i + ws.row_dimensions[row].height = 32 + row_fill = fill(COLOR_ROW_ODD) if i % 2 == 0 else fill("FFFFFF") + + # Phase label + c = ws.cell(row=row, column=1, value=phase_name) + c.font = Font(bold=True); c.fill = row_fill + c.alignment = left; c.border = border + + bench_results = all_data.get(bench_key) or {} + col = 2 + for fr in FILTER_RATES: + m = bench_results.get(fr) or {} + recall = m.get("recall") + qps = m.get("qps") + p99 = m.get("p99_latency") + + vals = [ + round(recall * 100, 2) if recall is not None else "N/A", + round(qps, 4) if qps is not None else "N/A", + round(p99, 6) if p99 is not None else "N/A", + ] + for v in vals: + c = ws.cell(row=row, column=col, value=v) + c.fill = row_fill; c.alignment = center; c.border = border + col += 1 + + del_dur = all_data.get(del_key) if del_key else None + ins_dur = all_data.get(ins_key) if ins_key else None + + c = ws.cell(row=row, column=col, + value=round(del_dur, 2) if del_dur is not None else "") + c.fill = row_fill; c.alignment = center; c.border = border + + c = ws.cell(row=row, column=col + 1, + value=round(ins_dur, 2) if ins_dur is not None else "") + c.fill = row_fill; c.alignment = center; c.border = border + + # --- Column widths --- + ws.column_dimensions["A"].width = 28 + metric_cols = ["B","C","D","E","F","G","H","I","J","K","L","M"] + for col_letter in metric_cols: + ws.column_dimensions[col_letter].width = 15 + ws.column_dimensions["N"].width = 22 + ws.column_dimensions["O"].width = 22 + + # Freeze top 2 header rows + phase column + ws.freeze_panes = "B3" + + wb.save(output_path) + print(f"\n[EXCEL] Saved: {output_path}") + + +# ============================================================ +# MAIN +# ============================================================ + +def main(): + print("=" * 65) + print(" Stability Test — GTE Filter Operator") + print(f" Index : {INDEX_NAME}") + print(f" Output: {OUTPUT_EXCEL}") + print("=" * 65) + + # Connect to index + client = EndeeClient(token=TOKEN) + client.set_base_url(BASE_URL) + index = client.get_index(INDEX_NAME) + print(f"\n[INIT] Connected to index '{INDEX_NAME}'") + print(f"[INIT] Current state: {index.describe()}") + + all_data: dict = {} + + # ---- Phase 1: Fresh run ---- + all_data["fresh"] = run_all_filter_rates("Fresh — 1M vectors") + + for rate_key in DELETE_RATES: + # ---- Delete ---- + all_data[f"delete_{rate_key}"] = delete_vectors(index, rate_key) + + # Verify count + verify_count(index, rate_key) + + # ---- Bench post-delete ---- + all_data[f"post_delete_{rate_key}"] = run_all_filter_rates( + f"Post-Delete {rate_key}" + ) + + # ---- Extra bench with prefilter=20000 only after 80p delete ---- + if rate_key == "80p": + all_data["post_delete_80p_prefilter"] = run_all_filter_rates( + "Post-Delete 80p (prefilter_cardinality_threshold=20000)", + prefilter=20000 + ) + + # Wait before reinsert + print(f"\n[WAIT] 10s pause before reinsert {rate_key} ...") + time.sleep(10) + + # ---- Reinsert ---- + all_data[f"insert_{rate_key}"] = reinsert_vectors(index, rate_key) + + # ---- Bench post-insert ---- + all_data[f"post_insert_{rate_key}"] = run_all_filter_rates( + f"Post-Reinsert {rate_key}" + ) + + # Wait before next delete (skip after last) + if rate_key != DELETE_RATES[-1]: + print(f"\n[WAIT] 10s pause before next delete cycle ...") + time.sleep(10) + + # ---- Save results ---- + write_excel(all_data, OUTPUT_EXCEL) + + print("\n" + "=" * 65) + print(" Stability test complete!") + print(f" Results: {OUTPUT_EXCEL}") + print("=" * 65) + + +if __name__ == "__main__": + main() diff --git a/stability_test_labelfilter.py b/stability_test_labelfilter.py new file mode 100644 index 000000000..002ae67b2 --- /dev/null +++ b/stability_test_labelfilter.py @@ -0,0 +1,393 @@ +#!/usr/bin/env python3 +""" +Stability Test Script - Label Filter +Index: test_1M_vaib_filter_1503_adup2 (1M vectors pre-loaded, no load step) + +Workflow: + 1. Fresh run -> 8 label percentages (0.001, 0.002, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5) + For each label in [label_1p, label_2p, label_5p, label_10p, label_20p, label_50p]: + 2. Delete