From 4bdde8eb6a54c9c2f2cf9882f11461f8fb7d50b6 Mon Sep 17 00:00:00 2001 From: aecsocket Date: Tue, 9 Jun 2026 19:04:07 +0100 Subject: [PATCH 1/2] Allow searching by project dependencies --- .../src/search/backend/typesense/mod.rs | 22 ++++-- apps/labrinth/src/search/indexing.rs | 67 +++++++++++++++++-- apps/labrinth/src/search/mod.rs | 19 ++++++ apps/labrinth/src/test/search.rs | 21 +++++- apps/labrinth/tests/search.rs | 42 +++++++++++- 5 files changed, 157 insertions(+), 14 deletions(-) diff --git a/apps/labrinth/src/search/backend/typesense/mod.rs b/apps/labrinth/src/search/backend/typesense/mod.rs index f5c44b51aa..d6e07c2160 100644 --- a/apps/labrinth/src/search/backend/typesense/mod.rs +++ b/apps/labrinth/src/search/backend/typesense/mod.rs @@ -325,16 +325,16 @@ impl TypesenseClient { filter_by: &str, ) -> Result<()> { let resp = self - .request( - Method::DELETE, - &format!( + .request( + Method::DELETE, + &format!( "/collections/{collection}/documents?filter_by={}&batch_size=1000", urlencoding::encode(filter_by) ), - ) - .send() - .await - .wrap_err("failed to DELETE Typesense documents by filter")?; + ) + .send() + .await + .wrap_err("failed to DELETE Typesense documents by filter")?; if resp.status() == reqwest::StatusCode::NOT_FOUND { return Ok(()); } @@ -478,6 +478,13 @@ impl SearchField { sort: false, optional: true, }, + SearchField::DependencyProjectId => TypesenseFieldSpec { + path: "dependency_project_id", + ty: "string[]", + facet: true, + sort: false, + optional: true, + }, } } } @@ -526,6 +533,7 @@ impl Typesense { json!({"name": "minecraft_java_server.verified_plays_2w", "type": "int64", "sort": true, "optional": true}), json!({"name": "minecraft_java_server.is_online", "type": "bool", "sort": true, "optional": true}), json!({"name": "minecraft_java_server.ping.data.players_online", "type": "int32", "sort": true, "optional": true}), + json!({"name": "dependencies", "type": "object[]", "optional": true}), ]; fields.extend(TYPESENSE_SEARCH_FIELDS.iter().cloned()); diff --git a/apps/labrinth/src/search/indexing.rs b/apps/labrinth/src/search/indexing.rs index ff2d734363..168342348a 100644 --- a/apps/labrinth/src/search/indexing.rs +++ b/apps/labrinth/src/search/indexing.rs @@ -5,6 +5,7 @@ use futures::TryStreamExt; use heck::ToKebabCase; use itertools::Itertools; use regex::Regex; +use sqlx::Row; use std::collections::HashMap; use std::sync::LazyLock; use tracing::{info, warn}; @@ -24,7 +25,7 @@ use crate::models::ids::ProjectId; use crate::models::projects::from_duplicate_version_fields; use crate::models::v2::projects::LegacyProject; use crate::routes::v2_reroute; -use crate::search::UploadSearchProject; +use crate::search::{SearchProjectDependency, UploadSearchProject}; use crate::util::error::Context; fn normalize_for_search(s: &str) -> String { @@ -65,6 +66,12 @@ pub async fn index_local( components: exp::ProjectSerial, } + let searchable_statuses = + crate::models::projects::ProjectStatus::iterator() + .filter(|x| x.is_searchable()) + .map(|x| x.to_string()) + .collect::>(); + let db_projects = sqlx::query!( r#" SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows, @@ -76,10 +83,7 @@ pub async fn index_local( ORDER BY m.id ASC LIMIT $2; "#, - &*crate::models::projects::ProjectStatus::iterator() - .filter(|x| x.is_searchable()) - .map(|x| x.to_string()) - .collect::>(), + &searchable_statuses, limit, cursor, ) @@ -118,6 +122,49 @@ pub async fn index_local( return Ok((vec![], i64::MAX)); }; + info!("Indexing local dependencies!"); + + let dependencies: DashMap> = + sqlx::query( + " + SELECT DISTINCT v.mod_id dependent_project_id, d.mod_dependency_id dependency_project_id, + m.name dependency_name, m.slug dependency_slug, m.icon_url dependency_icon_url + FROM versions v + INNER JOIN dependencies d ON d.dependent_id = v.id + INNER JOIN mods m ON m.id = d.mod_dependency_id + WHERE v.mod_id = ANY($1) + AND d.mod_dependency_id IS NOT NULL + AND m.status = ANY($2) + ", + ) + .bind(&project_ids) + .bind(&searchable_statuses) + .fetch(pool) + .try_fold( + DashMap::new(), + |acc: DashMap>, m| { + let dependency_project_id: Option = + m.get("dependency_project_id"); + if let Some(dependency_project_id) = dependency_project_id { + acc.entry(DBProjectId(m.get("dependent_project_id"))) + .or_default() + .push(SearchProjectDependency { + project_id: ProjectId::from(DBProjectId( + dependency_project_id, + )) + .to_string(), + name: m.get("dependency_name"), + slug: m.get("dependency_slug"), + icon_url: m.get("dependency_icon_url"), + }); + } + + async move { Ok(acc) } + }, + ) + .await + .wrap_err("failed to fetch project dependencies")?; + struct PartialGallery { url: String, featured: bool, @@ -346,6 +393,14 @@ pub async fn index_local( } else { (vec![], vec![]) }; + let dependencies = dependencies + .get(&project.id) + .map(|x| x.clone()) + .unwrap_or_default(); + let dependency_project_id = dependencies + .iter() + .map(|dependency| dependency.project_id.clone()) + .collect::>(); if let Some(versions) = versions.remove(&project.id) { // Aggregated project loader fields @@ -486,6 +541,8 @@ pub async fn index_local( featured_gallery: featured_gallery.clone(), open_source, color: project.color.map(|x| x as u32), + dependency_project_id: dependency_project_id.clone(), + dependencies: dependencies.clone(), loader_fields, project_loader_fields: project_loader_fields.clone(), // 'loaders' is aggregate of all versions' loaders diff --git a/apps/labrinth/src/search/mod.rs b/apps/labrinth/src/search/mod.rs index 1fd208a8ec..5a14ee19aa 100644 --- a/apps/labrinth/src/search/mod.rs +++ b/apps/labrinth/src/search/mod.rs @@ -195,6 +195,7 @@ pub enum SearchField { MinecraftJavaServerContentKind, MinecraftJavaServerContentSupportedGameVersions, MinecraftJavaServerPingData, + DependencyProjectId, } #[derive(Debug, Error)] @@ -248,6 +249,10 @@ pub struct UploadSearchProject { pub version_published_timestamp: i64, pub open_source: bool, pub color: Option, + #[serde(default)] + pub dependency_project_id: Vec, + #[serde(default)] + pub dependencies: Vec, // Hidden fields to get the Project model out of the search results. pub loaders: Vec, // Search uses loaders as categories- this is purely for the Project model. @@ -259,6 +264,14 @@ pub struct UploadSearchProject { pub loader_fields: HashMap>, } +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct SearchProjectDependency { + pub project_id: String, + pub name: String, + pub slug: Option, + pub icon_url: Option, +} + #[derive(Serialize, Deserialize, Debug)] pub struct SearchResults { pub hits: Vec, @@ -295,6 +308,10 @@ pub struct ResultSearchProject { pub gallery: Vec, pub featured_gallery: Option, pub color: Option, + #[serde(default)] + pub dependency_project_id: Vec, + #[serde(default)] + pub dependencies: Vec, // Hidden fields to get the Project model out of the search results. pub loaders: Vec, // Search uses loaders as categories- this is purely for the Project model. @@ -332,6 +349,8 @@ impl From for ResultSearchProject { gallery: source.gallery, featured_gallery: source.featured_gallery, color: source.color, + dependency_project_id: source.dependency_project_id, + dependencies: source.dependencies, loaders: source.loaders, project_loader_fields: source.project_loader_fields, components: source.components, diff --git a/apps/labrinth/src/test/search.rs b/apps/labrinth/src/test/search.rs index 45ce34f7f0..9159a5a14b 100644 --- a/apps/labrinth/src/test/search.rs +++ b/apps/labrinth/src/test/search.rs @@ -214,12 +214,31 @@ pub async fn setup_search_projects( USER_USER_PAT, ) .await; + let project_1 = api + .get_project_deserialized_common( + &format!("{test_name}-searchable-project-1"), + USER_USER_PAT, + ) + .await; + let modify_json = serde_json::from_value(json!([ + { + "op": "add", + "path": "/dependencies", + "value": [ + { + "project_id": project_1.id, + "dependency_type": "required" + } + ] + } + ])) + .unwrap(); api.add_public_version( project_7.id, "1.0.0", TestFile::build_random_jar(), None, - None, + Some(modify_json), USER_USER_PAT, ) .await; diff --git a/apps/labrinth/tests/search.rs b/apps/labrinth/tests/search.rs index 81a761a542..ce1f9ae8ce 100644 --- a/apps/labrinth/tests/search.rs +++ b/apps/labrinth/tests/search.rs @@ -4,7 +4,7 @@ use common::database::*; use common::dummy_data::DUMMY_CATEGORIES; -use ariadne::ids::base62_impl::parse_base62; +use ariadne::ids::base62_impl::{parse_base62, to_base62}; use common::environment::TestEnvironment; use common::environment::with_test_environment; use common::search::setup_search_projects; @@ -29,6 +29,12 @@ async fn search_projects() { let api = &test_env.api; let test_name = test_env.db.database_name.clone(); + let dependency_project_id = id_conversion + .iter() + .find_map(|(project_id, test_id)| { + (*test_id == 1).then_some(to_base62(*project_id)) + }) + .unwrap(); // Pairs of: // 1. vec of search facets @@ -83,6 +89,12 @@ async fn search_projects() { json!([["categories:fabric"], ["project_types:modpack"]]), vec![4], ), + ( + json!([[format!( + "dependency_project_id:{dependency_project_id}" + )]]), + vec![7], + ), ]; // TODO: versions, game versions // Untested: @@ -123,6 +135,34 @@ async fn search_projects() { } }) .await; + + let projects = api + .search_deserialized( + Some(&format!("&{test_name}")), + Some(json!([[format!( + "dependency_project_id:{dependency_project_id}" + )]])), + USER_USER_PAT, + ) + .await; + assert_eq!(projects.total_hits, 1); + assert_eq!(projects.hits[0].dependency_project_id.len(), 1); + assert_eq!( + projects.hits[0].dependency_project_id[0], + dependency_project_id + ); + assert_eq!(projects.hits[0].dependencies.len(), 1); + assert_eq!( + projects.hits[0].dependencies[0].project_id, + dependency_project_id + ); + assert!( + projects.hits[0].dependencies[0] + .slug + .as_ref() + .unwrap() + .contains("searchable-project-1") + ); }, ) .await; From 3e373999b61dfb003b5d7b275025c51cfa88f2b2 Mon Sep 17 00:00:00 2001 From: aecsocket Date: Tue, 9 Jun 2026 19:41:04 +0100 Subject: [PATCH 2/2] change field name --- apps/labrinth/src/search/backend/typesense/mod.rs | 4 ++-- apps/labrinth/src/search/indexing.rs | 4 ++-- apps/labrinth/src/search/mod.rs | 8 ++++---- apps/labrinth/tests/search.rs | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/apps/labrinth/src/search/backend/typesense/mod.rs b/apps/labrinth/src/search/backend/typesense/mod.rs index d6e07c2160..5959d95789 100644 --- a/apps/labrinth/src/search/backend/typesense/mod.rs +++ b/apps/labrinth/src/search/backend/typesense/mod.rs @@ -478,8 +478,8 @@ impl SearchField { sort: false, optional: true, }, - SearchField::DependencyProjectId => TypesenseFieldSpec { - path: "dependency_project_id", + SearchField::DependencyProjectIds => TypesenseFieldSpec { + path: "dependency_project_ids", ty: "string[]", facet: true, sort: false, diff --git a/apps/labrinth/src/search/indexing.rs b/apps/labrinth/src/search/indexing.rs index 168342348a..c8b8fb0910 100644 --- a/apps/labrinth/src/search/indexing.rs +++ b/apps/labrinth/src/search/indexing.rs @@ -397,7 +397,7 @@ pub async fn index_local( .get(&project.id) .map(|x| x.clone()) .unwrap_or_default(); - let dependency_project_id = dependencies + let dependency_project_ids = dependencies .iter() .map(|dependency| dependency.project_id.clone()) .collect::>(); @@ -541,7 +541,7 @@ pub async fn index_local( featured_gallery: featured_gallery.clone(), open_source, color: project.color.map(|x| x as u32), - dependency_project_id: dependency_project_id.clone(), + dependency_project_ids: dependency_project_ids.clone(), dependencies: dependencies.clone(), loader_fields, project_loader_fields: project_loader_fields.clone(), diff --git a/apps/labrinth/src/search/mod.rs b/apps/labrinth/src/search/mod.rs index 5a14ee19aa..90c0f9e3cd 100644 --- a/apps/labrinth/src/search/mod.rs +++ b/apps/labrinth/src/search/mod.rs @@ -195,7 +195,7 @@ pub enum SearchField { MinecraftJavaServerContentKind, MinecraftJavaServerContentSupportedGameVersions, MinecraftJavaServerPingData, - DependencyProjectId, + DependencyProjectIds, } #[derive(Debug, Error)] @@ -250,7 +250,7 @@ pub struct UploadSearchProject { pub open_source: bool, pub color: Option, #[serde(default)] - pub dependency_project_id: Vec, + pub dependency_project_ids: Vec, #[serde(default)] pub dependencies: Vec, @@ -309,7 +309,7 @@ pub struct ResultSearchProject { pub featured_gallery: Option, pub color: Option, #[serde(default)] - pub dependency_project_id: Vec, + pub dependency_project_ids: Vec, #[serde(default)] pub dependencies: Vec, @@ -349,7 +349,7 @@ impl From for ResultSearchProject { gallery: source.gallery, featured_gallery: source.featured_gallery, color: source.color, - dependency_project_id: source.dependency_project_id, + dependency_project_ids: source.dependency_project_ids, dependencies: source.dependencies, loaders: source.loaders, project_loader_fields: source.project_loader_fields, diff --git a/apps/labrinth/tests/search.rs b/apps/labrinth/tests/search.rs index ce1f9ae8ce..9bcba177c8 100644 --- a/apps/labrinth/tests/search.rs +++ b/apps/labrinth/tests/search.rs @@ -91,7 +91,7 @@ async fn search_projects() { ), ( json!([[format!( - "dependency_project_id:{dependency_project_id}" + "dependency_project_ids:{dependency_project_id}" )]]), vec![7], ), @@ -140,15 +140,15 @@ async fn search_projects() { .search_deserialized( Some(&format!("&{test_name}")), Some(json!([[format!( - "dependency_project_id:{dependency_project_id}" + "dependency_project_ids:{dependency_project_id}" )]])), USER_USER_PAT, ) .await; assert_eq!(projects.total_hits, 1); - assert_eq!(projects.hits[0].dependency_project_id.len(), 1); + assert_eq!(projects.hits[0].dependency_project_ids.len(), 1); assert_eq!( - projects.hits[0].dependency_project_id[0], + projects.hits[0].dependency_project_ids[0], dependency_project_id ); assert_eq!(projects.hits[0].dependencies.len(), 1);