Skip to content

Commit cd32bcf

Browse files
committed
Add monthly and weekly downloads to recent downloads calculation
We calculate the recent downloads using a materialized view that we refresh every day. But it's only for last 90 days. Most of the other ecosystems like NPM, PyPi, Packagist, Hex provide daily, weekly and monthly downloads. So, I added a weekly and monthly column in the recent downloads view and surfaced the data in the `/api/v1/crates/{name}` endpoint.
1 parent e8cf9f7 commit cd32bcf

File tree

48 files changed

+168
-7
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+168
-7
lines changed

crates/crates_io_api_types/src/lib.rs

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,14 @@ pub struct EncodableCrate {
314314
#[schema(example = 456_789)]
315315
pub recent_downloads: Option<i64>,
316316

317+
/// The total number of downloads for this crate in the last month.
318+
#[schema(example = 123_456)]
319+
pub monthly_downloads: Option<i64>,
320+
321+
/// The total number of downloads for this crate in the last week.
322+
#[schema(example = 56_789)]
323+
pub weekly_downloads: Option<i64>,
324+
317325
/// The "default" version of this crate.
318326
///
319327
/// This version will be displayed by default on the crate's page.
@@ -380,6 +388,8 @@ impl EncodableCrate {
380388
exact_match: bool,
381389
downloads: i64,
382390
recent_downloads: Option<i64>,
391+
monthly_downloads: Option<i64>,
392+
weekly_downloads: Option<i64>,
383393
) -> Self {
384394
let Crate {
385395
name,
@@ -440,6 +450,8 @@ impl EncodableCrate {
440450
created_at,
441451
downloads,
442452
recent_downloads,
453+
monthly_downloads,
454+
weekly_downloads,
443455
versions,
444456
keywords: keyword_ids,
445457
categories: category_ids,
@@ -477,6 +489,8 @@ impl EncodableCrate {
477489
exact_match: bool,
478490
downloads: i64,
479491
recent_downloads: Option<i64>,
492+
monthly_downloads: Option<i64>,
493+
weekly_downloads: Option<i64>,
480494
) -> Self {
481495
Self::from(
482496
krate,
@@ -490,6 +504,8 @@ impl EncodableCrate {
490504
exact_match,
491505
downloads,
492506
recent_downloads,
507+
monthly_downloads,
508+
weekly_downloads,
493509
)
494510
}
495511
}
@@ -1187,6 +1203,8 @@ mod tests {
11871203
.and_utc(),
11881204
downloads: 0,
11891205
recent_downloads: None,
1206+
monthly_downloads: None,
1207+
weekly_downloads: None,
11901208
default_version: None,
11911209
num_versions: 0,
11921210
yanked: false,

crates/crates_io_database/src/schema.patch

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
/// The `id` column of the `dependencies` table.
4141
///
4242
/// Its SQL type is `Int4`.
43-
@@ -773,6 +767,24 @@
43+
@@ -773,6 +767,32 @@
4444
}
4545

4646
diesel::table! {
@@ -58,14 +58,22 @@
5858
+ ///
5959
+ /// Its SQL type is `BigInt`.
6060
+ downloads -> BigInt,
61+
+ /// The `monthly` column of the `recent_crate_downloads` table.
62+
+ ///
63+
+ /// Its SQL type is `BigInt`.
64+
+ monthly -> BigInt,
65+
+ /// The `weekly` column of the `recent_crate_downloads` table.
66+
+ ///
67+
+ /// Its SQL type is `BigInt`.
68+
+ weekly -> BigInt,
6169
+ }
6270
+}
6371
+
6472
+diesel::table! {
6573
use diesel::sql_types::*;
6674
use diesel_full_text_search::Tsvector;
6775

68-
@@ -1214,7 +1226,8 @@
76+
@@ -1214,7 +1234,8 @@
6977
diesel::joinable!(crate_downloads -> crates (crate_id));
7078
diesel::joinable!(crate_owner_invitations -> crates (crate_id));
7179
diesel::joinable!(crate_owners -> crates (crate_id));
@@ -75,15 +83,15 @@
7583
diesel::joinable!(crates_categories -> categories (category_id));
7684
diesel::joinable!(crates_categories -> crates (crate_id));
7785
diesel::joinable!(crates_keywords -> crates (crate_id));
78-
@@ -1230,6 +1243,7 @@
86+
@@ -1230,6 +1251,7 @@
7987
diesel::joinable!(publish_limit_buckets -> users (user_id));
8088
diesel::joinable!(publish_rate_overrides -> users (user_id));
8189
diesel::joinable!(readme_renderings -> versions (version_id));
8290
+diesel::joinable!(recent_crate_downloads -> crates (crate_id));
8391
diesel::joinable!(trustpub_configs_github -> crates (crate_id));
8492
diesel::joinable!(trustpub_configs_gitlab -> crates (crate_id));
8593
diesel::joinable!(version_downloads -> versions (version_id));
86-
@@ -1262,6 +1276,7 @@
94+
@@ -1262,6 +1284,7 @@
8795
publish_limit_buckets,
8896
publish_rate_overrides,
8997
readme_renderings,

crates/crates_io_database/src/schema.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -783,6 +783,14 @@ diesel::table! {
783783
///
784784
/// Its SQL type is `BigInt`.
785785
downloads -> BigInt,
786+
/// The `monthly` column of the `recent_crate_downloads` table.
787+
///
788+
/// Its SQL type is `BigInt`.
789+
monthly -> BigInt,
790+
/// The `weekly` column of the `recent_crate_downloads` table.
791+
///
792+
/// Its SQL type is `BigInt`.
793+
weekly -> BigInt,
786794
}
787795
}
788796

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
DROP MATERIALIZED VIEW recent_crate_downloads;
2+
CREATE MATERIALIZED VIEW recent_crate_downloads (crate_id, downloads) AS
3+
SELECT crate_id, SUM(version_downloads.downloads) FROM version_downloads
4+
INNER JOIN versions
5+
ON version_downloads.version_id = versions.id
6+
WHERE version_downloads.date > date(CURRENT_TIMESTAMP - INTERVAL '90 days')
7+
GROUP BY crate_id;
8+
CREATE UNIQUE INDEX recent_crate_downloads_crate_id ON recent_crate_downloads (crate_id);
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
DROP MATERIALIZED VIEW recent_crate_downloads;
2+
CREATE MATERIALIZED VIEW recent_crate_downloads (crate_id, downloads, monthly, weekly) AS
3+
SELECT
4+
crate_id,
5+
SUM(version_downloads.downloads),
6+
SUM(version_downloads.downloads) FILTER (WHERE version_downloads.date > CURRENT_DATE - 30),
7+
SUM(version_downloads.downloads) FILTER (WHERE version_downloads.date > CURRENT_DATE - 7)
8+
FROM version_downloads
9+
INNER JOIN versions
10+
ON version_downloads.version_id = versions.id
11+
WHERE version_downloads.date > CURRENT_DATE - 90
12+
GROUP BY crate_id;
13+
CREATE UNIQUE INDEX recent_crate_downloads_crate_id ON recent_crate_downloads (crate_id);

src/controllers/krate/metadata.rs

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,9 @@ pub async fn find_crate(
187187
cats.as_deref(),
188188
false,
189189
downloads,
190-
recent_downloads,
190+
recent_downloads.map(|(d, _, _)| d),
191+
recent_downloads.map(|(_, m, _)| m),
192+
recent_downloads.map(|(_, _, w)| w),
191193
);
192194

193195
let encodable_versions = versions_publishers_and_audit_actions.map(|vpa| {
@@ -289,14 +291,18 @@ fn load_recent_downloads(
289291
conn: &mut AsyncPgConnection,
290292
crate_id: i32,
291293
includes: bool,
292-
) -> BoxFuture<'_, AppResult<Option<i64>>> {
294+
) -> BoxFuture<'_, AppResult<Option<(i64, i64, i64)>>> {
293295
if !includes {
294296
return always_ready(|| Ok(None)).boxed();
295297
}
296298

297299
let fut = recent_crate_downloads::table
298300
.filter(recent_crate_downloads::crate_id.eq(crate_id))
299-
.select(recent_crate_downloads::downloads)
301+
.select((
302+
recent_crate_downloads::downloads,
303+
recent_crate_downloads::monthly,
304+
recent_crate_downloads::weekly,
305+
))
300306
.get_result(conn);
301307
async move { Ok(fut.await.optional()?) }.boxed()
302308
}

src/controllers/krate/publish.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -701,6 +701,8 @@ pub async fn publish(app: AppState, req: Parts, body: Body) -> AppResult<Json<Go
701701
false,
702702
downloads,
703703
None,
704+
None,
705+
None,
704706
),
705707
warnings,
706708
}))

src/controllers/krate/search.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,8 @@ pub async fn list_crates(
258258
record.exact_match,
259259
record.downloads,
260260
Some(record.recent_downloads.unwrap_or(0)),
261+
None,
262+
None,
261263
)
262264
})
263265
.collect::<Vec<_>>();

src/controllers/krate/update.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,8 @@ async fn update_inner(
189189
false,
190190
downloads,
191191
recent_downloads,
192+
None,
193+
None,
192194
);
193195

194196
Ok(Json(PatchResponse {

src/controllers/summary.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,8 @@ fn encode_crates(
181181
false,
182182
record.total_downloads,
183183
record.recent_downloads,
184+
None,
185+
None,
184186
))
185187
})
186188
.collect()

0 commit comments

Comments
 (0)