From f51d3b4e737ce7f61eb8546713dca68cbaab3084 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Tue, 2 Dec 2025 14:26:46 -0800 Subject: [PATCH 01/24] add icons --- frontend/src/assets/icons/file-earmark-scan.svg | 8 ++++++++ frontend/src/assets/icons/file-earmark-scan2.svg | 7 +++++++ frontend/src/assets/icons/file-earmark-scan3.svg | 6 ++++++ 3 files changed, 21 insertions(+) create mode 100644 frontend/src/assets/icons/file-earmark-scan.svg create mode 100644 frontend/src/assets/icons/file-earmark-scan2.svg create mode 100644 frontend/src/assets/icons/file-earmark-scan3.svg diff --git a/frontend/src/assets/icons/file-earmark-scan.svg b/frontend/src/assets/icons/file-earmark-scan.svg new file mode 100644 index 0000000000..408f744403 --- /dev/null +++ b/frontend/src/assets/icons/file-earmark-scan.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/frontend/src/assets/icons/file-earmark-scan2.svg b/frontend/src/assets/icons/file-earmark-scan2.svg new file mode 100644 index 0000000000..4fa3f2d470 --- /dev/null +++ b/frontend/src/assets/icons/file-earmark-scan2.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/frontend/src/assets/icons/file-earmark-scan3.svg b/frontend/src/assets/icons/file-earmark-scan3.svg new file mode 100644 index 0000000000..a2988adca8 --- /dev/null +++ b/frontend/src/assets/icons/file-earmark-scan3.svg @@ -0,0 +1,6 @@ + + + + + + From 0a652ca11fb15e54f8635ee15aa5461927f4bd4b Mon Sep 17 00:00:00 2001 From: sua yoo Date: Tue, 2 Dec 2025 14:32:07 -0800 Subject: [PATCH 02/24] add state --- frontend/src/features/archived-items/crawl-status.ts | 5 ++++- frontend/src/types/crawlState.ts | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/features/archived-items/crawl-status.ts b/frontend/src/features/archived-items/crawl-status.ts index 9c7b70eb2a..d69921fcd5 100644 --- a/frontend/src/features/archived-items/crawl-status.ts +++ b/frontend/src/features/archived-items/crawl-status.ts @@ -95,6 +95,7 @@ export class CrawlStatus extends TailwindElement { case "waiting_capacity": case "waiting_org_limit": + case "waiting_dedupe_index": color = "var(--sl-color-violet-600)"; icon = html` Date: Wed, 3 Dec 2025 13:35:21 -0800 Subject: [PATCH 03/24] add badges --- .../src/assets/icons/file-earmark-scan.svg | 6 +- .../archived-item-list-item.ts | 24 ++++++- .../src/features/collections/dedupe-badge.ts | 72 +++++++++++++++++++ .../collections/dedupe-source-badge.ts | 25 +++++++ frontend/src/features/collections/index.ts | 2 + .../linked-collections-list-item.ts | 6 +- .../archived-item-detail/templates/badges.ts | 8 ++- frontend/src/pages/org/workflow-detail.ts | 18 ++++- .../src/stories/components/Badge.stories.ts | 7 ++ .../stories/components/DedupeBadge.stories.ts | 35 +++++++++ .../src/stories/components/DedupeBadge.ts | 14 ++++ frontend/src/types/crawler.ts | 2 + 12 files changed, 209 insertions(+), 10 deletions(-) create mode 100644 frontend/src/features/collections/dedupe-badge.ts create mode 100644 frontend/src/features/collections/dedupe-source-badge.ts create mode 100644 frontend/src/stories/components/DedupeBadge.stories.ts create mode 100644 frontend/src/stories/components/DedupeBadge.ts diff --git a/frontend/src/assets/icons/file-earmark-scan.svg b/frontend/src/assets/icons/file-earmark-scan.svg index 408f744403..ca3dc8cc7e 100644 --- a/frontend/src/assets/icons/file-earmark-scan.svg +++ b/frontend/src/assets/icons/file-earmark-scan.svg @@ -1,7 +1,7 @@ - - - + + + diff --git a/frontend/src/features/archived-items/archived-item-list/archived-item-list-item.ts b/frontend/src/features/archived-items/archived-item-list/archived-item-list-item.ts index 7cb3da3cd6..52993d23b5 100644 --- a/frontend/src/features/archived-items/archived-item-list/archived-item-list-item.ts +++ b/frontend/src/features/archived-items/archived-item-list/archived-item-list-item.ts @@ -1,5 +1,6 @@ import { localized, msg, str } from "@lit/localize"; import type { SlCheckbox, SlHideEvent } from "@shoelace-style/shoelace"; +import clsx from "clsx"; import { css, html, nothing } from "lit"; import { customElement, property, query } from "lit/decorators.js"; import { ifDefined } from "lit/directives/if-defined.js"; @@ -9,8 +10,9 @@ import type { ArchivedItemCheckedEvent } from "./types"; import { BtrixElement } from "@/classes/BtrixElement"; import { CrawlStatus } from "@/features/archived-items/crawl-status"; import { ReviewStatus, type ArchivedItem, type Crawl } from "@/types/crawler"; -import { renderName } from "@/utils/crawler"; +import { isCrawl, renderName } from "@/utils/crawler"; import { pluralOf } from "@/utils/pluralize"; +import { tw } from "@/utils/tailwind"; /** * @slot actionCell - Action cell @@ -100,6 +102,9 @@ export class ArchivedItemListItem extends BtrixElement { const qaStatus = CrawlStatus.getContent({ state: lastQAState || undefined, }); + const dedupeDependent = + isCrawl(this.item) && + (this.item.requiredByCrawls.length || this.item.requiresCrawls.length); return html` `} + + + + + + ${text} + + `; + } +} diff --git a/frontend/src/features/collections/dedupe-source-badge.ts b/frontend/src/features/collections/dedupe-source-badge.ts new file mode 100644 index 0000000000..f876eeb824 --- /dev/null +++ b/frontend/src/features/collections/dedupe-source-badge.ts @@ -0,0 +1,25 @@ +import { localized, msg } from "@lit/localize"; +import { html } from "lit"; +import { customElement } from "lit/decorators.js"; + +import { TailwindElement } from "@/classes/TailwindElement"; + +@customElement("btrix-dedupe-source-badge") +@localized() +export class DedupeSourceBadge extends TailwindElement { + render() { + return html` + + + ${msg("Dedupe")} + + `; + } +} diff --git a/frontend/src/features/collections/index.ts b/frontend/src/features/collections/index.ts index 2fd3fbe5fe..30f4f15c73 100644 --- a/frontend/src/features/collections/index.ts +++ b/frontend/src/features/collections/index.ts @@ -7,6 +7,8 @@ import("./collection-edit-dialog"); import("./collection-create-dialog"); import("./collection-initial-view-dialog"); import("./collection-workflow-list"); +import("./dedupe-badge"); +import("./dedupe-source-badge"); import("./linked-collections"); import("./select-collection-access"); import("./select-collection-page"); diff --git a/frontend/src/features/collections/linked-collections/linked-collections-list-item.ts b/frontend/src/features/collections/linked-collections/linked-collections-list-item.ts index 6a2d039426..78794d46a2 100644 --- a/frontend/src/features/collections/linked-collections/linked-collections-list-item.ts +++ b/frontend/src/features/collections/linked-collections/linked-collections-list-item.ts @@ -39,15 +39,15 @@ export class LinkedCollectionsListItem extends TailwindElement { const actual = isActualCollection(item); const dedupeEnabled = this.dedupeSource; + console.log(dedupeEnabled); + const content = [ html`
${item.name}
${dedupeEnabled - ? html` - ${msg("Dedupe Source")} - ` + ? html`` : nothing}
`, ]; diff --git a/frontend/src/pages/org/archived-item-detail/templates/badges.ts b/frontend/src/pages/org/archived-item-detail/templates/badges.ts index fdf2dd3c95..c2fb836f8e 100644 --- a/frontend/src/pages/org/archived-item-detail/templates/badges.ts +++ b/frontend/src/pages/org/archived-item-detail/templates/badges.ts @@ -48,7 +48,13 @@ const qaReviewBadge = (reviewStatus: ArchivedItem["reviewStatus"]) => html` export const badges = (item: ArchivedItem) => { return html`
${itemTypeBadge(item.type)} - ${isCrawl(item) ? html`${qaReviewBadge(item.reviewStatus)}` : nothing} + ${isCrawl(item) + ? html`${qaReviewBadge(item.reviewStatus)} + ` + : nothing} ${collectionBadge(item.collectionIds.length > 0)}
`; }; diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index aae3defed9..3dc7356b4d 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -683,7 +683,12 @@ export class WorkflowDetail extends BtrixElement {
-

${this.tabLabels[tab]}

+
+

${this.tabLabels[tab]}

+ ${tab === WorkflowTab.LatestCrawl + ? this.renderDedupeBadge() + : nothing} +
${this.renderPanelAction()}
@@ -1387,6 +1392,17 @@ export class WorkflowDetail extends BtrixElement { `; }; + private readonly renderDedupeBadge = () => { + const latestCrawl = this.latestCrawlTask.value; + + if (!latestCrawl) return; + + return html``; + }; + private readonly renderPausedNotice = ( { truncate } = { truncate: false }, ) => { diff --git a/frontend/src/stories/components/Badge.stories.ts b/frontend/src/stories/components/Badge.stories.ts index a0a2436115..b8afc7c9b0 100644 --- a/frontend/src/stories/components/Badge.stories.ts +++ b/frontend/src/stories/components/Badge.stories.ts @@ -6,6 +6,8 @@ import { renderComponent, type RenderProps } from "./Badge"; import "@/features/crawls/crawler-channel-badge"; import "@/features/crawls/proxy-badge"; +import "@/features/collections/dedupe-badge"; +import "@/features/collections/dedupe-source-badge"; const meta = { title: "Components/Badge", @@ -131,5 +133,10 @@ export const FeatureBadges: Story = { + + `, }; diff --git a/frontend/src/stories/components/DedupeBadge.stories.ts b/frontend/src/stories/components/DedupeBadge.stories.ts new file mode 100644 index 0000000000..67cc75f8e7 --- /dev/null +++ b/frontend/src/stories/components/DedupeBadge.stories.ts @@ -0,0 +1,35 @@ +import type { Meta, StoryObj } from "@storybook/web-components"; + +import { renderComponent, type RenderProps } from "./DedupeBadge"; + +const meta = { + title: "Features/Dedupe Badge", + component: "btrix-dedupe-badge", + tags: ["autodocs"], + decorators: [], + render: renderComponent, + argTypes: {}, + args: {}, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const Dependents: Story = { + args: { + dependents: ["crawl1", "crawl2"], + }, +}; + +export const Dependencies: Story = { + args: { + dependencies: ["crawl1"], + }, +}; + +export const Both: Story = { + args: { + dependents: ["crawl1", "crawl2"], + dependencies: ["crawl1"], + }, +}; diff --git a/frontend/src/stories/components/DedupeBadge.ts b/frontend/src/stories/components/DedupeBadge.ts new file mode 100644 index 0000000000..9eee109f16 --- /dev/null +++ b/frontend/src/stories/components/DedupeBadge.ts @@ -0,0 +1,14 @@ +import { html } from "lit"; + +import type { DedupeBadge } from "@/features/collections/dedupe-badge"; + +import "@/features/collections/dedupe-badge"; + +export type RenderProps = DedupeBadge; + +export const renderComponent = (props: Partial) => { + return html``; +}; diff --git a/frontend/src/types/crawler.ts b/frontend/src/types/crawler.ts index e511355aca..796d4a8741 100644 --- a/frontend/src/types/crawler.ts +++ b/frontend/src/types/crawler.ts @@ -207,6 +207,8 @@ export type Crawl = ArchivedItemBase & browserWindows: number; shouldPause: boolean | null; resources?: (StorageFile & { numReplicas: number })[]; + requiresCrawls: string[]; + requiredByCrawls: string[]; }; export type Upload = ArchivedItemBase & { From 79653839c0c09a358ef74a463a43bbd2d394ddf1 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Wed, 3 Dec 2025 19:04:16 -0800 Subject: [PATCH 04/24] add notice --- frontend/src/components/ui/link.ts | 7 ++- .../templates/dedupe-files-notice.ts | 41 ++++++++++++++++ .../templates/dedupe-qa-notice.ts | 25 ++++++++++ .../templates/dedupe-replay-notice.ts | 41 ++++++++++++++++ .../src/features/collections/dedupe-badge.ts | 8 +++- .../collections/dedupe-source-badge.ts | 8 +++- .../linked-collections-list-item.ts | 11 ++--- .../archived-item-detail.ts | 48 +++++++++++++------ frontend/src/pages/org/workflow-detail.ts | 13 ++++- 9 files changed, 176 insertions(+), 26 deletions(-) create mode 100644 frontend/src/features/archived-items/templates/dedupe-files-notice.ts create mode 100644 frontend/src/features/archived-items/templates/dedupe-qa-notice.ts create mode 100644 frontend/src/features/archived-items/templates/dedupe-replay-notice.ts diff --git a/frontend/src/components/ui/link.ts b/frontend/src/components/ui/link.ts index 73f1c46c45..6ad1869dff 100644 --- a/frontend/src/components/ui/link.ts +++ b/frontend/src/components/ui/link.ts @@ -5,6 +5,9 @@ import { ifDefined } from "lit/directives/if-defined.js"; import { BtrixElement } from "@/classes/BtrixElement"; +/** + * @cssPart base + */ @customElement("btrix-link") export class Link extends BtrixElement { @property({ type: String }) @@ -17,7 +20,7 @@ export class Link extends BtrixElement { rel?: HTMLAnchorElement["rel"]; @property({ type: String }) - variant: "primary" | "neutral" = "neutral"; + variant: "primary" | "warning" | "neutral" = "neutral"; @property({ type: Boolean }) hideIcon = false; @@ -31,6 +34,7 @@ export class Link extends BtrixElement { "group inline-flex items-center gap-1 transition-colors duration-fast", { primary: "text-primary-500 hover:text-primary-600", + warning: "text-warning-700 hover:text-warning-800", neutral: "text-blue-500 hover:text-blue-600", }[this.variant], )} @@ -40,6 +44,7 @@ export class Link extends BtrixElement { @click=${this.target === "_blank" || this.href.startsWith("http") ? () => {} : this.navigate.link} + part="base" > ${this.hideIcon diff --git a/frontend/src/features/archived-items/templates/dedupe-files-notice.ts b/frontend/src/features/archived-items/templates/dedupe-files-notice.ts new file mode 100644 index 0000000000..979d815b2d --- /dev/null +++ b/frontend/src/features/archived-items/templates/dedupe-files-notice.ts @@ -0,0 +1,41 @@ +import { msg } from "@lit/localize"; +import { html } from "lit"; +import { when } from "lit/directives/when.js"; + +export function dedupeFilesNotice({ href }: { href?: string } = {}) { + return html` +
+ + + + ${msg("This crawl is dependent on other crawls.")} + + +
+
+

+ ${msg( + "Files may contain incomplete or missing content due to deduplication.", + )} +

+ ${when( + href, + (href) => + html`

+ ${msg( + "Download the complete and deduplicated files in the collection.", + )} +

+ ${msg("Go to Collection")}`, + )} +
+ `; +} diff --git a/frontend/src/features/archived-items/templates/dedupe-qa-notice.ts b/frontend/src/features/archived-items/templates/dedupe-qa-notice.ts new file mode 100644 index 0000000000..7511387fb1 --- /dev/null +++ b/frontend/src/features/archived-items/templates/dedupe-qa-notice.ts @@ -0,0 +1,25 @@ +import { msg } from "@lit/localize"; +import { html } from "lit"; + +export function dedupeQANotice() { + return html` +
+ + + + ${msg("This crawl is dependent on other crawls.")} + + +
+
+

+ ${msg( + "Quality assurance tools are not currently supported for deduplication dependent crawls.", + )} +

+
+ `; +} diff --git a/frontend/src/features/archived-items/templates/dedupe-replay-notice.ts b/frontend/src/features/archived-items/templates/dedupe-replay-notice.ts new file mode 100644 index 0000000000..a8bdb8a4ea --- /dev/null +++ b/frontend/src/features/archived-items/templates/dedupe-replay-notice.ts @@ -0,0 +1,41 @@ +import { msg } from "@lit/localize"; +import { html } from "lit"; +import { when } from "lit/directives/when.js"; + +export function dedupeReplayNotice({ href }: { href?: string } = {}) { + return html` +
+ + + + ${msg("This crawl is dependent on other crawls.")} + + +
+
+

+ ${msg( + "Replay for this crawl may contain incomplete or missing pages due to its dependency of the deduplication source.", + )} +

+ ${when( + href, + (href) => + html`

+ ${msg( + "Replay the collection to view the complete and deduplicated crawl.", + )} +

+ ${msg("Go to Collection")}`, + )} +
+ `; +} diff --git a/frontend/src/features/collections/dedupe-badge.ts b/frontend/src/features/collections/dedupe-badge.ts index 730b7d82ae..35072e92a7 100644 --- a/frontend/src/features/collections/dedupe-badge.ts +++ b/frontend/src/features/collections/dedupe-badge.ts @@ -1,5 +1,5 @@ import { localized, msg, str } from "@lit/localize"; -import { html } from "lit"; +import { css, html } from "lit"; import { customElement, property } from "lit/decorators.js"; import { TailwindElement } from "@/classes/TailwindElement"; @@ -21,6 +21,12 @@ export const dedupeLabelFor = { @customElement("btrix-dedupe-badge") @localized() export class DedupeBadge extends TailwindElement { + static styles = css` + :host { + display: contents; + } + `; + @property({ type: Array }) dependents?: string[] = []; diff --git a/frontend/src/features/collections/dedupe-source-badge.ts b/frontend/src/features/collections/dedupe-source-badge.ts index f876eeb824..4bcf002c6e 100644 --- a/frontend/src/features/collections/dedupe-source-badge.ts +++ b/frontend/src/features/collections/dedupe-source-badge.ts @@ -1,5 +1,5 @@ import { localized, msg } from "@lit/localize"; -import { html } from "lit"; +import { css, html } from "lit"; import { customElement } from "lit/decorators.js"; import { TailwindElement } from "@/classes/TailwindElement"; @@ -7,6 +7,12 @@ import { TailwindElement } from "@/classes/TailwindElement"; @customElement("btrix-dedupe-source-badge") @localized() export class DedupeSourceBadge extends TailwindElement { + static styles = css` + :host { + display: contents; + } + `; + render() { return html` - ${item.crawlCount} - ${pluralOf("items", item.crawlCount)} - `, + html`${item.crawlCount} ${pluralOf("items", item.crawlCount)}`, ); } diff --git a/frontend/src/pages/org/archived-item-detail/archived-item-detail.ts b/frontend/src/pages/org/archived-item-detail/archived-item-detail.ts index a2d2fc7e3e..601176ab15 100644 --- a/frontend/src/pages/org/archived-item-detail/archived-item-detail.ts +++ b/frontend/src/pages/org/archived-item-detail/archived-item-detail.ts @@ -13,6 +13,9 @@ import { BtrixElement } from "@/classes/BtrixElement"; import { type Dialog } from "@/components/ui/dialog"; import { ClipboardController } from "@/controllers/clipboard"; import type { CrawlMetadataEditor } from "@/features/archived-items/item-metadata-editor"; +import { dedupeFilesNotice } from "@/features/archived-items/templates/dedupe-files-notice"; +import { dedupeQANotice } from "@/features/archived-items/templates/dedupe-qa-notice"; +import { dedupeReplayNotice } from "@/features/archived-items/templates/dedupe-replay-notice"; import { pageBack, pageHeader, @@ -313,6 +316,8 @@ export class ArchivedItemDetail extends BtrixElement { render() { const authToken = this.authState?.headers.Authorization.split(" ")[1]; const isSuccess = this.item && isSuccessfullyFinished(this.item); + const dedupeDependent = + this.item && isCrawl(this.item) && this.item.requiresCrawls.length; let sectionContent: string | TemplateResult<1> = ""; @@ -327,20 +332,22 @@ export class ArchivedItemDetail extends BtrixElement { html`${this.tabLabels.qa} `, )}
- ${when(this.qaRuns, this.renderQAHeader)} + ${when(!dedupeDependent && this.qaRuns, this.renderQAHeader)}
`, - html` - void this.fetchQARuns()} - > - `, + dedupeDependent + ? dedupeQANotice() + : html` + void this.fetchQARuns()} + > + `, ); break; } @@ -348,7 +355,6 @@ export class ArchivedItemDetail extends BtrixElement { sectionContent = this.renderPanel( this.tabLabels.replay, this.renderReplay(), - [tw`overflow-hidden rounded-lg border`], ); break; case "files": @@ -838,6 +844,16 @@ export class ArchivedItemDetail extends BtrixElement { } private renderReplay() { + const dedupeDependent = + this.item && isCrawl(this.item) && this.item.requiresCrawls.length; + + return html` + ${dedupeDependent ? dedupeReplayNotice() : nothing} +
${this.renderRWP()}
+ `; + } + + private renderRWP() { if (!this.item) return; const replaySource = `/api/orgs/${this.item.oid}/${ this.item.type === "upload" ? "uploads" : "crawls" @@ -1061,7 +1077,11 @@ export class ArchivedItemDetail extends BtrixElement { } private renderFiles() { + const dedupeDependent = + this.item && isCrawl(this.item) && this.item.requiresCrawls.length; + return html` + ${this.hasFiles && dedupeDependent ? dedupeFilesNotice() : nothing} ${this.hasFiles ? html`