Logs: new materialized view and some UI improvements

apps/webapp/app/components/logs/LogsTable.tsx

@@ -54,8 +54,6 @@ function getLevelBoxShadow(level: LogEntry["level"]): string {
   }
 }
 
-
-
 export function LogsTable({
   logs,
   searchTerm,
@@ -162,7 +160,7 @@ export function LogsTable({
                       boxShadow: getLevelBoxShadow(log.level),
                     }}
                   >
-                    <DateTimeAccurate date={log.triggeredTimestamp} />
+                    <DateTimeAccurate date={log.triggeredTimestamp} hour12={false} />
                   </TableCell>
                   <TableCell className="min-w-24">
                     <TruncatedCopyableValue value={log.runId} />
@@ -233,11 +231,7 @@ function BlankState({ isLoading, onRefresh }: { isLoading?: boolean; onRefresh?:
           No logs match your filters. Try refreshing or modifying your filters.
         </Paragraph>
         <div className="flex items-center gap-2">
-          <Button
-            LeadingIcon={ArrowPathIcon}
-            variant="tertiary/medium"
-            onClick={handleRefresh}
-          >
+          <Button LeadingIcon={ArrowPathIcon} variant="tertiary/medium" onClick={handleRefresh}>
             Refresh
           </Button>
         </div>

apps/webapp/app/components/navigation/SideMenu.tsx

@@ -433,18 +433,7 @@ export function SideMenu({
                 data-action="deployments"
                 isCollapsed={isCollapsed}
               />
-              {(user.admin || user.isImpersonating || featureFlags.hasLogsPageAccess) && (
-                <SideMenuItem
-                  name="Logs"
-                  icon={LogsIcon}
-                  activeIconColor="text-logs"
-                  inactiveIconColor="text-logs"
-                  to={v3LogsPath(organization, project, environment)}
-                  data-action="logs"
-                  badge={<AlphaBadge />}
-                  isCollapsed={isCollapsed}
-                />
-              )}
+
               <SideMenuItem
                 name="Test"
                 icon={BeakerIcon}
@@ -467,6 +456,18 @@ export function SideMenu({
                 )}
                 onCollapseToggle={handleSectionToggle("metrics")}
               >
+                {(user.admin || user.isImpersonating || featureFlags.hasLogsPageAccess) && (
+                  <SideMenuItem
+                    name="Logs"
+                    icon={LogsIcon}
+                    activeIconColor="text-logs"
+                    inactiveIconColor="text-logs"
+                    to={v3LogsPath(organization, project, environment)}
+                    data-action="logs"
+                    badge={<AlphaBadge />}
+                    isCollapsed={isCollapsed}
+                  />
+                )}
                 <SideMenuItem
                   name="Query"
                   icon={TableCellsIcon}

apps/webapp/app/components/primitives/DateTime.tsx

@@ -196,7 +196,11 @@ export const SmartDateTime = ({ date, previousDate = null, hour12 = true }: Date
     ? formatSmartDateTime(realDate, userTimeZone, locales, hour12)
     : formatTimeOnly(realDate, userTimeZone, locales, hour12);
 
-  return <span suppressHydrationWarning>{formattedDateTime.replace(/\s/g, String.fromCharCode(32))}</span>;
+  return (
+    <span suppressHydrationWarning>
+      {formattedDateTime.replace(/\s/g, String.fromCharCode(32))}
+    </span>
+  );
 };
 
 // Helper function to check if two dates are on the same day
@@ -270,14 +274,18 @@ const DateTimeAccurateInner = ({
     return hideDate
       ? formatTimeOnly(realDate, displayTimeZone, locales, hour12)
       : realPrevDate
-        ? isSameDay(realDate, realPrevDate)
-          ? formatTimeOnly(realDate, displayTimeZone, locales, hour12)
-          : formatDateTimeAccurate(realDate, displayTimeZone, locales, hour12)
-        : formatDateTimeAccurate(realDate, displayTimeZone, locales, hour12);
+      ? isSameDay(realDate, realPrevDate)
+        ? formatTimeOnly(realDate, displayTimeZone, locales, hour12)
+        : formatDateTimeAccurate(realDate, displayTimeZone, locales, hour12)
+      : formatDateTimeAccurate(realDate, displayTimeZone, locales, hour12);
   }, [realDate, displayTimeZone, locales, hour12, hideDate, previousDate]);
 
   if (!showTooltip)
-    return <span suppressHydrationWarning>{formattedDateTime.replace(/\s/g, String.fromCharCode(32))}</span>;
+    return (
+      <span suppressHydrationWarning>
+        {formattedDateTime.replace(/\s/g, String.fromCharCode(32))}
+      </span>
+    );
 
   const tooltipContent = (
     <TooltipContent
@@ -290,7 +298,11 @@ const DateTimeAccurateInner = ({
 
   return (
     <SimpleTooltip
-      button={<span suppressHydrationWarning>{formattedDateTime.replace(/\s/g, String.fromCharCode(32))}</span>}
+      button={
+        <span suppressHydrationWarning>
+          {formattedDateTime.replace(/\s/g, String.fromCharCode(32))}
+        </span>
+      }
       content={tooltipContent}
       side="right"
       asChild={true}
@@ -326,9 +338,13 @@ function formatDateTimeAccurate(
   locales: string[],
   hour12: boolean = true
 ): string {
-  const formattedDateTime = new Intl.DateTimeFormat(locales, {
+  const datePart = new Intl.DateTimeFormat(locales, {
     month: "short",
     day: "numeric",
+    timeZone,
+  }).format(date);
+
+  const timePart = new Intl.DateTimeFormat(locales, {
     hour: "numeric",
     minute: "numeric",
     second: "numeric",
@@ -338,7 +354,7 @@ function formatDateTimeAccurate(
     hour12,
   }).format(date);
 
-  return formattedDateTime;
+  return `${datePart} ${timePart}`;
 }
 
 export const DateTimeShort = ({ date, hour12 = true }: DateTimeProps) => {
@@ -347,7 +363,11 @@ export const DateTimeShort = ({ date, hour12 = true }: DateTimeProps) => {
   const realDate = typeof date === "string" ? new Date(date) : date;
   const formattedDateTime = formatDateTimeShort(realDate, userTimeZone, locales, hour12);
 
-  return <span suppressHydrationWarning>{formattedDateTime.replace(/\s/g, String.fromCharCode(32))}</span>;
+  return (
+    <span suppressHydrationWarning>
+      {formattedDateTime.replace(/\s/g, String.fromCharCode(32))}
+    </span>
+  );
 };
 
 function formatDateTimeShort(

apps/webapp/app/components/runs/v3/SharedFilters.tsx

@@ -219,7 +219,7 @@ export function timeFilterFromTo(props: {
   from?: string | number;
   to?: string | number;
   defaultPeriod: string;
-}): { from: Date; to: Date } {
+}): { from: Date; to: Date; isDefault: boolean } {
   const time = timeFilters(props);
 
   const periodMs = time.period ? parse(time.period) : undefined;
@@ -228,27 +228,31 @@ export function timeFilterFromTo(props: {
     return {
       from: new Date(Date.now() - periodMs),
       to: new Date(),
+      isDefault: time.isDefault,
     };
   }
 
   if (time.from && time.to) {
     return {
       from: time.from,
       to: time.to,
+      isDefault: time.isDefault,
     };
   }
 
   if (time.from) {
     return {
       from: time.from,
       to: new Date(),
+      isDefault: time.isDefault,
     };
   }
 
   const defaultPeriodMs = parse(props.defaultPeriod) ?? 24 * 60 * 60 * 1_000;
   return {
     from: new Date(Date.now() - defaultPeriodMs),
     to: time.to ?? new Date(),
+    isDefault: time.isDefault,
   };
 }
 

apps/webapp/app/presenters/v3/LogsListPresenter.server.ts

@@ -1,13 +1,11 @@
 import { z } from "zod";
-import { type ClickHouse } from "@internal/clickhouse";
-import {
-  type PrismaClientOrTransaction,
-} from "@trigger.dev/database";
+import { type ClickHouse, type WhereCondition } from "@internal/clickhouse";
+import { type PrismaClientOrTransaction } from "@trigger.dev/database";
 import { EVENT_STORE_TYPES, getConfiguredEventRepository } from "~/v3/eventRepository/index.server";
 
 import parseDuration from "parse-duration";
 import { type Direction } from "~/components/ListPagination";
-import { timeFilters } from "~/components/runs/v3/SharedFilters";
+import { timeFilterFromTo, timeFilters } from "~/components/runs/v3/SharedFilters";
 import { findDisplayableEnvironment } from "~/models/runtimeEnvironment.server";
 import { getAllTaskIdentifiers } from "~/models/task.server";
 import { ServiceValidationError } from "~/v3/services/baseService.server";
@@ -28,14 +26,9 @@ type ErrorAttributes = {
 };
 
 function escapeClickHouseString(val: string): string {
-  return val
-    .replace(/\\/g, "\\\\")
-    .replace(/\//g, "\\/")
-    .replace(/%/g, "\\%")
-    .replace(/_/g, "\\_");
+  return val.replace(/\\/g, "\\\\").replace(/\//g, "\\/").replace(/%/g, "\\%").replace(/_/g, "\\_");
 }
 
-
 export type LogsListOptions = {
   userId?: string;
   projectId: string;
@@ -153,24 +146,16 @@ export class LogsListPresenter extends BasePresenter {
       retentionLimitDays,
     }: LogsListOptions
   ) {
-    const time = timeFilters({
+    const time = timeFilterFromTo({
       period,
       from,
       to,
-      defaultPeriod,
+      defaultPeriod: defaultPeriod ?? "1h",
     });
 
     let effectiveFrom = time.from;
     let effectiveTo = time.to;
 
-    if (!effectiveFrom && !effectiveTo && time.period) {
-      const periodMs = parseDuration(time.period);
-      if (periodMs) {
-        effectiveFrom = new Date(Date.now() - periodMs);
-        effectiveTo = new Date();
-      }
-    }
-
     // Apply retention limit if provided
     let wasClampedByRetention = false;
     if (retentionLimitDays !== undefined && effectiveFrom) {
@@ -236,6 +221,11 @@ export class LogsListPresenter extends BasePresenter {
 
     const queryBuilder = this.clickhouse.taskEventsSearch.logsListQueryBuilder();
 
+    // This should be removed once we clear the old inserts, 30 DAYS, the materialized view excludes events without trace_id)
+    queryBuilder.where("trace_id != ''", {
+      environmentId,
+    });
+
     queryBuilder.where("environment_id = {environmentId: String}", {
       environmentId,
     });
@@ -245,11 +235,10 @@ export class LogsListPresenter extends BasePresenter {
     });
     queryBuilder.where("project_id = {projectId: String}", { projectId });
 
-
     if (effectiveFrom) {
-        queryBuilder.where("triggered_timestamp >= {triggeredAtStart: DateTime64(3)}", {
-          triggeredAtStart: convertDateToClickhouseDateTime(effectiveFrom),
-        });
+      queryBuilder.where("triggered_timestamp >= {triggeredAtStart: DateTime64(3)}", {
+        triggeredAtStart: convertDateToClickhouseDateTime(effectiveFrom),
+      });
     }
 
     if (effectiveTo) {
@@ -278,50 +267,43 @@ export class LogsListPresenter extends BasePresenter {
       queryBuilder.where(
         "(lower(message) like {searchPattern: String} OR lower(attributes_text) like {searchPattern: String})",
         {
-          searchPattern: `%${searchTerm}%`
+          searchPattern: `%${searchTerm}%`,
         }
       );
     }
 
     if (levels && levels.length > 0) {
-      const conditions: string[] = [];
-      const params: Record<string, string[]> = {};
+      const conditions: WhereCondition[] = [];
 
-      for (const level of levels) {
-        const filter = levelToKindsAndStatuses(level);
-        const levelConditions: string[] = [];
+      for (let i = 0; i < levels.length; i++) {
+        const filter = levelToKindsAndStatuses(levels[i]);
 
         if (filter.kinds && filter.kinds.length > 0) {
-          const kindsKey = `kinds_${level}`;
-          let kindCondition = `kind IN {${kindsKey}: Array(String)}`;
-
-
-          kindCondition += ` AND status NOT IN {excluded_statuses: Array(String)}`;
-          params["excluded_statuses"] = ["ERROR", "CANCELLED"];
-
-
-          levelConditions.push(kindCondition);
-          params[kindsKey] = filter.kinds;
+          conditions.push({
+            clause: `kind IN {kinds_${i}: Array(String)} AND status NOT IN {excluded_statuses: Array(String)}`,
+            params: {
+              [`kinds_${i}`]: filter.kinds,
+              excluded_statuses: ["ERROR", "CANCELLED"],
+            },
+          });
         }
 
         if (filter.statuses && filter.statuses.length > 0) {
-          const statusesKey = `statuses_${level}`;
-          levelConditions.push(`status IN {${statusesKey}: Array(String)}`);
-          params[statusesKey] = filter.statuses;
-        }
-
-        if (levelConditions.length > 0) {
-          conditions.push(`(${levelConditions.join(" OR ")})`);
+          conditions.push({
+            clause: `status IN {statuses_${i}: Array(String)}`,
+            params: { [`statuses_${i}`]: filter.statuses },
+          });
         }
       }
 
-      if (conditions.length > 0) {
-        queryBuilder.where(`(${conditions.join(" OR ")})`, params);
-      }
+      queryBuilder.whereOr(conditions);
     }
 
-    // Cursor pagination using explicit lexicographic comparison
-    // Must mirror the ORDER BY columns: (organization_id, environment_id, triggered_timestamp, trace_id)
+    // Cursor-based pagination using lexicographic comparison on (triggered_timestamp, trace_id).
+    // Since ORDER BY is DESC, "next page" means rows that sort *after* the cursor, i.e. less-than.
+    // The OR handles the tiebreaker: rows with an earlier timestamp always qualify, and rows
+    // with the *same* timestamp only qualify if their trace_id is also smaller.
+    // Equivalent to: WHERE (triggered_timestamp, trace_id) < (cursor.triggered_timestamp, cursor.trace_id)
     const decodedCursor = cursor ? decodeCursor(cursor) : null;
     if (decodedCursor) {
       queryBuilder.where(
@@ -423,10 +405,13 @@ export class LogsListPresenter extends BasePresenter {
       hasFilters,
       hasAnyLogs: transformedLogs.length > 0,
       searchTerm: search,
-      retention: retentionLimitDays !== undefined ? {
-        limitDays: retentionLimitDays,
-        wasClamped: wasClampedByRetention,
-      } : undefined,
+      retention:
+        retentionLimitDays !== undefined
+          ? {
+              limitDays: retentionLimitDays,
+              wasClamped: wasClampedByRetention,
+            }
+          : undefined,
     };
   }
 }

apps/webapp/tailwind.config.js

@@ -166,7 +166,7 @@ const deployments = colors.green[500];
 const concurrency = colors.amber[500];
 const limits = colors.purple[500];
 const regions = colors.green[500];
-const logs = colors.blue[500];
+const logs = colors.pink[500];
 const tests = colors.lime[500];
 const apiKeys = colors.amber[500];
 const environmentVariables = colors.pink[500];