From a23b5d6b06bccb9789fbf0da727719614a5ac3e2 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 3 Nov 2022 10:01:14 -0400 Subject: [PATCH 1/5] feat: Refactor Dispatched Action db entity to use full-fat Activity * Instead of storing limited info about an Activity in the table just persist the full Activity with a relationship * Fixes issue on CM init where snoowrap needs to fetch all activities for dispatched actions in order to get permalinks and simplifies things in general --- src/Common/Entities/Activity.ts | 43 +++++++++++++++---- src/Common/Entities/AuthorEntity.ts | 9 +++- src/Common/Entities/DispatchedEntity.ts | 39 +++++++---------- src/Common/Entities/Subreddit.ts | 18 +++++++- .../Server/1667415256831-delayedReset.ts | 19 ++++++++ src/Subreddit/Manager.ts | 4 +- src/Subreddit/SubredditResources.ts | 12 ++++-- src/Web/assets/views/status.ejs | 2 +- src/util.ts | 2 +- 9 files changed, 106 insertions(+), 42 deletions(-) create mode 100644 src/Common/Migrations/Database/Server/1667415256831-delayedReset.ts diff --git a/src/Common/Entities/Activity.ts b/src/Common/Entities/Activity.ts index 585f6a2a..f366ec3e 100644 --- a/src/Common/Entities/Activity.ts +++ b/src/Common/Entities/Activity.ts @@ -1,4 +1,4 @@ -import {Entity, Column, ManyToOne, PrimaryColumn, OneToMany, Index} from "typeorm"; +import {Entity, Column, ManyToOne, PrimaryColumn, OneToMany, Index, DataSource, JoinColumn} from "typeorm"; import {AuthorEntity} from "./AuthorEntity"; import {Subreddit} from "./Subreddit"; import {CMEvent} from "./CMEvent"; @@ -6,6 +6,8 @@ import {asComment, getActivityAuthorName, parseRedditFullname, redditThingTypeTo import {activityReports, ActivityType, Report, SnoowrapActivity} from "../Infrastructure/Reddit"; import {ActivityReport} from "./ActivityReport"; import dayjs, {Dayjs} from "dayjs"; +import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients"; +import {Comment, Submission} from 'snoowrap/dist/objects'; export interface ActivityEntityOptions { id: string @@ -45,7 +47,7 @@ export class Activity { @Column({name: 'name'}) name!: string; - @ManyToOne(type => Subreddit, sub => sub.activities, {cascade: ['insert']}) + @ManyToOne(type => Subreddit, sub => sub.activities, {cascade: ['insert'], eager: true}) subreddit!: Subreddit; @Column("varchar", {length: 20}) @@ -58,17 +60,18 @@ export class Activity { @Column("text") permalink!: string; - @ManyToOne(type => AuthorEntity, author => author.activities, {cascade: ['insert']}) + @ManyToOne(type => AuthorEntity, author => author.activities, {cascade: ['insert'], eager: true}) author!: AuthorEntity; @OneToMany(type => CMEvent, act => act.activity) // note: we will create author property in the Photo class below actionedEvents!: CMEvent[] - @ManyToOne(type => Activity, obj => obj.comments, {nullable: true}) + @ManyToOne('Activity', 'comments', {nullable: true, cascade: ['insert']}) + @JoinColumn({name: 'submission_id'}) submission?: Activity; - @OneToMany(type => Activity, obj => obj.submission, {nullable: true}) - comments!: Activity[]; + @OneToMany('Activity', 'submission', {nullable: true}) + comments?: Activity[]; @OneToMany(type => ActivityReport, act => act.activity, {cascade: ['insert'], eager: true}) reports: ActivityReport[] | undefined @@ -151,10 +154,12 @@ export class Activity { return false; } - static fromSnoowrapActivity(subreddit: Subreddit, activity: SnoowrapActivity, lastKnownStateTimestamp?: dayjs.Dayjs | undefined) { + static async fromSnoowrapActivity(activity: SnoowrapActivity, options: fromSnoowrapOptions | undefined = {}) { + let submission: Activity | undefined; let type: ActivityType = 'submission'; let content: string; + const subreddit = await Subreddit.fromSnoowrap(activity.subreddit, options?.db); if(asComment(activity)) { type = 'comment'; content = activity.body; @@ -179,8 +184,30 @@ export class Activity { submission }); - entity.syncReports(activity, lastKnownStateTimestamp); + entity.syncReports(activity, options.lastKnownStateTimestamp); return entity; } + + toSnoowrap(client: ExtendedSnoowrap): SnoowrapActivity { + let act: SnoowrapActivity; + if(this.type === 'submission') { + act = new Submission({name: this.id, id: this.name}, client, false); + act.title = this.content; + } else { + act = new Comment({name: this.id, id: this.name}, client, false); + act.link_id = this.submission?.id as string; + act.body = this.content; + } + act.permalink = this.permalink; + act.subreddit = this.subreddit.toSnoowrap(client); + act.author = this.author.toSnoowrap(client); + + return act; + } +} + +export interface fromSnoowrapOptions { + lastKnownStateTimestamp?: dayjs.Dayjs | undefined + db?: DataSource } diff --git a/src/Common/Entities/AuthorEntity.ts b/src/Common/Entities/AuthorEntity.ts index 3b773972..a1b96bf1 100644 --- a/src/Common/Entities/AuthorEntity.ts +++ b/src/Common/Entities/AuthorEntity.ts @@ -1,5 +1,8 @@ import {Entity, Column, PrimaryColumn, OneToMany} from "typeorm"; import {Activity} from "./Activity"; +import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients"; +import {SnoowrapActivity} from "../Infrastructure/Reddit"; +import {RedditUser} from "snoowrap/dist/objects"; @Entity({name: 'Author'}) export class AuthorEntity { @@ -11,11 +14,15 @@ export class AuthorEntity { name!: string; @OneToMany(type => Activity, act => act.author) - activities!: Activity[] + activities!: Promise constructor(data?: any) { if(data !== undefined) { this.name = data.name; } } + + toSnoowrap(client: ExtendedSnoowrap): RedditUser { + return new RedditUser({name: this.name, id: this.id}, client, false); + } } diff --git a/src/Common/Entities/DispatchedEntity.ts b/src/Common/Entities/DispatchedEntity.ts index 0c61ef09..848814c7 100644 --- a/src/Common/Entities/DispatchedEntity.ts +++ b/src/Common/Entities/DispatchedEntity.ts @@ -6,7 +6,7 @@ import { ManyToOne, PrimaryColumn, BeforeInsert, - AfterLoad + AfterLoad, JoinColumn } from "typeorm"; import { ActivityDispatch @@ -22,15 +22,15 @@ import Comment from "snoowrap/dist/objects/Comment"; import {ColumnDurationTransformer} from "./Transformers"; import { RedditUser } from "snoowrap/dist/objects"; import {ActivitySourceTypes, DurationVal, NonDispatchActivitySourceValue, onExistingFoundBehavior} from "../Infrastructure/Atomic"; +import {Activity} from "./Activity"; @Entity({name: 'DispatchedAction'}) export class DispatchedEntity extends TimeAwareRandomBaseEntity { - @Column() - activityId!: string - - @Column() - author!: string + //@ManyToOne(type => Activity, obj => obj.dispatched, {cascade: ['insert'], eager: true, nullable: false}) + @ManyToOne(type => Activity, undefined, {cascade: ['insert'], eager: true, nullable: false}) + @JoinColumn({name: 'activityId'}) + activity!: Activity @Column({ type: 'int', @@ -82,11 +82,10 @@ export class DispatchedEntity extends TimeAwareRandomBaseEntity { }}) tardyTolerant!: boolean | Duration - constructor(data?: ActivityDispatch & { manager: ManagerEntity }) { + constructor(data?: HydratedActivityDispatch) { super(); if (data !== undefined) { - this.activityId = data.activity.name; - this.author = getActivityAuthorName(data.activity.author); + this.activity = data.activity; this.delay = data.delay; this.createdAt = data.queuedAt; this.type = data.type; @@ -151,20 +150,7 @@ export class DispatchedEntity extends TimeAwareRandomBaseEntity { } async toActivityDispatch(client: ExtendedSnoowrap): Promise { - const redditThing = parseRedditFullname(this.activityId); - if(redditThing === undefined) { - throw new Error(`Could not parse reddit ID from value '${this.activityId}'`); - } - let activity: Comment | Submission; - if (redditThing?.type === 'comment') { - // @ts-ignore - activity = await client.getComment(redditThing.id); - } else { - // @ts-ignore - activity = await client.getSubmission(redditThing.id); - } - activity.author = new RedditUser({name: this.author}, client, false); - activity.id = redditThing.id; + let activity = this.activity.toSnoowrap(client); return { id: this.id, queuedAt: this.createdAt, @@ -176,8 +162,13 @@ export class DispatchedEntity extends TimeAwareRandomBaseEntity { cancelIfQueued: this.cancelIfQueued, identifier: this.identifier, type: this.type, - author: this.author, + author: activity.author.name, dryRun: this.dryRun } } } + +export interface HydratedActivityDispatch extends Omit { + activity: Activity + manager: ManagerEntity +} diff --git a/src/Common/Entities/Subreddit.ts b/src/Common/Entities/Subreddit.ts index 53bbcb1f..69adeebd 100644 --- a/src/Common/Entities/Subreddit.ts +++ b/src/Common/Entities/Subreddit.ts @@ -1,5 +1,7 @@ -import {Entity, Column, PrimaryColumn, OneToMany, Index} from "typeorm"; +import {Entity, Column, PrimaryColumn, OneToMany, Index, DataSource} from "typeorm"; import {Activity} from "./Activity"; +import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients"; +import {Subreddit as SnoowrapSubreddit} from "snoowrap/dist/objects"; export interface SubredditEntityOptions { id: string @@ -25,4 +27,18 @@ export class Subreddit { this.name = data.name; } } + + toSnoowrap(client: ExtendedSnoowrap): SnoowrapSubreddit { + return new SnoowrapSubreddit({display_name: this.name, name: this.id}, client, false); + } + + static async fromSnoowrap(subreddit: SnoowrapSubreddit, db?: DataSource) { + if(db !== undefined) { + const existing = await db.getRepository(Subreddit).findOneBy({name: subreddit.display_name}); + if(existing) { + return existing; + } + } + return new Subreddit({id: await subreddit.name, name: await subreddit.display_name}); + } } diff --git a/src/Common/Migrations/Database/Server/1667415256831-delayedReset.ts b/src/Common/Migrations/Database/Server/1667415256831-delayedReset.ts new file mode 100644 index 00000000..3681d2ee --- /dev/null +++ b/src/Common/Migrations/Database/Server/1667415256831-delayedReset.ts @@ -0,0 +1,19 @@ +import {MigrationInterface, QueryRunner, TableColumn} from "typeorm" + +export class delayedReset1667415256831 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + queryRunner.connection.logger.logSchemaBuild('Truncating (removing) existing Dispatched Actions due to internal structural changes'); + await queryRunner.clearTable('DispatchedAction'); + await queryRunner.changeColumn('DispatchedAction', 'author', new TableColumn({ + name: 'author', + type: 'varchar', + length: '150', + isNullable: true + })); + } + + public async down(queryRunner: QueryRunner): Promise { + } + +} diff --git a/src/Subreddit/Manager.ts b/src/Subreddit/Manager.ts index 77b6e6f6..d948a278 100644 --- a/src/Subreddit/Manager.ts +++ b/src/Subreddit/Manager.ts @@ -975,7 +975,7 @@ export class Manager extends EventEmitter implements RunningStates { let shouldPersistReports = false; if (existingEntity === null) { - activityEntity = Activity.fromSnoowrapActivity(this.managerEntity.subreddit, activity, lastKnownStateTimestamp); + activityEntity = await Activity.fromSnoowrapActivity(activity, {lastKnownStateTimestamp, db: this.resources.database}); // always persist if activity is not already persisted and any reports exist if (item.num_reports > 0) { shouldPersistReports = true; @@ -1189,7 +1189,7 @@ export class Manager extends EventEmitter implements RunningStates { // @ts-ignore const subProxy = await this.client.getSubmission((item as Comment).link_id); const sub = await this.resources.getActivity(subProxy); - subActivity = await this.activityRepo.save(Activity.fromSnoowrapActivity(this.managerEntity.subreddit, sub)); + subActivity = await this.activityRepo.save(await Activity.fromSnoowrapActivity(sub, {db: this.resources.database})); } event.activity.submission = subActivity; diff --git a/src/Subreddit/SubredditResources.ts b/src/Subreddit/SubredditResources.ts index 216f99cb..d41e8e91 100644 --- a/src/Subreddit/SubredditResources.ts +++ b/src/Subreddit/SubredditResources.ts @@ -162,6 +162,7 @@ import {ActivitySource} from "../Common/ActivitySource"; import {SubredditResourceOptions} from "../Common/Subreddit/SubredditResourceInterfaces"; import {SubredditStats} from "./Stats"; import {CMCache} from "../Common/Cache"; +import { Activity } from '../Common/Entities/Activity'; export const DEFAULT_FOOTER = '\r\n*****\r\nThis action was performed by [a bot.]({{botLink}}) Mention a moderator or [send a modmail]({{modmailLink}}) if you have any ideas, questions, or concerns about this action.'; @@ -404,7 +405,10 @@ export class SubredditResources { } }, relations: { - manager: true + manager: true, + activity: { + submission: true + } } }); const now = dayjs(); @@ -412,7 +416,7 @@ export class SubredditResources { const shouldDispatchAt = dAct.createdAt.add(dAct.delay.asSeconds(), 'seconds'); let tardyHint = ''; if(shouldDispatchAt.isBefore(now)) { - let tardyHint = `Activity ${dAct.activityId} queued at ${dAct.createdAt.format('YYYY-MM-DD HH:mm:ssZ')} for ${dAct.delay.humanize()} is now LATE`; + let tardyHint = `Activity ${dAct.activity.id} queued at ${dAct.createdAt.format('YYYY-MM-DD HH:mm:ssZ')} for ${dAct.delay.humanize()} is now LATE`; if(dAct.tardyTolerant === true) { tardyHint += ` but was configured as ALWAYS 'tardy tolerant' so will be dispatched immediately`; } else if(dAct.tardyTolerant === false) { @@ -439,14 +443,14 @@ export class SubredditResources { try { this.delayedItems.push(await dAct.toActivityDispatch(this.client)) } catch (e) { - this.logger.warn(new ErrorWithCause(`Unable to add Activity ${dAct.activityId} from database delayed activities to in-app delayed activities queue`, {cause: e})); + this.logger.warn(new ErrorWithCause(`Unable to add Activity ${dAct.activity.id} from database delayed activities to in-app delayed activities queue`, {cause: e})); } } } } async addDelayedActivity(data: ActivityDispatch) { - const dEntity = await this.dispatchedActivityRepo.save(new DispatchedEntity({...data, manager: this.managerEntity})); + const dEntity = await this.dispatchedActivityRepo.save(new DispatchedEntity({...data, manager: this.managerEntity, activity: await Activity.fromSnoowrapActivity(data.activity, {db: this.database})})); data.id = dEntity.id; this.delayedItems.push(data); } diff --git a/src/Web/assets/views/status.ejs b/src/Web/assets/views/status.ejs index b9aa0ae2..eb679b28 100644 --- a/src/Web/assets/views/status.ejs +++ b/src/Web/assets/views/status.ejs @@ -1297,7 +1297,7 @@ const durationDayjs = dayjs.duration(x.duration, 'seconds'); const durationDisplay = durationDayjs.humanize(); const cancelLink = `CANCEL`; - return `
A ${x.submissionId !== undefined ? 'Comment' : 'Submission'}${isAll ? ` in ${x.subreddit} ` : ''} by ${x.author} queued by ${x.source} at ${queuedAtDisplay} for ${durationDisplay} (dispatches ${durationUntilNow.humanize(true)}) -- ${cancelLink}
`; + return `
A ${x.submissionId !== undefined ? 'Comment' : 'Submission'} by ${x.author}${isAll ? `, dispatched in ${x.subreddit} ,` : ''} queued by ${x.source} at ${queuedAtDisplay} for ${durationDisplay} (dispatches ${durationUntilNow.humanize(true)}) -- ${cancelLink}
`; }); //let sub = resp.name; if(sub === 'All') { diff --git a/src/util.ts b/src/util.ts index bada909e..d4e2f259 100644 --- a/src/util.ts +++ b/src/util.ts @@ -2832,7 +2832,7 @@ export const generateSnoowrapEntityFromRedditThing = (data: RedditThing, client: case 'user': return new RedditUser({id: data.val}, client, false); case 'subreddit': - return new Subreddit({id: data.val}, client, false); + return new Subreddit({name: data.val}, client, false); case 'message': return new PrivateMessage({id: data.val}, client, false) From e05f350b37d8d7e8e28b48197b7232299c8b65f5 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 3 Nov 2022 13:42:01 -0400 Subject: [PATCH 2/5] feat: Implement orphaned activity cleanup on delayed activity deletion Make sure we delete Activities that were inserted on dispatched actions BUT ONLY if they are not used anywhere else (events or other delayed activities) --- src/Common/Entities/Activity.ts | 2 +- src/Subreddit/SubredditResources.ts | 116 +++++++++++++++++++++++++--- 2 files changed, 108 insertions(+), 10 deletions(-) diff --git a/src/Common/Entities/Activity.ts b/src/Common/Entities/Activity.ts index f366ec3e..194c47cf 100644 --- a/src/Common/Entities/Activity.ts +++ b/src/Common/Entities/Activity.ts @@ -63,7 +63,7 @@ export class Activity { @ManyToOne(type => AuthorEntity, author => author.activities, {cascade: ['insert'], eager: true}) author!: AuthorEntity; - @OneToMany(type => CMEvent, act => act.activity) // note: we will create author property in the Photo class below + @OneToMany(type => CMEvent, act => act.activity) actionedEvents!: CMEvent[] @ManyToOne('Activity', 'comments', {nullable: true, cascade: ['insert']}) diff --git a/src/Subreddit/SubredditResources.ts b/src/Subreddit/SubredditResources.ts index d41e8e91..957d9b07 100644 --- a/src/Subreddit/SubredditResources.ts +++ b/src/Subreddit/SubredditResources.ts @@ -69,7 +69,16 @@ import {cacheTTLDefaults, createHistoricalDisplayDefaults,} from "../Common/defa import {ExtendedSnoowrap} from "../Utils/SnoowrapClients"; import dayjs, {Dayjs} from "dayjs"; import ImageData from "../Common/ImageData"; -import {Between, DataSource, DeleteQueryBuilder, LessThan, Repository, SelectQueryBuilder} from "typeorm"; +import { + Between, Brackets, + DataSource, + DeleteQueryBuilder, + In, + LessThan, + NotBrackets, + Repository, + SelectQueryBuilder +} from "typeorm"; import {CMEvent as ActionedEventEntity, CMEvent} from "../Common/Entities/CMEvent"; import {RuleResultEntity} from "../Common/Entities/RuleResultEntity"; import globrex from 'globrex'; @@ -163,6 +172,7 @@ import {SubredditResourceOptions} from "../Common/Subreddit/SubredditResourceInt import {SubredditStats} from "./Stats"; import {CMCache} from "../Common/Cache"; import { Activity } from '../Common/Entities/Activity'; +import {FindOptionsWhere} from "typeorm/find-options/FindOptionsWhere"; export const DEFAULT_FOOTER = '\r\n*****\r\nThis action was performed by [a bot.]({{botLink}}) Mention a moderator or [send a modmail]({{modmailLink}}) if you have any ideas, questions, or concerns about this action.'; @@ -194,6 +204,7 @@ export class SubredditResources { botAccount?: string; dispatchedActivityRepo: Repository activitySourceRepo: Repository + activityRepo: Repository retention?: EventRetentionPolicyRange managerEntity: ManagerEntity botEntity: Bot @@ -230,6 +241,7 @@ export class SubredditResources { this.database = database; this.dispatchedActivityRepo = this.database.getRepository(DispatchedEntity); this.activitySourceRepo = this.database.getRepository(ActivitySourceEntity); + this.activityRepo = this.database.getRepository(Activity); this.retention = retention; //this.prefix = prefix; this.client = client; @@ -412,6 +424,7 @@ export class SubredditResources { } }); const now = dayjs(); + const toRemove = []; for(const dAct of dispatchedActivities) { const shouldDispatchAt = dAct.createdAt.add(dAct.delay.asSeconds(), 'seconds'); let tardyHint = ''; @@ -422,7 +435,7 @@ export class SubredditResources { } else if(dAct.tardyTolerant === false) { tardyHint += ` and was not configured as 'tardy tolerant' so will be dropped`; this.logger.warn(tardyHint); - await this.removeDelayedActivity(dAct.id); + toRemove.push(dAct.id); continue; } else { // see if its within tolerance @@ -430,7 +443,7 @@ export class SubredditResources { if(latest.isBefore(now)) { tardyHint += ` and IS NOT within tardy tolerance of ${dAct.tardyTolerant.humanize()} of planned dispatch time so will be dropped`; this.logger.warn(tardyHint); - await this.removeDelayedActivity(dAct.id); + toRemove.push(dAct.id); continue; } else { tardyHint += `but is within tardy tolerance of ${dAct.tardyTolerant.humanize()} of planned dispatch time so will be dispatched immediately`; @@ -446,24 +459,109 @@ export class SubredditResources { this.logger.warn(new ErrorWithCause(`Unable to add Activity ${dAct.activity.id} from database delayed activities to in-app delayed activities queue`, {cause: e})); } } + if(toRemove.length > 0) { + await this.removeDelayedActivity(toRemove); + } } } async addDelayedActivity(data: ActivityDispatch) { + // TODO merge this with getActivity or something... + if(asComment(data.activity)) { + const existingSub = await this.activityRepo.findOneBy({_id: data.activity.link_id}); + if(existingSub === null) { + const sub = await this.getActivity(new Submission({name: data.activity.link_id}, this.client, false)); + await this.activityRepo.save(await Activity.fromSnoowrapActivity(sub, {db: this.database})); + } + } const dEntity = await this.dispatchedActivityRepo.save(new DispatchedEntity({...data, manager: this.managerEntity, activity: await Activity.fromSnoowrapActivity(data.activity, {db: this.database})})); data.id = dEntity.id; this.delayedItems.push(data); } async removeDelayedActivity(val?: string | string[]) { - if(val === undefined) { - await this.dispatchedActivityRepo.delete({manager: {id: this.managerEntity.id}}); - this.delayedItems = []; - } else { + let dispatched: DispatchedEntity[] = []; + const where: FindOptionsWhere = { + manager: { + id: this.managerEntity.id + } + }; + + if(val !== undefined) { const ids = typeof val === 'string' ? [val] : val; - await this.dispatchedActivityRepo.delete(ids); - this.delayedItems = this.delayedItems.filter(x => !ids.includes(x.id)); + where.id = In(ids); + } + + dispatched = await this.dispatchedActivityRepo.find({ + where, + relations: { + manager: true, + activity: { + actionedEvents: true, + submission: { + actionedEvents: true + } + } + } + }); + + const actualDispatchedIds = dispatched.map(x => x.id); + this.logger.debug(`${actualDispatchedIds.length} marked for deletion`, {leaf: 'Delayed Activities'}); + + // get potential activities to delete + // but only include activities that don't have any actionedEvents + let activityIdsToDelete = Array.from(dispatched.reduce((acc, curr) => { + if(curr.activity.actionedEvents.length === 0) { + acc.add(curr.activity.id); + } + if(curr.activity.submission !== undefined && curr.activity.submission !== null) { + if(curr.activity.submission.actionedEvents.length === 0) { + acc.add(curr.activity.submission.id); + } + } + return acc; + }, new Set())); + const rawActCount = activityIdsToDelete.length; + let activeActCount = 0; + + // if we have any potential activities to delete we now need to get any dispatched actions that reference these activities + // that are NOT the ones we are going to delete + if(activityIdsToDelete.length > 0) { + const activeDispatchedQuery = this.dispatchedActivityRepo.createQueryBuilder('dis') + .leftJoinAndSelect('dis.activity', 'activity') + .leftJoinAndSelect('activity.submission', 'submission') + .where(new NotBrackets((qb) => { + qb.where('dis.id IN (:...currIds)', {currIds: actualDispatchedIds}); + })) + .andWhere(new Brackets((qb) => { + qb.where('activity._id IN (:...actMainIds)', {actMainIds: activityIdsToDelete}) + qb.orWhere('submission._id IN (:...actSubIds)', {actSubIds: activityIdsToDelete}) + })); + //const sql = activeDispatchedQuery.getSql(); + const activeDispatched = await activeDispatchedQuery.getMany(); + + // all activity ids, from the actions to delete, that are being used by dispatched actions that are NOT the ones we are going to delete + const activeDispatchedIds = Array.from(activeDispatched.reduce((acc, curr) => { + acc.add(curr.activity.id); + if(curr.activity.submission !== undefined && curr.activity.submission !== null) { + acc.add(curr.activity.submission.id); + } + return acc; + }, new Set())); + activeActCount = activeDispatchedIds.length; + + // filter out any that are still in use + activityIdsToDelete = activityIdsToDelete.filter(x => !activeDispatchedIds.includes(x)); } + + this.logger.debug(`Marked ${activityIdsToDelete.length} Activities created, by Delayed, for deletion (${rawActCount} w/o Events | ${activeActCount} used by other Delayed Activities)`, {leaf: 'Delayed Activities'}); + + await this.dispatchedActivityRepo.delete(actualDispatchedIds); + if(activityIdsToDelete.length > 0) { + await this.activityRepo.delete(activityIdsToDelete); + } + + this.delayedItems = this.delayedItems.filter(x => !actualDispatchedIds.includes(x.id)); } async initStats() { From 54917a562e4925766bd3ea7c8db3373944a8829c Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 15 Nov 2022 14:14:53 -0500 Subject: [PATCH 3/5] fix(delayed): Fix accessing non existent actioned events --- src/Subreddit/SubredditResources.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Subreddit/SubredditResources.ts b/src/Subreddit/SubredditResources.ts index 957d9b07..cf20b0aa 100644 --- a/src/Subreddit/SubredditResources.ts +++ b/src/Subreddit/SubredditResources.ts @@ -511,11 +511,11 @@ export class SubredditResources { // get potential activities to delete // but only include activities that don't have any actionedEvents let activityIdsToDelete = Array.from(dispatched.reduce((acc, curr) => { - if(curr.activity.actionedEvents.length === 0) { + if(curr.activity.actionedEvents === null || curr.activity.actionedEvents.length === 0) { acc.add(curr.activity.id); } if(curr.activity.submission !== undefined && curr.activity.submission !== null) { - if(curr.activity.submission.actionedEvents.length === 0) { + if(curr.activity.actionedEvents === null || curr.activity.submission.actionedEvents.length === 0) { acc.add(curr.activity.submission.id); } } From d5bba8ca873aab76e077f0036a937bfaa663f8a0 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 15 Nov 2022 14:22:08 -0500 Subject: [PATCH 4/5] fix(delayed): Fix missing submission accessor --- src/Subreddit/SubredditResources.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Subreddit/SubredditResources.ts b/src/Subreddit/SubredditResources.ts index cf20b0aa..911de4fd 100644 --- a/src/Subreddit/SubredditResources.ts +++ b/src/Subreddit/SubredditResources.ts @@ -515,7 +515,7 @@ export class SubredditResources { acc.add(curr.activity.id); } if(curr.activity.submission !== undefined && curr.activity.submission !== null) { - if(curr.activity.actionedEvents === null || curr.activity.submission.actionedEvents.length === 0) { + if(curr.activity.submission.actionedEvents === null || curr.activity.submission.actionedEvents.length === 0) { acc.add(curr.activity.submission.id); } } From 1956d04e79d6bfac47082bbbc8440e84a4a34618 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 15 Nov 2022 14:36:42 -0500 Subject: [PATCH 5/5] fix(delayed): Prevent delete call when no ids found --- src/Subreddit/SubredditResources.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Subreddit/SubredditResources.ts b/src/Subreddit/SubredditResources.ts index 911de4fd..954a6b7e 100644 --- a/src/Subreddit/SubredditResources.ts +++ b/src/Subreddit/SubredditResources.ts @@ -556,11 +556,14 @@ export class SubredditResources { this.logger.debug(`Marked ${activityIdsToDelete.length} Activities created, by Delayed, for deletion (${rawActCount} w/o Events | ${activeActCount} used by other Delayed Activities)`, {leaf: 'Delayed Activities'}); - await this.dispatchedActivityRepo.delete(actualDispatchedIds); + if(actualDispatchedIds.length > 0) { + await this.dispatchedActivityRepo.delete(actualDispatchedIds); + } else { + this.logger.warn('No dispatched ids found to delete'); + } if(activityIdsToDelete.length > 0) { await this.activityRepo.delete(activityIdsToDelete); } - this.delayedItems = this.delayedItems.filter(x => !actualDispatchedIds.includes(x.id)); }