Skip to content

Commit 30e1e4d

Browse files
th3raid0rth3raid0r
committed
Refactor and add joinfeeds
1 parent 3134adb commit 30e1e4d

File tree

1 file changed

+80
-60
lines changed

1 file changed

+80
-60
lines changed

‎main.js‎

Lines changed: 80 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -85,23 +85,24 @@ const feeds = [
8585
url: 'https://www.tucsonsentinel.com/local/rss/',
8686
content: 'description',
8787
exclude: [
88-
'localpolitics',
88+
'localpolitics', // the local feed contains politics, which we don't want. So we exclude the localpolitics feed to get local news only.
8989
]
9090
},
9191
{
9292
name: 'localpolitics',
9393
url: 'https://www.tucsonsentinel.com/category/rss/politics/',
9494
content: 'description',
95+
joinfeeds: [
96+
'localnews', // the politics feed contains national politics, which we don't want. So we join the local news feed to get local politics.
97+
]
9598
}
9699
]
97100

98-
const exclude = [
99-
{
100-
name: 'localpolitics',
101-
url: 'https://www.tucsonsentinel.com/category/rss/politics/',
102-
content: 'description',
103-
}
104-
]
101+
const sleepDuration = process.env.RATE_LIMIT_MS || 2000;
102+
103+
function sleep(ms) {
104+
return new Promise(resolve => setTimeout(resolve, ms));
105+
}
105106

106107
// -----------------------------------------------------------------------------
107108
// Main Bot Code
@@ -165,25 +166,44 @@ const bot = new LemmyBot.LemmyBot({
165166
},
166167
schedule: [
167168
{
168-
cronExpression: '0 */10 * * * *',
169+
cronExpression: '0 */30 * * * *',
169170
timezone: 'America/Phoenix',
170171
runAtStart: true,
171172
doTask: async ({getCommunityId, createPost}) => {
172173
console.log(`${chalk.green('STARTED:')} RSS Feed Fetcher.`);
173174
for (const feed of feeds) {
174175
const rss = await parser.parseURL(feed.url);
176+
175177
const cutoffDate = new Date();
176178
console.log(`${chalk.green('CURRENT DATE:')} ${cutoffDate}`);
177179
cutoffDate.setMonth(cutoffDate.getMonth() - 6); // set to 6 months ago
178180
console.log(`${chalk.green('CUTOFF DATE:')} ${cutoffDate}`);
181+
182+
let joinedItems = [];
183+
// gather all items from feeds to be joined
184+
if (feed.joinfeeds) {
185+
console.log(`${chalk.green('FETCHING:')} joining feeds for ${feed.name}`);
186+
for (const joinFeedName of feed.joinfeeds) {
187+
const joinFeed = feeds.find(f => f.name === joinFeedName);
188+
189+
if (joinFeed) {
190+
const joinRss = await parser.parseURL(joinFeed.url);
191+
joinedItems = joinedItems.concat(joinRss.items);
192+
}
193+
}
194+
}
195+
179196

180197
let excludeItems = [];
181-
console.log(`${chalk.green('FETCHING:')} exclusion feeds for ${feed.name}`);
198+
182199

183200
// exclude feeds
184201
if (feed.exclude) {
185-
for (const excludeFeed of exclude) {
186-
if (feed.exclude.includes(excludeFeed.name)) {
202+
console.log(`${chalk.green('FETCHING:')} exclusion feeds for ${feed.name}`);
203+
for (const excludeFeedName of feed.exclude) {
204+
const excludeFeed = feeds.find(f => f.name === excludeFeedName);
205+
206+
if (excludeFeed) {
187207
const excludeRss = await parser.parseURL(excludeFeed.url);
188208
for (const excludeItem of excludeRss.items) {
189209
excludeItems.push(excludeItem.link);
@@ -192,59 +212,59 @@ const bot = new LemmyBot.LemmyBot({
192212
}
193213
}
194214

195-
196-
for (const item of rss.items) {
197-
if (!excludeItems.includes(item.link)) {
198-
let pin_days = 0;
199-
const itemDate = new Date(item['dc:date'].trim());
200-
console.log(`${chalk.green('ITEM DATE:')} ${itemDate}`);
201-
//if item is newer than 6 months old, continue
202-
if (itemDate > cutoffDate) {
203-
console.log(`${chalk.green('RECENT:')} true`);
204-
console.log(`${chalk.green('LINK:')} ${item.link}`);
205-
// if has categories then see if it's a pin
206-
if (feed.pinCategories && item.categories) {
207-
for (const category of item.categories) {
208-
const found_category = feed.pinCategories.find(c => c.name === category);
209-
if (found_category) {
210-
pin_days = found_category.days;
211-
}
215+
let commonItems = rss.items.filter(item => joinedItems.map(i => i.link).includes(item.link) && !excludeItems.includes(item.link));
216+
217+
218+
for (const item of commonItems) {
219+
let pin_days = 0;
220+
const itemDate = new Date(item['dc:date'].trim());
221+
console.log(`${chalk.green('ITEM DATE:')} ${itemDate}`);
222+
//if item is newer than 6 months old, continue
223+
if (itemDate > cutoffDate) {
224+
console.log(`${chalk.green('RECENT:')} true`);
225+
console.log(`${chalk.green('LINK:')} ${item.link}`);
226+
// if has categories then see if it's a pin
227+
if (feed.pinCategories && item.categories) {
228+
for (const category of item.categories) {
229+
const found_category = feed.pinCategories.find(c => c.name === category);
230+
if (found_category) {
231+
pin_days = found_category.days;
212232
}
213233
}
234+
}
214235

215-
db.run(`INSERT INTO posts (link, pin_days, featured) VALUES (?, ?, ?)`, [item.link, pin_days, pin_days > 0 ? 1 : 0], async (err) => {
216-
if (err) {
217-
if (err.message.includes('UNIQUE constraint failed')) {
218-
// do nothing
219-
console.log(`${chalk.green('PRESENT:')} ${item.link} already present`);
220-
return;
221-
} else {
222-
return console.error(err.message);
223-
console.log(`${chalk.green('ERROR:')} ${err.message}`);
224-
}
236+
db.run(`INSERT INTO posts (link, pin_days, featured) VALUES (?, ?, ?)`, [item.link, pin_days, pin_days > 0 ? 1 : 0], async (err) => {
237+
if (err) {
238+
if (err.message.includes('UNIQUE constraint failed')) {
239+
// do nothing
240+
console.log(`${chalk.green('PRESENT:')} ${item.link} already present`);
241+
return;
242+
} else {
243+
console.log(`${chalk.green('ERROR:')} ${err.message}`);
244+
return console.error(err.message);
225245
}
226-
console.log(`${chalk.green('INSERTED:')} ${item.link} into database.`);
227-
228-
for (const community of communities) {
229-
if (community.feeds.includes(feed.name)) {
230-
231-
// Process the item only if its link is not in the excludeItems list
232-
if (!excludeItems.includes(item.link)) {
233-
console.log(`${chalk.green('CREATING:')} post for link ${item.link} in ${community.slug }`);
234-
const communityId = await getCommunityId({ name: community.slug, instance: community.instance });
235-
await createPost({
236-
name: item.title,
237-
body: ((feed.content && feed.content === 'summary') ? item.summary : item.content),
238-
url: item.link || undefined,
239-
community_id: communityId,
240-
});
241-
}
246+
}
247+
console.log(`${chalk.green('INSERTED:')} ${item.link} into database.`);
248+
249+
for (const community of communities) {
250+
if (community.feeds.includes(feed.name)) {
251+
252+
// Process the item only if its link is not in the excludeItems list
253+
if (!excludeItems.includes(item.link)) {
254+
console.log(`${chalk.green('CREATING:')} post for link ${item.link} in ${community.slug }`);
255+
const communityId = await getCommunityId({ name: community.slug, instance: community.instance });
256+
await createPost({
257+
name: item.title,
258+
body: ((feed.content && feed.content === 'summary') ? item.summary : item.content),
259+
url: item.link || undefined,
260+
community_id: communityId,
261+
});
262+
await sleep(sleepDuration);
242263
}
243264
}
244-
console.log(`${chalk.green('ADDED:')} ${item.link} for ${pin_days} days`);
245-
});
246-
247-
}
265+
}
266+
console.log(`${chalk.green('ADDED:')} ${item.link} for ${pin_days} days`);
267+
});
248268
}
249269

250270
}
@@ -253,7 +273,7 @@ const bot = new LemmyBot.LemmyBot({
253273
}
254274
},
255275
{
256-
cronExpression: '0 */5 * * * *',
276+
cronExpression: '0 */45 * * * *',
257277
timezone: 'America/Phoenix',
258278
doTask: async ({ featurePost }) => {
259279
const now = addMinutes(new Date(), 30);

0 commit comments

Comments
 (0)