From 5308950e50f903b327351accc4c36c7b2bd5da9e Mon Sep 17 00:00:00 2001 From: Douglas Eichelberger Date: Sat, 30 May 2026 20:54:58 -0700 Subject: [PATCH] Remove crawl API endpoint (#733) The crawl endpoint is no longer used by any known clients or crawlers (per investigation in #620), so remove it along with its view, test, and documentation. - Delete app/endpoints/crawl.js, app/views/crawl.html, app/test/crawl_test.js - Unregister the endpoint and drop the crawl config block - Remove the "Crawling an event" section and note from CALENDAR_API.md - Bump API version to 3.61.0 and add a changelog entry --- app/appEndpoints.js | 3 +-- app/config.js | 10 ++----- app/endpoints/crawl.js | 60 ------------------------------------------ app/test/crawl_test.js | 45 ------------------------------- app/views/crawl.html | 26 ------------------ docs/CALENDAR_API.md | 53 +------------------------------------ 6 files changed, 4 insertions(+), 193 deletions(-) delete mode 100644 app/endpoints/crawl.js delete mode 100644 app/test/crawl_test.js delete mode 100644 app/views/crawl.html diff --git a/app/appEndpoints.js b/app/appEndpoints.js index 2b2157268..f3302ac05 100644 --- a/app/appEndpoints.js +++ b/app/appEndpoints.js @@ -10,7 +10,7 @@ const app = express(); // and this says express can use it app.set('trust proxy', true); -// allows ex. res.render('crawl.html'); +// allows ex. res.render('email.njk'); nunjucks.express(app); // modify every request @@ -38,7 +38,6 @@ app.use(express.urlencoded({extended:false}), express.json()); // each of these is a javascript file // containing a get ( or post ) export. const endpoints = [ - "crawl", "delete_event", "events", "ical", diff --git a/app/config.js b/app/config.js index 12e2b92d3..4a65d4870 100644 --- a/app/config.js +++ b/app/config.js @@ -29,7 +29,7 @@ const config = { appPath, api: { header: 'Api-Version', - version: "3.60.0", + version: "3.61.0", }, db: getDatabaseConfig(dbType, isTesting), // maybe bad, but some code likes to know: @@ -40,7 +40,7 @@ const config = { name: "SHIFT to Bikes", listen, path: "/", - // used for crawl url, shareable links, and the manage url sent in email + // used for shareable links, and the manage url sent in email url(...parts) { const base = `${siteHost}${config.site.path}`; return base + parts.join("/"); @@ -115,12 +115,6 @@ const config = { 'image/png' : '.png', }, }, - crawl: { - image: 'https://www.shift2bikes.org/images/shiftLogo_plain.gif', - title: 'Shift/Pedalpalooza Calendar', - description: `Find fun bike events and make new friends!` + - `Shift helps groups and individuals to promote their "bike fun" events.`, - }, cal: { pedalp: { name: 'Pedalpalooza Bike Calendar', diff --git a/app/endpoints/crawl.js b/app/endpoints/crawl.js deleted file mode 100644 index 58346788f..000000000 --- a/app/endpoints/crawl.js +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Crawl: Returns a simple HTML rendering of ride data. - * Used by web crawlers such as search engines. - * - * Expects an (optional) TIME id using url query parameter; ex: - * https://api.shift2bikes.org/api/crawl.php?id=15229 - * https://localhost:3080/api/crawl.php?id=1893 - * - * See also: - * https://github.com/shift-org/shift-docs/blob/main/docs/CALENDAR_API.md#crawling-an-event - */ -const config = require("../config"); -const { CalEvent } = require("../models/calEvent"); -const { CalDaily } = require("../models/calDaily"); -const { to12HourString, from24HourString, friendlyDate } = require("../util/dateTime"); - -exports.get = function(req, res, next) { - let id = req.query.id; - const p = { - title: config.crawl.title, - description : config.crawl.description, - url : config.site.url(), - image: config.crawl.image, - siteName: config.site.name, - type: "website", - }; - if (!id) { - res.render('crawl.html', p); - } else { - return CalDaily.getByDailyID(id).then((at) => { - if (!at) { - res.sendStatus(404); // returns not found - } else { - return CalEvent.getByID(at.id).then((evt) => { - if (!evt) { - res.sendStatus(404); // returns not found - } else { - res.render('crawl.html',Object.assign(p, { - title: evt.title, - url: config.site.url("calendar", "event-${at.id}"), - image: evt.image || p.image, - type: "article", //FIXME: Does FB support 'event' yet? - description: evt.desc, - address: evt.address, - when : { - // ex. "Mon, Aug 8th" - // if the eventdate is invalid, the value here is 'null' - date: friendlyDate( at.eventdate ), - // note: the event time is stored as "19:00:00" - // and we want to report it as "7:00 PM" - // if the eventtime is invalid, the value here is 'null' - time: to12HourString( from24HourString(evt.eventtime) ), - }, - })); - } - }); - } - }).catch(next); - } -} diff --git a/app/test/crawl_test.js b/app/test/crawl_test.js deleted file mode 100644 index 82f990191..000000000 --- a/app/test/crawl_test.js +++ /dev/null @@ -1,45 +0,0 @@ -const app = require("../appEndpoints"); -const testdb = require("./testdb"); -// -const { describe, it, before, after } = require("node:test"); -const request = require('supertest'); - -describe("crawl testing", () => { - // runs before the first test in this block. - before(() => { - return testdb.setupTestData("crawl") - }); - // runs once after the last test in this block - after(() => { - return testdb.destroy(); - }); - // test: - it("handles a simple get", () => { - return request(app) - .get('/api/crawl.php') - .expect(200) - .expect('Content-Type', /html/) - // .then(logResponse); - }); - it("handles a valid daily id", () => { - return request(app) - .get('/api/crawl.php') - .query({id: 201}) - .expect(200) - .expect('Content-Type', /html/) - // .then(logResponse); - }); - it("errors on an invalid daily id", () => { - return request(app) - .get('/api/crawl.php') - .query({id: 999}) - .expect(404) // crawl returns an empty 404 - .expect('Content-Type', /text/) - // .then(logResponse); - }); -}); - - -function logResponse(res) { - console.log(res.text); -} \ No newline at end of file diff --git a/app/views/crawl.html b/app/views/crawl.html deleted file mode 100644 index 9ed1dbf3f..000000000 --- a/app/views/crawl.html +++ /dev/null @@ -1,26 +0,0 @@ - - - - {{title}} - - - - - - - - - - -{%- if when %} -

{{when.date}}, {{when.time}} - {{title}}

-{%- endif %} -

{{description}}

-{%- if address %} -

{{address}}

-{%- endif %} -{%- if type != 'website' %} - -{%- endif %} - - \ No newline at end of file diff --git a/docs/CALENDAR_API.md b/docs/CALENDAR_API.md index ba8ec0e04..71d947ddf 100644 --- a/docs/CALENDAR_API.md +++ b/docs/CALENDAR_API.md @@ -8,7 +8,6 @@ Base URL: Most responses are in JSON format, except for: * event export returns vCalendar format -* event crawl returns HTML ## Viewing events @@ -395,57 +394,6 @@ Errors: * date range too large (100 days maximum) -### Crawling an event - -Endpoint: -* GET `crawl` - -Example request: -* `/crawl.php?id=1234` - -URL parameters: -* `id`: `caldaily` event ID - -Unknown parameters are ignored. - -This endpoint is used by web crawlers such as search engines. - -Success: -* status code: `200` -* returns a simple HTML rendering of ride data -* if `id` parameter is not present, a short, general message about Shift - - -Example response: - - - - Shift to Pedalpalooza Ride - - - - - - - - - - -

Mon, Jun 5th, 6:00 PM - Shift to Pedalpalooza Ride

-

Have you ever wondered how Pedalpalooza happens every year...and did you know we have a team of programmers who work on the shift calendar and website. There is a lot of rewarding volunteer work that goes on behind the scenes and we are recruiting for new folks who are interested in helping out next year and beyond. Come on this ride and we will talk a little bit about the history of shift and try to find you a place to help out in the future. We will end at a family friendly watering hole. First round of drinks is on shift. We will be done by 8 so you can check out other rides.

-

877 SW park

- - - - -Errors: -* status code: `404` -* body of response is empty -* possible errors - * `id` not found - * `id` of a hidden (unpublished) event - - ## Managing events ### Retrieving all event data @@ -787,3 +735,4 @@ As with v1, there were probably revisions to v2 during this time, but changelog * 3.59.9: (2025-12-22) Updated dependencies: nodemailer * 3.59.10: (2026-01-15) Changed dependency management to only allow patch updates; updated dependencies: MySQL. Also removed unused example data. * 3.60.0: (2026-02-19) ICS export now supports either single occurrence (`event_id`; new default) or the series (`series_id`; previous default). The existing `id` parameter aliases to `series_id` for backwards compatibility, but clients are encouraged to specify the ID type explicitly. Also updated Node to v24.x (latest LTS). +* 3.61.0: (2026-05-30) Removed the `crawl` endpoint, which is no longer used by any known clients or crawlers.