feat: added prefix option to redis (#88)

* feat: added prefix option to redis

* feat: updated mapcolonies schemas package

* feat: added redis to workflow

* test: added redis prefix tests

Author: NatalieShaked

.github/workflows/pull_request.yaml

@@ -77,6 +77,16 @@ jobs:
           --health-timeout 5s
           --health-retries 5
 
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
     steps:
       - name: Check out Git repository
         uses: actions/checkout@v4

README.md

@@ -41,8 +41,9 @@ Almost all of our routes consists of the same common query parameters: `geo_cont
 | disable_fuzziness | Boolean  | false | Fuzziness is on by default. If you want exact match, you may set `disable_fuzziness: true`. |
 > [!IMPORTANT]
 > We also have Feedback API. Each request is sent back with x-req-id which is the identifier of the request. We kindly ask our users to provide us with a request to Feedback API which contains x-api-key and clicked response. It enables us to research the request and response to be more accurate.
-> Feedback API source code is built in a different repository at <TODO: ADD LINK TO FEEDBACK API REPO>.
+> Feedback API source code is built in a different repository at [feedback-api](https://github.com/MapColonies/feedback-api).
 > Geocoding API inserts the request and response to Redis before the response is sent.
+Speaking of Redis, in the redis config you can add a `prefix` flag, if you'd like to add keys with prefixes to redis.
 
 ## Installation
 Setup Elasticsearch and S3 provider (For local environment, Minio as a personal recommendation).

config/test.json

@@ -35,7 +35,7 @@
       "fileName": "table.json"
     },
     "redis": {
-      "host": "localhost",
+      "host": "redis",
       "port": 6379,
       "username": "",
       "password": "",
@@ -45,7 +45,8 @@
         "key": "",
         "cert": ""
       },
-      "dbIndex": 1
+      "dbIndex": 1,
+      "ttl": 300
     }
   },
   "application": {

package-lock.json

@@ -41,7 +41,7 @@
     "@map-colonies/js-logger": "^2.0.0",
     "@map-colonies/openapi-express-viewer": "^4.0.0",
     "@map-colonies/read-pkg": "^1.0.0",
-    "@map-colonies/schemas": "^1.12.2",
+    "@map-colonies/schemas": "^1.14.0",
     "@map-colonies/telemetry": "^10.0.1",
     "@opentelemetry/api": "^1.9.0",
     "@smithy/node-http-handler": "^3.1.4",

package.json

@@ -21,6 +21,7 @@ export class FeedbackApiMiddlewareManager {
     const reqId = res.getHeader(XApi.REQUEST);
     const redisClient = this.redis;
     const logger = this.logger;
+    const prefix = this.config.get(redisConfigPath).prefix;
 
     const drSite = this.config.get(siteConfig);
 
@@ -36,10 +37,11 @@ export class FeedbackApiMiddlewareManager {
     const { ttl: redisTtl } = this.config.get(redisConfigPath);
 
     const originalJson = res.json;
+    const fullRequestId = prefix !== undefined ? `${prefix}:${reqId as string}` : (reqId as string);
     const logJson = function (this: Response, body: JSON): Response {
       geocodingResponseDetails.response = body;
       redisClient
-        .setEx(reqId as string, redisTtl ?? defaultRedisTtl, JSON.stringify(geocodingResponseDetails))
+        .setEx(fullRequestId, redisTtl ?? defaultRedisTtl, JSON.stringify(geocodingResponseDetails))
         .then(() => {
           logger.info({ msg: `response ${reqId?.toString() ?? ''} saved to redis` });
         })

src/common/middlewares/feedbackApi.middleware.ts

@@ -0,0 +1,20 @@
+import jsLogger from '@map-colonies/js-logger';
+import { trace } from '@opentelemetry/api';
+import { SERVICES } from '@src/common/constants';
+import { S3_REPOSITORY_SYMBOL } from '@src/common/s3/s3Repository';
+import { cronLoadTileLatLonDataSymbol } from '@src/latLon/DAL/latLonDAL';
+import { GetBaseRegisterOptions } from '@tests/integration/helpers/types';
+
+export const getBaseRegisterOptions: GetBaseRegisterOptions = (options = []) => {
+  return {
+    override: [
+      { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } },
+      { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } },
+      { token: S3_REPOSITORY_SYMBOL, provider: { useValue: {} } },
+      { token: SERVICES.S3_CLIENT, provider: { useValue: {} } },
+      { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } },
+      ...options,
+    ],
+    useChild: true,
+  };
+};

tests/integration/control/item/helpers/index.ts

@@ -1,35 +1,26 @@
 /* eslint-disable @typescript-eslint/naming-convention */
-import jsLogger from '@map-colonies/js-logger';
-import { trace } from '@opentelemetry/api';
 import { CleanupRegistry } from '@map-colonies/cleanup-registry';
 import { DependencyContainer } from 'tsyringe';
 import httpStatusCodes from 'http-status-codes';
+import { IConfig } from 'config';
+import { RedisClient } from '@src/common/redis';
 import { getApp } from '../../../../src/app';
-import { SERVICES } from '../../../../src/common/constants';
+import { redisConfigPath, SERVICES } from '../../../../src/common/constants';
 import { GetItemsQueryParams } from '../../../../src/control/item/controllers/itemController';
 import { Item } from '../../../../src/control/item/models/item';
 import { CommonRequestParameters, GenericGeocodingResponse, GeoContext, GeoContextMode } from '../../../../src/common/interfaces';
-import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL';
-import { S3_REPOSITORY_SYMBOL } from '../../../../src/common/s3/s3Repository';
 import { expectedResponse } from '../utils';
 import { ITEM_1234, ITEM_1235, ITEM_1236 } from '../../../mockObjects/items';
 import { ItemRequestSender } from './helpers/requestSender';
+import { getBaseRegisterOptions } from './helpers';
 
 describe('/search/control/items', function () {
   let requestSender: ItemRequestSender;
   let depContainer: DependencyContainer;
 
   beforeEach(async function () {
-    const [app, container] = await getApp({
-      override: [
-        { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } },
-        { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } },
-        { token: S3_REPOSITORY_SYMBOL, provider: { useValue: {} } },
-        { token: SERVICES.S3_CLIENT, provider: { useValue: {} } },
-        { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } },
-      ],
-      useChild: true,
-    });
+    const [app, container] = await getApp(getBaseRegisterOptions());
+
     depContainer = container;
     requestSender = new ItemRequestSender(app);
   });
@@ -245,7 +236,47 @@ describe('/search/control/items', function () {
         expectedResponse(requestParams, [ITEM_1234, ITEM_1235], expect)
       );
     });
+
+    describe('Redis uses prefix key', () => {
+      it('should return 200 status code and add key to Redis with prefix', async function () {
+        const realConfig = depContainer.resolve<IConfig>(SERVICES.CONFIG);
+        const prefix = 'test-prefix-item';
+
+        const configWithPrefix: IConfig = {
+          ...realConfig,
+          get<T>(key: string): T {
+            if (key === redisConfigPath) {
+              const realRedisConfig = realConfig.get<RedisClient>(redisConfigPath);
+              return { ...realRedisConfig, prefix } as T;
+            }
+            return realConfig.get<T>(key);
+          },
+        };
+
+        const mockRegisterOptions = getBaseRegisterOptions([
+          {
+            token: SERVICES.CONFIG,
+            provider: { useValue: configWithPrefix },
+          },
+        ]);
+
+        const [mockApp, localContainer] = await getApp(mockRegisterOptions);
+        const localRequestSender = new ItemRequestSender(mockApp);
+
+        const redisConnection = localContainer.resolve<RedisClient>(SERVICES.REDIS);
+
+        const requestParams: GetItemsQueryParams = { command_name: '123', limit: 5, disable_fuzziness: false };
+        const response = await localRequestSender.getItems(requestParams);
+
+        const keys = await redisConnection.keys(prefix + '*');
+        expect(keys.length).toBeGreaterThanOrEqual(1);
+        expect(response.status).toBe(httpStatusCodes.OK);
+
+        await localContainer.dispose();
+      });
+    });
   });
+
   describe('Bad Path', function () {
     // All requests with status code of 400
     it("should return 400 status code and error message when item's command_name", async function () {

tests/integration/control/item/item.spec.ts

@@ -0,0 +1,20 @@
+import jsLogger from '@map-colonies/js-logger';
+import { trace } from '@opentelemetry/api';
+import { SERVICES } from '@src/common/constants';
+import { S3_REPOSITORY_SYMBOL } from '@src/common/s3/s3Repository';
+import { cronLoadTileLatLonDataSymbol } from '@src/latLon/DAL/latLonDAL';
+import { GetBaseRegisterOptions } from '@tests/integration/helpers/types';
+
+export const getBaseRegisterOptions: GetBaseRegisterOptions = (options = []) => {
+  return {
+    override: [
+      { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } },
+      { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } },
+      { token: S3_REPOSITORY_SYMBOL, provider: { useValue: {} } },
+      { token: SERVICES.S3_CLIENT, provider: { useValue: {} } },
+      { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } },
+      ...options,
+    ],
+    useChild: true,
+  };
+};

tests/integration/control/route/helpers/index.ts

@@ -1,16 +1,14 @@
 /* eslint-disable @typescript-eslint/naming-convention */
-import jsLogger from '@map-colonies/js-logger';
-import { trace } from '@opentelemetry/api';
 import httpStatusCodes from 'http-status-codes';
 import { DependencyContainer } from 'tsyringe';
 import { CleanupRegistry } from '@map-colonies/cleanup-registry';
+import { IConfig } from 'config';
+import { RedisClient } from '@src/common/redis';
 import { getApp } from '../../../../src/app';
-import { SERVICES } from '../../../../src/common/constants';
+import { redisConfigPath, SERVICES } from '../../../../src/common/constants';
 import { GetRoutesQueryParams } from '../../../../src/control/route/controllers/routeController';
 import { Route } from '../../../../src/control/route/models/route';
 import { CommonRequestParameters, GenericGeocodingResponse, GeoContext, GeoContextMode } from '../../../../src/common/interfaces';
-import { S3_REPOSITORY_SYMBOL } from '../../../../src/common/s3/s3Repository';
-import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL';
 import { expectedResponse } from '../utils';
 import {
   ROUTE_VIA_CAMILLUCCIA_A,
@@ -19,22 +17,14 @@ import {
   CONTROL_POINT_OLIMPIADE_112,
 } from '../../../mockObjects/routes';
 import { RouteRequestSender } from './helpers/requestSender';
+import { getBaseRegisterOptions } from './helpers';
 
 describe('/search/control/route', function () {
   let requestSender: RouteRequestSender;
   let depContainer: DependencyContainer;
 
   beforeEach(async function () {
-    const [app, container] = await getApp({
-      override: [
-        { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } },
-        { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } },
-        { token: S3_REPOSITORY_SYMBOL, provider: { useValue: {} } },
-        { token: SERVICES.S3_CLIENT, provider: { useValue: {} } },
-        { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } },
-      ],
-      useChild: true,
-    });
+    const [app, container] = await getApp(getBaseRegisterOptions());
 
     depContainer = container;
     requestSender = new RouteRequestSender(app);
@@ -382,7 +372,47 @@ describe('/search/control/route', function () {
         bbox: null,
       });
     });
+
+    describe('Redis uses prefix key', () => {
+      it('should return 200 status code and add key to Redis with prefix', async function () {
+        const realConfig = depContainer.resolve<IConfig>(SERVICES.CONFIG);
+        const prefix = 'test-prefix-route';
+
+        const configWithPrefix: IConfig = {
+          ...realConfig,
+          get<T>(key: string): T {
+            if (key === redisConfigPath) {
+              const realRedisConfig = realConfig.get<RedisClient>(redisConfigPath);
+              return { ...realRedisConfig, prefix } as T;
+            }
+            return realConfig.get<T>(key);
+          },
+        };
+
+        const mockRegisterOptions = getBaseRegisterOptions([
+          {
+            token: SERVICES.CONFIG,
+            provider: { useValue: configWithPrefix },
+          },
+        ]);
+
+        const [mockApp, localContainer] = await getApp(mockRegisterOptions);
+        const localRequestSender = new RouteRequestSender(mockApp);
+
+        const redisConnection = localContainer.resolve<RedisClient>(SERVICES.REDIS);
+
+        const requestParams: GetRoutesQueryParams = { command_name: 'via camilluccia', limit: 5, disable_fuzziness: false };
+        const response = await localRequestSender.getRoutes(requestParams);
+
+        const keys = await redisConnection.keys(prefix + '*');
+        expect(keys.length).toBeGreaterThanOrEqual(1);
+        expect(response.status).toBe(httpStatusCodes.OK);
+
+        await localContainer.dispose();
+      });
+    });
   });
+
   describe('Bad Path', function () {
     // All requests with status code of 400
     it('should return 400 status code and error message when empty object is passed', async function () {