fetchAndSaveLatestTokens.js

const splToken = require("@solana/spl-token");
const fs = require("fs");
const axios = require("axios");
const TokenAddress = require("./models/tokenAddressModel");
const { token } = require("@coral-xyz/anchor/dist/cjs/utils");

const fetchAndSaveLatestTokens = async () => {
  setInterval(async () => {
    try {
      // Get Latest Token list
      console.log("app starting");
      const data = await fetchLatest();
      if (!data || data?.length === 0) {
        console.log("No New Tokens");
        return;
      }
      // Save List
      saveToList(data);
    } catch (err) {
      console.log("Error occured: ", err);
    }
  }, [parseInt(process.env.FETCH_INTERVAL) * 1000]);
};

//fetch latest data and only take updated data compared to the last-saved data.
const fetchLatest = async () => {
  try {
    const res = await axios.get(process.env.FETCH_LATEST);
    const data = res.data;
    console.log("latest number", data.length);

    // Get the last token from DB
    const lastTokenData = await TokenAddress.find().sort({_id: -1}).limit(1);

    const index =
      data.length > 0 && lastTokenData.length > 0
        ? data.map((d) => d.tokenAddress).indexOf(lastTokenData[0].tokenAddress)
        : data.length;

    console.log("Index: ", index);
    let newData = [];
    if (index < 0) newData = data;
    else {
      newData = data.slice(0, index);
    }

    // Populate with timestamp
    newData = newData.map((d) => ({ ...d, timeStamp: new Date() })); //updated tokens
    newData = newData.reverse(); //Latest orders increase from right to left, so reverse them before storing in the database.
    console.log("New tokens: ", newData.length);

    // Populate with Pair
    const tokens = newData.map((d) => d.tokenAddress); //updated token addresses
    const pairs = await fetchPairByGroup(tokens); //pairs list
    return pairs;
  } catch (err) {
    console.log("fetchLatest error:", err);
    return false;
  }
};

const fetchPairByGroup = async (tokens) => {
  const promises = Promise.all(tokens.map((t) => fetchPair(t)));
  let res = await promises.then();
  res = res.flat();
  return res;
};

const fetchPair = async (token) => {
  const res = await axios.get(`${process.env.FETCH_PAIR}/${token}`);
  const data = res.data;
  return data.pairs;
};

const saveToList = async (data) => {
  try {
    for (let tokenInfo of data) {
      //now only save the solana tokens on database
      if (tokenInfo.chainId === "solana" && tokenInfo.dexId === "raydium" && !tokenInfo.labels) {
        const token = new TokenAddress({
          tokenAddress: tokenInfo.baseToken.address,
        });
        await token.save();
      }
    }
  } catch (err) {
    console.log("save error:", err);
  }
};

module.exports = fetchAndSaveLatestTokens;