diff --git a/.env b/.env index 01bb69f4a4..bb823d93d1 100644 --- a/.env +++ b/.env @@ -51,3 +51,17 @@ MAINNET_SEND_MANY_CONTRACT_ID=SP3FBR2AGK5H9QBDH3EEN6DF8EK8JY7RX8QJ5SVTE.send-man # Override the default file path for the proxy cache control file # STACKS_API_PROXY_CACHE_CONTROL_FILE=/path/to/.proxy-cache-control.json + +# Enable token metadata processing. Disabled by default. +# STACKS_API_ENABLE_FT_METADATA=1 +# STACKS_API_ENABLE_NFT_METADATA=1 + +# Configure a script to handle image URLs during token metadata processing. +# This example script uses the `imgix.net` service to create CDN URLs. +# Must be an executable script that accepts the URL as the first program argument +# and outputs a result URL to stdout. +# STACKS_API_IMAGE_CACHE_PROCESSOR=./config/token-metadata-image-cache-imgix.js +# Env vars needed for the above sample `imgix` script: +# IMGIX_DOMAIN=https://.imgix.net +# IMGIX_TOKEN= + diff --git a/.eslintignore b/.eslintignore index 06f558e607..26518cf8d5 100644 --- a/.eslintignore +++ b/.eslintignore @@ -14,3 +14,4 @@ src/tests-rosetta/ src/tests-rosetta-cli/ src/tests-bns/ client/src/ +config/ diff --git a/.github/workflows/stacks-blockchain-api.yml b/.github/workflows/stacks-blockchain-api.yml index a5134f2fff..a1fb730b23 100644 --- a/.github/workflows/stacks-blockchain-api.yml +++ b/.github/workflows/stacks-blockchain-api.yml @@ -266,6 +266,43 @@ jobs: uses: codecov/codecov-action@v1 if: always() + test-tokens: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Use Node.js + uses: actions/setup-node@v1 + with: + node-version: '14.x' + + - name: Install deps + run: npm install + + - name: Setup env vars + run: echo "STACKS_CORE_EVENT_HOST=http://0.0.0.0" >> $GITHUB_ENV + + - name: Setup integration environment + run: | + sudo ufw disable + npm run devenv:deploy -- -d + npm run devenv:logs -- --no-color &> docker-compose-logs.txt & + + - name: Run tokens tests + run: npm run test:tokens + + - name: Print integration environment logs + run: cat docker-compose-logs.txt + if: failure() + + - name: Teardown integration environment + run: npm run devenv:stop + if: always() + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + if: always() + build-publish: runs-on: ubuntu-latest needs: diff --git a/.vscode/launch.json b/.vscode/launch.json index cd1b7dae50..66daaf4ed4 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -143,6 +143,23 @@ "preLaunchTask": "stacks-node:deploy-dev", "postDebugTask": "stacks-node:stop-dev" }, + { + "type": "node", + "request": "launch", + "name": "Jest: Tokens", + "program": "${workspaceFolder}/node_modules/.bin/jest", + "args": [ + "--testTimeout=3600000", + "--runInBand", + "--no-cache", + "--config", + "${workspaceRoot}/jest.config.tokens.js" + ], + "outputCapture": "std", + "console": "integratedTerminal", + "preLaunchTask": "stacks-node:deploy-dev", + "postDebugTask": "stacks-node:stop-dev" + }, { "type": "node", "request": "launch", diff --git a/config/token-metadata-image-cache-imgix.js b/config/token-metadata-image-cache-imgix.js new file mode 100755 index 0000000000..e48d99d00d --- /dev/null +++ b/config/token-metadata-image-cache-imgix.js @@ -0,0 +1,7 @@ +#!/usr/bin/env node +const imgUrl = process.argv[2]; +const encodedUrl = encodeURIComponent(imgUrl); +const [imgixDomain, imgixToken] = [process.env['IMGIX_DOMAIN'], process.env['IMGIX_TOKEN']]; +const signature = require('crypto').createHash('md5').update(imgixToken + '/' + encodedUrl).digest('hex'); +const resultUrl = new URL(encodedUrl + '?s=' + signature, imgixDomain); +console.log(resultUrl.toString()); diff --git a/docs/api/tokens/get-fungible-tokens-metadata-list.example.schema.json b/docs/api/tokens/get-fungible-tokens-metadata-list.example.schema.json new file mode 100644 index 0000000000..d0f0c1ea4c --- /dev/null +++ b/docs/api/tokens/get-fungible-tokens-metadata-list.example.schema.json @@ -0,0 +1,18 @@ +{ + "limit": 1, + "offset": 0, + "total": 500, + "results": [ + { + "token_uri": "https://heystack.xyz/token-metadata.json", + "name": "Heystack", + "description": "Heystack is a SIP-010-compliant fungible token on the Stacks Blockchain, used on the Heystack app", + "image_uri": "https://heystack.xyz/assets/Stacks128w.png", + "image_canonical_uri": "https://heystack.xyz/assets/Stacks128w.png", + "tx_id": "0xef2ac1126e16f46843228b1dk4830e19eb7599129e4jf392cab9e65ae83a45c0", + "sender_address": "ST399W7Z9WS0GMSNQGJGME5JAENKN56D65VGMGKGA", + "symbol": "HEY", + "decimals": 5 + } + ] +} diff --git a/docs/api/tokens/get-fungible-tokens-metadata-list.schema.json b/docs/api/tokens/get-fungible-tokens-metadata-list.schema.json new file mode 100644 index 0000000000..332fd19f03 --- /dev/null +++ b/docs/api/tokens/get-fungible-tokens-metadata-list.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "List of fungible tokens metadata", + "title": "FungibleTokensMetadataList", + "type": "object", + "required": [ + "results", + "limit", + "offset", + "total" + ], + "properties": { + "limit": { + "type": "integer", + "maximum": 200, + "description": "The number of tokens metadata to return" + }, + "offset": { + "type": "integer", + "description": "The number to tokens metadata to skip (starting at `0`)" + }, + "total": { + "type": "integer", + "description": "The number of tokens metadata available" + }, + "results": { + "type": "array", + "items": { + "$ref": "../../entities/tokens/fungible-token.schema.json" + } + } + } +} diff --git a/docs/api/tokens/get-non-fungible-tokens-metadata-list.example.schema.json b/docs/api/tokens/get-non-fungible-tokens-metadata-list.example.schema.json new file mode 100644 index 0000000000..4130b33a76 --- /dev/null +++ b/docs/api/tokens/get-non-fungible-tokens-metadata-list.example.schema.json @@ -0,0 +1,16 @@ +{ + "limit": 1, + "offset": 0, + "total": 500, + "results": [ + { + "token_uri": "https://pool.friedger.de/nft.json", + "name": "Friedger Pool", + "description": "Enjoying the stacking pool.", + "image_uri": "https://pool.friedger.de/nft.webp", + "image_canonical_uri": "https://pool.friedger.de/nft.webp", + "tx_id": "0xef2ac1126e16f46843228b1dk4830e19eb7599129e4jf392cab9e65ae83a45c0", + "sender_address": "ST399W7Z9WS0GMSNQGJGME5JAENKN56D65VGMGKGA" + } + ] +} diff --git a/docs/api/tokens/get-non-fungible-tokens-metadata-list.schema.json b/docs/api/tokens/get-non-fungible-tokens-metadata-list.schema.json new file mode 100644 index 0000000000..23fa26ffdc --- /dev/null +++ b/docs/api/tokens/get-non-fungible-tokens-metadata-list.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "List of non fungible tokens metadata", + "title": "NonFungibleTokensMetadataList", + "type": "object", + "required": [ + "results", + "limit", + "offset", + "total" + ], + "properties": { + "limit": { + "type": "integer", + "maximum": 200, + "description": "The number of tokens metadata to return" + }, + "offset": { + "type": "integer", + "description": "The number to tokens metadata to skip (starting at `0`)" + }, + "total": { + "type": "integer", + "description": "The number of tokens metadata available" + }, + "results": { + "type": "array", + "items": { + "$ref": "../../entities/tokens/non-fungible-token.schema.json" + } + } + } +} diff --git a/docs/entities/tokens/fungible-token.schema.example.json b/docs/entities/tokens/fungible-token.schema.example.json new file mode 100644 index 0000000000..cffa6ab3a0 --- /dev/null +++ b/docs/entities/tokens/fungible-token.schema.example.json @@ -0,0 +1,11 @@ +{ + "token_uri": "https://heystack.xyz/token-metadata.json", + "name": "Heystack", + "description": "Heystack is a SIP-010-compliant fungible token on the Stacks Blockchain, used on the Heystack app", + "image_uri": "https://heystack.xyz/assets/Stacks128w.png", + "image_canonical_uri": "https://heystack.xyz/assets/Stacks128w.png", + "tx_id": "0xef2ac1126e16f46843228b1dk4830e19eb7599129e4jf392cab9e65ae83a45c0", + "sender_address": "ST399W7Z9WS0GMSNQGJGME5JAENKN56D65VGMGKGA", + "symbol": "HEY", + "decimals": 5 +} diff --git a/docs/entities/tokens/fungible-token.schema.json b/docs/entities/tokens/fungible-token.schema.json new file mode 100644 index 0000000000..32aca641fa --- /dev/null +++ b/docs/entities/tokens/fungible-token.schema.json @@ -0,0 +1,56 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "fungible-token-metadata", + "title": "FungibleTokenMetadata", + "type": "object", + "additionalProperties": false, + "required": [ + "token_uri", + "name", + "description", + "image_uri", + "image_canonical_uri", + "symbol", + "decimals", + "tx_id", + "sender_address" + ], + "properties": { + "token_uri": { + "type": "string", + "description": "An optional string that is a valid URI which resolves to this token's metadata. Can be empty." + }, + "name": { + "type": "string", + "description": "Identifies the asset to which this token represents" + }, + "description": { + "type": "string", + "description": "Describes the asset to which this token represents" + }, + "image_uri": { + "type": "string", + "description": "A URI pointing to a resource with mime type image/* representing the asset to which this token represents. The API may provide a URI to a cached resource, dependending on configuration. Otherwise, this can be the same value as the canonical image URI." + }, + "image_canonical_uri": { + "type": "string", + "description": "The original image URI specified by the contract. A URI pointing to a resource with mime type image/* representing the asset to which this token represents. Consider making any images at a width between 320 and 1080 pixels and aspect ratio between 1.91:1 and 4:5 inclusive." + }, + "symbol": { + "type": "string", + "description": "A shorter representation of a token. This is sometimes referred to as a \"ticker\". Examples: \"STX\", \"COOL\", etc. Typically, a token could be referred to as $SYMBOL when referencing it in writing." + }, + "decimals": { + "type": "number", + "description": "The number of decimal places in a token." + }, + "tx_id": { + "type": "string", + "description": "Tx id that deployed the contract" + }, + "sender_address": { + "type": "string", + "description": "principle that deployed the contract" + } + } +} diff --git a/docs/entities/tokens/non-fungible-token.schema.example.json b/docs/entities/tokens/non-fungible-token.schema.example.json new file mode 100644 index 0000000000..57f9613e1c --- /dev/null +++ b/docs/entities/tokens/non-fungible-token.schema.example.json @@ -0,0 +1,9 @@ +{ + "token_uri": "https://pool.friedger.de/nft.json", + "name": "Friedger Pool", + "description": "Enjoying the stacking pool.", + "image_uri": "https://pool.friedger.de/nft.webp", + "image_canonical_uri": "https://pool.friedger.de/nft.webp", + "tx_id": "0xef2ac1126e16f46843228b1dk4830e19eb7599129e4jf392cab9e65ae83a45c0", + "sender_address": "ST399W7Z9WS0GMSNQGJGME5JAENKN56D65VGMGKGA" +} diff --git a/docs/entities/tokens/non-fungible-token.schema.json b/docs/entities/tokens/non-fungible-token.schema.json new file mode 100644 index 0000000000..bc2801b889 --- /dev/null +++ b/docs/entities/tokens/non-fungible-token.schema.json @@ -0,0 +1,46 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "non-fungible-token-metadata", + "title": "NonFungibleTokenMetadata", + "type": "object", + "additionalProperties": false, + "required": [ + "token_uri", + "name", + "description", + "image_uri", + "image_canonical_uri", + "tx_id", + "sender_address" + ], + "properties": { + "token_uri": { + "type": "string", + "description": "An optional string that is a valid URI which resolves to this token's metadata. Can be empty." + }, + "name": { + "type": "string", + "description": "Identifies the asset to which this token represents" + }, + "description": { + "type": "string", + "description": "Describes the asset to which this token represents" + }, + "image_uri": { + "type": "string", + "description": "A URI pointing to a resource with mime type image/* representing the asset to which this token represents. The API may provide a URI to a cached resource, dependending on configuration. Otherwise, this can be the same value as the canonical image URI." + }, + "image_canonical_uri": { + "type": "string", + "description": "The original image URI specified by the contract. A URI pointing to a resource with mime type image/* representing the asset to which this token represents. Consider making any images at a width between 320 and 1080 pixels and aspect ratio between 1.91:1 and 4:5 inclusive." + }, + "tx_id": { + "type": "string", + "description": "Tx id that deployed the contract" + }, + "sender_address": { + "type": "string", + "description": "principle that deployed the contract" + } + } +} diff --git a/docs/generated.d.ts b/docs/generated.d.ts index ca1f1d06cc..b58b559e7e 100644 --- a/docs/generated.d.ts +++ b/docs/generated.d.ts @@ -86,6 +86,11 @@ export type SchemaMergeRootStub = | SearchSuccessResult | TxSearchResult | SearchResult + | { + [k: string]: unknown | undefined; + } + | FungibleTokensMetadataList + | NonFungibleTokensMetadataList | MempoolTransactionListResponse | GetRawTransactionResult | TransactionResults @@ -192,6 +197,8 @@ export type SchemaMergeRootStub = | RosettaSyncStatus | TransactionIdentifier | RosettaTransaction + | FungibleTokenMetadata + | NonFungibleTokenMetadata | { event_index: number; [k: string]: unknown | undefined; @@ -2915,6 +2922,112 @@ export interface TxSearchResult { }; }; } +/** + * List of fungible tokens metadata + */ +export interface FungibleTokensMetadataList { + /** + * The number of tokens metadata to return + */ + limit: number; + /** + * The number to tokens metadata to skip (starting at `0`) + */ + offset: number; + /** + * The number of tokens metadata available + */ + total: number; + results: FungibleTokenMetadata[]; + [k: string]: unknown | undefined; +} +export interface FungibleTokenMetadata { + /** + * An optional string that is a valid URI which resolves to this token's metadata. Can be empty. + */ + token_uri: string; + /** + * Identifies the asset to which this token represents + */ + name: string; + /** + * Describes the asset to which this token represents + */ + description: string; + /** + * A URI pointing to a resource with mime type image/* representing the asset to which this token represents. The API may provide a URI to a cached resource, dependending on configuration. Otherwise, this can be the same value as the canonical image URI. + */ + image_uri: string; + /** + * The original image URI specified by the contract. A URI pointing to a resource with mime type image/* representing the asset to which this token represents. Consider making any images at a width between 320 and 1080 pixels and aspect ratio between 1.91:1 and 4:5 inclusive. + */ + image_canonical_uri: string; + /** + * A shorter representation of a token. This is sometimes referred to as a "ticker". Examples: "STX", "COOL", etc. Typically, a token could be referred to as $SYMBOL when referencing it in writing. + */ + symbol: string; + /** + * The number of decimal places in a token. + */ + decimals: number; + /** + * Tx id that deployed the contract + */ + tx_id: string; + /** + * principle that deployed the contract + */ + sender_address: string; +} +/** + * List of non fungible tokens metadata + */ +export interface NonFungibleTokensMetadataList { + /** + * The number of tokens metadata to return + */ + limit: number; + /** + * The number to tokens metadata to skip (starting at `0`) + */ + offset: number; + /** + * The number of tokens metadata available + */ + total: number; + results: NonFungibleTokenMetadata[]; + [k: string]: unknown | undefined; +} +export interface NonFungibleTokenMetadata { + /** + * An optional string that is a valid URI which resolves to this token's metadata. Can be empty. + */ + token_uri: string; + /** + * Identifies the asset to which this token represents + */ + name: string; + /** + * Describes the asset to which this token represents + */ + description: string; + /** + * A URI pointing to a resource with mime type image/* representing the asset to which this token represents. The API may provide a URI to a cached resource, dependending on configuration. Otherwise, this can be the same value as the canonical image URI. + */ + image_uri: string; + /** + * The original image URI specified by the contract. A URI pointing to a resource with mime type image/* representing the asset to which this token represents. Consider making any images at a width between 320 and 1080 pixels and aspect ratio between 1.91:1 and 4:5 inclusive. + */ + image_canonical_uri: string; + /** + * Tx id that deployed the contract + */ + tx_id: string; + /** + * principle that deployed the contract + */ + sender_address: string; +} /** * GET request that returns transactions */ diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 674c33cfe9..dc1c7e2f92 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -2551,3 +2551,111 @@ paths: $ref: ./api/transaction/get-mempool-transactions.schema.json example: $ref: ./api/transaction/get-mempool-transactions.example.json + + /extended/v1/tokens/ft/metadata: + get: + operationId: get_ft_metadata_list + summary: Fungible tokens metadata list + description: Get list of fungible tokens metadata + tags: + - tokens + parameters: + - name: limit + in: query + description: max number of tokens to fetch + required: false + schema: + type: integer + - name: offset + in: query + description: index of first tokens to fetch + required: false + schema: + type: integer + responses: + 200: + description: List of fungible tokens metadata + content: + application/json: + schema: + $ref: ./api/tokens/get-fungible-tokens-metadata-list.schema.json + example: + $ref: ./api/tokens/get-fungible-tokens-metadata-list.example.schema.json + + /extended/v1/tokens/nft/metadata: + get: + operationId: get_nft_metadata_list + summary: Non fungible tokens metadata list + description: Get list of non fungible tokens metadata + tags: + - tokens + parameters: + - name: limit + in: query + description: max number of tokens to fetch + required: false + schema: + type: integer + - name: offset + in: query + description: index of first tokens to fetch + required: false + schema: + type: integer + responses: + 200: + description: List of non fungible tokens metadata + content: + application/json: + schema: + $ref: ./api/tokens/get-non-fungible-tokens-metadata-list.schema.json + example: + $ref: ./api/tokens/get-non-fungible-tokens-metadata-list.example.schema.json + + /extended/v1/tokens/{contractId}/nft/metadata: + get: + operationId: get_contract_nft_metadata + summary: Non fungible tokens metadata for contract id + description: Get non fungible tokens metadata for given contract id + tags: + - tokens + parameters: + - name: contractId + in: path + description: token's contract id + required: true + schema: + type: string + responses: + 200: + description: Non fungible tokens metadata for contract id + content: + application/json: + schema: + $ref: ./entities/tokens/non-fungible-token.schema.json + example: + $ref: ./entities/tokens/non-fungible-token.schema.example.json + + /extended/v1/tokens/{contractId}/ft/metadata: + get: + operationId: get_contract_ft_metadata + summary: Fungible tokens metadata for contract id + description: Get fungible tokens metadata for given contract id + tags: + - tokens + parameters: + - name: contractId + in: path + description: token's contract id + required: true + schema: + type: string + responses: + 200: + description: Fungible tokens metadata for contract id + content: + application/json: + schema: + $ref: ./entities/tokens/fungible-token.schema.json + example: + $ref: ./entities/tokens/fungible-token.schema.example.json diff --git a/jest.config.tokens.js b/jest.config.tokens.js new file mode 100644 index 0000000000..442a57e45d --- /dev/null +++ b/jest.config.tokens.js @@ -0,0 +1,13 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + rootDir: 'src', + testMatch: ['/tests-tokens/*.ts'], + testPathIgnorePatterns: ['/tests-tokens/setup.ts', '/tests-tokens/teardown.ts'], + collectCoverageFrom: ['/**/*.ts'], + coveragePathIgnorePatterns: ['/tests'], + coverageDirectory: '../coverage', + globalSetup: '/tests-tokens/setup.ts', + globalTeardown: '/tests-tokens/teardown.ts', + testTimeout: 60000, + } diff --git a/package-lock.json b/package-lock.json index 5328343434..997ed8898c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6395,6 +6395,15 @@ "safe-buffer": "^5.1.1" } }, + "evt": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/evt/-/evt-1.10.1.tgz", + "integrity": "sha512-0vkCFzH3Q2Qb9gs3yav4p3uu+l4mcIfKPTRFTO1WHYZd0+O/ZR7BgzpuF+FbqOJ6r9q20/sDL/5TQM+de0/hyg==", + "requires": { + "minimal-polyfills": "^2.1.5", + "run-exclusive": "^2.2.14" + } + }, "exec-sh": { "version": "0.3.6", "resolved": "https://registry.npmjs.org/exec-sh/-/exec-sh-0.3.6.tgz", @@ -12749,6 +12758,11 @@ "integrity": "sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==", "dev": true }, + "minimal-polyfills": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/minimal-polyfills/-/minimal-polyfills-2.2.1.tgz", + "integrity": "sha512-WLmHQrsZob4rVYf8yHapZPNJZ3sspGa/sN8abuSD59b0FifDEE7HMfLUi24z7mPZqTpBXy4Svp+iGvAmclCmXg==" + }, "minimalistic-assert": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", @@ -15196,6 +15210,14 @@ "integrity": "sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA==", "dev": true }, + "run-exclusive": { + "version": "2.2.14", + "resolved": "https://registry.npmjs.org/run-exclusive/-/run-exclusive-2.2.14.tgz", + "integrity": "sha512-NHaQfB3zPJFx7p4M06AcmoK8xz/h8YDMCdy3jxfyoC9VqIbl1U+DiVjUuAYZBRMwvj5qkQnOUGfsmyUC4k46dg==", + "requires": { + "minimal-polyfills": "^2.1.5" + } + }, "run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", diff --git a/package.json b/package.json index 0c6f054785..a7219dd14d 100644 --- a/package.json +++ b/package.json @@ -11,11 +11,13 @@ "test:rosetta": "cross-env NODE_ENV=development jest --config ./jest.config.rosetta.js --coverage --runInBand", "test:bns": "cross-env NODE_ENV=development jest --config ./jest.config.bns.js --coverage --runInBand", "test:microblocks": "cross-env NODE_ENV=development jest --config ./jest.config.microblocks.js --coverage --runInBand", + "test:tokens": "cross-env NODE_ENV=development jest --config ./jest.config.tokens.js --coverage --runInBand", "test:watch": "cross-env NODE_ENV=development jest --config ./jest.config.js --watch", "test:integration": "npm run devenv:deploy -- -d && cross-env NODE_ENV=development jest --config ./jest.config.js --coverage --no-cache --runInBand; npm run devenv:stop", "test:integration:rosetta": "npm run devenv:deploy -- -d && cross-env NODE_ENV=development jest --config ./jest.config.rosetta.js --coverage --no-cache --runInBand; npm run devenv:stop", "test:integration:bns": "npm run devenv:deploy -- -d && cross-env NODE_ENV=development jest --config ./jest.config.bns.js --coverage --no-cache --runInBand; npm run devenv:stop", "test:integration:microblocks": "npm run devenv:deploy:pg -- -d && cross-env NODE_ENV=development jest --config ./jest.config.microblocks.js --coverage --no-cache --runInBand; npm run devenv:stop:pg", + "test:integration:tokens": "npm run devenv:deploy -- -d && cross-env NODE_ENV=development jest --config ./jest.config.tokens.js --coverage --no-cache --runInBand; npm run devenv:stop", "git-info": "echo \"$(git rev-parse --abbrev-ref HEAD)\n$(git log -1 --pretty=format:%h)\n$(git describe --tags --abbrev=0)\" > ./.git-info", "build": "npm run git-info && rimraf ./lib && tsc -p tsconfig.build.json", "build:tests": "tsc -p tsconfig.json", @@ -121,6 +123,7 @@ "dotenv": "^8.2.0", "dotenv-flow": "^3.2.0", "escape-goat": "^3.0.0", + "evt": "^1.10.1", "express": "^4.17.1", "express-list-endpoints": "^5.0.0", "express-winston": "^4.1.0", diff --git a/src/api/init.ts b/src/api/init.ts index b5b6ff8130..97cc8324c8 100644 --- a/src/api/init.ts +++ b/src/api/init.ts @@ -41,6 +41,7 @@ import * as expressListEndpoints from 'express-list-endpoints'; import { createMiddleware as createPrometheusMiddleware } from '@promster/express'; import { createMicroblockRouter } from './routes/microblock'; import { createStatusRouter } from './routes/status'; +import { createTokenRouter } from './routes/tokens/tokens'; export interface ApiServer { expressApp: ExpressWithAsync; @@ -152,6 +153,7 @@ export async function startApiServer(opts: { router.use('/debug', createDebugRouter(datastore)); router.use('/status', createStatusRouter(datastore)); router.use('/faucets', createFaucetRouter(datastore)); + router.use('/tokens', createTokenRouter(datastore)); return router; })() ); diff --git a/src/api/routes/tokens/tokens.ts b/src/api/routes/tokens/tokens.ts new file mode 100644 index 0000000000..cb564c1e76 --- /dev/null +++ b/src/api/routes/tokens/tokens.ts @@ -0,0 +1,150 @@ +import { addAsync, RouterWithAsync } from '@awaitjs/express'; +import * as express from 'express'; +import { DataStore } from '../../../datastore/common'; +import { + FungibleTokenMetadata, + FungibleTokensMetadataList, + NonFungibleTokenMetadata, + NonFungibleTokensMetadataList, +} from '@stacks/stacks-blockchain-api-types'; +import { parseLimitQuery, parsePagingQueryInput } from './../../pagination'; +import { + isFtMetadataEnabled, + isNftMetadataEnabled, +} from '../../../event-stream/tokens-contract-handler'; + +const MAX_TOKENS_PER_REQUEST = 200; +const parseTokenQueryLimit = parseLimitQuery({ + maxItems: MAX_TOKENS_PER_REQUEST, + errorMsg: '`limit` must be equal to or less than ' + MAX_TOKENS_PER_REQUEST, +}); + +export function createTokenRouter(db: DataStore): RouterWithAsync { + const router = addAsync(express.Router()); + router.use(express.json()); + + router.getAsync('/ft/metadata', async (req, res) => { + if (!isFtMetadataEnabled()) { + return res.status(500).json({ + error: 'FT metadata processing is not enabled on this server', + }); + } + + const limit = parseTokenQueryLimit(req.query.limit ?? 96); + const offset = parsePagingQueryInput(req.query.offset ?? 0); + + const { results, total } = await db.getFtMetadataList({ offset, limit }); + + const response: FungibleTokensMetadataList = { + limit: limit, + offset: offset, + total: total, + results: results, + }; + + res.status(200).json(response); + }); + + router.getAsync('/nft/metadata', async (req, res) => { + if (!isNftMetadataEnabled()) { + return res.status(500).json({ + error: 'NFT metadata processing is not enabled on this server', + }); + } + + const limit = parseTokenQueryLimit(req.query.limit ?? 96); + const offset = parsePagingQueryInput(req.query.offset ?? 0); + + const { results, total } = await db.getNftMetadataList({ offset, limit }); + + const response: NonFungibleTokensMetadataList = { + limit: limit, + offset: offset, + total: total, + results: results, + }; + + res.status(200).json(response); + }); + + //router for fungible tokens + router.getAsync('/:contractId/ft/metadata', async (req, res) => { + if (!isFtMetadataEnabled()) { + return res.status(500).json({ + error: 'FT metadata processing is not enabled on this server', + }); + } + + const { contractId } = req.params; + + const metadata = await db.getFtMetadata(contractId); + if (!metadata.found) { + res.status(404).json({ error: 'tokens not found' }); + return; + } + + const { + token_uri, + name, + description, + image_uri, + image_canonical_uri, + symbol, + decimals, + tx_id, + sender_address, + } = metadata.result; + + const response: FungibleTokenMetadata = { + token_uri: token_uri, + name: name, + description: description, + image_uri: image_uri, + image_canonical_uri: image_canonical_uri, + symbol: symbol, + decimals: decimals, + tx_id: tx_id, + sender_address: sender_address, + }; + res.status(200).json(response); + }); + + //router for non-fungible tokens + router.getAsync('/:contractId/nft/metadata', async (req, res) => { + if (!isNftMetadataEnabled()) { + return res.status(500).json({ + error: 'NFT metadata processing is not enabled on this server', + }); + } + + const { contractId } = req.params; + const metadata = await db.getNftMetadata(contractId); + + if (!metadata.found) { + res.status(404).json({ error: 'tokens not found' }); + return; + } + const { + token_uri, + name, + description, + image_uri, + image_canonical_uri, + tx_id, + sender_address, + } = metadata.result; + + const response: NonFungibleTokenMetadata = { + token_uri: token_uri, + name: name, + description: description, + image_uri: image_uri, + image_canonical_uri: image_canonical_uri, + tx_id: tx_id, + sender_address: sender_address, + }; + res.status(200).json(response); + }); + + return router; +} diff --git a/src/datastore/common.ts b/src/datastore/common.ts index 4d15c641cd..ccb7cfa226 100644 --- a/src/datastore/common.ts +++ b/src/datastore/common.ts @@ -17,6 +17,7 @@ import { c32address } from 'c32check'; import { AddressTokenOfferingLocked, TransactionType } from '@stacks/stacks-blockchain-api-types'; import { getTxSenderAddress } from '../event-stream/reader'; import { RawTxQueryResult } from './postgres-store'; +import { ClarityAbi } from '@stacks/transactions'; export interface DbBlock { block_hash: string; @@ -353,6 +354,8 @@ export type DataStoreEventEmitter = StrictEventEmitter< ) => void; addressUpdate: (info: AddressTxUpdateInfo) => void; nameUpdate: (info: string) => void; + tokensUpdate: (contractID: string) => void; + tokenMetadataUpdateQueued: (entry: DbTokenMetadataQueueEntry) => void; } >; @@ -517,6 +520,39 @@ export type BlockIdentifier = | { burnBlockHash: string } | { burnBlockHeight: number }; +export interface DbNonFungibleTokenMetadata { + token_uri: string; + name: string; + description: string; + image_uri: string; + image_canonical_uri: string; + contract_id: string; + tx_id: string; + sender_address: string; +} + +export interface DbFungibleTokenMetadata { + token_uri: string; + name: string; + description: string; + image_uri: string; + image_canonical_uri: string; + contract_id: string; + symbol: string; + decimals: number; + tx_id: string; + sender_address: string; +} + +export interface DbTokenMetadataQueueEntry { + queueId: number; + txId: string; + contractId: string; + contractAbi: ClarityAbi; + blockHeight: number; + processed: boolean; +} + export interface DataStore extends DataStoreEventEmitter { storeRawEventRequest(eventPath: string, payload: string): Promise; getSubdomainResolver(name: { name: string }): Promise>; @@ -776,8 +812,29 @@ export interface DataStore extends DataStoreEventEmitter { address: string, blockHeight: number ): Promise>; - close(): Promise; getUnlockedAddressesAtBlock(block: DbBlock): Promise; + + getFtMetadata(contractId: string): Promise>; + getNftMetadata(contractId: string): Promise>; + + updateNFtMetadata(nftMetadata: DbNonFungibleTokenMetadata, dbQueueId: number): Promise; + updateFtMetadata(ftMetadata: DbFungibleTokenMetadata, dbQueueId: number): Promise; + + getFtMetadataList(args: { + limit: number; + offset: number; + }): Promise<{ results: DbFungibleTokenMetadata[]; total: number }>; + getNftMetadataList(args: { + limit: number; + offset: number; + }): Promise<{ results: DbNonFungibleTokenMetadata[]; total: number }>; + + getTokenMetadataQueue( + limit: number, + excludingEntries: number[] + ): Promise; + + close(): Promise; } export function getAssetEventId(event_index: number, event_tx_id: string): string { diff --git a/src/datastore/memory-store.ts b/src/datastore/memory-store.ts index 9c12133821..591a9242d1 100644 --- a/src/datastore/memory-store.ts +++ b/src/datastore/memory-store.ts @@ -35,6 +35,9 @@ import { DbGetBlockWithMetadataResponse, BlockIdentifier, StxUnlockEvent, + DbFungibleTokenMetadata, + DbNonFungibleTokenMetadata, + DbTokenMetadataQueueEntry, } from './common'; import { logger, FoundOrNot } from '../helpers'; import { AddressTokenOfferingLocked, TransactionType } from '@stacks/stacks-blockchain-api-types'; @@ -702,4 +705,37 @@ export class MemoryDataStore close() { return Promise.resolve(); } + getFtMetadata(contractId: string): Promise> { + throw new Error('Method not implemented.'); + } + getNftMetadata(contractId: string): Promise> { + throw new Error('Method not implemented.'); + } + updateNFtMetadata(nftMetadata: DbNonFungibleTokenMetadata): Promise { + throw new Error('Method not implemented.'); + } + updateFtMetadata(ftMetadata: DbFungibleTokenMetadata): Promise { + throw new Error('Method not implemented.'); + } + + getFtMetadataList(args: { + limit: number; + offset: number; + }): Promise<{ results: DbFungibleTokenMetadata[]; total: number }> { + throw new Error('Method not implemented.'); + } + + getNftMetadataList(args: { + limit: number; + offset: number; + }): Promise<{ results: DbNonFungibleTokenMetadata[]; total: number }> { + throw new Error('Method not implemented.'); + } + + getTokenMetadataQueue( + _limit: number, + _excludingEntries: number[] + ): Promise { + throw new Error('Method not implemented.'); + } } diff --git a/src/datastore/postgres-store.ts b/src/datastore/postgres-store.ts index ef61bcd5f6..2b6565c40b 100644 --- a/src/datastore/postgres-store.ts +++ b/src/datastore/postgres-store.ts @@ -72,6 +72,9 @@ import { DbRawEventRequest, BlockIdentifier, StxUnlockEvent, + DbNonFungibleTokenMetadata, + DbFungibleTokenMetadata, + DbTokenMetadataQueueEntry, } from './common'; import { AddressTokenOfferingLocked, @@ -79,6 +82,8 @@ import { AddressUnlockSchedule, } from '@stacks/stacks-blockchain-api-types'; import { getTxTypeId } from '../api/controllers/db-controller'; +import { isProcessableTokenMetadata } from '../event-stream/tokens-contract-handler'; +import { ClarityAbi } from '@stacks/transactions'; const MIGRATIONS_TABLE = 'pgmigrations'; const MIGRATIONS_DIR = path.join(APP_DIR, 'migrations'); @@ -467,6 +472,39 @@ interface TransferQueryResult { amount: string; } +interface NonFungibleTokenMetadataQueryResult { + token_uri: string; + name: string; + description: string; + image_uri: string; + image_canonical_uri: string; + contract_id: string; + tx_id: Buffer; + sender_address: string; +} + +interface FungibleTokenMetadataQueryResult { + token_uri: string; + name: string; + description: string; + image_uri: string; + image_canonical_uri: string; + contract_id: string; + symbol: string; + decimals: number; + tx_id: Buffer; + sender_address: string; +} + +interface DbTokenMetadataQueueEntryQuery { + queue_id: number; + tx_id: Buffer; + contract_id: string; + contract_abi: string; + block_height: number; + processed: boolean; +} + export interface RawTxQueryResult { raw_tx: Buffer; } @@ -887,6 +925,7 @@ export class PgDataStore } async update(data: DataStoreBlockUpdateData): Promise { + const tokenMetadataQueueEntries: DbTokenMetadataQueueEntry[] = []; await this.queryTx(async client => { const chainTip = await this.getChainTip(client); await this.handleReorg(client, data.block, chainTip.blockHeight); @@ -1013,6 +1052,28 @@ export class PgDataStore await this.updateNamespaces(client, entry.tx, namespace); } } + + const tokenContractDeployments = data.txs + .filter(entry => entry.tx.type_id === DbTxTypeId.SmartContract) + .filter(entry => entry.tx.status === DbTxStatus.Success) + .map(entry => { + const smartContract = entry.smartContracts[0]; + const contractAbi: ClarityAbi = JSON.parse(smartContract.abi); + const queueEntry: DbTokenMetadataQueueEntry = { + queueId: -1, + txId: entry.tx.tx_id, + contractId: smartContract.contract_id, + contractAbi: contractAbi, + blockHeight: entry.tx.block_height, + processed: false, + }; + return queueEntry; + }) + .filter(entry => isProcessableTokenMetadata(entry.contractAbi)); + for (const pendingQueueEntry of tokenContractDeployments) { + const queueEntry = await this.updateTokenMetadataQueue(client, pendingQueueEntry); + tokenMetadataQueueEntries.push(queueEntry); + } } }); @@ -1031,6 +1092,9 @@ export class PgDataStore this.emit('txUpdate', entry.tx); }); this.emitAddressTxUpdates(data); + for (const tokenMetadataQueueEntry of tokenMetadataQueueEntries) { + this.emit('tokenMetadataUpdateQueued', tokenMetadataQueueEntry); + } } async updateMicroCanonical( @@ -3845,6 +3909,64 @@ export class PgDataStore ); } + async getTokenMetadataQueue( + limit: number, + excludingEntries: number[] + ): Promise { + const result = await this.queryTx(async client => { + const queryResult = await client.query( + ` + SELECT * + FROM token_metadata_queue + WHERE NOT (queue_id = ANY($1)) + AND processed = false + ORDER BY block_height ASC, queue_id ASC + LIMIT $2 + `, + [excludingEntries, limit] + ); + return queryResult; + }); + const entries = result.rows.map(row => { + const entry: DbTokenMetadataQueueEntry = { + queueId: row.queue_id, + txId: bufferToHexPrefixString(row.tx_id), + contractId: row.contract_id, + contractAbi: JSON.parse(row.contract_abi), + blockHeight: row.block_height, + processed: row.processed, + }; + return entry; + }); + return entries; + } + + async updateTokenMetadataQueue( + client: ClientBase, + entry: DbTokenMetadataQueueEntry + ): Promise { + const queryResult = await client.query<{ queue_id: number }>( + ` + INSERT INTO token_metadata_queue( + tx_id, contract_id, contract_abi, block_height, processed + ) values($1, $2, $3, $4, $5) + RETURNING queue_id + `, + [ + hexToBuffer(entry.txId), + entry.contractId, + JSON.stringify(entry.contractAbi), + entry.blockHeight, + false, + ] + ); + const result: DbTokenMetadataQueueEntry = { + ...entry, + queueId: queryResult.rows[0].queue_id, + }; + return result; + } + async updateSmartContract(client: ClientBase, tx: DbTx, smartContract: DbSmartContract) { await client.query( ` @@ -5749,6 +5871,247 @@ export class PgDataStore return { found: false }; }); } + async getFtMetadata(contractId: string): Promise> { + return this.query(async client => { + const queryResult = await client.query( + ` + SELECT token_uri, name, description, image_uri, image_canonical_uri, symbol, decimals, contract_id, tx_id, sender_address + FROM ft_metadata + WHERE contract_id = $1 + LIMIT 1 + `, + [contractId] + ); + if (queryResult.rowCount > 0) { + const metadata: DbFungibleTokenMetadata = { + token_uri: queryResult.rows[0].token_uri, + name: queryResult.rows[0].name, + description: queryResult.rows[0].description, + image_uri: queryResult.rows[0].image_uri, + image_canonical_uri: queryResult.rows[0].image_canonical_uri, + symbol: queryResult.rows[0].symbol, + decimals: queryResult.rows[0].decimals, + contract_id: queryResult.rows[0].contract_id, + tx_id: bufferToHexPrefixString(queryResult.rows[0].tx_id), + sender_address: queryResult.rows[0].sender_address, + }; + return { + found: true, + result: metadata, + }; + } else { + return { found: false } as const; + } + }); + } + + async getNftMetadata(contractId: string): Promise> { + return this.query(async client => { + const queryResult = await client.query( + ` + SELECT token_uri, name, description, image_uri, image_canonical_uri, contract_id, tx_id, sender_address + FROM nft_metadata + WHERE contract_id = $1 + LIMIT 1 + `, + [contractId] + ); + if (queryResult.rowCount > 0) { + const metadata: DbNonFungibleTokenMetadata = { + token_uri: queryResult.rows[0].token_uri, + name: queryResult.rows[0].name, + description: queryResult.rows[0].description, + image_uri: queryResult.rows[0].image_uri, + image_canonical_uri: queryResult.rows[0].image_canonical_uri, + contract_id: queryResult.rows[0].contract_id, + tx_id: bufferToHexPrefixString(queryResult.rows[0].tx_id), + sender_address: queryResult.rows[0].sender_address, + }; + return { + found: true, + result: metadata, + }; + } else { + return { found: false } as const; + } + }); + } + + async updateFtMetadata(ftMetadata: DbFungibleTokenMetadata, dbQueueId: number): Promise { + const { + token_uri, + name, + description, + image_uri, + image_canonical_uri, + contract_id, + symbol, + decimals, + tx_id, + sender_address, + } = ftMetadata; + + const rowCount = await this.queryTx(async client => { + const result = await client.query( + ` + INSERT INTO ft_metadata( + token_uri, name, description, image_uri, image_canonical_uri, contract_id, symbol, decimals, tx_id, sender_address + ) values($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + `, + [ + token_uri, + name, + description, + image_uri, + image_canonical_uri, + contract_id, + symbol, + decimals, + hexToBuffer(tx_id), + sender_address, + ] + ); + await client.query( + ` + UPDATE token_metadata_queue + SET processed = true + WHERE queue_id = $1 + `, + [dbQueueId] + ); + return result.rowCount; + }); + this.emit('tokensUpdate', contract_id); + return rowCount; + } + + async updateNFtMetadata( + nftMetadata: DbNonFungibleTokenMetadata, + dbQueueId: number + ): Promise { + const { + token_uri, + name, + description, + image_uri, + image_canonical_uri, + contract_id, + tx_id, + sender_address, + } = nftMetadata; + const rowCount = await this.queryTx(async client => { + const result = await client.query( + ` + INSERT INTO nft_metadata( + token_uri, name, description, image_uri, image_canonical_uri, contract_id, tx_id, sender_address + ) values($1, $2, $3, $4, $5, $6, $7, $8) + `, + [ + token_uri, + name, + description, + image_uri, + image_canonical_uri, + contract_id, + hexToBuffer(tx_id), + sender_address, + ] + ); + await client.query( + ` + UPDATE token_metadata_queue + SET processed = true + WHERE queue_id = $1 + `, + [dbQueueId] + ); + return result.rowCount; + }); + this.emit('tokensUpdate', contract_id); + return rowCount; + } + + getFtMetadataList({ + limit, + offset, + }: { + limit: number; + offset: number; + }): Promise<{ results: DbFungibleTokenMetadata[]; total: number }> { + return this.queryTx(async client => { + const totalQuery = await client.query<{ count: number }>( + ` + SELECT COUNT(*)::integer + FROM ft_metadata + ` + ); + const resultQuery = await client.query( + ` + SELECT * + FROM ft_metadata + LIMIT $1 + OFFSET $2 + `, + [limit, offset] + ); + const parsed = resultQuery.rows.map(r => { + const metadata: DbFungibleTokenMetadata = { + name: r.name, + description: r.description, + token_uri: r.token_uri, + image_uri: r.image_uri, + image_canonical_uri: r.image_canonical_uri, + decimals: r.decimals, + symbol: r.symbol, + contract_id: r.contract_id, + tx_id: bufferToHexPrefixString(r.tx_id), + sender_address: r.sender_address, + }; + return metadata; + }); + return { results: parsed, total: totalQuery.rows[0].count }; + }); + } + + getNftMetadataList({ + limit, + offset, + }: { + limit: number; + offset: number; + }): Promise<{ results: DbNonFungibleTokenMetadata[]; total: number }> { + return this.queryTx(async client => { + const totalQuery = await client.query<{ count: number }>( + ` + SELECT COUNT(*)::integer + FROM nft_metadata + ` + ); + const resultQuery = await client.query( + ` + SELECT * + FROM nft_metadata + LIMIT $1 + OFFSET $2 + `, + [limit, offset] + ); + const parsed = resultQuery.rows.map(r => { + const metadata: DbNonFungibleTokenMetadata = { + name: r.name, + description: r.description, + token_uri: r.token_uri, + image_uri: r.image_uri, + image_canonical_uri: r.image_canonical_uri, + contract_id: r.contract_id, + tx_id: bufferToHexPrefixString(r.tx_id), + sender_address: r.sender_address, + }; + return metadata; + }); + return { results: parsed, total: totalQuery.rows[0].count }; + }); + } async close(): Promise { await this.pool.end(); diff --git a/src/event-stream/tokens-contract-handler.ts b/src/event-stream/tokens-contract-handler.ts new file mode 100644 index 0000000000..b34d3b3d68 --- /dev/null +++ b/src/event-stream/tokens-contract-handler.ts @@ -0,0 +1,855 @@ +import * as child_process from 'child_process'; +import { + DataStore, + DbFungibleTokenMetadata, + DbNonFungibleTokenMetadata, + DbTokenMetadataQueueEntry, +} from '../datastore/common'; +import { + callReadOnlyFunction, + ChainID, + ClarityAbi, + ClarityAbiFunction, + ClarityType, + ClarityValue, + getAddressFromPrivateKey, + makeRandomPrivKey, + ReadOnlyFunctionOptions, + TransactionVersion, + uintCV, + UIntCV, +} from '@stacks/transactions'; +import { GetStacksNetwork } from '../bns-helpers'; +import { logError, logger, parseDataUrl, REPO_DIR, stopwatch } from '../helpers'; +import { StacksNetwork } from '@stacks/network'; +import PQueue from 'p-queue'; +import * as querystring from 'querystring'; +import fetch from 'node-fetch'; +import { Evt } from 'evt'; + +/** + * The maximum number of token metadata parsing operations that can be ran concurrently before + * being added to a FIFO queue. + */ +const TOKEN_METADATA_PARSING_CONCURRENCY_LIMIT = 5; + +/** + * Amount of milliseconds to wait when fetching token metadata. + * If the fetch takes longer then it throws and the metadata is not processed. + */ +const METADATA_FETCH_TIMEOUT_MS: number = 10_000; // 10 seconds + +/** + * The maximum number of bytes of metadata to fetch. + * If the fetch encounters more bytes than this limit it throws and the metadata is not processed. + */ +const METADATA_MAX_PAYLOAD_BYTE_SIZE = 1_000_000; // 1 megabyte + +const PUBLIC_IPFS = 'https://ipfs.io'; + +export function isFtMetadataEnabled() { + const opt = process.env['STACKS_API_ENABLE_FT_METADATA']?.toLowerCase().trim(); + return opt === '1' || opt === 'true'; +} + +export function isNftMetadataEnabled() { + const opt = process.env['STACKS_API_ENABLE_NFT_METADATA']?.toLowerCase().trim(); + return opt === '1' || opt === 'true'; +} + +const FT_FUNCTIONS: ClarityAbiFunction[] = [ + { + access: 'public', + args: [ + { type: 'uint128', name: 'amount' }, + { type: 'principal', name: 'sender' }, + { type: 'principal', name: 'recipient' }, + { type: { optional: { buffer: { length: 34 } } }, name: 'memo' }, + ], + name: 'transfer', + outputs: { type: { response: { ok: 'bool', error: 'uint128' } } }, + }, + { + access: 'read_only', + args: [], + name: 'get-name', + outputs: { type: { response: { ok: { 'string-ascii': { length: 32 } }, error: 'uint128' } } }, + }, + { + access: 'read_only', + args: [], + name: 'get-symbol', + outputs: { type: { response: { ok: { 'string-ascii': { length: 32 } }, error: 'uint128' } } }, + }, + { + access: 'read_only', + args: [], + name: 'get-decimals', + outputs: { type: { response: { ok: 'uint128', error: 'uint128' } } }, + }, + { + access: 'read_only', + args: [{ type: 'principal', name: 'address' }], + name: 'get-balance', + outputs: { type: { response: { ok: 'uint128', error: 'uint128' } } }, + }, + { + access: 'read_only', + args: [], + name: 'get-total-supply', + outputs: { type: { response: { ok: 'uint128', error: 'uint128' } } }, + }, + { + access: 'read_only', + args: [], + name: 'get-token-uri', + outputs: { + type: { + response: { + ok: { + optional: { 'string-ascii': { length: 256 } }, + }, + error: 'uint128', + }, + }, + }, + }, +]; + +const NFT_FUNCTIONS: ClarityAbiFunction[] = [ + { + access: 'read_only', + args: [], + name: 'get-last-token-id', + outputs: { + type: { + response: { + ok: 'uint128', + error: 'uint128', + }, + }, + }, + }, + { + access: 'read_only', + args: [{ name: 'any', type: 'uint128' }], + name: 'get-token-uri', + outputs: { + type: { + response: { + ok: { + optional: { 'string-ascii': { length: 256 } }, + }, + error: 'uint128', + }, + }, + }, + }, + { + access: 'read_only', + args: [{ type: 'uint128', name: 'any' }], + name: 'get-owner', + outputs: { + type: { + response: { + ok: { + optional: 'principal', + }, + error: 'uint128', + }, + }, + }, + }, + { + access: 'public', + args: [ + { type: 'uint128', name: 'id' }, + { type: 'principal', name: 'sender' }, + { type: 'principal', name: 'recipient' }, + ], + name: 'transfer', + outputs: { + type: { + response: { + ok: 'bool', + error: { + tuple: [ + { type: { 'string-ascii': { length: 32 } }, name: 'kind' }, + { type: 'uint128', name: 'code' }, + ], + }, + }, + }, + }, + }, +]; + +interface NftTokenMetadata { + name: string; + imageUri: string; + description: string; +} + +interface FtTokenMetadata { + name: string; + imageUri: string; + description: string; +} + +export interface TokenHandlerArgs { + contractId: string; + smartContractAbi: ClarityAbi; + datastore: DataStore; + chainId: ChainID; + txId: string; + dbQueueId: number; +} + +/** + * Checks if the given ABI contains functions from FT or NFT metadata standards (e.g. sip-09, sip-10) which can be resolved. + * The function also checks if the server has FT and/or NFT metadata processing enabled. + */ +export function isProcessableTokenMetadata(abi: ClarityAbi): boolean { + return ( + (isFtMetadataEnabled() && isCompliantFt(abi)) || (isNftMetadataEnabled() && isCompliantNft(abi)) + ); +} + +function isCompliantNft(abi: ClarityAbi): boolean { + if (abi.non_fungible_tokens.length > 0) { + if (abiContains(abi, NFT_FUNCTIONS)) { + return true; + } + } + return false; +} + +function isCompliantFt(abi: ClarityAbi): boolean { + if (abi.fungible_tokens.length > 0) { + if (abiContains(abi, FT_FUNCTIONS)) { + return true; + } + } + return false; +} + +/** + * This method check if the contract is compliance with sip-09 and sip-10 + * Ref: https://github.com/stacksgov/sips/tree/main/sips + */ +function abiContains(abi: ClarityAbi, standardFunction: ClarityAbiFunction[]): boolean { + return standardFunction.every(abiFun => findFunction(abiFun, abi.functions)); +} + +/** + * check if the fun exist in the function list + * @param fun - function to be found + * @param functionList - list of functions + * @returns - true if function is in the list false otherwise + */ +function findFunction(fun: ClarityAbiFunction, functionList: ClarityAbiFunction[]): boolean { + const found = functionList.find(standardFunction => { + if (standardFunction.name !== fun.name || standardFunction.args.length !== fun.args.length) + return false; + for (let i = 0; i < fun.args.length; i++) { + if (standardFunction.args[i].type.toString() !== fun.args[i].type.toString()) { + return false; + } + } + return true; + }); + return found !== undefined; +} + +export class TokensContractHandler { + readonly contractAddress: string; + readonly contractName: string; + readonly contractId: string; + readonly txId: string; + readonly dbQueueId: number; + private readonly contractAbi: ClarityAbi; + private readonly db: DataStore; + private readonly randomPrivKey = makeRandomPrivKey(); + private readonly chainId: ChainID; + private readonly stacksNetwork: StacksNetwork; + private readonly address: string; + private readonly tokenKind: 'ft' | 'nft'; + + constructor(args: TokenHandlerArgs) { + [this.contractAddress, this.contractName] = args.contractId.split('.'); + this.contractId = args.contractId; + this.contractAbi = args.smartContractAbi; + this.db = args.datastore; + this.chainId = args.chainId; + this.txId = args.txId; + this.dbQueueId = args.dbQueueId; + + this.stacksNetwork = GetStacksNetwork(this.chainId); + this.address = getAddressFromPrivateKey( + this.randomPrivKey.data, + this.chainId === ChainID.Mainnet ? TransactionVersion.Mainnet : TransactionVersion.Testnet + ); + if (isCompliantFt(args.smartContractAbi)) { + this.tokenKind = 'ft'; + } else if (isCompliantNft(args.smartContractAbi)) { + this.tokenKind = 'nft'; + } else { + throw new Error( + `TokenContractHandler passed an ABI that isn't compliant to FT or NFT standards` + ); + } + } + + async start() { + logger.info( + `[token-metadata] found ${ + this.tokenKind === 'ft' ? 'sip-010-ft-standard' : 'sip-009-nft-standard' + } compliant contract ${this.contractId} in tx ${this.txId}, begin retrieving metadata...` + ); + const sw = stopwatch(); + try { + if (this.tokenKind === 'ft') { + await this.handleFtContract(); + } else if (this.tokenKind === 'nft') { + await this.handleNftContract(); + } else { + throw new Error(`Unexpected token kind '${this.tokenKind}'`); + } + } finally { + logger.info( + `[token-metadata] finished processing ${this.contractId} in ${sw.getElapsed()} ms` + ); + } + } + + /** + * Token metadata schema for 'image uri' is not well defined or adhered to. + * This function looks for a handful of possible properties that could be used to + * specify the image, and returns a metadata object with a normalized image property. + */ + private patchTokenMetadataImageUri(metadata: T): T { + // compare using lowercase + const allowedImageProperties = ['image', 'imageurl', 'imageuri', 'image_url', 'image_uri']; + const objectKeys = new Map(Object.keys(metadata).map(prop => [prop.toLowerCase(), prop])); + for (const possibleProp of allowedImageProperties) { + const existingProp = objectKeys.get(possibleProp); + if (existingProp) { + const imageUriVal = (metadata as Record)[existingProp]; + if (typeof imageUriVal !== 'string') { + continue; + } + return { + ...metadata, + imageUri: imageUriVal, + }; + } + } + return { ...metadata }; + } + + /** + * fetch Fungible contract metadata + */ + private async handleFtContract() { + let metadata: FtTokenMetadata | undefined; + let contractCallName: string | undefined; + let contractCallUri: string | undefined; + let contractCallSymbol: string | undefined; + let contractCallDecimals: number | undefined; + let imgUrl: string | undefined; + + try { + // get name value + contractCallName = await this.readStringFromContract('get-name', []); + + // get token uri + contractCallUri = await this.readStringFromContract('get-token-uri', []); + + // get token symbol + contractCallSymbol = await this.readStringFromContract('get-symbol', []); + + // get decimals + const decimalsResult = await this.readUIntFromContract('get-decimals', []); + if (decimalsResult) { + contractCallDecimals = Number(decimalsResult.toString()); + } + + if (contractCallUri) { + try { + metadata = await this.getMetadataFromUri(contractCallUri); + metadata = this.patchTokenMetadataImageUri(metadata); + } catch (error) { + logger.warn( + `[token-metadata] error fetching metadata while processing FT contract ${this.contractId}`, + error + ); + } + } + + if (metadata?.imageUri) { + try { + const normalizedUrl = this.getImageUrl(metadata.imageUri); + imgUrl = await this.processImageUrl(normalizedUrl); + } catch (error) { + logger.warn( + `[token-metadata] error handling image url while processing FT contract ${this.contractId}`, + error + ); + } + } + } catch (error) { + // Note: something is wrong with the above error handling if this is ever reached. + logError( + `[token-metadata] unexpected error processing FT contract ${this.contractId}`, + error + ); + } + + const fungibleTokenMetadata: DbFungibleTokenMetadata = { + token_uri: contractCallUri ?? '', + name: contractCallName ?? metadata?.name ?? '', // prefer the on-chain name + description: metadata?.description ?? '', + image_uri: imgUrl ?? '', + image_canonical_uri: metadata?.imageUri ?? '', + symbol: contractCallSymbol ?? '', + decimals: contractCallDecimals ?? 0, + contract_id: this.contractId, + tx_id: this.txId, + sender_address: this.contractAddress, + }; + + //store metadata in db + await this.storeFtMetadata(fungibleTokenMetadata); + } + + /** + * fetch Non Fungible contract metadata + */ + private async handleNftContract() { + let metadata: NftTokenMetadata | undefined; + let contractCallUri: string | undefined; + let imgUrl: string | undefined; + + try { + // TODO: This is incorrectly attempting to fetch the metadata for a specific + // NFT and applying it to the entire NFT type/contract. A new SIP needs created + // to define how generic metadata for an NFT type/contract should be retrieved. + // In the meantime, this will often fail or result in weird data, but at least + // the NFT type enumeration endpoints will have data like the contract ID and txid. + + // TODO: this should instead use the SIP-012 draft https://github.com/stacksgov/sips/pull/18 + // function `(get-nft-meta () (response (optional {name: (string-uft8 30), image: (string-ascii 255)}) uint))` + + contractCallUri = await this.readStringFromContract('get-token-uri', [uintCV(0)]); + if (contractCallUri) { + try { + metadata = await this.getMetadataFromUri(contractCallUri); + metadata = this.patchTokenMetadataImageUri(metadata); + } catch (error) { + logger.warn( + `[token-metadata] error fetching metadata while processing NFT contract ${this.contractId}`, + error + ); + } + } + + if (metadata?.imageUri) { + try { + const normalizedUrl = this.getImageUrl(metadata.imageUri); + imgUrl = await this.processImageUrl(normalizedUrl); + } catch (error) { + logger.warn( + `[token-metadata] error handling image url while processing NFT contract ${this.contractId}`, + error + ); + } + } + } catch (error) { + // Note: something is wrong with the above error handling if this is ever reached. + logError( + `[token-metadata] unexpected error processing NFT contract ${this.contractId}`, + error + ); + } + + const nonFungibleTokenMetadata: DbNonFungibleTokenMetadata = { + token_uri: contractCallUri ?? '', + name: metadata?.name ?? '', + description: metadata?.description ?? '', + image_uri: imgUrl ?? '', + image_canonical_uri: metadata?.imageUri ?? '', + contract_id: `${this.contractId}`, + tx_id: this.txId, + sender_address: this.contractAddress, + }; + await this.storeNftMetadata(nonFungibleTokenMetadata); + } + + /** + * If an external image processor script is configured, then it will process the given image URL for the purpose + * of caching on a CDN (or whatever else it may be created to do). The script is expected to return a new URL + * for the image. + * If the script is not configured, then the original URL is returned immediately. + * If a data-uri is passed, it is also immediately returned without being passed to the script. + */ + private async processImageUrl(imgUrl: string): Promise { + const imageCacheProcessor = process.env['STACKS_API_IMAGE_CACHE_PROCESSOR']; + if (!imageCacheProcessor) { + return imgUrl; + } + if (imgUrl.startsWith('data:')) { + return imgUrl; + } + const { code, stdout, stderr } = await new Promise<{ + code: number; + stdout: string; + stderr: string; + }>((resolve, reject) => { + const cp = child_process.spawn(imageCacheProcessor, [imgUrl], { cwd: REPO_DIR }); + let stdout = ''; + let stderr = ''; + cp.stdout.on('data', data => (stdout += data)); + cp.stderr.on('data', data => (stderr += data)); + cp.on('close', code => resolve({ code: code ?? 0, stdout, stderr })); + cp.on('error', error => reject(error)); + }); + if (code !== 0 && stderr) { + console.warn(`[token-metadata] stderr from STACKS_API_IMAGE_CACHE_PROCESSOR: ${stderr}`); + } + const result = stdout.trim(); + try { + const url = new URL(result); + return url.toString(); + } catch (error) { + throw new Error( + `Image processing script returned an invalid url for ${imgUrl}: ${result}, stderr: ${stderr}` + ); + } + } + + /** + * Helper method for creating http/s url for supported protocols. + * URLs with `http` or `https` protocols are returned as-is. + * URLs with `ipfs` or `ipns` protocols are returned with as an `https` url + * using a public IPFS gateway. + */ + private getFetchableUrl(uri: string): URL { + const parsedUri = new URL(uri); + if (parsedUri.protocol === 'http:' || parsedUri.protocol === 'https:') return parsedUri; + if (parsedUri.protocol === 'ipfs:') + return new URL(`${PUBLIC_IPFS}/${parsedUri.host}${parsedUri.pathname}`); + + if (parsedUri.protocol === 'ipns:') + return new URL(`${PUBLIC_IPFS}/${parsedUri.host}${parsedUri.pathname}`); + + throw new Error(`Unsupported uri protocol: ${uri}`); + } + + private getImageUrl(uri: string): string { + // Support images embedded in a Data URL + if (new URL(uri).protocol === 'data:') { + // const dataUrl = ParseDataUrl(uri); + const dataUrl = parseDataUrl(uri); + if (!dataUrl) { + throw new Error(`Data URL could not be parsed: ${uri}`); + } + if (!dataUrl.mediaType?.startsWith('image/')) { + throw new Error(`Token image is a Data URL with a non-image media type: ${uri}`); + } + return uri; + } + const fetchableUrl = this.getFetchableUrl(uri); + return fetchableUrl.toString(); + } + + /** + * Fetch metadata from uri + */ + private async getMetadataFromUri(token_uri: string): Promise { + // Support JSON embedded in a Data URL + if (new URL(token_uri).protocol === 'data:') { + const dataUrl = parseDataUrl(token_uri); + if (!dataUrl) { + throw new Error(`Data URL could not be parsed: ${token_uri}`); + } + let content: string; + // If media type is omitted it should default to percent-encoded `text/plain;charset=US-ASCII` + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs#syntax + // If media type is specified but without base64 then encoding is ambiguous, so check for + // percent-encoding or assume a literal string compatible with utf8. Because we're expecting + // a JSON object we can reliable check for a leading `%` char, otherwise assume unescaped JSON. + if (dataUrl.base64) { + content = Buffer.from(dataUrl.data, 'base64').toString('utf8'); + } else if (dataUrl.data.startsWith('%')) { + content = querystring.unescape(dataUrl.data); + } else { + content = dataUrl.data; + } + try { + return JSON.parse(content) as Type; + } catch (error) { + throw new Error(`Data URL could not be parsed as JSON: ${token_uri}`); + } + } + const httpUrl = this.getFetchableUrl(token_uri); + return await performFetch(httpUrl.toString(), { + timeoutMs: METADATA_FETCH_TIMEOUT_MS, + maxResponseBytes: METADATA_MAX_PAYLOAD_BYTE_SIZE, + }); + } + + /** + * Make readonly contract call + */ + private async makeReadOnlyContractCall( + functionName: string, + functionArgs: ClarityValue[] + ): Promise { + const txOptions: ReadOnlyFunctionOptions = { + senderAddress: this.address, + contractAddress: this.contractAddress, + contractName: this.contractName, + functionName: functionName, + functionArgs: functionArgs, + network: this.stacksNetwork, + }; + return await callReadOnlyFunction(txOptions); + } + + private async readStringFromContract( + functionName: string, + functionArgs: ClarityValue[] + ): Promise { + try { + const clarityValue = await this.makeReadOnlyContractCall(functionName, functionArgs); + const stringVal = this.checkAndParseString(clarityValue); + return stringVal; + } catch (error) { + logger.warn( + `[token-metadata] error extracting string with contract function call '${functionName}' while processing ${this.contractId}`, + error + ); + } + } + + private async readUIntFromContract( + functionName: string, + functionArgs: ClarityValue[] + ): Promise { + try { + const clarityValue = await this.makeReadOnlyContractCall(functionName, functionArgs); + const uintVal = this.checkAndParseUintCV(clarityValue); + return BigInt(uintVal.value.toString()); + } catch (error) { + logger.warn( + `[token-metadata] error extracting string with contract function call '${functionName}' while processing ${this.contractId}`, + error + ); + } + } + + /** + * Store ft metadata to db + */ + private async storeFtMetadata(ftMetadata: DbFungibleTokenMetadata) { + try { + await this.db.updateFtMetadata(ftMetadata, this.dbQueueId); + } catch (error) { + throw new Error(`Error occurred while updating FT metadata ${error}`); + } + } + + /** + * Store NFT Metadata to db + */ + private async storeNftMetadata(nftMetadata: DbNonFungibleTokenMetadata) { + try { + await this.db.updateNFtMetadata(nftMetadata, this.dbQueueId); + } catch (error) { + throw new Error(`Error occurred while updating NFT metadata ${error}`); + } + } + + private unwrapClarityType(clarityValue: ClarityValue): ClarityValue { + let unwrappedClarityValue: ClarityValue = clarityValue; + while ( + unwrappedClarityValue.type === ClarityType.ResponseOk || + unwrappedClarityValue.type === ClarityType.OptionalSome + ) { + unwrappedClarityValue = unwrappedClarityValue.value; + } + return unwrappedClarityValue; + } + + private checkAndParseUintCV(responseCV: ClarityValue): UIntCV { + const unwrappedClarityValue = this.unwrapClarityType(responseCV); + if (unwrappedClarityValue.type === ClarityType.UInt) { + return unwrappedClarityValue; + } + throw new Error( + `Unexpected Clarity type '${unwrappedClarityValue.type}' while unwrapping uint` + ); + } + + private checkAndParseString(responseCV: ClarityValue): string { + const unwrappedClarityValue = this.unwrapClarityType(responseCV); + if ( + unwrappedClarityValue.type === ClarityType.StringASCII || + unwrappedClarityValue.type === ClarityType.StringUTF8 + ) { + return unwrappedClarityValue.data; + } + throw new Error( + `Unexpected Clarity type '${unwrappedClarityValue.type}' while unwrapping string` + ); + } +} + +export class TokensProcessorQueue { + readonly queue: PQueue; + readonly db: DataStore; + readonly chainId: ChainID; + + readonly processStartedEvent: Evt<{ + contractId: string; + txId: string; + }> = new Evt(); + + readonly processEndEvent: Evt<{ + contractId: string; + txId: string; + }> = new Evt(); + + /** The entries currently queued for processing in memory, keyed by the queue entry db id. */ + readonly queuedEntries: Map = new Map(); + + readonly onTokenMetadataUpdateQueued: (entry: DbTokenMetadataQueueEntry) => void; + + constructor(db: DataStore, chainId: ChainID) { + this.db = db; + this.chainId = chainId; + this.queue = new PQueue({ concurrency: TOKEN_METADATA_PARSING_CONCURRENCY_LIMIT }); + this.onTokenMetadataUpdateQueued = entry => this.queueHandler(entry); + this.db.on('tokenMetadataUpdateQueued', this.onTokenMetadataUpdateQueued); + } + + close() { + this.db.off('tokenMetadataUpdateQueued', this.onTokenMetadataUpdateQueued); + this.queue.pause(); + this.queue.clear(); + } + + async drainDbQueue(): Promise { + let entries: DbTokenMetadataQueueEntry[] = []; + do { + if (this.queue.isPaused) { + return; + } + const queuedEntries = [...this.queuedEntries.keys()]; + entries = await this.db.getTokenMetadataQueue( + TOKEN_METADATA_PARSING_CONCURRENCY_LIMIT, + queuedEntries + ); + for (const entry of entries) { + this.queueHandler(entry); + } + await this.queue.onEmpty(); + // await this.queue.onIdle(); + } while (entries.length > 0 || this.queuedEntries.size > 0); + } + + async checkDbQueue(): Promise { + if (this.queue.isPaused) { + return; + } + const queuedEntries = [...this.queuedEntries.keys()]; + const limit = TOKEN_METADATA_PARSING_CONCURRENCY_LIMIT - this.queuedEntries.size; + if (limit > 0) { + const entries = await this.db.getTokenMetadataQueue( + TOKEN_METADATA_PARSING_CONCURRENCY_LIMIT, + queuedEntries + ); + for (const entry of entries) { + this.queueHandler(entry); + } + } + } + + queueHandler(queueEntry: DbTokenMetadataQueueEntry) { + if ( + this.queuedEntries.has(queueEntry.queueId) || + this.queuedEntries.size >= this.queue.concurrency + ) { + return; + } + logger.info( + `[token-metadata] queueing token contract for processing: ${queueEntry.contractId} from tx ${queueEntry.txId}` + ); + this.queuedEntries.set(queueEntry.queueId, queueEntry); + + const tokenContractHandler = new TokensContractHandler({ + contractId: queueEntry.contractId, + smartContractAbi: queueEntry.contractAbi, + datastore: this.db, + chainId: this.chainId, + txId: queueEntry.txId, + dbQueueId: queueEntry.queueId, + }); + + void this.queue + .add(async () => { + this.processStartedEvent.post({ + contractId: queueEntry.contractId, + txId: queueEntry.txId, + }); + await tokenContractHandler.start(); + }) + .catch(error => { + logError( + `[token-metadata] error processing token contract: ${tokenContractHandler.contractAddress} ${tokenContractHandler.contractName} from tx ${tokenContractHandler.txId}`, + error + ); + }) + .finally(() => { + this.queuedEntries.delete(queueEntry.queueId); + this.processEndEvent.post({ + contractId: queueEntry.contractId, + txId: queueEntry.txId, + }); + logger.info( + `[token-metadata] finished token contract processing for: ${queueEntry.contractId} from tx ${queueEntry.txId}` + ); + if (this.queuedEntries.size < this.queue.concurrency) { + void this.checkDbQueue(); + } + }); + } +} + +export async function performFetch( + url: string, + opts?: { + timeoutMs?: number; + maxResponseBytes?: number; + } +): Promise { + const result = await fetch(url, { + size: opts?.maxResponseBytes ?? METADATA_MAX_PAYLOAD_BYTE_SIZE, + timeout: opts?.timeoutMs ?? METADATA_FETCH_TIMEOUT_MS, + }); + if (!result.ok) { + let msg = ''; + try { + msg = await result.text(); + } catch (error) { + // ignore errors from fetching error text + } + throw new Error(`Response ${result.status}: ${result.statusText} fetching ${url} - ${msg}`); + } + const resultString = await result.text(); + try { + return JSON.parse(resultString) as Type; + } catch (error) { + throw new Error(`Error parsing response from ${url} as JSON: ${error}`); + } +} diff --git a/src/helpers.ts b/src/helpers.ts index c23018914c..1464eeeea7 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -829,6 +829,48 @@ export function normalizeHashString(input: string): string | false { return `0x${hashBuffer.toString('hex')}`; } +export function parseDataUrl( + s: string +): + | { mediaType?: string; contentType?: string; charset?: string; base64: boolean; data: string } + | false { + try { + const url = new URL(s); + if (url.protocol !== 'data:') { + return false; + } + const validDataUrlRegex = /^data:([a-z]+\/[a-z0-9-+.]+(;[a-z0-9-.!#$%*+.{}|~`]+=[a-z0-9-.!#$%*+.{}()|~`]+)*)?(;base64)?,(.*)$/i; + const parts = validDataUrlRegex.exec(s.trim()); + if (parts === null) { + return false; + } + const parsed: { + mediaType?: string; + contentType?: string; + charset?: string; + base64: boolean; + data: string; + } = { + base64: false, + data: '', + }; + if (parts[1]) { + parsed.mediaType = parts[1].toLowerCase(); + const mediaTypeParts = parts[1].split(';').map(x => x.toLowerCase()); + parsed.contentType = mediaTypeParts[0]; + mediaTypeParts.slice(1).forEach(attribute => { + const p = attribute.split('='); + Object.assign(parsed, { [p[0]]: p[1] }); + }); + } + parsed.base64 = !!parts[parts.length - 2]; + parsed.data = parts[parts.length - 1] || ''; + return parsed; + } catch (e) { + return false; + } +} + export function getSendManyContract(chainId: ChainID) { const contractId = chainId === ChainID.Mainnet diff --git a/src/index.ts b/src/index.ts index d33ee967c4..3a57417a8a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,6 +13,11 @@ import { cycleMigrations, dangerousDropAllTables, PgDataStore } from './datastor import { MemoryDataStore } from './datastore/memory-store'; import { startApiServer } from './api/init'; import { startEventServer } from './event-stream/event-server'; +import { + isFtMetadataEnabled, + isNftMetadataEnabled, + TokensProcessorQueue, +} from './event-stream/tokens-contract-handler'; import { StacksCoreRpcClient } from './core-rpc/client'; import { createServer as createPrometheusServer } from '@promster/server'; import { ChainID } from '@stacks/transactions'; @@ -112,6 +117,7 @@ async function init(): Promise { } const configuredChainID = getConfiguredChainID(); + const eventServer = await startEventServer({ datastore: db, chainId: configuredChainID, @@ -135,6 +141,17 @@ async function init(): Promise { monitorCoreRpcConnection().catch(error => { logger.error(`Error monitoring RPC connection: ${error}`, error); }); + + if (isFtMetadataEnabled() || isNftMetadataEnabled()) { + const tokenMetadataProcessor = new TokensProcessorQueue(db, configuredChainID); + registerShutdownConfig({ + name: 'Token Metadata Processor', + handler: () => tokenMetadataProcessor.close(), + forceKillable: true, + }); + // check if db has any non-processed token queues and await them all here + await tokenMetadataProcessor.drainDbQueue(); + } } if (isProdEnv && !fs.existsSync('.git-info')) { diff --git a/src/migrations/1621511823100_token-metadata-queue.ts b/src/migrations/1621511823100_token-metadata-queue.ts new file mode 100644 index 0000000000..a79247102c --- /dev/null +++ b/src/migrations/1621511823100_token-metadata-queue.ts @@ -0,0 +1,36 @@ +/* eslint-disable @typescript-eslint/camelcase */ +import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; + +export const shorthands: ColumnDefinitions | undefined = undefined; + +export async function up(pgm: MigrationBuilder): Promise { + pgm.createTable('token_metadata_queue', { + queue_id: { + type: 'serial', + primaryKey: true, + }, + tx_id: { + type: 'bytea', + notNull: true, + }, + contract_id: { + type: 'string', + notNull: true, + }, + contract_abi: { + type: 'string', + notNull: true, + }, + block_height: { + type: 'integer', + notNull: true, + }, + processed: { + type: 'boolean', + notNull: true, + } + }); + + pgm.createIndex('token_metadata_queue', 'block_height'); + pgm.createIndex('token_metadata_queue', 'processed'); +} diff --git a/src/migrations/1621511823381_nft-metadata.ts b/src/migrations/1621511823381_nft-metadata.ts new file mode 100644 index 0000000000..5038d812a2 --- /dev/null +++ b/src/migrations/1621511823381_nft-metadata.ts @@ -0,0 +1,53 @@ +/* eslint-disable @typescript-eslint/camelcase */ +import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; + +export const shorthands: ColumnDefinitions | undefined = undefined; + +export async function up(pgm: MigrationBuilder): Promise { + pgm.createTable('nft_metadata', { + id: { + type: 'serial', + primaryKey: true, + }, + name: { + type: 'string', + notNull: true, + }, + token_uri: { + type: 'string', + notNull: true, + }, + description: { + type: 'string', + notNull: true, + }, + image_uri: { + type: 'string', + notNull: true, + }, + image_canonical_uri: { + type: 'string', + notNull: true, + }, + contract_id: { + type: 'string', + notNull: true, + }, + tx_id: { + type: 'bytea', + notNull: true, + }, + sender_address: { + type: 'string', + notNull: true, + } + }); + + pgm.createIndex('nft_metadata', 'name'); + pgm.createIndex('nft_metadata', 'contract_id'); + pgm.createIndex('nft_metadata', 'tx_id'); +} + +export async function down(pgm: MigrationBuilder): Promise { + pgm.dropTable('nft_metadata') +} diff --git a/src/migrations/1621511832113_ft-metadata.ts b/src/migrations/1621511832113_ft-metadata.ts new file mode 100644 index 0000000000..0358fa752b --- /dev/null +++ b/src/migrations/1621511832113_ft-metadata.ts @@ -0,0 +1,62 @@ +/* eslint-disable @typescript-eslint/camelcase */ +import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; + +export const shorthands: ColumnDefinitions | undefined = undefined; + +export async function up(pgm: MigrationBuilder): Promise { + pgm.createTable('ft_metadata', { + id: { + type: 'serial', + primaryKey: true, + }, + name: { + type: 'string', + notNull: true, + }, + token_uri: { + type: 'string', + notNull: true, + }, + description: { + type: 'string', + notNull: true, + }, + image_uri: { + type: 'string', + notNull: true, + }, + image_canonical_uri: { + type: 'string', + notNull: true, + }, + contract_id: { + type: 'string', + notNull: true, + }, + symbol: { + type: 'string', + notNull: true, + }, + decimals: { + type: 'integer', + notNull: true, + }, + tx_id: { + type: 'bytea', + notNull: true, + }, + sender_address: { + type: 'string', + notNull: true, + } + }); + + pgm.createIndex('ft_metadata', 'name'); + pgm.createIndex('ft_metadata', 'symbol'); + pgm.createIndex('ft_metadata', 'contract_id'); + pgm.createIndex('ft_metadata', 'tx_id'); +} + +export async function down(pgm: MigrationBuilder): Promise { + pgm.dropTable('ft_metadata'); +} diff --git a/src/tests-tokens/setup.ts b/src/tests-tokens/setup.ts new file mode 100644 index 0000000000..266fa83083 --- /dev/null +++ b/src/tests-tokens/setup.ts @@ -0,0 +1,23 @@ +import { loadDotEnv } from '../helpers'; +import { StacksCoreRpcClient } from '../core-rpc/client'; +import { PgDataStore } from '../datastore/postgres-store'; + +export interface GlobalServices { + db: PgDataStore; +} + +export default async (): Promise => { + console.log('Jest - setup..'); + if (!process.env.NODE_ENV) { + process.env.NODE_ENV = 'test'; + } + loadDotEnv(); + const db = await PgDataStore.connect(true); + console.log('Waiting for RPC connection to core node..'); + await new StacksCoreRpcClient().waitForConnection(60000); + const globalServices: GlobalServices = { + db: db, + }; + Object.assign(global, globalServices); + console.log('Jest - setup done'); +}; diff --git a/src/tests-tokens/teardown.ts b/src/tests-tokens/teardown.ts new file mode 100644 index 0000000000..13d5b05702 --- /dev/null +++ b/src/tests-tokens/teardown.ts @@ -0,0 +1,8 @@ +import type { GlobalServices } from './setup'; + +export default async (): Promise => { + console.log('Jest - teardown..'); + const globalServices = (global as unknown) as GlobalServices; + await globalServices.db.close(); + console.log('Jest - teardown done'); +}; diff --git a/src/tests-tokens/test-contracts/beeple-data-url-a.clar b/src/tests-tokens/test-contracts/beeple-data-url-a.clar new file mode 100644 index 0000000000..a773a93d5f --- /dev/null +++ b/src/tests-tokens/test-contracts/beeple-data-url-a.clar @@ -0,0 +1,58 @@ +;; (impl-trait 'STB44HYPYAT2BB2QE513NSP81HTMYWBJP02HPGK6.nft-trait.nft-trait) +(define-non-fungible-token beeple uint) + +;; Public functions +(define-constant nft-not-owned-err (err u401)) ;; unauthorized +(define-constant nft-not-found-err (err u404)) ;; not found +(define-constant sender-equals-recipient-err (err u405)) ;; method not allowed + +(define-private (nft-transfer-err (code uint)) + (if (is-eq u1 code) + nft-not-owned-err + (if (is-eq u2 code) + sender-equals-recipient-err + (if (is-eq u3 code) + nft-not-found-err + (err code))))) + +;; Transfers tokens to a specified principal. +(define-public (transfer (token-id uint) (sender principal) (recipient principal)) + (if (and + (is-eq tx-sender (unwrap! (nft-get-owner? beeple token-id) nft-not-found-err)) + (is-eq tx-sender sender) + (not (is-eq recipient sender))) + (match (nft-transfer? beeple token-id sender recipient) + success (ok success) + error (nft-transfer-err error)) + nft-not-owned-err)) + +;; Gets the owner of the specified token ID. +(define-read-only (get-owner (token-id uint)) + (ok (nft-get-owner? beeple token-id))) + +;; Gets the owner of the specified token ID. +(define-read-only (get-last-token-id) + (ok u1)) + +(define-read-only (get-token-uri (token-id uint)) + (ok (some "data:,%7B%22name%22%3A%22Heystack%22%2C%22description%22%3A%22Heystack%20is%20a%20SIP-010-compliant%20fungible%20token%22%2C%22imageUrl%22%3A%22https%3A%2F%2Fheystack.xyz%2Fassets%2FStacks128w.png%22%7D"))) + +(define-read-only (get-meta (token-id uint)) + (if (is-eq token-id u1) + (ok (some {name: "EVERYDAYS: THE FIRST 5000 DAYS", uri: "https://ipfsgateway.makersplace.com/ipfs/QmZ15eQX8FPjfrtdX3QYbrhZxJpbLpvDpsgb2p3VEH8Bqq", mime-type: "image/jpeg"})) + (ok none))) + +(define-read-only (get-nft-meta) + (ok (some {name: "beeple", uri: "https://ipfsgateway.makersplace.com/ipfs/QmZ15eQX8FPjfrtdX3QYbrhZxJpbLpvDpsgb2p3VEH8Bqq", mime-type: "image/jpeg"}))) + +(define-read-only (get-errstr (code uint)) + (ok (if (is-eq u401 code) + "nft-not-owned" + (if (is-eq u404 code) + "nft-not-found" + (if (is-eq u405 code) + "sender-equals-recipient" + "unknown-error"))))) + +;; Initialize the contract +(try! (nft-mint? beeple u1 tx-sender)) diff --git a/src/tests-tokens/test-contracts/beeple-data-url-b.clar b/src/tests-tokens/test-contracts/beeple-data-url-b.clar new file mode 100644 index 0000000000..a8dfbf21c5 --- /dev/null +++ b/src/tests-tokens/test-contracts/beeple-data-url-b.clar @@ -0,0 +1,58 @@ +;; (impl-trait 'STB44HYPYAT2BB2QE513NSP81HTMYWBJP02HPGK6.nft-trait.nft-trait) +(define-non-fungible-token beeple uint) + +;; Public functions +(define-constant nft-not-owned-err (err u401)) ;; unauthorized +(define-constant nft-not-found-err (err u404)) ;; not found +(define-constant sender-equals-recipient-err (err u405)) ;; method not allowed + +(define-private (nft-transfer-err (code uint)) + (if (is-eq u1 code) + nft-not-owned-err + (if (is-eq u2 code) + sender-equals-recipient-err + (if (is-eq u3 code) + nft-not-found-err + (err code))))) + +;; Transfers tokens to a specified principal. +(define-public (transfer (token-id uint) (sender principal) (recipient principal)) + (if (and + (is-eq tx-sender (unwrap! (nft-get-owner? beeple token-id) nft-not-found-err)) + (is-eq tx-sender sender) + (not (is-eq recipient sender))) + (match (nft-transfer? beeple token-id sender recipient) + success (ok success) + error (nft-transfer-err error)) + nft-not-owned-err)) + +;; Gets the owner of the specified token ID. +(define-read-only (get-owner (token-id uint)) + (ok (nft-get-owner? beeple token-id))) + +;; Gets the owner of the specified token ID. +(define-read-only (get-last-token-id) + (ok u1)) + +(define-read-only (get-token-uri (token-id uint)) + (ok (some "data:;base64,eyJuYW1lIjoiSGV5c3RhY2siLCJkZXNjcmlwdGlvbiI6IkhleXN0YWNrIGlzIGEgU0lQLTAxMC1jb21wbGlhbnQgZnVuZ2libGUgdG9rZW4iLCJpbWFnZVVybCI6Imh0dHBzOi8vaGV5c3RhY2sueHl6L2Fzc2V0cy9TdGFja3MxMjh3LnBuZyJ9"))) + +(define-read-only (get-meta (token-id uint)) + (if (is-eq token-id u1) + (ok (some {name: "EVERYDAYS: THE FIRST 5000 DAYS", uri: "https://ipfsgateway.makersplace.com/ipfs/QmZ15eQX8FPjfrtdX3QYbrhZxJpbLpvDpsgb2p3VEH8Bqq", mime-type: "image/jpeg"})) + (ok none))) + +(define-read-only (get-nft-meta) + (ok (some {name: "beeple", uri: "https://ipfsgateway.makersplace.com/ipfs/QmZ15eQX8FPjfrtdX3QYbrhZxJpbLpvDpsgb2p3VEH8Bqq", mime-type: "image/jpeg"}))) + +(define-read-only (get-errstr (code uint)) + (ok (if (is-eq u401 code) + "nft-not-owned" + (if (is-eq u404 code) + "nft-not-found" + (if (is-eq u405 code) + "sender-equals-recipient" + "unknown-error"))))) + +;; Initialize the contract +(try! (nft-mint? beeple u1 tx-sender)) diff --git a/src/tests-tokens/test-contracts/beeple-data-url-c.clar b/src/tests-tokens/test-contracts/beeple-data-url-c.clar new file mode 100644 index 0000000000..fa335337ac --- /dev/null +++ b/src/tests-tokens/test-contracts/beeple-data-url-c.clar @@ -0,0 +1,58 @@ +;; (impl-trait 'STB44HYPYAT2BB2QE513NSP81HTMYWBJP02HPGK6.nft-trait.nft-trait) +(define-non-fungible-token beeple uint) + +;; Public functions +(define-constant nft-not-owned-err (err u401)) ;; unauthorized +(define-constant nft-not-found-err (err u404)) ;; not found +(define-constant sender-equals-recipient-err (err u405)) ;; method not allowed + +(define-private (nft-transfer-err (code uint)) + (if (is-eq u1 code) + nft-not-owned-err + (if (is-eq u2 code) + sender-equals-recipient-err + (if (is-eq u3 code) + nft-not-found-err + (err code))))) + +;; Transfers tokens to a specified principal. +(define-public (transfer (token-id uint) (sender principal) (recipient principal)) + (if (and + (is-eq tx-sender (unwrap! (nft-get-owner? beeple token-id) nft-not-found-err)) + (is-eq tx-sender sender) + (not (is-eq recipient sender))) + (match (nft-transfer? beeple token-id sender recipient) + success (ok success) + error (nft-transfer-err error)) + nft-not-owned-err)) + +;; Gets the owner of the specified token ID. +(define-read-only (get-owner (token-id uint)) + (ok (nft-get-owner? beeple token-id))) + +;; Gets the owner of the specified token ID. +(define-read-only (get-last-token-id) + (ok u1)) + +(define-read-only (get-token-uri (token-id uint)) + (ok (some "data:application/json,{\"name\":\"Heystack\",\"description\":\"Heystack is a SIP-010-compliant fungible token\",\"imageUrl\":\"https://heystack.xyz/assets/Stacks128w.png\"}"))) + ;; (ok (some "data:text/html,"))) +(define-read-only (get-meta (token-id uint)) + (if (is-eq token-id u1) + (ok (some {name: "EVERYDAYS: THE FIRST 5000 DAYS", uri: "https://ipfsgateway.makersplace.com/ipfs/QmZ15eQX8FPjfrtdX3QYbrhZxJpbLpvDpsgb2p3VEH8Bqq", mime-type: "image/jpeg"})) + (ok none))) + +(define-read-only (get-nft-meta) + (ok (some {name: "beeple", uri: "https://ipfsgateway.makersplace.com/ipfs/QmZ15eQX8FPjfrtdX3QYbrhZxJpbLpvDpsgb2p3VEH8Bqq", mime-type: "image/jpeg"}))) + +(define-read-only (get-errstr (code uint)) + (ok (if (is-eq u401 code) + "nft-not-owned" + (if (is-eq u404 code) + "nft-not-found" + (if (is-eq u405 code) + "sender-equals-recipient" + "unknown-error"))))) + +;; Initialize the contract +(try! (nft-mint? beeple u1 tx-sender)) diff --git a/src/tests-tokens/test-contracts/beeple.clar b/src/tests-tokens/test-contracts/beeple.clar new file mode 100644 index 0000000000..d7ea8b693b --- /dev/null +++ b/src/tests-tokens/test-contracts/beeple.clar @@ -0,0 +1,58 @@ +(impl-trait 'STB44HYPYAT2BB2QE513NSP81HTMYWBJP02HPGK6.nft-trait.nft-trait) +(define-non-fungible-token beeple uint) + +;; Public functions +(define-constant nft-not-owned-err (err u401)) ;; unauthorized +(define-constant nft-not-found-err (err u404)) ;; not found +(define-constant sender-equals-recipient-err (err u405)) ;; method not allowed + +(define-private (nft-transfer-err (code uint)) + (if (is-eq u1 code) + nft-not-owned-err + (if (is-eq u2 code) + sender-equals-recipient-err + (if (is-eq u3 code) + nft-not-found-err + (err code))))) + +;; Transfers tokens to a specified principal. +(define-public (transfer (token-id uint) (sender principal) (recipient principal)) + (if (and + (is-eq tx-sender (unwrap! (nft-get-owner? beeple token-id) nft-not-found-err)) + (is-eq tx-sender sender) + (not (is-eq recipient sender))) + (match (nft-transfer? beeple token-id sender recipient) + success (ok success) + error (nft-transfer-err error)) + nft-not-owned-err)) + +;; Gets the owner of the specified token ID. +(define-read-only (get-owner (token-id uint)) + (ok (nft-get-owner? beeple token-id))) + +;; Gets the owner of the specified token ID. +(define-read-only (get-last-token-id) + (ok u1)) + +(define-read-only (get-token-uri (token-id uint)) + (ok (some "ipfs://ipfs/QmPAg1mjxcEQPPtqsLoEcauVedaeMH81WXDPvPx3VC5zUz"))) + +(define-read-only (get-meta (token-id uint)) + (if (is-eq token-id u1) + (ok (some {name: "EVERYDAYS: THE FIRST 5000 DAYS", uri: "https://ipfsgateway.makersplace.com/ipfs/QmZ15eQX8FPjfrtdX3QYbrhZxJpbLpvDpsgb2p3VEH8Bqq", mime-type: "image/jpeg"})) + (ok none))) + +(define-read-only (get-nft-meta) + (ok (some {name: "beeple", uri: "https://ipfsgateway.makersplace.com/ipfs/QmZ15eQX8FPjfrtdX3QYbrhZxJpbLpvDpsgb2p3VEH8Bqq", mime-type: "image/jpeg"}))) + +(define-read-only (get-errstr (code uint)) + (ok (if (is-eq u401 code) + "nft-not-owned" + (if (is-eq u404 code) + "nft-not-found" + (if (is-eq u405 code) + "sender-equals-recipient" + "unknown-error"))))) + +;; Initialize the contract +(try! (nft-mint? beeple u1 tx-sender)) diff --git a/src/tests-tokens/test-contracts/ft-trait.clar b/src/tests-tokens/test-contracts/ft-trait.clar new file mode 100644 index 0000000000..69255cf9e9 --- /dev/null +++ b/src/tests-tokens/test-contracts/ft-trait.clar @@ -0,0 +1,24 @@ +(define-trait sip-010-trait + ( + ;; Transfer from the caller to a new principal + (transfer (uint principal principal (optional (buff 34))) (response bool uint)) + + ;; the human readable name of the token + (get-name () (response (string-ascii 32) uint)) + + ;; the ticker symbol, or empty if none + (get-symbol () (response (string-ascii 32) uint)) + + ;; the number of decimals used, e.g. 6 would mean 1_000_000 represents 1 token + (get-decimals () (response uint uint)) + + ;; the balance of the passed principal + (get-balance (principal) (response uint uint)) + + ;; the current total supply (which does not need to be a constant) + (get-total-supply () (response uint uint)) + + ;; an optional URI that represents metadata of this token + (get-token-uri () (response (optional (string-utf8 256)) uint)) + ) +) diff --git a/src/tests-tokens/test-contracts/hey-token.clar b/src/tests-tokens/test-contracts/hey-token.clar new file mode 100644 index 0000000000..bd9a50c5c1 --- /dev/null +++ b/src/tests-tokens/test-contracts/hey-token.clar @@ -0,0 +1,58 @@ +;; Implement the `ft-trait` trait defined in the `ft-trait` contract +;; https://github.com/hstove/stacks-fungible-token +(impl-trait 'STB44HYPYAT2BB2QE513NSP81HTMYWBJP02HPGK6.ft-trait.sip-010-trait) + +(define-constant contract-creator tx-sender) + +(define-fungible-token hey-token) + +;; Mint developer tokens +(ft-mint? hey-token u10000 contract-creator) +(ft-mint? hey-token u10000 'ST399W7Z9WS0GMSNQGJGME5JADNKN56R65VGM5KGA) ;; fara +(ft-mint? hey-token u10000 'ST1X6M947Z7E58CNE0H8YJVJTVKS9VW0PHEG3NHN3) ;; thomas +(ft-mint? hey-token u10000 'ST1NY8TXACV7D74886MK05SYW2XA72XJMDVPF3F3D) ;; kyran +(ft-mint? hey-token u10000 'ST34XEPDJJFJKFPT87CCZQCPGXR4PJ8ERFRP0F3GX) ;; jasper +(ft-mint? hey-token u10000 'ST3AGWHGAZKQS4JQ67WQZW5X8HZYZ4ZBWPPNWNMKF) ;; andres +(ft-mint? hey-token u10000 'ST17YZQB1228EK9MPHQXA8GC4G3HVWZ66X779FEBY) ;; esh +(ft-mint? hey-token u10000 'ST3Q0M9WAVBW633CG72VHNFZM2H82D2BJMBX85WP4) ;; mark + +;; get the token balance of owner +(define-read-only (get-balance (owner principal)) + (begin + (ok (ft-get-balance hey-token owner)))) + +;; returns the total number of tokens +(define-read-only (get-total-supply) + (ok (ft-get-supply hey-token))) + +;; returns the token name +(define-read-only (get-name) + (ok "Heystack Token")) + +;; the symbol or "ticker" for this token +(define-read-only (get-symbol) + (ok "HEY")) + +;; the number of decimals used +(define-read-only (get-decimals) + (ok u0)) + +;; Transfers tokens to a recipient +(define-public (transfer (amount uint) (sender principal) (recipient principal) (memo (optional (buff 34)))) + (if (is-eq tx-sender sender) + (begin + (try! (ft-transfer? hey-token amount sender recipient)) + (print memo) + (ok true) + ) + (err u4))) + +(define-read-only (get-token-uri) + (ok (some u"https://heystack.xyz/token-metadata.json"))) + +(define-public (gift-tokens (recipient principal)) + (begin + (asserts! (is-eq tx-sender recipient) (err u0)) + (ft-mint? hey-token u1 recipient) + ) +) diff --git a/src/tests-tokens/test-contracts/nft-trait.clar b/src/tests-tokens/test-contracts/nft-trait.clar new file mode 100644 index 0000000000..cc558fdb5a --- /dev/null +++ b/src/tests-tokens/test-contracts/nft-trait.clar @@ -0,0 +1,15 @@ +(define-trait nft-trait + ( + ;; Last token ID, limited to uint range + (get-last-token-id () (response uint uint)) + + ;; URI for metadata associated with the token + (get-token-uri (uint) (response (optional (string-ascii 256)) uint)) + + ;; Owner of a given token identifier + (get-owner (uint) (response (optional principal) uint)) + + ;; Transfer from the sender to a new principal + (transfer (uint principal principal) (response bool uint)) + ) +) diff --git a/src/tests-tokens/tokens-metadata-tests.ts b/src/tests-tokens/tokens-metadata-tests.ts new file mode 100644 index 0000000000..c72c223fec --- /dev/null +++ b/src/tests-tokens/tokens-metadata-tests.ts @@ -0,0 +1,394 @@ +import * as supertest from 'supertest'; +import { + makeContractDeploy, + ChainID, + getAddressFromPrivateKey, + PostConditionMode, +} from '@stacks/transactions'; +import * as BN from 'bn.js'; +import { + DbTx, + DbMempoolTx, + DbTxStatus, + DbFungibleTokenMetadata, + DbNonFungibleTokenMetadata, +} from '../datastore/common'; +import { startApiServer, ApiServer } from '../api/init'; +import { PgDataStore, cycleMigrations, runMigrations } from '../datastore/postgres-store'; +import { PoolClient } from 'pg'; +import * as fs from 'fs'; +import { EventStreamServer, startEventServer } from '../event-stream/event-server'; +import { getStacksTestnetNetwork } from '../rosetta-helpers'; +import { StacksCoreRpcClient } from '../core-rpc/client'; +import { logger, timeout } from '../helpers'; +import * as nock from 'nock'; +import { performFetch, TokensProcessorQueue } from './../event-stream/tokens-contract-handler'; + +const pKey = 'cb3df38053d132895220b9ce471f6b676db5b9bf0b4adefb55f2118ece2478df01'; +const stacksNetwork = getStacksTestnetNetwork(); +const HOST = 'localhost'; +const PORT = 20443; + +describe('api tests', () => { + let db: PgDataStore; + let client: PoolClient; + let api: ApiServer; + let eventServer: EventStreamServer; + let tokensProcessorQueue: TokensProcessorQueue; + + function standByForTx(expectedTxId: string): Promise { + const broadcastTx = new Promise(resolve => { + const listener: (info: DbTx | DbMempoolTx) => void = info => { + if ( + info.tx_id === expectedTxId && + (info.status === DbTxStatus.Success || + info.status === DbTxStatus.AbortByResponse || + info.status === DbTxStatus.AbortByPostCondition) + ) { + api.datastore.removeListener('txUpdate', listener); + resolve(info as DbTx); + } + }; + api.datastore.addListener('txUpdate', listener); + }); + + return broadcastTx; + } + + function standByForTokens(id: string): Promise { + const contractId = new Promise(resolve => { + tokensProcessorQueue.processEndEvent.attachOnce( + token => token.contractId === id, + () => resolve() + ); + }); + + return contractId; + } + + async function sendCoreTx(serializedTx: Buffer): Promise<{ txId: string }> { + try { + const submitResult = await new StacksCoreRpcClient({ + host: HOST, + port: PORT, + }).sendTransaction(serializedTx); + return submitResult; + } catch (error) { + logger.error('error: ', error); + } + return Promise.resolve({ txId: '' }); + } + + async function deployContract(contractName: string, senderPk: string, sourceFile: string) { + const senderAddress = getAddressFromPrivateKey(senderPk, stacksNetwork.version); + const source = fs.readFileSync(sourceFile).toString(); + const normalized_contract_source = source.replace(/\r/g, '').replace(/\t/g, ' '); + + const contractDeployTx = await makeContractDeploy({ + contractName: contractName, + codeBody: normalized_contract_source, + senderKey: senderPk, + network: stacksNetwork, + postConditionMode: PostConditionMode.Allow, + sponsored: false, + }); + + const contractId = senderAddress + '.' + contractName; + + const feeRateReq = await fetch(stacksNetwork.getTransferFeeEstimateApiUrl()); + const feeRateResult = await feeRateReq.text(); + const txBytes = new BN(contractDeployTx.serialize().byteLength); + const feeRate = new BN(feeRateResult); + const fee = feeRate.mul(txBytes); + contractDeployTx.setFee(fee); + const { txId } = await sendCoreTx(contractDeployTx.serialize()); + return { txId, contractId }; + } + + beforeAll(async () => { + process.env.PG_DATABASE = 'postgres'; + await cycleMigrations(); + db = await PgDataStore.connect(); + client = await db.pool.connect(); + eventServer = await startEventServer({ datastore: db, chainId: ChainID.Testnet }); + api = await startApiServer({ datastore: db, chainId: ChainID.Testnet }); + tokensProcessorQueue = new TokensProcessorQueue(db, ChainID.Testnet); + }); + + beforeEach(() => { + process.env['STACKS_API_ENABLE_FT_METADATA'] = '1'; + process.env['STACKS_API_ENABLE_NFT_METADATA'] = '1'; + nock.cleanAll(); + }); + + test('metadata disabled', async () => { + process.env['STACKS_API_ENABLE_FT_METADATA'] = '0'; + process.env['STACKS_API_ENABLE_NFT_METADATA'] = '0'; + const query1 = await supertest(api.server).get(`/extended/v1/tokens/nft/metadata`); + expect(query1.status).toBe(500); + expect(query1.body.error).toMatch(/not enabled/); + const query2 = await supertest(api.server).get(`/extended/v1/tokens/ft/metadata`); + expect(query2.status).toBe(500); + expect(query2.body.error).toMatch(/not enabled/); + const query3 = await supertest(api.server).get(`/extended/v1/tokens/example/nft/metadata`); + expect(query3.status).toBe(500); + expect(query3.body.error).toMatch(/not enabled/); + const query4 = await supertest(api.server).get(`/extended/v1/tokens/example/ft/metadata`); + expect(query4.status).toBe(500); + expect(query4.body.error).toMatch(/not enabled/); + }); + + test('token nft-metadata data URL plain percent-encoded', async () => { + const contract1 = await deployContract( + 'beeple-a', + pKey, + 'src/tests-tokens/test-contracts/beeple-data-url-a.clar' + ); + await standByForTokens(contract1.contractId); + + const query1 = await supertest(api.server).get( + `/extended/v1/tokens/${contract1.contractId}/nft/metadata` + ); + expect(query1.status).toBe(200); + expect(query1.body).toHaveProperty('token_uri'); + expect(query1.body).toHaveProperty('name'); + expect(query1.body).toHaveProperty('description'); + expect(query1.body).toHaveProperty('image_uri'); + expect(query1.body).toHaveProperty('image_canonical_uri'); + expect(query1.body).toHaveProperty('tx_id'); + expect(query1.body).toHaveProperty('sender_address'); + }); + + test('token nft-metadata data URL base64 w/o media type', async () => { + const contract1 = await deployContract( + 'beeple-b', + pKey, + 'src/tests-tokens/test-contracts/beeple-data-url-b.clar' + ); + + await standByForTokens(contract1.contractId); + + const query1 = await supertest(api.server).get( + `/extended/v1/tokens/${contract1.contractId}/nft/metadata` + ); + expect(query1.status).toBe(200); + expect(query1.body).toHaveProperty('token_uri'); + expect(query1.body).toHaveProperty('name'); + expect(query1.body).toHaveProperty('description'); + expect(query1.body).toHaveProperty('image_uri'); + expect(query1.body).toHaveProperty('image_canonical_uri'); + expect(query1.body).toHaveProperty('tx_id'); + expect(query1.body).toHaveProperty('sender_address'); + }); + + test('token nft-metadata data URL plain non-encoded', async () => { + const contract1 = await deployContract( + 'beeple-c', + pKey, + 'src/tests-tokens/test-contracts/beeple-data-url-c.clar' + ); + + await standByForTokens(contract1.contractId); + + const query1 = await supertest(api.server).get( + `/extended/v1/tokens/${contract1.contractId}/nft/metadata` + ); + expect(query1.status).toBe(200); + expect(query1.body).toHaveProperty('token_uri'); + expect(query1.body).toHaveProperty('name'); + expect(query1.body).toHaveProperty('description'); + expect(query1.body).toHaveProperty('image_uri'); + expect(query1.body).toHaveProperty('image_canonical_uri'); + expect(query1.body).toHaveProperty('tx_id'); + expect(query1.body).toHaveProperty('sender_address'); + }); + + test('token nft-metadata', async () => { + //mock the response + const nftMetadata = { + name: 'EVERYDAYS: THE FIRST 5000 DAYS', + imageUrl: + 'https://ipfsgateway.makersplace.com/ipfs/QmZ15eQX8FPjfrtdX3QYbrhZxJpbLpvDpsgb2p3VEH8Bqq', + description: + 'I made a picture from start to finish every single day from May 1st, 2007 - January 7th, 2021. This is every motherfucking one of those pictures.', + }; + nock('https://ipfs.io') + .get('/ipfs/QmPAg1mjxcEQPPtqsLoEcauVedaeMH81WXDPvPx3VC5zUz') + .reply(200, nftMetadata); + + const contract = await deployContract( + 'nft-trait', + pKey, + 'src/tests-tokens/test-contracts/nft-trait.clar' + ); + const tx = await standByForTx(contract.txId); + if (tx.status != 1) logger.error('contract deploy error', tx); + + const contract1 = await deployContract( + 'beeple', + pKey, + 'src/tests-tokens/test-contracts/beeple.clar' + ); + + await standByForTokens(contract1.contractId); + + const senderAddress = getAddressFromPrivateKey(pKey, stacksNetwork.version); + const query1 = await supertest(api.server).get( + `/extended/v1/tokens/${senderAddress}.beeple/nft/metadata` + ); + expect(query1.status).toBe(200); + expect(query1.body).toHaveProperty('token_uri'); + expect(query1.body.name).toBe(nftMetadata.name); + expect(query1.body.description).toBe(nftMetadata.description); + expect(query1.body.image_uri).toBe(nftMetadata.imageUrl); + expect(query1.body).toHaveProperty('image_canonical_uri'); + expect(query1.body).toHaveProperty('tx_id'); + expect(query1.body).toHaveProperty('sender_address'); + }); + + test('token ft-metadata tests', async () => { + //mock the response + const ftMetadata = { + name: 'Heystack', + description: + 'Heystack is a SIP-010-compliant fungible token on the Stacks Blockchain, used on the Heystack app', + image: 'https://heystack.xyz/assets/Stacks128w.png', + }; + + nock('https://heystack.xyz').get('/token-metadata.json').reply(200, ftMetadata); + + const contract = await deployContract( + 'ft-trait', + pKey, + 'src/tests-tokens/test-contracts/ft-trait.clar' + ); + + const tx = await standByForTx(contract.txId); + if (tx.status != 1) logger.error('contract deploy error', tx); + + const contract1 = await deployContract( + 'hey-token', + pKey, + 'src/tests-tokens/test-contracts/hey-token.clar' + ); + + await standByForTokens(contract1.contractId); + + const query1 = await supertest(api.server).get( + `/extended/v1/tokens/${contract1.contractId}/ft/metadata` + ); + + expect(query1.body).toHaveProperty('token_uri'); + expect(query1.body).toHaveProperty('name'); + expect(query1.body.description).toBe(ftMetadata.description); + expect(query1.body.image_uri).toBe(ftMetadata.image); + expect(query1.body).toHaveProperty('image_canonical_uri'); + expect(query1.body).toHaveProperty('tx_id'); + expect(query1.body).toHaveProperty('sender_address'); + }); + + test('token ft-metadata list', async () => { + for (let i = 0; i < 200; i++) { + const ftMetadata: DbFungibleTokenMetadata = { + token_uri: 'ft-token', + name: 'ft-metadata' + i, + description: 'ft -metadata description', + symbol: 'stx', + decimals: 5, + image_uri: 'ft-metadata image uri example', + image_canonical_uri: 'ft-metadata image canonical uri example', + contract_id: 'ABCDEFGHIJ.ft-metadata', + tx_id: '0x123456', + sender_address: 'ABCDEFGHIJ', + }; + await db.updateFtMetadata(ftMetadata, 0); + } + + const query = await supertest(api.server).get(`/extended/v1/tokens/ft/metadata`); + expect(query.status).toBe(200); + expect(query.body.total).toBeGreaterThan(96); + expect(query.body.limit).toStrictEqual(96); + expect(query.body.offset).toStrictEqual(0); + expect(query.body.results.length).toStrictEqual(96); + + const query1 = await supertest(api.server).get( + `/extended/v1/tokens/ft/metadata?limit=20&offset=10` + ); + expect(query1.status).toBe(200); + expect(query1.body.total).toBeGreaterThanOrEqual(200); + expect(query1.body.limit).toStrictEqual(20); + expect(query1.body.offset).toStrictEqual(10); + expect(query1.body.results.length).toStrictEqual(20); + }); + + test('token nft-metadata list', async () => { + for (let i = 0; i < 200; i++) { + const nftMetadata: DbNonFungibleTokenMetadata = { + token_uri: 'nft-tokenuri', + name: 'nft-metadata' + i, + description: 'nft -metadata description' + i, + image_uri: 'nft-metadata image uri example', + image_canonical_uri: 'nft-metadata image canonical uri example', + contract_id: 'ABCDEFGHIJ.nft-metadata' + i, + tx_id: '0x12345678', + sender_address: 'ABCDEFGHIJ', + }; + + await db.updateNFtMetadata(nftMetadata, 0); + } + + const query = await supertest(api.server).get(`/extended/v1/tokens/nft/metadata`); + expect(query.status).toBe(200); + expect(query.body.total).toBeGreaterThan(96); + expect(query.body.limit).toStrictEqual(96); + expect(query.body.offset).toStrictEqual(0); + expect(query.body.results.length).toStrictEqual(96); + + const query1 = await supertest(api.server).get( + `/extended/v1/tokens/nft/metadata?limit=20&offset=10` + ); + expect(query1.status).toBe(200); + expect(query1.body.total).toBeGreaterThanOrEqual(200); + expect(query1.body.limit).toStrictEqual(20); + expect(query1.body.offset).toStrictEqual(10); + expect(query1.body.results.length).toStrictEqual(20); + }); + + test('large metadata payload test', async () => { + //mock the response + const maxResponseBytes = 10_000; + const randomData = Buffer.alloc(maxResponseBytes + 100, 'x', 'utf8'); + nock('https://example.com').get('/large_payload').reply(200, randomData.toString()); + + await expect(async () => { + await performFetch('https://example.com/large_payload', { + maxResponseBytes: maxResponseBytes, + }); + }).rejects.toThrow(/over limit/); + }); + + test('timeout metadata payload test', async () => { + //mock the response + const responseTimeout = 100; + nock('https://example.com') + .get('/timeout_payload') + .reply(200, async (_uri, _requestBody, cb) => { + await timeout(responseTimeout + 200); + cb(null, '{"hello":"world"}'); + }); + + await expect(async () => { + await performFetch('https://example.com/timeout_payload', { + timeoutMs: responseTimeout, + }); + }).rejects.toThrow(/network timeout/); + }); + + afterAll(async () => { + await new Promise(resolve => eventServer.close(() => resolve(true))); + await api.terminate(); + client.release(); + await db?.close(); + await runMigrations(undefined, 'down'); + }); +}); diff --git a/src/tests/datastore-tests.ts b/src/tests/datastore-tests.ts index ceb2789c9d..f23429412c 100644 --- a/src/tests/datastore-tests.ts +++ b/src/tests/datastore-tests.ts @@ -19,6 +19,8 @@ import { DbBnsName, DbBnsSubdomain, DbTokenOfferingLocked, + DbNonFungibleTokenMetadata, + DbFungibleTokenMetadata, } from '../datastore/common'; import { PgDataStore, @@ -4087,6 +4089,48 @@ describe('postgres datastore', () => { expect(results.found).toBe(false); }); + test('pg token nft-metadata', async () => { + const nftMetadata: DbNonFungibleTokenMetadata = { + token_uri: 'nft-tokenuri', + name: 'nft-metadata', + description: 'nft -metadata description', + image_uri: 'nft-metadata image uri example', + image_canonical_uri: 'nft-metadata image canonical uri example', + contract_id: 'ABCDEFGHIJ.nft-metadata', + tx_id: '0x1234', + sender_address: 'sender-addr-test', + }; + + const rowCount = await db.updateNFtMetadata(nftMetadata, 0); + expect(rowCount).toBe(1); + + const query = await db.getNftMetadata(nftMetadata.contract_id); + expect(query.found).toBe(true); + if (query.found) expect(query.result).toStrictEqual(nftMetadata); + }); + + test('pg token ft-metadata', async () => { + const ftMetadata: DbFungibleTokenMetadata = { + token_uri: 'ft-token', + name: 'ft-metadata', + description: 'ft -metadata description', + symbol: 'stx', + decimals: 5, + image_uri: 'ft-metadata image uri example', + image_canonical_uri: 'ft-metadata image canonical uri example', + contract_id: 'ABCDEFGHIJ.ft-metadata', + tx_id: '0x1234', + sender_address: 'sender-addr-test', + }; + + const rowCount = await db.updateFtMetadata(ftMetadata, 0); + expect(rowCount).toBe(1); + + const query = await db.getFtMetadata(ftMetadata.contract_id); + expect(query.found).toBe(true); + if (query.found) expect(query.result).toStrictEqual(ftMetadata); + }); + afterEach(async () => { client.release(); await db?.close();