benchmarks (#298)

* init: benchmarks

* ci: remove cargo and cache

* ci: add setup project

* ci: add back cache

* ci: change to pnpm

* ci: *

* ci: change hash files

* ci: loud install debug

* ci: empty command

* ci: remove install for npm as pnpm manages this

* package: *

* fix: add peer graphql

* *

* *

Author: danstarns

.github/workflows/benchmarks.yml

@@ -0,0 +1,50 @@
+name: Run benchmarks
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - README.md
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-project
+      - uses: fregante/setup-git-user@v2
+      - uses: cachix/install-nix-action@v24
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.ivy2/cache
+            ~/.sbt
+          key: ${{ runner.os }}-sbt-${{ hashFiles('**/build.sbt') }}
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            async-graphql/target
+            juniper/target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+      - uses: actions/cache@v3
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package.json') }}
+      - name: Run benchmarks
+        working-directory: ./benchmarks
+        run: nix-shell --quiet --run ./run.cr
+      - name: Push readme
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          git add README.md
+          git commit -m "update readme"
+          git push

.turbo/daemon/b418ceec9a896940-turbo.log.2024-04-29

@@ -0,0 +1,27 @@
+<!-- README.md is generated from README.ecr, do not edit -->
+
+# GraphQL Debugger Benchmarks
+
+https://www.graphql-debugger.com
+
+Graphql server benchmarks in many frameworks.
+
+All servers implement a simple schema:
+
+```graphql
+type Query {
+  hello: String!
+}
+```
+
+The returned string is always `world`.
+
+The API is served over HTTP using a common web server and load tested using [bombardier](https://github.com/codesenberg/bombardier).
+
+### Results
+
+| Name                          | Language      | Server          | Latency avg      | Requests      |
+| ----------------------------  | ------------- | --------------- | ---------------- | ------------- |
+<%- benchmarks.each do |b| -%>
+| [<%= b.name %>](<%= b.url %>) | <%= b.lang %> | <%= b.server %> | <%= b.latency %> | <%= b.reqs %> |
+<%- end -%>

benchmarks/README.ecr

@@ -0,0 +1,31 @@
+<!-- README.md is generated from README.ecr, do not edit -->
+
+# GraphQL Debugger Benchmarks
+
+https://www.graphql-debugger.com
+
+Graphql server benchmarks in many frameworks.
+
+All servers implement a simple schema:
+
+```graphql
+type Query {
+  hello: String!
+}
+```
+
+The returned string is always `world`.
+
+The API is served over HTTP using a common web server and load tested using [bombardier](https://github.com/codesenberg/bombardier).
+
+### Results
+
+| Name                                                                 | Language | Server  | Latency avg | Requests |
+| -------------------------------------------------------------------- | -------- | ------- | ----------- | -------- |
+| [yoga](https://github.com/dotansimha/graphql-yoga)                   | Node.js  | http    | 15.25ms     | 13kps    |
+| [apollo](https://github.com/apollographql/apollo-server)             | Node.js  | Express | 33.20ms     | 6.0kps   |
+| [yoga-otel](https://github.com/open-telemetry/opentelemetry-js/)     | Node.js  | http    | 34.11ms     | 5.8kps   |
+| [helix](https://github.com/contra/graphql-helix)                     | Node.js  | http    | 46.29ms     | 4.3kps   |
+| [yoga-debugger](https://graphql-debugger.com/docs/plugins/yoga)      | Node.js  | http    | 89.22ms     | 2.2kps   |
+| [apollo-debugger](https://graphql-debugger.com/docs/plugins/apollo)  | Node.js  | Express | 100.57ms    | 2.0kps   |
+| [helix-debugger](https://github.com/rocket-connect/graphql-debugger) | Node.js  | http    | 106.46ms    | 1.9kps   |

benchmarks/README.md

@@ -0,0 +1,29 @@
+import { graphqlDebuggerPlugin } from "@graphql-debugger/apollo-server";
+
+import { ApolloServer } from "apollo-server";
+import cluster from "cluster";
+import { cpus } from "os";
+
+process.env.PORT = "8000";
+
+if (cluster.isPrimary) {
+  for (let i = 0; i < cpus().length; i++) {
+    cluster.fork();
+  }
+} else {
+  const server = new ApolloServer({
+    typeDefs: `
+        type Query {
+          hello: String!
+        }
+      `,
+    resolvers: {
+      Query: {
+        hello: () => "world",
+      },
+    },
+    plugins: [graphqlDebuggerPlugin()],
+  });
+
+  server.listen(8000);
+}

benchmarks/apollo-debugger/index.js

@@ -0,0 +1,10 @@
+{
+  "peerDependencies": {
+    "graphql": "^16.0.0 || ^17.0.0"
+  },
+  "dependencies": {
+    "@graphql-debugger/apollo-server": "workspace:^",
+    "apollo-server": "3.13.0"
+  },
+  "type": "module"
+}

benchmarks/apollo-debugger/package.json

@@ -0,0 +1,26 @@
+import { ApolloServer } from "apollo-server";
+import cluster from "cluster";
+import { cpus } from "os";
+
+process.env.PORT = "8000";
+
+if (cluster.isPrimary) {
+  for (let i = 0; i < cpus().length; i++) {
+    cluster.fork();
+  }
+} else {
+  const server = new ApolloServer({
+    typeDefs: `
+        type Query {
+          hello: String!
+        }
+      `,
+    resolvers: {
+      Query: {
+        hello: () => "world",
+      },
+    },
+  });
+
+  server.listen(8000);
+}

benchmarks/apollo/index.js

@@ -0,0 +1,9 @@
+{
+  "peerDependencies": {
+    "graphql": "^16.0.0 || ^17.0.0"
+  },
+  "dependencies": {
+    "apollo-server": "3.13.0"
+  },
+  "type": "module"
+}

benchmarks/apollo/package.json

@@ -0,0 +1,56 @@
+- id: helix
+  name: helix
+  url: https://github.com/contra/graphql-helix
+  lang: Node.js
+  server: http
+  run:
+    cmd: node
+    args: ["index.js"]
+- id: helix-debugger
+  name: helix-debugger
+  url: https://github.com/rocket-connect/graphql-debugger
+  lang: Node.js
+  server: http
+  run:
+    cmd: node
+    args: ["index.js"]
+- id: yoga
+  name: yoga
+  url: https://github.com/dotansimha/graphql-yoga
+  lang: Node.js
+  server: http
+  run:
+    cmd: node
+    args: ["--no-warnings", "index.js"]
+- id: yoga-debugger
+  name: yoga-debugger
+  url: https://graphql-debugger.com/docs/plugins/yoga
+  lang: Node.js
+  server: http
+  run:
+    cmd: node
+    args: ["--no-warnings", "index.js"]
+- id: yoga-otel
+  name: yoga-otel
+  url: https://github.com/open-telemetry/opentelemetry-js/
+  lang: Node.js
+  server: http
+  run:
+    cmd: node
+    args: ["--no-warnings", "index.js"]
+- id: apollo
+  name: apollo
+  url: https://github.com/apollographql/apollo-server
+  lang: Node.js
+  server: Express
+  run:
+    cmd: node
+    args: ["--no-warnings", "index.js"]
+- id: apollo-debugger
+  name: apollo-debugger
+  url: https://graphql-debugger.com/docs/plugins/apollo
+  lang: Node.js
+  server: Express
+  run:
+    cmd: node
+    args: ["--no-warnings", "index.js"]

benchmarks/benchmarks.yaml

@@ -0,0 +1,72 @@
+import { ProxyAdapter } from "@graphql-debugger/adapter-proxy";
+import {
+  GraphQLDebuggerContext,
+  traceSchema,
+} from "@graphql-debugger/trace-schema";
+
+import cluster from "cluster";
+import express from "express";
+import { GraphQLObjectType, GraphQLSchema, GraphQLString } from "graphql";
+import {
+  getGraphQLParameters,
+  processRequest,
+  sendResult,
+} from "graphql-helix";
+import { cpus } from "os";
+
+if (cluster.isPrimary) {
+  for (let i = 0; i < cpus().length; i++) {
+    cluster.fork();
+  }
+} else {
+  var schema = new GraphQLSchema({
+    query: new GraphQLObjectType({
+      name: "Query",
+      fields: {
+        hello: {
+          type: GraphQLString,
+          resolve() {
+            return "world";
+          },
+        },
+      },
+    }),
+  });
+
+  var tracedSchema = traceSchema({
+    schema,
+    adapter: new ProxyAdapter(),
+    shouldExportSchema: false,
+  });
+
+  var app = express();
+  app.use(express.json());
+
+  app.use("/graphql", async (req, res) => {
+    const request = {
+      body: req.body,
+      headers: req.headers,
+      method: req.method,
+      query: req.query,
+    };
+
+    const { operationName, query, variables } = getGraphQLParameters(request);
+
+    const result = await processRequest({
+      operationName,
+      query,
+      variables,
+      request,
+      schema: tracedSchema,
+      contextFactory: () => ({
+        GraphQLDebuggerContext: new GraphQLDebuggerContext({
+          schema,
+        }),
+      }),
+    });
+
+    sendResult(result, res);
+  });
+
+  app.listen(8000);
+}