From e6e5cce7b098a2ccc8511ea9291f33d270ea54b6 Mon Sep 17 00:00:00 2001
From: diwei sun <diwei.sun@intel.com>
Date: Sun, 21 Jul 2024 20:46:59 -0700
Subject: [PATCH 01/11] [CI] enable operator benchmark on CPU

---
 .ci/pytorch/test.sh                      | 12 +++++++
 .github/workflows/operator_benchmark.yml | 42 ++++++++++++++++++++++++
 2 files changed, 54 insertions(+)
 create mode 100644 .github/workflows/operator_benchmark.yml

diff --git a/.ci/pytorch/test.sh b/.ci/pytorch/test.sh
index e443b0713177a..f10826611baaf 100755
--- a/.ci/pytorch/test.sh
+++ b/.ci/pytorch/test.sh
@@ -1368,6 +1368,16 @@ test_linux_aarch64(){
        inductor/test_torchinductor_codegen_dynamic_shapes inductor/test_torchinductor_dynamic_shapes --verbose
 }
 
+test_operator_benchmark() {
+  TEST_REPORTS_DIR=$(pwd)/test/test-reports
+  TEST_DIR=$(pwd)
+  cd benchmarks/operator_benchmark/pt_extension
+  python setup.py install
+  cd "${TEST_DIR}"
+  taskset -c 0-"$end_core" python -m benchmarks/operator_benchmark/benchmark_all_test.py --output "${TEST_REPORTS_DIR}/operator_benchmark.csv"
+}
+
+
 if ! [[ "${BUILD_ENVIRONMENT}" == *libtorch* || "${BUILD_ENVIRONMENT}" == *-bazel-* ]]; then
   (cd test && python -c "import torch; print(torch.__config__.show())")
   (cd test && python -c "import torch; print(torch.__config__.parallel_info())")
@@ -1394,6 +1404,8 @@ elif [[ "$TEST_CONFIG" == distributed ]]; then
   if [[ "${SHARD_NUMBER}" == 1 ]]; then
     test_rpc
   fi
+elif [[ "${TEST_CONFIG}" == *cpu_operator_benchmark* ]]; then
+  test_operator_benchmark
 elif [[ "${TEST_CONFIG}" == *inductor_distributed* ]]; then
   test_inductor_distributed
 elif [[ "${TEST_CONFIG}" == *inductor-halide* ]]; then
diff --git a/.github/workflows/operator_benchmark.yml b/.github/workflows/operator_benchmark.yml
new file mode 100644
index 0000000000000..c308d8c7ddfc1
--- /dev/null
+++ b/.github/workflows/operator_benchmark.yml
@@ -0,0 +1,42 @@
+name: operator_benchmark
+
+on:
+  pull_request:
+    branches-ignore:
+      - nightly
+  push:
+    branches:
+      - main
+      - release/*
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
+  cancel-in-progress: true
+
+permissions: read-all
+
+jobs:
+  linux-jammy-cpu-py3_8-gcc11-opbenchmark-build:
+    name: linux-jammy-cpu-py3.8-gcc11-opbenchmark
+    uses: ./.github/workflows/_linux-build.yml
+    with:
+      build-environment: linux-jammy-py3.8-gcc11-build
+      docker-image-name: pytorch-linux-jammy-py3.8-gcc11-opbenchmark-build
+      test-matrix: |
+        { include: [
+          { config: "cpu_operator_benchmark", shard: 1, num_shards: 1, runner: "linux.12xlarge" },
+        ]}
+    secrets:
+      HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
+
+  linux-jammy-cpu-py3_8-gcc11-opbenchmark-test:
+    name: linux-jammy-cpu-py3.8-gcc11-opbenchmark
+    uses: ./.github/workflows/_linux-test.yml
+    needs: linux-jammy-cpu-py3_8-gcc11-opbenchmark-build
+    with:
+      build-environment: linux-jammy-py3.8-gcc11-build
+      docker-image: ${{ needs.linux-jammy-cpu-py3_8-gcc11-opbenchmark-build.outputs.docker-image }}
+      test-matrix: ${{ needs.linux-jammy-cpu-py3_8-gcc11-opbenchmark-build.outputs.test-matrix }}
+    secrets:
+      HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}

From 5bbfbd4f0ca340f2f95420d95688247d55febcaf Mon Sep 17 00:00:00 2001
From: diwei sun <diwei.sun@intel.com>
Date: Mon, 22 Jul 2024 00:20:39 -0700
Subject: [PATCH 02/11] bugfix for operator benchmark

---
 .ci/pytorch/test.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.ci/pytorch/test.sh b/.ci/pytorch/test.sh
index f10826611baaf..b958af6a7bab3 100755
--- a/.ci/pytorch/test.sh
+++ b/.ci/pytorch/test.sh
@@ -1371,6 +1371,7 @@ test_linux_aarch64(){
 test_operator_benchmark() {
   TEST_REPORTS_DIR=$(pwd)/test/test-reports
   TEST_DIR=$(pwd)
+  end_core=$(( CORES-1 ))
   cd benchmarks/operator_benchmark/pt_extension
   python setup.py install
   cd "${TEST_DIR}"

From 67fb05722de61f253e196a46ea613b9001615e3f Mon Sep 17 00:00:00 2001
From: diwei sun <diwei.sun@intel.com>
Date: Wed, 24 Jul 2024 04:29:54 -0700
Subject: [PATCH 03/11] add numcatl in operator benchmark

---
 .ci/pytorch/test.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.ci/pytorch/test.sh b/.ci/pytorch/test.sh
index b958af6a7bab3..6403b75601a07 100755
--- a/.ci/pytorch/test.sh
+++ b/.ci/pytorch/test.sh
@@ -1371,6 +1371,7 @@ test_linux_aarch64(){
 test_operator_benchmark() {
   TEST_REPORTS_DIR=$(pwd)/test/test-reports
   TEST_DIR=$(pwd)
+  CORES=$(lscpu | grep Core | awk '{print $4}')
   end_core=$(( CORES-1 ))
   cd benchmarks/operator_benchmark/pt_extension
   python setup.py install

From 953da241a617e0320b3fc209b02ec957c7688f5a Mon Sep 17 00:00:00 2001
From: diwei sun <diwei.sun@intel.com>
Date: Wed, 24 Jul 2024 23:01:41 -0700
Subject: [PATCH 04/11] print out operator benchmark results

---
 .ci/pytorch/test.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.ci/pytorch/test.sh b/.ci/pytorch/test.sh
index 6403b75601a07..442f772ea5dfd 100755
--- a/.ci/pytorch/test.sh
+++ b/.ci/pytorch/test.sh
@@ -1375,8 +1375,9 @@ test_operator_benchmark() {
   end_core=$(( CORES-1 ))
   cd benchmarks/operator_benchmark/pt_extension
   python setup.py install
-  cd "${TEST_DIR}"
-  taskset -c 0-"$end_core" python -m benchmarks/operator_benchmark/benchmark_all_test.py --output "${TEST_REPORTS_DIR}/operator_benchmark.csv"
+  cd "${TEST_DIR}"/benchmarks/operator_benchmark/pt_extension
+  taskset -c 0-"$end_core" python -m benchmark_all_test --output "${TEST_REPORTS_DIR}/operator_benchmark.csv"
+  cat ${TEST_REPORTS_DIR}/operator_benchmark.csv
 }
 
 

From ff35dbc82b6c7db170da24783cade10a50f4e48d Mon Sep 17 00:00:00 2001
From: diwei sun <diwei.sun@intel.com>
Date: Thu, 25 Jul 2024 01:42:30 -0700
Subject: [PATCH 05/11] operator direction update

---
 .ci/pytorch/test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/pytorch/test.sh b/.ci/pytorch/test.sh
index 442f772ea5dfd..a0c819b02e9ff 100755
--- a/.ci/pytorch/test.sh
+++ b/.ci/pytorch/test.sh
@@ -1375,7 +1375,7 @@ test_operator_benchmark() {
   end_core=$(( CORES-1 ))
   cd benchmarks/operator_benchmark/pt_extension
   python setup.py install
-  cd "${TEST_DIR}"/benchmarks/operator_benchmark/pt_extension
+  cd "${TEST_DIR}"/benchmarks/operator_benchmark
   taskset -c 0-"$end_core" python -m benchmark_all_test --output "${TEST_REPORTS_DIR}/operator_benchmark.csv"
   cat ${TEST_REPORTS_DIR}/operator_benchmark.csv
 }

From 20fc41c3ceaf08d8df1742926a2d0936e0bdffe3 Mon Sep 17 00:00:00 2001
From: diwei sun <diwei.sun@intel.com>
Date: Thu, 1 Aug 2024 05:24:53 -0700
Subject: [PATCH 06/11] enable operator benchmark outputs

---
 .ci/pytorch/test.sh                           |  2 +-
 .../operator_benchmark/benchmark_core.py      | 50 +++++++++++++++++--
 .../operator_benchmark/benchmark_runner.py    |  6 +++
 3 files changed, 53 insertions(+), 5 deletions(-)

diff --git a/.ci/pytorch/test.sh b/.ci/pytorch/test.sh
index a0c819b02e9ff..092df2c96d120 100755
--- a/.ci/pytorch/test.sh
+++ b/.ci/pytorch/test.sh
@@ -1376,7 +1376,7 @@ test_operator_benchmark() {
   cd benchmarks/operator_benchmark/pt_extension
   python setup.py install
   cd "${TEST_DIR}"/benchmarks/operator_benchmark
-  taskset -c 0-"$end_core" python -m benchmark_all_test --output "${TEST_REPORTS_DIR}/operator_benchmark.csv"
+  taskset -c 0-"$end_core" python -m benchmark_all_test --output-dir "${TEST_REPORTS_DIR} --device cpu"
   cat ${TEST_REPORTS_DIR}/operator_benchmark.csv
 }
 
diff --git a/benchmarks/operator_benchmark/benchmark_core.py b/benchmarks/operator_benchmark/benchmark_core.py
index bc340ae17f67d..0d5dd9475c9f6 100644
--- a/benchmarks/operator_benchmark/benchmark_core.py
+++ b/benchmarks/operator_benchmark/benchmark_core.py
@@ -1,8 +1,10 @@
 import ast
 import copy
+import csv
 import functools
 import json
 import timeit
+import os
 from collections import namedtuple
 
 import benchmark_utils
@@ -30,7 +32,7 @@
 
 
 BENCHMARK_TESTER = []
-
+disable_output = False
 
 def _register_test(*test_metainfo):
     """save the metainfo needed to create a test. Currently test_metainfo
@@ -188,7 +190,9 @@ def __init__(self, args):
         self.use_jit = args.use_jit
         self.num_runs = args.num_runs
         self.print_per_iter = False
+        self.output_dir = args.output_dir
         self.operator_range = benchmark_utils.get_operator_range(args.operator_range)
+        self.disable_output = args.disable_output
         # 100 is the default warmup iterations
         if self.args.warmup_iterations == -1:
             self.args.warmup_iterations = 100
@@ -395,10 +399,40 @@ def _print_test_case_info(self, test_case):
             return True
 
         return False
-
+    
+    def _output_csv(self, filename, headers, row):
+        global disable_output
+        if disable_output:
+            return
+        if os.path.exists(filename):
+            with open(filename) as fd:
+                lines = list(csv.reader(fd)) or [[]]
+                if headers and len(headers) > len(lines[0]):
+                    # if prior results failed the header might not be filled in yet
+                    lines[0] = headers
+                else:
+                    headers = lines[0]
+        else:
+            lines = [headers]
+        lines.append([(f"{x:.6f}" if isinstance(x, float) else x) for x in row])
+        with open(filename, "w") as fd:
+            writer = csv.writer(fd, lineterminator="\n")
+            for line in lines:
+                writer.writerow(list(line) + ["0"] * (len(headers) - len(line)))
+    
+    def _generate_csv_name(self, dtype, suite, device, compiler):
+        return f"operator_benchmark_{suite}_{dtype}_{compiler}_{device}.csv"          
+    
     def run(self):
         self._print_header()
-
+        DEFAULT_OUTPUT_DIR = "benchmark_logs"
+        mode = "JIT" if self.use_jit else "Eager"
+        output_filename = f"{self.args.output_dir}/{self._generate_csv_name('default', 'all', self.args.device, mode)}"
+        headers = ["Benchmarking Framework", "Benchamrking Module Name", "Case Name", "tag", "run_backward", "Execution Time"]
+        
+        if self.args.disable_output:
+            disable_output = True
+        
         for test_metainfo in BENCHMARK_TESTER:
             for test in _build_test(*test_metainfo):
                 full_test_id, test_case = test
@@ -437,5 +471,13 @@ def run(self):
                     )
                     for _ in range(self.num_runs)
                 ]
-
                 self._print_perf_result(reported_time, test_case)
+
+                #output results to csv
+                self._output_csv(
+                    output_filename,
+                    headers,
+                    [test_case.framework, test_case.op_bench.module_name(), test_case.test_config.test_name, test_case.test_config.tag, test_case.test_config.run_backward, reported_time[0]]
+                )
+
+
diff --git a/benchmarks/operator_benchmark/benchmark_runner.py b/benchmarks/operator_benchmark/benchmark_runner.py
index 0695e4847e76a..d9e409c74fd1a 100644
--- a/benchmarks/operator_benchmark/benchmark_runner.py
+++ b/benchmarks/operator_benchmark/benchmark_runner.py
@@ -143,6 +143,12 @@ def parse_args():
         default="None",
     )
 
+    parser.add_argument(
+        "--output-dir",
+        help="Choose the output directory to save the logs",
+        default="DEFAULT_OUTPUT_DIR",
+    )
+
     args, _ = parser.parse_known_args()
 
     if args.omp_num_threads:

From b848622ce002b9877d103ceb7a322ed2ab0bdacb Mon Sep 17 00:00:00 2001
From: diwei sun <diwei.sun@intel.com>
Date: Thu, 1 Aug 2024 05:31:23 -0700
Subject: [PATCH 07/11] refine: lint format check

---
 .ci/pytorch/test.sh                           |  3 +-
 .../operator_benchmark/benchmark_core.py      | 38 ++++++++++++-------
 2 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/.ci/pytorch/test.sh b/.ci/pytorch/test.sh
index 092df2c96d120..584ffe224557e 100755
--- a/.ci/pytorch/test.sh
+++ b/.ci/pytorch/test.sh
@@ -1376,8 +1376,7 @@ test_operator_benchmark() {
   cd benchmarks/operator_benchmark/pt_extension
   python setup.py install
   cd "${TEST_DIR}"/benchmarks/operator_benchmark
-  taskset -c 0-"$end_core" python -m benchmark_all_test --output-dir "${TEST_REPORTS_DIR} --device cpu"
-  cat ${TEST_REPORTS_DIR}/operator_benchmark.csv
+  taskset -c 0-"$end_core" python -m benchmark_all_test --device cpu --output-dir "${TEST_REPORTS_DIR}"
 }
 
 
diff --git a/benchmarks/operator_benchmark/benchmark_core.py b/benchmarks/operator_benchmark/benchmark_core.py
index 0d5dd9475c9f6..2da686d8de1ac 100644
--- a/benchmarks/operator_benchmark/benchmark_core.py
+++ b/benchmarks/operator_benchmark/benchmark_core.py
@@ -3,8 +3,8 @@
 import csv
 import functools
 import json
-import timeit
 import os
+import timeit
 from collections import namedtuple
 
 import benchmark_utils
@@ -32,7 +32,7 @@
 
 
 BENCHMARK_TESTER = []
-disable_output = False
+
 
 def _register_test(*test_metainfo):
     """save the metainfo needed to create a test. Currently test_metainfo
@@ -399,7 +399,7 @@ def _print_test_case_info(self, test_case):
             return True
 
         return False
-    
+
     def _output_csv(self, filename, headers, row):
         global disable_output
         if disable_output:
@@ -419,20 +419,27 @@ def _output_csv(self, filename, headers, row):
             writer = csv.writer(fd, lineterminator="\n")
             for line in lines:
                 writer.writerow(list(line) + ["0"] * (len(headers) - len(line)))
-    
+
     def _generate_csv_name(self, dtype, suite, device, compiler):
-        return f"operator_benchmark_{suite}_{dtype}_{compiler}_{device}.csv"          
-    
+        return f"operator_benchmark_{suite}_{dtype}_{compiler}_{device}.csv"
+
     def run(self):
         self._print_header()
         DEFAULT_OUTPUT_DIR = "benchmark_logs"
         mode = "JIT" if self.use_jit else "Eager"
         output_filename = f"{self.args.output_dir}/{self._generate_csv_name('default', 'all', self.args.device, mode)}"
-        headers = ["Benchmarking Framework", "Benchamrking Module Name", "Case Name", "tag", "run_backward", "Execution Time"]
-        
+        headers = [
+            "Benchmarking Framework",
+            "Benchamrking Module Name",
+            "Case Name",
+            "tag",
+            "run_backward",
+            "Execution Time",
+        ]
+
         if self.args.disable_output:
             disable_output = True
-        
+
         for test_metainfo in BENCHMARK_TESTER:
             for test in _build_test(*test_metainfo):
                 full_test_id, test_case = test
@@ -473,11 +480,16 @@ def run(self):
                 ]
                 self._print_perf_result(reported_time, test_case)
 
-                #output results to csv
+                # output results to csv
                 self._output_csv(
                     output_filename,
                     headers,
-                    [test_case.framework, test_case.op_bench.module_name(), test_case.test_config.test_name, test_case.test_config.tag, test_case.test_config.run_backward, reported_time[0]]
+                    [
+                        test_case.framework,
+                        test_case.op_bench.module_name(),
+                        test_case.test_config.test_name,
+                        test_case.test_config.tag,
+                        test_case.test_config.run_backward,
+                        reported_time[0],
+                    ],
                 )
-
-

From 572434132f638fd5bbcb9a4902e63a138f38c64d Mon Sep 17 00:00:00 2001
From: diwei sun <diwei.sun@intel.com>
Date: Thu, 1 Aug 2024 22:29:02 -0700
Subject: [PATCH 08/11] diable output args

---
 benchmarks/operator_benchmark/benchmark_runner.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/benchmarks/operator_benchmark/benchmark_runner.py b/benchmarks/operator_benchmark/benchmark_runner.py
index d9e409c74fd1a..f34a4e39e4dbf 100644
--- a/benchmarks/operator_benchmark/benchmark_runner.py
+++ b/benchmarks/operator_benchmark/benchmark_runner.py
@@ -148,6 +148,11 @@ def parse_args():
         help="Choose the output directory to save the logs",
         default="DEFAULT_OUTPUT_DIR",
     )
+    parser.add_argument(
+        "--disable-output",
+        help="Disable log output to csv file",
+        default="False",
+    )
 
     args, _ = parser.parse_known_args()
 

From 4c862ac42aeb64f2ede9a6219cb4be34c91ed78c Mon Sep 17 00:00:00 2001
From: diwei sun <diwei.sun@intel.com>
Date: Fri, 2 Aug 2024 03:32:45 -0700
Subject: [PATCH 09/11] debug for output

---
 benchmarks/operator_benchmark/benchmark_core.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/benchmarks/operator_benchmark/benchmark_core.py b/benchmarks/operator_benchmark/benchmark_core.py
index 2da686d8de1ac..8e581da37b68f 100644
--- a/benchmarks/operator_benchmark/benchmark_core.py
+++ b/benchmarks/operator_benchmark/benchmark_core.py
@@ -401,8 +401,7 @@ def _print_test_case_info(self, test_case):
         return False
 
     def _output_csv(self, filename, headers, row):
-        global disable_output
-        if disable_output:
+        if self.args.disable_output:
             return
         if os.path.exists(filename):
             with open(filename) as fd:

From e04e43cd91fd8b8358316f9fb6005c1dee83be44 Mon Sep 17 00:00:00 2001
From: diwei sun <diwei.sun@intel.com>
Date: Sat, 3 Aug 2024 10:20:29 -0700
Subject: [PATCH 10/11] refine output path

---
 benchmarks/operator_benchmark/benchmark_core.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/benchmarks/operator_benchmark/benchmark_core.py b/benchmarks/operator_benchmark/benchmark_core.py
index 8e581da37b68f..5f12f35ad43aa 100644
--- a/benchmarks/operator_benchmark/benchmark_core.py
+++ b/benchmarks/operator_benchmark/benchmark_core.py
@@ -401,7 +401,7 @@ def _print_test_case_info(self, test_case):
         return False
 
     def _output_csv(self, filename, headers, row):
-        if self.args.disable_output:
+        if self.args.disable_output is False:
             return
         if os.path.exists(filename):
             with open(filename) as fd:

From 0c2dea8b435071676bc260bb8a307d875916bf67 Mon Sep 17 00:00:00 2001
From: diwei sun <diwei.sun@intel.com>
Date: Sat, 3 Aug 2024 11:15:14 -0700
Subject: [PATCH 11/11] enable perf regression/imporvement check for operator
 benchmark

---
 .ci/pytorch/test.sh                             | 9 ++++++++-
 benchmarks/operator_benchmark/benchmark_core.py | 5 +----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/.ci/pytorch/test.sh b/.ci/pytorch/test.sh
index 584ffe224557e..27d28e0e0a378 100755
--- a/.ci/pytorch/test.sh
+++ b/.ci/pytorch/test.sh
@@ -1373,10 +1373,17 @@ test_operator_benchmark() {
   TEST_DIR=$(pwd)
   CORES=$(lscpu | grep Core | awk '{print $4}')
   end_core=$(( CORES-1 ))
+
   cd benchmarks/operator_benchmark/pt_extension
   python setup.py install
+
   cd "${TEST_DIR}"/benchmarks/operator_benchmark
-  taskset -c 0-"$end_core" python -m benchmark_all_test --device cpu --output-dir "${TEST_REPORTS_DIR}"
+  taskset -c 0-"$end_core" python -m benchmark_all_test --device cpu --output-dir "${TEST_REPORTS_DIR}/operator_benchmark_eager_float32_cpu.csv"
+
+  python benchmarks/operator_benchmark/check_perf_csv.py \
+  --actual "${TEST_REPORTS_DIR}/operator_benchmark_eager_float32_cpu.csv" \
+  --expected "benchmarks/operator_benchmark_eager_float32_cpu_expected.csv"
+
 }
 
 
diff --git a/benchmarks/operator_benchmark/benchmark_core.py b/benchmarks/operator_benchmark/benchmark_core.py
index 5f12f35ad43aa..9e7b223e1b77c 100644
--- a/benchmarks/operator_benchmark/benchmark_core.py
+++ b/benchmarks/operator_benchmark/benchmark_core.py
@@ -419,14 +419,11 @@ def _output_csv(self, filename, headers, row):
             for line in lines:
                 writer.writerow(list(line) + ["0"] * (len(headers) - len(line)))
 
-    def _generate_csv_name(self, dtype, suite, device, compiler):
-        return f"operator_benchmark_{suite}_{dtype}_{compiler}_{device}.csv"
-
     def run(self):
         self._print_header()
         DEFAULT_OUTPUT_DIR = "benchmark_logs"
         mode = "JIT" if self.use_jit else "Eager"
-        output_filename = f"{self.args.output_dir}/{self._generate_csv_name('default', 'all', self.args.device, mode)}"
+        output_filename = self.args.output_dir
         headers = [
             "Benchmarking Framework",
             "Benchamrking Module Name",