Merge pull request #12 from intel/tmp

WSF release 23.1

Author: xwu2intel

CMakeLists.txt

@@ -2,25 +2,39 @@
 cmake_minimum_required (VERSION 2.8...3.24)
 
 Project(workloads NONE)
+STRING(ASCII 27 esc)
+set(red "${esc}[31m")
+set(green "${esc}[32m")
+set(strikethrough "${esc}[9m")
+set(reset "${esc}[0m")
+
 set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/script/")
 include(prerequisite)
 
 add_custom_target(bom)
-add_custom_target(kpi)
 enable_testing()
 
+set(PLATFORM_CONV_FROM "")
+include(platform OPTIONAL)
+
+if(EXISTS "${CMAKE_SOURCE_DIR}/workload/platforms")
+  set(platforms "${CMAKE_SOURCE_DIR}/workload/platforms")
+else()
+  set(platforms "${CMAKE_SOURCE_DIR}/platforms")
+endif()
+
 if(NOT DEFINED PLATFORM)
-    execute_process(COMMAND bash -c "head -n 1 '${CMAKE_SOURCE_DIR}/workload/platforms'" OUTPUT_VARIABLE PLATFORM OUTPUT_STRIP_TRAILING_WHITESPACE)
+    execute_process(COMMAND bash -c "head -n 1 '${platforms}'" OUTPUT_VARIABLE PLATFORM OUTPUT_STRIP_TRAILING_WHITESPACE)
 else()
-    execute_process(COMMAND bash -c "grep -E '^${PLATFORM}$' '${CMAKE_SOURCE_DIR}/workload/platforms' | head -n 1" OUTPUT_VARIABLE platform OUTPUT_STRIP_TRAILING_WHITESPACE)
+    execute_process(COMMAND bash -c "grep -E '^${PLATFORM}$' '${platforms}' | head -n 1" OUTPUT_VARIABLE platform OUTPUT_STRIP_TRAILING_WHITESPACE)
     if(NOT platform)
         message(FATAL_ERROR "Platform ${PLATFORM} not recognized!")
     endif()
     set(PLATFORM "${platform}")
 endif()
 
 include(registry OPTIONAL)
-execute_process(COMMAND bash -c "GIT_ASKPASS=echo git fetch --tags > /dev/null && git tag" TIMEOUT 5 OUTPUT_VARIABLE tmp RESULT_VARIABLE status OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
+execute_process(COMMAND bash -c "GIT_SSH_COMMAND='ssh -o BatchMode=yes' GIT_ASKPASS=echo git fetch --tags > /dev/null && GIT_ASKPASS=echo git tag" TIMEOUT 5 OUTPUT_VARIABLE tmp RESULT_VARIABLE status OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
 if (RELEASE MATCHES "^:?v?[0-9]+[.][0-9]+$")
     string(REPLACE ":" "" release1 "${RELEASE}")
     if ((status EQUAL 0) AND (tmp MATCHES "${release1}"))
@@ -36,7 +50,8 @@ else ()
         message("")
         message("*************************************************************")
         message("")
-        message("Release ${tmp} detected. Switch to release ${tmp} as follows:")
+        message("${green}INFO:${reset} Detected release ${tmp}.")
+        message("  Switch to release ${tmp} as follows:")
         message("")
         message("git checkout ${tmp}")
         message("cmake -DRELEASE=v${tmp} ..")
@@ -47,8 +62,8 @@ else ()
         message("")
         message("****************************************************************************")
         message("")
-        message("Failed to detect official releases.")
-        message("Switch to any release as follows:")
+        message("${red}WARNING:${reset} Failed to detect any official release.")
+        message("  Switch to any release as follows:")
         message("")
         message("git checkout <VERSION>")
         message("cmake -DREGISTRY=${DEFAULT_REGISTRY} -DRELEASE=v<VERSION> ..")
@@ -61,11 +76,11 @@ if (NOT ${REGISTRY} MATCHES "/$")
     set(REGISTRY "${REGISTRY}/")
 endif()
 
-set(ENABLE_BUILD ON)
+set(BUILDSH_OPTIONS "")
 if (REGISTRY STREQUAL "${DEFAULT_REGISTRY}/")
     execute_process(COMMAND bash -c "grep -q -F $(echo ${REGISTRY} | cut -f1 -d/) ~/.docker/config.json" RESULT_VARIABLE disable_build OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
     if(disable_build)
-        set(ENABLE_BUILD OFF)
+        set(BUILDSH_OPTIONS "--read-only-registry")
     endif()
 endif()
 
@@ -80,10 +95,23 @@ elseif (NOT ${RELEASE} MATCHES "^:")
 endif()
 string(TOLOWER "${RELEASE}" RELEASE)
 
+if ((NOT DEFINED BENCHMARK) AND (EXISTS "${CMAKE_SOURCE_DIR}/workload/dummy"))
+    set(BENCHMARK "dummy")
+    message("${red}WARNING:${reset} Default to the dummy workload for quick evaluation.")
+    message("  Enable specific workload with cmake -DBENCHMARK=<name> .. or")
+    message("  Enable all workloads with cmake -DBENCHMARK= ..")
+    message("")
+endif()
+
 if (NOT DEFINED BACKEND)
-    set(BACKEND "terraform")
+    if(EXISTS "${PROJECT_SOURCE_DIR}/script/terraform.cmake")
+        set(BACKEND "terraform")
+    endif()
+endif()
+
+if(BACKEND)
+    include(${BACKEND} OPTIONAL)
 endif()
-include(${BACKEND} OPTIONAL)
 set(IMAGEARCH "linux/amd64")
 include(multi-arch OPTIONAL)
 
@@ -94,28 +122,35 @@ foreach(subdir image stack workload dist)
 endforeach()
 
 message("")
-message("-- Setting: PLATFORM=${PLATFORM}, ARCH=${IMAGEARCH}")
+message("-- Setting: PLATFORM=${PLATFORM}${PLATFORM_CONV_FROM}, ARCH=${IMAGEARCH}")
 if(NOT DEFINED REGISTRY_AUTH)
     message("-- Setting: REGISTRY=${REGISTRY}")
 else()
     message("-- Setting: REGISTRY=${REGISTRY}, AUTH=${REGISTRY_AUTH}")
 endif()
 message("-- Setting: RELEASE=${RELEASE}")
-message("-- Setting: TIMEOUT=${TIMEOUT}")
+if(BACKEND)
+    message("-- Setting: TIMEOUT=${TIMEOUT}")
+endif()
 if(ACCEPT_LICENSE)
     message("-- Setting: ACCEPT_LICENSE=${ACCEPT_LICENSE}")
 endif()
-message("-- Setting: BENCHMARK=${BENCHMARK}")
-message("-- Setting: BACKEND=${BACKEND}")
+if(BACKEND)
+    message("-- Setting: BENCHMARK=${BENCHMARK}")
+    message("-- Setting: BACKEND=${BACKEND}")
+endif()
 if(COMMAND show_backend_settings)
     show_backend_settings()
 endif()
 message("")
-if(NOT ENABLE_BUILD)
-    STRING(ASCII 27 esc)
-    message("${esc}[32mINFO:${esc}[m Build is disabled as ${DEFAULT_REGISTRY} is readonly.")
+if(BUILDSH_OPTIONS MATCHES "--read-only-registry")
+    message("${red}INFO:${reset} Docker build is disabled as ${DEFAULT_REGISTRY} is readonly.")
     message("")
 endif()
 
-execute_process(COMMAND bash -c "ln -s -r -f '${PROJECT_SOURCE_DIR}'/script/benchmark/*.sh ." WORKING_DIRECTORY "${CMAKE_BINARY_DIR}")
+if(EXISTS "${PROJECT_SOURCE_DIR}/script/benchmark")
+    execute_process(COMMAND bash -c "ln -s -r -f '${PROJECT_SOURCE_DIR}'/script/benchmark/*.sh ." WORKING_DIRECTORY "${CMAKE_BINARY_DIR}")
+endif()
+
+include(legalnotice OPTIONAL)
 

doc/cmake.md

@@ -1,4 +1,3 @@
-
 ### Customize the Build Process:
 
 You can use the following build options to customize the build process:  
@@ -11,12 +10,12 @@ You can use the following build options to customize the build process:
   > `RELEASE` must begin with colon `:`
 - **BACKEND**: Specify the validation backend: [`docker`](setup-docker.md), [`kubernetes`](setup-kubernetes.md), or [`terraform`](setup-terraform.md).    
   - **TERRAFORM_OPTIONS**: Specify the `terraform` options.  
-  - **TERRAFORM_SUT**: Specify the target SUT (Sytem Under Test) list.
+  - **TERRAFORM_SUT**: Specify the target SUT (System Under Test) list.
 - **TIMEOUT**: Specify the validation timeout, which contains the execution timeout and docker pull timeout. Default to 28800,300 seconds.   
 - **BENCHMARK**: Specify a workload pattern. Workloads not matching the pattern will be disabled. 
 - **SPOT_INSTANCE**: If specified, overwrite the `spot_instance` variable in the Cloud configuration files.   
 
-Build examples:   
+### Build examples:   
 
 ```bash
 cd build
@@ -39,4 +38,3 @@ make bom
 - [Docker Engine](setup-docker.md)  
 - [Kubernetes Cluster](setup-kubernetes.md)  
 - [Terraform Setup](setup-terraform.md)  
-

doc/ctest.md

@@ -1,49 +1,48 @@
 
 ### Run Test
 
-Use `ctest` to run a single test or batch of tests. You can do this at the top-level `build` directory or under each workload directory. In the latter case, only the tests of the workload will be executed. 
+Use `./ctest.sh` to run a single test or batch of tests. You can do this at the top-level `build` directory or under each workload directory. In the latter case, only the tests of the workload will be executed. 
 
 ```
 cd build
 cd workload/dummy
-ctest
+./ctest.sh -N
 ```
 
 ### CTest Options
 
-There is extensive list of options in `ctest` to control how tests can be executed. See the `ctest` manpage. The followings are most common options.  
+There is extensive list of options in `./ctest.sh` to control how tests can be executed. See the `./ctest.sh` manpage. The followings are most common options.  
 
 - *`-R`*: Select tests based on a regular expression string.   
 - *`-E`*: Exclude tests based on a regular expression string.  
 - *`-V`*: Show test execution with details.  
 - *`-N`*: Dry-run the tests only.  
 
 Example: list tests with `boringssl` in name excluding those with `_gated`
+
 ```
-ctest -R boringssl -E _gated -N
-```
+./ctest.sh -R boringssl -E _gated -N
 
+```
 Example: run only `test_static_boringssl` (exact match)
+
 ```
-ctest -R '^test_static_boringssl$'
+./ctest.sh -R '^test_static_boringssl$'
 ```
 
 ### Customize Configurations
 
-It is possible to specify a global test configuration file to overwrite any configuration parameter of a test case:   
+It is possible to specify a test configuration file to overwrite any configuration parameter of a test case:   
 
 ```
-TEST_CONFIG=$(pwd)/test_config.yaml ctest -V
+./ctest.sh --config=test_config.yaml -V
 ```
 
-where `TEST_CONFIG` specifies the test configuration file.  
-
 The configuration file uses the following format:
 
 ```
 *_dummy_pi:
     SCALE: 3000
-
 ```
 
 where `*_dummy_pi` specifies the test case name. You can use `*` to specify a wildcard match. The subsection underneath specifies the configuration variables and values. Any parameters specified in each test case [`validate.sh`](validate.md) can be overwritten. 
@@ -62,17 +61,19 @@ A set of utility scripts are linked under your workload build directory to make
 Usage: [options]
 --nohup          Run ctest in the daemon mode for long benchmark 
 --loop           Run the benchmark multiple times sequentially.
---run            Run the benchmark multiple times on the same SUT(s), only for cumulus.  
+--run            Run the benchmark multiple times on the same SUT(s).  
 --burst          Run the benchmark multiple times simultaneously.
 --config         Specify the test-config file.  
---options        Specify additioanl validation backend options.  
+--options        Specify additional validation backend options.  
 --set            Set the workload parameter values during loop and burst iterations.  
 --stop           Kill all ctest sessions.  
 --continue       Ignore any errors and continue the loop and burst iterations.  
 --prepare-sut    Prepare cloud SUT instances for reuse.
 --reuse-sut      Reuse previously prepared cloud SUT instances. 
 --cleanup-sut    Cleanup cloud SUT instances. 
 --dry-run        Generate the testcase configurations and then exit.  
+--testcase       Specify the exact testcase name to be executed.  
+--noenv          Clean any external environment variables before proceeding with the tests.    
 ```
 
 The followings are some examples:
@@ -129,7 +130,6 @@ The followings are some examples:
 ./ctest.sh -R aws --set "AWS_DISK_SIZE=500 1000" --loop 2 --nohup
 ./ctest.sh -R aws --set "AWS_IOPS=16000 32000" --loop 2 --nohup
 ./ctest.sh -R aws --set "AWS_NUM_STRIPED_DISKS=1 2" --loop 2 --nohup
-
 ```
 
 See Also: [Cloud SUT Reuse](#cloud-sut-reuse)
@@ -151,6 +151,7 @@ Usage: [options] [logs-directory]
 --filter _(real|throughput)
                       Specify a trim filter to shorten spreadsheet name.  
 --file <filename>     Specify the spread sheet filename. 
+--uri                 Show the WSF portal URI if present.   
 ```
 
 > The `xls-ai` option writes the KPI data in the `kpi-report.xls` spread sheet as follows:
@@ -171,7 +172,7 @@ Usage: [options] [logs-directory]
 
 ### Cloud SUT Reuse
 
-With the cumulus backend, it is possible to reuse the Cloud SUT instances during the benchmark process. This is especially useful in tuning parameters for any workload.   
+It is possible to reuse the Cloud SUT instances during the benchmark process. This is especially useful in tuning parameters for any workload.   
 
 To reuse any SUT instances, you need to first prepare (provision) the Cloud instances, using the `ctest.sh` `--prepare-sut` command as follows:  
 
@@ -205,6 +206,6 @@ SUT reuse is subject to the following limitations:
 
 ---
 
-Please cleanup the Cloud instances after use. 
+After using the Cloud instances, please clean them up.
 
 --- 

doc/setup-auth.md

@@ -1,23 +1,35 @@
 
 ### Introduction
 
-A private docker registry is optional in most of the validation scenarios except if you want to run the workloads on an On-Premises  Kubernetes cluster, or you explicitly setup a docker registry to store any newly built workload images. 
+A private docker registry is optional in most of the validation scenarios except if you want to run the workloads on an On-Premises  Kubernetes cluster, or you explicitly setup a docker registry to store any newly built workload images.
 
-This document describes how to authenticate to a docker registry if the registry requires authentication. Skip this document if there is no authentication requirement.   
+This document describes how to authenticate to a docker registry if the registry requires authentication. Skip this document if there is no authentication requirement.
 
 ### `REGISTRY_AUTH`
 
-The [`cmake`](cmake.md) `REGISTRY_AUTH` option specifies how to authenticate to a private docker registry. Currently, `docker` is the only supported value, which uses the docker authentication mechanism. 
+The [`cmake`](cmake.md) `REGISTRY_AUTH` option specifies how to authenticate to a private docker registry. Currently, `docker` and `pass` are the only supported values, which uses the docker authentication mechanism or docker-credential-pass with pass as password store.
 
-Enable the `REGISTRY_AUTH` option: 
+#### Docker
+
+Enable the `REGISTRY_AUTH` option:
 
 ```
 cmake -DREGISTRY=<url> -DREGISTRY_AUTH=docker ..
 ```
 
-With the above command, the validation scripts will upload the docker authentication information specified in `.docker/config.json` as a Kubernetes `imagePullSecret` to the validation cluster, On-Premises or in Cloud.  
+With the above command, the validation scripts will upload the docker authentication information specified in `.docker/config.json` as a Kubernetes `imagePullSecret` to the validation cluster, On-Premises or in Cloud.
+
+> `CredHelpers` or `CredStore` in `.docker/config.json` is not supported.
+
+#### Pass
+
+Enable the `REGISTRY_AUTH` option:
 
-> `CredHelpers` or `CredStore` in `.docker/config.json` is not supported. 
+```
+cmake -DREGISTRY=<url> -DREGISTRY_AUTH=pass ..
+```
+
+Version for both pass and docker-credential-pass binaries is read automatically. With the above command, the validation scripts will make the `pass` and `gpg` session available on execution machine.
 
 ### Authenticate to Cloud Private Registry
 
@@ -29,15 +41,13 @@ $ aws ecr get-login-password --region <region> | docker login --username AWS --p
 $ exit
 ```
 
-> Note that the build script will auto-create the image repository namespaces.  
+> Note that the build script will auto-create the image repository namespaces.
 
 #### Google Cloud Container Registry
 
 ```
 make gcp
-$ gcloud iam service-accounts create <NAME>
-$ gcloud projects add-iam-policy-binding <PROJECT_ID> --member "serviceAccount:<NAME>@<PROJECT_ID>.iam.gserviceaccount.com" --role "roles/ROLE"
-$ gcloud iam service-accounts keys create keyfile.json --iam-account <NAME>@<PROJECT_ID>.iam.gserviceaccount.com
+$ gcloud auth activate-service-account --key-file <key-file>
 $ gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin <registry-url>
 $ exit
 ```
@@ -52,4 +62,3 @@ make azure
 $ az acr login --name <registry-name> --expose-token --output tsv --query accessToken | docker login -username 00000000-0000-0000-0000-000000000000 --password-stdin <registry-url>
 $ exit
 ```
-

doc/setup-docker.md

@@ -1,5 +1,5 @@
 
-The `docker` engine is a prerequisite to build the workload images. It is also one of the validation backends that can be used to run single-container workloads on your local machine.  
+The `docker` engine is a prerequisite to build the docker images. 
 
 ### Setup Docker