Tx setup on server

scripts/sosp24-experiments/vm_tx_flows_exp.sh

@@ -0,0 +1,177 @@
+#!/bin/bash
+cd ..
+echo "Running flow experiment... this may take a few minutes"
+
+# Add dry run flag
+DRY_RUN=0
+
+while [[ $# -gt 0 ]]; do
+    key="$1"
+    case $key in
+        --dry)
+        DRY_RUN=1
+        shift # past argument
+        ;;
+        *)
+        shift # past unknown argument
+        ;;
+    esac
+done
+
+GUEST_INTF="enp0s1np0"
+GUEST_IP="192.168.101.11"
+GUEST_NIC_BUS="0x0"
+GUEST_HOME="/home/schai"
+# for some reason, public domain name doesn't work
+HOST_IP="192.17.101.97"
+HOST_UNAME="lbalara"
+HOST_HOME="/home/lbalara"
+CLIENT_HOME="/home/siyuanc3"
+CLIENT_INTF="ens1006np0"
+CLIENT_IP="192.168.101.3"
+CLIENT_SSH_UNAME="siyuanc3"
+CLIENT_SSH_HOST="nexus03.csl.illinois.edu" # Public IP or hostname for SSH "genie12.cs.cornell.edu"
+CLIENT_SSH_PASSWORD="saksham"
+CLIENT_USE_PASS_AUTH=0 # 1 to use password, 0 to use identity file
+CLIENT_SSH_IDENTITY_FILE="/home/schai/.ssh/id_rsa"
+
+function detect_virt_tech() {
+    local virsh_out=$(ssh -i "$CLIENT_SSH_IDENTITY_FILE" "${HOST_UNAME}@${HOST_IP}" 'bash -l -c "virsh list --all"')
+    local running_vms=$(echo "$virsh_out" | grep running | awk '{print $2}')
+
+    if [ -z "$running_vms" ]; then
+        echo "ERROR: No VMs are currently running" >&2
+        return 1
+    fi
+
+    echo "Found running VMs: $running_vms" >&2
+
+    local detected_tech=""
+    if [[ "$running_vms" == *"nested"* ]]; then
+        detected_tech="nested"
+    elif [[ "$running_vms" == *"shadow"* ]]; then
+        detected_tech="shadow"
+    elif [[ "$running_vms" == *"off"* ]]; then
+        detected_tech="off"
+    else
+        echo "ERROR: Could not detect virtualization technology from VM name: $running_vms" >&2
+        echo "Expected VM name to contain 'nested', 'shadow', or 'off'" >&2
+        return 1
+    fi
+
+    echo "$detected_tech"
+    return 0
+}
+
+parse_iommu_mode() {
+	local cmdline="${1:-$(</proc/cmdline)}"
+	local cl
+	cl="$(printf '%s' "$cmdline" | tr '[:upper:]' '[:lower:]')"
+
+	# Passthrough (separate case)
+	if [[ "$cl" =~ (^|[[:space:]])(iommu=pt|iommu\.passthrough=(1|on|y|yes|true))($|[[:space:]]) ]]; then
+		echo passthrough
+		return
+	fi
+
+	# Off
+	if [[ "$cl" =~ (^|[[:space:]])(noiommu|iommu=off|intel_iommu=off|amd_iommu=off)($|[[:space:]]) ]]; then
+		echo off
+		return
+	fi
+
+	# Strict
+	if [[ "$cl" =~ (^|[[:space:]])iommu\.strict=(1|on|y|yes|true)($|[[:space:]]) ]] || \
+	   [[ "$cl" =~ (^|[[:space:]])intel_iommu=([^[:space:]]*,)?strict([^[:space:]]*)($|[[:space:]]) ]] || \
+	   [[ "$cl" =~ (^|[[:space:]])amd_iommu=([^[:space:]]*,)?strict([^[:space:]]*)($|[[:space:]]) ]]; then
+		echo strict
+		return
+	fi
+
+	# Lazy (non-strict)
+	if [[ "$cl" =~ (^|[[:space:]])iommu\.strict=(0|off|n|no|false)($|[[:space:]]) ]] || \
+	   [[ "$cl" =~ (^|[[:space:]])intel_iommu=([^[:space:]]*,)?nonstrict([^[:space:]]*)($|[[:space:]]) ]] || \
+	   [[ "$cl" =~ (^|[[:space:]])amd_iommu=([^[:space:]]*,)?nonstrict([^[:space:]]*)($|[[:space:]]) ]]; then
+		echo lazy
+		return
+	fi
+
+	# Explicitly enabled but no strictness specified → assume strict
+	if [[ "$cl" =~ (^|[[:space:]])(iommu=on|intel_iommu=on|amd_iommu=on)($|[[:space:]]) ]]; then
+		echo strict
+		return
+	fi
+
+	# Default if unspecified
+	echo strict
+}
+
+if [ "$CLIENT_USE_PASS_AUTH" -eq 1 ]; then
+	SSH_CLIENT_CMD="sshpass -p $CLIENT_SSH_PASSWORD ssh ${CLIENT_SSH_UNAME}@${CLIENT_SSH_HOST}"
+else
+	SSH_CLIENT_CMD="ssh -i $CLIENT_SSH_IDENTITY_FILE ${CLIENT_SSH_UNAME}@${CLIENT_SSH_HOST}"
+fi
+
+guest_cmdline=$(cat /proc/cmdline)
+guest_iommu_config=$(parse_iommu_mode "$guest_cmdline")
+host_cmdline=$(ssh -i "$CLIENT_SSH_IDENTITY_FILE" "${HOST_UNAME}@${HOST_IP}" 'cat /proc/cmdline')
+host_iommu_config=$(parse_iommu_mode "$host_cmdline")
+
+virt_tech=$(detect_virt_tech)
+if [ $? -ne 0 ]; then
+    echo "Failed to detect virtualization technology"
+    exit 1
+fi
+
+iommu_config="host-${guest_iommu_config}-guest-${host_iommu_config}-$virt_tech"
+
+echo "iommu_config: $iommu_config"
+# exit 0
+# pause the frame
+sudo ethtool --pause $GUEST_INTF tx off rx off
+$SSH_CLIENT_CMD "sudo ethtool --pause $CLIENT_INTF tx off rx off"
+$SSH_CLIENT_CMD "echo off | sudo tee /sys/devices/system/cpu/smt/control"
+
+sleep 1
+
+client_cores="0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31"
+server_cores="0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31"
+
+timestamp=$(date '+%Y-%m-%d-%H-%M-%S')
+for socket_buf in 1; do
+    for ring_buffer in 512; do
+        #for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 20 24 28; do
+        for i in 1; do
+            num_cores=$i
+            client_cores_mask=($(echo $client_cores | tr ',' '\n' | head -n $num_cores | tr '\n' ','))
+            server_cores_mask=($(echo $server_cores | tr ',' '\n' | head -n $num_cores | tr '\n' ','))
+
+      	    format_i=$(printf "%02d\n" $i)
+            exp_name="${timestamp}-$(uname -r)-flow${format_i}-${iommu_config}-${num_cores}cores"
+            echo $exp_name
+
+            if [ "$DRY_RUN" -eq 1 ]; then
+                continue
+            fi
+
+            sudo bash vm-tx-run-dctcp-tput-experiment.sh \
+            --guest-home "$GUEST_HOME" --guest-ip "$GUEST_IP" --guest-intf "$GUEST_INTF" --guest-bus "$GUEST_NIC_BUS" -n "$i" -c $server_cores_mask \
+            --client-home "$CLIENT_HOME" --client-ip "$CLIENT_IP" --client-intf "$CLIENT_INTF" -N "$i" -C $client_cores_mask \
+            --host-home "$HOST_HOME" --host-ip "$HOST_IP" \
+            --client-ssh-name "$CLIENT_SSH_UNAME" --client-ssh-pass "$CLIENT_SSH_PASSWORD" --client-ssh-host "$CLIENT_SSH_HOST" --client-ssh-use-pass "$CLIENT_USE_PASS_AUTH" --client-ssh-ifile "$CLIENT_SSH_IDENTITY_FILE" \
+            -e "$exp_name" -m 4000 -r $ring_buffer -b "400g" -d 1\
+            --socket-buf $socket_buf --mlc-cores 'none' --runs 1
+
+            python3 report-tput-metrics.py $exp_name tput,drops,acks,iommu,cpu | sudo tee ../utils/reports/$exp_name/summary.txt
+            echo $PWD
+            cd ../utils/reports/$exp_name
+
+            sudo bash -c "cat /sys/kernel/debug/tracing/trace > iova.log"
+
+            cd -
+            sudo chmod +666 -R ../utils/reports/$exp_name
+
+        done
+    done
+done
+

scripts/vm-tx-collect-tput-stats.py

@@ -0,0 +1,184 @@
+import sys
+import numpy as np
+import statistics
+import subprocess
+
+# TODO: Leshna, Combine both vm and baremetal stat collector with file names as parameters.
+
+EXP_NAME = sys.argv[1]
+NUM_RUNS = int(sys.argv[2])
+COLLECT_MLC_TPUT = int(sys.argv[3])
+
+FILE_NAME = "../utils/reports/" + EXP_NAME
+command = 'mkdir -p ' + FILE_NAME
+result = subprocess.run(command, shell=True, capture_output=True, text=True)
+
+net_tputs = []
+retx_rates = []
+sent_packets = []
+mem_bws = []
+cpu_utils = []
+mlc_tputs = []
+
+pcie_wr_tput = []
+iotlb_first_lookup = []
+iotlb_all_lookup = []
+iotlb_miss = []
+iommu_mem_access = []
+iotlb_inv = []
+pwt_occupancy = []
+
+# iotlb_hits = []
+# ctx_lookup = []
+# ctx_hits   = []
+# ctxt_misses = []
+# cache_lookup   = []
+# cache_hit_256T = []
+# cache_hit_512G = []
+# cache_hit_1G   = []
+# cache_hit_2M   = []
+# cache_fills    = []
+
+for i in range(NUM_RUNS):
+    with open(FILE_NAME + '-RUN-' + str(i) + '/iperf.bw.rpt') as f1:
+        for line in f1:
+            tput = float(line.split()[-1])
+            if (tput > 0):
+                net_tputs.append(tput)
+            break
+
+    with open(FILE_NAME + '-RUN-' + str(i) + '/retx.rpt') as f1:
+        for line in f1:
+            line_str = line.split()
+            if (line_str[0] == 'Retx_percent:'):  # always come last so we can break
+                retx_pct = float(line_str[-1])
+                if (retx_pct >= 0):
+                    retx_rates.append(retx_pct)
+                break
+            elif (line_str[0] == "Recv:"):
+                sent = float(line_str[-1])
+                sent_packets.append(sent)
+
+    with open(FILE_NAME + '-RUN-' + str(i) + '/host-membw.rpt') as f1:
+        try:
+            for line in f1:
+                line_str = line.split()
+                if (line_str[0] != 'Node0_total_bw:'):
+                    continue
+                else:
+                    membw = float(line_str[-1])
+                    if (membw >= 0):
+                        mem_bws.append(membw)
+                    break
+        except Exception as e:
+            mem_bws.append(0)
+
+    with open(FILE_NAME + '-RUN-' + str(i) + '/cpu_util.rpt') as f1:
+        for line in f1:
+            line_str = line.split()
+            if (line_str[0] != 'avg_cpu_util:'):
+                continue
+            else:
+                cpu_util = float(line_str[-1])
+                if (cpu_util >= 0):
+                    cpu_utils.append(cpu_util)
+                break
+    try: 
+        with open(FILE_NAME + '-RUN-' + str(i) + '/host-pcie.rpt') as f1:
+            for line in f1:
+                line_str = line.split()
+                if (line_str[0] == 'PCIe_wr_tput:'):
+                    pcie_tput = float(line_str[-1])
+                    if (pcie_tput >= 0):
+                        pcie_wr_tput.append(pcie_tput)
+                elif (line_str[0] == 'IOTLB_first_lookup_avg:'):
+                    iotlb_first_lookup_ = float(line_str[-1])
+                    iotlb_first_lookup.append(iotlb_first_lookup_)
+                elif (line_str[0] == 'IOTLB_all_lookup_avg:'):
+                    iotlb_all_lookup_ = float(line_str[-1])
+                    iotlb_all_lookup.append(iotlb_all_lookup_)
+                elif (line_str[0] == 'IOTLB_miss_avg:'):
+                    iotlb_miss_ = float(line_str[-1])
+                    iotlb_miss.append(iotlb_miss_)
+                elif (line_str[0] == 'IOMMU_mem_access:'):
+                    iommu_mem_access_ = float(line_str[-1])
+                    iommu_mem_access.append(iommu_mem_access_)
+                elif (line_str[0] == 'IOTLB_inv_avg:'):
+                    iotlb_inv_ = float(line_str[-1])
+                    iotlb_inv.append(iotlb_inv_)
+                elif (line_str[0] == 'PWT_occupancy_avg:'):
+                    pwt_occupancy_ = float(line_str[-1])
+                    pwt_occupancy.append(pwt_occupancy_)
+    except Exception as e:
+        pcie_wr_tput.append(0) 
+        iotlb_first_lookup.append(0)
+        iotlb_all_lookup.append(0)
+        iotlb_miss.append(0)
+        iommu_mem_access.append(0)
+        iotlb_inv.append(0)
+        pwt_occupancy.append(0)
+
+    if (COLLECT_MLC_TPUT > 0 and False):
+        with open(FILE_NAME + '-MLCRUN-' + str(i) + '/mlc.log') as f1:
+            for line in f1:
+                if line.startswith(' 00000'):
+                    tput = float(line.split()[-1])
+                    if (tput >= 0):
+                        mlc_tputs.append(tput)
+                    break
+
+
+def mean_or_zero(arr): return statistics.mean(arr) if arr else 0
+def stdev_or_zero(arr): return statistics.stdev(arr) if len(arr) > 1 else 0
+
+
+cpu_utils_mean = mean_or_zero(cpu_utils);               cpu_utils_stddev = stdev_or_zero(cpu_utils)
+net_tput_mean = mean_or_zero(net_tputs);                net_tput_stddev = stdev_or_zero(net_tputs)
+retx_rate_mean = mean_or_zero(retx_rates);              retx_rate_stddev = stdev_or_zero(retx_rates)
+sent_packets_mean = mean_or_zero(sent_packets);         sent_packets_stddev = stdev_or_zero(sent_packets)
+mem_bw_mean = mean_or_zero(mem_bws);                    mem_bw_stddev = stdev_or_zero(mem_bws)
+pcie_wr_tput_mean = mean_or_zero(pcie_wr_tput);         pcie_wr_tput_stddev = stdev_or_zero(pcie_wr_tput)
+
+iotlb_first_lookup_mean = mean_or_zero(iotlb_first_lookup);  iotlb_first_lookup_stddev = stdev_or_zero(iotlb_first_lookup)
+iotlb_all_lookup_mean  = mean_or_zero(iotlb_all_lookup);     iotlb_all_lookup_stddev  = stdev_or_zero(iotlb_all_lookup)
+iotlb_miss_mean        = mean_or_zero(iotlb_miss);           iotlb_miss_stddev        = stdev_or_zero(iotlb_miss)
+iommu_mem_access_mean  = mean_or_zero(iommu_mem_access);     iommu_mem_access_stddev  = stdev_or_zero(iommu_mem_access)
+iotlb_inv_mean         = mean_or_zero(iotlb_inv);            iotlb_inv_stddev         = stdev_or_zero(iotlb_inv)
+pwt_occupancy_mean     = mean_or_zero(pwt_occupancy);        pwt_occupancy_stddev     = stdev_or_zero(pwt_occupancy)
+
+mlc_tput_mean = 0
+mlc_tput_stddev = 0
+
+if (COLLECT_MLC_TPUT > 0):
+    mlc_tput_mean = statistics.mean(mlc_tputs)
+    if NUM_RUNS > 1:
+        mlc_tput_stddev = statistics.stdev(mlc_tputs)
+    else:
+        mlc_tput_stddev = 0
+
+output_list = [
+    ("cpu_utils_mean", cpu_utils_mean), ("cpu_utils_stddev", cpu_utils_stddev),
+    ("net_tput_mean", net_tput_mean), ("net_tput_stddev", net_tput_stddev),
+    ("retx_rate_mean", retx_rate_mean), ("retx_rate_stddev", retx_rate_stddev),
+    ("mem_bw_mean", mem_bw_mean), ("mem_bw_stddev", mem_bw_stddev),
+    ("pcie_wr_tput_mean", pcie_wr_tput_mean), ("pcie_wr_tput_stddev", pcie_wr_tput_stddev),
+
+    ("iotlb_first_lookup_mean", iotlb_first_lookup_mean), ("iotlb_first_lookup_stddev", iotlb_first_lookup_stddev),
+    ("iotlb_all_lookup_mean",  iotlb_all_lookup_mean),   ("iotlb_all_lookup_stddev",  iotlb_all_lookup_stddev),
+    ("iotlb_miss_mean", iotlb_miss_mean), ("iotlb_miss_stddev", iotlb_miss_stddev),
+    ("iommu_mem_access_mean", iommu_mem_access_mean), ("iommu_mem_access_stddev", iommu_mem_access_stddev),
+    ("iotlb_inv_mean", iotlb_inv_mean), ("iotlb_inv_stddev", iotlb_inv_stddev),
+    ("pwt_occupancy_mean", pwt_occupancy_mean), ("pwt_occupancy_stddev", pwt_occupancy_stddev),
+
+    ("mlc_tput_mean", 0 if not mlc_tputs else mean_or_zero(mlc_tputs)),
+    ("mlc_tput_stddev", 0 if len(mlc_tputs) < 2 else stdev_or_zero(mlc_tputs)),
+    ("sent_packets_mean", sent_packets_mean), ("sent_packets_stddev", sent_packets_stddev),
+]
+
+headers, outputs = zip(*output_list)
+headers = ",".join(headers) 
+outputs = list(outputs)
+
+# Save array to DAT file
+np.savetxt(FILE_NAME + '/tput_metrics.dat',
+           [outputs], delimiter=",", header=headers, comments='', fmt='%.10f')