Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from .base import BaseCheck
from ..constants import *
from ..constants import MAX_ACCURACY_LOG_SIZE
from ..loader import SubmissionLogs
from ..configuration.configuration import Config
from ..utils import check_extra_files
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@

from .base import BaseCheck
from ..constants import *
from ..constants import REQUIRED_TEST01_ACC_FILES, REQUIRED_COMP_PER_FILES
from ..loader import SubmissionLogs
from ..configuration.configuration import Config
from .performance_check import PerformanceCheck
from .accuracy_check import AccuracyCheck
from ..utils import *
from ..utils import files_diff, list_files
import re
import os

Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
from .base import BaseCheck
from ..constants import *
from ..constants import REQUIRED_MEASURE_FILES, SYSTEM_IMP_REQUIRED_FILES
from ..loader import SubmissionLogs
from ..configuration.configuration import Config
from ..utils import *
from ..utils import list_files
import os


Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from .base import BaseCheck
from ..constants import *
from ..constants import OFFLINE_MIN_SPQ, NETWORK_MODE_REQUIRED_SUBSTRING_IN_SUT_NAME, LLM_LATENCY_LIMITS, RESULT_FIELD_NEW, RESULT_FIELD_BENCHMARK_OVERWRITE, MS_TO_NS, S_TO_MS
from ..loader import SubmissionLogs
from ..configuration.configuration import Config
import os
Expand Down
4 changes: 2 additions & 2 deletions tools/submission/submission_checker/checks/power_check.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
from .base import BaseCheck
from ..constants import *
from ..constants import REQUIRED_PERF_FILES, REQUIRED_PERF_POWER_FILES, REQUIRED_POWER_FILES, OPTIONAL_PERF_FILES
from ..loader import SubmissionLogs
from ..configuration.configuration import Config
from .power.power_checker import check as check_power_more
from ..utils import *
from ..utils import files_diff, list_files
import os
import sys
import datetime
Expand Down
4 changes: 2 additions & 2 deletions tools/submission/submission_checker/checks/system_check.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
from .base import BaseCheck
from ..constants import *
from ..constants import VALID_AVAILABILITIES, SYSTEM_DESC_IS_NETWORK_MODE, SYSTEM_DESC_REQUIRED_FIELDS, SYSTEM_DESC_REQUIRED_FIELDS_NETWORK_MODE, SYSTEM_DESC_MEANINGFUL_RESPONSE_REQUIRED_FIELDS, SYSTEM_DESC_NUMERIC_RESPONSE_REQUIRED_FIELDS
from ..loader import SubmissionLogs
from ..configuration.configuration import Config
from ..utils import *
from ..utils import is_number


class SystemCheck(BaseCheck):
Expand Down
8 changes: 4 additions & 4 deletions tools/submission/submission_checker/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -1132,12 +1132,12 @@
"84",
"59",
"12",
"31",
"31",
"86",
"122",
"233",
"122",
"233",
"96",
]
]
},
}
}
Expand Down
4 changes: 2 additions & 2 deletions tools/submission/submission_checker/loader.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import os
from .constants import *
from .constants import PERFORMANCE_LOG_PATH, PERFORMANCE_SUMMARY_PATH, ACCURACY_LOG_PATH, ACCURACY_RESULT_PATH, ACCURACY_JSON_PATH, SYSTEM_PATH, MEASUREMENTS_PATH, COMPLIANCE_PATH, TEST01_PERF_PATH, TEST01_ACC_PATH, TEST04_PERF_PATH, TEST04_ACC_PATH, TEST06_ACC_PATH, TEST07_ACC_PATH, TEST08_ACC_PATH, TEST09_ACC_PATH, POWER_DIR_PATH, SRC_PATH, VALID_DIVISIONS
from .utils import list_dir
from .parsers.loadgen_parser import LoadgenParser
from typing import Generator, Literal
from .utils import *
from .utils import list_files
from .configuration.configuration import Config
import logging
import json
Expand Down
3 changes: 1 addition & 2 deletions tools/submission/submission_checker/results.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@

from .loader import SubmissionLogs
from .configuration.configuration import Config
import os
import csv
from .constants import *
from .constants import UNIT_DICT, POWER_UNIT_DICT, SPECIAL_UNIT_DICT
import json


Expand Down
5 changes: 3 additions & 2 deletions tools/submission/submission_checker/utils.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import os
from .constants import *
import logging
from .constants import REQUIRED_COMP_PER_FILES, RESULT_FIELD_NEW, RESULT_FIELD_BENCHMARK_OVERWRITE, MS_TO_NS, S_TO_MS
from .parsers.loadgen_parser import LoadgenParser


Expand Down Expand Up @@ -199,6 +200,7 @@ def get_inferred_result(
if uses_early_stopping:
early_stopping_latency_ms = mlperf_log["early_stopping_latency_ms"]
if early_stopping_latency_ms == 0 and log_error:
log = logging.getLogger("main")
log.error(
"Not enough samples were processed for early stopping to make an estimate"
)
Expand All @@ -213,7 +215,6 @@ def get_inferred_result(

def check_compliance_perf_dir(test_dir):
is_valid = False
import logging
log = logging.getLogger("main")

fname = os.path.join(test_dir, "verify_performance.txt")
Expand Down