diff --git a/docs/competition.zip b/docs/competition.zip deleted file mode 100644 index e49d15f07..000000000 Binary files a/docs/competition.zip and /dev/null differ diff --git a/docs/example_scripts/README.md b/docs/example_scripts/README.md deleted file mode 100644 index fd7311e46..000000000 --- a/docs/example_scripts/README.md +++ /dev/null @@ -1,7 +0,0 @@ -## Example Scripts - -These scripts have been built solely to improve users' understanding of the API -and expressly not as utility scripts. They can serve as starting places, but they -ought not be solely relied upon for automation. - -They exist here for easy testing of robot submissions outlined [here](https://docs.codabench.org/latest/Developers_and_Administrators/Robot-submissions/). diff --git a/docs/example_scripts/example_submission.py b/docs/example_scripts/example_submission.py deleted file mode 100755 index 337a4d2ce..000000000 --- a/docs/example_scripts/example_submission.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3 -""" -Overview -======== -This script leverages the "robot submissions" feature which allows someone to make -unlimited submissions to a competition. An example use case is the ChaGrade teaching -website: student submissions are made and scores returned via robot submissions. - -Usage -===== - -1. Create a competition that allows robots, and create a user marked as a robot - user. Use that username and password below. - -2. Get into a python3 environment with requests installed - -3. Review this script and edit the applicable variables, like... - - CODALAB_URL - USERNAME - PASSWORD - ... - -4. Then execute the contents of this script: - - ./example_submission.py -""" -# ---------------------------------------------------------------------------- -# Configure these -# ---------------------------------------------------------------------------- -CODALAB_URL = 'http://localhost/' -USERNAME = 'admin' -PASSWORD = 'admin' -PHASE_ID = None -TASK_LIST = [] -SUBMISSION_ZIP_PATH = '../../src/tests/functional/test_files/submission.zip' - - -# ---------------------------------------------------------------------------- -# Script start.. -# ---------------------------------------------------------------------------- -from urllib.parse import urljoin # noqa: E402 -import requests # noqa: E402,E261 # Ignore E261 to line up these noqa -import os - -# Check someone updated PHASE_ID argument! -assert PHASE_ID, "PHASE_ID must be set at the top of this script" - -# Login -login_url = urljoin(CODALAB_URL, '/api/api-token-auth/') -resp = requests.post(login_url, {"username": USERNAME, "password": PASSWORD}) -if resp.status_code != 200: - print(f"Failed to login: {resp.content}") - print('Is the url correct? (http vs https)') - exit(-1) - -# Setup auth headers for the rest of communication -token = resp.json()["token"] -headers = { - "Authorization": f"Token {token}" -} - -# Check if we can make a submission -can_make_sub_url = urljoin(CODALAB_URL, f'/api/can_make_submission/{PHASE_ID}/') -resp = requests.get(can_make_sub_url, headers=headers) - -if not resp.json()['can']: - print(f"Failed to create submission: {resp.json()['reason']}") - exit(-2) - - - -file_size = os.path.getsize(SUBMISSION_ZIP_PATH) - -# Create + Upload our dataset -datasets_url = urljoin(CODALAB_URL, '/api/datasets/') -datasets_payload = { - "type": "submission", - "request_sassy_file_name": "submission.zip", - "file_size": file_size, -} -resp = requests.post(datasets_url, datasets_payload, headers=headers) -if resp.status_code != 201: - print(f"Failed to create dataset: {resp.content}") - exit(-3) - -dataset_data = resp.json() -sassy_url = dataset_data["sassy_url"].replace('docker.for.mac.localhost', 'localhost') # switch URLs for local testing -resp = requests.put(sassy_url, data=open(SUBMISSION_ZIP_PATH, 'rb'), headers={'Content-Type': 'application/zip'}) -if resp.status_code != 200: - print(f"Failed to upload dataset: {resp.content} to {sassy_url}") - exit(-4) - -# Submit it to the competition -submission_url = urljoin(CODALAB_URL, '/api/submissions/') -submission_payload = { - "phase": PHASE_ID, - "tasks": TASK_LIST, - "data": dataset_data["key"], -} - -print(f"Making submission using data: {submission_payload}") -resp = requests.post(submission_url, submission_payload, headers=headers) - -if resp.status_code in (200, 201): - print(f"Successfully submitted: {resp.content}") -else: - print(f"Error submitting ({resp.status_code}): {resp.content}") diff --git a/docs/example_scripts/get_competition_details.py b/docs/example_scripts/get_competition_details.py deleted file mode 100755 index 125a64b93..000000000 --- a/docs/example_scripts/get_competition_details.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python3 -""" -Overview -======== -This script is designed to list competition information. It's main purpose is to help find -the phase id to be used in the 'example_submission.py' script. - -Usage -===== - -1. Run the script with no arguments to list all competitions. - - ./get_competition_details.py - -2. Find the competition you would like to test on. - -3. Run the script again with competition id as the only argument to find the phase ID. - - ./get_competition_details.py 1 - -4. If you want to use the task selection feature, run the script a third time with competition id as the first argument, - and the phase ID as the second argument to get a list of tasks on the phase. - - ./get_competition_details.py 1 1 -""" - -from sys import argv # noqa: E402,E261 # Ignore E261 to line up these noqa -from operator import itemgetter # noqa: E402,E261 -from urllib.parse import urljoin # noqa: E402,E261 -import requests # noqa: E402,E261 - - -# ---------------------------------------------------------------------------- -# Configure this -# ---------------------------------------------------------------------------- -CODALAB_URL = 'http://localhost/' - - -# ---------------------------------------------------------------------------- -# Script start.. -# ---------------------------------------------------------------------------- -COMPETITION_ID = None -PHASE_ID = None -MODE = 0 - -if len(argv) > 1: - COMPETITION_ID = int(argv[1]) - MODE = 1 -if len(argv) > 2: - PHASE_ID = int(argv[2]) - MODE = 2 - -if COMPETITION_ID and not PHASE_ID: - comp_detail_url = urljoin(CODALAB_URL, f'/api/competitions/{COMPETITION_ID}') - resp = requests.get(comp_detail_url) - if resp.status_code != 200: - print(f"Failed to get competitions: {resp.content}") - exit(-1) - data = resp.json() - comp_title = data['title'] - phases = sorted(data['phases'], key=itemgetter('id')) - - print(f"\nCompetition: {comp_title}\n") - print('---------- Phases ----------') - print(' id | name') - print('----------------------------') - for p in phases: - print(f"{p['id']:>4} | {p['name']}") - print() - -elif PHASE_ID: - comp_detail_url = urljoin(CODALAB_URL, f'/api/competitions/{COMPETITION_ID}') - resp = requests.get(comp_detail_url) - if resp.status_code != 200: - print(f"Failed to get phase: {resp.content}") - exit(-2) - data = resp.json() - - selected_phase = None - for phase in data['phases']: - if phase['id'] == PHASE_ID: - selected_phase = phase - break - - comp_title = selected_phase['name'] - phases = sorted(data['phases'], key=itemgetter('id')) - - print(f"\nCompetition: {comp_title}\n") - print('----------------- Tasks ----------------') - print(' id | name') - print('----------------------------------------') - for t in selected_phase['tasks']: - print(f"{t['id']:>4} | {t['name']}") - print() - -elif not COMPETITION_ID and not PHASE_ID: - comp_list_url = urljoin(CODALAB_URL, f'/api/competitions/') - resp = requests.get(comp_list_url) - if resp.status_code != 200: - print(f"Failed to get competitions: {resp.content}") - exit(-3) - competitions = sorted(resp.json(), key=itemgetter('id')) - - print('\n------------------ Competitions ------------------') - print(' id | creator | name') - print('--------------------------------------------------') - for c in competitions: - print(f"{c['id']:>4} | {c['created_by']:<20} | {c['title']}") - print() diff --git a/docs/example_scripts/get_submission_details.py b/docs/example_scripts/get_submission_details.py deleted file mode 100755 index 6fc8e21dc..000000000 --- a/docs/example_scripts/get_submission_details.py +++ /dev/null @@ -1,222 +0,0 @@ -#!/usr/bin/env python3 -""" -Overview -======== -This script is designed to find submission information. It's main purpose is to demonstrate -how to programmatically find submission information. - -Tip -======== -If you don't know a phase ID, start with the get_competition_details.py script. - - -Usage -======== - get_submission_details.py -p Show table of submissions on a phase - get_submission_details.py -s Get Details of selected submission - get_submission_details.py -s -v Download Submission, Metadata, and logs to zip file - -Arguments -======== - -h, --help Print Help (this message) and exit - -p, --phase Phase ID/PK to select - -s, --submission Submission ID/PK to select - -v, --verbose Download Verbose Output - -o, --output Change directory to save Verbose Output - - - -""" -import json -from sys import argv # noqa: E402,E261 # Ignore E261 to line up these noqa -import os -import getopt -from operator import itemgetter # noqa: E402,E261 -from urllib.parse import urljoin # noqa: E402,E261 -import requests # noqa: E402,E261 -from pprint import pprint # noqa: E402,E261 -import tempfile -import zipfile - - -# ---------------------------------------------------------------------------- -# Configure these -# ---------------------------------------------------------------------------- -CODALAB_URL = 'http://localhost/' -USERNAME = 'admin' -PASSWORD = 'admin' - - -# ---------------------------------------------------------------------------- -# Help (-h, --help) -# ---------------------------------------------------------------------------- -def print_help(): - help_commands = [ - ["-h, --help", "Print Help (this message) and exit"], - ["-p, --phase ", "Phase ID/PK to select"], - ["-s, --submission ", "Submission ID/PK to select"], - ["-v, --verbose", "Download Verbose Output"], - ["-o, --output", "Change directory to save Verbose Output"], - ] - usage = [ - [f"{argv[0]} -p ", "Show table of submissions on a phase"], - [f"{argv[0]} -s ", "Get Details of selected submission"], - [f"{argv[0]} -s -v", "Download Submission, Metadata, and logs to zip file"], - ] - print("Overview:\n This script is designed to find submission information.\n " - "It's main purpose is to demonstrate how to programmatically find submission information.\n") - print("Tip:\n If you don't know a phase ID, start with the get_competition_details.py script.\n") - print("Usage:") - for use in usage: - print(" %-55s %-45s" % (use[0], use[1])) - print("\nArguments:") - for command in help_commands: - print(" %-55s %-45s" % (command[0], command[1])) - exit(0) - - -# ---------------------------------------------------------------------------- -# Script start.. -# ---------------------------------------------------------------------------- -PHASE_ID = SUBMISSION_ID = None -VERBOSE = False -CURRENT_DIR = os.getcwd() - -short_options = "hp:s:vo:" -long_options = ["help", "phase=", "submission=", "verbose", "output="] -argument_list = argv[1:] - -try: - arguments, values = getopt.getopt(argument_list, short_options, long_options) -except getopt.error as err: - # Output error, and return with an error code - print(str(err)) - exit(2) - -# Evaluate given options -for current_argument, current_value in arguments: - if current_argument in ("-v", "--verbose"): - VERBOSE = True - elif current_argument in ("-h", "--help"): - print_help() - elif current_argument in ("-p", "--phase"): - PHASE_ID = int(current_value) - elif current_argument in ("-s", "--submission"): - SUBMISSION_ID = int(current_value) - elif current_argument in ("-o", "--output"): - os.chdir(current_value) - CURRENT_DIR = os.getcwd() - -# Login -login_url = urljoin(CODALAB_URL, '/api/api-token-auth/') -resp = requests.post(login_url, {"username": USERNAME, "password": PASSWORD}) -if resp.status_code != 200: - print(f"Failed to login: {resp.content}") - print('Is the url correct? (http vs https)') - exit(-1) - -# Setup auth headers for the rest of communication -token = resp.json()["token"] -headers = { - "Authorization": f"Token {token}" -} - - -# If importing this, make sure to call .cleanup() in response -def get_verbose(SUBMISSION_ID): - submissions_detail_url = urljoin(CODALAB_URL, f'/api/submissions/{SUBMISSION_ID}/') - resp = requests.get(submissions_detail_url, headers=headers) - if resp.status_code != 200: - print(f"Failed to get submission: {resp.content}") - exit(-3) - resp_json = resp.json() - - submissions_get_details_url = urljoin(CODALAB_URL, f'/api/submissions/{SUBMISSION_ID}/get_details/') - detail_resp = requests.get(submissions_get_details_url, headers=headers) - if detail_resp.status_code != 200: - print(f"Failed to get submission: {detail_resp.content}") - exit(-3) - detail_resp_json = detail_resp.json() - - url = detail_resp_json["data_file"] - # NEEDED FOR DEV ENVIRONMENT - url = url.replace("docker.for.mac.", '') - r_zip = requests.get(url) - temp_dir = tempfile.TemporaryDirectory() - with open(f'{temp_dir.__enter__()}/submission.zip', 'wb') as file: - file.write(r_zip.content) - with open(f'{temp_dir.__enter__()}/submission.json', 'w', encoding='utf-8') as file: - file.write(json.dumps(resp_json, ensure_ascii=False, indent=4)) - with open(f'{temp_dir.__enter__()}/submission_detail.json', 'w', encoding='utf-8') as file: - file.write(json.dumps(detail_resp_json, ensure_ascii=False, indent=4)) - for log in detail_resp_json['logs']: - with open(f'{temp_dir.__enter__()}/{log["name"]}.txt', 'w', encoding='utf-8') as file: - url = log["data_file"] - # NEEDED FOR DEV ENVIRONMENT - url = url.replace("docker.for.mac.", '') - resp = requests.get(url) - resp.encoding = 'utf-8' - file.write(resp.text) - - return temp_dir - - -# ---------------------------------------------------------------------------- -# MAIN -# ---------------------------------------------------------------------------- - -if PHASE_ID and not SUBMISSION_ID: - submissions_list_url = urljoin(CODALAB_URL, f'/api/submissions/') - resp = requests.get(submissions_list_url, {"phase": PHASE_ID}, headers=headers) - if resp.status_code != 200: - print(f"Failed to get submissions: {resp.content}") - exit(-2) - - submissions = sorted(resp.json(), key=itemgetter('id')) - - print('\n--------------------- Submissions -----------------------') - print(' id | creator | creation date') - print('---------------------------------------------------------') - for s in submissions: - print(f"{s['id']:>4} | {s['owner']:<20} | {s['created_when']}") - print() - -elif SUBMISSION_ID and VERBOSE: - temp_dir = get_verbose(SUBMISSION_ID) - files = os.listdir(temp_dir.__enter__()) - zip_dir = f'{CURRENT_DIR}/submission-{SUBMISSION_ID}.zip' - with zipfile.ZipFile(zip_dir, 'w') as zipObj: - for file in files: - zipObj.write(f'{temp_dir.__enter__()}/{file}', arcname=file) - print(f"Saved Output to: {zip_dir}") - temp_dir.cleanup() - - -elif SUBMISSION_ID: - submissions_detail_url = urljoin(CODALAB_URL, f'/api/submissions/{SUBMISSION_ID}/') - resp = requests.get(submissions_detail_url, headers=headers) - if resp.status_code != 200: - print(f"Failed to get submission: {resp.content}") - exit(-3) - - print('-------------------') - print('Submission Object:') - print('-------------------\n') - pprint(resp.json()) - print() - - submissions_get_details_url = urljoin(CODALAB_URL, f'/api/submissions/{SUBMISSION_ID}/get_details/') - resp = requests.get(submissions_get_details_url, headers=headers) - if resp.status_code != 200: - print(f"Failed to get submission: {resp.content}") - exit(-3) - - print('-------------------------------') - print('Submission get_details Object:') - print('-------------------------------\n') - pprint(resp.json()) - print() - -else: - print('No command given. Here is some --help') - print_help() diff --git a/docs/example_scripts/rerun_submission.py b/docs/example_scripts/rerun_submission.py deleted file mode 100755 index 7d4642cd4..000000000 --- a/docs/example_scripts/rerun_submission.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python3 -""" -Overview -======== - This script is built to guide a user through the selection of both a submission id and a task id. - These ids are used to demonstrate the ability of a robot user to re-run a submission on a specific - task. This can be used to enable clinicians to run pre-built algorithms on private datasets that - are not attached to any competition. - - -Usage -===== - To re-run a submission on a task, this script can be run with arguments of a submission id and a - task id as shown below: - - ./rerun_submission.py - - The following instruction list will provide a detailed guide in selecting a submission and task id. - - -1. Create a competition that allows robots, and create a user marked as a robot - user. Use that username and password below. - - -2. Get into a python3 environment with requests installed - - -3. Review this script and edit the applicable variables, like... - - CODALAB_URL - USERNAME - PASSWORD - ... - - -4. Execute the contents of this script with no additional command line arguments with - the command shown below: - - ./rerun_submission.py - - The script is built to assist the user in the selection of the submission that will be re-run. - - -5. After selecting a submission id from the list shown in the previous step, add that id to - the command as a positional argument as shown below. - - ./rerun_submission.py 42 - - The script will assist the user in the selection of a task id. - - -5. After selecting a submission id and a task id, run the command again with both arguments to - see a demonstration of a robot user re-running a submission on a specific task. - - e.g. - - ./rerun_submission.py 42 a217a322-6ddf-400c-ac7d-336a42863724 -""" -# ---------------------------------------------------------------------------- -# Configure these -# ---------------------------------------------------------------------------- -CODALAB_URL = 'http://localhost/' -USERNAME = 'admin' -PASSWORD = 'admin' - - -# ---------------------------------------------------------------------------- -# Script start.. -# ---------------------------------------------------------------------------- -from urllib.parse import urljoin # noqa: E402 -from sys import argv, exit # noqa: E402,E261 # Ignore E261 to line up these noqa -import requests # noqa: E402,E261 # Ignore E261 to line up these noqa - - -SUBMISSION_ID = None -TASK_KEY = None -MODE = 0 - -if len(argv) > 1: - SUBMISSION_ID = int(argv[1]) - MODE = 1 -if len(argv) > 2: - TASK_KEY = argv[2] - MODE = 2 - - -# Login -login_url = urljoin(CODALAB_URL, '/api/api-token-auth/') -resp = requests.post(login_url, {"username": USERNAME, "password": PASSWORD}) -if resp.status_code != 200: - print(f"Failed to login: {resp.content}") - print('Is the url correct? (http vs https)') - exit(-1) - -# Setup auth headers for the rest of communication -token = resp.json()["token"] -headers = { - "Authorization": f"Token {token}" -} - -if MODE == 0: - submission_list_url = urljoin(CODALAB_URL, f'/api/submissions/') - resp = requests.get(submission_list_url, headers=headers) - if resp.status_code != 200: - print(f"Failed to get submissions: {resp.content}") - exit(-1) - submissions = resp.json() - message = """ -\n\nPlease select a submission id from the list to be used as a command line argument as such:\n - ./rerun_submission.py \n -It does not fundamentally matter which submission you choose.\n\n - """ - print(message) - print('------------------------- Submissions ----------------------') - print(f"{'id':^10}|{'owner':^20}|{'task id':^10}") - print('------------------------------------------------------------') - for sub in submissions: - task = sub['task'] - if task is not None: - task = task['id'] - print(f"{sub['id']:^10}|{sub['owner']:^20}|{str(task):^10}") - print(message) - exit(0) - -elif MODE == 1: - task_list_url = urljoin(CODALAB_URL, f'/api/tasks/') - resp = requests.get(task_list_url, headers=headers) - if resp.status_code != 200: - print(f"Failed to get tasks: {resp.content}") - exit(-1) - tasks = resp.json()['results'] - message = """ -\n\nPlease select a task key from the list to be used as a command line argument as such:\n - ./rerun_submission.py \n -It does not fundamentally matter which task you choose.\n\n - """ - print(message) - print('--------------------------------------- Tasks -------------------------------------------') - print(f"{'key':^50}|{'name':^30}") - print('-----------------------------------------------------------------------------------------') - for task in tasks: - name = task['name'][:20] - print(f"{task['key']:^50}|{name:^30}") - print(message) - exit(0) - -""" -Submit it to the competition without creating new dataset. -The submission data is retrieved from the original submission, -And the task data is retrieved from the task passed as a task_pk. -""" -rerun_submission_url = urljoin(CODALAB_URL, f'/api/submissions/{SUBMISSION_ID}/re_run_submission/') -submission_params = { - "task_key": TASK_KEY, -} - -print(f"Rerunning submission {SUBMISSION_ID} using data: {submission_params}") -resp = requests.post(rerun_submission_url, headers=headers, params=submission_params) - -NEW_SUBMISSION_ID = None -if resp.status_code in (200, 201): - print(f"Successfully submitted: {resp.content}") - NEW_SUBMISSION_ID = resp.json()['id'] -else: - print(f"Error submitting ({resp.status_code}): {resp.content}") - print("Are you sure this user has `is_bot` checked in the Django Admin?") - exit(1) - -# Get information of new submission -submission_detail_url = urljoin(CODALAB_URL, f'/api/submissions/{NEW_SUBMISSION_ID}/') -resp = requests.get(submission_detail_url, headers=headers) - -if resp.status_code in (200, 201): - NEW_SUBMISSION_TASK_KEY = resp.json()['task']['key'] - print(f'\n\nOld submission pk: {SUBMISSION_ID}') - print(f'New submission pk: {NEW_SUBMISSION_ID}') - print(f'\n\nThe new submission should have the same task as the one specified') - assert TASK_KEY == NEW_SUBMISSION_TASK_KEY - print(f'Task key specified: {TASK_KEY}') - print(f'New submission task key: {NEW_SUBMISSION_TASK_KEY}') -else: - print(f"Error retrieving details ({resp.status_code}): {resp.content}") - print("Are you sure this user has `is_bot` checked in the Django Admin?") diff --git a/docs/readme.md b/docs/readme.md deleted file mode 100755 index 5d987cd41..000000000 --- a/docs/readme.md +++ /dev/null @@ -1,57 +0,0 @@ -## GET READY - -- This new platform is an upgraded version of [Codalab](https://competitions.codalab.org/) allowing you to create either competitions or benchmarks -- This getting started tutorial shows you a simple example of how to create a competition (fancier examples can be found [here](https://github.com/codalab/competitions-v2/tree/codabench/sample_bundle/src/tests/functional/test_files/AutoWSL_sample); the full documentation is [here](https://github.com/codalab/competitions-v2/wiki)) -- Create a Codalab account (if not done yet) -- Download the [sample competition bundle](https://github.com/codalab/competitions-v2/tree/develop/docs/competition.zip) and a [sample submission](https://github.com/codalab/competitions-v2/tree/develop/docs/submission.zip) -- Do not unzip them. - -## WHAT'S EXPECTED AFTER THIS TUTORIAL - -- a working [competition/benchmark](https://www.codabench.org/competitions/214/) -- you can submit [submission.zip](https://github.com/codalab/competitions-v2/tree/develop/docs/submission.zip) for playing with -- we have enabled "Auto approve registration requests" so that anyone could join without approval - -## CREATE A COMPETITION - -- From the front page [https://www.codabench.org/](https://www.codabench.org/) top menu, go to the Benchmark > Management -- Click the green “Upload” button at the top right. -- Upload the sample competition bundle => this will create your competition. - -## MAKE A SUBMISSION - -- In your competition page, go to the tab “My submissions” -- Submit the sample submission bundle. -- When your submission finishes, go to the Result tab to check it shows up on the leaderboard. - -## MAKE CHANGES - -- Click on the Edit gray button at the top the enter the editor -- Change the logo -- Save your changes - -## Composition of the competition bundle - -### metadata file - -- competition.yaml: contains ALL the configurations of your benchmark/competition - -### datasets - -- input_data: contains datasets for training models and datasets for testing models (i.e. X_train, y_train, X_test) -- reference_data: answers to training Datasets (i.e. y_test) - -### core logic - -- ingestion_program: defines the logic of how to read user submissions, read your data, train the model and inferences outputs -- scoring_program: defines the logic of how to calculate user submissions score - -### static files - -- participate.md: markdown file about how to participate in this benchmark/competition -- terms.md: about the terms and conditions of the benchmark, this will show up when other participants join the benchmark/competition -- wheat.jpg: logo for this benchmark/competition - -You are done with this simple tutorial. - -Next, check the more [advanced tutorial](https://github.com/codalab/competitions-v2/tree/develop/docs/tutorial) \ No newline at end of file diff --git a/docs/submission.zip b/docs/submission.zip deleted file mode 100644 index 4863303ff..000000000 Binary files a/docs/submission.zip and /dev/null differ diff --git a/docs/tutorial/Codabench_quickstart_documentation.pdf b/docs/tutorial/Codabench_quickstart_documentation.pdf deleted file mode 100644 index 3141200b2..000000000 Binary files a/docs/tutorial/Codabench_quickstart_documentation.pdf and /dev/null differ