Merge pull request #139 from motional/nuplan-devkit-v1.0

nuplan-devkit-v1.0

Author: patk-motional

.bazelrc

@@ -1,23 +1,33 @@
+# NOTE:
+# If you are adding something under `test`, check if it exists as a command under `build` instead:
+#    https://bazel.build/reference/command-line-reference
+# If it does, add it to `build` as all test options inherit from `build`. Same for `run`.
+# This prevents invalidation of the analysis cache when switching between bazel build/test/run. Example warning:
+#    INFO: Build options --action_env and --test_env have changed, discarding analysis cache.
 build --remote_cache=http://bazel-cache.ci.motional.com:80
 build --remote_upload_local_results=false
+build --sandbox_tmpfs_path=/dev/shm
+build --incompatible_strict_action_env
+
+build --action_env=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
+build --action_env=CC=gcc # force Cython to use gcc
+build --action_env=CACHE=/tmp/pyxbld
+
+# make sure you have defined all environment veriables listed below
+build --action_env=AWS_ACCESS_KEY_ID
+build --action_env=AWS_SECRET_ACCESS_KEY
+build --action_env=NUPLAN_DATA_ROOT
+build --action_env=NUPLAN_MAPS_ROOT
+build --action_env=NUPLAN_EXP_ROOT
+build --action_env=NUPLAN_DATA_STORE
+build --action_env=NUPLAN_S3_PROFILE
+build --action_env=NUPLAN_CACHE_FROM_S3
+build --action_env=NUPLAN_DATA_ROOT_S3_URL
+build --action_env=NUPLAN_MAPS_ROOT_S3_URL
+build --action_env=NUPLAN_DB_FILES
+build --action_env=NUPLAN_MAP_VERSION
+build --action_env=NUPLAN_HYDRA_CONFIG_PATH
 
 test --test_output=errors
 test --test_timeout=2400
-test --test_output=all
-test --test_env=WORKSPACE
-test --test_env=CACHE=/tmp/pyxbld
-test --test_env=PYTHONPATH
-test --test_env=AWS_ACCESS_KEY_ID
-test --test_env=AWS_SECRET_ACCESS_KEY
-
-test --test_env=NUPLAN_DATA_ROOT
-test --test_env=NUPLAN_MAPS_ROOT
-test --test_env=NUPLAN_EXP_ROOT
-test --test_env=NUPLAN_DATA_STORE
-test --test_env=NUPLAN_S3_PROFILE
-test --test_env=NUPLAN_CACHE_FROM_S3
-test --test_env=NUPLAN_DATA_ROOT_S3_URL
-test --test_env=NUPLAN_MAPS_ROOT_S3_URL
-test --test_env=NUPLAN_DB_FILES
-test --test_env=NUPLAN_MAP_VERSION
-test --test_env=NUPLAN_HYDRA_CONFIG_PATH
+test --incompatible_exclusive_test_sandboxed=true

.gitignore

@@ -19,7 +19,6 @@ _ext
 *.o
 *.out
 *.pdf
-*.png
 .project
 *.pth.tar
 *.pyc

BUILD

@@ -1,7 +1,31 @@
 load("@com_github_bazelbuild_buildtools//buildifier:def.bzl", "buildifier")
+load("@rules_python//python:pip.bzl", "compile_pip_requirements")
+load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
 
 package(default_visibility = ["//visibility:public"])
 
+buildifier(
+    name = "buildifier_nuplan",
+    lint_mode = "fix",
+    lint_warnings = ["+native-py"],
+)
+
+string_flag(
+    name = "ubuntu_distro",
+    build_setting_default = "focal",
+    values = [
+        "bionic",
+        "focal",
+    ],
+)
+
+config_setting(
+    name = "focal",
+    flag_values = {
+        ":ubuntu_distro": "focal",
+    },
+)
+
 filegroup(
     name = "requirements",
     data = [
@@ -16,3 +40,34 @@ buildifier(
     lint_warnings = ["+native-py"],
     mode = "check",
 )
+
+# See the 'pip_deps' pip_parse() in WORKSPACE for further details
+# Validate: bazel test <targetname>_update
+# Generate: bazel run <targetname>.update
+compile_pip_requirements(
+    name = "pip_nuplan_devkit_deps",
+    timeout = "moderate",  # Increase timeout for underlying py_tests
+    extra_args = [
+        "--allow-unsafe",
+        "--index-url=https://pypi.org/simple",
+    ],
+    requirements_in = "requirements.txt",
+    requirements_txt = "requirements_lock.txt",
+    tags = [
+        "nuplan_devkit_local",
+    ],
+)
+
+compile_pip_requirements(
+    name = "pip_deps_torch",
+    timeout = "moderate",  # Increase timeout for underlying py_tests
+    extra_args = [
+        "--allow-unsafe",
+        "--index-url=https://pypi.org/simple",
+    ],
+    requirements_in = "requirements_torch.txt",
+    requirements_txt = "requirements_torch_lock.txt",
+    tags = [
+        "nuplan_devkit_local",
+    ],
+)

Dockerfile

@@ -60,7 +60,7 @@ ARG NUPLAN_DATA_STORE
 
 ENV NUPLAN_DATA_ROOT $NUPLAN_DATA_ROOT
 ENV NUPLAN_MAPS_ROOT $NUPLAN_MAPS_ROOT
-ENV NUPLAN_DB_FILES  /data/sets/nuplan/nuplan-v1.0/mini
+ENV NUPLAN_DB_FILES  /data/sets/nuplan/nuplan-v1.1/mini
 ENV NUPLAN_MAP_VERSION "nuplan-maps-v1.0"
 ENV NUPLAN_DATA_STORE $NUPLAN_DATA_STORE
 ENV NUPLAN_S3_PROFILE "default"

Dockerfile.result_processor

@@ -2,8 +2,14 @@ FROM 300189864575.dkr.ecr.us-east-1.amazonaws.com/nuplan-evalservice-client
 
 ARG EVALAI_CHALLENGE_PK
 ARG EVALAI_PERSONAL_AUTH_TOKEN
+ARG NUSCENES_SERVER_AWS_ACCESS_KEY_ID
+ARG NUSCENES_SERVER_AWS_SECRET_ACCESS_KEY
+ARG NUSCENES_SERVER_S3_ROOT_URL
 
 ENV EVALAI_CHALLENGE_PK $EVALAI_CHALLENGE_PK
 ENV EVALAI_PERSONAL_AUTH_TOKEN $EVALAI_PERSONAL_AUTH_TOKEN
+ENV NUSCENES_SERVER_AWS_ACCESS_KEY_ID $NUSCENES_SERVER_AWS_ACCESS_KEY_ID
+ENV NUSCENES_SERVER_AWS_SECRET_ACCESS_KEY $NUSCENES_SERVER_AWS_SECRET_ACCESS_KEY
+ENV NUSCENES_SERVER_S3_ROOT_URL $NUSCENES_SERVER_S3_ROOT_URL
 
 CMD ["/nuplan_devkit/nuplan/entrypoint_result_processor.sh"]

Jenkinsfile

@@ -40,7 +40,7 @@ pipeline{
     BAZEL_OPTS = "--local_cpu_resources=8 --jobs=8 --remote_cache=http://bazel-cache.ci.motional.com:80 --remote_upload_local_results=true"
 
     NUPLAN_DATA_ROOT         = "/data/sets/nuplan"
-    NUPLAN_DB_FILES          = "/data/sets/nuplan/nuplan-v1.0/mini"
+    NUPLAN_DB_FILES          = "/data/sets/nuplan/nuplan-v1.1/mini"
     NUPLAN_MAPS_ROOT         = "/data/sets/nuplan/maps"
     NUPLAN_MAP_VERSION       = "nuplan-maps-v1.0"
     NUPLAN_EXP_ROOT          = "/tmp/exp/nuplan"

WORKSPACE

@@ -4,17 +4,27 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 http_archive(
     name = "rules_python",
-    sha256 = "895fa3b03898d7708eb50ed34dcfb71c07866433df6912a6ff4f4fb473048f99",
-    strip_prefix = "rules_python-2b1d6beb4d5d8f59d629597e30e9aa519182d9a9",
+    sha256 = "9fcf91dbcc31fde6d1edb15f117246d912c33c36f44cf681976bd886538deba6",
+    strip_prefix = "rules_python-0.8.0",
     urls = [
-        "https://github.com/bazelbuild/rules_python/archive/2b1d6beb4d5d8f59d629597e30e9aa519182d9a9.tar.gz",
+        "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.8.0.tar.gz",
     ],
 )
 
 load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies")
 
 pip_install_dependencies()
 
+load("@rules_python//python:repositories.bzl", "python_register_toolchains")
+
+python_register_toolchains(
+    name = "python3_9",
+    python_version = "3.9",
+)
+
+load("@python3_9//:defs.bzl", PYTHON_INTERPRETER_TARGET = "interpreter")
+load("@rules_python//python:pip.bzl", "pip_parse")
+
 http_archive(
     name = "com_github_bazelbuild_buildtools",
     sha256 = "932160d5694e688cb7a05ac38efba4b9a90470c75f39716d85fb1d2f95eec96d",
@@ -61,3 +71,69 @@ http_archive(
 load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
 
 protobuf_deps()
+
+# pip_parse() is used instead of pip_install() so that we can move pip installation to the build stage
+# instead of prior to the Bazel analysis phase (which is the case for pip_install()). This change will
+# help to speed up the overall build time and only download the dependencies as and when needed.
+#
+# pip_parse() also requires us to specify a fully resolved lock file for our Python dependencies. The
+# advantage to this is that it enforces fully deterministic builds. Without fully locked dependencies,
+# we cannot guarantee that two builds on the same commit use the same package versions.
+#
+# See:
+# - https://github.com/bazelbuild/rules_python#fetch-pip-dependencies-lazily
+# - https://github.com/bazelbuild/rules_python/blob/main/docs/pip.md#pip_parse
+# - https://github.com/bazelbuild/rules_python/blob/main/docs/pip.md#compile_pip_requirements
+#
+# The helper compile_pip_requirements() is used for regenerating the locked requirements.txt files.
+# Steps:
+# 1) Add/remove/modify package in requirements.txt.
+# 2) Validate locked requirements.txt can be generated:
+#     bazel test //path/to/package_deps:package_deps_test
+# 3) Update requirements_lock.txt:
+#     bazel run //path/to/package_deps.update
+# 4) Commit updated requirements_lock.txt
+
+PIP_INSTALL_TIMEOUT_SECONDS = 3600  # 60 minutes
+
+# "--only-binary=:all:" parameter below is to enforce pip to use prebuilt whl only,
+# to reduce external package preparation time.
+# For all packages with missing prebuilt binary, wheels are created manually and uploaded to Artifactory.
+# All such packages have `+av` suffix, just in case.
+# The project https://github.com/pypa/manylinux has been used - `quay.io/pypa/manylinux_2_24_x86_64` image
+#
+# To create and upload package, using `quay.io/pypa/manylinux_2_24_x86_64` docker image:
+# - download tarball from https://pypi.org/project/
+# - update `setup.py` and add `+av` suffix to version variable
+# - run `python setup.py sdist bdist_wheel upload -r nutonomypip`
+# Please refer to https://packaging.python.org/en/latest/tutorials/packaging-projects/ for further details
+
+PIP_EXTRA_ARGS = [
+    "--require-hashes",
+    "--index-url=https://pypi.org/simple",
+]
+
+# Base Python pip dependencies
+pip_parse(
+    name = "pip_nuplan_devkit_deps",
+    timeout = PIP_INSTALL_TIMEOUT_SECONDS,
+    extra_pip_args = PIP_EXTRA_ARGS,
+    python_interpreter_target = PYTHON_INTERPRETER_TARGET,
+    requirements_lock = "//:requirements_lock.txt",
+)
+
+load("@pip_nuplan_devkit_deps//:requirements.bzl", install_pip_deps = "install_deps")
+
+install_pip_deps()
+
+pip_parse(
+    name = "pip_torch_deps",
+    timeout = PIP_INSTALL_TIMEOUT_SECONDS,
+    extra_pip_args = PIP_EXTRA_ARGS,
+    python_interpreter_target = PYTHON_INTERPRETER_TARGET,
+    requirements_lock = "//:requirements_torch_lock.txt",
+)
+
+load("@pip_torch_deps//:requirements.bzl", install_pip_torch_deps = "install_deps")
+
+install_pip_torch_deps()

docs/README.md

@@ -10,9 +10,11 @@ ______________________________________________________________________
   <a href="https://www.nuscenes.org/nuplan#download">Download</a> •
   <a href="#citation">Citation</a><br>
   <a href="#changelog">Changelog</a> •
-  <a href="#devkit-structure">Structure</a><br>
-  <a href="https://github.com/motional/nuplan-devkit/blob/master/docs/installation.md">Setup</a> •
-  <a href="https://github.com/motional/nuplan-devkit/blob/master/tutorials/nuplan_framework.ipynb">Tutorial</a>
+  <a href="#devkit-structure">Structure</a> •
+  <a href="https://github.com/motional/nuplan-devkit/blob/master/docs/installation.md">Setup</a> <br>
+  <a href="https://github.com/motional/nuplan-devkit/blob/master/tutorials/nuplan_framework.ipynb">Tutorial</a> •
+  <a href="https://nuplan-devkit.readthedocs.io/en/latest/">Documentation</a> •
+  <a href="https://eval.ai/web/challenges/challenge-page/1856/overview">Competition</a>
 </p>
 
 [![python](https://img.shields.io/badge/python-%20%203.9-blue.svg)]()
@@ -28,13 +30,17 @@ ______________________________________________________________________
 </div>
 
 ______________________________________________________________________
-## nuPlan Planning Challenge is live!
-The nuPlan Planning Challenge is now live! The warm-up stage has begun. Please check our competition landing [page](https://nuplan-devkit.readthedocs.io/en/latest/).
+## Planning challenges
+#### The nuPlan Planning Competition has been postponed to January 2023.
+#### The nuPlan Dataset v1.1 has been released. Please download the latest dataset from the nuPlan [page](https://www.nuscenes.org/nuplan#download).
 ______________________________________________________________________
 
 ## Changelog
+- Oct 13th 2022
+  * v1.1 Dataset: Full nuPlan dataset - improved route plan, traffic light status, mission goal and more!
+  * v1.0 Devkit: Update to nuplan-v1.1 dataset, metrics improvements, route plan fixes, documentation, IDMPlanner
 - Sep 09 2022
-  * v0.6 Devkit: COMPETITION WARM-UP PHASE RELEASE - smart agents optimizations, nuBoard improvements, metrics improvements, submission pipeline deployment and documentation.
+  * v0.6 Devkit: Smart agents optimizations, nuBoard improvements, metrics improvements, submission pipeline deployment and documentation.
 - Aug 26 2022
   * v0.5 Devkit: New map features, simulation improvements, open-loop detections with smart agents, iLQR tracker, metrics improvements and documentation.
 - Aug 05 2022
@@ -54,9 +60,9 @@ ______________________________________________________________________
 ______________________________________________________________________
 
 ## Devkit and dataset setup
-Please refer to the [installation page](https://github.com/motional/nuplan-devkit/blob/master/docs/installation.md) for detailed instructions on how to setup the devkit.
+Please refer to the [installation page](https://nuplan-devkit.readthedocs.io/en/latest/installation.html) for detailed instructions on how to setup the devkit.
 
-Please refer to the [dataset page](https://github.com/motional/nuplan-devkit/blob/master/docs/dataset_setup.md) for detailed instructions on how to download and setup the dataset.
+Please refer to the [dataset page](https://nuplan-devkit.readthedocs.io/en/latest/dataset_setup.html) for detailed instructions on how to download and setup the dataset.
 
 ______________________________________________________________________
 
@@ -83,6 +89,11 @@ Please follow these steps to make yourself familiar with the nuPlan dataset:
 
 ______________________________________________________________________
 
+## Performance tuning guide
+Training configurations are important to ensure your expected system performance, for example preprocessing cost, training speed, and numerical stability. If you encounter problems related to aforementioned aspects, please refer to [performance tuning guide](https://github.com/motional/nuplan-devkit/blob/master/docs/performance_tuning_guide.md) to find potential solutions.
+
+______________________________________________________________________
+
 ## Devkit structure
 Our code is organized in these directories:
 
@@ -93,12 +104,14 @@ nuplan_devkit
 ├── nuplan          - The main source folder.
 │   ├── common      - Code shared by "database" and "planning".
 │   ├── database    - The core devkit used to load and render nuPlan dataset and maps.
-│   └── planning    - The stand-alone planning framework for simulation, training and evaluation.
+│   ├── planning    - The stand-alone planning framework for simulation, training and evaluation.
+│   ├── submission  - The submission engine used for the planning challenge.
 │   └── cli         - Command line interface tools for the nuPlan database.
 └── tutorials       - Interactive tutorials, see "Getting started".
 ```
 ______________________________________________________________________
 
+
 ## Citation
 Please use the following citation when referencing [nuPlan](https://arxiv.org/abs/2106.11810):
 ```

docs/baselines.md

@@ -0,0 +1,30 @@
+# Baselines
+
+We provide several baselines within the devkit. These baselines are standard comparison points in which to compare new 
+planners. Moreover, the baselines serve as a starting point for users to prototype their planner or simply tinker with it. 
+
+## SimplePlanner
+The SimplePlanner, as the name suggests, has little planning capability. The planner plans a straight line at a constant
+speed. The only logic of this planner is to decelerate if the current velocity exceeds the `max_velocity`.
+
+Link to the [code](https://github.com/motional/nuplan-devkit/blob/master/nuplan/planning/simulation/planner/simple_planner.py)
+
+## IDMPlanner
+The Intelligent Driver Model Planner (IDMPlanner) consists of two parts:
+
+1. Path planning
+2. Longitudinal control (IDM policy)
+
+### Path planning
+The path planning is a breadth-first search algorithm. It finds a path towards the mission goal.
+The path consists of a serie of lane and lane connectors that leads to the roadblock containing the mission goal.
+The baseline is then extracted from the found path and is used as the reference path for the planner.
+
+### IDM Policy
+Now that the planner has a reference path, it must then decide how fast to go along this path. For this, it follows
+the [IDM policy](https://en.wikipedia.org/wiki/Intelligent_driver_model). The policy describes how fast the planner should
+go based on the distance between itself and a given agent. Of course, it is wise to choose the closest agent in the path of the planner.
+
+Hence, the IDMPlanner uses breadth-first search to find the path towards the mission goal, and the IDM policy describes how far along that path the planner should be.
+
+Link to the [code](https://github.com/motional/nuplan-devkit/blob/master/nuplan/planning/simulation/planner/idm_planner.py)