Merge pull request #120 from motional/nuplan-devkit-v0.6

Nuplan devkit v0.6

Author: patk-motional

.dockerignore

@@ -0,0 +1,9 @@
+tutorials
+.pytest_cache
+docs
+.git
+bazel-bin
+bazel-nuplan_devkit
+bazel-out
+bazel-testlogs
+.ipynb_checkpoints

.env

@@ -1,10 +1,13 @@
 # Common variables
-NUPLAN_TEST_MAPS_VERSION="nuplan-maps-v1.0"
-
+export NUPLAN_TEST_MAPS_VERSION="nuplan-maps-v1.0"
+export NUPLAN_DATA_STORE='local'
 # Simulation variables
-SIMULATION_CONTAINER_PORT=50050
-SIMULATION_IMAGE="nuplan/nuplan"
+export SIMULATION_CONTAINER_PORT=50050
+export SIMULATION_IMAGE="nuplan/nuplan"
 
 # Submission variables
-SUBMISSION_CONTAINER_PORT=50051
-SUBMISSION_IMAGE="test-contestant/nuplan-test-submission"
+export SUBMISSION_CONTAINER_PORT=50051
+export CONTESTANT_ID="test"
+export SUBMISSION_ID="contestant"
+
+export SUBMISSION_IMAGE="nuplan/test-submission"

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,6 +1,6 @@
 ---
 name: Bug report
-about: Create a report to help us improve
+about: Create a report to help us improve.
 title: ''
 labels: ''
 assignees: ''

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,6 +1,6 @@
 ---
 name: Feature request
-about: Suggest an idea for this project
+about: Suggest an idea for this project.
 title: ''
 labels: ''
 assignees: ''

README.md .github/README.mdREADME.md renamed to .github/README.md

@@ -30,3 +30,5 @@ _ext
 *.swp
 .tox/*
 .pytest_cache/*
+docs/_build/
+docs/generated/

.gitignore

@@ -16,6 +16,7 @@ RUN curl -fsSL https://bazel.build/bazel-release.pub.gpg | apt-key add - \
     zip \
     nvidia-container-toolkit \
     software-properties-common \
+    awscli \
     && rm -rf /var/lib/apt/lists/*
 
 # Download miniconda and install silently.
@@ -29,22 +30,49 @@ RUN curl -fsSLo Miniconda3-latest-Linux-x86_64.sh https://repo.anaconda.com/mini
 ARG NUPLAN_HOME=/nuplan_devkit
 WORKDIR $NUPLAN_HOME
 
-COPY requirements.txt requirements_torch.txt /nuplan_devkit/
-RUN bash -c "python -m pip install --upgrade pip --pre && \
-    pip install --no-cache-dir -r $NUPLAN_HOME/requirements_torch.txt -f https://download.pytorch.org/whl/torch_stable.html \
-    pip install --no-cache-dir -r $NUPLAN_HOME/requirements.txt -f https://download.pytorch.org/whl/torch_stable.html"
+COPY requirements.txt requirements_torch.txt environment.yml /nuplan_devkit/
+RUN conda env create -f $NUPLAN_HOME/environment.yml
 
 RUN mkdir -p $NUPLAN_HOME/nuplan
 
 COPY setup.py $NUPLAN_HOME
 COPY nuplan $NUPLAN_HOME/nuplan
 
+SHELL ["conda", "run", "-n", "nuplan", "/bin/bash", "-c"]
+
 RUN bash -c "pip install -e ."
 
 ENV NUPLAN_MAPS_ROOT=/data/sets/nuplan/maps \
     NUPLAN_DATA_ROOT=/data/sets/nuplan \
     NUPLAN_EXP_ROOT=/data/exp/nuplan
 
-RUN mkdir -p {$NUPLAN_MAPS_ROOT, $NUPLAN_DATA_ROOT, $NUPLAN_EXP_ROOT}
+RUN bash -c 'mkdir -p {$NUPLAN_MAPS_ROOT,$NUPLAN_DATA_ROOT,$NUPLAN_EXP_ROOT}'
+
+ARG NUPLAN_DATA_ROOT_S3_URL
+ARG NUPLAN_MAPS_ROOT_S3_URL
+ARG AWS_ACCESS_KEY_ID
+ARG AWS_SECRET_ACCESS_KEY
+ARG NUPLAN_SERVER_S3_ROOT_URL
+ARG NUPLAN_SERVER_AWS_ACCESS_KEY_ID
+ARG NUPLAN_SERVER_AWS_SECRET_ACCESS_KEY
+ARG S3_TOKEN_DIR
+ARG NUPLAN_DATA_STORE
+
+ENV NUPLAN_DATA_ROOT $NUPLAN_DATA_ROOT
+ENV NUPLAN_MAPS_ROOT $NUPLAN_MAPS_ROOT
+ENV NUPLAN_DB_FILES  /data/sets/nuplan/nuplan-v1.0/mini
+ENV NUPLAN_MAP_VERSION "nuplan-maps-v1.0"
+ENV NUPLAN_DATA_STORE $NUPLAN_DATA_STORE
+ENV NUPLAN_S3_PROFILE "default"
+ENV NUPLAN_DATA_ROOT_S3_URL $NUPLAN_DATA_ROOT_S3_URL
+ENV NUPLAN_MAPS_ROOT_S3_URL $NUPLAN_MAPS_ROOT_S3_URL
+ENV AWS_ACCESS_KEY_ID $AWS_ACCESS_KEY_ID
+ENV AWS_SECRET_ACCESS_KEY $AWS_SECRET_ACCESS_KEY
+ENV NUPLAN_SERVER_AWS_ACCESS_KEY_ID $NUPLAN_SERVER_AWS_ACCESS_KEY_ID
+ENV NUPLAN_SERVER_AWS_SECRET_ACCESS_KEY $NUPLAN_SERVER_AWS_SECRET_ACCESS_KEY
+ENV NUPLAN_SERVER_S3_ROOT_URL $NUPLAN_SERVER_S3_ROOT_URL
+ENV S3_TOKEN_DIR $S3_TOKEN_DIR
+
+RUN bash -c 'mkdir -p $NUPLAN_DB_FILES'
 
-CMD ["/bin/bash"]
+CMD ["/nuplan_devkit/nuplan/entrypoint_simulation.sh"]

Dockerfile

@@ -0,0 +1,9 @@
+FROM 300189864575.dkr.ecr.us-east-1.amazonaws.com/nuplan-evalservice-client
+
+ARG EVALAI_CHALLENGE_PK
+ARG EVALAI_PERSONAL_AUTH_TOKEN
+
+ENV EVALAI_CHALLENGE_PK $EVALAI_CHALLENGE_PK
+ENV EVALAI_PERSONAL_AUTH_TOKEN $EVALAI_PERSONAL_AUTH_TOKEN
+
+CMD ["/nuplan_devkit/nuplan/entrypoint_result_processor.sh"]

Dockerfile.result_processor

@@ -29,21 +29,27 @@ RUN curl -fsSLo Miniconda3-latest-Linux-x86_64.sh https://repo.anaconda.com/mini
 ARG NUPLAN_HOME=/nuplan_devkit
 WORKDIR $NUPLAN_HOME
 
-COPY requirements.txt requirements_submission.txt $NUPLAN_HOME/
+COPY requirements.txt requirements_torch.txt requirements_submission.txt $NUPLAN_HOME/
 
 RUN bash -c "python -m pip install --upgrade pip --pre && \
+    pip install --no-cache-dir -r $NUPLAN_HOME/requirements_torch.txt -f https://download.pytorch.org/whl/torch_stable.html && \
     pip install --no-cache-dir -r $NUPLAN_HOME/requirements.txt -f https://download.pytorch.org/whl/torch_stable.html && \
     pip install --no-cache-dir -r $NUPLAN_HOME/requirements_submission.txt"
 
 RUN mkdir -p $NUPLAN_HOME/nuplan
 
 COPY setup.py $NUPLAN_HOME
 COPY nuplan $NUPLAN_HOME/nuplan
+# You can copy your own files with the following command.
+# $NUPLAN_HOME Here represents the absolute path /nuplan_devkit inside the container
+# COPY my_fabulous_model.ckpt $NUPLAN_HOME
 
 RUN bash -c "pip install -e ."
 
 ENV NUPLAN_MAPS_ROOT=/data/sets/nuplan/maps \
     NUPLAN_DATA_ROOT=/data/sets/nuplan \
     NUPLAN_EXP_ROOT=/data/exp/nuplan
 
-CMD ["/bin/bash", "-c", "python nuplan/submission/submission_planner.py"]
+RUN bash -c 'mkdir -p {$NUPLAN_MAPS_ROOT,$NUPLAN_DATA_ROOT,$NUPLAN_EXP_ROOT}'
+
+CMD ["/nuplan_devkit/nuplan/entrypoint_submission.sh"]

Dockerfile.submission

@@ -3,31 +3,47 @@ version: "3.7"
 x-nuplan-volumes: &x-nuplan-volumes
   volumes:
     - "$NUPLAN_DATA_ROOT:/data/sets/nuplan:rw"
+    - "$NUPLAN_EXP_ROOT:/data/exp/nuplan:rw"
+
+
+x-nuplan-maps: &x-nuplan-maps
+  volumes:
+    - "$NUPLAN_MAPS_ROOT:/data/sets/nuplan/maps:rw"
 
 
 services:
   submission:
-    <<: *x-nuplan-volumes
-    container_name: "test-contestant_nuplan-test-submission"
+    <<: *x-nuplan-maps
     image: ${SUBMISSION_IMAGE}
+    build:
+      context: ./
+      dockerfile: Dockerfile.submission
     env_file:
       - .env
-    ports:
-      - ${SUBMISSION_CONTAINER_PORT}:${SUBMISSION_CONTAINER_PORT}
-
+    network_mode: "host"
     tty: true
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: ["gpu"]
+    entrypoint: /nuplan_devkit/nuplan/entrypoint_submission.sh
 
   simulation:
     <<: *x-nuplan-volumes
+    image: ${SIMULATION_IMAGE}
+    build:
+      context: ./
+      dockerfile: Dockerfile
     env_file:
       - .env
     environment:
       'NUPLAN_CHALLENGE': $NUPLAN_CHALLENGE
       'NUPLAN_PLANNER': $NUPLAN_PLANNER
-    image: ${SIMULATION_IMAGE}
-    ports:
-      - ${SIMULATION_CONTAINER_PORT}:${SIMULATION_CONTAINER_PORT}
-    tty: true
+
+    network_mode: "host"
     depends_on:
       - submission
-    entrypoint: /nuplan_devkit/nuplan/entrypoint.sh
+    entrypoint: /nuplan_devkit/nuplan/entrypoint_simulation.sh