[Work In Progress / Do Not Merge] Docker machine support

README.md

@@ -54,6 +54,10 @@
         cd acronym && pip install -e .
         ```
 
+## Docker Setup
+
+Refer to [docker documenation](./docker/README.md) for details.
+
 ## Folder structure
 ```bash
 NGDF

docker/README.md

@@ -0,0 +1,31 @@
+# Introduction
+
+This folder contains a mechanism to launch `NGDF` using docker as well as a development 
+environment for it, which includes VSCode, ROS, Open3D and nVidia support among others.
+
+## Installation
+
+For the docker machine to work it is necesary to install `docker` and `nVidia docker`.
+
+If you are using `Ubuntu`, you can install the requirements running:
+```bash
+./bin/install-docker.sh
+```
+
+Then, build the docker images running:
+```bash
+./bin/build.sh
+```
+
+Finally, copy the `HOME` folder of the docker machine into a local
+folder so that you can develop using the docker machine in a way that changes are persisted.
+```bash
+./bin/install.sh
+```
+
+## Launch
+
+After the installation is sucessful, run:
+```bash
+./bin/launch.py
+```

docker/bin/build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+docker build -t ros-base:v1.0 --build-arg BUILD_USER_ID=$(id -u) --build-arg BUILD_GROUP_ID=$(id -g) dockerfiles/base
+docker build --build-arg SSH_PRIVATE_KEY="$(cat $HOME/.ssh/id_rsa)" --build-arg SSH_PUBLIC_KEY="$(cat $HOME/.ssh/id_rsa.pub)" -t ros-desktop:v1.0 dockerfiles/desktop

docker/bin/install-docker.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+sudo apt-get uninstall -y docker docker-engine docker.io containerd runc nvidia-docker2
+sudo apt-get install -y \
+    apt-transport-https \
+    ca-certificates \
+    curl \
+    gnupg \
+    lsb-release
+
+sudo groupadd docker
+sudo usermod -aG docker $USER
+newgrp docker 
+pip3 install click
+
+distribution=$(. /etc/os-release;echo $ID$VERSION_ID) \
+      && curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
+      && curl -s -L https://nvidia.github.io/libnvidia-container/$distribution/libnvidia-container.list | \
+            sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
+            sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
+
+sudo apt-get update
+sudo apt-get install -y nvidia-docker2
+sudo systemctl restart docker
+# THIS IS NECESSARY AS OTHERWISE NVIDIA REPOS TAKE OVER UBUNTU-ONES AND THE VERSIONING GETS BROKEN
+sudo rm -rf /etc/apt/sources.list.d/nvidia-*

docker/bin/install.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+./bin/run-desktop.sh dt
+sleep 5
+container_id=$(docker ps -aqf "name=ros-desktop" | tr -d '\n')
+echo "Docker container $container_id"
+destination_folder="$HOME/docker-dir"
+mkdir -p $destination_folder/home/ros
+mkdir -p $destination_folder/home/ros/.ssh
+docker cp $container_id:/home/ros $destination_folder/home
+cp $HOME/.ssh/* $destination_folder/home/ros/.ssh/
+docker stop -t 1 $container_id
+pip3 install click

docker/bin/launch.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+import sys
+import os 
+import click
+import subprocess
+
+@click.command()
+@click.option("--data-dir", default=lambda: f"{os.environ.get('HOME', '')}/docker-dir", help="Data directory")
+@click.option("--desktop", default=False, help="Desktop mode", type=bool)
+@click.option("--gpu", default=True, help="GPU Support", type=bool)
+def main(data_dir, desktop, gpu):
+  entrypoint = "" if desktop else "--entrypoint /bin/bash"
+  gpu_support = "-e NVIDIA_DRIVER_CAPABILITIES=all --gpus all \\" if gpu else "\\"
+  commands = [
+  "xhost +",
+  "export LIBGL_ALWAYS_INDIRECT=1",
+  f"""
+  docker container run --rm -it \
+    --cap-add=SYS_PTRACE \
+    --security-opt=seccomp:unconfined \
+    --security-opt=apparmor:unconfined \
+    --privileged {gpu_support}
+    --device-cgroup-rule "c 81:* rmw" \
+    --device-cgroup-rule "c 189:* rmw" \
+    --volume=/dev:/dev \
+    --device=/dev:/dev \
+    --user $(id -u) {entrypoint} \
+    --workdir /home/ros \
+    --mount type=bind,source="{data_dir}/home/ros",target=/home/ros \
+    --name ros-desktop-nvidia \
+    --security-opt apparmor:unconfined \
+    --net=host \
+    --env="DISPLAY" \
+    --volume="$HOME/.Xauthority:/home/ros/.Xauthority:rw" \
+    ros-desktop:v1.0
+    """
+    ]
+  for command in commands:
+    results = subprocess.run(command, shell=True, text=True)
+
+if __name__ == "__main__":
+  main()

docker/bin/stop.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+container_id=$(docker ps -aqf "name=ros-desktop" | tr -d '\n')
+echo "$container_id"
+docker stop $container_id
+

docker/dockerfiles/base/Dockerfile

@@ -0,0 +1,108 @@
+##
+## How do I select the base image?
+## Check the host nvidia version by running: nvcc --version
+## That will give you your host's cuda version.
+## Then try to find one that matches from the release notes
+## of nVidia's images: https://docs.nvidia.com/deeplearning/frameworks/tensorflow-release-notes/rel_21-12.html#rel_21-12
+## Finally, the tag in: https://catalog.ngc.nvidia.com/orgs/nvidia/containers/l4t-tensorflow/tags
+## You will have to also change the installation of pytorch so that the cuda version also fits
+##
+## For the current version, the host, the docker image and pytorch are CUDA 11.5
+##
+## First you can try the base by using: docker run --runtime=nvidia --rm nvcr.io/nvidia/tensorflow:21.12-tf2-py3 /bin/bash
+##
+FROM nvcr.io/nvidia/tensorflow:21.12-tf2-py3
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN echo 'Etc/UTC' > /etc/timezone && \
+    apt-get update && \
+    apt-get install -q -y --no-install-recommends tzdata && \
+    apt-get autoclean
+
+RUN apt-get update && apt-get install -q -y --no-install-recommends \
+    dirmngr \
+    gnupg2 && \
+    apt-get autoclean
+
+##
+## What's at the end of this line "cu115" should be consistent with both:
+##  (1) The cuda version in the host that shall be checked with "nvcc --version"
+##  (2) The cuda version in the base image which you can check in 
+##      https://docs.nvidia.com/deeplearning/frameworks/tensorflow-release-notes/rel_21-12.html#rel_21-12
+RUN pip install torch==1.13.0 torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113
+
+RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654
+
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
+ARG BUILD_GROUP_ID
+ARG BUILD_USER_ID
+
+# Create 'ros' user with sudo powers
+ARG USERNAME=ros
+RUN groupadd --gid $BUILD_GROUP_ID $USERNAME && \
+    useradd -s /bin/bash --uid $BUILD_USER_ID --gid $BUILD_GROUP_ID -m $USERNAME && \
+    mkdir -p /etc/sudoers.d && \
+    echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME && \
+    chmod 0440 /etc/sudoers.d/$USERNAME && \
+    echo "source /usr/share/bash-completion/completions/git" >> /home/$USERNAME/.bashrc
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    git vim tree net-tools \
+    python3-pip \
+    apt-file \
+    graphicsmagick-libmagick-dev-compat \
+    libceres-dev \
+    libzmq3-dev \
+    libboost-dev \
+    jq \
+    wget \
+    python3-tk \
+    curl \
+    libssl-dev \
+    libusb-1.0-0-dev \
+    usb-1.0-0 \
+    libudev-dev \
+    udev \
+    pkg-config  \
+    libgtk-3-dev \
+    freeglut3 \
+    freeglut3-dev \
+    libglfw3-dev \
+    libgl1-mesa-dev \
+    libglu1-mesa-dev \
+    fswebcam \
+    xdg-utils && \
+    apt-get autoclean
+
+RUN apt-get update && \
+    apt install -y lsb-release  && \
+    sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list'  && \
+    curl -s https://raw.githubusercontent.com/ros/rosdistro/master/ros.asc | apt-key add - && \
+    apt-get autoclean
+
+RUN apt-get update && \
+    apt install -y ros-noetic-desktop-full python3-rosdep python3-rosinstall python3-rosinstall-generator python3-wstool build-essential && \
+    apt-get autoclean
+
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    python3 get-pip.py --force-reinstall
+
+USER $USERNAME
+WORKDIR /home/$USERNAME
+
+RUN sudo mkdir /etc/ros && \
+    sudo chown -R ros:ros /etc/ros && \
+    sudo chown -R ros:ros /usr/local/lib/python3.8/dist-packages
+
+RUN wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > packages.microsoft.gpg && \
+  sudo install -o root -g root -m 644 packages.microsoft.gpg /etc/apt/trusted.gpg.d/ && \
+  sudo sh -c 'echo "deb [arch=amd64,arm64,armhf signed-by=/etc/apt/trusted.gpg.d/packages.microsoft.gpg] https://packages.microsoft.com/repos/code stable main" > /etc/apt/sources.list.d/vscode.list' && \
+  rm -f packages.microsoft.gpg
+
+RUN sudo apt update && \
+    sudo apt install --no-install-recommends -y apt-transport-https  code ninja-build && \
+    sudo apt-get autoclean