[WIP] Running BOSH and CF with Docker

bosh-lite-docker.yml

@@ -20,20 +20,6 @@
     sha1: 78d79f08ff5001cc2a24f572837c7a9c59a0e796
     url: https://bosh.io/d/github.com/cloudfoundry/os-conf-release?v=18
     version: 18
-- path: /instance_groups/name=bosh/jobs/-
-  type: replace
-  value:
-    name: docker_cpi
-    properties:
-      docker_cpi:
-        agent:
-          blobstore: null
-          mbus: nats://nats:((nats_password))@10.254.50.4:4222
-        docker:
-          host: tcp://127.0.0.1:4243
-          tls: ((docker_client_tls))
-        host_ip: 10.254.50.4
-    release: bosh-docker-cpi
 - path: /instance_groups/name=bosh/jobs/-
   type: replace
   value:
@@ -82,7 +68,7 @@
     type: certificate
 - path: /variables/name=nats_server_tls/options/alternative_names?/-
   type: replace
-  value: 10.254.50.4
+  value: ((static_ip))
 - path: /variables/name=blobstore_server_tls/options/alternative_names/-
   type: replace
-  value: 10.254.50.4
+  value: ((static_ip))

bosh.yml

@@ -102,7 +102,7 @@ instance_groups:
         user: hm
       resurrector_enabled: true
     nats:
-      address: ((internal_ip))
+      address: ((static_ip))
       director_account:
         ca_cert: ((director_ssl.ca))
         password: ((nats_sync_password))
@@ -140,7 +140,7 @@ networks:
     gateway: ((internal_gw))
     range: ((internal_cidr))
     static:
-    - ((internal_ip))
+    - ((static_ip))
   type: manual
 releases:
 - name: bosh

docker/bosh-lite.yml

@@ -1,6 +1,3 @@
-- type: replace
-  path: /instance_groups/name=bosh/jobs/name=garden/properties/garden/apparmor_profile?
-  value: "" # overwrites garden-default which is the default
 - path: /instance_groups/name=bosh/properties/agent/env/bosh/blobstores/provider=dav/options/endpoint
   type: replace
-  value: https://10.254.50.4:25250
+  value: https://((static_ip)):25250

docker/cpi.yml

@@ -1,11 +1,3 @@
-- name: cpi
-  path: /releases/-
-  type: replace
-  value:
-    name: bosh-docker-cpi
-    sha1: b6f492596b89dd34d110785be01b52d19cd27a22
-    url: https://bosh.io/d/github.com/cloudfoundry/bosh-docker-cpi-release?v=0.0.15
-    version: 0.0.15
 - name: stemcell
   path: /resource_pools/name=vms/stemcell?
   type: replace
@@ -26,6 +18,7 @@
     - 4222/tcp
     - 25250/tcp
     - 8443/tcp
+    - 8844/tcp
     - 8080/tcp
 - path: /instance_groups/name=bosh/jobs/-
   type: replace
@@ -45,7 +38,7 @@
   value:
     agent:
       blobstore: null
-      mbus: nats://((internal_ip)):4222
+      mbus: nats://((static_ip)):4222
     docker:
       host: ((docker_host))
       tls: ((docker_tls))

docker/create-env.sh

@@ -6,6 +6,8 @@ STEP() { echo ; echo ; echo "==\\" ; echo "===>" "$@" ; echo "==/" ; echo ; }
 
 bosh_deployment="$(cd "$(dirname "${BASH_SOURCE[0]}")"; cd ..; pwd)"
 bosh_deployment_sha="$(cd "${bosh_deployment}"; git rev-parse --short HEAD)"
+bosh_stemcell_version="1.651"
+cf_deployment="$(cd "${bosh_deployment}"; cd ../cf-deployment; pwd)"
 
 if [ "${PWD##${bosh_deployment}}" != "${PWD}" ] || [ -e docker/create-env.sh ] || [ -e ../docker/create-env.sh ]; then
   echo "It looks like you are running this within the ${bosh_deployment} repository."
@@ -19,12 +21,12 @@ fi
 STEP "Creating Docker Network"
 ####
 
-docker_network=bosh
-docker_network_ip=10.245.0.10
-docker_network_gw=10.245.0.1
-docker_network_cidr=10.245.0.0/16
+docker_network=random
+docker_network_ip=10.244.0.2
+docker_network_gw=10.244.0.1
+docker_network_cidr=10.244.0.0/20
 
-if docker network ls | grep -q "${docker_network}"; then
+if [ "$(docker network ls | grep -c "${docker_network}")" -eq 0 ]; then
     echo "Creating docker network: ${docker_network} with range: ${docker_network_cidr}"
     docker network create -d bridge --subnet=${docker_network_cidr} ${docker_network} --attachable 1>/dev/null
 else
@@ -36,15 +38,15 @@ fi
 STEP "Creating BOSH Director"
 ####
 
-docker_host=$(docker context inspect | jq -r '.[0].Endpoints.docker.Host')
+docker_host="unix:///var/run/docker.sock"
 docker_tls=$(docker context inspect | jq -r '.[0].Endpoints.docker.SkipTLSVerify')
 
-#time bosh create-env "${bosh_deployment}/bosh.yml" \
-time ~/workspace/bosh-cli/out/bosh create-env "${bosh_deployment}/bosh.yml" \
+bosh create-env "${bosh_deployment}/bosh.yml" \
   --state "${PWD}/state.json" \
   --ops-file "${bosh_deployment}/docker/cpi.yml" \
+  --ops-file "${bosh_deployment}/bosh-lite-docker.yml" \
   --ops-file "${bosh_deployment}/docker/localhost.yml" \
-  --ops-file "${bosh_deployment}/bosh-lite.yml" \
+  --ops-file "${bosh_deployment}/docker/unix-sock.yml" \
   --ops-file "${bosh_deployment}/docker/bosh-lite.yml" \
   --ops-file "${bosh_deployment}/uaa.yml" \
   --ops-file "${bosh_deployment}/credhub.yml" \
@@ -60,12 +62,27 @@ time ~/workspace/bosh-cli/out/bosh create-env "${bosh_deployment}/bosh.yml" \
   --var internal_cidr="${docker_network_cidr}" "$@"
 
 
+# Find BOSH director's docker container by the exposed port 6868
+director_container_id=$(docker ps --filter "expose=6868" --format "{{.ID}}")
+if [ -n "${director_container_id}" ]; then
+  echo "Found director container with ID: ${director_container_id} and will modify docker socket permissions"
+  docker container exec -it ${director_container_id} bash -c "chmod 777 /var/run/docker.sock"
+else
+  echo "No director container ID found"
+fi
+
+
 ####
 STEP "Adding Network Routes (sudo is required)"
 ####
 
 if [ "$(uname)" = "Darwin" ]; then
-  sudo route add -net 10.244.0.0/16 192.168.56.6
+  if [ "netstat -rn | grep 10.244" -eq 0 ]; then
+    echo "Adding new route "
+    sudo route add -net 10.244.0.0/16 192.168.56.6
+  else
+      echo "Using existing route"
+  fi
 elif [ "$(uname)" = "Linux" ]; then
   if type ip > /dev/null 2>&1; then
     sudo ip route add 10.244.0.0/16 via 192.168.56.6
@@ -116,7 +133,7 @@ bosh \
 STEP "Updating Cloud Config"
 ####
 
-bosh -n update-cloud-config "${bosh_deployment}/warden/cloud-config.yml" \
+bosh -n update-cloud-config "../cf-deployment/iaas-support/bosh-lite/cloud-config.yml" \
   > /dev/null
 
 echo Succeeded
@@ -142,3 +159,19 @@ echo "BOSH Director is now running. You may need to run the following before usi
 echo
 echo "    source .envrc"
 echo
+
+####
+STEP "Upload Stemcell"
+####
+
+bosh upload-stemcell "https://storage.googleapis.com/bosh-core-stemcells/${bosh_stemcell_version}/bosh-stemcell-${bosh_stemcell_version}-warden-boshlite-ubuntu-jammy-go_agent.tgz"
+
+
+####
+STEP "Deploy CF"
+####
+ bosh -n -d cf deploy ${cf_deployment}/cf-deployment.yml -o ${cf_deployment}/operations/bosh-lite.yml \
+ -o ${cf_deployment}/operations/use-postgres.yml -o ${cf_deployment}/operations/use-compiled-releases.yml \
+ -o ${cf_deployment}/operations/enable-cpu-throttling.yml -o ${cf_deployment}/operations/experimental/use-native-garden-runc-runner.yml \
+ -o ${cf_deployment}/operations/experimental/disable-interpolate-service-bindings.yml -o ${cf_deployment}/operations/experimental/disable-cf-credhub.yml \
+ -v system_domain=bosh-lite.com