From 42ee3755fe18acfbf1fde24698b24f527876d3bb Mon Sep 17 00:00:00 2001 From: Fabio Ziviello Date: Wed, 23 Jul 2025 12:44:15 +0200 Subject: [PATCH 1/5] Beta Version --- README.md | 14 +++++++++++ cloud-init.yaml | 65 ++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 236ee1e..ad5b923 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,20 @@ NAME STATUS ROLES AGE VERSION opsv Ready control-plane,master 4h58m v1.29.6+k3s1 ``` +## (Optional) Add block storage rook + +``` +multipass exec opsv -- bash -c 'export KUBECONFIG=/home/ubuntu/.kube/config && /home/ubuntu/add-rook' + +``` + +## (Optional) Add block storage longhorn + +``` +multipass exec opsv -- bash -c 'export KUBECONFIG=/home/ubuntu/.kube/config && /home/ubuntu/add-longhorn' + +``` + # Development Environment Overview If you only want to test OpenServerless, stop here. You should have a working environment. diff --git a/cloud-init.yaml b/cloud-init.yaml index d0e0244..131d0a3 100644 --- a/cloud-init.yaml +++ b/cloud-init.yaml @@ -92,6 +92,69 @@ write_files: # TODO add a .git-hooks directory with hooks #git config core.hooksPath .git-hooks + - path: /home/ubuntu/add-rook + permissions: '0755' + defer: true + owner: ubuntu:ubuntu + content: | + #!/bin/bash + set -e + echo "๐Ÿ“ฆ Installing Rook..." + cd /home/ubuntu + git clone --depth 1 https://github.com/rook/rook.git || (cd rook && git pull) + cd rook/deploy/examples + echo "๐Ÿ”ง Patching Ceph cluster.yaml for single node..." + if ! command -v yq >/dev/null 2>&1; then + mkdir -p /home/ubuntu/bin + wget -qO /home/ubuntu/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + chmod +x /home/ubuntu/bin/yq + export PATH=/home/ubuntu/bin:$PATH + fi + yq e '.spec.mon.count = 1 | .spec.mon.allowMultiplePerNode = true' -i cluster.yaml + + kubectl apply -f crds.yaml + kubectl apply -f common.yaml + kubectl apply -f operator.yaml + sleep 15 + kubectl apply -f cluster.yaml + sleep 90 + kubectl apply -f ceph-client.yaml + echo "๐Ÿ“ฆ Installing RBD CSI manifests for StorageClass..." + kubectl apply -f csi/rbd/ + + for i in {1..90}; do + status=$(kubectl -n rook-ceph get cephcluster rook-ceph -o jsonpath='{.status.phase}' 2>/dev/null || echo "notfound") + echo "โณ Attempt $i: Rook status = $status" + [[ "$status" == "Ready" ]] && echo "โœ… Rook Ready" && break + sleep 15 + done + + echo "๐Ÿ“ฆ Setting rook-ceph-block as default storage class (if exists)..." + if kubectl get storageclass rook-ceph-block >/dev/null 2>&1; then + kubectl annotate storageclass rook-ceph-block storageclass.kubernetes.io/is-default-class=true --overwrite + fi + + - path: /home/ubuntu/add-longhorn + permissions: '0755' + defer: true + owner: ubuntu:ubuntu + content: | + #!/bin/bash + echo "๐Ÿ“ฆ Installing Longhorn..." + kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/v1.4.1/deploy/longhorn.yaml + for i in {1..90}; do + ready=$(kubectl -n longhorn-system get pods -l app=longhorn-manager -o jsonpath='{.items[*].status.containerStatuses[0].ready}' | grep -o true | wc -l) + total=$(kubectl -n longhorn-system get pods -l app=longhorn-manager --no-headers | wc -l) + echo "โณ Attempt $i: $ready/$total pods ready" + [[ "$ready" == "$total" && "$total" -gt 0 ]] && echo "โœ… Longhorn Ready" && break + sleep 15 + done + + echo "๐Ÿ“ฆ Setting Longhorn as default storage class (if exists)..." + if kubectl get storageclass longhorn >/dev/null 2>&1; then + kubectl annotate storageclass longhorn storageclass.kubernetes.io/is-default-class=true --overwrite + fi + runcmd: - | cd /home/ubuntu @@ -100,4 +163,4 @@ runcmd: curl -sL https://raw.githubusercontent.com/sciabarracom/openserverless/refs/heads/main/bash_aliases -o ".bash_aliases" echo 'source $HOME/.z.sh' >> .bashrc chown ubuntu:ubuntu .z.sh .bashrc .bash_aliases - echo -e '#!/bin/bash\n/etc/cloud-init.sh 2>&1 >/tmp/cloud-init.log\n' | at now + echo -e '#!/bin/bash\n/etc/cloud-init.sh 2>&1 >/tmp/cloud-init.log\n' | at now \ No newline at end of file From c6d50dfbd84775ffb1c9f7eac8823a2cdb9ee1c2 Mon Sep 17 00:00:00 2001 From: Fabio Ziviello Date: Wed, 23 Jul 2025 13:24:02 +0200 Subject: [PATCH 2/5] update used raw url --- cloud-init.yaml | 46 ++++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/cloud-init.yaml b/cloud-init.yaml index 131d0a3..9d39f06 100644 --- a/cloud-init.yaml +++ b/cloud-init.yaml @@ -99,28 +99,34 @@ write_files: content: | #!/bin/bash set -e - echo "๐Ÿ“ฆ Installing Rook..." - cd /home/ubuntu - git clone --depth 1 https://github.com/rook/rook.git || (cd rook && git pull) - cd rook/deploy/examples - echo "๐Ÿ”ง Patching Ceph cluster.yaml for single node..." + echo "๐Ÿ“ฆ Installing Rook v1.14.1..." + + BASE=https://raw.githubusercontent.com/rook/rook/v1.14.1/deploy/examples + + echo "๐Ÿ”ง Applying CRDs and common resources..." + kubectl apply -f $BASE/crds.yaml + kubectl apply -f $BASE/common.yaml + + echo "๐Ÿš€ Deploying Rook Operator..." + kubectl apply -f $BASE/operator.yaml + sleep 15 + + echo "๐Ÿ”ง Configuring cluster.yaml for single node..." + curl -s $BASE/cluster.yaml -o /tmp/cluster.yaml + if ! command -v yq >/dev/null 2>&1; then mkdir -p /home/ubuntu/bin wget -qO /home/ubuntu/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 chmod +x /home/ubuntu/bin/yq export PATH=/home/ubuntu/bin:$PATH fi - yq e '.spec.mon.count = 1 | .spec.mon.allowMultiplePerNode = true' -i cluster.yaml - kubectl apply -f crds.yaml - kubectl apply -f common.yaml - kubectl apply -f operator.yaml - sleep 15 - kubectl apply -f cluster.yaml + yq e '.spec.mon.count = 1 | .spec.mon.allowMultiplePerNode = true' -i /tmp/cluster.yaml + kubectl apply -f /tmp/cluster.yaml sleep 90 - kubectl apply -f ceph-client.yaml - echo "๐Ÿ“ฆ Installing RBD CSI manifests for StorageClass..." - kubectl apply -f csi/rbd/ + + echo "๐Ÿ“„ Applying Ceph clients..." + kubectl apply -f $BASE/ceph-client.yaml for i in {1..90}; do status=$(kubectl -n rook-ceph get cephcluster rook-ceph -o jsonpath='{.status.phase}' 2>/dev/null || echo "notfound") @@ -129,9 +135,11 @@ write_files: sleep 15 done - echo "๐Ÿ“ฆ Setting rook-ceph-block as default storage class (if exists)..." + echo "๐Ÿ“ฆ Trying to set rook-ceph-block as default storage class (non-destructive)..." if kubectl get storageclass rook-ceph-block >/dev/null 2>&1; then kubectl annotate storageclass rook-ceph-block storageclass.kubernetes.io/is-default-class=true --overwrite + else + echo "โš ๏ธ rook-ceph-block not found, skipping annotation" fi - path: /home/ubuntu/add-longhorn @@ -140,8 +148,10 @@ write_files: owner: ubuntu:ubuntu content: | #!/bin/bash - echo "๐Ÿ“ฆ Installing Longhorn..." + set -e + echo "๐Ÿ“ฆ Installing Longhorn v1.4.1..." kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/v1.4.1/deploy/longhorn.yaml + for i in {1..90}; do ready=$(kubectl -n longhorn-system get pods -l app=longhorn-manager -o jsonpath='{.items[*].status.containerStatuses[0].ready}' | grep -o true | wc -l) total=$(kubectl -n longhorn-system get pods -l app=longhorn-manager --no-headers | wc -l) @@ -150,9 +160,9 @@ write_files: sleep 15 done - echo "๐Ÿ“ฆ Setting Longhorn as default storage class (if exists)..." + echo "๐Ÿ“ฆ Trying to set Longhorn as default storage class (non-destructive)..." if kubectl get storageclass longhorn >/dev/null 2>&1; then - kubectl annotate storageclass longhorn storageclass.kubernetes.io/is-default-class=true --overwrite + kubectl annotate storageclass longhorn storageclass.kubernetes.io/is-default-class=true --overwrite || true fi runcmd: From 30ac5cf3d4671aae02e6de53fec65fc21cd7f57d Mon Sep 17 00:00:00 2001 From: Fabio Ziviello Date: Wed, 23 Jul 2025 13:37:15 +0200 Subject: [PATCH 3/5] optimize --- cloud-init.yaml | 51 +++++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/cloud-init.yaml b/cloud-init.yaml index 9d39f06..d72004a 100644 --- a/cloud-init.yaml +++ b/cloud-init.yaml @@ -99,34 +99,29 @@ write_files: content: | #!/bin/bash set -e - echo "๐Ÿ“ฆ Installing Rook v1.14.1..." - - BASE=https://raw.githubusercontent.com/rook/rook/v1.14.1/deploy/examples - - echo "๐Ÿ”ง Applying CRDs and common resources..." - kubectl apply -f $BASE/crds.yaml - kubectl apply -f $BASE/common.yaml - - echo "๐Ÿš€ Deploying Rook Operator..." - kubectl apply -f $BASE/operator.yaml - sleep 15 - - echo "๐Ÿ”ง Configuring cluster.yaml for single node..." - curl -s $BASE/cluster.yaml -o /tmp/cluster.yaml - + echo "๐Ÿ“ฆ Installing Rook..." + cd /home/ubuntu + git clone --depth 1 https://github.com/rook/rook.git || (cd rook && git pull) + cd rook/deploy/examples + echo "๐Ÿ”ง Patching Ceph cluster.yaml for single node..." if ! command -v yq >/dev/null 2>&1; then mkdir -p /home/ubuntu/bin wget -qO /home/ubuntu/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 chmod +x /home/ubuntu/bin/yq export PATH=/home/ubuntu/bin:$PATH fi + yq e '.spec.mon.count = 1 | .spec.mon.allowMultiplePerNode = true' -i cluster.yaml - yq e '.spec.mon.count = 1 | .spec.mon.allowMultiplePerNode = true' -i /tmp/cluster.yaml - kubectl apply -f /tmp/cluster.yaml + kubectl apply -f crds.yaml + kubectl apply -f common.yaml + kubectl apply -f operator.yaml + sleep 15 + kubectl apply -f cluster.yaml sleep 90 + kubectl apply -f ceph-client.yaml - echo "๐Ÿ“„ Applying Ceph clients..." - kubectl apply -f $BASE/ceph-client.yaml + echo "๐Ÿ“ฆ Installing RBD CSI manifests (except storageclass.yaml)..." + kubectl apply -f csi/rbd/ for i in {1..90}; do status=$(kubectl -n rook-ceph get cephcluster rook-ceph -o jsonpath='{.status.phase}' 2>/dev/null || echo "notfound") @@ -135,11 +130,21 @@ write_files: sleep 15 done - echo "๐Ÿ“ฆ Trying to set rook-ceph-block as default storage class (non-destructive)..." + echo "๐Ÿ“ฆ Waiting for csi-rbdplugin pods to be ready..." + for i in {1..90}; do + notready=$(kubectl -n rook-ceph get pods -l app=csi-rbdplugin -o jsonpath='{.items[*].status.containerStatuses[*].ready}' | grep -v true | wc -l) + total=$(kubectl -n rook-ceph get pods -l app=csi-rbdplugin --no-headers 2>/dev/null | wc -l) + echo "โณ Attempt $i: $((total - notready))/$total pods ready" + if [[ "$notready" == "0" && "$total" -gt 0 ]]; then + echo "โœ… CSI RBD Plugin Ready" + break + fi + sleep 15 + done + if kubectl get storageclass rook-ceph-block >/dev/null 2>&1; then - kubectl annotate storageclass rook-ceph-block storageclass.kubernetes.io/is-default-class=true --overwrite - else - echo "โš ๏ธ rook-ceph-block not found, skipping annotation" + echo "๐Ÿ“ฆ Setting rook-ceph-block as default storage class" + kubectl annotate storageclass rook-ceph-block storageclass.kubernetes.io/is-default-class=true --overwrite >/dev/null 2>&1 fi - path: /home/ubuntu/add-longhorn From 6245e57e7c0fac8f1cd48a8d12a81e3988e3140a Mon Sep 17 00:00:00 2001 From: Fabio Ziviello Date: Wed, 23 Jul 2025 13:37:15 +0200 Subject: [PATCH 4/5] fix for rook # Conflicts: # cloud-init.yaml --- cloud-init.yaml | 105 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 71 insertions(+), 34 deletions(-) diff --git a/cloud-init.yaml b/cloud-init.yaml index d72004a..cd3d178 100644 --- a/cloud-init.yaml +++ b/cloud-init.yaml @@ -91,7 +91,7 @@ write_files: bash direnv-init.sh # TODO add a .git-hooks directory with hooks #git config core.hooksPath .git-hooks - + - path: /home/ubuntu/add-rook permissions: '0755' defer: true @@ -99,54 +99,89 @@ write_files: content: | #!/bin/bash set -e - echo "๐Ÿ“ฆ Installing Rook..." - cd /home/ubuntu - git clone --depth 1 https://github.com/rook/rook.git || (cd rook && git pull) - cd rook/deploy/examples - echo "๐Ÿ”ง Patching Ceph cluster.yaml for single node..." + + ROOK_VERSION="v1.14.6" + RAW_BASE="https://raw.githubusercontent.com/rook/rook/${ROOK_VERSION}/deploy/examples" + + echo "๐Ÿ“ฆ Installing Rook ${ROOK_VERSION}" + echo "๐Ÿ“„ Applying CRDs and operator..." + kubectl apply -f ${RAW_BASE}/crds.yaml + kubectl apply -f ${RAW_BASE}/common.yaml + kubectl apply -f ${RAW_BASE}/operator.yaml + sleep 10 + + echo "๐Ÿ”Ž Checking for yq..." if ! command -v yq >/dev/null 2>&1; then + echo "๐Ÿ“ฅ Installing yq..." mkdir -p /home/ubuntu/bin - wget -qO /home/ubuntu/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + curl -sL https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -o /home/ubuntu/bin/yq chmod +x /home/ubuntu/bin/yq export PATH=/home/ubuntu/bin:$PATH + else + echo "โœ… yq already installed." fi - yq e '.spec.mon.count = 1 | .spec.mon.allowMultiplePerNode = true' -i cluster.yaml - kubectl apply -f crds.yaml - kubectl apply -f common.yaml - kubectl apply -f operator.yaml - sleep 15 - kubectl apply -f cluster.yaml - sleep 90 - kubectl apply -f ceph-client.yaml + echo "๐Ÿ› ๏ธ Patching and applying Ceph cluster for single-node setup..." + curl -sL ${RAW_BASE}/cluster.yaml | \ + yq e '.spec.mon.count = 1 | .spec.mon.allowMultiplePerNode = true' - | \ + kubectl apply -f - + sleep 30 - echo "๐Ÿ“ฆ Installing RBD CSI manifests (except storageclass.yaml)..." - kubectl apply -f csi/rbd/ + echo "๐Ÿ“„ Applying Ceph client..." + kubectl apply -f ${RAW_BASE}/ceph-client.yaml + echo "๐Ÿ” Waiting for Rook to be Ready..." for i in {1..90}; do status=$(kubectl -n rook-ceph get cephcluster rook-ceph -o jsonpath='{.status.phase}' 2>/dev/null || echo "notfound") echo "โณ Attempt $i: Rook status = $status" - [[ "$status" == "Ready" ]] && echo "โœ… Rook Ready" && break - sleep 15 - done - - echo "๐Ÿ“ฆ Waiting for csi-rbdplugin pods to be ready..." - for i in {1..90}; do - notready=$(kubectl -n rook-ceph get pods -l app=csi-rbdplugin -o jsonpath='{.items[*].status.containerStatuses[*].ready}' | grep -v true | wc -l) - total=$(kubectl -n rook-ceph get pods -l app=csi-rbdplugin --no-headers 2>/dev/null | wc -l) - echo "โณ Attempt $i: $((total - notready))/$total pods ready" - if [[ "$notready" == "0" && "$total" -gt 0 ]]; then - echo "โœ… CSI RBD Plugin Ready" + if [[ "$status" == "Ready" ]]; then + echo "โœ… Rook is healthy!" break fi sleep 15 done - if kubectl get storageclass rook-ceph-block >/dev/null 2>&1; then - echo "๐Ÿ“ฆ Setting rook-ceph-block as default storage class" - kubectl annotate storageclass rook-ceph-block storageclass.kubernetes.io/is-default-class=true --overwrite >/dev/null 2>&1 + if [[ "$status" != "Ready" ]]; then + echo "โŒ Timeout waiting for Rook to become healthy." + exit 1 fi + echo "๐Ÿ“„ Installing VolumeSnapshot CRDs..." + kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v6.3.3/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml + kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v6.3.3/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml + kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v6.3.3/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml + + echo "๐Ÿงน Re-creating rook-ceph-block StorageClass..." + kubectl delete storageclass rook-ceph-block --ignore-not-found=true + + cat </dev/null 2>&1; then kubectl annotate storageclass longhorn storageclass.kubernetes.io/is-default-class=true --overwrite || true fi From b5c98ba28ced1b7e27c74071d0cb125c51b26eb7 Mon Sep 17 00:00:00 2001 From: Fabio Ziviello Date: Wed, 23 Jul 2025 16:07:33 +0200 Subject: [PATCH 5/5] stable version --- cloud-init.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloud-init.yaml b/cloud-init.yaml index cd3d178..8f00f83 100644 --- a/cloud-init.yaml +++ b/cloud-init.yaml @@ -110,7 +110,6 @@ write_files: kubectl apply -f ${RAW_BASE}/operator.yaml sleep 10 - echo "๐Ÿ”Ž Checking for yq..." if ! command -v yq >/dev/null 2>&1; then echo "๐Ÿ“ฅ Installing yq..." mkdir -p /home/ubuntu/bin @@ -181,7 +180,6 @@ write_files: echo "โœ… Rook installed and configured with default StorageClass rook-ceph-block." - - path: /home/ubuntu/add-longhorn permissions: '0755' defer: true @@ -189,6 +187,7 @@ write_files: content: | #!/bin/bash set -e + LONGHORN_VERSION="v1.4.1" echo "๐Ÿ“ฆ Installing Longhorn ${LONGHORN_VERSION}"