This repository is currently being migrated. It's locked while the migration is in progress.
Corrupted state - Unable to list / modify volumes #234
Description
In my cluster I've been experimenting with adding / removing nodes. Not 100% sure, but I believe I drained all nodes before deleting them from the cluster. When I deployed a new workload with some volumes I noticed I could no longer use the CLI to fetch any node or volume information in that namespace. In #229 (comment) I describe the issues I was getting.
In order to finally get it working again I needed to go into the etcd cluster and forcefully remove all keys of which the volume id was in the key or in it's value. I used the api to fetch which volumes were not attached to any host and started removing them one by one until the CLI was responding to my requests again. By doing this there might be some residue left on some nodes tho.
Below I've include the api response I was using the forcefully remove the volumes from the state. Note that none of the detached volumes listed here were showing up in Kubernetes. They only references to them were in the Ondat api.
GET: /v2/namespaces/a92a8f34-4cfa-4db1-bca2-a266e0906acc/volumes
[
{
"id": "3b107eb1-37a7-467e-b77f-d58a03560045",
"namespaceID": "a92a8f34-4cfa-4db1-bca2-a266e0906acc",
"attachedOn": "",
"attachmentType": "detached",
"attachedOnHost": "",
"nfs": {},
"labels": {
"app.kubernetes.io/component": "primary",
"app.kubernetes.io/instance": "state",
"app.kubernetes.io/name": "mariadb",
"csi.storage.k8s.io/pv/name": "pvc-e6fca42d-5f14-461c-aa6c-280499a59e74",
"csi.storage.k8s.io/pvc/name": "data-state-mariadb-primary-0",
"csi.storage.k8s.io/pvc/namespace": "main",
"storageos.com/nocompress": "true",
"storageos.com/replicas": "2",
"storageos.com/topology-aware": "true"
},
"topologyLabels": {
"requisiteZones": [
"nuremberg",
"us-east-ash-dc1",
"eu-central-nbg1-dc3",
"us-west-hil-dc1",
"falkenstein",
"ap-southeast-syd1"
],
"preferredZones": [
"eu-central-nbg1-dc3",
"falkenstein",
"nuremberg",
"us-east-ash-dc1",
"us-west-hil-dc1",
"ap-southeast-syd1"
]
},
"name": "pvc-e6fca42d-5f14-461c-aa6c-280499a59e74",
"fsType": "ext4",
"sizeBytes": 10737418240,
"description": "",
"master": {
"id": "9d4dc98d-064e-4a45-96dc-f4562e6a89c2",
"nodeID": "102339c1-1e5d-464b-8bbb-0dfb6b5361b2",
"hostname": "",
"health": "unknown",
"promotable": true
},
"replicas": [
{
"id": "0918c75e-7ce7-47ff-b83b-281209ea3ae6",
"nodeID": "19413d46-0075-47ec-95a4-53e708aad8c5",
"hostname": "",
"health": "unknown",
"promotable": true
}
],
"version": "Ng",
"createdAt": "2023-03-15T16:10:29Z",
"updatedAt": "2023-03-16T18:04:11Z",
"snapshotConfig": {},
"snapshotVolumes": {},
"parentInfo": null
},
{
"id": "40dd4e6f-9aaf-4cab-b414-9de13cd9a86a",
"namespaceID": "a92a8f34-4cfa-4db1-bca2-a266e0906acc",
"attachedOn": "",
"attachmentType": "detached",
"attachedOnHost": "",
"nfs": {},
"labels": {
"app.kubernetes.io/component": "primary",
"app.kubernetes.io/instance": "state",
"app.kubernetes.io/name": "mariadb",
"csi.storage.k8s.io/pv/name": "pvc-4c602647-2d01-4ab8-9377-2d7011bf10ea",
"csi.storage.k8s.io/pvc/name": "data-state-mariadb-0",
"csi.storage.k8s.io/pvc/namespace": "main",
"storageos.com/nocompress": "true",
"storageos.com/replicas": "2",
"storageos.com/topology-aware": "true"
},
"topologyLabels": {
"requisiteZones": [
"eu-central-nbg1-dc3",
"ap-southeast-syd1",
"us-west-hil-dc1",
"falkenstein",
"nuremberg",
"us-east-ash-dc1"
],
"preferredZones": [
"us-west-hil-dc1",
"ap-southeast-syd1",
"eu-central-nbg1-dc3",
"falkenstein",
"nuremberg",
"us-east-ash-dc1"
]
},
"name": "pvc-4c602647-2d01-4ab8-9377-2d7011bf10ea",
"fsType": "ext4",
"sizeBytes": 10737418240,
"description": "",
"master": {
"id": "9d031c5d-dd43-4190-a6db-3eed65ad871a",
"nodeID": "102339c1-1e5d-464b-8bbb-0dfb6b5361b2",
"hostname": "",
"health": "unknown",
"promotable": true
},
"replicas": [
{
"id": "141aa981-e351-4260-b187-4574fc14526a",
"nodeID": "eb3c26e4-6b12-433a-9104-012a328c1631",
"hostname": "",
"health": "unknown",
"promotable": false
},
{
"id": "1fb54270-a651-4795-a658-5f1bbb1bae6a",
"nodeID": "6d1434f5-1829-4c33-a28f-8f72b5df7054",
"hostname": "",
"health": "unknown",
"promotable": false
}
],
"version": "MTE",
"createdAt": "2023-03-15T16:19:47Z",
"updatedAt": "2023-03-16T18:06:11Z",
"snapshotConfig": {},
"snapshotVolumes": {},
"parentInfo": null
},
{
"id": "718779c7-cd67-4fa9-8bce-808e96ccb007",
"namespaceID": "a92a8f34-4cfa-4db1-bca2-a266e0906acc",
"attachedOn": "",
"attachmentType": "detached",
"attachedOnHost": "",
"nfs": {},
"labels": {
"app.kubernetes.io/component": "primary",
"app.kubernetes.io/instance": "state",
"app.kubernetes.io/name": "mariadb",
"csi.storage.k8s.io/pv/name": "pvc-bccc381d-b074-4326-bcf5-cbabc1cdd359",
"csi.storage.k8s.io/pvc/name": "data-state-mariadb-primary-0",
"csi.storage.k8s.io/pvc/namespace": "main",
"storageos.com/nocompress": "true",
"storageos.com/replicas": "2",
"storageos.com/topology-aware": "true"
},
"topologyLabels": {
"requisiteZones": [
"falkenstein",
"ap-southeast-syd1",
"nuremberg",
"us-east-ash-dc1",
"eu-central-nbg1-dc3",
"us-west-hil-dc1"
],
"preferredZones": [
"eu-central-nbg1-dc3",
"falkenstein",
"nuremberg",
"us-east-ash-dc1",
"us-west-hil-dc1",
"ap-southeast-syd1"
]
},
"name": "pvc-bccc381d-b074-4326-bcf5-cbabc1cdd359",
"fsType": "ext4",
"sizeBytes": 10737418240,
"description": "",
"master": {
"id": "10e097dd-ad03-4b64-af99-bcde27f8962a",
"nodeID": "6d1434f5-1829-4c33-a28f-8f72b5df7054",
"hostname": "",
"health": "online",
"promotable": true
},
"replicas": [
{
"id": "7746f2be-96fd-40f5-83c3-7c566635ec10",
"nodeID": "b6932610-5cb2-416d-bf11-ab52396caa67",
"hostname": "",
"health": "ready",
"promotable": true
},
{
"id": "11b172d8-cf40-4701-a207-46cef3ac58c7",
"nodeID": "97f53391-97d1-4db6-817a-6b902fe0caee",
"hostname": "",
"health": "ready",
"promotable": true
}
],
"version": "MTE",
"createdAt": "2023-03-15T15:38:10Z",
"updatedAt": "2023-03-17T15:21:53Z",
"snapshotConfig": {},
"snapshotVolumes": {},
"parentInfo": null
},
{
"id": "772e746a-6a18-4b94-b601-5c909d0cb85a",
"namespaceID": "a92a8f34-4cfa-4db1-bca2-a266e0906acc",
"attachedOn": "",
"attachmentType": "detached",
"attachedOnHost": "",
"nfs": {},
"labels": {
"app.kubernetes.io/component": "primary",
"app.kubernetes.io/instance": "state",
"app.kubernetes.io/name": "mariadb",
"csi.storage.k8s.io/pv/name": "pvc-08d6a2e3-2741-4012-8790-c3e04a194cc0",
"csi.storage.k8s.io/pvc/name": "data-state-mariadb-0",
"csi.storage.k8s.io/pvc/namespace": "main",
"storageos.com/nocompress": "true",
"storageos.com/replicas": "2",
"storageos.com/topology-aware": "true"
},
"topologyLabels": {
"requisiteZones": [
"nuremberg",
"us-east-ash-dc1",
"us-west-hil-dc1",
"ap-southeast-syd1",
"eu-central-nbg1-dc3",
"falkenstein"
],
"preferredZones": [
"us-west-hil-dc1",
"ap-southeast-syd1",
"eu-central-nbg1-dc3",
"falkenstein",
"nuremberg",
"us-east-ash-dc1"
]
},
"name": "pvc-08d6a2e3-2741-4012-8790-c3e04a194cc0",
"fsType": "ext4",
"sizeBytes": 10737418240,
"description": "",
"master": {
"id": "7c5706b0-837c-40f5-ab4b-c8af683808db",
"nodeID": "b6932610-5cb2-416d-bf11-ab52396caa67",
"hostname": "",
"health": "online",
"promotable": true
},
"replicas": [
{
"id": "102d6258-5836-4363-9843-905fc9b8c433",
"nodeID": "97f53391-97d1-4db6-817a-6b902fe0caee",
"hostname": "",
"health": "ready",
"promotable": true
},
{
"id": "fcc4340b-9c53-46a8-a315-20dea8ee73df",
"nodeID": "6d1434f5-1829-4c33-a28f-8f72b5df7054",
"hostname": "",
"health": "ready",
"promotable": true
}
],
"version": "MTQ",
"createdAt": "2023-03-15T16:54:28Z",
"updatedAt": "2023-03-17T15:22:04Z",
"snapshotConfig": {},
"snapshotVolumes": {},
"parentInfo": null
},
{
"id": "9a385145-8613-47c2-9816-dcbcb6ff604c",
"namespaceID": "a92a8f34-4cfa-4db1-bca2-a266e0906acc",
"attachedOn": "",
"attachmentType": "detached",
"attachedOnHost": "",
"nfs": {},
"labels": {
"app.kubernetes.io/component": "primary",
"app.kubernetes.io/instance": "state",
"app.kubernetes.io/name": "mariadb",
"csi.storage.k8s.io/pv/name": "pvc-8d37dd78-d43a-4625-be17-eb26ad7ee407",
"csi.storage.k8s.io/pvc/name": "data-state-mariadb-0",
"csi.storage.k8s.io/pvc/namespace": "main",
"storageos.com/nocompress": "true",
"storageos.com/replicas": "2",
"storageos.com/topology-aware": "true"
},
"topologyLabels": {
"requisiteZones": [
"nuremberg",
"us-west-hil-dc1",
"falkenstein",
"ap-southeast-syd1",
"us-east-ash-dc1",
"eu-central-nbg1-dc3"
],
"preferredZones": [
"us-west-hil-dc1",
"ap-southeast-syd1",
"eu-central-nbg1-dc3",
"falkenstein",
"nuremberg",
"us-east-ash-dc1"
]
},
"name": "pvc-8d37dd78-d43a-4625-be17-eb26ad7ee407",
"fsType": "ext4",
"sizeBytes": 10737418240,
"description": "",
"master": {
"id": "1b28d52a-9aa3-4c8b-905b-38d6c1d0b0d5",
"nodeID": "6d1434f5-1829-4c33-a28f-8f72b5df7054",
"hostname": "",
"health": "online",
"promotable": true
},
"replicas": [
{
"id": "69e5c914-14fa-4d95-b1f8-9405f867129c",
"nodeID": "b6932610-5cb2-416d-bf11-ab52396caa67",
"hostname": "",
"health": "ready",
"promotable": true
},
{
"id": "cee5b623-b996-4f72-bd91-dd035455f6e0",
"nodeID": "97f53391-97d1-4db6-817a-6b902fe0caee",
"hostname": "",
"health": "ready",
"promotable": true
}
],
"version": "MTY",
"createdAt": "2023-03-15T17:04:33Z",
"updatedAt": "2023-03-17T15:21:54Z",
"snapshotConfig": {},
"snapshotVolumes": {},
"parentInfo": null
},
{
"id": "a3058662-1ed2-463c-9979-2ac72a25980b",
"namespaceID": "a92a8f34-4cfa-4db1-bca2-a266e0906acc",
"attachedOn": "",
"attachmentType": "detached",
"attachedOnHost": "",
"nfs": {},
"labels": {
"app.kubernetes.io/component": "primary",
"app.kubernetes.io/instance": "state",
"app.kubernetes.io/name": "mariadb",
"csi.storage.k8s.io/pv/name": "pvc-e3b36bad-14a8-40ab-bb76-3160dca41ed0",
"csi.storage.k8s.io/pvc/name": "data-state-mariadb-0",
"csi.storage.k8s.io/pvc/namespace": "main",
"storageos.com/nocompress": "true",
"storageos.com/replicas": "1"
},
"topologyLabels": {
"requisiteZones": [
""
],
"preferredZones": [
""
]
},
"name": "pvc-e3b36bad-14a8-40ab-bb76-3160dca41ed0",
"fsType": "ext4",
"sizeBytes": 10737418240,
"description": "",
"master": {
"id": "4519ccaa-2617-48c1-8f18-bc0ace3e769f",
"nodeID": "5ca08388-71d4-48b2-9d40-2d202c39e94e",
"hostname": "",
"health": "online",
"promotable": true
},
"replicas": [
{
"id": "a76fd3b5-22db-460e-a132-0c3401f9c216",
"nodeID": "6d1434f5-1829-4c33-a28f-8f72b5df7054",
"hostname": "",
"health": "ready",
"promotable": true
}
],
"version": "MTM",
"createdAt": "2023-03-16T11:12:41Z",
"updatedAt": "2023-03-17T15:21:46Z",
"snapshotConfig": {},
"snapshotVolumes": {},
"parentInfo": null
},
{
"id": "a5774c83-b846-4ad8-8219-c361fe30b707",
"namespaceID": "a92a8f34-4cfa-4db1-bca2-a266e0906acc",
"attachedOn": "",
"attachmentType": "detached",
"attachedOnHost": "",
"nfs": {},
"labels": {
"app.kubernetes.io/component": "primary",
"app.kubernetes.io/instance": "state",
"app.kubernetes.io/name": "mariadb",
"csi.storage.k8s.io/pv/name": "pvc-941f5b55-810f-4b16-b324-edf21f718e98",
"csi.storage.k8s.io/pvc/name": "data-state-mariadb-primary-0",
"csi.storage.k8s.io/pvc/namespace": "main",
"storageos.com/nocompress": "true",
"storageos.com/replicas": "2",
"storageos.com/topology-aware": "true"
},
"topologyLabels": {
"requisiteZones": [
"us-west-hil-dc1",
"ap-southeast-syd1",
"falkenstein",
"nuremberg",
"us-east-ash-dc1",
"eu-central-nbg1-dc3"
],
"preferredZones": [
"eu-central-nbg1-dc3",
"falkenstein",
"nuremberg",
"us-east-ash-dc1",
"us-west-hil-dc1",
"ap-southeast-syd1"
]
},
"name": "pvc-941f5b55-810f-4b16-b324-edf21f718e98",
"fsType": "ext4",
"sizeBytes": 10737418240,
"description": "",
"master": {
"id": "a12b114d-f79b-4d4c-ae4b-f1c92938e173",
"nodeID": "b6932610-5cb2-416d-bf11-ab52396caa67",
"hostname": "",
"health": "online",
"promotable": true
},
"replicas": [
{
"id": "9ac7fba7-0528-4728-827c-4cdc6e4641a2",
"nodeID": "97f53391-97d1-4db6-817a-6b902fe0caee",
"hostname": "",
"health": "ready",
"promotable": true
},
{
"id": "97990e7d-ef3e-4f62-9800-f7a411b2331a",
"nodeID": "6d1434f5-1829-4c33-a28f-8f72b5df7054",
"hostname": "",
"health": "ready",
"promotable": true
}
],
"version": "MTE",
"createdAt": "2023-03-15T15:24:11Z",
"updatedAt": "2023-03-17T15:22:12Z",
"snapshotConfig": {},
"snapshotVolumes": {},
"parentInfo": null
},
{
"id": "bf186dd6-ca68-4232-8dd8-b098933b0a26",
"namespaceID": "a92a8f34-4cfa-4db1-bca2-a266e0906acc",
"attachedOn": "af2c5174-7f95-40ca-a937-96002d98eb67",
"attachmentType": "host",
"attachedOnHost": "",
"nfs": {},
"labels": {
"app.kubernetes.io/component": "primary",
"app.kubernetes.io/instance": "state",
"app.kubernetes.io/name": "mariadb",
"csi.storage.k8s.io/pv/name": "pvc-056e1cda-30eb-4c87-984a-a6bbc4de4211",
"csi.storage.k8s.io/pvc/name": "data-state-mariadb-0",
"csi.storage.k8s.io/pvc/namespace": "main",
"storageos.com/nocompress": "true",
"storageos.com/replicas": "1",
"storageos.com/topology-aware": "true"
},
"topologyLabels": {
"requisiteZones": [
"eu-central-fsn1-dc14",
"nuremberg",
"falkenstein",
"us-west-hil-dc1",
"ap-southeast-syd1",
"eu-central-nbg1-dc3"
],
"preferredZones": [
"us-west-hil-dc1",
"ap-southeast-syd1",
"eu-central-fsn1-dc14",
"eu-central-nbg1-dc3",
"falkenstein",
"nuremberg"
]
},
"name": "pvc-056e1cda-30eb-4c87-984a-a6bbc4de4211",
"fsType": "ext4",
"sizeBytes": 10737418240,
"description": "",
"master": {
"id": "70e2799b-ccf0-4e4f-9fbd-17c0330b64ef",
"nodeID": "af2c5174-7f95-40ca-a937-96002d98eb67",
"hostname": "",
"health": "online",
"promotable": true
},
"replicas": [
{
"id": "4e30ae43-9e4e-4f20-a461-292f920295b4",
"nodeID": "5ca08388-71d4-48b2-9d40-2d202c39e94e",
"hostname": "",
"health": "ready",
"promotable": true
}
],
"version": "Ng",
"createdAt": "2023-03-17T15:47:46Z",
"updatedAt": "2023-03-17T21:07:39Z",
"snapshotConfig": {},
"snapshotVolumes": {},
"parentInfo": null
},
{
"id": "e455a29a-2a32-42c2-9234-21a613f84522",
"namespaceID": "a92a8f34-4cfa-4db1-bca2-a266e0906acc",
"attachedOn": "",
"attachmentType": "detached",
"attachedOnHost": "",
"nfs": {},
"labels": {
"app.kubernetes.io/component": "primary",
"app.kubernetes.io/instance": "state",
"app.kubernetes.io/name": "mariadb",
"csi.storage.k8s.io/pv/name": "pvc-8a4d6583-23c7-4869-9325-ba0bf42a174f",
"csi.storage.k8s.io/pvc/name": "data-state-mariadb-0",
"csi.storage.k8s.io/pvc/namespace": "main",
"storageos.com/nocompress": "true",
"storageos.com/replicas": "1"
},
"topologyLabels": {
"requisiteZones": [
""
],
"preferredZones": [
""
]
},
"name": "pvc-8a4d6583-23c7-4869-9325-ba0bf42a174f",
"fsType": "ext4",
"sizeBytes": 10737418240,
"description": "",
"master": {
"id": "8b7b8aed-225f-4d25-a856-9152768e8db6",
"nodeID": "97f53391-97d1-4db6-817a-6b902fe0caee",
"hostname": "",
"health": "online",
"promotable": true
},
"replicas": [
{
"id": "129267f4-5ac4-466b-80dc-ef2ce64f2720",
"nodeID": "5ca08388-71d4-48b2-9d40-2d202c39e94e",
"hostname": "",
"health": "ready",
"promotable": true
}
],
"version": "MjM",
"createdAt": "2023-03-16T13:00:51Z",
"updatedAt": "2023-03-17T15:21:47Z",
"snapshotConfig": {},
"snapshotVolumes": {},
"parentInfo": null
},
{
"id": "f43ca95c-bdbd-4650-88b6-b143d7e9e50c",
"namespaceID": "a92a8f34-4cfa-4db1-bca2-a266e0906acc",
"attachedOn": "",
"attachmentType": "detached",
"attachedOnHost": "",
"nfs": {},
"labels": {
"app.kubernetes.io/component": "primary",
"app.kubernetes.io/instance": "state",
"app.kubernetes.io/name": "mariadb",
"csi.storage.k8s.io/pv/name": "pvc-05177deb-5634-43fb-8950-aac94d39f8c2",
"csi.storage.k8s.io/pvc/name": "data-state-mariadb-primary-0",
"csi.storage.k8s.io/pvc/namespace": "main",
"storageos.com/nocompress": "true",
"storageos.com/replicas": "2",
"storageos.com/topology-aware": "true"
},
"topologyLabels": {
"requisiteZones": [
"us-west-hil-dc1",
"ap-southeast-syd1",
"falkenstein",
"nuremberg",
"us-east-ash-dc1",
"eu-central-nbg1-dc3"
],
"preferredZones": [
"eu-central-nbg1-dc3",
"falkenstein",
"nuremberg",
"us-east-ash-dc1",
"us-west-hil-dc1",
"ap-southeast-syd1"
]
},
"name": "pvc-05177deb-5634-43fb-8950-aac94d39f8c2",
"fsType": "ext4",
"sizeBytes": 10737418240,
"description": "",
"master": {
"id": "65e5df8e-f7a7-40e7-975f-c7a2f716a59f",
"nodeID": "97f53391-97d1-4db6-817a-6b902fe0caee",
"hostname": "",
"health": "online",
"promotable": true
},
"replicas": [
{
"id": "4ca3000c-5316-47a8-9b3c-43826e89acdb",
"nodeID": "b6932610-5cb2-416d-bf11-ab52396caa67",
"hostname": "",
"health": "ready",
"promotable": true
},
{
"id": "d07ae817-31cb-42c7-a7af-e14476d6231a",
"nodeID": "5ca08388-71d4-48b2-9d40-2d202c39e94e",
"hostname": "",
"health": "ready",
"promotable": true
}
],
"version": "MTk",
"createdAt": "2023-03-15T15:45:42Z",
"updatedAt": "2023-03-17T15:21:51Z",
"snapshotConfig": {},
"snapshotVolumes": {},
"parentInfo": null
}
]I used the following bash script to remove the keys in the etcd cluster:
#!/bin/bash
# Check if an argument is provided
if [ $# -eq 0 ]; then
echo "Usage: $0 <search_string>"
exit 1
fi
# Store the search string
search_string=$1
# Set the etcd endpoint
endpoint="https://storageos-etcd-1.storageos-etcd.storageos:2379"
# Get all keys containing the search string
matching_keys=$(etcdctl --endpoints=$endpoint get "" --prefix --keys-only | grep "${search_string}")
# Print the matching keys and their values
echo "Matching keys and values:"
for key in $matching_keys; do
value=$(etcdctl --endpoints=$endpoint get "$key" --print-value-only)
echo -e "${key}\n${value}"
done
# Get all keys and values, then check if the value contains the search string
all_keys=$(etcdctl --endpoints=$endpoint get "" --prefix --keys-only)
for key in $all_keys; do
value=$(etcdctl --endpoints=$endpoint get "$key" --print-value-only)
if [[ "$value" == *"${search_string}"* ]]; then
matching_keys+=$'\n'"$key"
echo -e "${key}\n${value}"
fi
done
# Delete all matching keys
for key in $matching_keys; do
echo "Deleting $key"
etcdctl --endpoints=$endpoint del "$key"
done
echo "All keys with '${search_string}' in key or value have been deleted."