Skip to content

Commit 4a33f36

Browse files
committed
Retry instance directory deletion on ENOTEMPTY
1 parent 4dc8607 commit 4a33f36

File tree

2 files changed

+82
-1
lines changed

2 files changed

+82
-1
lines changed

lib/instances/manager_test.go

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1289,6 +1289,58 @@ func TestStorageOperations(t *testing.T) {
12891289
assert.ErrorIs(t, err, ErrNotFound)
12901290
}
12911291

1292+
func TestRemoveAllWithRetryRetriesDirectoryNotEmpty(t *testing.T) {
1293+
t.Parallel()
1294+
1295+
path := "/tmp/test-instance"
1296+
attempts := 0
1297+
var slept []time.Duration
1298+
1299+
err := removeAllWithRetry(path, func(got string) error {
1300+
attempts++
1301+
require.Equal(t, path, got)
1302+
if attempts <= 3 {
1303+
return &os.PathError{Op: "unlinkat", Path: got, Err: syscall.ENOTEMPTY}
1304+
}
1305+
return nil
1306+
}, func(d time.Duration) {
1307+
slept = append(slept, d)
1308+
})
1309+
1310+
require.NoError(t, err)
1311+
assert.Equal(t, 4, attempts)
1312+
assert.Equal(t, []time.Duration{
1313+
10 * time.Millisecond,
1314+
20 * time.Millisecond,
1315+
40 * time.Millisecond,
1316+
}, slept)
1317+
}
1318+
1319+
func TestRemoveAllWithRetryStopsAfterMaxRetries(t *testing.T) {
1320+
t.Parallel()
1321+
1322+
attempts := 0
1323+
var slept []time.Duration
1324+
1325+
err := removeAllWithRetry("/tmp/test-instance", func(string) error {
1326+
attempts++
1327+
return &os.PathError{Op: "unlinkat", Path: "/tmp/test-instance", Err: syscall.ENOTEMPTY}
1328+
}, func(d time.Duration) {
1329+
slept = append(slept, d)
1330+
})
1331+
1332+
require.Error(t, err)
1333+
assert.ErrorIs(t, err, syscall.ENOTEMPTY)
1334+
assert.Equal(t, deleteInstanceDataMaxRetries+1, attempts)
1335+
assert.Equal(t, []time.Duration{
1336+
10 * time.Millisecond,
1337+
20 * time.Millisecond,
1338+
40 * time.Millisecond,
1339+
80 * time.Millisecond,
1340+
100 * time.Millisecond,
1341+
}, slept)
1342+
}
1343+
12921344
func TestStandbyAndRestore(t *testing.T) {
12931345
t.Parallel()
12941346
// Require KVM access (don't skip, fail informatively)

lib/instances/storage.go

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,23 @@ package instances
22

33
import (
44
"encoding/json"
5+
"errors"
56
"fmt"
67
"os"
78
"path/filepath"
9+
"syscall"
10+
"time"
811

912
"github.com/kernel/hypeman/lib/autostandby"
1013
"github.com/kernel/hypeman/lib/images"
1114
)
1215

16+
const (
17+
deleteInstanceDataMaxRetries = 5
18+
deleteInstanceDataInitialBackoff = 10 * time.Millisecond
19+
deleteInstanceDataMaxBackoff = 100 * time.Millisecond
20+
)
21+
1322
// Filesystem structure:
1423
// {dataDir}/guests/{instance-id}/
1524
// metadata.json # Instance metadata
@@ -111,7 +120,7 @@ func (m *manager) createVolumeOverlayDisk(instanceID, volumeID string, sizeBytes
111120
func (m *manager) deleteInstanceData(id string) error {
112121
instDir := m.paths.InstanceDir(id)
113122

114-
if err := os.RemoveAll(instDir); err != nil {
123+
if err := removeAllWithRetry(instDir, os.RemoveAll, time.Sleep); err != nil {
115124
return fmt.Errorf("remove instance directory: %w", err)
116125
}
117126

@@ -120,6 +129,26 @@ func (m *manager) deleteInstanceData(id string) error {
120129
return nil
121130
}
122131

132+
func removeAllWithRetry(path string, removeAll func(string) error, sleep func(time.Duration)) error {
133+
backoff := deleteInstanceDataInitialBackoff
134+
135+
for attempt := 0; ; attempt++ {
136+
err := removeAll(path)
137+
if err == nil {
138+
return nil
139+
}
140+
if !errors.Is(err, syscall.ENOTEMPTY) || attempt >= deleteInstanceDataMaxRetries {
141+
return err
142+
}
143+
144+
sleep(backoff)
145+
backoff *= 2
146+
if backoff > deleteInstanceDataMaxBackoff {
147+
backoff = deleteInstanceDataMaxBackoff
148+
}
149+
}
150+
}
151+
123152
// listMetadataFiles returns paths to all instance metadata files
124153
func (m *manager) listMetadataFiles() ([]string, error) {
125154
guestsDir := m.paths.GuestsDir()

0 commit comments

Comments
 (0)