Skip to content

Commit f09cd84

Browse files
knollearyknolleary
authored
Merge pull request #335 from FlowFuse/retry-fixed-again
Checking correct error field for retry
2 parents 83ca90c + 57527e9 commit f09cd84

4 files changed

Lines changed: 125 additions & 48 deletions

File tree

‎kubernetes.js‎

Lines changed: 58 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -419,43 +419,54 @@ const createCustomIngress = async (project, hostname, options) => {
419419

420420
const createPersistentVolumeClaim = async (project, options) => {
421421
const namespace = this._app.config.driver.options?.projectNamespace || 'flowforge'
422-
const pvc = JSON.parse(JSON.stringify(persistentVolumeClaimTemplate))
422+
const name = `${project.id}-pvc`
423+
try {
424+
await this._k8sApi.readNamespacedPersistentVolumeClaim({ name, namespace })
425+
// exists no need to recreate
426+
return undefined
427+
} catch (err) {
428+
if (err.code === 404 || err.response?.statusCode === 404) {
429+
const pvc = JSON.parse(JSON.stringify(persistentVolumeClaimTemplate))
423430

424-
const drvOptions = this._app.config.driver.options
425-
const allowedAccessModes = new Set(['ReadWriteOnce', 'ReadWriteMany', 'ReadWriteOncePod'])
426-
const configuredAccessMode = drvOptions?.storage?.accessMode
431+
const drvOptions = this._app.config.driver.options
432+
const allowedAccessModes = new Set(['ReadWriteOnce', 'ReadWriteMany', 'ReadWriteOncePod'])
433+
const configuredAccessMode = drvOptions?.storage?.accessMode
427434

428-
if (configuredAccessMode !== undefined) {
429-
if (!allowedAccessModes.has(configuredAccessMode)) {
430-
throw new Error(`Unsupported storage.accessMode '${configuredAccessMode}'. Allowed values: ${Array.from(allowedAccessModes).join(', ')}`)
431-
}
432-
pvc.spec.accessModes = [configuredAccessMode]
433-
}
435+
if (configuredAccessMode !== undefined) {
436+
if (!allowedAccessModes.has(configuredAccessMode)) {
437+
throw new Error(`Unsupported storage.accessMode '${configuredAccessMode}'. Allowed values: ${Array.from(allowedAccessModes).join(', ')}`)
438+
}
439+
pvc.spec.accessModes = [configuredAccessMode]
440+
}
434441

435-
if (drvOptions?.storage?.storageClass) {
436-
pvc.spec.storageClassName = drvOptions.storage.storageClass
437-
} else if (drvOptions?.storage?.storageClassEFSTag) {
438-
pvc.spec.storageClassName = await awsEFS.lookupStorageClass(drvOptions?.storage?.storageClassEFSTag)
439-
}
442+
if (drvOptions?.storage?.storageClass) {
443+
pvc.spec.storageClassName = drvOptions.storage.storageClass
444+
} else if (drvOptions?.storage?.storageClassEFSTag) {
445+
pvc.spec.storageClassName = await awsEFS.lookupStorageClass(drvOptions?.storage?.storageClassEFSTag)
446+
}
440447

441-
if (drvOptions?.storage?.size) {
442-
pvc.spec.resources.requests.storage = drvOptions.storage.size
443-
}
448+
if (drvOptions?.storage?.size) {
449+
pvc.spec.resources.requests.storage = drvOptions.storage.size
450+
}
444451

445-
pvc.metadata.namespace = namespace
446-
pvc.metadata.name = `${project.id}-pvc`
447-
pvc.metadata.labels = {
448-
'ff-project-id': project.id,
449-
'ff-project-name': project.safeName
450-
}
451-
if (this._app.config.driver.options?.projectLabels) {
452-
pvc.metadata.labels = {
453-
...pvc.metadata.labels,
454-
...this._app.config.driver.options.projectLabels
452+
pvc.metadata.namespace = namespace
453+
pvc.metadata.name = name
454+
pvc.metadata.labels = {
455+
'ff-project-id': project.id,
456+
'ff-project-name': project.safeName
457+
}
458+
if (this._app.config.driver.options?.projectLabels) {
459+
pvc.metadata.labels = {
460+
...pvc.metadata.labels,
461+
...this._app.config.driver.options.projectLabels
462+
}
463+
}
464+
console.error(`PVC: ${JSON.stringify(pvc, null, 2)}`)
465+
return pvc
466+
} else {
467+
throw err
455468
}
456469
}
457-
console.error(`PVC: ${JSON.stringify(pvc, null, 2)}`)
458-
return pvc
459470
}
460471

461472
const createProject = async (project, options) => {
@@ -468,17 +479,19 @@ const createProject = async (project, options) => {
468479
if (this._app.config.driver.options?.storage?.enabled) {
469480
const localPVC = await createPersistentVolumeClaim(project, options)
470481
// console.log(JSON.stringify(localPVC, null, 2))
471-
try {
472-
await this._k8sApi.createNamespacedPersistentVolumeClaim({ namespace, body: localPVC })
473-
} catch (err) {
474-
console.error(JSON.stringify(err))
475-
if (err.code === 409) {
476-
this._app.log.warn(`[k8s] PVC for instance ${project.id} already exists, proceeding...`)
477-
} else {
478-
if (project.state !== 'suspended') {
479-
this._app.log.error(`[k8s] Instance ${project.id} - error creating PVC: ${err.toString()} ${err.code} ${err.stack}`)
480-
// console.log(err)
481-
throw err
482+
if (localPVC !== undefined) {
483+
try {
484+
await this._k8sApi.createNamespacedPersistentVolumeClaim({ namespace, body: localPVC })
485+
} catch (err) {
486+
console.error(JSON.stringify(err))
487+
if (err.code === 409) {
488+
this._app.log.warn(`[k8s] PVC for instance ${project.id} already exists, proceeding...`)
489+
} else {
490+
if (project.state !== 'suspended') {
491+
this._app.log.error(`[k8s] Instance ${project.id} - error creating PVC: ${err.toString()} ${err.code} ${err.stack}`)
492+
// console.log(err)
493+
throw err
494+
}
482495
}
483496
}
484497
}
@@ -735,8 +748,8 @@ const waitForInstanceRunning = async (endpoint) => {
735748
// functions to wrap k8s api functions in retry logic
736749
const retry = (driver, api, func, args, delay, times) => {
737750
return func.apply(api, args).catch(err => {
738-
driver._app.log.error(`[k8s] API call to ${func.name} failed. attempt=${driver._k8sRetries - times + 1}/${driver._k8sRetries + 1} statusCode=${err.response?.statusCode || 'N/A'} ${err.toString()}`)
739-
if (times > 0 && err.response && err.response.statusCode === 429) {
751+
driver._app.log.error(`[k8s] API call to ${func.name} failed. attempt=${driver._k8sRetries - times + 1}/${driver._k8sRetries + 1} statusCode=${err.code || 'N/A'} ${err.toString()}`)
752+
if (times > 0 && err.code === 429) {
740753
return new Promise(resolve => {
741754
setTimeout(() => { resolve(retry(driver, api, func, args, delay * 2, times - 1)) }, delay)
742755
})
@@ -807,7 +820,8 @@ module.exports = {
807820
this._k8sApi.deleteNamespacedPod,
808821
this._k8sApi.deleteNamespacedSecret,
809822
this._k8sApi.deleteNamespacedService,
810-
this._k8sApi.deleteNamespacedPersistentVolumeClaim
823+
this._k8sApi.deleteNamespacedPersistentVolumeClaim,
824+
this._k8sApi.readNamespacedPersistentVolumeClaim
811825
], this)
812826
wrapClient(this._k8sAppApi, [
813827
this._k8sAppApi.createNamespacedDeployment,

‎lib/aws-efs.js‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
const { EFSClient, DescribeFileSystemsCommand, DescribeAccessPointsCommand } = require('@aws-sdk/client-efs')
1+
const { EFSClient, DescribeFileSystemsCommand, DescribeAccessPointsCommand, ThrottlingException } = require('@aws-sdk/client-efs')
2+
const retry = require('async-retry')
23

34
let client
45

@@ -10,7 +11,22 @@ async function lookupStorageClass (tagName) {
1011
}
1112

1213
const fsCommand = new DescribeFileSystemsCommand()
13-
const fsList = await client.send(fsCommand)
14+
const fsList = await retry(async (bail) => {
15+
try {
16+
const list = await client.send(fsCommand)
17+
return list
18+
} catch (err) {
19+
if (err instanceof ThrottlingException) {
20+
throw err // retry after delay
21+
} else {
22+
return bail(err) // not Throttling, time to fail
23+
}
24+
}
25+
},
26+
{
27+
retries: 5,
28+
minTimeout: 500
29+
})
1430
// console.log(JSON.stringify(fsList, null, 2))
1531

1632
const fileSystems = []
@@ -31,11 +47,25 @@ async function lookupStorageClass (tagName) {
3147
// console.log(storageClass)
3248
const apParams = {
3349
FileSystemId: fsList.FileSystems[i].FileSystemId,
34-
MaxResults: 999
50+
MaxResults: 9999 // max access points per filesystem is now 10,000
3551
}
3652
// console.log(apParams)
3753
const apListCommand = new DescribeAccessPointsCommand(apParams)
38-
const apList = await client.send(apListCommand)
54+
const apList = await retry(async (bail) => {
55+
try {
56+
const list = await client.send(apListCommand)
57+
return list
58+
} catch (err) {
59+
if (err instanceof ThrottlingException) {
60+
throw err // retry after delay
61+
} else {
62+
return bail(err) // not Throttling, time to fail
63+
}
64+
}
65+
}, {
66+
retries: 5,
67+
minTimeout: 500
68+
})
3969
// fileSystems[fsList.FileSystems[i].FileSystemId]
4070
fileSystems.push({
4171
apCount: apList.AccessPoints.length,

‎package-lock.json‎

Lines changed: 32 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
"dependencies": {
2424
"@aws-sdk/client-efs": "^3.600.0",
2525
"@kubernetes/client-node": "^1.3.0",
26+
"async-retry": "^1.3.3",
2627
"form-data": "^4.0.0",
2728
"got": "^11.8.0",
2829
"lodash": "^4.17.21"

0 commit comments

Comments
 (0)