Orphaned Assets Fix

client/modules/IDE/actions/files.js

@@ -191,6 +191,16 @@ export function deleteFile(id, parentId) {
       apiClient
         .delete(`/projects/${state.project.id}/files/${id}`, deleteConfig)
         .then((response) => {
+          const file = state.files.find((f) => f.id === id);
+          if (file?.url && file.url.includes('/pending/')) {
+            const pendingIndex = file.url.indexOf('/pending/');
+            const objectKey = file.url
+              .substring(pendingIndex + 1)
+              .split('?')[0];
+            apiClient
+              .delete(`/S3/delete?objectKey=${objectKey}`)
+              .catch(() => {});
+          }
           dispatch(setProjectSavedTime(response.data.project.updatedAt));
           dispatch({
             type: ActionTypes.DELETE_FILE,

client/modules/IDE/actions/project.js

@@ -306,6 +306,8 @@ export function cloneProject(project) {
     generateNewIdsForChildren(rootFile, newFiles);
 
     // duplicate all files hosted on S3
+    const copiedAssetKeys = [];
+
     each(
       newFiles,
       (file, callback) => {
@@ -316,24 +318,34 @@ export function cloneProject(project) {
           (file.url.includes(S3_BUCKET_URL_BASE) ||
             file.url.includes(S3_BUCKET))
         ) {
-          const formParams = {
-            url: file.url
-          };
-          apiClient.post('/S3/copy', formParams).then((response) => {
-            file.url = response.data.url;
-            callback(null);
-          });
+          apiClient
+            .post('/S3/copy', { url: file.url })
+            .then((response) => {
+              file.url = response.data.url;
+              const objectKey = response.data.url.split('/').pop();
+              copiedAssetKeys.push(objectKey);
+              callback(null);
+            })
+            .catch(callback);
         } else {
           callback(null);
         }
       },
       (err) => {
-        // if not errors in duplicating the files on S3, then duplicate it
-        const formParams = Object.assign(
-          {},
-          { name: `${projectName} copy` },
-          { files: newFiles }
-        );
+        if (err) {
+          copiedAssetKeys.forEach((key) => {
+            apiClient.delete(`/S3/delete?objectKey=${key}`).catch(() => {});
+          });
+          dispatch({
+            type: ActionTypes.PROJECT_SAVE_FAIL,
+            error: err?.response?.data
+          });
+          return;
+        }
+        const formParams = {
+          name: `${projectName} copy`,
+          files: newFiles
+        };
         apiClient
           .post('/projects', formParams)
           .then((response) => {
@@ -343,6 +355,9 @@ export function cloneProject(project) {
             dispatch(setNewProject(response.data));
           })
           .catch((error) => {
+            copiedAssetKeys.forEach((key) => {
+              apiClient.delete(`/S3/delete?objectKey=${key}`).catch(() => {});
+            });
             dispatch({
               type: ActionTypes.PROJECT_SAVE_FAIL,
               error: error?.response?.data

server/controllers/aws.controller.js

@@ -32,7 +32,12 @@ function getExtension(filename) {
 }
 
 export function getObjectKey(url) {
-  const urlArray = url.split('/');
+  const pendingMarker = '/pending/';
+  const pendingIndex = url.indexOf(pendingMarker);
+  if (pendingIndex !== -1) {
+    return url.substring(pendingIndex + 1).split('?')[0];
+  }
+  const urlArray = url.split('?')[0].split('/');
   const objectKey = urlArray.pop();
   const userId = urlArray.pop();
   if (ObjectId.isValid(userId) && userId === new ObjectId(userId).toString()) {
@@ -67,7 +72,9 @@ export async function deleteObjectsFromS3(keyList) {
 export async function deleteObjectFromS3(req, res) {
   const userId = req.user.id;
   const { objectKey } = req.query;
-  const fullObjectKey = `${userId}/${objectKey}`;
+  const fullObjectKey = objectKey.startsWith('pending/')
+    ? objectKey
+    : `${userId}/${objectKey}`;
 
   try {
     await deleteObjectsFromS3([fullObjectKey]);
@@ -80,18 +87,31 @@ export async function deleteObjectFromS3(req, res) {
 
 export async function listObjectsInS3ForUser(userId) {
   try {
-    let assets = [];
-    const params = {
-      Bucket: process.env.S3_BUCKET,
-      Prefix: `${userId}/`
-    };
-
-    const data = await s3Client.send(new ListObjectsCommand(params));
-
-    assets = data.Contents?.map((object) => ({
-      key: object.Key,
-      size: object.Size
-    }));
+    const [savedData, pendingData] = await Promise.all([
+      s3Client.send(
+        new ListObjectsCommand({
+          Bucket: process.env.S3_BUCKET,
+          Prefix: `${userId}/`
+        })
+      ),
+      s3Client.send(
+        new ListObjectsCommand({
+          Bucket: process.env.S3_BUCKET,
+          Prefix: `pending/${userId}/`
+        })
+      )
+    ]);
+
+    const assets = [
+      ...(savedData.Contents?.map((object) => ({
+        key: object.Key,
+        size: object.Size
+      })) ?? []),
+      ...(pendingData.Contents?.map((object) => ({
+        key: object.Key,
+        size: object.Size
+      })) ?? [])
+    ];
 
     const projects = await Project.getProjectsForUserId(userId);
     const projectAssets = [];
@@ -156,7 +176,7 @@ export async function signS3(req, res) {
     const acl = 'public-read';
     const policy = S3Policy.generate({
       acl,
-      key: `${req.body.userId}/${filename}`,
+      key: `pending/${req.user.id}/${filename}`,
       bucket: process.env.S3_BUCKET,
       contentType: req.body.type,
       region: process.env.AWS_REGION,

server/controllers/project.controller.js

@@ -11,6 +11,10 @@ import Project from '../models/project';
 import { User } from '../models/user';
 import { resolvePathToFile } from '../utils/filePath';
 import { generateFileSystemSafeName } from '../utils/generateFileSystemSafeName';
+import {
+  commitPendingAssets,
+  rewritePendingFileUrls
+} from '../utils/pendingAssets';
 
 const s3Client = new S3Client({
   credentials: {
@@ -65,6 +69,12 @@ export async function updateProject(req, res) {
         updateData[field] = req.body[field];
       }
     });
+
+    if (updateData.files) {
+      await commitPendingAssets(req.user.id, updateData.files);
+      updateData.files = rewritePendingFileUrls(updateData.files, req.user.id);
+    }
+
     const updatedProject = await Project.findByIdAndUpdate(
       req.params.project_id,
       {
@@ -77,6 +87,7 @@ export async function updateProject(req, res) {
     )
       .populate('user', 'username')
       .exec();
+
     if (
       req.body.files &&
       updatedProject.files.length !== req.body.files.length

server/controllers/project.controller/createProject.js

@@ -4,26 +4,33 @@ import {
   FileValidationError,
   ProjectValidationError
 } from '../../domain-objects/Project';
+import {
+  commitPendingAssets,
+  rewritePendingFileUrls
+} from '../../utils/pendingAssets';
 
-export default function createProject(req, res) {
-  const projectValues = Object.assign({}, req.body, { user: req.user._id });
+export default async function createProject(req, res) {
+  try {
+    const projectValues = Object.assign({}, req.body, { user: req.user._id });
 
-  function sendFailure(err) {
-    res.status(400).json({ success: false });
-  }
+    if (projectValues.files) {
+      await commitPendingAssets(req.user.id, projectValues.files);
+      projectValues.files = rewritePendingFileUrls(
+        projectValues.files,
+        req.user.id
+      );
+    }
 
-  function populateUserData(newProject) {
-    return Project.populate(newProject, {
+    const newProject = await Project.create(projectValues);
+    const newProjectWithUser = await Project.populate(newProject, {
       path: 'user',
       select: 'username'
-    }).then((newProjectWithUser) => {
-      res.json(newProjectWithUser);
     });
-  }
 
-  return Project.create(projectValues)
-    .then(populateUserData)
-    .catch(sendFailure);
+    res.json(newProjectWithUser);
+  } catch (err) {
+    res.status(400).json({ success: false });
+  }
 }
 
 // TODO: What happens if you don't supply any files?
@@ -86,7 +93,13 @@ export async function apiCreateProject(req, res) {
       throw error;
     }
 
+    if (model.files) {
+      await commitPendingAssets(req.user.id, model.files);
+      model.files = rewritePendingFileUrls(model.files, req.user.id);
+    }
+
     const newProject = await model.save();
+
     res.status(201).json({ id: newProject.id });
   } catch (err) {
     handleErrors(err);

server/utils/pendingAssets.js

@@ -0,0 +1,76 @@
+import {
+  S3Client,
+  CopyObjectCommand,
+  DeleteObjectsCommand
+} from '@aws-sdk/client-s3';
+
+const s3Client = new S3Client({
+  credentials: {
+    accessKeyId: process.env.AWS_ACCESS_KEY,
+    secretAccessKey: process.env.AWS_SECRET_KEY
+  },
+  region: process.env.AWS_REGION
+});
+
+function getPendingKeyFromUrl(url, userId) {
+  const marker = `pending/${userId}/`;
+  if (!url || !url.includes(marker)) {
+    return null;
+  }
+  const filename = url.split('?')[0].split('/').pop();
+  return `pending/${userId}/${filename}`;
+}
+
+export function rewritePendingFileUrls(files, userId) {
+  const marker = `pending/${userId}/`;
+  const replacement = `${userId}/`;
+  return files.map((file) => {
+    if (file.url && file.url.includes(marker)) {
+      return Object.assign({}, file, {
+        url: file.url.replace(marker, replacement)
+      });
+    }
+    return file;
+  });
+}
+
+async function moveAssetFromPending(pendingKey, userId) {
+  const filename = pendingKey.split('/').pop();
+  const destinationKey = `${userId}/${filename}`;
+
+  await s3Client.send(
+    new CopyObjectCommand({
+      Bucket: process.env.S3_BUCKET,
+      CopySource: `${process.env.S3_BUCKET}/${pendingKey}`,
+      Key: destinationKey,
+      ACL: 'public-read'
+    })
+  );
+
+  await s3Client.send(
+    new DeleteObjectsCommand({
+      Bucket: process.env.S3_BUCKET,
+      Delete: { Objects: [{ Key: pendingKey }] }
+    })
+  );
+
+  return destinationKey;
+}
+
+export async function commitPendingAssets(userId, files = []) {
+  const pendingKeys = [
+    ...new Set(
+      files
+        .map((file) => getPendingKeyFromUrl(file.url, userId))
+        .filter(Boolean)
+    )
+  ];
+
+  if (pendingKeys.length === 0) {
+    return [];
+  }
+
+  return Promise.all(
+    pendingKeys.map((key) => moveAssetFromPending(key, userId))
+  );
+}