Test/rest list objects v2 queries

tests/commands/create_bucket.sh

@@ -25,8 +25,7 @@ create_bucket() {
     return 1
   fi
 
-  local exit_code=0
-  local error
+  local exit_code=0 error
   if [[ $1 == 's3' ]]; then
     error=$(send_command aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == 's3api' ]]; then

tests/commands/head_object.sh

@@ -59,10 +59,11 @@ head_object_rest_expect_success_callback() {
     return 1
   fi
   env_vars="BUCKET_NAME=$1 OBJECT_KEY=$2 $3"
-  if ! send_rest_command_expect_success_callback "$env_vars" "./tests/rest_scripts/head_object.sh" "200" "$4"; then
-    log 2 "error sending REST command and checking error"
+  if ! callback_result=$(send_rest_command_expect_success_callback "$env_vars" "./tests/rest_scripts/head_object.sh" "200" "$4" 2>&1); then
+    log 2 "error sending REST command and checking error: $callback_result"
     return 1
   fi
+  echo "$callback_result"
   return 0
 }
 

tests/commands/list_buckets.sh

@@ -118,9 +118,12 @@ list_buckets_rest() {
   if ! check_param_count_v2 "params, callback" 2 $#; then
     return 1
   fi
-  if ! send_rest_command_expect_success_callback "$1" "./tests/rest_scripts/list_buckets.sh" "200" "$2"; then
-    log 2 "error sending REST command and checking error"
+  if ! callback_result=$(send_rest_command_expect_success_callback "$1" "./tests/rest_scripts/list_buckets.sh" "200" "$2" 2>&1); then
+    log 2 "error sending REST command and checking error: $callback_result"
     return 1
+  else
+    buckets="$callback_result"
   fi
+  echo "$buckets"
   return 0
 }

tests/commands/upload_part_copy.sh

@@ -20,7 +20,7 @@ upload_part_copy() {
     return 1
   fi
   local etag_json
-  log 5 "parameters:  $1 $2 $3 $4 $5"
+  log 5 "parameters:  '$1' '$2' '$3' '$4' '$5'"
   etag_json=$(send_command aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))") || local uploaded=$?
   if [[ $uploaded -ne 0 ]]; then
     log 2 "Error uploading part $5: $etag_json"

tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh

@@ -193,10 +193,12 @@ complete_multipart_upload_invalid_object_size_string() {
   if ! check_param_count_v2 "bucket, key, file" 3 $#; then
     return 1
   fi
-  if ! multipart_upload_rest_before_completion "$1" "$2" "$3" 2; then
-    log 2 "error performing multipart upload before completion"
+  local response upload_id parts_payload
+  if ! response=$(multipart_upload_rest_before_completion "$1" "$2" "$3" 2 2>&1); then
+    log 2 "error performing multipart upload before completion: $response"
     return 1
   fi
+  read -r upload_id parts_payload <<< "$response"
   if ! complete_multipart_upload_rest_expect_error "$1" "$2" "$upload_id" "$parts_payload" "MULTIPART_OBJECT_SIZE=size" "400" "InvalidRequest" "Value for x-amz-mp-object-size header is invalid"; then
     log 2 "error completing multipart upload"
     return 1

tests/drivers/delete_bucket/delete_bucket_rest.sh

@@ -29,12 +29,12 @@ delete_buckets_with_prefix() {
     log 2 "delete_buckets_with_prefix requires non-empty prefix"
     return 1
   fi
-  if ! list_buckets_rest "PREFIX=$1" "parse_bucket_list"; then
-    log 2 "error listing buckets with prefix"
+  local response
+  if ! response=$(list_buckets_rest "PREFIX=$1" "parse_bucket_list" 2>&1); then
+    log 2 "error listing buckets with prefix: $response"
     return 1
   fi
-  # shellcheck disable=SC2154
-  log 5 "buckets: ${bucket_array[*]}"
+  read -r -a bucket_array <<< "$response"
   for bucket in "${bucket_array[@]}"; do
     if ! delete_bucket_recursive "$bucket"; then
       log 2 "error with recursive bucket delete of bucket '$bucket'"

tests/drivers/delete_object/delete_object_rest.sh

@@ -24,13 +24,16 @@ list_and_delete_objects() {
   if ! check_param_count "list_and_delete_objects" "bucket" 1 $#; then
     return 1
   fi
-  if ! list_objects_rest "$1" "parse_objects_list_rest"; then
-    log 2 "error getting object list"
+  local response
+  if ! response=$(list_objects_rest "$1" "parse_objects_list_rest" 2>&1); then
+    log 2 "error getting object list: $response"
     return 1
   fi
-  # shellcheck disable=SC2154
-  log 5 "objects: ${object_array[*]}"
+  mapfile -t object_array <<< "$response"
   for object in "${object_array[@]}"; do
+    if [ "$object" == "" ]; then
+      break
+    fi
     if ! clear_object_in_bucket "$1" "$object"; then
       log 2 "error deleting object $object"
       return 1
@@ -114,11 +117,16 @@ put_object_with_lock_mode_and_delete_latest_version() {
       log 2 "error sending put object command with object lock"
       return 1
   fi
-  if ! send_rest_go_command_callback "200" "parse_latest_version_id" \
-      "-method" "GET" "-bucketName" "$2" "-query" "versions="; then
-    log 2 "error checking versions before deletion"
+
+  local response
+  if ! response=$(send_rest_go_command_callback "200" "parse_latest_version_id" \
+      "-method" "GET" "-bucketName" "$2" "-query" "versions=" 2>&1); then
+    log 2 "error checking versions before deletion: $response"
     return 1
+  else
+    version_id="$response"
   fi
+  log 5 "version ID: $version_id"
   if ! delete_object_version_rest_expect_error "$2" "$3" "$version_id" "403" "AccessDenied" "object protected by object lock"; then
     log 2 "shouldn't have been able to delete"
     return 1
@@ -140,11 +148,15 @@ attempt_to_delete_version_after_retention_policy() {
       log 2 "error sending put object command"
       return 1
   fi
-  if ! send_rest_go_command_callback "200" "parse_latest_version_id" \
-      "-method" "GET" "-bucketName" "$2" "-query" "versions="; then
-    log 2 "error checking versions before deletion"
+
+  local response
+  if ! response=$(send_rest_go_command_callback "200" "parse_latest_version_id" \
+      "-method" "GET" "-bucketName" "$2" "-query" "versions=" 2>&1); then
+    log 2 "error checking versions before deletion: $response"
     return 1
   fi
+
+  version_id="$response"
   if ! delete_object_version_rest_expect_error "$2" "$3" "$version_id" "403" "AccessDenied" "object protected by object lock"; then
     log 2 "shouldn't have been able to delete"
     return 1

tests/drivers/file.sh

@@ -163,15 +163,25 @@ setup_bucket_and_large_file_v2() {
 }
 
 setup_bucket_and_large_file_v3() {
-  if ! check_param_count_v2 "bucket env var" 1 $#; then
+  if ! check_param_count_ge_le "bucket env var, file size in MBs (optional)" 1 2 $#; then
     return 1
   fi
-  if ! bucket_name=$(setup_bucket_v3 "$1" 2>&1); then
-    log 2 "error setting up bucket: $bucket_name"
+
+  local response bucket_name file_name error
+  if ! response=$(setup_bucket_v3 "$1" 2>&1); then
+    log 2 "error setting up bucket: $response"
     return 1
   fi
-  if ! file_name=$(create_large_file "$file_name" 2>&1); then
-    log 2 "error creating large file: $file_name"
+  bucket_name="$response"
+
+  if ! response=$(get_file_name 2>&1); then
+    log 2 "error getting file name: $response"
+    return 1
+  fi
+  file_name="$response"
+
+  if ! error=$(create_large_file "$file_name" "$2" 2>&1); then
+    log 2 "error creating large file: $error"
     return 1
   fi
   echo "$bucket_name $file_name"
@@ -521,65 +531,147 @@ compare_files() {
   return 1
 }
 
-# Usage: create_large_file [filename] [size_in_mb]
-# If filename is omitted, it generates one. Defaults to 160MB.
+# generate 160MB file
+# input: filename
+# fail on error
 create_large_file() {
-  if ! check_param_count_le "filename (optional), size in MB (optional)" 2 $#; then
+  log 6 "create_large_file"
+  if ! check_param_count_ge_le "file name, size in MBs (optional)" 1 2 $#; then
+    return 1
+  fi
+  file_size=${2:-160}
+  if ! error=$(create_large_file_with_size "$1" "$file_size" 2>&1); then
+    log 2 "error creating 160MB file: $error"
     return 1
   fi
+  return 0
+}
 
-  local file_name="$1"
-  local size_mb="${2:-160}"
-  local error
+create_large_file_v2() {
+  local response file_name
+  if ! response=$(get_file_name 2>&1); then
+    log 2 "error getting file name: $response"
+    return 1
+  fi
+  file_name="$response"
 
-  if [ -z "$TEST_FILE_FOLDER" ]; then
-    log 2 "TEST_FILE_FOLDER must be defined"
+  if ! error=$(create_large_file_with_size "$file_name" 160 2>&1); then
+    log 2 "error creating 160MB file with name '$file_name': $error"
     return 1
   fi
+  echo "$file_name"
+  return 0
+}
 
-  if [[ -z "$file_name" ]]; then
-    if ! file_name=$(get_file_name 2>&1); then
-      log 2 "error generating automatic file name: $file_name"
-      return 1
-    fi
+create_large_file_with_size() {
+  if ! check_param_count_v2 "file name, size in MB" 2 $#; then
+    return 1
+  fi
+  file_size=$(($2*1024*1024))
+  if ! error=$(create_test_file_base "$1" "$file_size" 2>&1); then
+    log 2 "error creating large file with size ${2}MB: $error"
+    return 1
+  fi
+  return 0
+}
+
+create_test_file_base() {
+  if ! check_param_count_ge_le "file name, size (optional), block size (optional)" 1 3 $#; then
+    return 1
   fi
+  log 5 "params: $*"
 
-  log 6 "Creating ${size_mb}MB file: $file_name"
-  # bs=1M is significantly faster than bs=1024 for large files
-  if ! error=$(dd if=/dev/urandom of="${TEST_FILE_FOLDER}/${file_name}" bs=1M count="$size_mb" 2>&1); then
-    log 2 "error creating ${size_mb}MB file at ${file_name}: $error"
+  local file_name=$1
+  local requested_size=${2:-10}
+  local block_size=${3:-1048576}
+
+  if [ "$file_name" == "" ]; then
+    log 2 "no file name specified"
     return 1
   fi
+  local full_path="$TEST_FILE_FOLDER/$file_name"
+
+  if [[ "$requested_size" -eq 0 ]]; then
+    touch "$full_path"
+  elif [[ "$requested_size" -lt "$block_size" ]]; then
+    # For small files, use the size as the block size (1 write)
+    if ! error=$(dd if=/dev/urandom of="$full_path" bs="$requested_size" count=1 conv=notrunc 2>&1); then
+      log 2 "error writing file: $error"
+      return 1
+    fi
+  else
+    # For large files, use 1MB chunks to save RAM
+    count=$(( requested_size / block_size ))
+    remainder=$(( requested_size % block_size ))
 
+    # Write the big 1MB chunks
+    if ! error=$(dd if=/dev/urandom of="$full_path" bs="$block_size" count="$count" conv=notrunc 2>&1); then
+      log 2 "error writing file chunk: $error"
+      return 1
+    fi
+
+    # If there's a remainder (e.g., 1.5MB), append the last few bytes
+    if [[ $remainder -gt 0 ]] && ! error=$(dd if=/dev/urandom of="$full_path" bs=1 count="$remainder" oflag=append conv=notrunc 2>&1); then
+      log 2 "error writing final file chunk: $error"
+      return 1
+    fi
+  fi
   echo "$file_name"
   return 0
 }
 
+create_and_split_large_file() {
+if ! check_param_count_v2 "file name, size in MB, pieces" 3 $#; then
+  return 1
+fi
+if ! error=$(create_large_file_with_size "$1" "$2" 2>&1); then
+  log 2 "error creating large file: $error"
+  return 1
+fi
+if ! split_file "$TEST_FILE_FOLDER/$1" "$3"; then
+  log 2 "error splitting file"
+  return 1
+fi
+}
+
 # param: number of files
 # fail on error
 create_test_file_count() {
   if ! check_param_count_v2 "number of files" 1 $#; then
     return 1
   fi
+  if [ -z "$TEST_FILE_FOLDER" ]; then
+    log 2 "TEST_FILE_FOLDER not defined"
+    return 1
+  fi
+
+  local response file_prefix
+  if ! response=$(get_file_name 2>&1); then
+    log 2 "error getting file prefix: $response"
+    return 1
+  fi
+  file_prefix="$response"
+
   for ((i=1;i<=$1;i++)) {
-    if ! error=$(touch "$TEST_FILE_FOLDER/file_$i" 2>&1); then
+    if ! error=$(touch "$TEST_FILE_FOLDER/${file_prefix}_${i}" 2>&1); then
       log 2 "error creating file_$i: $error"
       return 1
     fi
   }
   # shellcheck disable=SC2153
   if [[ $LOG_LEVEL -ge 5 ]]; then
-    ls_result=$(ls "$TEST_FILE_FOLDER/file_*")
+    ls_result=$(ls "$TEST_FILE_FOLDER/${file_prefix}_*" 2>&1)
     log 5 "$ls_result"
   fi
+  echo "$file_prefix"
   return 0
 }
 
 download_and_compare_file_with_user() {
   if ! check_param_count_gt "original file, bucket, key, destination, username, password, chunk size (optional)" 6 $#; then
     return 1
   fi
-  if [ -e "$4" ] && ! error=$(rm -f "$4"); then
+  if [ -e "$4" ] && ! error=$(rm -f "$4" 2>&1); then
     log 2 "error deleting local file at download destination before download: $error"
     return 1
   fi

tests/drivers/get_object_tagging/get_object_tagging_rest.sh

@@ -88,10 +88,10 @@ get_check_object_tags_empty() {
 }
 
 check_header_version_id() {
-  if ! check_param_count_v2 "data file" 1 $#; then
+  if ! check_param_count_v2 "data file, version ID" 2 $#; then
     return 1
   fi
-  if ! check_for_header_key_and_value "$1" "x-amz-version-id" "$version_id"; then
+  if ! check_for_header_key_and_value "$1" "x-amz-version-id" "$2"; then
     log 2 "error checking for x-amz-version-id header"
     return 1
   fi
@@ -107,7 +107,7 @@ add_version_tags_check_version_id() {
     return 1
   fi
   if ! send_rest_go_command_callback "200" "check_header_version_id" "-bucketName" "$1" "-objectKey" "$2" "-debug" "-logFile" "signature.log" \
-        "-method" "GET" "-query" "tagging=&versionId=$version_id" "-tagKey" "key" "-tagValue" "value" "-contentMD5"; then
+        "-method" "GET" "-query" "tagging=&versionId=$version_id" "-tagKey" "key" "-tagValue" "value" "-contentMD5" "--" "$version_id"; then
     log 2 "error tagging object"
     return 1
   fi