Skip to content

Commit daeabb6

Browse files
whispinwhispin
committed
Add robust verification and logging for archive operations
1 parent 1571745 commit daeabb6

2 files changed

Lines changed: 173 additions & 23 deletions

File tree

‎scripts/backup-functions.sh‎

Lines changed: 114 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -290,36 +290,96 @@ compress_and_upload() {
290290
return 0
291291
fi
292292

293-
log "INFO" "Compressing $source_file..."
294-
temp_compressed_file=$(mktemp)
293+
log "INFO" "Compressing $source_file (size: $(du -h "$source_file" | cut -f1))..."
294+
temp_compressed_file=$(mktemp --suffix=.gz)
295295

296296
# Ensure cleanup on exit
297297
trap "rm -f '$temp_compressed_file'" EXIT
298298

299-
if ! gzip -c "$source_file" > "$temp_compressed_file"; then
299+
# Use gzip with better compression and verification
300+
if ! gzip -9 -c "$source_file" > "$temp_compressed_file"; then
300301
log "ERROR" "Failed to compress $source_file."
301302
rm -f "$temp_compressed_file"
302303
return 1
303304
fi
304305

305-
log "INFO" "Verifying compressed file..."
306-
if ! gzip -t "$temp_compressed_file"; then
307-
log "ERROR" "Compressed file is corrupted. Aborting upload."
306+
# Verify the compressed file is not empty
307+
if [[ ! -s "$temp_compressed_file" ]]; then
308+
log "ERROR" "Compressed file is empty."
308309
rm -f "$temp_compressed_file"
309310
return 1
310311
fi
311-
log "INFO" "Compressed file verified successfully."
312312

313-
log "INFO" "Uploading $source_file to ${remote_path}/${remote_filename}"
313+
log "INFO" "Verifying compressed file integrity..."
314+
if ! gzip -t "$temp_compressed_file" 2>/dev/null; then
315+
log "ERROR" "Compressed file integrity check failed."
316+
rm -f "$temp_compressed_file"
317+
return 1
318+
fi
319+
320+
# Log compression ratio
321+
local original_size=$(stat -c%s "$source_file")
322+
local compressed_size=$(stat -c%s "$temp_compressed_file")
323+
local ratio=$(( (original_size - compressed_size) * 100 / original_size ))
324+
log "INFO" "Compression successful. Ratio: ${ratio}% ($(du -h "$temp_compressed_file" | cut -f1))"
325+
326+
log "INFO" "Uploading compressed file to ${remote_path}/${remote_filename}"
327+
328+
# Create remote directory first
329+
if ! rclone mkdir "${REMOTE_NAME}:${remote_path}/" --config "$RCLONE_CONFIG_PATH"; then
330+
log "WARN" "Failed to create remote directory (may already exist)"
331+
fi
332+
333+
# Upload with retry mechanism and verification
334+
local upload_attempts=3
335+
local attempt=1
336+
337+
while [ $attempt -le $upload_attempts ]; do
338+
log "INFO" "Upload attempt $attempt/$upload_attempts"
339+
340+
if rclone copy "$temp_compressed_file" "${REMOTE_NAME}:${remote_path}/" \
341+
--config "$RCLONE_CONFIG_PATH" \
342+
--progress \
343+
--checksum \
344+
--timeout=300s; then
345+
346+
# Verify upload by checking file size
347+
local remote_size=$(rclone size "${REMOTE_NAME}:${remote_path}/$(basename "$temp_compressed_file")" --config "$RCLONE_CONFIG_PATH" 2>/dev/null | grep "Total size:" | awk '{print $3}' || echo "0")
348+
local local_size=$(stat -c%s "$temp_compressed_file")
349+
350+
if [[ "$remote_size" == "$local_size" ]]; then
351+
log "INFO" "Upload verified successfully"
352+
break
353+
else
354+
log "WARN" "Upload size mismatch (local: $local_size, remote: $remote_size)"
355+
attempt=$((attempt + 1))
356+
fi
357+
else
358+
log "WARN" "Upload attempt $attempt failed"
359+
attempt=$((attempt + 1))
360+
if [ $attempt -le $upload_attempts ]; then
361+
sleep 5
362+
fi
363+
fi
364+
done
314365

315-
if ! rclone rcat "${REMOTE_NAME}:${remote_path}/${remote_filename}" --config "$RCLONE_CONFIG_PATH" < "$temp_compressed_file"; then
316-
log "ERROR" "Failed to upload $source_file."
366+
if [ $attempt -gt $upload_attempts ]; then
367+
log "ERROR" "Failed to upload $source_file after $upload_attempts attempts."
317368
rm -f "$temp_compressed_file"
318369
return 1
319370
fi
320371

372+
# Rename to final filename if needed
373+
if [[ "$(basename "$temp_compressed_file")" != "$remote_filename" ]]; then
374+
if ! rclone move "${REMOTE_NAME}:${remote_path}/$(basename "$temp_compressed_file")" "${REMOTE_NAME}:${remote_path}/${remote_filename}" --config "$RCLONE_CONFIG_PATH"; then
375+
log "ERROR" "Failed to rename uploaded file to $remote_filename"
376+
rm -f "$temp_compressed_file"
377+
return 1
378+
fi
379+
fi
380+
321381
rm -f "$temp_compressed_file"
322-
log "INFO" "Successfully uploaded $remote_filename."
382+
log "INFO" "Successfully uploaded and verified $remote_filename"
323383

324384
# Clear the trap since we've cleaned up manually
325385
trap - EXIT
@@ -711,26 +771,62 @@ create_compressed_repository_archive() {
711771

712772
log "INFO" "Creating compressed repository archive: $archive_name"
713773

714-
# Check if repository directory exists
774+
# Check if repository directory exists and has content
715775
if [[ ! -d "$repo_path" ]]; then
716776
log "ERROR" "pgBackRest repository directory does not exist: $repo_path"
717777
return 1
718778
fi
719779

720-
# Create compressed archive of the entire repository
721-
if ! tar -czf "$archive_path" -C "$(dirname "$repo_path")" "$(basename "$repo_path")"; then
780+
# Check if repository has content
781+
if [[ ! "$(ls -A "$repo_path" 2>/dev/null)" ]]; then
782+
log "WARN" "pgBackRest repository directory is empty: $repo_path"
783+
# Create a minimal archive to avoid issues
784+
mkdir -p "${repo_path}/empty"
785+
echo "Repository was empty during backup" > "${repo_path}/empty/README.txt"
786+
fi
787+
788+
# Create compressed archive with better compression and error handling
789+
log "INFO" "Creating tar.gz archive from $repo_path"
790+
if ! tar --create \
791+
--gzip \
792+
--file="$archive_path" \
793+
--directory="$(dirname "$repo_path")" \
794+
--verbose \
795+
--exclude="*.lock" \
796+
--exclude="*.tmp" \
797+
"$(basename "$repo_path")"; then
722798
log "ERROR" "Failed to create compressed repository archive"
799+
rm -f "$archive_path"
723800
return 1
724801
fi
725802

726-
# Verify archive integrity
727-
if ! tar -tzf "$archive_path" > /dev/null; then
728-
log "ERROR" "Compressed repository archive verification failed"
803+
# Verify archive integrity with multiple checks
804+
log "INFO" "Verifying archive integrity..."
805+
806+
# Check if file exists and has size
807+
if [[ ! -s "$archive_path" ]]; then
808+
log "ERROR" "Archive file is empty or does not exist"
809+
rm -f "$archive_path"
810+
return 1
811+
fi
812+
813+
# Test gzip integrity
814+
if ! gzip -t "$archive_path" 2>/dev/null; then
815+
log "ERROR" "Archive gzip integrity check failed"
816+
rm -f "$archive_path"
817+
return 1
818+
fi
819+
820+
# Test tar listing
821+
if ! tar -tzf "$archive_path" >/dev/null 2>&1; then
822+
log "ERROR" "Archive tar listing check failed"
729823
rm -f "$archive_path"
730824
return 1
731825
fi
732826

733-
log "INFO" "Compressed repository archive created and verified: $archive_path"
827+
# Get file size for logging
828+
local file_size=$(du -h "$archive_path" | cut -f1)
829+
log "INFO" "Compressed repository archive created successfully: $archive_path (size: $file_size)"
734830
echo "$archive_path"
735831
return 0
736832
}

‎scripts/recovery.sh‎

Lines changed: 59 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -117,24 +117,78 @@ download_and_extract_repository_archive() {
117117

118118
local temp_archive="/tmp/${archive_name}"
119119

120-
# Download the archive
121-
if ! rclone copy "${REMOTE_NAME}:${remote_repo_path}/${archive_name}" "/tmp/" --config "$RCLONE_CONFIG_PATH"; then
120+
# Download the archive with verification
121+
recovery_log "INFO" "Downloading archive..."
122+
if ! rclone copy "${REMOTE_NAME}:${remote_repo_path}/${archive_name}" "/tmp/" \
123+
--config "$RCLONE_CONFIG_PATH" \
124+
--progress \
125+
--checksum; then
122126
recovery_log "ERROR" "Failed to download repository archive: $archive_name"
123127
return 1
124128
fi
125129

126-
# Extract the archive
130+
# Verify downloaded file exists and has content
131+
if [[ ! -s "$temp_archive" ]]; then
132+
recovery_log "ERROR" "Downloaded archive is empty or missing: $temp_archive"
133+
rm -f "$temp_archive"
134+
return 1
135+
fi
136+
137+
# Get file size for logging
138+
local file_size=$(du -h "$temp_archive" | cut -f1)
139+
recovery_log "INFO" "Downloaded archive size: $file_size"
140+
141+
# Verify archive integrity before extraction
142+
recovery_log "INFO" "Verifying archive integrity..."
143+
if ! gzip -t "$temp_archive" 2>/dev/null; then
144+
recovery_log "ERROR" "Archive gzip integrity check failed"
145+
rm -f "$temp_archive"
146+
return 1
147+
fi
148+
149+
if ! tar -tzf "$temp_archive" >/dev/null 2>&1; then
150+
recovery_log "ERROR" "Archive tar integrity check failed"
151+
rm -f "$temp_archive"
152+
return 1
153+
fi
154+
155+
recovery_log "INFO" "Archive integrity verified successfully"
156+
157+
# Remove existing repository if it exists
158+
if [[ -d "$local_repo_path" ]]; then
159+
recovery_log "INFO" "Removing existing repository directory"
160+
rm -rf "$local_repo_path"
161+
fi
162+
163+
# Create parent directory
164+
mkdir -p "$(dirname "$local_repo_path")"
165+
166+
# Extract the archive with verbose output
127167
recovery_log "INFO" "Extracting repository archive..."
128-
if ! tar -xzf "$temp_archive" -C "$(dirname "$local_repo_path")"; then
168+
if ! tar --extract \
169+
--gzip \
170+
--file="$temp_archive" \
171+
--directory="$(dirname "$local_repo_path")" \
172+
--verbose; then
129173
recovery_log "ERROR" "Failed to extract repository archive"
130174
rm -f "$temp_archive"
131175
return 1
132176
fi
133177

178+
# Verify extraction was successful
179+
if [[ ! -d "$local_repo_path" ]]; then
180+
recovery_log "ERROR" "Repository directory not found after extraction: $local_repo_path"
181+
rm -f "$temp_archive"
182+
return 1
183+
fi
184+
134185
# Clean up temporary archive
135186
rm -f "$temp_archive"
136187

137-
recovery_log "INFO" "Repository archive extracted successfully"
188+
# Log extraction results
189+
local extracted_size=$(du -sh "$local_repo_path" | cut -f1)
190+
recovery_log "INFO" "Repository archive extracted successfully (size: $extracted_size)"
191+
138192
return 0
139193
}
140194

0 commit comments

Comments
 (0)