commit e9175bc1391f4f6c2ed8e0929b135b986ced7f36 Author: adam Date: Mon Oct 21 16:58:32 2024 +0100 initial commit diff --git a/space-transfer.sh b/space-transfer.sh new file mode 100644 index 0000000..e69f497 --- /dev/null +++ b/space-transfer.sh @@ -0,0 +1,172 @@ +#!/bin/bash + +LINUX_DIR="/mnt/linux" +WINDOWS_DIR="/mnt/windows" +LANDING_DIR="/mnt/os-transfer/space-landing/Import" +EXPORT_DIR="/mnt/os-transfer/space-landing/Export" + +TODAY_EGRESS="/root/scripts/"$(date +%Y-%m-%d)"_space_egress.list" +TODAY_INGRESS="/root/scripts/"$(date +%Y-%m-%d)"_space_ingress.list" +TODAY_FAILURES="/root/scripts/"$(date +%Y-%m-%d)"_space_failures.list" +TODAY_TRANSFER="/root/scripts/"$(date +%Y-%m-%d)"_space_transfer.list" + +SPACE_EMAIL_FILE="/root/scripts/space-email" + +function import_files { + for filepath in $LANDING_DIR/*; do + [[ -e "$filepath" ]] || continue + [[ -L "$filepath" ]] && continue # Prevent Symlinks MAYBE LOG IN EMAIL + [[ -d "$filepath" ]] && continue # Prevent Directory MAYBE LOG IN EMAIL + + local file=${filepath##*/} + + echo '--------------------------------------------------------------------------------------------------------------' + + + if [[ "$file" =~ ^eidf[a-zA-Z0-9]{2,5}-[a-zA-Z0-9]{3,8}-.+$ ]] && [[ -f "$filepath" ]]; then + echo "File '$file' is formatted correctly" + else + echo "IMPORT FAILED FOR '$file'. Incorrect formatting." | tee -a $TODAY_FAILURES + continue + fi + + # example file: eidf123-abc01-file.csv + local project="${file%%-*}" # eidf123 + local rest="${file#*-}" # abc01-file.csv + local study="${rest%%-*}" # abc01 + local filename="${rest#*-}" # file.csv + # Worth check if above variables are set??? + + echo "File '$filename' in project $project for study $study" + + local pattern="/mnt/*/$project*" + local matching_projects=($(compgen -G "$pattern")) + + if [[ -n "${project:+VALID}" ]]; then :; else continue; fi # check $project is valid POSSIBLY NOT NEEDED (above regex) + + case ${#matching_projects[@]} in + 0) + echo "IMPORT FAILED FOR FILE: $file - No project directory could be identified from the filename." | tee -a $TODAY_FAILURES + continue + ;; + 1) + echo $matching_projects + project_mount=$matching_projects + ;; + *) + echo "IMPORT FAILED FOR FILE: $file - A mount for project $project exists in multiple places." | tee -a $TODAY_FAILURES + continue + ;; + esac + + if (( $(date +%s) - $(stat -c %Y "$LANDING_DIR/$file") < (6*3) )); then + echo "File $file modified in the last 3 minutes. Waiting for upload to complete" + continue + fi + + echo $project 'directory mounted at' $project_mount + echo "Moving $file to $project_mount" + + read -r sha256 _ < <(/usr/bin/sha256sum "$filepath") + printf '%s | %s\n' "$sha256" "$filepath" | tee -a $TODAY_INGRESS + + local environment="${project_mount#/mnt/}" + local environment="${environment%%/*}" + + if [[ $environment == "windows" ]]; then + if [[ -z $study ]]; then + echo "IMPORT FAILED FOR FILE $file - Study could not be identified" + continue + else + echo "$LANDING_DIR/$file $project_mount/Import/$study" + mv $LANDING_DIR/$file $project_mount/Import/$study + fi + elif [[ $environment == "linux" ]]; then + echo "$filepath $project_mount/Import" + mv "$filepath" $project_mount/Import + chown root:$project "$project_mount/Import/$file" + else + echo "IMPORT FAILED FOR FILE $file - Environment could not be identified" | tee -a $TODAY_FAILURES + fi + done +} + +function export_files { + for filepath in $LINUX_DIR/*/Export/*; do + [[ -e "$filepath" ]] || continue + [[ -L "$filepath" ]] && continue # Prevent Symlinks MAYBE LOG IN EMAIL + [[ -d "$filepath" ]] && continue # Prevent Directory MAYBE LOG IN EMAIL + + local file=${filepath##*/} + + local project="${filepath#/mnt/linux/}" + local project="${project%%/Export/*}" + + if (( $(date +%s) - $(stat -c %Y $filepath) < (60*3) )); then + echo "File $file has been modified in the last 3 minutes. Waiting for write to complete" + continue + fi + + echo "$file from $project will be exported" + + zone=$(grep $project /root/scripts/space-project-map | cut -d' ' -f 2) + + [[ -z $zone ]] && echo $project not in space-project-map && continue + + new_filename="$zone-$project-$file" + echo $new_filename + + read -r sha256 _ < <(/usr/bin/sha256sum "$filepath") + printf '%s | %s\n' "$sha256" "$new_filename" | tee -a $TODAY_EGRESS + + mv "$filepath" "$EXPORT_DIR/$new_filename" + chgrp $project "$EXPORT_DIR/$new_filename" + chmod 664 "$EXPORT_DIR/$new_filename" + done + + readarray -t SPACE_WINDOWS_DOMAINS < /root/scripts/space-domains.txt + + for filepath in $WINDOWS_DIR/*/Export/*/*; do + [[ -e "$filepath" ]] || continue + [[ -L "$filepath" ]] && continue # Prevent Symlinks MAYBE LOG IN EMAIL + [[ -d "$filepath" ]] && continue # Prevent Directory MAYBE LOG IN EMAIL + + local file=${filepath##*/} + + local base="${filepath#/mnt/windows/}" + local study="${base#*/Export/}" + local study="${study%%/$file}" + local project="${base%%/Export/*}" + + local eidf_project="${project%%-*}" + + [[ ! "${SPACE_WINDOWS_DOMAINS[*]}" =~ "$project" ]] && echo $project not ins space-domains.txt && continue + + if (( $(date +%s) - $(stat -c %Y $filepath) < (60*3) )); then + echo "File $file has been modified in the last 3 minutes. Waiting for write to complete" + continue + fi + + echo "$file from $project will be exported" + + zone=$(grep $project /root/scripts/space-project-map | cut -d' ' -f 2) + + [[ -z $zone ]] && echo $project not in space-project-map && continue + + new_filename="$zone-$project-$file" + echo "$file will be renamed $new_filename" + + read -r sha256 _ < <(/usr/bin/sha256sum "$filepath") + printf '%s | %s\n' "$sha256" "$new_filename" | tee -a $TODAY_EGRESS + + if [[ "$file" == mft-* ]]; then + mv "$filepath" "/mnt/os-transfer/windows-mft/$project/$study/Export/$new_filename" + else + mv "$filepath" "$EXPORT_DIR/$new_filename" + chmod 774 "$EXPORT_DIR/$new_filename" + fi + done +} + +import_files +export_files