diff --git a/transfer.sh b/transfer.sh new file mode 100755 index 0000000..6d3116b --- /dev/null +++ b/transfer.sh @@ -0,0 +1,382 @@ +#!/usr/bin/env bash + +function main() { + shopt -s nullglob # * returns nothing for empty directory + + # ansi escaped codes for color output + local GREEN='\033[32m' + local YELLOW='\033[33m' + local BLUE='\033[34m' + local RED='\033[31m' + local RESET='\033[0m' + + # Files uploaded to serv-u + local SERVU_BASE="/mnt/os-transfer" + local IMPORT_DIR="$SERVU_BASE/space-landing/Import" + local EXPORT_DIR="$SERVU_BASE/space-landing/Export" + local LIN_DIR="/mnt/linux" + local WIN_DIR="/mnt/windows" + + local SPACE_PROJECT_MAP="./space-project-map" + [[ -f "$SPACE_PROJECT_MAP" ]] || \ + { + logging::debug "Missing: $SPACE_PROJECT_MAP" + exit 1 + } + + local SPACE_DOMAINS_FILE="./space-domains.txt" + [[ -f "$SPACE_DOMAINS_FILE" ]] || \ + { + logging::debug "Missing: $SPACE_DOMAINS_FILE" + exit 1 + } + + local SPACE_EMAIL_FILE="/tmp/space-email" + + local EMAIL_ARCHIVE="." + [[ -d "/opt/space-transfer/emails" ]] && { + local EMAIL_ARCHIVE="/opt/space-transfer/emails" + } + + local LOG_DIR="." + [[ -d "/opt/space-transfer/log" ]] && { + local LOG_DIR="/opt/space-transfer/log" + } + local TODAY_FAILURES="$LOG_DIR/failure.log" + local TODAY_INGRESS="$LOG_DIR/ingress.log" + local TODAY_EGRESS="$LOG_DIR/egress.log" + + while [[ $# -gt 0 ]]; do + case "$1" in + -m|--email) + run_email=true + ;; + -p|--purge) + run_purge=true + ;; + -e|--export) + run_export=true + ;; + -i|--import) + run_import=true + ;; + -h|--help) + echo "Usage: script.sh [-m|--email] [-p|--purge] [-e|--export] [-i|--import]" + exit 0 + ;; + -*) + echo "Unknown option: $1" >&2 + exit 1 + ;; + *) + echo "Ignoring unexpected argument: $1" >&2 + ;; + esac + shift + done + + # Execute actions in desired order + [[ $run_import ]] && transfer::import + [[ $run_export ]] && transfer::export + [[ $run_purge ]] && purge_files + [[ $run_email ]] && email +} + +function transfer::import() { + echo -e "${BLUE}Importing Files...${RESET}" + + for filepath in "$IMPORT_DIR"/*; do + [[ -f "$filepath" ]] || continue # Regular file + [[ -L "$filepath" ]] && continue # Skip Symlinks + [[ -d "$filepath" ]] && continue # Skip Directory + + local file=${filepath##*/} # strip directories from path + + # ensure file has correct format + if [[ ! "$file" =~ ^eidf[a-zA-Z0-9]{2,5}-[a-zA-Z0-9]{3,8}-.+$ ]]; then + logging::failure "[IMPORT] $file — Filename format invalid" + continue + fi + + # Ensure filename matches expected format + if [[ ! "$file" =~ ^eidf[a-zA-Z0-9]{2,5}-[a-zA-Z0-9]{3,8}-.+ ]]; then + logging::failure "[IMPORT] $file — Filename format invalid" + continue + fi + + + # example file: eidf123-abc01-file.csv + local project="${file%%-*}" # eidf123 + local rest="${file#*-}" # abc01-file.csv + local study="${rest%%-*}" # abc01 + local filename="${rest#*-}" # file.csv + + # fail if any variable are empty + if [[ -z "$project" || -z "$study" || -z "$filename" ]]; then + logging::failure "[IMPORT] $file — Filename format invalid" + continue + fi + + #echo "File '$filename' in project $project for study $study" + + local pattern="/mnt/*/$project*" + local matching_projects + mapfile -t matching_projects < <(compgen -G "$pattern") + + case ${#matching_projects[@]} in + 0) + logging::failure "[IMPORT] $file — $project does not exist" + continue + ;; + 1) + local project_mount="${matching_projects[0]}" + ;; + *) + logging::failure "[IMPORT] $file — $project exists in multiple locations" + continue + ;; + esac + + check_mtitme "import" "$filepath" || continue + + local sha256 + read -r sha256 _ < <(sha256sum "$filepath") + + # Extracts Windows/Linux from $project_mount and sets the 'environment' vairable + # This changes the moving path depending if window or Linux + # /mnt/windows/eidfsp02-sgstd + local environment="${project_mount#/mnt/}" # windows/eidfsp02-sgstd + local environment="${environment%%/*}" # windows + + if [[ $environment == "windows" ]]; then + # Check destination directory exists + [[ -d "$project_mount/Import/$study" ]] || { + logging::failure "[IMPORT] $file — $study project does not exist" + continue + } + + logging::move "[IMPORT] $file ===> $project_mount/Import/$study" + mv "$filepath" "$project_mount/Import/$study" + logging::transfer "import" "sha256:$sha256 — $file" "$TODAY_INGRESS" + + elif [[ $environment == "linux" ]]; then + # Check destination directory exists + [[ -d "$project_mount/Import/" ]] || { + logging::failure "[IMPORT] $file — $project does not exist" + continue + } + + logging::move "[IMPORT] $file ==MOVE==> $project_mount/Import/" + mv "$filepath" "$project_mount/Import/" + logging::ingress "sha256:$sha256 — $file" + + else + logging::failure "[IMPORT] $file — Environment could not be identified" + fi + done +} + +function transfer::export() { + echo -e "${BLUE}Exporting Files...${RESET}" + + # LINUX EXPORT + + for filepath in "$LIN_DIR"/*/Export/*; do + [[ -f "$filepath" ]] || continue # Regular file + [[ -L "$filepath" ]] && continue # Skip Symlinks + [[ -d "$filepath" ]] && continue # Skip Directory + + local file=${filepath##*/} + + local project="${filepath#/mnt/linux/}" + local project="${project%%/Export/*}" + + local zone + local lookup_key="$project" + while read -r key value; do + if [[ "$key" == "$lookup_key" ]]; then + zone="$value" + break + fi + done < "$SPACE_PROJECT_MAP" + + [[ -z "$zone" ]] && { + logging::failure "[EXPORT] $filepath — $project has not been mapped to standard or protected" + continue + } + + check_mtitme "export" "$filepath" || continue + + new_filename="$zone-$project-$file" + logging::debug "[EXPORT] $filepath ==RENAME==> $new_filename" + + read -r sha256 _ < <(sha256sum "$filepath") + logging::transfer "export" "sha256:$sha256 — $file" "$TODAY_EGRESS" + + logging::move "[EXPORT] $file ==MOVE==> $EXPORT_DIR/$new_filename" + mv "$filepath" "$EXPORT_DIR/$new_filename" + chgrp "$project" "$EXPORT_DIR/$new_filename" + chmod 664 "$EXPORT_DIR/$new_filename" + done + + # WINDOWS EXPORT + + readarray -t SPACE_WINDOWS_DOMAINS < "$SPACE_DOMAINS_FILE" + + for filepath in "$WIN_DIR"/*/Export/*/*; do + [[ -f "$filepath" ]] || continue # Regular file + [[ -L "$filepath" ]] && continue # Skip Symlinks + [[ -d "$filepath" ]] && continue # Skip Directory + local file=${filepath##*/} + + local base="${filepath#/mnt/windows/}" + local study="${base#*/Export/}" + local study="${study%%/"$file"}" + local project="${base%%/Export/*}" + # local eidf_project="${project%%-*}" + + # Check that windows environment is in domain list + [[ ! " ${SPACE_WINDOWS_DOMAINS[*]} " =~ (^|[[:space:]])$project([[:space:]]|$) ]] && { + logging::failure "[EXPORT] $project is not in windows domain list" + } + + check_mtitme "export" "$filepath" || continue + + local zone + local lookup_key="$project" + while read -r key value; do + if [[ "$key" == "$lookup_key" ]]; then + zone="$value" + break + fi + done < "$SPACE_PROJECT_MAP" + + [[ -z "$zone" ]] && { + logging::failure "[EXPORT] $filepath — $project has not been mapped to standard or protected" + continue + } + + new_filename="$zone-$project-$file" + logging::debug "[EXPORT] $filepath ==RENAME==> $new_filename" + + read -r sha256 _ < <(sha256sum "$filepath") + logging::transfer "export" "sha256:$sha256 — $file" "$TODAY_EGRESS" + + if [[ "$file" == mft-* ]]; then + logging::move "[EXPORT] $filepath ==MOVE==> /mnt/os-transfer/windows-mft/$project/$study/Export/$new_filename" + mv "$filepath" "$SERVU_BASE/windows-mft/$project/$study/Export/$new_filename" + else + logging::move "[EXPORT] $filepath ==MOVE==> $EXPORT_DIR/$new_filename" + mv "$filepath" "$EXPORT_DIR/$new_filename" + chmod 774 "$EXPORT_DIR/$new_filename" + fi + done +} + +function logging::transfer() { + local action=$1 + local log_line=$2 + local log_file=$3 + + # Create log file if missing + [[ -f "$log_file" ]] || : > "$log_file" + + # Always print to terminal + echo -e "${GREEN}[${action^^}] ${log_line}${RESET}" + + # Append to file only if the exact line isn't already logged + if ! grep -qxF "$log_line" "$log_file"; then + echo "$log_line" >> "$log_file" + fi +} + +function logging::failure() { + local log_line="$1" + + # Create log file if missing + [[ -f "$TODAY_FAILURES" ]] || : > "$TODAY_FAILURES" + + # Always print to terminal + echo -e "${RED}${log_line}${RESET}" + + # Append to file only if the exact line isn't already logged + if ! grep -qxF "$log_line" "$TODAY_FAILURES"; then + echo "$log_line" >> "$TODAY_FAILURES" + fi +} + +function logging::move() { echo -e "${YELLOW}$1${RESET}"; } +function logging::debug() { echo -e "${BLUE}$1${RESET}"; } + +function check_mtitme() { + local action=$1 + local filepath=$2 + local threshold=3 + if find "$filepath" -mmin -"$threshold" | grep -q .; then + logging::debug "[${action^^}] $file — file has been modified in last $(( threshold * 60 )) seconds" + return 1 + fi +} + +function purge_files() { + logging::debug "[PRUGE] Removing files older than two weeks" + for filepath in "$IMPORT_DIR"/*; do + #local file=${filepath##*/} + if find "$filepath" -mtime +14 | grep -q .; then + logging::move "[PURGE] Deleting $filepath" + rm "$filepath" + fi + done +} + +function email() { + local SUBJECT="Space Openstack File Transfer" + local MAIL_FROM="os-transfer@adbraun.lol" + local RCPT_TO="adam@adbraun.lol" + + : > "$SPACE_EMAIL_FILE" + + # Build email + { + echo "Subject: $SUBJECT" + echo "From: $MAIL_FROM" + echo + + if [[ ! -s "$TODAY_EGRESS" ]] && [[ ! -s "$TODAY_INGRESS" ]]; then + echo "No files transferred within the last day." + echo + else + if [[ -s "$TODAY_EGRESS" ]]; then + echo "Files exported in the last day" + echo "---------------------------------" + cat "$TODAY_EGRESS" + echo + else echo "No files exported in the last day"; echo + fi + + if [[ -s "$TODAY_INGRESS" ]]; then + echo "Files Imported in the last day" + echo "---------------------------------" + cat "$TODAY_INGRESS" + echo + else echo "No files imported in the last day"; echo + fi + fi + + if [[ -s "$TODAY_FAILURES" ]]; then + echo "File Transfer Failures in the last day" + echo "---------------------------------" + cat "$TODAY_FAILURES" + echo + fi + } >> "$SPACE_EMAIL_FILE" + + sendmail "$RCPT_TO" < "$SPACE_EMAIL_FILE" + + : > "$TODAY_INGRESS" + : > "$TODAY_EGRESS" + : > "$TODAY_FAILURES" + mv "$SPACE_EMAIL_FILE" "$EMAIL_ARCHIVE/$(date +%Y-%m-%d)-space-email" +} + +main "$@" \ No newline at end of file