#!/usr/bin/env bash function main() { shopt -s nullglob # * returns nothing for empty directory # ansi escape codes for color output local GREEN='\033[32m' local YELLOW='\033[33m' local BLUE='\033[34m' local RED='\033[31m' local RESET='\033[0m' # Files uploaded to serv-u local SERVU_BASE="/mnt/os-transfer" local IMPORT_DIR="$SERVU_BASE/space-landing/Import" local EXPORT_DIR="$SERVU_BASE/space-landing/Export" local LIN_DIR="/mnt/linux" local WIN_DIR="/mnt/windows" cd "/opt/os-transfer" || { echo "Cannot cd to /opt/os-transfer"; exit; } local SPACE_PROJECT_MAP="./space-project-map" [[ -f "$SPACE_PROJECT_MAP" ]] || \ { logging::debug "Missing: $SPACE_PROJECT_MAP" exit 1 } local SPACE_DOMAINS_FILE="./space-domains.txt" [[ -f "$SPACE_DOMAINS_FILE" ]] || \ { logging::debug "Missing: $SPACE_DOMAINS_FILE" exit 1 } local SPACE_EMAIL_FILE="/tmp/space-email" local EMAIL_ARCHIVE="." [[ -d "/opt/os-transfer/emails" ]] && { local EMAIL_ARCHIVE="/opt/os-transfer/emails" } local LOG_DIR="." [[ -d "/opt/os-transfer/log" ]] && { local LOG_DIR="/opt/os-transfer/log" } local TODAY_FAILURES="$LOG_DIR/failure.log" local TODAY_INGRESS="$LOG_DIR/ingress.log" local TODAY_EGRESS="$LOG_DIR/egress.log" while [[ $# -gt 0 ]]; do case "$1" in -m|--email) run_email=true ;; -p|--purge) run_purge=true ;; -e|--export) run_export=true ;; -i|--import) run_import=true ;; -h|--help) echo "Usage: script.sh [-m|--email] [-p|--purge] [-e|--export] [-i|--import]" exit 0 ;; -*) echo "Unknown option: $1" >&2 exit 1 ;; *) echo "Ignoring unexpected argument: $1" >&2 ;; esac shift done # Execute actions in desired order [[ $run_import ]] && transfer::import [[ $run_export ]] && transfer::export [[ $run_purge ]] && purge_files [[ $run_email ]] && email exit 0 } function transfer::import() { echo -e "${BLUE}Importing Files...${RESET}" for filepath in "$IMPORT_DIR"/*; do [[ -f "$filepath" ]] || continue # Regular file [[ -L "$filepath" ]] && continue # Skip Symlinks [[ -d "$filepath" ]] && continue # Skip Directory local file=${filepath##*/} # strip directories from path # ensure file has correct format if [[ ! "$file" =~ ^eidf[a-zA-Z0-9]{2,5}-[a-zA-Z0-9]{3,8}-.+$ ]]; then logging::failure "[IMPORT] $file — Filename format invalid" continue fi # Ensure filename matches expected format if [[ ! "$file" =~ ^eidf[a-zA-Z0-9]{2,5}-[a-zA-Z0-9]{3,8}-.+ ]]; then logging::failure "[IMPORT] $file — Filename format invalid" continue fi # example file: eidf123-abc01-file.csv local project="${file%%-*}" # eidf123 local rest="${file#*-}" # abc01-file.csv local study="${rest%%-*}" # abc01 local filename="${rest#*-}" # file.csv # fail if any variable are empty if [[ -z "$project" || -z "$study" || -z "$filename" ]]; then logging::failure "[IMPORT] $file — Filename format invalid" continue fi #echo "File '$filename' in project $project for study $study" local pattern="/mnt/*/$project*" local matching_projects mapfile -t matching_projects < <(compgen -G "$pattern") case ${#matching_projects[@]} in 0) logging::failure "[IMPORT] $file — $project does not exist" continue ;; 1) local project_mount="${matching_projects[0]}" ;; *) logging::failure "[IMPORT] $file — $project exists in multiple locations" continue ;; esac check_mtitme "import" "$filepath" || continue local sha256 read -r sha256 _ < <(sha256sum "$filepath") # Extracts Windows/Linux from $project_mount and sets the 'environment' vairable # This changes the moving path depending if window or Linux # /mnt/windows/eidfsp02-sgstd local environment="${project_mount#/mnt/}" # windows/eidfsp02-sgstd local environment="${environment%%/*}" # windows if [[ $environment == "windows" ]]; then # Check destination directory exists [[ -d "$project_mount/Import/$study" ]] || { logging::failure "[IMPORT] $file — $study project does not exist" continue } logging::move "[IMPORT] $file ===> $project_mount/Import/$study" mv "$filepath" "$project_mount/Import/$study" logging::transfer "import" "sha256:$sha256 — $file" "$TODAY_INGRESS" elif [[ $environment == "linux" ]]; then # Check destination directory exists [[ -d "$project_mount/Import/" ]] || { logging::failure "[IMPORT] $file — $project does not exist" continue } logging::move "[IMPORT] $file ==MOVE==> $project_mount/Import/" mv "$filepath" "$project_mount/Import/" else logging::failure "[IMPORT] $file — Environment could not be identified" fi done } function transfer::export() { echo -e "${BLUE}Exporting Files...${RESET}" # LINUX EXPORT for filepath in "$LIN_DIR"/*/Export/*; do [[ -f "$filepath" ]] || continue # Regular file [[ -L "$filepath" ]] && continue # Skip Symlinks [[ -d "$filepath" ]] && continue # Skip Directory local file=${filepath##*/} local project="${filepath#/mnt/linux/}" local project="${project%%/Export/*}" local zone local lookup_key="$project" while read -r key value; do if [[ "$key" == "$lookup_key" ]]; then zone="$value" break fi done < "$SPACE_PROJECT_MAP" [[ -z "$zone" ]] && { logging::failure "[EXPORT] $filepath — $project has not been mapped to standard or protected" continue } check_mtitme "export" "$filepath" || continue new_filename="$zone-$project-$file" logging::debug "[EXPORT] $filepath ==RENAME==> $new_filename" read -r sha256 _ < <(sha256sum "$filepath") logging::transfer "export" "sha256:$sha256 — $file" "$TODAY_EGRESS" logging::move "[EXPORT] $file ==MOVE==> $EXPORT_DIR/$new_filename" mv "$filepath" "$EXPORT_DIR/$new_filename" chgrp "$project" "$EXPORT_DIR/$new_filename" chmod 664 "$EXPORT_DIR/$new_filename" done # WINDOWS EXPORT readarray -t SPACE_WINDOWS_DOMAINS < "$SPACE_DOMAINS_FILE" for filepath in "$WIN_DIR"/*/Export/*/*; do [[ -f "$filepath" ]] || continue # Regular file [[ -L "$filepath" ]] && continue # Skip Symlinks [[ -d "$filepath" ]] && continue # Skip Directory local file=${filepath##*/} local base="${filepath#/mnt/windows/}" local study="${base#*/Export/}" local study="${study%%/"$file"}" local project="${base%%/Export/*}" # local eidf_project="${project%%-*}" # Check that windows environment is in domain list [[ ! " ${SPACE_WINDOWS_DOMAINS[*]} " =~ (^|[[:space:]])$project([[:space:]]|$) ]] && { logging::failure "[EXPORT] $project is not in windows domain list" } check_mtitme "export" "$filepath" || continue local zone local lookup_key="$project" while read -r key value; do if [[ "$key" == "$lookup_key" ]]; then zone="$value" break fi done < "$SPACE_PROJECT_MAP" [[ -z "$zone" ]] && { logging::failure "[EXPORT] $filepath — $project has not been mapped to standard or protected" continue } new_filename="$zone-$project-$study-$file" logging::debug "[EXPORT] $filepath ==RENAME==> $new_filename" read -r sha256 _ < <(sha256sum "$filepath") logging::transfer "export" "sha256:$sha256 — $file" "$TODAY_EGRESS" if [[ "$file" == mft-* ]]; then logging::move "[EXPORT] $filepath ==MOVE==> /mnt/os-transfer/windows-mft/$project/$study/Export/$new_filename" mv "$filepath" "$SERVU_BASE/windows-mft/$project/$study/Export/$new_filename" else logging::move "[EXPORT] $filepath ==MOVE==> $EXPORT_DIR/$new_filename" mv "$filepath" "$EXPORT_DIR/$new_filename" chmod 774 "$EXPORT_DIR/$new_filename" fi done } function logging::transfer() { local action=$1 local log_line=$2 local log_file=$3 # Create log file if missing [[ -f "$log_file" ]] || : > "$log_file" # Always print to terminal echo -e "${GREEN}[${action^^}] ${log_line}${RESET}" # Append to file only if the exact line isn't already logged if ! grep -qxF "$log_line" "$log_file"; then echo "$log_line" >> "$log_file" fi } function logging::failure() { local log_line="$1" # Create log file if missing [[ -f "$TODAY_FAILURES" ]] || : > "$TODAY_FAILURES" # Always print to terminal echo -e "${RED}${log_line}${RESET}" # Append to file only if the exact line isn't already logged if ! grep -qxF "$log_line" "$TODAY_FAILURES"; then echo "$log_line" >> "$TODAY_FAILURES" fi } function logging::move() { echo -e "${YELLOW}$1${RESET}"; } function logging::debug() { echo -e "${BLUE}$1${RESET}"; } function check_mtitme() { local action=$1 local filepath=$2 local threshold=3 if find "$filepath" -mmin -"$threshold" | grep -q .; then logging::debug "[${action^^}] $file — file has been modified in last $(( threshold * 60 )) seconds" return 1 fi } function purge_files() { logging::debug "[PRUGE] Removing files older than two weeks" for filepath in "$IMPORT_DIR"/*; do #local file=${filepath##*/} if find "$filepath" -mtime +14 | grep -q .; then logging::move "[PURGE] Deleting $filepath" rm "$filepath" fi done } function email() { local SUBJECT="Space Openstack File Transfer" local MAIL_FROM="hpcsys-bayes@epcc.ed.ac.uk" local RCPT_TO="hpcsys-bayes@epcc.ed.ac.uk r.davies@epcc.ed.ac.uk analytical.workbench@gov.scot a.brown@epcc.ed.ac.uk" : > "$SPACE_EMAIL_FILE" # Build email { echo "Subject: $SUBJECT" echo "From: $MAIL_FROM" echo if [[ ! -s "$TODAY_EGRESS" ]] && [[ ! -s "$TODAY_INGRESS" ]]; then echo "No files transferred within the last day." echo else if [[ -s "$TODAY_EGRESS" ]]; then echo "Files exported in the last day" echo "---------------------------------" cat "$TODAY_EGRESS" echo else echo "No files exported in the last day"; echo fi if [[ -s "$TODAY_INGRESS" ]]; then echo "Files Imported in the last day" echo "---------------------------------" cat "$TODAY_INGRESS" echo else echo "No files imported in the last day"; echo fi fi if [[ -s "$TODAY_FAILURES" ]]; then echo "File Transfer Failures in the last day" echo "---------------------------------" cat "$TODAY_FAILURES" echo fi } >> "$SPACE_EMAIL_FILE" sendmail "$RCPT_TO" < "$SPACE_EMAIL_FILE" : > "$TODAY_INGRESS" : > "$TODAY_EGRESS" : > "$TODAY_FAILURES" mv "$SPACE_EMAIL_FILE" "$EMAIL_ARCHIVE/$(date +%Y-%m-%d)-space-email" } main "$@"