382 lines
11 KiB
Bash
Executable File
382 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
function main() {
|
|
shopt -s nullglob # * returns nothing for empty directory
|
|
|
|
# ansi escaped codes for color output
|
|
local GREEN='\033[32m'
|
|
local YELLOW='\033[33m'
|
|
local BLUE='\033[34m'
|
|
local RED='\033[31m'
|
|
local RESET='\033[0m'
|
|
|
|
# Files uploaded to serv-u
|
|
local SERVU_BASE="/mnt/os-transfer"
|
|
local IMPORT_DIR="$SERVU_BASE/space-landing/Import"
|
|
local EXPORT_DIR="$SERVU_BASE/space-landing/Export"
|
|
local LIN_DIR="/mnt/linux"
|
|
local WIN_DIR="/mnt/windows"
|
|
|
|
local SPACE_PROJECT_MAP="./space-project-map"
|
|
[[ -f "$SPACE_PROJECT_MAP" ]] || \
|
|
{
|
|
logging::debug "Missing: $SPACE_PROJECT_MAP"
|
|
exit 1
|
|
}
|
|
|
|
local SPACE_DOMAINS_FILE="./space-domains.txt"
|
|
[[ -f "$SPACE_DOMAINS_FILE" ]] || \
|
|
{
|
|
logging::debug "Missing: $SPACE_DOMAINS_FILE"
|
|
exit 1
|
|
}
|
|
|
|
local SPACE_EMAIL_FILE="/tmp/space-email"
|
|
|
|
local EMAIL_ARCHIVE="."
|
|
[[ -d "/opt/space-transfer/emails" ]] && {
|
|
local EMAIL_ARCHIVE="/opt/space-transfer/emails"
|
|
}
|
|
|
|
local LOG_DIR="."
|
|
[[ -d "/opt/space-transfer/log" ]] && {
|
|
local LOG_DIR="/opt/space-transfer/log"
|
|
}
|
|
local TODAY_FAILURES="$LOG_DIR/failure.log"
|
|
local TODAY_INGRESS="$LOG_DIR/ingress.log"
|
|
local TODAY_EGRESS="$LOG_DIR/egress.log"
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
-m|--email)
|
|
run_email=true
|
|
;;
|
|
-p|--purge)
|
|
run_purge=true
|
|
;;
|
|
-e|--export)
|
|
run_export=true
|
|
;;
|
|
-i|--import)
|
|
run_import=true
|
|
;;
|
|
-h|--help)
|
|
echo "Usage: script.sh [-m|--email] [-p|--purge] [-e|--export] [-i|--import]"
|
|
exit 0
|
|
;;
|
|
-*)
|
|
echo "Unknown option: $1" >&2
|
|
exit 1
|
|
;;
|
|
*)
|
|
echo "Ignoring unexpected argument: $1" >&2
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
# Execute actions in desired order
|
|
[[ $run_import ]] && transfer::import
|
|
[[ $run_export ]] && transfer::export
|
|
[[ $run_purge ]] && purge_files
|
|
[[ $run_email ]] && email
|
|
}
|
|
|
|
function transfer::import() {
|
|
echo -e "${BLUE}Importing Files...${RESET}"
|
|
|
|
for filepath in "$IMPORT_DIR"/*; do
|
|
[[ -f "$filepath" ]] || continue # Regular file
|
|
[[ -L "$filepath" ]] && continue # Skip Symlinks
|
|
[[ -d "$filepath" ]] && continue # Skip Directory
|
|
|
|
local file=${filepath##*/} # strip directories from path
|
|
|
|
# ensure file has correct format
|
|
if [[ ! "$file" =~ ^eidf[a-zA-Z0-9]{2,5}-[a-zA-Z0-9]{3,8}-.+$ ]]; then
|
|
logging::failure "[IMPORT] $file — Filename format invalid"
|
|
continue
|
|
fi
|
|
|
|
# Ensure filename matches expected format
|
|
if [[ ! "$file" =~ ^eidf[a-zA-Z0-9]{2,5}-[a-zA-Z0-9]{3,8}-.+ ]]; then
|
|
logging::failure "[IMPORT] $file — Filename format invalid"
|
|
continue
|
|
fi
|
|
|
|
|
|
# example file: eidf123-abc01-file.csv
|
|
local project="${file%%-*}" # eidf123
|
|
local rest="${file#*-}" # abc01-file.csv
|
|
local study="${rest%%-*}" # abc01
|
|
local filename="${rest#*-}" # file.csv
|
|
|
|
# fail if any variable are empty
|
|
if [[ -z "$project" || -z "$study" || -z "$filename" ]]; then
|
|
logging::failure "[IMPORT] $file — Filename format invalid"
|
|
continue
|
|
fi
|
|
|
|
#echo "File '$filename' in project $project for study $study"
|
|
|
|
local pattern="/mnt/*/$project*"
|
|
local matching_projects
|
|
mapfile -t matching_projects < <(compgen -G "$pattern")
|
|
|
|
case ${#matching_projects[@]} in
|
|
0)
|
|
logging::failure "[IMPORT] $file — $project does not exist"
|
|
continue
|
|
;;
|
|
1)
|
|
local project_mount="${matching_projects[0]}"
|
|
;;
|
|
*)
|
|
logging::failure "[IMPORT] $file — $project exists in multiple locations"
|
|
continue
|
|
;;
|
|
esac
|
|
|
|
check_mtitme "import" "$filepath" || continue
|
|
|
|
local sha256
|
|
read -r sha256 _ < <(sha256sum "$filepath")
|
|
|
|
# Extracts Windows/Linux from $project_mount and sets the 'environment' vairable
|
|
# This changes the moving path depending if window or Linux
|
|
# /mnt/windows/eidfsp02-sgstd
|
|
local environment="${project_mount#/mnt/}" # windows/eidfsp02-sgstd
|
|
local environment="${environment%%/*}" # windows
|
|
|
|
if [[ $environment == "windows" ]]; then
|
|
# Check destination directory exists
|
|
[[ -d "$project_mount/Import/$study" ]] || {
|
|
logging::failure "[IMPORT] $file — $study project does not exist"
|
|
continue
|
|
}
|
|
|
|
logging::move "[IMPORT] $file ===> $project_mount/Import/$study"
|
|
mv "$filepath" "$project_mount/Import/$study"
|
|
logging::transfer "import" "sha256:$sha256 — $file" "$TODAY_INGRESS"
|
|
|
|
elif [[ $environment == "linux" ]]; then
|
|
# Check destination directory exists
|
|
[[ -d "$project_mount/Import/" ]] || {
|
|
logging::failure "[IMPORT] $file — $project does not exist"
|
|
continue
|
|
}
|
|
|
|
logging::move "[IMPORT] $file ==MOVE==> $project_mount/Import/"
|
|
mv "$filepath" "$project_mount/Import/"
|
|
logging::ingress "sha256:$sha256 — $file"
|
|
|
|
else
|
|
logging::failure "[IMPORT] $file — Environment could not be identified"
|
|
fi
|
|
done
|
|
}
|
|
|
|
function transfer::export() {
|
|
echo -e "${BLUE}Exporting Files...${RESET}"
|
|
|
|
# LINUX EXPORT
|
|
|
|
for filepath in "$LIN_DIR"/*/Export/*; do
|
|
[[ -f "$filepath" ]] || continue # Regular file
|
|
[[ -L "$filepath" ]] && continue # Skip Symlinks
|
|
[[ -d "$filepath" ]] && continue # Skip Directory
|
|
|
|
local file=${filepath##*/}
|
|
|
|
local project="${filepath#/mnt/linux/}"
|
|
local project="${project%%/Export/*}"
|
|
|
|
local zone
|
|
local lookup_key="$project"
|
|
while read -r key value; do
|
|
if [[ "$key" == "$lookup_key" ]]; then
|
|
zone="$value"
|
|
break
|
|
fi
|
|
done < "$SPACE_PROJECT_MAP"
|
|
|
|
[[ -z "$zone" ]] && {
|
|
logging::failure "[EXPORT] $filepath — $project has not been mapped to standard or protected"
|
|
continue
|
|
}
|
|
|
|
check_mtitme "export" "$filepath" || continue
|
|
|
|
new_filename="$zone-$project-$file"
|
|
logging::debug "[EXPORT] $filepath ==RENAME==> $new_filename"
|
|
|
|
read -r sha256 _ < <(sha256sum "$filepath")
|
|
logging::transfer "export" "sha256:$sha256 — $file" "$TODAY_EGRESS"
|
|
|
|
logging::move "[EXPORT] $file ==MOVE==> $EXPORT_DIR/$new_filename"
|
|
mv "$filepath" "$EXPORT_DIR/$new_filename"
|
|
chgrp "$project" "$EXPORT_DIR/$new_filename"
|
|
chmod 664 "$EXPORT_DIR/$new_filename"
|
|
done
|
|
|
|
# WINDOWS EXPORT
|
|
|
|
readarray -t SPACE_WINDOWS_DOMAINS < "$SPACE_DOMAINS_FILE"
|
|
|
|
for filepath in "$WIN_DIR"/*/Export/*/*; do
|
|
[[ -f "$filepath" ]] || continue # Regular file
|
|
[[ -L "$filepath" ]] && continue # Skip Symlinks
|
|
[[ -d "$filepath" ]] && continue # Skip Directory
|
|
local file=${filepath##*/}
|
|
|
|
local base="${filepath#/mnt/windows/}"
|
|
local study="${base#*/Export/}"
|
|
local study="${study%%/"$file"}"
|
|
local project="${base%%/Export/*}"
|
|
# local eidf_project="${project%%-*}"
|
|
|
|
# Check that windows environment is in domain list
|
|
[[ ! " ${SPACE_WINDOWS_DOMAINS[*]} " =~ (^|[[:space:]])$project([[:space:]]|$) ]] && {
|
|
logging::failure "[EXPORT] $project is not in windows domain list"
|
|
}
|
|
|
|
check_mtitme "export" "$filepath" || continue
|
|
|
|
local zone
|
|
local lookup_key="$project"
|
|
while read -r key value; do
|
|
if [[ "$key" == "$lookup_key" ]]; then
|
|
zone="$value"
|
|
break
|
|
fi
|
|
done < "$SPACE_PROJECT_MAP"
|
|
|
|
[[ -z "$zone" ]] && {
|
|
logging::failure "[EXPORT] $filepath — $project has not been mapped to standard or protected"
|
|
continue
|
|
}
|
|
|
|
new_filename="$zone-$project-$file"
|
|
logging::debug "[EXPORT] $filepath ==RENAME==> $new_filename"
|
|
|
|
read -r sha256 _ < <(sha256sum "$filepath")
|
|
logging::transfer "export" "sha256:$sha256 — $file" "$TODAY_EGRESS"
|
|
|
|
if [[ "$file" == mft-* ]]; then
|
|
logging::move "[EXPORT] $filepath ==MOVE==> /mnt/os-transfer/windows-mft/$project/$study/Export/$new_filename"
|
|
mv "$filepath" "$SERVU_BASE/windows-mft/$project/$study/Export/$new_filename"
|
|
else
|
|
logging::move "[EXPORT] $filepath ==MOVE==> $EXPORT_DIR/$new_filename"
|
|
mv "$filepath" "$EXPORT_DIR/$new_filename"
|
|
chmod 774 "$EXPORT_DIR/$new_filename"
|
|
fi
|
|
done
|
|
}
|
|
|
|
function logging::transfer() {
|
|
local action=$1
|
|
local log_line=$2
|
|
local log_file=$3
|
|
|
|
# Create log file if missing
|
|
[[ -f "$log_file" ]] || : > "$log_file"
|
|
|
|
# Always print to terminal
|
|
echo -e "${GREEN}[${action^^}] ${log_line}${RESET}"
|
|
|
|
# Append to file only if the exact line isn't already logged
|
|
if ! grep -qxF "$log_line" "$log_file"; then
|
|
echo "$log_line" >> "$log_file"
|
|
fi
|
|
}
|
|
|
|
function logging::failure() {
|
|
local log_line="$1"
|
|
|
|
# Create log file if missing
|
|
[[ -f "$TODAY_FAILURES" ]] || : > "$TODAY_FAILURES"
|
|
|
|
# Always print to terminal
|
|
echo -e "${RED}${log_line}${RESET}"
|
|
|
|
# Append to file only if the exact line isn't already logged
|
|
if ! grep -qxF "$log_line" "$TODAY_FAILURES"; then
|
|
echo "$log_line" >> "$TODAY_FAILURES"
|
|
fi
|
|
}
|
|
|
|
function logging::move() { echo -e "${YELLOW}$1${RESET}"; }
|
|
function logging::debug() { echo -e "${BLUE}$1${RESET}"; }
|
|
|
|
function check_mtitme() {
|
|
local action=$1
|
|
local filepath=$2
|
|
local threshold=3
|
|
if find "$filepath" -mmin -"$threshold" | grep -q .; then
|
|
logging::debug "[${action^^}] $file — file has been modified in last $(( threshold * 60 )) seconds"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
function purge_files() {
|
|
logging::debug "[PRUGE] Removing files older than two weeks"
|
|
for filepath in "$IMPORT_DIR"/*; do
|
|
#local file=${filepath##*/}
|
|
if find "$filepath" -mtime +14 | grep -q .; then
|
|
logging::move "[PURGE] Deleting $filepath"
|
|
rm "$filepath"
|
|
fi
|
|
done
|
|
}
|
|
|
|
function email() {
|
|
local SUBJECT="Space Openstack File Transfer"
|
|
local MAIL_FROM="os-transfer@adbraun.lol"
|
|
local RCPT_TO="adam@adbraun.lol"
|
|
|
|
: > "$SPACE_EMAIL_FILE"
|
|
|
|
# Build email
|
|
{
|
|
echo "Subject: $SUBJECT"
|
|
echo "From: $MAIL_FROM"
|
|
echo
|
|
|
|
if [[ ! -s "$TODAY_EGRESS" ]] && [[ ! -s "$TODAY_INGRESS" ]]; then
|
|
echo "No files transferred within the last day."
|
|
echo
|
|
else
|
|
if [[ -s "$TODAY_EGRESS" ]]; then
|
|
echo "Files exported in the last day"
|
|
echo "---------------------------------"
|
|
cat "$TODAY_EGRESS"
|
|
echo
|
|
else echo "No files exported in the last day"; echo
|
|
fi
|
|
|
|
if [[ -s "$TODAY_INGRESS" ]]; then
|
|
echo "Files Imported in the last day"
|
|
echo "---------------------------------"
|
|
cat "$TODAY_INGRESS"
|
|
echo
|
|
else echo "No files imported in the last day"; echo
|
|
fi
|
|
fi
|
|
|
|
if [[ -s "$TODAY_FAILURES" ]]; then
|
|
echo "File Transfer Failures in the last day"
|
|
echo "---------------------------------"
|
|
cat "$TODAY_FAILURES"
|
|
echo
|
|
fi
|
|
} >> "$SPACE_EMAIL_FILE"
|
|
|
|
sendmail "$RCPT_TO" < "$SPACE_EMAIL_FILE"
|
|
|
|
: > "$TODAY_INGRESS"
|
|
: > "$TODAY_EGRESS"
|
|
: > "$TODAY_FAILURES"
|
|
mv "$SPACE_EMAIL_FILE" "$EMAIL_ARCHIVE/$(date +%Y-%m-%d)-space-email"
|
|
}
|
|
|
|
main "$@" |