first commit version 1 complete
This commit is contained in:
adam committed
1 parent
e5f034b394
commit
3edea42cff
1 file changed
+382
Executable
+382
@@ -0,0 +1,382 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
function main() {
|
||||
shopt -s nullglob # * returns nothing for empty directory
|
||||
|
||||
# ansi escaped codes for color output
|
||||
local GREEN='\033[32m'
|
||||
local YELLOW='\033[33m'
|
||||
local BLUE='\033[34m'
|
||||
local RED='\033[31m'
|
||||
local RESET='\033[0m'
|
||||
|
||||
# Files uploaded to serv-u
|
||||
local SERVU_BASE="/mnt/os-transfer"
|
||||
local IMPORT_DIR="$SERVU_BASE/space-landing/Import"
|
||||
local EXPORT_DIR="$SERVU_BASE/space-landing/Export"
|
||||
local LIN_DIR="/mnt/linux"
|
||||
local WIN_DIR="/mnt/windows"
|
||||
|
||||
local SPACE_PROJECT_MAP="./space-project-map"
|
||||
[[ -f "$SPACE_PROJECT_MAP" ]] || \
|
||||
{
|
||||
logging::debug "Missing: $SPACE_PROJECT_MAP"
|
||||
exit 1
|
||||
}
|
||||
|
||||
local SPACE_DOMAINS_FILE="./space-domains.txt"
|
||||
[[ -f "$SPACE_DOMAINS_FILE" ]] || \
|
||||
{
|
||||
logging::debug "Missing: $SPACE_DOMAINS_FILE"
|
||||
exit 1
|
||||
}
|
||||
|
||||
local SPACE_EMAIL_FILE="/tmp/space-email"
|
||||
|
||||
local EMAIL_ARCHIVE="."
|
||||
[[ -d "/opt/space-transfer/emails" ]] && {
|
||||
local EMAIL_ARCHIVE="/opt/space-transfer/emails"
|
||||
}
|
||||
|
||||
local LOG_DIR="."
|
||||
[[ -d "/opt/space-transfer/log" ]] && {
|
||||
local LOG_DIR="/opt/space-transfer/log"
|
||||
}
|
||||
local TODAY_FAILURES="$LOG_DIR/failure.log"
|
||||
local TODAY_INGRESS="$LOG_DIR/ingress.log"
|
||||
local TODAY_EGRESS="$LOG_DIR/egress.log"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
-m|--email)
|
||||
run_email=true
|
||||
;;
|
||||
-p|--purge)
|
||||
run_purge=true
|
||||
;;
|
||||
-e|--export)
|
||||
run_export=true
|
||||
;;
|
||||
-i|--import)
|
||||
run_import=true
|
||||
;;
|
||||
-h|--help)
|
||||
echo "Usage: script.sh [-m|--email] [-p|--purge] [-e|--export] [-i|--import]"
|
||||
exit 0
|
||||
;;
|
||||
-*)
|
||||
echo "Unknown option: $1" >&2
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
echo "Ignoring unexpected argument: $1" >&2
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# Execute actions in desired order
|
||||
[[ $run_import ]] && transfer::import
|
||||
[[ $run_export ]] && transfer::export
|
||||
[[ $run_purge ]] && purge_files
|
||||
[[ $run_email ]] && email
|
||||
}
|
||||
|
||||
function transfer::import() {
|
||||
echo -e "${BLUE}Importing Files...${RESET}"
|
||||
|
||||
for filepath in "$IMPORT_DIR"/*; do
|
||||
[[ -f "$filepath" ]] || continue # Regular file
|
||||
[[ -L "$filepath" ]] && continue # Skip Symlinks
|
||||
[[ -d "$filepath" ]] && continue # Skip Directory
|
||||
|
||||
local file=${filepath##*/} # strip directories from path
|
||||
|
||||
# ensure file has correct format
|
||||
if [[ ! "$file" =~ ^eidf[a-zA-Z0-9]{2,5}-[a-zA-Z0-9]{3,8}-.+$ ]]; then
|
||||
logging::failure "[IMPORT] $file — Filename format invalid"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Ensure filename matches expected format
|
||||
if [[ ! "$file" =~ ^eidf[a-zA-Z0-9]{2,5}-[a-zA-Z0-9]{3,8}-.+ ]]; then
|
||||
logging::failure "[IMPORT] $file — Filename format invalid"
|
||||
continue
|
||||
fi
|
||||
|
||||
|
||||
# example file: eidf123-abc01-file.csv
|
||||
local project="${file%%-*}" # eidf123
|
||||
local rest="${file#*-}" # abc01-file.csv
|
||||
local study="${rest%%-*}" # abc01
|
||||
local filename="${rest#*-}" # file.csv
|
||||
|
||||
# fail if any variable are empty
|
||||
if [[ -z "$project" || -z "$study" || -z "$filename" ]]; then
|
||||
logging::failure "[IMPORT] $file — Filename format invalid"
|
||||
continue
|
||||
fi
|
||||
|
||||
#echo "File '$filename' in project $project for study $study"
|
||||
|
||||
local pattern="/mnt/*/$project*"
|
||||
local matching_projects
|
||||
mapfile -t matching_projects < <(compgen -G "$pattern")
|
||||
|
||||
case ${#matching_projects[@]} in
|
||||
0)
|
||||
logging::failure "[IMPORT] $file — $project does not exist"
|
||||
continue
|
||||
;;
|
||||
1)
|
||||
local project_mount="${matching_projects[0]}"
|
||||
;;
|
||||
*)
|
||||
logging::failure "[IMPORT] $file — $project exists in multiple locations"
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
|
||||
check_mtitme "import" "$filepath" || continue
|
||||
|
||||
local sha256
|
||||
read -r sha256 _ < <(sha256sum "$filepath")
|
||||
|
||||
# Extracts Windows/Linux from $project_mount and sets the 'environment' vairable
|
||||
# This changes the moving path depending if window or Linux
|
||||
# /mnt/windows/eidfsp02-sgstd
|
||||
local environment="${project_mount#/mnt/}" # windows/eidfsp02-sgstd
|
||||
local environment="${environment%%/*}" # windows
|
||||
|
||||
if [[ $environment == "windows" ]]; then
|
||||
# Check destination directory exists
|
||||
[[ -d "$project_mount/Import/$study" ]] || {
|
||||
logging::failure "[IMPORT] $file — $study project does not exist"
|
||||
continue
|
||||
}
|
||||
|
||||
logging::move "[IMPORT] $file ===> $project_mount/Import/$study"
|
||||
mv "$filepath" "$project_mount/Import/$study"
|
||||
logging::transfer "import" "sha256:$sha256 — $file" "$TODAY_INGRESS"
|
||||
|
||||
elif [[ $environment == "linux" ]]; then
|
||||
# Check destination directory exists
|
||||
[[ -d "$project_mount/Import/" ]] || {
|
||||
logging::failure "[IMPORT] $file — $project does not exist"
|
||||
continue
|
||||
}
|
||||
|
||||
logging::move "[IMPORT] $file ==MOVE==> $project_mount/Import/"
|
||||
mv "$filepath" "$project_mount/Import/"
|
||||
logging::ingress "sha256:$sha256 — $file"
|
||||
|
||||
else
|
||||
logging::failure "[IMPORT] $file — Environment could not be identified"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
function transfer::export() {
|
||||
echo -e "${BLUE}Exporting Files...${RESET}"
|
||||
|
||||
# LINUX EXPORT
|
||||
|
||||
for filepath in "$LIN_DIR"/*/Export/*; do
|
||||
[[ -f "$filepath" ]] || continue # Regular file
|
||||
[[ -L "$filepath" ]] && continue # Skip Symlinks
|
||||
[[ -d "$filepath" ]] && continue # Skip Directory
|
||||
|
||||
local file=${filepath##*/}
|
||||
|
||||
local project="${filepath#/mnt/linux/}"
|
||||
local project="${project%%/Export/*}"
|
||||
|
||||
local zone
|
||||
local lookup_key="$project"
|
||||
while read -r key value; do
|
||||
if [[ "$key" == "$lookup_key" ]]; then
|
||||
zone="$value"
|
||||
break
|
||||
fi
|
||||
done < "$SPACE_PROJECT_MAP"
|
||||
|
||||
[[ -z "$zone" ]] && {
|
||||
logging::failure "[EXPORT] $filepath — $project has not been mapped to standard or protected"
|
||||
continue
|
||||
}
|
||||
|
||||
check_mtitme "export" "$filepath" || continue
|
||||
|
||||
new_filename="$zone-$project-$file"
|
||||
logging::debug "[EXPORT] $filepath ==RENAME==> $new_filename"
|
||||
|
||||
read -r sha256 _ < <(sha256sum "$filepath")
|
||||
logging::transfer "export" "sha256:$sha256 — $file" "$TODAY_EGRESS"
|
||||
|
||||
logging::move "[EXPORT] $file ==MOVE==> $EXPORT_DIR/$new_filename"
|
||||
mv "$filepath" "$EXPORT_DIR/$new_filename"
|
||||
chgrp "$project" "$EXPORT_DIR/$new_filename"
|
||||
chmod 664 "$EXPORT_DIR/$new_filename"
|
||||
done
|
||||
|
||||
# WINDOWS EXPORT
|
||||
|
||||
readarray -t SPACE_WINDOWS_DOMAINS < "$SPACE_DOMAINS_FILE"
|
||||
|
||||
for filepath in "$WIN_DIR"/*/Export/*/*; do
|
||||
[[ -f "$filepath" ]] || continue # Regular file
|
||||
[[ -L "$filepath" ]] && continue # Skip Symlinks
|
||||
[[ -d "$filepath" ]] && continue # Skip Directory
|
||||
local file=${filepath##*/}
|
||||
|
||||
local base="${filepath#/mnt/windows/}"
|
||||
local study="${base#*/Export/}"
|
||||
local study="${study%%/"$file"}"
|
||||
local project="${base%%/Export/*}"
|
||||
# local eidf_project="${project%%-*}"
|
||||
|
||||
# Check that windows environment is in domain list
|
||||
[[ ! " ${SPACE_WINDOWS_DOMAINS[*]} " =~ (^|[[:space:]])$project([[:space:]]|$) ]] && {
|
||||
logging::failure "[EXPORT] $project is not in windows domain list"
|
||||
}
|
||||
|
||||
check_mtitme "export" "$filepath" || continue
|
||||
|
||||
local zone
|
||||
local lookup_key="$project"
|
||||
while read -r key value; do
|
||||
if [[ "$key" == "$lookup_key" ]]; then
|
||||
zone="$value"
|
||||
break
|
||||
fi
|
||||
done < "$SPACE_PROJECT_MAP"
|
||||
|
||||
[[ -z "$zone" ]] && {
|
||||
logging::failure "[EXPORT] $filepath — $project has not been mapped to standard or protected"
|
||||
continue
|
||||
}
|
||||
|
||||
new_filename="$zone-$project-$file"
|
||||
logging::debug "[EXPORT] $filepath ==RENAME==> $new_filename"
|
||||
|
||||
read -r sha256 _ < <(sha256sum "$filepath")
|
||||
logging::transfer "export" "sha256:$sha256 — $file" "$TODAY_EGRESS"
|
||||
|
||||
if [[ "$file" == mft-* ]]; then
|
||||
logging::move "[EXPORT] $filepath ==MOVE==> /mnt/os-transfer/windows-mft/$project/$study/Export/$new_filename"
|
||||
mv "$filepath" "$SERVU_BASE/windows-mft/$project/$study/Export/$new_filename"
|
||||
else
|
||||
logging::move "[EXPORT] $filepath ==MOVE==> $EXPORT_DIR/$new_filename"
|
||||
mv "$filepath" "$EXPORT_DIR/$new_filename"
|
||||
chmod 774 "$EXPORT_DIR/$new_filename"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
function logging::transfer() {
|
||||
local action=$1
|
||||
local log_line=$2
|
||||
local log_file=$3
|
||||
|
||||
# Create log file if missing
|
||||
[[ -f "$log_file" ]] || : > "$log_file"
|
||||
|
||||
# Always print to terminal
|
||||
echo -e "${GREEN}[${action^^}] ${log_line}${RESET}"
|
||||
|
||||
# Append to file only if the exact line isn't already logged
|
||||
if ! grep -qxF "$log_line" "$log_file"; then
|
||||
echo "$log_line" >> "$log_file"
|
||||
fi
|
||||
}
|
||||
|
||||
function logging::failure() {
|
||||
local log_line="$1"
|
||||
|
||||
# Create log file if missing
|
||||
[[ -f "$TODAY_FAILURES" ]] || : > "$TODAY_FAILURES"
|
||||
|
||||
# Always print to terminal
|
||||
echo -e "${RED}${log_line}${RESET}"
|
||||
|
||||
# Append to file only if the exact line isn't already logged
|
||||
if ! grep -qxF "$log_line" "$TODAY_FAILURES"; then
|
||||
echo "$log_line" >> "$TODAY_FAILURES"
|
||||
fi
|
||||
}
|
||||
|
||||
function logging::move() { echo -e "${YELLOW}$1${RESET}"; }
|
||||
function logging::debug() { echo -e "${BLUE}$1${RESET}"; }
|
||||
|
||||
function check_mtitme() {
|
||||
local action=$1
|
||||
local filepath=$2
|
||||
local threshold=3
|
||||
if find "$filepath" -mmin -"$threshold" | grep -q .; then
|
||||
logging::debug "[${action^^}] $file — file has been modified in last $(( threshold * 60 )) seconds"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
function purge_files() {
|
||||
logging::debug "[PRUGE] Removing files older than two weeks"
|
||||
for filepath in "$IMPORT_DIR"/*; do
|
||||
#local file=${filepath##*/}
|
||||
if find "$filepath" -mtime +14 | grep -q .; then
|
||||
logging::move "[PURGE] Deleting $filepath"
|
||||
rm "$filepath"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
function email() {
|
||||
local SUBJECT="Space Openstack File Transfer"
|
||||
local MAIL_FROM="os-transfer@adbraun.lol"
|
||||
local RCPT_TO="adam@adbraun.lol"
|
||||
|
||||
: > "$SPACE_EMAIL_FILE"
|
||||
|
||||
# Build email
|
||||
{
|
||||
echo "Subject: $SUBJECT"
|
||||
echo "From: $MAIL_FROM"
|
||||
echo
|
||||
|
||||
if [[ ! -s "$TODAY_EGRESS" ]] && [[ ! -s "$TODAY_INGRESS" ]]; then
|
||||
echo "No files transferred within the last day."
|
||||
echo
|
||||
else
|
||||
if [[ -s "$TODAY_EGRESS" ]]; then
|
||||
echo "Files exported in the last day"
|
||||
echo "---------------------------------"
|
||||
cat "$TODAY_EGRESS"
|
||||
echo
|
||||
else echo "No files exported in the last day"; echo
|
||||
fi
|
||||
|
||||
if [[ -s "$TODAY_INGRESS" ]]; then
|
||||
echo "Files Imported in the last day"
|
||||
echo "---------------------------------"
|
||||
cat "$TODAY_INGRESS"
|
||||
echo
|
||||
else echo "No files imported in the last day"; echo
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -s "$TODAY_FAILURES" ]]; then
|
||||
echo "File Transfer Failures in the last day"
|
||||
echo "---------------------------------"
|
||||
cat "$TODAY_FAILURES"
|
||||
echo
|
||||
fi
|
||||
} >> "$SPACE_EMAIL_FILE"
|
||||
|
||||
sendmail "$RCPT_TO" < "$SPACE_EMAIL_FILE"
|
||||
|
||||
: > "$TODAY_INGRESS"
|
||||
: > "$TODAY_EGRESS"
|
||||
: > "$TODAY_FAILURES"
|
||||
mv "$SPACE_EMAIL_FILE" "$EMAIL_ARCHIVE/$(date +%Y-%m-%d)-space-email"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in new issue
Block a user