Add test for tar deletion

.gitignore

@@ -2,6 +2,7 @@ build/
 dist/
 tests/utils/test_follow_symlinks
 tests/utils/test_follow_symlinks_non_archived
+tests/utils/globus_tar_deletion
 zstash.egg-info/
 *.pyc
 *~

tests/integration/bash_tests/run_from_chrysalis/test_globus_tar_deletion.bash

@@ -0,0 +1,170 @@
+# Assertions ##################################################################
+check_log_has()
+{
+    local expected_grep="${1}"
+    local log_file="${2}"
+    grep -q "${expected_grep}" ${log_file}
+    if [ $? != 0 ]; then
+        echo "Expected grep '${expected_grep}' not found in ${log_file}. Test failed."
+        exit 2
+    fi
+}
+
+check_log_does_not_have()
+{
+    local not_expected_grep="${1}"
+    local log_file="${2}"
+    grep -q "${not_expected_grep}" ${log_file}
+    if [ $? == 0 ]; then
+        echo "Not-expected grep '${not_expected_grep}' was found in ${log_file}. Test failed."
+        exit 2
+    fi
+}
+
+# Helper functions ############################################################
+setup()
+{
+  echo "##########################################################################################################"
+  local case_name="${1}"
+  local src_dir="${2}"
+  echo "Testing: ${case_name}"
+  full_dir="${src_dir}/${case_name}"
+  rm -rf ${full_dir}
+  mkdir -p ${full_dir}
+  cd ${full_dir}
+
+  mkdir zstash_demo
+  mkdir zstash_demo/empty_dir
+  mkdir zstash_demo/dir
+  echo -n '' > zstash_demo/file_empty.txt
+  echo 'file0 stuff' > zstash_demo/dir/file0.txt
+}
+
+get_endpoint()
+{
+    # Usage example:
+    # uuid=$(get_endpoint NERSC_PERLMUTTER_ENDPOINT)
+
+    local endpoint_name=$1
+    # Define endpoints; see https://app.globus.org/collections
+    LCRC_IMPROV_DTN_ENDPOINT=15288284-7006-4041-ba1a-6b52501e49f1
+    case ${endpoint_name} in
+        LCRC_IMPROV_DTN_ENDPOINT)
+            echo ${LCRC_IMPROV_DTN_ENDPOINT}
+            ;;
+        *)
+            echo "Unknown endpoint name: ${endpoint_name}" >&2
+            exit 1
+            ;;
+    esac
+}
+
+confirm() {
+    read -p "$1 (y/n): " -n 1 -r
+    echo
+    [[ $REPLY =~ ^[Yy]$ ]]
+}
+
+# Tests #######################################################################
+test_globus_tar_deletion()
+{
+    local path_to_repo=$1
+    local dst_endpoint=$2
+    local dst_dir=$3
+    local case_name=$4
+
+    src_dir=${path_to_repo}/tests/utils/globus_tar_deletion
+    rm -rf ${src_dir} # Start fresh
+    mkdir -p ${src_dir}
+    dst_endpoint_uuid=$(get_endpoint ${dst_endpoint})
+    globus_path=globus://${dst_endpoint_uuid}/${dst_dir}
+
+    # GLOBUS_CFG=${HOME}/.globus-native-apps.cfg
+    # INI_PATH=${HOME}/.zstash.ini
+    # TOKEN_FILE=${HOME}/.zstash_globus_tokens.json
+
+    # Start fresh
+    # echo "Reset Globus consents:"
+    # echo "https://auth.globus.org/v2/web/consents > Globus Endpoint Performance Monitoring > rescind all"
+    # if ! confirm "Have you revoked Globus consents?"; then
+    #     exit 1
+    # fi
+    # rm -rf ${GLOBUS_CFG}
+    # rm -rf ${INI_PATH}
+    # rm -rf ${TOKEN_FILE}
+
+    echo "Running test_globus_tar_deletion on case=${case_name}"
+    echo "Exit codes: 0 -- success, 1 -- zstash failed, 2 -- grep check failed"
+
+    setup ${case_name} "${src_dir}"
+
+    if [ "$case_name" == "non-blocking" ]; then
+        blocking_flag="--non-blocking"
+    else
+        blocking_flag=""
+    fi
+
+    zstash create ${blocking_flag} --hpss=${globus_path}/${case_name} --maxsize 128 zstash_demo 2>&1 | tee ${case_name}.log
+    if [ $? != 0 ]; then
+        echo "${case_name} failed. Check ${case_name}_create.log for details. Cannot continue."
+        exit 1
+    fi
+    echo "${case_name} completed successfully. Checking ${case_name}.log now."
+    check_log_has "Creating new tar archive 000000.tar" ${case_name}.log
+
+    echo "Checking directory status after 'zstash create' has completed. src should only have index.db. dst should have tar and index.db."
+
+    echo "Checking src"
+    ls ${src_dir}/${case_name}/ 2>&1 | tee ls_${case_name}_src_output.log
+    check_log_has "${case_name}.log" ls_${case_name}_src_output.log
+    check_log_has "zstash_demo" ls_${case_name}_src_output.log
+    ls ${src_dir}/${case_name}/zstash_demo 2>&1 | tee ls_${case_name}_src2_output.log
+    check_log_has "zstash" ls_${case_name}_src2_output.log
+    ls ${src_dir}/${case_name}/zstash_demo/zstash 2>&1 | tee ls_${case_name}_src3_output.log
+    check_log_has "index.db" ls_${case_name}_src3_output.log
+    check_log_does_not_have "tar" ls_${case_name}_src3_output.log # tar is deleted at src
+
+    echo "Checking dst"
+    ls ${dst_dir}/ 2>&1 | tee ls_${case_name}_dst_output.log
+    check_log_has "${case_name}" ls_${case_name}_dst_output.log
+    ls ${dst_dir}/${case_name}/ 2>&1 | tee ls_${case_name}_dst2_output.log
+    check_log_has "index.db" ls_${case_name}_dst2_output.log
+    check_log_has "000000.tar" ls_${case_name}_dst2_output.log # tar should be at dst
+
+    # Cleanup:
+    #cd ${path_to_repo}/tests/integration/bash_tests/run_from_chrysalis
+    #rm -rf ${path_to_repo}/tests/utils/globus_tar_deletion
+}
+
+# Follow these directions #####################################################
+
+# Example usage:
+# ./test_globus_tar_deletion.bash run1 /home/ac.forsyth2/ez/zstash /home/ac.forsyth2/zstash_tests
+
+# Command line parameters:
+unique_id="$1"
+path_to_repo="$2" # /home/ac.forsyth2/ez/zstash
+chrysalis_dst_basedir="$3" # /home/ac.forsyth2/zstash_tests
+chrysalis_dst_dir=${chrysalis_dst_basedir}/test_globus_tar_deletion_${unique_id}
+
+
+echo "You may wish to clear your dst directories for a fresh start:"
+echo "Chrysalis: rm -rf ${chrysalis_dst_basedir}/test_globus_tar_deletion*"
+echo "It is advisable to just set a unique_id to avoid directory conflicts."
+echo "Currently, unique_id=${unique_id}"
+if ! confirm "Is the unique_id correct?"; then
+    exit 1
+fi
+
+echo "Go to https://app.globus.org/file-manager?two_pane=true > For "Collection", choose each of the following endpoints and, if needed, authenticate:"
+echo "LCRC Improv DTN"
+if ! confirm "Have you authenticated into all endpoints?"; then
+    exit 1
+fi
+
+echo "Primary tests: single authentication code tests for each endpoint"
+echo "If a test hangs, check if https://app.globus.org/activity reports any errors on your transfers."
+test_globus_tar_deletion ${path_to_repo} LCRC_IMPROV_DTN_ENDPOINT ${chrysalis_dst_dir} "blocking"
+test_globus_tar_deletion ${path_to_repo} LCRC_IMPROV_DTN_ENDPOINT ${chrysalis_dst_dir} "non-blocking"
+
+echo "All globus tar deletion tests completed successfully."