diff --git a/README.md b/README.md index 36a11b15..7d43edd5 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ Restart the SSH service: sudo systemctl restart ssh ``` -Keep your private key (`~/.ssh/id_rsa`) safe. You will need it for all future logins. +Keep your private key (`~/.ssh/id_rsa`) safe. You will need it for all future logins. **Do not upload it to the server** --- @@ -189,6 +189,17 @@ sudo ufw allow 8989 Once your server is secured and accessible, proceed with the standard masternode setup. +### Log Retention Settings + +The startup scripts for `mainnet`, `testnet`, and `devnet` support two log retention environment variables: + +* `LOG_KEEP_DAYS`: total number of calendar days of node logs to retain. Minimum value: `3`. +* `LOG_UNCOMPRESSED_DAYS`: number of recent calendar days to keep as plain `.log` files before older retained logs are compressed to `.log.gz`. Minimum value: `2`. + +Default values are `LOG_KEEP_DAYS=30` and `LOG_UNCOMPRESSED_DAYS=3`. If `LOG_KEEP_DAYS` is below `3`, or `LOG_UNCOMPRESSED_DAYS` is below `2`, or `LOG_UNCOMPRESSED_DAYS` is greater than `LOG_KEEP_DAYS`, the node exits during startup with an error. + +Mainnet and testnet read these values from their `.env` files. Devnet reads them from the per-node files under `devnet/envs/${HOSTIP}-1` and `devnet/envs/${HOSTIP}-2`. Old `.log` files are compressed only after rotation, and the original file is removed only after the `.log.gz` file has been written successfully. + --- ## How to Setup XinFin Masternode diff --git a/devnet/.env b/devnet/.env index 312cf719..406bef74 100755 --- a/devnet/.env +++ b/devnet/.env @@ -4,7 +4,7 @@ SYNC_MODE=full GC_MODE=archive NETWORK=devnet PRIVATE_KEY=3333333333333333333333333333333333333333333333333333333333333333 -LOG_LEVEL=2 +LOG_LEVEL=3 ENABLE_RPC=false # RPC API setting diff --git a/devnet/docker-compose-group.yml b/devnet/docker-compose-group.yml index f25e8f5b..551d86ee 100644 --- a/devnet/docker-compose-group.yml +++ b/devnet/docker-compose-group.yml @@ -1,8 +1,8 @@ services: xinfinnetwork1: build: - context: . - dockerfile: . + context: . + dockerfile: . volumes: - "./xdcchain-1:/work/xdcchain" - "./genesis.json:/work/genesis.json" @@ -11,6 +11,7 @@ services: - "./.pwd:/work/.pwd" - "/etc/localtime:/etc/localtime:ro" restart: "always" + stop_grace_period: 60s env_file: "./envs/${HOSTIP}-1" ports: - "30303:30303" @@ -19,8 +20,8 @@ services: xinfinnetwork2: build: - context: . - dockerfile: . + context: . + dockerfile: . volumes: - "./xdcchain-2:/work/xdcchain" - "./genesis.json:/work/genesis.json" @@ -29,6 +30,7 @@ services: - "./.pwd:/work/.pwd" - "/etc/localtime:/etc/localtime:ro" restart: "always" + stop_grace_period: 60s env_file: "./envs/${HOSTIP}-2" ports: - "30304:30304" diff --git a/devnet/docker-compose-hash-rpc.yml b/devnet/docker-compose-hash-rpc.yml index 2e17d1f9..0f55a1ba 100644 --- a/devnet/docker-compose-hash-rpc.yml +++ b/devnet/docker-compose-hash-rpc.yml @@ -4,6 +4,7 @@ services: volumes: - "./xdcchain-1:/work/xdcchain" restart: "always" + stop_grace_period: 60s env_file: "./.env" ports: - "30303:30303" diff --git a/devnet/docker-compose.yml b/devnet/docker-compose.yml index 45c5d7fd..570cbbe8 100644 --- a/devnet/docker-compose.yml +++ b/devnet/docker-compose.yml @@ -11,6 +11,7 @@ services: - "./.pwd:/work/.pwd" - "/etc/localtime:/etc/localtime:ro" restart: "always" + stop_grace_period: 60s env_file: "./.env" ports: - "30303:30303" diff --git a/devnet/envs/144.126.140.3-1 b/devnet/envs/144.126.140.3-1 index aa269831..e802d601 100644 --- a/devnet/envs/144.126.140.3-1 +++ b/devnet/envs/144.126.140.3-1 @@ -5,3 +5,5 @@ PRIVATE_KEY=77e0b4a7aabef3b07db42f1f2aa43130ba5a3175ec47f2ff8891d49ad87cbed9 PORT=30303 RPCPORT=8545 WSPORT=8555 +LOG_KEEP_DAYS=30 +LOG_UNCOMPRESSED_DAYS=3 diff --git a/devnet/envs/144.126.140.3-2 b/devnet/envs/144.126.140.3-2 index f52eb8b1..02052ce4 100644 --- a/devnet/envs/144.126.140.3-2 +++ b/devnet/envs/144.126.140.3-2 @@ -5,3 +5,5 @@ PRIVATE_KEY=31b0fbcba7b60ea9974ae1bac0523af8cd72661de47ea2d3569344c975b93801 PORT=30303 RPCPORT=8545 WSPORT=8555 +LOG_KEEP_DAYS=30 +LOG_UNCOMPRESSED_DAYS=3 diff --git a/devnet/envs/194.163.167.177-1 b/devnet/envs/194.163.167.177-1 index f51c7a8d..647f8614 100644 --- a/devnet/envs/194.163.167.177-1 +++ b/devnet/envs/194.163.167.177-1 @@ -5,3 +5,5 @@ PRIVATE_KEY=5a3457e9323ef7f9351d7b6d8f4d5c2c7c66a5e094142d6e186c86402b29a787 PORT=30303 RPCPORT=8545 WSPORT=8555 +LOG_KEEP_DAYS=30 +LOG_UNCOMPRESSED_DAYS=3 diff --git a/devnet/envs/194.163.167.177-2 b/devnet/envs/194.163.167.177-2 index 0e1bfc07..b26f0056 100644 --- a/devnet/envs/194.163.167.177-2 +++ b/devnet/envs/194.163.167.177-2 @@ -5,3 +5,5 @@ PRIVATE_KEY=363f48b205f95859e13ea1acf6b047631ab34e69d193bbf5eb0df871decca69e PORT=30303 RPCPORT=8545 WSPORT=8555 +LOG_KEEP_DAYS=30 +LOG_UNCOMPRESSED_DAYS=3 diff --git a/devnet/envs/194.233.77.19-1 b/devnet/envs/194.233.77.19-1 index 21a6d8d7..9ad86a98 100644 --- a/devnet/envs/194.233.77.19-1 +++ b/devnet/envs/194.233.77.19-1 @@ -5,3 +5,5 @@ PRIVATE_KEY=bde24b587c04ab8d8cc2acc176a4b85e646d3c89f4d98d570629d756aea68303 PORT=30303 RPCPORT=8545 WSPORT=8555 +LOG_KEEP_DAYS=30 +LOG_UNCOMPRESSED_DAYS=3 diff --git a/devnet/envs/194.233.77.19-2 b/devnet/envs/194.233.77.19-2 index f6664acc..1d3ca4ae 100644 --- a/devnet/envs/194.233.77.19-2 +++ b/devnet/envs/194.233.77.19-2 @@ -5,3 +5,5 @@ PRIVATE_KEY=3efdb44088929167487da052125162b48d8d54fe8f7b7db11b5d5cc3b9a1c14b PORT=30304 RPCPORT=8546 WSPORT=8556 +LOG_KEEP_DAYS=30 +LOG_UNCOMPRESSED_DAYS=3 diff --git a/devnet/envs/66.94.121.151-1 b/devnet/envs/66.94.121.151-1 index a49a767f..16e3a588 100644 --- a/devnet/envs/66.94.121.151-1 +++ b/devnet/envs/66.94.121.151-1 @@ -5,3 +5,5 @@ PRIVATE_KEY=1c40ebf394c9c9db15f60528f6a030ba9f465a7c615acd9b9d79792175b6bcd6 PORT=30303 RPCPORT=8545 WSPORT=8555 +LOG_KEEP_DAYS=30 +LOG_UNCOMPRESSED_DAYS=3 diff --git a/devnet/envs/66.94.121.151-2 b/devnet/envs/66.94.121.151-2 index d7dd3c4f..2e0bab65 100644 --- a/devnet/envs/66.94.121.151-2 +++ b/devnet/envs/66.94.121.151-2 @@ -5,3 +5,5 @@ PRIVATE_KEY=58fbe847ab6faa2fb5559b4d1f1e02573e222d2524b6f4598a301897c0881e71 PORT=30303 RPCPORT=8545 WSPORT=8555 +LOG_KEEP_DAYS=30 +LOG_UNCOMPRESSED_DAYS=3 diff --git a/devnet/envs/66.94.98.186-1 b/devnet/envs/66.94.98.186-1 index 63fe8202..ba9cc859 100644 --- a/devnet/envs/66.94.98.186-1 +++ b/devnet/envs/66.94.98.186-1 @@ -5,3 +5,5 @@ PRIVATE_KEY=64651f33879becd32391e3cf802680f3621500c55fb53db7b6b041ff74c3a62f PORT=30303 RPCPORT=8545 WSPORT=8555 +LOG_KEEP_DAYS=30 +LOG_UNCOMPRESSED_DAYS=3 diff --git a/devnet/envs/66.94.98.186-2 b/devnet/envs/66.94.98.186-2 index bfbbf356..812a1536 100644 --- a/devnet/envs/66.94.98.186-2 +++ b/devnet/envs/66.94.98.186-2 @@ -5,3 +5,5 @@ PRIVATE_KEY=e754b95280b2232ffb4398de0cdda06c2be24ef8aa5c6aba090802e0cd706022 PORT=30303 RPCPORT=8545 WSPORT=8555 +LOG_KEEP_DAYS=30 +LOG_UNCOMPRESSED_DAYS=3 diff --git a/devnet/start.sh b/devnet/start.sh index 972d7543..ca5a4997 100755 --- a/devnet/start.sh +++ b/devnet/start.sh @@ -1,6 +1,6 @@ #!/bin/bash if [ ! -d /work/xdcchain/XDC/chaindata ]; then - echo $PRIVATE_KEY >>/tmp/key + echo "$PRIVATE_KEY" >>/tmp/key wallet=$(XDC account import --password .pwd --datadir /work/xdcchain /tmp/key | awk -F '[{}]' '{print $2}') XDC init --datadir /work/xdcchain /work/genesis.json else @@ -17,7 +17,7 @@ while IFS= read -r line; do fi done <"$input" -log_level=2 +log_level=3 if test -z "$LOG_LEVEL" then echo "Log level not set, default to verbosity of $log_level" @@ -26,9 +26,467 @@ else log_level=$LOG_LEVEL fi -# create log file with timestamp -DATE="$(date +%Y%m%d-%H%M%S)" -LOG_FILE="/work/xdcchain/xdc-${DATE}.log" +log_dir=/work/xdcchain +log_keep_days=30 +log_uncompressed_days=3 +log_compress_lock_dir="${log_dir}/.log-compress.lock" +log_compress_lock_ready_file="${log_compress_lock_dir}/ready" +log_compress_lock_partial_stale_minutes=1 +stream_chunk_size=4096 +stream_chunk_timeout=0.2 +initial_log_day="" +initial_log_path="" + +if [[ -z "${LOG_KEEP_DAYS}" ]]; then + echo "LOG_KEEP_DAYS not set, default to ${log_keep_days}" +elif [[ "${LOG_KEEP_DAYS}" =~ ^[0-9]+$ && "${LOG_KEEP_DAYS}" -ge 3 ]]; then + echo "LOG_KEEP_DAYS found, set to ${LOG_KEEP_DAYS}" + log_keep_days=${LOG_KEEP_DAYS} +else + echo "Invalid LOG_KEEP_DAYS: ${LOG_KEEP_DAYS}. Expected an integer greater than or equal to 3." >&2 + exit 1 +fi + +if [[ -z "${LOG_UNCOMPRESSED_DAYS}" ]]; then + echo "LOG_UNCOMPRESSED_DAYS not set, default to ${log_uncompressed_days}" +elif [[ "${LOG_UNCOMPRESSED_DAYS}" =~ ^[0-9]+$ && "${LOG_UNCOMPRESSED_DAYS}" -ge 2 ]]; then + echo "LOG_UNCOMPRESSED_DAYS found, set to ${LOG_UNCOMPRESSED_DAYS}" + log_uncompressed_days=${LOG_UNCOMPRESSED_DAYS} +else + echo "Invalid LOG_UNCOMPRESSED_DAYS: ${LOG_UNCOMPRESSED_DAYS}. Expected an integer greater than or equal to 2." >&2 + exit 1 +fi + +if [[ "${log_uncompressed_days}" -gt "${log_keep_days}" ]]; then + echo "Invalid log retention settings: LOG_UNCOMPRESSED_DAYS (${log_uncompressed_days}) cannot be greater than LOG_KEEP_DAYS (${log_keep_days})." >&2 + exit 1 +fi + +read_stream_chunk() { + local stream_fd=$1 + local result_var=$2 + local chunk_data="" + local read_status=0 + + IFS= read -r -N "${stream_chunk_size}" -t "${stream_chunk_timeout}" chunk_data <"/proc/self/fd/${stream_fd}" + read_status=$? + printf -v "${result_var}" '%s' "${chunk_data}" + + return "${read_status}" +} + +extract_log_date() { + local log_name=$1 + + parsed_log_date="" + if [[ "${log_name}" =~ ^xdc-([0-9]{8})-[0-9]{6}\.log(\.gz)?$ ]]; then + parsed_log_date="${BASH_REMATCH[1]}" + return 0 + fi + + return 1 +} + +get_process_start_time() { + local pid=$1 + local start_time="" + + start_time="$(awk '{print $22}' "/proc/${pid}/stat" 2>/dev/null)" + if [[ ! "${start_time}" =~ ^[0-9]+$ ]]; then + return 1 + fi + + printf '%s\n' "${start_time}" +} + +get_process_pid_namespace() { + local pid=$1 + local pid_namespace="" + + pid_namespace="$(readlink "/proc/${pid}/ns/pid" 2>/dev/null)" + if [[ -z "${pid_namespace}" ]]; then + return 1 + fi + + printf '%s\n' "${pid_namespace}" +} + +lock_process_is_active() { + local pid=$1 + local expected_start_time=$2 + local expected_pid_namespace=$3 + local current_start_time="" + local current_pid_namespace="" + + if [[ ! "${pid}" =~ ^[0-9]+$ ]] || ! kill -0 "${pid}" 2>/dev/null; then + return 1 + fi + + current_start_time="$(get_process_start_time "${pid}")" || return 1 + current_pid_namespace="$(get_process_pid_namespace "${pid}")" || return 1 + + if [[ "${current_start_time}" != "${expected_start_time}" || "${current_pid_namespace}" != "${expected_pid_namespace}" ]]; then + return 1 + fi + + return 0 +} + +cleanup_old_logs() { + local retention_start_date="" + local deleted_logs=0 + local log_file="" + local log_name="" + local log_date="" + + retention_start_date="$(date -d "-$((log_keep_days - 1)) days" +%Y%m%d)" + + while IFS= read -r -d '' log_file; do + log_name="$(basename "${log_file}")" + if extract_log_date "${log_name}"; then + log_date="${parsed_log_date}" + if [[ "${log_date}" < "${retention_start_date}" ]]; then + echo "Deleting expired log file: ${log_file}" + rm -f -- "${log_file}" + deleted_logs=$((deleted_logs + 1)) + fi + fi + done < <(find "${log_dir}" -maxdepth 1 -type f \( -name 'xdc-*.log' -o -name 'xdc-*.log.gz' \) -print0) + + echo "Log retention: kept ${log_keep_days} day(s), removed ${deleted_logs} expired log file(s)" +} + +compress_old_logs() { + local active_log_path=$1 + local retention_start_date="" + local uncompressed_start_date="" + local compressed_logs=0 + local failed_logs=0 + local recovered_logs=0 + local log_file="" + local log_name="" + local log_date="" + local final_gz="" + local tmp_gz="" + local invalid_archive_removed=0 + + retention_start_date="$(date -d "-$((log_keep_days - 1)) days" +%Y%m%d)" + uncompressed_start_date="$(date -d "-$((log_uncompressed_days - 1)) days" +%Y%m%d)" + + while IFS= read -r -d '' log_file; do + if [[ "${log_file}" == "${active_log_path}" ]]; then + continue + fi + + log_name="$(basename "${log_file}")" + if ! extract_log_date "${log_name}"; then + continue + fi + + log_date="${parsed_log_date}" + if [[ "${log_date}" < "${retention_start_date}" || "${log_date}" > "${uncompressed_start_date}" || "${log_date}" == "${uncompressed_start_date}" ]]; then + continue + fi + + final_gz="${log_file}.gz" + invalid_archive_removed=0 + if [[ -e "${final_gz}" ]]; then + if gzip -t -- "${final_gz}" >/dev/null 2>&1; then + if rm -f -- "${log_file}"; then + echo "Compressed log already exists for ${log_name}; removed leftover original log file" + recovered_logs=$((recovered_logs + 1)) + else + echo "Unable to remove leftover original log file for ${log_name}" + failed_logs=$((failed_logs + 1)) + fi + else + echo "Existing compressed log invalid for ${log_name}; recreating archive" + if ! rm -f -- "${final_gz}"; then + echo "Unable to remove invalid compressed log for ${log_name}" + failed_logs=$((failed_logs + 1)) + continue + fi + invalid_archive_removed=1 + fi + + if [[ "${invalid_archive_removed}" -eq 0 ]]; then + continue + fi + fi + + tmp_gz="${final_gz}.tmp.${BASHPID:-$$}" + rm -f -- "${tmp_gz}" + echo "Compressing log file: ${log_name}" + if gzip -c -- "${log_file}" >"${tmp_gz}" && mv -f -- "${tmp_gz}" "${final_gz}" && rm -f -- "${log_file}"; then + compressed_logs=$((compressed_logs + 1)) + else + rm -f -- "${tmp_gz}" + echo "Log compression failed for ${log_name}; keeping original log file" + failed_logs=$((failed_logs + 1)) + fi + done < <(find "${log_dir}" -maxdepth 1 -type f -name 'xdc-*.log' -print0) + + echo "Log compression: compressed ${compressed_logs} file(s), recovered ${recovered_logs} file(s), failed ${failed_logs} file(s)" +} + +log_compression_lock_is_ready() { + [[ -f "${log_compress_lock_ready_file}" ]] +} + +lock_directory_is_stale_partial() { + find "${log_compress_lock_dir}" -maxdepth 0 -type d -mmin "+${log_compress_lock_partial_stale_minutes}" | grep -q . +} + +cleanup_stale_log_temp_files() { + local removed_temp_files=0 + local temp_file="" + + while IFS= read -r -d '' temp_file; do + if rm -f -- "${temp_file}"; then + removed_temp_files=$((removed_temp_files + 1)) + fi + done < <(find "${log_dir}" -maxdepth 1 -type f -name 'xdc-*.log.gz.tmp.*' -print0) + + echo "Log compression cleanup: removed ${removed_temp_files} stale temp file(s)" +} + +acquire_log_compression_lock() { + local attempt=0 + local existing_pid="" + local existing_start_time="" + local existing_pid_namespace="" + local current_pid="" + local current_start_time="" + local current_pid_namespace="" + + while [[ "${attempt}" -lt 2 ]]; do + if mkdir "${log_compress_lock_dir}" 2>/dev/null; then + current_pid="${BASHPID:-$$}" + printf '%s\n' "${current_pid}" >"${log_compress_lock_dir}/pid" + current_start_time="$(get_process_start_time "${current_pid}")" && printf '%s\n' "${current_start_time}" >"${log_compress_lock_dir}/start_time" + current_pid_namespace="$(get_process_pid_namespace "${current_pid}")" && printf '%s\n' "${current_pid_namespace}" >"${log_compress_lock_dir}/pid_namespace" + date -u +%Y-%m-%dT%H:%M:%SZ >"${log_compress_lock_dir}/started_at" + : >"${log_compress_lock_ready_file}" + return 0 + fi + + if ! log_compression_lock_is_ready; then + if lock_directory_is_stale_partial; then + echo "Recovering stale partial log compression lock" + rm -rf -- "${log_compress_lock_dir}" + attempt=$((attempt + 1)) + continue + fi + + echo "Log compression lock is still initializing, skipping this cycle" + return 1 + fi + + if [[ -f "${log_compress_lock_dir}/pid" ]]; then + existing_pid="$(cat "${log_compress_lock_dir}/pid" 2>/dev/null)" + fi + if [[ -f "${log_compress_lock_dir}/start_time" ]]; then + existing_start_time="$(cat "${log_compress_lock_dir}/start_time" 2>/dev/null)" + fi + if [[ -f "${log_compress_lock_dir}/pid_namespace" ]]; then + existing_pid_namespace="$(cat "${log_compress_lock_dir}/pid_namespace" 2>/dev/null)" + fi + + if [[ -n "${existing_start_time}" && -n "${existing_pid_namespace}" ]] && lock_process_is_active "${existing_pid}" "${existing_start_time}" "${existing_pid_namespace}"; then + echo "Log compression already running with pid ${existing_pid}, skipping this cycle" + return 1 + fi + + echo "Recovering stale log compression lock" + rm -rf -- "${log_compress_lock_dir}" + attempt=$((attempt + 1)) + done + + echo "Unable to acquire log compression lock, skipping this cycle" + return 1 +} + +release_log_compression_lock() { + rm -rf -- "${log_compress_lock_dir}" +} + +compress_old_logs_with_lock() { + local active_log_path=$1 + + if ! command -v gzip >/dev/null 2>&1; then + echo "gzip not available, skipping log compression" + return 0 + fi + + if ! acquire_log_compression_lock; then + return 0 + fi + + trap 'release_log_compression_lock' EXIT + cleanup_stale_log_temp_files + compress_old_logs "${active_log_path}" +} + +run_log_compression_async() { + local active_log_path=$1 + + ( + if command -v ionice >/dev/null 2>&1; then + ionice -c3 -p "${BASHPID:-$$}" >/dev/null 2>&1 || true + fi + if command -v renice >/dev/null 2>&1; then + renice -n 19 -p "${BASHPID:-$$}" >/dev/null 2>&1 || true + fi + compress_old_logs_with_lock "${active_log_path}" + ) & +} + +prepare_initial_log_file() { + local timestamp="" + local log_fd="" + + mkdir -p "${log_dir}" || return 1 + + timestamp="$(date +%Y%m%d-%H%M%S)" + initial_log_day="${timestamp%%-*}" + initial_log_path="${log_dir}/xdc-${timestamp}.log" + + exec {log_fd}>>"${initial_log_path}" || return 1 + exec {log_fd}>&- + cleanup_old_logs + run_log_compression_async "${initial_log_path}" +} + +write_rotated_log_stream() { + local current_day="${initial_log_day}" + local log_fd="" + local log_path="${initial_log_path}" + local timestamp="" + local chunk="" + local read_status=0 + + if [[ -n "${log_path}" ]]; then + exec {log_fd}>>"${log_path}" || return 1 + fi + + while true; do + read_stream_chunk 0 chunk + read_status=$? + + if [[ "${read_status}" -gt 128 && -z "${chunk}" ]]; then + continue + fi + + if [[ -z "${chunk}" && "${read_status}" -ne 0 ]]; then + break + fi + + timestamp="$(date +%Y%m%d-%H%M%S)" + + if [[ "${timestamp%%-*}" != "${current_day}" ]]; then + if [[ -n "${log_fd}" ]]; then + exec {log_fd}>&- + fi + + current_day="${timestamp%%-*}" + log_path="${log_dir}/xdc-${timestamp}.log" + exec {log_fd}>>"${log_path}" || return 1 + cleanup_old_logs + run_log_compression_async "${log_path}" + fi + + printf '%s' "${chunk}" >&${log_fd} || return 1 + done + + if [[ -n "${log_fd}" ]]; then + exec {log_fd}>&- + fi +} + +log_writer_fd="" +log_writer_pid="" +xdc_read_fd="" +xdc_write_fd="" +xdc_pid="" +shutdown_requested=0 +shutdown_exit_code=0 + +start_log_writer() { + prepare_initial_log_file || return 1 + exec {log_writer_fd}> >(write_rotated_log_stream) + log_writer_pid=$! +} + +stop_log_writer() { + local log_writer_exit_code=0 + + if [[ -n "${log_writer_fd}" ]]; then + exec {log_writer_fd}>&- + log_writer_fd="" + fi + + if [[ -n "${log_writer_pid}" ]]; then + wait "${log_writer_pid}" + log_writer_exit_code=$? + log_writer_pid="" + fi + + return "${log_writer_exit_code}" +} + +start_xdc_stream() { + coproc XDC_STREAM { exec XDC "${args[@]}" 2>&1; } + xdc_pid=$XDC_STREAM_PID + xdc_read_fd=${XDC_STREAM[0]} + xdc_write_fd=${XDC_STREAM[1]} + exec {xdc_write_fd}>&- + xdc_write_fd="" +} + +stop_xdc_stream() { + local xdc_exit_code=0 + + if [[ -n "${xdc_pid}" ]]; then + wait "${xdc_pid}" + xdc_exit_code=$? + xdc_pid="" + fi + + if [[ -n "${xdc_read_fd}" ]]; then + exec {xdc_read_fd}<&- + xdc_read_fd="" + fi + + return "${xdc_exit_code}" +} + +terminate_xdc_stream() { + if [[ -n "${xdc_read_fd}" ]]; then + exec {xdc_read_fd}<&- + xdc_read_fd="" + fi + + if [[ -n "${xdc_pid}" ]]; then + kill "${xdc_pid}" 2>/dev/null || true + wait "${xdc_pid}" 2>/dev/null || true + xdc_pid="" + fi +} + +request_shutdown() { + local signal_name=$1 + + if [[ "${shutdown_requested}" -ne 0 ]]; then + return + fi + + shutdown_requested=1 + shutdown_exit_code=$((128 + $(kill -l "${signal_name}"))) + + if [[ -n "${xdc_pid}" ]]; then + kill -s "${signal_name}" "${xdc_pid}" 2>/dev/null || true + fi +} # Set sync_mode from SYNC_MODE env or default to 'full' sync_mode=full @@ -86,4 +544,62 @@ if echo "${ENABLE_RPC}" | grep -iq "true"; then ) fi -XDC "${args[@]}" 2>&1 >>"${LOG_FILE}" | tee -a "${LOG_FILE}" +set -o pipefail +trap 'request_shutdown TERM' TERM +trap 'request_shutdown INT' INT +start_log_writer || exit $? +start_xdc_stream +stream_exit_code=0 +while true; do + chunk="" + read_stream_chunk "${xdc_read_fd}" chunk + read_status=$? + + if [[ "${read_status}" -gt 128 && -z "${chunk}" ]]; then + continue + fi + + if [[ -z "${chunk}" && "${read_status}" -ne 0 ]]; then + break + fi + + printf '%s' "${chunk}" || { + stream_exit_code=$? + break + } + + printf '%s' "${chunk}" >"/proc/self/fd/${log_writer_fd}" || { + stream_exit_code=$? + break + } +done + +xdc_exit_code=0 +if [[ "${shutdown_requested}" -ne 0 ]]; then + stop_xdc_stream + xdc_exit_code=$? +elif [[ "${stream_exit_code}" -eq 0 ]]; then + stop_xdc_stream + xdc_exit_code=$? +else + terminate_xdc_stream +fi + +stop_log_writer +log_writer_exit_code=$? + +trap - TERM INT + +if [[ "${shutdown_requested}" -ne 0 ]]; then + exit "${shutdown_exit_code}" +fi + +if [[ "${xdc_exit_code}" -ne 0 ]]; then + exit "${xdc_exit_code}" +fi + +if [[ "${stream_exit_code}" -ne 0 ]]; then + exit "${stream_exit_code}" +fi + +exit "${log_writer_exit_code}" diff --git a/mainnet/.env b/mainnet/.env index e0eb5e00..a7eede22 100644 --- a/mainnet/.env +++ b/mainnet/.env @@ -4,7 +4,8 @@ SYNC_MODE=full GC_MODE=full NETWORK=mainnet PRIVATE_KEY=3333333333333333333333333333333333333333333333333333333333333333 -LOG_LEVEL=2 +LOG_LEVEL=3 +LOG_KEEP_DAYS=30 ENABLE_RPC=false # RPC API setting diff --git a/mainnet/docker-compose-hash-rpc.yml b/mainnet/docker-compose-hash-rpc.yml index 2e17d1f9..0f55a1ba 100644 --- a/mainnet/docker-compose-hash-rpc.yml +++ b/mainnet/docker-compose-hash-rpc.yml @@ -4,6 +4,7 @@ services: volumes: - "./xdcchain-1:/work/xdcchain" restart: "always" + stop_grace_period: 60s env_file: "./.env" ports: - "30303:30303" diff --git a/mainnet/docker-compose.yml b/mainnet/docker-compose.yml index 65b12681..98b3e4ba 100644 --- a/mainnet/docker-compose.yml +++ b/mainnet/docker-compose.yml @@ -5,11 +5,13 @@ services: volumes: - "./xdcchain:/work/xdcchain" - "./genesis.json:/work/genesis.json" + - "./entry.sh:/work/entry.sh" - "./start-node.sh:/work/start.sh" - "./bootnodes.list:/work/bootnodes.list" - "./.pwd:/work/.pwd" - "/etc/localtime:/etc/localtime:ro" restart: "always" + stop_grace_period: 60s env_file: - .env ports: diff --git a/mainnet/entry.sh b/mainnet/entry.sh new file mode 100755 index 00000000..58d54f78 --- /dev/null +++ b/mainnet/entry.sh @@ -0,0 +1,13 @@ +#!/bin/bash +if test -z "$NETWORK" +then + echo "NETWORK env Must be set, mainnet/testnet/devnet/local" + exit 1 +fi + +echo "Select to run $NETWORK..." +ln -s /usr/bin/XDC-"$NETWORK" /usr/bin/XDC +cp -n /work/"$NETWORK"/* /work + +echo "Start Node..." +exec /work/start.sh diff --git a/mainnet/env.example b/mainnet/env.example index 4bfb04a2..7da97403 100644 --- a/mainnet/env.example +++ b/mainnet/env.example @@ -2,7 +2,9 @@ INSTANCE_NAME=YOUR_NODE_NAME CONTACT_DETAILS=YOUR_EMAIL_ADDRESS NODE_TYPE=full NETWORK=mainnet -LOG_LEVEL=2 +LOG_LEVEL=3 +LOG_KEEP_DAYS=30 +LOG_UNCOMPRESSED_DAYS=3 ENABLE_RPC=false # RPC API setting diff --git a/mainnet/start-node.sh b/mainnet/start-node.sh index 4cec61a9..427fee46 100755 --- a/mainnet/start-node.sh +++ b/mainnet/start-node.sh @@ -24,7 +24,7 @@ while IFS= read -r line; do fi done <"$input" -log_level=2 +log_level=3 if test -z "$LOG_LEVEL" then echo "Log level not set, default to verbosity of $log_level" @@ -33,9 +33,467 @@ else log_level=$LOG_LEVEL fi -# create log file with timestamp -DATE="$(date +%Y%m%d-%H%M%S)" -LOG_FILE="/work/xdcchain/xdc-${DATE}.log" +log_dir=/work/xdcchain +log_keep_days=30 +log_uncompressed_days=3 +log_compress_lock_dir="${log_dir}/.log-compress.lock" +log_compress_lock_ready_file="${log_compress_lock_dir}/ready" +log_compress_lock_partial_stale_minutes=1 +stream_chunk_size=4096 +stream_chunk_timeout=0.2 +initial_log_day="" +initial_log_path="" + +if [[ -z "${LOG_KEEP_DAYS}" ]]; then + echo "LOG_KEEP_DAYS not set, default to ${log_keep_days}" +elif [[ "${LOG_KEEP_DAYS}" =~ ^[0-9]+$ && "${LOG_KEEP_DAYS}" -ge 3 ]]; then + echo "LOG_KEEP_DAYS found, set to ${LOG_KEEP_DAYS}" + log_keep_days=${LOG_KEEP_DAYS} +else + echo "Invalid LOG_KEEP_DAYS: ${LOG_KEEP_DAYS}. Expected an integer greater than or equal to 3." >&2 + exit 1 +fi + +if [[ -z "${LOG_UNCOMPRESSED_DAYS}" ]]; then + echo "LOG_UNCOMPRESSED_DAYS not set, default to ${log_uncompressed_days}" +elif [[ "${LOG_UNCOMPRESSED_DAYS}" =~ ^[0-9]+$ && "${LOG_UNCOMPRESSED_DAYS}" -ge 2 ]]; then + echo "LOG_UNCOMPRESSED_DAYS found, set to ${LOG_UNCOMPRESSED_DAYS}" + log_uncompressed_days=${LOG_UNCOMPRESSED_DAYS} +else + echo "Invalid LOG_UNCOMPRESSED_DAYS: ${LOG_UNCOMPRESSED_DAYS}. Expected an integer greater than or equal to 2." >&2 + exit 1 +fi + +if [[ "${log_uncompressed_days}" -gt "${log_keep_days}" ]]; then + echo "Invalid log retention settings: LOG_UNCOMPRESSED_DAYS (${log_uncompressed_days}) cannot be greater than LOG_KEEP_DAYS (${log_keep_days})." >&2 + exit 1 +fi + +read_stream_chunk() { + local stream_fd=$1 + local result_var=$2 + local chunk_data="" + local read_status=0 + + IFS= read -r -N "${stream_chunk_size}" -t "${stream_chunk_timeout}" chunk_data <"/proc/self/fd/${stream_fd}" + read_status=$? + printf -v "${result_var}" '%s' "${chunk_data}" + + return "${read_status}" +} + +extract_log_date() { + local log_name=$1 + + parsed_log_date="" + if [[ "${log_name}" =~ ^xdc-([0-9]{8})-[0-9]{6}\.log(\.gz)?$ ]]; then + parsed_log_date="${BASH_REMATCH[1]}" + return 0 + fi + + return 1 +} + +get_process_start_time() { + local pid=$1 + local start_time="" + + start_time="$(awk '{print $22}' "/proc/${pid}/stat" 2>/dev/null)" + if [[ ! "${start_time}" =~ ^[0-9]+$ ]]; then + return 1 + fi + + printf '%s\n' "${start_time}" +} + +get_process_pid_namespace() { + local pid=$1 + local pid_namespace="" + + pid_namespace="$(readlink "/proc/${pid}/ns/pid" 2>/dev/null)" + if [[ -z "${pid_namespace}" ]]; then + return 1 + fi + + printf '%s\n' "${pid_namespace}" +} + +lock_process_is_active() { + local pid=$1 + local expected_start_time=$2 + local expected_pid_namespace=$3 + local current_start_time="" + local current_pid_namespace="" + + if [[ ! "${pid}" =~ ^[0-9]+$ ]] || ! kill -0 "${pid}" 2>/dev/null; then + return 1 + fi + + current_start_time="$(get_process_start_time "${pid}")" || return 1 + current_pid_namespace="$(get_process_pid_namespace "${pid}")" || return 1 + + if [[ "${current_start_time}" != "${expected_start_time}" || "${current_pid_namespace}" != "${expected_pid_namespace}" ]]; then + return 1 + fi + + return 0 +} + +cleanup_old_logs() { + local retention_start_date="" + local deleted_logs=0 + local log_file="" + local log_name="" + local log_date="" + + retention_start_date="$(date -d "-$((log_keep_days - 1)) days" +%Y%m%d)" + + while IFS= read -r -d '' log_file; do + log_name="$(basename "${log_file}")" + if extract_log_date "${log_name}"; then + log_date="${parsed_log_date}" + if [[ "${log_date}" < "${retention_start_date}" ]]; then + echo "Deleting expired log file: ${log_file}" + rm -f -- "${log_file}" + deleted_logs=$((deleted_logs + 1)) + fi + fi + done < <(find "${log_dir}" -maxdepth 1 -type f \( -name 'xdc-*.log' -o -name 'xdc-*.log.gz' \) -print0) + + echo "Log retention: kept ${log_keep_days} day(s), removed ${deleted_logs} expired log file(s)" +} + +compress_old_logs() { + local active_log_path=$1 + local retention_start_date="" + local uncompressed_start_date="" + local compressed_logs=0 + local failed_logs=0 + local recovered_logs=0 + local log_file="" + local log_name="" + local log_date="" + local final_gz="" + local tmp_gz="" + local invalid_archive_removed=0 + + retention_start_date="$(date -d "-$((log_keep_days - 1)) days" +%Y%m%d)" + uncompressed_start_date="$(date -d "-$((log_uncompressed_days - 1)) days" +%Y%m%d)" + + while IFS= read -r -d '' log_file; do + if [[ "${log_file}" == "${active_log_path}" ]]; then + continue + fi + + log_name="$(basename "${log_file}")" + if ! extract_log_date "${log_name}"; then + continue + fi + + log_date="${parsed_log_date}" + if [[ "${log_date}" < "${retention_start_date}" || "${log_date}" > "${uncompressed_start_date}" || "${log_date}" == "${uncompressed_start_date}" ]]; then + continue + fi + + final_gz="${log_file}.gz" + invalid_archive_removed=0 + if [[ -e "${final_gz}" ]]; then + if gzip -t -- "${final_gz}" >/dev/null 2>&1; then + if rm -f -- "${log_file}"; then + echo "Compressed log already exists for ${log_name}; removed leftover original log file" + recovered_logs=$((recovered_logs + 1)) + else + echo "Unable to remove leftover original log file for ${log_name}" + failed_logs=$((failed_logs + 1)) + fi + else + echo "Existing compressed log invalid for ${log_name}; recreating archive" + if ! rm -f -- "${final_gz}"; then + echo "Unable to remove invalid compressed log for ${log_name}" + failed_logs=$((failed_logs + 1)) + continue + fi + invalid_archive_removed=1 + fi + + if [[ "${invalid_archive_removed}" -eq 0 ]]; then + continue + fi + fi + + tmp_gz="${final_gz}.tmp.${BASHPID:-$$}" + rm -f -- "${tmp_gz}" + echo "Compressing log file: ${log_name}" + if gzip -c -- "${log_file}" >"${tmp_gz}" && mv -f -- "${tmp_gz}" "${final_gz}" && rm -f -- "${log_file}"; then + compressed_logs=$((compressed_logs + 1)) + else + rm -f -- "${tmp_gz}" + echo "Log compression failed for ${log_name}; keeping original log file" + failed_logs=$((failed_logs + 1)) + fi + done < <(find "${log_dir}" -maxdepth 1 -type f -name 'xdc-*.log' -print0) + + echo "Log compression: compressed ${compressed_logs} file(s), recovered ${recovered_logs} file(s), failed ${failed_logs} file(s)" +} + +log_compression_lock_is_ready() { + [[ -f "${log_compress_lock_ready_file}" ]] +} + +lock_directory_is_stale_partial() { + find "${log_compress_lock_dir}" -maxdepth 0 -type d -mmin "+${log_compress_lock_partial_stale_minutes}" | grep -q . +} + +cleanup_stale_log_temp_files() { + local removed_temp_files=0 + local temp_file="" + + while IFS= read -r -d '' temp_file; do + if rm -f -- "${temp_file}"; then + removed_temp_files=$((removed_temp_files + 1)) + fi + done < <(find "${log_dir}" -maxdepth 1 -type f -name 'xdc-*.log.gz.tmp.*' -print0) + + echo "Log compression cleanup: removed ${removed_temp_files} stale temp file(s)" +} + +acquire_log_compression_lock() { + local attempt=0 + local existing_pid="" + local existing_start_time="" + local existing_pid_namespace="" + local current_pid="" + local current_start_time="" + local current_pid_namespace="" + + while [[ "${attempt}" -lt 2 ]]; do + if mkdir "${log_compress_lock_dir}" 2>/dev/null; then + current_pid="${BASHPID:-$$}" + printf '%s\n' "${current_pid}" >"${log_compress_lock_dir}/pid" + current_start_time="$(get_process_start_time "${current_pid}")" && printf '%s\n' "${current_start_time}" >"${log_compress_lock_dir}/start_time" + current_pid_namespace="$(get_process_pid_namespace "${current_pid}")" && printf '%s\n' "${current_pid_namespace}" >"${log_compress_lock_dir}/pid_namespace" + date -u +%Y-%m-%dT%H:%M:%SZ >"${log_compress_lock_dir}/started_at" + : >"${log_compress_lock_ready_file}" + return 0 + fi + + if ! log_compression_lock_is_ready; then + if lock_directory_is_stale_partial; then + echo "Recovering stale partial log compression lock" + rm -rf -- "${log_compress_lock_dir}" + attempt=$((attempt + 1)) + continue + fi + + echo "Log compression lock is still initializing, skipping this cycle" + return 1 + fi + + if [[ -f "${log_compress_lock_dir}/pid" ]]; then + existing_pid="$(cat "${log_compress_lock_dir}/pid" 2>/dev/null)" + fi + if [[ -f "${log_compress_lock_dir}/start_time" ]]; then + existing_start_time="$(cat "${log_compress_lock_dir}/start_time" 2>/dev/null)" + fi + if [[ -f "${log_compress_lock_dir}/pid_namespace" ]]; then + existing_pid_namespace="$(cat "${log_compress_lock_dir}/pid_namespace" 2>/dev/null)" + fi + + if [[ -n "${existing_start_time}" && -n "${existing_pid_namespace}" ]] && lock_process_is_active "${existing_pid}" "${existing_start_time}" "${existing_pid_namespace}"; then + echo "Log compression already running with pid ${existing_pid}, skipping this cycle" + return 1 + fi + + echo "Recovering stale log compression lock" + rm -rf -- "${log_compress_lock_dir}" + attempt=$((attempt + 1)) + done + + echo "Unable to acquire log compression lock, skipping this cycle" + return 1 +} + +release_log_compression_lock() { + rm -rf -- "${log_compress_lock_dir}" +} + +compress_old_logs_with_lock() { + local active_log_path=$1 + + if ! command -v gzip >/dev/null 2>&1; then + echo "gzip not available, skipping log compression" + return 0 + fi + + if ! acquire_log_compression_lock; then + return 0 + fi + + trap 'release_log_compression_lock' EXIT + cleanup_stale_log_temp_files + compress_old_logs "${active_log_path}" +} + +run_log_compression_async() { + local active_log_path=$1 + + ( + if command -v ionice >/dev/null 2>&1; then + ionice -c3 -p "${BASHPID:-$$}" >/dev/null 2>&1 || true + fi + if command -v renice >/dev/null 2>&1; then + renice -n 19 -p "${BASHPID:-$$}" >/dev/null 2>&1 || true + fi + compress_old_logs_with_lock "${active_log_path}" + ) & +} + +prepare_initial_log_file() { + local timestamp="" + local log_fd="" + + mkdir -p "${log_dir}" || return 1 + + timestamp="$(date +%Y%m%d-%H%M%S)" + initial_log_day="${timestamp%%-*}" + initial_log_path="${log_dir}/xdc-${timestamp}.log" + + exec {log_fd}>>"${initial_log_path}" || return 1 + exec {log_fd}>&- + cleanup_old_logs + run_log_compression_async "${initial_log_path}" +} + +write_rotated_log_stream() { + local current_day="${initial_log_day}" + local log_fd="" + local log_path="${initial_log_path}" + local timestamp="" + local chunk="" + local read_status=0 + + if [[ -n "${log_path}" ]]; then + exec {log_fd}>>"${log_path}" || return 1 + fi + + while true; do + read_stream_chunk 0 chunk + read_status=$? + + if [[ "${read_status}" -gt 128 && -z "${chunk}" ]]; then + continue + fi + + if [[ -z "${chunk}" && "${read_status}" -ne 0 ]]; then + break + fi + + timestamp="$(date +%Y%m%d-%H%M%S)" + + if [[ "${timestamp%%-*}" != "${current_day}" ]]; then + if [[ -n "${log_fd}" ]]; then + exec {log_fd}>&- + fi + + current_day="${timestamp%%-*}" + log_path="${log_dir}/xdc-${timestamp}.log" + exec {log_fd}>>"${log_path}" || return 1 + cleanup_old_logs + run_log_compression_async "${log_path}" + fi + + printf '%s' "${chunk}" >&${log_fd} || return 1 + done + + if [[ -n "${log_fd}" ]]; then + exec {log_fd}>&- + fi +} + +log_writer_fd="" +log_writer_pid="" +xdc_read_fd="" +xdc_write_fd="" +xdc_pid="" +shutdown_requested=0 +shutdown_exit_code=0 + +start_log_writer() { + prepare_initial_log_file || return 1 + exec {log_writer_fd}> >(write_rotated_log_stream) + log_writer_pid=$! +} + +stop_log_writer() { + local log_writer_exit_code=0 + + if [[ -n "${log_writer_fd}" ]]; then + exec {log_writer_fd}>&- + log_writer_fd="" + fi + + if [[ -n "${log_writer_pid}" ]]; then + wait "${log_writer_pid}" + log_writer_exit_code=$? + log_writer_pid="" + fi + + return "${log_writer_exit_code}" +} + +start_xdc_stream() { + coproc XDC_STREAM { exec XDC "${args[@]}" 2>&1; } + xdc_pid=$XDC_STREAM_PID + xdc_read_fd=${XDC_STREAM[0]} + xdc_write_fd=${XDC_STREAM[1]} + exec {xdc_write_fd}>&- + xdc_write_fd="" +} + +stop_xdc_stream() { + local xdc_exit_code=0 + + if [[ -n "${xdc_pid}" ]]; then + wait "${xdc_pid}" + xdc_exit_code=$? + xdc_pid="" + fi + + if [[ -n "${xdc_read_fd}" ]]; then + exec {xdc_read_fd}<&- + xdc_read_fd="" + fi + + return "${xdc_exit_code}" +} + +terminate_xdc_stream() { + if [[ -n "${xdc_read_fd}" ]]; then + exec {xdc_read_fd}<&- + xdc_read_fd="" + fi + + if [[ -n "${xdc_pid}" ]]; then + kill "${xdc_pid}" 2>/dev/null || true + wait "${xdc_pid}" 2>/dev/null || true + xdc_pid="" + fi +} + +request_shutdown() { + local signal_name=$1 + + if [[ "${shutdown_requested}" -ne 0 ]]; then + return + fi + + shutdown_requested=1 + shutdown_exit_code=$((128 + $(kill -l "${signal_name}"))) + + if [[ -n "${xdc_pid}" ]]; then + kill -s "${signal_name}" "${xdc_pid}" 2>/dev/null || true + fi +} # Set sync_mode from SYNC_MODE env or default to 'full' sync_mode=full @@ -94,4 +552,62 @@ if echo "${ENABLE_RPC}" | grep -iq "true"; then ) fi -XDC "${args[@]}" 2>&1 >>"${LOG_FILE}" | tee -a "${LOG_FILE}" +set -o pipefail +trap 'request_shutdown TERM' TERM +trap 'request_shutdown INT' INT +start_log_writer || exit $? +start_xdc_stream +stream_exit_code=0 +while true; do + chunk="" + read_stream_chunk "${xdc_read_fd}" chunk + read_status=$? + + if [[ "${read_status}" -gt 128 && -z "${chunk}" ]]; then + continue + fi + + if [[ -z "${chunk}" && "${read_status}" -ne 0 ]]; then + break + fi + + printf '%s' "${chunk}" || { + stream_exit_code=$? + break + } + + printf '%s' "${chunk}" >"/proc/self/fd/${log_writer_fd}" || { + stream_exit_code=$? + break + } +done + +xdc_exit_code=0 +if [[ "${shutdown_requested}" -ne 0 ]]; then + stop_xdc_stream + xdc_exit_code=$? +elif [[ "${stream_exit_code}" -eq 0 ]]; then + stop_xdc_stream + xdc_exit_code=$? +else + terminate_xdc_stream +fi + +stop_log_writer +log_writer_exit_code=$? + +trap - TERM INT + +if [[ "${shutdown_requested}" -ne 0 ]]; then + exit "${shutdown_exit_code}" +fi + +if [[ "${xdc_exit_code}" -ne 0 ]]; then + exit "${xdc_exit_code}" +fi + +if [[ "${stream_exit_code}" -ne 0 ]]; then + exit "${stream_exit_code}" +fi + +exit "${log_writer_exit_code}" diff --git a/testnet/.env b/testnet/.env index d54e736e..23a6d9a7 100644 --- a/testnet/.env +++ b/testnet/.env @@ -4,7 +4,9 @@ SYNC_MODE=full GC_MODE=archive NETWORK=testnet PRIVATE_KEY=3333333333333333333333333333333333333333333333333333333333333333 -LOG_LEVEL=2 +LOG_LEVEL=3 +LOG_KEEP_DAYS=30 +LOG_UNCOMPRESSED_DAYS=3 ENABLE_RPC=false # RPC API setting diff --git a/testnet/docker-compose-hash-rpc.yml b/testnet/docker-compose-hash-rpc.yml index 2e17d1f9..0f55a1ba 100644 --- a/testnet/docker-compose-hash-rpc.yml +++ b/testnet/docker-compose-hash-rpc.yml @@ -4,6 +4,7 @@ services: volumes: - "./xdcchain-1:/work/xdcchain" restart: "always" + stop_grace_period: 60s env_file: "./.env" ports: - "30303:30303" diff --git a/testnet/docker-compose.yml b/testnet/docker-compose.yml index d91a8074..b6f9d050 100644 --- a/testnet/docker-compose.yml +++ b/testnet/docker-compose.yml @@ -4,7 +4,9 @@ services: image: xinfinorg/xdposchain:v2.6.7-testnet volumes: - "./xdcchain-testnet:/work/xdcchain" + - "./entry.sh:/work/entry.sh" - "./start-apothem.sh:/work/start.sh" restart: "always" + stop_grace_period: 60s env_file: .env network_mode: "host" diff --git a/testnet/entry.sh b/testnet/entry.sh new file mode 100755 index 00000000..58d54f78 --- /dev/null +++ b/testnet/entry.sh @@ -0,0 +1,13 @@ +#!/bin/bash +if test -z "$NETWORK" +then + echo "NETWORK env Must be set, mainnet/testnet/devnet/local" + exit 1 +fi + +echo "Select to run $NETWORK..." +ln -s /usr/bin/XDC-"$NETWORK" /usr/bin/XDC +cp -n /work/"$NETWORK"/* /work + +echo "Start Node..." +exec /work/start.sh diff --git a/testnet/start-apothem.sh b/testnet/start-apothem.sh index 188f36e0..0844c0f9 100755 --- a/testnet/start-apothem.sh +++ b/testnet/start-apothem.sh @@ -27,7 +27,7 @@ while IFS= read -r line; do fi done <"$input" -log_level=2 +log_level=3 if test -z "$LOG_LEVEL" then echo "Log level not set, default to verbosity of $log_level" @@ -36,9 +36,467 @@ else log_level=$LOG_LEVEL fi -# create log file with timestamp -DATE="$(date +%Y%m%d-%H%M%S)" -LOG_FILE="/work/xdcchain/xdc-${DATE}.log" +log_dir=/work/xdcchain +log_keep_days=30 +log_uncompressed_days=3 +log_compress_lock_dir="${log_dir}/.log-compress.lock" +log_compress_lock_ready_file="${log_compress_lock_dir}/ready" +log_compress_lock_partial_stale_minutes=1 +stream_chunk_size=4096 +stream_chunk_timeout=0.2 +initial_log_day="" +initial_log_path="" + +if [[ -z "${LOG_KEEP_DAYS}" ]]; then + echo "LOG_KEEP_DAYS not set, default to ${log_keep_days}" +elif [[ "${LOG_KEEP_DAYS}" =~ ^[0-9]+$ && "${LOG_KEEP_DAYS}" -ge 3 ]]; then + echo "LOG_KEEP_DAYS found, set to ${LOG_KEEP_DAYS}" + log_keep_days=${LOG_KEEP_DAYS} +else + echo "Invalid LOG_KEEP_DAYS: ${LOG_KEEP_DAYS}. Expected an integer greater than or equal to 3." >&2 + exit 1 +fi + +if [[ -z "${LOG_UNCOMPRESSED_DAYS}" ]]; then + echo "LOG_UNCOMPRESSED_DAYS not set, default to ${log_uncompressed_days}" +elif [[ "${LOG_UNCOMPRESSED_DAYS}" =~ ^[0-9]+$ && "${LOG_UNCOMPRESSED_DAYS}" -ge 2 ]]; then + echo "LOG_UNCOMPRESSED_DAYS found, set to ${LOG_UNCOMPRESSED_DAYS}" + log_uncompressed_days=${LOG_UNCOMPRESSED_DAYS} +else + echo "Invalid LOG_UNCOMPRESSED_DAYS: ${LOG_UNCOMPRESSED_DAYS}. Expected an integer greater than or equal to 2." >&2 + exit 1 +fi + +if [[ "${log_uncompressed_days}" -gt "${log_keep_days}" ]]; then + echo "Invalid log retention settings: LOG_UNCOMPRESSED_DAYS (${log_uncompressed_days}) cannot be greater than LOG_KEEP_DAYS (${log_keep_days})." >&2 + exit 1 +fi + +read_stream_chunk() { + local stream_fd=$1 + local result_var=$2 + local chunk_data="" + local read_status=0 + + IFS= read -r -N "${stream_chunk_size}" -t "${stream_chunk_timeout}" chunk_data <"/proc/self/fd/${stream_fd}" + read_status=$? + printf -v "${result_var}" '%s' "${chunk_data}" + + return "${read_status}" +} + +extract_log_date() { + local log_name=$1 + + parsed_log_date="" + if [[ "${log_name}" =~ ^xdc-([0-9]{8})-[0-9]{6}\.log(\.gz)?$ ]]; then + parsed_log_date="${BASH_REMATCH[1]}" + return 0 + fi + + return 1 +} + +get_process_start_time() { + local pid=$1 + local start_time="" + + start_time="$(awk '{print $22}' "/proc/${pid}/stat" 2>/dev/null)" + if [[ ! "${start_time}" =~ ^[0-9]+$ ]]; then + return 1 + fi + + printf '%s\n' "${start_time}" +} + +get_process_pid_namespace() { + local pid=$1 + local pid_namespace="" + + pid_namespace="$(readlink "/proc/${pid}/ns/pid" 2>/dev/null)" + if [[ -z "${pid_namespace}" ]]; then + return 1 + fi + + printf '%s\n' "${pid_namespace}" +} + +lock_process_is_active() { + local pid=$1 + local expected_start_time=$2 + local expected_pid_namespace=$3 + local current_start_time="" + local current_pid_namespace="" + + if [[ ! "${pid}" =~ ^[0-9]+$ ]] || ! kill -0 "${pid}" 2>/dev/null; then + return 1 + fi + + current_start_time="$(get_process_start_time "${pid}")" || return 1 + current_pid_namespace="$(get_process_pid_namespace "${pid}")" || return 1 + + if [[ "${current_start_time}" != "${expected_start_time}" || "${current_pid_namespace}" != "${expected_pid_namespace}" ]]; then + return 1 + fi + + return 0 +} + +cleanup_old_logs() { + local retention_start_date="" + local deleted_logs=0 + local log_file="" + local log_name="" + local log_date="" + + retention_start_date="$(date -d "-$((log_keep_days - 1)) days" +%Y%m%d)" + + while IFS= read -r -d '' log_file; do + log_name="$(basename "${log_file}")" + if extract_log_date "${log_name}"; then + log_date="${parsed_log_date}" + if [[ "${log_date}" < "${retention_start_date}" ]]; then + echo "Deleting expired log file: ${log_file}" + rm -f -- "${log_file}" + deleted_logs=$((deleted_logs + 1)) + fi + fi + done < <(find "${log_dir}" -maxdepth 1 -type f \( -name 'xdc-*.log' -o -name 'xdc-*.log.gz' \) -print0) + + echo "Log retention: kept ${log_keep_days} day(s), removed ${deleted_logs} expired log file(s)" +} + +compress_old_logs() { + local active_log_path=$1 + local retention_start_date="" + local uncompressed_start_date="" + local compressed_logs=0 + local failed_logs=0 + local recovered_logs=0 + local log_file="" + local log_name="" + local log_date="" + local final_gz="" + local tmp_gz="" + local invalid_archive_removed=0 + + retention_start_date="$(date -d "-$((log_keep_days - 1)) days" +%Y%m%d)" + uncompressed_start_date="$(date -d "-$((log_uncompressed_days - 1)) days" +%Y%m%d)" + + while IFS= read -r -d '' log_file; do + if [[ "${log_file}" == "${active_log_path}" ]]; then + continue + fi + + log_name="$(basename "${log_file}")" + if ! extract_log_date "${log_name}"; then + continue + fi + + log_date="${parsed_log_date}" + if [[ "${log_date}" < "${retention_start_date}" || "${log_date}" > "${uncompressed_start_date}" || "${log_date}" == "${uncompressed_start_date}" ]]; then + continue + fi + + final_gz="${log_file}.gz" + invalid_archive_removed=0 + if [[ -e "${final_gz}" ]]; then + if gzip -t -- "${final_gz}" >/dev/null 2>&1; then + if rm -f -- "${log_file}"; then + echo "Compressed log already exists for ${log_name}; removed leftover original log file" + recovered_logs=$((recovered_logs + 1)) + else + echo "Unable to remove leftover original log file for ${log_name}" + failed_logs=$((failed_logs + 1)) + fi + else + echo "Existing compressed log invalid for ${log_name}; recreating archive" + if ! rm -f -- "${final_gz}"; then + echo "Unable to remove invalid compressed log for ${log_name}" + failed_logs=$((failed_logs + 1)) + continue + fi + invalid_archive_removed=1 + fi + + if [[ "${invalid_archive_removed}" -eq 0 ]]; then + continue + fi + fi + + tmp_gz="${final_gz}.tmp.${BASHPID:-$$}" + rm -f -- "${tmp_gz}" + echo "Compressing log file: ${log_name}" + if gzip -c -- "${log_file}" >"${tmp_gz}" && mv -f -- "${tmp_gz}" "${final_gz}" && rm -f -- "${log_file}"; then + compressed_logs=$((compressed_logs + 1)) + else + rm -f -- "${tmp_gz}" + echo "Log compression failed for ${log_name}; keeping original log file" + failed_logs=$((failed_logs + 1)) + fi + done < <(find "${log_dir}" -maxdepth 1 -type f -name 'xdc-*.log' -print0) + + echo "Log compression: compressed ${compressed_logs} file(s), recovered ${recovered_logs} file(s), failed ${failed_logs} file(s)" +} + +log_compression_lock_is_ready() { + [[ -f "${log_compress_lock_ready_file}" ]] +} + +lock_directory_is_stale_partial() { + find "${log_compress_lock_dir}" -maxdepth 0 -type d -mmin "+${log_compress_lock_partial_stale_minutes}" | grep -q . +} + +cleanup_stale_log_temp_files() { + local removed_temp_files=0 + local temp_file="" + + while IFS= read -r -d '' temp_file; do + if rm -f -- "${temp_file}"; then + removed_temp_files=$((removed_temp_files + 1)) + fi + done < <(find "${log_dir}" -maxdepth 1 -type f -name 'xdc-*.log.gz.tmp.*' -print0) + + echo "Log compression cleanup: removed ${removed_temp_files} stale temp file(s)" +} + +acquire_log_compression_lock() { + local attempt=0 + local existing_pid="" + local existing_start_time="" + local existing_pid_namespace="" + local current_pid="" + local current_start_time="" + local current_pid_namespace="" + + while [[ "${attempt}" -lt 2 ]]; do + if mkdir "${log_compress_lock_dir}" 2>/dev/null; then + current_pid="${BASHPID:-$$}" + printf '%s\n' "${current_pid}" >"${log_compress_lock_dir}/pid" + current_start_time="$(get_process_start_time "${current_pid}")" && printf '%s\n' "${current_start_time}" >"${log_compress_lock_dir}/start_time" + current_pid_namespace="$(get_process_pid_namespace "${current_pid}")" && printf '%s\n' "${current_pid_namespace}" >"${log_compress_lock_dir}/pid_namespace" + date -u +%Y-%m-%dT%H:%M:%SZ >"${log_compress_lock_dir}/started_at" + : >"${log_compress_lock_ready_file}" + return 0 + fi + + if ! log_compression_lock_is_ready; then + if lock_directory_is_stale_partial; then + echo "Recovering stale partial log compression lock" + rm -rf -- "${log_compress_lock_dir}" + attempt=$((attempt + 1)) + continue + fi + + echo "Log compression lock is still initializing, skipping this cycle" + return 1 + fi + + if [[ -f "${log_compress_lock_dir}/pid" ]]; then + existing_pid="$(cat "${log_compress_lock_dir}/pid" 2>/dev/null)" + fi + if [[ -f "${log_compress_lock_dir}/start_time" ]]; then + existing_start_time="$(cat "${log_compress_lock_dir}/start_time" 2>/dev/null)" + fi + if [[ -f "${log_compress_lock_dir}/pid_namespace" ]]; then + existing_pid_namespace="$(cat "${log_compress_lock_dir}/pid_namespace" 2>/dev/null)" + fi + + if [[ -n "${existing_start_time}" && -n "${existing_pid_namespace}" ]] && lock_process_is_active "${existing_pid}" "${existing_start_time}" "${existing_pid_namespace}"; then + echo "Log compression already running with pid ${existing_pid}, skipping this cycle" + return 1 + fi + + echo "Recovering stale log compression lock" + rm -rf -- "${log_compress_lock_dir}" + attempt=$((attempt + 1)) + done + + echo "Unable to acquire log compression lock, skipping this cycle" + return 1 +} + +release_log_compression_lock() { + rm -rf -- "${log_compress_lock_dir}" +} + +compress_old_logs_with_lock() { + local active_log_path=$1 + + if ! command -v gzip >/dev/null 2>&1; then + echo "gzip not available, skipping log compression" + return 0 + fi + + if ! acquire_log_compression_lock; then + return 0 + fi + + trap 'release_log_compression_lock' EXIT + cleanup_stale_log_temp_files + compress_old_logs "${active_log_path}" +} + +run_log_compression_async() { + local active_log_path=$1 + + ( + if command -v ionice >/dev/null 2>&1; then + ionice -c3 -p "${BASHPID:-$$}" >/dev/null 2>&1 || true + fi + if command -v renice >/dev/null 2>&1; then + renice -n 19 -p "${BASHPID:-$$}" >/dev/null 2>&1 || true + fi + compress_old_logs_with_lock "${active_log_path}" + ) & +} + +prepare_initial_log_file() { + local timestamp="" + local log_fd="" + + mkdir -p "${log_dir}" || return 1 + + timestamp="$(date +%Y%m%d-%H%M%S)" + initial_log_day="${timestamp%%-*}" + initial_log_path="${log_dir}/xdc-${timestamp}.log" + + exec {log_fd}>>"${initial_log_path}" || return 1 + exec {log_fd}>&- + cleanup_old_logs + run_log_compression_async "${initial_log_path}" +} + +write_rotated_log_stream() { + local current_day="${initial_log_day}" + local log_fd="" + local log_path="${initial_log_path}" + local timestamp="" + local chunk="" + local read_status=0 + + if [[ -n "${log_path}" ]]; then + exec {log_fd}>>"${log_path}" || return 1 + fi + + while true; do + read_stream_chunk 0 chunk + read_status=$? + + if [[ "${read_status}" -gt 128 && -z "${chunk}" ]]; then + continue + fi + + if [[ -z "${chunk}" && "${read_status}" -ne 0 ]]; then + break + fi + + timestamp="$(date +%Y%m%d-%H%M%S)" + + if [[ "${timestamp%%-*}" != "${current_day}" ]]; then + if [[ -n "${log_fd}" ]]; then + exec {log_fd}>&- + fi + + current_day="${timestamp%%-*}" + log_path="${log_dir}/xdc-${timestamp}.log" + exec {log_fd}>>"${log_path}" || return 1 + cleanup_old_logs + run_log_compression_async "${log_path}" + fi + + printf '%s' "${chunk}" >&${log_fd} || return 1 + done + + if [[ -n "${log_fd}" ]]; then + exec {log_fd}>&- + fi +} + +log_writer_fd="" +log_writer_pid="" +xdc_read_fd="" +xdc_write_fd="" +xdc_pid="" +shutdown_requested=0 +shutdown_exit_code=0 + +start_log_writer() { + prepare_initial_log_file || return 1 + exec {log_writer_fd}> >(write_rotated_log_stream) + log_writer_pid=$! +} + +stop_log_writer() { + local log_writer_exit_code=0 + + if [[ -n "${log_writer_fd}" ]]; then + exec {log_writer_fd}>&- + log_writer_fd="" + fi + + if [[ -n "${log_writer_pid}" ]]; then + wait "${log_writer_pid}" + log_writer_exit_code=$? + log_writer_pid="" + fi + + return "${log_writer_exit_code}" +} + +start_xdc_stream() { + coproc XDC_STREAM { exec XDC "${args[@]}" 2>&1; } + xdc_pid=$XDC_STREAM_PID + xdc_read_fd=${XDC_STREAM[0]} + xdc_write_fd=${XDC_STREAM[1]} + exec {xdc_write_fd}>&- + xdc_write_fd="" +} + +stop_xdc_stream() { + local xdc_exit_code=0 + + if [[ -n "${xdc_pid}" ]]; then + wait "${xdc_pid}" + xdc_exit_code=$? + xdc_pid="" + fi + + if [[ -n "${xdc_read_fd}" ]]; then + exec {xdc_read_fd}<&- + xdc_read_fd="" + fi + + return "${xdc_exit_code}" +} + +terminate_xdc_stream() { + if [[ -n "${xdc_read_fd}" ]]; then + exec {xdc_read_fd}<&- + xdc_read_fd="" + fi + + if [[ -n "${xdc_pid}" ]]; then + kill "${xdc_pid}" 2>/dev/null || true + wait "${xdc_pid}" 2>/dev/null || true + xdc_pid="" + fi +} + +request_shutdown() { + local signal_name=$1 + + if [[ "${shutdown_requested}" -ne 0 ]]; then + return + fi + + shutdown_requested=1 + shutdown_exit_code=$((128 + $(kill -l "${signal_name}"))) + + if [[ -n "${xdc_pid}" ]]; then + kill -s "${signal_name}" "${xdc_pid}" 2>/dev/null || true + fi +} # Set sync_mode from SYNC_MODE env or default to 'full' sync_mode=full @@ -96,4 +554,62 @@ if echo "${ENABLE_RPC}" | grep -iq "true"; then ) fi -XDC "${args[@]}" 2>&1 >>"${LOG_FILE}" | tee -a "${LOG_FILE}" +set -o pipefail +trap 'request_shutdown TERM' TERM +trap 'request_shutdown INT' INT +start_log_writer || exit $? +start_xdc_stream +stream_exit_code=0 +while true; do + chunk="" + read_stream_chunk "${xdc_read_fd}" chunk + read_status=$? + + if [[ "${read_status}" -gt 128 && -z "${chunk}" ]]; then + continue + fi + + if [[ -z "${chunk}" && "${read_status}" -ne 0 ]]; then + break + fi + + printf '%s' "${chunk}" || { + stream_exit_code=$? + break + } + + printf '%s' "${chunk}" >"/proc/self/fd/${log_writer_fd}" || { + stream_exit_code=$? + break + } +done + +xdc_exit_code=0 +if [[ "${shutdown_requested}" -ne 0 ]]; then + stop_xdc_stream + xdc_exit_code=$? +elif [[ "${stream_exit_code}" -eq 0 ]]; then + stop_xdc_stream + xdc_exit_code=$? +else + terminate_xdc_stream +fi + +stop_log_writer +log_writer_exit_code=$? + +trap - TERM INT + +if [[ "${shutdown_requested}" -ne 0 ]]; then + exit "${shutdown_exit_code}" +fi + +if [[ "${xdc_exit_code}" -ne 0 ]]; then + exit "${xdc_exit_code}" +fi + +if [[ "${stream_exit_code}" -ne 0 ]]; then + exit "${stream_exit_code}" +fi + +exit "${log_writer_exit_code}"