DAOS-623 test: instrument NLT fault-injection sweep hang

Jenkinsfile

@@ -993,12 +993,17 @@ pipeline {
                     agent {
                         label params.CI_NLT_1_LABEL
                     }
+                    environment {
+                        // Temporary, for this branch only: shorten stall detection while
+                        // hunting an intermittent hang.  The deliberate wedge that proved
+                        // the watchdog is gone, so a failure here now means a real one.
+                        NLT_FI_STALL_SECS = '120'
+                    }
                     steps {
                         // NLT memchecks the valgrind-tagged build, not the shared -race one.
                         unstash 'opt-daos-valgrind'
                         job_step_update(
-                            unitTest(timeout_time: 60 * cachedCommitPragma(pragma: 'NLT-repeat',
-                                                                           def_val: '1').toInteger(),
+                            unitTest(timeout_time: 360,
                                      inst_repos: daosRepos(),
                                      test_script: 'ci/unit/test_nlt.sh' +
                                                   ' --system-ram-reserved 4' +

ci/unit/test_nlt.sh

@@ -22,4 +22,6 @@ rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" .build_vars* opt-daos-install.tar utils re
 ssh -T "$SSH_KEY_ARGS" jenkins@"$NODE" \
     "DAOS_HTTPS_PROXY=\"${DAOS_HTTPS_PROXY:-}\" \
      DAOS_NO_PROXY=\"${DAOS_NO_PROXY:-}\" \
+     NLT_FI_STALL_SECS=\"${NLT_FI_STALL_SECS:-}\" \
+     NLT_FI_WEDGE_AT=\"${NLT_FI_WEDGE_AT:-}\" \
      bash -s -- $*" < "$mydir/test_nlt_node.sh"

ci/unit/test_nlt_node.sh

@@ -51,7 +51,11 @@ mkdir -p nlt_logs
 sudo mount -t tmpfs tmpfs nlt_logs
 sudo chown jenkins:jenkins nlt_logs
 
+# Unbuffered so the console shows exactly where a hang occurs.
 TMPDIR="$(pwd)/nlt_logs" \
+    PYTHONUNBUFFERED=1 \
+    NLT_FI_STALL_SECS="${NLT_FI_STALL_SECS:-}" \
+    NLT_FI_WEDGE_AT="${NLT_FI_WEDGE_AT:-}" \
     HTTPS_PROXY="${DAOS_HTTPS_PROXY:-}" \
     NO_PROXY="${DAOS_NO_PROXY:-}" \
     exec ./utils/node_local_test.py "$@"