From 4d44143d5ede549b4f76594ded2aead8540b6c53 Mon Sep 17 00:00:00 2001 From: Brandon Holt Date: Wed, 22 Jan 2014 22:56:24 -0800 Subject: [PATCH 01/11] use RUNPATH again, and prepend LD_LIBRARY_PATH correctly, but... doesn't solve the problem of being able to specify libs, like we wanted to do on Pal, because whatever's in LD_LIBRARY_PATH will still be prepended by srun_prolog --- CMakeLists.txt | 6 ++--- bin/CMakeLists.txt | 31 +++++++++++++++++++++-- bin/{srun_prolog.rb => srun_prolog.rb.in} | 3 +++ 3 files changed, 35 insertions(+), 5 deletions(-) rename bin/{srun_prolog.rb => srun_prolog.rb.in} (93%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f692abca..b7cbdc36c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,9 +69,9 @@ endif("${isSystemDir}" STREQUAL "-1") ## Not setting runpath because having MPI libs in LD_LIBRARY_PATH was messing up VampirTrace's ability to find its own libs. Maybe there's another way to fix this, but it just seems more robust (for now) to not allow LD_LIBRARY_PATH to affect our libs (after configure uses it to find them). # set runpath, too -# if(NOT APPLE) -# set(CMAKE_EXE_LINKER_FLAGS "-Wl,--enable-new-dtags") -# endif() +if(NOT APPLE) + set(CMAKE_EXE_LINKER_FLAGS "-Wl,--enable-new-dtags") +endif() set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${THIRD_PARTY_ROOT}/lib") diff --git a/bin/CMakeLists.txt b/bin/CMakeLists.txt index 7bc795332..6eb6467a7 100644 --- a/bin/CMakeLists.txt +++ b/bin/CMakeLists.txt @@ -1,9 +1,36 @@ # file(COPY . DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) file(GLOB scripts "*") -foreach(file ${scripts}) + +set(symlinked_scripts + distcc_make + distcc_ninja + grappa_srun + launch_distcc.sh + srun_epilog.sh +) + +set(configured_scripts + srun_prolog.rb +) + +foreach(file ${symlinked_scripts}) get_filename_component(name ${file} NAME) - file(RELATIVE_PATH relative_file ${CMAKE_CURRENT_BINARY_DIR} ${file}) + file(RELATIVE_PATH relative_file ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/${file}) execute_process(COMMAND ln -sf ${relative_file} ${CMAKE_CURRENT_BINARY_DIR}/${name} ) endforeach() + +set(GRAPPA_LD_DIRS "") +if(VAMPIR_ROOT) + set(GRAPPA_LD_DIRS "${GRAPPA_LD_DIRS}:${VAMPIR_ROOT}/lib") +endif() +if(THIRD_PARTY_ROOT) + set(GRAPPA_LD_DIRS "${GRAPPA_LD_DIRS}:${THIRD_PARTY_ROOT}/lib") +else() + set(GRAPPA_LD_DIRS "${GRAPPA_LD_DIRS}:${EXTERN_BUILD_DIR}/lib") +endif() + +foreach(f ${configured_scripts}) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${f}.in ${CMAKE_CURRENT_BINARY_DIR}/${f} @ONLY) +endforeach() diff --git a/bin/srun_prolog.rb b/bin/srun_prolog.rb.in similarity index 93% rename from bin/srun_prolog.rb rename to bin/srun_prolog.rb.in index 9c52ee721..4fbbe4d05 100755 --- a/bin/srun_prolog.rb +++ b/bin/srun_prolog.rb.in @@ -4,6 +4,9 @@ interactive = true end +# prepend VampirTrace & third-party dirs to LD_LIBRARY_PATH +ENV["LD_LIBRARY_PATH"].prepend "@GRAPPA_LD_DIRS@" + ## set up Google logging defaults ENV["GLOG_logtostderr"] = "1" ENV["GLOG_v"] = "0" From 0469afcd55da99a0dcb6ef3c20b9fdd3dcdc0722 Mon Sep 17 00:00:00 2001 From: Brandon Holt Date: Wed, 22 Jan 2014 23:08:48 -0800 Subject: [PATCH 02/11] remove old glob in bin/CMakeLists.txt --- bin/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/CMakeLists.txt b/bin/CMakeLists.txt index 6eb6467a7..c5a53a256 100644 --- a/bin/CMakeLists.txt +++ b/bin/CMakeLists.txt @@ -1,5 +1,3 @@ -# file(COPY . DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) -file(GLOB scripts "*") set(symlinked_scripts distcc_make From 69caac25652b114b06acf613742e2a39b1b0f7f9 Mon Sep 17 00:00:00 2001 From: Brandon Holt Date: Mon, 24 Feb 2014 15:14:32 -0800 Subject: [PATCH 03/11] fix symlinking --- bin/CMakeLists.txt | 3 +-- bin/grappa_srun | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/bin/CMakeLists.txt b/bin/CMakeLists.txt index c5a53a256..e469340e7 100644 --- a/bin/CMakeLists.txt +++ b/bin/CMakeLists.txt @@ -13,9 +13,8 @@ set(configured_scripts foreach(file ${symlinked_scripts}) get_filename_component(name ${file} NAME) - file(RELATIVE_PATH relative_file ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/${file}) execute_process(COMMAND - ln -sf ${relative_file} ${CMAKE_CURRENT_BINARY_DIR}/${name} + ln -sf ${CMAKE_CURRENT_SOURCE_DIR}/${file} ${CMAKE_CURRENT_BINARY_DIR}/${name} ) endforeach() diff --git a/bin/grappa_srun b/bin/grappa_srun index 778238014..71f8d0d3f 100755 --- a/bin/grappa_srun +++ b/bin/grappa_srun @@ -37,7 +37,7 @@ srun_flags = %w[ --cpu_bind=rank --label --kill-on-bad-exit ] \ << "--task-epilog=#{DIR}/srun_epilog.sh" # "source" prolog -require_relative "srun_prolog.rb" +require "#{DIR}/srun_prolog.rb" setarch = "" From ab5b1050cbbc5d02e0ba97b50c71854a533d0d15 Mon Sep 17 00:00:00 2001 From: Brandon Holt Date: Mon, 24 Feb 2014 15:15:28 -0800 Subject: [PATCH 04/11] prepend LD_LIBRARY_PATH unless Igor says it's overriding it --- bin/srun_prolog.rb.in | 6 ++++-- util/igor_common.rb | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/bin/srun_prolog.rb.in b/bin/srun_prolog.rb.in index 4fbbe4d05..23ffb4282 100755 --- a/bin/srun_prolog.rb.in +++ b/bin/srun_prolog.rb.in @@ -4,8 +4,10 @@ if ARGV.size > 1 && ARGV[0] == "--" interactive = true end -# prepend VampirTrace & third-party dirs to LD_LIBRARY_PATH -ENV["LD_LIBRARY_PATH"].prepend "@GRAPPA_LD_DIRS@" +unless ENV["IGOR_OVERRIDE_LIBRARY_PATH"] + # prepend VampirTrace & third-party dirs to LD_LIBRARY_PATH + ENV["LD_LIBRARY_PATH"] = "@GRAPPA_LD_DIRS@:#{ENV["LD_LIBRARY_PATH"]}" +end ## set up Google logging defaults ENV["GLOG_logtostderr"] = "1" diff --git a/util/igor_common.rb b/util/igor_common.rb index d9b9e45ab..7cd2bd6ad 100644 --- a/util/igor_common.rb +++ b/util/igor_common.rb @@ -105,7 +105,7 @@ def command(c=nil) echo $l; sbcast #{@ldir}/$l #{tdir}/${l}; done; fi; - #{c} + LD_LIBRARY_PATH=#{tdir} IGOR_OVERRIDE_LIBRARY_PATH=1 #{c} ].tr("\n "," ") end return super(c) From 934233deb7864ca4b063c7217987591cb80ddcdb Mon Sep 17 00:00:00 2001 From: Brandon Holt Date: Mon, 24 Feb 2014 16:32:56 -0800 Subject: [PATCH 05/11] use new Igor command: 'require_relative_to_symlink' in igor_new_bfs.rb --- applications/graph500/grappa/igor_new_bfs.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/graph500/grappa/igor_new_bfs.rb b/applications/graph500/grappa/igor_new_bfs.rb index e09b091cc..f747fbd1c 100755 --- a/applications/graph500/grappa/igor_new_bfs.rb +++ b/applications/graph500/grappa/igor_new_bfs.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../../../util/igor_common.rb' +require_relative_to_symlink '../../../util/igor_common.rb' Igor do include Isolatable From afee9e0aeaa2296f9e4843ce3c9c47e3d48ad4c9 Mon Sep 17 00:00:00 2001 From: Brandon Holt Date: Mon, 24 Feb 2014 17:25:19 -0800 Subject: [PATCH 06/11] append to LD_LIBRARY_PATH so we can find libstdc++ --- util/igor_common.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/igor_common.rb b/util/igor_common.rb index 1d1699178..60561a3e1 100644 --- a/util/igor_common.rb +++ b/util/igor_common.rb @@ -69,7 +69,7 @@ def isolate(exes, shared_dir=nil) end end FileUtils.cp(exe, @ldir) - libs = `bash -c "LD_LIBRARY_PATH=#{$GRAPPA_LIBPATH} ldd #{exe}"` + libs = `bash -c "LD_LIBRARY_PATH=#{$GRAPPA_LIBPATH}:$LD_LIBRARY_PATH ldd #{exe}"` .split(/\n/) .map{|s| s[/.*> (.*) \(.*/,1] } .select{|s| s != nil and s != "" and From d9f37942ebb0d0a9fc25a7a2dc35807ab1427d8a Mon Sep 17 00:00:00 2001 From: Brandon Holt Date: Mon, 24 Feb 2014 17:25:47 -0800 Subject: [PATCH 07/11] add path to MPI libs to RUNPATH --- CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index a994eb8fc..335e9e182 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -203,6 +203,11 @@ endif() # MPI (mostly for booting, but we also use MPI communication in some places now) find_package(MPI REQUIRED) include_directories(${MPI_INCLUDE_PATH}) +# Add MPI libs dirs to RPATH +foreach(lib ${MPI_CXX_LIBRARIES}) + get_filename_component(libdir "${lib}" PATH) + set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${libdir}") +endforeach() macro(add_grappa_exe target exe ) add_executable(${target} EXCLUDE_FROM_ALL ${ARGN}) From edae49f62086f9d6429716d7f5826c56cd4e1b1f Mon Sep 17 00:00:00 2001 From: Brandon Holt Date: Mon, 24 Feb 2014 18:02:00 -0800 Subject: [PATCH 08/11] fix signal handling on segfault (use google stacktrace) --- system/Grappa.cpp | 24 +++++++++++++++++------- system/LocaleSharedMemory.cpp | 2 +- system/RDMAAggregator.cpp | 2 +- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/system/Grappa.cpp b/system/Grappa.cpp index 88915f56f..6e2f53fe1 100644 --- a/system/Grappa.cpp +++ b/system/Grappa.cpp @@ -185,13 +185,18 @@ static void stats_dump_sighandler( int signum ) { // function to call when google logging library detect a failure namespace Grappa { namespace impl { + + bool freeze_on_error = false; + /// called on failures to backtrace and pause for debugger - void failure_function() { + void failure_function(int ignore = 0) { + fprintf(stderr, "!! failure function\n"); google::FlushLogFiles(google::GLOG_INFO); - google::DumpStackTrace(); - gasnett_freezeForDebuggerErr(); + // google::DumpStackTrace(); + if (freeze_on_error) gasnett_freezeForDebuggerErr(); gasnet_exit(1); } + } } @@ -207,7 +212,7 @@ void Grappa_init( int * argc_p, char ** argv_p[], int64_t global_memory_size_byt // } // make sure gasnet is ready to backtrace - gasnett_backtrace_init( (*argv_p)[0] ); + // gasnett_backtrace_init( (*argv_p)[0] ); // help generate unique profile filename Grappa::impl::set_exe_name( (*argv_p)[0] ); @@ -217,8 +222,11 @@ void Grappa_init( int * argc_p, char ** argv_p[], int64_t global_memory_size_byt // activate logging google::InitGoogleLogging( *argv_p[0] ); - google::InstallFailureFunction( &Grappa::impl::failure_function ); - google::OverrideDefaultSignalHandler( &gasnet_pause_sighandler ); + + char * env_freeze = getenv("GASNET_FREEZE_ON_ERROR"); + if (env_freeze && strncmp(env_freeze,"1",1) == 0) freeze_on_error = true; + + google::OverrideDefaultSignalHandler( &Grappa::impl::failure_function ); DVLOG(1) << "Initializing Grappa library...."; #ifdef HEAPCHECK_ENABLE @@ -248,7 +256,7 @@ void Grappa_init( int * argc_p, char ** argv_p[], int64_t global_memory_size_byt sigemptyset( &sigabrt_sa.sa_mask ); sigabrt_sa.sa_flags = 0; sigabrt_sa.sa_handler = &gasnet_pause_sighandler; - CHECK_EQ( 0, sigaction( SIGABRT, &sigabrt_sa, 0 ) ) << "SIGABRT signal handler installation failed."; + // CHECK_EQ( 0, sigaction( SIGABRT, &sigabrt_sa, 0 ) ) << "SIGABRT signal handler installation failed."; // Asynchronous IO // initialize completed stack @@ -385,6 +393,8 @@ void Grappa_activate() { DVLOG(1) << "Activating Grappa library...."; global_communicator.activate(); + google::InstallFailureSignalHandler(); + locale_shared_memory.activate(); global_task_manager.activate(); Grappa::comm_barrier(); diff --git a/system/LocaleSharedMemory.cpp b/system/LocaleSharedMemory.cpp index ff0cee30c..c9c2f2a40 100644 --- a/system/LocaleSharedMemory.cpp +++ b/system/LocaleSharedMemory.cpp @@ -38,7 +38,7 @@ namespace Grappa { namespace impl { /// called on failures to backtrace and pause for debugger -extern void failure_function(); +extern void failure_function(int ignore = 0); /// how much memory do we expect to allocate? extern int64_t global_memory_size_bytes; diff --git a/system/RDMAAggregator.cpp b/system/RDMAAggregator.cpp index d27504272..24b50336c 100644 --- a/system/RDMAAggregator.cpp +++ b/system/RDMAAggregator.cpp @@ -33,7 +33,7 @@ namespace Grappa { namespace impl { // defined in Grappa.cpp - extern void failure_function(); + extern void failure_function(int ignore = 0); } } From ff5c18e69dd6291b54a18e551e314c41ae5548b9 Mon Sep 17 00:00:00 2001 From: Brandon Holt Date: Mon, 24 Feb 2014 21:18:56 -0800 Subject: [PATCH 09/11] update igor scripts to use 'require_relative_to_symlink' --- applications/NPB/GRAPPA/IS/igor_grappa_intsort.rb | 2 +- applications/NPB/MPI/igor_mpi_intsort.rb | 2 +- applications/graph500/grappa/igor_ckpt_gen.rb | 2 +- applications/graph500/grappa/igor_grappa_bfs.rb | 2 +- applications/graph500/grappa/igor_new_cc.rb | 2 +- applications/graph500/mpi/igor_mpi_bfs.rb | 2 +- applications/pagerank/igor_grappa_pagerank.rb | 2 +- applications/uts/igor_grappa_uts.rb | 2 +- system/tests/igor_context_switch.rb | 2 +- system/tests/igor_datastructs.rb | 2 +- system/tests/igor_tests.rb | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/applications/NPB/GRAPPA/IS/igor_grappa_intsort.rb b/applications/NPB/GRAPPA/IS/igor_grappa_intsort.rb index 63f69065d..d78f7a85b 100755 --- a/applications/NPB/GRAPPA/IS/igor_grappa_intsort.rb +++ b/applications/NPB/GRAPPA/IS/igor_grappa_intsort.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../../../../util/igor_common.rb' +require_relative_to_symlink '../../../../util/igor_common.rb' Igor do include Isolatable diff --git a/applications/NPB/MPI/igor_mpi_intsort.rb b/applications/NPB/MPI/igor_mpi_intsort.rb index f1bb30e36..04aad8764 100755 --- a/applications/NPB/MPI/igor_mpi_intsort.rb +++ b/applications/NPB/MPI/igor_mpi_intsort.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../../../util/igor_common.rb' +require_relative_to_symlink '../../../util/igor_common.rb' Igor do include Isolatable diff --git a/applications/graph500/grappa/igor_ckpt_gen.rb b/applications/graph500/grappa/igor_ckpt_gen.rb index ea37ff51a..872d06a80 100755 --- a/applications/graph500/grappa/igor_ckpt_gen.rb +++ b/applications/graph500/grappa/igor_ckpt_gen.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../../../util/igor_common.rb' +require_relative_to_symlink '../../../util/igor_common.rb' def expand_flags(*names) names.map{|n| "--#{n}=%{#{n}}"}.join(' ') diff --git a/applications/graph500/grappa/igor_grappa_bfs.rb b/applications/graph500/grappa/igor_grappa_bfs.rb index f4caa2e42..a44605bb2 100755 --- a/applications/graph500/grappa/igor_grappa_bfs.rb +++ b/applications/graph500/grappa/igor_grappa_bfs.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../../../util/igor_common.rb' +require_relative_to_symlink '../../../util/igor_common.rb' Igor do include Isolatable diff --git a/applications/graph500/grappa/igor_new_cc.rb b/applications/graph500/grappa/igor_new_cc.rb index 6edc8079e..55d540e09 100755 --- a/applications/graph500/grappa/igor_new_cc.rb +++ b/applications/graph500/grappa/igor_new_cc.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../../../util/igor_common.rb' +require_relative_to_symlink '../../../util/igor_common.rb' Igor do include Isolatable diff --git a/applications/graph500/mpi/igor_mpi_bfs.rb b/applications/graph500/mpi/igor_mpi_bfs.rb index 1a842c724..eaf89787a 100755 --- a/applications/graph500/mpi/igor_mpi_bfs.rb +++ b/applications/graph500/mpi/igor_mpi_bfs.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../../../util/igor_common.rb' +require_relative_to_symlink '../../../util/igor_common.rb' Igor do include Isolatable diff --git a/applications/pagerank/igor_grappa_pagerank.rb b/applications/pagerank/igor_grappa_pagerank.rb index 6e09f97d6..8e479efb8 100755 --- a/applications/pagerank/igor_grappa_pagerank.rb +++ b/applications/pagerank/igor_grappa_pagerank.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../../util/igor_common.rb' +require_relative_to_symlink '../../util/igor_common.rb' def expand_flags(*names) names.map{|n| "--#{n}=%{#{n}}"}.join(' ') diff --git a/applications/uts/igor_grappa_uts.rb b/applications/uts/igor_grappa_uts.rb index eb600da65..92c3fb270 100755 --- a/applications/uts/igor_grappa_uts.rb +++ b/applications/uts/igor_grappa_uts.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../../util/igor_common.rb' +require_relative_to_symlink '../../util/igor_common.rb' def expand_flags(*names) names.map{|n| "--#{n}=%{#{n}}"}.join(' ') diff --git a/system/tests/igor_context_switch.rb b/system/tests/igor_context_switch.rb index 69cd44575..b907cc8b5 100755 --- a/system/tests/igor_context_switch.rb +++ b/system/tests/igor_context_switch.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../util/igor_common.rb' +require_relative_to_symlink '../util/igor_common.rb' Igor do include Isolatable diff --git a/system/tests/igor_datastructs.rb b/system/tests/igor_datastructs.rb index f6d7399ee..88612ae36 100755 --- a/system/tests/igor_datastructs.rb +++ b/system/tests/igor_datastructs.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../util/igor_common.rb' +require_relative_to_symlink '../util/igor_common.rb' Igor do database '~/exp/pgas.sqlite', :queue diff --git a/system/tests/igor_tests.rb b/system/tests/igor_tests.rb index b189aed15..879979586 100755 --- a/system/tests/igor_tests.rb +++ b/system/tests/igor_tests.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../util/igor_common.rb' +require_relative_to_symlink '../util/igor_common.rb' Igor do database '~/exp/test.sqlite', :vector From 0222a348c08716fc3969b7c2d6be6d980f206e08 Mon Sep 17 00:00:00 2001 From: Brandon Holt Date: Mon, 24 Feb 2014 23:01:30 -0800 Subject: [PATCH 10/11] igor_common: fix sbcast --- util/igor_common.rb | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/util/igor_common.rb b/util/igor_common.rb index 60561a3e1..8570b4758 100644 --- a/util/igor_common.rb +++ b/util/igor_common.rb @@ -97,14 +97,15 @@ def command(c=nil) @params[:tdir] = tdir ignore :tdir, :command c = %Q[ - if [[ ! -d "#{tdir}" ]]; then - srun mkdir -p #{tdir}; - ls #{tdir}; - echo $(hostname); - for l in $(ls #{@ldir}); do - echo $l; sbcast #{@ldir}/$l #{tdir}/${l}; - done; - fi; + srun bash -c ' + if [[ ! -d "#{tdir}" ]]; then + mkdir -p #{tdir}; + ls #{tdir}; + echo $(hostname); + fi'; + for l in $(ls #{@ldir}); do + echo $l; sbcast -f #{@ldir}/$l #{tdir}/${l}; + done; LD_LIBRARY_PATH=#{tdir} IGOR_OVERRIDE_LIBRARY_PATH=1 #{c} ].tr("\n "," ") end From c8c5899ccbcf1a1a57301650bc619bb81aa39bdd Mon Sep 17 00:00:00 2001 From: Brandon Holt Date: Mon, 24 Feb 2014 23:03:48 -0800 Subject: [PATCH 11/11] get rid of debug print --- system/Grappa.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/system/Grappa.cpp b/system/Grappa.cpp index 6e2f53fe1..a35a616a5 100644 --- a/system/Grappa.cpp +++ b/system/Grappa.cpp @@ -190,7 +190,6 @@ namespace Grappa { /// called on failures to backtrace and pause for debugger void failure_function(int ignore = 0) { - fprintf(stderr, "!! failure function\n"); google::FlushLogFiles(google::GLOG_INFO); // google::DumpStackTrace(); if (freeze_on_error) gasnett_freezeForDebuggerErr();