From ca44dbd3eee30f2a37be6cafee86373e2681a1e8 Mon Sep 17 00:00:00 2001 From: kennyzzhang Date: Wed, 29 Apr 2026 21:11:52 -0400 Subject: [PATCH 1/5] Reset call stack on cilk scheduler Signed-off-by: kennyzzhang --- src/trace_writer.ml | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/src/trace_writer.ml b/src/trace_writer.ml index 6293daf4e..b2bd7fdac 100644 --- a/src/trace_writer.ml +++ b/src/trace_writer.ml @@ -558,12 +558,6 @@ let create_thread t event = } ;; -let call t thread_info ~time ~location = - let ev = Pending_event.create_call location ~from_untraced:false in - add_event t thread_info time ev; - Callstack.push thread_info.callstack location -;; - let ret_without_checking_for_go_hacks t (thread_info : _ Thread_info.t) ~time = match Callstack.pop thread_info.callstack with | Some { symbol; _ } -> add_event t thread_info time { symbol; kind = Ret } @@ -610,6 +604,34 @@ let end_of_thread t (thread_info : _ Thread_info.t) ~time ~is_kernel_address : u Thread_info.set_callstack thread_info ~is_kernel_address ~time ;; +module Cilk_hacks : sig + val call_clear_if_scheduler + : 'a inner + -> 'a Thread_info.t + -> time:Mapped_time.t + -> location: Event.Location.t + -> unit +end = struct + let is_cilk_scheduler (symbol : Symbol.t) = + match symbol with + | From_perf "worker_scheduler(__cilkrts_worker*, history_t*)" -> true + | _ -> false + ;; + + let call_clear_if_scheduler t thread_info ~time ~location = + let call_symbol = Event.Location.symbol location in + if is_cilk_scheduler call_symbol + then clear_callstack t thread_info ~time; + ;; +end + +let call t thread_info ~time ~location = + Cilk_hacks.call_clear_if_scheduler t thread_info ~time ~location; + let ev = Pending_event.create_call location ~from_untraced:false in + add_event t thread_info time ev; + Callstack.push thread_info.callstack location +;; + (* Go (the programming language) has coroutines known as goroutines. The function [gogo] jumps from one goroutine to the next. Since [gogo] can jump anywhere, it's a shining example of what magic-trace can't handle out of the box. So, we hack it. From 6e71222967e76abb935a696b05e879cfc099d375 Mon Sep 17 00:00:00 2001 From: kennyzzhang Date: Fri, 1 May 2026 16:05:26 -0400 Subject: [PATCH 2/5] Change to better approach on longjmps longjmp_to_user_code pushes to inactive stacks, longjmp_to_runtime pops from inactive stacks Signed-off-by: kennyzzhang --- src/trace_writer.ml | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/src/trace_writer.ml b/src/trace_writer.ml index b2bd7fdac..d08fb623b 100644 --- a/src/trace_writer.ml +++ b/src/trace_writer.ml @@ -605,31 +605,45 @@ let end_of_thread t (thread_info : _ Thread_info.t) ~time ~is_kernel_address : u ;; module Cilk_hacks : sig - val call_clear_if_scheduler + val call_handle_stack_switch : 'a inner -> 'a Thread_info.t -> time:Mapped_time.t -> location: Event.Location.t -> unit end = struct - let is_cilk_scheduler (symbol : Symbol.t) = - match symbol with - | From_perf "worker_scheduler(__cilkrts_worker*, history_t*)" -> true - | _ -> false + let ret = ret_without_checking_for_go_hacks + + let call_switch_to_user_code t (thread_info : _ Thread_info.t) ~time = + ret t thread_info ~time; + Stack.push thread_info.inactive_callstacks thread_info.callstack; + thread_info.callstack <- Callstack.create ~create_time:time + ;; + + let call_switch_to_runtime t (thread_info : _ Thread_info.t) ~time = + ret t thread_info ~time; + clear_callstack t thread_info ~time; + match Stack.pop thread_info.inactive_callstacks with + | Some callstack -> thread_info.callstack <- callstack + | None -> thread_info.callstack <- Callstack.create ~create_time:time ;; - let call_clear_if_scheduler t thread_info ~time ~location = + let call_handle_stack_switch t thread_info ~time ~location = let call_symbol = Event.Location.symbol location in - if is_cilk_scheduler call_symbol - then clear_callstack t thread_info ~time; + match call_symbol with + | From_perf "longjmp_to_user_code(__cilkrts_worker*, Closure*)" -> + call_switch_to_user_code t thread_info ~time; + | From_perf "longjmp_to_runtime(__cilkrts_worker*)" -> + call_switch_to_runtime t thread_info ~time; + | _ -> () ;; end let call t thread_info ~time ~location = - Cilk_hacks.call_clear_if_scheduler t thread_info ~time ~location; let ev = Pending_event.create_call location ~from_untraced:false in add_event t thread_info time ev; - Callstack.push thread_info.callstack location + Callstack.push thread_info.callstack location; + Cilk_hacks.call_handle_stack_switch t thread_info ~time ~location ;; (* Go (the programming language) has coroutines known as goroutines. The function [gogo] jumps From 484af551388991893d388d40529f24b600abe6a7 Mon Sep 17 00:00:00 2001 From: kennyzzhang Date: Fri, 1 May 2026 17:06:32 -0400 Subject: [PATCH 3/5] Fix __cilkrts_sync, add documentation Signed-off-by: kennyzzhang --- src/trace_writer.ml | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/trace_writer.ml b/src/trace_writer.ml index d08fb623b..8c5aee1f2 100644 --- a/src/trace_writer.ml +++ b/src/trace_writer.ml @@ -604,7 +604,13 @@ let end_of_thread t (thread_info : _ Thread_info.t) ~time ~is_kernel_address : u Thread_info.set_callstack thread_info ~is_kernel_address ~time ;; -module Cilk_hacks : sig +(* OpenCilk's runtime cheetah (https://github.com/OpenCilk/cheetah) uses + longjmp to switch between user and runtime stacks. + + To deal with this stack switching, when jumping into the runtime, we need + to clear the user's stack frames. The inactive_callstacks mechanism + is perfect for keeping track of this stack data. *) +module OpenCilk_hacks : sig val call_handle_stack_switch : 'a inner -> 'a Thread_info.t @@ -615,13 +621,26 @@ end = struct let ret = ret_without_checking_for_go_hacks let call_switch_to_user_code t (thread_info : _ Thread_info.t) ~time = + (* Pop the sysdep_longjmp_to_sf frame *) ret t thread_info ~time; - Stack.push thread_info.inactive_callstacks thread_info.callstack; - thread_info.callstack <- Callstack.create ~create_time:time + (* The next stack frame is either __cilkrts_sync or longjmp_to_user_code. + In either case, we want to pop this frame as well. *) + match Callstack.top thread_info.callstack with + | Some { symbol = From_perf symbol; _ } -> + (match symbol with + | "longjmp_to_user_code(__cilkrts_worker*, Closure*)" -> + ret t thread_info ~time; + Stack.push thread_info.inactive_callstacks thread_info.callstack; + thread_info.callstack <- Callstack.create ~create_time:time + | "__cilkrts_sync" -> + ret t thread_info ~time + | _ -> ()) + | _ ->() ;; let call_switch_to_runtime t (thread_info : _ Thread_info.t) ~time = - ret t thread_info ~time; + (* Even though we want to pop the longjmp_to_runtime frame, we're clearing + the entire user stack anyways, so no point in manually popping. *) clear_callstack t thread_info ~time; match Stack.pop thread_info.inactive_callstacks with | Some callstack -> thread_info.callstack <- callstack @@ -631,7 +650,7 @@ end = struct let call_handle_stack_switch t thread_info ~time ~location = let call_symbol = Event.Location.symbol location in match call_symbol with - | From_perf "longjmp_to_user_code(__cilkrts_worker*, Closure*)" -> + | From_perf "sysdep_longjmp_to_sf(__cilkrts_stack_frame*)" -> call_switch_to_user_code t thread_info ~time; | From_perf "longjmp_to_runtime(__cilkrts_worker*)" -> call_switch_to_runtime t thread_info ~time; @@ -643,7 +662,7 @@ let call t thread_info ~time ~location = let ev = Pending_event.create_call location ~from_untraced:false in add_event t thread_info time ev; Callstack.push thread_info.callstack location; - Cilk_hacks.call_handle_stack_switch t thread_info ~time ~location + OpenCilk_hacks.call_handle_stack_switch t thread_info ~time ~location ;; (* Go (the programming language) has coroutines known as goroutines. The function [gogo] jumps From 9e5b4ea1369d790feec0830fdc0c8cdc47888ce6 Mon Sep 17 00:00:00 2001 From: kennyzzhang Date: Fri, 1 May 2026 17:58:13 -0400 Subject: [PATCH 4/5] Minor rename Signed-off-by: kennyzzhang --- src/trace_writer.ml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/trace_writer.ml b/src/trace_writer.ml index 8c5aee1f2..eea226e36 100644 --- a/src/trace_writer.ml +++ b/src/trace_writer.ml @@ -620,7 +620,7 @@ module OpenCilk_hacks : sig end = struct let ret = ret_without_checking_for_go_hacks - let call_switch_to_user_code t (thread_info : _ Thread_info.t) ~time = + let switch_to_user_code t (thread_info : _ Thread_info.t) ~time = (* Pop the sysdep_longjmp_to_sf frame *) ret t thread_info ~time; (* The next stack frame is either __cilkrts_sync or longjmp_to_user_code. @@ -634,11 +634,11 @@ end = struct thread_info.callstack <- Callstack.create ~create_time:time | "__cilkrts_sync" -> ret t thread_info ~time - | _ -> ()) - | _ ->() + | _ -> Printf.printf "OpenCilk_hacks: Unexpected symbol %s\n" symbol) + | _ -> Printf.printf "OpenCilk_hacks: Unexpected symbol [unknown]" ;; - let call_switch_to_runtime t (thread_info : _ Thread_info.t) ~time = + let switch_to_runtime t (thread_info : _ Thread_info.t) ~time = (* Even though we want to pop the longjmp_to_runtime frame, we're clearing the entire user stack anyways, so no point in manually popping. *) clear_callstack t thread_info ~time; @@ -651,9 +651,9 @@ end = struct let call_symbol = Event.Location.symbol location in match call_symbol with | From_perf "sysdep_longjmp_to_sf(__cilkrts_stack_frame*)" -> - call_switch_to_user_code t thread_info ~time; + switch_to_user_code t thread_info ~time; | From_perf "longjmp_to_runtime(__cilkrts_worker*)" -> - call_switch_to_runtime t thread_info ~time; + switch_to_runtime t thread_info ~time; | _ -> () ;; end From c31a2f752fbdf41f786b87b3c0fd5ff4869b05eb Mon Sep 17 00:00:00 2001 From: kennyzzhang Date: Fri, 1 May 2026 18:02:00 -0400 Subject: [PATCH 5/5] Autoformat Signed-off-by: kennyzzhang --- src/trace_writer.ml | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/src/trace_writer.ml b/src/trace_writer.ml index eea226e36..a780fe824 100644 --- a/src/trace_writer.ml +++ b/src/trace_writer.ml @@ -604,18 +604,18 @@ let end_of_thread t (thread_info : _ Thread_info.t) ~time ~is_kernel_address : u Thread_info.set_callstack thread_info ~is_kernel_address ~time ;; -(* OpenCilk's runtime cheetah (https://github.com/OpenCilk/cheetah) uses - longjmp to switch between user and runtime stacks. +(* OpenCilk's runtime cheetah (https://github.com/OpenCilk/cheetah) uses longjmp to switch + between user and runtime stacks. - To deal with this stack switching, when jumping into the runtime, we need - to clear the user's stack frames. The inactive_callstacks mechanism - is perfect for keeping track of this stack data. *) + To deal with this stack switching, when jumping into the runtime, we need to clear the + user's stack frames. The inactive_callstacks mechanism is perfect for keeping track of + this stack data. *) module OpenCilk_hacks : sig val call_handle_stack_switch - : 'a inner + : 'a inner -> 'a Thread_info.t -> time:Mapped_time.t - -> location: Event.Location.t + -> location:Event.Location.t -> unit end = struct let ret = ret_without_checking_for_go_hacks @@ -623,24 +623,23 @@ end = struct let switch_to_user_code t (thread_info : _ Thread_info.t) ~time = (* Pop the sysdep_longjmp_to_sf frame *) ret t thread_info ~time; - (* The next stack frame is either __cilkrts_sync or longjmp_to_user_code. - In either case, we want to pop this frame as well. *) + (* The next stack frame is either __cilkrts_sync or longjmp_to_user_code. In either + case, we want to pop this frame as well. *) match Callstack.top thread_info.callstack with | Some { symbol = From_perf symbol; _ } -> (match symbol with - | "longjmp_to_user_code(__cilkrts_worker*, Closure*)" -> - ret t thread_info ~time; - Stack.push thread_info.inactive_callstacks thread_info.callstack; - thread_info.callstack <- Callstack.create ~create_time:time - | "__cilkrts_sync" -> - ret t thread_info ~time - | _ -> Printf.printf "OpenCilk_hacks: Unexpected symbol %s\n" symbol) + | "longjmp_to_user_code(__cilkrts_worker*, Closure*)" -> + ret t thread_info ~time; + Stack.push thread_info.inactive_callstacks thread_info.callstack; + thread_info.callstack <- Callstack.create ~create_time:time + | "__cilkrts_sync" -> ret t thread_info ~time + | _ -> Printf.printf "OpenCilk_hacks: Unexpected symbol %s\n" symbol) | _ -> Printf.printf "OpenCilk_hacks: Unexpected symbol [unknown]" ;; let switch_to_runtime t (thread_info : _ Thread_info.t) ~time = - (* Even though we want to pop the longjmp_to_runtime frame, we're clearing - the entire user stack anyways, so no point in manually popping. *) + (* Even though we want to pop the longjmp_to_runtime frame, we're clearing the entire + user stack anyways, so no point in manually popping. *) clear_callstack t thread_info ~time; match Stack.pop thread_info.inactive_callstacks with | Some callstack -> thread_info.callstack <- callstack @@ -651,9 +650,9 @@ end = struct let call_symbol = Event.Location.symbol location in match call_symbol with | From_perf "sysdep_longjmp_to_sf(__cilkrts_stack_frame*)" -> - switch_to_user_code t thread_info ~time; + switch_to_user_code t thread_info ~time | From_perf "longjmp_to_runtime(__cilkrts_worker*)" -> - switch_to_runtime t thread_info ~time; + switch_to_runtime t thread_info ~time | _ -> () ;; end