DRAFT: CIRC-4436 - Allow stacktrace across an eventer asynch

src/eventer/eventer.h

@@ -49,6 +49,8 @@ extern "C" {
 #include <ck_pr.h>
 #include <ck_spinlock.h>
 
+#define EVENTER_STACKTRACE_SIZE   128
+
 /* We use ETIMEDOUT, which might be ETIME on some platforms */
 #ifndef ETIME
 #define ETIME ETIMEDOUT

src/eventer/eventer_impl.c

@@ -1029,6 +1029,7 @@ int eventer_impl_init(void) {
 }
 
 void eventer_add_asynch_subqueue(eventer_jobq_t *q, eventer_t e, uint64_t subqueue) {
+  e->frames = mtev_backtrace_ucontext(e->callstack, NULL, EVENTER_STACKTRACE_SIZE, true);
   eventer_job_t *job;
   /* always use 0, if unspecified */
   if(eventer_is_loop(e->thr_owner) < 0) e->thr_owner = eventer_impl_tls_data[0].tid;
@@ -1059,6 +1060,7 @@ void eventer_add_asynch(eventer_jobq_t *q, eventer_t e) {
 }
 
 void eventer_add_asynch_dep_subqueue(eventer_jobq_t *q, eventer_t e, uint64_t subqueue) {
+  e->frames = mtev_backtrace_ucontext(e->callstack, NULL, EVENTER_STACKTRACE_SIZE, true);
   eventer_job_t *job;
   /* always use 0, if unspecified */
   if(eventer_is_loop(e->thr_owner) < 0) e->thr_owner = eventer_impl_tls_data[0].tid;
@@ -1089,6 +1091,7 @@ void eventer_add_asynch_dep(eventer_jobq_t *q, eventer_t e) {
 }
 
 mtev_boolean eventer_try_add_asynch_subqueue(eventer_jobq_t *q, eventer_t e, uint64_t subqueue) {
+  e->frames = mtev_backtrace_ucontext(e->callstack, NULL, EVENTER_STACKTRACE_SIZE, true);
   eventer_t timeout = NULL;
   eventer_job_t *job;
   /* always use 0, if unspecified */
@@ -1125,6 +1128,7 @@ mtev_boolean eventer_try_add_asynch(eventer_jobq_t *q, eventer_t e) {
 }
 
 mtev_boolean eventer_try_add_asynch_dep_subqueue(eventer_jobq_t *q, eventer_t e, uint64_t subqueue) {
+  e->frames = mtev_backtrace_ucontext(e->callstack, NULL, EVENTER_STACKTRACE_SIZE, true);
   eventer_t timeout = NULL;
   eventer_job_t *job;
   /* always use 0, if unspecified */
@@ -1464,6 +1468,7 @@ int eventer_run_callback(eventer_func_t f, eventer_t e, int m, void *c, struct t
   eventer_ref(e);
   eventer_set_this_event(e);
   eventer_callback_prep(e, m, c, n);
+  e->eventer_run_callback_end = &&end;
   rmask = f(e, m, c, n);
   eventer_callback_cleanup(e, rmask);
   mtev_boolean freed = eventer_deref(e);
@@ -1485,6 +1490,8 @@ int eventer_run_callback(eventer_func_t f, eventer_t e, int m, void *c, struct t
     }
   }
   return rmask;
+end:
+  ;
 }
 
 static void *eventer_thread_harness(void *ve) {

src/eventer/eventer_impl_private.h

@@ -77,6 +77,9 @@ struct name { \
   pthread_t           thr_owner; \
   uint32_t            refcnt; \
   eventer_context_t   ctx[MAX_EVENT_CTXS]; \
+  int                 frames; \
+  void               *callstack[EVENTER_STACKTRACE_SIZE]; \
+  void               *eventer_run_callback_end; \
 }
 
 EVENTER_STRUCT(_event);

src/utils/mtev_stacktrace.c

@@ -34,6 +34,7 @@
 #include "mtev_sort.h"
 #include "mtev_skiplist.h"
 #include "mtev_hash.h"
+#include "../eventer/eventer.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <signal.h>
@@ -969,7 +970,7 @@ mtev_stacktrace_internal(mtev_log_stream_t ls, void *caller,
   crash_in_crash = NULL;
 }
 
-int mtev_backtrace_ucontext(void **callstack, ucontext_t *ctx, int cnt) {
+int mtev_backtrace_ucontext(void **callstack, ucontext_t *ctx, int cnt, bool cross_eventer) {
   int frames = 0;
 #if defined(HAVE_LIBUNWIND)
   unw_cursor_t cursor;
@@ -987,13 +988,28 @@ int mtev_backtrace_ucontext(void **callstack, ucontext_t *ctx, int cnt) {
   }
   unw_init_local(&cursor, (unw_context_t *)ctx);
 
-  int step_result = 1;
+  // this works but only allows us to go back through one asynch layer because we are pulling the
+  // event object for the current thread
+  eventer_t e = eventer_get_this_event();
+
+  // when backtracing from before eventer, remove the current function
+  int step_result = cross_eventer ? unw_step(&cursor) : 1;
   while (step_result > 0 && frames<cnt) {
     unw_word_t pc;
     unw_get_reg(&cursor, UNW_REG_IP, &pc);
     if (pc == 0) {
       break;
     }
+    // if we hit an address inside eventer_run_callback, then we can try to roll in the pre-eventer
+    // callstack starting from where we run into the address within eventer_run_callback()
+    if (e && pc >= (unw_word_t)eventer_run_callback &&
+        pc <= (unw_word_t)e->eventer_run_callback_end) {
+      int eventer_frame = 0;
+      while(frames<cnt && eventer_frame < e->frames) {
+        callstack[frames++] = e->callstack[eventer_frame++]; 
+      }
+      break;
+    }
     callstack[frames++] = (void *)pc;
     step_result = unw_step(&cursor);
   }
@@ -1008,15 +1024,15 @@ void mtev_log_backtrace(mtev_log_stream_t ls, void **callstack, int cnt) {
   mtev_stacktrace_internal(ls, mtev_stacktrace, NULL, NULL, callstack, cnt);
 }
 int mtev_backtrace(void **callstack, int cnt) {
-  return mtev_backtrace_ucontext(callstack, NULL, cnt);
+  return mtev_backtrace_ucontext(callstack, NULL, cnt, false);
 }
 int global_in_stacktrace = 0;
 #if defined(__sun__)
 void mtev_stacktrace_ucontext(mtev_log_stream_t ls, ucontext_t *ucp) {
   if (ck_pr_fas_int(&global_in_stacktrace, 1))
     return;
   void *callstack[128];
-  int frames = mtev_backtrace_ucontext(callstack, ucp, 128);
+  int frames = mtev_backtrace_ucontext(callstack, ucp, 128, false);
   mtev_stacktrace_internal(ls, mtev_stacktrace, NULL, (void *)ucp, callstack, frames);
   global_in_stacktrace = 0;
 }
@@ -1029,7 +1045,7 @@ void mtev_stacktrace_ucontext_skip(mtev_log_stream_t ls, ucontext_t *ucp, int ig
   struct sigaction sa, sasegv, saill, sabus;
   sigset_t smprev;
   void *callstack[128];
-  int frames = mtev_backtrace_ucontext(callstack, ucp, 128);
+  int frames = mtev_backtrace_ucontext(callstack, ucp, 128, false);
   ignore = MIN(ignore, frames);
 
   memset(&sa, 0, sizeof(sa));

src/utils/mtev_stacktrace.h

@@ -68,7 +68,7 @@ API_EXPORT(int)
   mtev_backtrace(void **ips, int cnt);
 
 API_EXPORT(int)
-  mtev_backtrace_ucontext(void **ips, ucontext_t *ucp, int cnt);
+  mtev_backtrace_ucontext(void **ips, ucontext_t *ucp, int cnt, bool cross_eventer);
 
 API_EXPORT(int)
   mtev_aco_backtrace(aco_t *co, void **addrs, int addrs_len);