1  /* Copyright (C) 2021-2023 Free Software Foundation, Inc.
       2     Contributed by Oracle.
       3  
       4     This file is part of GNU Binutils.
       5  
       6     This program is free software; you can redistribute it and/or modify
       7     it under the terms of the GNU General Public License as published by
       8     the Free Software Foundation; either version 3, or (at your option)
       9     any later version.
      10  
      11     This program is distributed in the hope that it will be useful,
      12     but WITHOUT ANY WARRANTY; without even the implied warranty of
      13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      14     GNU General Public License for more details.
      15  
      16     You should have received a copy of the GNU General Public License
      17     along with this program; if not, write to the Free Software
      18     Foundation, 51 Franklin Street - Fifth Floor, Boston,
      19     MA 02110-1301, USA.  */
      20  
      21  /*
      22   *	Synchronization events
      23   */
      24  #include "config.h"
      25  #include <alloca.h>
      26  #include <dlfcn.h>
      27  #include <unistd.h>
      28  #include <semaphore.h>		/* sem_wait() */
      29  #include <stdlib.h>
      30  #include <string.h>
      31  #include <sys/param.h>
      32  #include <pthread.h>
      33  
      34  #include "gp-defs.h"
      35  #include "collector.h"
      36  #include "gp-experiment.h"
      37  #include "data_pckts.h"
      38  #include "tsd.h"
      39  #include "cc_libcollector.h"
      40  
      41  /* define the packet that will be written out */
      42  typedef struct Sync_packet
      43  { /* Synchronization delay tracing packet */
      44    Common_packet comm;
      45    hrtime_t requested;       /* time of synchronization request */
      46    Vaddr_type objp;          /* vaddr of synchronization object */
      47  } Sync_packet;
      48  
      49  static int open_experiment (const char *);
      50  static int start_data_collection (void);
      51  static int stop_data_collection (void);
      52  static int close_experiment (void);
      53  static int detach_experiment (void);
      54  static int init_thread_intf ();
      55  static int sync_calibrate ();
      56  
      57  static ModuleInterface module_interface ={
      58    SP_SYNCTRACE_FILE,        /* description */
      59    NULL,                     /* initInterface */
      60    open_experiment,          /* openExperiment */
      61    start_data_collection,    /* startDataCollection */
      62    stop_data_collection,     /* stopDataCollection */
      63    close_experiment,         /* closeExperiment */
      64    detach_experiment         /* detachExperiment (fork child) */
      65  };
      66  
      67  static CollectorInterface *collector_interface = NULL;
      68  static int sync_mode = 0;
      69  static long sync_scope = 0;
      70  static int sync_native = 0;
      71  static int sync_java = 0;
      72  static CollectorModule sync_hndl = COLLECTOR_MODULE_ERR;
      73  static unsigned sync_key = COLLECTOR_TSD_INVALID_KEY;
      74  static long sync_threshold = -1; /* calibrate the value */
      75  static int init_thread_intf_started = 0;
      76  static int init_thread_intf_finished = 0;
      77  
      78  #define CHCK_NREENTRANCE(x)     (!sync_native || !sync_mode || ((x) = collector_interface->getKey( sync_key )) == NULL || (*(x) != 0))
      79  #define RECHCK_NREENTRANCE(x)   (!sync_native || !sync_mode || ((x) = collector_interface->getKey( sync_key )) == NULL || (*(x) == 0))
      80  #define CHCK_JREENTRANCE(x)     (!sync_java || !sync_mode || ((x) = collector_interface->getKey( sync_key )) == NULL || (*(x) != 0))
      81  #define RECHCK_JREENTRANCE(x)   (!sync_java || !sync_mode || ((x) = collector_interface->getKey( sync_key )) == NULL || (*(x) == 0))
      82  #define PUSH_REENTRANCE(x)      ((*(x))++)
      83  #define POP_REENTRANCE(x)       ((*(x))--)
      84  #define gethrtime	collector_interface->getHiResTime
      85  
      86  /*
      87   * In most cases, the functions which require interposition are implemented as
      88   * weak symbols corresponding to an associated internal function named with a
      89   * leading underscore: e.g., mutex_lock() is simply an alias for _mutex_lock().
      90   * For the wait functions, however, the published version (used by applications)
      91   * is distinct from the internal version (used by system libraries), i.e.,
      92   * cond_wait() is an alias for _cond_wait_cancel() rather than _cond_wait().
      93   */
      94  static long int (*__real_strtol)(const char *nptr, char **endptr, int base) = NULL;
      95  static int (*__real_fprintf) (FILE *stream, const char *format, ...) = NULL;
      96  static void (*__real___collector_jprofile_enable_synctrace) (void) = NULL;
      97  static int (*__real_pthread_mutex_lock) (pthread_mutex_t *mutex) = NULL;
      98  static int (*__real_pthread_mutex_lock_2_17) (pthread_mutex_t *mutex) = NULL;
      99  static int (*__real_pthread_mutex_lock_2_2_5) (pthread_mutex_t *mutex) = NULL;
     100  static int (*__real_pthread_mutex_lock_2_0) (pthread_mutex_t *mutex) = NULL;
     101  static int (*__real_pthread_mutex_unlock) (pthread_mutex_t *mutex) = NULL;
     102  static int (*__real_pthread_cond_wait) (pthread_cond_t *restrict cond,
     103  				pthread_mutex_t *restrict mutex) = NULL;
     104  static int (*__real_pthread_cond_timedwait) (pthread_cond_t *restrict cond,
     105  			pthread_mutex_t *restrict mutex,
     106  			const struct timespec *restrict abstime) = NULL;
     107  static int (*__real_pthread_join) (pthread_t thread, void **retval) = NULL;
     108  static int (*__real_pthread_join_2_34) (pthread_t thread, void **retval) = NULL;
     109  static int (*__real_pthread_join_2_17) (pthread_t thread, void **retval) = NULL;
     110  static int (*__real_pthread_join_2_2_5) (pthread_t thread, void **retval) = NULL;
     111  static int (*__real_pthread_join_2_0) (pthread_t thread, void **retval) = NULL;
     112  static int (*__real_sem_wait) (sem_t *sem) = NULL;
     113  static int (*__real_sem_wait_2_34) (sem_t *sem) = NULL;
     114  static int (*__real_sem_wait_2_17) (sem_t *sem) = NULL;
     115  static int (*__real_sem_wait_2_2_5) (sem_t *sem) = NULL;
     116  static int (*__real_sem_wait_2_1) (sem_t *sem) = NULL;
     117  static int (*__real_sem_wait_2_0) (sem_t *sem) = NULL;
     118  static int (*__real_pthread_cond_wait_2_17) (pthread_cond_t *restrict cond,
     119  				pthread_mutex_t *restrict mutex) = NULL;
     120  static int (*__real_pthread_cond_wait_2_3_2) (pthread_cond_t *restrict cond,
     121  				pthread_mutex_t *restrict mutex) = NULL;
     122  static int (*__real_pthread_cond_wait_2_2_5) (pthread_cond_t *restrict cond,
     123  				pthread_mutex_t *restrict mutex) = NULL;
     124  static int (*__real_pthread_cond_wait_2_0) (pthread_cond_t *restrict cond,
     125  				pthread_mutex_t *restrict mutex) = NULL;
     126  static int (*__real_pthread_cond_timedwait_2_17) (pthread_cond_t *restrict cond,
     127  			pthread_mutex_t *restrict mutex,
     128  			const struct timespec *restrict abstime) = NULL;
     129  static int (*__real_pthread_cond_timedwait_2_3_2) (pthread_cond_t *restrict cond,
     130  			pthread_mutex_t *restrict mutex,
     131  			const struct timespec *restrict abstime) = NULL;
     132  static int (*__real_pthread_cond_timedwait_2_2_5) (pthread_cond_t *restrict cond,
     133  			pthread_mutex_t *restrict mutex,
     134  			const struct timespec *restrict abstime) = NULL;
     135  static int (*__real_pthread_cond_timedwait_2_0) (pthread_cond_t *restrict cond,
     136  			pthread_mutex_t *restrict mutex,
     137  			const struct timespec *restrict abstime) = NULL;
     138  
     139  
     140  static void
     141  collector_memset (void *s, int c, size_t n)
     142  {
     143    unsigned char *s1 = s;
     144    while (n--)
     145      *s1++ = (unsigned char) c;
     146  }
     147  
     148  void
     149  __collector_module_init (CollectorInterface *_collector_interface)
     150  {
     151    if (_collector_interface == NULL)
     152      return;
     153    collector_interface = _collector_interface;
     154    TprintfT (0, "synctrace: __collector_module_init\n");
     155    sync_hndl = collector_interface->registerModule (&module_interface);
     156  
     157    /* Initialize next module */
     158    ModuleInitFunc next_init = (ModuleInitFunc) dlsym (RTLD_NEXT, "__collector_module_init");
     159    if (next_init != NULL)
     160      next_init (_collector_interface);
     161  }
     162  
     163  static int
     164  open_experiment (const char *exp)
     165  {
     166    long thresh = 0;
     167    if (init_thread_intf_finished == 0)
     168      init_thread_intf ();
     169    if (collector_interface == NULL)
     170      {
     171        Tprintf (0, "synctrace: collector_interface is null.\n");
     172        return COL_ERROR_SYNCINIT;
     173      }
     174    if (sync_hndl == COLLECTOR_MODULE_ERR)
     175      {
     176        Tprintf (0, "synctrace: handle create failed.\n");
     177        collector_interface->writeLog ("<event kind=\"%s\" id=\"%d\">data handle not created</event>\n",
     178  				     SP_JCMD_CERROR, COL_ERROR_SYNCINIT);
     179        return COL_ERROR_SYNCINIT;
     180      }
     181    TprintfT (0, "synctrace: open_experiment %s\n", exp);
     182  
     183    char *params = (char *) collector_interface->getParams ();
     184    while (params)
     185      {
     186        if ((params[0] == 's') && (params[1] == ':'))
     187  	{
     188  	  char *ptr = params + 2;
     189  	  Tprintf (DBG_LT1, "synctrace: open_experiment s: parameter = %s\n", ptr);
     190  	  while (*ptr != ',' && *ptr != ';')
     191  	    ptr++;
     192  	  sync_scope = 0;
     193  	  if (*ptr == ',')
     194  	    {
     195  	      sync_scope = CALL_REAL (strtol) (ptr + 1, NULL, 0);
     196  	      switch (sync_scope)
     197  		{
     198  		case 1:
     199  		  sync_java = 0;
     200  		  sync_native = 1;
     201  		  break;
     202  		case 2:
     203  		  sync_java = 1;
     204  		  sync_native = 0;
     205  		  break;
     206  		default:
     207  		case 3:
     208  		  sync_native = 1;
     209  		  sync_java = 1;
     210  		  break;
     211  		}
     212  	      Tprintf (0, "\tsynctrace: sync_scope found as %ld\n", sync_scope);
     213  	    }
     214  	  else
     215  	    {
     216  	      /* the old-style descriptor, without scope */
     217  	      /* if there was no comma, use the old default */
     218  	      sync_scope = 3;
     219  	      sync_java = 1;
     220  	      sync_native = 1;
     221  	      Tprintf (0, "\tsynctrace: sync_scope not found set to %ld\n", sync_scope);
     222  	    }
     223  	  if (__real___collector_jprofile_enable_synctrace == NULL)
     224  	    sync_java = 0;
     225  	  thresh = CALL_REAL (strtol)(params + 2, NULL, 0);
     226  	  break; /* from the loop to find the "s:thresh,scope" entry */
     227  	}
     228        else
     229  	params++;
     230      }
     231    if (params == NULL)  /* Sync data collection not specified */
     232      return COL_ERROR_SYNCINIT;
     233    if (thresh < 0)  /* calibrate the threshold, keep it as a negative number */
     234      thresh = -sync_calibrate ();
     235  
     236    sync_key = collector_interface->createKey (sizeof ( int), NULL, NULL);
     237    if (sync_key == (unsigned) - 1)
     238      {
     239        Tprintf (0, "synctrace: TSD key create failed.\n");
     240        collector_interface->writeLog ("<event kind=\"%s\" id=\"%d\">TSD key not created</event>\n",
     241  				     SP_JCMD_CERROR, COL_ERROR_SYNCINIT);
     242        return COL_ERROR_SYNCINIT;
     243      }
     244    /* if Java synctrace was requested, tell the jprofile module */
     245    if (sync_java)
     246      {
     247        TprintfT (0, "synctrace: enabling Java synctrace\n");
     248        CALL_REAL (__collector_jprofile_enable_synctrace)();
     249      }
     250    collector_interface->writeLog ("<profile name=\"%s\" threshold=\"%ld\" scope=\"%ld\">\n",
     251  				 SP_JCMD_SYNCTRACE, thresh, sync_scope);
     252    collector_interface->writeLog ("  <profdata fname=\"%s\"/>\n",
     253  				 module_interface.description);
     254    /* Record Sync_packet description */
     255    Sync_packet *pp = NULL;
     256    collector_interface->writeLog ("  <profpckt kind=\"%d\" uname=\"Synchronization tracing data\">\n", SYNC_PCKT);
     257    collector_interface->writeLog ("    <field name=\"LWPID\" uname=\"Lightweight process id\" offset=\"%d\" type=\"%s\"/>\n",
     258  				 &pp->comm.lwp_id, sizeof (pp->comm.lwp_id) == 4 ? "INT32" : "INT64");
     259    collector_interface->writeLog ("    <field name=\"THRID\" uname=\"Thread number\" offset=\"%d\" type=\"%s\"/>\n",
     260  				 &pp->comm.thr_id, sizeof (pp->comm.thr_id) == 4 ? "INT32" : "INT64");
     261    collector_interface->writeLog ("    <field name=\"CPUID\" uname=\"CPU id\" offset=\"%d\" type=\"%s\"/>\n",
     262  				 &pp->comm.cpu_id, sizeof (pp->comm.cpu_id) == 4 ? "INT32" : "INT64");
     263    collector_interface->writeLog ("    <field name=\"TSTAMP\" uname=\"High resolution timestamp\" offset=\"%d\" type=\"%s\"/>\n",
     264  				 &pp->comm.tstamp, sizeof (pp->comm.tstamp) == 4 ? "INT32" : "INT64");
     265    collector_interface->writeLog ("    <field name=\"FRINFO\" offset=\"%d\" type=\"%s\"/>\n",
     266  				 &pp->comm.frinfo, sizeof (pp->comm.frinfo) == 4 ? "INT32" : "INT64");
     267    collector_interface->writeLog ("    <field name=\"SRQST\" uname=\"Synchronization start time\" offset=\"%d\" type=\"%s\"/>\n",
     268  				 &pp->requested, sizeof (pp->requested) == 4 ? "INT32" : "INT64");
     269    collector_interface->writeLog ("    <field name=\"SOBJ\" uname=\"Synchronization object address\" offset=\"%d\" type=\"%s\"/>\n",
     270  				 &pp->objp, sizeof (pp->objp) == 4 ? "INT32" : "INT64");
     271    collector_interface->writeLog ("  </profpckt>\n");
     272    collector_interface->writeLog ("</profile>\n");
     273  
     274    /* Convert threshold from microsec to nanosec */
     275    sync_threshold = (thresh > 0 ? thresh : -thresh) * 1000;
     276    TprintfT (0, "synctrace: open_experiment complete %ld\n", sync_threshold);
     277    return COL_ERROR_NONE;
     278  }
     279  
     280  static int
     281  start_data_collection (void)
     282  {
     283    sync_mode = 1;
     284    TprintfT (0, "synctrace: start_data_collection\n");
     285    return 0;
     286  }
     287  
     288  static int
     289  stop_data_collection (void)
     290  {
     291    sync_mode = 0;
     292    TprintfT (0, "synctrace: stop_data_collection\n");
     293    return 0;
     294  }
     295  
     296  static int
     297  close_experiment (void)
     298  {
     299    sync_mode = 0;
     300    sync_threshold = -1;
     301    sync_key = COLLECTOR_TSD_INVALID_KEY;
     302    TprintfT (0, "synctrace: close_experiment\n");
     303    return 0;
     304  }
     305  
     306  /* fork child.  Clean up state but don't write to experiment */
     307  static int
     308  detach_experiment (void)
     309  {
     310    sync_mode = 0;
     311    sync_threshold = -1;
     312    sync_key = COLLECTOR_TSD_INVALID_KEY;
     313    TprintfT (0, "synctrace: detach_experiment\n");
     314    return 0;
     315  }
     316  
     317  #define NUM_ITER    100     /* number of iterations in calibration */
     318  #define NUM_WARMUP    3     /* number of warm up iterations */
     319  
     320  static int
     321  sync_calibrate ()
     322  {
     323    pthread_mutex_t mt = PTHREAD_MUTEX_INITIALIZER;
     324    hrtime_t bt, at, delta;
     325    hrtime_t avg, max, min;
     326    int i;
     327    int ret;
     328    avg = (hrtime_t) 0;
     329    min = max = (hrtime_t) 0;
     330    for (i = 0; i < NUM_ITER + NUM_WARMUP; i++)
     331      {
     332        /* Here we simulate a real call */
     333        bt = gethrtime ();
     334        ret = CALL_REAL (pthread_mutex_lock)(&mt);
     335        at = gethrtime ();
     336        CALL_REAL (pthread_mutex_unlock)(&mt);
     337        if (i < NUM_WARMUP)   /* skip these iterations */
     338  	continue;
     339        /* add the time of this one */
     340        delta = at - bt;
     341        avg += delta;
     342        if (min == 0)
     343  	min = delta;
     344        if (delta < min)
     345  	min = delta;
     346        if (delta > max)
     347  	max = delta;
     348      }
     349    /* compute average time */
     350    avg = avg / NUM_ITER;
     351  
     352    /* pretty simple, let's see how it works */
     353    if (max < 6 * avg)
     354      max = 6 * avg;
     355    /* round up to the nearest microsecond */
     356    ret = (int) ((max + 999) / 1000);
     357    return ret;
     358  }
     359  
     360  static int
     361  init_pthread_mutex_lock (void *dlflag)
     362  {
     363    __real_pthread_mutex_lock_2_17 = dlvsym (dlflag, "pthread_mutex_lock", "GLIBC_2.17");
     364    __real_pthread_mutex_lock_2_2_5 = dlvsym (dlflag, "pthread_mutex_lock", "GLIBC_2.2.5");
     365    __real_pthread_mutex_lock_2_0 = dlvsym (dlflag, "pthread_mutex_lock", "GLIBC_2.0");
     366    if (__real_pthread_mutex_lock_2_17)
     367      __real_pthread_mutex_lock = __real_pthread_mutex_lock_2_17;
     368    else if (__real_pthread_mutex_lock_2_2_5)
     369      __real_pthread_mutex_lock = __real_pthread_mutex_lock_2_2_5;
     370    else if (__real_pthread_mutex_lock_2_0)
     371      __real_pthread_mutex_lock = __real_pthread_mutex_lock_2_0;
     372    else
     373      __real_pthread_mutex_lock = dlsym (dlflag, "pthread_mutex_lock");
     374    return __real_pthread_mutex_lock ? 1 : 0;
     375  }
     376  
     377  static int
     378  init_thread_intf ()
     379  {
     380    void *dlflag = RTLD_NEXT;
     381    int err = 0;
     382    /* if we detect recursion/reentrance, SEGV so we can get a stack */
     383    init_thread_intf_started++;
     384    if (!init_thread_intf_finished && init_thread_intf_started >= 3)
     385      {
     386        /* pull the plug if recursion occurs... */
     387        abort ();
     388      }
     389    /* lookup fprint to print fatal error message */
     390    void *ptr = dlsym (RTLD_DEFAULT, "fprintf");
     391    if (ptr)
     392      {
     393        __real_fprintf = (void *) ptr;
     394      }
     395    else
     396      {
     397        abort ();
     398      }
     399  
     400    /* find the __collector_jprofile_enable_synctrace routine in jprofile module */
     401    ptr = dlsym (RTLD_DEFAULT, "__collector_jprofile_enable_synctrace");
     402    if (ptr)
     403      __real___collector_jprofile_enable_synctrace = (void *) ptr;
     404    else
     405      {
     406  #if defined(GPROFNG_JAVA_PROFILING)
     407        CALL_REAL (fprintf)(stderr, "synctrace_init COL_ERROR_SYNCINIT __collector_jprofile_enable_synctrace\n");
     408        err = COL_ERROR_SYNCINIT;
     409  #endif
     410        sync_java = 0;
     411      }
     412  
     413    dlflag = RTLD_NEXT;
     414    if (init_pthread_mutex_lock (dlflag) == 0)
     415      {
     416        /* We are probably dlopened after libthread/libc,
     417         * try to search in the previously loaded objects
     418         */
     419        dlflag = RTLD_DEFAULT;
     420        if (init_pthread_mutex_lock (dlflag) == 0)
     421  	{
     422  	  CALL_REAL (fprintf)(stderr, "synctrace_init COL_ERROR_SYNCINIT pthread_mutex_lock\n");
     423  	  err = COL_ERROR_SYNCINIT;
     424  	}
     425      }
     426  
     427    if ((ptr = dlvsym (dlflag, "pthread_mutex_unlock", "GLIBC_2.17")) != NULL)
     428      __real_pthread_mutex_unlock = ptr;
     429    else if ((ptr = dlvsym (dlflag, "pthread_mutex_unlock", "GLIBC_2.2.5")) != NULL)
     430      __real_pthread_mutex_unlock = ptr;
     431    else if ((ptr = dlvsym (dlflag, "pthread_mutex_unlock", "GLIBC_2.0")) != NULL)
     432      __real_pthread_mutex_unlock = ptr;
     433    else
     434      __real_pthread_mutex_unlock = dlsym (dlflag, "pthread_mutex_unlock");
     435    if (__real_pthread_mutex_unlock == NULL)
     436      {
     437        CALL_REAL (fprintf)(stderr, "synctrace_init COL_ERROR_SYNCINIT pthread_mutex_unlock\n");
     438        err = COL_ERROR_SYNCINIT;
     439      }
     440  
     441    __real_pthread_cond_wait_2_17 = dlvsym (dlflag, "pthread_cond_wait", "GLIBC_2.17");
     442    __real_pthread_cond_wait_2_3_2 = dlvsym (dlflag, "pthread_cond_wait", "GLIBC_2.3.2");
     443    __real_pthread_cond_wait_2_2_5 = dlvsym (dlflag, "pthread_cond_wait", "GLIBC_2.2.5");
     444    __real_pthread_cond_wait_2_0 = dlvsym (dlflag, "pthread_cond_wait", "GLIBC_2.0");
     445    if (__real_pthread_cond_wait_2_17)
     446      __real_pthread_cond_wait = __real_pthread_cond_wait_2_17;
     447    else if (__real_pthread_cond_wait_2_3_2)
     448      __real_pthread_cond_wait = __real_pthread_cond_wait_2_3_2;
     449    else if (__real_pthread_cond_wait_2_2_5)
     450      __real_pthread_cond_wait = __real_pthread_cond_wait_2_2_5;
     451    else if (__real_pthread_cond_wait_2_0)
     452      __real_pthread_cond_wait = __real_pthread_cond_wait_2_0;
     453    else
     454      __real_pthread_cond_wait = dlsym (dlflag, "pthread_cond_wait");
     455    if (__real_pthread_cond_wait == NULL)
     456      {
     457        CALL_REAL (fprintf)(stderr, "synctrace_init COL_ERROR_SYNCINIT pthread_cond_wait\n");
     458        err = COL_ERROR_SYNCINIT;
     459      }
     460  
     461    __real_pthread_cond_timedwait_2_17 = dlvsym (dlflag, "pthread_cond_timedwait", "GLIBC_2.17");
     462    __real_pthread_cond_timedwait_2_3_2 = dlvsym (dlflag, "pthread_cond_timedwait", "GLIBC_2.3.2");
     463    __real_pthread_cond_timedwait_2_2_5 = dlvsym (dlflag, "pthread_cond_timedwait", "GLIBC_2.2.5");
     464    __real_pthread_cond_timedwait_2_0 = dlvsym (dlflag, "pthread_cond_timedwait", "GLIBC_2.0");
     465    if (__real_pthread_cond_timedwait_2_17)
     466      __real_pthread_cond_timedwait = __real_pthread_cond_timedwait_2_17;
     467    else if (__real_pthread_cond_timedwait_2_3_2)
     468      __real_pthread_cond_timedwait = __real_pthread_cond_timedwait_2_3_2;
     469    else if (__real_pthread_cond_timedwait_2_2_5)
     470      __real_pthread_cond_timedwait = __real_pthread_cond_timedwait_2_2_5;
     471    else if (__real_pthread_cond_timedwait_2_0)
     472      __real_pthread_cond_timedwait = __real_pthread_cond_timedwait_2_0;
     473    else
     474      __real_pthread_cond_timedwait = dlsym (dlflag, "pthread_cond_timedwait");
     475    if (__real_pthread_cond_timedwait == NULL)
     476      {
     477        CALL_REAL (fprintf)(stderr, "synctrace_init COL_ERROR_SYNCINIT pthread_cond_timedwait\n");
     478        err = COL_ERROR_SYNCINIT;
     479      }
     480  
     481    __real_pthread_join_2_34 = dlvsym (dlflag, "pthread_join", "GLIBC_2.34");
     482    __real_pthread_join_2_17 = dlvsym (dlflag, "pthread_join", "GLIBC_2.17");
     483    __real_pthread_join_2_2_5 = dlvsym (dlflag, "pthread_join", "GLIBC_2.2.5");
     484    __real_pthread_join_2_0 = dlvsym (dlflag, "pthread_join", "GLIBC_2.0");
     485    if (__real_pthread_join_2_34)
     486      __real_pthread_join = __real_pthread_join_2_34;
     487    else if (__real_pthread_join_2_17)
     488      __real_pthread_join = __real_pthread_join_2_17;
     489    else if (__real_pthread_join_2_2_5)
     490      __real_pthread_join = __real_pthread_join_2_2_5;
     491    else if (__real_pthread_join_2_0)
     492      __real_pthread_join = __real_pthread_join_2_0;
     493    else
     494      __real_pthread_join = dlsym (dlflag, "pthread_join");
     495    if (__real_pthread_join == NULL)
     496      {
     497        CALL_REAL (fprintf)(stderr, "synctrace_init COL_ERROR_SYNCINIT pthread_join\n");
     498        err = COL_ERROR_SYNCINIT;
     499      }
     500  
     501    __real_sem_wait_2_34 = dlvsym (dlflag, "sem_wait", "GLIBC_2.34");
     502    __real_sem_wait_2_17 = dlvsym (dlflag, "sem_wait", "GLIBC_2.17");
     503    __real_sem_wait_2_2_5 = dlvsym (dlflag, "sem_wait", "GLIBC_2.2.5");
     504    __real_sem_wait_2_1 = dlvsym (dlflag, "sem_wait", "GLIBC_2.1");
     505    __real_sem_wait_2_0 = dlvsym (dlflag, "sem_wait", "GLIBC_2.0");
     506    if (__real_sem_wait_2_34)
     507      __real_sem_wait = __real_sem_wait_2_34;
     508    else if (__real_sem_wait_2_17)
     509      __real_sem_wait = __real_sem_wait_2_17;
     510    else if (__real_sem_wait_2_2_5)
     511      __real_sem_wait = __real_sem_wait_2_2_5;
     512    else if (__real_sem_wait_2_1)
     513      __real_sem_wait = __real_sem_wait_2_1;
     514    else if (__real_sem_wait_2_0)
     515      __real_sem_wait = __real_sem_wait_2_0;
     516    else
     517      __real_sem_wait = dlsym (dlflag, "sem_wait");
     518    if (__real_sem_wait == NULL)
     519      {
     520        CALL_REAL (fprintf)(stderr, "synctrace_init COL_ERROR_SYNCINIT sem_wait\n");
     521        err = COL_ERROR_SYNCINIT;
     522      }
     523  
     524  
     525    ptr = dlsym (dlflag, "strtol");
     526    if (ptr)
     527      __real_strtol = (void *) ptr;
     528    else
     529      {
     530        CALL_REAL (fprintf)(stderr, "synctrace_init COL_ERROR_SYNCINIT strtol\n");
     531        err = COL_ERROR_SYNCINIT;
     532      }
     533    init_thread_intf_finished++;
     534    TprintfT (0, "synctrace init_thread_intf complete\n");
     535    return err;
     536  }
     537  
     538  /* These next two routines are used from jprofile to record Java synctrace data */
     539  void
     540  __collector_jsync_begin ()
     541  {
     542    int *guard;
     543    if (CHCK_JREENTRANCE (guard))
     544      {
     545        Tprintf (DBG_LT1, "__collector_jsync_begin: skipped\n");
     546        return;
     547      }
     548    Tprintf (DBG_LT1, "__collector_jsync_begin: start event\n");
     549    PUSH_REENTRANCE (guard);
     550  }
     551  
     552  void
     553  __collector_jsync_end (hrtime_t reqt, void *object)
     554  {
     555    int *guard;
     556    if (RECHCK_JREENTRANCE (guard))
     557      {
     558        Tprintf (DBG_LT1, "__collector_jsync_end: skipped\n");
     559        return;
     560      }
     561    hrtime_t grnt = gethrtime ();
     562    if (grnt - reqt >= sync_threshold)
     563      {
     564        Sync_packet spacket;
     565        collector_memset (&spacket, 0, sizeof (Sync_packet));
     566        spacket.comm.tsize = sizeof (Sync_packet);
     567        spacket.comm.tstamp = grnt;
     568        spacket.requested = reqt;
     569        spacket.objp = (intptr_t) object;
     570        spacket.comm.frinfo = collector_interface->getFrameInfo (sync_hndl,
     571  			spacket.comm.tstamp, FRINFO_FROM_STACK_ARG, &spacket);
     572        collector_interface->writeDataRecord (sync_hndl, (Common_packet*) &spacket);
     573      }
     574    Tprintf (DBG_LT1, "__collector_jsync_begin: end event\n");
     575    POP_REENTRANCE (guard);
     576  }
     577  /*-------------------------------------------------------- pthread_mutex_lock */
     578  static int
     579  gprofng_pthread_mutex_lock (int (real_func) (pthread_mutex_t *),
     580  			    pthread_mutex_t *mp)
     581  {
     582    int *guard;
     583    if (CHCK_NREENTRANCE (guard))
     584      return (real_func) (mp);
     585    PUSH_REENTRANCE (guard);
     586    hrtime_t reqt = gethrtime ();
     587    int ret = (real_func) (mp);
     588    if (RECHCK_NREENTRANCE (guard))
     589      {
     590        POP_REENTRANCE (guard);
     591        return ret;
     592      }
     593    hrtime_t grnt = gethrtime ();
     594    if (grnt - reqt >= sync_threshold)
     595      {
     596        Sync_packet spacket;
     597        collector_memset (&spacket, 0, sizeof (Sync_packet));
     598        spacket.comm.tsize = sizeof (Sync_packet);
     599        spacket.comm.tstamp = grnt;
     600        spacket.requested = reqt;
     601        spacket.objp = (intptr_t) mp;
     602        spacket.comm.frinfo = collector_interface->getFrameInfo (sync_hndl,
     603  			spacket.comm.tstamp, FRINFO_FROM_STACK_ARG, &spacket);
     604        collector_interface->writeDataRecord (sync_hndl, (Common_packet*) &spacket);
     605      }
     606    POP_REENTRANCE (guard);
     607    return ret;
     608  }
     609  
     610  #define DCL_PTHREAD_MUTEX_LOCK(dcl_f) \
     611    int dcl_f (pthread_mutex_t *mp) \
     612    { \
     613      if (__real_pthread_mutex_lock == NULL) \
     614        init_thread_intf (); \
     615      return gprofng_pthread_mutex_lock (__real_pthread_mutex_lock, mp); \
     616    }
     617  
     618  DCL_FUNC_VER (DCL_PTHREAD_MUTEX_LOCK, pthread_mutex_lock_2_17, pthread_mutex_lock@GLIBC_2.17)
     619  DCL_FUNC_VER (DCL_PTHREAD_MUTEX_LOCK, pthread_mutex_lock_2_2_5, pthread_mutex_lock@GLIBC_2.2.5)
     620  DCL_FUNC_VER (DCL_PTHREAD_MUTEX_LOCK, pthread_mutex_lock_2_0, pthread_mutex_lock@GLIBC_2.0)
     621  DCL_PTHREAD_MUTEX_LOCK (pthread_mutex_lock)
     622  
     623  /*------------------------------------------------------------- pthread_cond_wait */
     624  static int
     625  gprofng_pthread_cond_wait (int(real_func) (pthread_cond_t *, pthread_mutex_t *),
     626  			   pthread_cond_t *cond, pthread_mutex_t *mutex)
     627  {
     628    int *guard;
     629    if (CHCK_NREENTRANCE (guard))
     630      return (real_func) (cond, mutex);
     631    PUSH_REENTRANCE (guard);
     632    hrtime_t reqt = gethrtime ();
     633    int ret = -1;
     634    ret = (real_func) (cond, mutex);
     635    if (RECHCK_NREENTRANCE (guard))
     636      {
     637        POP_REENTRANCE (guard);
     638        return ret;
     639      }
     640    hrtime_t grnt = gethrtime ();
     641    if (grnt - reqt >= sync_threshold)
     642      {
     643        Sync_packet spacket;
     644        collector_memset (&spacket, 0, sizeof (Sync_packet));
     645        spacket.comm.tsize = sizeof (Sync_packet);
     646        spacket.comm.tstamp = grnt;
     647        spacket.requested = reqt;
     648        spacket.objp = (intptr_t) mutex;
     649        spacket.comm.frinfo = collector_interface->getFrameInfo (sync_hndl,
     650  			spacket.comm.tstamp, FRINFO_FROM_STACK_ARG, &spacket);
     651        collector_interface->writeDataRecord (sync_hndl, (Common_packet*) &spacket);
     652      }
     653    POP_REENTRANCE (guard);
     654    return ret;
     655  }
     656  
     657  #define DCL_PTHREAD_COND_WAIT(dcl_f) \
     658    int dcl_f (pthread_cond_t *cond, pthread_mutex_t *mutex) \
     659    { \
     660      if (__real_pthread_cond_wait == NULL) \
     661        init_thread_intf (); \
     662      return gprofng_pthread_cond_wait (__real_pthread_cond_wait, cond, mutex); \
     663    }
     664  
     665  DCL_FUNC_VER (DCL_PTHREAD_COND_WAIT, pthread_cond_wait_2_17, pthread_cond_wait@GLIBC_2.17)
     666  DCL_FUNC_VER (DCL_PTHREAD_COND_WAIT, pthread_cond_wait_2_3_2, pthread_cond_wait@GLIBC_2.3.2)
     667  DCL_FUNC_VER (DCL_PTHREAD_COND_WAIT, pthread_cond_wait_2_2_5, pthread_cond_wait@GLIBC_2.2.5)
     668  DCL_FUNC_VER (DCL_PTHREAD_COND_WAIT, pthread_cond_wait_2_0, pthread_cond_wait@GLIBC_2.0)
     669  DCL_PTHREAD_COND_WAIT (pthread_cond_wait)
     670  
     671  /*---------------------------------------------------- pthread_cond_timedwait */
     672  static int
     673  gprofng_pthread_cond_timedwait (int(real_func) (pthread_cond_t *,
     674  				    pthread_mutex_t*, const struct timespec *),
     675  				pthread_cond_t *cond, pthread_mutex_t *mutex,
     676  				const struct timespec *abstime)
     677  {
     678    int *guard;
     679    if (CHCK_NREENTRANCE (guard))
     680      return (real_func) (cond, mutex, abstime);
     681    PUSH_REENTRANCE (guard);
     682    hrtime_t reqt = gethrtime ();
     683    int ret = -1;
     684    ret = (real_func) (cond, mutex, abstime);
     685    if (RECHCK_NREENTRANCE (guard))
     686      {
     687        POP_REENTRANCE (guard);
     688        return ret;
     689      }
     690    hrtime_t grnt = gethrtime ();
     691    if (grnt - reqt >= sync_threshold)
     692      {
     693        Sync_packet spacket;
     694        collector_memset (&spacket, 0, sizeof (Sync_packet));
     695        spacket.comm.tsize = sizeof (Sync_packet);
     696        spacket.comm.tstamp = grnt;
     697        spacket.requested = reqt;
     698        spacket.objp = (intptr_t) mutex;
     699        spacket.comm.frinfo = collector_interface->getFrameInfo (sync_hndl,
     700  			spacket.comm.tstamp, FRINFO_FROM_STACK_ARG, &spacket);
     701        collector_interface->writeDataRecord (sync_hndl, (Common_packet*) &spacket);
     702      }
     703    POP_REENTRANCE (guard);
     704    return ret;
     705  }
     706  
     707  #define DCL_PTHREAD_COND_TIMEDWAIT(dcl_f) \
     708    int dcl_f (pthread_cond_t *cond, pthread_mutex_t *mutex, \
     709  	     const struct timespec *abstime) \
     710    { \
     711      if (__real_pthread_cond_timedwait == NULL) \
     712        init_thread_intf (); \
     713      return gprofng_pthread_cond_timedwait (__real_pthread_cond_timedwait, cond, mutex, abstime); \
     714    }
     715  
     716  DCL_FUNC_VER (DCL_PTHREAD_COND_TIMEDWAIT, pthread_cond_timedwait_2_17, pthread_cond_timedwait@GLIBC_2.17)
     717  DCL_FUNC_VER (DCL_PTHREAD_COND_TIMEDWAIT, pthread_cond_timedwait_2_3_2, pthread_cond_timedwait@GLIBC_2.3.2)
     718  DCL_FUNC_VER (DCL_PTHREAD_COND_TIMEDWAIT, pthread_cond_timedwait_2_2_5, pthread_cond_timedwait@GLIBC_2.2.5)
     719  DCL_FUNC_VER (DCL_PTHREAD_COND_TIMEDWAIT, pthread_cond_timedwait_2_0, pthread_cond_timedwait@GLIBC_2.0)
     720  DCL_PTHREAD_COND_TIMEDWAIT (pthread_cond_timedwait)
     721  
     722  
     723  /*------------------------------------------------------------- pthread_join */
     724  static int
     725  gprofng_pthread_join (int(real_func) (pthread_t, void **),
     726  		      pthread_t target_thread, void **status)
     727  {
     728    int *guard;
     729    if (CHCK_NREENTRANCE (guard))
     730      return real_func (target_thread, status);
     731    PUSH_REENTRANCE (guard);
     732    hrtime_t reqt = gethrtime ();
     733    int ret = real_func(target_thread, status);
     734    if (RECHCK_NREENTRANCE (guard))
     735      {
     736        POP_REENTRANCE (guard);
     737        return ret;
     738      }
     739    hrtime_t grnt = gethrtime ();
     740    if (grnt - reqt >= sync_threshold)
     741      {
     742        Sync_packet spacket;
     743        collector_memset (&spacket, 0, sizeof (Sync_packet));
     744        spacket.comm.tsize = sizeof (Sync_packet);
     745        spacket.comm.tstamp = grnt;
     746        spacket.requested = reqt;
     747        spacket.objp = (Vaddr_type) target_thread;
     748        spacket.comm.frinfo = collector_interface->getFrameInfo (sync_hndl,
     749  			spacket.comm.tstamp, FRINFO_FROM_STACK_ARG, &spacket);
     750        collector_interface->writeDataRecord (sync_hndl, (Common_packet*) &spacket);
     751      }
     752    POP_REENTRANCE (guard);
     753    return ret;
     754  }
     755  
     756  #define DCL_PTHREAD_JOIN(dcl_f) \
     757    int dcl_f (pthread_t target_thread, void **status) \
     758    { \
     759      if (__real_pthread_join == NULL) \
     760        init_thread_intf (); \
     761      return gprofng_pthread_join (__real_pthread_join, target_thread, status); \
     762    }
     763  
     764  DCL_FUNC_VER (DCL_PTHREAD_JOIN, pthread_join_2_34, pthread_join@GLIBC_2.34)
     765  DCL_FUNC_VER (DCL_PTHREAD_JOIN, pthread_join_2_17, pthread_join@GLIBC_2.17)
     766  DCL_FUNC_VER (DCL_PTHREAD_JOIN, pthread_join_2_2_5, pthread_join@GLIBC_2.2.5)
     767  DCL_FUNC_VER (DCL_PTHREAD_JOIN, pthread_join_2_0, pthread_join@GLIBC_2.0)
     768  DCL_PTHREAD_JOIN (pthread_join)
     769  
     770  /*------------------------------------------------------------- sem_wait */
     771  static int
     772  gprofng_sem_wait (int (real_func) (sem_t *), sem_t *sp)
     773  {
     774    int *guard;
     775    if (CHCK_NREENTRANCE (guard))
     776      return real_func (sp);
     777    PUSH_REENTRANCE (guard);
     778    hrtime_t reqt = gethrtime ();
     779    int ret = -1;
     780    ret = real_func (sp);
     781    if (RECHCK_NREENTRANCE (guard))
     782      {
     783        POP_REENTRANCE (guard);
     784        return ret;
     785      }
     786    hrtime_t grnt = gethrtime ();
     787    if (grnt - reqt >= sync_threshold)
     788      {
     789        Sync_packet spacket;
     790        collector_memset (&spacket, 0, sizeof (Sync_packet));
     791        spacket.comm.tsize = sizeof (Sync_packet);
     792        spacket.comm.tstamp = grnt;
     793        spacket.requested = reqt;
     794        spacket.objp = (intptr_t) sp;
     795        spacket.comm.frinfo = collector_interface->getFrameInfo (sync_hndl,
     796  			spacket.comm.tstamp, FRINFO_FROM_STACK_ARG, &spacket);
     797        collector_interface->writeDataRecord (sync_hndl, (Common_packet*) &spacket);
     798      }
     799    POP_REENTRANCE (guard);
     800    return ret;
     801  }
     802  
     803  #define DCL_SEM_WAIT(dcl_f) \
     804    int dcl_f (sem_t *sp) \
     805    { \
     806      if (__real_sem_wait == NULL) \
     807        init_thread_intf (); \
     808      return gprofng_sem_wait (__real_sem_wait, sp); \
     809    }
     810  
     811  DCL_FUNC_VER (DCL_SEM_WAIT, sem_wait_2_34, sem_wait@GLIBC_2.34)
     812  DCL_FUNC_VER (DCL_SEM_WAIT, sem_wait_2_17, sem_wait@GLIBC_2.17)
     813  DCL_FUNC_VER (DCL_SEM_WAIT, sem_wait_2_2_5, sem_wait@GLIBC_2.2.5)
     814  DCL_FUNC_VER (DCL_SEM_WAIT, sem_wait_2_0, sem_wait@GLIBC_2.0)
     815  DCL_FUNC_VER (DCL_SEM_WAIT, sem_wait_2_1, sem_wait@GLIBC_2.1)
     816  DCL_SEM_WAIT (sem_wait)