How can I count the times a particular call chain occurs?

I have a probe that calls ap_LogTrackback in several different places. I'm trying to figure out what it all means.

I think what I'd prefer to see is the number of times that a particular call chain is exercised. What's the best way to achieve this? Are there any predefined probes or utility functions I could call, or should I try storing the call chain myself somehow in order to count what I want?

This probe, tentatively called callers.apc, will do pretty much what you require.  Edit it to name the routines you want (see example calls to AddFunction) and recompile it using this command: 

   $ apc callers.apc

Here is callers.apc:

// Demonstrate capturing the callers and merging
// the stack traces for later display.
// Note that getting the stack trace on Solaris
// is very slow due to the need to flush the
// CPU's register windows, so don't go overboard!

// *** Note that no attempt has been made to make
//     this thread-safe. 
// *** 

// Change this parameter to increase the maximum
// levels we will capture.  It's very unlikely this
// default will be too small but you might want to
// decrease it to a much smaller value to speed
// things up a little.
//
#define MAX_LEVELS 99

// We store the information in an "upside down"
// tree hierarchy. The top level tree holds a list
// of addresses, counts and sub-trees. The count is
// only stored at the "top" of a particular tree.
//
typedef struct
{
   ap_AddressT   Address;
   ap_RbTreePtrT SubTree;
   int           Count;
} AddressEntryT, *AddressEntryPtrT;

// This is the top-level tree (a red-black tree):
DECLARE_RB_TREE (AddressTree);

// This is the list of routines to which we will
// apply the probe:
typedef struct _FunctionList_
{
   ap_FunctionIdT        FunctionId;
   struct _FunctionList_ *Next;
} FunctionListT, *FunctionListPtrT;

static FunctionListPtrT FunctionList = NULL;

// Adds a function to the list and instruments it:
static void AddFunction (ap_NameT FunctionName,
                         ap_NameT Filename,
                         ap_NameT ModuleName)
{
   ap_ModuleIdT     ModuleId;
   ap_FunctionIdT   FunctionId;
   FunctionListPtrT NewEntry;
   

   if (ap_IsNoName (ModuleName))
   {
     ModuleId = ap_ApplicationModuleId ();
   }
   else
   {
     ModuleId = ap_ModuleNameToId (ModuleName);
     if (ap_IsNoModuleId (ModuleId))
     {
       ap_Error
         (ap_WarningSev,
          "Cannot find module %s "
          "for function %s\n",
          ModuleName,
          FunctionName);
       return;
     }
   }
   

   FunctionId = ap_SymbolToFunction
     (ap_SymbolNameToId (ModuleId,
                         FunctionName,
                         Filename,
                         ap_FunctionSymbol));
   if (ap_IsNoFunctionId (FunctionId))
   {
     ap_Error
       (ap_WarningSev,
        "Cannot find function %s\n",
        FunctionName);
     return;
   }
   

   // Try to instrument it:
   if (ap_InstrumentFunction (FunctionId) == 
       ap_NoProbeVectorIndex)
   {
     ap_Error
       (ap_WarningSev,
        "Cannot instrument function %s\n"
        "    Use -v on the command line"
        " for further information.");
     return;
   }
   

   // Create a new entry for the given function
   // in the list:
   NewEntry = ap_Malloc (sizeof (FunctionListT));
   NewEntry->FunctionId = FunctionId;
   NewEntry->Next       = FunctionList;
   FunctionList         = NewEntry;
} /* AddFunction */

// Compares two addresses in the list:
static int CompareAddresses
           (void *L, void *R, void *EP)
{
   AddressEntryPtrT Left;
   AddressEntryPtrT Right;
   

   Left  = (AddressEntryPtrT) L;
   Right = (AddressEntryPtrT) R;
   

   if (Left->Address > Right->Address)
   {
      return 1;
   }
   else if (Left->Address < Right->Address)
   {
      return -1;
   }
   else
   {
      return 0;
   }
} /* CompareAddresses */

// This adds a traceback to the list:
static void AddTraceback
            (ap_AddressT *Buffer,
             int         Depth)
{
   ap_RbTreePtrT    Tree = NULL;
   AddressEntryT    MatchingEntry;
   int              CurrentDepth = 0;
   AddressEntryPtrT CurrentEntry;
   

   while (CurrentDepth < Depth)
   {
     ap_RbtNodePtrT NodePtr;
     

     // Get or create the tree:
     if (Tree == NULL)
     {
       Tree = &AddressTree;
     }
     else
     {
       // Is there a sub-tree?
       if (CurrentEntry->SubTree == NULL)
       {
         // No, so create one:
         CurrentEntry->SubTree = 
           ap_Malloc (sizeof (ap_RbTreeT));
         ap_RbtInitializeTree
           (CurrentEntry->SubTree,
            ap_Malloc,
            ap_Free);
       }
       Tree = CurrentEntry->SubTree;
     }
     

     // Find this address in the tree:
     MatchingEntry.Address = Buffer [CurrentDepth];
     NodePtr = ap_RbtFind
       (Tree,
        (void *) &MatchingEntry,
        CompareAddresses,
        NULL);
     

     if (NodePtr == NULL)
     {
       // No entry so create a new one:
       CurrentEntry
         = ap_Malloc (sizeof (AddressEntryT));
       CurrentEntry->Address
         = Buffer [CurrentDepth];
       CurrentEntry->SubTree = NULL;
       CurrentEntry->Count   = 0;
       

       NodePtr = ap_RbtInsert
         (Tree,
          (void *) CurrentEntry,
          0,
          CompareAddresses,
          NULL);   
     }
     

     CurrentEntry = (AddressEntryPtrT)
       ap_RbtContents (NodePtr);
     

     // Move to the next level:
     CurrentDepth++;
   } /* while */
   

   // Increment the count for the current entry:
   CurrentEntry->Count++;

} /* AddTraceback */

// This is the format routine to print
// a traceback:
static void TracebackFormat
   (ap_AddressT *Buffer,
    int         *Depth,
    int         *Count)
{
   if (*Count > 1)
   {
     printf 
      ("   Traceback captured %d times:\n",
       *Count);
   }
   else
   {
     printf 
      ("   Traceback captured once:\n");
   }
   printf 
    ("  -----------------------------"
     "----------------------------\n");
   ap_PrintTracebackFunc (Buffer, *Depth);
   printf 
    ("  -----------------------------"
     "----------------------------\n");
   printf ("\n");
} /* TracebackFormat */

// This is the type we use for iterating through
// the tree to log the tracebacks:
typedef struct
{
   ap_AddressT Buffer [MAX_LEVELS];
   int         Depth;
} IterateDataT, *IterateDataPtrT;

// This is the iterator function we use to log
// the tracebacks. It recurses down the tree,
// storing a traceback each time it reaches a 
// non-zero count:
static ap_BooleanT DumpAddresses
                   (void *Entry, void *EP)
{
   AddressEntryPtrT AddressEntry;
   IterateDataPtrT  Data;
   

   Data         = (IterateDataPtrT) EP;
   AddressEntry = (AddressEntryPtrT) Entry;
   

   // Store the current depth
   Data->Buffer [Data->Depth]
     = AddressEntry->Address;
   

   // Do we have a count here?
   if (AddressEntry->Count != 0)
   {
     log (Data->Buffer [0 .. Data->Depth],
          Data->Depth + 1,
          AddressEntry->Count)
     with TracebackFormat;
   }
   

   // If there is a subtree, recurse into it:
   if (AddressEntry->SubTree)
   {
     Data->Depth++;
     ap_RbtIterate
       (AddressEntry->SubTree,
        DumpAddresses,
        ap_Forward,
        EP);
     Data->Depth--;
   }
   

   return TRUE;
} /* DumpAddresses */

// ************************************
// Finally, here's the probe itself:   
// ************************************

probe program
{
   probe thread
   {
     // This is the probe that gets applied
     // to each function for which we want
     // to capture traceback information:
     typedef probe
     {
       on_entry
       {
          ap_AddressT Buffer [MAX_LEVELS];
          int         Depth;
          

          // Get the traceback:
          Depth = 
            ap_GetTraceback 
               (ap_CurrentLocation,
                Buffer,
                MAX_LEVELS);
          

          // Store it:
          AddTraceback (Buffer, Depth);
       }
     } TracebackProbeT;
     

     on_entry /* to probe thread */ 
     {
       FunctionListPtrT Current;
       

       // Create a probe instance for all
       // functions we want:
       Current = FunctionList;
       while (Current)
       {
         new TracebackProbeT
             (Current->FunctionId);
         Current = Current->Next;
       }
     }
   } /* probe thread */
   

   on_entry /* to probe program */ 
   {
     ap_RbtInitializeTree 
        (&AddressTree,
         ap_Malloc,
         ap_Free);
     

     // Add the functions you want here.
     // The following examples add
     // main() and printf() in libc.so
     // respectively. Use ap_ExternSymbol
     // for the filename field (the second
     // parameter) if you're sure that
     // the function is extern, ap_NoName if
     // you're not sure and give the
     // name of the file if it's static 
     // (e.g. "myfile.c"):
     // 
     AddFunction ("main()",
                  ap_ExternSymbol,
                  NULL);
     AddFunction ("printf()",
                  ap_ExternSymbol,
                  "libc.so");
   }
   

   on_exit /* from probe program */ 
   {
     IterateDataT Data;
     

     // Log the trees out to APD file:
     Data.Depth = 0;
     ap_RbtIterate 
       (&AddressTree,
        DumpAddresses,
        ap_Forward,
        (void *) &Data);
   }
} /* probe program */