Difference between revisions of "Brcov Predefined Probe"

From OC Systems Wiki!
Jump to: navigation, search
m (Branch Coverage Predefined Probe: brcov.ual)
m (Branch Coverage Report)
Line 297: Line 297:
  
  
Generated: 12 May 2017 14:49:38 -0400
+
Generated: 27 Nov 2017 09:19:05 -0500
  
  
Line 320: Line 320:
  
 
TimeStamp:
 
TimeStamp:
Program  : /home/swn/aprobe/test/brcov/example/coverage_example
+
Program  : /home/swn/aprobe/test/brcov/tnamec/main.exe
 
Host      : centos7
 
Host      : centos7
PID      : 32198
+
PID      : 13805
Start Time: 2017-05-12 14:48:06
+
Start Time: 2017-11-27 09:19:06
End Time  : 2017-05-12 14:48:08
+
End Time  : 2017-11-27 09:19:06
 
 
TimeStamp:
 
Program  : /home/swn/aprobe/test/brcov/example/coverage_example
 
Host      : centos7
 
PID      : 32198
 
Start Time: 2017-05-12 14:48:06
 
End Time  : 2017-05-12 14:48:08
 
 
 
TimeStamp:
 
Program  : /home/swn/aprobe/test/brcov/example/coverage_example
 
Host      : centos7
 
PID      : 13281
 
Start Time: 2017-05-12 14:36:27
 
End Time  : 2017-05-12 14:36:28
 
  
  
Line 350: Line 336:
 
Total    Total  Not    Not        Coverage  Total    Not      Not        Branches Not      Coverage
 
Total    Total  Not    Not        Coverage  Total    Not      Not        Branches Not      Coverage
 
Calls    Lines  Probed  Executed  Percent  Branches  Probed    Executed  Taken    Taken    Percent  Name
 
Calls    Lines  Probed  Executed  Percent  Branches  Probed    Executed  Taken    Taken    Percent  Name
------- ------- -------- --------- --------- --------- --------- ---------- --------- --------- -------- --------- - -
+
------- ------- -------- --------- ========= --------- --------- ---------- --------- --------- ======== --------- - -
     3      16       0        0      100        4         0          0        3         3     100  extern:"DoProcessing()"
+
     2      5       0        0      100        1         0          0        1         1     100  extern:"pkg.p1[1]"
     3       4       0        0      100        0         0          0        0        0       0  extern:"main()"
+
     2       6       0        0      100        2         0          0        1        2      100  extern:"pkg.p2[1]"
 +
    10      19        0         1        94        5         0         0         5        4      100 extern:"pkg.td[1]"
  
 
Note: Negative counts indicate the function/line was not instrumented.
 
Note: Negative counts indicate the function/line was not instrumented.
Line 359: Line 346:
 
  Total    Not    Coverage    Total    Not      Not        Coverage  Total      Not        Not        Branches  Not        Coverage
 
  Total    Not    Coverage    Total    Not      Not        Coverage  Total      Not        Not        Branches  Not        Coverage
 
  Funcs    Called  Percent    Lines    Probed    Executed  Percent    Branches  Probed    Executed  Taken      Taken      Percent  
 
  Funcs    Called  Percent    Lines    Probed    Executed  Percent    Branches  Probed    Executed  Taken      Taken      Percent  
-------- -------- -------- ---------- ---------- ---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------
+
-------- -------- ======== ---------- ---------- ---------- ========== ---------- ---------- ---------- ---------- ---------- ==========
     12      10      16         20         0          0        100         4         0          0          3         3       100
+
      3        0     100         30         0          1        96         8         0          0          7         7       100
  
  
Line 368: Line 355:
 
--------------------------------------------------------------------------
 
--------------------------------------------------------------------------
  
extern:"DoProcessing()" (/home/swn/aprobe/test/brcov/example/coverage_example.c)
+
extern:"pkg.p1[1]" (/home/swn/aprobe/test/brcov/tnamec/pkg.adb)
extern:"main()" (/home/swn/aprobe/test/brcov/example/coverage_example.c)
+
extern:"pkg.p2[1]" (/home/swn/aprobe/test/brcov/tnamec/pkg.adb)
 +
extern:"pkg.td[1]" (/home/swn/aprobe/test/brcov/tnamec/pkg-td.adb)
  
  
 
--------------------------------------------------------------------------
 
--------------------------------------------------------------------------
Files TOC
+
File TOC
 
--------------------------------------------------------------------------
 
--------------------------------------------------------------------------
  
coverage_example.c
+
pkg.ads
 +
pkg.adb
 +
pkg-td.adb
 +
main.adb
  
  
Line 383: Line 374:
 
--------------------------------------------------------------------------
 
--------------------------------------------------------------------------
  
extern:"DoProcessing()" (/home/swn/aprobe/test/brcov/example/coverage_example.c)
 
  
 +
extern:"pkg.p1[1]" (/home/swn/aprobe/test/brcov/tnamec/pkg.adb)
  
Calls:  3
 
  
 +
Calls:  2
 +
-----
 +
 +
Test Names: [ test p1 ]
 +
----------
 +
 +
Lines:
 +
-----
 
           Lines  Probed  Not Probed  Hit  
 
           Lines  Probed  Not Probed  Hit  
 
         ------- -------- ---------- ----------
 
         ------- -------- ---------- ----------
Count:       16       16         0       16
+
Count:        5        5         0         5
 
Percent:              100        0      100
 
Percent:              100        0      100
  
  
 +
Branches:
 +
--------
 
           Branches    Probed      Hit      Taken    NotTaken
 
           Branches    Probed      Hit      Taken    NotTaken
 
         ---------- ---------- ---------- ---------- ----------
 
         ---------- ---------- ---------- ---------- ----------
Count:            4         4         4         3         3
+
Count:            1         1         1         1         1
Percent:                  100        100         75        75
+
Percent:                  100        100       100        100
 +
 
  
 +
extern:"pkg.p2[1]" (/home/swn/aprobe/test/brcov/tnamec/pkg.adb)
  
extern:"main()" (/home/swn/aprobe/test/brcov/example/coverage_example.c)
 
  
 +
Calls:  2
 +
-----
  
Calls: 3
+
Test Names: [ test p1, test p2 ]
 +
----------
  
 +
Lines:
 +
-----
 
           Lines  Probed  Not Probed  Hit  
 
           Lines  Probed  Not Probed  Hit  
 
         ------- -------- ---------- ----------
 
         ------- -------- ---------- ----------
Count:        4         4         0        4
+
Count:        6         6         0        6
 
Percent:              100        0      100
 
Percent:              100        0      100
  
  
 +
Branches:
 +
--------
 
           Branches    Probed      Hit      Taken    NotTaken
 
           Branches    Probed      Hit      Taken    NotTaken
 
         ---------- ---------- ---------- ---------- ----------
 
         ---------- ---------- ---------- ---------- ----------
Count:            0         0         0         0         0
+
Count:            2          2         2         1         2
Percent:                     0          0         0          0
+
Percent:                  100        100        50        100
 +
 
 +
 
 +
extern:"pkg.td[1]" (/home/swn/aprobe/test/brcov/tnamec/pkg-td.adb)
 +
 
 +
 
 +
Calls:  10
 +
-----
 +
 
 +
Test Names: [ test p1, test p2 ]
 +
----------
 +
 
 +
Lines:
 +
-----
 +
          Lines  Probed  Not Probed  Hit
 +
         ------- -------- ---------- ----------
 +
Count:      19        19        0       18
 +
Percent:             100        0       94
 +
 
 +
    Lines Not Executed:
 +
    ------------------
 +
    21 / pkg-td.adb
 +
 
 +
Branches:
 +
--------
 +
          Branches    Probed      Hit      Taken    NotTaken
 +
        ---------- ---------- ---------- ---------- ----------
 +
Count:            5          5          5          5          4
 +
Percent:                  100        100        100        80
 +
 
 +
 
 +
 
 +
--------------------------------------------------------------------------
 +
Function Coverage Details
 +
--------------------------------------------------------------------------
 +
 
 +
 
 +
extern:"pkg.p1[1]" (/home/swn/aprobe/test/brcov/tnamec/pkg.adb)
 +
 
 +
Lines:
 +
-----
 +
      Line    I    Count      File  [ Test Names ]
 +
    ---------- -  -----------  ---------------------
 +
            11 Y :          2 : pkg.adb [ test p1 ]
 +
            13 Y :          2 : pkg.adb [ test p1 ]
 +
            14 Y :          1 : pkg.adb [ test p1 ]
 +
            16 Y :          1 : pkg.adb [ test p1 ]
 +
            18 Y :          2 : pkg.adb [ test p1 ]
 +
 
 +
Branches:
 +
--------
 +
    Offset  I      Hit      Taken    NotTaken    Line/File -> Line/File  [ Test Names ]
 +
    -------- -  ---------- ---------- ----------  --------------------------------------
 +
    0x00000b Y :          2          1          1 : 13 / pkg.adb -> 16 / pkg.adb [ test p1 ]
 +
 
 +
Edges:
 +
------
 +
      Count    Offset/Line/File -> Offset/Line/File [ Test Names ]
 +
    ----------  ---------------------------------------------------
 +
 
 +
extern:"pkg.p2[1]" (/home/swn/aprobe/test/brcov/tnamec/pkg.adb)
 +
 
 +
Lines:
 +
-----
 +
      Line    I    Count      File  [ Test Names ]
 +
    ---------- -  -----------  ---------------------
 +
            20 Y :          2 : pkg.adb [ test p2 ]
 +
            22 Y :          2 : pkg.adb [ test p2 ]
 +
            22 Y :          1 : pkg.adb [ test p2 ]
 +
            23 Y :          1 : pkg.adb [ test p2 ]
 +
            25 Y :          1 : pkg.adb [ test p2 ]
 +
            27 Y :          2 : pkg.adb [ test p2 ]
 +
 
 +
Branches:
 +
--------
 +
    Offset  I      Hit      Taken    NotTaken    Line/File -> Line/File  [ Test Names ]
 +
    -------- -  ---------- ---------- ----------  --------------------------------------
 +
    0x000013 Y :          2          1          1 : 22 / pkg.adb -> 25 / pkg.adb [ test p2 ]
 +
    0x000021 Y :         1         0          1 : 22 / pkg.adb -> 25 / pkg.adb [ test p2 ]
 +
 
 +
Edges:
 +
------
 +
      Count    Offset/Line/File -> Offset/Line/File [ Test Names ]
 +
    ----------  ---------------------------------------------------
  
 +
extern:"pkg.td[1]" (/home/swn/aprobe/test/brcov/tnamec/pkg-td.adb)
 +
 +
Lines:
 +
-----
 +
      Line    I    Count      File  [ Test Names ]
 +
    ---------- -  -----------  ---------------------
 +
            4 Y :        10 : pkg-td.adb [ test p1, test p2 ]
 +
            5 Y :        10 : pkg-td.adb [ test p1, test p2 ]
 +
            6 Y :        10 : pkg-td.adb [ test p1, test p2 ]
 +
            7 Y :        10 : pkg-td.adb [ test p1, test p2 ]
 +
            8 Y :        10 : pkg-td.adb [ test p1, test p2 ]
 +
            9 Y :        10 : pkg-td.adb [ test p1, test p2 ]
 +
            11 Y :        10 : pkg-td.adb [ test p1, test p2 ]
 +
            21 Y :          0 : pkg-td.adb [  ]
 +
            11 Y :          8 : pkg-td.adb [ test p1, test p2 ]
 +
            21 Y :          6 : pkg-td.adb [ test p2 ]
 +
            13 Y :          1 : pkg-td.adb [  ]
 +
            22 Y :          1 : pkg-td.adb [ test p1 ]
 +
            15 Y :          1 : pkg-td.adb [ test p1 ]
 +
            22 Y :          1 : pkg-td.adb [ test p1 ]
 +
            17 Y :          1 : pkg-td.adb [ test p1 ]
 +
            22 Y :          1 : pkg-td.adb [ test p2 ]
 +
            19 Y :          1 : pkg-td.adb [ test p2 ]
 +
            22 Y :          1 : pkg-td.adb [ test p2 ]
 +
            23 Y :        10 : pkg-td.adb [ test p1, test p2 ]
 +
 +
Branches:
 +
--------
 +
    Offset  I      Hit      Taken    NotTaken    Line/File -> Line/File  [ Test Names ]
 +
    -------- -  ---------- ---------- ----------  --------------------------------------
 +
    0x000037 Y :        10          1          9 : 11 / pkg-td.adb -> 15 / pkg-td.adb [ test p2, test p1 ]
 +
    0x00003c Y :          9          8          1 : 11 / pkg-td.adb -> 11 / pkg-td.adb [ test p2, test p1 ]
 +
    0x000041 Y :          1          1          0 : 11 / pkg-td.adb -> 13 / pkg-td.adb [  ]
 +
    0x000048 Y :          8          1          7 : 11 / pkg-td.adb -> 17 / pkg-td.adb [ test p2, test p1 ]
 +
    0x00004d Y :          7          1          6 : 11 / pkg-td.adb -> 19 / pkg-td.adb [ test p2 ]
 +
 +
Edges:
 +
------
 +
      Count    Offset/Line/File -> Offset/Line/File [ Test Names ]
 +
    ----------  ---------------------------------------------------
  
  
Line 422: Line 553:
 
--------------------------------------------------------------------------
 
--------------------------------------------------------------------------
  
coverage_example.c:
+
pkg.ads:
 +
 
  
   Line    Br O/I/T/N    Line Cnt  Source
+
+  - This line/branch/edge was not executed.
 +
.  - This internal branch/edge was not executed but may not be significant.
 +
*  - This line/branch/edge was not probed.
 +
 
 +
   Line    Edge/Branch  Line Cnt  Source
 
   ------  -----------  ---------  ------- - -
 
   ------  -----------  ---------  ------- - -
       1 :              :        : /* This is a simple example to demonstrate using coverage.ual */
+
       1 :              :        :  
       2 :              :        : #include <stdio.h>
+
      2 :              :        : package pkg is
       3 :              :        : #include <stdlib.h>
+
      3 :              :        :
 +
      4 :              :        :  procedure p1(I : in out integer);
 +
      5 :              :        :  procedure p2(I : in out float);
 +
      6 :              :        :
 +
      7 :              :        :  procedure td(I : in integer);
 +
      8 :              :        :
 +
      9 :              :        : end pkg;
 +
      10 :              :        :
 +
 
 +
pkg.adb:
 +
 
 +
 
 +
+  - This line/branch/edge was not executed.
 +
.  - This internal branch/edge was not executed but may not be significant.
 +
* - This line/branch/edge was not probed.
 +
 
 +
  Line    Edge/Branch  Line Cnt  Source
 +
  ------  -----------  ---------  ------- - -
 +
      1 :              :        :
 +
      2 :              :        : with Text_Io;
 +
      3 :              :        :
 +
      4 :              :        : package body pkg is
 +
      5 :              :        :
 +
      6 :              :        :  function Ident(i : integer) return integer is
 +
      7 :              :        :  begin
 +
      8 :              :        :      return i;
 +
      9 :              :        :  end;
 +
      10 :              :        :
 +
      11 :              :      2 :  procedure p1(I : in out integer) is
 +
      12 :              :        :  begin
 +
      13 :              :      2 :    if I = 0 then
 +
          B      1x -> 16
 +
      14 :              :      1 :      Text_Io.Put_Line("p1, 0!");
 +
      15 :              :        :    else
 +
      16 :              :      1 :      Text_Io.Put_Line("p1, not zero");
 +
      17 :              :        :    end if;
 +
      18 :              :      2 :  end p1;
 +
      19 :              :        :
 +
      20 :              :      2 :  procedure p2(I : in out float) is
 +
      21 :              :        :  begin
 +
      22 :              :      2 :    if I < 0.000001 and then I > -0.000001 then
 +
          B      1x -> 25
 +
+        B      0x -> 25
 +
      23 :              :      1 :      Text_Io.Put_Line("p2, 0!");
 +
      24 :              :        :    else
 +
      25 :              :      1 :      Text_Io.Put_Line("p2, not zero");
 +
      26 :              :        :    end if;
 +
      27 :              :      2 :  end p2;
 +
      28 :              :        :
 +
      29 :              :        :  procedure td(i : in integer) is separate;
 +
      30 :              :        :
 +
      31 :              :        : end pkg;
 +
      32 :              :        :
 +
 
 +
pkg-td.adb:
 +
 
 +
 
 +
+  - This line/branch/edge was not executed.
 +
.  - This internal branch/edge was not executed but may not be significant.
 +
* - This line/branch/edge was not probed.
 +
 
 +
  Line    Edge/Branch  Line Cnt  Source
 +
  ------  -----------  ---------  ------- - -
 +
      1 :              :        :
 +
       2 :              :        : separate(pkg)
 +
      3 :              :        :
 +
      4 :              :    10 : procedure td(i : in integer) is
 +
      5 :              :    10 :  i0 : integer := 0;
 +
      6 :              :    10 :  i1 : integer := 1;
 +
      7 :              :    10 :  f0 : float := 0.0;
 +
      8 :              :    10 :  f1 : float := 1.0;
 +
      9 :              :    10 :  l  : integer := 0;
 +
      10 :              :        : begin
 +
      11 :              :    10 :  case i is
 +
          B      1x -> 15
 +
          B      1x -> 13
 +
          B      1x -> 17
 +
          B      1x -> 19
 +
      12 :              :        :  when 1 =>
 +
      13 :              :      1 :    p1(i0);
 +
      14 :              :       :  when 2 =>
 +
      15 :              :      1 :    p1(i1);
 +
      16 :              :        :  when 3 =>
 +
      17 :              :      1 :    p2(f0);
 +
      18 :              :        :  when 4 =>
 +
      19 :              :      1 :    p2(f1);
 +
      20 :              :        :  when others =>
 +
      21 :              :      6 :    null;
 +
      22 :              :      1 :  end case;
 +
      23 :              :    10 : end td;
 +
      24 :              :        :  
 +
 
 +
main.adb:
 +
 
 +
 
 +
+  - This line/branch/edge was not executed.
 +
.  - This internal branch/edge was not executed but may not be significant.
 +
*  - This line/branch/edge was not probed.
 +
 
 +
  Line    Edge/Branch  Line Cnt  Source
 +
  ------  -----------  ---------  ------- - -
 +
      1 :              :        :
 +
      2 :              :        : with Text_Io;
 +
      3 :              :        : with pkg;
 
       4 :              :        :  
 
       4 :              :        :  
       5 :              :        : /* This is the function we are interested in. */
+
       5 :              :        : procedure Main is
       6 :              :        : void DoProcessing (void)
+
       6 :              :        : begin
       7 :              :     3 : {
+
       7 :              :        :    for i in 1..10 loop
       8 :              :      3 :    int  Option = 0;
+
       8 :              :        :      pkg.td(i);
      9 :              :        :    char Buffer [100];
+
       9 :              :        :    end loop;
      10 :              :        :   
+
       10 :              :        : end Main;
      11 :              :      3 :    printf ("1. Print string #1\n");
+
       11 :              :        :  
      12 :              :      3 :    printf ("2. Print string #2\n");
 
      13 :              :       :   
 
+    14 :      2/2/1/2 :      6 :    while (Option < 1 || Option > 2)
 
                3 -> 16
 
                0 -> 16
 
      15 :              :        :   {
 
       16 :              :      3 :      printf ("      Enter your choice (1 or 2): \n");
 
      17 :              :      3 :      fgets (Buffer, 99, stdin);
 
      18 :              :      3 :      Option = atoi (Buffer);
 
      19 :              :       :    }
 
      20 :              :        :     
 
+    21 :      2/2/2/1 :      3 :    switch (Option)
 
                1 -> 24
 
                2 -> 28
 
      22 :              :        :    {
 
      23 :              :        :    case 1:
 
      24 :              :      1 :      printf ("This is string #1 - pretty exciting, huh?!\n");
 
       25 :              :      1 :      break;
 
      26 :              :        :  
 
      27 :              :        :    case 2:
 
      28 :              :      2 :      printf ("This is string #2 - it doesn't get any better than this!\n");
 
      29 :              :      2 :      break;
 
       30 :              :        :   }
 
      31 :              :      3 : }
 
      32 :              :        :
 
      33 :              :        : int main (int argc, char **argv)
 
      34 :              :      3 : {
 
      35 :              :      3 :    DoProcessing ();
 
      36 :              :      3 :    return 0;
 
      37 :              :      3 : }
 
  
O  - Number of offsets for line
 
B  - Number of branches for line
 
I  - Number of offsets/branches instrumented
 
I% - Percentage of offsets/branches instrumented
 
H  - Number of offsets/branches hit
 
H% - Percentage of offsets/branches hit
 
T  - Number of branches taken
 
N  - Number of branches not taken
 
+  - Partial coverage on this line
 
*  - No coverage on this line
 
 
</pre>
 
</pre>
  

Revision as of 18:13, 21 December 2017

Branch Coverage Predefined Probe: brcov.ual

NOTE: This predefined probe is still under development and not all features and platforms are supported.

The brcov.ual predefined probe supports branch coverage analysis on user applications. Branch coverage tells you exactly which branches within a function were actually hit during program execution. This allows you to develop unit test cases which can be used to confirm that all the branches of code you have written are executed and taken.

While this probe is a unique and powerful tool for collecting branch coverage information, it is only a tool. Getting complete branch coverage on a function can be a difficult task, and requires the user to compose test cases which will cause all paths to be executed, including error conditions and exception propagation. The brcov.ual probe can be used with the coverage.ual to produce comprehensive test coverage information.

But it should be noted that it is in testing these tricky cases that Aprobe can especially useful, since you as a user can write a probe that modifies the data on_entry to a function to force a specific path through the function, without worrying about how to get the normal caller of that function to pass that erroneous data.

You can find an example that works through some of the issues that arise from using Aprobe to do this in the $APROBE/examples/learn/branch_coverage directory. Additionally you are encouraged to contact OC Systems to discuss using Aprobe on your specific application.

Usage

This probe is applied at run time using aprobe as described under Branch Coverage UAL Parameters below. The only functions for which data will be collected are functions selected in the configuration file (see Branch Coverage Configuration File). Selecting a function means that branches in the function will be instrumented. Conversely, branches in functions that are not selected will not be instrumented, and no record will be kept of invocations of these functions or branches.

The instrumentation of branches requires that there be accurate source line debugging information recorded in the executable. This generally means that the application or library containing the covered functions:

  • must have been compiled with the appropriate debug flag (-g);
  • should have been compiled without optimization; and
  • must have the debug information still available.

A snapshot of the current state of collected branch coverage data may be written to the APD file while the program is running (see Configuration of Snapshots). Taking such periodic snapshots at known events, like the entry to or exit from a particular function, may help correlate state data with the execution path recorded by the brcov probe. A final snapshot occurs automatically at program termination.

After the application program terminates, the apformat command must be run to produce an XML branch coverage .brc file from the collected data. Use the abrmerge tool to produce reports and from the .brc files. Data from multiple runs may be merged into a single .brc file and a single report using Abrmerge (see abrcmerge below). Coverage data from brcov.ual and coverage.ual can be combined using Abrmerge to produce very extensive reports.

The report produced by the abrcmerge consists of several parts: the "Overall Summary," the "Subprogram Summary," the "Subprogram Details," and the "Source Annotation." (A subprogram is another name for a function, procedure, subroutine -- any callable entity.) The summaries lists the total lines and branches in the subprogram, how many lines/branches were not instrumented nor executed, and the percentage of instrumented lines/branches that were executed for the subprogram. The details section reports whether the subprogram was called, and if it was, it will report how many of the instrumented lines/branches were executed. In cases where coverage is less than 100% for a particular subprogram, the Details section will also list instrumented lines/branches which were not executed. The source section will annotate any provided source files with branch/line coverage information which will assist with test case creation. Some parts of the report are options. The coverage report will look similar to that in Branch Coverage Summary Report.

Branch Coverage UAL Parameters

brcov.ual is specified on the aprobe command line or in an APO file, and with apformat. The specific options are:

aprobe   -u brcov[.ual]    [-p " [-c config_filename] [-h] [-v]"   your_program 

For example:

aprobe -u brcov -p "-c foo1.brcov.cfg" foo.exe
aprobe -u brcov -p 

where:

-c config_filename

specifies that the name of the probe configuration options file will follow immediately after -c. The default file name is your_program.brcov.cfg. For example, if your executable program is called wilbur.exe, then the default file name would be wilbur.exe.brcov.cfg.
-h
produces brief help text.
-v
verbose mode, which produces additional progress messages.

Note that if you use no UAL parameters, you need not specify UAL names at all. For example:

apformat progname

will format progname.apd with all the UALs with which it was created.

Branch Coverage Configuration File

The Branch Coverage configuration file is used to specify what subprograms are to be analyzed, when snapshots are to be taken, and other options, as described in Configuration File. The example below shows one possible Branch Coverage configuration file.

PROBE CONFIGURATION FILE FOR BRCOV VERSION 1.0.0

CoverageEnabledInitially TRUE

// Here we select which subprograms we want to cover:
COVERAGE "main"
COVERAGE extern:"YET_ANOTHER_LOCAL_PROC"
COVERAGE "STILL_ANOTHER_LOCAL_PROC"

// Here we define which subprogram invocations cause snapshots
// to be saved.
SNAPSHOT "printf" IN "libc.a(shr.o)" ON ENTRY IS "printf called"

Example D-2. brcov.cfg File

Note that if you do not provide a configuration file, the probe will create a default configuration file for you. This is a good way to get started.

Configuration Variables

The following are the only valid keywords that identify lines to set configuration variables. Each such line must begin with one of these keywords, and the keyword must be followed by a value. Nothing else is allowed on the same line.

CoverageEnabledInitially

This must be followed by the value TRUE or FALSE. The default is TRUE which indicates that data logging will begin as soon as the application program starts running. If set to FALSE, data logging will begin only after a call is made to the probe's function ap_Brcov_Enable(), rather than as soon as the application program starts running.

ShowAllSnapshots

This must be followed by TRUE or FALSE. The default is FALSE. When FALSE, the data from all intermediate snapshots are rolled into the one final report. When TRUE the data for every snapshot will be reported in separate summaries.

ResetSnapshotCounts

This must be followed by TRUE or FALSE. The default is FALSE. When FALSE, the data from all intermediate snapshots accumulate into the one final report. When TRUE the counts are reset for each separate snapshot.

ShowUnconditionalBranches

This must be followed by TRUE or FALSE. The default is FALSE, which indicates that unconditional branches will not be tracked. In general, conditional branches are more interesting than unconditional branches.

Configuration of Branch Coverage Functions

Each function for which branch coverage data is to be collected must be specified explicitly using the COVERAGE, COVERFILE, or COVERUNIT keywords.

The COVERAGE keyword is followed by the name of the function, as described in Configuration of Selected Functions. This directive selects specific functions by name (with possible wildcards).

The COVERFILE keyword is followed by the name of a file and an option module name. This keyword selects the functions originating from the designated source file (with possible wildcards).

The COVERUNIT keyword is followed by the name of a PowerAda unit. This keyword selects the functions originating from the designated PowerAda unit (with possible wildcards).

By default, if no COVERAGE/COVERFILE/COVERUNIT lines exist, then coverage data will only be collected for the "main()" function of the application module.

The REMOVE keyword allows you to specify subprograms that should not be instrumented for logging coverage data. This is useful when used in conjunction with a wildcard ("*"), to gather data about everything except certain routines.

Note: COVERAGE "*" is not a particuarly useful concept since it is important for you to identify and isolate your functions carefully in order to use test coverage for its intended purpose.

Configuration of Snapshots

The branch coverage probe configuration file allows you to specify the name of some subprograms for which snapshots of the coverage data are to be automatically taken. This is done with lines beginning with the keyword SNAPSHOT.

Each SNAPSHOT line must specify a particular subprogram in the usual manner, just as is done for a COVERAGE line.

The remainder of the SNAPSHOT line contains pairs, where each pair has a special identifier keyword followed by its own associated value. These pairs give supplementary information about the snapshot.

  • ON - This optional special identifier must be followed by the value ENTRY or EXIT, to denote that the snapshot is to be taken on entry to or exit from the subprogram respectively.
  • IS - This optional special identifier must be followed by an (arbitrarily long) string enclosed within quotation marks (""). It specifies a textual description or title that is to be logged along with the snapshot.

Here is an example of a SNAPSHOT directive:

SNAPSHOT "my_function" ON EXIT IS "my on exit snapshot"

Configuration of TestNames

The branch coverage probe configuration file allows you to specify the name of some subprograms for which test names can be set. This is done with lines beginning with the keyword TESTNAME.

Each TESTNAME line must specify a particular subprogram in the usual manner, just as is done for a COVERAGE line.

The remainder of the TESTNAME line contains pairs, where each pair has a special identifier keyword followed by its own associated value. These pairs give supplementary information about the test name change.

  • ON - This optional special identifier must be followed by the value ENTRY or EXIT, to denote that the test name is to be set on entry to or exit from the subprogram respectively.
  • IS - This optional special identifier must be followed by an (arbitrarily long) string enclosed within quotation marks (""). It specifies a test name that is to be logged along with the snapshot.

Here is an example of a TESTNAME directive:

TESTNAME "my_function" ON ENTRY IS "test 1"

Configuration of TestName Hook Function

The branch coverage probe configuration file allows you to specify the name of some subprograms which are called when test name changes. The designated function should have a single (in) parameter which is a pointer to a nil-terminated string which is the test name. A value of NULL indicates no test name.

Each TESTNAMEHOOK line must specify a particular subprogram in the usual manner, just as is done for a COVERAGE line.

Here is an example of a TESTNAMEHOOK directive:

TESTNAMEHOOK "my_function"

Test Coverage API

Users can control the behavior of the coverage probe by calls from within their own probes. The API for the trace probe is defined by [../include/coverage.h $APROBE/include/brcov.h]. Some of the functions exported by brcov.ual are:

  • ap_Brcov_Enable - Enables logging of coverage data.
  • ap_Brcov_Disable - Disables logging of coverage data.
  • ap_Brcov_DoSnapshot - Dumps current coverage information.

See Snapshots for more information and an example.

Branch Coverage Performance Issues

See Performance Issues for a general discussion of factors that affect performance.

The branch coverage probe is applied to the functions specified in the configuration file, so the run-time overhead is directly proportional to the number of calls to the functions selected and the number of branches executed in each function. The branch coverage probe simply increments an integer for each branch executed, so the amount of data is a constant multiple of the number of branches being covered, though additional snapshots increase this.

As branch coverage analysis is generally a unit-testing activity, it is mostly done on the small number of related functions under test, so performance should not be an issue. Application-wide branch coverage analysis during integration testing is not recommended.

Abrmerge

The results of multiple runs of brcov.ual (and coverage.ual) over the same executable may be merged into a single "combined summary" report using abrmerge.

A merged summary report is useful when running many tests, each with different input data or perhaps different probes, to force all logic paths to be executed. abrmerge can also merge branch data files to accumulate data from multiple runs into a single file.

A simple example

This example of the brcov.ual (and coverage.ual) predefined probe can be found in $APROBE/examples/predefined_probes/brcov.

The test application is very simple. It just prints out a mini-menu and waits for you to select 1 or 2. Depending on your selection, it does one of two actions. You can see this in the DoProcessing() function in coverage_example.c. We want to exercise every branch in this function -- we refer to this as getting 100% coverage.

Note that, unlike some other uses of Aprobe, branch and line coverage requires you to be familiar with (the lines in) your source code to make most effective use of the probe.

As before, we'll use make to build the example application:

cd $APROBE/examples/predefined_probes/brcov
make
aprobe -u brcov -u coverage coverage_example

When the example application prompts you for a choice, enter the number 1. That will print out the first string and then the application will exit normally.

As before, aprobe stored the data in .apd files, so use apformat to format it:

apformat coverage_example

and abrmerge to merge and view it:

abrmerge *.brc *.c

Here's a portion of the report showing the line and branch coverage we've achieved:

coverage_example.c:

   Line    Br O/I/T/N    Line Cnt   Source
  ------   -----------   ---------  ------- - -
       1 :              :        : /* This is a simple example to demonstrate using coverage.ual */
       2 :              :        : #include <stdio.h>
       3 :              :        : #include <stdlib.h>
       4 :              :        : 
       5 :              :        : /* This is the function we are interested in. */
       6 :              :        : void DoProcessing (void)
       7 :              :      1 : {
       8 :              :      1 :    int  Option = 0;
       9 :              :        :    char Buffer [100];
      10 :              :        :    
      11 :              :      1 :    printf ("1. Print string #1\n");
      12 :              :      1 :    printf ("2. Print string #2\n");
      13 :              :        :    
+     14 :      2/2/1/2 :      2 :    while (Option < 1 || Option > 2)
                1 -> 16
                0 -> 16
      15 :              :        :    {
      16 :              :      1 :       printf ("      Enter your choice (1 or 2): \n");
      17 :              :      1 :       fgets (Buffer, 99, stdin);
      18 :              :      1 :       Option = atoi (Buffer);
      19 :              :        :    }
      20 :              :        :       
+     21 :      2/2/1/0 :      1 :    switch (Option)
                1 -> 24
                0 -> 28
      22 :              :        :    {
      23 :              :        :    case 1:
      24 :              :      1 :       printf ("This is string #1 - pretty exciting, huh?!\n");
      25 :              :      1 :       break;
      26 :              :        : 
      27 :              :        :    case 2:
*     28 :              :      0 :       printf ("This is string #2 - it doesn't get any better than this!\n");
*     29 :              :      0 :       break;
      30 :              :        :    }
      31 :              :      1 : }
      32 :              :        : 
      33 :              :        : int main (int argc, char **argv)
      34 :              :      1 : {
      35 :              :      1 :    DoProcessing ();
      36 :              :      1 :    return 0;
      37 :              :      1 : }

O  - Number of offsets for line
B  - Number of branches for line
I  - Number of offsets/branches instrumented
I% - Percentage of offsets/branches instrumented
H  - Number of offsets/branches hit
H% - Percentage of offsets/branches hit
T  - Number of branches taken
N  - Number of branches not taken
+  - Partial coverage on this line
*  - No coverage on this line

Using abrmerge

Sometimes we want to combine several test and branch coverage reports into a single report. Use abrmerge to accomplish this.

The above 2 lines that were not executed are specific to choice number 2. We could easily execute the lines of choice number 2 if we ran it again, but then we wouldn't be able to cover the choice 1 lines. If you look in the local directory, you'll see coverage_example.*.brc files. These were created at format time by the brcov and coverage predefined probes. These contain the branch and line coverage data obtained during choice 1, so let's save it.

We typically use abrmerge to merge results from many .brc files and display the combined data:

abrmerge -m combined.brc coverage_example.*.brc 

This "merges" the data in the input .brc files and outputs it in combined.brc.

Now run the application again using aprobe:

aprobe -u brcov -u coverage coverage_example

Select choice 2, then after the application ends re-run apformat:

apformat coverage_example

This time we get a different set of lines executed, as expected. Use the abrmerge command to combine the data in combined.brc (from the first run) with the data from this run (in a new coverage_example.*.brc):

abrmerge -m combined.brc coverage_example.*.brc combined.brc

Now we get the 100% coverage we wanted!

The on-line example in $APROBE/examples/learn/TBD extends this by illustrating how to use your own probes in combination with brcov.ual and coverage.ual to ensure all lines are executed.

Branch Coverage Report

The final output of the learn/branch_coverage example looks like this:

Aprobe Coverage Summary Report


Generated: 27 Nov 2017 09:19:05 -0500



--------------------------------------------------------------------------
Contents
--------------------------------------------------------------------------

TimeStamps
Overall Summary
Function TOC
File TOC
Function Summaries
Function Details
File Coverage



--------------------------------------------------------------------------
Timestmaps
--------------------------------------------------------------------------

TimeStamp:
Program   : /home/swn/aprobe/test/brcov/tnamec/main.exe
Host      : centos7
PID       : 13805
Start Time: 2017-11-27 09:19:06
End Time  : 2017-11-27 09:19:06



--------------------------------------------------------------------------
Overall Summary
--------------------------------------------------------------------------


                 Lines   Lines      Line                Branches  Branches            Branches  Branch
Total    Total   Not     Not        Coverage  Total     Not       Not        Branches Not       Coverage
Calls    Lines   Probed  Executed   Percent   Branches  Probed    Executed   Taken    Taken     Percent   Name
------- ------- -------- --------- ========= --------- --------- ---------- --------- --------- ======== --------- - -
     2       5        0         0       100         1         0          0         1         1      100  extern:"pkg.p1[1]"
     2       6        0         0       100         2         0          0         1         2      100  extern:"pkg.p2[1]"
    10      19        0         1        94         5         0          0         5         4      100  extern:"pkg.td[1]"

Note: Negative counts indicate the function/line was not instrumented.

          Funcs   Func                  Lines     Lines      Lines                 Branches   Branches              Branches   Branch
 Total    Not     Coverage    Total     Not       Not        Coverage   Total      Not        Not        Branches   Not        Coverage
 Funcs    Called  Percent     Lines     Probed    Executed   Percent    Branches   Probed     Executed   Taken      Taken      Percent 
-------- -------- ======== ---------- ---------- ---------- ========== ---------- ---------- ---------- ---------- ---------- ==========
      3        0      100         30          0          1         96          8          0          0          7          7        100



--------------------------------------------------------------------------
Function TOC
--------------------------------------------------------------------------

extern:"pkg.p1[1]" (/home/swn/aprobe/test/brcov/tnamec/pkg.adb)
extern:"pkg.p2[1]" (/home/swn/aprobe/test/brcov/tnamec/pkg.adb)
extern:"pkg.td[1]" (/home/swn/aprobe/test/brcov/tnamec/pkg-td.adb)


--------------------------------------------------------------------------
File TOC
--------------------------------------------------------------------------

pkg.ads
pkg.adb
pkg-td.adb
main.adb


--------------------------------------------------------------------------
Function Coverage Summaries
--------------------------------------------------------------------------


extern:"pkg.p1[1]"	(/home/swn/aprobe/test/brcov/tnamec/pkg.adb)


Calls:  2
-----

Test Names: [ test p1 ]
----------

Lines:
-----
          Lines   Probed  Not Probed   Hit 
         ------- -------- ---------- ----------
Count:        5         5         0         5
Percent:              100         0       100


Branches:
--------
          Branches    Probed      Hit       Taken     NotTaken
         ---------- ---------- ---------- ---------- ----------
Count:            1          1          1          1          1
Percent:                   100        100        100        100


extern:"pkg.p2[1]"	(/home/swn/aprobe/test/brcov/tnamec/pkg.adb)


Calls:  2
-----

Test Names: [ test p1, test p2 ]
----------

Lines:
-----
          Lines   Probed  Not Probed   Hit 
         ------- -------- ---------- ----------
Count:        6         6         0         6
Percent:              100         0       100


Branches:
--------
          Branches    Probed      Hit       Taken     NotTaken
         ---------- ---------- ---------- ---------- ----------
Count:            2          2          2          1          2
Percent:                   100        100         50        100


extern:"pkg.td[1]"	(/home/swn/aprobe/test/brcov/tnamec/pkg-td.adb)


Calls:  10
-----

Test Names: [ test p1, test p2 ]
----------

Lines:
-----
          Lines   Probed  Not Probed   Hit 
         ------- -------- ---------- ----------
Count:       19        19         0        18
Percent:              100         0        94

    Lines Not Executed:
    ------------------
    21 / pkg-td.adb

Branches:
--------
          Branches    Probed      Hit       Taken     NotTaken
         ---------- ---------- ---------- ---------- ----------
Count:            5          5          5          5          4
Percent:                   100        100        100         80



--------------------------------------------------------------------------
Function Coverage Details
--------------------------------------------------------------------------


extern:"pkg.p1[1]"	(/home/swn/aprobe/test/brcov/tnamec/pkg.adb)

Lines:
-----
      Line     I     Count      File  [ Test Names ]
    ---------- -  -----------  ---------------------
            11 Y :          2 : pkg.adb [ test p1 ]
            13 Y :          2 : pkg.adb [ test p1 ]
            14 Y :          1 : pkg.adb [ test p1 ]
            16 Y :          1 : pkg.adb [ test p1 ]
            18 Y :          2 : pkg.adb [ test p1 ]

Branches:
--------
     Offset  I      Hit       Taken     NotTaken    Line/File -> Line/File  [ Test Names ]
    -------- -   ---------- ---------- ----------   --------------------------------------
    0x00000b Y :          2          1          1 : 13 / pkg.adb -> 16 / pkg.adb [ test p1 ]

Edges:
------
       Count     Offset/Line/File -> Offset/Line/File [ Test Names ]
    ----------  ---------------------------------------------------

extern:"pkg.p2[1]"	(/home/swn/aprobe/test/brcov/tnamec/pkg.adb)

Lines:
-----
      Line     I     Count      File  [ Test Names ]
    ---------- -  -----------  ---------------------
            20 Y :          2 : pkg.adb [ test p2 ]
            22 Y :          2 : pkg.adb [ test p2 ]
            22 Y :          1 : pkg.adb [ test p2 ]
            23 Y :          1 : pkg.adb [ test p2 ]
            25 Y :          1 : pkg.adb [ test p2 ]
            27 Y :          2 : pkg.adb [ test p2 ]

Branches:
--------
     Offset  I      Hit       Taken     NotTaken    Line/File -> Line/File  [ Test Names ]
    -------- -   ---------- ---------- ----------   --------------------------------------
    0x000013 Y :          2          1          1 : 22 / pkg.adb -> 25 / pkg.adb [ test p2 ]
    0x000021 Y :          1          0          1 : 22 / pkg.adb -> 25 / pkg.adb [ test p2 ]

Edges:
------
       Count     Offset/Line/File -> Offset/Line/File [ Test Names ]
    ----------  ---------------------------------------------------

extern:"pkg.td[1]"	(/home/swn/aprobe/test/brcov/tnamec/pkg-td.adb)

Lines:
-----
      Line     I     Count      File  [ Test Names ]
    ---------- -  -----------  ---------------------
             4 Y :         10 : pkg-td.adb [ test p1, test p2 ]
             5 Y :         10 : pkg-td.adb [ test p1, test p2 ]
             6 Y :         10 : pkg-td.adb [ test p1, test p2 ]
             7 Y :         10 : pkg-td.adb [ test p1, test p2 ]
             8 Y :         10 : pkg-td.adb [ test p1, test p2 ]
             9 Y :         10 : pkg-td.adb [ test p1, test p2 ]
            11 Y :         10 : pkg-td.adb [ test p1, test p2 ]
            21 Y :          0 : pkg-td.adb [  ]
            11 Y :          8 : pkg-td.adb [ test p1, test p2 ]
            21 Y :          6 : pkg-td.adb [ test p2 ]
            13 Y :          1 : pkg-td.adb [  ]
            22 Y :          1 : pkg-td.adb [ test p1 ]
            15 Y :          1 : pkg-td.adb [ test p1 ]
            22 Y :          1 : pkg-td.adb [ test p1 ]
            17 Y :          1 : pkg-td.adb [ test p1 ]
            22 Y :          1 : pkg-td.adb [ test p2 ]
            19 Y :          1 : pkg-td.adb [ test p2 ]
            22 Y :          1 : pkg-td.adb [ test p2 ]
            23 Y :         10 : pkg-td.adb [ test p1, test p2 ]

Branches:
--------
     Offset  I      Hit       Taken     NotTaken    Line/File -> Line/File  [ Test Names ]
    -------- -   ---------- ---------- ----------   --------------------------------------
    0x000037 Y :         10          1          9 : 11 / pkg-td.adb -> 15 / pkg-td.adb [ test p2, test p1 ]
    0x00003c Y :          9          8          1 : 11 / pkg-td.adb -> 11 / pkg-td.adb [ test p2, test p1 ]
    0x000041 Y :          1          1          0 : 11 / pkg-td.adb -> 13 / pkg-td.adb [  ]
    0x000048 Y :          8          1          7 : 11 / pkg-td.adb -> 17 / pkg-td.adb [ test p2, test p1 ]
    0x00004d Y :          7          1          6 : 11 / pkg-td.adb -> 19 / pkg-td.adb [ test p2 ]

Edges:
------
       Count     Offset/Line/File -> Offset/Line/File [ Test Names ]
    ----------  ---------------------------------------------------


--------------------------------------------------------------------------
File Coverage
--------------------------------------------------------------------------

pkg.ads:


+  - This line/branch/edge was not executed.
.  - This internal branch/edge was not executed but may not be significant.
*  - This line/branch/edge was not probed.

   Line    Edge/Branch   Line Cnt   Source
  ------   -----------   ---------  ------- - -
       1 :              :        : 
       2 :              :        : package pkg is
       3 :              :        : 
       4 :              :        :   procedure p1(I : in out integer);
       5 :              :        :   procedure p2(I : in out float);
       6 :              :        : 
       7 :              :        :   procedure td(I : in integer);
       8 :              :        : 
       9 :              :        : end pkg;
      10 :              :        : 

pkg.adb:


+  - This line/branch/edge was not executed.
.  - This internal branch/edge was not executed but may not be significant.
*  - This line/branch/edge was not probed.

   Line    Edge/Branch   Line Cnt   Source
  ------   -----------   ---------  ------- - -
       1 :              :        : 
       2 :              :        : with Text_Io;
       3 :              :        : 
       4 :              :        : package body pkg is
       5 :              :        : 
       6 :              :        :   function Ident(i : integer) return integer is
       7 :              :        :   begin
       8 :              :        :      return i;
       9 :              :        :   end;
      10 :              :        : 
      11 :              :      2 :   procedure p1(I : in out integer) is
      12 :              :        :   begin
      13 :              :      2 :     if I = 0 then
          B       1x -> 16
      14 :              :      1 :       Text_Io.Put_Line("p1, 0!");
      15 :              :        :     else
      16 :              :      1 :       Text_Io.Put_Line("p1, not zero");
      17 :              :        :     end if;
      18 :              :      2 :   end p1;
      19 :              :        : 
      20 :              :      2 :   procedure p2(I : in out float) is
      21 :              :        :   begin
      22 :              :      2 :     if I < 0.000001 and then I > -0.000001 then
          B       1x -> 25
+         B       0x -> 25
      23 :              :      1 :       Text_Io.Put_Line("p2, 0!");
      24 :              :        :     else
      25 :              :      1 :       Text_Io.Put_Line("p2, not zero");
      26 :              :        :     end if;
      27 :              :      2 :   end p2;
      28 :              :        : 
      29 :              :        :   procedure td(i : in integer) is separate;
      30 :              :        : 
      31 :              :        : end pkg;
      32 :              :        : 

pkg-td.adb:


+  - This line/branch/edge was not executed.
.  - This internal branch/edge was not executed but may not be significant.
*  - This line/branch/edge was not probed.

   Line    Edge/Branch   Line Cnt   Source
  ------   -----------   ---------  ------- - -
       1 :              :        : 
       2 :              :        : separate(pkg)
       3 :              :        : 
       4 :              :     10 : procedure td(i : in integer) is
       5 :              :     10 :   i0 : integer := 0;
       6 :              :     10 :   i1 : integer := 1;
       7 :              :     10 :   f0 : float := 0.0;
       8 :              :     10 :   f1 : float := 1.0;
       9 :              :     10 :   l  : integer := 0;
      10 :              :        : begin
      11 :              :     10 :   case i is
          B       1x -> 15
          B       1x -> 13
          B       1x -> 17
          B       1x -> 19
      12 :              :        :   when 1 =>
      13 :              :      1 :     p1(i0);
      14 :              :        :   when 2 =>
      15 :              :      1 :     p1(i1);
      16 :              :        :   when 3 =>
      17 :              :      1 :     p2(f0);
      18 :              :        :   when 4 =>
      19 :              :      1 :     p2(f1);
      20 :              :        :   when others =>
      21 :              :      6 :     null;
      22 :              :      1 :   end case;
      23 :              :     10 : end td;
      24 :              :        :   

main.adb:


+  - This line/branch/edge was not executed.
.  - This internal branch/edge was not executed but may not be significant.
*  - This line/branch/edge was not probed.

   Line    Edge/Branch   Line Cnt   Source
  ------   -----------   ---------  ------- - -
       1 :              :        : 
       2 :              :        : with Text_Io;
       3 :              :        : with pkg;
       4 :              :        : 
       5 :              :        : procedure Main is
       6 :              :        : begin
       7 :              :        :    for i in 1..10 loop
       8 :              :        :       pkg.td(i);
       9 :              :        :    end loop;
      10 :              :        : end Main;
      11 :              :        : 

Example D-3. Test Coverage Summary Report