Trace NVidia’s ioctl calls with valgrind-mmt

Valgrind-mmt is a Valgrind modification which allows tracing application accesses to mmaped memory (which is how userspace parts of graphics drivers communicate with hardware). It was created by Dave Airlie and then extended/fixed by others.

In order to trace ioctl calls made by the blob’s userspace, I used a modified version of valgrind-mmt to get a trace of the registers modified by CUPTI to monitor the wanted signals. I applied the following patch of Christoph Bumiller (calim) :

diff --git a/mmt/mmt_nv_ioctl.c b/mmt/mmt_nv_ioctl.c
index 23682e7..11890b0 100644
--- a/mmt/mmt_nv_ioctl.c
+++ b/mmt/mmt_nv_ioctl.c
@@ -386,6 +386,24 @@ void mmt_nv_ioctl_pre(UWord *args)
 				UInt *addr2 = (*(UInt **) (&data[4]));
 				dumpmem("in2 ", addr2[2], 0x3c);
 			}
+         else if (data[2] == 0x20800122)
+         {
+            UInt k;
+            UInt *in = (UInt *)mmt_2x4to8(data[5], data[4]);
+            UInt cnt = in[5];
+            UInt *tx = (UInt *)mmt_2x4to8(in[7], in[6]);
+            VG_(message) (Vg_DebugMsg, "<==(%u at %p)\n", cnt, tx);
+            for (k = 0; k < cnt; ++k)
+               VG_(message) (Vg_DebugMsg, "REQUEST: DIR=%x MMIO=%x VALUE=%08x MASK=%08x UNK=%08x,%08x,%08x,%08x\n",
+                             tx[k * 8 + 0],
+                             tx[k * 8 + 3],
+                             tx[k * 8 + 5],
+                             tx[k * 8 + 7],
+                             tx[k * 8 + 1],
+                             tx[k * 8 + 2],
+                             tx[k * 8 + 4],
+                             tx[k * 8 + 6]);
+         }
 			break;

 		case 0xc040464d:
@@ -565,6 +583,23 @@ void mmt_nv_ioctl_post(UWord *args)
 				UInt *addr2 = (*(UInt **) (&data[4]));
 				dumpmem("out2 ", addr2[2], 0x3c);
 			}
+         else if (data[2] == 0x20800122)
+         {
+            UInt *out = (UInt *)mmt_2x4to8(data[5], data[4]);
+            UInt cnt = out[5];
+            UInt *tx = (UInt *)mmt_2x4to8(out[7], out[6]);
+            UInt k;
+            for (k = 0; k < cnt; ++k)
+               VG_(message) (Vg_DebugMsg, "RETURND: DIR=%x MMIO=%x VALUE=%08x MASK=%08x UNK=%08x,%08x,%08x,%08x\n",
+                             tx[k * 8 + 0],
+                             tx[k * 8 + 3],
+                             tx[k * 8 + 5],
+                             tx[k * 8 + 7],
+                             tx[k * 8 + 1],
+                             tx[k * 8 + 2],
+                             tx[k * 8 + 4],
+                             tx[k * 8 + 6]);
+         }
 			break;
 			// 0x37 read configuration parameter
 		case 0xc0204638:

That patch displays MMIO register of pre/post ioctl calls made by the blob. In order to trace these calls, you have to use valgrind-mmt as this way :

valgrind --tool=mmt --mmt-trace-file=/dev/nvidia0 --mmt-trace-nvidia-ioctls

For example, if I want to see the post ioctl calls of the vectorAddDrv CUDA sample when I trace the inst_executed event, I’ll use :

valgrind --tool=mmt --mmt-trace-file=/dev/nvidia0 --mmt-trace-nvidia-ioctls ./vectorAddDrv 2>&1 | grep RETURND

And the trace looks like this:

--4803-- RETURND: DIR=1 MMIO=504600 VALUE=00000000 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=1 MMIO=504e00 VALUE=00000000 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=0 MMIO=504600 VALUE=00000000 MASK=00000000 UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=1 MMIO=504600 VALUE=80000000 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=101 MMIO=504604 VALUE=002d2d2d MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=101 MMIO=504608 VALUE=00000000 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=101 MMIO=50465c VALUE=00000000 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=101 MMIO=504660 VALUE=00000000 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=101 MMIO=504664 VALUE=00000000 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=101 MMIO=504668 VALUE=00000000 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=101 MMIO=50466c VALUE=00000000 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=101 MMIO=504730 VALUE=00000000 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=101 MMIO=504734 VALUE=00000011 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=101 MMIO=504738 VALUE=00000022 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=100 MMIO=504674 VALUE=0000137c MASK=00000000 UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=100 MMIO=504678 VALUE=00001208 MASK=00000000 UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=100 MMIO=50467c VALUE=000003e7 MASK=00000000 UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=100 MMIO=504670 VALUE=00000000 MASK=00000000 UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=101 MMIO=504674 VALUE=00000000 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=101 MMIO=504678 VALUE=00000000 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=101 MMIO=50467c VALUE=00000000 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=101 MMIO=504680 VALUE=00000000 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=101 MMIO=504684 VALUE=00000000 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=101 MMIO=504688 VALUE=00000000 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=101 MMIO=50468c VALUE=00000000 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=101 MMIO=504690 VALUE=00000000 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=0 MMIO=504600 VALUE=80000000 MASK=00000000 UNK=00000000,00000000,00000000,00000000
--4803-- RETURND: DIR=1 MMIO=504600 VALUE=00000000 MASK=ffffffff UNK=00000000,00000000,00000000,00000000
Advertisements

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s