mirror of
https://github.com/ROCm/jax.git
synced 2025-04-16 03:46:06 +00:00
Add a heap profiler API and document it. (#3576)
This commit is contained in:
Peter Hawkins
GitHub
parent
63ff6cb8e9
commit
5116fd47aa
435
docs/_static/device_memory_profile.svg
vendored
Normal file
435
docs/_static/device_memory_profile.svg
vendored
Normal file
@ -0,0 +1,435 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Generated by graphviz version 2.44.0 (20200408.0750)
|
||||
-->
|
||||
<!-- Title: unnamed Pages: 1 -->
|
||||
<svg width="685pt" height="956pt"
|
||||
viewBox="0.00 0.00 685.00 956.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 952)">
|
||||
<title>unnamed</title>
|
||||
<polygon fill="white" stroke="transparent" points="-4,4 -4,-952 681,-952 681,4 -4,4"/>
|
||||
<g id="clust1" class="cluster">
|
||||
<title>cluster_L</title>
|
||||
<polygon fill="none" stroke="black" points="8,-825 8,-940 462,-940 462,-825 8,-825"/>
|
||||
</g>
|
||||
<!-- Type: space -->
|
||||
<g id="node1" class="node">
|
||||
<title>Type: space</title>
|
||||
<polygon fill="#f8f8f8" stroke="black" points="453.5,-932 16.5,-932 16.5,-833 453.5,-833 453.5,-932"/>
|
||||
<text text-anchor="start" x="24.5" y="-915.2" font-family="Times,serif" font-size="16.00">Type: space</text>
|
||||
<text text-anchor="start" x="24.5" y="-897.2" font-family="Times,serif" font-size="16.00">Showing nodes accounting for 80.11MB, 100% of 80.13MB total</text>
|
||||
<text text-anchor="start" x="24.5" y="-879.2" font-family="Times,serif" font-size="16.00">Dropped 26 nodes (cum <= 0.40MB)</text>
|
||||
<text text-anchor="start" x="24.5" y="-842.2" font-family="Times,serif" font-size="16.00">See https://git.io/JfYMW for how to read the graph</text>
|
||||
</g>
|
||||
<!-- N1 -->
|
||||
<g id="node1" class="node">
|
||||
<title>N1</title>
|
||||
<g id="a_node1"><a xlink:title="_execute_compiled_primitive (76.29MB)">
|
||||
<polygon fill="#edd5d5" stroke="#b20200" points="568.5,-173 265.5,-173 265.5,-87 568.5,-87 568.5,-173"/>
|
||||
<text text-anchor="middle" x="417" y="-149.8" font-family="Times,serif" font-size="24.00">_execute_compiled_primitive</text>
|
||||
<text text-anchor="middle" x="417" y="-123.8" font-family="Times,serif" font-size="24.00">76.29MB (95.21%)</text>
|
||||
<text text-anchor="middle" x="417" y="-97.8" font-family="Times,serif" font-size="24.00">of 76.29MB (95.21%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N1_0 -->
|
||||
<g id="N1_0" class="node">
|
||||
<title>N1_0</title>
|
||||
<g id="a_N1_0"><a xlink:title="76.29MB">
|
||||
<polygon fill="#f8f8f8" stroke="black" points="444,-36 394,-36 390,-32 390,0 440,0 444,-4 444,-36"/>
|
||||
<polyline fill="none" stroke="black" points="440,-32 390,-32 "/>
|
||||
<polyline fill="none" stroke="black" points="440,-32 440,0 "/>
|
||||
<polyline fill="none" stroke="black" points="440,-32 444,-36 "/>
|
||||
<text text-anchor="middle" x="417" y="-16.1" font-family="Times,serif" font-size="8.00">kind:buffer</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N1->N1_0 -->
|
||||
<g id="edge1" class="edge">
|
||||
<title>N1->N1_0</title>
|
||||
<g id="a_edge1"><a xlink:title="76.29MB">
|
||||
<path fill="none" stroke="black" d="M417,-86.79C417,-73.2 417,-58.5 417,-46.28"/>
|
||||
<polygon fill="black" stroke="black" points="420.5,-46.27 417,-36.27 413.5,-46.27 420.5,-46.27"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge1-label"><a xlink:title="76.29MB">
|
||||
<text text-anchor="middle" x="445.5" y="-57.8" font-family="Times,serif" font-size="14.00"> 76.29MB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N2 -->
|
||||
<g id="node2" class="node">
|
||||
<title>N2</title>
|
||||
<g id="a_node2"><a xlink:title="<unknown> (80.13MB)">
|
||||
<polygon fill="#edd5d5" stroke="#b20000" points="558.5,-900.5 471.5,-900.5 471.5,-864.5 558.5,-864.5 558.5,-900.5"/>
|
||||
<text text-anchor="middle" x="515" y="-885.1" font-family="Times,serif" font-size="8.00"><unknown></text>
|
||||
<text text-anchor="middle" x="515" y="-876.1" font-family="Times,serif" font-size="8.00">0 of 80.13MB (100%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N4 -->
|
||||
<g id="node4" class="node">
|
||||
<title>N4</title>
|
||||
<g id="a_node4"><a xlink:title="func2 (76.30MB)">
|
||||
<polygon fill="#edd5d5" stroke="#b20200" points="561.5,-782 468.5,-782 468.5,-746 561.5,-746 561.5,-782"/>
|
||||
<text text-anchor="middle" x="515" y="-766.6" font-family="Times,serif" font-size="8.00">func2</text>
|
||||
<text text-anchor="middle" x="515" y="-757.6" font-family="Times,serif" font-size="8.00">0 of 76.30MB (95.22%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N2->N4 -->
|
||||
<g id="edge4" class="edge">
|
||||
<title>N2->N4</title>
|
||||
<g id="a_edge4"><a xlink:title="<unknown> -> func2 (76.30MB)">
|
||||
<path fill="none" stroke="#b20200" stroke-width="5" d="M515,-864.31C515,-845.38 515,-814.68 515,-792.42"/>
|
||||
<polygon fill="#b20200" stroke="#b20200" stroke-width="5" points="519.38,-792.26 515,-782.26 510.63,-792.26 519.38,-792.26"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge4-label"><a xlink:title="<unknown> -> func2 (76.30MB)">
|
||||
<text text-anchor="middle" x="543.5" y="-803.8" font-family="Times,serif" font-size="14.00"> 76.30MB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N15 -->
|
||||
<g id="node15" class="node">
|
||||
<title>N15</title>
|
||||
<g id="a_node15"><a xlink:title="normal (3.83MB)">
|
||||
<polygon fill="#edeae7" stroke="#b2a187" points="666,-782 580,-782 580,-746 666,-746 666,-782"/>
|
||||
<text text-anchor="middle" x="623" y="-766.6" font-family="Times,serif" font-size="8.00">normal</text>
|
||||
<text text-anchor="middle" x="623" y="-757.6" font-family="Times,serif" font-size="8.00">0 of 3.83MB (4.78%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N2->N15 -->
|
||||
<g id="edge14" class="edge">
|
||||
<title>N2->N15</title>
|
||||
<g id="a_edge14"><a xlink:title="<unknown> -> normal (3.83MB)">
|
||||
<path fill="none" stroke="#b2a187" d="M530.94,-864.31C549.2,-844.61 579.29,-812.15 600.07,-789.73"/>
|
||||
<polygon fill="#b2a187" stroke="#b2a187" points="602.77,-791.97 607,-782.26 597.63,-787.21 602.77,-791.97"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge14-label"><a xlink:title="<unknown> -> normal (3.83MB)">
|
||||
<text text-anchor="middle" x="614" y="-803.8" font-family="Times,serif" font-size="14.00"> 3.83MB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N3 -->
|
||||
<g id="node3" class="node">
|
||||
<title>N3</title>
|
||||
<g id="a_node3"><a xlink:title="bind (76.30MB)">
|
||||
<polygon fill="#edd5d5" stroke="#b20200" points="463.5,-347 370.5,-347 370.5,-311 463.5,-311 463.5,-347"/>
|
||||
<text text-anchor="middle" x="417" y="-331.6" font-family="Times,serif" font-size="8.00">bind</text>
|
||||
<text text-anchor="middle" x="417" y="-322.6" font-family="Times,serif" font-size="8.00">0 of 76.30MB (95.22%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N11 -->
|
||||
<g id="node11" class="node">
|
||||
<title>N11</title>
|
||||
<g id="a_node11"><a xlink:title="apply_primitive (76.30MB)">
|
||||
<polygon fill="#edd5d5" stroke="#b20200" points="463.5,-260 370.5,-260 370.5,-224 463.5,-224 463.5,-260"/>
|
||||
<text text-anchor="middle" x="417" y="-244.6" font-family="Times,serif" font-size="8.00">apply_primitive</text>
|
||||
<text text-anchor="middle" x="417" y="-235.6" font-family="Times,serif" font-size="8.00">0 of 76.30MB (95.22%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N3->N11 -->
|
||||
<g id="edge3" class="edge">
|
||||
<title>N3->N11</title>
|
||||
<g id="a_edge3"><a xlink:title="bind -> apply_primitive (76.30MB)">
|
||||
<path fill="none" stroke="#b20200" stroke-width="5" d="M417,-310.8C417,-299.16 417,-283.55 417,-270.24"/>
|
||||
<polygon fill="#b20200" stroke="#b20200" stroke-width="5" points="421.38,-270.18 417,-260.18 412.63,-270.18 421.38,-270.18"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge3-label"><a xlink:title="bind -> apply_primitive (76.30MB)">
|
||||
<text text-anchor="middle" x="445.5" y="-281.8" font-family="Times,serif" font-size="14.00"> 76.30MB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N5 -->
|
||||
<g id="node5" class="node">
|
||||
<title>N5</title>
|
||||
<g id="a_node5"><a xlink:title="deferring_binary_op (76.30MB)">
|
||||
<polygon fill="#edd5d5" stroke="#b20200" points="494.5,-608 401.5,-608 401.5,-572 494.5,-572 494.5,-608"/>
|
||||
<text text-anchor="middle" x="448" y="-592.6" font-family="Times,serif" font-size="8.00">deferring_binary_op</text>
|
||||
<text text-anchor="middle" x="448" y="-583.6" font-family="Times,serif" font-size="8.00">0 of 76.30MB (95.22%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N4->N5 -->
|
||||
<g id="edge12" class="edge">
|
||||
<title>N4->N5</title>
|
||||
<g id="a_edge12"><a xlink:title="func2 -> deferring_binary_op (38.15MB)">
|
||||
<path fill="none" stroke="#b22300" stroke-width="3" d="M468.45,-747.76C444.01,-737.11 416.25,-720.21 402,-695 387.27,-668.94 406.12,-637.27 423.95,-615.77"/>
|
||||
<polygon fill="#b22300" stroke="#b22300" stroke-width="3" points="426.73,-617.9 430.66,-608.06 421.45,-613.31 426.73,-617.9"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge12-label"><a xlink:title="func2 -> deferring_binary_op (38.15MB)">
|
||||
<text text-anchor="middle" x="430.5" y="-673.3" font-family="Times,serif" font-size="14.00"> 38.15MB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N8 -->
|
||||
<g id="node8" class="node">
|
||||
<title>N8</title>
|
||||
<g id="a_node8"><a xlink:title="func1 (38.15MB)">
|
||||
<polygon fill="#eddad5" stroke="#b22300" points="561.5,-695 468.5,-695 468.5,-659 561.5,-659 561.5,-695"/>
|
||||
<text text-anchor="middle" x="515" y="-679.6" font-family="Times,serif" font-size="8.00">func1</text>
|
||||
<text text-anchor="middle" x="515" y="-670.6" font-family="Times,serif" font-size="8.00">0 of 38.15MB (47.61%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N4->N8 -->
|
||||
<g id="edge7" class="edge">
|
||||
<title>N4->N8</title>
|
||||
<g id="a_edge7"><a xlink:title="func2 -> func1 (38.15MB)">
|
||||
<path fill="none" stroke="#b22300" stroke-width="3" d="M515,-745.8C515,-734.16 515,-718.55 515,-705.24"/>
|
||||
<polygon fill="#b22300" stroke="#b22300" stroke-width="3" points="518.5,-705.18 515,-695.18 511.5,-705.18 518.5,-705.18"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge7-label"><a xlink:title="func2 -> func1 (38.15MB)">
|
||||
<text text-anchor="middle" x="543.5" y="-716.8" font-family="Times,serif" font-size="14.00"> 38.15MB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N6 -->
|
||||
<g id="node6" class="node">
|
||||
<title>N6</title>
|
||||
<g id="a_node6"><a xlink:title="fn (76.30MB)">
|
||||
<polygon fill="#edd5d5" stroke="#b20200" points="494.5,-521 401.5,-521 401.5,-485 494.5,-485 494.5,-521"/>
|
||||
<text text-anchor="middle" x="448" y="-505.6" font-family="Times,serif" font-size="8.00">fn</text>
|
||||
<text text-anchor="middle" x="448" y="-496.6" font-family="Times,serif" font-size="8.00">0 of 76.30MB (95.22%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N5->N6 -->
|
||||
<g id="edge5" class="edge">
|
||||
<title>N5->N6</title>
|
||||
<g id="a_edge5"><a xlink:title="deferring_binary_op -> fn (76.30MB)">
|
||||
<path fill="none" stroke="#b20200" stroke-width="5" d="M448,-571.8C448,-560.16 448,-544.55 448,-531.24"/>
|
||||
<polygon fill="#b20200" stroke="#b20200" stroke-width="5" points="452.38,-531.18 448,-521.18 443.63,-531.18 452.38,-531.18"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge5-label"><a xlink:title="deferring_binary_op -> fn (76.30MB)">
|
||||
<text text-anchor="middle" x="476.5" y="-542.8" font-family="Times,serif" font-size="14.00"> 76.30MB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N10 -->
|
||||
<g id="node10" class="node">
|
||||
<title>N10</title>
|
||||
<g id="a_node10"><a xlink:title="add (38.15MB)">
|
||||
<polygon fill="#eddad5" stroke="#b22300" points="411.5,-434 318.5,-434 318.5,-398 411.5,-398 411.5,-434"/>
|
||||
<text text-anchor="middle" x="365" y="-418.6" font-family="Times,serif" font-size="8.00">add</text>
|
||||
<text text-anchor="middle" x="365" y="-409.6" font-family="Times,serif" font-size="8.00">0 of 38.15MB (47.61%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N6->N10 -->
|
||||
<g id="edge9" class="edge">
|
||||
<title>N6->N10</title>
|
||||
<g id="a_edge9"><a xlink:title="fn -> add (38.15MB)">
|
||||
<path fill="none" stroke="#b22300" stroke-width="3" d="M421.33,-484.94C414.01,-479.64 406.36,-473.48 400,-467 392.92,-459.78 386.25,-450.99 380.67,-442.81"/>
|
||||
<polygon fill="#b22300" stroke="#b22300" stroke-width="3" points="383.49,-440.72 375.08,-434.27 377.63,-444.55 383.49,-440.72"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge9-label"><a xlink:title="fn -> add (38.15MB)">
|
||||
<text text-anchor="middle" x="428.5" y="-455.8" font-family="Times,serif" font-size="14.00"> 38.15MB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N14 -->
|
||||
<g id="node14" class="node">
|
||||
<title>N14</title>
|
||||
<g id="a_node14"><a xlink:title="mul (38.15MB)">
|
||||
<polygon fill="#eddad5" stroke="#b22300" points="522.5,-434 429.5,-434 429.5,-398 522.5,-398 522.5,-434"/>
|
||||
<text text-anchor="middle" x="476" y="-418.6" font-family="Times,serif" font-size="8.00">mul</text>
|
||||
<text text-anchor="middle" x="476" y="-409.6" font-family="Times,serif" font-size="8.00">0 of 38.15MB (47.61%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N6->N14 -->
|
||||
<g id="edge10" class="edge">
|
||||
<title>N6->N14</title>
|
||||
<g id="a_edge10"><a xlink:title="fn -> mul (38.15MB)">
|
||||
<path fill="none" stroke="#b22300" stroke-width="3" d="M453.67,-484.8C457.54,-473.05 462.75,-457.24 467.16,-443.84"/>
|
||||
<polygon fill="#b22300" stroke="#b22300" stroke-width="3" points="470.54,-444.77 470.34,-434.18 463.89,-442.58 470.54,-444.77"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge10-label"><a xlink:title="fn -> mul (38.15MB)">
|
||||
<text text-anchor="middle" x="492.5" y="-455.8" font-family="Times,serif" font-size="14.00"> 38.15MB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N7 -->
|
||||
<g id="node7" class="node">
|
||||
<title>N7</title>
|
||||
<g id="a_node7"><a xlink:title="_execute_compiled (3.81MB)">
|
||||
<polygon fill="#edeae7" stroke="#b2a188" points="677,-434 569,-434 569,-398 677,-398 677,-434"/>
|
||||
<text text-anchor="middle" x="623" y="-419.4" font-family="Times,serif" font-size="12.00">_execute_compiled</text>
|
||||
<text text-anchor="middle" x="623" y="-406.4" font-family="Times,serif" font-size="12.00">3.81MB (4.76%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N7_0 -->
|
||||
<g id="N7_0" class="node">
|
||||
<title>N7_0</title>
|
||||
<g id="a_N7_0"><a xlink:title="3.81MB">
|
||||
<polygon fill="#f8f8f8" stroke="black" points="650,-347 600,-347 596,-343 596,-311 646,-311 650,-315 650,-347"/>
|
||||
<polyline fill="none" stroke="black" points="646,-343 596,-343 "/>
|
||||
<polyline fill="none" stroke="black" points="646,-343 646,-311 "/>
|
||||
<polyline fill="none" stroke="black" points="646,-343 650,-347 "/>
|
||||
<text text-anchor="middle" x="623" y="-327.1" font-family="Times,serif" font-size="8.00">kind:buffer</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N7->N7_0 -->
|
||||
<g id="edge2" class="edge">
|
||||
<title>N7->N7_0</title>
|
||||
<g id="a_edge2"><a xlink:title="3.81MB">
|
||||
<path fill="none" stroke="black" d="M623,-397.8C623,-386.16 623,-370.55 623,-357.24"/>
|
||||
<polygon fill="black" stroke="black" points="626.5,-357.18 623,-347.18 619.5,-357.18 626.5,-357.18"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge2-label"><a xlink:title="3.81MB">
|
||||
<text text-anchor="middle" x="648" y="-368.8" font-family="Times,serif" font-size="14.00"> 3.81MB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N8->N5 -->
|
||||
<g id="edge11" class="edge">
|
||||
<title>N8->N5</title>
|
||||
<g id="a_edge11"><a xlink:title="func1 -> deferring_binary_op (38.15MB)">
|
||||
<path fill="none" stroke="#b22300" stroke-width="3" d="M501.44,-658.8C491.82,-646.59 478.73,-629.99 467.92,-616.28"/>
|
||||
<polygon fill="#b22300" stroke="#b22300" stroke-width="3" points="470.48,-613.86 461.54,-608.18 464.98,-618.2 470.48,-613.86"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge11-label"><a xlink:title="func1 -> deferring_binary_op (38.15MB)">
|
||||
<text text-anchor="middle" x="514.5" y="-629.8" font-family="Times,serif" font-size="14.00"> 38.15MB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N9 -->
|
||||
<g id="node9" class="node">
|
||||
<title>N9</title>
|
||||
<g id="a_node9"><a xlink:title="_xla_call_impl (3.83MB)">
|
||||
<polygon fill="#edeae7" stroke="#b2a187" points="666,-521 580,-521 580,-485 666,-485 666,-521"/>
|
||||
<text text-anchor="middle" x="623" y="-505.6" font-family="Times,serif" font-size="8.00">_xla_call_impl</text>
|
||||
<text text-anchor="middle" x="623" y="-496.6" font-family="Times,serif" font-size="8.00">0 of 3.83MB (4.78%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N9->N7 -->
|
||||
<g id="edge18" class="edge">
|
||||
<title>N9->N7</title>
|
||||
<g id="a_edge18"><a xlink:title="_xla_call_impl -> _execute_compiled (3.81MB)">
|
||||
<path fill="none" stroke="#b2a188" d="M623,-484.8C623,-473.16 623,-457.55 623,-444.24"/>
|
||||
<polygon fill="#b2a188" stroke="#b2a188" points="626.5,-444.18 623,-434.18 619.5,-444.18 626.5,-444.18"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge18-label"><a xlink:title="_xla_call_impl -> _execute_compiled (3.81MB)">
|
||||
<text text-anchor="middle" x="648" y="-455.8" font-family="Times,serif" font-size="14.00"> 3.81MB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N10->N3 -->
|
||||
<g id="edge8" class="edge">
|
||||
<title>N10->N3</title>
|
||||
<g id="a_edge8"><a xlink:title="add -> bind (38.15MB)">
|
||||
<path fill="none" stroke="#b22300" stroke-width="3" d="M368.72,-397.94C371.42,-387.86 375.7,-375.12 382,-365 384.25,-361.39 386.93,-357.86 389.81,-354.53"/>
|
||||
<polygon fill="#b22300" stroke="#b22300" stroke-width="3" points="392.45,-356.83 396.73,-347.14 387.34,-352.05 392.45,-356.83"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge8-label"><a xlink:title="add -> bind (38.15MB)">
|
||||
<text text-anchor="middle" x="410.5" y="-368.8" font-family="Times,serif" font-size="14.00"> 38.15MB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N11->N1 -->
|
||||
<g id="edge6" class="edge">
|
||||
<title>N11->N1</title>
|
||||
<g id="a_edge6"><a xlink:title="apply_primitive -> _execute_compiled_primitive (76.29MB)">
|
||||
<path fill="none" stroke="#b20200" stroke-width="5" d="M417,-223.81C417,-212.82 417,-197.91 417,-183.26"/>
|
||||
<polygon fill="#b20200" stroke="#b20200" stroke-width="5" points="421.38,-183.21 417,-173.21 412.63,-183.21 421.38,-183.21"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge6-label"><a xlink:title="apply_primitive -> _execute_compiled_primitive (76.29MB)">
|
||||
<text text-anchor="middle" x="445.5" y="-194.8" font-family="Times,serif" font-size="14.00"> 76.29MB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N12 -->
|
||||
<g id="node12" class="node">
|
||||
<title>N12</title>
|
||||
<g id="a_node12"><a xlink:title="call_bind (3.83MB)">
|
||||
<polygon fill="#edeae7" stroke="#b2a187" points="666,-608 580,-608 580,-572 666,-572 666,-608"/>
|
||||
<text text-anchor="middle" x="623" y="-592.6" font-family="Times,serif" font-size="8.00">call_bind</text>
|
||||
<text text-anchor="middle" x="623" y="-583.6" font-family="Times,serif" font-size="8.00">0 of 3.83MB (4.78%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N12->N9 -->
|
||||
<g id="edge15" class="edge">
|
||||
<title>N12->N9</title>
|
||||
<g id="a_edge15"><a xlink:title="call_bind -> _xla_call_impl (3.83MB)">
|
||||
<path fill="none" stroke="#b2a187" d="M623,-571.8C623,-560.16 623,-544.55 623,-531.24"/>
|
||||
<polygon fill="#b2a187" stroke="#b2a187" points="626.5,-531.18 623,-521.18 619.5,-531.18 626.5,-531.18"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge15-label"><a xlink:title="call_bind -> _xla_call_impl (3.83MB)">
|
||||
<text text-anchor="middle" x="648" y="-542.8" font-family="Times,serif" font-size="14.00"> 3.83MB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N13 -->
|
||||
<g id="node13" class="node">
|
||||
<title>N13</title>
|
||||
<g id="a_node13"><a xlink:title="f_jitted (3.83MB)">
|
||||
<polygon fill="#edeae7" stroke="#b2a187" points="666,-695 580,-695 580,-659 666,-659 666,-695"/>
|
||||
<text text-anchor="middle" x="623" y="-679.6" font-family="Times,serif" font-size="8.00">f_jitted</text>
|
||||
<text text-anchor="middle" x="623" y="-670.6" font-family="Times,serif" font-size="8.00">0 of 3.83MB (4.78%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N13->N12 -->
|
||||
<g id="edge16" class="edge">
|
||||
<title>N13->N12</title>
|
||||
<g id="a_edge16"><a xlink:title="f_jitted -> call_bind (3.83MB)">
|
||||
<path fill="none" stroke="#b2a187" d="M623,-658.8C623,-647.16 623,-631.55 623,-618.24"/>
|
||||
<polygon fill="#b2a187" stroke="#b2a187" points="626.5,-618.18 623,-608.18 619.5,-618.18 626.5,-618.18"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge16-label"><a xlink:title="f_jitted -> call_bind (3.83MB)">
|
||||
<text text-anchor="middle" x="648" y="-629.8" font-family="Times,serif" font-size="14.00"> 3.83MB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N14->N3 -->
|
||||
<g id="edge13" class="edge">
|
||||
<title>N14->N3</title>
|
||||
<g id="a_edge13"><a xlink:title="mul -> bind (38.15MB)">
|
||||
<path fill="none" stroke="#b22300" stroke-width="3" d="M464.06,-397.8C455.66,-385.7 444.28,-369.3 434.82,-355.67"/>
|
||||
<polygon fill="#b22300" stroke="#b22300" stroke-width="3" points="437.5,-353.39 428.92,-347.18 431.75,-357.39 437.5,-353.39"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge13-label"><a xlink:title="mul -> bind (38.15MB)">
|
||||
<text text-anchor="middle" x="479.5" y="-368.8" font-family="Times,serif" font-size="14.00"> 38.15MB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N15->N13 -->
|
||||
<g id="edge17" class="edge">
|
||||
<title>N15->N13</title>
|
||||
<g id="a_edge17"><a xlink:title="normal -> f_jitted (3.83MB)">
|
||||
<path fill="none" stroke="#b2a187" d="M623,-745.8C623,-734.16 623,-718.55 623,-705.24"/>
|
||||
<polygon fill="#b2a187" stroke="#b2a187" points="626.5,-705.18 623,-695.18 619.5,-705.18 626.5,-705.18"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge17-label"><a xlink:title="normal -> f_jitted (3.83MB)">
|
||||
<text text-anchor="middle" x="648" y="-716.8" font-family="Times,serif" font-size="14.00"> 3.83MB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 20 KiB |
307
docs/_static/device_memory_profile_leak1.svg
vendored
Normal file
307
docs/_static/device_memory_profile_leak1.svg
vendored
Normal file
@ -0,0 +1,307 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Generated by graphviz version 2.44.0 (20200408.0750)
|
||||
-->
|
||||
<!-- Title: unnamed Pages: 1 -->
|
||||
<svg width="791pt" height="851pt"
|
||||
viewBox="0.00 0.00 791.00 851.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 847)">
|
||||
<title>unnamed</title>
|
||||
<polygon fill="white" stroke="transparent" points="-4,4 -4,-847 787,-847 787,4 -4,4"/>
|
||||
<g id="clust1" class="cluster">
|
||||
<title>cluster_L</title>
|
||||
<polygon fill="none" stroke="black" points="8,-720 8,-835 496,-835 496,-720 8,-720"/>
|
||||
</g>
|
||||
<!-- Type: space -->
|
||||
<g id="node1" class="node">
|
||||
<title>Type: space</title>
|
||||
<polygon fill="#f8f8f8" stroke="black" points="487.5,-827 16.5,-827 16.5,-728 487.5,-728 487.5,-827"/>
|
||||
<text text-anchor="start" x="24.5" y="-810.2" font-family="Times,serif" font-size="16.00">Type: space</text>
|
||||
<text text-anchor="start" x="24.5" y="-792.2" font-family="Times,serif" font-size="16.00">Showing nodes accounting for 5806.95kB, 99.79% of 5819.22kB total</text>
|
||||
<text text-anchor="start" x="24.5" y="-774.2" font-family="Times,serif" font-size="16.00">Dropped 25 nodes (cum <= 29.10kB)</text>
|
||||
<text text-anchor="start" x="24.5" y="-737.2" font-family="Times,serif" font-size="16.00">See https://git.io/JfYMW for how to read the graph</text>
|
||||
</g>
|
||||
<!-- N1 -->
|
||||
<g id="node1" class="node">
|
||||
<title>N1</title>
|
||||
<g id="a_node1"><a xlink:title="_execute_compiled (5664.06kB)">
|
||||
<polygon fill="#edd5d5" stroke="#b20100" points="661.5,-242 442.5,-242 442.5,-182 661.5,-182 661.5,-242"/>
|
||||
<text text-anchor="middle" x="552" y="-218.8" font-family="Times,serif" font-size="24.00">_execute_compiled</text>
|
||||
<text text-anchor="middle" x="552" y="-192.8" font-family="Times,serif" font-size="24.00">5664.06kB (97.33%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N1_0 -->
|
||||
<g id="N1_0" class="node">
|
||||
<title>N1_0</title>
|
||||
<g id="a_N1_0"><a xlink:title="5664.06kB">
|
||||
<polygon fill="#f8f8f8" stroke="black" points="579,-127 529,-127 525,-123 525,-91 575,-91 579,-95 579,-127"/>
|
||||
<polyline fill="none" stroke="black" points="575,-123 525,-123 "/>
|
||||
<polyline fill="none" stroke="black" points="575,-123 575,-91 "/>
|
||||
<polyline fill="none" stroke="black" points="575,-123 579,-127 "/>
|
||||
<text text-anchor="middle" x="552" y="-107.1" font-family="Times,serif" font-size="8.00">kind:buffer</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N1->N1_0 -->
|
||||
<g id="edge1" class="edge">
|
||||
<title>N1->N1_0</title>
|
||||
<g id="a_edge1"><a xlink:title="5664.06kB">
|
||||
<path fill="none" stroke="black" d="M552,-181.79C552,-167.81 552,-151.17 552,-137.44"/>
|
||||
<polygon fill="black" stroke="black" points="555.5,-137.15 552,-127.15 548.5,-137.15 555.5,-137.15"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge1-label"><a xlink:title="5664.06kB">
|
||||
<text text-anchor="middle" x="584" y="-152.8" font-family="Times,serif" font-size="14.00"> 5664.06kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N2 -->
|
||||
<g id="node2" class="node">
|
||||
<title>N2</title>
|
||||
<g id="a_node2"><a xlink:title="<unknown> (5819.22kB)">
|
||||
<polygon fill="#edd5d5" stroke="#b20000" points="598,-795.5 506,-795.5 506,-759.5 598,-759.5 598,-795.5"/>
|
||||
<text text-anchor="middle" x="552" y="-780.1" font-family="Times,serif" font-size="8.00"><unknown></text>
|
||||
<text text-anchor="middle" x="552" y="-771.1" font-family="Times,serif" font-size="8.00">0 of 5819.22kB (100%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N6 -->
|
||||
<g id="node6" class="node">
|
||||
<title>N6</title>
|
||||
<g id="a_node6"><a xlink:title="afunction (3925.29kB)">
|
||||
<polygon fill="#edd8d5" stroke="#b21300" points="601,-677 503,-677 503,-641 601,-641 601,-677"/>
|
||||
<text text-anchor="middle" x="552" y="-661.6" font-family="Times,serif" font-size="8.00">afunction</text>
|
||||
<text text-anchor="middle" x="552" y="-652.6" font-family="Times,serif" font-size="8.00">0 of 3925.29kB (67.45%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N2->N6 -->
|
||||
<g id="edge7" class="edge">
|
||||
<title>N2->N6</title>
|
||||
<g id="a_edge7"><a xlink:title="<unknown> -> afunction (3925.29kB)">
|
||||
<path fill="none" stroke="#b21300" stroke-width="4" d="M552,-759.31C552,-740.38 552,-709.68 552,-687.42"/>
|
||||
<polygon fill="#b21300" stroke="#b21300" stroke-width="4" points="555.5,-687.26 552,-677.26 548.5,-687.26 555.5,-687.26"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge7-label"><a xlink:title="<unknown> -> afunction (3925.29kB)">
|
||||
<text text-anchor="middle" x="584" y="-698.8" font-family="Times,serif" font-size="14.00"> 3925.29kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N7 -->
|
||||
<g id="node7" class="node">
|
||||
<title>N7</title>
|
||||
<g id="a_node7"><a xlink:title="anotherfunc (1893.93kB)">
|
||||
<polygon fill="#eddcd5" stroke="#b23300" points="717,-677 619,-677 619,-641 717,-641 717,-677"/>
|
||||
<text text-anchor="middle" x="668" y="-661.6" font-family="Times,serif" font-size="8.00">anotherfunc</text>
|
||||
<text text-anchor="middle" x="668" y="-652.6" font-family="Times,serif" font-size="8.00">0 of 1893.93kB (32.55%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N2->N7 -->
|
||||
<g id="edge9" class="edge">
|
||||
<title>N2->N7</title>
|
||||
<g id="a_edge9"><a xlink:title="<unknown> -> anotherfunc (1893.93kB)">
|
||||
<path fill="none" stroke="#b23300" stroke-width="2" d="M569.76,-759.43C583.57,-746.08 603.15,-727.01 620,-710 628,-701.92 636.64,-692.99 644.35,-684.95"/>
|
||||
<polygon fill="#b23300" stroke="#b23300" stroke-width="2" points="647.32,-686.9 651.69,-677.25 642.26,-682.07 647.32,-686.9"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge9-label"><a xlink:title="<unknown> -> anotherfunc (1893.93kB)">
|
||||
<text text-anchor="middle" x="666" y="-698.8" font-family="Times,serif" font-size="14.00"> 1893.93kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N3 -->
|
||||
<g id="node3" class="node">
|
||||
<title>N3</title>
|
||||
<g id="a_node3"><a xlink:title="normal (5817.23kB)">
|
||||
<polygon fill="#edd5d5" stroke="#b20000" points="598,-590 506,-590 506,-554 598,-554 598,-590"/>
|
||||
<text text-anchor="middle" x="552" y="-574.6" font-family="Times,serif" font-size="8.00">normal</text>
|
||||
<text text-anchor="middle" x="552" y="-565.6" font-family="Times,serif" font-size="8.00">0 of 5817.23kB (100%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N9 -->
|
||||
<g id="node9" class="node">
|
||||
<title>N9</title>
|
||||
<g id="a_node9"><a xlink:title="f_jitted (5817.23kB)">
|
||||
<polygon fill="#edd5d5" stroke="#b20000" points="598,-503 506,-503 506,-467 598,-467 598,-503"/>
|
||||
<text text-anchor="middle" x="552" y="-487.6" font-family="Times,serif" font-size="8.00">f_jitted</text>
|
||||
<text text-anchor="middle" x="552" y="-478.6" font-family="Times,serif" font-size="8.00">0 of 5817.23kB (100%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N3->N9 -->
|
||||
<g id="edge5" class="edge">
|
||||
<title>N3->N9</title>
|
||||
<g id="a_edge5"><a xlink:title="normal -> f_jitted (5817.23kB)">
|
||||
<path fill="none" stroke="#b20000" stroke-width="5" d="M552,-553.8C552,-542.16 552,-526.55 552,-513.24"/>
|
||||
<polygon fill="#b20000" stroke="#b20000" stroke-width="5" points="556.38,-513.18 552,-503.18 547.63,-513.18 556.38,-513.18"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge5-label"><a xlink:title="normal -> f_jitted (5817.23kB)">
|
||||
<text text-anchor="middle" x="584" y="-524.8" font-family="Times,serif" font-size="14.00"> 5817.23kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N4 -->
|
||||
<g id="node4" class="node">
|
||||
<title>N4</title>
|
||||
<g id="a_node4"><a xlink:title="_xla_call_impl (5817.23kB)">
|
||||
<polygon fill="#edd5d5" stroke="#b20000" points="598,-329 506,-329 506,-293 598,-293 598,-329"/>
|
||||
<text text-anchor="middle" x="552" y="-313.6" font-family="Times,serif" font-size="8.00">_xla_call_impl</text>
|
||||
<text text-anchor="middle" x="552" y="-304.6" font-family="Times,serif" font-size="8.00">0 of 5817.23kB (100%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N4->N1 -->
|
||||
<g id="edge6" class="edge">
|
||||
<title>N4->N1</title>
|
||||
<g id="a_edge6"><a xlink:title="_xla_call_impl -> _execute_compiled (5664.06kB)">
|
||||
<path fill="none" stroke="#b20100" stroke-width="5" d="M552,-292.66C552,-281.54 552,-266.64 552,-252.73"/>
|
||||
<polygon fill="#b20100" stroke="#b20100" stroke-width="5" points="556.38,-252.37 552,-242.37 547.63,-252.37 556.38,-252.37"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge6-label"><a xlink:title="_xla_call_impl -> _execute_compiled (5664.06kB)">
|
||||
<text text-anchor="middle" x="584" y="-263.8" font-family="Times,serif" font-size="14.00"> 5664.06kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N10 -->
|
||||
<g id="node10" class="node">
|
||||
<title>N10</title>
|
||||
<g id="a_node10"><a xlink:title="memoized_fun (153.16kB)">
|
||||
<polygon fill="#edecea" stroke="#b2aa9b" points="770,-230 680,-230 680,-194 770,-194 770,-230"/>
|
||||
<text text-anchor="middle" x="725" y="-214.6" font-family="Times,serif" font-size="8.00">memoized_fun</text>
|
||||
<text text-anchor="middle" x="725" y="-205.6" font-family="Times,serif" font-size="8.00">0 of 153.16kB (2.63%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N4->N10 -->
|
||||
<g id="edge11" class="edge">
|
||||
<title>N4->N10</title>
|
||||
<g id="a_edge11"><a xlink:title="_xla_call_impl -> memoized_fun (153.16kB)">
|
||||
<path fill="none" stroke="#b2aa9b" d="M586.1,-292.94C597,-287.35 609.07,-281.03 620,-275 642.98,-262.31 668.34,-247.37 688.37,-235.33"/>
|
||||
<polygon fill="#b2aa9b" stroke="#b2aa9b" points="690.42,-238.19 697.18,-230.02 686.81,-232.19 690.42,-238.19"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge11-label"><a xlink:title="_xla_call_impl -> memoized_fun (153.16kB)">
|
||||
<text text-anchor="middle" x="675" y="-263.8" font-family="Times,serif" font-size="14.00"> 153.16kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N5 -->
|
||||
<g id="node5" class="node">
|
||||
<title>N5</title>
|
||||
<g id="a_node5"><a xlink:title="_xla_callable (153.16kB)">
|
||||
<polygon fill="#edecea" stroke="#b2aa9b" points="779.5,-131 670.5,-131 670.5,-87 779.5,-87 779.5,-131"/>
|
||||
<text text-anchor="middle" x="725" y="-118.2" font-family="Times,serif" font-size="11.00">_xla_callable</text>
|
||||
<text text-anchor="middle" x="725" y="-106.2" font-family="Times,serif" font-size="11.00">142.89kB (2.46%)</text>
|
||||
<text text-anchor="middle" x="725" y="-94.2" font-family="Times,serif" font-size="11.00">of 153.16kB (2.63%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N5_0 -->
|
||||
<g id="N5_0" class="node">
|
||||
<title>N5_0</title>
|
||||
<g id="a_N5_0"><a xlink:title="153.16kB">
|
||||
<polygon fill="#f8f8f8" stroke="black" points="758.5,-36 695.5,-36 691.5,-32 691.5,0 754.5,0 758.5,-4 758.5,-36"/>
|
||||
<polyline fill="none" stroke="black" points="754.5,-32 691.5,-32 "/>
|
||||
<polyline fill="none" stroke="black" points="754.5,-32 754.5,0 "/>
|
||||
<polyline fill="none" stroke="black" points="754.5,-32 758.5,-36 "/>
|
||||
<text text-anchor="middle" x="725" y="-16.1" font-family="Times,serif" font-size="8.00">kind:executable</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N5->N5_0 -->
|
||||
<g id="edge2" class="edge">
|
||||
<title>N5->N5_0</title>
|
||||
<g id="a_edge2"><a xlink:title="153.16kB">
|
||||
<path fill="none" stroke="black" d="M725,-86.91C725,-74.78 725,-59.41 725,-46.36"/>
|
||||
<polygon fill="black" stroke="black" points="728.5,-46.07 725,-36.07 721.5,-46.07 728.5,-46.07"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge2-label"><a xlink:title="153.16kB">
|
||||
<text text-anchor="middle" x="754" y="-57.8" font-family="Times,serif" font-size="14.00"> 153.16kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N6->N3 -->
|
||||
<g id="edge8" class="edge">
|
||||
<title>N6->N3</title>
|
||||
<g id="a_edge8"><a xlink:title="afunction -> normal (3923.30kB)">
|
||||
<path fill="none" stroke="#b21300" stroke-width="4" d="M552,-640.8C552,-629.16 552,-613.55 552,-600.24"/>
|
||||
<polygon fill="#b21300" stroke="#b21300" stroke-width="4" points="555.5,-600.18 552,-590.18 548.5,-600.18 555.5,-600.18"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge8-label"><a xlink:title="afunction -> normal (3923.30kB)">
|
||||
<text text-anchor="middle" x="584" y="-611.8" font-family="Times,serif" font-size="14.00"> 3923.30kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N7->N3 -->
|
||||
<g id="edge10" class="edge">
|
||||
<title>N7->N3</title>
|
||||
<g id="a_edge10"><a xlink:title="anotherfunc -> normal (1893.93kB)">
|
||||
<path fill="none" stroke="#b23300" stroke-width="2" d="M654.09,-640.85C645.1,-630.49 632.73,-617.49 620,-608 613.71,-603.31 606.71,-598.93 599.65,-594.95"/>
|
||||
<polygon fill="#b23300" stroke="#b23300" stroke-width="2" points="601.19,-591.81 590.73,-590.14 597.87,-597.97 601.19,-591.81"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge10-label"><a xlink:title="anotherfunc -> normal (1893.93kB)">
|
||||
<text text-anchor="middle" x="669" y="-611.8" font-family="Times,serif" font-size="14.00"> 1893.93kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N8 -->
|
||||
<g id="node8" class="node">
|
||||
<title>N8</title>
|
||||
<g id="a_node8"><a xlink:title="call_bind (5817.23kB)">
|
||||
<polygon fill="#edd5d5" stroke="#b20000" points="598,-416 506,-416 506,-380 598,-380 598,-416"/>
|
||||
<text text-anchor="middle" x="552" y="-400.6" font-family="Times,serif" font-size="8.00">call_bind</text>
|
||||
<text text-anchor="middle" x="552" y="-391.6" font-family="Times,serif" font-size="8.00">0 of 5817.23kB (100%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N8->N4 -->
|
||||
<g id="edge3" class="edge">
|
||||
<title>N8->N4</title>
|
||||
<g id="a_edge3"><a xlink:title="call_bind -> _xla_call_impl (5817.23kB)">
|
||||
<path fill="none" stroke="#b20000" stroke-width="5" d="M552,-379.8C552,-368.16 552,-352.55 552,-339.24"/>
|
||||
<polygon fill="#b20000" stroke="#b20000" stroke-width="5" points="556.38,-339.18 552,-329.18 547.63,-339.18 556.38,-339.18"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge3-label"><a xlink:title="call_bind -> _xla_call_impl (5817.23kB)">
|
||||
<text text-anchor="middle" x="584" y="-350.8" font-family="Times,serif" font-size="14.00"> 5817.23kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N9->N8 -->
|
||||
<g id="edge4" class="edge">
|
||||
<title>N9->N8</title>
|
||||
<g id="a_edge4"><a xlink:title="f_jitted -> call_bind (5817.23kB)">
|
||||
<path fill="none" stroke="#b20000" stroke-width="5" d="M552,-466.8C552,-455.16 552,-439.55 552,-426.24"/>
|
||||
<polygon fill="#b20000" stroke="#b20000" stroke-width="5" points="556.38,-426.18 552,-416.18 547.63,-426.18 556.38,-426.18"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge4-label"><a xlink:title="f_jitted -> call_bind (5817.23kB)">
|
||||
<text text-anchor="middle" x="584" y="-437.8" font-family="Times,serif" font-size="14.00"> 5817.23kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N10->N5 -->
|
||||
<g id="edge12" class="edge">
|
||||
<title>N10->N5</title>
|
||||
<g id="a_edge12"><a xlink:title="memoized_fun -> _xla_callable (153.16kB)">
|
||||
<path fill="none" stroke="#b2aa9b" d="M725,-193.87C725,-179.51 725,-158.65 725,-141.32"/>
|
||||
<polygon fill="#b2aa9b" stroke="#b2aa9b" points="728.5,-141.08 725,-131.08 721.5,-141.08 728.5,-141.08"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge12-label"><a xlink:title="memoized_fun -> _xla_callable (153.16kB)">
|
||||
<text text-anchor="middle" x="754" y="-152.8" font-family="Times,serif" font-size="14.00"> 153.16kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 14 KiB |
271
docs/_static/device_memory_profile_leak2.svg
vendored
Normal file
271
docs/_static/device_memory_profile_leak2.svg
vendored
Normal file
@ -0,0 +1,271 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Generated by graphviz version 2.44.0 (20200408.0750)
|
||||
-->
|
||||
<!-- Title: unnamed Pages: 1 -->
|
||||
<svg width="801pt" height="857pt"
|
||||
viewBox="0.00 0.00 800.50 857.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 853)">
|
||||
<title>unnamed</title>
|
||||
<polygon fill="white" stroke="transparent" points="-4,4 -4,-853 796.5,-853 796.5,4 -4,4"/>
|
||||
<g id="clust1" class="cluster">
|
||||
<title>cluster_L</title>
|
||||
<polygon fill="none" stroke="black" points="8,-726 8,-841 496,-841 496,-726 8,-726"/>
|
||||
</g>
|
||||
<!-- Type: space -->
|
||||
<g id="node1" class="node">
|
||||
<title>Type: space</title>
|
||||
<polygon fill="#f8f8f8" stroke="black" points="487.5,-833 16.5,-833 16.5,-734 487.5,-734 487.5,-833"/>
|
||||
<text text-anchor="start" x="24.5" y="-816.2" font-family="Times,serif" font-size="16.00">Type: space</text>
|
||||
<text text-anchor="start" x="24.5" y="-798.2" font-family="Times,serif" font-size="16.00">Showing nodes accounting for 1832.09kB, 46.05% of 3978.91kB total</text>
|
||||
<text text-anchor="start" x="24.5" y="-780.2" font-family="Times,serif" font-size="16.00">Dropped 13 nodes (cum <= 19.89kB)</text>
|
||||
<text text-anchor="start" x="24.5" y="-743.2" font-family="Times,serif" font-size="16.00">See https://git.io/JfYMW for how to read the graph</text>
|
||||
</g>
|
||||
<!-- N1 -->
|
||||
<g id="node1" class="node">
|
||||
<title>N1</title>
|
||||
<g id="a_node1"><a xlink:title="_execute_compiled (1718.75kB)">
|
||||
<polygon fill="#eddad5" stroke="#b22700" points="664.5,-248 445.5,-248 445.5,-188 664.5,-188 664.5,-248"/>
|
||||
<text text-anchor="middle" x="555" y="-224.8" font-family="Times,serif" font-size="24.00">_execute_compiled</text>
|
||||
<text text-anchor="middle" x="555" y="-198.8" font-family="Times,serif" font-size="24.00">1718.75kB (43.20%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N1_0 -->
|
||||
<g id="N1_0" class="node">
|
||||
<title>N1_0</title>
|
||||
<g id="a_N1_0"><a xlink:title="1718.75kB">
|
||||
<polygon fill="#f8f8f8" stroke="black" points="582,-130 532,-130 528,-126 528,-94 578,-94 582,-98 582,-130"/>
|
||||
<polyline fill="none" stroke="black" points="578,-126 528,-126 "/>
|
||||
<polyline fill="none" stroke="black" points="578,-126 578,-94 "/>
|
||||
<polyline fill="none" stroke="black" points="578,-126 582,-130 "/>
|
||||
<text text-anchor="middle" x="555" y="-110.1" font-family="Times,serif" font-size="8.00">kind:buffer</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N1->N1_0 -->
|
||||
<g id="edge1" class="edge">
|
||||
<title>N1->N1_0</title>
|
||||
<g id="a_edge1"><a xlink:title="1718.75kB">
|
||||
<path fill="none" stroke="black" d="M555,-187.75C555,-172.91 555,-154.99 555,-140.42"/>
|
||||
<polygon fill="black" stroke="black" points="558.5,-140.03 555,-130.03 551.5,-140.03 558.5,-140.03"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge1-label"><a xlink:title="1718.75kB">
|
||||
<text text-anchor="middle" x="587" y="-158.8" font-family="Times,serif" font-size="14.00"> 1718.75kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N2 -->
|
||||
<g id="node2" class="node">
|
||||
<title>N2</title>
|
||||
<g id="a_node2"><a xlink:title="_xla_call_impl (1840.31kB)">
|
||||
<polygon fill="#eddad5" stroke="#b22400" points="604,-335 506,-335 506,-299 604,-299 604,-335"/>
|
||||
<text text-anchor="middle" x="555" y="-319.6" font-family="Times,serif" font-size="8.00">_xla_call_impl</text>
|
||||
<text text-anchor="middle" x="555" y="-310.6" font-family="Times,serif" font-size="8.00">0 of 1840.31kB (46.25%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N2->N1 -->
|
||||
<g id="edge8" class="edge">
|
||||
<title>N2->N1</title>
|
||||
<g id="a_edge8"><a xlink:title="_xla_call_impl -> _execute_compiled (1718.75kB)">
|
||||
<path fill="none" stroke="#b22700" stroke-width="3" d="M555,-298.66C555,-287.54 555,-272.64 555,-258.73"/>
|
||||
<polygon fill="#b22700" stroke="#b22700" stroke-width="3" points="558.5,-258.37 555,-248.37 551.5,-258.37 558.5,-258.37"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge8-label"><a xlink:title="_xla_call_impl -> _execute_compiled (1718.75kB)">
|
||||
<text text-anchor="middle" x="587" y="-269.8" font-family="Times,serif" font-size="14.00"> 1718.75kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N7 -->
|
||||
<g id="node7" class="node">
|
||||
<title>N7</title>
|
||||
<g id="a_node7"><a xlink:title="memoized_fun (121.56kB)">
|
||||
<polygon fill="#edebe9" stroke="#b2a897" points="773,-236 683,-236 683,-200 773,-200 773,-236"/>
|
||||
<text text-anchor="middle" x="728" y="-220.6" font-family="Times,serif" font-size="8.00">memoized_fun</text>
|
||||
<text text-anchor="middle" x="728" y="-211.6" font-family="Times,serif" font-size="8.00">0 of 121.56kB (3.06%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N2->N7 -->
|
||||
<g id="edge9" class="edge">
|
||||
<title>N2->N7</title>
|
||||
<g id="a_edge9"><a xlink:title="_xla_call_impl -> memoized_fun (121.56kB)">
|
||||
<path fill="none" stroke="#b2a897" d="M589.1,-298.94C600,-293.35 612.07,-287.03 623,-281 645.98,-268.31 671.34,-253.37 691.37,-241.33"/>
|
||||
<polygon fill="#b2a897" stroke="#b2a897" points="693.42,-244.19 700.18,-236.02 689.81,-238.19 693.42,-244.19"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge9-label"><a xlink:title="_xla_call_impl -> memoized_fun (121.56kB)">
|
||||
<text text-anchor="middle" x="678" y="-269.8" font-family="Times,serif" font-size="14.00"> 121.56kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N3 -->
|
||||
<g id="node3" class="node">
|
||||
<title>N3</title>
|
||||
<g id="a_node3"><a xlink:title="_xla_callable (121.56kB)">
|
||||
<polygon fill="#edebe9" stroke="#b2a897" points="792.5,-137 663.5,-137 663.5,-87 792.5,-87 792.5,-137"/>
|
||||
<text text-anchor="middle" x="728" y="-122.6" font-family="Times,serif" font-size="13.00">_xla_callable</text>
|
||||
<text text-anchor="middle" x="728" y="-108.6" font-family="Times,serif" font-size="13.00">113.34kB (2.85%)</text>
|
||||
<text text-anchor="middle" x="728" y="-94.6" font-family="Times,serif" font-size="13.00">of 121.56kB (3.06%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N3_0 -->
|
||||
<g id="N3_0" class="node">
|
||||
<title>N3_0</title>
|
||||
<g id="a_N3_0"><a xlink:title="121.56kB">
|
||||
<polygon fill="#f8f8f8" stroke="black" points="761.5,-36 698.5,-36 694.5,-32 694.5,0 757.5,0 761.5,-4 761.5,-36"/>
|
||||
<polyline fill="none" stroke="black" points="757.5,-32 694.5,-32 "/>
|
||||
<polyline fill="none" stroke="black" points="757.5,-32 757.5,0 "/>
|
||||
<polyline fill="none" stroke="black" points="757.5,-32 761.5,-36 "/>
|
||||
<text text-anchor="middle" x="728" y="-16.1" font-family="Times,serif" font-size="8.00">kind:executable</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N3->N3_0 -->
|
||||
<g id="edge2" class="edge">
|
||||
<title>N3->N3_0</title>
|
||||
<g id="a_edge2"><a xlink:title="121.56kB">
|
||||
<path fill="none" stroke="black" d="M728,-86.83C728,-74.35 728,-59.08 728,-46.19"/>
|
||||
<polygon fill="black" stroke="black" points="731.5,-46.04 728,-36.04 724.5,-46.04 731.5,-46.04"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge2-label"><a xlink:title="121.56kB">
|
||||
<text text-anchor="middle" x="757" y="-57.8" font-family="Times,serif" font-size="14.00"> 121.56kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N4 -->
|
||||
<g id="node4" class="node">
|
||||
<title>N4</title>
|
||||
<g id="a_node4"><a xlink:title="anotherfunc (1840.31kB)">
|
||||
<polygon fill="#eddad5" stroke="#b22400" points="604,-683 506,-683 506,-647 604,-647 604,-683"/>
|
||||
<text text-anchor="middle" x="555" y="-667.6" font-family="Times,serif" font-size="8.00">anotherfunc</text>
|
||||
<text text-anchor="middle" x="555" y="-658.6" font-family="Times,serif" font-size="8.00">0 of 1840.31kB (46.25%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N9 -->
|
||||
<g id="node9" class="node">
|
||||
<title>N9</title>
|
||||
<g id="a_node9"><a xlink:title="normal (1840.31kB)">
|
||||
<polygon fill="#eddad5" stroke="#b22400" points="604,-596 506,-596 506,-560 604,-560 604,-596"/>
|
||||
<text text-anchor="middle" x="555" y="-580.6" font-family="Times,serif" font-size="8.00">normal</text>
|
||||
<text text-anchor="middle" x="555" y="-571.6" font-family="Times,serif" font-size="8.00">0 of 1840.31kB (46.25%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N4->N9 -->
|
||||
<g id="edge4" class="edge">
|
||||
<title>N4->N9</title>
|
||||
<g id="a_edge4"><a xlink:title="anotherfunc -> normal (1840.31kB)">
|
||||
<path fill="none" stroke="#b22400" stroke-width="3" d="M555,-646.8C555,-635.16 555,-619.55 555,-606.24"/>
|
||||
<polygon fill="#b22400" stroke="#b22400" stroke-width="3" points="558.5,-606.18 555,-596.18 551.5,-606.18 558.5,-606.18"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge4-label"><a xlink:title="anotherfunc -> normal (1840.31kB)">
|
||||
<text text-anchor="middle" x="587" y="-617.8" font-family="Times,serif" font-size="14.00"> 1840.31kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N5 -->
|
||||
<g id="node5" class="node">
|
||||
<title>N5</title>
|
||||
<g id="a_node5"><a xlink:title="call_bind (1840.31kB)">
|
||||
<polygon fill="#eddad5" stroke="#b22400" points="604,-422 506,-422 506,-386 604,-386 604,-422"/>
|
||||
<text text-anchor="middle" x="555" y="-406.6" font-family="Times,serif" font-size="8.00">call_bind</text>
|
||||
<text text-anchor="middle" x="555" y="-397.6" font-family="Times,serif" font-size="8.00">0 of 1840.31kB (46.25%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N5->N2 -->
|
||||
<g id="edge5" class="edge">
|
||||
<title>N5->N2</title>
|
||||
<g id="a_edge5"><a xlink:title="call_bind -> _xla_call_impl (1840.31kB)">
|
||||
<path fill="none" stroke="#b22400" stroke-width="3" d="M555,-385.8C555,-374.16 555,-358.55 555,-345.24"/>
|
||||
<polygon fill="#b22400" stroke="#b22400" stroke-width="3" points="558.5,-345.18 555,-335.18 551.5,-345.18 558.5,-345.18"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge5-label"><a xlink:title="call_bind -> _xla_call_impl (1840.31kB)">
|
||||
<text text-anchor="middle" x="587" y="-356.8" font-family="Times,serif" font-size="14.00"> 1840.31kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N6 -->
|
||||
<g id="node6" class="node">
|
||||
<title>N6</title>
|
||||
<g id="a_node6"><a xlink:title="f_jitted (1840.31kB)">
|
||||
<polygon fill="#eddad5" stroke="#b22400" points="604,-509 506,-509 506,-473 604,-473 604,-509"/>
|
||||
<text text-anchor="middle" x="555" y="-493.6" font-family="Times,serif" font-size="8.00">f_jitted</text>
|
||||
<text text-anchor="middle" x="555" y="-484.6" font-family="Times,serif" font-size="8.00">0 of 1840.31kB (46.25%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N6->N5 -->
|
||||
<g id="edge6" class="edge">
|
||||
<title>N6->N5</title>
|
||||
<g id="a_edge6"><a xlink:title="f_jitted -> call_bind (1840.31kB)">
|
||||
<path fill="none" stroke="#b22400" stroke-width="3" d="M555,-472.8C555,-461.16 555,-445.55 555,-432.24"/>
|
||||
<polygon fill="#b22400" stroke="#b22400" stroke-width="3" points="558.5,-432.18 555,-422.18 551.5,-432.18 558.5,-432.18"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge6-label"><a xlink:title="f_jitted -> call_bind (1840.31kB)">
|
||||
<text text-anchor="middle" x="587" y="-443.8" font-family="Times,serif" font-size="14.00"> 1840.31kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N7->N3 -->
|
||||
<g id="edge10" class="edge">
|
||||
<title>N7->N3</title>
|
||||
<g id="a_edge10"><a xlink:title="memoized_fun -> _xla_callable (121.56kB)">
|
||||
<path fill="none" stroke="#b2a897" d="M728,-199.83C728,-185.68 728,-165.18 728,-147.65"/>
|
||||
<polygon fill="#b2a897" stroke="#b2a897" points="731.5,-147.2 728,-137.2 724.5,-147.2 731.5,-147.2"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge10-label"><a xlink:title="memoized_fun -> _xla_callable (121.56kB)">
|
||||
<text text-anchor="middle" x="757" y="-158.8" font-family="Times,serif" font-size="14.00"> 121.56kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N8 -->
|
||||
<g id="node8" class="node">
|
||||
<title>N8</title>
|
||||
<g id="a_node8"><a xlink:title="<unknown> (1840.31kB)">
|
||||
<polygon fill="#eddad5" stroke="#b22400" points="604,-801.5 506,-801.5 506,-765.5 604,-765.5 604,-801.5"/>
|
||||
<text text-anchor="middle" x="555" y="-786.1" font-family="Times,serif" font-size="8.00"><unknown></text>
|
||||
<text text-anchor="middle" x="555" y="-777.1" font-family="Times,serif" font-size="8.00">0 of 1840.31kB (46.25%)</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N8->N4 -->
|
||||
<g id="edge3" class="edge">
|
||||
<title>N8->N4</title>
|
||||
<g id="a_edge3"><a xlink:title="<unknown> -> anotherfunc (1840.31kB)">
|
||||
<path fill="none" stroke="#b22400" stroke-width="3" d="M555,-765.31C555,-746.38 555,-715.68 555,-693.42"/>
|
||||
<polygon fill="#b22400" stroke="#b22400" stroke-width="3" points="558.5,-693.26 555,-683.26 551.5,-693.26 558.5,-693.26"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge3-label"><a xlink:title="<unknown> -> anotherfunc (1840.31kB)">
|
||||
<text text-anchor="middle" x="587" y="-704.8" font-family="Times,serif" font-size="14.00"> 1840.31kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- N9->N6 -->
|
||||
<g id="edge7" class="edge">
|
||||
<title>N9->N6</title>
|
||||
<g id="a_edge7"><a xlink:title="normal -> f_jitted (1840.31kB)">
|
||||
<path fill="none" stroke="#b22400" stroke-width="3" d="M555,-559.8C555,-548.16 555,-532.55 555,-519.24"/>
|
||||
<polygon fill="#b22400" stroke="#b22400" stroke-width="3" points="558.5,-519.18 555,-509.18 551.5,-519.18 558.5,-519.18"/>
|
||||
</a>
|
||||
</g>
|
||||
<g id="a_edge7-label"><a xlink:title="normal -> f_jitted (1840.31kB)">
|
||||
<text text-anchor="middle" x="587" y="-530.8" font-family="Times,serif" font-size="14.00"> 1840.31kB</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 12 KiB |
142
docs/device_memory_profiling.md
Normal file
142
docs/device_memory_profiling.md
Normal file
@ -0,0 +1,142 @@
|
||||
# Device Memory Profiling
|
||||
|
||||
The JAX Device Memory Profiler allows us to explore how and why JAX programs are
|
||||
using GPU or TPU memory. For example, it can be used to:
|
||||
|
||||
* Figure out which arrays and executables are in GPU memory at a given time, or
|
||||
* Track down memory leaks.
|
||||
|
||||
## Installation
|
||||
|
||||
The JAX device memory profiler emits output that can be interpreted using
|
||||
pprof (<https://github.com/google/pprof>). Start by installing `pprof`,
|
||||
by following its
|
||||
[installation instructions](https://github.com/google/pprof#building-pprof).
|
||||
At the time of writing, installing `pprof` requires first installing
|
||||
[Go](https://golang.org/) and [Graphviz](http://www.graphviz.org/), and then
|
||||
running
|
||||
|
||||
```shell
|
||||
go get -u github.com/google/pprof
|
||||
```
|
||||
|
||||
which installs `pprof` as `$GOPATH/bin/pprof`, where `GOPATH` defaults to
|
||||
`~/go`.
|
||||
|
||||
```{note}
|
||||
The version of `pprof` from <https://github.com/google/pprof> is not the same as
|
||||
the older tool of the same name distributed as part of the `gperftools` package.
|
||||
The `gperftools` version of `pprof` will not work with JAX.
|
||||
```
|
||||
|
||||
## Understanding how a JAX program is using GPU or TPU memory
|
||||
|
||||
A common use of the device memory profiler is to figure out why a JAX program is
|
||||
using a large amount of GPU or TPU memory, for example if trying to debug an
|
||||
out-of-memory problem.
|
||||
|
||||
To capture a device memory profile to disk, use
|
||||
{func}`jax.profiler.save_device_memory_profile`. For example, consider the
|
||||
following Python program:
|
||||
|
||||
```python
|
||||
import jax
|
||||
import jax.numpy as jnp
|
||||
import jax.profiler
|
||||
|
||||
def func1(x):
|
||||
return jnp.tile(x, 10) * 0.5
|
||||
|
||||
def func2(x):
|
||||
y = func1(x)
|
||||
return y, jnp.tile(x, 10) + 1
|
||||
|
||||
x = jax.random.normal(jax.random.PRNGKey(42), (1000, 1000))
|
||||
y, z = func2(x)
|
||||
|
||||
z.block_until_ready()
|
||||
|
||||
jax.profiler.save_device_memory_profile("memory.prof")
|
||||
```
|
||||
|
||||
If we first run the program above and then execute
|
||||
|
||||
```shell
|
||||
pprof --web memory.prof
|
||||
```
|
||||
|
||||
`pprof` opens a web browser containing the following visualization of the device
|
||||
memory profile in callgraph format:
|
||||
|
||||
|
||||
|
||||
The callgraph is a visualization of
|
||||
the Python stack at the point the allocation of each live buffer was made.
|
||||
For example, in this specific case, the visualization shows that
|
||||
`func2` and its callees were responsible for allocating 76.30MB, of which
|
||||
38.15MB was allocated inside the call from `func1` to `func2`.
|
||||
For more information about how to interpret callgraph visualizations, see the
|
||||
[pprof documentation](https://github.com/google/pprof/blob/master/doc/README.md#interpreting-the-callgraph).
|
||||
|
||||
Functions compiled with {func}`jax.jit` are opaque to the device memory profiler.
|
||||
That is, any memory allocated inside a `jit`-compiled function will be
|
||||
attributed to the function as whole.
|
||||
|
||||
In the example, the call to `block_until_ready()` is to ensure that `func2`
|
||||
completes before the device memory profile is collected. See
|
||||
{doc}`async_dispatch` for more details.
|
||||
|
||||
## Debugging memory leaks
|
||||
|
||||
We can also use the JAX device memory profiler to track down memory leaks by using
|
||||
`pprof` to visualize the change in memory usage between two device memory profiles
|
||||
taken at different times. For example consider the following program which
|
||||
accumulates JAX arrays into a constantly-growing Python list.
|
||||
|
||||
```python
|
||||
import jax
|
||||
import jax.numpy as jnp
|
||||
import jax.profiler
|
||||
|
||||
def afunction():
|
||||
return jax.random.normal(jax.random.PRNGKey(77), (1000000,))
|
||||
|
||||
z = afunction()
|
||||
|
||||
def anotherfunc():
|
||||
arrays = []
|
||||
for i in range(1, 10):
|
||||
x = jax.random.normal(jax.random.PRNGKey(42), (i, 10000))
|
||||
arrays.append(x)
|
||||
x.block_until_ready()
|
||||
jax.profiler.save_device_memory_profile(f"memory{i}.prof")
|
||||
|
||||
anotherfunc()
|
||||
```
|
||||
|
||||
If we simply visualize the device memory profile at the end of execution
|
||||
(`memory9.prof`), it may not be obvious that each iteration of the loop in
|
||||
`anotherfunc` accumulates more device memory allocations:
|
||||
|
||||
```shell
|
||||
pprof --web memory9.prof
|
||||
```
|
||||
|
||||
|
||||
|
||||
The large but fixed allocation inside `afunction` dominates the profile but does
|
||||
not grow over time.
|
||||
|
||||
By using `pprof`'s
|
||||
[`--diff_base` feature](https://github.com/google/pprof/blob/master/doc/README.md#comparing-profiles) to visualize the change in memory usage
|
||||
across loop iterations, we can identify why the memory usage of the
|
||||
program increases over time:
|
||||
|
||||
```shell
|
||||
pprof --web --diff_base memory1.prof memory9.prof
|
||||
```
|
||||
|
||||
|
||||
|
||||
The visualization shows that the memory growth can be attributed to the call to
|
||||
`normal` inside `anotherfunc`.
|
||||
@ -38,6 +38,7 @@ For an introduction to JAX, start at the
|
||||
concurrency
|
||||
gpu_memory_allocation
|
||||
profiling
|
||||
device_memory_profiling
|
||||
pytrees
|
||||
rank_promotion_warning
|
||||
type_promotion
|
||||
|
||||
@ -1,6 +1,33 @@
|
||||
.. currentmodule:: jax.profiler
|
||||
|
||||
jax.profiler module
|
||||
===================
|
||||
|
||||
.. automodule:: jax.profiler
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
Tracing and time profiling
|
||||
--------------------------
|
||||
|
||||
:doc:`profiling` describes how to make use of JAX's tracing and time profiling
|
||||
features.
|
||||
|
||||
.. autosummary::
|
||||
:toctree: _autosummary
|
||||
|
||||
start_server
|
||||
trace_function
|
||||
TraceContext
|
||||
|
||||
|
||||
Device memory profiling
|
||||
-----------------------
|
||||
|
||||
See :doc:`device_memory_profiling` for an introduction to JAX's device memory
|
||||
profiling features.
|
||||
|
||||
.. autosummary::
|
||||
:toctree: _autosummary
|
||||
|
||||
device_memory_profile
|
||||
save_device_memory_profile
|
||||
|
||||
@ -113,7 +113,7 @@ sudo update-initramfs -u
|
||||
sudo reboot now
|
||||
```
|
||||
|
||||
See [Nvidia's documentation on this
|
||||
See [NVIDIA's documentation on this
|
||||
error](https://developer.nvidia.com/nvidia-development-tools-solutions-err-nvgpuctrperm-cupti)
|
||||
for more information.
|
||||
|
||||
@ -133,7 +133,7 @@ ssh -L 6006:localhost:6006 <remote server address>
|
||||
|
||||
## Nsight
|
||||
|
||||
Nvidia's `Nsight` tools can be used to trace and profile JAX code on GPU. For
|
||||
NVIDIA's `Nsight` tools can be used to trace and profile JAX code on GPU. For
|
||||
details, see the [`Nsight`
|
||||
documentation](https://developer.nvidia.com/tools-overview).
|
||||
|
||||
|
||||
@ -13,8 +13,9 @@
|
||||
# limitations under the License.
|
||||
|
||||
from functools import wraps
|
||||
from typing import Callable
|
||||
from typing import Callable, Optional
|
||||
|
||||
from .lib import xla_bridge
|
||||
from .lib import xla_client
|
||||
|
||||
|
||||
@ -86,3 +87,48 @@ def trace_function(func: Callable, name: str = None, **kwargs):
|
||||
return func(*args, **kwargs)
|
||||
return wrapper
|
||||
return wrapper
|
||||
|
||||
|
||||
def device_memory_profile(backend: Optional[str] = None) -> bytes:
|
||||
"""Captures a JAX device memory profile as ``pprof``-format protocol buffer.
|
||||
|
||||
A device memory profile is a snapshot of the state of memory, that describes the JAX
|
||||
:class:`jax.DeviceArray` and executable objects present in memory and their
|
||||
allocation sites.
|
||||
|
||||
For more information how to use the device memory profiler, see
|
||||
:doc:`/device_memory_profiling`.
|
||||
|
||||
The profiling system works by instrumenting JAX on-device allocations,
|
||||
capturing a Python stack trace for each allocation. The instrumentation is
|
||||
always enabled; :func:`device_memory_profile` provides an API to capture it.
|
||||
|
||||
The output of :func:`device_memory_profile` is a binary protocol buffer that
|
||||
can be interpreted and visualized by the `pprof tool
|
||||
<https://github.com/google/pprof>`_.
|
||||
|
||||
Args:
|
||||
backend: optional; the name of the JAX backend for which the device memory
|
||||
profile should be collected.
|
||||
|
||||
Returns:
|
||||
A byte string containing a binary `pprof`-format protocol buffer.
|
||||
"""
|
||||
return xla_client.heap_profile(xla_bridge.get_backend(backend))
|
||||
|
||||
|
||||
def save_device_memory_profile(filename, backend: Optional[str] = None):
|
||||
"""Collects a device memory profile and writes it to a file.
|
||||
|
||||
:func:`save_device_memory_profile` is a convenience wrapper around :func:`device_memory_profile`
|
||||
that saves its output to a ``filename``. See the
|
||||
:func:`device_memory_profile` documentation for more information.
|
||||
|
||||
Args:
|
||||
filename: the filename to which the profile should be written.
|
||||
backend: optional; the name of the JAX backend for which the device memory
|
||||
profile should be collected.
|
||||
"""
|
||||
profile = device_memory_profile(backend)
|
||||
with open(filename, "wb") as f:
|
||||
f.write(profile)
|
||||
|
||||
@ -18,6 +18,7 @@ import unittest
|
||||
from absl.testing import absltest
|
||||
|
||||
import jax
|
||||
import jax.numpy as jnp
|
||||
import jax.profiler
|
||||
from jax.config import config
|
||||
import jax.test_util
|
||||
@ -61,6 +62,10 @@ class ProfilerTest(unittest.TestCase):
|
||||
return x + 2
|
||||
self.assertEqual(h(7), 9)
|
||||
|
||||
def testDeviceMemoryProfile(self):
|
||||
x = jnp.ones((20,)) + 7.
|
||||
self.assertTrue(isinstance(jax.profiler.device_memory_profile(), bytes))
|
||||
del x
|
||||
|
||||
if __name__ == "__main__":
|
||||
absltest.main()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user