

<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>metrics package &mdash; LEAF 0.1 documentation</title>
  

  
  
  
  

  

  
  
    

  

  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/customheader.css" type="text/css" />
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="next" title="model module" href="model.html" />
    <link rel="prev" title="main module" href="main.html" /> 

  
  <script src="../_static/js/modernizr.min.js"></script>

</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">

    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search">
          

          
            <a href="../index.html" class="icon icon-home"> LEAF
          

          
          </a>

          
            
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">Getting Started</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../install/get_leaf.html">Installation and Contribution</a></li>
</ul>
<p class="caption"><span class="caption-text">User Guide</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/index.html">General Examples</a></li>
</ul>
<p class="caption"><span class="caption-text">API Documentation</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="modules.html">models</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="baseline_constants.html">baseline_constants module</a></li>
<li class="toctree-l2"><a class="reference internal" href="client.html">client module</a></li>
<li class="toctree-l2"><a class="reference internal" href="main.html">main module</a></li>
<li class="toctree-l2 current"><a class="current reference internal" href="#">metrics package</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#submodules">Submodules</a></li>
<li class="toctree-l3"><a class="reference internal" href="#module-metrics.visualization_utils">metrics.visualization_utils module</a></li>
<li class="toctree-l3"><a class="reference internal" href="#module-metrics.writer">metrics.writer module</a></li>
<li class="toctree-l3"><a class="reference internal" href="#module-metrics">Module contents</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="model.html">model module</a></li>
<li class="toctree-l2"><a class="reference internal" href="server.html">server module</a></li>
<li class="toctree-l2"><a class="reference internal" href="utils.html">utils package</a></li>
</ul>
</li>
</ul>
<p class="caption"><span class="caption-text">Additional Information</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../contact.html">Contact</a></li>
<li class="toctree-l1"><a class="reference internal" href="../citations.html">Citing LEAF</a></li>
</ul>

            
          
        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../index.html">LEAF</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../index.html">Docs</a> &raquo;</li>
        
          <li><a href="modules.html">models</a> &raquo;</li>
        
      <li>metrics package</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
            <a href="../_sources/autodoc/metrics.rst.txt" rel="nofollow"> View page source</a>
          
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="metrics-package">
<h1>metrics package<a class="headerlink" href="#metrics-package" title="Permalink to this headline">¶</a></h1>
<div class="section" id="submodules">
<h2>Submodules<a class="headerlink" href="#submodules" title="Permalink to this headline">¶</a></h2>
</div>
<div class="section" id="module-metrics.visualization_utils">
<span id="metrics-visualization-utils-module"></span><h2>metrics.visualization_utils module<a class="headerlink" href="#module-metrics.visualization_utils" title="Permalink to this headline">¶</a></h2>
<p>Helper to visualize metrics.</p>
<dl class="function">
<dt id="metrics.visualization_utils.get_longest_flops_path">
<code class="descclassname">metrics.visualization_utils.</code><code class="descname">get_longest_flops_path</code><span class="sig-paren">(</span><em>sys_metrics</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/metrics/visualization_utils.html#get_longest_flops_path"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#metrics.visualization_utils.get_longest_flops_path" title="Permalink to this definition">¶</a></dt>
<dd><p>Prints the largest amount of flops required to complete training.</p>
<dl class="docutils">
<dt>To calculate this metric, we:</dt>
<dd><ol class="first last arabic simple">
<li><dl class="first docutils">
<dt>For each round, pick the client that required the largest amount</dt>
<dd>of local training.</dd>
</dl>
</li>
<li>Sum the FLOPS from the clients picked in step 1 across rounds.</li>
</ol>
</dd>
</dl>
<p>TODO: This metric would make more sense with seconds instead of FLOPS.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>sys_metrics</strong> – pd.DataFrame as written by writer.py.</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt id="metrics.visualization_utils.load_data">
<code class="descclassname">metrics.visualization_utils.</code><code class="descname">load_data</code><span class="sig-paren">(</span><em>stat_metrics_file='stat_metrics.csv'</em>, <em>sys_metrics_file='sys_metrics.csv'</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/metrics/visualization_utils.html#load_data"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#metrics.visualization_utils.load_data" title="Permalink to this definition">¶</a></dt>
<dd><p>Loads the data from the given stat_metric and sys_metric files.</p>
</dd></dl>

<dl class="function">
<dt id="metrics.visualization_utils.plot_accuracy_vs_round_number">
<code class="descclassname">metrics.visualization_utils.</code><code class="descname">plot_accuracy_vs_round_number</code><span class="sig-paren">(</span><em>stat_metrics</em>, <em>weighted=False</em>, <em>plot_stds=False</em>, <em>figsize=(10</em>, <em>8)</em>, <em>title_fontsize=16</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/metrics/visualization_utils.html#plot_accuracy_vs_round_number"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#metrics.visualization_utils.plot_accuracy_vs_round_number" title="Permalink to this definition">¶</a></dt>
<dd><p>Plots the clients’ average test accuracy vs. the round number.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>stat_metrics</strong> – pd.DataFrame as written by writer.py.</li>
<li><strong>weighted</strong> – Whether the average across clients should be weighted by number of
test samples.</li>
<li><strong>plot_stds</strong> – Whether to plot error bars corresponding to the std between users.</li>
<li><strong>figsize</strong> – Size of the plot as specified by plt.figure().</li>
<li><strong>title_fontsize</strong> – Font size for the plot’s title.</li>
<li><strong>kwargs</strong> – Arguments to be passed to _set_plot_properties.</li>
</ul>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt id="metrics.visualization_utils.plot_accuracy_vs_round_number_per_client">
<code class="descclassname">metrics.visualization_utils.</code><code class="descname">plot_accuracy_vs_round_number_per_client</code><span class="sig-paren">(</span><em>stat_metrics</em>, <em>sys_metrics</em>, <em>max_num_clients</em>, <em>figsize=(15</em>, <em>12)</em>, <em>title_fontsize=16</em>, <em>max_name_len=10</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/metrics/visualization_utils.html#plot_accuracy_vs_round_number_per_client"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#metrics.visualization_utils.plot_accuracy_vs_round_number_per_client" title="Permalink to this definition">¶</a></dt>
<dd><p>Plots the clients’ test accuracy vs. the round number.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>stat_metrics</strong> – pd.DataFrame as written by writer.py.</li>
<li><strong>sys_metrics</strong> – pd.DataFrame as written by writer.py. Allows us to know which client actually performed training
in each round. If None, then no indication is given of when was each client trained.</li>
<li><strong>max_num_clients</strong> – Maximum number of clients to plot.</li>
<li><strong>figsize</strong> – Size of the plot as specified by plt.figure().</li>
<li><strong>title_fontsize</strong> – Font size for the plot’s title.</li>
<li><strong>max_name_len</strong> – Maximum length for a client’s id.</li>
<li><strong>kwargs</strong> – Arguments to be passed to _set_plot_properties.</li>
</ul>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt id="metrics.visualization_utils.plot_bytes_written_and_read">
<code class="descclassname">metrics.visualization_utils.</code><code class="descname">plot_bytes_written_and_read</code><span class="sig-paren">(</span><em>sys_metrics</em>, <em>rolling_window=10</em>, <em>figsize=(10</em>, <em>8)</em>, <em>title_fontsize=16</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/metrics/visualization_utils.html#plot_bytes_written_and_read"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#metrics.visualization_utils.plot_bytes_written_and_read" title="Permalink to this definition">¶</a></dt>
<dd><p>Plots the cumulative sum of the bytes written and read by the server.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>sys_metrics</strong> – pd.DataFrame as written by writer.py.</li>
<li><strong>rolling_window</strong> – Number of previous rounds to consider in the cumulative sum.</li>
<li><strong>figsize</strong> – Size of the plot as specified by plt.figure().</li>
<li><strong>title_fontsize</strong> – Font size for the plot’s title.</li>
<li><strong>kwargs</strong> – Arguments to be passed to _set_plot_properties.</li>
</ul>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt id="metrics.visualization_utils.plot_client_computations_vs_round_number">
<code class="descclassname">metrics.visualization_utils.</code><code class="descname">plot_client_computations_vs_round_number</code><span class="sig-paren">(</span><em>sys_metrics</em>, <em>aggregate_window=20</em>, <em>max_num_clients=20</em>, <em>figsize=(25</em>, <em>15)</em>, <em>title_fontsize=16</em>, <em>max_name_len=10</em>, <em>range_rounds=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/metrics/visualization_utils.html#plot_client_computations_vs_round_number"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#metrics.visualization_utils.plot_client_computations_vs_round_number" title="Permalink to this definition">¶</a></dt>
<dd><p>Plots the clients’ local computations against round number.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>sys_metrics</strong> – pd.DataFrame as written by writer.py.</li>
<li><strong>aggregate_window</strong> – Number of rounds that are aggregated. e.g. If set to 20, then
rounds 0-19, 20-39, etc. will be added together.</li>
<li><strong>max_num_clients</strong> – Maximum number of clients to plot.</li>
<li><strong>figsize</strong> – Size of the plot as specified by plt.figure().</li>
<li><strong>title_fontsize</strong> – Font size for the plot’s title.</li>
<li><strong>max_name_len</strong> – Maximum length for a client’s id.</li>
<li><strong>range_rounds</strong> – Tuple representing the range of rounds to be plotted. The rounds
are subsampled before aggregation. If None, all rounds are considered.</li>
</ul>
</td>
</tr>
</tbody>
</table>
</dd></dl>

</div>
<div class="section" id="module-metrics.writer">
<span id="metrics-writer-module"></span><h2>metrics.writer module<a class="headerlink" href="#module-metrics.writer" title="Permalink to this headline">¶</a></h2>
<p>Writes the given metrics in a csv.</p>
<dl class="function">
<dt id="metrics.writer.get_metrics_names">
<code class="descclassname">metrics.writer.</code><code class="descname">get_metrics_names</code><span class="sig-paren">(</span><em>metrics</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/metrics/writer.html#get_metrics_names"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#metrics.writer.get_metrics_names" title="Permalink to this definition">¶</a></dt>
<dd><p>Gets the names of the metrics.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>metrics</strong> – Dict keyed by client id. Each element is a dict of metrics
for that client in the specified round. The dicts for all clients
are expected to have the same set of keys.</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt id="metrics.writer.print_dataframe">
<code class="descclassname">metrics.writer.</code><code class="descname">print_dataframe</code><span class="sig-paren">(</span><em>df</em>, <em>path</em>, <em>mode='w'</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/metrics/writer.html#print_dataframe"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#metrics.writer.print_dataframe" title="Permalink to this definition">¶</a></dt>
<dd><p>Writes the given dataframe in path as a csv</p>
</dd></dl>

<dl class="function">
<dt id="metrics.writer.print_metrics">
<code class="descclassname">metrics.writer.</code><code class="descname">print_metrics</code><span class="sig-paren">(</span><em>round_number</em>, <em>client_ids</em>, <em>metrics</em>, <em>hierarchies</em>, <em>num_samples</em>, <em>path</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/metrics/writer.html#print_metrics"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#metrics.writer.print_metrics" title="Permalink to this definition">¶</a></dt>
<dd><p>Prints or appends the given metrics in a csv.</p>
<dl class="docutils">
<dt>The resulting dataframe is of the form:</dt>
<dd>client_id, round_number, hierarchy, num_samples, metric1, metric2
twebbstack, 0, , 18, 0.5, 0.89</dd>
</dl>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>round_number</strong> – Number of the round the metrics correspond to. If
0, then the file in path is overwritten. If not 0, we append to
that file.</li>
<li><strong>client_ids</strong> – Ids of the clients. Not all ids must be in the following
dicts.</li>
<li><strong>metrics</strong> – Dict keyed by client id. Each element is a dict of metrics
for that client in the specified round. The dicts for all clients
are expected to have the same set of keys.</li>
<li><strong>hierarchies</strong> – Dict keyed by client id. Each element is a list of hierarchies
to which the client belongs.</li>
<li><strong>num_samples</strong> – Dict keyed by client id. Each element is the number of test
samples for the client.</li>
</ul>
</td>
</tr>
</tbody>
</table>
</dd></dl>

</div>
<div class="section" id="module-metrics">
<span id="module-contents"></span><h2>Module contents<a class="headerlink" href="#module-metrics" title="Permalink to this headline">¶</a></h2>
</div>
</div>


           </div>
           
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="model.html" class="btn btn-neutral float-right" title="model module" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="main.html" class="btn btn-neutral" title="main module" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2019, Talwalkar Lab

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  


  

    
    
      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
        <script type="text/javascript" src="../_static/jquery.js"></script>
        <script type="text/javascript" src="../_static/underscore.js"></script>
        <script type="text/javascript" src="../_static/doctools.js"></script>
    

  

  <script type="text/javascript" src="../_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script> 

</body>
</html>