
@article{abadiTensorFlowLargeScaleMachine,
  title = {{{TensorFlow}}: {{Large}}-{{Scale Machine Learning}} on {{Heterogeneous Distributed Systems}}},
  author = {<NAME> and Agarwal, Ashish and Barham, Paul and <NAME> and <NAME> and Citro, Craig and Corrado, <NAME> and Davis, Andy and Dean, Jeffrey and <NAME> and <NAME> and Goodfellow, Ian and Harp, Andrew and Irving, Geoffrey and Isard, Michael and <NAME> and Jozefowicz, Rafal and <NAME> and <NAME> and Levenberg, <NAME> and <NAME>, <NAME>, Mike and Shlens, Jonathon and Steiner, Benoit and Sutskever, Ilya and <NAME> and Tucker, Paul and Vanhoucke, <NAME>, <NAME>, <NAME>, Oriol and Warden, Pete and Wattenberg, <NAME>, Martin and <NAME> and Zheng, Xiaoqiang},
  pages = {19},
  file = {/Users/hudson_home/Zotero/storage/3LHCLTLG/Abadi et al. - TensorFlow Large-Scale Machine Learning on Hetero.pdf},
  langid = {english}
}

@article{allen-zhuConvergenceRateTraining2019,
  title = {On the {{Convergence Rate}} of {{Training Recurrent Neural Networks}}},
  author = {<NAME> and <NAME> and <NAME>},
  date = {2019-05-27},
  url = {http://arxiv.org/abs/1810.12065},
  urldate = {2020-01-21},
  abstract = {How can local-search methods such as stochastic gradient descent (SGD) avoid bad local minima in training multi-layer neural networks? Why can they fit random labels even given non-convex and non-smooth architectures? Most existing theory only covers networks with one hidden layer, so can we go deeper? In this paper, we focus on recurrent neural networks (RNNs) which are multi-layer networks widely used in natural language processing. They are harder to analyze than feedforward neural networks, because the same recurrent unit is repeatedly applied across the entire time horizon of length L, which is analogous to feedforward networks of depth L. We show when the number of neurons is sufficiently large, meaning polynomial in the training data size and in L, then SGD is capable of minimizing the regression loss in the linear convergence rate. This gives theoretical evidence of how RNNs can memorize data.},
  archivePrefix = {arXiv},
  eprint = {1810.12065},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/SPXNEVCE/Allen-Zhu et al. - 2019 - On the Convergence Rate of Training Recurrent Neur.pdf},
  keywords = {Computer Science - Data Structures and Algorithms,Computer Science - Machine Learning,Computer Science - Neural and Evolutionary Computing,Mathematics - Optimization and Control,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, math, stat}
}

@article{allen-zhuConvergenceTheoryDeep2019,
  title = {A {{Convergence Theory}} for {{Deep Learning}} via {{Over}}-{{Parameterization}}},
  author = {<NAME> and <NAME> and <NAME>},
  date = {2019-06-17},
  url = {http://arxiv.org/abs/1811.03962},
  urldate = {2020-01-15},
  abstract = {Deep neural networks (DNNs) have demonstrated dominating performance in many fields; since AlexNet, networks used in practice are going wider and deeper. On the theoretical side, a long line of works has been focusing on training neural networks with one hidden layer. The theory of multi-layer networks remains largely unsettled.},
  archivePrefix = {arXiv},
  eprint = {1811.03962},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/FWDYDK2F/Allen-Zhu et al. - 2019 - A Convergence Theory for Deep Learning via Over-Pa.pdf},
  keywords = {Computer Science - Data Structures and Algorithms,Computer Science - Machine Learning,Computer Science - Neural and Evolutionary Computing,Mathematics - Optimization and Control,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, math, stat}
}

@article{allen-zhuLearningGeneralizationOverparameterized2019,
  title = {Learning and {{Generalization}} in {{Overparameterized Neural Networks}}, {{Going Beyond Two Layers}}},
  author = {<NAME> <NAME> and <NAME>},
  date = {2019-05-28},
  url = {http://arxiv.org/abs/1811.04918},
  urldate = {2020-01-15},
  abstract = {The fundamental learning theory behind neural networks remains largely open. What classes of functions can neural networks actually learn? Why doesn’t the trained neural networks overfit when the it is overparameterized (namely, having more parameters than statistically needed to overfit training data)? In this work, we prove that overparameterized neural networks can learn some notable concept classes, including two and three-layer networks with fewer parameters and smooth activations. Moreover, the learning can be simply done by SGD (stochastic gradient descent) or its variants in polynomial time using polynomially many samples. The sample complexity can also be almost independent of the number of parameters in the overparameterized network.},
  archivePrefix = {arXiv},
  eprint = {1811.04918},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/G5RKIHYQ/Allen-Zhu et al. - 2019 - Learning and Generalization in Overparameterized N.pdf},
  keywords = {Computer Science - Data Structures and Algorithms,Computer Science - Machine Learning,Computer Science - Neural and Evolutionary Computing,Mathematics - Optimization and Control,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, math, stat}
}

@article{baikPhaseTransitionLargest2004,
  title = {Phase Transition of the Largest Eigenvalue for Non-Null Complex Sample Covariance Matrices},
  author = {<NAME> and <NAME> and <NAME>},
  date = {2004-10-20},
  url = {http://arxiv.org/abs/math/0403022},
  urldate = {2020-03-12},
  abstract = {We compute the limiting distributions of the largest eigenvalue of a complex Gaussian sample covariance matrix when both the number of samples and the number of variables in each sample become large. When all but finitely many, say r, eigenvalues of the covariance matrix are the same, the dependence of the limiting distribution of the largest eigenvalue of the sample covariance matrix on those distinguished r eigenvalues of the covariance matrix is completely characterized in terms of an infinite sequence of new distribution functions that generalize the Tracy-Widom distributions of the random matrix theory. Especially a phase transition phenomena is observed. Our results also apply to a last passage percolation model and a queuing model.},
  archivePrefix = {arXiv},
  eprint = {math/0403022},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/9K4MG9SP/Baik et al. - 2004 - Phase transition of the largest eigenvalue for non.pdf},
  keywords = {Mathematics - Probability},
  langid = {english}
}

@article{baity-jesiComparingDynamicsDeep2019,
  title = {Comparing {{Dynamics}}: {{Deep Neural Networks}} versus {{Glassy Systems}}},
  shorttitle = {Comparing {{Dynamics}}},
  author = {<NAME>. and <NAME> <NAME>.},
  date = {2019-12-20},
  journaltitle = {Journal of Statistical Mechanics: Theory and Experiment},
  shortjournal = {J. Stat. Mech.},
  volume = {2019},
  pages = {124013},
  issn = {1742-5468},
  doi = {10.1088/1742-5468/ab3281},
  url = {http://arxiv.org/abs/1803.06969},
  urldate = {2020-01-15},
  abstract = {We analyze numerically the training dynamics of deep neural networks (DNN) by using methods developed in statistical physics of glassy systems. The two main issues we address are (1) the complexity of the loss landscape and of the dynamics within it, and (2) to what extent DNNs share similarities with glassy systems. Our findings, obtained for different architectures and datasets, suggest that during the training process the dynamics slows down because of an increasingly large number of flat directions. At large times, when the loss is approaching zero, the system diffuses at the bottom of the landscape. Despite some similarities with the dynamics of mean-field glassy systems, in particular, the absence of barrier crossing, we find distinctive dynamical behaviors in the two cases, showing that the statistical properties of the corresponding loss and energy landscapes are different. In contrast, when the network is under-parametrized we observe a typical glassy behavior, thus suggesting the existence of different phases depending on whether the network is under-parametrized or over-parametrized.},
  archivePrefix = {arXiv},
  eprint = {1803.06969},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/GR625E3F/Baity-Jesi et al. - 2019 - Comparing Dynamics Deep Neural Networks versus Gl.pdf},
  keywords = {Computer Science - Machine Learning,Condensed Matter - Disordered Systems and Neural Networks,Statistics - Machine Learning},
  langid = {english},
  number = {12}
}

@article{belkinReconcilingModernMachine2019,
  title = {Reconciling Modern Machine Learning Practice and the Bias-Variance Trade-Off},
  author = {<NAME> <NAME> and <NAME> and <NAME>},
  date = {2019-09-10},
  url = {http://arxiv.org/abs/1812.11118},
  urldate = {2020-08-16},
  abstract = {Breakthroughs in machine learning are rapidly changing science and society, yet our fundamental understanding of this technology has lagged far behind. Indeed, one of the central tenets of the field, the bias-variance trade-off, appears to be at odds with the observed behavior of methods used in the modern machine learning practice. The bias-variance trade-off implies that a model should balance under-fitting and over-fitting: rich enough to express underlying structure in data, simple enough to avoid fitting spurious patterns. However, in the modern practice, very rich models such as neural networks are trained to exactly fit (i.e., interpolate) the data. Classically, such models would be considered over-fit, and yet they often obtain high accuracy on test data. This apparent contradiction has raised questions about the mathematical foundations of machine learning and their relevance to practitioners.},
  archivePrefix = {arXiv},
  eprint = {1812.11118},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/ZEDICVQH/Belkin et al. - 2019 - Reconciling modern machine learning practice and t.pdf},
  keywords = {Computer Science - Machine Learning,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, stat}
}

@article{belkinUnderstandDeepLearning2018,
  title = {To Understand Deep Learning We Need to Understand Kernel Learning},
  author = {<NAME> <NAME> and <NAME>},
  date = {2018-06-14},
  url = {http://arxiv.org/abs/1802.01396},
  urldate = {2020-08-16},
  abstract = {Generalization performance of classifiers in deep learning has recently become a subject of intense study. Deep models, which are typically heavily over-parametrized, tend to fit the training data exactly. Despite this “overfitting", they perform well on test data, a phenomenon not yet fully understood.},
  archivePrefix = {arXiv},
  eprint = {1802.01396},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/6NIYTPAJ/Belkin et al. - 2018 - To understand deep learning we need to understand .pdf},
  keywords = {Computer Science - Machine Learning,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, stat}
}

@article{bianconiBoseEinsteinCondensationComplex2001,
  title = {Bose-{{Einstein}} Condensation in Complex Networks},
  author = {<NAME>. and <NAME>.},
  date = {2001-06-11},
  journaltitle = {Physical Review Letters},
  shortjournal = {Phys. Rev. Lett.},
  volume = {86},
  pages = {5632--5635},
  issn = {0031-9007, 1079-7114},
  doi = {10.1103/PhysRevLett.86.5632},
  url = {http://arxiv.org/abs/cond-mat/0011224},
  urldate = {2020-02-21},
  abstract = {The evolution of many complex systems, including the world wide web, business and citation networks is encoded in the dynamic web describing the interactions between the system’s constituents. Despite their irreversible and non-equilibrium nature these networks follow Bose statistics and can undergo Bose-Einstein condensation. Addressing the dynamical properties of these non-equilibrium systems within the framework of equilibrium quantum gases predicts that the ’first-mover-advantage’, ’fit-get-rich’ and ’winner-takes-all’ phenomena observed in competitive systems are thermodynamically distinct phases of the underlying evolving networks.},
  archivePrefix = {arXiv},
  eprint = {cond-mat/0011224},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/GVBV4WVW/Bianconi and Barabási - 2001 - Bose-Einstein condensation in complex networks.pdf},
  keywords = {Condensed Matter - Disordered Systems and Neural Networks,Condensed Matter - Statistical Mechanics},
  langid = {english},
  number = {24}
}

@article{biettiInductiveBiasNeural2019,
  title = {On the {{Inductive Bias}} of {{Neural Tangent Kernels}}},
  author = {<NAME> and <NAME>},
  date = {2019-10-31},
  url = {http://arxiv.org/abs/1905.12173},
  urldate = {2020-03-12},
  abstract = {State-of-the-art neural networks are heavily over-parameterized, making the optimization algorithm a crucial ingredient for learning predictive models with good generalization properties. A recent line of work has shown that in a certain overparameterized regime, the learning dynamics of gradient descent are governed by a certain kernel obtained at initialization, called the neural tangent kernel. We study the inductive bias of learning in such a regime by analyzing this kernel and the corresponding function space (RKHS). In particular, we study smoothness, approximation, and stability properties of functions with finite norm, including stability to image deformations in the case of convolutional networks, and compare to other known kernels for similar architectures.},
  archivePrefix = {arXiv},
  eprint = {1905.12173},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/JJLSILZ2/Bietti and Mairal - 2019 - On the Inductive Bias of Neural Tangent Kernels.pdf},
  keywords = {Computer Science - Machine Learning,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, stat}
}

@article{canatarStatisticalMechanicsGeneralization2020,
  title = {Statistical {{Mechanics}} of {{Generalization}} in {{Kernel Regression}}},
  author = {<NAME> and <NAME> and <NAME>},
  date = {2020-06-23},
  url = {http://arxiv.org/abs/2006.13198},
  urldate = {2020-07-01},
  abstract = {Generalization beyond a training dataset is a main goal of machine learning. We investigate generalization error in kernel regression using statistical mechanics, deriving an analytical expression applicable to any kernel. We discuss applications to a kernel with finite number of spectral modes. Then, focusing on the broad class of rotation invariant kernels, which is relevant to training deep neural networks in the infinite-width limit, we show several phenomena. When data is drawn from a spherically symmetric distribution and the number of input dimensions, \$D\$, is large, we find that multiple learning stages exist, one for each scaling of the number of training samples with \$\textbackslash mathcal\{O\}\_D(D\^K)\$ where \$K\textbackslash in Z\^+\$. The behavior of the learning curve in each stage is related to an \textbackslash textit\{effective\} noise and regularizer that are related to the tail of the kernel and target function spectra. When effective regularization is zero, we identify a first order phase transition that corresponds to a divergence in the generalization error. Each learning stage can exhibit sample-wise \textbackslash textit\{double descent\}, where learning curves show non-monotonic sample size dependence. For each stage an optimal value of effective regularizer exists, equal to the effective noise variance, that gives minimum generalization error.},
  archivePrefix = {arXiv},
  eprint = {2006.13198},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/KRTNQX9R/Canatar et al. - 2020 - Statistical Mechanics of Generalization in Kernel .pdf},
  keywords = {Computer Science - Machine Learning,Condensed Matter - Disordered Systems and Neural Networks,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cond-mat, stat}
}

@article{chizatLazyTrainingDifferentiable2020,
  title = {On {{Lazy Training}} in {{Differentiable Programming}}},
  author = {<NAME> and <NAME> and <NAME>},
  date = {2020-01-07},
  url = {http://arxiv.org/abs/1812.07956},
  urldate = {2020-06-19},
  abstract = {In a series of recent theoretical works, it was shown that strongly overparameterized neural networks trained with gradient-based methods could converge exponentially fast to zero training loss, with their parameters hardly varying. In this work, we show that this “lazy training” phenomenon is not specific to overparameterized neural networks, and is due to a choice of scaling, often implicit, that makes the model behave as its linearization around the initialization, thus yielding a model equivalent to learning with positive-definite kernels. Through a theoretical analysis, we exhibit various situations where this phenomenon arises in non-convex optimization and we provide bounds on the distance between the lazy and linearized optimization paths. Our numerical experiments bring a critical note, as we observe that the performance of commonly used non-linear deep convolutional neural networks in computer vision degrades when trained in the lazy regime. This makes it unlikely that “lazy training” is behind the many successes of neural networks in difficult high dimensional tasks.},
  archivePrefix = {arXiv},
  eprint = {1812.07956},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/S5PWYM7I/Chizat et al. - 2020 - On Lazy Training in Differentiable Programming.pdf},
  keywords = {Computer Science - Machine Learning,Mathematics - Optimization and Control},
  langid = {english},
  primaryClass = {cs, math}
}

@article{choKernelMethodsDeep,
  title = {Kernel {{Methods}} for {{Deep Learning}}},
  author = {Cho, Youngmin and Saul, <NAME>},
  pages = {9},
  abstract = {We introduce a new family of positive-definite kernel functions that mimic the computation in large, multilayer neural nets. These kernel functions can be used in shallow architectures, such as support vector machines (SVMs), or in deep kernel-based architectures that we call multilayer kernel machines (MKMs). We evaluate SVMs and MKMs with these kernel functions on problems designed to illustrate the advantages of deep architectures. On several problems, we obtain better results than previous, leading benchmarks from both SVMs with Gaussian kernels as well as deep belief nets.},
  file = {/Users/hudson_home/Zotero/storage/AD3DJ2QG/Cho and Saul - Kernel Methods for Deep Learning.pdf},
  langid = {english}
}

@article{choromanskaLossSurfacesMultilayer,
  title = {The {{Loss Surfaces}} of {{Multilayer Networks}}},
  author = {Choromanska, Anna and Henaff, Mikael and Mathieu, <NAME> Arous, <NAME> and LeCun, Yann},
  pages = {13},
  file = {/Users/hudson_home/Zotero/storage/NU57JRB4/Choromanska et al. - The Loss Surfaces of Multilayer Networks.pdf},
  langid = {english}
}

@article{danielyDeeperUnderstandingNeural2017,
  title = {Toward {{Deeper Understanding}} of {{Neural Networks}}: {{The Power}} of {{Initialization}} and a {{Dual View}} on {{Expressivity}}},
  shorttitle = {Toward {{Deeper Understanding}} of {{Neural Networks}}},
  author = {<NAME> and <NAME> and <NAME>},
  date = {2017-05-19},
  url = {http://arxiv.org/abs/1602.05897},
  urldate = {2020-06-08},
  abstract = {We develop a general duality between neural networks and compositional kernels, striving towards a better understanding of deep learning. We show that initial representations generated by common random initializations are sufficiently rich to express all functions in the dual kernel space. Hence, though the training objective is hard to optimize in the worst case, the initial weights form a good starting point for optimization. Our dual view also reveals a pragmatic and aesthetic perspective of neural networks and underscores their expressive power.},
  archivePrefix = {arXiv},
  eprint = {1602.05897},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/XJID36TZ/Daniely et al. - 2017 - Toward Deeper Understanding of Neural Networks Th.pdf},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Computational Complexity,Computer Science - Data Structures and Algorithms,Computer Science - Machine Learning,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, stat}
}

@article{dascoliDoubleTroubleDouble2020,
  title = {Double {{Trouble}} in {{Double Descent}} : {{Bias}} and {{Variance}}(s) in the {{Lazy Regime}}},
  shorttitle = {Double {{Trouble}} in {{Double Descent}}},
  author = {<NAME>, <NAME>, <NAME> Krzakala, Florent},
  date = {2020-04-03},
  url = {http://arxiv.org/abs/2003.01054},
  urldate = {2020-06-18},
  abstract = {Deep neural networks can achieve remarkable generalization performances while interpolating the training data perfectly. Rather than the U-curve emblematic of the bias-variance trade-off, their test error often follows a “double descent”— a mark of the beneficial role of overparametrization. In this work, we develop a quantitative theory for this phenomenon in the so-called lazy learning regime of neural networks, by considering the problem of learning a high-dimensional function with random features regression. We obtain a precise asymptotic expression for the bias-variance decomposition of the test error, and show that the bias displays a phase transition at the interpolation threshold, beyond which it remains constant. We disentangle the variances stemming from the sampling of the dataset, from the additive noise corrupting the labels, and from the initialization of the weights. Following up on Geiger et al. [1], we first show that the latter two contributions are the crux of the double descent: they lead to the overfitting peak at the interpolation threshold and to the decay of the test error upon overparametrization. We then quantify how they are suppressed by ensembling the outputs of K independently initialized estimators. When K is sent to infinity, the test error remains constant beyond the interpolation threshold. We further compare the effects of overparametrizing, ensembling and regularizing. Finally, we present numerical experiments on classic deep learning setups to show that our results hold qualitatively in realistic lazy learning scenarios.},
  archivePrefix = {arXiv},
  eprint = {2003.01054},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/GNN23PW4/d'Ascoli et al. - 2020 - Double Trouble in Double Descent  Bias and Varian.pdf},
  keywords = {Computer Science - Machine Learning,Condensed Matter - Disordered Systems and Neural Networks,Statistics - Machine Learning},
  langid = {english},
  options = {useprefix=true},
  primaryClass = {cond-mat, stat}
}

@article{dengModelDoubleDescent2020,
  title = {A {{Model}} of {{Double Descent}} for {{High}}-Dimensional {{Binary Linear Classification}}},
  author = {<NAME> <NAME> <NAME>},
  date = {2020-05-10},
  url = {http://arxiv.org/abs/1911.05822},
  urldate = {2020-08-23},
  abstract = {We consider a model for logistic regression where only a subset of features of size p is used for training a linear classifier over n training samples. The classifier is obtained by running gradient descent (GD) on logistic loss. For this model, we investigate the dependence of the classification error on the overparameterization ratio κ = p/n. First, building on known deterministic results on the implicit bias of GD, we uncover a phase-transition phenomenon for the case of Gaussian features: the classification error of GD is the same as that of the maximum-likelihood (ML) solution when κ {$<$} κ⋆, and that of the max-margin (SVM) solution when κ {$>$} κ⋆. Next, using the convex Gaussian min-max theorem (CGMT), we sharply characterize the performance of both the ML and the SVM solutions. Combining these results, we obtain curves that explicitly characterize the classification error for varying values of κ. The numerical results validate the theoretical predictions and unveil double-descent phenomena that complement similar recent findings in linear regression settings as well as empirical observations in more complex learning scenarios.},
  archivePrefix = {arXiv},
  eprint = {1911.05822},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/28XVHG8T/Deng et al. - 2020 - A Model of Double Descent for High-dimensional Bin.pdf},
  keywords = {Computer Science - Machine Learning,Electrical Engineering and Systems Science - Signal Processing,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, eess, stat}
}

@article{dietrichStatisticalMechanicsSupport1999,
  title = {Statistical {{Mechanics}} of {{Support Vector Networks}}},
  author = {<NAME> Opper, <NAME> <NAME>},
  date = {1999-04-05},
  journaltitle = {Physical Review Letters},
  shortjournal = {Phys. Rev. Lett.},
  volume = {82},
  pages = {2975--2978},
  issn = {0031-9007, 1079-7114},
  doi = {10.1103/PhysRevLett.82.2975},
  url = {http://arxiv.org/abs/cond-mat/9811421},
  urldate = {2020-07-07},
  abstract = {Using methods of Statistical Physics, we investigate the generalization performance of support vector machines (SVMs), which have been recently introduced as a general alternative to neural networks. For nonlinear classification rules, the generalization error saturates on a plateau, when the number of examples is too small to properly estimate the coefficients of the nonlinear part. When trained on simple rules, we find that SVMs overfit only weakly. The performance of SVMs is strongly enhanced, when the distribution of the inputs has a gap in feature space.},
  archivePrefix = {arXiv},
  eprint = {cond-mat/9811421},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/DTCE7XSL/Dietrich et al. - 1999 - Statistical Mechanics of Support Vector Networks.pdf},
  keywords = {Condensed Matter - Disordered Systems and Neural Networks},
  langid = {english},
  number = {14}
}

@article{dinosejdinovicWhatRKHS2014,
  title = {What Is an {{RKHS}}?},
  author = {{<NAME>} and <NAME>},
  date = {2014},
  file = {/Users/hudson_home/Zotero/storage/X4NCQQ4P/What is an RKHS.pdf}
}

@article{draxlerEssentiallyNoBarriers2019,
  title = {Essentially {{No Barriers}} in {{Neural Network Energy Landscape}}},
  author = {<NAME> <NAME> Salmhofer, <NAME> Hamprecht, <NAME>.},
  date = {2019-02-22},
  url = {http://arxiv.org/abs/1803.00885},
  urldate = {2020-03-02},
  abstract = {Training neural networks involves finding minima of a high-dimensional non-convex loss function. Knowledge of the structure of this energy landscape is sparse. Relaxing from linear interpolations, we construct continuous paths between minima of recent neural network architectures on CIFAR10 and CIFAR100. Surprisingly, the paths are essentially flat in both the training and test landscapes. This implies that neural networks have enough capacity for structural changes, or that these changes are small between minima. Also, each minimum has at least one vanishing Hessian eigenvalue in addition to those resulting from trivial invariance.},
  archivePrefix = {arXiv},
  eprint = {1803.00885},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/DC2KIYYZ/Draxler et al. - 2019 - Essentially No Barriers in Neural Network Energy L.pdf},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Machine Learning,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, stat}
}

@article{duGradientDescentFinds2019,
  title = {Gradient {{Descent Finds Global Minima}} of {{Deep Neural Networks}}},
  author = {<NAME>. and Lee, <NAME>. and <NAME> and <NAME> and <NAME>},
  date = {2019-05-28},
  url = {http://arxiv.org/abs/1811.03804},
  urldate = {2020-01-15},
  abstract = {Gradient descent finds a global minimum in training deep neural networks despite the objective function being non-convex. The current paper proves gradient descent achieves zero training loss in polynomial time for a deep overparameterized neural network with residual connections (ResNet). Our analysis relies on the particular structure of the Gram matrix induced by the neural network architecture. This structure allows us to show the Gram matrix is stable throughout the training process and this stability implies the global optimality of the gradient descent algorithm. We further extend our analysis to deep residual convolutional neural networks and obtain a similar convergence result.},
  archivePrefix = {arXiv},
  eprint = {1811.03804},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/F5KSRC5J/Du et al. - 2019 - Gradient Descent Finds Global Minima of Deep Neura.pdf},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Computer Vision and Pattern Recognition,Computer Science - Machine Learning,Mathematics - Optimization and Control,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, math, stat}
}

@article{dyerAsymptoticsWideNetworks2019,
  title = {Asymptotics of {{Wide Networks}} from {{Feynman Diagrams}}},
  author = {<NAME> and <NAME>},
  date = {2019-09-25},
  url = {http://arxiv.org/abs/1909.11304},
  urldate = {2020-02-21},
  abstract = {Understanding the asymptotic behavior of wide networks is of considerable interest. In this work, we present a general method for analyzing this large width behavior. The method is an adaptation of Feynman diagrams, a standard tool for computing multivariate Gaussian integrals. We apply our method to study training dynamics, improving existing bounds and deriving new results on wide network evolution during stochastic gradient descent. Going beyond the strict large width limit, we present closed-form expressions for higher-order terms governing wide network training, and test these predictions empirically.},
  archivePrefix = {arXiv},
  eprint = {1909.11304},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/JT3KB3FR/Dyer and Gur-Ari - 2019 - Asymptotics of Wide Networks from Feynman Diagrams.pdf},
  keywords = {Computer Science - Machine Learning,High Energy Physics - Theory,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {hep-th, stat}
}

@article{fanLIBLINEARLibraryLarge,
  title = {{{LIBLINEAR}}: {{A Library}} for {{Large Linear Classiﬁcation}}},
  author = {<NAME> and <NAME> and <NAME> and <NAME> and <NAME>},
  pages = {4},
  abstract = {LIBLINEAR is an open source library for large-scale linear classification. It supports logistic regression and linear support vector machines. We provide easy-to-use command-line tools and library calls for users and developers. Comprehensive documents are available for both beginners and advanced users. Experiments demonstrate that LIBLINEAR is very efficient on large sparse data sets.},
  file = {/Users/hudson_home/Zotero/storage/9Y9269C8/Fan et al. - LIBLINEAR A Library for Large Linear Classiﬁcatio.pdf},
  langid = {english}
}

@article{fanSpectraConjugateKernel2020,
  title = {Spectra of the {{Conjugate Kernel}} and {{Neural Tangent Kernel}} for Linear-Width Neural Networks},
  author = {<NAME> and <NAME>},
  date = {2020-06-02},
  url = {http://arxiv.org/abs/2005.11879},
  urldate = {2020-06-04},
  abstract = {We study the eigenvalue distributions of the Conjugate Kernel and Neural Tangent Kernel associated to multi-layer feedforward neural networks. In an asymptotic regime where network width is increasing linearly in sample size, under random initialization of the weights, and for input samples satisfying a notion of approximate pairwise orthogonality, we show that the eigenvalue distributions of the CK and NTK converge to deterministic limits. The limit for the CK is described by iterating the Marcenko-Pastur map across the hidden layers. The limit for the NTK is equivalent to that of a linear combination of the CK matrices across layers, and may be described by recursive fixed-point equations that extend this Marcenko-Pastur map. We demonstrate the agreement of these asymptotic predictions with the observed spectra for both synthetic and CIFAR-10 training data, and we perform a small simulation to investigate the evolutions of these spectra over training.},
  archivePrefix = {arXiv},
  eprint = {2005.11879},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/2ZWNHFRX/Fan and Wang - 2020 - Spectra of the Conjugate Kernel and Neural Tangent.pdf},
  keywords = {Computer Science - Machine Learning,Mathematics - Probability,Mathematics - Statistics Theory,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, math, stat}
}

@article{franzSimplestModelJamming2016,
  title = {The Simplest Model of Jamming},
  author = {<NAME> and <NAME>},
  date = {2016-04-08},
  journaltitle = {Journal of Physics A: Mathematical and Theoretical},
  shortjournal = {J. Phys. A: Math. Theor.},
  volume = {49},
  pages = {145001},
  issn = {1751-8113, 1751-8121},
  doi = {10.1088/1751-8113/49/14/145001},
  url = {https://iopscience.iop.org/article/10.1088/1751-8113/49/14/145001},
  urldate = {2020-06-01},
  abstract = {We study a well known neural network model—the perceptron—as a simple statistical physics model of jamming of hard objects. We exhibit two regimes: (1) a convex optimization regime where jamming is hypostatic and non-critical; (2) a non-convex optimization regime where jamming is isostatic and critical. We characterize the critical jamming phase through exponents describing the distribution laws of forces and gaps. Surprisingly we find that these exponents coincide with the corresponding ones recently computed in high dimensional hard spheres. In addition, modifying the perceptron to a random linear programming problem, we show that isostaticity is not a sufficient condition for singular force and gap distributions. For that, fragmentation of the space of solutions (replica symmetry breaking) appears to be a crucial ingredient. We hypothesize universality for a large class of non-convex constrained satisfaction problems with continuous variables.},
  file = {/Users/hudson_home/Zotero/storage/PHZRKCTW/Franz and Parisi - 2016 - The simplest model of jamming.pdf},
  langid = {english},
  number = {14}
}

@article{franzUniversalitySATUNSATJamming2017,
  title = {Universality of the {{SAT}}-{{UNSAT}} (Jamming) Threshold in Non-Convex Continuous Constraint Satisfaction Problems},
  author = {<NAME> <NAME> <NAME> <NAME> <NAME>},
  date = {2017-06-02},
  journaltitle = {SciPost Physics},
  shortjournal = {SciPost Phys.},
  volume = {2},
  pages = {019},
  issn = {2542-4653},
  doi = {10.21468/SciPostPhys.2.3.019},
  url = {http://arxiv.org/abs/1702.06919},
  urldate = {2020-08-26},
  abstract = {Random constraint satisfaction problems (CSP) have been studied extensively using statistical physics techniques. They provide a benchmark to study average case scenarios instead of the worst case one. The interplay between statistical physics of disordered systems and computer science has brought new light into the realm of computational complexity theory, by introducing the notion of clustering of solutions, related to replica symmetry breaking. However, the class of problems in which clustering has been studied often involve discrete degrees of freedom: standard random CSPs are random K-SAT (aka disordered Ising models) or random coloring problems (aka disordered Potts models). In this work we consider instead problems that involve continuous degrees of freedom. The simplest prototype of these problems is the perceptron. Here we discuss in detail the full phase diagram of the model. In the regions of parameter space where the problem is non-convex, leading to multiple disconnected clusters of solutions, the solution is critical at the SAT/UNSAT threshold and lies in the same universality class of the jamming transition of soft spheres. We show how the critical behavior at the satisfiability threshold emerges, and we compute the critical exponents associated to the approach to the transition from both the SAT and UNSAT phase. We conjecture that there is a large universality class of non-convex continuous CSPs whose SAT-UNSAT threshold is described by the same scaling solution.},
  archivePrefix = {arXiv},
  eprint = {1702.06919},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/43ALHGBZ/Franz et al. - 2017 - Universality of the SAT-UNSAT (jamming) threshold .pdf},
  keywords = {Condensed Matter - Disordered Systems and Neural Networks},
  langid = {english},
  number = {3}
}

@article{geigerDisentanglingFeatureLazy2020,
  title = {Disentangling Feature and Lazy Training in Deep Neural Networks},
  author = {<NAME> <NAME> Jacot, <NAME> <NAME>},
  date = {2020-06-01},
  url = {http://arxiv.org/abs/1906.08034},
  urldate = {2020-06-27},
  abstract = {Two distinct limits for deep learning have been derived as the network width h → ∞, depending on how the weights of the last layer scale with h. In the Neural Tangent Kernel (NTK) limit, the dynamics becomes linear in the weights and is described by a frozen kernel Θ (the NTK). By contrast, in the Mean-Field limit, the dynamics can be expressed in terms of the distribution of the parameters associated with a neuron, that follows a partial differential equation. In this work we consider deep networks where the weights in the last layer scale as αh−1/2 at initialization. By varying α and h, we probe the crossover between the two limits. We observe the two previously identified regimes of “lazy training” and “feature training”. In the lazytraining regime, the dynamics is almost linear and the NTK barely changes after initialization. The featuretraining regime includes the mean-field formulation as a limiting case and is characterized by a kernel that evolves in time, and thus learns some features. We perform numerical experiments on MNIST, FashionMNIST, EMNIST and CIFAR10 and consider various architectures. We find that: (i) The two regimes are separated by an α∗ that scales as √1 . (ii) Network architecture and data structure play an important role h in determining which regime is better: in our tests, fully-connected networks perform generally better in the lazy-training regime, unlike convolutional networks. (iii) In√both regimes, the fluctuations δF induced on the learned function by initial conditions decay as δF ∼ 1/ h, leading to a performance that increases with h. The same improvement can also be obtained at an intermediate width by ensemble-averaging severa√l networks that are trained independently. (iv) In the feature-training regime we identify a time sc√ale t1 ∼ hα, such that for t t1 the dynamics is linear. At t ∼ t1, the output has grown by a magni√tude h and the changes of the tangent kernel ||∆Θ|| become significant. Ultimately, it follows ||∆Θ|| ∼ ( hα)−a for ReLU and Softplus activation functions, with a {$<$} 2 and a → 2 as depth grows. We provide scaling arguments supporting these findings.},
  archivePrefix = {arXiv},
  eprint = {1906.08034},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/P5B4BWP4/Geiger et al. - 2020 - Disentangling feature and lazy training in deep ne.pdf},
  keywords = {Computer Science - Machine Learning,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, stat}
}

@article{geigerJammingTransitionParadigm2019,
  title = {The Jamming Transition as a Paradigm to Understand the Loss Landscape of Deep Neural Networks},
  author = {<NAME> <NAME> and <NAME>, Stéphane and Sagun, Levent and Baity-Jesi, Marco and <NAME> and <NAME>},
  date = {2019-07-11},
  journaltitle = {Physical Review E},
  shortjournal = {Phys. Rev. E},
  volume = {100},
  pages = {012115},
  issn = {2470-0045, 2470-0053},
  doi = {10.1103/PhysRevE.100.012115},
  url = {http://arxiv.org/abs/1809.09349},
  urldate = {2020-01-21},
  abstract = {Deep learning has been immensely successful at a variety of tasks, ranging from classification to AI. Learning corresponds to fitting training data, which is implemented by descending a very high-dimensional loss function. Understanding under which conditions neural networks do not get stuck in poor minima of the loss, and how the landscape of that loss evolves as depth is increased remains a challenge. Here we predict, and test empirically, an analogy between this landscape and the energy landscape of repulsive ellipses. We argue that in FC networks a phase transition delimits the over- and under-parametrized regimes where fitting can or cannot be achieved. In the vicinity of this transition, properties of the curvature of the minima of the loss are critical. This transition shares direct similarities with the jamming transition by which particles form a disordered solid as the density is increased, which also occurs in certain classes of computational optimization and learning problems such as the perceptron. Our analysis gives a simple explanation as to why poor minima of the loss cannot be encountered in the overparametrized regime, and puts forward the surprising result that the ability of fully connected networks to fit random data is independent of their depth. Our observations suggests that this independence also holds for real data. We also study a quantity \$\textbackslash Delta\$ which characterizes how well (\$\textbackslash Delta{$<$}0\$) or badly (\$\textbackslash Delta{$>$}0\$) a datum is learned. At the critical point it is power-law distributed, \$P\_+(\textbackslash Delta)\textbackslash sim\textbackslash Delta\^\textbackslash theta\$ for \$\textbackslash Delta{$>$}0\$ and \$P\_-(\textbackslash Delta)\textbackslash sim(-\textbackslash Delta)\^\{-\textbackslash gamma\}\$ for \$\textbackslash Delta{$<$}0\$, with \$\textbackslash theta\textbackslash approx0.3\$ and \$\textbackslash gamma\textbackslash approx0.2\$. This observation suggests that near the transition the loss landscape has a hierarchical structure and that the learning dynamics is prone to avalanche-like dynamics, with abrupt changes in the set of patterns that are learned.},
  archivePrefix = {arXiv},
  eprint = {1809.09349},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/6AWIQNZQ/Geiger et al. - 2019 - The jamming transition as a paradigm to understand.pdf},
  keywords = {Computer Science - Machine Learning,Condensed Matter - Disordered Systems and Neural Networks},
  langid = {english},
  number = {1},
  options = {useprefix=true}
}

@article{geigerScalingDescriptionGeneralization2019,
  title = {Scaling Description of Generalization with Number of Parameters in Deep Learning},
  author = {<NAME> <NAME> <NAME>, <NAME> Hongler, Clément and <NAME>},
  date = {2019-10-08},
  url = {http://arxiv.org/abs/1901.01608},
  urldate = {2020-02-11},
  abstract = {Supervised deep learning involves the training of neural networks with a large number \$N\$ of parameters. For large enough \$N\$, in the so-called over-parametrized regime, one can essentially fit the training data points. Sparsity-based arguments would suggest that the generalization error increases as \$N\$ grows past a certain threshold \$N\^\{*\}\$. Instead, empirical studies have shown that in the over-parametrized regime, generalization error keeps decreasing with \$N\$. We resolve this paradox through a new framework. We rely on the so-called Neural Tangent Kernel, which connects large neural nets to kernel methods, to show that the initialization causes finite-size random fluctuations \$\textbackslash |f\_\{N\}-\textbackslash bar\{f\}\_\{N\}\textbackslash |\textbackslash sim N\^\{-1/4\}\$ of the neural net output function \$f\_\{N\}\$ around its expectation \$\textbackslash bar\{f\}\_\{N\}\$. These affect the generalization error \$\textbackslash epsilon\_\{N\}\$ for classification: under natural assumptions, it decays to a plateau value \$\textbackslash epsilon\_\{\textbackslash infty\}\$ in a power-law fashion \$\textbackslash sim N\^\{-1/2\}\$. This description breaks down at a so-called jamming transition \$N=N\^\{*\}\$. At this threshold, we argue that \$\textbackslash |f\_\{N\}\textbackslash |\$ diverges. This result leads to a plausible explanation for the cusp in test error known to occur at \$N\^\{*\}\$. Our results are confirmed by extensive empirical observations on the MNIST and CIFAR image datasets. Our analysis finally suggests that, given a computational envelope, the smallest generalization error is obtained using several networks of intermediate sizes, just beyond \$N\^\{*\}\$, and averaging their outputs.},
  archivePrefix = {arXiv},
  eprint = {1901.01608},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/LRQHSNVZ/Geiger et al. - 2019 - Scaling description of generalization with number .pdf;/Users/hudson_home/Zotero/storage/SCSBAMNI/Geiger et al. - 2019 - Scaling description of generalization with number .pdf},
  keywords = {Computer Science - Machine Learning,Condensed Matter - Disordered Systems and Neural Networks},
  langid = {english},
  options = {useprefix=true},
  primaryClass = {cond-mat}
}

@article{goelEigenvalueDecayImplies,
  title = {Eigenvalue {{Decay Implies Polynomial}}-{{Time Learnability}} for {{Neural Networks}}},
  author = {<NAME> and <NAME>},
  pages = {11},
  abstract = {We consider the problem of learning function classes computed by neural networks with various activations (e.g. ReLU or Sigmoid), a task believed to be computationally intractable in the worst-case. A major open problem is to understand the minimal assumptions under which these classes admit provably efficient algorithms. In this work we show that a natural distributional assumption corresponding to eigenvalue decay of the Gram matrix yields polynomial-time algorithms in the non-realizable setting for expressive classes of networks (e.g. feed-forward networks of ReLUs). We make no assumptions on the structure of the network or the labels. Given sufficiently-strong eigenvalue decay, we obtain fully-polynomial time algorithms in all the relevant parameters with respect to square-loss. This is the first purely distributional assumption that leads to polynomial-time algorithms for networks of ReLUs. Further, unlike prior distributional assumptions (e.g., the marginal distribution is Gaussian), eigenvalue decay has been observed in practice on common data sets.},
  file = {/Users/hudson_home/Zotero/storage/P956KTJ8/Goel and Klivans - Eigenvalue Decay Implies Polynomial-Time Learnabil.pdf},
  langid = {english}
}

@article{goodfellowQualitativelyCharacterizingNeural2015,
  title = {Qualitatively Characterizing Neural Network Optimization Problems},
  author = {Goodfellow, <NAME>. and Vinyals, <NAME>, <NAME>.},
  date = {2015-05-21},
  url = {http://arxiv.org/abs/1412.6544},
  urldate = {2020-08-23},
  abstract = {Training neural networks involves solving large-scale non-convex optimization problems. This task has long been believed to be extremely difficult, with fear of local minima and other obstacles motivating a variety of schemes to improve optimization, such as unsupervised pretraining. However, modern neural networks are able to achieve negligible training error on complex tasks, using only direct training with stochastic gradient descent. We introduce a simple analysis technique to look for evidence that such networks are overcoming local optima. We find that, in fact, on a straight path from initialization to solution, a variety of state of the art neural networks never encounter any significant obstacles.},
  archivePrefix = {arXiv},
  eprint = {1412.6544},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/CTXARUS9/Goodfellow et al. - 2015 - Qualitatively characterizing neural network optimi.pdf},
  keywords = {Computer Science - Machine Learning,Computer Science - Neural and Evolutionary Computing,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, stat}
}

@article{haninFiniteDepthWidth2019,
  title = {Finite {{Depth}} and {{Width Corrections}} to the {{Neural Tangent Kernel}}},
  author = {<NAME> and <NAME>},
  date = {2019-09-12},
  url = {http://arxiv.org/abs/1909.05989},
  urldate = {2020-02-21},
  abstract = {We prove the precise scaling, at finite depth and width, for the mean and variance of the neural tangent kernel (NTK) in a randomly initialized ReLU network. The standard deviation is exponential in the ratio of network depth to width. Thus, even in the limit of infinite overparameterization, the NTK is not deterministic if depth and width simultaneously tend to infinity. Moreover, we prove that for such deep and wide networks, the NTK has a non-trivial evolution during training by showing that the mean of its first SGD update is also exponential in the ratio of network depth to width. This is sharp contrast to the regime where depth is fixed and network width is very large. Our results suggest that, unlike relatively shallow and wide networks, deep and wide ReLU networks are capable of learning data-dependent features even in the so-called lazy training regime.},
  archivePrefix = {arXiv},
  eprint = {1909.05989},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/FL3Z8ZN7/Hanin and Nica - 2019 - Finite Depth and Width Corrections to the Neural T.pdf},
  keywords = {Computer Science - Machine Learning,Mathematics - Probability,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, math, stat}
}

@article{hayouImpactActivationFunction2019,
  title = {On the {{Impact}} of the {{Activation Function}} on {{Deep Neural Networks Training}}},
  author = {<NAME> and <NAME> <NAME>},
  date = {2019-05-26},
  url = {http://arxiv.org/abs/1902.06853},
  urldate = {2020-03-02},
  abstract = {The weight initialization and the activation function of deep neural networks have a crucial impact on the performance of the training procedure. An inappropriate selection can lead to the loss of information of the input during forward propagation and the exponential vanishing/exploding of gradients during back-propagation. Understanding the theoretical properties of untrained random networks is key to identifying which deep networks may be trained successfully as recently demonstrated by (Schoenholz et al., 2017) who showed that for deep feedforward neural networks only a specific choice of hyperparameters known as the ‘Edge of Chaos’ can lead to good performance. While the work by (Schoenholz et al., 2017) discuss trainability issues, we focus here on training acceleration and overall performance. We give a comprehensive theoretical analysis of the Edge of Chaos and show that we can indeed tune the initialization parameters and the activation function in order to accelerate the training and improve performance.},
  archivePrefix = {arXiv},
  eprint = {1902.06853},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/6P2TA23M/Hayou et al. - 2019 - On the Impact of the Activation Function on Deep N.pdf},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Machine Learning,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, stat}
}

@article{hayouMeanfieldBehaviourNeural2020,
  title = {Mean-Field {{Behaviour}} of {{Neural Tangent Kernel}} for {{Deep Neural Networks}}},
  author = {<NAME> <NAME>},
  date = {2020-02-18},
  url = {http://arxiv.org/abs/1905.13654},
  urldate = {2020-03-02},
  abstract = {Recent influential work by Jacot et al. [2018] has shown that training a neural network of any kind with gradient descent in parameter space is strongly related to kernel gradient descent in function space with respect to the Neural Tangent Kernel (NTK). Lee et al. [2019] built on this result by establishing that the output of a neural network trained using gradient descent can be approximated by a linear model for wide networks. In parallel, a recent line of studies Schoenholz et al. [2017], Hayou et al. [2019] has suggested that a special initialization known as the Edge of Chaos improves training. In this paper, we bridge the gap between these two concepts by quantifying the impact of the initialization and the activation function on the NTK when the network depth becomes large. We provide experiments illustrating our theoretical results.},
  archivePrefix = {arXiv},
  eprint = {1905.13654},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/TMPKRPZQ/Hayou et al. - 2020 - Mean-field Behaviour of Neural Tangent Kernel for .pdf},
  keywords = {Computer Science - Machine Learning,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, stat}
}

@article{jacotAsymptoticSpectrumHessian2020,
  title = {The Asymptotic Spectrum of the {{Hessian}} of {{DNN}} throughout Training},
  author = {<NAME> and <NAME> and <NAME>},
  date = {2020-02-10},
  url = {http://arxiv.org/abs/1910.02875},
  urldate = {2020-06-04},
  abstract = {The dynamics of DNNs during gradient descent is described by the so-called Neural Tangent Kernel (NTK). In this article, we show that the NTK allows one to gain precise insight into the Hessian of the cost of DNNs. When the NTK is fixed during training, we obtain a full characterization of the asymptotics of the spectrum of the Hessian, at initialization and during training. In the so-called mean-field limit, where the NTK is not fixed during training, we describe the first two moments of the Hessian at initialization.},
  archivePrefix = {arXiv},
  eprint = {1910.02875},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/8RZF9HGR/Jacot et al. - 2020 - The asymptotic spectrum of the Hessian of DNN thro.pdf},
  keywords = {Computer Science - Machine Learning,Computer Science - Neural and Evolutionary Computing,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, stat}
}

@article{jacotNeuralTangentKernel2018,
  title = {Neural {{Tangent Kernel}}: {{Convergence}} and {{Generalization}} in {{Neural Networks}}},
  shorttitle = {Neural {{Tangent Kernel}}},
  author = {<NAME> and <NAME> and <NAME>},
  date = {2018-11-26},
  url = {http://arxiv.org/abs/1806.07572},
  urldate = {2020-01-21},
  abstract = {At initialization, artificial neural networks (ANNs) are equivalent to Gaussian processes in the infinite-width limit (14; 11), thus connecting them to kernel methods. We prove that the evolution of an ANN during training can also be described by a kernel: during gradient descent on the parameters of an ANN, the network function fθ (which maps input vectors to output vectors) follows the kernel gradient of the functional cost (which is convex, in contrast to the parameter cost) w.r.t. a new kernel: the Neural Tangent Kernel (NTK). This kernel is central to describe the generalization features of ANNs. While the NTK is random at initialization and varies during training, in the infinite-width limit it converges to an explicit limiting kernel and it stays constant during training. This makes it possible to study the training of ANNs in function space instead of parameter space. Convergence of the training can then be related to the positive-definiteness of the limiting NTK. We prove the positive-definiteness of the limiting NTK when the data is supported on the sphere and the non-linearity is non-polynomial.},
  archivePrefix = {arXiv},
  eprint = {1806.07572},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/457PIYHM/Jacot et al. - 2018 - Neural Tangent Kernel Convergence and Generalizat.pdf},
  keywords = {Computer Science - Machine Learning,Computer Science - Neural and Evolutionary Computing,Mathematics - Probability,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, math, stat}
}

@article{jacquinMicroscopicMeanfieldTheory2011,
  title = {A Microscopic Mean-Field Theory of the Jamming Transition},
  author = {<NAME> and <NAME> and <NAME>},
  date = {2011-03-31},
  journaltitle = {Physical Review Letters},
  shortjournal = {Phys. Rev. Lett.},
  volume = {106},
  pages = {135702},
  issn = {0031-9007, 1079-7114},
  doi = {10.1103/PhysRevLett.106.135702},
  url = {http://arxiv.org/abs/1011.5638},
  urldate = {2020-03-10},
  abstract = {Dense particle packings acquire rigidity through a nonequilibrium jamming transition commonly observed in materials from emulsions to sandpiles. We describe athermal packings and their observed geometric phase transitions using fully equilibrium statistical mechanics and develop a microscopic many-body mean-field theory of the jamming transition for soft repulsive spherical particles. We derive analytically some of the scaling laws and exponents characterizing the transition and obtain predictions for microscopic correlation functions of jammed states that are amenable to experimental verifications, and whose accuracy we confirm using computer simulations.},
  archivePrefix = {arXiv},
  eprint = {1011.5638},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/JV28FTJ5/Jacquin et al. - 2011 - A microscopic mean-field theory of the jamming tra.pdf},
  keywords = {Condensed Matter - Statistical Mechanics},
  langid = {english},
  number = {13}
}

@article{karakidaUniversalStatisticsFisher2019,
  title = {Universal {{Statistics}} of {{Fisher Information}} in {{Deep Neural Networks}}: {{Mean Field Approach}}},
  shorttitle = {Universal {{Statistics}} of {{Fisher Information}} in {{Deep Neural Networks}}},
  author = {<NAME> <NAME>},
  date = {2019-10-08},
  url = {http://arxiv.org/abs/1806.01316},
  urldate = {2020-03-02},
  abstract = {The Fisher information matrix (FIM) is a fundamental quantity to represent the characteristics of a stochastic model, including deep neural networks (DNNs). The present study reveals novel statistics of FIM that are universal among a wide class of DNNs. To this end, we use random weights and large width limits, which enables us to utilize mean field theories. We investigate the asymptotic statistics of the FIM’s eigenvalues and reveal that most of them are close to zero while the maximum eigenvalue takes a huge value. Because the landscape of the parameter space is defined by the FIM, it is locally flat in most dimensions, but strongly distorted in others. Moreover, we demonstrate the potential usage of the derived statistics in learning strategies. First, small eigenvalues that induce flatness can be connected to a norm-based capacity measure of generalization ability. Second, the maximum eigenvalue that induces the distortion enables us to quantitatively estimate an appropriately sized learning rate for gradient methods to converge.},
  archivePrefix = {arXiv},
  eprint = {1806.01316},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/MKPNEYQ3/Karakida et al. - 2019 - Universal Statistics of Fisher Information in Deep.pdf},
  keywords = {Computer Science - Machine Learning,Condensed Matter - Disordered Systems and Neural Networks,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cond-mat, stat}
}

@article{krzakalaLandscapeAnalysisConstraint2007,
  title = {A {{Landscape Analysis}} of {{Constraint Satisfaction Problems}}},
  author = {<NAME> and <NAME>},
  date = {2007-08-27},
  journaltitle = {Physical Review E},
  shortjournal = {Phys. Rev. E},
  volume = {76},
  pages = {021122},
  issn = {1539-3755, 1550-2376},
  doi = {10.1103/PhysRevE.76.021122},
  url = {http://arxiv.org/abs/cond-mat/0702546},
  urldate = {2020-08-26},
  abstract = {We discuss an analysis of Constraint Satisfaction problems, such as Sphere Packing, K-SAT and Graph Coloring, in terms of an effective energy landscape. Several intriguing geometrical properties of the solution space become in this light familiar in terms of the well-studied ones of rugged (glassy) energy landscapes. A `benchmark' algorithm naturally suggested by this construction finds solutions in polynomial time up to a point beyond the `clustering' and in some cases even the `thermodynamic' transitions. This point has a simple geometric meaning and can be in principle determined with standard Statistical Mechanical methods, thus pushing the analytic bound up to which problems are guaranteed to be easy. We illustrate this for the graph three and four-coloring problem. For Packing problems the present discussion allows to better characterize the `J-point', proposed as a systematic definition of Random Close Packing, and to place it in the context of other theories of glasses.},
  archivePrefix = {arXiv},
  eprint = {cond-mat/0702546},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/TV5NWZT6/Krzakala and Kurchan - 2007 - A Landscape Analysis of Constraint Satisfaction Pr.pdf},
  keywords = {Computer Science - Computational Complexity,Condensed Matter - Disordered Systems and Neural Networks,Condensed Matter - Statistical Mechanics,Nonlinear Sciences - Chaotic Dynamics},
  langid = {english},
  number = {2}
}

@article{lecunDeepLearning2015,
  title = {Deep Learning},
  author = {<NAME> and <NAME> and <NAME>},
  date = {2015-05},
  journaltitle = {Nature},
  shortjournal = {Nature},
  volume = {521},
  pages = {436--444},
  issn = {0028-0836, 1476-4687},
  doi = {10.1038/nature14539},
  url = {http://www.nature.com/articles/nature14539},
  urldate = {2020-08-24},
  file = {/Users/hudson_home/Zotero/storage/FHI2UU7Z/LeCun et al. - 2015 - Deep learning.pdf},
  langid = {english},
  number = {7553}
}

@article{leeDeepNeuralNetworks2018,
  title = {Deep {{Neural Networks}} as {{Gaussian Processes}}},
  author = {<NAME> <NAME> Schoenholz, <NAME>. and <NAME> <NAME>},
  date = {2018-03-02},
  url = {http://arxiv.org/abs/1711.00165},
  urldate = {2020-08-27},
  abstract = {It has long been known that a single-layer fully-connected neural network with an i.i.d. prior over its parameters is equivalent to a Gaussian process (GP), in the limit of infinite network width. This correspondence enables exact Bayesian inference for infinite width neural networks on regression tasks by means of evaluating the corresponding GP. Recently, kernel functions which mimic multi-layer random neural networks have been developed, but only outside of a Bayesian framework. As such, previous work has not identified that these kernels can be used as covariance functions for GPs and allow fully Bayesian prediction with a deep neural network.},
  archivePrefix = {arXiv},
  eprint = {1711.00165},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/APDLR7QZ/Lee et al. - 2018 - Deep Neural Networks as Gaussian Processes.pdf},
  keywords = {Computer Science - Machine Learning,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, stat}
}

@article{leeWideNeuralNetworks2019,
  title = {Wide {{Neural Networks}} of {{Any Depth Evolve}} as {{Linear Models Under Gradient Descent}}},
  author = {<NAME> <NAME> and <NAME>. and <NAME> and <NAME> and <NAME> <NAME>},
  date = {2019-12-08},
  url = {http://arxiv.org/abs/1902.06720},
  urldate = {2020-02-12},
  abstract = {A longstanding goal in deep learning research has been to precisely characterize training and generalization. However, the often complex loss landscapes of neural networks have made a theory of learning dynamics elusive. In this work, we show that for wide neural networks the learning dynamics simplify considerably and that, in the infinite width limit, they are governed by a linear model obtained from the first-order Taylor expansion of the network around its initial parameters. Furthermore, mirroring the correspondence between wide Bayesian neural networks and Gaussian processes, gradient-based training of wide neural networks with a squared loss produces test set predictions drawn from a Gaussian process with a particular compositional kernel. While these theoretical results are only exact in the infinite width limit, we nevertheless find excellent empirical agreement between the predictions of the original network and those of the linearized version even for finite practically-sized networks. This agreement is robust across different architectures, optimization methods, and loss functions.},
  archivePrefix = {arXiv},
  eprint = {1902.06720},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/CTX65ZL9/Lee et al. - 2019 - Wide Neural Networks of Any Depth Evolve as Linear.pdf},
  keywords = {Computer Science - Machine Learning,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, stat}
}

@article{liaoSpectrumRandomFeatures2018,
  title = {On the {{Spectrum}} of {{Random Features Maps}} of {{High Dimensional Data}}},
  author = {<NAME> and <NAME>},
  date = {2018-07-20},
  url = {http://arxiv.org/abs/1805.11916},
  urldate = {2020-06-04},
  abstract = {Random feature maps are ubiquitous in modern statistical machine learning, where they generalize random projections by means of powerful, yet often difficult to analyze nonlinear operators. In this paper, we leverage the “concentration” phenomenon induced by random matrix theory to perform a spectral analysis on the Gram matrix of these random feature maps, here for Gaussian mixture models of simultaneously large dimension and size. Our results are instrumental to a deeper understanding on the interplay of the nonlinearity and the statistics of the data, thereby allowing for a better tuning of random featurebased techniques.},
  archivePrefix = {arXiv},
  eprint = {1805.11916},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/87BW98HR/Liao and Couillet - 2018 - On the Spectrum of Random Features Maps of High Di.pdf},
  keywords = {Computer Science - Machine Learning,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, stat}
}

@article{liuExactHighdimensionalAsymptotics2019,
  title = {Exact High-Dimensional Asymptotics for {{Support Vector Machine}}},
  author = {<NAME>},
  date = {2019-07-31},
  url = {http://arxiv.org/abs/1905.05125},
  urldate = {2020-06-28},
  abstract = {The Support Vector Machine (SVM) is one of the most widely used classification methods. In this paper, we consider the soft-margin SVM used on data points with independent features, where the sample size n and the feature dimension p grows to ∞ in a fixed ratio p/n → δ. We propose a set of equations that exactly characterizes the asymptotic behavior of support vector machine. In particular, we give exact formulas for (1) the variability of the optimal coefficients, (2) the proportion of data points lying on the margin boundary (i.e. number of support vectors), (3) the final objective function value, and (4) the expected misclassification error on new data points, which in particular implies the exact formula for the optimal tuning parameter given a data generating mechanism. We first establish these formulas in the case where the label y ∈ \{+1, −1\} is independent of the feature x. Then the results are generalized to the case where the label y ∈ \{+1, −1\} is allowed to have a general dependence on the feature x through a linear combination a0T x. These formulas for the non-smooth hinge loss are analogous to the recent results in [Sur and Cand`es, 2018] for smooth logistic loss. Our approach is based on heuristic leave-one-out calculations.},
  archivePrefix = {arXiv},
  eprint = {1905.05125},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/RTGGLDG8/Liu - 2019 - Exact high-dimensional asymptotics for Support Vec.pdf},
  keywords = {Computer Science - Machine Learning,Mathematics - Statistics Theory,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, math, stat}
}

@article{liuJammingScenarioIntroduction2011,
  title = {The Jamming Scenario - an Introduction and Outlook},
  author = {<NAME>. and <NAME>. and <NAME>, <NAME> <NAME>},
  date = {2011-01-27},
  url = {http://arxiv.org/abs/1006.2365},
  urldate = {2020-03-15},
  abstract = {The jamming scenario of disordered media, formulated about 10 years ago, has in recent years been advanced by analyzing model systems of granular media. This has led to various new concepts that are increasingly being explored in in a variety of systems. This chapter contains an introductory review of these recent developments and provides an outlook on their applicability to different physical systems and on future directions. The first part of the paper is devoted to an overview of the findings for model systems of frictionless spheres, focussing on the excess of low-frequency modes as the jamming point is approached. Particular attention is paid to a discussion of the cross-over frequency and length scales that govern this approach. We then discuss the effects of particle asphericity and static friction, the applicability to bubble models for wet foams in which the friction is dynamic, the dynamical arrest in colloids, and the implications for molecular glasses.},
  archivePrefix = {arXiv},
  eprint = {1006.2365},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/LBHW63QL/Liu et al. - 2011 - The jamming scenario - an introduction and outlook.pdf},
  keywords = {Condensed Matter - Materials Science,Condensed Matter - Soft Condensed Matter,Condensed Matter - Statistical Mechanics},
  langid = {english},
  options = {useprefix=true},
  primaryClass = {cond-mat}
}

@article{louartRandomMatrixApproach2017,
  title = {A {{Random Matrix Approach}} to {{Neural Networks}}},
  author = {<NAME> and <NAME> and <NAME>},
  date = {2017-06-29},
  url = {http://arxiv.org/abs/1702.05419},
  urldate = {2020-06-04},
  abstract = {This article studies the Gram random matrix model \$G=\textbackslash frac1T\textbackslash Sigma\^\{\textbackslash rm T\}\textbackslash Sigma\$, \$\textbackslash Sigma=\textbackslash sigma(WX)\$, classically found in the analysis of random feature maps and random neural networks, where \$X=[x\_1,\textbackslash ldots,x\_T]\textbackslash in\{\textbackslash mathbb R\}\^\{p\textbackslash times T\}\$ is a (data) matrix of bounded norm, \$W\textbackslash in\{\textbackslash mathbb R\}\^\{n\textbackslash times p\}\$ is a matrix of independent zero-mean unit variance entries, and \$\textbackslash sigma:\{\textbackslash mathbb R\}\textbackslash to\{\textbackslash mathbb R\}\$ is a Lipschitz continuous (activation) function --- \$\textbackslash sigma(WX)\$ being understood entry-wise. By means of a key concentration of measure lemma arising from non-asymptotic random matrix arguments, we prove that, as \$n,p,T\$ grow large at the same rate, the resolvent \$Q=(G+\textbackslash gamma I\_T)\^\{-1\}\$, for \$\textbackslash gamma{$>$}0\$, has a similar behavior as that met in sample covariance matrix models, involving notably the moment \$\textbackslash Phi=\textbackslash frac\{T\}n\{\textbackslash mathbb E\}[G]\$, which provides in passing a deterministic equivalent for the empirical spectral measure of \$G\$. Application-wise, this result enables the estimation of the asymptotic performance of single-layer random neural networks. This in turn provides practical insights into the underlying mechanisms into play in random neural networks, entailing several unexpected consequences, as well as a fast practical means to tune the network hyperparameters.},
  archivePrefix = {arXiv},
  eprint = {1702.05419},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/ZU8J6EGB/Louart et al. - 2017 - A Random Matrix Approach to Neural Networks.pdf},
  keywords = {Computer Science - Machine Learning,Mathematics - Probability},
  langid = {english},
  primaryClass = {cs, math}
}

@article{meiGeneralizationErrorRandom2019,
  title = {The Generalization Error of Random Features Regression: {{Precise}} Asymptotics and Double Descent Curve},
  shorttitle = {The Generalization Error of Random Features Regression},
  author = {<NAME> and Montanari, Andrea},
  date = {2019-10-22},
  url = {http://arxiv.org/abs/1908.05355},
  urldate = {2020-06-18},
  abstract = {Deep learning methods operate in regimes that defy the traditional statistical mindset. Neural network architectures often contain more parameters than training samples, and are so rich that they can interpolate the observed labels, even if the latter are replaced by pure noise. Despite their huge complexity, the same architectures achieve small generalization error on real data.},
  archivePrefix = {arXiv},
  eprint = {1908.05355},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/GQZPL8F3/Mei and Montanari - 2019 - The generalization error of random features regres.pdf},
  keywords = {62J99,Mathematics - Statistics Theory,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {math, stat}
}

@article{meiMeanFieldView2018,
  title = {A {{Mean Field View}} of the {{Landscape}} of {{Two}}-{{Layers Neural Networks}}},
  author = {<NAME> and Montanari, Andrea and <NAME>},
  date = {2018-08-28},
  url = {http://arxiv.org/abs/1804.06561},
  urldate = {2020-08-22},
  abstract = {Multi-layer neural networks are among the most powerful models in machine learning, yet the fundamental reasons for this success defy mathematical understanding. Learning a neural network requires to optimize a non-convex high-dimensional objective (risk function), a problem which is usually attacked using stochastic gradient descent (SGD). Does SGD converge to a global optimum of the risk or only to a local optimum? In the first case, does this happen because local minima are absent, or because SGD somehow avoids them? In the second, why do local minima reached by SGD have good generalization properties? In this paper we consider a simple case, namely two-layers neural networks, and prove that –in a suitable scaling limit– SGD dynamics is captured by a certain non-linear partial differential equation (PDE) that we call distributional dynamics (DD). We then consider several specific examples, and show how DD can be used to prove convergence of SGD to networks with nearlyideal generalization error. This description allows to ‘average-out’ some of the complexities of the landscape of neural networks, and can be used to prove a general convergence result for noisy SGD.},
  archivePrefix = {arXiv},
  eprint = {1804.06561},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/C5SX8DWI/Mei et al. - 2018 - A Mean Field View of the Landscape of Two-Layers N.pdf},
  keywords = {Computer Science - Machine Learning,Condensed Matter - Statistical Mechanics,Mathematics - Statistics Theory,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cond-mat, stat}
}

@article{mullerMarginalStabilityStructural2015,
  title = {Marginal {{Stability}} in {{Structural}}, {{Spin}}, and {{Electron Glasses}}},
  author = {<NAME> and Wyart, Matthieu},
  date = {2015-03},
  journaltitle = {Annual Review of Condensed Matter Physics},
  shortjournal = {Annu. Rev. Condens. Matter Phys.},
  volume = {6},
  pages = {177--200},
  issn = {1947-5454, 1947-5462},
  doi = {10.1146/annurev-conmatphys-031214-014614},
  url = {http://www.annualreviews.org/doi/10.1146/annurev-conmatphys-031214-014614},
  urldate = {2020-06-25},
  abstract = {We revisit the concept of marginal stability in glasses and determine its range of applicability in the context of an avalanche-type response to slow external driving. We argue that there is an intimate connection between a pseudogap in the distribution of local fields and crackling in systems with long-range interactions. We classify glassy systems according to the presence or absence of marginal stability, providing a unifying perspective on the phenomenology of systems as diverse as spin and electron glasses, hard spheres, pinned elastic interfaces, and soft amorphous solids undergoing plastic deformation.},
  file = {/Users/hudson_home/Zotero/storage/RKS3VTPZ/Müller and Wyart - 2015 - Marginal Stability in Structural, Spin, and Electr.pdf},
  langid = {english},
  number = {1}
}

@article{nealModernTakeBiasVariance2019,
  title = {A {{Modern Take}} on the {{Bias}}-{{Variance Tradeoff}} in {{Neural Networks}}},
  author = {<NAME> <NAME> Baratin, <NAME> Tantia, <NAME>, <NAME> Lacoste-Julien, <NAME> Mitliagkas, Ioannis},
  date = {2019-12-18},
  url = {http://arxiv.org/abs/1810.08591},
  urldate = {2020-08-24},
  abstract = {The bias-variance tradeoff tells us that as model complexity increases, bias falls and variances increases, leading to a U-shaped test error curve. However, recent empirical results with over-parameterized neural networks are marked by a striking absence of the classic U-shaped test error curve: test error keeps decreasing in wider networks. This suggests that there might not be a bias-variance tradeoff in neural networks with respect to network width, unlike was originally claimed by, e.g., Geman et al. (1992). Motivated by the shaky evidence used to support this claim in neural networks, we measure bias and variance in the modern setting. We find that both bias and variance can decrease as the number of parameters grows. To better understand this, we introduce a new decomposition of the variance to disentangle the effects of optimization and data sampling. We also provide theoretical analysis in a simplified setting that is consistent with our empirical findings.},
  archivePrefix = {arXiv},
  eprint = {1810.08591},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/RP6JRBNR/Neal et al. - 2019 - A Modern Take on the Bias-Variance Tradeoff in Neu.pdf},
  keywords = {Computer Science - Machine Learning,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, stat}
}

@article{neyshaburGeometryOptimizationImplicit2017,
  title = {Geometry of {{Optimization}} and {{Implicit Regularization}} in {{Deep Learning}}},
  author = {<NAME> <NAME> Salakhutdinov, <NAME> <NAME>},
  date = {2017-05-08},
  url = {http://arxiv.org/abs/1705.03071},
  urldate = {2020-08-26},
  abstract = {We argue that the optimization plays a crucial role in generalization of deep learning models through implicit regularization. We do this by demonstrating that generalization ability is not controlled by network size but rather by some other implicit control. We then demonstrate how changing the empirical optimization procedure can improve generalization, even if actual optimization quality is not affected. We do so by studying the geometry of the parameter space of deep networks, and devising an optimization algorithm attuned to this geometry.},
  archivePrefix = {arXiv},
  eprint = {1705.03071},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/REARE29G/Neyshabur et al. - 2017 - Geometry of Optimization and Implicit Regularizati.pdf},
  keywords = {Computer Science - Machine Learning},
  langid = {english},
  primaryClass = {cs}
}

@article{novakNeuralTangentsFast2019,
  title = {Neural {{Tangents}}: {{Fast}} and {{Easy Infinite Neural Networks}} in {{Python}}},
  shorttitle = {Neural {{Tangents}}},
  author = {<NAME>, <NAME>. and Sohl-Dickstein, <NAME>, <NAME>.},
  date = {2019-12-05},
  url = {http://arxiv.org/abs/1912.02803},
  urldate = {2020-03-02},
  abstract = {NEURAL TANGENTS is a library designed to enable research into infinite-width neural networks. It provides a high-level API for specifying complex and hierarchical neural network architectures. These networks can then be trained and evaluated either at finite-width as usual or in their infinite-width limit. Infinite-width networks can be trained analytically using exact Bayesian inference or using gradient descent via the Neural Tangent Kernel. Additionally, NEURAL TANGENTS provides tools to study gradient descent training dynamics of wide but finite networks in either function space or weight space.},
  archivePrefix = {arXiv},
  eprint = {1912.02803},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/BJ8A8RIU/Novak et al. - 2019 - Neural Tangents Fast and Easy Infinite Neural Net.pdf},
  keywords = {Computer Science - Machine Learning,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, stat}
}

@article{pedregosaScikitlearnMachineLearning,
  title = {Scikit-Learn: {{Machine Learning}} in {{Python}}},
  author = {<NAME> and Varoquaux, <NAME> Gramfort, Alexandre and <NAME>, Bertrand and Grisel, Olivier and Blondel, Mathieu and Prettenhofer, Peter and Weiss, Ron and Dubourg, Vincent and Vanderplas, Jake and Passos, Alexandre and Cournapeau, David},
  journaltitle = {MACHINE LEARNING IN PYTHON},
  pages = {6},
  abstract = {Scikit-learn is a Python module integrating a wide range of state-of-the-art machine learning algorithms for medium-scale supervised and unsupervised problems. This package focuses on bringing machine learning to non-specialists using a general-purpose high-level language. Emphasis is put on ease of use, performance, documentation, and API consistency. It has minimal dependencies and is distributed under the simplified BSD license, encouraging its use in both academic and commercial settings. Source code, binaries, and documentation can be downloaded from http://scikit-learn.sourceforge.net.},
  file = {/Users/hudson_home/Zotero/storage/HGZ5TV84/Pedregosa et al. - Scikit-learn Machine Learning in Python.pdf},
  langid = {english}
}

@article{pellegriniAnalyticTheoryShallow2020,
  title = {An Analytic Theory of Shallow Networks Dynamics for Hinge Loss Classification},
  author = {Pellegrini, Franco and Biroli, Giulio},
  date = {2020-06-19},
  url = {http://arxiv.org/abs/2006.11209},
  urldate = {2020-08-22},
  abstract = {Neural networks have been shown to perform incredibly well in classification tasks over structured high-dimensional datasets. However, the learning dynamics of such networks is still poorly understood. In this paper we study in detail the training dynamics of a simple type of neural network: a single hidden layer trained to perform a classification task. We show that in a suitable mean-field limit this case maps to a single-node learning problem with a time-dependent dataset determined self-consistently from the average nodes population. We specialize our theory to the prototypical case of a linearly separable dataset and a linear hinge loss, for which the dynamics can be explicitly solved. This allow us to address in a simple setting several phenomena appearing in modern networks such as slowing down of training dynamics, crossover between rich and lazy learning, and overfitting. Finally, we asses the limitations of mean-field theory by studying the case of large but finite number of nodes and of training samples.},
  archivePrefix = {arXiv},
  eprint = {2006.11209},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/566YWLER/Pellegrini and Biroli - 2020 - An analytic theory of shallow networks dynamics fo.pdf},
  keywords = {Computer Science - Machine Learning,Condensed Matter - Disordered Systems and Neural Networks,Condensed Matter - Statistical Mechanics,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cond-mat, stat}
}

@article{penningtonNonlinearRandomMatrix2019,
  title = {Nonlinear Random Matrix Theory for Deep Learning},
  author = {<NAME> and <NAME>},
  date = {2019-12-20},
  journaltitle = {Journal of Statistical Mechanics: Theory and Experiment},
  shortjournal = {J. Stat. Mech.},
  volume = {2019},
  pages = {124005},
  issn = {1742-5468},
  doi = {10.1088/1742-5468/ab3bc3},
  url = {https://iopscience.iop.org/article/10.1088/1742-5468/ab3bc3},
  urldate = {2020-06-08},
  file = {/Users/hudson_home/Zotero/storage/5JAPHNC2/Pennington and Worah - 2019 - Nonlinear random matrix theory for deep learning.pdf},
  langid = {english},
  number = {12}
}

@article{rahimiRandomFeaturesLargeScale2008,
  title = {Random {{Features}} for {{Large}}-{{Scale Kernel Machines}}},
  author = {<NAME> Recht, Benjamin},
  date = {2008},
  journaltitle = {Advances in neural information processing systems},
  pages = {1177--1184},
  abstract = {To accelerate the training of kernel machines, we propose to map the input data to a randomized low-dimensional feature space and then apply existing fast linear methods. The features are designed so that the inner products of the transformed data are approximately equal to those in the feature space of a user specified shiftinvariant kernel. We explore two sets of random features, provide convergence bounds on their ability to approximate various radial basis kernels, and show that in large-scale classification and regression tasks linear machine learning algorithms applied to these features outperform state-of-the-art large-scale kernel machines.},
  file = {/Users/hudson_home/Zotero/storage/JLGZUBTJ/Rahimi and Recht - Random Features for Large-Scale Kernel Machines.pdf},
  langid = {english}
}

@online{rajatUnderstandingNeuralTangent,
  title = {Understanding the {{Neural Tangent Kernel}}},
  author = {Rajat},
  file = {/Users/hudson_home/Zotero/storage/VMDUUPH4/github_io.pdf}
}

@article{rotskoffTrainabilityAccuracyNeural2019,
  title = {Trainability and {{Accuracy}} of {{Neural Networks}}: {{An Interacting Particle System Approach}}},
  shorttitle = {Trainability and {{Accuracy}} of {{Neural Networks}}},
  author = {<NAME>. and <NAME>},
  date = {2019-07-30},
  url = {http://arxiv.org/abs/1805.00915},
  urldate = {2020-01-15},
  abstract = {Neural networks, a central tool in machine learning, have demonstrated remarkable, high fidelity performance on image recognition and classification tasks. These successes evince an ability to accurately represent high dimensional functions, but rigorous results about the approximation error of neural networks after training are few. Here we establish conditions for global convergence of the standard optimization algorithm used in machine learning applications, stochastic gradient descent (SGD), and quantify the scaling of its error with the size of the network. This is done by reinterpreting SGD as the evolution of a particle system with interactions governed by a potential related to the objective or “loss” function used to train the network. We show that, when the number n of units is large, the empirical distribution of the particles descends on a convex landscape towards the global minimum at a rate independent of n, with a resulting approximation error that universally scales as O(n−1). These properties are established in the form of a Law of Large Numbers and a Central Limit Theorem for the empirical distribution. Our analysis also quantifies the scale and nature of the noise introduced by SGD and provides guidelines for the step size and batch size to use when training a neural network. We illustrate our findings on examples in which we train neural networks to learn the energy function of the continuous 3-spin model on the sphere. The approximation error scales as our analysis predicts in as high a dimension as d = 25.},
  archivePrefix = {arXiv},
  eprint = {1805.00915},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/2NY6LCWZ/Rotskoff and Vanden-Eijnden - 2019 - Trainability and Accuracy of Neural Networks An I.pdf},
  keywords = {Computer Science - Machine Learning,Condensed Matter - Statistical Mechanics,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cond-mat, stat}
}

@article{spiglerJammingTransitionOverparametrization2019,
  title = {A Jamming Transition from Under- to over-Parametrization Affects Loss Landscape and Generalization},
  author = {<NAME> and <NAME> <NAME>},
  date = {2019-11-22},
  journaltitle = {Journal of Physics A: Mathematical and Theoretical},
  shortjournal = {<NAME>. A: Math. Theor.},
  volume = {52},
  pages = {474001},
  issn = {1751-8113, 1751-8121},
  doi = {10.1088/1751-8121/ab4c8b},
  url = {http://arxiv.org/abs/1810.09665},
  urldate = {2020-01-21},
  abstract = {In this paper we first recall the recent result that in deep networks a phase transition, analogous to the jamming transition of granular media, delimits the over- and under-parametrized regimes where fitting can or cannot be achieved. The analysis leading to this result support that for proper initialization and architectures, in the whole over-parametrized regime poor minima of the loss are not encountered during training, because the number of constraints that hinders the dynamics is insufficient to allow for the emergence of stable minima. Next, we study systematically how this transition affects generalization properties of the network (i.e. its predictive power). As we increase the number of parameters of a given model, starting from an under-parametrized network, we observe for gradient descent that the generalization error displays three phases: (i) initial decay, (ii) increase until the transition point — where it displays a cusp — and (iii) slow decay toward an asymptote as the network width diverges. However if early stopping is used, the cusp signaling the jamming transition disappears. Thereby we identify the region where the classical phenomenon of over-fitting takes place as the vicinity of the jamming transition, and the region where the model keeps improving with increasing the number of parameters, thus organizing previous empirical observations made in modern neural networks.},
  archivePrefix = {arXiv},
  eprint = {1810.09665},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/8S8IYISB/Spigler et al. - 2019 - A jamming transition from under- to over-parametri.pdf},
  keywords = {Computer Science - Machine Learning,Condensed Matter - Disordered Systems and Neural Networks,Statistics - Machine Learning},
  langid = {english},
  number = {47},
  options = {useprefix=true}
}

@article{surModernMaximumlikelihoodTheory2018,
  title = {A Modern Maximum-Likelihood Theory for High-Dimensional Logistic Regression},
  author = {<NAME> and Candes, <NAME>.},
  date = {2018-06-16},
  url = {http://arxiv.org/abs/1803.06964},
  urldate = {2020-06-29},
  abstract = {Every student in statistics or data science learns early on that when the sample size n largely exceeds the number p of variables, fitting a logistic model produces estimates that are approximately unbiased. Every student also learns that there are formulas to predict the variability of these estimates which are used for the purpose of statistical inference; for instance, to produce p-values for testing the significance of regression coefficients. Although these formulas come from large sample asymptotics, we are often told that we are on reasonably safe grounds when n is large in such a way that n ≥ 5p or n ≥ 10p. This paper shows that this is far from the case, and consequently, inferences routinely produced by common software packages are often unreliable.},
  archivePrefix = {arXiv},
  eprint = {1803.06964},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/2Y3LHSH5/Sur and Candes - 2018 - A modern maximum-likelihood theory for high-dimens.pdf},
  keywords = {Mathematics - Statistics Theory,Statistics - Methodology},
  langid = {english},
  primaryClass = {math, stat}
}

@article{wilsonBayesianDeepLearning2020,
  title = {Bayesian {{Deep Learning}} and a {{Probabilistic Perspective}} of {{Generalization}}},
  author = {<NAME> and <NAME>},
  date = {2020-02-20},
  url = {http://arxiv.org/abs/2002.08791},
  urldate = {2020-02-21},
  abstract = {The key distinguishing property of a Bayesian approach is marginalization, rather than using a single setting of weights. Bayesian marginalization can particularly improve the accuracy and calibration of modern deep neural networks, which are typically underspecified by the data, and can represent many compelling but different solutions. We show that deep ensembles provide an effective mechanism for approximate Bayesian marginalization, and propose a related approach that further improves the predictive distribution by marginalizing within basins of attraction, without significant overhead. We also investigate the prior over functions implied by a vague distribution over neural network weights, explaining the generalization properties of such models from a probabilistic perspective. From this perspective, we explain results that have been presented as mysterious and distinct to neural network generalization, such as the ability to fit images with random labels, and show that these results can be reproduced with Gaussian processes. Finally, we provide a Bayesian perspective on tempering for calibrating predictive distributions.},
  archivePrefix = {arXiv},
  eprint = {2002.08791},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/LWMMXZWU/Wilson and Izmailov - 2020 - Bayesian Deep Learning and a Probabilistic Perspec.pdf},
  keywords = {Computer Science - Machine Learning,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, stat}
}

@article{wyartEffectsCompressionVibrational2005,
  title = {Effects of Compression on the Vibrational Modes of Marginally Jammed Solids},
  author = {<NAME> Silbert, <NAME>. and Nagel, <NAME>. and Witten, <NAME>.},
  date = {2005-11-30},
  journaltitle = {Physical Review E},
  shortjournal = {Phys. Rev. E},
  volume = {72},
  pages = {051306},
  issn = {1539-3755, 1550-2376},
  doi = {10.1103/PhysRevE.72.051306},
  url = {http://arxiv.org/abs/cond-mat/0508415},
  urldate = {2020-02-25},
  abstract = {Glasses have a large excess of low-frequency vibrational modes in comparison with most crystalline solids. We show that such a feature is a necessary consequence of the weak connectivity of the solid, and that the frequency of modes in excess is very sensitive to the pressure. We analyze in particular two systems whose density D(w) of vibrational modes of angular frequency w display scaling behaviors with the packing fraction: (i) simulations of jammed packings of particles interacting through finite-range, purely repulsive potentials, comprised of weakly compressed spheres at zero temperature and (ii) a system with the same network of contacts, but where the force between any particles in contact (and therefore the total pressure) is set to zero. We account in the two cases for the observed a) convergence of D(w) toward a non-zero constant as w goes to 0, b) appearance of a low-frequency cutoff w*, and c) power-law increase of w* with compression. Differences between these two systems occur at lower frequency. The density of states of the modified system displays an abrupt plateau that appears at w*, below which we expect the system to behave as a normal, continuous, elastic body. In the unmodified system, the pressure lowers the frequency of the modes in excess. The requirement of stability despite the destabilizing effect of pressure yields a lower bound on the number of extra contact per particle dz: dz {$>$} p\^(1/2), which generalizes the Maxwell criterion for rigidity when pressure is present. This scaling behavior is observed in the simulations. We finally discuss how the cooling procedure can affect the microscopic structure and the density of normal modes.},
  archivePrefix = {arXiv},
  eprint = {cond-mat/0508415},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/NAMYBDMT/Wyart et al. - 2005 - Effects of compression on the vibrational modes of.pdf},
  keywords = {Condensed Matter - Disordered Systems and Neural Networks,Condensed Matter - Materials Science},
  langid = {english},
  number = {5}
}

@article{yangFineGrainedSpectralPerspective2020,
  title = {A {{Fine}}-{{Grained Spectral Perspective}} on {{Neural Networks}}},
  author = {<NAME> and <NAME>},
  date = {2020-04-09},
  url = {http://arxiv.org/abs/1907.10599},
  urldate = {2020-06-03},
  abstract = {Are neural networks biased toward simple functions? Does depth always help learn more complex features? Is training the last layer of a network as good as training all layers? How to set the range for learning rate tuning? These questions seem unrelated at face value, but in this work we give all of them a common treatment from the spectral perspective. We will study the spectra of the Conjugate Kernel, CK, (also called the Neural Network-Gaussian Process Kernel), and the Neural Tangent Kernel, NTK. Roughly, the CK and the NTK tell us respectively “what a network looks like at initialization” and “what a network looks like during and after training.” Their spectra then encode valuable information about the initial distribution and the training and generalization properties of neural networks. By analyzing the eigenvalues, we lend novel insights into the questions put forth at the beginning, and we verify these insights by extensive experiments of neural networks. We derive fast algorithms for computing the spectra of CK and NTK when the data is uniformly distributed over the boolean cube, and show this spectra is the same in high dimensions when data is drawn from isotropic Gaussian or uniformly over the sphere. Code replicating our results is available at https://github.com/ thegregyang/NNspectra.},
  archivePrefix = {arXiv},
  eprint = {1907.10599},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/SNQ53Z9G/Yang and Salman - 2020 - A Fine-Grained Spectral Perspective on Neural Netw.pdf},
  keywords = {Computer Science - Machine Learning,Computer Science - Neural and Evolutionary Computing,Statistics - Machine Learning},
  langid = {english},
  primaryClass = {cs, stat}
}

@article{yaoEarlyStoppingGradient2007,
  title = {On {{Early Stopping}} in {{Gradient Descent Learning}}},
  author = {<NAME> <NAME> <NAME>},
  date = {2007-08},
  journaltitle = {Constructive Approximation},
  shortjournal = {Constr Approx},
  volume = {26},
  pages = {289--315},
  issn = {0176-4276, 1432-0940},
  doi = {10.1007/s00365-006-0663-2},
  url = {http://link.springer.com/10.1007/s00365-006-0663-2},
  urldate = {2020-08-27},
  abstract = {In this paper, we study a family of gradient descent algorithms to approximate the regression function from Reproducing Kernel Hilbert Spaces (RKHSs), the family being characterized by a polynomial decreasing rate of step sizes (or learning rate). By solving a bias-variance trade-off we obtain an early stopping rule and some probabilistic upper bounds for the convergence of the algorithms. These upper bounds have improved rates where the usual regularized least square algorithm fails and achieve the minimax optimal rate O(m−1/2) in some cases. We also discuss the implication of these results in the context of classification. Some connections are addressed with Boosting, Landweber iterations, and the on-line learning algorithms as stochastic approximations of the gradient descent method.},
  file = {/Users/hudson_home/Zotero/storage/NXEZ7AAR/Yao et al. - 2007 - On Early Stopping in Gradient Descent Learning.pdf},
  langid = {english},
  number = {2}
}

@article{zdeborovaPhaseTransitionsColoring2007,
  title = {Phase {{Transitions}} in the {{Coloring}} of {{Random Graphs}}},
  author = {Zdeborová, Lenka and Krzakala, Florent},
  date = {2007-09-26},
  journaltitle = {Physical Review E},
  shortjournal = {Phys. Rev. E},
  volume = {76},
  pages = {031131},
  issn = {1539-3755, 1550-2376},
  doi = {10.1103/PhysRevE.76.031131},
  url = {http://arxiv.org/abs/0704.1269},
  urldate = {2020-08-26},
  abstract = {We consider the problem of coloring the vertices of a large sparse random graph with a given number of colors so that no adjacent vertices have the same color. Using the cavity method, we present a detailed and systematic analytical study of the space of proper colorings (solutions). We show that for a fixed number of colors and as the average vertex degree (number of constraints) increases, the set of solutions undergoes several phase transitions similar to those observed in the mean field theory of glasses. First, at the clustering transition, the entropically dominant part of the phase space decomposes into an exponential number of pure states so that beyond this transition a uniform sampling of solutions becomes hard. Afterward, the space of solutions condenses over a finite number of the largest states and consequently the total entropy of solutions becomes smaller than the annealed one. Another transition takes place when in all the entropically dominant states a finite fraction of nodes freezes so that each of these nodes is allowed a single color in all the solutions inside the state. Eventually, above the coloring threshold, no more solutions are available. We compute all the critical connectivities for Erdos-Renyi and regular random graphs and determine their asymptotic values for large number of colors. Finally, we discuss the algorithmic consequences of our findings. We argue that the onset of computational hardness is not associated with the clustering transition and we suggest instead that the freezing transition might be the relevant phenomenon. We also discuss the performance of a simple local Walk-COL algorithm and of the belief propagation algorithm in the light of our results.},
  archivePrefix = {arXiv},
  eprint = {0704.1269},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/DGVEC49C/Zdeborová and Krzakala - 2007 - Phase Transitions in the Coloring of Random Graphs.pdf},
  keywords = {Computer Science - Computational Complexity,Condensed Matter - Disordered Systems and Neural Networks,Condensed Matter - Statistical Mechanics},
  langid = {english},
  number = {3}
}

@article{zhangUnderstandingDeepLearning2017,
  title = {Understanding Deep Learning Requires Rethinking Generalization},
  author = {<NAME>},
  date = {2017-02-26},
  url = {http://arxiv.org/abs/1611.03530},
  urldate = {2020-01-21},
  abstract = {Despite their massive size, successful deep artificial neural networks can exhibit a remarkably small difference between training and test performance. Conventional wisdom attributes small generalization error either to properties of the model family, or to the regularization techniques used during training.},
  archivePrefix = {arXiv},
  eprint = {1611.03530},
  eprinttype = {arxiv},
  file = {/Users/hudson_home/Zotero/storage/KGVXYJUM/Zhang et al. - 2017 - Understanding deep learning requires rethinking ge.pdf},
  keywords = {Computer Science - Machine Learning},
  langid = {english},
  primaryClass = {cs}
}

@preamble{ "\ifdefined\DeclarePrefChars\DeclarePrefChars{'’-}\else\fi " }

