@inproceedings{Zhou2018NonvacuousGB,
  title={Non-vacuous Generalization Bounds at the ImageNet Scale: a PAC-Bayesian Compression Approach},
  author={<NAME> and <NAME> and <NAME> and <NAME> and <NAME>},
  booktitle={ICLR},
  year={2018}
}

@inproceedings{Conneau2018Cram,
    title = "What you can cram into a single {\$}{\&}!{\#}* vector: Probing sentence embeddings for linguistic properties",
    author = {Conneau, Alexis  and
      <NAME>  and
      <NAME>  and
      <NAME>{\"\i}c  and
      <NAME>},
    booktitle = "Annual Meeting of the Association for Computational Linguistics",
    year = "2018",
    pages = "2126--2136"
}

@article{Oord2018RepresentationLW,
  title={Representation Learning with Contrastive Predictive Coding},
  author={<NAME> and <NAME> and <NAME>},
  journal={arXiv preprint arXiv:1807.03748},
  year={2018}
}

@article{He2019MomentumCF,
  title={Momentum Contrast for Unsupervised Visual Representation Learning},
  author={<NAME> and <NAME> and <NAME> and <NAME> and <NAME>},
  journal={arXiv preprint arXiv:1911.05722},
  year={2019}
}

@inproceedings{Jiang2019FantasticGM,
  title={Fantastic Generalization Measures and Where to Find Them},
  author={<NAME> and <NAME> and <NAME> and <NAME> and <NAME>},
  year={2019}
}

@article{McAllester2013APT,
  title={A PAC-Bayesian Tutorial with A Dropout Bound},
  author={<NAME>},
  journal={arXiv preprint arXiv:1307.2118},
  year={2013}
}

@misc{Dziugaite2017Nonvacuous,
    title={Computing Nonvacuous Generalization Bounds for Deep (Stochastic) Neural Networks with Many More Parameters than Training Data},
    author={<NAME> and <NAME>},
    year={2017},
    eprint={1703.11008},
    archivePrefix={arXiv},
    primaryClass={cs.LG}
}

@article{Rivasplata2019PACBayesWB,
  title={PAC-Bayes with Backprop},
  author={<NAME> and <NAME> and <NAME>},
  journal={arXiv preprint arXiv:1908.07380},
  year={2019}
}

@article{Chen2020ASF,
  title={A Simple Framework for Contrastive Learning of Visual Representations},
  author={<NAME> and <NAME> and <NAME> and <NAME>},
  journal={arXiv preprint arXiv:2002.05709},
  year={2020}
}

@article{Voita2020InformationTheoreticPW,
  title={Information-Theoretic Probing with Minimum Description Length},
  author={<NAME> and <NAME>},
  journal={arXiv preprint arXiv:2003.12298},
  year={2020}
}

@inproceedings{Amit2017MetaLearningBA,
  title={Meta-Learning by Adjusting Priors Based on Extended PAC-Bayes Theory},
  author={<NAME> and <NAME>},
  booktitle={International Conference on Machine Learning},
  year={2017}
}

@book{Cover2006ElementsOI,
  title={Elements of information theory},
  author={<NAME> and <NAME>},
  year={2006},
  publisher={Wiley}
}

@article{Song2020UnderstandingTL,
  title={Understanding the Limitations of Variational Mutual Information Estimators},
  author={<NAME> and <NAME>},
  journal={International Conference on Learning Representations},
  year={2020},
}

@article{McAllester2018FormalLO,
  title={Formal Limitations on the Measurement of Mutual Information},
  author={<NAME> and <NAME>},
  journal={International Conference on Artificial Intelligence and Statistics},
  year={2020}
}


@inproceedings{Tschannen2020OnMI,
title={On Mutual Information Maximization for Representation Learning},
author={<NAME> and <NAME> and <NAME> and <NAME> and <NAME>},
booktitle={International Conference on Learning Representations},
year={2020}
}

@article{Pimentel2020InformationTheoreticPF,
  title={Information-Theoretic Probing for Linguistic Structure},
  author={<NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME>},
  journal={arXiv preprint arXiv:2004.03061},
  year={2020}
}

@article{Grnwald2004ATI,
  title={A tutorial introduction to the minimum description length principle},
  author={<NAME>}nwald},
  journal={arXiv preprint math:0406077},
  year={2004}
}

@article{Rissanen1978ModelingBS,
  title={Modeling By Shortest Data Description},
  author={<NAME>},
  journal={Automatica},
  year={1978},
  volume={14},
  pages={465-471}
}

@article{Yogatama2019LinguisticIntel,
  title={Learning and evaluating general linguistic intelligence},
  author={<NAME>, <NAME> and Connor, Jerome and Kocisky, Tomas and Chrzanowski, Mike and Kong, Lingpeng and Lazaridou, Angeliki and Ling, Wang and Yu, Lei and Dyer, Chris and others},
  journal={arXiv preprint arXiv:1901.11373},
  year={2019}
}

@article{Belinkov2019AnalysisNLP,
  title={Analysis methods in neural language processing: A survey},
  author={<NAME> and <NAME>},
  journal={Transactions of the Association for Computational Linguistics},
  volume={7},
  pages={49--72},
  year={2019},
  publisher={MIT Press}
}

@article{Talmor2019oLMpics,
  title={o{LM}pics--On what Language Model Pre-training Captures},
  author={<NAME> and <NAME> and <NAME> and <NAME>},
  journal={arXiv preprint arXiv:1912.13283},
  year={2019}
}

@inproceedings{Blier2018TheDL,
  title={The Description Length of Deep Learning models},
  author={L{\'<NAME> and <NAME>},
  booktitle={Advances in Neural Information Processing Systems},
  year={2018}
}

@inproceedings{Hewitt2019DesigningProbes,
    title = "Designing and Interpreting Probes with Control Tasks",
    author = "Hewitt, John  and
      Liang, Percy",
    booktitle = "Empirical Methods in Natural Language Processing and International Joint Conference on Natural Language Processing",
    year = "2019",
    pages = "2733-2743"
}

@inproceedings{
    Xu2020ATheory,
    title={A Theory of Usable Information under Computational Constraints},
    author={<NAME> and <NAME> and <NAME> and <NAME> and <NAME>mon},
    booktitle={International Conference on Learning Representations},
    year={2020}
}

@article{Shannon1948TheMT,
  title={A mathematical theory of communication},
  author={<NAME>},
  journal={Bell Syst. Tech. J.},
  year={1948},
  volume={27},
  pages={379-423}
}

@article{Dawid1984Prequential,
 author = {<NAME>},
 journal = {Journal of the Royal Statistical Society. Series A (General)},
 number = {2},
 pages = {278--292},
 publisher = {[Royal Statistical Society, Wiley]},
 title = {Present Position and Potential Developments: Some Personal Views: Statistical Theory: The Prequential Approach},
 volume = {147},
 year = {1984}
}

@inproceedings{Zhang2018LanguageMT,
    title = "Language Modeling Teaches You More than Translation Does: Lessons Learned Through Auxiliary Syntactic Task Analysis",
    author = "<NAME>  and
      <NAME>",
    booktitle = "{EMNLP} Workshop {B}lackbox{NLP}: Analyzing and Interpreting Neural Networks for {NLP}",
    year = "2018",
    pages = "359--361"
}

@article{he2019momentum,
  title={Momentum contrast for unsupervised visual representation learning},
  author={<NAME> and <NAME> and <NAME> and <NAME> and <NAME>},
  journal={arXiv preprint arXiv:1911.05722},
  year={2019}
}

@inproceedings{bachman2019learning,
  title={Learning representations by maximizing mutual information across views},
  author={<NAME> <NAME> and <NAME>},
  booktitle={Advances in Neural Information Processing Systems},
  year={2019}
}

@inproceedings{devlin2018bert,
    title = "{BERT}: Pre-training of Deep Bidirectional Transformers for Language Understanding",
    author = "<NAME>  and
      <NAME>  and
      Lee, Kenton  and
      Toutanova, Kristina",
    booktitle = "North {A}merican Chapter of the Association for Computational Linguistics",
    year = "2019",
    pages = "4171--4186"
}

@article{liu2019roberta,
  title={{RoBERTa}: A robustly optimized {BERT} pretraining approach},
  author={<NAME> and Ott, Myle and <NAME> and <NAME> and <NAME> and <NAME> and Levy, Omer and Lewis, Mike and Zettlemoyer, <NAME> Stoyanov, Veselin},
  journal={arXiv preprint arXiv:1907.11692},
  year={2019}
}

@article{raffel2019exploring,
  title={Exploring the limits of transfer learning with a unified text-to-text transformer},
  author={<NAME> and Shazeer, <NAME> Roberts, Adam and Lee, Katherine and Narang, Sharan and Matena, Michael and Zhou, Yanqi and <NAME> and Liu, <NAME>},
  journal={arXiv preprint arXiv:1910.10683},
  year={2019}
}

@ARTICLE{gpt3,
       author = {{Brown}, <NAME>. and {Mann}, Benjamin and {Ryder}, Nick and
         {Subbiah}, Melanie and {Kaplan}, Jared and {Dhariwal}, Prafulla and
         {Neelakantan}, Arvind and {Shyam}, Pranav and {Sastry}, Girish and
         {Askell}, Amanda and {Agarwal}, Sandhini and {Herbert-Voss}, Ariel and
         {Krueger}, Gretchen and {Henighan}, Tom and {Child}, Rewon and
         {Ramesh}, Aditya and {Ziegler}, <NAME>. and {Wu}, Jeffrey and
         {Winter}, Clemens and {Hesse}, Christopher and {Chen}, Mark and
         {Sigler}, Eric and {Litwin}, Mateusz and {Gray}, Scott and
         {Chess}, Benjamin and {Clark}, Jack and {Berner}, Christopher and {McCand
        lish}, Sam and {Radford}, Alec and {Sutskever}, Ilya and {Amodei}, Dario},
        title = "{Language Models are Few-Shot Learners}",
      journal = {arXiv preprint arXiv:2005.14165},
         year = {2020}
}


@article{lecun2015deep,
  title={Deep learning},
  author={<NAME> and <NAME> and <NAME>},
  journal={Nature},
  volume={521},
  number={7553},
  pages={436--444},
  year={2015},
  publisher={Nature Publishing Group}
}

@article{resnick2019probing,
  title={Probing the State of the Art: A Critical Look at Visual Representation Evaluation},
  author={<NAME> and <NAME> and <NAME>},
  journal={arXiv preprint arXiv:1912.00215},
  year={2019}
}

@inproceedings{ettinger2016probing,
  title={Probing for semantic evidence of composition by means of simple classification tasks},
  author={Ettinger, Allyson and <NAME> and Resnik, Philip},
  booktitle={Workshop on Evaluating Vector-Space Representations for NLP},
  pages={134--139},
  year={2016}
}

@article{Alain2016Understanding,
  title={Understanding intermediate layers using linear classifier probes},
  author={<NAME> and <NAME>},
  journal={International Conference on Learning Representations},
  year={2016}
}

@inproceedings{Shi2016String,
    title = "Does String-Based Neural {MT} Learn Source Syntax?",
    author = "<NAME>  and
      <NAME>  and
      <NAME>",
    booktitle = "Empirical Methods in Natural Language Processing",
    year = "2016",
    pages = "1526--1534"
}

@article{Kingma2014AutoEncodingVB,
  title={Auto-Encoding Variational {B}ayes},
  author={<NAME> and <NAME>},
  journal={International Conference on Learning Representations},
  year={2014}
}

@inproceedings{Rezende2014StochasticBA,
  title={Stochastic Backpropagation and Approximate Inference in Deep Generative Models},
  author={<NAME> and <NAME> and <NAME>},
  booktitle={International Conference on Machine Learning},
  year={2014}
}

@inproceedings{Peters2018DeepCW,
    title = "Deep Contextualized Word Representations",
    author = "<NAME>  and
      <NAME>  and
      <NAME>  and
      Gardner, Matt  and
      <NAME>  and
      <NAME>  and
      <NAME>",
    booktitle = "North {A}merican Chapter of the Association for Computational Linguistics",
    year = "2018",
    pages = "2227--2237"
}

@article{Erven2012CatchingUF,
author = {<NAME> and Grünwald, Peter and <NAME>},
year = {2012},
month = {06},
pages = {},
title = {Catching up faster by switching sooner: A predictive approach to adaptive stimation with an application to the AIC-BIC dilemma},
volume = {74},
journal = {Journal of the Royal Statistical Society. Series B (Statistical Methodology)}
}

@inproceedings{Strubell2019EnergyAP,
  title={Energy and Policy Considerations for Deep Learning in NLP},
  author={<NAME> and <NAME> and <NAME>},
  booktitle={ACL},
  year={2019}
}

@inproceedings{kurita-etal-2019-measuring,
    title = "Measuring Bias in Contextualized Word Representations",
    author = "<NAME>  and
      <NAME>  and
      Pareek, Ayush  and
      <NAME>  and
      <NAME>",
    booktitle = "Workshop on Gender Bias in Natural Language Processing",
    year = "2019",
    pages = "166--172"
}

@inproceedings{bordia-bowman-2019-identifying,
    title = "Identifying and Reducing Gender Bias in Word-Level Language Models",
    author = "<NAME>  and
      <NAME>.",
    booktitle = "North {A}merican Chapter of the Association for Computational Linguistics: Student Research Workshop",
    year = "2019",
    pages = "7--15"
}

@inproceedings{Paszke2019PyTorchAI,
  title={PyTorch: An Imperative Style, High-Performance Deep Learning Library},
  author={<NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME>},
  booktitle={NeurIPS},
  year={2019}
}

@article{Kingma2015AdamAM,
  title={Adam: A Method for Stochastic Optimization},
  author={<NAME> and <NAME>},
  journal={CoRR},
  year={2015},
  volume={abs/1412.6980}
}

@software{jax2018github,
  author = {<NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME>},
  title = {{JAX}: composable transformations of {P}ython+{N}um{P}y programs},
  url = {http://github.com/google/jax},
  version = {0.1.55},
  year = {2018},
}

@Misc{embedtheworld,
author = {<NAME>},
title = {Deep Learning \& Convolutional Networks},
howpublished = {\textsc{url:}~\url{https://www.hotchips.org/wp-content/uploads/hc_archives/hc27/HC27.24-Monday-Epub/HC27.24.19-Key1-Neural-Nets-Epub/HC27.24.190-Convolutional-Neural-LeCun-Facebook.pdf}},
month = {8},
year = {2015}
}

@article{He2016DeepRL,
  title={Deep Residual Learning for Image Recognition},
  author={<NAME> and <NAME> and <NAME> and <NAME>},
  journal={2016 IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
  year={2016},
  pages={770-778}
}

@inproceedings{Hnaff2020DataEfficientIR,
  title={Data-Efficient Image Recognition with Contrastive Predictive Coding},
  author={<NAME>{\'e}naff and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME>},
  booktitle={ICML},
  year={2020}
}

@inproceedings{Valiant1984ATO,
  title={A theory of the learnable},
  author={<NAME>},
  booktitle={STOC '84},
  year={1984}
}

@inproceedings{Kiros2015SkipThoughtV,
  title={Skip-Thought Vectors},
  author={<NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME>},
  booktitle={NIPS},
  year={2015}
}

@inproceedings{Hill2016LearningDR,
  title={Learning Distributed Representations of Sentences from Unlabelled Data},
  author={<NAME> and <NAME> and <NAME>},
  booktitle={HLT-NAACL},
  year={2016}
}

@article{Dubois2020LearningOR,
  title={Learning Optimal Representations with the Decodable Information Bottleneck},
  author={<NAME> and <NAME> and <NAME> and <NAME>},
  journal={ArXiv},
  year={2020},
  volume={abs/2009.12789}
}

@article{Kaplan2020ScalingLF,
  title={Scaling Laws for Neural Language Models},
  author={<NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME> and <NAME>},
  journal={ArXiv},
  year={2020},
  volume={abs/2001.08361}
}
