{ "count": 156, "categories": [ "architectures", "language-models", "image-generation", "multimodal", "robotics", "techniques", "essays" ], "papers": [ { "number": 1, "title": "Attention Is All You Need", "slug": "01-attention-is-all-you-need", "category": "architectures", "authors": "Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Łukasz Kaiser, Illia Polosukhin", "published": "June 2017 (NeurIPS 2017)", "year": 2017, "url": "https://arxiv.org/abs/1706.03762", "path": "papers/architectures/01-attention-is-all-you-need/summary.md", "topics": [ "transformers", "attention", "architecture" ] }, { "number": 2, "title": "Generative Adversarial Networks (GANs)", "slug": "02-generative-adversarial-networks", "category": "image-generation", "authors": "Ian Goodfellow, Jean Pouget-Abadie, Mehdi Mirza, Bing Xu, David Warde-Farley, Sherjil Ozair, Aaron Courville, Yoshua Bengio", "published": "June 2014 (NeurIPS 2014)", "year": 2014, "url": "https://arxiv.org/abs/1406.2661", "path": "papers/image-generation/02-generative-adversarial-networks/summary.md", "topics": [ "image-generation", "gan" ] }, { "number": 3, "title": "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding", "slug": "03-bert", "category": "language-models", "authors": "Jacob Devlin, Ming-Wei Chang, Kenton Lee, Kristina Toutanova (Google AI Language)", "published": "October 2018 (NAACL 2019)", "year": 2018, "url": "https://arxiv.org/abs/1810.04805", "path": "papers/language-models/03-bert/summary.md", "topics": [ "language-model", "pretraining" ] }, { "number": 4, "title": "Language Models are Few-Shot Learners (GPT-3)", "slug": "04-gpt3-few-shot-learners", "category": "language-models", "authors": "Tom B. Brown et al. (OpenAI - 31 authors total)", "published": "May 2020 (NeurIPS 2020)", "year": 2020, "url": "https://arxiv.org/abs/2005.14165", "path": "papers/language-models/04-gpt3-few-shot-learners/summary.md", "topics": [ "language-model", "scaling", "pretraining" ] }, { "number": 5, "title": "Training Language Models to Follow Instructions with Human Feedback (InstructGPT)", "slug": "05-instructgpt-rlhf", "category": "language-models", "authors": "Long Ouyang, Jeff Wu, Xu Jiang, et al. (OpenAI)", "published": "March 2022", "year": 2022, "url": "https://arxiv.org/abs/2203.02155", "path": "papers/language-models/05-instructgpt-rlhf/summary.md", "topics": [ "alignment", "rlhf", "instruction-tuning" ] }, { "number": 6, "title": "Denoising Diffusion Probabilistic Models (DDPM)", "slug": "06-diffusion-models", "category": "image-generation", "authors": "Jonathan Ho, Ajay Jain, Pieter Abbeel (UC Berkeley)", "published": "June 2020 (NeurIPS 2020)", "year": 2020, "url": "https://arxiv.org/abs/2006.11239", "path": "papers/image-generation/06-diffusion-models/summary.md", "topics": [ "image-generation", "diffusion" ] }, { "number": 7, "title": "High-Resolution Image Synthesis with Latent Diffusion Models (Stable Diffusion)", "slug": "07-stable-diffusion", "category": "image-generation", "authors": "Robin Rombach, Andreas Blattmann, Dominik Lorenz, Patrick Esser, Björn Ommer (Ludwig Maximilian University of Munich & Runway)", "published": "December 2021 (CVPR 2022)", "year": 2021, "url": "https://arxiv.org/abs/2112.10752", "path": "papers/image-generation/07-stable-diffusion/summary.md", "topics": [ "image-generation", "diffusion", "efficiency" ] }, { "number": 8, "title": "Learning Transferable Visual Models From Natural Language Supervision (CLIP)", "slug": "08-clip", "category": "multimodal", "authors": "Alec Radford, Jong Wook Kim, Chris Hallacy, et al. (OpenAI)", "published": "February 2021 (ICML 2021)", "year": 2021, "url": "https://arxiv.org/abs/2103.00020", "path": "papers/multimodal/08-clip/summary.md", "topics": [ "multimodal", "vision" ] }, { "number": 9, "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models", "slug": "09-chain-of-thought", "category": "techniques", "authors": "Jason Wei, Xuezhi Wang, Dale Schuurmans, et al. (Google Research, Brain Team)", "published": "January 2022 (NeurIPS 2022)", "year": 2022, "url": "https://arxiv.org/abs/2201.11903", "path": "papers/techniques/09-chain-of-thought/summary.md", "topics": [ "reasoning", "chain-of-thought" ] }, { "number": 10, "title": "LoRA: Low-Rank Adaptation of Large Language Models", "slug": "10-lora", "category": "techniques", "authors": "Edward Hu, Yelong Shen, Phillip Wallis, et al. (Microsoft)", "published": "June 2021 (ICLR 2022)", "year": 2021, "url": "https://arxiv.org/abs/2106.09685", "path": "papers/techniques/10-lora/summary.md", "topics": [ "efficiency", "fine-tuning" ] }, { "number": 11, "title": "An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale (Vision Transformer)", "slug": "11-vision-transformer", "category": "architectures", "authors": "Alexey Dosovitskiy, Lucas Beyer, Alexander Kolesnikov, et al. (Google Research)", "published": "October 2020 (ICLR 2021)", "year": 2020, "url": "https://arxiv.org/abs/2010.11929", "path": "papers/architectures/11-vision-transformer/summary.md", "topics": [ "vision", "transformers", "architecture" ] }, { "number": 12, "title": "Scaling Laws for Neural Language Models", "slug": "12-scaling-laws", "category": "techniques", "authors": "Jared Kaplan, Sam McCandlish, Tom Henighan, et al. (OpenAI)", "published": "January 2020", "year": 2020, "url": "https://arxiv.org/abs/2001.08361", "path": "papers/techniques/12-scaling-laws/summary.md", "topics": [ "scaling" ] }, { "number": 13, "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks (RAG)", "slug": "13-rag", "category": "techniques", "authors": "Patrick Lewis, Ethan Perez, Aleksandra Piktus, et al. (Facebook AI Research, UCL, NYU)", "published": "May 2020 (NeurIPS 2020)", "year": 2020, "url": "https://arxiv.org/abs/2005.11401", "path": "papers/techniques/13-rag/summary.md", "topics": [ "retrieval" ] }, { "number": 14, "title": "Constitutional AI: Harmlessness from AI Feedback", "slug": "14-constitutional-ai", "category": "language-models", "authors": "Yuntao Bai, Saurav Kadavath, Sandipan Kundu, et al. (Anthropic)", "published": "December 2022", "year": 2022, "url": "https://arxiv.org/abs/2212.08073", "path": "papers/language-models/14-constitutional-ai/summary.md", "topics": [ "alignment", "safety" ] }, { "number": 15, "title": "LLaMA: Open and Efficient Foundation Language Models", "slug": "15-llama", "category": "language-models", "authors": "Hugo Touvron, Thibaut Lavril, Gautier Izacard, et al. (Meta AI)", "published": "February 2023", "year": 2023, "url": "https://arxiv.org/abs/2302.13971", "path": "papers/language-models/15-llama/summary.md", "topics": [ "language-model", "pretraining" ] }, { "number": 16, "title": "FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness", "slug": "16-flash-attention", "category": "techniques", "authors": "Tri Dao, Daniel Y. Fu, Stefano Ermon, Atri Rudra, Christopher Ré (Stanford, CMU)", "published": "May 2022 (NeurIPS 2022)", "year": 2022, "url": "https://arxiv.org/abs/2205.14135", "path": "papers/techniques/16-flash-attention/summary.md", "topics": [ "efficiency", "attention", "inference-optimization" ] }, { "number": 17, "title": "LLaMA 2: Open Foundation and Fine-Tuned Chat Models", "slug": "17-llama2", "category": "language-models", "authors": "Hugo Touvron, Louis Martin, et al. (Meta AI)", "published": "July 2023", "year": 2023, "url": "https://arxiv.org/abs/2307.09288", "path": "papers/language-models/17-llama2/summary.md", "topics": [ "language-model", "alignment", "rlhf" ] }, { "number": 18, "title": "Training Compute-Optimal Large Language Models (Chinchilla)", "slug": "18-chinchilla", "category": "techniques", "authors": "Jordan Hoffmann, Sebastian Borgeaud, et al. (DeepMind)", "published": "March 2022 (NeurIPS 2022)", "year": 2022, "url": "https://arxiv.org/abs/2203.15556", "path": "papers/techniques/18-chinchilla/summary.md", "topics": [ "scaling" ] }, { "number": 19, "title": "Direct Preference Optimization (DPO): Your Language Model is Secretly a Reward Model", "slug": "19-dpo", "category": "language-models", "authors": "Rafael Rafailov, Archit Sharma, et al. (Stanford)", "published": "May 2023 (NeurIPS 2023)", "year": 2023, "url": "https://arxiv.org/abs/2305.18290", "path": "papers/language-models/19-dpo/summary.md", "topics": [ "alignment", "preference-optimization" ] }, { "number": 20, "title": "Mamba: Linear-Time Sequence Modeling with Selective State Spaces", "slug": "20-mamba", "category": "architectures", "authors": "Albert Gu, Tri Dao (Carnegie Mellon, Princeton)", "published": "December 2023", "year": 2023, "url": "https://arxiv.org/abs/2312.00752", "path": "papers/architectures/20-mamba/summary.md", "topics": [ "architecture", "state-space", "efficiency", "long-context" ] }, { "number": 21, "title": "ReAct: Synergizing Reasoning and Acting in Language Models", "slug": "21-react", "category": "techniques", "authors": "Shunyu Yao, Jeffrey Zhao, Dian Yu, et al. (Google Research, Princeton)", "published": "October 2022 (ICLR 2023)", "year": 2022, "url": "https://arxiv.org/abs/2210.03629", "path": "papers/techniques/21-react/summary.md", "topics": [ "agents", "tool-use", "reasoning" ] }, { "number": 22, "title": "QLoRA: Efficient Finetuning of Quantized LLMs", "slug": "22-qlora", "category": "techniques", "authors": "Tim Dettmers, Artidoro Pagnoni, et al. (University of Washington)", "published": "May 2023 (NeurIPS 2023)", "year": 2023, "url": "https://arxiv.org/abs/2305.14314", "path": "papers/techniques/22-qlora/summary.md", "topics": [ "efficiency", "fine-tuning", "quantization" ] }, { "number": 23, "title": "GPT-4V(ision): System Card", "slug": "23-gpt4v", "category": "multimodal", "authors": "OpenAI", "published": "September 2023", "year": 2023, "url": "https://cdn.openai.com/papers/GPTV_System_Card.pdf", "path": "papers/multimodal/23-gpt4v/summary.md", "topics": [ "multimodal", "vision" ] }, { "number": 24, "title": "Toolformer: Language Models Can Teach Themselves to Use Tools", "slug": "24-toolformer", "category": "techniques", "authors": "Timo Schick, Jane Dwivedi-Yu, et al. (Meta AI Research)", "published": "February 2023", "year": 2023, "url": "https://arxiv.org/abs/2302.04761", "path": "papers/techniques/24-toolformer/summary.md", "topics": [ "agents", "tool-use" ] }, { "number": 25, "title": "Tree of Thoughts: Deliberate Problem Solving with Large Language Models", "slug": "25-tree-of-thoughts", "category": "techniques", "authors": "Shunyu Yao, Dian Yu, Jeffrey Zhao, et al. (Princeton, Google DeepMind)", "published": "May 2023 (NeurIPS 2023)", "year": 2023, "url": "https://arxiv.org/abs/2305.10601", "path": "papers/techniques/25-tree-of-thoughts/summary.md", "topics": [ "reasoning" ] }, { "number": 26, "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning", "slug": "26-deepseek-r1", "category": "language-models", "authors": "DeepSeek-AI", "published": "January 20, 2025", "year": 2025, "url": "https://arxiv.org/abs/2501.12948", "path": "papers/language-models/26-deepseek-r1/summary.md", "topics": [ "reasoning", "reinforcement-learning" ] }, { "number": 27, "title": "DeepSeek-V3 Technical Report", "slug": "27-deepseek-v3", "category": "language-models", "authors": "DeepSeek-AI", "published": "December 27, 2024", "year": 2024, "url": "https://arxiv.org/abs/2412.19437", "path": "papers/language-models/27-deepseek-v3/summary.md", "topics": [ "language-model", "moe", "efficiency" ] }, { "number": 28, "title": "Qwen3: Technical Report", "slug": "28-qwen3", "category": "language-models", "authors": "Qwen Team (Alibaba Cloud)", "published": "May 14, 2025", "year": 2025, "url": "https://arxiv.org/abs/2505.09388", "path": "papers/language-models/28-qwen3/summary.md", "topics": [ "language-model", "reasoning" ] }, { "number": 29, "title": "Gemini 2.5: Pushing the Frontier with Advanced Reasoning, Multimodality, Long Context, and Next Generation Agentic Capabilities", "slug": "29-gemini-2.5", "category": "multimodal", "authors": "Google DeepMind", "published": "July 7, 2025", "year": 2025, "url": "https://arxiv.org/abs/2507.06261", "path": "papers/multimodal/29-gemini-2.5/summary.md", "topics": [ "multimodal", "long-context" ] }, { "number": 30, "title": "Claude 3.5 Sonnet: Computer Use and Enhanced Capabilities", "slug": "30-claude-3.5-sonnet", "category": "language-models", "authors": "Anthropic", "published": "October 22, 2024", "year": 2024, "url": "https://www.anthropic.com/news/3-5-models-and-computer-use", "path": "papers/language-models/30-claude-3.5-sonnet/summary.md", "topics": [ "language-model", "agents" ] }, { "number": 31, "title": "OpenAI o1: Learning to Reason with Reinforcement Learning", "slug": "31-openai-o1", "category": "language-models", "authors": "OpenAI", "published": "September 12, 2024", "year": 2024, "url": "https://openai.com/index/learning-to-reason-with-llms/", "path": "papers/language-models/31-openai-o1/summary.md", "topics": [ "reasoning", "test-time-compute" ] }, { "number": 32, "title": "SAM 2: Segment Anything in Images and Videos", "slug": "32-sam2", "category": "multimodal", "authors": "Meta AI (FAIR)", "published": "August 2024", "year": 2024, "url": "https://arxiv.org/abs/2408.00714", "path": "papers/multimodal/32-sam2/summary.md", "topics": [ "vision" ] }, { "number": 33, "title": "LLaMA 3.3: Matching 405B Performance with 70B Parameters", "slug": "33-llama3.3", "category": "language-models", "authors": "Meta AI", "published": "December 2024", "year": 2024, "url": "https://www.meta.ai/blog/meta-llama-3-3/", "path": "papers/language-models/33-llama3.3/summary.md", "topics": [ "language-model", "efficiency" ] }, { "number": 34, "title": "Meta Chain-of-Thought: Towards System 2 Reasoning in LLMs", "slug": "34-meta-cot", "category": "techniques", "authors": "Multiple research teams", "published": "January 2025", "year": 2025, "url": "https://arxiv.org/abs/2501.04682", "path": "papers/techniques/34-meta-cot/summary.md", "topics": [ "reasoning", "chain-of-thought" ] }, { "number": 35, "title": "rStar-Math: Small LLMs Can Master Math Reasoning with Self-Evolved Deep Thinking", "slug": "35-rstar-math", "category": "techniques", "authors": "Microsoft Research, Tsinghua University", "published": "January 2025", "year": 2025, "url": "https://arxiv.org/abs/2501.04519", "path": "papers/techniques/35-rstar-math/summary.md", "topics": [ "reasoning" ] }, { "number": 36, "title": "GPT-4 Technical Report", "slug": "36-gpt4", "category": "language-models", "authors": "OpenAI", "published": "March 14, 2023", "year": 2023, "url": "https://arxiv.org/abs/2303.08774", "path": "papers/language-models/36-gpt4/summary.md", "topics": [ "language-model", "multimodal" ] }, { "number": 37, "title": "Mixtral of Experts (and the Mixture-of-Experts Architecture)", "slug": "37-mixture-of-experts", "category": "architectures", "authors": "Albert Jiang, Alexandre Sablayrolles, et al. (Mistral AI)", "published": "January 8, 2024", "year": 2024, "url": "https://arxiv.org/abs/2401.04088", "path": "papers/architectures/37-mixture-of-experts/summary.md", "topics": [ "moe", "architecture", "efficiency" ] }, { "number": 38, "title": "GRPO: Group Relative Policy Optimization", "slug": "38-grpo", "category": "techniques", "authors": "Zhihong Shao, Peiyi Wang, et al. (DeepSeek-AI)", "published": "February 5, 2024 (in DeepSeekMath paper)", "year": 2024, "url": "https://arxiv.org/abs/2402.03300", "path": "papers/techniques/38-grpo/summary.md", "topics": [ "reinforcement-learning", "alignment" ] }, { "number": 39, "title": "RLVR: Reinforcement Learning from Verifiable Rewards", "slug": "39-rlvr", "category": "techniques", "authors": "Multiple research groups (paradigm, not a single paper)", "published": "", "year": 2024, "url": "https://arxiv.org/abs/2501.12948", "path": "papers/techniques/39-rlvr/summary.md", "topics": [ "reinforcement-learning", "reasoning" ] }, { "number": 40, "title": "GPT-4o: The First Omni Model", "slug": "40-gpt4o", "category": "language-models", "authors": "OpenAI", "published": "May 13, 2024", "year": 2024, "url": "https://cdn.openai.com/gpt-4o-system-card.pdf", "path": "papers/language-models/40-gpt4o/summary.md", "topics": [ "multimodal", "audio", "vision" ] }, { "number": 41, "title": "Llama 4: Natively Multimodal Open-Source AI", "slug": "41-llama4", "category": "language-models", "authors": "Meta AI", "published": "April 5, 2025", "year": 2025, "url": "https://ai.meta.com/blog/llama-4-multimodal-intelligence/", "path": "papers/language-models/41-llama4/summary.md", "topics": [ "language-model", "moe", "multimodal" ] }, { "number": 42, "title": "GPT-5: Unified Intelligence", "slug": "42-gpt5", "category": "language-models", "authors": "OpenAI", "published": "August 7, 2025", "year": 2025, "url": "https://cdn.openai.com/gpt-5-system-card.pdf", "path": "papers/language-models/42-gpt5/summary.md", "topics": [ "language-model", "reasoning" ] }, { "number": 43, "title": "Claude 4 Family: The Agentic AI Leader", "slug": "43-claude4", "category": "language-models", "authors": "Anthropic", "published": "Claude 4 (May 2025), Opus 4.1 (August 2025), Sonnet 4.5 (September 2025), Opus 4.5 (November 2025), Opus 4.6 (February 2026)", "year": 2025, "url": "https://www.anthropic.com/news/claude-4", "path": "papers/language-models/43-claude4/summary.md", "topics": [ "language-model", "agents" ] }, { "number": 44, "title": "Sora and Diffusion Transformers (DiT): Video Generation as World Simulation", "slug": "44-sora-dit", "category": "image-generation", "authors": "William Peebles, Saining Xie (DiT); OpenAI (Sora)", "published": "DiT: December 2022; Sora: February 2024", "year": 2022, "url": "https://arxiv.org/abs/2212.09748", "path": "papers/image-generation/44-sora-dit/summary.md", "topics": [ "video-generation", "diffusion" ] }, { "number": 45, "title": "Speculative Decoding: Fast Inference from Transformers", "slug": "45-speculative-decoding", "category": "techniques", "authors": "Yaniv Leviathan, Matan Kalman, Yossi Matias (Google Research)", "published": "November 2022 (ICML 2023)", "year": 2022, "url": "https://arxiv.org/abs/2211.17192", "path": "papers/techniques/45-speculative-decoding/summary.md", "topics": [ "efficiency", "inference-optimization" ] }, { "number": 46, "title": "LLaVA: Visual Instruction Tuning", "slug": "46-llava", "category": "multimodal", "authors": "Haotian Liu, Chunyuan Li, Qingyang Wu, Yong Jae Lee", "published": "April 2023 (NeurIPS 2023 Oral)", "year": 2023, "url": "https://arxiv.org/abs/2304.08485", "path": "papers/multimodal/46-llava/summary.md", "topics": [ "multimodal", "vision", "instruction-tuning" ] }, { "number": 47, "title": "Gemini 3: Google's Most Capable AI Model", "slug": "47-gemini3", "category": "multimodal", "authors": "Google DeepMind", "published": "November 2025", "year": 2025, "url": "https://blog.google/products-and-platforms/products/gemini/gemini-3/", "path": "papers/multimodal/47-gemini3/summary.md", "topics": [ "multimodal" ] }, { "number": 48, "title": "DALL-E 3: Improving Image Generation with Better Captions", "slug": "48-dalle3", "category": "image-generation", "authors": "James Betker, Gabriel Goh, Li Jing, et al. (OpenAI)", "published": "September 2023", "year": 2023, "url": "https://cdn.openai.com/papers/dall-e-3.pdf", "path": "papers/image-generation/48-dalle3/summary.md", "topics": [ "image-generation", "diffusion" ] }, { "number": 49, "title": "Whisper: Robust Speech Recognition via Large-Scale Weak Supervision", "slug": "49-whisper", "category": "multimodal", "authors": "Alec Radford, Jong Wook Kim, Tao Xu, et al. (OpenAI)", "published": "December 2022", "year": 2022, "url": "https://arxiv.org/abs/2212.04356", "path": "papers/multimodal/49-whisper/summary.md", "topics": [ "audio" ] }, { "number": 50, "title": "Scaling LLM Test-Time Compute: The Theoretical Foundation for Reasoning Models", "slug": "50-test-time-compute", "category": "techniques", "authors": "Charlie Snell, Jaehoon Lee, Kelvin Xu, Aviral Kumar (Google DeepMind)", "published": "August 2024", "year": 2024, "url": "https://arxiv.org/abs/2408.03314", "path": "papers/techniques/50-test-time-compute/summary.md", "topics": [ "reasoning", "test-time-compute", "scaling" ] }, { "number": 51, "title": "Let's Verify Step by Step: Process Reward Models", "slug": "51-process-reward-models", "category": "techniques", "authors": "Hunter Lightman, Vineet Kosaraju, Yura Burda, et al. (OpenAI)", "published": "May 2023 (ICLR 2024 Oral)", "year": 2023, "url": "https://arxiv.org/abs/2305.20050", "path": "papers/techniques/51-process-reward-models/summary.md", "topics": [ "reasoning", "alignment" ] }, { "number": 52, "title": "PagedAttention: Efficient LLM Serving with vLLM", "slug": "52-pagedattention-vllm", "category": "techniques", "authors": "Woosuk Kwon, Zhuohan Li, Sicheng Zhuang, et al. (UC Berkeley)", "published": "September 2023 (SOSP 2023)", "year": 2023, "url": "https://arxiv.org/abs/2309.06180", "path": "papers/techniques/52-pagedattention-vllm/summary.md", "topics": [ "efficiency", "inference-optimization" ] }, { "number": 53, "title": "Efficient Estimation of Word Representations in Vector Space (Word2Vec)", "slug": "53-word2vec", "category": "techniques", "authors": "Tomas Mikolov, Kai Chen, Greg Corrado, Jeffrey Dean (Google)", "published": "January 2013 (arXiv); companion paper \"Distributed Representations of Words and Phrases and their Compositionality\" at NeurIPS 2013", "year": 2013, "url": "https://arxiv.org/abs/1301.3781", "path": "papers/techniques/53-word2vec/summary.md", "topics": [ "embeddings" ] }, { "number": 54, "title": "RoFormer: Enhanced Transformer with Rotary Position Embedding (RoPE)", "slug": "54-rope-rotary-position-embedding", "category": "techniques", "authors": "Jianlin Su, Yu Lu, Shengfeng Pan, Bo Wen, Yunfeng Liu (Zhuiyi Technology)", "published": "April 2021 (revised through 2023)", "year": 2021, "url": "https://arxiv.org/abs/2104.09864", "path": "papers/techniques/54-rope-rotary-position-embedding/summary.md", "topics": [ "position-encoding", "attention" ] }, { "number": 55, "title": "Sequence to Sequence Learning with Neural Networks (Seq2Seq)", "slug": "55-seq2seq", "category": "architectures", "authors": "Ilya Sutskever, Oriol Vinyals, Quoc V. Le (Google)", "published": "September 2014 (NeurIPS 2014)", "year": 2014, "url": "https://arxiv.org/abs/1409.3215", "path": "papers/architectures/55-seq2seq/summary.md", "topics": [ "architecture" ] }, { "number": 56, "title": "Codex: Evaluating Large Language Models Trained on Code", "slug": "56-codex", "category": "language-models", "authors": "Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, et al. (OpenAI)", "published": "July 2021", "year": 2021, "url": "https://arxiv.org/abs/2107.03374", "path": "papers/language-models/56-codex/summary.md", "topics": [ "code", "language-model" ] }, { "number": 57, "title": "Auto-Encoding Variational Bayes (VAE)", "slug": "57-vae", "category": "image-generation", "authors": "Diederik P. Kingma, Max Welling", "published": "December 2013 (arXiv); ICLR 2014", "year": 2013, "url": "https://arxiv.org/abs/1312.6114", "path": "papers/image-generation/57-vae/summary.md", "topics": [ "image-generation", "vae" ] }, { "number": 58, "title": "Generative Agents: Interactive Simulacra of Human Behavior", "slug": "58-generative-agents", "category": "techniques", "authors": "Joon Sung Park, Joseph C. O'Brien, Carrie J. Cai, Meredith Ringel Morris, Percy Liang, Michael S. Bernstein (Stanford University, Google Research)", "published": "April 2023 (UIST 2023 Best Paper)", "year": 2023, "url": "https://arxiv.org/abs/2304.03442", "path": "papers/techniques/58-generative-agents/summary.md", "topics": [ "agents" ] }, { "number": 59, "title": "Model Context Protocol (MCP): An Open Standard for AI Tool Integration", "slug": "59-model-context-protocol", "category": "techniques", "authors": "Anthropic", "published": "November 25, 2024", "year": 2024, "url": "https://www.anthropic.com/news/model-context-protocol", "path": "papers/techniques/59-model-context-protocol/summary.md", "topics": [ "agents", "tool-use" ] }, { "number": 60, "title": "GraphRAG: From Local to Global - A Graph RAG Approach to Query-Focused Summarization", "slug": "60-graph-rag", "category": "techniques", "authors": "Darren Edge, Ha Trinh, Newman Cheng, Joshua Bradley, Alex Chao, Apurva Mody, Steven Truitt, Jonathan Larson (Microsoft Research)", "published": "April 2024", "year": 2024, "url": "https://arxiv.org/abs/2404.16130", "path": "papers/techniques/60-graph-rag/summary.md", "topics": [ "retrieval" ] }, { "number": 61, "title": "AlphaGeometry: Solving Olympiad Geometry Without Human Demonstrations", "slug": "61-alphageometry", "category": "techniques", "authors": "Trieu H. Trinh, Yuhuai Wu, Quoc V. Le, He He, Thang Luong (Google DeepMind, NYU)", "published": "January 2024 (Nature, vol. 625)", "year": 2024, "url": "https://www.nature.com/articles/s41586-023-06747-5", "path": "papers/techniques/61-alphageometry/summary.md", "topics": [ "reasoning", "science" ] }, { "number": 62, "title": "AlphaEvolve: A Gemini-Powered Coding Agent for Designing Advanced Algorithms", "slug": "62-alphaevolve", "category": "techniques", "authors": "Alexander Novikov, Ngan Vu, Marvin Eisenberger, et al. (Google DeepMind)", "published": "May 2025", "year": 2025, "url": "https://deepmind.google/discover/blog/alphaevolve-a-gemini-powered-coding-agent-for-designing-advanced-algorithms/", "path": "papers/techniques/62-alphaevolve/summary.md", "topics": [ "agents", "science", "code" ] }, { "number": 63, "title": "Proximal Policy Optimization Algorithms (PPO)", "slug": "63-ppo", "category": "techniques", "authors": "John Schulman, Filip Wolski, Prafulla Dhariwal, Alec Radford, Oleg Klimov (OpenAI)", "published": "July 2017", "year": 2017, "url": "https://arxiv.org/abs/1707.06347", "path": "papers/techniques/63-ppo/summary.md", "topics": [ "reinforcement-learning", "alignment" ] }, { "number": 64, "title": "Language Models are Unsupervised Multitask Learners (GPT-2)", "slug": "64-gpt2", "category": "language-models", "authors": "Alec Radford, Jeffrey Wu, Rewon Child, David Luan, Dario Amodei, Ilya Sutskever (OpenAI)", "published": "February 2019", "year": 2019, "url": "https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf", "path": "papers/language-models/64-gpt2/summary.md", "topics": [ "language-model", "scaling", "pretraining" ] }, { "number": 65, "title": "Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer (T5)", "slug": "65-t5", "category": "language-models", "authors": "Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu (Google)", "published": "October 2019 (JMLR 2020)", "year": 2019, "url": "https://arxiv.org/abs/1910.10683", "path": "papers/language-models/65-t5/summary.md", "topics": [ "language-model", "architecture", "pretraining" ] }, { "number": 66, "title": "Neural Machine Translation by Jointly Learning to Align and Translate (Bahdanau Attention)", "slug": "66-bahdanau-attention", "category": "architectures", "authors": "Dzmitry Bahdanau, Kyunghyun Cho, Yoshua Bengio", "published": "September 2014 (ICLR 2015)", "year": 2014, "url": "https://arxiv.org/abs/1409.0473", "path": "papers/architectures/66-bahdanau-attention/summary.md", "topics": [ "attention", "architecture" ] }, { "number": 67, "title": "Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity", "slug": "67-switch-transformer", "category": "architectures", "authors": "William Fedus, Barret Zoph, Noam Shazeer (Google)", "published": "January 2021 (JMLR 2022)", "year": 2021, "url": "https://arxiv.org/abs/2101.03961", "path": "papers/architectures/67-switch-transformer/summary.md", "topics": [ "moe", "architecture", "scaling" ] }, { "number": 68, "title": "Highly Accurate Protein Structure Prediction with AlphaFold (AlphaFold 2)", "slug": "68-alphafold", "category": "techniques", "authors": "John Jumper, Richard Evans, Alexander Pritzel, Tim Green, Michael Figurnov, Olaf Ronneberger, Kathryn Tunyasuvunakool, Russ Bates, Augustin Zidek, Anna Potapenko, Alex Bridgland, Clemens Meyer, Simon A. A. Kohl, Andrew J. Ballard, Andrew Cowie, Bernardino Romera-Paredes, Stanislav Nikolov, Rishub Jain, Jonas Adler, Trevor Back, Stig Petersen, David Reiman, Ellen Clancy, Michal Zielinski, Martin Steinegger, Michalina Pacholska, Tamas Berghammer, Sebastian Bodenstein, David Silver, Oriol Vinyals, Andrew W. Senior, Koray Kavukcuoglu, Pushmeet Kohli, Demis Hassabis (DeepMind)", "published": "2021 (Nature, volume 596)", "year": 2021, "url": "https://www.nature.com/articles/s41586-021-03819-2", "path": "papers/techniques/68-alphafold/summary.md", "topics": [ "science", "attention" ] }, { "number": 69, "title": "Classifier-Free Diffusion Guidance", "slug": "69-classifier-free-guidance", "category": "image-generation", "authors": "Jonathan Ho, Tim Salimans (Google Research, Brain Team)", "published": "December 2021 (NeurIPS 2021 Workshop on Deep Generative Models); arXiv July 2022", "year": 2021, "url": "https://arxiv.org/abs/2207.12598", "path": "papers/image-generation/69-classifier-free-guidance/summary.md", "topics": [ "image-generation", "diffusion", "guidance" ] }, { "number": 70, "title": "Denoising Diffusion Implicit Models (DDIM)", "slug": "70-ddim", "category": "image-generation", "authors": "Jiaming Song, Chenlin Meng, Stefano Ermon (Stanford University)", "published": "October 2020 (ICLR 2021)", "year": 2020, "url": "https://arxiv.org/abs/2010.02502", "path": "papers/image-generation/70-ddim/summary.md", "topics": [ "image-generation", "diffusion", "sampling", "inference-optimization" ] }, { "number": 71, "title": "ControlNet: Adding Conditional Control to Text-to-Image Diffusion Models", "slug": "71-controlnet", "category": "image-generation", "authors": "Lvmin Zhang, Anyi Rao, Maneesh Agrawala (Stanford University)", "published": "February 2023 (ICCV 2023, Marr Prize / Best Paper)", "year": 2023, "url": "https://arxiv.org/abs/2302.05543", "path": "papers/image-generation/71-controlnet/summary.md", "topics": [ "image-generation", "diffusion", "controllable-generation", "fine-tuning" ] }, { "number": 72, "title": "Flow Matching and Rectified Flow: The New Default for Image Generation (Stable Diffusion 3)", "slug": "72-flow-matching-sd3", "category": "image-generation", "authors": "Yaron Lipman, Ricky T. Q. Chen, Heli Ben-Hamu, Maximilian Nickel, Matt Le (Meta AI) - Flow Matching; Xingchao Liu, Chengyue Gong, Qiang Liu (UT Austin) - Rectified Flow; Patrick Esser, Sumith Kulal, Andreas Blattmann, Rahim Entezari, Jonas Muller et al. (Stability AI) - Stable Diffusion 3", "published": "October 2022 (Flow Matching, ICLR 2023); September 2022 (Rectified Flow, ICLR 2023); March 2024 (Stable Diffusion 3, ICML 2024)", "year": 2022, "url": "https://arxiv.org/abs/2210.02747", "path": "papers/image-generation/72-flow-matching-sd3/summary.md", "topics": [ "image-generation", "diffusion", "flow-matching", "architecture" ] }, { "number": 73, "title": "Deep Residual Learning for Image Recognition (ResNet)", "slug": "73-resnet", "category": "architectures", "authors": "Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun (Microsoft Research)", "published": "December 2015 (CVPR 2016, Best Paper Award)", "year": 2015, "url": "https://arxiv.org/abs/1512.03385", "path": "papers/architectures/73-resnet/summary.md", "topics": [ "architecture", "vision", "computer-vision" ] }, { "number": 74, "title": "U-Net: Convolutional Networks for Biomedical Image Segmentation", "slug": "74-unet", "category": "architectures", "authors": "Olaf Ronneberger, Philipp Fischer, Thomas Brox (University of Freiburg)", "published": "May 2015 (MICCAI 2015)", "year": 2015, "url": "https://arxiv.org/abs/1505.04597", "path": "papers/architectures/74-unet/summary.md", "topics": [ "architecture", "vision", "computer-vision", "diffusion" ] }, { "number": 75, "title": "GQA: Grouped-Query Attention (and Multi-Query Attention)", "slug": "75-grouped-query-attention", "category": "architectures", "authors": "Joshua Ainslie, James Lee-Thorp, Michiel de Jong, Yury Zemlyanskiy, Federico Lebron, Sumit Sanghai (Google Research) - GQA; Noam Shazeer (Google) - Multi-Query Attention", "published": "May 2023 (GQA, EMNLP 2023); November 2019 (MQA)", "year": 2023, "url": "https://arxiv.org/abs/2305.13245", "path": "papers/architectures/75-grouped-query-attention/summary.md", "topics": [ "attention", "architecture", "efficiency", "inference-optimization" ] }, { "number": 76, "title": "ZeRO and Megatron-LM: How Trillion-Parameter Models Are Actually Trained", "slug": "76-zero-megatron", "category": "techniques", "authors": "Samyam Rajbhandari, Jeff Rasley, Olatunji Ruwase, Yuxiong He (Microsoft) - ZeRO; Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper, Bryan Catanzaro (NVIDIA) - Megatron-LM", "published": "October 2019 (ZeRO, SC 2020); September 2019 (Megatron-LM)", "year": 2019, "url": "https://arxiv.org/abs/1910.02054", "path": "papers/techniques/76-zero-megatron/summary.md", "topics": [ "distributed-training", "systems", "scaling", "efficiency" ] }, { "number": 77, "title": "Self-Consistency Improves Chain of Thought Reasoning in Language Models", "slug": "77-self-consistency", "category": "techniques", "authors": "Xuezhi Wang, Jason Wei, Dale Schuurmans, Quoc Le, Ed Chi, Sharan Narang, Aakanksha Chowdhery, Denny Zhou (Google Research, Brain Team)", "published": "March 2022 (ICLR 2023)", "year": 2022, "url": "https://arxiv.org/abs/2203.11171", "path": "papers/techniques/77-self-consistency/summary.md", "topics": [ "reasoning", "chain-of-thought", "test-time-compute", "prompting" ] }, { "number": 78, "title": "Reflexion: Language Agents with Verbal Reinforcement Learning", "slug": "78-reflexion", "category": "techniques", "authors": "Noah Shinn, Federico Cassano, Edward Berman, Ashwin Gopinath, Karthik Narasimhan, Shunyu Yao (Northeastern University, MIT, Princeton University)", "published": "March 2023 (NeurIPS 2023)", "year": 2023, "url": "https://arxiv.org/abs/2303.11366", "path": "papers/techniques/78-reflexion/summary.md", "topics": [ "agents", "reasoning", "tool-use", "prompting" ] }, { "number": 79, "title": "Self-Instruct: Aligning Language Models with Self-Generated Instructions", "slug": "79-self-instruct", "category": "techniques", "authors": "Yizhong Wang, Yeganeh Kordi, Swaroop Mishra, Alisa Liu, Noah A. Smith, Daniel Khashabi, Hannaneh Hajishirzi (University of Washington, Tehran Polytechnic, Arizona State University, Johns Hopkins University, Allen Institute for AI)", "published": "December 2022 (ACL 2023)", "year": 2022, "url": "https://arxiv.org/abs/2212.10560", "path": "papers/techniques/79-self-instruct/summary.md", "topics": [ "instruction-tuning", "synthetic-data", "alignment" ] }, { "number": 80, "title": "FLAN: Finetuned Language Models Are Zero-Shot Learners (Instruction Tuning)", "slug": "80-flan", "category": "techniques", "authors": "Jason Wei, Maarten Bosma, Vincent Y. Zhao, Kelvin Guu, Adams Wei Yu, Brian Lester, Nan Du, Andrew M. Dai, Quoc V. Le (Google Research); scaling follow-up by Hyung Won Chung, Le Hou, Shayne Longpre et al.", "published": "September 2021 (ICLR 2022); Scaling Instruction-Finetuned Language Models, October 2022", "year": 2021, "url": "https://arxiv.org/abs/2109.01652", "path": "papers/techniques/80-flan/summary.md", "topics": [ "instruction-tuning", "pretraining", "scaling" ] }, { "number": 81, "title": "Emergent Abilities of Large Language Models (and the Mirage Rebuttal)", "slug": "81-emergent-abilities", "category": "techniques", "authors": "Jason Wei, Yi Tay, Rishi Bommasani, Colin Raffel, Barret Zoph, Sebastian Borgeaud, Dani Yogatama, Maarten Bosma, Denny Zhou, Donald Metzler, Ed H. Chi, Tatsunori Hashimoto, Oriol Vinyals, Percy Liang, Jeff Dean, William Fedus (Google Research, Stanford, UNC Chapel Hill, DeepMind); rebuttal by Rylan Schaeffer, Brando Miranda, Sanmi Koyejo (Stanford)", "published": "June 2022 (TMLR 2022); rebuttal April 2023 (NeurIPS 2023, Outstanding Paper)", "year": 2022, "url": "https://arxiv.org/abs/2206.07682", "path": "papers/techniques/81-emergent-abilities/summary.md", "topics": [ "scaling", "evaluation", "reasoning" ] }, { "number": 82, "title": "Sparse Autoencoders and Monosemanticity: Reading the Features Inside a Model", "slug": "82-sparse-autoencoders", "category": "techniques", "authors": "Trenton Bricken, Adly Templeton, Joshua Batson, Brian Chen, Adam Jermyn, Tom Conerly, Nick Turner, Cem Anil, Carson Denison, Amanda Askell, Chris Olah et al. (Anthropic); superposition groundwork by Nelson Elhage, Tristan Hume, Catherine Olsson et al. (Anthropic); scaling work by Leo Gao et al. (OpenAI)", "published": "May 2024 (Scaling Monosemanticity, Claude 3 Sonnet); earlier: September 2022 (Toy Models of Superposition), October 2023 (Towards Monosemanticity); concurrent: June 2024 (OpenAI, Scaling and Evaluating Sparse Autoencoders)", "year": 2024, "url": "https://transformer-circuits.pub/2024/scaling-monosemanticity/index.html", "path": "papers/techniques/82-sparse-autoencoders/summary.md", "topics": [ "interpretability", "safety" ] }, { "number": 83, "title": "Sleeper Agents: Training Deceptive LLMs that Persist Through Safety Training", "slug": "83-sleeper-agents", "category": "techniques", "authors": "Evan Hubinger, Carson Denison, Jesse Mu, Mike Lambert, Meg Tong, Monte MacDiarmid, Tamera Lanham, Daniel M. Ziegler, Tim Maxwell, Newton Cheng, Adam Jermyn, Amanda Askell, Ansh Radhakrishnan, Cem Anil, David Duvenaud, Deep Ganguli, Fazl Barez, Jack Clark, Kamal Ndousse, Nicholas Schiefer, Ethan Perez et al. (Anthropic)", "published": "January 2024", "year": 2024, "url": "https://arxiv.org/abs/2401.05566", "path": "papers/techniques/83-sleeper-agents/summary.md", "topics": [ "safety", "alignment", "interpretability" ] }, { "number": 84, "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "slug": "84-swe-bench", "category": "techniques", "authors": "Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan (Princeton University, University of Chicago)", "published": "October 2023 (ICLR 2024)", "year": 2023, "url": "https://arxiv.org/abs/2310.06770", "path": "papers/techniques/84-swe-bench/summary.md", "topics": [ "evaluation", "benchmarks", "agents", "code" ] }, { "number": 85, "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "slug": "85-llm-as-judge", "category": "techniques", "authors": "Lianmin Zheng, Wei-Lin Chiang, Ying Sheng, Siyuan Zhuang, Zhanghao Wu, Yonghao Zhuang, Zi Lin, Zhuohan Li, Dacheng Li, Eric P. Xing, Hao Zhang, Joseph E. Gonzalez, Ion Stoica (UC Berkeley, UCSD, CMU, Stanford, MBZUAI)", "published": "June 2023 (NeurIPS 2023 Datasets and Benchmarks Track)", "year": 2023, "url": "https://arxiv.org/abs/2306.05685", "path": "papers/techniques/85-llm-as-judge/summary.md", "topics": [ "evaluation", "benchmarks", "alignment" ] }, { "number": 86, "title": "GPTQ and AWQ: Post-Training Quantization for Large Language Models", "slug": "86-gptq-awq-quantization", "category": "techniques", "authors": "Elias Frantar, Saleh Ashkboos, Torsten Hoefler, Dan Alistarh (IST Austria, ETH Zurich, Neural Magic) - GPTQ; Ji Lin, Jiaming Tang, Haotian Tang, Shang Yang, Xingyu Dang, Chuang Gan, Song Han (MIT, SJTU, MIT-IBM Watson AI Lab) - AWQ", "published": "October 2022 (GPTQ, ICLR 2023); June 2023 (AWQ, MLSys 2024 Best Paper)", "year": 2022, "url": "https://arxiv.org/abs/2210.17323", "path": "papers/techniques/86-gptq-awq-quantization/summary.md", "topics": [ "quantization", "efficiency", "inference-optimization" ] }, { "number": 87, "title": "Dense Passage Retrieval, ColBERT, and Sentence-BERT: The Retrieval Half of RAG", "slug": "87-dense-retrieval", "category": "techniques", "authors": "Vladimir Karpukhin, Barlas Oguz, Sewon Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, Wen-tau Yih (Facebook AI, University of Washington, Princeton) - DPR; Omar Khattab, Matei Zaharia (Stanford) - ColBERT; Nils Reimers, Iryna Gurevych (TU Darmstadt) - Sentence-BERT", "published": "April 2020 (DPR, EMNLP 2020); April 2020 (ColBERT, SIGIR 2020); August 2019 (Sentence-BERT, EMNLP 2019)", "year": 2020, "url": "https://arxiv.org/abs/2004.04906", "path": "papers/techniques/87-dense-retrieval/summary.md", "topics": [ "retrieval", "embeddings", "search" ] }, { "number": 88, "title": "Masked Autoencoders Are Scalable Vision Learners (MAE)", "slug": "88-mae", "category": "architectures", "authors": "Kaiming He, Xinlei Chen, Saining Xie, Yanghao Li, Piotr Dollar, Ross Girshick (Facebook AI Research)", "published": "November 2021 (arXiv 2111.06377), CVPR 2022", "year": 2021, "url": "https://arxiv.org/abs/2111.06377", "path": "papers/architectures/88-mae/summary.md", "topics": [ "vision", "pretraining", "self-supervised", "architecture" ] }, { "number": 89, "title": "Neural Discrete Representation Learning (VQ-VAE)", "slug": "89-vq-vae", "category": "image-generation", "authors": "Aaron van den Oord, Oriol Vinyals, Koray Kavukcuoglu (DeepMind)", "published": "November 2017 (arXiv 1711.00937), NeurIPS 2017", "year": 2017, "url": "https://arxiv.org/abs/1711.00937", "path": "papers/image-generation/89-vq-vae/summary.md", "topics": [ "image-generation", "vae", "discrete-representation" ] }, { "number": 90, "title": "Taming Transformers for High-Resolution Image Synthesis (VQ-GAN)", "slug": "90-vq-gan", "category": "image-generation", "authors": "Patrick Esser, Robin Rombach, Björn Ommer (Heidelberg University)", "published": "December 2020 (CVPR 2021 oral)", "year": 2020, "url": "https://arxiv.org/abs/2012.09841", "path": "papers/image-generation/90-vq-gan/summary.md", "topics": [ "image-generation", "gan", "transformers", "discrete-representation" ] }, { "number": 91, "title": "Photorealistic Text-to-Image Diffusion Models with Deep Language Understanding (Imagen)", "slug": "91-imagen", "category": "image-generation", "authors": "Chitwan Saharia, William Chan, Saurabh Saxena, Lala Li, Jay Whang, Emily Denton, Seyed Kamyar Seyed Ghasemabadi, Burcu Karagol Ayan, S. Sara Mahdavi, Rapha Gontijo Lopes, Tim Salimans, Jonathan Ho, David J. Fleet, Mohammad Norouzi (Google Research, Brain Team)", "published": "May 2022 (NeurIPS 2022)", "year": 2022, "url": "https://arxiv.org/abs/2205.11487", "path": "papers/image-generation/91-imagen/summary.md", "topics": [ "image-generation", "diffusion", "text-to-image" ] }, { "number": 92, "title": "DreamBooth: Fine Tuning Text-to-Image Diffusion Models for Subject-Driven Generation", "slug": "92-dreambooth", "category": "image-generation", "authors": "Nataniel Ruiz, Yuanzhen Li, Varun Jampani, Yael Pritch, Michael Rubinstein, Kfir Aberman (Google Research, Boston University)", "published": "August 2022 (CVPR 2023)", "year": 2022, "url": "https://arxiv.org/abs/2208.12242", "path": "papers/image-generation/92-dreambooth/summary.md", "topics": [ "image-generation", "diffusion", "fine-tuning", "controllable-generation" ] }, { "number": 93, "title": "Improving Language Understanding by Generative Pre-Training (GPT-1)", "slug": "93-gpt1", "category": "language-models", "authors": "Alec Radford, Karthik Narasimhan, Tim Salimans, Ilya Sutskever (OpenAI)", "published": "June 2018 (OpenAI Technical Report)", "year": 2018, "url": "https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf", "path": "papers/language-models/93-gpt1/summary.md", "topics": [ "language-model", "pretraining", "transfer-learning" ] }, { "number": 94, "title": "PaLM: Scaling Language Modeling with Pathways", "slug": "94-palm", "category": "language-models", "authors": "Aakanksha Chowdhery, Sharan Narang, Jacob Devlin, et al. (Google Research) - 67 authors", "published": "April 2022 (arXiv 2204.02311)", "year": 2022, "url": "https://arxiv.org/abs/2204.02311", "path": "papers/language-models/94-palm/summary.md", "topics": [ "language-model", "scaling", "pretraining" ] }, { "number": 95, "title": "Mistral 7B", "slug": "95-mistral-7b", "category": "language-models", "authors": "Albert Q. Jiang, Alexandre Sablayrolles, Arthur Mensch, et al. (Mistral AI)", "published": "October 2023 (arXiv 2310.06825)", "year": 2023, "url": "https://arxiv.org/abs/2310.06825", "path": "papers/language-models/95-mistral-7b/summary.md", "topics": [ "language-model", "efficiency", "attention" ] }, { "number": 96, "title": "Llama Guard: LLM-based Input-Output Safeguard for Human-AI Conversations", "slug": "96-llama-guard", "category": "language-models", "authors": "Hakan Inan, Kartikeya Upasani, Jianfeng Chi, Rashi Rungta, Krithika Iyer, Yuning Mao, Michael Tontchev, Qing Hu, Brian Fuller, Davide Testuggine, Madian Khabsa (Meta GenAI / Responsible AI)", "published": "December 2023", "year": 2023, "url": "https://arxiv.org/abs/2312.06674", "path": "papers/language-models/96-llama-guard/summary.md", "topics": [ "safety", "alignment", "evaluation" ] }, { "number": 97, "title": "STaR: Bootstrapping Reasoning With Reasoning (Self-Taught Reasoner)", "slug": "97-star", "category": "techniques", "authors": "Eric Zelikman, Yuhuai Wu, Jesse Mu, Noah D. Goodman (Stanford University, Google Research)", "published": "March 2022 (NeurIPS 2022)", "year": 2022, "url": "https://arxiv.org/abs/2203.14465", "path": "papers/techniques/97-star/summary.md", "topics": [ "reasoning", "self-improvement", "synthetic-data" ] }, { "number": 98, "title": "Quiet-STaR: Language Models Can Teach Themselves to Think Before Speaking", "slug": "98-quiet-star", "category": "techniques", "authors": "Eric Zelikman, Georges Harik, Yijia Shao, Varuna Jayasiri, Nick Haber, Noah D. Goodman (Stanford University, Notbad AI)", "published": "March 2024 (COLM 2024)", "year": 2024, "url": "https://arxiv.org/abs/2403.09629", "path": "papers/techniques/98-quiet-star/summary.md", "topics": [ "reasoning", "self-improvement", "pretraining" ] }, { "number": 99, "title": "Self-Refine: Iterative Refinement with Self-Feedback", "slug": "99-self-refine", "category": "techniques", "authors": "Aman Madaan, Niket Tandon, Prakhar Gupta, Skyler Hallinan, Luyu Gao, Sarah Wiegreffe, Uri Alon, Nouha Dziri, Shrimai Prabhumoye, Yiming Yang, Shashank Gupta, Bodhisattwa Prasad Majumder, Katherine Hermann, Sean Welleck, Amir Yazdanbakhsh, Peter Clark (CMU, Allen AI, University of Washington, NVIDIA, UC San Diego, Google Research)", "published": "March 2023 (NeurIPS 2023)", "year": 2023, "url": "https://arxiv.org/abs/2303.17651", "path": "papers/techniques/99-self-refine/summary.md", "topics": [ "reasoning", "prompting", "self-improvement" ] }, { "number": 100, "title": "Voyager: An Open-Ended Embodied Agent with Large Language Models", "slug": "100-voyager", "category": "techniques", "authors": "Guanzhi Wang, Yuqi Xie, Yunfan Jiang, Ajay Mandlekar, Chaowei Xiao, Yuke Zhu, Linxi Fan, Anima Anandkumar (NVIDIA, Caltech, UT Austin, Stanford, ASU)", "published": "May 2023 (TMLR 2024)", "year": 2023, "url": "https://arxiv.org/abs/2305.16291", "path": "papers/techniques/100-voyager/summary.md", "topics": [ "agents", "tool-use", "self-improvement", "code" ] }, { "number": 101, "title": "Accurate Structure Prediction of Biomolecular Interactions with AlphaFold 3", "slug": "101-alphafold3", "category": "techniques", "authors": "Josh Abramson, Jonas Adler, Jack Dunger, Richard Evans, Tim Green, Alexander Pritzel, Olaf Ronneberger, Lindsay Willmore, Andrew J. Ballard, Joshua Bambrick, Sebastian W. Bodenstein, David A. Evans, Chia-Chun Hung, Michael O'Neill, David Reiman, Kathryn Tunyasuvunakool, Zachary Wu, Akvile Zemgulyte, Eirini Arvaniti, Charles Beattie, Ottavia Bertolli, Alex Bridgland, Alexey Cherepanov, Miles Congreve, Alexander I. Cowen-Rivers, Andrew Cowie, Michael Figurnov, Fabian B. Fuchs, Hannah Gladman, Rishub Jain, Yousuf A. Khan, Caroline M. R. Low, Kuba Perlin, Anna Potapenko, Pascal Savy, Sukhdeep Singh, Adrian Stecula, Ashok Thillaisundaram, Catherine Tong, Sergei Yakneen, Ellen D. Zhong, Michal Zielinski, Augustin Zidek, Victor Bapst, Pushmeet Kohli, Max Jaderberg, Demis Hassabis, John M. Jumper (DeepMind and Isomorphic Labs)", "published": "May 2024 (Nature, vol. 630, pp. 493-500)", "year": 2024, "url": "https://www.nature.com/articles/s41586-024-07487-w", "path": "papers/techniques/101-alphafold3/summary.md", "topics": [ "science", "diffusion" ] }, { "number": 102, "title": "Mastering Chess and Shogi by Self-Play with a General Reinforcement Learning Algorithm", "slug": "102-alphazero", "category": "techniques", "authors": "David Silver, Thomas Hubert, Julian Schrittwieser, Ioannis Antonoglou, Matthew Lai, Arthur Guez, Marc Lanctot, Laurent Sifre, Dharshan Kumaran, Thore Graepel, Timothy Lillicrap, Karen Simonyan, Demis Hassabis (DeepMind)", "published": "December 2017 (later in Science, 2018)", "year": 2017, "url": "https://arxiv.org/abs/1712.01815", "path": "papers/techniques/102-alphazero/summary.md", "topics": [ "reinforcement-learning", "search", "self-play" ] }, { "number": 103, "title": "KTO: Model Alignment as Prospect Theoretic Optimization", "slug": "103-kto", "category": "techniques", "authors": "Kawin Ethayarajh, Winnie Xu, Niklas Muennighoff, Dan Jurafsky, Douwe Kiela (Stanford / Contextual AI)", "published": "February 2024", "year": 2024, "url": "https://arxiv.org/abs/2402.01306", "path": "papers/techniques/103-kto/summary.md", "topics": [ "alignment", "preference-optimization" ] }, { "number": 104, "title": "Genie: Generative Interactive Environments", "slug": "104-genie", "category": "techniques", "authors": "Jake Bruce, Michael Dennis, Ashley Edwards, Jack Parker-Holder, Yuge (Jimmy) Shi, Edward Hughes, Matthew Lai, Aditi Mavalankar, Richie Steigerwald, Chris Apps, Yusuf Aytar, Sarah Bechtle, Feryal Behbahani, Stephanie Chan, Nicolas Heess, Lucy Gonzalez, Simon Osindero, Sherjil Ozair, Scott Reed, Jingwei Zhang, Konrad Zolna, Jeff Clune, Nando de Freitas, Satinder Singh, Tim Rocktaeschel (Google DeepMind)", "published": "February 2024", "year": 2024, "url": "https://arxiv.org/abs/2402.15391", "path": "papers/techniques/104-genie/summary.md", "topics": [ "video-generation", "world-models", "self-supervised" ] }, { "number": 105, "title": "Mastering Diverse Domains through World Models (DreamerV3)", "slug": "105-dreamerv3", "category": "techniques", "authors": "Danijar Hafner, Jurgis Pasukonis, Jimmy Ba, Timothy Lillicrap (University of Toronto / DeepMind)", "published": "January 2023 (Nature, 2025 for journal version)", "year": 2023, "url": "https://arxiv.org/abs/2301.04104", "path": "papers/techniques/105-dreamerv3/summary.md", "topics": [ "reinforcement-learning", "world-models" ] }, { "number": 106, "title": "Evolutionary-Scale Prediction of Atomic-Level Protein Structure with a Language Model (ESM-2 / ESMFold)", "slug": "106-esm", "category": "techniques", "authors": "Zeming Lin, Halil Akin, Roshan Rao, Brian Hie, Zhongkai Zhu, Wenting Lu, Nikita Smetanin, Robert Verkuil, Ori Kabeli, Yaniv Shmueli, Allan dos Santos Costa, Maryam Fazel-Zarandi, Tom Sercu, Salvatore Candido, Alexander Rives (Meta AI, Fundamental AI Research)", "published": "March 2023, Science vol. 379, issue 6637 (DOI: 10.1126/science.ade2574)", "year": 2023, "url": "https://doi.org/10.1126/science.ade2574", "path": "papers/techniques/106-esm/summary.md", "topics": [ "science", "language-model", "embeddings" ] }, { "number": 107, "title": "Human-Level Play in the Game of Diplomacy by Combining Language Models with Strategic Reasoning (CICERO)", "slug": "107-cicero", "category": "techniques", "authors": "Meta Fundamental AI Research Diplomacy Team (FAIR), including Anton Bakhtin, Noam Brown, David Wu, Adam Lerer, Hengyuan Hu, et al.", "published": "November 2022, Science vol. 378, issue 6624 (DOI: 10.1126/science.ade9097)", "year": 2022, "url": "https://doi.org/10.1126/science.ade9097", "path": "papers/techniques/107-cicero/summary.md", "topics": [ "agents", "reinforcement-learning", "reasoning" ] }, { "number": 108, "title": "Computing Machinery and Intelligence (The Imitation Game / Turing Test)", "slug": "108-computing-machinery-and-intelligence", "category": "essays", "authors": "Alan M. Turing (University of Manchester)", "published": "October 1950 (Mind, Vol. 59, No. 236, pp. 433-460)", "year": 1950, "url": "https://academic.oup.com/mind/article/LIX/236/433/986238", "path": "papers/essays/108-computing-machinery-and-intelligence/summary.md", "topics": [ "essay", "history", "evaluation" ] }, { "number": 109, "title": "The Unreasonable Effectiveness of Recurrent Neural Networks (char-rnn)", "slug": "109-unreasonable-effectiveness-of-rnns", "category": "essays", "authors": "Andrej Karpathy (Stanford University)", "published": "May 2015 (blog post, karpathy.github.io)", "year": 2015, "url": "https://karpathy.github.io/2015/05/21/rnn-effectiveness/", "path": "papers/essays/109-unreasonable-effectiveness-of-rnns/summary.md", "topics": [ "essay", "language-model", "history" ] }, { "number": 110, "title": "Software 2.0", "slug": "110-software-2", "category": "essays", "authors": "Andrej Karpathy (Tesla)", "published": "November 2017 (Medium)", "year": 2017, "url": "https://karpathy.medium.com/software-2-0-a64152b37c35", "path": "papers/essays/110-software-2/summary.md", "topics": [ "essay", "code" ] }, { "number": 111, "title": "The Bitter Lesson", "slug": "111-bitter-lesson", "category": "essays", "authors": "Richard S. Sutton (University of Alberta; DeepMind)", "published": "March 2019 (essay, incompleteideas.net)", "year": 2019, "url": "http://www.incompleteideas.net/IncIdeas/BitterLesson.html", "path": "papers/essays/111-bitter-lesson/summary.md", "topics": [ "essay", "scaling" ] }, { "number": 112, "title": "The Scaling Hypothesis", "slug": "112-scaling-hypothesis", "category": "essays", "authors": "Gwern Branwen (independent researcher)", "published": "May 2020 (gwern.net; revised through 2022, with later editorial notes)", "year": 2020, "url": "https://gwern.net/scaling-hypothesis", "path": "papers/essays/112-scaling-hypothesis/summary.md", "topics": [ "essay", "scaling" ] }, { "number": 113, "title": "Situational Awareness: The Decade Ahead (Situational Awareness)", "slug": "113-situational-awareness", "category": "essays", "authors": "Leopold Aschenbrenner (independent; formerly OpenAI Superalignment team)", "published": "June 2024 (self-published essay series)", "year": 2024, "url": "https://situational-awareness.ai/", "path": "papers/essays/113-situational-awareness/summary.md", "topics": [ "essay", "scaling", "policy" ] }, { "number": 114, "title": "Machines of Loving Grace: How AI Could Transform the World for the Better (Machines of Loving Grace)", "slug": "114-machines-of-loving-grace", "category": "essays", "authors": "Dario Amodei (Anthropic)", "published": "October 2024 (personal essay, darioamodei.com)", "year": 2024, "url": "https://www.darioamodei.com/essay/machines-of-loving-grace", "path": "papers/essays/114-machines-of-loving-grace/summary.md", "topics": [ "essay", "science", "policy" ] }, { "number": 115, "title": "Building Effective Agents (Building Effective Agents)", "slug": "115-building-effective-agents", "category": "essays", "authors": "Erik Schluntz, Barry Zhang (Anthropic)", "published": "December 2024 (Anthropic engineering blog)", "year": 2024, "url": "https://www.anthropic.com/engineering/building-effective-agents", "path": "papers/essays/115-building-effective-agents/summary.md", "topics": [ "essay", "agents", "tool-use" ] }, { "number": 116, "title": "Welcome to the Era of Experience (Era of Experience)", "slug": "116-era-of-experience", "category": "essays", "authors": "David Silver (Google DeepMind), Richard S. Sutton (University of Alberta)", "published": "April 2025 (preprint of a chapter for \"Designing an Intelligence\", ed. George Konidaris, MIT Press)", "year": 2025, "url": "https://storage.googleapis.com/deepmind-media/Era-of-Experience%20/The%20Era%20of%20Experience%20Paper.pdf", "path": "papers/essays/116-era-of-experience/summary.md", "topics": [ "essay", "reinforcement-learning", "agents" ] }, { "number": 117, "title": "NeRF: Representing Scenes as Neural Radiance Fields for View Synthesis (NeRF)", "slug": "117-nerf", "category": "image-generation", "authors": "Ben Mildenhall, Pratul P. Srinivasan, Matthew Tancik, Jonathan T. Barron, Ravi Ramamoorthi, Ren Ng (UC Berkeley, Google Research, UC San Diego)", "published": "March 2020 (ECCV 2020, oral; Best Paper Honorable Mention)", "year": 2020, "url": "https://arxiv.org/abs/2003.08934", "path": "papers/image-generation/117-nerf/summary.md", "topics": [ "3d", "image-generation", "computer-vision" ] }, { "number": 118, "title": "3D Gaussian Splatting for Real-Time Radiance Field Rendering (3D Gaussian Splatting)", "slug": "118-3d-gaussian-splatting", "category": "image-generation", "authors": "Bernhard Kerbl, Georgios Kopanas, Thomas Leimkühler, George Drettakis (Inria and Université Côte d'Azur, Max-Planck-Institut für Informatik)", "published": "August 2023 (SIGGRAPH 2023, ACM Transactions on Graphics 42(4); Best Paper Award)", "year": 2023, "url": "https://arxiv.org/abs/2308.04079", "path": "papers/image-generation/118-3d-gaussian-splatting/summary.md", "topics": [ "3d", "computer-vision", "efficiency" ] }, { "number": 119, "title": "Hierarchical Text-Conditional Image Generation with CLIP Latents (DALL-E 2 / unCLIP)", "slug": "119-dalle2-unclip", "category": "image-generation", "authors": "Aditya Ramesh, Prafulla Dhariwal, Alex Nichol, Casey Chu, Mark Chen (OpenAI)", "published": "April 2022", "year": 2022, "url": "https://arxiv.org/abs/2204.06125", "path": "papers/image-generation/119-dalle2-unclip/summary.md", "topics": [ "image-generation", "diffusion", "text-to-image" ] }, { "number": 120, "title": "Consistency Models (Consistency Models)", "slug": "120-consistency-models", "category": "image-generation", "authors": "Yang Song, Prafulla Dhariwal, Mark Chen, Ilya Sutskever (OpenAI)", "published": "March 2023 (ICML 2023)", "year": 2023, "url": "https://arxiv.org/abs/2303.01469", "path": "papers/image-generation/120-consistency-models/summary.md", "topics": [ "image-generation", "diffusion", "inference-optimization" ] }, { "number": 121, "title": "AudioLM: a Language Modeling Approach to Audio Generation (AudioLM)", "slug": "121-audiolm", "category": "multimodal", "authors": "Zalán Borsos, Raphaël Marinier, Damien Vincent, Eugene Kharitonov, Olivier Pietquin, Matt Sharifi, Dominik Roblek, Olivier Teboul, David Grangier, Marco Tagliasacchi, Neil Zeghidour (Google Research)", "published": "September 2022 (IEEE/ACM Transactions on Audio, Speech, and Language Processing, 2023)", "year": 2022, "url": "https://arxiv.org/abs/2209.03143", "path": "papers/multimodal/121-audiolm/summary.md", "topics": [ "audio", "language-model", "discrete-representation" ] }, { "number": 122, "title": "Neural Codec Language Models are Zero-Shot Text to Speech Synthesizers (VALL-E)", "slug": "122-vall-e", "category": "multimodal", "authors": "Chengyi Wang, Sanyuan Chen, Yu Wu, Ziqiang Zhang, Long Zhou, Shujie Liu, Zhuo Chen, Yanqing Liu, Huaming Wang, Jinyu Li, Lei He, Sheng Zhao, Furu Wei (Microsoft)", "published": "January 2023", "year": 2023, "url": "https://arxiv.org/abs/2301.02111", "path": "papers/multimodal/122-vall-e/summary.md", "topics": [ "audio", "language-model", "safety" ] }, { "number": 123, "title": "RT-2: Vision-Language-Action Models Transfer Web Knowledge to Robotic Control (RT-2)", "slug": "123-rt2", "category": "robotics", "authors": "Anthony Brohan, Noah Brown, ... Brianna Zitkovich (54 authors, Google DeepMind)", "published": "July 2023 (CoRL 2023)", "year": 2023, "url": "https://arxiv.org/abs/2307.15818", "path": "papers/robotics/123-rt2/summary.md", "topics": [ "robotics", "multimodal", "transfer-learning" ] }, { "number": 124, "title": "pi0: A Vision-Language-Action Flow Model for General Robot Control (pi0)", "slug": "124-pi0", "category": "robotics", "authors": "Kevin Black, Noah Brown, Danny Driess, ... Ury Zhilinsky (24 authors, Physical Intelligence)", "published": "October 2024 (RSS 2025)", "year": 2024, "url": "https://arxiv.org/abs/2410.24164", "path": "papers/robotics/124-pi0/summary.md", "topics": [ "robotics", "flow-matching", "multimodal" ] }, { "number": 125, "title": "Open X-Embodiment: Robotic Learning Datasets and RT-X Models (Open X-Embodiment / RT-X)", "slug": "125-open-x-embodiment", "category": "robotics", "authors": "Open X-Embodiment Collaboration - more than 290 researchers from 21 institutions, including Google DeepMind, Stanford, UC Berkeley and many others", "published": "October 2023 (ICRA 2024, Best Conference Paper)", "year": 2023, "url": "https://arxiv.org/abs/2310.08864", "path": "papers/robotics/125-open-x-embodiment/summary.md", "topics": [ "robotics", "scaling", "benchmarks" ] }, { "number": 126, "title": "In-context Learning and Induction Heads (Induction Heads)", "slug": "126-induction-heads", "category": "techniques", "authors": "Catherine Olsson, Nelson Elhage, Neel Nanda, Nicholas Joseph, Nova DasSarma, Tom Henighan, Ben Mann, Amanda Askell, Yuntao Bai, Anna Chen, Tom Conerly, Dawn Drain, Deep Ganguli, Zac Hatfield-Dodds, Danny Hernandez, Scott Johnston, Andy Jones, Jackson Kernion, Liane Lovitt, Kamal Ndousse, Dario Amodei, Tom Brown, Jack Clark, Jared Kaplan, Sam McCandlish, Chris Olah (Anthropic)", "published": "March 2022 (Transformer Circuits Thread; arXiv September 2022)", "year": 2022, "url": "https://arxiv.org/abs/2209.11895", "path": "papers/techniques/126-induction-heads/summary.md", "topics": [ "interpretability", "attention", "transformers" ] }, { "number": 127, "title": "Universal and Transferable Adversarial Attacks on Aligned Language Models (GCG)", "slug": "127-gcg-adversarial-attacks", "category": "techniques", "authors": "Andy Zou, Zifan Wang, Nicholas Carlini, Milad Nasr, J. Zico Kolter, Matt Fredrikson (Carnegie Mellon University, Center for AI Safety, Google DeepMind, Bosch Center for AI)", "published": "July 2023 (arXiv)", "year": 2023, "url": "https://arxiv.org/abs/2307.15043", "path": "papers/techniques/127-gcg-adversarial-attacks/summary.md", "topics": [ "safety", "alignment", "evaluation" ] }, { "number": 128, "title": "Weak-to-Strong Generalization: Eliciting Strong Capabilities With Weak Supervision (Weak-to-Strong)", "slug": "128-weak-to-strong", "category": "techniques", "authors": "Collin Burns, Pavel Izmailov, Jan Hendrik Kirchner, Bowen Baker, Leo Gao, Leopold Aschenbrenner, Yining Chen, Adrien Ecoffet, Manas Joglekar, Jan Leike, Ilya Sutskever, Jeff Wu (OpenAI)", "published": "December 2023 (ICML 2024)", "year": 2023, "url": "https://arxiv.org/abs/2312.09390", "path": "papers/techniques/128-weak-to-strong/summary.md", "topics": [ "alignment", "safety", "scaling" ] }, { "number": 129, "title": "Red Teaming Language Models with Language Models (LM Red Teaming)", "slug": "129-red-teaming-lms", "category": "techniques", "authors": "Ethan Perez, Saffron Huang, Francis Song, Trevor Cai, Roman Ring, John Aslanides, Amelia Glaese, Nat McAleese, Geoffrey Irving (DeepMind; Perez also New York University)", "published": "February 2022 (EMNLP 2022)", "year": 2022, "url": "https://arxiv.org/abs/2202.03286", "path": "papers/techniques/129-red-teaming-lms/summary.md", "topics": [ "safety", "evaluation", "alignment" ] }, { "number": 130, "title": "YaRN: Efficient Context Window Extension of Large Language Models (YaRN)", "slug": "130-yarn-context-extension", "category": "techniques", "authors": "Bowen Peng, Jeffrey Quesnelle, Honglu Fan, Enrico Shippole (Nous Research, EleutherAI, University of Geneva)", "published": "August 2023 (ICLR 2024)", "year": 2023, "url": "https://arxiv.org/abs/2309.00071", "path": "papers/techniques/130-yarn-context-extension/summary.md", "topics": [ "long-context", "position-encoding", "fine-tuning" ] }, { "number": 131, "title": "Longformer: The Long-Document Transformer (Longformer)", "slug": "131-longformer", "category": "architectures", "authors": "Iz Beltagy, Matthew E. Peters, Arman Cohan (Allen Institute for Artificial Intelligence)", "published": "April 2020 (arXiv; v2 December 2020 added the encoder-decoder LED)", "year": 2020, "url": "https://arxiv.org/abs/2004.05150", "path": "papers/architectures/131-longformer/summary.md", "topics": [ "attention", "long-context", "efficiency" ] }, { "number": 132, "title": "RULER: What's the Real Context Size of Your Long-Context Language Models? (RULER)", "slug": "132-ruler", "category": "techniques", "authors": "Cheng-Ping Hsieh, Simeng Sun, Samuel Kriman, Shantanu Acharya, Dima Rekesh, Fei Jia, Yang Zhang, Boris Ginsburg (NVIDIA)", "published": "April 2024 (COLM 2024)", "year": 2024, "url": "https://arxiv.org/abs/2404.06654", "path": "papers/techniques/132-ruler/summary.md", "topics": [ "long-context", "evaluation", "benchmarks" ] }, { "number": 133, "title": "The FineWeb Datasets: Decanting the Web for the Finest Text Data at Scale (FineWeb)", "slug": "133-fineweb", "category": "techniques", "authors": "Guilherme Penedo, Hynek Kydlíček, Loubna Ben allal, Anton Lozhkov, Margaret Mitchell, Colin Raffel, Leandro Von Werra, Thomas Wolf (Hugging Face)", "published": "June 2024 (NeurIPS 2024 Datasets and Benchmarks Track)", "year": 2024, "url": "https://arxiv.org/abs/2406.17557", "path": "papers/techniques/133-fineweb/summary.md", "topics": [ "pretraining", "datasets", "data-curation" ] }, { "number": 134, "title": "Distilling the Knowledge in a Neural Network (Knowledge Distillation)", "slug": "134-knowledge-distillation", "category": "techniques", "authors": "Geoffrey Hinton, Oriol Vinyals, Jeff Dean (Google)", "published": "March 2015 (NIPS 2014 Deep Learning Workshop)", "year": 2015, "url": "https://arxiv.org/abs/1503.02531", "path": "papers/techniques/134-knowledge-distillation/summary.md", "topics": [ "efficiency", "distillation", "transfer-learning" ] }, { "number": 135, "title": "Textbooks Are All You Need (phi-1)", "slug": "135-phi-1-textbooks", "category": "language-models", "authors": "Suriya Gunasekar, Yi Zhang, Jyoti Aneja, Caio César Teodoro Mendes, Allie Del Giorno, Sivakanth Gopi, Mojan Javaheripi, Piero Kauffmann, Gustavo de Rosa, Olli Saarikivi, Adil Salim, Shital Shah, Harkirat Singh Behl, Xin Wang, Sébastien Bubeck, Ronen Eldan, Adam Tauman Kalai, Yin Tat Lee, Yuanzhi Li (Microsoft Research)", "published": "June 2023 (arXiv preprint)", "year": 2023, "url": "https://arxiv.org/abs/2306.11644", "path": "papers/language-models/135-phi-1-textbooks/summary.md", "topics": [ "language-model", "synthetic-data", "code" ] }, { "number": 136, "title": "Neural Machine Translation of Rare Words with Subword Units (BPE)", "slug": "136-bpe-subword-units", "category": "techniques", "authors": "Rico Sennrich, Barry Haddow, Alexandra Birch (University of Edinburgh)", "published": "August 2015 (arXiv); August 2016 (ACL 2016)", "year": 2015, "url": "https://arxiv.org/abs/1508.07909", "path": "papers/techniques/136-bpe-subword-units/summary.md", "topics": [ "tokenization", "language-model", "embeddings" ] }, { "number": 137, "title": "Measuring Massive Multitask Language Understanding (MMLU)", "slug": "137-mmlu", "category": "techniques", "authors": "Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt (UC Berkeley, Columbia University, University of Chicago, UIUC)", "published": "September 2020 (ICLR 2021)", "year": 2020, "url": "https://arxiv.org/abs/2009.03300", "path": "papers/techniques/137-mmlu/summary.md", "topics": [ "evaluation", "benchmarks" ] }, { "number": 138, "title": "On the Measure of Intelligence (ARC)", "slug": "138-arc-agi", "category": "techniques", "authors": "François Chollet (Google)", "published": "November 2019 (arXiv preprint)", "year": 2019, "url": "https://arxiv.org/abs/1911.01547", "path": "papers/techniques/138-arc-agi/summary.md", "topics": [ "evaluation", "benchmarks", "reasoning" ] }, { "number": 139, "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments (OSWorld)", "slug": "139-osworld", "category": "techniques", "authors": "Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu (University of Hong Kong, Carnegie Mellon University, Salesforce Research, University of Waterloo)", "published": "April 2024 (NeurIPS 2024 Datasets and Benchmarks Track)", "year": 2024, "url": "https://arxiv.org/abs/2404.07972", "path": "papers/techniques/139-osworld/summary.md", "topics": [ "evaluation", "benchmarks", "agents" ] }, { "number": 140, "title": "AI Models Collapse When Trained on Recursively Generated Data (Model Collapse)", "slug": "140-model-collapse", "category": "techniques", "authors": "Ilia Shumailov, Zakhar Shumaylov, Yiren Zhao, Nicolas Papernot, Ross Anderson, Yarin Gal (University of Oxford, University of Cambridge, Imperial College London, University of Toronto and Vector Institute, University of Edinburgh)", "published": "July 2024 (Nature 631, 755-759); earlier preprint May 2023 as \"The Curse of Recursion\"", "year": 2024, "url": "https://www.nature.com/articles/s41586-024-07566-y", "path": "papers/techniques/140-model-collapse/summary.md", "topics": [ "synthetic-data", "pretraining", "evaluation" ] }, { "number": 141, "title": "DeepSeek-V2: A Strong, Economical, and Efficient Mixture-of-Experts Language Model (Multi-head Latent Attention)", "slug": "141-multi-head-latent-attention", "category": "architectures", "authors": "DeepSeek-AI", "published": "May 2024 (arXiv)", "year": 2024, "url": "https://arxiv.org/abs/2405.04434", "path": "papers/architectures/141-multi-head-latent-attention/summary.md", "topics": [ "attention", "architecture", "efficiency", "inference-optimization" ] }, { "number": 142, "title": "Adam: A Method for Stochastic Optimization (Adam)", "slug": "142-adam", "category": "techniques", "authors": "Diederik P. Kingma, Jimmy Lei Ba (University of Amsterdam and OpenAI; University of Toronto)", "published": "December 2014 (ICLR 2015)", "year": 2014, "url": "https://arxiv.org/abs/1412.6980", "path": "papers/techniques/142-adam/summary.md", "topics": [ "optimization", "training" ] }, { "number": 143, "title": "Mixed Precision Training", "slug": "143-mixed-precision-training", "category": "techniques", "authors": "Paulius Micikevicius, Sharan Narang, Jonah Alben, Gregory Diamos, Erich Elsen, David Garcia, Boris Ginsburg, Michael Houston, Oleksii Kuchaiev, Ganesh Venkatesh, Hao Wu (NVIDIA, Baidu Research)", "published": "October 2017 (ICLR 2018)", "year": 2017, "url": "https://arxiv.org/abs/1710.03740", "path": "papers/techniques/143-mixed-precision-training/summary.md", "topics": [ "efficiency", "training", "systems" ] }, { "number": 144, "title": "Muon: An Optimizer for Hidden Layers, and Muon is Scalable for LLM Training (Muon)", "slug": "144-muon", "category": "techniques", "authors": "Keller Jordan, with Yuchen Jin, Vlado Boza, Jiacheng You, Franz Cesista, Laker Newhouse, Jeremy Bernstein (independent researchers and academic collaborators) - Muon post; Jingyuan Liu, Jianlin Su, Xingcheng Yao, Zhejun Jiang, Guokun Lai, Yulun Du, Yidao Qin, Weixin Xu, Enzhe Lu, Junjie Yan, Yanru Chen, Huabin Zheng, Yibo Liu, Shaowei Liu, Bohong Yin, Weiran He, Han Zhu, Yuzhi Wang, Jianzhou Wang, Mengnan Dong, Zheng Zhang, Yongsheng Kang, Hao Zhang, Xinran Xu, Yutao Zhang, Yuxin Wu, Xinyu Zhou, Zhilin Yang (Moonshot AI, UCLA) - scaling paper", "published": "December 2024 (blog post); February 2025 (arXiv preprint)", "year": 2024, "url": "https://arxiv.org/abs/2502.16982", "path": "papers/techniques/144-muon/summary.md", "topics": [ "optimization", "training", "efficiency" ] }, { "number": 145, "title": "Measuring AI Ability to Complete Long Tasks (METR Time Horizons)", "slug": "145-metr-time-horizons", "category": "techniques", "authors": "Thomas Kwa, Ben West, Joel Becker, Amy Deng, Katharyn Garcia, Max Hasin, Sami Jawhar, Megan Kinniment, Nate Rush, Sydney Von Arx, Ryan Bloom, Thomas Broadley, Haoxing Du, Brian Goodrich, Nikola Jurkovic, Luke Harold Miles, Seraphina Nix, Tao Lin, Chris Painter, Neev Parikh, David Rein, Lucas Jun Koba Sato, Hjalmar Wijk, Daniel M. Ziegler, Elizabeth Barnes, Lawrence Chan (METR)", "published": "March 2025 (arXiv v1, 18 March 2025; later revised - the current arXiv title is \"Measuring AI Ability to Complete Long Software Tasks\")", "year": 2025, "url": "https://arxiv.org/abs/2503.14499", "path": "papers/techniques/145-metr-time-horizons/summary.md", "topics": [ "evaluation", "agents", "scaling" ] }, { "number": 146, "title": "Deep Reinforcement Learning from Human Preferences", "slug": "146-deep-rl-from-human-preferences", "category": "techniques", "authors": "Paul Christiano, Jan Leike, Tom B. Brown, Miljan Martic, Shane Legg, Dario Amodei (OpenAI, DeepMind)", "published": "June 2017 (arXiv v1, 12 June 2017; NIPS 2017)", "year": 2017, "url": "https://arxiv.org/abs/1706.03741", "path": "papers/techniques/146-deep-rl-from-human-preferences/summary.md", "topics": [ "alignment", "reinforcement-learning", "rlhf" ] }, { "number": 147, "title": "Learning to Summarize from Human Feedback", "slug": "147-learning-to-summarize-from-human-feedback", "category": "techniques", "authors": "Nisan Stiennon, Long Ouyang, Jeff Wu, Daniel M. Ziegler, Ryan Lowe, Chelsea Voss, Alec Radford, Dario Amodei, Paul Christiano (OpenAI)", "published": "September 2020 (NeurIPS 2020)", "year": 2020, "url": "https://arxiv.org/abs/2009.01325", "path": "papers/techniques/147-learning-to-summarize-from-human-feedback/summary.md", "topics": [ "alignment", "reinforcement-learning", "rlhf" ] }, { "number": 148, "title": "Tulu 3: Pushing Frontiers in Open Language Model Post-Training (Tulu 3)", "slug": "148-tulu-3", "category": "language-models", "authors": "Nathan Lambert, Jacob Morrison, Valentina Pyatkin, Shengyi Huang, Hamish Ivison, Faeze Brahman, Lester James V. Miranda, Alisa Liu, Nouha Dziri, Shane Lyu, Yuling Gu, Saumya Malik, Victoria Graf, Jena D. Hwang, Jiangjiang Yang, Ronan Le Bras, Oyvind Tafjord, Chris Wilhelm, Luca Soldaini, Noah A. Smith, Yizhong Wang, Pradeep Dasigi, Hannaneh Hajishirzi (Allen Institute for AI and University of Washington)", "published": "November 2024", "year": 2024, "url": "https://arxiv.org/abs/2411.15124", "path": "papers/language-models/148-tulu-3/summary.md", "topics": [ "alignment", "reinforcement-learning", "open-models" ] }, { "number": 149, "title": "Why Language Models Hallucinate", "slug": "149-why-language-models-hallucinate", "category": "techniques", "authors": "Adam Tauman Kalai, Ofir Nachum, Santosh S. Vempala, Edwin Zhang (OpenAI, Georgia Tech)", "published": "September 2025 (arXiv preprint, v1 submitted 4 September 2025)", "year": 2025, "url": "https://arxiv.org/abs/2509.04664", "path": "papers/techniques/149-why-language-models-hallucinate/summary.md", "topics": [ "safety", "evaluation" ] }, { "number": 150, "title": "On the Dangers of Stochastic Parrots: Can Language Models Be Too Big?", "slug": "150-stochastic-parrots", "category": "essays", "authors": "Emily M. Bender (University of Washington), Timnit Gebru (Black in AI), Angelina McMillan-Major (University of Washington), Shmargaret Shmitchell (listed as \"The Aether\"; a pseudonym, see Context)", "published": "March 2021 (FAccT '21, ACM Conference on Fairness, Accountability, and Transparency, held virtually 3-10 March 2021)", "year": 2021, "url": "https://dl.acm.org/doi/10.1145/3442188.3445922", "path": "papers/essays/150-stochastic-parrots/summary.md", "topics": [ "essay", "safety", "scaling" ] }, { "number": 151, "title": "Circuit Tracing and On the Biology of a Large Language Model (Attribution Graphs)", "slug": "151-circuit-tracing", "category": "techniques", "authors": "Emmanuel Ameisen, Jack Lindsey, Adam Pearce, Wes Gurnee, Nicholas L. Turner, Brian Chen, Craig Citro, and 19 further Anthropic colleagues including Chris Olah and Joshua Batson (Anthropic). The companion paper \"On the Biology of a Large Language Model\" is led by Jack Lindsey with the same team.", "published": "March 2025 (March 27, 2025; transformer-circuits.pub)", "year": 2025, "url": "https://transformer-circuits.pub/2025/attribution-graphs/methods.html", "path": "papers/techniques/151-circuit-tracing/summary.md", "topics": [ "interpretability", "safety" ] }, { "number": 152, "title": "Reasoning Models Don't Always Say What They Think (CoT Faithfulness)", "slug": "152-reasoning-models-dont-say-what-they-think", "category": "techniques", "authors": "Yanda Chen, Joe Benton, Ansh Radhakrishnan, Jonathan Uesato, Carson Denison, John Schulman, Arushi Somani, Peter Hase, Misha Wagner, Fabien Roger, Vlad Mikulik, Samuel R. Bowman, Jan Leike, Jared Kaplan, Ethan Perez (Anthropic Alignment Science)", "published": "May 2025 (arXiv, submitted May 8, 2025)", "year": 2025, "url": "https://arxiv.org/abs/2505.05410", "path": "papers/techniques/152-reasoning-models-dont-say-what-they-think/summary.md", "topics": [ "reasoning", "safety", "interpretability" ] }, { "number": 153, "title": "Emergent Misalignment: Narrow Finetuning Can Produce Broadly Misaligned LLMs", "slug": "153-emergent-misalignment", "category": "techniques", "authors": "Jan Betley, Daniel Tan, Niels Warncke, Anna Sztyber-Betley, Xuchan Bao, Martin Soto, Nathan Labenz, Owain Evans (Truthful AI, UCL, Center on Long-Term Risk, Warsaw University of Technology, University of Toronto, UK AISI, UC Berkeley, independent)", "published": "February 2025 (arXiv v1, 24 February 2025; the arXiv listing notes ICML 2025 acceptance of an earlier revision, and an extended version appeared in Nature in 2026)", "year": 2025, "url": "https://arxiv.org/abs/2502.17424", "path": "papers/techniques/153-emergent-misalignment/summary.md", "topics": [ "safety", "alignment" ] }, { "number": 154, "title": "s1: Simple Test-Time Scaling (s1)", "slug": "154-s1-simple-test-time-scaling", "category": "techniques", "authors": "Niklas Muennighoff, Zitong Yang, Weijia Shi, Xiang Lisa Li, Li Fei-Fei, Hannaneh Hajishirzi, Luke Zettlemoyer, Percy Liang, Emmanuel Candes, Tatsunori Hashimoto (Stanford University, Allen Institute for AI, University of Washington, Contextual AI)", "published": "January 2025 (arXiv v1, 31 January 2025; latest version v3, 1 March 2025)", "year": 2025, "url": "https://arxiv.org/abs/2501.19393", "path": "papers/techniques/154-s1-simple-test-time-scaling/summary.md", "topics": [ "reasoning", "scaling", "test-time-compute" ] }, { "number": 155, "title": "DAPO: An Open-Source LLM Reinforcement Learning System at Scale (DAPO)", "slug": "155-dapo", "category": "techniques", "authors": "Qiying Yu, Zheng Zhang, Ruofei Zhu, Yufeng Yuan, Xiaochen Zuo, Yu Yue, et al. (ByteDance Seed, Tsinghua University AIR, The University of Hong Kong; project lead Qiying Yu, senior authors include Hao Zhou and Mingxuan Wang)", "published": "March 2025 (arXiv v1, 18 March 2025; revised May 2025)", "year": 2025, "url": "https://arxiv.org/abs/2503.14476", "path": "papers/techniques/155-dapo/summary.md", "topics": [ "reinforcement-learning", "reasoning", "alignment" ] }, { "number": 156, "title": "Mamba-3: Improved Sequence Modeling using State Space Principles (Mamba-3)", "slug": "156-mamba-3", "category": "architectures", "authors": "Aakash Lahoti, Kevin Y. Li, Berlin Chen, Caitlin Wang, Aviv Bick, J. Zico Kolter, Tri Dao, Albert Gu (Carnegie Mellon University, Princeton University, Together AI, Cartesia AI)", "published": "March 2026 (arXiv v1, 16 March 2026; the arXiv listing marks it ICLR 2026)", "year": 2026, "url": "https://arxiv.org/abs/2603.15569", "path": "papers/architectures/156-mamba-3/summary.md", "topics": [ "architecture", "state-space", "efficiency" ] } ] }