{"dataset":"epoch_ai_models","title":"Модели ИИ: масштабы, вычисления и стоимость обучения","page":0,"pages":82,"rows_in_page":100,"total_rows":8120,"columns":["entity_type","entity_id","iso3","period_start","period_end","time_basis","measure","value","unit","dimensions","quality","source_url","raw_sha256"],"rows":[["ai_model","00056d2eb6cd23da5633","","2024-06-07","2024-06-07","model_publication_date","ai_parameters",7000000000,"parameters","{\"Country (of organization)\": \"China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Qwen2-7B\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"\", \"Organization\": \"Alibaba\", \"Task\": \"Chat,Language modeling/generation,Question answering\", \"field\": \"Parameters\", \"notes\": \"7B parameters (table 1)\", \"reference_url\": \"https://qwenlm.github.io/blog/qwen2/ \\nhttps://arxiv.org/abs/2407.10671 \"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","00056d2eb6cd23da5633","","2024-06-07","2024-06-07","model_publication_date","ai_training_compute",2.9400000000001e+23,"FLOP","{\"Country (of organization)\": \"China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Qwen2-7B\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"\", \"Organization\": \"Alibaba\", \"Task\": \"Chat,Language modeling/generation,Question answering\", \"field\": \"Training compute (FLOP)\", \"notes\": \"7 billion params, 7 trillion tokens\\n\\n6 FLOP * 7 billion * 7 trillion ~= 2.94e23 FLOP\", \"reference_url\": \"https://qwenlm.github.io/blog/qwen2/ \\nhttps://arxiv.org/abs/2407.10671 \"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","00056d2eb6cd23da5633","","2024-06-07","2024-06-07","model_publication_date","ai_training_dataset_size",7000000000000,"source-defined data units","{\"Country (of organization)\": \"China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Qwen2-7B\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"\", \"Organization\": \"Alibaba\", \"Task\": \"Chat,Language modeling/generation,Question answering\", \"field\": \"Training dataset size (total)\", \"notes\": \"\\\"All models were pre-trained on a high-quality, large-scale dataset comprising over 7 trillion tokens, covering a wide range of domains and languages.\\\"\", \"reference_url\": \"https://qwenlm.github.io/blog/qwen2/ \\nhttps://arxiv.org/abs/2407.10671 \"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","000f0e8e1a7d68cbdd17","","2024-09-24","2024-09-24","model_publication_date","ai_parameters",3210000000,"parameters","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Llama 3.2 3B\", \"Model accessibility\": \"Open weights (restricted use)\", \"Notability criteria\": \"\", \"Organization\": \"Meta AI\", \"Task\": \"Language modeling/generation,Text summarization,Question answering,Quantitative reasoning,Translation\", \"field\": \"Parameters\", \"notes\": \"https://huggingface.co/meta-llama/Llama-3.2-1B\", \"reference_url\": \"https://ai.meta.com/blog/llama-3-2-connect-2024-vision-edge-mobile-devices/\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","000f0e8e1a7d68cbdd17","","2024-09-24","2024-09-24","model_publication_date","ai_training_chip_hours",460000,"chip-hours","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Llama 3.2 3B\", \"Model accessibility\": \"Open weights (restricted use)\", \"Notability criteria\": \"\", \"Organization\": \"Meta AI\", \"Task\": \"Language modeling/generation,Text summarization,Question answering,Quantitative reasoning,Translation\", \"field\": \"Training chip-hours\", \"notes\": \"6ND = 6*3210000000.00*9000000000000 = 1.7334e+23\\n\\n460000 hours * 3600 s * 133800000000000 FLOPS/s* 0.3 = 6.647184e+22\", \"reference_url\": \"https://ai.meta.com/blog/llama-3-2-connect-2024-vision-edge-mobile-devices/\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","000f0e8e1a7d68cbdd17","","2024-09-24","2024-09-24","model_publication_date","ai_training_compute",1.7334e+23,"FLOP","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Llama 3.2 3B\", \"Model accessibility\": \"Open weights (restricted use)\", \"Notability criteria\": \"\", \"Organization\": \"Meta AI\", \"Task\": \"Language modeling/generation,Text summarization,Question answering,Quantitative reasoning,Translation\", \"field\": \"Training compute (FLOP)\", \"notes\": \"6ND = 6*3210000000.00*9000000000000 = 1.7334e+23\\n\\n460000 hours * 3600 s * 133800000000000 FLOPS/s* 0.3 = 6.647184e+22\", \"reference_url\": \"https://ai.meta.com/blog/llama-3-2-connect-2024-vision-edge-mobile-devices/\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","000f0e8e1a7d68cbdd17","","2024-09-24","2024-09-24","model_publication_date","ai_training_dataset_size",9000000000000,"source-defined data units","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Llama 3.2 3B\", \"Model accessibility\": \"Open weights (restricted use)\", \"Notability criteria\": \"\", \"Organization\": \"Meta AI\", \"Task\": \"Language modeling/generation,Text summarization,Question answering,Quantitative reasoning,Translation\", \"field\": \"Training dataset size (total)\", \"notes\": \"\\\"Llama 3.2 was pretrained on up to 9 trillion tokens of data from publicly available sources.\\\"\", \"reference_url\": \"https://ai.meta.com/blog/llama-3-2-connect-2024-vision-edge-mobile-devices/\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","004b0cbfdcac0b782de7","","2024-02-27","2024-02-27","model_publication_date","ai_parameters",7000000000,"parameters","{\"Country (of organization)\": \"United States of America,United States of America,United States of America\", \"Domain\": \"Biology\", \"Frontier model\": \"\", \"Model\": \"Evo\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"\", \"Organization\": \"Stanford University,University of California (UC) Berkeley,Together\", \"Task\": \"Protein or nucleotide language model (pLM/nLM)\", \"field\": \"Parameters\", \"notes\": \"Based on a StripedHyena architecture, context length of 131 kilobases\", \"reference_url\": \"https://arcinstitute.org/news/blog/evo\\nhttps://www.biorxiv.org/content/10.1101/2024.02.27.582234v1\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","004b0cbfdcac0b782de7","","2024-02-27","2024-02-27","model_publication_date","ai_training_compute",2.0000000001e+22,"FLOP","{\"Country (of organization)\": \"United States of America,United States of America,United States of America\", \"Domain\": \"Biology\", \"Frontier model\": \"\", \"Model\": \"Evo\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"\", \"Organization\": \"Stanford University,University of California (UC) Berkeley,Together\", \"Task\": \"Protein or nucleotide language model (pLM/nLM)\", \"field\": \"Training compute (FLOP)\", \"notes\": \"\\\"In total, Evo was trained on approximately 340B tokens, using approximately 2e22 FLOPS\\\"\", \"reference_url\": \"https://arcinstitute.org/news/blog/evo\\nhttps://www.biorxiv.org/content/10.1101/2024.02.27.582234v1\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","004b0cbfdcac0b782de7","","2024-02-27","2024-02-27","model_publication_date","ai_training_dataset_size",300000000000,"source-defined data units","{\"Country (of organization)\": \"United States of America,United States of America,United States of America\", \"Domain\": \"Biology\", \"Frontier model\": \"\", \"Model\": \"Evo\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"\", \"Organization\": \"Stanford University,University of California (UC) Berkeley,Together\", \"Task\": \"Protein or nucleotide language model (pLM/nLM)\", \"field\": \"Training dataset size (total)\", \"notes\": \"300 billion nucleotides. Since these are tokenized at the single-nucleotide level, 300B tokens.\", \"reference_url\": \"https://arcinstitute.org/news/blog/evo\\nhttps://www.biorxiv.org/content/10.1101/2024.02.27.582234v1\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","004b0cbfdcac0b782de7","","2024-02-27","2024-02-27","model_publication_date","ai_training_hours",2968,"hours","{\"Country (of organization)\": \"United States of America,United States of America,United States of America\", \"Domain\": \"Biology\", \"Frontier model\": \"\", \"Model\": \"Evo\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"\", \"Organization\": \"Stanford University,University of California (UC) Berkeley,Together\", \"Task\": \"Protein or nucleotide language model (pLM/nLM)\", \"field\": \"Training time (hours)\", \"notes\": \"Trained for first stage on 64 H100s, then for second stage on 128 A100s. Trained primarily with BF16 precision; FP32 used for \\\"long convolutional parameters\\\". I use BF16 performance for simplicity.\\n\\nThis gives a plausible range for completing 2e22 FLOPs.\\n\\nIf all 2e22 FLOPs were performed on H100s:\\n2e22 FLOP / (64 GPUs * 1.3e14 FLOP/GPU-sec) * 1/0.3 efficiency *  1/3600 hour/sec  = 2226 hours\\n\\nIf all 2e22 FLOPs were performed on A100s:\\n2e22 FLOP / (128 GPUs * 3.9e13 FLOP/GPU-sec) * 1/0.3 efficiency * 1/3600 hour/sec = 3710 hours\\n\\nAssume each stage is roughly half the total time, so:\\n(2226+3710)/2 = 2968 hours\", \"reference_url\": \"https://arcinstitute.org/news/blog/evo\\nhttps://www.biorxiv.org/content/10.1101/2024.02.27.582234v1\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","00b76dace233c9cffca2","","2024-02-15","2024-02-15","model_publication_date","ai_finetune_compute",35000000000000000000,"FLOP","{\"Country (of organization)\": \"Australia,China\", \"Domain\": \"Biology\", \"Frontier model\": \"\", \"Model\": \"ProtChatGPT\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"\", \"Organization\": \"University of Technology Sydney,Zhejiang University (ZJU)\", \"Task\": \"Protein question answering\", \"field\": \"Finetune compute (FLOP)\", \"notes\": \"\", \"reference_url\": \"https://arxiv.org/abs/2402.09649\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","00b76dace233c9cffca2","","2024-02-15","2024-02-15","model_publication_date","ai_parameters",8000000000,"parameters","{\"Country (of organization)\": \"Australia,China\", \"Domain\": \"Biology\", \"Frontier model\": \"\", \"Model\": \"ProtChatGPT\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"\", \"Organization\": \"University of Technology Sydney,Zhejiang University (ZJU)\", \"Task\": \"Protein question answering\", \"field\": \"Parameters\", \"notes\": \"\", \"reference_url\": \"https://arxiv.org/abs/2402.09649\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","00b76dace233c9cffca2","","2024-02-15","2024-02-15","model_publication_date","ai_training_compute",7.200353201209069e+23,"FLOP","{\"Country (of organization)\": \"Australia,China\", \"Domain\": \"Biology\", \"Frontier model\": \"\", \"Model\": \"ProtChatGPT\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"\", \"Organization\": \"University of Technology Sydney,Zhejiang University (ZJU)\", \"Task\": \"Protein question answering\", \"field\": \"Training compute (FLOP)\", \"notes\": \"GPU hours: 86400s*9.9e+14*2*0.4=6.8e+19\\n\\n6ND: 6*8000000000*380000001=18240000048000000000\\n\\nGMean(1.8e+19,6.8e+19)=3.5e+19\\n\\nBase model: 7.2e+23\\n\\nTotal: 720035320120906900000000\", \"reference_url\": \"https://arxiv.org/abs/2402.09649\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","00b76dace233c9cffca2","","2024-02-15","2024-02-15","model_publication_date","ai_training_hardware_quantity",2,"devices","{\"Country (of organization)\": \"Australia,China\", \"Domain\": \"Biology\", \"Frontier model\": \"\", \"Model\": \"ProtChatGPT\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"\", \"Organization\": \"University of Technology Sydney,Zhejiang University (ZJU)\", \"Task\": \"Protein question answering\", \"field\": \"Hardware quantity\", \"notes\": \"NVIDIA H100 SXM5 80GB\", \"reference_url\": \"https://arxiv.org/abs/2402.09649\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","00b76dace233c9cffca2","","2024-02-15","2024-02-15","model_publication_date","ai_training_hours",24,"hours","{\"Country (of organization)\": \"Australia,China\", \"Domain\": \"Biology\", \"Frontier model\": \"\", \"Model\": \"ProtChatGPT\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"\", \"Organization\": \"University of Technology Sydney,Zhejiang University (ZJU)\", \"Task\": \"Protein question answering\", \"field\": \"Training time (hours)\", \"notes\": \"\", \"reference_url\": \"https://arxiv.org/abs/2402.09649\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","00b76dace233c9cffca2","","2024-02-15","2024-02-15","model_publication_date","ai_training_power",2771.3350442941023,"watts","{\"Country (of organization)\": \"Australia,China\", \"Domain\": \"Biology\", \"Frontier model\": \"\", \"Model\": \"ProtChatGPT\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"\", \"Organization\": \"University of Technology Sydney,Zhejiang University (ZJU)\", \"Task\": \"Protein question answering\", \"field\": \"Training power draw (W)\", \"notes\": \"GPU hours: 86400s*9.9e+14*2*0.4=6.8e+19\\n\\n6ND: 6*8000000000*380000001=18240000048000000000\\n\\nGMean(1.8e+19,6.8e+19)=3.5e+19\\n\\nBase model: 7.2e+23\\n\\nTotal: 720035320120906900000000\", \"reference_url\": \"https://arxiv.org/abs/2402.09649\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","00c076196b24a6017486","","2023-05-03","2023-05-03","model_publication_date","ai_parameters",16000000000,"parameters","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"CodeGen2\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"\", \"Organization\": \"Salesforce\", \"Task\": \"Code generation\", \"field\": \"Parameters\", \"notes\": \"16B for largest CodeGen2 model\", \"reference_url\": \"https://arxiv.org/abs/2305.02309\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","00c40f8a6136a79a2b61","","2023-07-18","2023-07-18","model_publication_date","ai_parameters",34000000000,"parameters","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Llama 2-34B\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"Highly cited\", \"Organization\": \"Meta AI\", \"Task\": \"Language modeling\", \"field\": \"Parameters\", \"notes\": \"Llama has been released in 7B, 13B, 34B, and 70B variants.\", \"reference_url\": \"https://ai.meta.com/research/publications/llama-2-open-foundation-and-fine-tuned-chat-models/\\nhttps://arxiv.org/abs/2307.09288\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","00c40f8a6136a79a2b61","","2023-07-18","2023-07-18","model_publication_date","ai_training_compute",4.08e+23,"FLOP","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Llama 2-34B\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"Highly cited\", \"Organization\": \"Meta AI\", \"Task\": \"Language modeling\", \"field\": \"Training compute (FLOP)\", \"notes\": \"All models sizes trained on 2.0T tokens, per table 1\\n2T * 34b * 6 = 4.08e23\\n\\nAlso trained on 1038336 A100-hours, which is 3.5e23 at 30% utilization. So the utilization was probably around 35%.\", \"reference_url\": \"https://ai.meta.com/research/publications/llama-2-open-foundation-and-fine-tuned-chat-models/\\nhttps://arxiv.org/abs/2307.09288\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","00c40f8a6136a79a2b61","","2023-07-18","2023-07-18","model_publication_date","ai_training_cost",600469.8720490151,"2023 US dollars","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Llama 2-34B\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"Highly cited\", \"Organization\": \"Meta AI\", \"Task\": \"Language modeling\", \"field\": \"Training compute cost (2023 USD)\", \"notes\": \"All models sizes trained on 2.0T tokens, per table 1\\n2T * 34b * 6 = 4.08e23\\n\\nAlso trained on 1038336 A100-hours, which is 3.5e23 at 30% utilization. So the utilization was probably around 35%.\", \"reference_url\": \"https://ai.meta.com/research/publications/llama-2-open-foundation-and-fine-tuned-chat-models/\\nhttps://arxiv.org/abs/2307.09288\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","00c40f8a6136a79a2b61","","2023-07-18","2023-07-18","model_publication_date","ai_training_dataset_size",2000000000000,"source-defined data units","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Llama 2-34B\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"Highly cited\", \"Organization\": \"Meta AI\", \"Task\": \"Language modeling\", \"field\": \"Training dataset size (total)\", \"notes\": \"\", \"reference_url\": \"https://ai.meta.com/research/publications/llama-2-open-foundation-and-fine-tuned-chat-models/\\nhttps://arxiv.org/abs/2307.09288\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","00e4c4ab45c6689ea565","","2023-07-12","2023-07-12","model_publication_date","ai_parameters",169000000,"parameters","{\"Country (of organization)\": \"China\", \"Domain\": \"Biology\", \"Frontier model\": \"\", \"Model\": \"Uni-RNA-L16\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"\", \"Organization\": \"DP Technology\", \"Task\": \"Protein or nucleotide language model (pLM/nLM)\", \"field\": \"Parameters\", \"notes\": \"Table 8: Model architecture parameters of different Uni-RNA models\\n\\nEstimating sequence length at 500 tokens.\\n\\n500000000*500=250000000000\", \"reference_url\": \"https://www.biorxiv.org/content/10.1101/2023.07.11.548588v1.abstract\"}","Likely","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","00e6f2a11bf68628d249","","2025-08-01","2025-08-01","model_publication_date","ai_finetune_compute",232300000000000000000,"FLOP","{\"Country (of organization)\": \"China,China,China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"rStar-Math (Qwen2-Math-7B base)\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"\", \"Organization\": \"Microsoft Research Asia,Peking University,Tsinghua University\", \"Task\": \"Language modeling/generation,Mathematical reasoning\", \"field\": \"Finetune compute (FLOP)\", \"notes\": \"5530000000 * 7000000000 * 6 = 2.323e20\", \"reference_url\": \"https://arxiv.org/abs/2501.04519\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","00e6f2a11bf68628d249","","2025-08-01","2025-08-01","model_publication_date","ai_parameters",7000000000,"parameters","{\"Country (of organization)\": \"China,China,China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"rStar-Math (Qwen2-Math-7B base)\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"\", \"Organization\": \"Microsoft Research Asia,Peking University,Tsinghua University\", \"Task\": \"Language modeling/generation,Mathematical reasoning\", \"field\": \"Parameters\", \"notes\": \"\", \"reference_url\": \"https://arxiv.org/abs/2501.04519\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","00e6f2a11bf68628d249","","2025-08-01","2025-08-01","model_publication_date","ai_training_hardware_quantity",80,"devices","{\"Country (of organization)\": \"China,China,China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"rStar-Math (Qwen2-Math-7B base)\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"\", \"Organization\": \"Microsoft Research Asia,Peking University,Tsinghua University\", \"Task\": \"Language modeling/generation,Mathematical reasoning\", \"field\": \"Hardware quantity\", \"notes\": \"NVIDIA H100 SXM5 80GB\", \"reference_url\": \"https://arxiv.org/abs/2501.04519\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","00e6f2a11bf68628d249","","2025-08-01","2025-08-01","model_publication_date","ai_training_power",109545.39710400862,"watts","{\"Country (of organization)\": \"China,China,China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"rStar-Math (Qwen2-Math-7B base)\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"\", \"Organization\": \"Microsoft Research Asia,Peking University,Tsinghua University\", \"Task\": \"Language modeling/generation,Mathematical reasoning\", \"field\": \"Training power draw (W)\", \"notes\": \"\", \"reference_url\": \"https://arxiv.org/abs/2501.04519\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","012f5806a34dbc58d5ea","","2021-12-20","2021-12-20","model_publication_date","ai_parameters",564000000,"parameters","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"\", \"Frontier model\": \"\", \"Model\": \"XGLM\", \"Model accessibility\": \"\", \"Notability criteria\": \"\", \"Organization\": \"Meta AI\", \"Task\": \"\", \"field\": \"Parameters\", \"notes\": \"XGLM-564M is a multilingual autoregressive language model (with 564 million parameters) trained on a balanced corpus of a diverse set of 30 languages totaling 500 billion sub-tokens.\", \"reference_url\": \"https://arxiv.org/abs/2112.10668\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","014b6559b229e00774e9","","2022-05-25","2022-05-25","model_publication_date","ai_parameters",7000000,"parameters","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"TRIMELMext (7M)\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Princeton University\", \"Task\": \"Language modeling/generation,Translation\", \"field\": \"Parameters\", \"notes\": \"\", \"reference_url\": \"https://arxiv.org/abs/2205.12674\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","014b6559b229e00774e9","","2022-05-25","2022-05-25","model_publication_date","ai_training_compute",206000000000000000,"FLOP","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"TRIMELMext (7M)\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Princeton University\", \"Task\": \"Language modeling/generation,Translation\", \"field\": \"Training compute (FLOP)\", \"notes\": \"24576 tokens per update *  200000 updates = 4915200000 tokens\\n\\nUsing 6ND formula:\\n6*7000000*4915200000=2.064384e+17\", \"reference_url\": \"https://arxiv.org/abs/2205.12674\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","014b6559b229e00774e9","","2022-05-25","2022-05-25","model_publication_date","ai_training_dataset_size",4915200000,"source-defined data units","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"TRIMELMext (7M)\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Princeton University\", \"Task\": \"Language modeling/generation,Translation\", \"field\": \"Training dataset size (total)\", \"notes\": \"\\\"WIKITEXT-103 (Merity et al., 2017) is a word-\\nlevel language modeling dataset consisting of\\n103M training tokens. \\\"\", \"reference_url\": \"https://arxiv.org/abs/2205.12674\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","0152928183fbb8c2706c","","2025-11-17","2025-11-17","model_publication_date","ai_parameters",235000000000,"parameters","{\"Country (of organization)\": \"China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"P1-235B-A22B\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"Discretionary\", \"Organization\": \"Shanghai AI Lab\", \"Task\": \"Question answering\", \"field\": \"Parameters\", \"notes\": \"MoE: P1-235B-A22B has 235B total, 22B active parameters. Lightweight variant P1-30B-A3B has 30B total, 3B active. (https://arxiv.org/abs/2511.13612)\", \"reference_url\": \"https://arxiv.org/abs/2511.13612, https://github.com/PRIME-RL/P1\"}","Likely","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01592bd3fcaec13ac3c6","","2021-05-04","2021-05-04","model_publication_date","ai_parameters",3000000000,"parameters","{\"Country (of organization)\": \"Germany,China,United States of America,United States of America,United States of America,South Korea\", \"Domain\": \"Biology\", \"Frontier model\": \"\", \"Model\": \"ProtT5-XL-U50\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"Highly cited,SOTA improvement\", \"Organization\": \"Technical University of Munich,Med AI Technology,Nvidia,Oak Ridge National Laboratory,Google,Seoul National University\", \"Task\": \"Proteins,Protein or nucleotide language model (pLM/nLM),Protein contact and distance prediction\", \"field\": \"Parameters\", \"notes\": \"Table 2\", \"reference_url\": \"https://www.biorxiv.org/content/10.1101/2020.07.12.199554v3.full.pdf\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01592bd3fcaec13ac3c6","","2021-05-04","2021-05-04","model_publication_date","ai_training_compute",1.8704498688e+22,"FLOP","{\"Country (of organization)\": \"Germany,China,United States of America,United States of America,United States of America,South Korea\", \"Domain\": \"Biology\", \"Frontier model\": \"\", \"Model\": \"ProtT5-XL-U50\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"Highly cited,SOTA improvement\", \"Organization\": \"Technical University of Munich,Med AI Technology,Nvidia,Oak Ridge National Laboratory,Google,Seoul National University\", \"Task\": \"Proteins,Protein or nucleotide language model (pLM/nLM),Protein contact and distance prediction\", \"field\": \"Training compute (FLOP)\", \"notes\": \"991K steps, 2048 batch, 512 sequence length (Table 2)\\n\\nTotal tokens: 991K*2048*512=1039138816000\\n\\nFLOP: 6*1039138816000*3000000000=18704498688000000000000\", \"reference_url\": \"https://www.biorxiv.org/content/10.1101/2020.07.12.199554v3.full.pdf\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01592bd3fcaec13ac3c6","","2021-05-04","2021-05-04","model_publication_date","ai_training_dataset_size",19582371375,"source-defined data units","{\"Country (of organization)\": \"Germany,China,United States of America,United States of America,United States of America,South Korea\", \"Domain\": \"Biology\", \"Frontier model\": \"\", \"Model\": \"ProtT5-XL-U50\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"Highly cited,SOTA improvement\", \"Organization\": \"Technical University of Munich,Med AI Technology,Nvidia,Oak Ridge National Laboratory,Google,Seoul National University\", \"Task\": \"Proteins,Protein or nucleotide language model (pLM/nLM),Protein contact and distance prediction\", \"field\": \"Training dataset size (total)\", \"notes\": \"\", \"reference_url\": \"https://www.biorxiv.org/content/10.1101/2020.07.12.199554v3.full.pdf\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01592bd3fcaec13ac3c6","","2021-05-04","2021-05-04","model_publication_date","ai_training_hardware_quantity",256,"devices","{\"Country (of organization)\": \"Germany,China,United States of America,United States of America,United States of America,South Korea\", \"Domain\": \"Biology\", \"Frontier model\": \"\", \"Model\": \"ProtT5-XL-U50\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"Highly cited,SOTA improvement\", \"Organization\": \"Technical University of Munich,Med AI Technology,Nvidia,Oak Ridge National Laboratory,Google,Seoul National University\", \"Task\": \"Proteins,Protein or nucleotide language model (pLM/nLM),Protein contact and distance prediction\", \"field\": \"Hardware quantity\", \"notes\": \"\", \"reference_url\": \"https://www.biorxiv.org/content/10.1101/2020.07.12.199554v3.full.pdf\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01641014fc93f2e15eb0","","2026-08-11","2026-08-11","model_publication_date","ai_parameters",30000000000,"parameters","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Nemotron 3.5 Lightning\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"\", \"Organization\": \"Nvidia\", \"Task\": \"Language modeling/generation\", \"field\": \"Parameters\", \"notes\": \"30B total, 3B active\", \"reference_url\": \"https://developer.nvidia.com/blog/nvidia-nemotron-3-5-lightning-delivers-fast-accurate-specialized-task-execution-for-long-running-agents/\"}","Speculative","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01641014fc93f2e15eb0","","2026-08-11","2026-08-11","model_publication_date","ai_training_dataset_size",20000000000000,"source-defined data units","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Nemotron 3.5 Lightning\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"\", \"Organization\": \"Nvidia\", \"Task\": \"Language modeling/generation\", \"field\": \"Training dataset size (total)\", \"notes\": \"\", \"reference_url\": \"https://developer.nvidia.com/blog/nvidia-nemotron-3-5-lightning-delivers-fast-accurate-specialized-task-execution-for-long-running-agents/\"}","Speculative","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01647dad736df1c8528a","","2024-02-20","2024-02-20","model_publication_date","ai_finetune_compute",4.2727617e+22,"FLOP","{\"Country (of organization)\": \"United States of America,United States of America,United States of America\", \"Domain\": \"Language,Medicine\", \"Frontier model\": \"\", \"Model\": \"Me Llama 70B\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Yale School of Medicine,University of Florida,University of Texas Health Science Center\", \"Task\": \"Language modeling/generation,Question answering,Medical diagnosis,Named entity recognition (NER),Text classification,Text summarization\", \"field\": \"Finetune compute (FLOP)\", \"notes\": \"6 FLOP / parameter / token * 70 * 10^9 parameters * 129 * 10^9 pre-training tokens [instruction-tuning is negligible] = 5.418e+22 FLOP\\n\\n312000000000000 FLOP / GPU / sec [A100, bf16] * 100000 GPU-hours * 3600 sec/ hour * 0.3 [assumed utilization] = 3.3696e+22 FLOP\\n\\nsqrt(3.3696e+22 * 5.418e+22) = 4.2727617e+22 FLOP\", \"reference_url\": \"https://arxiv.org/abs/2402.12749\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01647dad736df1c8528a","","2024-02-20","2024-02-20","model_publication_date","ai_parameters",70000000000,"parameters","{\"Country (of organization)\": \"United States of America,United States of America,United States of America\", \"Domain\": \"Language,Medicine\", \"Frontier model\": \"\", \"Model\": \"Me Llama 70B\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Yale School of Medicine,University of Florida,University of Texas Health Science Center\", \"Task\": \"Language modeling/generation,Question answering,Medical diagnosis,Named entity recognition (NER),Text classification,Text summarization\", \"field\": \"Parameters\", \"notes\": \"70B\", \"reference_url\": \"https://arxiv.org/abs/2402.12749\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01647dad736df1c8528a","","2024-02-20","2024-02-20","model_publication_date","ai_training_chip_hours",100000,"chip-hours","{\"Country (of organization)\": \"United States of America,United States of America,United States of America\", \"Domain\": \"Language,Medicine\", \"Frontier model\": \"\", \"Model\": \"Me Llama 70B\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Yale School of Medicine,University of Florida,University of Texas Health Science Center\", \"Task\": \"Language modeling/generation,Question answering,Medical diagnosis,Named entity recognition (NER),Text classification,Text summarization\", \"field\": \"Training chip-hours\", \"notes\": \"\", \"reference_url\": \"https://arxiv.org/abs/2402.12749\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01647dad736df1c8528a","","2024-02-20","2024-02-20","model_publication_date","ai_training_dataset_size",129000000000,"source-defined data units","{\"Country (of organization)\": \"United States of America,United States of America,United States of America\", \"Domain\": \"Language,Medicine\", \"Frontier model\": \"\", \"Model\": \"Me Llama 70B\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Yale School of Medicine,University of Florida,University of Texas Health Science Center\", \"Task\": \"Language modeling/generation,Question answering,Medical diagnosis,Named entity recognition (NER),Text classification,Text summarization\", \"field\": \"Training dataset size (total)\", \"notes\": \"\\\"129B pre-training tokens and 214K instruction tuning samples from diverse biomedical and clinical data sources\\\"\", \"reference_url\": \"https://arxiv.org/abs/2402.12749\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","0198d35adc2a4e5dc799","","2024-05-31","2024-05-31","model_publication_date","ai_parameters",20000000000,"parameters","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Granite 20B\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"\", \"Organization\": \"IBM Research\", \"Task\": \"Language modeling/generation\", \"field\": \"Parameters\", \"notes\": \"\", \"reference_url\": \"https://www.ibm.com/downloads/documents/us-en/10a99803c92fdb35\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","0198d35adc2a4e5dc799","","2024-05-31","2024-05-31","model_publication_date","ai_training_compute",3.0000000000001e+23,"FLOP","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Granite 20B\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"\", \"Organization\": \"IBM Research\", \"Task\": \"Language modeling/generation\", \"field\": \"Training compute (FLOP)\", \"notes\": \"6*2500000000000*20000000000=3e+23\", \"reference_url\": \"https://www.ibm.com/downloads/documents/us-en/10a99803c92fdb35\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","0198d35adc2a4e5dc799","","2024-05-31","2024-05-31","model_publication_date","ai_training_dataset_size",2500000000000,"source-defined data units","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Granite 20B\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"\", \"Organization\": \"IBM Research\", \"Task\": \"Language modeling/generation\", \"field\": \"Training dataset size (total)\", \"notes\": \"For pre-training, we used 0.5 trillion English, 0.4 trillion multilingual (es, fr, de, pt), and 1.6 trillion code tokens.\", \"reference_url\": \"https://www.ibm.com/downloads/documents/us-en/10a99803c92fdb35\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01b86913e1b616e7afaa","","2021-06-09","2021-06-09","model_publication_date","ai_parameters",440000000,"parameters","{\"Country (of organization)\": \"Canada,Canada,United Kingdom\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"EMDR\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"SOTA improvement\", \"Organization\": \"Mila - Quebec AI (originally Montreal Institute for Learning Algorithms),McGill University,DeepMind\", \"Task\": \"Question answering\", \"field\": \"Parameters\", \"notes\": \"Table 2\", \"reference_url\": \"https://arxiv.org/abs/2106.05346v2\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01b86913e1b616e7afaa","","2021-06-09","2021-06-09","model_publication_date","ai_training_compute",1.91e+21,"FLOP","{\"Country (of organization)\": \"Canada,Canada,United Kingdom\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"EMDR\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"SOTA improvement\", \"Organization\": \"Mila - Quebec AI (originally Montreal Institute for Learning Algorithms),McGill University,DeepMind\", \"Task\": \"Question answering\", \"field\": \"Training compute (FLOP)\", \"notes\": \"\\\"We run all of our experiments on a machine with 96 CPUs, 1.3TB physical memory, and 16 A100 GPUs. We use PyTorch (Paszke et al., 2019) to implement our proposed model. With this hardware setup, our experiments on NQ and TriviaQA took approximately 25 hours to complete, while experiments on WebQ took roughly 8 hours to complete. Before supervised training, we also perform a one-time unsupervised MSS pre-training for 82,000 steps that took roughly 1 week.\\\"\\n\\n1 week + 25 hours * 16 A100s\\n= ~193 * 16 A100-hours\\n= 193 * 16 * 3600 * 312 trillion * 0.3 = 1.04e21\\n\\nAdditionally, the model uses BERT, ICT, and T5 models. These required:\\n- BERT: 6 * 110M parameters * (1M * 256 * 256) inputs = 4.33e19 FLOP\\n- ICT: 6 * 220M parameters * (100k * 4096 * 256) inputs = 1.38e20 FLOP\\n- T5: 6 * 220M parameters * (1M * 2048 * 256) inputs = 6.92e20 FLOP\\n\\nTotal: 1.04e21 + 4.33e19 + 1.38e20 + 6.92e20 = 1.91e21\", \"reference_url\": \"https://arxiv.org/abs/2106.05346v2\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01b86913e1b616e7afaa","","2021-06-09","2021-06-09","model_publication_date","ai_training_cost",2773.3124307816734,"2023 US dollars","{\"Country (of organization)\": \"Canada,Canada,United Kingdom\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"EMDR\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"SOTA improvement\", \"Organization\": \"Mila - Quebec AI (originally Montreal Institute for Learning Algorithms),McGill University,DeepMind\", \"Task\": \"Question answering\", \"field\": \"Training compute cost (2023 USD)\", \"notes\": \"\\\"We run all of our experiments on a machine with 96 CPUs, 1.3TB physical memory, and 16 A100 GPUs. We use PyTorch (Paszke et al., 2019) to implement our proposed model. With this hardware setup, our experiments on NQ and TriviaQA took approximately 25 hours to complete, while experiments on WebQ took roughly 8 hours to complete. Before supervised training, we also perform a one-time unsupervised MSS pre-training for 82,000 steps that took roughly 1 week.\\\"\\n\\n1 week + 25 hours * 16 A100s\\n= ~193 * 16 A100-hours\\n= 193 * 16 * 3600 * 312 trillion * 0.3 = 1.04e21\\n\\nAdditionally, the model uses BERT, ICT, and T5 models. These required:\\n- BERT: 6 * 110M parameters * (1M * 256 * 256) inputs = 4.33e19 FLOP\\n- ICT: 6 * 220M parameters * (100k * 4096 * 256) inputs = 1.38e20 FLOP\\n- T5: 6 * 220M parameters * (1M * 2048 * 256) inputs = 6.92e20 FLOP\\n\\nTotal: 1.04e21 + 4.33e19 + 1.38e20 + 6.92e20 = 1.91e21\", \"reference_url\": \"https://arxiv.org/abs/2106.05346v2\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01b86913e1b616e7afaa","","2021-06-09","2021-06-09","model_publication_date","ai_training_dataset_size",171600000000,"source-defined data units","{\"Country (of organization)\": \"Canada,Canada,United Kingdom\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"EMDR\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"SOTA improvement\", \"Organization\": \"Mila - Quebec AI (originally Montreal Institute for Learning Algorithms),McGill University,DeepMind\", \"Task\": \"Question answering\", \"field\": \"Training dataset size (total)\", \"notes\": \"At the time of publication there were about 4B words (5.3B tokens) on English Wikipedia: https://en.wikipedia.org/wiki/Wikipedia:Size_of_Wikipedia#Yearly_statistics\\nBookCorpus has about 1B words (1.3B tokens), C4 has about 156B tokens, and OpenWebText has about 9B tokens.\\n\\nFrom Table 6, it looks like all datasets were trained on for over one epoch.\\n\\nBERT: 1M steps, batches of 256, sequence length 256 = 65.5B tokens vs 6.6B in Wikipedia + BookCorpus\\nICT: 100k steps, batches of 4096, sequence length 256 = 104.9B tokens vs 5.3B in Wikipedia\\nT5: 1M steps, batches of 2048, sequence length 256 = 524.3B tokens vs 170.3B tokens in C4 + Wikipedia + OpenWebText\\n\\nTotal tokens: 171.6 billion\\n\\nSome tokens were probably seen more times than others, but overall this corresponds to 4.05 epochs on the pre-training data. \", \"reference_url\": \"https://arxiv.org/abs/2106.05346v2\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01b86913e1b616e7afaa","","2021-06-09","2021-06-09","model_publication_date","ai_training_hours",355,"hours","{\"Country (of organization)\": \"Canada,Canada,United Kingdom\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"EMDR\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"SOTA improvement\", \"Organization\": \"Mila - Quebec AI (originally Montreal Institute for Learning Algorithms),McGill University,DeepMind\", \"Task\": \"Question answering\", \"field\": \"Training time (hours)\", \"notes\": \"\\\"We run all of our experiments on a machine with 96 CPUs, 1.3TB physical memory, and 16 A100 GPUs [...] our experiments on NQ and TriviaQA took approximately 25 hours to complete, while experiments on WebQ took roughly 8 hours to complete. Before supervised training, we also perform a one-time unsupervised MSS pre-training for 82,000 steps that took roughly 1 week\\\"\\n\\nAdditionally, they pre-trained BERT, ICT, and T5 models, which took a combined 8.733e20 FLOPs. On 16 A100s at 0.3 utilization, that would have taken approximately 162 hours.\\n\\nSo total time for the largest experiment (NQ or TriviaQA) is around:\\n25 + 168 + 162 = 355\", \"reference_url\": \"https://arxiv.org/abs/2106.05346v2\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01b9128301d6a34ed6eb","","2022-05-05","2022-05-05","model_publication_date","ai_training_chip_hours",9600,"chip-hours","{\"Country (of organization)\": \"Germany,Germany\", \"Domain\": \"Image generation\", \"Frontier model\": \"\", \"Model\": \"StyleGAN-XL\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"\", \"Organization\": \"Max Planck Institute for Intelligent Systems,University of Tübingen\", \"Task\": \"Image generation\", \"field\": \"Training chip-hours\", \"notes\": \"\\\" to match the prior state-of-the-art performance of ADM [Dhariwal and Nichol 2021] at a resolution of 512^2 pixels,\\ntraining the models on a single NVIDIA Tesla V100 takes 400 days compared to the previously required 1914 V100-days.\\\"\\n\\n125000000000000 FLOP / GPU / sec * 9600 GPU-hours [for a 512 resolution SOTA result] * 3600 sec / hour * 0.3 [assumed utilization] = 1.296e+21 FLOP\", \"reference_url\": \"https://arxiv.org/abs/2202.00273\"}","Likely","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01b9128301d6a34ed6eb","","2022-05-05","2022-05-05","model_publication_date","ai_training_compute",1.296e+21,"FLOP","{\"Country (of organization)\": \"Germany,Germany\", \"Domain\": \"Image generation\", \"Frontier model\": \"\", \"Model\": \"StyleGAN-XL\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"\", \"Organization\": \"Max Planck Institute for Intelligent Systems,University of Tübingen\", \"Task\": \"Image generation\", \"field\": \"Training compute (FLOP)\", \"notes\": \"\\\" to match the prior state-of-the-art performance of ADM [Dhariwal and Nichol 2021] at a resolution of 512^2 pixels,\\ntraining the models on a single NVIDIA Tesla V100 takes 400 days compared to the previously required 1914 V100-days.\\\"\\n\\n125000000000000 FLOP / GPU / sec * 9600 GPU-hours [for a 512 resolution SOTA result] * 3600 sec / hour * 0.3 [assumed utilization] = 1.296e+21 FLOP\", \"reference_url\": \"https://arxiv.org/abs/2202.00273\"}","Likely","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01e67eafd3c0e6eb3058","","2022-03-28","2022-03-28","model_publication_date","ai_parameters",173900000000000,"parameters","{\"Country (of organization)\": \"China,China,China,China\", \"Domain\": \"Multimodal,Language,Vision\", \"Frontier model\": \"\", \"Model\": \"BaGuaLu\", \"Model accessibility\": \"\", \"Notability criteria\": \"\", \"Organization\": \"Tsinghua University,Zhejiang Lab,Beijing Academy of Artificial Intelligence / BAAI,Alibaba\", \"Task\": \"Language modeling,Image classification\", \"field\": \"Parameters\", \"notes\": \"Table 3, MoDa-174T has 173.9 trillion parameters\", \"reference_url\": \"https://dl.acm.org/doi/abs/10.1145/3503221.3508417\"}","confidence_not_given","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01ea488da3e556562a36","","2021-11-15","2021-11-15","model_publication_date","ai_parameters",18000000000,"parameters","{\"Country (of organization)\": \"South Korea\", \"Domain\": \"Language,Speech\", \"Frontier model\": \"\", \"Model\": \"A.X (Adot) 18B\", \"Model accessibility\": \"\", \"Notability criteria\": \"\", \"Organization\": \"SK Telecom\", \"Task\": \"Chat\", \"field\": \"Parameters\", \"notes\": \"\", \"reference_url\": \"https://aclanthology.org/2023.acl-industry.68.pdf\"}","Speculative","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01ea488da3e556562a36","","2021-11-15","2021-11-15","model_publication_date","ai_training_compute",1350000000000000000,"FLOP","{\"Country (of organization)\": \"South Korea\", \"Domain\": \"Language,Speech\", \"Frontier model\": \"\", \"Model\": \"A.X (Adot) 18B\", \"Model accessibility\": \"\", \"Notability criteria\": \"\", \"Organization\": \"SK Telecom\", \"Task\": \"Chat\", \"field\": \"Training compute (FLOP)\", \"notes\": \"=18000000000 parameters *12500000 training tokens * 6 FLOP/token/parameter=1350000000000000000=1.35 × 10^18 FLOP\\n\\nunsure about training dataset size->speculative confidence level\", \"reference_url\": \"https://aclanthology.org/2023.acl-industry.68.pdf\"}","Speculative","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","01ef2b7e034dcd9d36e2","","2018-09-18","2018-09-18","model_publication_date","ai_parameters",24000000,"parameters","{\"Country (of organization)\": \"China,China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"AWD-LSTM-MoS + dynamic evaluation (PTB, 2018)\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Peking University,Microsoft Research Asia\", \"Task\": \"Language modeling,Translation,Text classification\", \"field\": \"Parameters\", \"notes\": \"\", \"reference_url\": \"https://arxiv.org/abs/1809.06858\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","021ec8ef9adb22acc286","","2024-09-24","2024-09-24","model_publication_date","ai_training_dataset_size",165428,"source-defined data units","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Biology\", \"Frontier model\": \"\", \"Model\": \"Importance of higher-order epistasis in large protein sequence-function relationships\", \"Model accessibility\": \"\", \"Notability criteria\": \"\", \"Organization\": \"University of Florida\", \"Task\": \"Protein function prediction\", \"field\": \"Training dataset size (total)\", \"notes\": \"Initial calculations by dataset:\\n1. GRB-1: 129,320 * 0.8 * 33 = 3,414,048\\n2. GRB-3-abundance: 31,936 * 0.8 * 15 = 383,235\\n3. GRB-3-binding: 25,967 * 0.8 * 15 = 311,610\\n4. AAV2-Capsid: 42,328 * 0.8 * 28 = 948,136\\n5. CreiLOV: 165,428 * 0.8 * 15 = 1,985,130\\n6. cgreGFP: 26,165 * 0.8 * 234 = 4,898,088\\n7. ppluGFP: 32,260 * 0.8 * 221 = 5,703,568\\n8. His3-S2: 116,935 * 0.8 * 28 = 2,619,344\\n9. His3-S5: 92,408 * 0.8 * 31 = 2,291,706\\n10. His3-S12: 62,305 * 0.8 * 19 = 947,036\\n\\nFinal sum: 3,414,048 + 383,235 + 311,610 + 948,136 + 1,985,130 + 4,898,088 + 5,703,568 + 2,619,344 + 2,291,706 + 947,036 = 23,501,901\", \"reference_url\": \"https://pmc.ncbi.nlm.nih.gov/articles/PMC11463489/\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","021ec8ef9adb22acc286","","2024-09-24","2024-09-24","model_publication_date","ai_training_hardware_quantity",1,"devices","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Biology\", \"Frontier model\": \"\", \"Model\": \"Importance of higher-order epistasis in large protein sequence-function relationships\", \"Model accessibility\": \"\", \"Notability criteria\": \"\", \"Organization\": \"University of Florida\", \"Task\": \"Protein function prediction\", \"field\": \"Hardware quantity\", \"notes\": \"NVIDIA A100\", \"reference_url\": \"https://pmc.ncbi.nlm.nih.gov/articles/PMC11463489/\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","021ec8ef9adb22acc286","","2024-09-24","2024-09-24","model_publication_date","ai_training_power",432.9149024352496,"watts","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Biology\", \"Frontier model\": \"\", \"Model\": \"Importance of higher-order epistasis in large protein sequence-function relationships\", \"Model accessibility\": \"\", \"Notability criteria\": \"\", \"Organization\": \"University of Florida\", \"Task\": \"Protein function prediction\", \"field\": \"Training power draw (W)\", \"notes\": \"\", \"reference_url\": \"https://pmc.ncbi.nlm.nih.gov/articles/PMC11463489/\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","0233eb41c0ae9644e51c","","2024-09-26","2024-09-26","model_publication_date","ai_training_dataset_size",77940000,"source-defined data units","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Biology\", \"Frontier model\": \"\", \"Model\": \"Loop-Diffusion\", \"Model accessibility\": \"\", \"Notability criteria\": \"\", \"Organization\": \"University of Washington\", \"Task\": \"Protein design\", \"field\": \"Training dataset size (total)\", \"notes\": \"433,000 total datapoints = 433k atomic neighborhoods from 20,000 protein structures\\nFinal result: 4.33e5 datapoints\", \"reference_url\": \"https://arxiv.org/abs/2409.18201\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","025917ee72c442788022","","2023-02-20","2023-02-20","model_publication_date","ai_parameters",243000000.00000003,"parameters","{\"Country (of organization)\": \"United Kingdom,United Kingdom\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"E-SPA\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"\", \"Organization\": \"University of Oxford,DeepMind\", \"Task\": \"Language modeling/generation,Question answering\", \"field\": \"Parameters\", \"notes\": \"\", \"reference_url\": \"https://arxiv.org/abs/2302.10322\"}","confidence_not_given","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","0272c1219e37e58ba436","","2023-10-13","2023-10-13","model_publication_date","ai_parameters",55000000000,"parameters","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Robotics\", \"Frontier model\": \"\", \"Model\": \"RT-2-X\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"SOTA improvement\", \"Organization\": \"Google DeepMind\", \"Task\": \"Robotic manipulation\", \"field\": \"Parameters\", \"notes\": \"55B\", \"reference_url\": \"https://arxiv.org/abs/2310.08864\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","027a08f3dc052363ce15","","2022-12-22","2022-12-22","model_publication_date","ai_finetune_compute",2.1e+21,"FLOP","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"OPT-IML (175B)\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Meta AI\", \"Task\": \"Language modeling\", \"field\": \"Finetune compute (FLOP)\", \"notes\": \"\\\"We fine-tune all 30B models on 64 40GB A100s, and 175B models on 128 40GB A100s\\\", no timeframe specified\\n\\nfine-tuned on 2B tokens. 2B * 175B * 6 = 2.1e21\\n\\n\", \"reference_url\": \"https://arxiv.org/abs/2212.12017\"}","Likely","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","027a08f3dc052363ce15","","2022-12-22","2022-12-22","model_publication_date","ai_parameters",175000000000,"parameters","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"OPT-IML (175B)\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Meta AI\", \"Task\": \"Language modeling\", \"field\": \"Parameters\", \"notes\": \"\", \"reference_url\": \"https://arxiv.org/abs/2212.12017\"}","Likely","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","027a08f3dc052363ce15","","2022-12-22","2022-12-22","model_publication_date","ai_training_chip_hours",9216,"chip-hours","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"OPT-IML (175B)\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Meta AI\", \"Task\": \"Language modeling\", \"field\": \"Training chip-hours\", \"notes\": \"fine-tuned from OPT-175B (4.3e23) with an estimate 2.1e21 FLOP for fine-tuning. \\n\\\"During fine-tuning, our models saw approximately 2 billion tokens, which is only 0.6% of the pre-training budget of OPT\\\"\", \"reference_url\": \"https://arxiv.org/abs/2212.12017\"}","Likely","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","027a08f3dc052363ce15","","2022-12-22","2022-12-22","model_publication_date","ai_training_compute",4.3e+23,"FLOP","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"OPT-IML (175B)\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Meta AI\", \"Task\": \"Language modeling\", \"field\": \"Training compute (FLOP)\", \"notes\": \"fine-tuned from OPT-175B (4.3e23) with an estimate 2.1e21 FLOP for fine-tuning. \\n\\\"During fine-tuning, our models saw approximately 2 billion tokens, which is only 0.6% of the pre-training budget of OPT\\\"\", \"reference_url\": \"https://arxiv.org/abs/2212.12017\"}","Likely","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","027a08f3dc052363ce15","","2022-12-22","2022-12-22","model_publication_date","ai_training_dataset_size",2000000000,"source-defined data units","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"OPT-IML (175B)\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Meta AI\", \"Task\": \"Language modeling\", \"field\": \"Training dataset size (total)\", \"notes\": \"\\\"During fine-tuning, our models saw approximately 2 billion tokens, which is only 0.6% of the pre-training budget of OPT\\\"\", \"reference_url\": \"https://arxiv.org/abs/2212.12017\"}","Likely","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","027a08f3dc052363ce15","","2022-12-22","2022-12-22","model_publication_date","ai_training_hardware_quantity",128,"devices","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"OPT-IML (175B)\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Meta AI\", \"Task\": \"Language modeling\", \"field\": \"Hardware quantity\", \"notes\": \"NVIDIA A100\", \"reference_url\": \"https://arxiv.org/abs/2212.12017\"}","Likely","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","027a08f3dc052363ce15","","2022-12-22","2022-12-22","model_publication_date","ai_training_hours",72,"hours","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"OPT-IML (175B)\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Meta AI\", \"Task\": \"Language modeling\", \"field\": \"Training time (hours)\", \"notes\": \"Table 3\", \"reference_url\": \"https://arxiv.org/abs/2212.12017\"}","Likely","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","027a08f3dc052363ce15","","2022-12-22","2022-12-22","model_publication_date","ai_training_power",102304.08471008124,"watts","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"OPT-IML (175B)\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Meta AI\", \"Task\": \"Language modeling\", \"field\": \"Training power draw (W)\", \"notes\": \"fine-tuned from OPT-175B (4.3e23) with an estimate 2.1e21 FLOP for fine-tuning. \\n\\\"During fine-tuning, our models saw approximately 2 billion tokens, which is only 0.6% of the pre-training budget of OPT\\\"\", \"reference_url\": \"https://arxiv.org/abs/2212.12017\"}","Likely","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","02b3ef51bda45a9d45e8","","2023-10-30","2023-10-30","model_publication_date","ai_parameters",500000000,"parameters","{\"Country (of organization)\": \"China,China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Spec-Drafter\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Peking University,Microsoft Research Asia\", \"Task\": \"Translation\", \"field\": \"Parameters\", \"notes\": \"0.5B\\n12-layer encoder + 2-layer decoder, d=512/2048\", \"reference_url\": \"https://arxiv.org/abs/2203.16487\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","02b3ef51bda45a9d45e8","","2023-10-30","2023-10-30","model_publication_date","ai_training_compute",117964800000000000000,"FLOP","{\"Country (of organization)\": \"China,China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Spec-Drafter\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Peking University,Microsoft Research Asia\", \"Task\": \"Translation\", \"field\": \"Training compute (FLOP)\", \"notes\": \"6 FLOP/parameter/token * 500000000 parameters * 39321600000 tokens = 117964800000000000000 FLOP\", \"reference_url\": \"https://arxiv.org/abs/2203.16487\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","02b3ef51bda45a9d45e8","","2023-10-30","2023-10-30","model_publication_date","ai_training_dataset_size",39321600000,"source-defined data units","{\"Country (of organization)\": \"China,China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Spec-Drafter\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Peking University,Microsoft Research Asia\", \"Task\": \"Translation\", \"field\": \"Training dataset size (total)\", \"notes\": \"# max tokens 4096\\nupdate frequency 4\\n8 GPUs\\nmax updates 300K\\n\\n4096 * 4 * 8 * 300000 = 39321600000 tokens\", \"reference_url\": \"https://arxiv.org/abs/2203.16487\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","02b3ef51bda45a9d45e8","","2023-10-30","2023-10-30","model_publication_date","ai_training_hardware_quantity",8,"devices","{\"Country (of organization)\": \"China,China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Spec-Drafter\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Peking University,Microsoft Research Asia\", \"Task\": \"Translation\", \"field\": \"Hardware quantity\", \"notes\": \"NVIDIA V100\", \"reference_url\": \"https://arxiv.org/abs/2203.16487\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","02b3ef51bda45a9d45e8","","2023-10-30","2023-10-30","model_publication_date","ai_training_power",4762.3000835803105,"watts","{\"Country (of organization)\": \"China,China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Spec-Drafter\", \"Model accessibility\": \"Open weights (non-commercial)\", \"Notability criteria\": \"\", \"Organization\": \"Peking University,Microsoft Research Asia\", \"Task\": \"Translation\", \"field\": \"Training power draw (W)\", \"notes\": \"6 FLOP/parameter/token * 500000000 parameters * 39321600000 tokens = 117964800000000000000 FLOP\", \"reference_url\": \"https://arxiv.org/abs/2203.16487\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","02b7200ca27bec4c826d","","2026-04-23","2026-04-23","model_publication_date","ai_parameters",1020000000000,"parameters","{\"Country (of organization)\": \"China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"MiMo-V2.5-Pro\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"Discretionary\", \"Organization\": \"Xiaomi Corp\", \"Task\": \"Language modeling/generation,Question answering\", \"field\": \"Parameters\", \"notes\": \"\\\"MiMo-V2.5-Pro is a 1.02T-parameter Mixture-of-Experts model with 42B active parameters, \\\"\", \"reference_url\": \"https://mimo.xiaomi.com/mimo-v2-5-pro/\"}","Likely","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","02b7200ca27bec4c826d","","2026-04-23","2026-04-23","model_publication_date","ai_training_compute",6.804e+24,"FLOP","{\"Country (of organization)\": \"China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"MiMo-V2.5-Pro\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"Discretionary\", \"Organization\": \"Xiaomi Corp\", \"Task\": \"Language modeling/generation,Question answering\", \"field\": \"Training compute (FLOP)\", \"notes\": \"6 FLOPS/token/active parameter * 27e12 tokens * 42e9 active parameters = 6.804E24 FLOP\", \"reference_url\": \"https://mimo.xiaomi.com/mimo-v2-5-pro/\"}","Likely","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","02b7200ca27bec4c826d","","2026-04-23","2026-04-23","model_publication_date","ai_training_dataset_size",27000000000000,"source-defined data units","{\"Country (of organization)\": \"China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"MiMo-V2.5-Pro\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"Discretionary\", \"Organization\": \"Xiaomi Corp\", \"Task\": \"Language modeling/generation,Question answering\", \"field\": \"Training dataset size (total)\", \"notes\": \"\", \"reference_url\": \"https://mimo.xiaomi.com/mimo-v2-5-pro/\"}","Likely","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","02c958e4729ff2b4e663","","2026-08-19","2026-08-19","model_publication_date","ai_training_hours",5760,"hours","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Robotics,Language,Multimodal\", \"Frontier model\": \"\", \"Model\": \"GEN-1.5\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"Discretionary\", \"Organization\": \"Generalist\", \"Task\": \"Robotic manipulation,System control,Instruction interpretation\", \"field\": \"Training time (hours)\", \"notes\": \"\\\"GEN-1.5’s initial pretraining began in parallel — it has now been training continuously for over eight months. We left it running [...]\\\"\", \"reference_url\": \"https://generalistai.com/blog/gen-1.5\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","031f2a61a47844ae3cc8","","2024-03-27","2024-03-27","model_publication_date","ai_parameters",75000000000,"parameters","{\"Country (of organization)\": \"\", \"Domain\": \"\", \"Frontier model\": \"\", \"Model\": \"AInno-75B 2.0\", \"Model accessibility\": \"\", \"Notability criteria\": \"\", \"Organization\": \"\", \"Task\": \"\", \"field\": \"Parameters\", \"notes\": \"\", \"reference_url\": \"https://www.php.cn/faq/727953.html\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","032d9f5ed9178197a8a4","","2025-03-11","2025-03-11","model_publication_date","ai_parameters",7000000000,"parameters","{\"Country (of organization)\": \"Hong Kong,China\", \"Domain\": \"Speech\", \"Frontier model\": \"\", \"Model\": \"YuE\", \"Model accessibility\": \"\", \"Notability criteria\": \"\", \"Organization\": \"Hong Kong University of Science and Technology (HKUST),Multimodal Art Projection (MAP)\", \"Task\": \"Music generation,Text-to-speech (TTS)\", \"field\": \"Parameters\", \"notes\": \"7B\", \"reference_url\": \"https://arxiv.org/abs/2503.08638\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","032d9f5ed9178197a8a4","","2025-03-11","2025-03-11","model_publication_date","ai_training_hardware_quantity",512,"devices","{\"Country (of organization)\": \"Hong Kong,China\", \"Domain\": \"Speech\", \"Frontier model\": \"\", \"Model\": \"YuE\", \"Model accessibility\": \"\", \"Notability criteria\": \"\", \"Organization\": \"Hong Kong University of Science and Technology (HKUST),Multimodal Art Projection (MAP)\", \"Task\": \"Music generation,Text-to-speech (TTS)\", \"field\": \"Hardware quantity\", \"notes\": \"NVIDIA H800 SXM5\", \"reference_url\": \"https://arxiv.org/abs/2503.08638\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","032d9f5ed9178197a8a4","","2025-03-11","2025-03-11","model_publication_date","ai_training_power",703326.7370966855,"watts","{\"Country (of organization)\": \"Hong Kong,China\", \"Domain\": \"Speech\", \"Frontier model\": \"\", \"Model\": \"YuE\", \"Model accessibility\": \"\", \"Notability criteria\": \"\", \"Organization\": \"Hong Kong University of Science and Technology (HKUST),Multimodal Art Projection (MAP)\", \"Task\": \"Music generation,Text-to-speech (TTS)\", \"field\": \"Training power draw (W)\", \"notes\": \"\", \"reference_url\": \"https://arxiv.org/abs/2503.08638\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","035127e1ef910421dfa9","","2021-11-01","2021-11-01","model_publication_date","ai_training_compute",10500000000000000000,"FLOP","{\"Country (of organization)\": \"Germany\", \"Domain\": \"Image generation\", \"Frontier model\": \"\", \"Model\": \"Projected GAN\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"SOTA improvement\", \"Organization\": \"Heidelberg University\", \"Task\": \"Image generation\", \"field\": \"Training compute (FLOP)\", \"notes\": \"\\\"With this setting, each experiment takes roughly 100-200 GPU hours on a NVIDIA V100,\\nfor more details we refer to the appendix.\\\"\\n\\n\\\"We conduct our experiments on an internal cluster with several nodes, each with up to 8 Quadro RTX\\n6000 or NVIDIA V100 using PyTorch 1.7.1 and CUDA 11.0.\\\"\\n\\nIn appendix table 7, takes 10.1 seconds per 1k images on 8 Quadro RTX 6000s. Longest training run for Projected GAN appears to be in Figure 4 (left), at 14M images, though this is overtrained and the largest checkpoint used for evaluations was 10M.\\n10M images * 10.1 s/1000 images * 8 * 3.26e13 FLOP/s * 0.4 = 1.05e19\", \"reference_url\": \"https://proceedings.neurips.cc/paper/2021/hash/9219adc5c42107c4911e249155320648-Abstract.html\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","035127e1ef910421dfa9","","2021-11-01","2021-11-01","model_publication_date","ai_training_dataset_size",3000000,"source-defined data units","{\"Country (of organization)\": \"Germany\", \"Domain\": \"Image generation\", \"Frontier model\": \"\", \"Model\": \"Projected GAN\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"SOTA improvement\", \"Organization\": \"Heidelberg University\", \"Task\": \"Image generation\", \"field\": \"Training dataset size (total)\", \"notes\": \"They experiment with 22 image datasets. Largest appears to be LSUN-Bedroom at 3M images.\", \"reference_url\": \"https://proceedings.neurips.cc/paper/2021/hash/9219adc5c42107c4911e249155320648-Abstract.html\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","03513fbd038280b5dea5","","2024-04-15","2024-04-15","model_publication_date","ai_parameters",67000000000,"parameters","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Multimodal,Language,Vision,Video,Speech\", \"Frontier model\": \"\", \"Model\": \"Reka Core\", \"Model accessibility\": \"API access\", \"Notability criteria\": \"Training cost\", \"Organization\": \"Reka AI\", \"Task\": \"Chat,Language modeling/generation,Image captioning,Code generation,Code autocompletion,Question answering,Visual question answering,Video description,Speech recognition (ASR),Speech-to-text,Quantitative reasoning\", \"field\": \"Parameters\", \"notes\": \"\", \"reference_url\": \"https://arxiv.org/abs/2404.12387\"}","Speculative","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","03513fbd038280b5dea5","","2024-04-15","2024-04-15","model_publication_date","ai_training_compute",8.400010000000001e+24,"FLOP","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Multimodal,Language,Vision,Video,Speech\", \"Frontier model\": \"\", \"Model\": \"Reka Core\", \"Model accessibility\": \"API access\", \"Notability criteria\": \"Training cost\", \"Organization\": \"Reka AI\", \"Task\": \"Chat,Language modeling/generation,Image captioning,Code generation,Code autocompletion,Question answering,Visual question answering,Video description,Speech recognition (ASR),Speech-to-text,Quantitative reasoning\", \"field\": \"Training compute (FLOP)\", \"notes\": \"No direct information about Reka Core model (\\\"Reka Core has not finished training and is still improving.\\\")\\n\\nThe smaller dense model Reka Flash has 21B parameters and was trained on 5 trillion language tokens.\\n\\nThere is information about compute: \\\"Our setup comprises of clusters from a mixture of vendors with our peak compute being approximately\\n2.5K H100s and 2.5K A100s.\\\"\\n\\nIf we assume 2 months of training with 2.5k H100s and 2.5k A100s at utilization 0.5 we get 8.4e24 FLOP (2500*9.9e14+2500*3.12e14)*60*60*24*60*0.5.\", \"reference_url\": \"https://arxiv.org/abs/2404.12387\"}","Speculative","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","03513fbd038280b5dea5","","2024-04-15","2024-04-15","model_publication_date","ai_training_compute_lower_bound",6.3e+23,"FLOP","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Multimodal,Language,Vision,Video,Speech\", \"Frontier model\": \"\", \"Model\": \"Reka Core\", \"Model accessibility\": \"API access\", \"Notability criteria\": \"Training cost\", \"Organization\": \"Reka AI\", \"Task\": \"Chat,Language modeling/generation,Image captioning,Code generation,Code autocompletion,Question answering,Visual question answering,Video description,Speech recognition (ASR),Speech-to-text,Quantitative reasoning\", \"field\": \"Training compute lower bound\", \"notes\": \"No direct information about Reka Core model (\\\"Reka Core has not finished training and is still improving.\\\")\\n\\nThe smaller dense model Reka Flash has 21B parameters and was trained on 5 trillion language tokens.\\n\\nThere is information about compute: \\\"Our setup comprises of clusters from a mixture of vendors with our peak compute being approximately\\n2.5K H100s and 2.5K A100s.\\\"\\n\\nIf we assume 2 months of training with 2.5k H100s and 2.5k A100s at utilization 0.5 we get 8.4e24 FLOP (2500*9.9e14+2500*3.12e14)*60*60*24*60*0.5.\", \"reference_url\": \"https://arxiv.org/abs/2404.12387\"}","Speculative","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","03513fbd038280b5dea5","","2024-04-15","2024-04-15","model_publication_date","ai_training_compute_upper_bound",2.6563132345e+26,"FLOP","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Multimodal,Language,Vision,Video,Speech\", \"Frontier model\": \"\", \"Model\": \"Reka Core\", \"Model accessibility\": \"API access\", \"Notability criteria\": \"Training cost\", \"Organization\": \"Reka AI\", \"Task\": \"Chat,Language modeling/generation,Image captioning,Code generation,Code autocompletion,Question answering,Visual question answering,Video description,Speech recognition (ASR),Speech-to-text,Quantitative reasoning\", \"field\": \"Training compute upper bound\", \"notes\": \"No direct information about Reka Core model (\\\"Reka Core has not finished training and is still improving.\\\")\\n\\nThe smaller dense model Reka Flash has 21B parameters and was trained on 5 trillion language tokens.\\n\\nThere is information about compute: \\\"Our setup comprises of clusters from a mixture of vendors with our peak compute being approximately\\n2.5K H100s and 2.5K A100s.\\\"\\n\\nIf we assume 2 months of training with 2.5k H100s and 2.5k A100s at utilization 0.5 we get 8.4e24 FLOP (2500*9.9e14+2500*3.12e14)*60*60*24*60*0.5.\", \"reference_url\": \"https://arxiv.org/abs/2404.12387\"}","Speculative","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","035abcdace2803131aae","","2022-06-06","2022-06-06","model_publication_date","ai_parameters",5600000000,"parameters","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Multimodal,Vision,Language\", \"Frontier model\": \"\", \"Model\": \"LIMoE-H/14\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"\", \"Organization\": \"Google\", \"Task\": \"Image classification\", \"field\": \"Parameters\", \"notes\": \"Section 1: \\\"We scale this up to a large 5.6B parameter LIMoE-H/14\\\"\", \"reference_url\": \"https://arxiv.org/abs/2206.02770\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","035abcdace2803131aae","","2022-06-06","2022-06-06","model_publication_date","ai_training_compute",1.8e+22,"FLOP","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Multimodal,Vision,Language\", \"Frontier model\": \"\", \"Model\": \"LIMoE-H/14\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"\", \"Organization\": \"Google\", \"Task\": \"Image classification\", \"field\": \"Training compute (FLOP)\", \"notes\": \"Section 3.2: \\\"The model contains 5.6B parameters in total, but only applies 675M parameters per token\\\"\\n\\nFrom Section A.3, \\\"batch size 21502 with resolution 288 and text sequence length16\\\". \\\"The model was trained for 700k steps pre-cooldown. There was one cooldown of length 125k steps\\nfrom the final step, and 3 of length 40k steps starting from step 650k\\\". Patch size 14 for images.\\n\\nAssume C = 6*N*D. \\nC = 6*675e6*21.5e3*1e6*(16+(288/14)**2)/2 = 1.8e22\\n\\nThis is broadly consistent with ViT-H/14's compute\\n\\n\\nupd Figure 11(c) suggests 1.4*10^4*10^9 FLOPS = 1.4*10^13 FLOPS\", \"reference_url\": \"https://arxiv.org/abs/2206.02770\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","035abcdace2803131aae","","2022-06-06","2022-06-06","model_publication_date","ai_training_dataset_size",4575625612000,"source-defined data units","{\"Country (of organization)\": \"United States of America\", \"Domain\": \"Multimodal,Vision,Language\", \"Frontier model\": \"\", \"Model\": \"LIMoE-H/14\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"\", \"Organization\": \"Google\", \"Task\": \"Image classification\", \"field\": \"Training dataset size (total)\", \"notes\": \"\\\"By default, all models are trained on paired image-text data used in [16], consisting of 3.6B images and alt-texts scraped from the web. For large LIMoE-H/14 experiment, we also co-train with JFT-4B [17].\\\"\\n\\n\\\"The largest scale model is trained at batch size 21502, with resolution 288 and text sequence length 16.\\\"\\n\\n\\\"The model was trained for 700k steps pre-cooldown. There was one cooldown of length 125k steps from the final step, and 3 of length 40k steps starting from step 650k\\\"\\n\\n21502*288*700000 = 4.3348032e+12 image tokens\\n21502*16*700000 = 240822400000 text tokens\\n\\n240822400000 + 4.3348032e+12 = 4.5756256e+12 total tokens\", \"reference_url\": \"https://arxiv.org/abs/2206.02770\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","037628219290197bd4d8","","2025-09-01","2025-09-01","model_publication_date","ai_parameters",560000000000,"parameters","{\"Country (of organization)\": \"China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"LongCat-Flash\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"SOTA improvement\", \"Organization\": \"Meituan Inc\", \"Task\": \"Language modeling/generation,Question answering,Chat,Code generation,Quantitative reasoning,Instruction interpretation\", \"field\": \"Parameters\", \"notes\": \"\\\"560 billion total parameters, featuring an innovative Mixture-of-Experts (MoE) architecture. The model incorporates a dynamic computation mechanism that activates 18.6B∼31.3B parameters (averaging∼27B)\\\"\", \"reference_url\": \"https://arxiv.org/abs/2509.01322\"}","Likely","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","037628219290197bd4d8","","2025-09-01","2025-09-01","model_publication_date","ai_training_compute",3.726e+24,"FLOP","{\"Country (of organization)\": \"China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"LongCat-Flash\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"SOTA improvement\", \"Organization\": \"Meituan Inc\", \"Task\": \"Language modeling/generation,Question answering,Chat,Code generation,Quantitative reasoning,Instruction interpretation\", \"field\": \"Training compute (FLOP)\", \"notes\": \"6 FLOP/parameter/token * 27000000000 active parameters * 23000000000000 tokens [\\\"Likely\\\" confidence, see dataset size notes] = 3.726e+24 FLOP\", \"reference_url\": \"https://arxiv.org/abs/2509.01322\"}","Likely","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","037628219290197bd4d8","","2025-09-01","2025-09-01","model_publication_date","ai_training_dataset_size",23000000000000,"source-defined data units","{\"Country (of organization)\": \"China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"LongCat-Flash\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"SOTA improvement\", \"Organization\": \"Meituan Inc\", \"Task\": \"Language modeling/generation,Question answering,Chat,Code generation,Quantitative reasoning,Instruction interpretation\", \"field\": \"Training dataset size (total)\", \"notes\": \"\\\"(1) We train the model on approximately 20 trillion tokens with 8192 sequence length to establish a robust base model. \\n(2) Reasoning and coding capabilities are further enhanced using trillions of data. \\n(3) The context length is extended to 128k through training on long context corpora.\\\"\\n\\nWith \\\"Likely\\\" confidence we may assume theytrained on total of 23T tokens\", \"reference_url\": \"https://arxiv.org/abs/2509.01322\"}","Likely","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","037628219290197bd4d8","","2025-09-01","2025-09-01","model_publication_date","ai_training_hours",720,"hours","{\"Country (of organization)\": \"China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"LongCat-Flash\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"SOTA improvement\", \"Organization\": \"Meituan Inc\", \"Task\": \"Language modeling/generation,Question answering,Chat,Code generation,Quantitative reasoning,Instruction interpretation\", \"field\": \"Training time (hours)\", \"notes\": \"\\\"completing training within 30 days\\\" (=720 hours)\", \"reference_url\": \"https://arxiv.org/abs/2509.01322\"}","Likely","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","0379321947fe55ec3e2d","","2023-04-05","2023-04-05","model_publication_date","ai_parameters",7000000000,"parameters","{\"Country (of organization)\": \"China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"BELLE-7B-1M\", \"Model accessibility\": \"Open weights (unrestricted)\", \"Notability criteria\": \"\", \"Organization\": \"KE Holdings Inc. (“Beike”)\", \"Task\": \"Language modeling/generation\", \"field\": \"Parameters\", \"notes\": \"\", \"reference_url\": \"https://huggingface.co/BelleGroup/BELLE-7B-1M\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","038c5c7445900e268c7d","","2019-06-24","2019-06-24","model_publication_date","ai_parameters",160000000,"parameters","{\"Country (of organization)\": \"China,China,China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Tensorized Transformer (OBW)\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"SOTA improvement\", \"Organization\": \"Tianjin University,Microsoft Research Asia,Beijing Institute of Technology\", \"Task\": \"Language modeling,Translation\", \"field\": \"Parameters\", \"notes\": \"\", \"reference_url\": \"https://arxiv.org/abs/1906.09777\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","038c5c7445900e268c7d","","2019-06-24","2019-06-24","model_publication_date","ai_training_compute",23880000000000000000,"FLOP","{\"Country (of organization)\": \"China,China,China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Tensorized Transformer (OBW)\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"SOTA improvement\", \"Organization\": \"Tianjin University,Microsoft Research Asia,Beijing Institute of Technology\", \"Task\": \"Language modeling,Translation\", \"field\": \"Training compute (FLOP)\", \"notes\": \"FLOP calculation: 6*829250940*30*160000000=2.3882427e+19\", \"reference_url\": \"https://arxiv.org/abs/1906.09777\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"],["ai_model","038c5c7445900e268c7d","","2019-06-24","2019-06-24","model_publication_date","ai_training_hardware_quantity",2,"devices","{\"Country (of organization)\": \"China,China,China\", \"Domain\": \"Language\", \"Frontier model\": \"\", \"Model\": \"Tensorized Transformer (OBW)\", \"Model accessibility\": \"Unreleased\", \"Notability criteria\": \"SOTA improvement\", \"Organization\": \"Tianjin University,Microsoft Research Asia,Beijing Institute of Technology\", \"Task\": \"Language modeling,Translation\", \"field\": \"Hardware quantity\", \"notes\": \"NVIDIA P40\", \"reference_url\": \"https://arxiv.org/abs/1906.09777\"}","Confident","https://epoch.ai/data/ai-models-documentation","0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67"]],"units":{"ai_parameters":["parameters"],"ai_training_compute":["FLOP"],"ai_training_dataset_size":["source-defined data units"],"ai_training_chip_hours":["chip-hours"],"ai_training_hours":["hours"],"ai_finetune_compute":["FLOP"],"ai_training_hardware_quantity":["devices"],"ai_training_power":["watts"],"ai_training_cost":["2023 US dollars"],"ai_training_compute_lower_bound":["FLOP"],"ai_training_compute_upper_bound":["FLOP"],"ai_post_training_compute":["FLOP"]},"time_basis":["model_publication_date"],"source_url":"https://epoch.ai/data/ai-models-documentation","license":"Creative Commons Attribution; credit Epoch AI and dataset authors","notes":"Epoch AI и авторы набора: оценки с уровнем уверенности, а не полные отчёты компаний. Набор отобран и не является переписью всех моделей. Размер обучающего набора имеет разные единицы по доменам, их нельзя механически сравнивать. Страна организации не страна обучения. Событие датировано публикацией модели; личные авторы и аннотации не перепубликуются.","inputs":[{"url":"https://epoch.ai/data/all_ai_models.csv","resolved_url":"https://epoch.ai/data/all_ai_models.csv","retrieved_at":"2026-10-05T15:00:47.883449+00:00","sha256":"0606548dbc1d2db6b57b62459a12b2b92add3ce31acb671bab7b6208fc4ddf67","bytes":6826881,"content_type":"text/csv"}],"metadata_url":"https://macroscope.is/data/indicators/life/meta/epoch_ai_models.json","csv_files":["https://macroscope.is/data/indicators/life/datasets/epoch_ai_models_part1.csv","https://macroscope.is/data/indicators/life/datasets/epoch_ai_models_part2.csv"],"next_page":"https://macroscope.is/api/v1/datasets/epoch_ai_models?page=1&lang=ru"}