{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2021:CWMWO2EYH7EEUUEYIPJQFD2BZN","short_pith_number":"pith:CWMWO2EY","schema_version":"1.0","canonical_sha256":"15996768983fc84a509843d3028f41cb42aea8ecd4db6eb86e53ba32b0297976","source":{"kind":"arxiv","id":"2109.10686","version":2},"attestation_state":"computed","paper":{"title":"Scale Efficiently: Insights from Pre-training and Fine-tuning Transformers","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CV","cs.LG"],"primary_cat":"cs.CL","authors_text":"Ashish Vaswani, Dani Yogatama, Donald Metzler, Hyung Won Chung, Jinfeng Rao, Mostafa Dehghani, Samira Abnar, Sharan Narang, William Fedus, Yi Tay","submitted_at":"2021-09-22T12:29:15Z","abstract_excerpt":"There remain many open questions pertaining to the scaling behaviour of Transformer architectures. These scaling decisions and findings can be critical, as training runs often come with an associated computational cost which have both financial and/or environmental impact. The goal of this paper is to present scaling insights from pretraining and finetuning Transformers. While Kaplan et al. presents a comprehensive study of the scaling behaviour of Transformer language models, the scope is only on the upstream (pretraining) loss. Therefore, it is still unclear if these set of findings transfer"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2109.10686","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-09-22T12:29:15Z","cross_cats_sorted":["cs.AI","cs.CV","cs.LG"],"title_canon_sha256":"b3c634bcab5b138f5a832f79fd5d2d95995b2811adebb6760fce5c3f24f41f21","abstract_canon_sha256":"31e2c6bf265c360d8f2f99c01899a2c260f6d62585614230f3188d2a27ffd71d"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T03:52:31.180789Z","signature_b64":"qQ1WzH2BtYAPDieRFgocsuxWKCH1rxh8D5GRfdJ08L314YuMnmkiK2WVN/iQ3UrRRdBxZT/KtADrNgF1oZa6Bw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"15996768983fc84a509843d3028f41cb42aea8ecd4db6eb86e53ba32b0297976","last_reissued_at":"2026-07-05T03:52:31.180367Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T03:52:31.180367Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Scale Efficiently: Insights from Pre-training and Fine-tuning Transformers","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CV","cs.LG"],"primary_cat":"cs.CL","authors_text":"Ashish Vaswani, Dani Yogatama, Donald Metzler, Hyung Won Chung, Jinfeng Rao, Mostafa Dehghani, Samira Abnar, Sharan Narang, William Fedus, Yi Tay","submitted_at":"2021-09-22T12:29:15Z","abstract_excerpt":"There remain many open questions pertaining to the scaling behaviour of Transformer architectures. These scaling decisions and findings can be critical, as training runs often come with an associated computational cost which have both financial and/or environmental impact. The goal of this paper is to present scaling insights from pretraining and finetuning Transformers. While Kaplan et al. presents a comprehensive study of the scaling behaviour of Transformer language models, the scope is only on the upstream (pretraining) loss. Therefore, it is still unclear if these set of findings transfer"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2109.10686","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2109.10686/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2109.10686","created_at":"2026-07-05T03:52:31.180428+00:00"},{"alias_kind":"arxiv_version","alias_value":"2109.10686v2","created_at":"2026-07-05T03:52:31.180428+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2109.10686","created_at":"2026-07-05T03:52:31.180428+00:00"},{"alias_kind":"pith_short_12","alias_value":"CWMWO2EYH7EE","created_at":"2026-07-05T03:52:31.180428+00:00"},{"alias_kind":"pith_short_16","alias_value":"CWMWO2EYH7EEUUEY","created_at":"2026-07-05T03:52:31.180428+00:00"},{"alias_kind":"pith_short_8","alias_value":"CWMWO2EY","created_at":"2026-07-05T03:52:31.180428+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":7,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.18246","citing_title":"Variable-Width Transformers","ref_index":39,"is_internal_anchor":false},{"citing_arxiv_id":"2510.18245","citing_title":"Scaling Laws Meet Model Architecture: Toward Inference-Efficient LLMs","ref_index":39,"is_internal_anchor":false},{"citing_arxiv_id":"2305.16264","citing_title":"Scaling Data-Constrained Language Models","ref_index":113,"is_internal_anchor":false},{"citing_arxiv_id":"2303.17564","citing_title":"BloombergGPT: A Large Language Model for Finance","ref_index":114,"is_internal_anchor":false},{"citing_arxiv_id":"2604.02320","citing_title":"Large-scale Codec Avatars: The Unreasonable Effectiveness of Large-scale Avatar Pretraining","ref_index":59,"is_internal_anchor":false},{"citing_arxiv_id":"2403.07815","citing_title":"Chronos: Learning the Language of Time Series","ref_index":83,"is_internal_anchor":false},{"citing_arxiv_id":"2202.08906","citing_title":"ST-MoE: Designing Stable and Transferable Sparse Expert Models","ref_index":203,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/CWMWO2EYH7EEUUEYIPJQFD2BZN","json":"https://pith.science/pith/CWMWO2EYH7EEUUEYIPJQFD2BZN.json","graph_json":"https://pith.science/api/pith-number/CWMWO2EYH7EEUUEYIPJQFD2BZN/graph.json","events_json":"https://pith.science/api/pith-number/CWMWO2EYH7EEUUEYIPJQFD2BZN/events.json","paper":"https://pith.science/paper/CWMWO2EY"},"agent_actions":{"view_html":"https://pith.science/pith/CWMWO2EYH7EEUUEYIPJQFD2BZN","download_json":"https://pith.science/pith/CWMWO2EYH7EEUUEYIPJQFD2BZN.json","view_paper":"https://pith.science/paper/CWMWO2EY","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2109.10686&json=true","fetch_graph":"https://pith.science/api/pith-number/CWMWO2EYH7EEUUEYIPJQFD2BZN/graph.json","fetch_events":"https://pith.science/api/pith-number/CWMWO2EYH7EEUUEYIPJQFD2BZN/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/CWMWO2EYH7EEUUEYIPJQFD2BZN/action/timestamp_anchor","attest_storage":"https://pith.science/pith/CWMWO2EYH7EEUUEYIPJQFD2BZN/action/storage_attestation","attest_author":"https://pith.science/pith/CWMWO2EYH7EEUUEYIPJQFD2BZN/action/author_attestation","sign_citation":"https://pith.science/pith/CWMWO2EYH7EEUUEYIPJQFD2BZN/action/citation_signature","submit_replication":"https://pith.science/pith/CWMWO2EYH7EEUUEYIPJQFD2BZN/action/replication_record"}},"created_at":"2026-07-05T03:52:31.180428+00:00","updated_at":"2026-07-05T03:52:31.180428+00:00"}