@inproceedings{e4fbf0e759bc44a5acfee620ad50f670,
title = "RecShard: Statistical Feature-Based Memory Optimization for Industry-Scale Neural Recommendation",
abstract = "We propose RecShard, a fine-grained embedding table (EMB) partitioning and placement technique for deep learning recommendation models (DLRMs). RecShard is designed based on two key observations. First, not all EMBs are equal, nor all rows within an EMB are equal in terms of access patterns. EMBs exhibit distinct memory characteristics, providing performance optimization opportunities for intelligent EMB partitioning and placement across a tiered memory hierarchy. Second, in modern DLRMs, EMBs function as hash tables. As a result, EMBs display interesting phenomena, such as the birthday paradox, leaving EMBs severely under-utilized. RecShard determines an optimal EMB sharding strategy for a set of EMBs based on training data distributions and model characteristics, along with the bandwidth characteristics of the underlying tiered memory hierarchy. In doing so, RecShard achieves over 6 times higher EMB training throughput on average for capacity constrained DLRMs. The throughput increase comes from improved EMB load balance by over 12 times and from the reduced access to the slower memory by over 87 times.",
keywords = "AI training systems, Deep learning recommendation models, Memory optimization, Neural networks",
author = "Geet Sethi and Bilge Acun and Niket Agarwal and Christos Kozyrakis and Caroline Trippel and Wu, {Carole Jean}",
note = "Funding Information: We would like to thank Jade Nie, Jianyu Huang, Jongsoo Park, Andrew Tulloch, Xing Liu, Benny Chen, Ying Liu, Liu Ke, Udit Gupta, Newsha Ardalani, Hsien-Hsin S. Lee, and Kim Hazelwood at Meta for their valuable feedback and various discussions on this work, as well as Fan Yang and the anonymous reviewers for their constructive feedback. This work was supported in part by the Stanford Platform Lab and its affiliates for Geet Sethi and Christos Kozyrakis. Publisher Copyright: {\textcopyright} 2022 ACM.; 27th ACM International Conference on Architectural Support for Programming Languages and Operating Systems, ASPLOS 2022 ; Conference date: 28-02-2022 Through 04-03-2022",
year = "2022",
month = feb,
day = "28",
doi = "10.1145/3503222.3507777",
language = "English (US)",
series = "International Conference on Architectural Support for Programming Languages and Operating Systems - ASPLOS",
publisher = "Association for Computing Machinery",
pages = "344--358",
editor = "Babak Falsafi and Michael Ferdman and Shan Lu and Wenisch, {Thomas F.}",
booktitle = "ASPLOS 2022 - Proceedings of the 27th ACM International Conference on Architectural Support for Programming Languages and Operating Systems",
}