@article{Sakaguchi_Le Bras_Bhagavatula_Choi_2020, title={WinoGrande: An Adversarial Winograd Schema Challenge at Scale}, volume={34}, url={https://ojs.aaai.org/index.php/AAAI/article/view/6399}, DOI={10.1609/aaai.v34i05.6399}, abstractNote={<p>The Winograd Schema Challenge (WSC) (Levesque, Davis, and Morgenstern 2011), a benchmark for commonsense reasoning, is a set of 273 expert-crafted pronoun resolution problems originally designed to be unsolvable for statistical models that rely on selectional preferences or word associations. However, recent advances in neural language models have already reached around 90% accuracy on variants of WSC. This raises an important question whether these models have truly acquired robust commonsense capabilities or whether they rely on spurious biases in the datasets that lead to an overestimation of the true capabilities of machine commonsense.</p><p>To investigate this question, we introduce <strong>W<span style="font-variant: small-caps;">ino</span>G<span style="font-variant: small-caps;">rande</span></strong>, a large-scale dataset of 44k problems, inspired by the original WSC design, but adjusted to improve both the scale and the hardness of the dataset. The key steps of the dataset construction consist of (1) a carefully designed crowdsourcing procedure, followed by (2) systematic bias reduction using a novel A<span style="font-variant: small-caps;">f</span>L<span style="font-variant: small-caps;">ite</span> algorithm that generalizes human-detectable <em>word associations</em> to machine-detectable <em>embedding associations</em>. The best state-of-the-art methods on W<span style="font-variant: small-caps;">ino</span>G<span style="font-variant: small-caps;">rande</span> achieve 59.4 – 79.1%, which are ∼15-35% (absolute) below human performance of 94.0%, depending on the amount of the training data allowed (2% – 100% respectively).</p><p>Furthermore, we establish new state-of-the-art results on <em>five</em> related benchmarks — WSC (→ <strong>90.1%</strong>), DPR (→ <strong>93.1%</strong>), COPA(→ <strong>90.6%</strong>), KnowRef (→ <strong>85.6%</strong>), and Winogender (→ <strong>97.1%</strong>). These results have dual implications: on one hand, they demonstrate the effectiveness of W<span style="font-variant: small-caps;">ino</span>G<span style="font-variant: small-caps;">rande</span> when used as a resource for transfer learning. On the other hand, they raise a concern that we are likely to be overestimating the true capabilities of machine commonsense across all these benchmarks. We emphasize the importance of algorithmic bias reduction in existing and future benchmarks to mitigate such overestimation.</p>}, number={05}, journal={Proceedings of the AAAI Conference on Artificial Intelligence}, author={Sakaguchi, Keisuke and Le Bras, Ronan and Bhagavatula, Chandra and Choi, Yejin}, year={2020}, month={Apr.}, pages={8732-8740} }