{
  "abstract": "Between April 1 and May 15, 2026, a group of 49 mathematicians compiled a dataset of research-level mathematics questions with known answers. Most of the work was done during the three-day workshop Benchmarks in Leipzig with 35 participants at the Max Planck Institute for Mathematics in the Sciences in Leipzig, Germany. We present the resulting collection of 100~questions. We evaluated these questions in three stages: a single attempt by five state-of-the-art LLMs and their predecessors, followed by a 20-runs-per-model evaluation with three of these models, and finally a 3-run attempt with two heavy-thinking models. After Stage 1, 41 questions remained completely unsolved; after Stage 2, this count dropped to 16; and we concluded Stage 3 with only 2 unsolved questions. This demonstrates that the mathematical reasoning capabilities of LLMs are becoming impressive. In September 2026, we added a fourth stage in which the next generation of models attempted all 100 questions once more, after which only 1 question remains unsolved.",
  "abstract_shown": true,
  "affiliations": [],
  "article_type": null,
  "arxiv": "2606.05818",
  "author_total": 48,
  "authors": [
    {
      "name": "Andrei Balakin"
    },
    {
      "name": "Miklós Bóna"
    },
    {
      "name": "Marie-Charlotte Brandenburg"
    },
    {
      "name": "Clara Briand"
    },
    {
      "name": "Veronica Calvo Cortes"
    },
    {
      "name": "Shelby Cox"
    },
    {
      "name": "Jesus A. De Loera"
    },
    {
      "name": "Danai Deligeorgaki"
    },
    {
      "name": "Hannah Friedman"
    },
    {
      "name": "Tim Gehrunger"
    },
    {
      "name": "Chiara Giardino"
    },
    {
      "name": "Stephen Griffeth"
    },
    {
      "name": "Baran Hashemi"
    },
    {
      "name": "Elena Hoster"
    },
    {
      "name": "Alexander Ivanov"
    },
    {
      "name": "Nupur Jain"
    },
    {
      "name": "Aryaman Jal"
    },
    {
      "name": "Leonie Kayser"
    },
    {
      "name": "Joris Koefler"
    },
    {
      "name": "Kevin Kühn"
    },
    {
      "name": "Mario Kummer"
    },
    {
      "name": "Felix Lotter"
    },
    {
      "name": "René Marczinzik"
    },
    {
      "name": "Victor S. Miller"
    },
    {
      "name": "Alejandro Morales"
    },
    {
      "name": "Greta Panova"
    },
    {
      "name": "Gianni Petrella"
    },
    {
      "name": "Nathan Pflueger"
    },
    {
      "name": "Lakshmi Ramesh"
    },
    {
      "name": "Nikolas Rieke"
    },
    {
      "name": "Carlos Rodriguez"
    },
    {
      "name": "Andrea Rosana"
    },
    {
      "name": "Flavio Salizzoni"
    },
    {
      "name": "Otto T. P. Schmidt"
    },
    {
      "name": "Sven Ulf Schmitz"
    },
    {
      "name": "Lina Maria Simbaqueba Marin"
    },
    {
      "name": "Luca Sodomaco"
    },
    {
      "name": "Christian Stump"
    },
    {
      "name": "Bernd Sturmfels"
    },
    {
      "name": "Alexander Taveira Blomenhofer"
    },
    {
      "name": "Simon Telen"
    },
    {
      "name": "Philipp Tuchel"
    },
    {
      "name": "Emil Verkama"
    },
    {
      "name": "Carl Felix Waller"
    },
    {
      "name": "Julian Weigert"
    },
    {
      "name": "Annette Werner"
    },
    {
      "name": "Nathan Williams"
    },
    {
      "name": "Claudius Zibrowius"
    }
  ],
  "bibtex": "https://research.draconitic.com/paper/9sqrryz2ncd7f.bib",
  "cited_by": 0,
  "doi": "10.48550/arxiv.2606.05818",
  "fulltext_url": null,
  "id": "9sqrryz2ncd7f",
  "issns": [],
  "landing_url": "https://arxiv.org/abs/2606.05818",
  "language": null,
  "licence": "LicenseRef-arXiv-nonexclusive-1.0",
  "licence_label": "arXiv licence",
  "licence_source": "article",
  "licence_url": "http://arxiv.org/licenses/nonexclusive-distrib/1.0/",
  "metadata_licence": "CC0-1.0",
  "native_subjects": [
    [
      "arxiv",
      "cs.AI"
    ],
    [
      "arxiv",
      "math.AG"
    ],
    [
      "arxiv",
      "math.CO"
    ],
    [
      "arxiv",
      "math.HO"
    ],
    [
      "arxiv",
      "math.RT"
    ],
    [
      "openalex",
      "1702|Topic Modeling"
    ]
  ],
  "openalex": "W7163676334",
  "pdf_url": "https://arxiv.org/pdf/2606.05818",
  "pmcid": null,
  "pmid": null,
  "published": "2026-10-05",
  "publisher": null,
  "ris": "https://research.draconitic.com/paper/9sqrryz2ncd7f.ris",
  "sources": [
    {
      "first_seen": "2026-10-30T12:27:10Z",
      "native_id": "2606.05818",
      "source": "arxiv",
      "url": "https://arxiv.org/abs/2606.05818"
    }
  ],
  "subjects": [
    {
      "field": "mathematics",
      "origin": "arxiv:math.AG",
      "subfield": "algebra-number-theory"
    },
    {
      "field": "mathematics",
      "origin": "arxiv:math.CO",
      "subfield": "combinatorics"
    }
  ],
  "title": "Benchmarks in Leipzig",
  "title_text": "Benchmarks in Leipzig",
  "type": "preprint",
  "updated_at": "2026-10-30T12:27:10Z",
  "url": "https://research.draconitic.com/paper/9sqrryz2ncd7f",
  "venue": "arXiv",
  "versions": []
}