{
  "abstract": "Spatial transcriptomics maps gene expression across tissue sections and, when integrated with whole slide images, jointly captures both morphological architecture and molecular activity to provide a rich, multimodal view of tissue biology. ML approaches combining spatial transcriptomics and WSIs typically learn a mapping from morphological features to molecular profiles via contrastive alignment or supervised fine-tuning of pathology foundation models. In both cases, transcriptomics serves as a training signal rather than a true input modality, leaving inference fundamentally image-driven. We propose an early-fusion vision transformer architecture in which transcript tokens are constructed at arbitrary spatial resolution, merged with image patch tokens, and jointly processed through a shared transformer stack for deep cross-modal interaction. Starting from a pretrained vision-only pathology foundation model, we train HistoTx by continuing self-supervised pretraining on paired image and spatial transcriptomics data and demonstrate that: 1) at inference with image only, performance remains on par with the baseline on most tasks while improving on tasks that inherently benefit from molecular knowledge, such as gene expression prediction; 2) when both modalities are available at inference, jointly providing image and transcript inputs outperforms either modality alone.",
  "abstract_shown": true,
  "affiliations": [],
  "article_type": null,
  "arxiv": "2611.02681",
  "author_total": 14,
  "authors": [
    {
      "name": "Alice Driessen"
    },
    {
      "name": "Emil Malmsten"
    },
    {
      "name": "Steffen Wolf"
    },
    {
      "name": "Bernd Illing"
    },
    {
      "name": "Isinsu Katircioglu"
    },
    {
      "name": "Adriano Martinelli"
    },
    {
      "name": "Rolf Stahel"
    },
    {
      "name": "Solange Peters"
    },
    {
      "name": "Enriqueta Felip"
    },
    {
      "name": "Sanjay Popat"
    },
    {
      "name": "Krisztian Homicsko"
    },
    {
      "name": "Raphael Gottardo"
    },
    {
      "name": "Marianna Rapsomaniki"
    },
    {
      "name": "Fei Tang"
    }
  ],
  "bibtex": "https://research.draconitic.com/paper/ekg29ggcfj4sr.bib",
  "cited_by": 0,
  "doi": "10.48550/arxiv.2611.02681",
  "fulltext_url": null,
  "id": "ekg29ggcfj4sr",
  "issns": [],
  "landing_url": "https://arxiv.org/abs/2611.02681",
  "language": null,
  "licence": "CC-BY-4.0",
  "licence_label": "CC BY 4.0",
  "licence_source": "article",
  "licence_url": "http://creativecommons.org/licenses/by/4.0/",
  "metadata_licence": "CC0-1.0",
  "native_subjects": [
    [
      "arxiv",
      "cs.AI"
    ],
    [
      "arxiv",
      "cs.LG"
    ],
    [
      "arxiv",
      "q-bio.GN"
    ],
    [
      "arxiv",
      "q-bio.QM"
    ]
  ],
  "openalex": null,
  "pdf_url": "https://arxiv.org/pdf/2611.02681",
  "pmcid": null,
  "pmid": null,
  "published": "2026-11-02",
  "publisher": null,
  "ris": "https://research.draconitic.com/paper/ekg29ggcfj4sr.ris",
  "sources": [
    {
      "first_seen": "2026-11-03T12:21:15Z",
      "native_id": "2611.02681",
      "source": "arxiv",
      "url": "https://arxiv.org/abs/2611.02681"
    }
  ],
  "subjects": [
    {
      "field": "biology",
      "origin": "arxiv:q-bio.QM",
      "subfield": "bioinformatics"
    },
    {
      "field": "biology",
      "origin": "arxiv:q-bio.GN",
      "subfield": "genetics-genomics"
    }
  ],
  "title": "HistoTx: Early fusion of H&E images and spatial transcriptomics at varying spatial transcriptomics resolution with self-supervised learning",
  "title_text": "HistoTx: Early fusion of H&E images and spatial transcriptomics at varying spatial transcriptomics resolution with self-supervised learning",
  "type": "preprint",
  "updated_at": "2026-11-05T15:20:29Z",
  "url": "https://research.draconitic.com/paper/ekg29ggcfj4sr",
  "venue": "arXiv",
  "versions": []
}