@inbook{da2cb5efa4e8474dadf4f82e94590fcc,
title = "Unified Brain MR-Ultrasound Synthesis Using Multi-modal Hierarchical Representations",
abstract = "We introduce MHVAE, a deep hierarchical variational auto-encoder (VAE) that synthesizes missing images from various modalities. Extending multi-modal VAEs with a hierarchical latent structure, we introduce a probabilistic formulation for fusing multi-modal images in a common latent representation while having the flexibility to handle incomplete image sets as input. Moreover, adversarial learning is employed to generate sharper images. Extensive experiments are performed on the challenging problem of joint intra-operative ultrasound (iUS) and Magnetic Resonance (MR) synthesis. Our model outperformed multi-modal VAEs, conditional GANs, and the current state-of-the-art unified method (ResViT) for synthesizing missing images, demonstrating the advantage of using a hierarchical latent representation and a principled probabilistic fusion operation. Our code is publicly available (https://github.com/ReubenDo/MHVAE ).",
keywords = "Brain Resection, Image Synthesis, Ultrasound, Variational Auto-Encoder",
author = "Reuben Dorent and Nazim Haouchine and Fryderyk Kogl and Samuel Joutard and Parikshit Juvekar and Erickson Torio and Golby, {Alexandra J.} and Sebastien Ourselin and Sarah Frisken and Tom Vercauteren and Tina Kapur and {Wells III}, {William M.}",
note = "Funding Information: Acknowledgement. This work was supported by the National Institutes of Health (R01EB032387, R01EB027134, P41EB028741, R03EB032050), the McMahon Family Brain Tumor Research Fund and by core funding from the Wellcome/EPSRC [WT203148/Z/16/Z; NS/A000049/1]. For the purpose of open access, the authors have applied a CC BY public copyright licence to any Author Accepted Manuscript version arising from this submission. Publisher Copyright: {\textcopyright} The Author(s), under exclusive license to Springer Nature Switzerland AG 2023.; 26th International Conference on Medical Image Computing and Computer-Assisted Intervention, MICCAI 2023 ; Conference date: 08-10-2023 Through 12-10-2023",
year = "2023",
month = oct,
day = "2",
doi = "10.1007/978-3-031-43999-5_43",
language = "English",
isbn = "9783031439988",
series = "Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)",
publisher = "Springer Nature",
pages = "448--458",
editor = "Hayit Greenspan and Anant Madabhushi and Parvin Mousavi and Septimiu Salcudean and James Duncan and Tanveer Syeda-Mahmood and Russell Taylor",
booktitle = "Medical Image Computing and Computer Assisted Intervention – MICCAI 2023",
}