Support gallery messages (#6519)

* Support gallery messages * Address review * Remove unused function * Fix indent * Add "Gallery" message prefix * Support galleries for image editing * Fix tapping on a tile opens the last item * Fix overflow count * Add caption editing to gallery messages * Use the new gallery string for prefix * Lock sending galleries behind a feature flag * Fix detekts * Fix * Ensure image edition is saved when navigating * Fix sending media broken on edited gallery. * Fix tests * Order imports * Remove unused parameters. * Fix copyright header of new files. * Fix Konsist test * Extract new previews to a dedicated file. * Sync strings * Remove unused import * Update screenshots * Trigger CI * Remove parameters with default value. * More cleanup * Restore sendAsFile behavior. * Improve Preview. * Improve Preview. * Improve Preview. * Fix gallery sending cancel and retry issue * Ensure any previous job is cancelled. * Fix issue in summary message * Gallery feature is disabled by default. * Kotlin convention * Remove useless parenthesis * Update screenshots * Fix test * List -> ImmutableList * Remove useless code. * Render formatted caption for attachment list. * Replace set of Booleans by an enum * Remove unused model for individual caption in a gallery Event. * Fix tests * Fix tests * Rework MediaViewer entry point. And ensure that the clicked image from the gallery is displayed first. * Ensure gallery item can be click in the pinned message list Improve the gallery item click handling code. * Improve code and fix separator color Closes #7101 * React on attachment item click Improve code * Improve code and support 0 items in gallery. * Fix click on attachment item not rendering anything. --------- Co-authored-by: Benoit Marty <benoitm@element.io> Co-authored-by: Benoit Marty <benoit@matrix.org> Co-authored-by: ElementBot <android@element.io>
2026-06-26 17:00:56 +03:00
parent 9e7ae34728
commit 87aff40b26
180 changed files with 3470 additions and 637 deletions
@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2026 Element Creations Ltd.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
+ * Please see LICENSE files in the repository root for full details.
+ */
+
+package io.element.android.libraries.matrix.impl.media
+
+import io.element.android.libraries.matrix.api.timeline.item.event.FormattedBody
+import org.matrix.rustcomponents.sdk.FormattedBody as RustFormattedBody
+import org.matrix.rustcomponents.sdk.MessageFormat as RustMessageFormat
+
+fun FormattedBody.map(): RustFormattedBody = RustFormattedBody(
+    format = format.map(),
+    body = body,
+)
+
+private fun io.element.android.libraries.matrix.api.timeline.item.event.MessageFormat.map(): RustMessageFormat {
+    return when (this) {
+        io.element.android.libraries.matrix.api.timeline.item.event.MessageFormat.HTML -> RustMessageFormat.Html
+        io.element.android.libraries.matrix.api.timeline.item.event.MessageFormat.UNKNOWN -> RustMessageFormat.Unknown("")
+    }
+}
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2026 Element Creations Ltd.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
+ * Please see LICENSE files in the repository root for full details.
+ */
+
+package io.element.android.libraries.matrix.impl.media
+
+import io.element.android.libraries.matrix.api.media.GalleryItemInfo
+import org.matrix.rustcomponents.sdk.GalleryItemInfo as RustGalleryItemInfo
+import org.matrix.rustcomponents.sdk.UploadSource as RustUploadSource
+
+fun GalleryItemInfo.map(): RustGalleryItemInfo = when (this) {
+    is GalleryItemInfo.Image -> {
+        RustGalleryItemInfo.Image(
+            imageInfo = imageInfo.map(),
+            source = RustUploadSource.File(file.path),
+            caption = null,
+            formattedCaption = null,
+            thumbnailSource = thumbnailFile?.path?.let(RustUploadSource::File),
+        )
+    }
+    is GalleryItemInfo.Video -> {
+        RustGalleryItemInfo.Video(
+            videoInfo = videoInfo.map(),
+            source = RustUploadSource.File(file.path),
+            caption = null,
+            formattedCaption = null,
+            thumbnailSource = thumbnailFile?.path?.let(RustUploadSource::File),
+        )
+    }
+    is GalleryItemInfo.Audio -> {
+        RustGalleryItemInfo.Audio(
+            audioInfo = audioInfo.map(),
+            source = RustUploadSource.File(file.path),
+            caption = null,
+            formattedCaption = null,
+        )
+    }
+    is GalleryItemInfo.MediaFile -> {
+        RustGalleryItemInfo.File(
+            fileInfo = fileInfo.map(),
+            source = RustUploadSource.File(file.path),
+            caption = null,
+            formattedCaption = null,
+        )
+    }
+}
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2026 Element Creations Ltd.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
+ * Please see LICENSE files in the repository root for full details.
+ */
+
+package io.element.android.libraries.matrix.impl.media
+
+import io.element.android.libraries.androidutils.file.safeDelete
+import io.element.android.libraries.core.extensions.runCatchingExceptions
+import io.element.android.libraries.matrix.api.media.MediaUploadHandler
+import org.matrix.rustcomponents.sdk.SendGalleryJoinHandle
+import java.io.File
+
+class GalleryMediaUploadHandlerImpl(
+    private val filesToUpload: List<File>,
+    private val sendGalleryJoinHandle: SendGalleryJoinHandle,
+) : MediaUploadHandler {
+    override suspend fun await(): Result<Unit> =
+        runCatchingExceptions {
+            sendGalleryJoinHandle.join()
+        }
+            .also { cleanUpFiles() }
+
+    override fun cancel() {
+        sendGalleryJoinHandle.cancel()
+        cleanUpFiles()
+    }
+
+    private fun cleanUpFiles() {
+        filesToUpload.forEach { file -> file.safeDelete() }
+    }
+}
@@ -97,7 +97,10 @@ private fun MessageLikeEventContent.toContent(senderId: UserId): NotificationCon
            is MessageLikeEventContent.ReactionContent -> NotificationContent.MessageLike.ReactionContent(relatedEventId)
            MessageLikeEventContent.RoomEncrypted -> NotificationContent.MessageLike.RoomEncrypted
            is MessageLikeEventContent.RoomMessage -> {
-                NotificationContent.MessageLike.RoomMessage(senderId, EventMessageMapper().mapMessageType(messageType))
+                NotificationContent.MessageLike.RoomMessage(
+                    senderId,
+                    EventMessageMapper().mapMessageType(messageType)
+                )
            }
            is MessageLikeEventContent.RoomRedaction -> NotificationContent.MessageLike.RoomRedaction(
                redactedEventId = redactedEventId?.let(::EventId),
@@ -219,6 +219,7 @@ class JoinedRustRoom(
                    RoomMessageEventMessageType.IMAGE,
                    RoomMessageEventMessageType.VIDEO,
                    RoomMessageEventMessageType.AUDIO,
+                    RoomMessageEventMessageType.GALLERY,
                )
            )
            is CreateTimelineParams.Focused,
@@ -15,6 +15,7 @@ import io.element.android.libraries.matrix.api.core.RoomId
 import io.element.android.libraries.matrix.api.core.UserId
 import io.element.android.libraries.matrix.api.media.AudioInfo
 import io.element.android.libraries.matrix.api.media.FileInfo
+import io.element.android.libraries.matrix.api.media.GalleryItemInfo
 import io.element.android.libraries.matrix.api.media.ImageInfo
 import io.element.android.libraries.matrix.api.media.MediaUploadHandler
 import io.element.android.libraries.matrix.api.media.VideoInfo
@@ -30,6 +31,7 @@ import io.element.android.libraries.matrix.api.timeline.Timeline
 import io.element.android.libraries.matrix.api.timeline.TimelineException
 import io.element.android.libraries.matrix.api.timeline.item.event.EventOrTransactionId
 import io.element.android.libraries.matrix.api.timeline.item.event.InReplyTo
+import io.element.android.libraries.matrix.impl.media.GalleryMediaUploadHandlerImpl
 import io.element.android.libraries.matrix.impl.media.MediaUploadHandlerImpl
 import io.element.android.libraries.matrix.impl.media.map
 import io.element.android.libraries.matrix.impl.poll.toInner
@@ -67,6 +69,7 @@ import kotlinx.coroutines.launch
 import kotlinx.coroutines.withContext
 import org.matrix.rustcomponents.sdk.EditedContent
 import org.matrix.rustcomponents.sdk.FormattedBody
+import org.matrix.rustcomponents.sdk.GalleryUploadParameters
 import org.matrix.rustcomponents.sdk.MessageFormat
 import org.matrix.rustcomponents.sdk.PollData
 import org.matrix.rustcomponents.sdk.SendAttachmentJoinHandle
@@ -535,6 +538,42 @@ class RustTimeline(
        }
    }

+    override suspend fun sendGallery(
+        items: List<GalleryItemInfo>,
+        caption: String?,
+        formattedCaption: String?,
+        inReplyToEventId: EventId?,
+    ): Result<MediaUploadHandler> {
+        Timber.d("Sending gallery with ${items.size} items")
+        val allFiles = items.flatMap { item ->
+            when (item) {
+                is GalleryItemInfo.Image -> listOfNotNull(item.file, item.thumbnailFile)
+                is GalleryItemInfo.Video -> listOfNotNull(item.file, item.thumbnailFile)
+                is GalleryItemInfo.Audio -> listOf(item.file)
+                is GalleryItemInfo.MediaFile -> listOf(item.file)
+            }
+        }
+        return sendGalleryAttachment(allFiles) {
+            inner.sendGallery(
+                params = GalleryUploadParameters(
+                    caption = caption,
+                    formattedCaption = formattedCaption?.let {
+                        FormattedBody(body = it, format = MessageFormat.Html)
+                    },
+                    mentions = null,
+                    inReplyTo = inReplyToEventId?.value,
+                ),
+                itemInfos = items.map { it.map() },
+            )
+        }
+    }
+
+    private fun sendGalleryAttachment(files: List<File>, handle: () -> org.matrix.rustcomponents.sdk.SendGalleryJoinHandle): Result<MediaUploadHandler> {
+        return runCatchingExceptions {
+            GalleryMediaUploadHandlerImpl(files, handle())
+        }
+    }
+
    override suspend fun createPoll(
        question: String,
        answers: List<String>,
@@ -13,6 +13,8 @@ import io.element.android.libraries.matrix.api.timeline.item.event.AudioMessageT
 import io.element.android.libraries.matrix.api.timeline.item.event.EmoteMessageType
 import io.element.android.libraries.matrix.api.timeline.item.event.FileMessageType
 import io.element.android.libraries.matrix.api.timeline.item.event.FormattedBody
+import io.element.android.libraries.matrix.api.timeline.item.event.GalleryItemType
+import io.element.android.libraries.matrix.api.timeline.item.event.GalleryMessageType
 import io.element.android.libraries.matrix.api.timeline.item.event.ImageMessageType
 import io.element.android.libraries.matrix.api.timeline.item.event.InReplyTo
 import io.element.android.libraries.matrix.api.timeline.item.event.LocationMessageType
@@ -31,12 +33,10 @@ import org.matrix.rustcomponents.sdk.MessageType
 import org.matrix.rustcomponents.sdk.MsgLikeKind
 import org.matrix.rustcomponents.sdk.use
 import org.matrix.rustcomponents.sdk.FormattedBody as RustFormattedBody
+import org.matrix.rustcomponents.sdk.GalleryItemType as RustGalleryItemType
 import org.matrix.rustcomponents.sdk.MessageFormat as RustMessageFormat
 import org.matrix.rustcomponents.sdk.MessageType as RustMessageType

-// https://github.com/Johennes/matrix-spec-proposals/blob/johannes/msgtype-galleries/proposals/4274-inline-media-galleries.md#unstable-prefix
-private const val MSG_TYPE_GALLERY_UNSTABLE = "dm.filament.gallery"
-
 class EventMessageMapper {
    private val inReplyToMapper by lazy { InReplyToMapper(TimelineEventContentMapper()) }

@@ -124,8 +124,64 @@ class EventMessageMapper {
            OtherMessageType(type.msgtype, type.body)
        }
        is MessageType.Gallery -> {
-            // TODO expose the GalleryType.
-            OtherMessageType(MSG_TYPE_GALLERY_UNSTABLE, type.content.body)
+            GalleryMessageType(
+                body = type.content.body,
+                formatted = type.content.formatted?.map(),
+                items = type.content.itemtypes.map { mapGalleryItemType(it) },
+            )
+        }
+    }
+
+    private fun mapGalleryItemType(type: RustGalleryItemType): GalleryItemType = when (type) {
+        is RustGalleryItemType.Image -> {
+            GalleryItemType.Image(
+                content = ImageMessageType(
+                    filename = type.content.filename,
+                    caption = type.content.caption,
+                    formattedCaption = type.content.formattedCaption?.map(),
+                    source = type.content.source.map(),
+                    info = type.content.info?.map(),
+                )
+            )
+        }
+        is RustGalleryItemType.Audio -> {
+            GalleryItemType.Audio(
+                content = AudioMessageType(
+                    filename = type.content.filename,
+                    caption = type.content.caption,
+                    formattedCaption = type.content.formattedCaption?.map(),
+                    source = type.content.source.map(),
+                    info = type.content.info?.map(),
+                )
+            )
+        }
+        is RustGalleryItemType.Video -> {
+            GalleryItemType.Video(
+                content = VideoMessageType(
+                    filename = type.content.filename,
+                    caption = type.content.caption,
+                    formattedCaption = type.content.formattedCaption?.map(),
+                    source = type.content.source.map(),
+                    info = type.content.info?.map(),
+                )
+            )
+        }
+        is RustGalleryItemType.File -> {
+            GalleryItemType.File(
+                content = FileMessageType(
+                    filename = type.content.filename,
+                    caption = type.content.caption,
+                    formattedCaption = type.content.formattedCaption?.map(),
+                    source = type.content.source.map(),
+                    info = type.content.info?.map(),
+                )
+            )
+        }
+        is RustGalleryItemType.Other -> {
+            GalleryItemType.Other(
+                itemType = type.itemtype,
+                body = type.body,
+            )
        }
    }
 }