Feature/clean job

Mixin.xcodeproj/project.pbxproj

@@ -571,6 +571,8 @@
 		7C359DCE26A6C15A001D3AE4 /* StickerStorePreviewCell.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C359DCD26A6C15A001D3AE4 /* StickerStorePreviewCell.swift */; };
 		7C359DD026A6C173001D3AE4 /* StickerStoreBannerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C359DCF26A6C173001D3AE4 /* StickerStoreBannerView.swift */; };
 		7C36BE88274F248C00089B6D /* StickerStoreBannerCell.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C36BE87274F248C00089B6D /* StickerStoreBannerCell.swift */; };
+		7C3E19952A06455C0066D342 /* CleanUpLargeQuoteContentJob.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C3E19942A06455C0066D342 /* CleanUpLargeQuoteContentJob.swift */; };
+		7C3E19972A0646FF0066D342 /* CleanUpLargeThumbImageJob.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C3E19962A0646FF0066D342 /* CleanUpLargeThumbImageJob.swift */; };
 		7C427BC428373F8000FFDE12 /* Wallpaper.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C427BC328373F8000FFDE12 /* Wallpaper.swift */; };
 		7C4733A128533EB800ECD293 /* PhoneContactSearchResult.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C4733A028533EB800ECD293 /* PhoneContactSearchResult.swift */; };
 		7C47352928571CC900ECD293 /* AccessPhoneContactHintWindow.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C47352828571CC900ECD293 /* AccessPhoneContactHintWindow.swift */; };
@@ -1645,6 +1647,8 @@
 		7C359DCD26A6C15A001D3AE4 /* StickerStorePreviewCell.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StickerStorePreviewCell.swift; sourceTree = "<group>"; };
 		7C359DCF26A6C173001D3AE4 /* StickerStoreBannerView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StickerStoreBannerView.swift; sourceTree = "<group>"; };
 		7C36BE87274F248C00089B6D /* StickerStoreBannerCell.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StickerStoreBannerCell.swift; sourceTree = "<group>"; };
+		7C3E19942A06455C0066D342 /* CleanUpLargeQuoteContentJob.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CleanUpLargeQuoteContentJob.swift; sourceTree = "<group>"; };
+		7C3E19962A0646FF0066D342 /* CleanUpLargeThumbImageJob.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CleanUpLargeThumbImageJob.swift; sourceTree = "<group>"; };
 		7C427BC328373F8000FFDE12 /* Wallpaper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Wallpaper.swift; sourceTree = "<group>"; };
 		7C4733A028533EB800ECD293 /* PhoneContactSearchResult.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PhoneContactSearchResult.swift; sourceTree = "<group>"; };
 		7C47352828571CC900ECD293 /* AccessPhoneContactHintWindow.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AccessPhoneContactHintWindow.swift; sourceTree = "<group>"; };
@@ -3927,6 +3931,8 @@
 				947F4AD525866D6C00B0A5F9 /* InitializeFTSJob.swift */,
 				842347ED2695BA6400009A39 /* InitializeBotJob.swift */,
 				94FCB83A264683D900CCC8FD /* TranscriptAttachmentUploadJob.swift */,
+				7C3E19942A06455C0066D342 /* CleanUpLargeQuoteContentJob.swift */,
+				7C3E19962A0646FF0066D342 /* CleanUpLargeThumbImageJob.swift */,
 			);
 			path = Job;
 			sourceTree = "<group>";
@@ -5039,6 +5045,7 @@
 				7BA931F51FC08DF4005DF478 /* TextMessageCell.swift in Sources */,
 				7CBD2EFA268FF6BB00AA0847 /* HomeAppsStorage.swift in Sources */,
 				7B34A19F22C0E73D00665F41 /* GalleryVideoView.swift in Sources */,
+				7C3E19952A06455C0066D342 /* CleanUpLargeQuoteContentJob.swift in Sources */,
 				7BFD3458228589ED00524EA0 /* ContactSelectorViewController.swift in Sources */,
 				7BD38B6220BEA12800D06E5C /* AudioMessageViewModel.swift in Sources */,
 				94B8D185266E37ED00F43CBB /* DiagnoseViewController.swift in Sources */,
@@ -5250,6 +5257,7 @@
 				7B5BC7B720AA1BB300DA5296 /* AttachmentLoadingMessageCell.swift in Sources */,
 				7B88655B20AC3417002D15E4 /* PhotoRepresentableMessageCell.swift in Sources */,
 				7BF42E4122DDF0D9005066E6 /* GalleryAnimatable.swift in Sources */,
+				7C3E19972A0646FF0066D342 /* CleanUpLargeThumbImageJob.swift in Sources */,
 				E09C5EC123703FEE0095F729 /* ProfileShortcutView.swift in Sources */,
 				7B2AFEE220FE022C00C747BB /* MXMFastURLDetector.m in Sources */,
 				DF1F277C21A53585009A74C6 /* BackupViewController.swift in Sources */,

Mixin/Service/DeviceTransfer/TransferMessage/DeviceTransferMessage.swift

@@ -100,6 +100,12 @@ struct DeviceTransferMessage {
         } else {
             duration = nil
         }
+        let messageThumbImage: String?
+        if let thumbImage, thumbImage.utf8.count > maxThumbImageLength {
+            messageThumbImage = defaultThumbImage
+        } else {
+            messageThumbImage = thumbImage
+        }
         return Message(messageId: messageId,
                        conversationId: conversationId,
                        userId: userId,
@@ -117,7 +123,7 @@ struct DeviceTransferMessage {
                        mediaStatus: mediaStatus,
                        mediaWaveform: mediaWaveform,
                        mediaLocalIdentifier: mediaLocalIdentifier,
-                       thumbImage: thumbImage,
+                       thumbImage: messageThumbImage,
                        thumbUrl: thumbUrl,
                        status: MessageStatus.READ.rawValue,
                        action: action,

Mixin/Service/Job/CleanUpLargeQuoteContentJob.swift

@@ -0,0 +1,49 @@
+import Foundation
+import MixinServices
+
+class CleanUpLargeQuoteContentJob: AsynchronousJob {
+
+    private let limit = 100
+    private var rowId: Int?
+
+    override func getJobId() -> String {
+        return "clean-up-large-quote-content"
+    }
+
+    override func execute() -> Bool {
+        while true {
+            let messages = MessageDAO.shared.largeQuoteContentMessages(limit: limit, after: rowId)
+            if messages.isEmpty {
+                AppGroupUserDefaults.User.hasCleanedUpLargeQuoteContent = true
+                Logger.general.info(category: "CleanUpLargeQuoteContentJob", message: "Cleaned up done")
+                return true
+            }
+            for message in messages {
+                let quotedMessage: MessageItem
+                if let m = MessageDAO.shared.getNonFailedMessage(messageId: message.quoteMessageId) {
+                    quotedMessage = m
+                } else if let m = try? JSONDecoder.default.decode(MessageItem.self, from: message.quoteContent) {
+                    quotedMessage = m
+                } else {
+                    continue
+                }
+                if let thumbImage = quotedMessage.thumbImage, thumbImage.utf8.count > maxThumbImageLength {
+                    quotedMessage.thumbImage = defaultThumbImage
+                }
+                quotedMessage.quoteContent = nil
+                quotedMessage.quoteMessageId = nil
+                if let content = try? JSONEncoder.default.encode(quotedMessage) {
+                    MessageDAO.shared.update(quoteContent: content, for: message.messageId)
+                }
+            }
+            Logger.general.info(category: "CleanUpLargeQuoteContentJob", message: "Cleaned up \(messages.count)")
+            if messages.count < limit {
+                AppGroupUserDefaults.User.hasCleanedUpLargeQuoteContent = true
+                Logger.general.info(category: "CleanUpLargeQuoteContentJob", message: "Cleaned up done")
+                return true
+            }
+            rowId = messages.last?.rowId
+        }
+    }
+
+}

Mixin/Service/Job/CleanUpLargeThumbImageJob.swift

@@ -0,0 +1,17 @@
+import Foundation
+import MixinServices
+
+class CleanUpLargeThumbImageJob: AsynchronousJob {
+
+    override func getJobId() -> String {
+        return "clean-up-large-thumb-image"
+    }
+
+    override func execute() -> Bool {
+        MessageDAO.shared.cleanUpLargeThumbImage()
+        AppGroupUserDefaults.User.hasCleanedUpLargeThumbImage = true
+        Logger.general.info(category: "CleanUpLargeQuoteContentJob", message: "Cleaned up large thumb image")
+        return true
+    }
+
+}

Mixin/UserInterface/Controllers/Home/HomeViewController.swift

@@ -152,6 +152,12 @@ class HomeViewController: UIViewController {
             if SpotlightManager.isAvailable {
                 SpotlightManager.shared.indexIfNeeded()
             }
+            if !AppGroupUserDefaults.User.hasCleanedUpLargeThumbImage {
+                ConcurrentJobQueue.shared.addJob(job: CleanUpLargeThumbImageJob())
+            }
+            if !AppGroupUserDefaults.User.hasCleanedUpLargeQuoteContent {
+                ConcurrentJobQueue.shared.addJob(job: CleanUpLargeQuoteContentJob())
+            }
         }
         UIApplication.homeContainerViewController?.clipSwitcher.loadClipsFromPreviousSession()
     }

MixinServices/MixinServices/Database/User/DAO/MessageDAO.swift

@@ -562,6 +562,20 @@ public final class MessageDAO: UserDatabaseDAO {
         return db.select(with: MessageDAO.sqlQueryQuoteMessageById, arguments: [messageId])
     }
 
+    public func largeQuoteContentMessages(limit: Int, after rowId: Int?) -> [LargeQuoteMessage] {
+        var sql = "SELECT rowid, id, quote_message_id, quote_content FROM messages WHERE"
+        if let rowId {
+            sql += " ROWID > \(rowId) AND"
+        }
+        sql += " quote_message_id IS NOT NULL AND quote_message_id != '' AND LENGTH(quote_content) > ? GROUP BY quote_message_id LIMIT ?"
+        return db.select(with: sql, arguments: [maxQuoteContentLength, limit])
+    }
+
+    public func cleanUpLargeThumbImage() {
+        let sql = "UPDATE messages SET thumb_image = ? WHERE LENGTH(thumb_image) > ?"
+        db.execute(sql: sql, arguments: [defaultThumbImage, maxThumbImageLength])
+    }
+
     public func insertMessage(
         message: Message,
         children: [TranscriptMessage]? = nil,

MixinServices/MixinServices/Database/User/Model/LargeQuoteMessage.swift

@@ -0,0 +1,22 @@
+import Foundation
+import GRDB
+
+public struct LargeQuoteMessage {
+
+    public let rowId: Int
+    public let messageId: String
+    public let quoteMessageId: String
+    public let quoteContent: Data
+
+}
+
+extension LargeQuoteMessage: Codable, DatabaseColumnConvertible, MixinFetchableRecord, MixinEncodableRecord {
+
+    public enum CodingKeys: String, CodingKey {
+        case rowId = "rowid"
+        case messageId = "id"
+        case quoteMessageId = "quote_message_id"
+        case quoteContent = "quote_content"
+    }
+
+}

MixinServices/MixinServices/Foundation/Constant.swift

@@ -8,6 +8,10 @@ public let secondsPerDay: TimeInterval = 24 * secondsPerHour
 public let bytesPerKiloByte: UInt = 1024
 public let bytesPerMegaByte: UInt = bytesPerKiloByte * 1024
 
+public let maxQuoteContentLength = 10 * bytesPerKiloByte
+public let maxThumbImageLength = 5 * bytesPerKiloByte
+public let defaultThumbImage = "K0OWvn_3fQ~qj[fQfQfQfQ"
+
 public enum JPEGCompressionQuality {
     public static let max: CGFloat = 1
     public static let high: CGFloat = 0.85

MixinServices/MixinServices/Foundation/User Defaults/AppGroupUserDefaults+User.swift

@@ -67,6 +67,9 @@ extension AppGroupUserDefaults {
             case wallpapers = "wallpapers"
             case chatFontSize = "chat_font_size"
             case useSystemFont = "use_system_font"
+
+            case hasCleanedUpLargeThumbImage = "has_cleaned_up_large_thumb_image"
+            case hasCleanedUpLargeQuoteContent = "has_cleaned_up_large_quote_content"
         }
 
         public static let version = 31
@@ -261,6 +264,12 @@ extension AppGroupUserDefaults {
             }
         }
 
+        @Default(namespace: .user, key: Key.hasCleanedUpLargeThumbImage, defaultValue: false)
+        public static var hasCleanedUpLargeThumbImage: Bool
+
+        @Default(namespace: .user, key: Key.hasCleanedUpLargeQuoteContent, defaultValue: false)
+        public static var hasCleanedUpLargeQuoteContent: Bool
+
         public static func insertRecentlyUsedAppId(id: String) {
             let maxNumberOfIds = 12
             var ids = recentlyUsedAppIds

MixinServices/MixinServices/Services/WebSocket/Service/ReceiveMessageService.swift

@@ -724,8 +724,15 @@ public class ReceiveMessageService: MixinService {
         let quoteMessage = MessageDAO.shared.getNonFailedMessage(messageId: data.quoteMessageId)
 
         defer {
-            if let quoteMessage = quoteMessage, let quoteContent = try? JSONEncoder.default.encode(quoteMessage) {
-                MessageDAO.shared.update(quoteContent: quoteContent, for: messageId)
+            if let quoteMessage {
+                if let thumbImage = quoteMessage.thumbImage, thumbImage.utf8.count > maxThumbImageLength {
+                    quoteMessage.thumbImage = defaultThumbImage
+                }
+                quoteMessage.quoteContent = nil
+                quoteMessage.quoteMessageId = nil
+                if let quoteContent = try? JSONEncoder.default.encode(quoteMessage) {
+                    MessageDAO.shared.update(quoteContent: quoteContent, for: messageId)
+                }
             }
         }