From bec7fe226d841f4cebafccb916b3a37daecb0d44 Mon Sep 17 00:00:00 2001 From: SomesH S Date: Wed, 8 Sep 2021 17:18:31 +0530 Subject: [PATCH 01/11] update progress in `GoogleDriveHelper` --- tgmb/__init__.py | 54 +++++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/tgmb/__init__.py b/tgmb/__init__.py index 76b76ab..40d9fff 100644 --- a/tgmb/__init__.py +++ b/tgmb/__init__.py @@ -1294,16 +1294,16 @@ def addDownload(self, mirrorInfo: 'MirrorInfo') -> None: isFolder = True if mirrorInfo.isGoogleDriveUpload and not (mirrorInfo.isCompress or mirrorInfo.isDecompress): if isFolder: - folderId = self.cloneFolder(sourceFolderId=sourceId, parentFolderId=mirrorInfo.googleDriveUploadFolderId) + folderId = self.cloneFolder(sourceFolderId=sourceId, parentFolderId=mirrorInfo.googleDriveUploadFolderId, uid=mirrorInfo.uid) self.botHelper.mirrorHelper.mirrorInfos[mirrorInfo.uid].uploadUrl = self.baseFolderDownloadUrl.format(folderId) else: - fileId = self.cloneFile(sourceFileId=sourceId, parentFolderId=mirrorInfo.googleDriveUploadFolderId) + fileId = self.cloneFile(sourceFileId=sourceId, parentFolderId=mirrorInfo.googleDriveUploadFolderId, uid=mirrorInfo.uid) self.botHelper.mirrorHelper.mirrorInfos[mirrorInfo.uid].uploadUrl = self.baseFileDownloadUrl.format(fileId) else: if isFolder: - self.downloadFolder(sourceFolderId=sourceId, dlPath=mirrorInfo.path) + self.downloadFolder(sourceFolderId=sourceId, dlPath=mirrorInfo.path, uid=mirrorInfo.uid) else: - self.downloadFile(sourceFileId=sourceId, dlPath=mirrorInfo.path) + self.downloadFile(sourceFileId=sourceId, dlPath=mirrorInfo.path, uid=mirrorInfo.uid) self.botHelper.mirrorListenerHelper.updateStatus(mirrorInfo.uid, MirrorStatus.downloadComplete) def cancelDownload(self, uid: str) -> None: @@ -1313,10 +1313,10 @@ def addUpload(self, mirrorInfo: 'MirrorInfo') -> None: if not (mirrorInfo.isGoogleDriveDownload and not (mirrorInfo.isCompress or mirrorInfo.isDecompress)): uploadPath = os.path.join(mirrorInfo.path, os.listdir(mirrorInfo.path)[0]) if os.path.isdir(uploadPath): - folderId = self.uploadFolder(folderPath=uploadPath, parentFolderId=mirrorInfo.googleDriveUploadFolderId) + folderId = self.uploadFolder(folderPath=uploadPath, parentFolderId=mirrorInfo.googleDriveUploadFolderId, uid=mirrorInfo.uid) self.botHelper.mirrorHelper.mirrorInfos[mirrorInfo.uid].uploadUrl = self.baseFolderDownloadUrl.format(folderId) if os.path.isfile(uploadPath): - fileId = self.uploadFile(filePath=uploadPath, parentFolderId=mirrorInfo.googleDriveUploadFolderId) + fileId = self.uploadFile(filePath=uploadPath, parentFolderId=mirrorInfo.googleDriveUploadFolderId, uid=mirrorInfo.uid) self.botHelper.mirrorHelper.mirrorInfos[mirrorInfo.uid].uploadUrl = self.baseFileDownloadUrl.format(fileId) else: time.sleep(self.botHelper.statusHelper.statusUpdateInterval) @@ -1344,7 +1344,7 @@ def buildService(self) -> None: self.service = googleapiclient.discovery.build(serviceName='drive', version='v3', credentials=self.oauthCreds, cache_discovery=False) - def uploadFile(self, filePath: str, parentFolderId: str) -> str: + def uploadFile(self, filePath: str, parentFolderId: str, uid: str) -> str: upStatus: googleapiclient.http.MediaUploadProgress fileName, fileMimeType, fileMetadata, mediaBody = self.getUpData(filePath, isResumable=True) fileMetadata['parents'] = [parentFolderId] @@ -1352,9 +1352,10 @@ def uploadFile(self, filePath: str, parentFolderId: str) -> str: upResponse = None while not upResponse: upStatus, upResponse = fileOp.next_chunk() + self.updateProgress(self.chunkSize, uid) return upResponse['id'] - def uploadFolder(self, folderPath: str, parentFolderId: str) -> str: + def uploadFolder(self, folderPath: str, parentFolderId: str, uid: str) -> str: folderName = folderPath.split('/')[-1] folderId = self.createFolder(folderName, parentFolderId) folderContents = os.listdir(folderPath) @@ -1362,28 +1363,30 @@ def uploadFolder(self, folderPath: str, parentFolderId: str) -> str: for contentName in folderContents: contentPath = os.path.join(folderPath, contentName) if os.path.isdir(contentPath): - self.uploadFolder(contentPath, folderId) + self.uploadFolder(folderPath=contentPath, parentFolderId=folderId, uid=uid) if os.path.isfile(contentPath): - self.uploadFile(contentPath, folderId) + self.uploadFile(filePath=contentPath, parentFolderId=folderId, uid=uid) return folderId - def cloneFile(self, sourceFileId: str, parentFolderId: str) -> str: + def cloneFile(self, sourceFileId: str, parentFolderId: str, uid: str) -> str: fileMetadata = {'parents': [parentFolderId]} - return self.service.files().copy(supportsAllDrives=True, fileId=sourceFileId, body=fileMetadata).execute()['id'] + fileOp = self.service.files().copy(supportsAllDrives=True, fileId=sourceFileId, body=fileMetadata).execute() + self.updateProgress(self.getSizeById(sourceFileId), uid) + return fileOp['id'] - def cloneFolder(self, sourceFolderId: str, parentFolderId: str) -> str: + def cloneFolder(self, sourceFolderId: str, parentFolderId: str, uid: str) -> str: sourceFolderName = self.getMetadataById(sourceFolderId, 'name') folderId = self.createFolder(sourceFolderName, parentFolderId) folderContents = self.getFolderContentsById(sourceFolderId) if len(folderContents) != 0: for content in folderContents: if content.get('mimeType') == self.googleDriveFolderMimeType: - self.cloneFolder(content.get('id'), folderId) + self.cloneFolder(sourceFolderId=content.get('id'), parentFolderId=folderId, uid=uid) else: - self.cloneFile(content.get('id'), folderId) + self.cloneFile(sourceFileId=content.get('id'), parentFolderId=folderId, uid=uid) return folderId - def downloadFile(self, sourceFileId: str, dlPath: str) -> None: + def downloadFile(self, sourceFileId: str, dlPath: str, uid: str) -> None: fileName = self.getMetadataById(sourceFileId, 'name') filePath = os.path.join(dlPath, fileName) downStatus: googleapiclient.http.MediaDownloadProgress @@ -1393,9 +1396,10 @@ def downloadFile(self, sourceFileId: str, dlPath: str) -> None: downResponse = None while not downResponse: downStatus, downResponse = fileOp.next_chunk() + self.updateProgress(self.chunkSize, uid) return - def downloadFolder(self, sourceFolderId: str, dlPath: str) -> None: + def downloadFolder(self, sourceFolderId: str, dlPath: str, uid: str) -> None: folderName = self.getMetadataById(sourceFolderId, 'name') folderPath = os.path.join(dlPath, folderName) os.mkdir(folderPath) @@ -1403,9 +1407,9 @@ def downloadFolder(self, sourceFolderId: str, dlPath: str) -> None: if len(folderContents) != 0: for content in folderContents: if content.get('mimeType') == self.googleDriveFolderMimeType: - self.downloadFolder(content.get('id'), folderPath) + self.downloadFolder(sourceFolderId=content.get('id'), dlPath=folderPath, uid=uid) else: - self.downloadFile(content.get('id'), folderPath) + self.downloadFile(sourceFileId=content.get('id'), dlPath=folderPath, uid=uid) return def createFolder(self, folderName: str, parentFolderId: str) -> str: @@ -1475,6 +1479,14 @@ def patchFile(self, filePath: str, fileId: str) -> str: fileOp = self.service.files().update(fileId=fileId, body=fileMetadata, media_body=mediaBody).execute() return f"Patched: [{fileOp['id']}] [{fileName}] [{os.path.getsize(fileName)} bytes]" + def updateProgress(self, sizeUpdate: int, uid: str): + timeCurrent = int(time.time()) + sizeCurrent = self.botHelper.mirrorHelper.mirrorInfos[uid].sizeCurrent + sizeUpdate + speedCurrent = int(sizeUpdate / (timeCurrent - self.botHelper.mirrorHelper.mirrorInfos[uid].timeCurrent)) + self.botHelper.mirrorHelper.mirrorInfos[uid].updateVars({MirrorInfo.updatableVars[1]: sizeCurrent, + MirrorInfo.updatableVars[2]: speedCurrent, + MirrorInfo.updatableVars[3]: timeCurrent}) + class MegaHelper(BaseHelper): def __init__(self, botHelper: BotHelper): @@ -1734,8 +1746,8 @@ def getStatusMsgTxt(self) -> str: for uid in self.botHelper.mirrorHelper.mirrorInfos.keys(): mirrorInfo: MirrorInfo = self.botHelper.mirrorHelper.mirrorInfos[uid] statusMsgTxt += f'{mirrorInfo.uid} | {mirrorInfo.status}\n' - if mirrorInfo.status == MirrorStatus.downloadProgress: - if mirrorInfo.isAriaDownload: + if mirrorInfo.status in [MirrorStatus.downloadProgress, MirrorStatus.uploadProgress]: + if mirrorInfo.status == MirrorStatus.downloadProgress and mirrorInfo.isAriaDownload: self.botHelper.ariaHelper.updateProgress(mirrorInfo.uid) statusMsgTxt += f'S: {self.botHelper.getHelper.readableSize(mirrorInfo.sizeCurrent)} | ' \ f'{self.botHelper.getHelper.readableSize(mirrorInfo.sizeTotal)} | ' \ From 190941d7c2419fe5c6b47a8439676ad30146f1ee Mon Sep 17 00:00:00 2001 From: SomesH S Date: Wed, 8 Sep 2021 22:59:00 +0530 Subject: [PATCH 02/11] add method resetVars() in `MirrorInfo` and refactor --- tgmb/__init__.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/tgmb/__init__.py b/tgmb/__init__.py index 40d9fff..dff6df2 100644 --- a/tgmb/__init__.py +++ b/tgmb/__init__.py @@ -389,6 +389,14 @@ def fileHash(filePath: str) -> str: fileChunk = fileStream.read(blockSize) return hashSum.hexdigest() + @staticmethod + def folderSize(folderPath: str) -> int: + size: int = 0 + for path, dirs, files in os.walk(folderPath): + for file in files: + size += os.path.getsize(os.path.join(path, file)) + return size + def progressBar(self, progress: float) -> str: progressRounded = round(progress) numFull = progressRounded // 8 @@ -1311,6 +1319,8 @@ def cancelDownload(self, uid: str) -> None: def addUpload(self, mirrorInfo: 'MirrorInfo') -> None: if not (mirrorInfo.isGoogleDriveDownload and not (mirrorInfo.isCompress or mirrorInfo.isDecompress)): + currVars = {MirrorInfo.updatableVars[0]: self.botHelper.getHelper.folderSize(mirrorInfo.path)} + self.botHelper.mirrorHelper.mirrorInfos[mirrorInfo.uid].updateVars(currVars) uploadPath = os.path.join(mirrorInfo.path, os.listdir(mirrorInfo.path)[0]) if os.path.isdir(uploadPath): folderId = self.uploadFolder(folderPath=uploadPath, parentFolderId=mirrorInfo.googleDriveUploadFolderId, uid=mirrorInfo.uid) @@ -1318,8 +1328,6 @@ def addUpload(self, mirrorInfo: 'MirrorInfo') -> None: if os.path.isfile(uploadPath): fileId = self.uploadFile(filePath=uploadPath, parentFolderId=mirrorInfo.googleDriveUploadFolderId, uid=mirrorInfo.uid) self.botHelper.mirrorHelper.mirrorInfos[mirrorInfo.uid].uploadUrl = self.baseFileDownloadUrl.format(fileId) - else: - time.sleep(self.botHelper.statusHelper.statusUpdateInterval) self.botHelper.mirrorListenerHelper.updateStatus(mirrorInfo.uid, MirrorStatus.uploadComplete) def cancelUpload(self, uid: str) -> None: @@ -1480,9 +1488,11 @@ def patchFile(self, filePath: str, fileId: str) -> str: return f"Patched: [{fileOp['id']}] [{fileName}] [{os.path.getsize(fileName)} bytes]" def updateProgress(self, sizeUpdate: int, uid: str): + sizeLast = self.botHelper.mirrorHelper.mirrorInfos[uid].sizeCurrent + timeLast = self.botHelper.mirrorHelper.mirrorInfos[uid].timeCurrent timeCurrent = int(time.time()) - sizeCurrent = self.botHelper.mirrorHelper.mirrorInfos[uid].sizeCurrent + sizeUpdate - speedCurrent = int(sizeUpdate / (timeCurrent - self.botHelper.mirrorHelper.mirrorInfos[uid].timeCurrent)) + sizeCurrent = sizeLast + sizeUpdate + speedCurrent = int(sizeUpdate / (timeCurrent - timeLast)) self.botHelper.mirrorHelper.mirrorInfos[uid].updateVars({MirrorInfo.updatableVars[1]: sizeCurrent, MirrorInfo.updatableVars[2]: speedCurrent, MirrorInfo.updatableVars[3]: timeCurrent}) @@ -1582,6 +1592,8 @@ def cancelDownload(self, uid: str) -> None: raise NotImplementedError def addUpload(self, mirrorInfo: 'MirrorInfo') -> None: + currVars = {MirrorInfo.updatableVars[0]: self.botHelper.getHelper.folderSize(mirrorInfo.path)} + self.botHelper.mirrorHelper.mirrorInfos[mirrorInfo.uid].updateVars(currVars) uploadPath = os.path.join(mirrorInfo.path, os.listdir(mirrorInfo.path)[0]) upResponse: bool = True if os.path.isfile(uploadPath): @@ -2028,8 +2040,7 @@ def onUploadError(self, mirrorInfo: 'MirrorInfo') -> None: self.checkUploadQueue() def resetMirrorProgress(self, uid: str) -> None: - self.botHelper.mirrorHelper.mirrorInfos[uid].timeEnd = 0 - self.botHelper.mirrorHelper.mirrorInfos[uid].progressPercent = 0 + self.botHelper.mirrorHelper.mirrorInfos[uid].resetVars() class MegaApiWrapper: @@ -2194,6 +2205,12 @@ def __init__(self, msg: telegram.Message, botHelper: BotHelper): self.isCompress: bool = False self.isDecompress: bool = False + def resetVars(self): + self.sizeTotal, self.sizeCurrent = 0, 0 + self.timeEnd, self.timeCurrent = 0, 0 + self.speedCurrent = 0 + self.progressPercent = 0.0 + def updateVars(self, currVars: typing.Dict[str, typing.Union[int, float, str]]) -> None: currVarsKeys = list(currVars.keys()) if self.updatableVars[0] in currVarsKeys: From 6e99c4f0c6c8d091939da046596aa2ecdb47d10e Mon Sep 17 00:00:00 2001 From: SomesH S Date: Wed, 8 Sep 2021 23:07:53 +0530 Subject: [PATCH 03/11] rearrange attributes in `MirrorInfo` --- tgmb/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tgmb/__init__.py b/tgmb/__init__.py index dff6df2..c498ba7 100644 --- a/tgmb/__init__.py +++ b/tgmb/__init__.py @@ -2180,14 +2180,14 @@ def __init__(self, msg: telegram.Message, botHelper: BotHelper): self.uid: str = botHelper.getHelper.randomString(8) self.path: str = os.path.join(botHelper.envVars['dlRootDirPath'], self.uid) self.status: str = '' - self.downloadUrl: str = '' self.tag: str = '' + self.downloadUrl: str = '' + self.sizeTotal: int = 0 + self.sizeCurrent: int = 0 self.timeStart: int = 0 self.timeCurrent: int = 0 self.timeEnd: int = 0 self.speedCurrent: int = 0 - self.sizeTotal: int = 0 - self.sizeCurrent: int = 0 self.progressPercent: float = 0.0 self.isTorrent: bool = False self.numSeeders: int = 0 From a6cc895eacb8ecc16c678c3428b6edd632bbc201 Mon Sep 17 00:00:00 2001 From: SomesH S Date: Wed, 8 Sep 2021 23:27:51 +0530 Subject: [PATCH 04/11] fix ZeroDivisionError in GoogleDriveHelper.updateProgress() --- tgmb/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tgmb/__init__.py b/tgmb/__init__.py index c498ba7..1a107e5 100644 --- a/tgmb/__init__.py +++ b/tgmb/__init__.py @@ -1490,9 +1490,11 @@ def patchFile(self, filePath: str, fileId: str) -> str: def updateProgress(self, sizeUpdate: int, uid: str): sizeLast = self.botHelper.mirrorHelper.mirrorInfos[uid].sizeCurrent timeLast = self.botHelper.mirrorHelper.mirrorInfos[uid].timeCurrent - timeCurrent = int(time.time()) + speedLast = self.botHelper.mirrorHelper.mirrorInfos[uid].speedCurrent sizeCurrent = sizeLast + sizeUpdate - speedCurrent = int(sizeUpdate / (timeCurrent - timeLast)) + timeCurrent = int(time.time()) + timeDiff = timeCurrent - timeLast + speedCurrent = (int(sizeUpdate / timeDiff) if timeDiff else speedLast) self.botHelper.mirrorHelper.mirrorInfos[uid].updateVars({MirrorInfo.updatableVars[1]: sizeCurrent, MirrorInfo.updatableVars[2]: speedCurrent, MirrorInfo.updatableVars[3]: timeCurrent}) From e0ac43337e7f546d841c5b3f7df0caa0f5c2c13a Mon Sep 17 00:00:00 2001 From: SomesH S Date: Thu, 9 Sep 2021 10:32:20 +0530 Subject: [PATCH 05/11] (timeStart, timeEnd, timeCurrent) : int -> float in `MirrorInfo` and refactor --- tgmb/__init__.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tgmb/__init__.py b/tgmb/__init__.py index 1a107e5..95fc510 100644 --- a/tgmb/__init__.py +++ b/tgmb/__init__.py @@ -1083,7 +1083,7 @@ def initHelper(self) -> None: def addMirror(self, mirrorInfo: 'MirrorInfo') -> None: self.logger.debug(vars(mirrorInfo)) self.mirrorInfos[mirrorInfo.uid] = mirrorInfo - self.mirrorInfos[mirrorInfo.uid].timeStart = int(time.time()) + self.mirrorInfos[mirrorInfo.uid].timeStart = time.time() self.botHelper.mirrorListenerHelper.updateStatus(mirrorInfo.uid, MirrorStatus.addMirror) self.botHelper.threadingHelper.initThread(target=self.botHelper.statusHelper.addStatus, name=f'{mirrorInfo.uid}-addStatus', chatId=mirrorInfo.chatId, msgId=mirrorInfo.msgId) @@ -1239,7 +1239,7 @@ def updateProgress(self, uid: str) -> None: {MirrorInfo.updatableVars[0]: dlObj.total_length, MirrorInfo.updatableVars[1]: dlObj.completed_length, MirrorInfo.updatableVars[2]: dlObj.download_speed, - MirrorInfo.updatableVars[3]: int(time.time())} + MirrorInfo.updatableVars[3]: time.time()} if dlObj.is_torrent: currVars[MirrorInfo.updatableVars[4]] = True currVars[MirrorInfo.updatableVars[5]] = dlObj.num_seeders @@ -1492,7 +1492,7 @@ def updateProgress(self, sizeUpdate: int, uid: str): timeLast = self.botHelper.mirrorHelper.mirrorInfos[uid].timeCurrent speedLast = self.botHelper.mirrorHelper.mirrorInfos[uid].speedCurrent sizeCurrent = sizeLast + sizeUpdate - timeCurrent = int(time.time()) + timeCurrent = time.time() timeDiff = timeCurrent - timeLast speedCurrent = (int(sizeUpdate / timeDiff) if timeDiff else speedLast) self.botHelper.mirrorHelper.mirrorInfos[uid].updateVars({MirrorInfo.updatableVars[1]: sizeCurrent, @@ -1769,8 +1769,8 @@ def getStatusMsgTxt(self) -> str: f'P: {self.botHelper.getHelper.progressBar(mirrorInfo.progressPercent)} | ' \ f'{mirrorInfo.progressPercent}% | ' \ f'{self.botHelper.getHelper.readableSize(mirrorInfo.speedCurrent)}/s\n' \ - f'T: {self.botHelper.getHelper.readableTime(mirrorInfo.timeCurrent - mirrorInfo.timeStart)} | ' \ - f'{self.botHelper.getHelper.readableTime(mirrorInfo.timeEnd - mirrorInfo.timeCurrent)}\n' + f'T: {self.botHelper.getHelper.readableTime(int(mirrorInfo.timeCurrent - mirrorInfo.timeStart))} | ' \ + f'{self.botHelper.getHelper.readableTime(int(mirrorInfo.timeEnd - mirrorInfo.timeCurrent))}\n' statusMsgTxt += (f'nS: {mirrorInfo.numSeeders} nL: {mirrorInfo.numLeechers}\n' if mirrorInfo.isTorrent else '') return statusMsgTxt @@ -2152,7 +2152,7 @@ def onTransferUpdate(self, api: mega.MegaApi, transfer: mega.MegaTransfer): {MirrorInfo.updatableVars[0]: transfer.getTotalBytes(), MirrorInfo.updatableVars[1]: transfer.getTransferredBytes(), MirrorInfo.updatableVars[2]: transfer.getSpeed(), - MirrorInfo.updatableVars[3]: int(time.time())} + MirrorInfo.updatableVars[3]: time.time()} self.megaHelper.botHelper.mirrorHelper.mirrorInfos[uid].updateVars(currVars) self.logger.debug(f'Transfer Update ({transfer} {transfer.getFileName()}); ' f'Progress: {transfer.getTransferredBytes() / 1024} KB of {transfer.getTotalBytes() / 1024} KB, ' @@ -2186,9 +2186,9 @@ def __init__(self, msg: telegram.Message, botHelper: BotHelper): self.downloadUrl: str = '' self.sizeTotal: int = 0 self.sizeCurrent: int = 0 - self.timeStart: int = 0 - self.timeCurrent: int = 0 - self.timeEnd: int = 0 + self.timeStart: float = 0.0 + self.timeCurrent: float = 0.0 + self.timeEnd: float = 0.0 self.speedCurrent: int = 0 self.progressPercent: float = 0.0 self.isTorrent: bool = False @@ -2209,7 +2209,7 @@ def __init__(self, msg: telegram.Message, botHelper: BotHelper): def resetVars(self): self.sizeTotal, self.sizeCurrent = 0, 0 - self.timeEnd, self.timeCurrent = 0, 0 + self.timeEnd, self.timeCurrent = 0.0, 0.0 self.speedCurrent = 0 self.progressPercent = 0.0 @@ -2224,7 +2224,7 @@ def updateVars(self, currVars: typing.Dict[str, typing.Union[int, float, str]]) if self.sizeTotal != 0: self.progressPercent = round(((self.sizeCurrent / self.sizeTotal) * 100), ndigits=2) if self.speedCurrent != 0: - self.timeEnd = self.timeCurrent + int((self.sizeTotal - self.sizeCurrent) / self.speedCurrent) + self.timeEnd = self.timeCurrent + ((self.sizeTotal - self.sizeCurrent) / self.speedCurrent) if self.updatableVars[4] in currVarsKeys: self.isTorrent = True self.numSeeders = currVars[self.updatableVars[5]] From cb853a707caa1fa76e5cd2d1751db48471996075 Mon Sep 17 00:00:00 2001 From: SomesH S Date: Thu, 9 Sep 2021 11:21:21 +0530 Subject: [PATCH 06/11] improve sizeUpdate precision with updateProgress() method in `GoogleDriveHelper` --- tgmb/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tgmb/__init__.py b/tgmb/__init__.py index 95fc510..985fdb8 100644 --- a/tgmb/__init__.py +++ b/tgmb/__init__.py @@ -1360,7 +1360,8 @@ def uploadFile(self, filePath: str, parentFolderId: str, uid: str) -> str: upResponse = None while not upResponse: upStatus, upResponse = fileOp.next_chunk() - self.updateProgress(self.chunkSize, uid) + sizeUpdate = (self.chunkSize if not upResponse else (os.path.getsize(filePath) % self.chunkSize)) + self.updateProgress(sizeUpdate, uid) return upResponse['id'] def uploadFolder(self, folderPath: str, parentFolderId: str, uid: str) -> str: @@ -1404,7 +1405,8 @@ def downloadFile(self, sourceFileId: str, dlPath: str, uid: str) -> None: downResponse = None while not downResponse: downStatus, downResponse = fileOp.next_chunk() - self.updateProgress(self.chunkSize, uid) + sizeUpdate = (self.chunkSize if not downResponse else (self.getSizeById(sourceFileId) % self.chunkSize)) + self.updateProgress(sizeUpdate, uid) return def downloadFolder(self, sourceFolderId: str, dlPath: str, uid: str) -> None: From 55c1270462234ccd1025237df0621623662cb195 Mon Sep 17 00:00:00 2001 From: SomesH S Date: Thu, 9 Sep 2021 11:26:49 +0530 Subject: [PATCH 07/11] revert-fix 'else' clauses in methods authorizeApi(), addUpload() in `GoogleDriveHelper` --- tgmb/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tgmb/__init__.py b/tgmb/__init__.py index 985fdb8..521d07f 100644 --- a/tgmb/__init__.py +++ b/tgmb/__init__.py @@ -1328,6 +1328,8 @@ def addUpload(self, mirrorInfo: 'MirrorInfo') -> None: if os.path.isfile(uploadPath): fileId = self.uploadFile(filePath=uploadPath, parentFolderId=mirrorInfo.googleDriveUploadFolderId, uid=mirrorInfo.uid) self.botHelper.mirrorHelper.mirrorInfos[mirrorInfo.uid].uploadUrl = self.baseFileDownloadUrl.format(fileId) + else: + time.sleep(self.botHelper.statusHelper.statusUpdateInterval) self.botHelper.mirrorListenerHelper.updateStatus(mirrorInfo.uid, MirrorStatus.uploadComplete) def cancelUpload(self, uid: str) -> None: @@ -1347,6 +1349,8 @@ def authorizeApi(self) -> None: else: self.logger.info('Google Drive API User Token Needs to Refreshed Manually ! Exiting...') exit(1) + else: + self.buildService() def buildService(self) -> None: self.service = googleapiclient.discovery.build(serviceName='drive', version='v3', credentials=self.oauthCreds, From 9bbbfd46928da440ee94d6bbb4857a0cd4f579da Mon Sep 17 00:00:00 2001 From: SomesH S Date: Thu, 9 Sep 2021 13:39:16 +0530 Subject: [PATCH 08/11] add method progressHook() in `YouTubeHelper` and refactor --- tgmb/__init__.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tgmb/__init__.py b/tgmb/__init__.py index 521d07f..3609bf1 100644 --- a/tgmb/__init__.py +++ b/tgmb/__init__.py @@ -1659,10 +1659,9 @@ def initHelper(self) -> None: super().initHelper() def addDownload(self, mirrorInfo: 'MirrorInfo') -> None: - ytdlOpts: dict = {'format': 'best/bestvideo+bestaudio', 'logger': self.logger, + ytdlOpts: dict = {'quiet': True, 'format': 'best/bestvideo+bestaudio', 'progress_hooks': [self.progressHook], 'outtmpl': f'{mirrorInfo.path}/%(title)s-%(id)s.f%(format_id)s.%(ext)s'} self.downloadVideo(mirrorInfo.downloadUrl, ytdlOpts) - self.botHelper.mirrorListenerHelper.updateStatus(mirrorInfo.uid, MirrorStatus.downloadComplete) def cancelDownload(self, uid: str) -> None: raise NotImplementedError @@ -1672,6 +1671,18 @@ def downloadVideo(videoUrl: str, ytdlOpts: dict) -> None: with youtube_dl.YoutubeDL(ytdlOpts) as ytdl: ytdl.download([videoUrl]) + def progressHook(self, progressUpdate: dict): + uid = progressUpdate['filename'].replace(self.botHelper.envVars['dlRootDirPath'], '').split('/')[1] + if progressUpdate['status'] == 'downloading': + currVars: typing.Dict[str, typing.Union[int, float, str]] = \ + {MirrorInfo.updatableVars[0]: int(progressUpdate['total_bytes']), + MirrorInfo.updatableVars[1]: int(progressUpdate['downloaded_bytes']), + MirrorInfo.updatableVars[2]: int(progressUpdate['speed']), + MirrorInfo.updatableVars[3]: time.time()} + self.botHelper.mirrorHelper.mirrorInfos[uid].updateVars(currVars) + if progressUpdate['status'] == 'finished': + self.botHelper.mirrorListenerHelper.updateStatus(uid, MirrorStatus.downloadComplete) + class CompressionHelper(BaseHelper): def __init__(self, botHelper: BotHelper): From 6ee9cc4f0c28ff076f3a4190f9f5629abe5b3d5d Mon Sep 17 00:00:00 2001 From: SomesH S Date: Thu, 9 Sep 2021 15:37:02 +0530 Subject: [PATCH 09/11] fix NoneTypeException in method progressHook() in `YouTubeHelper` --- tgmb/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tgmb/__init__.py b/tgmb/__init__.py index 3609bf1..c6ca8a5 100644 --- a/tgmb/__init__.py +++ b/tgmb/__init__.py @@ -1675,9 +1675,9 @@ def progressHook(self, progressUpdate: dict): uid = progressUpdate['filename'].replace(self.botHelper.envVars['dlRootDirPath'], '').split('/')[1] if progressUpdate['status'] == 'downloading': currVars: typing.Dict[str, typing.Union[int, float, str]] = \ - {MirrorInfo.updatableVars[0]: int(progressUpdate['total_bytes']), - MirrorInfo.updatableVars[1]: int(progressUpdate['downloaded_bytes']), - MirrorInfo.updatableVars[2]: int(progressUpdate['speed']), + {MirrorInfo.updatableVars[0]: int((sizeTotal if (sizeTotal := progressUpdate['total_bytes']) else 0)), + MirrorInfo.updatableVars[1]: int((sizeCurrent if (sizeCurrent := progressUpdate['downloaded_bytes']) else 0)), + MirrorInfo.updatableVars[2]: int((speedCurrent if (speedCurrent := progressUpdate['speed']) else 0)), MirrorInfo.updatableVars[3]: time.time()} self.botHelper.mirrorHelper.mirrorInfos[uid].updateVars(currVars) if progressUpdate['status'] == 'finished': From 9158502395f8390490606d8b8ec580812b2a8420 Mon Sep 17 00:00:00 2001 From: SomesH S Date: Thu, 9 Sep 2021 16:35:01 +0530 Subject: [PATCH 10/11] add configVar 'ytdlFormat' and refactor --- deploy/setup.py | 3 ++- tgmb/__init__.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/deploy/setup.py b/deploy/setup.py index 10afcbb..8e71b99 100644 --- a/deploy/setup.py +++ b/deploy/setup.py @@ -138,7 +138,8 @@ def fileUpload(fileName: str) -> str: 'max-connection-per-server': '8', 'max-overall-upload-limit': '1K', 'min-split-size': '10M', 'seed-time': '0.01', 'split': '10'}, 'authorizedChats': {}, 'dlRootDir': 'dl', 'logLevel': 'INFO', 'megaAuth': {'apiKey': '', 'emailId': '', 'passPhrase': ''}, - 'statusUpdateInterval': '5', 'trackersListUrl': 'https://trackerslist.com/all_aria2.txt'} + 'statusUpdateInterval': '5', 'trackersListUrl': 'https://trackerslist.com/all_aria2.txt', + 'ytdlFormat': 'best/bestvideo+bestaudio'} envVars: typing.Dict = {'dlWaitTime': '5'} if __name__ == '__main__': diff --git a/tgmb/__init__.py b/tgmb/__init__.py index c6ca8a5..3284baf 100644 --- a/tgmb/__init__.py +++ b/tgmb/__init__.py @@ -210,12 +210,12 @@ def initHelper(self) -> None: self.reqVars: [str] = ['botToken', 'botOwnerId', 'telegramApiId', 'telegramApiHash', 'googleDriveAuth', 'googleDriveUploadFolderIds'] self.optVars: typing.List[str] = ['ariaGlobalOpts', 'authorizedChats', 'dlRootDir', 'logLevel', - 'megaAuth', 'statusUpdateInterval', 'trackersListUrl'] + 'megaAuth', 'statusUpdateInterval', 'trackersListUrl', 'ytdlFormat'] self.optVals: typing.List[typing.Union[str, typing.Dict]] = \ [{'allow-overwrite': 'true', 'bt-max-peers': '0', 'follow-torrent': 'mem', 'max-connection-per-server': '8', 'max-overall-upload-limit': '1K', 'min-split-size': '10M', 'seed-time': '0.01', 'split': '10'}, - {}, 'dl', 'INFO', {}, '5', 'https://trackerslist.com/all_aria2.txt'] + {}, 'dl', 'INFO', {}, '5', 'https://trackerslist.com/all_aria2.txt', 'best/bestvideo+bestaudio'] self.emptyVals: typing.List[typing.Union[str, typing.Dict]] = ['', ' ', {}] self.isFixConfigJson: bool = False self.configVarsLoad() @@ -1659,7 +1659,7 @@ def initHelper(self) -> None: super().initHelper() def addDownload(self, mirrorInfo: 'MirrorInfo') -> None: - ytdlOpts: dict = {'quiet': True, 'format': 'best/bestvideo+bestaudio', 'progress_hooks': [self.progressHook], + ytdlOpts: dict = {'quiet': True, 'format': mirrorInfo.ytdlFormat, 'progress_hooks': [self.progressHook], 'outtmpl': f'{mirrorInfo.path}/%(title)s-%(id)s.f%(format_id)s.%(ext)s'} self.downloadVideo(mirrorInfo.downloadUrl, ytdlOpts) @@ -2201,6 +2201,7 @@ def __init__(self, msg: telegram.Message, botHelper: BotHelper): self.status: str = '' self.tag: str = '' self.downloadUrl: str = '' + self.ytdlFormat: str = botHelper.configHelper.configVars[botHelper.configHelper.optVars[7]] self.sizeTotal: int = 0 self.sizeCurrent: int = 0 self.timeStart: float = 0.0 From b8c1137c31fb39950a6f97ee5168ac729348e6a8 Mon Sep 17 00:00:00 2001 From: SomesH S Date: Thu, 9 Sep 2021 16:57:21 +0530 Subject: [PATCH 11/11] update TODOs in tgmb/__init__.py --- tgmb/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tgmb/__init__.py b/tgmb/__init__.py index 3284baf..720aad0 100644 --- a/tgmb/__init__.py +++ b/tgmb/__init__.py @@ -1,7 +1,5 @@ # TODO: add sufficient documentation to the functions and classes in this module -# TODO: Code for Download from Mega # TODO: Code for Upload to Mega -# TODO: Helper functions - bot_utils.py, fs_utils.py, message_utils.py # TODO: Code for user filters # TODO: Add and Handle Exceptions # TODO: Code for direct link generation @@ -1042,7 +1040,7 @@ def stageFive(self, update: telegram.Update, _: telegram.ext.CallbackContext) -> query.edit_message_text(text='addMirror Cancelled !') return telegram.ext.ConversationHandler.END - # TODO: reduce this function code if possible + # TODO: reduce this method code if possible def getMirrorInfoStr(self): mirrorInfoStr = f'[uid | {self.mirrorInfo.uid}]\n' if self.mirrorInfo.isAriaDownload: