perf(exhentai): tweak request time for restart a connection (#187)

orzyyyy · Aug 24, 2019 · 2e4bb9a · 2e4bb9a
1 parent dafe366
commit 2e4bb9a
Show file tree

Hide file tree

Showing 4 changed files with 61 additions and 19 deletions.
diff --git a/server/controller/ExhentaiController.ts b/server/controller/ExhentaiController.ts
@@ -12,7 +12,9 @@ import {
   getLatestListFileName,
   getBaseNameOfImage,
   getListFiles,
+  getEmptyRestDetailUrlInfo,
 } from '../utils/exhentai';
+import path from 'path';
 
 export interface ExHentaiInfoItem {
   name?: string;
@@ -53,20 +55,26 @@ export default class ExhentaiController {
 
     info(`start fetching thumbnai urls from: ${url}`);
 
-    const thumbnailUrls = await service.getThumbnaiUrlFromDetailPage();
+    const restDetailUrls: string[] = [
+      url,
+      ...(await service.getUrlFromPaginationInfo()),
+    ];
+    const thumbnailUrls = await service.getThumbnailUrlFromDetailPage(
+      restDetailUrls,
+    );
     writeIntoJsonFile(`${prefixPath}/restDetailUrls`, thumbnailUrls);
 
     info(`start fetching target images`);
 
     const images = await service.fetchImageUrls(thumbnailUrls);
 
-    success('fetch all image urls');
+    success('fetch all image completed');
 
     writeIntoJsonFile(`${prefixPath}/detailImageUrls`, images);
 
     await service.downloadImages(images, prefixPath);
 
-    ctx.response.body = true;
+    ctx.response.body = 'success';
   }
 
   @Request({ url: '/download/target', method: 'get' })
@@ -78,7 +86,7 @@ export default class ExhentaiController {
       joinWithRootPath(`${prefixPath}/detailImageUrls.json`),
     );
     service.downloadImages(detailImageUrls, prefixPath);
-    ctx.response.body = true;
+    ctx.response.body = 'success';
   }
 
   @Request({ url: '/download/stat', method: 'get' })
@@ -116,4 +124,21 @@ export default class ExhentaiController {
 
     ctx.response.body = listFiles;
   }
+
+  @Request({ url: '/sync', method: 'get' })
+  async sync(ctx: any) {
+    const service = new ExhentaiService();
+    await service.initBrowser();
+    const targetComic = getEmptyRestDetailUrlInfo();
+    for (const jsonUrl of targetComic) {
+      const thumbnailUrls = readJsonFile(jsonUrl);
+      info(`start fetching target images`);
+
+      const images = await service.fetchImageUrls(thumbnailUrls);
+      await service.downloadImages(images, path.dirname(jsonUrl));
+
+      success('fetch all image completed');
+    }
+    ctx.response.body = 'succcess';
+  }
 }
diff --git a/server/resource/server.toml b/server/resource/server.toml
@@ -7,6 +7,7 @@ winChromePath = 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.ex
 linuxChromePath = '/opt/google/chrome/chrome'
 waitTime = 1000
 waitTimeAfterError = 10000
+requestTime = 90000
 maxPageIndex = 40
 downloadPath = 'exhentai'
 winProxy = 'http://127.0.0.1:1080'

diff --git a/server/service/ExhentaiService.ts b/server/service/ExhentaiService.ts
@@ -104,7 +104,7 @@ export default class ExhentaiService {
     const timer = Date.now();
     try {
       await this.gotoTargetPage(targetUrl, true);
-      if (Date.now() - timer >= 30 * 1000) {
+      if (Date.now() - timer >= this.config.requestTime) {
         throw Error('time out at page index: ' + pageIndex);
       }
     } catch (err) {
@@ -244,6 +244,7 @@ export default class ExhentaiService {
       const item = imageUrl[i];
       const pageIndex = i + 1;
       const targetUrl = joinWithRootPath(`${prefixPath}/${pageIndex}.jpg`);
+      fs.ensureFileSync(targetUrl);
 
       const handleDownloadStream = async () => {
         trace('download begin: ' + item);
@@ -256,7 +257,7 @@ export default class ExhentaiService {
           .on('data', () => {
             const newTimer = Date.now();
             if (
-              newTimer - timer >= 30 * 1000 &&
+              newTimer - timer >= this.config.requestTime &&
               fs.existsSync(targetUrl) &&
               status
             ) {
@@ -266,6 +267,12 @@ export default class ExhentaiService {
               status = false;
             }
           })
+          .on('error', () => {
+            error('unexpected error occar, will re-request later');
+            fs.unlinkSync(targetUrl);
+            trace(`unlink: ${pageIndex}.jpg`);
+            handleDownloadStream();
+          })
           .pipe(
             imageStream.on('finish', () => {
               if (status) {
@@ -291,21 +298,18 @@ export default class ExhentaiService {
     }
   };
 
-  getThumbnaiUrlFromDetailPage = async () => {
-    const restDetailUrls: string[] = await this.getUrlFromPaginationInfo();
-    const firstPageThumbnailUrls: string[] = await this.getAllThumbnaiUrls();
-    await this.page.waitFor(this.config.waitTime);
-
+  getThumbnailUrlFromDetailPage = async (restDetailUrls: string[]) => {
+    const result: string[] = [];
     for (const item of restDetailUrls) {
-      await this.gotoTargetPage(item);
-      const thumbnailUrlsFromNextPage: string[] = await this.getAllThumbnaiUrls();
-      firstPageThumbnailUrls.push(...thumbnailUrlsFromNextPage);
+      await this.gotoTargetPage(item, true);
+      const thumbnailUrls: string[] = await this.getAllThumbnaiUrls();
+      result.push(...thumbnailUrls);
 
-      info('image length: ' + firstPageThumbnailUrls.length);
+      info('image length: ' + result.length);
 
       await this.page.waitFor(this.config.waitTime);
     }
-    return firstPageThumbnailUrls;
+    return result;
   };
 
   fetchImageUrls = async (thumbnailUrls: string[]) => {
@@ -339,7 +343,7 @@ export default class ExhentaiService {
     const timer = Date.now();
     try {
       await this.gotoTargetPage(targetUrl, true);
-      if (Date.now() - timer >= 30 * 1000) {
+      if (Date.now() - timer >= this.config.requestTime) {
         throw Error('time out at page index: ' + pageIndex);
       }
     } catch (err) {

diff --git a/server/utils/exhentai.ts b/server/utils/exhentai.ts
@@ -1,9 +1,10 @@
 import { joinWithRootPath, readJsonFile } from './common';
 import fs from 'fs-extra';
-import path from 'path';
+import path, { dirname, join } from 'path';
+import glob from 'glob';
 import { getTargetResource } from './resource';
 
-const { listInfoPath } = getTargetResource('server').exhentai;
+const { listInfoPath, downloadPath } = getTargetResource('server').exhentai;
 
 export const getLatestListFileName = () => {
   const infoPath = joinWithRootPath(listInfoPath);
@@ -46,3 +47,14 @@ export const getBaseNameOfImage = (dir: string) =>
     .readdirSync(joinWithRootPath(dir))
     .filter(f => path.extname(f) !== 'json')
     .map(f => path.basename(f, '.jpg'));
+
+export const getEmptyRestDetailUrlInfo = () =>
+  glob
+    .sync(joinWithRootPath(downloadPath) + '/**/restDetailUrls.json')
+    .filter(item => {
+      const dirName = dirname(item);
+      if (!fs.existsSync(join(dirName, 'detailImageUrls.json'))) {
+        return true;
+      }
+      return false;
+    });