From 722d2d3c6b8477e46711a1ca1fd63effd4816205 Mon Sep 17 00:00:00 2001 From: Openbyte Date: Fri, 4 May 2018 23:34:50 +0200 Subject: [PATCH] updated docs, updated domains for gogoanime; updated dependencies --- docs/classes/hoster.html | 10 +-- docs/classes/hosterinfo.html | 6 +- docs/classes/hosterscrapper.html | 26 +++---- docs/classes/runner.html | 6 +- docs/classes/runnerlist.html | 10 +-- docs/classes/scrap.html | 8 +- docs/classes/scrapper.html | 26 +++---- docs/classes/scrapperlist.html | 14 ++-- docs/classes/source.html | 10 +-- docs/classes/sourceinfo.html | 8 +- docs/classes/sourcescrapper.html | 26 +++---- docs/globals.html | 76 +++++++++---------- docs/index.html | 21 +++-- docs/interfaces/array.html | 2 +- docs/interfaces/domrunnerargs.html | 6 +- docs/interfaces/domrunnerscrapperargs.html | 14 ++-- docs/interfaces/htmlrunnerargs.html | 6 +- docs/interfaces/htmlrunnerscrapperargs.html | 10 +-- docs/interfaces/puppeteerrunnerargs.html | 6 +- .../puppeteerrunnerscrapperargs.html | 10 +-- docs/interfaces/regexp.html | 2 +- docs/interfaces/runnerargs.html | 6 +- docs/interfaces/runnerscrapperargs.html | 6 +- lib/scrappers.ts | 18 ++++- package-lock.json | 15 ++-- package.json | 2 +- 26 files changed, 181 insertions(+), 169 deletions(-) diff --git a/docs/classes/hoster.html b/docs/classes/hoster.html index 857dd324..5e78a476 100644 --- a/docs/classes/hoster.html +++ b/docs/classes/hoster.html @@ -106,7 +106,7 @@

constructor

  • Parameters

    @@ -142,7 +142,7 @@

    Optional data

    data: undefined | string
    @@ -152,7 +152,7 @@

    name

    name: string
    @@ -162,7 +162,7 @@

    Optional quality

    quality: string | number
    @@ -172,7 +172,7 @@

    url

    url: string
    diff --git a/docs/classes/hosterinfo.html b/docs/classes/hosterinfo.html index 26796173..203aa396 100644 --- a/docs/classes/hosterinfo.html +++ b/docs/classes/hosterinfo.html @@ -109,7 +109,7 @@

    constructor

  • Parameters

    @@ -139,7 +139,7 @@

    hoster

    hoster: Hoster[]
    @@ -149,7 +149,7 @@

    Optional title

    title: undefined | string
    diff --git a/docs/classes/hosterscrapper.html b/docs/classes/hosterscrapper.html index 797ea342..9d332a9e 100644 --- a/docs/classes/hosterscrapper.html +++ b/docs/classes/hosterscrapper.html @@ -128,7 +128,7 @@

    constructor

    Parameters

    @@ -191,7 +191,7 @@

    exec

    @@ -202,7 +202,7 @@

    name

    @@ -213,7 +213,7 @@

    Optional runner

    @@ -224,7 +224,7 @@

    Optional runnerOptions

    Inherited from Scrapper.runnerOptions

    @@ -243,7 +243,7 @@

    domain

    Returns string[]

    @@ -252,7 +252,7 @@

    Returns string

    Inherited from Scrapper.domain

    Parameters

    @@ -277,7 +277,7 @@

    type

    Returns string

    @@ -286,7 +286,7 @@

    Returns string

    Inherited from Scrapper.type

    Parameters

    @@ -313,7 +313,7 @@

    isApplicable

    Parameters

    @@ -337,7 +337,7 @@

    run

    Parameters

    @@ -361,7 +361,7 @@

    setRunnerByType

    Parameters

    @@ -385,7 +385,7 @@

    toJSON

    Returns object

    diff --git a/docs/classes/runner.html b/docs/classes/runner.html index f3f92912..9982a0e4 100644 --- a/docs/classes/runner.html +++ b/docs/classes/runner.html @@ -104,7 +104,7 @@

    constructor

  • Parameters

    @@ -129,7 +129,7 @@

    run

    run: RunnerFunction
    @@ -139,7 +139,7 @@

    type

    type: string
    diff --git a/docs/classes/runnerlist.html b/docs/classes/runnerlist.html index 66d37919..7d74cffd 100644 --- a/docs/classes/runnerlist.html +++ b/docs/classes/runnerlist.html @@ -152,7 +152,7 @@

    constructor

  • Parameters

    @@ -166,7 +166,7 @@

    Returns

    Parameters

    @@ -221,7 +221,7 @@

    getByType

  • Parameters

    @@ -247,7 +247,7 @@

    toMap

  • Returns Map<string, Runner>

    @@ -881,7 +881,7 @@

    Static flatMap

    diff --git a/docs/classes/scrap.html b/docs/classes/scrap.html index eaa2c154..a13abca5 100644 --- a/docs/classes/scrap.html +++ b/docs/classes/scrap.html @@ -105,7 +105,7 @@

    constructor

  • Parameters

    @@ -138,7 +138,7 @@

    info

    info: Info | null
    @@ -148,7 +148,7 @@

    scrapper

    scrapper: string
    @@ -158,7 +158,7 @@

    url

    url: string
    diff --git a/docs/classes/scrapper.html b/docs/classes/scrapper.html index 1140a9b5..038b1d1b 100644 --- a/docs/classes/scrapper.html +++ b/docs/classes/scrapper.html @@ -130,7 +130,7 @@

    constructor

  • Parameters

    @@ -195,7 +195,7 @@

    exec

    exec: ScrapperExec
    @@ -205,7 +205,7 @@

    name

    name: string
    @@ -215,7 +215,7 @@

    Optional runner

    runner: Runner
    @@ -225,7 +225,7 @@

    Optional runnerOptionsrunnerOptions: undefined | object @@ -243,7 +243,7 @@

    domain

  • Returns string[]

    @@ -251,7 +251,7 @@

    Returns string

    Parameters

    @@ -275,7 +275,7 @@

    type

  • Returns string

    @@ -283,7 +283,7 @@

    Returns string

    Parameters

    @@ -309,7 +309,7 @@

    isApplicable

  • Parameters

    @@ -332,7 +332,7 @@

    run

  • Parameters

    @@ -355,7 +355,7 @@

    setRunnerByType

  • Parameters

    @@ -378,7 +378,7 @@

    toJSON

  • Returns object

    diff --git a/docs/classes/scrapperlist.html b/docs/classes/scrapperlist.html index fb1a5e13..ee6b5807 100644 --- a/docs/classes/scrapperlist.html +++ b/docs/classes/scrapperlist.html @@ -159,7 +159,7 @@

    constructor

  • Parameters

    @@ -173,7 +173,7 @@

    Returns

    Parameters

    @@ -228,7 +228,7 @@

    hosters

  • Returns string[]

    @@ -248,7 +248,7 @@

    getAllApplicable

  • Parameters

    @@ -271,7 +271,7 @@

    getByName

  • Parameters

    @@ -297,7 +297,7 @@

    getFirstApplicable

  • Parameters

    @@ -940,7 +940,7 @@

    Static flatMap

    diff --git a/docs/classes/source.html b/docs/classes/source.html index 61bba42c..2775a7c9 100644 --- a/docs/classes/source.html +++ b/docs/classes/source.html @@ -106,7 +106,7 @@

    constructor

  • Parameters

    @@ -142,7 +142,7 @@

    Optional codec

    codec: undefined | string
    @@ -152,7 +152,7 @@

    Optional resolution

    resolution: undefined | string
    @@ -162,7 +162,7 @@

    Optional type

    type: undefined | string
    @@ -172,7 +172,7 @@

    url

    url: string
    diff --git a/docs/classes/sourceinfo.html b/docs/classes/sourceinfo.html index 7249168e..444daefb 100644 --- a/docs/classes/sourceinfo.html +++ b/docs/classes/sourceinfo.html @@ -110,7 +110,7 @@

    constructor

  • Parameters

    @@ -143,7 +143,7 @@

    Optional poster

    poster: undefined | string
    @@ -153,7 +153,7 @@

    source

    source: Source[]
    @@ -163,7 +163,7 @@

    Optional title

    title: undefined | string
    diff --git a/docs/classes/sourcescrapper.html b/docs/classes/sourcescrapper.html index 10a95c6d..e474b0f9 100644 --- a/docs/classes/sourcescrapper.html +++ b/docs/classes/sourcescrapper.html @@ -128,7 +128,7 @@

    constructor

    Parameters

    @@ -191,7 +191,7 @@

    exec

    @@ -202,7 +202,7 @@

    name

    @@ -213,7 +213,7 @@

    Optional runner

    @@ -224,7 +224,7 @@

    Optional runnerOptions

    Inherited from Scrapper.runnerOptions

    @@ -243,7 +243,7 @@

    domain

    Returns string[]

    @@ -252,7 +252,7 @@

    Returns string

    Inherited from Scrapper.domain

    Parameters

    @@ -277,7 +277,7 @@

    type

    Returns string

    @@ -286,7 +286,7 @@

    Returns string

    Inherited from Scrapper.type

    Parameters

    @@ -313,7 +313,7 @@

    isApplicable

    Parameters

    @@ -337,7 +337,7 @@

    run

    Parameters

    @@ -361,7 +361,7 @@

    setRunnerByType

    Parameters

    @@ -385,7 +385,7 @@

    toJSON

    Returns object

    diff --git a/docs/globals.html b/docs/globals.html index 2f430af2..2fd05a2c 100644 --- a/docs/globals.html +++ b/docs/globals.html @@ -149,7 +149,7 @@

    DOMRunnerScrapper

    DOMRunnerScrapper: RunnerScrapper<DOMRunnerScrapperArgs>
    @@ -159,7 +159,7 @@

    HTMLRunnerScrapper

    HTMLRunnerScrapper: RunnerScrapper<HTMLRunnerScrapperArgs>
    @@ -169,7 +169,7 @@

    HosterScrapperExec

    HosterScrapperExec: function
    @@ -205,7 +205,7 @@

    PuppeteerRunnerScrapper

    PuppeteerRunnerScrapper: RunnerScrapper<PuppeteerRunnerScrapperArgs>
    @@ -215,7 +215,7 @@

    RunnerFunction

    RunnerFunction: function
    @@ -262,7 +262,7 @@

    RunnerScrapper

    RunnerScrapper: function
    @@ -293,7 +293,7 @@

    ScrapperExec

    ScrapperExec: function
    @@ -329,7 +329,7 @@

    SourceScrapperExec

    SourceScrapperExec: function
    @@ -365,7 +365,7 @@

    UrlRunnerScrapper

    UrlRunnerScrapper: function
    @@ -410,7 +410,7 @@

    Const _private

    _private: WeakMap<object, any> = new WeakMap()
    @@ -420,7 +420,7 @@

    Const defaultScrappers

    defaultScrappers: any = new ScrapperList(new SourceScrapper({name: 'dom',runner: 'dom',exec: ({dom}) => {const info = new SourceInfo();const videos = dom.getElementsByTagName('video');for (const video of videos) {const vsrc = video.attrs.src;if (vsrc)info.source.push(new Source({url: vsrc,type: video.attrs.type}));const poster = video.attrs.poster;if (poster)info.poster = poster;const sources = video.getElementsByTagName('source');for (const source of sources) {const ssrc = source.attrs.src;if (ssrc)info.source.push(new Source({url: ssrc,type: source.attrs.type}));}}const titles = dom.getElementsByTagName('title');if (titles.length >= 1)info.title = titles[0].getText();return info;}}),new SourceScrapper({name: 'puppeteer',runner: 'puppeteer',exec: async ({page}) => {return page.evaluate((scrapper) => {return eval('(' + scrapper + ')')(document);}, defaultScrappers.getByName('dom').toString());}}),new SourceScrapper({name: 'html',runner: 'html',exec: async (args) => {return defaultScrappers.getByName('dom').exec(htmlToDomArgs(args));}}))
    @@ -430,7 +430,7 @@

    Const domRunner

    domRunner: Runner = new Runner('dom',async ({url, scrapper, options= {}}: DOMRunnerArgs) => {return htmlRunner.run({url,scrapper: async (args) => scrapper(htmlToDomArgs(args)),options});})
    @@ -440,7 +440,7 @@

    Const htmlRunner

    htmlRunner: Runner = new Runner('html',async ({url, scrapper, options= {}}: HTMLRunnerArgs) => {let _options: any = {config: {url,method: 'get',headers: {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0'}}};if (options.axios)_options = objectMerge(_options, options.axios);const response = await axios.request(_options.config);return scrapper({url,html: response.data,response,scrapper,runners});})
    @@ -450,7 +450,7 @@

    Const puppeteerRunner

    puppeteerRunner: Runner = new Runner('puppeteer',async ({url, scrapper, options= {}}: PuppeteerRunnerArgs) => {let _options: any = {config: {headless: false},requestInterception: {active: true,block: ({ request }: { request: any, resourceType: string, url: string, page: any, browser: any }) =>request.resourceType === 'font'},init: null};if (options.puppeteer)_options = objectMerge(_options, options.puppeteer);_options.config = objectMerge(_options.config, config.puppeteer);const browser = await puppeteerLaunch(_options.config);const page = await browser.newPage();await page.emulateMedia('screen');await page.setRequestInterception(_options.requestInterception && _options.requestInterception.active);page.on('request', (request: any) => {const block =_options.requestInterception &&_options.requestInterception.block &&_options.requestInterception.block({request,resourceType: request.resourceType(),url: request.url(),page,browser});if (typeof block === 'undefined')return;if (block)request.abort();else request.continue();});if (typeOf(_options.init) === 'function') {_options.init({page,browser});}/* await page.evaluateOnNewDocument(async () => {HTMLVideoElement.prototype.canPlayType = function () { return "probably"; };}) ;*/await page.goto(url);try {return await scrapper({url,page,browser,scrapper,runners});} catch (e) {return null;} finally {await browser.close();}})
    @@ -460,7 +460,7 @@

    Const runners

    runners: RunnerList = new RunnerList()
    @@ -470,7 +470,7 @@

    Const urlRunner

    urlRunner: Runner = new Runner('url',async ({url, scrapper, options= {}}: RunnerArgs<UrlRunnerScrapper>) => {return scrapper({url,scrapper,runners});})
    @@ -487,7 +487,7 @@

    all

  • Returns ScrapperList

    @@ -504,7 +504,7 @@

    htmlToDomArgs

  • Parameters

    @@ -526,7 +526,7 @@

    Const classes

    classes: object
    @@ -535,7 +535,7 @@

    Hoster

    Hoster: Hoster
    @@ -545,7 +545,7 @@

    HosterInfo

    HosterInfo: HosterInfo
    @@ -555,7 +555,7 @@

    HosterScrapper

    HosterScrapper: HosterScrapper
    @@ -565,7 +565,7 @@

    Info

    Info: Info
    @@ -575,7 +575,7 @@

    Runner

    Runner: Runner
    @@ -585,7 +585,7 @@

    Scrap

    Scrap: Scrap
    @@ -595,7 +595,7 @@

    Scrapper

    Scrapper: Scrapper
    @@ -605,7 +605,7 @@

    ScrapperList

    ScrapperList: ScrapperList
    @@ -615,7 +615,7 @@

    Source

    Source: Source
    @@ -625,7 +625,7 @@

    SourceInfo

    SourceInfo: SourceInfo
    @@ -635,7 +635,7 @@

    SourceScrapper

    SourceScrapper: SourceScrapper
    @@ -646,7 +646,7 @@

    Let config

    config: object
    @@ -655,7 +655,7 @@

    showError

    showError: boolean = false
    @@ -665,7 +665,7 @@

    puppeteer

    puppeteer: object
    @@ -674,7 +674,7 @@

    headless

    headless: boolean = true
    @@ -686,7 +686,7 @@

    Const scrappers

    scrappers: object
    @@ -695,17 +695,17 @@

    hoster

    hoster: ScrapperList = new ScrapperList(new HosterScrapper({name: 'Gogoanime',domain: 'gogoanime.io',runner: 'dom',exec: ({dom}) => {const bodys = dom.getElementsByTagName('body');if (!(bodys.length >= 1))return null;const body = bodys[0];const containers = body.getElementsByClassName('anime_muti_link'); // <-- no typo hereif (!(containers.length >= 1))return null;const container = containers[0];const items = container.getElementsByTagName('li');const info = new HosterInfo();for (const item of items) {const links = item.getElementsByTagName('a');if (!(links.length >= 1))continue;const link = links[0];const url = link.attrs['data-video'];if (!url)continue;let name = link.getText();if (name) {const spans = link.getElementsByTagName('span');if (spans.length >= 1) {const span = spans[0];name = name.replace(span.textContent, '');}}info.hoster.push(new Hoster({url,name: name.trim()}));}const titles = dom.getElementsByTagName('title');if (titles.length >= 1)info.title = titles[0].getText();return info;}}),new HosterScrapper({name: 'Kissanime',domain: 'kissanime.ru',runner: 'puppeteer',runnerOptions: {puppeteer: {requestInterception: {active: true,block: ({url, resourceType}) =>!url.includes('kissanime.ru') ||resourceType === 'image' ||resourceType === 'font' ||resourceType === 'stylesheet'// ||// url.includes('2mdnsys.com') ||// url.includes('ti553.com')}}},exec: async ({page, url}) => {const gotoIfNotTarget = async (_page) => {if (normalizeUrl(_page.url()) !== normalizeUrl(url)) {await _page.goto(url, {waitUntil: 'domcontentloaded'});}};await page.waitForNavigation({waitUntil: 'domcontentloaded'});await gotoIfNotTarget(page);// await page.waitForSelector('#formVerify img');while (page.url().toLowerCase().includes('special/areyouhuman2?reurl')) {const verifyOptions = await page.$$('#formVerify img');const r = random.generator({min: 0,max: verifyOptions.length - 1,integer: true});const r1 = r();let r2;do { r2 = r(); }while (r1 === r2);await verifyOptions[r1].click();await verifyOptions[r2].click();const response = await page.waitForNavigation({waitUntil: 'domcontentloaded'});if (response.status() === 500)await page.goto(url, {waitUntil: 'domcontentloaded'});if (page.url().toLowerCase().includes('special/areyouhuman2') &&(await page.$eval('body', (body) => body.innerText)).startsWith('Wrong')) {const link = await page.$('a');await link.click();await page.waitForSelector('#formVerify img');}}await gotoIfNotTarget(page);const waitAndGetVideoFrameSource = async (_page) => {await _page.waitForSelector('#divContentVideo iframe');return _page.$eval('#divContentVideo iframe', (frame) => frame.src);};const defaultUrl = await waitAndGetVideoFrameSource(page);const hosterOptionsHandles = await page.$$('#selectServer option');const hosterOptions: Array<{ name: string, url: string, default: boolean }> = [];const info = new HosterInfo();for (const hosterHandle of hosterOptionsHandles)hosterOptions.push({name: await (await hosterHandle.getProperty('innerText')).jsonValue(),url: await (await hosterHandle.getProperty('value')).jsonValue(),default: await (await hosterHandle.getProperty('selected')).jsonValue()});const u = urlparser.parse(url);const baseUrl = `${u.host.protocol}://${u.host.hostname}`;for (const hoster of hosterOptions) {if (hoster.name.toLowerCase() === 'beta server')continue;if (hoster.default) {info.hoster.push(new Hoster({name: hoster.name,url: defaultUrl}));continue;}await page.goto(baseUrl + hoster.url, {waitUntil: 'domcontentloaded'});info.hoster.push(new Hoster({name: hoster.name,url: await waitAndGetVideoFrameSource(page)}));}return info;}}),new HosterScrapper({name: 'MasterAnime',domain: 'masterani.me',runner: 'html',exec: ({ html }) => {const argsRegex = /<script[^>]*>\s*(?:(?:var|let|const)\s*)?args\s*=\s*({.*?})\s*(;\s*)?<\/script>/;const argsData = argsRegex.exec(html);if (argsData === null || argsData.length < 2)return null;const argsString = argsData[1];const args = objectLiteralStringToObject.parse(argsString) as {anime: {info: {id: number,title: string,slug: string,episode_length: number},poster: string,episodes: {current: {id: number,episode: string,subbed: number,dubbed: number,type: number,title: string,duration: number,created_at: string,tvdb_id: number,description: string | null,aired: string,users: Array<{id: number,name: string,last_time_seen: string,is_online: boolean,avatar: {id: string,path: string,extension: string,file: string}}> | null,extra_viewers: number},next: {id: number,episode: string},prev: {id: number,episode: string}}},mirrors: Array<{id: number,host_id: number,embed_id: string,quality: number,type: number,host: {id: number,name: string,embed_prefix: string,embed_suffix: string | null}}>,auto_update: number[]};return new HosterInfo({title: args.anime.info.title,hoster: args.mirrors.map(e => new Hoster({name: e.host.name,url:e.host.embed_prefix.replace(/\\\//g, '/') +e.embed_id +e.host.embed_suffix || '',quality: e.quality}))});}}))

    stream

    -
    stream: ScrapperList = new ScrapperList(new SourceScrapper({name: 'Openload',domain: ['openload.co', 'oload.tv'],runner: 'puppeteer',exec: async ({page}) => {const streamurl = await page.$eval('[id*=stream]',(e) => e.innerText);const title = await page.$eval('meta[name="description"], meta[name="og:title"], meta[name="twitter:title"]',(e) => e.content);const thumb = await page.$eval('meta[name="og:image"], meta[name="twitter:image"]',(e) => e.content);return new SourceInfo({source: new Source({url: `https://openload.co/stream/${await streamurl}?mime=true`}),title,poster: thumb});}}),new SourceScrapper({name: 'Streamcloud',domain: 'streamcloud.eu',runner: 'url',exec: async ({url, runners}) => {const dataRegex = new RegExp(/\/([^\/.]+)\/(.*)\.[^.]+/, 'i');const data = dataRegex.exec(url);if (data === null || data.length <= 2)return null;const id = data[1];const name = data[2];if (!id || !name)return null;const htmlRunner = runners.getByType('html');if (!(htmlRunner instanceof Runner))return null;const html = await htmlRunner.run({url,scrapper: ({html: _html}) => removeNewline(_html),options: {axios: {config: {method: 'post',headers: {'Referer': url,'Content-Type': 'application/x-www-form-urlencoded','Cache-Control': 'no-cache','Connection': 'Keep-Alive'},data: queryString.stringify({fname: name,hash: '',id,op: 'download1',referer: '',usr_login: '',imhuman: 'Weiter+zum+Video'})}}}}) as string;const configRegex = new RegExp(/jwplayer\("[^"]+"\).setup\({(.*?)}\);/, 'i');const propRegex = new RegExp(/(\w+)\s*:\s*"?(.*?)"?\s*,/, 'ig');const tabRegex = /(?:\\t|\t)/g;const configData = configRegex.exec(html);if (!configData || configData.length < 1)return null;const configRAW = configData[1].replace(tabRegex, '');const propData = propRegex.execAll(configRAW);const props: any = {};for (const e of propData)if (e && e.length >= 2)props[e[1]] = e[2];return new SourceInfo({poster: props.image,title: name,source: new Source({url: props.file,resolution: props.width + 'x' + props.height})});}}),new SourceScrapper({name: 'Vidzi',domain: 'vidzi.tv',runner: 'puppeteer',exec: async ({page}) => {// tslint:disable-next-linelet jwplayer; // remove typescript error "cannot find name 'jwplayer'"const config = await page.evaluate(() => jwplayer().getConfig().playlistItem);const title = await page.$eval('.video-title', (t) => t.innerText);return new SourceInfo({poster: config.image,title,source: config.allSources.map((s) => new Source({url: s.file,type: s.type}))});}}),new SourceScrapper({name: 'Vidstreaming',domain: 'vidstreaming.io',runner: 'html',exec: async ({html}) => {const titleregex =new RegExp(/<title>([^<]+)<\/title>/);const setupregex =new RegExp(/playerInstance\.setup\({\s*sources\s*:\s*\[\s*{\s*(.*?)\s*}\s*]\s*,?\s*}\);/, 'gi');const loadregex =new RegExp(/playerInstance\.load\({\s*(.*?)\s*,?\s*}\)\s*;/, 'gi');html = removeNewline(html);const setupjson ='[' + setupregex.execAll(html).map((match) => match.length > 1 ? match[1] : null).filter((e) => e !== null).map((s) => s.replace(/'/g, '"')).map((s) => s.replace(/(^|,|{)\s*(file|label|type)\s*:/g, '$1"$2":')).map((s) => '{' + s + '}').reduce((a, c) => a + ',' + c) + ']';const setupdata = JSON.parse(setupjson);const loadjson ='[' + loadregex.execAll(html).map((match) => match.length > 1 ? match[1] : null).filter((e) => e !== null).map((s) => s.replace(/'/g, '"')).map((s) => s.replace(/(^|,|{)\s*(file|label|type)\s*:/g, '$1"$2":')).map((s) => '{' + s + '}').reduce((a, c) => a + ',' + c) + ']';const loaddata = JSON.parse(loadjson);const srcdata = setupdata.concat(loaddata).filter((e) => typeOf(e.file) === 'string' && !e.file.includes('error.com'));const titles = titleregex.exec(html);const title = titles !== null && titles.length >= 1 ? titles[0] : undefined;return new SourceInfo({title,source: srcdata.map((e) => new Source({url: e.file}))});}}),new SourceScrapper({name: 'Streamango',domain: 'streamango.com',runner: 'puppeteer',exec: async ({page}) => {// tslint:disable-next-linelet srces; // remove typescript error "cannot find name 'srces'"const srcs = await page.evaluate(() => {for (const script of document.getElementsByTagName('script')) {if (script.src)continue;if (script.text.includes('srces.push')) {eval(script.innerText);return srces;}}});return new SourceInfo({source: srcs.map((s) => new Source({url: s.src,type: s.type,resolution: s.height}))});}}),new SourceScrapper({name: 'Rapidvideo',domain: 'rapidvideo.com',runner: 'dom',exec: (args= {}) => defaultScrappers.getByName('dom').exec(args)}),new SourceScrapper({name: 'Stream.Moe',domain: 'stream.moe',runner: 'html',exec: async args => {const html = args.html;const encodedDataRegex = /atob\((['"])(.*?)\1\)/i;const encodedData = encodedDataRegex.exec(html);if (encodedData === null || encodedData.length < 3)return null;const encoded = encodedData[2];const decoded = Buffer.from(encoded, 'base64').toString('ascii');const info = await defaultScrappers.getByName('html').exec({ ...args, html: decoded });const titleRegex = /<title[^>]*>\s*(?:\[.*?])\s*(.*?)\s*(?:\[.*?]).*?<\/title>/i;const titleData = titleRegex.exec(html);if (titleData !== null && titleData.length >= 1)info.title = titleData[1];return info;}}))
    +
    stream: ScrapperList = new ScrapperList(new SourceScrapper({name: 'Openload',domain: ['openload.co', 'oload.tv', 'oload.win'],runner: 'puppeteer',exec: async ({page}) => {const streamurl = await page.$eval('[id*=stream], div[style*="display:none"] p:last-of-type',e => e.innerText);const title = await page.$eval('meta[name="description"], meta[name="og:title"], meta[name="twitter:title"]',e => e.content).catch(e => undefined);const thumb = await page.$eval('meta[name="og:image"], meta[name="twitter:image"]',e => e.content).catch(e => undefined);return new SourceInfo({source: new Source({url: `https://openload.co/stream/${await streamurl}?mime=true`}),title,poster: thumb});}}),new SourceScrapper({name: 'Streamcloud',domain: 'streamcloud.eu',runner: 'url',exec: async ({url, runners}) => {const dataRegex = new RegExp(/\/([^\/.]+)\/(.*)\.[^.]+/, 'i');const data = dataRegex.exec(url);if (data === null || data.length <= 2)return null;const id = data[1];const name = data[2];if (!id || !name)return null;const htmlRunner = runners.getByType('html');if (!(htmlRunner instanceof Runner))return null;const html = await htmlRunner.run({url,scrapper: ({html: _html}) => removeNewline(_html),options: {axios: {config: {method: 'post',headers: {'Referer': url,'Content-Type': 'application/x-www-form-urlencoded','Cache-Control': 'no-cache','Connection': 'Keep-Alive'},data: queryString.stringify({fname: name,hash: '',id,op: 'download1',referer: '',usr_login: '',imhuman: 'Weiter+zum+Video'})}}}}) as string;const configRegex = new RegExp(/jwplayer\("[^"]+"\).setup\({(.*?)}\);/, 'i');const propRegex = new RegExp(/(\w+)\s*:\s*"?(.*?)"?\s*,/, 'ig');const tabRegex = /(?:\\t|\t)/g;const configData = configRegex.exec(html);if (!configData || configData.length < 1)return null;const configRAW = configData[1].replace(tabRegex, '');const propData = propRegex.execAll(configRAW);const props: any = {};for (const e of propData)if (e && e.length >= 2)props[e[1]] = e[2];return new SourceInfo({poster: props.image,title: name,source: new Source({url: props.file,resolution: props.width + 'x' + props.height})});}}),new SourceScrapper({name: 'Vidzi',domain: 'vidzi.tv',runner: 'puppeteer',exec: async ({page}) => {// tslint:disable-next-linelet jwplayer; // remove typescript error "cannot find name 'jwplayer'"const config = await page.evaluate(() => jwplayer().getConfig().playlistItem);const title = await page.$eval('.video-title', (t) => t.innerText);return new SourceInfo({poster: config.image,title,source: config.allSources.map((s) => new Source({url: s.file,type: s.type}))});}}),new SourceScrapper({name: 'Vidstreaming',domain: 'vidstreaming.io',runner: 'html',exec: async ({html}) => {const titleregex = /<title>([^<]+)<\/title>/;const dataregex = /playerInstance\.(setup|load)\(({.*?})\)/gi;html = removeNewline(html);const data: Array<{sources: Array<{file: string,type: string,label: string}>,tracks: Array<{kind: 'thumbnail' | string,file: string}>,image: string,file: string,type: string,label: string}> = dataregex.execAll(html).map(match => match.length >= 3 ? match[2] : null).filter(e => e).map(e => objectLiteralStringToObject.parse(e)).filter(e => e);const sources = new Set(data.filter(e => Array.isArray(e.sources) && e.sources.length > 0).flatMap(e => e.sources).filter(e => e && typeOf(e.file) === 'string' && !e.file.includes('error.com')));const images = new Set(data.filter(e => e.image).map(e => e.image));const titles = titleregex.exec(html);return new SourceInfo({title: titles && titles[0] || undefined,source: [...sources].map(e => new Source({url: e.file,type: e.type || undefined})),poster: images[0] || undefined});}}),new SourceScrapper({name: 'Streamango',domain: 'streamango.com',runner: 'puppeteer',exec: async ({page}) => {// tslint:disable-next-linelet srces; // remove typescript error "cannot find name 'srces'"const srcs = await page.evaluate(() => {for (const script of document.getElementsByTagName('script')) {if (script.src)continue;if (script.text.includes('srces.push')) {eval(script.innerText);return srces;}}});return new SourceInfo({source: srcs.map((s) => new Source({url: s.src,type: s.type,resolution: s.height}))});}}),new SourceScrapper({name: 'Rapidvideo',domain: 'rapidvideo.com',runner: 'dom',exec: (args= {}) => defaultScrappers.getByName('dom').exec(args)}),new SourceScrapper({name: 'Stream.Moe',domain: 'stream.moe',runner: 'html',exec: async args => {const html = args.html;const encodedDataRegex = /atob\((['"])(.*?)\1\)/i;const encodedData = encodedDataRegex.exec(html);if (encodedData === null || encodedData.length < 3)return null;const encoded = encodedData[2];const decoded = Buffer.from(encoded, 'base64').toString('ascii');const info = await defaultScrappers.getByName('html').exec({ ...args, html: decoded });const titleRegex = /<title[^>]*>\s*(?:\[.*?])\s*(.*?)\s*(?:\[.*?]).*?<\/title>/i;const titleData = titleRegex.exec(html);if (titleData !== null && titleData.length >= 1)info.title = titleData[1];return info;}}))
    diff --git a/docs/index.html b/docs/index.html index cb5d0903..fa136dbe 100644 --- a/docs/index.html +++ b/docs/index.html @@ -74,20 +74,19 @@

    Source Scrapper

    Supported Sites

    Source

      -
    • openload.co
      • -
    • oload.tv
      • -
    • streamcloud.eu
      • -
    • vidzi.tv
      • -
    • vidstreaming.io
      • -
    • streamango.com
      • -
    • rapidvideo.com
      • -
    • stream.moe
      • +
    • Openload (openload.co, oload.tv, oload.win)
      • +
    • Streamcloud (streamcloud.eu)
      • +
    • Vidzi (vidzi.tv)
      • +
    • Vidstreaming (vidstreaming.io)
      • +
    • Streamango (streamango.com)
      • +
    • Rapidvideo (rapidvideo.com)
      • +
    • StreamMoe (stream.moe)

    Hoster

      -
    • masterani.me
      • -
    • gogoanime.io
      • -
    • kissanime.ru
      +
    • MasterAnime (masterani.me)
      • +
    • Gogoanime (gogoanime.io, gogoanime.se)
      • +
    • Kissanime (kissanime.ru)

    Getting Started

    diff --git a/docs/interfaces/array.html b/docs/interfaces/array.html index 6ffa3f56..37361ffd 100644 --- a/docs/interfaces/array.html +++ b/docs/interfaces/array.html @@ -105,7 +105,7 @@

    flatMap

  • diff --git a/docs/interfaces/domrunnerargs.html b/docs/interfaces/domrunnerargs.html index 39a4ad72..68c9c265 100644 --- a/docs/interfaces/domrunnerargs.html +++ b/docs/interfaces/domrunnerargs.html @@ -101,7 +101,7 @@

    Optional options

    @@ -112,7 +112,7 @@

    scrapper

    @@ -123,7 +123,7 @@

    url

    diff --git a/docs/interfaces/domrunnerscrapperargs.html b/docs/interfaces/domrunnerscrapperargs.html index 4e638acc..94f2c3c8 100644 --- a/docs/interfaces/domrunnerscrapperargs.html +++ b/docs/interfaces/domrunnerscrapperargs.html @@ -104,7 +104,7 @@

    dom

    dom: any
    @@ -115,7 +115,7 @@

    html

    @@ -125,7 +125,7 @@

    parser

    parser: any
    @@ -136,7 +136,7 @@

    response

    @@ -147,7 +147,7 @@

    runners

    @@ -158,7 +158,7 @@

    scrapper

    @@ -169,7 +169,7 @@

    url

    diff --git a/docs/interfaces/htmlrunnerargs.html b/docs/interfaces/htmlrunnerargs.html index 6057f992..fe36da09 100644 --- a/docs/interfaces/htmlrunnerargs.html +++ b/docs/interfaces/htmlrunnerargs.html @@ -101,7 +101,7 @@

    Optional options

    @@ -112,7 +112,7 @@

    scrapper

    @@ -123,7 +123,7 @@

    url

    diff --git a/docs/interfaces/htmlrunnerscrapperargs.html b/docs/interfaces/htmlrunnerscrapperargs.html index 54fc9cd6..0c5a1fd6 100644 --- a/docs/interfaces/htmlrunnerscrapperargs.html +++ b/docs/interfaces/htmlrunnerscrapperargs.html @@ -107,7 +107,7 @@

    html

    html: string
    @@ -117,7 +117,7 @@

    response

    response: AxiosResponse
    @@ -128,7 +128,7 @@

    runners

    @@ -139,7 +139,7 @@

    scrapper

    @@ -150,7 +150,7 @@

    url

    diff --git a/docs/interfaces/puppeteerrunnerargs.html b/docs/interfaces/puppeteerrunnerargs.html index c62f5c11..e79ee66f 100644 --- a/docs/interfaces/puppeteerrunnerargs.html +++ b/docs/interfaces/puppeteerrunnerargs.html @@ -101,7 +101,7 @@

    Optional options

    @@ -112,7 +112,7 @@

    scrapper

    @@ -123,7 +123,7 @@

    url

    diff --git a/docs/interfaces/puppeteerrunnerscrapperargs.html b/docs/interfaces/puppeteerrunnerscrapperargs.html index 53754a98..880f489d 100644 --- a/docs/interfaces/puppeteerrunnerscrapperargs.html +++ b/docs/interfaces/puppeteerrunnerscrapperargs.html @@ -102,7 +102,7 @@

    browser

    browser: Browser
    @@ -112,7 +112,7 @@

    page

    page: Page
    @@ -123,7 +123,7 @@

    runners

    @@ -134,7 +134,7 @@

    scrapper

    @@ -145,7 +145,7 @@

    url

    diff --git a/docs/interfaces/regexp.html b/docs/interfaces/regexp.html index c6697d49..9e9803ee 100644 --- a/docs/interfaces/regexp.html +++ b/docs/interfaces/regexp.html @@ -93,7 +93,7 @@

    execAll

    execAll: function
    diff --git a/docs/interfaces/runnerargs.html b/docs/interfaces/runnerargs.html index dbb8c949..8aaa14d2 100644 --- a/docs/interfaces/runnerargs.html +++ b/docs/interfaces/runnerargs.html @@ -114,7 +114,7 @@

    Optional options

    options: any
    @@ -124,7 +124,7 @@

    scrapper

    scrapper: T
    @@ -134,7 +134,7 @@

    url

    url: string
    diff --git a/docs/interfaces/runnerscrapperargs.html b/docs/interfaces/runnerscrapperargs.html index 46514311..887f4b44 100644 --- a/docs/interfaces/runnerscrapperargs.html +++ b/docs/interfaces/runnerscrapperargs.html @@ -103,7 +103,7 @@

    runners

    runners: RunnerList
    @@ -113,7 +113,7 @@

    scrapper

    scrapper: RunnerScrapper<any>
    @@ -123,7 +123,7 @@

    url

    url: string
    diff --git a/lib/scrappers.ts b/lib/scrappers.ts index 909e22cd..5bdf37b3 100644 --- a/lib/scrappers.ts +++ b/lib/scrappers.ts @@ -188,7 +188,21 @@ export const scrappers: { stream: ScrapperList, hoster: ScrapperList } = { html = removeNewline(html); - const data = dataregex + const data: Array<{ + sources: Array<{ + file: string, + type: string, + label: string + }>, + tracks: Array<{ + kind: 'thumbnail' | string, + file: string + }>, + image: string, + file: string, + type: string, + label: string + }> = dataregex .execAll(html) .map(match => match.length >= 3 ? match[2] : null) .filter(e => e) @@ -272,7 +286,7 @@ export const scrappers: { stream: ScrapperList, hoster: ScrapperList } = { hoster: new ScrapperList( new HosterScrapper({ name: 'Gogoanime', - domain: 'gogoanime.io', + domain: ['gogoanime.io', 'gogoanime.se'], runner: 'dom', exec: ({dom}) => { const bodys = dom.getElementsByTagName('body'); diff --git a/package-lock.json b/package-lock.json index f907cd4b..d1bcf50c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "integrity": "sha512-h3wnflb+jMTipvbbZnClgA2BexrT4w0GcfoCz5qyxd0IRsbqhLSyesM6mqZTAnhbVmhyTm5tuxfRu9R+8l+lGw==", "dev": true, "requires": { - "@types/node": "9.6.11" + "@types/node": "10.0.3" } }, "@types/glob": { @@ -27,7 +27,7 @@ "requires": { "@types/events": "1.2.0", "@types/minimatch": "3.0.3", - "@types/node": "9.6.11" + "@types/node": "10.0.3" } }, "@types/handlebars": { @@ -67,10 +67,9 @@ "dev": true }, "@types/node": { - "version": "9.6.11", - "resolved": "https://registry.npmjs.org/@types/node/-/node-9.6.11.tgz", - "integrity": "sha512-Zm2W8b3ouIP/rB4ZgIDBMsdi1a/5NuKG3IguCM6aUwXtT+h6g1/Lh59Td7S1QtHMRXU7AbrGRDJ7lbxW4MEDmg==", - "dev": true + "version": "10.0.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.0.3.tgz", + "integrity": "sha512-J7nx6JzxmtT4zyvfLipYL7jNaxvlCWpyG7JhhCQ4fQyG+AGfovAHoYR55TFx+X8akfkUJYpt5JG6GPeFMjZaCQ==" }, "@types/normalize-url": { "version": "1.9.1", @@ -91,7 +90,7 @@ "dev": true, "requires": { "@types/events": "1.2.0", - "@types/node": "9.6.11" + "@types/node": "10.0.3" } }, "@types/query-string": { @@ -113,7 +112,7 @@ "dev": true, "requires": { "@types/glob": "5.0.35", - "@types/node": "9.6.11" + "@types/node": "10.0.3" } }, "@types/typeof": { diff --git a/package.json b/package.json index 12886e80..e5fe676b 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ }, "devDependencies": { "@types/newline-remove": "^1.0.0", - "@types/node": "^9.6.11", + "@types/node": "^10.0.3", "@types/normalize-url": "^1.9.1", "@types/object-merge": "^2.5.0", "@types/puppeteer": "^1.3.0",