Skip to content

Commit

Permalink
add cloudflare error
Browse files Browse the repository at this point in the history
  • Loading branch information
gigobyte committed Jan 16, 2021
1 parent 61253dc commit f252865
Show file tree
Hide file tree
Showing 21 changed files with 62 additions and 12 deletions.
4 changes: 2 additions & 2 deletions __tests__/getEvent.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@ function sleep(ms: number): Promise<void> {
}

test('getEvent', async () => {
await sleep(1000)
await sleep(3000)
expect(await HLTV.getEvent({ id: 3452 })).toMatchSnapshot()
expect(await HLTV.getEvent({ id: 3552 })).toMatchSnapshot()
expect(await HLTV.getEvent({ id: 355 })).toMatchSnapshot()
await sleep(3000) // Sleep to prevent accidental ban by hltv
expect(await HLTV.getEvent({ id: 3005 })).toMatchSnapshot()
expect(await HLTV.getEvent({ id: 4356 })).toMatchSnapshot()
expect(await HLTV.getEvent({ id: 2870 })).toMatchSnapshot()
await sleep(1000)
await sleep(3000)
}, 30000)
1 change: 1 addition & 0 deletions __tests__/getMatch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ function sleep(ms: number): Promise<void> {
}

test('getMatch', async () => {
await sleep(3000)
expect(await HLTV.getMatch({ id: 2325765 })).toMatchSnapshot()
expect(await HLTV.getMatch({ id: 2302345 })).toMatchSnapshot()
await sleep(3000)
Expand Down
4 changes: 2 additions & 2 deletions __tests__/getMatchStats.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@ function sleep(ms: number): Promise<void> {
}

test('getMatchStats', async () => {
await sleep(1000)
await sleep(3000)
expect(await HLTV.getMatchStats({ id: 62979 })).toMatchSnapshot()
expect(await HLTV.getMatchStats({ id: 63146 })).toMatchSnapshot()
await sleep(3000)
expect(await HLTV.getMatchStats({ id: 63095 })).toMatchSnapshot()
await sleep(1000)
await sleep(3000)
}, 30000)
4 changes: 2 additions & 2 deletions __tests__/getPlayer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@ function sleep(ms: number): Promise<void> {
}

test('getPlayer', async () => {
await sleep(1000)
await sleep(3000)
expect(await HLTV.getPlayer({ id: 8000 })).toMatchSnapshot()
expect(await HLTV.getPlayer({ id: 7000 })).toMatchSnapshot()
await sleep(3000)
expect(await HLTV.getPlayer({ id: 3000 })).toMatchSnapshot()
expect(await HLTV.getPlayer({ id: 9123 })).toMatchSnapshot()
await sleep(1000)
await sleep(3000)
}, 30000)
2 changes: 1 addition & 1 deletion __tests__/getPlayerRankings.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ function sleep(ms: number): Promise<void> {
}

test('getMatchStats', async () => {
await sleep(1000)
await sleep(3000)
expect(
await HLTV.getPlayerRanking({
startDate: '2019-10-15',
Expand Down
3 changes: 3 additions & 0 deletions src/endpoints/getEvent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { FullEvent } from '../models/FullEvent'
import { HLTVConfig } from '../config'
import { fetchPage, toArray, getMapSlug } from '../utils/mappers'
import { popSlashSource } from '../utils/parsing'
import { checkForRateLimiting } from '../utils/checkForRateLimiting'

export const getEvent = (config: HLTVConfig) => async ({
id
Expand All @@ -10,6 +11,8 @@ export const getEvent = (config: HLTVConfig) => async ({
}): Promise<FullEvent> => {
const $ = await fetchPage(`${config.hltvUrl}/events/${id}/-`, config.loadPage)

checkForRateLimiting($)

const name = $('.eventname').text()
const dateStart =
Number($('td.eventdate span[data-unix]').first().attr('data-unix')) ||
Expand Down
3 changes: 3 additions & 0 deletions src/endpoints/getEvents.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { EventSize } from '../enums/EventSize'
import { EventType } from '../enums/EventType'
import { SimpleEvent } from '../models/SimpleEvent'
import { popSlashSource } from '../utils/parsing'
import { checkForRateLimiting } from '../utils/checkForRateLimiting'

export const getEvents = (config: HLTVConfig) => async ({
size,
Expand All @@ -15,6 +16,8 @@ export const getEvents = (config: HLTVConfig) => async ({
} = {}): Promise<EventResult[]> => {
const $ = await fetchPage(`${config.hltvUrl}/events`, config.loadPage)

checkForRateLimiting($)

const events = toArray($('.events-month'))
.map((eventEl) => {
const checkMonth = new Date(
Expand Down
3 changes: 3 additions & 0 deletions src/endpoints/getMatch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import {
getMapSlug,
getMatchPlayer
} from '../utils/mappers'
import { checkForRateLimiting } from '../utils/checkForRateLimiting'

const getTeamId = (el: cheerio.Cheerio): number | undefined => {
if (el.find('a').length) {
Expand All @@ -39,6 +40,8 @@ export const getMatch = (config: HLTVConfig) => async ({
config.loadPage
)

checkForRateLimiting($)

const title = $('.timeAndEvent .text').text().trim() || undefined
const date = Number($('.timeAndEvent .date').attr('data-unix'))
const format = $('.preformatted-text').text().split('\n')[0]
Expand Down
4 changes: 4 additions & 0 deletions src/endpoints/getMatchMapStats.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {
getMapSlug,
mapRoundElementToModel
} from '../utils/mappers'
import { checkForRateLimiting } from '../utils/checkForRateLimiting'

export type PlayerPerformanceStatsMap = {
[key: number]: PlayerPerformanceStats
Expand Down Expand Up @@ -74,6 +75,9 @@ export const getMatchMapStats = (config: HLTVConfig) => async ({
)
])

checkForRateLimiting(m$)
checkForRateLimiting(p$)

const matchPageID = Number(m$('.match-page-link').attr('href')!.split('/')[2])
const matchScore = [
Number(m$('.team-left .bold').text()),
Expand Down
3 changes: 3 additions & 0 deletions src/endpoints/getMatchStats.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
import { Event } from '../models/Event'
import { HLTVConfig } from '../config'
import { fetchPage, toArray } from '../utils/mappers'
import { checkForRateLimiting } from '../utils/checkForRateLimiting'

export const getMatchStats = (config: HLTVConfig) => async ({
id
Expand Down Expand Up @@ -49,6 +50,8 @@ export const getMatchStats = (config: HLTVConfig) => async ({
config.loadPage
)

checkForRateLimiting($)

const matchPageID = Number($('.match-page-link').attr('href')!.split('/')[2])
const matchScore = [
Number($('.team-left .bold').text()),
Expand Down
3 changes: 3 additions & 0 deletions src/endpoints/getMatches.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,15 @@ import { Team } from '../models/Team'
import { popSlashSource } from '../utils/parsing'
import { HLTVConfig } from '../config'
import { fetchPage, toArray } from '../utils/mappers'
import { checkForRateLimiting } from '../utils/checkForRateLimiting'

export const getMatches = (config: HLTVConfig) => async (): Promise<
(UpcomingMatch | LiveMatch)[]
> => {
const $ = await fetchPage(`${config.hltvUrl}/matches`, config.loadPage)

checkForRateLimiting($)

const liveMatches: LiveMatch[] = toArray($('.liveMatch-container')).map(
(matchEl) => {
const id = Number(matchEl.find('.a-reset').attr('href')!.split('/')[2])
Expand Down
4 changes: 4 additions & 0 deletions src/endpoints/getMatchesStats.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { Map } from '../enums/Map'
import { MapSlug } from '../enums/MapSlug'
import { HLTVConfig } from '../config'
import { fetchPage, toArray } from '../utils/mappers'
import { checkForRateLimiting } from '../utils/checkForRateLimiting'

export const getMatchesStats = (config: HLTVConfig) => async ({
startDate,
Expand All @@ -32,6 +33,9 @@ export const getMatchesStats = (config: HLTVConfig) => async ({
`${config.hltvUrl}/stats/matches?${query}&offset=${page * 50}`,
config.loadPage
)

checkForRateLimiting($)

page++

matches = matches.concat(
Expand Down
3 changes: 3 additions & 0 deletions src/endpoints/getOngoingEvents.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
import { HLTVConfig } from '../config'
import { fetchPage, toArray } from '../utils/mappers'
import { OngoingEventResult } from '../models/OngoingEventResult'
import { checkForRateLimiting } from '../utils/checkForRateLimiting'

export const getOngoingEvents = (config: HLTVConfig) => async (): Promise<
OngoingEventResult[]
> => {
const $ = await fetchPage(`${config.hltvUrl}/events`, config.loadPage)

checkForRateLimiting($)

const ongoingEventsToday = toArray(
$('.tab-content').first().next().find('a')
).map((eventEl) => {
Expand Down
3 changes: 3 additions & 0 deletions src/endpoints/getPlayer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { Team } from '../models/Team'
import { HLTVConfig } from '../config'
import { fetchPage, toArray } from '../utils/mappers'
import { popSlashSource } from '../utils/parsing'
import { checkForRateLimiting } from '../utils/checkForRateLimiting'

export const getPlayer = (config: HLTVConfig) => async ({
id
Expand All @@ -11,6 +12,8 @@ export const getPlayer = (config: HLTVConfig) => async ({
}): Promise<FullPlayer> => {
const $ = await fetchPage(`${config.hltvUrl}/player/${id}/-`, config.loadPage)

checkForRateLimiting($)

const isStandardPlayer = $('.standard-box.profileTopBox').length !== 0

const name = isStandardPlayer
Expand Down
6 changes: 5 additions & 1 deletion src/endpoints/getPlayerRanking.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { HLTVConfig } from '../config'
import { fetchPage, toArray } from '../utils/mappers'
import { BestOfFilter } from '../enums/BestOfFilter'
import { Team } from '../models/Team'
import { checkForRateLimiting } from '../utils/checkForRateLimiting'

export const getPlayerRanking = (config: HLTVConfig) => async ({
startDate,
Expand Down Expand Up @@ -43,11 +44,14 @@ export const getPlayerRanking = (config: HLTVConfig) => async ({
config.loadPage
)

checkForRateLimiting($)

return toArray($('.player-ratings-table tbody tr')).map((playerRow) => {
const id = Number(
playerRow.find('.playerCol a').first().attr('href')!.split('/')[3]
)
const country = playerRow.find('.playerCol img.flag').eq(0).attr('alt') || ''
const country =
playerRow.find('.playerCol img.flag').eq(0).attr('alt') || ''
const name = playerRow.find('.playerCol').text()
const rating = Number(playerRow.find('.ratingCol').text())
const teams: Team[] = toArray(playerRow.find('.teamCol a')).map(
Expand Down
3 changes: 3 additions & 0 deletions src/endpoints/getPlayerStats.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { MatchType } from '../enums/MatchType'
import { RankingFilter } from '../enums/RankingFilter'
import { fetchPage } from '../utils/mappers'
import { popSlashSource } from '../utils/parsing'
import { checkForRateLimiting } from '../utils/checkForRateLimiting'

export const getPlayerStats = (config: HLTVConfig) => async ({
id,
Expand All @@ -32,6 +33,8 @@ export const getPlayerStats = (config: HLTVConfig) => async ({
config.loadPage
)

checkForRateLimiting($)

const name = $('.summaryRealname div').text() || undefined
const ign = $('.context-item-name').text()

Expand Down
4 changes: 4 additions & 0 deletions src/endpoints/getTeam.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { FullTeam, Result, Achievement } from '../models/FullTeam'
import { HLTVConfig } from '../config'
import { fetchPage, toArray } from '../utils/mappers'
import { checkForRateLimiting } from '../utils/checkForRateLimiting'

export const getTeam = (config: HLTVConfig) => async ({
id
Expand All @@ -13,6 +14,9 @@ export const getTeam = (config: HLTVConfig) => async ({
config.loadPage
)

checkForRateLimiting(t$)
checkForRateLimiting(e$)

const name = t$('.profile-team-name').text()
const logo = `${config.hltvStaticUrl}/images/team/logo/${id}`
const coverImage = t$('.coverImage').attr('data-bg-image')
Expand Down
3 changes: 3 additions & 0 deletions src/endpoints/getTeamRanking.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { TeamRanking } from '../models/TeamRanking'
import { Team } from '../models/Team'
import { HLTVConfig } from '../config'
import { fetchPage, toArray } from '../utils/mappers'
import { checkForRateLimiting } from '../utils/checkForRateLimiting'

export const getTeamRanking = (config: HLTVConfig) => async ({
year = '',
Expand All @@ -28,6 +29,8 @@ export const getTeamRanking = (config: HLTVConfig) => async ({
)
}

checkForRateLimiting($)

const teams = toArray($('.ranked-team')).map((teamEl) => {
const points = Number(
teamEl.find('.points').text().replace(/\(|\)/g, '').split(' ')[0]
Expand Down
3 changes: 3 additions & 0 deletions src/endpoints/getTeamStats.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { FullTeamStats } from '../models/FullTeamStats'
import { HLTVConfig } from '../config'
import { fetchPage, toArray, getTimestamp, getMapSlug } from '../utils/mappers'
import { popSlashSource } from '../utils/parsing'
import { checkForRateLimiting } from '../utils/checkForRateLimiting'

export const getTeamStats = (config: HLTVConfig) => async ({
id,
Expand All @@ -20,6 +21,8 @@ export const getTeamStats = (config: HLTVConfig) => async ({
config.loadPage
)

checkForRateLimiting($)

const getContainerByText = (text) =>
$('.standard-headline')
.filter((_, el) => $(el).text() === text)
Expand Down
7 changes: 7 additions & 0 deletions src/utils/checkForRateLimiting.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
export const checkForRateLimiting = ($: cheerio.Root) => {
if ($.html().includes('error code: 1015')) {
throw new Error(
'Access denied | www.hltv.org used Cloudflare to restrict access'
)
}
}
4 changes: 0 additions & 4 deletions tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
"lib": ["es2017", "dom"],
"declaration": true,
"outDir": "./lib",
"rootDir": "./src",
"removeComments": true,
"strictNullChecks": true,
"noImplicitThis": true,
Expand All @@ -14,9 +13,6 @@
"noImplicitReturns": true,
"alwaysStrict": true,
"noFallthroughCasesInSwitch": true,
"baseUrl": "./src/",
"experimentalDecorators": true,
"emitDecoratorMetadata": true,
"skipLibCheck": true,
"isolatedModules": true
},
Expand Down

0 comments on commit f252865

Please sign in to comment.