Skip to content

Commit

Permalink
feat: use search endpoint from legacy tweetdeck
Browse files Browse the repository at this point in the history 
fix: rss for most timeline endpoints
fix: use .mjs for build exports
fix: wrong bearer token to update in online endpoints
remove: cfworkers builder for github actions
update: use tweetdeck bearer token for crawler
update: use package crypto-es to replace create-hmac
update: archive script with new search endpoint
  • Loading branch information
@BANKA2017
BANKA2017 committed Jul 12, 2023
1 parent 6aaed70 commit 11e3088
Show file tree
Hide file tree
Showing 32 changed files with 578 additions and 191 deletions.
22 changes: 11 additions & 11 deletions .github/workflows/build_and_deploy.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,12 @@ jobs:
uses: actions/setup-node@v3
with:
node-version: 18.x
- name: Yarn install and build && Replace Workers' id
- name: Yarn install and build
run: |
yarn
yarn run build:rollup
sed -i 's/<YOUR_ID>/${{ secrets.CF_KV_ID }}/g' apps/cfworkers/wrangler.toml
sed -i 's/\[\"\*\"\]/${{ secrets.CF_ENV_WORKERS_ALLOW_ORIGIN }}/g' apps/cfworkers/wrangler.toml
# sed -i 's/<YOUR_ID>/${{ secrets.CF_KV_ID }}/g' apps/cfworkers/wrangler.toml
# sed -i 's/\[\"\*\"\]/${{ secrets.CF_ENV_WORKERS_ALLOW_ORIGIN }}/g' apps/cfworkers/wrangler.toml
# - name: Upload dist
# uses: actions/[email protected]
# with:
Expand All @@ -35,15 +35,15 @@ jobs:
uses: actions/[email protected]
with:
name: 'archive_node.js'
path: dist/archiver/archive_node.js
path: dist/archiver/archive_node.mjs
- name: Upload online backend
uses: actions/[email protected]
with:
name: 'app_online.js'
path: dist/backend/app_online.js
- name: Deploy to workers
uses: cloudflare/[email protected]
with:
apiToken: ${{ secrets.CF_API_TOKEN }}
accountId: ${{ secrets.CF_ACCOUNT_ID }}
workingDirectory: 'apps/cfworkers'
path: dist/backend/app_online.mjs
#- name: Deploy to workers
# uses: cloudflare/[email protected]
# with:
# apiToken: ${{ secrets.CF_API_TOKEN }}
# accountId: ${{ secrets.CF_ACCOUNT_ID }}
# workingDirectory: 'apps/cfworkers'
20 changes: 10 additions & 10 deletions apps/archiver/archive.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,16 +145,16 @@ if (existsSync(basePath + '/savedata/data.json')) {
let uid = UserData.account_info?.uid || null

//get init token
global.guest_token = new GuestToken('android')
global.guest_token = new GuestToken()
global.legacy_guest_token = new GuestToken()
await global.guest_token.updateGuestToken(global.guest_token?.open_account?.authorization)
await global.guest_token.updateGuestToken(4)
await global.legacy_guest_token.updateGuestToken(0)

if (global.guest_token.token.nextActiveTime) {
//await TGPush(`[${new Date()}]: #Crawler #GuestToken #429 Wait until ${global.guest_token.token.nextActiveTime}`)
console.error(`[${new Date()}]: #Crawler #GuestToken #429 Wait until ${global.guest_token.token.nextActiveTime}`)
await Sleep(global.guest_token.token.nextActiveTime - Date.now())
await global.guest_token.updateGuestToken(global.guest_token?.open_account?.authorization)
await global.guest_token.updateGuestToken(4)
if (!global.guest_token.token.success) {
process.exit()
}
Expand Down Expand Up @@ -204,7 +204,7 @@ if (activeFlags.tweet) {
const query = [`from:${name}`, 'include:replies', 'include:nativeretweets', 'include:retweets', 'include:quote', `since_id:${cursor.tweets.tmpId ? cursor.tweets.tmpId : 0}`].join(' ')
console.log(`archiver: query string -->${query}<--`)
do {
await global.guest_token.updateGuestToken(global.guest_token?.open_account?.authorization)
await global.guest_token.updateGuestToken(4)
if (global.guest_token.token.nextActiveTime) {
//await TGPush(`[${new Date()}]: #Crawler #GuestToken #429 Wait until ${global.guest_token.token.nextActiveTime}`)
console.error(`[${new Date()}]: #Crawler #GuestToken #429 Wait until ${global.guest_token.token.nextActiveTime}`)
Expand All @@ -216,7 +216,7 @@ if (activeFlags.tweet) {
queryString: forceTimelineForUpdate ? uid : query,
cursor: forceTimelineForUpdate ? '' : cursor.tweets.cursor || '',
guest_token: global.guest_token.token,
count: 999,
count: !forceTimelineForUpdate ? 100 : 999,
online: true,
graphqlMode: forceTimelineForUpdate,
searchMode: !forceTimelineForUpdate,
Expand All @@ -232,7 +232,7 @@ if (activeFlags.tweet) {
queryString: forceTimelineForUpdate ? uid : query,
cursor: forceTimelineForUpdate ? '' : cursor.tweets.cursor || '',
guest_token: global.guest_token.token,
count: 999,
count: !forceTimelineForUpdate ? 100 : 999,
online: true,
graphqlMode: forceTimelineForUpdate,
searchMode: !forceTimelineForUpdate,
Expand All @@ -251,7 +251,7 @@ if (activeFlags.tweet) {
}

//const tmpTweetsInfo = TweetsInfo(tweets.data, true)
const tmpTweetsInfo = GenerateData(tweets, false, '', true)
const tmpTweetsInfo = GenerateData(tweets, false, '', forceTimelineForUpdate)
if (tmpTweetsInfo.tweetsInfo.errors.code !== 0) {
console.log(`archiver: error #${tmpTweetsInfo.tweetsInfo.errors.code} , ${tmpTweetsInfo.tweetsInfo.errors.message}`)
//TGPush(`archiver: error #${tmpTweetsInfo.tweetsInfo.errors.code} , ${tmpTweetsInfo.tweetsInfo.errors.message}`)
Expand All @@ -266,7 +266,7 @@ if (activeFlags.tweet) {
//restful
//uid = Object.values(tmpTweetsInfo.tweetsInfo.users).filter(user => user.screen_name.toLocaleLowerCase() === name.toLocaleLowerCase())[0]?.id_str || null
//graphql
uid = Object.values(tmpTweetsInfo.tweetsInfo.users).find((user) => user.legacy.screen_name.toLocaleLowerCase() === name.toLocaleLowerCase())?.rest_id || null
uid = (user => user?.rest_id || user?.id_str || null)(Object.values(tmpTweetsInfo.tweetsInfo.users).find((user) => (user?.legacy?.screen_name || user.screen_name || '').toLocaleLowerCase() === name.toLocaleLowerCase()))

if (!uid) {
console.error(`archiver: no such account!!!`)
Expand Down Expand Up @@ -423,7 +423,7 @@ if (activeFlags.broadcast) {
console.log(`archiver: broadcast (${broadcastsCards.length})`)

const tmpCardsList = broadcastsCards.splice(0, 100)
await global.guest_token.updateGuestToken(global.guest_token?.open_account?.authorization)
await global.guest_token.updateGuestToken(4)
const broadcasts = await getBroadcast({
id: tmpCardsList.map((card) => card.url.replaceAll(/.*\/([^\/\?#]+)(?:$|\?.*|#.*)/gm, '$1')),
guest_token: global.guest_token.token
Expand Down Expand Up @@ -530,7 +530,7 @@ if (activeFlags.space) {
console.log(`archiver: audiospace (${audiospacesCards.length})`)

const tmpCardsList = audiospacesCards.splice(0, 100)
await global.guest_token.updateGuestToken(global.guest_token?.open_account?.authorization)
await global.guest_token.updateGuestToken(4)
const audiospaces = await getAudioSpace({ id: tmpCardsList.map((card) => card.url), guest_token: global.guest_token.token })
global.guest_token.updateRateLimit('AudioSpaceById', 100)

Expand Down
68 changes: 34 additions & 34 deletions apps/archiver/archive_lite.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ import { GenerateData } from '../backend/CoreFunctions/online/OnlineTweet.mjs'
import { AudioSpace, Broadcast } from '../../libs/core/Core.tweet.mjs'
import { Parser } from 'm3u8-parser'

let name = window.name //readFileSync('./screen_name.txt').toString().trim()
let forceTimelineForUpdate = window.timelineFirst || false
let name = globalThis.name //readFileSync('./screen_name.txt').toString().trim()
let forceTimelineForUpdate = globalThis.timelineFirst || false

//save date
let now = Date.now()
Expand All @@ -25,39 +25,39 @@ let range = {
let rawTweetData = {}
let rawUserInfoData = {}
let resetCursor = false
if (window.UserData?.account_info?.name !== name) {
window.UserData = {
if (globalThis.UserData?.account_info?.name !== name) {
globalThis.UserData = {
account_info: null,
account_list: {},
tweets: {},
type: 'lite'
}
resetCursor = true
}
let UserData = window.UserData
let UserData = globalThis.UserData
let uid = UserData.account_info?.uid || null

//get init token
window.guest_token = new GuestToken()
await window.guest_token.updateGuestToken(1)

if (window.guest_token.token.nextActiveTime) {
await Sleep(window.guest_token.token.nextActiveTime - Date.now())
await window.guest_token.updateGuestToken(1)
if (!window.guest_token.token.success) {
throw window.guest_token
globalThis.guest_token = new GuestToken()
await globalThis.guest_token.updateGuestToken(4)

if (globalThis.guest_token.token.nextActiveTime) {
await Sleep(globalThis.guest_token.token.nextActiveTime - Date.now())
await globalThis.guest_token.updateGuestToken(4)
if (!globalThis.guest_token.token.success) {
throw globalThis.guest_token
}
}

// cursor
if (resetCursor || !Object.values(window.cursor || {}).some((v) => v.cursor)) {
window.cursor = {
if (resetCursor || !Object.values(globalThis.cursor || {}).some((v) => v.cursor)) {
globalThis.cursor = {
tweets: { maxId: '', tmpId: '', cursor: '' },
broadcast: { maxId: '', tmpId: '', cursor: '' },
space: { maxId: '', tmpId: '', cursor: '' }
}
}
let cursor = window.cursor
let cursor = globalThis.cursor

//userinfo and tweets
if (cursor.tweets.cursor === '') {
Expand All @@ -80,18 +80,18 @@ if (cursor.tweets.cursor !== 'complete') {
let query = [`from:${name}`, 'include:replies', 'include:nativeretweets', 'include:retweets', 'include:quote', `since_id:${cursor.tweets.tmpId ? cursor.tweets.tmpId : 0}`].join(' ')
console.log(`archiver: query string -->${query}<--`)
do {
await window.guest_token.updateGuestToken(1)
if (window.guest_token.token.nextActiveTime) {
console.error(`[${new Date()}]: #Crawler #GuestToken #429 Wait until ${window.guest_token.token.nextActiveTime}`)
await globalThis.guest_token.updateGuestToken(4)
if (globalThis.guest_token.token.nextActiveTime) {
console.error(`[${new Date()}]: #Crawler #GuestToken #429 Wait until ${globalThis.guest_token.token.nextActiveTime}`)
break //只处理现有的
}
let tweets = null
try {
tweets = await getTweets({
queryString: forceTimelineForUpdate ? uid : query,
cursor: forceTimelineForUpdate ? '' : cursor.tweets.cursor || '',
guest_token: window.guest_token.token,
count: 999,
guest_token: globalThis.guest_token.token,
count: !forceTimelineForUpdate ? 100 : 999,
online: true,
graphqlMode: forceTimelineForUpdate,
searchMode: !forceTimelineForUpdate,
Expand All @@ -103,22 +103,22 @@ if (cursor.tweets.cursor !== 'complete') {
.filter((cookie) => cookie.length >= 2)
)
})
window.guest_token.updateRateLimit('Search')
globalThis.guest_token.updateRateLimit('Search')
} catch (e) {
//try again
console.log('archiver: first retry...')
try {
tweets = await getTweets({
queryString: forceTimelineForUpdate ? uid : query,
cursor: forceTimelineForUpdate ? '' : cursor.tweets.cursor || '',
guest_token: window.guest_token.token,
count: 999,
guest_token: globalThis.guest_token.token,
count: !forceTimelineForUpdate ? 100 : 999,
online: true,
graphqlMode: forceTimelineForUpdate,
searchMode: !forceTimelineForUpdate,
withReply: true
})
window.guest_token.updateRateLimit('Search')
globalThis.guest_token.updateRateLimit('Search')
} catch (e1) {
console.log('archiver: retry failed...')
throw e1
Expand All @@ -129,7 +129,7 @@ if (cursor.tweets.cursor !== 'complete') {
}

//let tmpTweetsInfo = TweetsInfo(tweets.data, true)
let tmpTweetsInfo = GenerateData(tweets, false, '', true)
let tmpTweetsInfo = GenerateData(tweets, false, '', forceTimelineForUpdate)
if (tmpTweetsInfo.tweetsInfo.errors.code !== 0) {
console.log(`archiver: error #${tmpTweetsInfo.tweetsInfo.errors.code} , ${tmpTweetsInfo.tweetsInfo.errors.message}`)
continue
Expand All @@ -143,7 +143,7 @@ if (cursor.tweets.cursor !== 'complete') {
//restful
//uid = Object.values(tmpTweetsInfo.tweetsInfo.users).filter(user => user.screen_name.toLocaleLowerCase() === name.toLocaleLowerCase())[0]?.id_str || null
//graphql
uid = Object.values(tmpTweetsInfo.tweetsInfo.users).find((user) => user.legacy.screen_name.toLocaleLowerCase() === name.toLocaleLowerCase())?.rest_id || null
uid = (user => user?.rest_id || user?.id_str || null)(Object.values(tmpTweetsInfo.tweetsInfo.users).find((user) => (user?.legacy?.screen_name || user.screen_name || '').toLocaleLowerCase() === name.toLocaleLowerCase()))

if (!uid) {
throw `archiver: no such account!!!`
Expand Down Expand Up @@ -287,12 +287,12 @@ if (broadcastsCards.length <= 0) {
while (broadcastsCards.length > 0) {
console.log(`archiver: broadcast (${broadcastsCards.length})`)
let tmpCardsList = broadcastsCards.splice(0, 100)
await window.guest_token.updateGuestToken(1)
await globalThis.guest_token.updateGuestToken(4)
let broadcasts = await getBroadcast({
id: tmpCardsList.map((card) => card.url.replaceAll(/.*\/([^\/\?#]+)(?:$|\?.*|#.*)/gm, '$1')),
guest_token: window.guest_token.token
guest_token: globalThis.guest_token.token
})
window.guest_token.updateRateLimit('BroadCast', 100)
globalThis.guest_token.updateRateLimit('BroadCast', 100)

for (let index in broadcasts) {
let broadcastResponse = broadcasts[index]
Expand Down Expand Up @@ -358,9 +358,9 @@ if (audiospacesCards.length <= 0) {
console.log(`archiver: audiospace (${audiospacesCards.length})`)

let tmpCardsList = audiospacesCards.splice(0, 100)
await window.guest_token.updateGuestToken(1)
let audiospaces = await getAudioSpace({ id: tmpCardsList.map((card) => card.url), guest_token: window.guest_token.token })
window.guest_token.updateRateLimit('AudioSpaceById', 100)
await globalThis.guest_token.updateGuestToken(4)
let audiospaces = await getAudioSpace({ id: tmpCardsList.map((card) => card.url), guest_token: globalThis.guest_token.token })
globalThis.guest_token.updateRateLimit('AudioSpaceById', 100)

for (let index in audiospaces) {
let audiospaceResponse = audiospaces[index]
Expand Down Expand Up @@ -412,4 +412,4 @@ const Download = (url, fileName) => {
document.body.removeChild(element)
}

Download(URL.createObjectURL(new Blob([JSON.stringify(window.UserData, null, 4)], { type: 'application/json' })), `${name}.json`)
Download(URL.createObjectURL(new Blob([JSON.stringify(globalThis.UserData, null, 4)], { type: 'application/json' })), `${name}.json`)
6 changes: 3 additions & 3 deletions apps/backend/CoreFunctions/album/Album.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ const AlbumSearch = async (req, env) => {
//TODO fix tokens
tweets = await getConversation({ tweet_id: tweetId, guest_token: env.guest_token2, graphqlMode: true })
//updateGuestToken
await env.updateGuestToken(env, 'guest_token2', 1, tweets.headers.get('x-rate-limit-remaining') < 1, 'TweetDetail')
await env.updateGuestToken(env, 'guest_token2', 4, tweets.headers.get('x-rate-limit-remaining') < 1, 'TweetDetail')
} else {
tweets = await getTweets({
queryString: queryArray.join(' '),
Expand All @@ -51,14 +51,14 @@ const AlbumSearch = async (req, env) => {
searchMode: true
})
//updateGuestToken
await env.updateGuestToken(env, 'guest_token2', 1, tweets.headers.get('x-rate-limit-remaining') < 1, 'Search')
await env.updateGuestToken(env, 'guest_token2', 4, tweets.headers.get('x-rate-limit-remaining') < 1, 'Search')
}
} catch (e) {
console.error(`[${new Date()}]: #Album #${e.code} ${e.message}`)
return env.json(apiTemplate(e.code, e.message, {}, 'album'))
}

let { tweetsInfo, tweetsContent } = GenerateData(tweets, isPhotos, '', true)
let { tweetsInfo, tweetsContent } = GenerateData(tweets, isPhotos, '', false)
if (tweetsInfo.errors.code !== 0) {
return env.json(apiTemplate(tweetsInfo.errors.code, tweetsInfo.errors.message, {}, 'album'))
}
Expand Down
Loading

1 comment on commit 11e3088

@BANKA2017
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#4 Known issues

  • The search endpoint is unstable and often returns empty results.
  • The bearer token of the new TweetDeck does not support the typeahead endpoint.
  • more...

Please sign in to comment.