From 31374a4ac730bc456c9fb3f71d1c5b663e53475d Mon Sep 17 00:00:00 2001 From: Kayo Souza Date: Sun, 20 Aug 2023 19:01:49 -0300 Subject: [PATCH] Add stories download --- README.md | 1 + src/Downloader.js | 93 +++++++++++++++------ src/typings/api.d.ts | 192 +++++++++++++++++++++---------------------- 3 files changed, 164 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index 36410ec..c14b690 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Arguments: -o, --output Output directory -f, --force Force creation of output directory +--no-stories Disable stories download --no-timeline Disable timeline download --no-highlights Disable hightlights download --no-cover Disable hightlights' cover download diff --git a/src/Downloader.js b/src/Downloader.js index f7b1d84..c59205e 100644 --- a/src/Downloader.js +++ b/src/Downloader.js @@ -80,8 +80,6 @@ export default class Downloader { this.GetConfig() this.UpdateHeaders() - // TODO: Add stories download - do{ try{ await this.CheckServerConfig() @@ -130,7 +128,8 @@ export default class Downloader { const results = await Promise.allSettled([ timeline && this.DownloadTimeline(username, folder), - highlights && this.DownloadHighlights(user_id, folder, cover) + highlights && this.DownloadHighlights(user_id, folder, cover), + stories && this.DownloadStories(user_id, folder) ]) for(const result of results){ @@ -179,18 +178,15 @@ export default class Downloader { /** @param {string} user_id */ async GetHighlights(user_id){ const url = new URL(API_QUERY, BASE_URL) - const data = new URLSearchParams - - data.set("query_hash", this.config.queryHash) - data.set("user_id", user_id) - data.set("include_chaining", "false") - data.set("include_reel", "false") - data.set("include_suggested_users", "false") - data.set("include_logged_out_extras", "false") - data.set("include_live_status", "false") - data.set("include_highlight_reels", "true") - url.search = data.toString() + url.searchParams.set("query_hash", this.config.queryHash) + url.searchParams.set("user_id", user_id) + url.searchParams.set("include_chaining", "false") + url.searchParams.set("include_reel", "false") + url.searchParams.set("include_suggested_users", "false") + url.searchParams.set("include_logged_out_extras", "false") + url.searchParams.set("include_live_status", "false") + url.searchParams.set("include_highlight_reels", "true") /** @type {import("axios").AxiosResponse} */ const response = await this.Request(url, "GET", { @@ -200,14 +196,11 @@ export default class Downloader { return response.data.data.user.edge_highlight_reels.edges.map(({ node }) => node) } - /** @param {string[]} reelsIds */ + /** @param {`${number}`[]} reelsIds */ async GetHighlightsContents(reelsIds){ const url = new URL(API_REELS, BASE_URL) - const data = new URLSearchParams - - for(const id of reelsIds) data.append("reel_ids", "highlight:" + id) - url.search = data.toString() + for(const id of reelsIds) url.searchParams.append("reel_ids", "highlight:" + id) /** @type {import("axios").AxiosResponse} */ const response = await this.Request(url, "GET", { @@ -216,6 +209,23 @@ export default class Downloader { return response.data.reels_media } + /** @param {`${number}`} userId */ + async GetStories(userId){ + const url = new URL(API_REELS, BASE_URL) + + url.searchParams.set("reel_ids", userId) + + /** @type {import("axios").AxiosResponse} */ + const response = await this.Request(url, "GET", { + responseType: "json" + }) + + const { reels, reels_media } = response.data + + if(!reels_media.length) return null + + return response.data.reels[userId] + } /** * @param {string} user_id * @param {string} folder @@ -233,7 +243,6 @@ export default class Downloader { if(hasHighlights && !existsSync(folder)) await mkdir(folder, { recursive: true }) - this.Log("Downloading highlights") while(highlights.length && limit > count){ const ids = highlights.splice(0, 10).map(({ id }) => id) const highlightsContents = await this.GetHighlightsContents(ids) @@ -246,10 +255,7 @@ export default class Downloader { } for(const { id, items } of highlightsContents){ - if(count > limit){ - console.log("Should not be here") - break - } + if(count > limit) throw new Error("Unexpected error") const highlightId = /** @type {`${number}`} */ (id.substring(id.indexOf(":") + 1)) @@ -292,6 +298,35 @@ export default class Downloader { if(count === 0) this.Log("No content found in the highlights") }else this.Log("No highlights found") } + /** + * @param {string} user_id + * @param {string} folder + * @param {number} [limit] + */ + async DownloadStories(user_id, folder, limit = Infinity){ + const results = await this.GetStories(/** @type {`${number}`} */ (user_id)) + + if(!results) return this.Log("No stories found") + + const { items: stories } = results + + if(stories.length){ + if(!existsSync(folder)) await mkdir(folder, { recursive: true }) + this.Log("Downloading stories") + } + + let count = 0 + + while(stories.length && limit > count){ + const items = stories.splice(0, 10) + const data = { count, limit } + const { limited } = await this.DownloadItems(items, folder, data) + + count = data.count + + if(limited) break + } + } /** * @param {string} username * @param {string} folder @@ -304,10 +339,10 @@ export default class Downloader { /** @type {string} */ let lastId + let first = true let count = 0 let hasMore = true - this.Log("Downloading timeline") while(hasMore && limit > count){ if(lastId) url.searchParams.set("max_id", lastId) @@ -317,6 +352,8 @@ export default class Downloader { if(num_results === 0) break + if(first) this.Log("Downloading timeline"), first = false + if(!existsSync(folder)) await mkdir(folder, { recursive: true }) const data = { count, limit } @@ -505,7 +542,11 @@ export default class Downloader { const date = new Date().toLocaleString("pt-BR").split(", ")[1] - if(args.length === 1 && args[0] instanceof Error) return console.error(chalk.redBright(`[${date}] ${args[0].message}`)) + if(args.length === 1){ + const arg = args[0] + if(arg instanceof Error) return console.error(chalk.redBright(`[${date}] ${args[0].message}`)) + if(typeof arg === "string") return console.log(`${chalk.blackBright(`[${date}]`)} ${arg}`) + } console.log(chalk.blackBright(`[${date}] `), ...args) } diff --git a/src/typings/api.d.ts b/src/typings/api.d.ts index bebc99e..d356656 100644 --- a/src/typings/api.d.ts +++ b/src/typings/api.d.ts @@ -1,5 +1,9 @@ export interface APIHeaders { [key: string]: string + "viewport-width"?: string + "X-Ig-Www-Claim"?: string + "X-Requested-With"?: "XMLHttpRequest" + "X-Asbd-Id"?: string "X-Ig-App-Id"?: string "X-Csrftoken"?: string Cookie?: string @@ -169,11 +173,11 @@ export interface FeedItem { id: string media_type: number image_versions2: { - candidates: Array<{ + candidates: { width: number height: number url: string - }> + }[] } original_width: number original_height: number @@ -181,7 +185,7 @@ export interface FeedItem { pk: string carousel_parent_id: string usertags?: { - in: Array<{ + in: { user: { pk: string pk_id: string @@ -192,10 +196,10 @@ export interface FeedItem { profile_pic_url: string profile_pic_id?: string } - position: Array + position: number[] start_time_in_video_in_sec: any duration_in_video_in_sec: any - }> + }[] } commerciality_status: string sharing_friction_info: { @@ -203,13 +207,13 @@ export interface FeedItem { bloks_app_url: any sharing_friction_payload: any } - video_versions?: Array<{ + video_versions?: { type: number width: number height: number url: string id: string - }> + }[] video_duration?: number is_dash_eligible?: number video_dash_manifest?: string @@ -411,8 +415,8 @@ export interface Reel { __typename: "GraphReel" } -export interface ReelNode { - id: string +export interface ReelChainNode { + id: `${number}` blocked_by_viewer: boolean restricted_by_viewer: boolean followed_by_viewer: boolean @@ -460,7 +464,7 @@ export interface QueryHighlightsResponse { /** Suggested users */ edge_chaining: { edges: { - node: ReelNode + node: ReelChainNode }[] } edge_highlight_reels: { @@ -479,6 +483,18 @@ export interface QueryHighlightsResponse { } }[] } + has_public_story?: boolean + is_live?: boolean + reel?: { + expiring_at: number + has_pride_media: boolean + id: `${number}` + latest_reel_media: number + owner: ReelOwner + seen: boolean | null + user: ReelUser + __typename: "GraphReel" + } } } extensions?: { @@ -495,7 +511,7 @@ export interface QueryTimelineAPIResponse { has_previous_page: boolean has_next_page: boolean } - edges: Array<{ + edges: { node: { media?: { taken_at: number @@ -514,13 +530,13 @@ export interface QueryTimelineAPIResponse { commerciality_status: string is_paid_partnership: boolean is_visual_reply_commenter_notice_enabled: boolean - clips_tab_pinned_user_ids: Array + clips_tab_pinned_user_ids: any[] has_delayed_metadata: boolean comment_likes_enabled: boolean comment_threading_enabled: boolean max_num_visible_preview_comments: number has_more_comments: boolean - preview_comments: Array + preview_comments: any[] photo_of_you: boolean is_organic_product_tagging_eligible: boolean can_see_insights_as_brand: boolean @@ -553,14 +569,14 @@ export interface QueryTimelineAPIResponse { } profile_pic_id: string profile_pic_url: string - account_badges: Array + account_badges: any[] latest_reel_media: number } can_viewer_reshare: boolean like_count: number fb_like_count?: number has_liked: boolean - top_likers: Array + top_likers: string[] facepile_top_likers: GenericUser[] preview: string image_versions2: { @@ -586,13 +602,13 @@ export interface QueryTimelineAPIResponse { video_dash_manifest?: string video_codec?: string number_of_qualities?: number - video_versions?: Array<{ + video_versions?: { type: number width: number height: number url: string id: string - }> + }[] has_audio?: boolean video_duration?: number can_viewer_save: boolean @@ -645,7 +661,7 @@ export interface QueryTimelineAPIResponse { connected_group_id: string remember_group_choice: boolean style: any - groups: Array<{ + groups: { id: string title: string show_group_text: string @@ -715,21 +731,21 @@ export interface QueryTimelineAPIResponse { } profile_pic_id: string profile_pic_url: string - account_badges: Array + account_badges: any[] latest_reel_media: number } can_viewer_reshare: boolean like_count: number has_liked: boolean - top_likers: Array - facepile_top_likers: Array + top_likers: any[] + facepile_top_likers: any[] preview: string image_versions2: { - candidates: Array<{ + candidates: { width: number height: number url: string - }> + }[] } original_width: number original_height: number @@ -798,7 +814,7 @@ export interface QueryTimelineAPIResponse { ranking_weight: number can_view_more_preview_comments: boolean hide_view_all_comment_entrypoint: boolean - comments: Array + comments: any[] comment_count: number inline_composer_display_condition: string } @@ -821,13 +837,13 @@ export interface QueryTimelineAPIResponse { commerciality_status: string is_paid_partnership: boolean is_visual_reply_commenter_notice_enabled: boolean - clips_tab_pinned_user_ids: Array + clips_tab_pinned_user_ids: any[] has_delayed_metadata: boolean comment_likes_enabled: boolean comment_threading_enabled: boolean max_num_visible_preview_comments: number has_more_comments: boolean - preview_comments: Array + preview_comments: any[] photo_of_you: boolean is_organic_product_tagging_eligible: boolean can_see_insights_as_brand: boolean @@ -860,20 +876,20 @@ export interface QueryTimelineAPIResponse { } profile_pic_id: string profile_pic_url: string - account_badges: Array + account_badges: any[] } can_viewer_reshare: boolean like_count: number has_liked: boolean - top_likers: Array - facepile_top_likers: Array + top_likers: any[] + facepile_top_likers: any[] preview: string image_versions2: { - candidates: Array<{ + candidates: { width: number height: number url: string - }> + }[] } original_width: number original_height: number @@ -951,7 +967,7 @@ export interface QueryTimelineAPIResponse { confirmation_title_style: string undo_style: string confirmation_style: string - followup_options: Array<{ + followup_options: { text: string style: any id: string @@ -963,16 +979,16 @@ export interface QueryTimelineAPIResponse { confirmation_body: string undo_style: string confirmation_title?: string - followup_options?: Array<{ + followup_options?: { text: string id: string style: string show_icon: boolean data: any demotion_control: {} - }> + }[] } - }> + }[] } recommendation_data: string explore: { @@ -980,10 +996,10 @@ export interface QueryTimelineAPIResponse { } can_view_more_preview_comments: boolean hide_view_all_comment_entrypoint: boolean - comments: Array + comments: any[] comment_count: number inline_composer_display_condition: string - timeline_pinned_user_ids?: Array + timeline_pinned_user_ids?: number[] } id: string inventory_source: string @@ -991,12 +1007,12 @@ export interface QueryTimelineAPIResponse { }[] next_max_id: string pagination_source: string - }> + }[] } } } cursor: string - }> + }[] } } status: APIStatus @@ -1081,6 +1097,7 @@ export interface FacebookAccountAPIResponse { export type ReplyTypes = "story_selfie_reply" | "story_remix_reply" export type HighlightId = `highlight:${number}` +export type StoryId = `${number}` export interface HighlightCoverMedia { cropped_image_version: ImageVersion & { @@ -1096,64 +1113,47 @@ export interface HighlightUser extends Omit { +interface ReelMedia { + id: T + strong_id__: T + latest_reel_media: number + seen: boolean | null + can_reply: boolean + can_gif_quick_reply: boolean + can_reshare: boolean + can_react_with_avatar: boolean + reel_type: "highlight_reel" + ad_expiry_timestamp_in_millis: any + is_cta_sticker_available: any + app_sticker_info: any + should_treat_link_sticker_as_cta: any + cover_media: HighlightCoverMedia[] + user: HighlightUser[] + items: FeedItem[] + title: string + created_at: number + is_pinned_highlight: boolean + prefetch_count: number + media_count: number + media_ids: string[] + is_cacheable: boolean + is_converted_to_clips: boolean + disabled_reply_types: ReplyTypes[] + highlight_reel_type: string +} + +export interface HighlightsAPIResponse { reels: { - [highlightId: HighlightId]: { - id: T - strong_id__: T - latest_reel_media: number - seen: boolean | null - can_reply: boolean - can_gif_quick_reply: boolean - can_reshare: boolean - can_react_with_avatar: boolean - reel_type: "highlight_reel" - ad_expiry_timestamp_in_millis: any - is_cta_sticker_available: any - app_sticker_info: any - should_treat_link_sticker_as_cta: any - cover_media: HighlightCoverMedia[] - user: HighlightUser[] - items: FeedItem[] - title: string - created_at: number - is_pinned_highlight: boolean - prefetch_count: number - media_count: number - media_ids: string[] - is_cacheable: boolean - is_converted_to_clips: boolean - disabled_reply_types: ReplyTypes[] - highlight_reel_type: string - } + [highlightId: HighlightId]: ReelMedia } - reels_media: { - id: HighlightId - strong_id__: HighlightId - latest_reel_media: number - seen: boolean | null - can_reply: boolean - can_gif_quick_reply: boolean - can_reshare: boolean - can_react_with_avatar: boolean - reel_type: string - ad_expiry_timestamp_in_millis: any - is_cta_sticker_available: any - app_sticker_info: any - should_treat_link_sticker_as_cta: any - cover_media: HighlightCoverMedia[] - user: HighlightUser[] - items: FeedItem[] - title: string - created_at: number - is_pinned_highlight: boolean - prefetch_count: number - media_count: number - media_ids: string[] - is_cacheable: boolean - is_converted_to_clips: boolean - disabled_reply_types: ReplyTypes[] - highlight_reel_type: string - }[] + reels_media: ReelMedia[] + status: APIStatus +} + +export interface StoriesAPIResponse { + reels: { + [storyId: StoryId]: ReelMedia + } + reels_media: ReelMedia[] status: APIStatus }