From 077174eb753a8fb5a9a8bf002baaad4ecb6f6ece Mon Sep 17 00:00:00 2001 From: Kayo Souza Date: Sun, 18 Feb 2024 19:59:31 -0300 Subject: [PATCH] Add download limits and queue and bug fixes --- package.json | 20 +- pnpm-lock.yaml | 538 ++++++++++++++++++++--------------------- src/Downloader.js | 210 ++++++++++------ src/Queue.js | 74 ++++++ src/config.js | 6 + src/index.js | 7 +- src/typings/api.d.ts | 13 + src/typings/index.d.ts | 2 +- tests/index.test.js | 41 ++-- 9 files changed, 516 insertions(+), 395 deletions(-) create mode 100644 src/Queue.js diff --git a/package.json b/package.json index a43ea67..c415e12 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "author": "Kayo Souza", "name": "insta-downloader", - "version": "2.1.0", + "version": "3.0.0", "description": "An application to download content from Instagram", "main": "src/index.js", "scripts": { @@ -19,18 +19,18 @@ "videos" ], "dependencies": { - "axios": "^1.6.0", + "axios": "~1.6.7", "chalk": "^5.3.0", - "commander": "~11.1.0", - "dotenv": "~16.3.1", + "commander": "~12.0.0", + "dotenv": "~16.4.4", "mime": "~3.0.0", - "sharp": "~0.32.6" + "sharp": "~0.33.2" }, "devDependencies": { - "@types/mime": "~3.0.3", - "@types/node": "^20.8.9", + "@types/mime": "~3.0.4", + "@types/node": "20.x", "@types/sharp": "~0.31.1", - "typescript": "^5.2.2" + "typescript": "^5.3.3" }, "repository": { "type": "git", @@ -44,8 +44,8 @@ "instadl": "src/index.js" }, "engines": { - "npm": "^8.19.4 || >=9.8.1", - "node": "^16.19 || ^18.13 || ^19.2 || 20 || 21" + "npm": ">=9.8.1", + "node": "^18.17 || ^19.2 || ^20.3 || 21" }, "license": "ISC", "type": "module" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1afcbad..4d11125 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,46 +6,242 @@ settings: dependencies: axios: - specifier: ^1.6.0 - version: 1.6.0 + specifier: ~1.6.7 + version: 1.6.7 chalk: specifier: ^5.3.0 version: 5.3.0 commander: - specifier: ~11.1.0 - version: 11.1.0 + specifier: ~12.0.0 + version: 12.0.0 dotenv: - specifier: ~16.3.1 - version: 16.3.1 + specifier: ~16.4.4 + version: 16.4.4 mime: specifier: ~3.0.0 version: 3.0.0 sharp: - specifier: ~0.32.6 - version: 0.32.6 + specifier: ~0.33.2 + version: 0.33.2 devDependencies: '@types/mime': - specifier: ~3.0.3 - version: 3.0.3 + specifier: ~3.0.4 + version: 3.0.4 '@types/node': - specifier: ^20.8.9 - version: 20.8.9 + specifier: 20.x + version: 20.11.19 '@types/sharp': specifier: ~0.31.1 version: 0.31.1 typescript: - specifier: ^5.2.2 - version: 5.2.2 + specifier: ^5.3.3 + version: 5.3.3 packages: - /@types/mime@3.0.3: - resolution: {integrity: sha512-i8MBln35l856k5iOhKk2XJ4SeAWg75mLIpZB4v6imOagKL6twsukBZGDMNhdOVk7yRFTMPpfILocMos59Q1otQ==} + /@emnapi/runtime@0.45.0: + resolution: {integrity: sha512-Txumi3td7J4A/xTTwlssKieHKTGl3j4A1tglBx72auZ49YK7ePY6XZricgIg9mnZT4xPfA+UPCUdnhRuEFDL+w==} + requiresBuild: true + dependencies: + tslib: 2.6.2 + dev: false + optional: true + + /@img/sharp-darwin-arm64@0.33.2: + resolution: {integrity: sha512-itHBs1rPmsmGF9p4qRe++CzCgd+kFYktnsoR1sbIAfsRMrJZau0Tt1AH9KVnufc2/tU02Gf6Ibujx+15qRE03w==} + engines: {glibc: '>=2.26', node: ^18.17.0 || ^20.3.0 || >=21.0.0, npm: '>=9.6.5', pnpm: '>=7.1.0', yarn: '>=3.2.0'} + cpu: [arm64] + os: [darwin] + requiresBuild: true + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.0.1 + dev: false + optional: true + + /@img/sharp-darwin-x64@0.33.2: + resolution: {integrity: sha512-/rK/69Rrp9x5kaWBjVN07KixZanRr+W1OiyKdXcbjQD6KbW+obaTeBBtLUAtbBsnlTTmWthw99xqoOS7SsySDg==} + engines: {glibc: '>=2.26', node: ^18.17.0 || ^20.3.0 || >=21.0.0, npm: '>=9.6.5', pnpm: '>=7.1.0', yarn: '>=3.2.0'} + cpu: [x64] + os: [darwin] + requiresBuild: true + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.0.1 + dev: false + optional: true + + /@img/sharp-libvips-darwin-arm64@1.0.1: + resolution: {integrity: sha512-kQyrSNd6lmBV7O0BUiyu/OEw9yeNGFbQhbxswS1i6rMDwBBSX+e+rPzu3S+MwAiGU3HdLze3PanQ4Xkfemgzcw==} + engines: {macos: '>=11', npm: '>=9.6.5', pnpm: '>=7.1.0', yarn: '>=3.2.0'} + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /@img/sharp-libvips-darwin-x64@1.0.1: + resolution: {integrity: sha512-eVU/JYLPVjhhrd8Tk6gosl5pVlvsqiFlt50wotCvdkFGf+mDNBJxMh+bvav+Wt3EBnNZWq8Sp2I7XfSjm8siog==} + engines: {macos: '>=10.13', npm: '>=9.6.5', pnpm: '>=7.1.0', yarn: '>=3.2.0'} + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /@img/sharp-libvips-linux-arm64@1.0.1: + resolution: {integrity: sha512-bnGG+MJjdX70mAQcSLxgeJco11G+MxTz+ebxlz8Y3dxyeb3Nkl7LgLI0mXupoO+u1wRNx/iRj5yHtzA4sde1yA==} + engines: {glibc: '>=2.26', npm: '>=9.6.5', pnpm: '>=7.1.0', yarn: '>=3.2.0'} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@img/sharp-libvips-linux-arm@1.0.1: + resolution: {integrity: sha512-FtdMvR4R99FTsD53IA3LxYGghQ82t3yt0ZQ93WMZ2xV3dqrb0E8zq4VHaTOuLEAuA83oDawHV3fd+BsAPadHIQ==} + engines: {glibc: '>=2.28', npm: '>=9.6.5', pnpm: '>=7.1.0', yarn: '>=3.2.0'} + cpu: [arm] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@img/sharp-libvips-linux-s390x@1.0.1: + resolution: {integrity: sha512-3+rzfAR1YpMOeA2zZNp+aYEzGNWK4zF3+sdMxuCS3ey9HhDbJ66w6hDSHDMoap32DueFwhhs3vwooAB2MaK4XQ==} + engines: {glibc: '>=2.28', npm: '>=9.6.5', pnpm: '>=7.1.0', yarn: '>=3.2.0'} + cpu: [s390x] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@img/sharp-libvips-linux-x64@1.0.1: + resolution: {integrity: sha512-3NR1mxFsaSgMMzz1bAnnKbSAI+lHXVTqAHgc1bgzjHuXjo4hlscpUxc0vFSAPKI3yuzdzcZOkq7nDPrP2F8Jgw==} + engines: {glibc: '>=2.26', npm: '>=9.6.5', pnpm: '>=7.1.0', yarn: '>=3.2.0'} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@img/sharp-libvips-linuxmusl-arm64@1.0.1: + resolution: {integrity: sha512-5aBRcjHDG/T6jwC3Edl3lP8nl9U2Yo8+oTl5drd1dh9Z1EBfzUKAJFUDTDisDjUwc7N4AjnPGfCA3jl3hY8uDg==} + engines: {musl: '>=1.2.2', npm: '>=9.6.5', pnpm: '>=7.1.0', yarn: '>=3.2.0'} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@img/sharp-libvips-linuxmusl-x64@1.0.1: + resolution: {integrity: sha512-dcT7inI9DBFK6ovfeWRe3hG30h51cBAP5JXlZfx6pzc/Mnf9HFCQDLtYf4MCBjxaaTfjCCjkBxcy3XzOAo5txw==} + engines: {musl: '>=1.2.2', npm: '>=9.6.5', pnpm: '>=7.1.0', yarn: '>=3.2.0'} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@img/sharp-linux-arm64@0.33.2: + resolution: {integrity: sha512-pz0NNo882vVfqJ0yNInuG9YH71smP4gRSdeL09ukC2YLE6ZyZePAlWKEHgAzJGTiOh8Qkaov6mMIMlEhmLdKew==} + engines: {glibc: '>=2.26', node: ^18.17.0 || ^20.3.0 || >=21.0.0, npm: '>=9.6.5', pnpm: '>=7.1.0', yarn: '>=3.2.0'} + cpu: [arm64] + os: [linux] + requiresBuild: true + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.0.1 + dev: false + optional: true + + /@img/sharp-linux-arm@0.33.2: + resolution: {integrity: sha512-Fndk/4Zq3vAc4G/qyfXASbS3HBZbKrlnKZLEJzPLrXoJuipFNNwTes71+Ki1hwYW5lch26niRYoZFAtZVf3EGA==} + engines: {glibc: '>=2.28', node: ^18.17.0 || ^20.3.0 || >=21.0.0, npm: '>=9.6.5', pnpm: '>=7.1.0', yarn: '>=3.2.0'} + cpu: [arm] + os: [linux] + requiresBuild: true + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.0.1 + dev: false + optional: true + + /@img/sharp-linux-s390x@0.33.2: + resolution: {integrity: sha512-MBoInDXDppMfhSzbMmOQtGfloVAflS2rP1qPcUIiITMi36Mm5YR7r0ASND99razjQUpHTzjrU1flO76hKvP5RA==} + engines: {glibc: '>=2.28', node: ^18.17.0 || ^20.3.0 || >=21.0.0, npm: '>=9.6.5', pnpm: '>=7.1.0', yarn: '>=3.2.0'} + cpu: [s390x] + os: [linux] + requiresBuild: true + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.0.1 + dev: false + optional: true + + /@img/sharp-linux-x64@0.33.2: + resolution: {integrity: sha512-xUT82H5IbXewKkeF5aiooajoO1tQV4PnKfS/OZtb5DDdxS/FCI/uXTVZ35GQ97RZXsycojz/AJ0asoz6p2/H/A==} + engines: {glibc: '>=2.26', node: ^18.17.0 || ^20.3.0 || >=21.0.0, npm: '>=9.6.5', pnpm: '>=7.1.0', yarn: '>=3.2.0'} + cpu: [x64] + os: [linux] + requiresBuild: true + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.0.1 + dev: false + optional: true + + /@img/sharp-linuxmusl-arm64@0.33.2: + resolution: {integrity: sha512-F+0z8JCu/UnMzg8IYW1TMeiViIWBVg7IWP6nE0p5S5EPQxlLd76c8jYemG21X99UzFwgkRo5yz2DS+zbrnxZeA==} + engines: {musl: '>=1.2.2', node: ^18.17.0 || ^20.3.0 || >=21.0.0, npm: '>=9.6.5', pnpm: '>=7.1.0', yarn: '>=3.2.0'} + cpu: [arm64] + os: [linux] + requiresBuild: true + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.0.1 + dev: false + optional: true + + /@img/sharp-linuxmusl-x64@0.33.2: + resolution: {integrity: sha512-+ZLE3SQmSL+Fn1gmSaM8uFusW5Y3J9VOf+wMGNnTtJUMUxFhv+P4UPaYEYT8tqnyYVaOVGgMN/zsOxn9pSsO2A==} + engines: {musl: '>=1.2.2', node: ^18.17.0 || ^20.3.0 || >=21.0.0, npm: '>=9.6.5', pnpm: '>=7.1.0', yarn: '>=3.2.0'} + cpu: [x64] + os: [linux] + requiresBuild: true + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.0.1 + dev: false + optional: true + + /@img/sharp-wasm32@0.33.2: + resolution: {integrity: sha512-fLbTaESVKuQcpm8ffgBD7jLb/CQLcATju/jxtTXR1XCLwbOQt+OL5zPHSDMmp2JZIeq82e18yE0Vv7zh6+6BfQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0, npm: '>=9.6.5', pnpm: '>=7.1.0', yarn: '>=3.2.0'} + cpu: [wasm32] + requiresBuild: true + dependencies: + '@emnapi/runtime': 0.45.0 + dev: false + optional: true + + /@img/sharp-win32-ia32@0.33.2: + resolution: {integrity: sha512-okBpql96hIGuZ4lN3+nsAjGeggxKm7hIRu9zyec0lnfB8E7Z6p95BuRZzDDXZOl2e8UmR4RhYt631i7mfmKU8g==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0, npm: '>=9.6.5', pnpm: '>=7.1.0', yarn: '>=3.2.0'} + cpu: [ia32] + os: [win32] + requiresBuild: true + dev: false + optional: true + + /@img/sharp-win32-x64@0.33.2: + resolution: {integrity: sha512-E4magOks77DK47FwHUIGH0RYWSgRBfGdK56kIHSVeB9uIS4pPFr4N2kIVsXdQQo4LzOsENKV5KAhRlRL7eMAdg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0, npm: '>=9.6.5', pnpm: '>=7.1.0', yarn: '>=3.2.0'} + cpu: [x64] + os: [win32] + requiresBuild: true + dev: false + optional: true + + /@types/mime@3.0.4: + resolution: {integrity: sha512-iJt33IQnVRkqeqC7PzBHPTC6fDlRNRW8vjrgqtScAhrmMwe8c4Eo7+fUGTa+XdWrpEgpyKWMYmi2dIwMAYRzPw==} dev: true - /@types/node@20.8.9: - resolution: {integrity: sha512-UzykFsT3FhHb1h7yD4CA4YhBHq545JC0YnEz41xkipN88eKQtL6rSgocL5tbAP6Ola9Izm/Aw4Ora8He4x0BHg==} + /@types/node@20.11.19: + resolution: {integrity: sha512-7xMnVEcZFu0DikYjWOlRq7NTPETrm7teqUT2WkQjrTIkEgUyyGdWsj/Zg8bEJt5TNklzbPD1X3fqfsHw3SpapQ==} dependencies: undici-types: 5.26.5 dev: true @@ -53,55 +249,28 @@ packages: /@types/sharp@0.31.1: resolution: {integrity: sha512-5nWwamN9ZFHXaYEincMSuza8nNfOof8nmO+mcI+Agx1uMUk4/pQnNIcix+9rLPXzKrm1pS34+6WRDbDV0Jn7ag==} dependencies: - '@types/node': 20.8.9 + '@types/node': 20.11.19 dev: true /asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} dev: false - /axios@1.6.0: - resolution: {integrity: sha512-EZ1DYihju9pwVB+jg67ogm+Tmqc6JmhamRN6I4Zt8DfZu5lbcQGw3ozH9lFejSJgs/ibaef3A9PMXPLeefFGJg==} + /axios@1.6.7: + resolution: {integrity: sha512-/hDJGff6/c7u0hDkvkGxR/oy6CbCs8ziCsC7SqmhjfozqiJGc8Z11wrv9z9lYfY4K8l+H9TpjcMDX0xOZmx+RA==} dependencies: - follow-redirects: 1.15.3 + follow-redirects: 1.15.5 form-data: 4.0.0 proxy-from-env: 1.1.0 transitivePeerDependencies: - debug dev: false - /b4a@1.6.4: - resolution: {integrity: sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw==} - dev: false - - /base64-js@1.5.1: - resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - dev: false - - /bl@4.1.0: - resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} - dependencies: - buffer: 5.7.1 - inherits: 2.0.4 - readable-stream: 3.6.2 - dev: false - - /buffer@5.7.1: - resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} - dependencies: - base64-js: 1.5.1 - ieee754: 1.2.1 - dev: false - /chalk@5.3.0: resolution: {integrity: sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==} engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} dev: false - /chownr@1.1.4: - resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} - dev: false - /color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -135,21 +304,9 @@ packages: delayed-stream: 1.0.0 dev: false - /commander@11.1.0: - resolution: {integrity: sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==} - engines: {node: '>=16'} - dev: false - - /decompress-response@6.0.0: - resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} - engines: {node: '>=10'} - dependencies: - mimic-response: 3.1.0 - dev: false - - /deep-extend@0.6.0: - resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} - engines: {node: '>=4.0.0'} + /commander@12.0.0: + resolution: {integrity: sha512-MwVNWlYjDTtOjX5PiD7o5pK0UrFU/OYgcJfjjK4RaHZETNtjJqrZa9Y9ds88+A+f+d5lv+561eZ+yCKoS3gbAA==} + engines: {node: '>=18'} dev: false /delayed-stream@1.0.0: @@ -162,28 +319,13 @@ packages: engines: {node: '>=8'} dev: false - /dotenv@16.3.1: - resolution: {integrity: sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==} + /dotenv@16.4.4: + resolution: {integrity: sha512-XvPXc8XAQThSjAbY6cQ/9PcBXmFoWuw1sQ3b8HqUCR6ziGXjkTi//kB9SWa2UwqlgdAIuRqAa/9hVljzPehbYg==} engines: {node: '>=12'} dev: false - /end-of-stream@1.4.4: - resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} - dependencies: - once: 1.4.0 - dev: false - - /expand-template@2.0.3: - resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} - engines: {node: '>=6'} - dev: false - - /fast-fifo@1.3.2: - resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} - dev: false - - /follow-redirects@1.15.3: - resolution: {integrity: sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==} + /follow-redirects@1.15.5: + resolution: {integrity: sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==} engines: {node: '>=4.0'} peerDependencies: debug: '*' @@ -201,26 +343,6 @@ packages: mime-types: 2.1.35 dev: false - /fs-constants@1.0.0: - resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} - dev: false - - /github-from-package@0.0.0: - resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} - dev: false - - /ieee754@1.2.1: - resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} - dev: false - - /inherits@2.0.4: - resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} - dev: false - - /ini@1.3.8: - resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} - dev: false - /is-arrayish@0.3.2: resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==} dev: false @@ -250,130 +372,46 @@ packages: hasBin: true dev: false - /mimic-response@3.1.0: - resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} - engines: {node: '>=10'} - dev: false - - /minimist@1.2.8: - resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} - dev: false - - /mkdirp-classic@0.5.3: - resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} - dev: false - - /napi-build-utils@1.0.2: - resolution: {integrity: sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==} - dev: false - - /node-abi@3.51.0: - resolution: {integrity: sha512-SQkEP4hmNWjlniS5zdnfIXTk1x7Ome85RDzHlTbBtzE97Gfwz/Ipw4v/Ryk20DWIy3yCNVLVlGKApCnmvYoJbA==} - engines: {node: '>=10'} - dependencies: - semver: 7.5.4 - dev: false - - /node-addon-api@6.1.0: - resolution: {integrity: sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==} - dev: false - - /once@1.4.0: - resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} - dependencies: - wrappy: 1.0.2 - dev: false - - /prebuild-install@7.1.1: - resolution: {integrity: sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==} - engines: {node: '>=10'} - hasBin: true - dependencies: - detect-libc: 2.0.2 - expand-template: 2.0.3 - github-from-package: 0.0.0 - minimist: 1.2.8 - mkdirp-classic: 0.5.3 - napi-build-utils: 1.0.2 - node-abi: 3.51.0 - pump: 3.0.0 - rc: 1.2.8 - simple-get: 4.0.1 - tar-fs: 2.1.1 - tunnel-agent: 0.6.0 - dev: false - /proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} dev: false - /pump@3.0.0: - resolution: {integrity: sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==} - dependencies: - end-of-stream: 1.4.4 - once: 1.4.0 - dev: false - - /queue-tick@1.0.1: - resolution: {integrity: sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==} - dev: false - - /rc@1.2.8: - resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} - hasBin: true - dependencies: - deep-extend: 0.6.0 - ini: 1.3.8 - minimist: 1.2.8 - strip-json-comments: 2.0.1 - dev: false - - /readable-stream@3.6.2: - resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} - engines: {node: '>= 6'} - dependencies: - inherits: 2.0.4 - string_decoder: 1.3.0 - util-deprecate: 1.0.2 - dev: false - - /safe-buffer@5.2.1: - resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} - dev: false - - /semver@7.5.4: - resolution: {integrity: sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==} + /semver@7.6.0: + resolution: {integrity: sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==} engines: {node: '>=10'} hasBin: true dependencies: lru-cache: 6.0.0 dev: false - /sharp@0.32.6: - resolution: {integrity: sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w==} - engines: {node: '>=14.15.0'} + /sharp@0.33.2: + resolution: {integrity: sha512-WlYOPyyPDiiM07j/UO+E720ju6gtNtHjEGg5vovUk1Lgxyjm2LFO+37Nt/UI3MMh2l6hxTWQWi7qk3cXJTutcQ==} + engines: {libvips: '>=8.15.1', node: ^18.17.0 || ^20.3.0 || >=21.0.0} requiresBuild: true dependencies: color: 4.2.3 detect-libc: 2.0.2 - node-addon-api: 6.1.0 - prebuild-install: 7.1.1 - semver: 7.5.4 - simple-get: 4.0.1 - tar-fs: 3.0.4 - tunnel-agent: 0.6.0 - dev: false - - /simple-concat@1.0.1: - resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} - dev: false - - /simple-get@4.0.1: - resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} - dependencies: - decompress-response: 6.0.0 - once: 1.4.0 - simple-concat: 1.0.1 + semver: 7.6.0 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.33.2 + '@img/sharp-darwin-x64': 0.33.2 + '@img/sharp-libvips-darwin-arm64': 1.0.1 + '@img/sharp-libvips-darwin-x64': 1.0.1 + '@img/sharp-libvips-linux-arm': 1.0.1 + '@img/sharp-libvips-linux-arm64': 1.0.1 + '@img/sharp-libvips-linux-s390x': 1.0.1 + '@img/sharp-libvips-linux-x64': 1.0.1 + '@img/sharp-libvips-linuxmusl-arm64': 1.0.1 + '@img/sharp-libvips-linuxmusl-x64': 1.0.1 + '@img/sharp-linux-arm': 0.33.2 + '@img/sharp-linux-arm64': 0.33.2 + '@img/sharp-linux-s390x': 0.33.2 + '@img/sharp-linux-x64': 0.33.2 + '@img/sharp-linuxmusl-arm64': 0.33.2 + '@img/sharp-linuxmusl-x64': 0.33.2 + '@img/sharp-wasm32': 0.33.2 + '@img/sharp-win32-ia32': 0.33.2 + '@img/sharp-win32-x64': 0.33.2 dev: false /simple-swizzle@0.2.2: @@ -382,68 +420,14 @@ packages: is-arrayish: 0.3.2 dev: false - /streamx@2.15.1: - resolution: {integrity: sha512-fQMzy2O/Q47rgwErk/eGeLu/roaFWV0jVsogDmrszM9uIw8L5OA+t+V93MgYlufNptfjmYR1tOMWhei/Eh7TQA==} - dependencies: - fast-fifo: 1.3.2 - queue-tick: 1.0.1 - dev: false - - /string_decoder@1.3.0: - resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} - dependencies: - safe-buffer: 5.2.1 - dev: false - - /strip-json-comments@2.0.1: - resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} - engines: {node: '>=0.10.0'} - dev: false - - /tar-fs@2.1.1: - resolution: {integrity: sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==} - dependencies: - chownr: 1.1.4 - mkdirp-classic: 0.5.3 - pump: 3.0.0 - tar-stream: 2.2.0 - dev: false - - /tar-fs@3.0.4: - resolution: {integrity: sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w==} - dependencies: - mkdirp-classic: 0.5.3 - pump: 3.0.0 - tar-stream: 3.1.6 - dev: false - - /tar-stream@2.2.0: - resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} - engines: {node: '>=6'} - dependencies: - bl: 4.1.0 - end-of-stream: 1.4.4 - fs-constants: 1.0.0 - inherits: 2.0.4 - readable-stream: 3.6.2 - dev: false - - /tar-stream@3.1.6: - resolution: {integrity: sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==} - dependencies: - b4a: 1.6.4 - fast-fifo: 1.3.2 - streamx: 2.15.1 - dev: false - - /tunnel-agent@0.6.0: - resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} - dependencies: - safe-buffer: 5.2.1 + /tslib@2.6.2: + resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} + requiresBuild: true dev: false + optional: true - /typescript@5.2.2: - resolution: {integrity: sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==} + /typescript@5.3.3: + resolution: {integrity: sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==} engines: {node: '>=14.17'} hasBin: true dev: true @@ -452,14 +436,6 @@ packages: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} dev: true - /util-deprecate@1.0.2: - resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} - dev: false - - /wrappy@1.0.2: - resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} - dev: false - /yallist@4.0.0: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} dev: false diff --git a/src/Downloader.js b/src/Downloader.js index cf965f3..7625267 100644 --- a/src/Downloader.js +++ b/src/Downloader.js @@ -6,6 +6,7 @@ import { fileURLToPath } from "url" import GetCorrectContent from "./helpers/GetCorrectContent.js" import GetURLFilename from "./helpers/GetURLFilename.js" import Question from "./helpers/Question.js" +import Queue from "./Queue.js" import axios, { AxiosError } from "axios" import dotenv from "dotenv" import chalk from "chalk" @@ -20,33 +21,46 @@ const configPath = join(root, "config.json") const isTesting = process.env.npm_command === "test" || process.env.npm_lifecycle_event === "test" +Object.assign(axios.defaults.headers.common, { + "Sec-Ch-Prefers-Color-Scheme": "dark", + "Sec-Ch-Ua": '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"', + "Sec-Ch-Ua-Full-version-list": '"Not A(Brand";v="99.0.0.0", "Google Chrome";v="121.0.6167.185", "Chromium";v="121.0.6167.185"', + "Sec-Ch-Ua-Mobile": "?0", + "Sec-Ch-Ua-Model": '""', + "Sec-Ch-Ua-Platform": '"Windows"', + "Sec-Ch-Ua-Platform-version": '"15.0.0"', + "Sec-Fetch-Site": "same-origin", + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36" +}) + export default class Downloader { - /** @type {import("./typings/api.js").APIHeaders} */ headers + /** @type {import("./typings/api.js").APIHeaders} */ headers = { + Accept: "*/*", + Origin: BASE_URL + } /** @type {import("./typings/index.js").Config} */ config /** @type {string[]} */ usernames - /** @type {number} */ limit + /** @type {number | undefined} */ limit /** @type {string} */ output + /** @type {Queue>} */ queue /** * @param {string | string[]} usernames + * @param {number} queue * @param {number} [limit] */ - constructor(usernames, limit = 15){ + constructor(usernames, queue, limit){ this.usernames = Array.isArray(usernames) ? usernames : [usernames] this.limit = limit - - // TODO: Add an array with the downloads queue - - this.headers = { - Accept: "*/*", - Origin: BASE_URL - } + this.queue = new Queue(queue) } GetConfig(){ if(!existsSync(configPath)){ /** @type {import("./typings/index.js").Config} */ const config = this.config = { cookie: {} } - if(!isTesting) this.SetConfig(true) + this.SetConfig(true) return config } @@ -57,7 +71,9 @@ export default class Downloader { return this.config = /** @type {import("./typings/index.js").Config} */ (config) } - SetConfig(sync = false){ + async SetConfig(sync = false){ + if(isTesting) return + const data = JSON.stringify(this.config, null, "\t") + "\n" if(sync){ @@ -66,10 +82,12 @@ export default class Downloader { return } - return writeFile(configPath, data, "utf8") - .then(() => this.SetEnv()) + await writeFile(configPath, data, "utf8") + await this.SetEnv(sync) } SetEnv(sync = false){ + if(isTesting) return + const { config } = this const envPath = join(root, ".env") const data = existsSync(envPath) ? dotenv.parse(envPath) : {} @@ -96,7 +114,7 @@ export default class Downloader { if(csrftoken) headers["X-Csrftoken"] = csrftoken if(app_id) headers["X-Ig-App-Id"] = app_id - headers.Cookie = Object.entries(cookie).map(([key, value]) => `${key}=${value ?? ""}`).join("; ") + headers.Cookie = Object.entries(cookie).map(([key, value]) => `${key}=${value || ""}`).join("; ") } /** @param {{ * output: string @@ -157,10 +175,12 @@ export default class Downloader { const folder = join(output, username) + if(highlights) this.Log(new Error("Highlights download disabled temporarily")) + const results = await Promise.allSettled([ timeline && this.DownloadTimeline(username, folder), - highlights && this.DownloadHighlights(user_id, folder, hcover), - stories && this.DownloadStories(user_id, folder) + // highlights && this.DownloadHighlights(user_id, folder, hcover, this.limit, username), + stories && this.DownloadStories(user_id, folder, this.limit, username) ]) for(const result of results){ @@ -206,11 +226,13 @@ export default class Downloader { const { id } = await this.GetUser(username) return id } - /** @param {string} user_id */ - async GetHighlights(user_id){ + /** + * @param {string} user_id + * @param {string} username + */ + async GetHighlights(user_id, username){ const url = new URL(API_QUERY, BASE_URL) - url.searchParams.set("query_hash", this.config.queryHash) url.searchParams.set("user_id", user_id) url.searchParams.set("include_chaining", "false") url.searchParams.set("include_reel", "false") @@ -221,33 +243,50 @@ export default class Downloader { /** @type {import("axios").AxiosResponse} */ const response = await this.Request(url, "GET", { - headers: { "X-Requested-With": "XMLHttpRequest" }, + headers: { + Referer: username ? `${BASE_URL}/${username}/` : BASE_URL + "/", + "X-Requested-With": "XMLHttpRequest" + }, responseType: "json" }) - return response.data.data.user.edge_highlight_reels.edges.map(({ node }) => node) + try{ + return response.data.data.user.edge_highlight_reels.edges.map(({ node }) => node) + }catch(error){ + throw new Error(`Failed to get user (${username || user_id}) highlights`, { + cause: /** @type {Error} */ (error).message.replace(/\[?Error\]?:? ?/, "") + }) + } } - /** @param {`${number}`[]} reelsIds */ - async GetHighlightsContents(reelsIds){ + /** + * @param {`${number}`[]} reelsIds + * @param {string} [username] + */ + async GetHighlightsContents(reelsIds, username){ const url = new URL(API_REELS, BASE_URL) for(const id of reelsIds) url.searchParams.append("reel_ids", "highlight:" + id) /** @type {import("axios").AxiosResponse} */ const response = await this.Request(url, "GET", { + headers: { Referer: username ? `${BASE_URL}/${username}/` : BASE_URL + "/" }, responseType: "json" }) return response.data.reels_media } - /** @param {`${number}`} userId */ - async GetStories(userId){ + /** + * @param {`${number}`} userId + * @param {string} [username] + */ + async GetStories(userId, username){ const url = new URL(API_REELS, BASE_URL) url.searchParams.set("reel_ids", userId) /** @type {import("axios").AxiosResponse} */ const response = await this.Request(url, "GET", { + headers: { Referer: username ? `${BASE_URL}/${username}/` : BASE_URL + "/" }, responseType: "json" }) @@ -263,9 +302,10 @@ export default class Downloader { * @param {string} folder * @param {boolean} [hcover] * @param {number} [limit] + * @param {string} [username] */ - async DownloadHighlights(user_id, folder, hcover, limit = Infinity){ - const highlights = await this.GetHighlights(user_id) + async DownloadHighlights(user_id, folder, hcover, limit = Infinity, username){ + const highlights = await this.GetHighlights(user_id, username) const highlightsMap = new Map(highlights.map(reel => [reel.id, reel])) @@ -277,7 +317,7 @@ export default class Downloader { while(highlights.length && limit > count){ const ids = highlights.splice(0, 10).map(({ id }) => id) - const highlightsContents = await this.GetHighlightsContents(ids) + const highlightsContents = await this.GetHighlightsContents(ids, username) if(!highlightsContents) throw new Error("No highlights found. The request might have been forbidden") @@ -307,7 +347,7 @@ export default class Downloader { } const data = { count, limit } - const { urls, limited } = await this.DownloadItems(items, folder, data) + const { urls, limited } = await this.DownloadItems(items, folder, data, username) count = data.count @@ -339,9 +379,10 @@ export default class Downloader { * @param {string} user_id * @param {string} folder * @param {number} [limit] + * @param {string} [username] */ - async DownloadStories(user_id, folder, limit = Infinity){ - const results = await this.GetStories(/** @type {`${number}`} */ (user_id)) + async DownloadStories(user_id, folder, limit = Infinity, username){ + const results = await this.GetStories(/** @type {`${number}`} */ (user_id), username) if(!results) return this.Log("No stories found") @@ -357,7 +398,7 @@ export default class Downloader { while(stories.length && limit > count){ const items = stories.splice(0, 10) const data = { count, limit } - const { limited } = await this.DownloadItems(items, folder, data) + const { limited } = await this.DownloadItems(items, folder, data, username) count = data.count @@ -396,12 +437,10 @@ export default class Downloader { first = false } - if(!existsSync(folder)){ - await mkdir(folder, { recursive: true }) - } + await mkdir(folder, { recursive: true }) const data = { count, limit } - const { limited } = await this.DownloadItems(items, folder, data) + const { limited } = await this.DownloadItems(items, folder, data, username) if(limited) break @@ -416,11 +455,10 @@ export default class Downloader { /** * @param {import("./typings/api.js").FeedItem[]} items * @param {string} folder - * @param {object} [data] - * @param {number} data.count - * @param {number} data.limit + * @param {{ count: number, limit: number }} [data] + * @param {string} [username] */ - async DownloadItems(items, folder, data){ + async DownloadItems(items, folder, data, username){ /** @type {Map} */ const urls = new Map @@ -472,9 +510,11 @@ export default class Downloader { await Promise.all(Array.from(urls.entries()).map(async ([url, date]) => { try{ - await this.Download(url, folder, date) + await this.Download(url, folder, date, undefined, { + headers: { Referer: username ? `${BASE_URL}/${username}/` : BASE_URL + "/" } + }) }catch(error){ - this.Log(error) + this.Log(error instanceof Error ? error : new Error(String(error))) urls.delete(url) } })) @@ -487,45 +527,48 @@ export default class Downloader { /** * @param {string} url * @param {string} folder - * @param {Date} [date] + * @param {Date | number} [date] * @param {string} [filename] * @param {import("axios").AxiosRequestConfig} [config] * @returns {Promise} */ - async Download(url, folder, date = new Date, filename, config){ - if(!config) config = {} + async Download(url, folder, date = new Date, filename = "", config = {}){ if(!filename) filename = GetURLFilename(url) const { name, ext } = parse(filename) if(/^image\/.+$/.test(mime.getType(ext))){ - Object.assign(config, { responseType: "arraybuffer" }) + return this.queue.add(async () => { + Object.assign(config, { responseType: "arraybuffer" }) - /** @type {import("axios").AxiosResponse} */ - const { data, status } = await this.Request(url, "GET", config) + /** @type {import("axios").AxiosResponse} */ + const { data, status } = await this.Request(url, "GET", config) - if(status < 200 || status >= 300) throw new Error(`Request to media ${filename} failed with status ${status}`) + if(status < 200 || status >= 300) throw new Error(`Request to media ${filename} failed with status ${status}`) - const { format } = await sharp(data).metadata() - const path = join(folder, `${name}.${format === "jpeg" ? "jpg" : format}`) + const { format } = await sharp(data).metadata() + const path = join(folder, `${name}.${format === "jpeg" ? "jpg" : format}`) - await writeFile(path, data) - await utimes(path, date, date) + await writeFile(path, data) + await utimes(path, new Date, date) - return path + return path + }) } - Object.assign(config, { responseType: "stream" }) + return this.queue.add(async () => { + Object.assign(config, { responseType: "stream" }) - /** @type {import("axios").AxiosResponse} */ - const { data } = await this.Request(url, "GET", config) - const path = join(folder, filename) - const file = createWriteStream(path) + /** @type {import("axios").AxiosResponse} */ + const { data } = await this.Request(url, "GET", config) + const path = join(folder, filename) + const file = createWriteStream(path) - return new Promise((resolve, reject) => { - file.on("close", () => utimes(path, date, date).then(() => resolve(path)).catch(reject)) - file.on("error", reject) - data.pipe(file) + return new Promise((resolve, reject) => { + file.on("close", () => utimes(path, date, date).then(() => resolve(path)).catch(reject)) + file.on("error", reject) + data.pipe(file) + }) }) } /** @@ -535,7 +578,7 @@ export default class Downloader { * @param {Omit} [config] */ async Request(url, method = "GET", config = {}){ - config.headers = Object.assign({}, this.headers, config.headers ?? {}) + config.headers = Object.assign({}, this.headers, config.headers || {}) /** @type {import("axios").AxiosResponse} */ let response @@ -553,12 +596,25 @@ export default class Downloader { }) this.UpdateHeaders() - if(!isTesting) await this.SetConfig() + await this.SetConfig() } + let _url + + if(url instanceof URL){ + _url = url + url = url.href + }else{ + _url = new URL(url) + } + + Object.assign(config.headers, { + "Sec-Fetch-Site": BASE_URL === _url.origin ? "same-origin" : "cross-site" + }) + try{ response = await axios({ - url: url instanceof URL ? url.href : url, + url, method, ...config }) @@ -567,7 +623,6 @@ export default class Downloader { return response }catch(error){ - // TODO: Handle axios errors in test mode to prevent token's leakage if(error instanceof AxiosError){ if(error.response){ response = error.response @@ -576,22 +631,18 @@ export default class Downloader { } } - throw error + throw error instanceof Error ? new Error(error.name, { cause: error.message }) : error } } async CheckServerConfig(){ const { config, usernames } = this - if(config.app_id && config.queryHash) return + if(config.app_id) return const response = await this.Request(new URL(usernames[0], BASE_URL), "GET", { responseType: "text" }) if(typeof response?.data === "string"){ - const appId = response.data.match(/"X-IG-App-ID":"(\d+)"/)?.[1] - const queryHash = response.data.match(/"query_hash":"([a-z0-9]+)"/)?.[1] - - config.queryHash = queryHash - config.app_id = appId + config.app_id = response.data.match(/"X-IG-App-ID":"(\d+)"/)?.[1] } } Log(...args){ @@ -601,10 +652,15 @@ export default class Downloader { if(args.length === 1){ const arg = args[0] - if(arg instanceof Error) return console.error(chalk.redBright(`[${date}] ${args[0].message}`)) + + if(arg instanceof Error){ + const message = arg.cause ? `${arg.message} (${arg.cause})` : arg.message + return console.error(chalk.redBright(`[${date}] ${message}`)) + } + if(typeof arg === "string") return console.log(`${chalk.blackBright(`[${date}]`)} ${arg}`) } - console.log(chalk.blackBright(`[${date}] `), ...args) + console.log(chalk.blackBright(`[${date}]`), ...args) } } diff --git a/src/Queue.js b/src/Queue.js new file mode 100644 index 0000000..ef78b6b --- /dev/null +++ b/src/Queue.js @@ -0,0 +1,74 @@ +/** @template {Promise} T */ +export default class Queue { + /** @param {number} limit */ + constructor(limit){ + this.limit = limit + this.running = false + + /** @type {Set<() => T>} */ this._queue = new Set + /** @type {Set} */ this._queuePromises = new Set + /** @type {Map<() => T, (value: any) => void>} */ this._queueCallbacks = new Map + } + /** @param {() => T} item */ + add(item){ + const { size } = this._queuePromises + + if(size < this.limit){ + return this.HandleItem(item) + }else{ + this._queue.add(item) + return this.AwaitItem(item) + } + } + + /** @param {() => T} item */ + async HandleItem(item){ + const promise = item() + + this._queuePromises.add(promise) + + try{ + const value = await promise + if(this._queueCallbacks.has(item)) this._queueCallbacks.get(item)(value) + }catch(error){ + if(this._queueCallbacks.has(item)) this._queueCallbacks.get(item)(error) + } + + this._queuePromises.delete(promise) + this.RunQueue() + + return promise + } + /** + * @param {() => T} item + * @returns {Promise} + */ + async AwaitItem(item){ + let resolve, reject + + const promise = new Promise((res, rej) => { + resolve = res + reject = rej + }) + + this._queueCallbacks.set(item, value => (value instanceof Error ? reject : resolve)(value)) + + const value = await promise + + return value + } + RunQueue(){ + if(this.running) return + + this.running = true + + const queue = Array.from(this._queue).splice(0, this.limit - this._queuePromises.size) + + for(const item of queue){ + this._queue.delete(item) + this.HandleItem(item) + } + + this.running = false + } +} diff --git a/src/config.js b/src/config.js index 6521d41..f6b9b1b 100644 --- a/src/config.js +++ b/src/config.js @@ -30,6 +30,12 @@ const config = { defaultValue: 12, syntax: "" }, + { + option: "limit", + alternative: "l", + description: "Set how many items to download in total", + syntax: "" + }, { option: "no-stories", alternative: "ns", diff --git a/src/index.js b/src/index.js index 21b4be0..afae1cd 100644 --- a/src/index.js +++ b/src/index.js @@ -13,7 +13,6 @@ const __filename = fileURLToPath(import.meta.url) const __dirname = dirname(__filename) const cwd = process.cwd() -const args = process.argv.slice(2) const root = join(__dirname, "..") /** @@ -21,7 +20,7 @@ const root = join(__dirname, "..") * @param {boolean} force */ function GetOutputDirectory(directory, force){ - if(!directory) return GetOutputDirectory(process.cwd(), force) + if(!directory) return GetOutputDirectory(cwd, force) const path = resolve(cwd, directory) const relativePath = relative(root, path) @@ -61,7 +60,7 @@ const command = program const output = GetOutputDirectory(options.output, options.force) - new Downloader(command.args, isNumber(options.queue) ? Number(options.queue) : 12).Init({ + new Downloader(command.args, isNumber(options.queue) ? Number(options.queue) : 12, isNumber(options.limit) ? Number(options.limit) : undefined).Init({ output, ...options }) @@ -84,4 +83,4 @@ config.options.forEach(({ option, alternative, description, defaultValue, syntax command.option(flags, description, defaultValue) }) -command.parse(process.argv) +command.parse() diff --git a/src/typings/api.d.ts b/src/typings/api.d.ts index d356656..fe4a85f 100644 --- a/src/typings/api.d.ts +++ b/src/typings/api.d.ts @@ -6,6 +6,19 @@ export interface APIHeaders { "X-Asbd-Id"?: string "X-Ig-App-Id"?: string "X-Csrftoken"?: string + "X-Fb-Friendly-Name"?: string + "X-Fb-Lsd"?: string + "Sec-Ch-Prefers-Color-Scheme"?: string + "Sec-Ch-Ua"?: string + "Sec-Ch-Ua-Full-version-list"?: string + "Sec-Ch-Ua-Mobile"?: string + "Sec-Ch-Ua-Model"?: string + "Sec-Ch-Ua-Platform"?: string + "Sec-Ch-Ua-Platform-version"?: string + "Sec-Fetch-Dest"?: string + "Sec-Fetch-Mode"?: string + "Sec-Fetch-Site"?: string + DPR?: "1" Cookie?: string } diff --git a/src/typings/index.d.ts b/src/typings/index.d.ts index 60331fe..df33798 100644 --- a/src/typings/index.d.ts +++ b/src/typings/index.d.ts @@ -1,7 +1,6 @@ export interface Config { app_id?: string csrftoken?: string - queryHash?: string cookie: { [key: string]: string } @@ -11,6 +10,7 @@ export interface Options { output?: string force: boolean queue: 12 | string + limit?: string stories: boolean timeline: boolean highlights: boolean diff --git a/tests/index.test.js b/tests/index.test.js index d15bf40..a7575ad 100644 --- a/tests/index.test.js +++ b/tests/index.test.js @@ -26,7 +26,7 @@ const output = join(root, "output") const folder = join(output, username) test("Instagram API", async t => { - const downloader = new Downloader(username) + const downloader = new Downloader(username, 12) await t.test("should reject if user is not logged in", async () => { await assert.rejects(downloader.CheckLogin.bind(downloader)) @@ -46,10 +46,10 @@ test("Instagram API", async t => { await t.test("should update headers", () => { downloader.UpdateHeaders() - assert.ok(typeof downloader.headers === "object") - assert.ok(typeof downloader.headers.Cookie === "string") - assert.ok(downloader.headers.Cookie.includes("ds_user_id")) - assert.ok(downloader.headers.Cookie.includes("sessionid")) + assert.strictEqual(typeof downloader.headers, "object") + assert.strictEqual(typeof downloader.headers.Cookie, "string") + assert.match(downloader.headers.Cookie, /\bds_user_id=\d/) + assert.match(downloader.headers.Cookie, /\bsessionid=\d/) assert.strictEqual(downloader.headers["X-Csrftoken"], TOKEN) }) @@ -57,15 +57,12 @@ test("Instagram API", async t => { await downloader.CheckServerConfig() downloader.UpdateHeaders() - const { app_id, queryHash } = downloader.config + const { app_id } = downloader.config - assert.ok(typeof app_id === "string") + assert.strictEqual(typeof app_id, "string") assert.strictEqual(downloader.headers["X-Ig-App-Id"], app_id) - assert.ok(typeof queryHash === "string") - assert.ok(/^[a-z0-9]+$/.test(queryHash)) - - if(app_id !== "936619743392459") t.diagnostic("App id has changed") + if(app_id !== "936619743392459") t.diagnostic("App ID has changed: " + app_id) }) await t.test("should check if user is logged in", async () => { @@ -75,20 +72,20 @@ test("Instagram API", async t => { await t.test("should get user id", async () => { const userId = await downloader.GetUserId("instagram") - assert.ok(typeof userId === "string") + assert.strictEqual(typeof userId, "string") assert.strictEqual(userId, "25025320") }) - async function EmptyFolder(){ - const contents = await readdir(folder, { withFileTypes: true }) + await t.test("Download", async t => { + async function EmptyFolder(){ + const contents = await readdir(folder, { withFileTypes: true }) - await Promise.all(contents.map(content => { - const path = join(folder, content.name) - return rm(path, { recursive: content.isFile() }) - })) - } + await Promise.all(contents.map(content => { + const path = join(folder, content.name) + return rm(path, { recursive: content.isFile() }) + })) + } - await t.test("Download", async t => { if(existsSync(folder)) await EmptyFolder() else await mkdir(folder, { recursive: true }) @@ -97,12 +94,12 @@ test("Instagram API", async t => { assert.strictEqual((await readdir(folder)).length, 20) }) - await EmptyFolder() + /* await EmptyFolder() await t.test("should download 20 items from highlights", async () => { await downloader.DownloadHighlights("25025320", folder, true, 20) assert.strictEqual((await readdir(folder)).length, 20) - }) + }) */ await rm(folder, { recursive: true }) })