Merge branch 'main' of https://github.com/star-nox/gpt-crawler

.github/workflows/pr.yml

@@ -0,0 +1,57 @@
+name: Pull request workflow
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize, edited]
+
+permissions:
+  pull-requests: read
+
+jobs:
+  build:
+    name: Build and test
+    runs-on: ubuntu-latest
+    env:
+      CI_JOB_NUMBER: 1
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          cache: npm
+          node-version: 18
+      - run: npm i
+      - run: npm run build
+      - run: npm run test
+      - uses: preactjs/compressed-size-action@v2
+        with:
+          pattern: ".dist/**/*.{js,ts,json}"
+  static-tests:
+    name: Static tests
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          cache: npm
+          node-version: 18
+      - run: npm i
+      - run: npm run prettier:check
+  build-docker:
+    name: Build Docker image
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile
+          push: false
+  semantic-pr:
+    name: Validate PR title for semantic versioning
+    runs-on: ubuntu-latest
+    steps:
+      - uses: amannn/action-semantic-pull-request@v5
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+npm run fmt

.releaserc

@@ -4,6 +4,7 @@
   ],
   "plugins": [
     "@semantic-release/commit-analyzer",
+    "@semantic-release/release-notes-generator",
     "@semantic-release/changelog",
     "@semantic-release/npm",
     "@semantic-release/git",

CHANGELOG.md

@@ -0,0 +1,5 @@
+# [1.3.0](https://github.com/BuilderIO/gpt-crawler/compare/v1.2.1...v1.3.0) (2024-01-06)
+
+### Features
+
+- add exclude pattern for links in config ([16443ed](https://github.com/BuilderIO/gpt-crawler/commit/16443ed9501624de40d921b8e47e4c35f15bf6b4))

Dockerfile

@@ -7,6 +7,9 @@ FROM apify/actor-node-playwright-chrome:18 AS builder
 # to speed up the build using Docker layer cache.
 COPY --chown=myuser package*.json ./
 
+# Delete the prepare script. It's not needed in the final image.
+RUN npm pkg delete scripts.prepare
+
 # Install all dependencies. Don't audit to speed up the installation.
 RUN npm install --include=dev --audit=false
 
@@ -31,7 +34,8 @@ COPY --chown=myuser package*.json ./
 # Install NPM packages, skip optional and development dependencies to
 # keep the image small. Avoid logging too much and print the dependency
 # tree for debugging
-RUN npm --quiet set progress=false \
+RUN npm pkg delete scripts.prepare \
+    && npm --quiet set progress=false \
     && npm install --omit=dev --omit=optional \
     && echo "Installed NPM packages:" \
     && (npm list --omit=dev --all || true) \

config.ts

@@ -5,4 +5,5 @@ export const defaultConfig: Config = {
   match: "",
   maxPagesToCrawl: 50,
   outputFileName: "output.json",
+  maxTokens: 2000000,
 };