algolia-crawler-config.js

new Crawler({
  appId: "N5FBG9I0SH",
  apiKey: "",
  rateLimit: 8,
  maxDepth: 10,
  startUrls: [
    "https://aframe.io/docs/master/introduction/",
    "https://aframe.io/docs/0.1.0/guide/",
    "https://aframe.io/docs/0.2.0/guide/",
    "https://aframe.io/docs/0.3.0/introduction/",
    "https://aframe.io/docs/0.4.0/introduction/",
    "https://aframe.io/docs/0.5.0/introduction/",
    "https://aframe.io/docs/0.6.0/introduction/",
    "https://aframe.io/docs/0.7.0/introduction/",
    "https://aframe.io/docs/0.8.0/introduction/",
    "https://aframe.io/docs/1.0.0/introduction/",
    "https://aframe.io/docs/1.1.0/introduction/",
    "https://aframe.io/docs/1.2.0/introduction/",
    "https://aframe.io/docs/1.3.0/introduction/",
    "https://aframe.io/docs/1.4.0/introduction/",
    "https://aframe.io/docs/1.5.0/introduction/",
    "https://aframe.io/docs/1.6.0/introduction/",
  ],
  renderJavaScript: false,
  sitemaps: [],
  ignoreCanonicalTo: true,
  discoveryPatterns: ["https://aframe.io/docs/**"],
  actions: [
    {
      indexName: "aframe",
      pathsToMatch: ["https://aframe.io/docs/**"],
      recordExtractor: ({ $, url, helpers }) => {
        $(".mobile-header").remove();
        $(".docs-header").remove();
        return helpers.docsearch({
          recordProps: {
            lvl0: {
              selectors: "",
              defaultValue: "Documentation",
            },
            lvl1: [".content h1"],
            lvl2: [".content h"],
            lvl3: [".content h3"],
            lvl4: [".content h4"],
            lvl5: [".content h5"],
            content: [".content p", ".content li", ".content td"],
            version: {
              defaultValue: url.pathname.split("/")[2],
            },
          },
          aggregateContent: true,
          recordVersion: "v2",
        });
      },
    },
  ],
  initialIndexSettings: {
    aframe: {
      attributesForFaceting: ["version"],
      attributesToRetrieve: [
        "hierarchy",
        "content",
        "anchor",
        "url",
        "url_without_anchor",
        "type",
        "version",
      ],
      attributesToHighlight: ["hierarchy", "content"],
      attributesToSnippet: ["content:10"],
      camelCaseAttributes: ["hierarchy", "content"],
      searchableAttributes: [
        "unordered(hierarchy.lvl0)",
        "unordered(hierarchy.lvl1)",
        "unordered(hierarchy.lvl2)",
        "unordered(hierarchy.lvl3)",
        "unordered(hierarchy.lvl4)",
        "unordered(hierarchy.lvl5)",
        "unordered(hierarchy.lvl6)",
        "content",
      ],
      distinct: true,
      attributeForDistinct: "url",
      customRanking: [
        "desc(weight.pageRank)",
        "desc(weight.level)",
        "asc(weight.position)",
      ],
      ranking: [
        "words",
        "filters",
        "typo",
        "attribute",
        "proximity",
        "exact",
        "custom",
      ],
      highlightPreTag: '<span class="algolia-docsearch-suggestion--highlight">',
      highlightPostTag: "</span>",
      minWordSizefor1Typo: 3,
      minWordSizefor2Typos: 7,
      allowTyposOnNumericTokens: false,
      minProximity: 1,
      ignorePlurals: true,
      advancedSyntax: true,
      attributeCriteriaComputedByMinProximity: true,
      removeWordsIfNoResults: "allOptional",
    },
  },
});