check-links.js

import { parse } from 'node-html-parser';
import fs from 'fs';
import fetch from 'node-fetch';
import { URL, fileURLToPath } from 'url';
import path from 'path';
import pLimit from 'p-limit';

export async function checkLinksInHtml(
  htmlContent,
  brokenLinksMap,
  baseUrl,
  documentPath,
  checkedLinks = new Map(),
  distPath = '',
  astroConfigRedirects = {},
  logger,
  checkExternalLinks = true
) {
  const root = parse(htmlContent);
  const linkElements = root.querySelectorAll('a[href]');
  const links = linkElements.map((el) => el.getAttribute('href'));
  // add img src
  const imgElements = root.querySelectorAll('img[src]');
  const imgLinks = imgElements.map((el) => el.getAttribute('src'));
  links.push(...imgLinks);

  const limit = pLimit(50); // Limit to 10 concurrent link checks

  const checkLinkPromises = links.map((link) =>
    limit(async () => {
      if (!isValidUrl(link)) {
        return;
      }

      let absoluteLink;
      try {
        
        // Differentiate between absolute, domain-relative, and relative links
        if (/^https?:\/\//i.test(link) || /^:\/\//i.test(link)) {
          // Absolute URL
          absoluteLink = link;
        } else {
          absoluteLink = new URL(link, "https://localhost" + baseUrl).pathname;
          // if (link !== absoluteLink) {
          //   logger.info(`Link ${link} was resolved to ${absoluteLink}`);
          // }
        }
      } catch (err) {
        // Invalid URL, skip
        logger.error(`Invalid URL in ${normalizePath(documentPath)} ${link} ${err}`);
        return;
      }

      let fetchLink = link;
      if (absoluteLink.startsWith('/') && distPath) {
        fetchLink = absoluteLink;
      }

      if (astroConfigRedirects[fetchLink]) {
        // Check if the link is a redirect
        const redirect = astroConfigRedirects[fetchLink];
        if (redirect) {
          fetchLink = redirect.destination ? redirect.destination : redirect;
        }
      }

      if (checkedLinks.has(fetchLink)) {
        const isBroken = !checkedLinks.get(fetchLink);
        if (isBroken) {
          addBrokenLink(brokenLinksMap, documentPath, link, distPath);
        }
        return;
      }

      let isBroken = false;
     

      if (fetchLink.startsWith('/') && distPath) {
        // Internal link in build mode, check if file exists
        const relativePath = fetchLink;
        // Potential file paths to check
        const possiblePaths = [
          path.join(distPath, relativePath),
          path.join(distPath, relativePath, 'index.html'),
          path.join(distPath, `${relativePath}.html`),
        ];

        // Check if any of the possible paths exist
        if (!possiblePaths.some((p) => fs.existsSync(p))) {
          // console.log('Failed paths', possiblePaths);
          isBroken = true;
          // Fall back to checking a redirect file if it exists. 

        }
      } else  {
        // External link, check via HTTP request. Retry 3 times if ECONNRESET
        if (checkExternalLinks) {
          let retries = 0;
          while (retries < 3) {
            try {
              const response = await fetch(fetchLink, { method: 'GET' });
              isBroken = !response.ok;
              if (isBroken) {
                logger.error(`${response.status} Error fetching ${fetchLink}`);
              }
              break;
            } catch (error) {
              isBroken = true;
              let statusCodeNumber = error.errno == 'ENOTFOUND' ? 404 : (error.errno);
              logger.error(`${statusCodeNumber} error fetching ${fetchLink}`);
              if (error.errno === 'ECONNRESET') {
                retries++;
                continue;
              }
              break;
              }
            }
          }
        }
      

      // Cache the link's validity
      checkedLinks.set(fetchLink, !isBroken);
      checkedLinks.set(absoluteLink, !isBroken);

      if (isBroken) {
        addBrokenLink(brokenLinksMap, documentPath, link, distPath);
      }
    })
  );

  await Promise.all(checkLinkPromises);
}

function isValidUrl(url) {
  // Skip mailto:, tel:, javascript:, and empty links
  if (
    url.startsWith('mailto:') ||
    url.startsWith('tel:') ||
    url.startsWith('javascript:') ||
    url.startsWith('#') ||
    url.trim() === ''
  ) {
    return false;
  }
  return true;
}

function normalizePath(p) {
  p = p.toString();
  // Remove query parameters and fragments
  p = p.split('?')[0].split('#')[0];

  // Remove '/index.html' or '.html' suffixes
  if (p.endsWith('/index.html')) {
    p = p.slice(0, -'index.html'.length);
  } else if (p.endsWith('.html')) {
    p = p.slice(0, -'.html'.length);
  }

  // Ensure leading '/'
  if (!p.startsWith('/')) {
    p = '/' + p;
  }

  return p;
}

export function normalizeHtmlFilePath(filePath, distPath = '') {
  return normalizePath(distPath ? path.relative(distPath, filePath) : filePath);
}

function addBrokenLink(brokenLinksMap, documentPath, brokenLink, distPath) {
  // Normalize document path
  documentPath = normalizeHtmlFilePath(documentPath, distPath);

  // Normalize broken link for reporting
  let normalizedBrokenLink = brokenLink;


  if (!brokenLinksMap.has(normalizedBrokenLink)) {
    brokenLinksMap.set(normalizedBrokenLink, new Set());
  }
  brokenLinksMap.get(normalizedBrokenLink).add(documentPath);
}