Make the nightly link check trustworthy: stop failing on no-response, fix the one real dead link

Author: shoumikhin

.ci/docker/common/install_openssl.sh

@@ -5,8 +5,10 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-# NB: This script is needed for sccache and is adopted from PyTorch core repo at
-# https://github.com/pytorch/pytorch/blob/main/.ci/docker/common/install_openssl.sh
+# NB: This script is needed for sccache and is adopted from PyTorch core repo.
+# Core deleted its copy in pytorch/pytorch#179513, so this points at the last
+# revision that still had it instead of at a branch.
+# https://github.com/pytorch/pytorch/blob/9274b93eac9f026e9f51e282449b0300b8e05482/.ci/docker/common/install_openssl.sh
 set -ex
 
 OPENSSL=openssl-1.1.1k

scripts/lint_urls.sh

@@ -26,31 +26,10 @@ while IFS=: read -r filepath url; do
       sleep 1
       code=$(curl -k -gsLm30 --retry 3 --retry-delay 3 --retry-connrefused -o /dev/null -w "%{http_code}" -r 0-0 -A "$user_agent" "$url") || code=000
     fi
-    if [ "$code" -lt 200 ] || [ "$code" -ge 400 ]; then
-      sleep 1
-      request_id=$(curl -sS -G -H 'Accept: application/json' \
-        --data-urlencode "host=$url" \
-        --data-urlencode "max_nodes=1" \
-        --data-urlencode "node=us3.node.check-host.net" \
-        https://check-host.net/check-http \
-        | jq -r .request_id) || request_id=""
-      if [ -n "$request_id" ]; then
-        sleep 5
-        for _ in {1..5}; do
-          new_code=$(curl -sS -H 'Accept: application/json' \
-            "https://check-host.net/check-result/$request_id" \
-            | jq -r -e '.[][0][3]') || new_code=000
-          [[ "$new_code" =~ ^[0-9]+$ ]] || new_code=000
-          if [ "$new_code" -ge 200 ] && [ "$new_code" -lt 400 ]; then
-            code=$new_code
-            break
-          fi
-          sleep 5
-        done
-      fi
-    fi
-    # Treat Cloudflare JS-challenge and rate-limit as success.
-    if [[ "$code" == "403" || "$code" == "429" || "$code" == "503" ]]; then
+    # 403/429/503 are Cloudflare challenges and rate limits. 000 means curl never
+    # got an HTTP response at all, which on a CI runner is a DNS, TLS or egress
+    # problem far more often than a dead link, so it must not fail the job.
+    if [[ "$code" == "000" || "$code" == "403" || "$code" == "429" || "$code" == "503" ]]; then
       printf "${yellow}WARN %s${reset} ${cyan}%s${reset} %s\n" "$code" "$url" "$filepath"
       exit 0
     fi