Separate render and lint functionality into subdirs

.remarkrc

@@ -0,0 +1,7 @@
+{
+  "plugins": [
+    "remark-preset-lint-recommended",
+    ["remark-lint-list-item-indent", false],
+    "remark-validate-links"
+  ]
+}

extract-docs.sh

@@ -80,8 +80,10 @@ function extract {
         fi
 
         # These repos have docs in the main dir, not docs/
-        if [[ "$AMWA_ID" == "NMOS" || "$AMWA_ID" == "BCP-002" || "$AMWA_ID" == "BCP-003" ]]; then
-            cp -- *.md "../$target_dir"
+        if [[ "$AMWA_ID" == "SPECS" || "$AMWA_ID" == "NMOS" || "$AMWA_ID" == "BCP-002" || "$AMWA_ID" == "BCP-003" ]]; then
+            for i in *.md; do
+                cp "$i" "../$target_dir"
+            done
             if [ -d images ] ; then
                 cp -r images "../$target_dir" 
             fi
@@ -151,7 +153,9 @@ EOF
                         [ -e "$i.bak" ] && mv "$i.bak" "$i" # Otherwise next checkout will fail
                     done
                     mkdir "../../$target_dir/APIs"
-                    mv -- *.html "../../$target_dir/APIs/"
+                    for i in *.html; do
+                        mv "$i" "../../$target_dir/APIs/"
+                    done
 
                     cp ../../.scripts/json-formatter.js "../../$target_dir/APIs/"
 
@@ -203,7 +207,9 @@ EOF
                         done
                         echo "Moving examples..."
                         mkdir "../../$target_dir/examples"
-                        mv -- *.html "../../$target_dir/examples"
+                        for i in *.html; do
+                            mv "$i" "../../$target_dir/examples"
+                        done
                         cp ../../.scripts/json-formatter.js "../../$target_dir/examples"
                         cp -r ../../.scripts/codemirror "../../$target_dir/examples"
                 )
@@ -217,20 +223,16 @@ for branch in $(cd source-repo; git branch -r | sed 's:origin/::' | grep -v HEAD
     if [[ "$branch" =~ $SHOW_BRANCHES ]]; then
         extract "$branch" "branches/$branch"
     else
-        echo Skipping branch "$branch"
+        echo "Skipping branch $branch"
     fi
 done
 
-# Unfortunately NMOS-PARAMETER-REGISTERS already has a "tags" register, which has grabbed the /tags/ dir.
-# For now just skip in this case, but if we ever want to have releases of param-regs we'll need a better way...
-
-if [[ "$AMWA_ID" != "NMOS-PARAMETER-REGISTERS" ]]; then
-    mkdir tags
-    for tag in $(cd source-repo; git tag); do
-        if [[ "$tag" =~ $SHOW_TAGS ]]; then
-            extract "tags/$tag" "tags/$tag"
-        else
-            echo Skipping tag "$tag"
-        fi
-    done
-fi
+# tag means git tag, release means NMOS/GitHub release
+mkdir releases
+for tag in $(cd source-repo; git tag); do
+    if [[ "$tag" =~ $SHOW_RELEASES ]]; then
+        extract "tags/$tag" "releases/$tag"
+    else
+        echo "Skipping tag/release $tag"
+    fi
+done

fix-links.sh

@@ -25,7 +25,7 @@ echo "Fixing links in documents"
 
 function process_file {
     # Fix overview links
-    perl -pi -e 's~https://github.com/AMWA-TV/nmos/blob/master/NMOS%20Technical%20Overview.md~https://amwa-tv.github.io/nmos/branches/master/NMOS_Technical_Overview.html~gi;' "$1" 
+    perl -pi -e 's~https://github.com/AMWA-TV/nmos/blob/master/NMOS%20Technical%20Overview.md~https://specs.amwa.tv/nmos/branches/master/NMOS_Technical_Overview.html~gi;' "$1" 
     # Change .raml links to .html
     perl -pi -e 's:\.raml\):.html\):g;' "$1"
 
@@ -44,19 +44,19 @@ function process_file {
 
 # These repos have docs in the main dir, not docs/
 if [[ "$AMWA_ID" == "NMOS" || "$AMWA_ID" == "BCP-002" || "$AMWA_ID" == "BCP-003" ]]; then
-    for file in {branches,tags}/*/*.md index.md; do
+    for file in {branches,releases}/*/*.md index.md; do
         process_file "$file"
     done
 
 # NMOS-PARAMETER-REGISTERS has individual dir for each register
 elif [[ "$AMWA_ID" == "NMOS-PARAMETER-REGISTERS" ]]; then
-    for file in branches/*/*/*.md index.md; do  # NO GIT tags (see comment in extract-docs.sh)
+    for file in {branches,releases}/*/*/*.md index.md; do
         process_file "$file"
     done
 
 # Other repos have some or all of docs/, APIs/, examples/
 else
-    for file in {branches,tags}/*/docs/*.md; do
+    for file in {branches,releases}/*/docs/*.md; do
         process_file "$file"
     done
 fi
@@ -71,7 +71,7 @@ if [[ "$AMWA_ID" == "NMOS-TESTING" ]]; then
         perl -pi -e "s~\]\($1~]($2~g;" "$3"
    }
 
-    for tree in {branches,tags}/*; do
+    for tree in {branches,releases}/*; do
         linkto="$REPO_ADDRESS/blob/${tree##*/}"
         cd "$tree"
             for file in docs/*.md; do

get-config.sh

@@ -19,6 +19,9 @@ AMWA_ID="$(awk '/amwa_id/ { print $2 }' $_CONFIG_YML)"
 REPO_ORIGIN="$(git remote get-url origin)"
 REPO_ADDRESS="${REPO_ORIGIN%.git}"
 REPO_NAME="${REPO_ADDRESS##*/}"
+BASEURL="$(awk '/baseurl/ { print $2 }' $_CONFIG_YML)"
+SITE_NAME="${BASEURL#/}"
 DEFAULT_TREE="$(awk '/default_tree:/ { print $2 }' $_CONFIG_YML)"
-SHOW_TAGS="$(awk '/show_tags:/ { print $2 }' $_CONFIG_YML)"
+SHOW_RELEASES="$(awk '/show_releases:/ { print $2 }' $_CONFIG_YML)"
 SHOW_BRANCHES="$(awk '/show_branches:/ { print $2 }' $_CONFIG_YML)"
+SPEC_SERVER="$(awk '/spec_server/ { print $2 }' $_CONFIG_YML)"

install-dependencies.sh

@@ -2,20 +2,20 @@
 
 set -o errexit
 
-[ ! -e README.md ] && echo Run this from the top-level directory && exit 1
+[[ "${PWD##*/}" != "render" ]] && echo Run this from the top-level directory && exit 1
 
 # shellcheck source=get-config.sh
 . .scripts/get-config.sh
 
-
-if [[ ! "$AMWA_ID" =~ "IS-" ]]; then
-    echo Nothing to install
-    exit 0
+# Just install these for specs that need them
+if [[ -d ../APIs ]]; then
+    rm -rf raml2html-nmos-theme
+    git clone https://${GITHUB_TOKEN:+${GITHUB_TOKEN}@}github.com/AMWA-TV/raml2html-nmos-theme
+    cp .scripts/package.json .
+    yarn install
+    pip3 install setuptools
+    pip3 install jsonref pathlib
 fi
 
-rm -rf raml2html-nmos-theme
-git clone https://${GITHUB_TOKEN:+${GITHUB_TOKEN}@}github.com/AMWA-TV/raml2html-nmos-theme
-cp .scripts/package.json .
-yarn install
-pip3 install setuptools jsonref pathlib
+# All specs need Jekyll
 bundle install

intro_common.md

@@ -1,7 +1,7 @@
 %AMWA_ID% has been developed by the [Advanced Media Workflow Association](https://www.amwa.tv) as part of the [Networked Media Open Specifications](https://www.nmos.tv) initiative.
 
-See [here](https://amwa-tv.github.io/nmos) for an overview of NMOS specifications.
+See [here](https://specs.amwa.tv/nmos) for an overview of NMOS specifications.
 
 The formal specification is provided in [this GitHub repository](%REPO_ADDRESS%). These pages provide supporting documentation and HTML renders of the APIs (which are specified in RAML and JSON Schema).
 
-The links to documentation, API and examples below and for DOCS and SPEC in the page menu bar are for this release (tag) or branch: **%DEFAULT_TREE%**. Links to [other releases (tags)](tags/) and [other branches](branches/) appear later in the page, or in the VERSIONS menu.
+The links to documentation, API and examples below and for DOCS and SPEC in the page menu bar are for this release or branch: **%DEFAULT_TREE%**. Links to [other releases](releases/) and [other branches](branches/) appear later in the page, or in the VERSIONS menu.

lint.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+shopt -s globstar nullglob
+
+echo Linting Markdown...
+if ! find . -name node_modules -prune -o -name '*.md' -print0 | xargs -0 ./node_modules/.bin/remark --rc-path .scripts/.remarkrc --frail; then
+    failed=y
+fi
+
+if [ -d APIs ]; then
+    echo Linting APIs...
+    for i in APIs/*.raml; do
+        perl -pi.bak -e 's/!include//' "$i"
+        if ./node_modules/.bin/yamllint "$i" > output; then
+            echo "$i" ok
+        else
+            cat output
+            echo -e "\033[31m$i failed\033[0m"
+            failed=y
+            rm output
+        fi
+        mv "$i.bak" "$i"
+    done
+fi
+
+if [ -d APIs/schemas ]; then
+    echo Linting schemas...
+    for i in APIs/schemas/*.json ; do
+        if ./node_modules/.bin/jsonlint "$i" > /dev/null; then
+            echo "$i" ok
+        else
+            echo -e "\033[31m$i failed\033[0m"
+        failed=y
+      fi
+    done
+fi
+
+if [ -d examples ]; then
+    echo Linting examples...
+    for i in examples/**/*.json ; do
+        if ./node_modules/.bin/jsonlint "$i" > /dev/null; then
+            echo "$i" ok
+        else
+            echo -e "\033[31m$i failed\033[0m"
+        failed=y
+      fi
+    done
+fi
+
+if [ "$failed" == "y" ]; then
+    exit 1
+else
+    exit 0
+fi

make-clean.sh

@@ -2,4 +2,4 @@
 
 set -o errexit
 
-rm -rf source-repo branches tags index.md index-contents.md
+rm -rf source-repo branches releases index.md index-contents.md _site assets _layouts

make-indexes.sh

@@ -24,7 +24,7 @@ set -o errexit
 shopt -s nullglob
 
 # Text in this file will appear at the start of the top-level index
-INTRO=intro.md
+INTRO=../intro.md
 INTRO_COMMON=.scripts/intro_common.md
 
 # Filename for index in each dir
@@ -80,26 +80,28 @@ function add_possibly_nested_example {
     echo "- [${doc%%.md}]($underscore_space_doc)" >> "$INDEX"
 };
 
-function do_b_or_t {
-    b_or_t=$1
-    label=$2
+function do_tree {
+    tree=$1
+    label=$2 # because of spelling of plurals
 
-    echo "Processing $b_or_t $INDEX..."
+    echo "Processing $tree $INDEX..."
     (
-        cd "$b_or_t" || exit 1
+        cd "$tree" || exit 1
         for dir in */; do
             dirname="${dir%%/}"
             echo "Making $dirname/$INDEX"
             (
                 cd "$dir" || exit 1
 
                 # These repos have unnumbered docs in the main dir
-                if [[ "$AMWA_ID" == "NMOS" || "$AMWA_ID" == "BCP-002" || "$AMWA_ID" == "BCP-003" ]]; then
+                if [[  "$AMWA_ID" == "SPECS" || "$AMWA_ID" == "NMOS" || "$AMWA_ID" == "BCP-002" || "$AMWA_ID" == "BCP-003" ]]; then
                     for doc in *.md; do
                         if [[ "$doc" != "index.md" &&
+                            "$doc" != "intro.md" &&
                             "$doc" != "README.md" &&
                             "$doc" != "CHANGELOG.md" &&
-                            "$doc" != "CONTRIBUTING.md" ]]; then
+                            "$doc" != "CONTRIBUTING.md" &&
+                            "$doc" != "404.md" ]]; then
                             add_unnumbered_doc "$doc"
                         fi
 
@@ -171,18 +173,15 @@ function do_b_or_t {
     )
 }
 
-do_b_or_t branches branch
-
-# NMOS-PARAMETER-REGISTERS has NO GIT TAGS (see comment in extract-docs.sh)
-if [[ "$AMWA_ID" != "NMOS-PARAMETER-REGISTERS" ]]; then
-    do_b_or_t tags release/tag
-fi
+do_tree branches branch
+do_tree releases release
 
 echo "Making top level $INDEX"
 
-# Add lint and render status badges -- with GitHub Actions these default to default branch
-default_branch="$(git remote show origin | awk '/HEAD branch/ { print $3 }')"
-cat << EOF > "$INDEX"
+if [[ "$AMWA_ID" != "SPECS" ]]; then
+    # Add lint and render status badges -- with GitHub Actions these default to default branch
+    default_branch="$(git remote show origin | awk '/HEAD branch/ { print $3 }')"
+    cat << EOF > "$INDEX"
 | Repository | Default Branch | Lint (default) | Render (all) |
 | --- | --- | --- | --- |
 | [${REPO_ADDRESS##*/}]($REPO_ADDRESS) \
@@ -191,16 +190,17 @@ cat << EOF > "$INDEX"
 | [![Render Status]($REPO_ADDRESS/workflows/Render/badge.svg)]($REPO_ADDRESS/actions?query=workflow%3ARender) \
 |
 EOF
+fi
 
 # Repo-specific About: section...
-{ 
-    echo -e "\n\n---\n\n## About ${AMWA_ID}\n\n"
+{
+    [[ "$AMWA_ID" != "SPECS" ]] && echo -e "\n\n---\n\n## About ${AMWA_ID}\n\n"
     cat "$INTRO"
     echo -e "\n\n---\n\n"
 } >> "$INDEX"
 
 # Heading/intro depends on repo type
-if [[ "$AMWA_ID" == "NMOS" || "$AMWA_ID" == "BCP-003" || "$AMWA_ID" == "NMOS-TESTING" ]]; then
+if [[ "$AMWA_ID" == "SPECS" || "$AMWA_ID" == "NMOS" || "$AMWA_ID" == "BCP-003" || "$AMWA_ID" == "NMOS-TESTING" ]]; then
     echo "## Documentation" >> "$INDEX"
 elif [[ "$AMWA_ID" == "NMOS-PARAMETER-REGISTERS" ]]; then
     echo "## Parameter Registers" >> "$INDEX"
@@ -220,8 +220,8 @@ fi
 # TODO: DRY on the following...
 
 
-# These excluded repos don't have branch and tags indexes
-if [[ ! "$AMWA_ID" == "NMOS" && ! "$AMWA_ID" == "BCP-002" && ! "$AMWA_ID" == "BCP-003" ]]; then
+# These excluded repos don't have branch and releases indexes
+if [[ ! "$AMWA_ID" == "SPECS" && ! "$AMWA_ID" == "NMOS" && ! "$AMWA_ID" == "BCP-002" && ! "$AMWA_ID" == "BCP-003" ]]; then
     echo Adding branches index...
     INDEX_BRANCHES="branches/index.md"
     # Parameter Registers use branches for published and dev versions
@@ -239,15 +239,15 @@ if [[ ! "$AMWA_ID" == "NMOS" && ! "$AMWA_ID" == "BCP-002" && ! "$AMWA_ID" == "BC
         echo -e "\n[$branch]($branch/)" >>  "$INDEX_BRANCHES"
     done
 
-    echo Adding tags index...
-    INDEX_TAGS="tags/index.md"
-    echo "## Published Releases/Tags" > "$INDEX_TAGS"
-    echo -e "\n##  Published Releases/Tags" >> "$INDEX"
-    for dir in tags/*; do
+    echo Adding releases index...
+    INDEX_RELEASES="releases/index.md"
+    echo "## Published Releases" > "$INDEX_RELEASES"
+    echo -e "\n##  Published Releases" >> "$INDEX"
+    for dir in releases/*; do
         [ ! -d "$dir" ] && continue
-        tag="${dir##*/}"
-        echo -e "\n[$tag](tags/$tag/)" >>  "$INDEX"
-        echo -e "\n[$tag]($tag/)" >>  "$INDEX_TAGS"
+        release="${dir##*/}"
+        echo -e "\n[$release](releases/$release/)" >>  "$INDEX"
+        echo -e "\n[$release]($release/)" >>  "$INDEX_RELEASES"
     done
 
 fi

make-layouts.sh

@@ -2,7 +2,7 @@
 
 set -o errexit
 
-git clone https://${GITHUB_TOKEN:+${GITHUB_TOKEN}@}github.com/AMWA-TV/nmos-doc-layouts .layouts
+git clone --single-branch --branch external-site https://${GITHUB_TOKEN:+${GITHUB_TOKEN}@}github.com/AMWA-TV/nmos-doc-layouts .layouts
 rm -rf _layouts assets
 mv .layouts/_layouts .
 mv .layouts/assets .