Adapt for myst_parser>=0.18

Author: lesteve

build_tools/generate-index.py

@@ -9,14 +9,20 @@
 
 from sphinx_external_toc.parsing import parse_toc_yaml
 
-from myst_parser.main import to_tokens
+from markdown_it.renderer import RendererHTML
+
+from myst_parser.config.main import MdParserConfig
+from myst_parser.parsers.mdit import create_md_parser
+
 
 # This hard-code the git repo root directory relative to this script
 root_dir = Path(__file__).parents[1]
 
 
 def get_first_title_from_md_str(md_str):
-    tokens = to_tokens(md_str)
+    parser = create_md_parser(MdParserConfig(), RendererHTML)
+    tokens = parser.parse(md_str)
+
     is_title_token = False
     for t in tokens:
         if is_title_token:

environment-dev.yml

@@ -14,7 +14,3 @@ dependencies:
   - pip
   - pip:
     - jupyter-book >= 0.11
-    # TODO pin myst_parser temporarily because myst_parser 0.18 removed
-    # to_tokens needed in build_tools/generate-index.py. See
-    # https://myst-parser.readthedocs.io/en/latest/develop/_changelog.html#breaking-changes
-    - myst_parser<0.18