diff --git a/.quartoignore b/.quartoignore index 95db102..0632f33 100644 --- a/.quartoignore +++ b/.quartoignore @@ -1,2 +1,6 @@ README.md +NEWS.md LICENSE +docs/ +.quarto/ +!.gitignore diff --git a/NEWS.md b/NEWS.md new file mode 100644 index 0000000..3cf7997 --- /dev/null +++ b/NEWS.md @@ -0,0 +1,12 @@ +## Sverto 0.0.2 + +- Bump minimum Quarto version to 1.3.0. +- Fixes for compatibility with newer Quarto 1.3 pre-releases + - Quarto's switch from Pandoc 2 to Pandoc 3 caused some issues with the way Sverto identifies Svelte import statements. This should no longer be a problem. +- We now take advantage of the improved `.quartoignore` functionality in Quarto 1.3 to: + 1. avoid copying the `docs` folder in with the project template; and + 2. include the `.gitignore` with the template + +## 0.0.1 + +- Initial release diff --git a/_extensions/sverto/_extension.yml b/_extensions/sverto/_extension.yml index 2522b6e..1e08544 100644 --- a/_extensions/sverto/_extension.yml +++ b/_extensions/sverto/_extension.yml @@ -1,7 +1,7 @@ title: Sverto author: 360info -version: 0.0.1 -quarto-version: ">=1.2.0" +version: 0.0.2 +quarto-version: ">=1.3.0" contributes: project: project: diff --git a/_extensions/sverto/create-imports.lua b/_extensions/sverto/create-imports.lua index 62c714a..4e6d462 100644 --- a/_extensions/sverto/create-imports.lua +++ b/_extensions/sverto/create-imports.lua @@ -1,3 +1,11 @@ +-- create-imports: a project pre-render script that: +-- * replaces svelte_import() ojs statements in qmd files, saving to /.sverto +-- * writes svelte import paths to /.sverto s othey can be compiled + +-- some content from quarto's qmd-reader.lua +-- (c) 2023 rstudio, pbc. licensed under gnu gpl v2: +-- https://github.com/quarto-dev/quarto-cli/blob/main/COPYRIGHT + -- return contents of named file function read_file(name) local file = io.open(name, "r") @@ -53,10 +61,114 @@ function path_dir(path) return path:match("(.*".. get_path_sep() ..")") or "" end --- TODO - +-- content following from quarto's qmd-reader.lua +-- (c) 2023 rstudio, pbc. licensed under gnu gpl v2: +-- https://github.com/quarto-dev/quarto-cli/blob/main/COPYRIGHT + +function random_string(size) + -- we replace invalid tags with random strings of the same size + -- to safely allow code blocks inside pipe tables + -- note that we can't use uppercase letters here + -- because pandoc canonicalizes classes to lowercase. + local chars = "abcdefghijklmnopqrstuvwxyz" + local lst = {} + for _ = 1,size do + local ix = math.random(1, #chars) + table.insert(lst, string.sub(chars, ix, ix)) + end + return table.concat(lst, "") +end + +function find_invalid_tags(str) + -- [^.=\n] + -- we disallow "." to avoid catching {.python} + -- we disallow "=" to avoid catching {foo="bar"} + -- we disallow "\n" to avoid multiple lines + + -- no | in lua patterns... + + -- (c standard, 7.4.1.10, isspace function) + -- %s catches \n and \r, so we must use [ \t\f\v] instead + + local patterns = { + "^[ \t\f\v]*(```+[ \t\f\v]*)(%{+[^.=\n\r]*%}+)", + "\n[ \t\f\v]*(```+[ \t\f\v]*)(%{+[^.=\n\r]+%}+)" + } + local function find_it(init) + for _, pattern in ipairs(patterns) do + local range_start, range_end, ticks, tag = str:find(pattern, init) + if range_start ~= nil then + return range_start, range_end, ticks, tag + end + end + return nil + end + + local init = 1 + local range_start, range_end, ticks, tag = find_it(init) + local tag_set = {} + local tags = {} + while tag ~= nil do + init = range_end + 1 + if not tag_set[tag] then + tag_set[tag] = true + table.insert(tags, tag) + end + range_start, range_end, ticks, tag = find_it(init) + end + return tags +end + +function escape_invalid_tags(str) + local tags = find_invalid_tags(str) + -- we must now replace the tags in a careful order. Specifically, + -- we can't replace a key that's a substring of a larger key without + -- first replacing the larger key. + -- + -- ie. if we replace {python} before {{python}}, Bad Things Happen. + -- so we sort the tags by descending size, which suffices + table.sort(tags, function(a, b) return #b < #a end) + + local replacements = {} + for _, k in ipairs(tags) do + local replacement + local attempts = 1 + repeat + replacement = random_string(#k) + attempts = attempts + 1 + until str:find(replacement, 1, true) == nil or attempts == 100 + if attempts == 100 then + print("Internal error, could not find safe replacement for "..k.." after 100 tries") + print("Please file a bug at https://github.com/quarto-dev/quarto-cli") + os.exit(1) + end + -- replace all lua special pattern characters with their + -- escaped versions + local safe_pattern = k:gsub("([%^%$%(%)%%%.%[%]%*%+%-%?])", "%%%1") + replacements[replacement] = k + local patterns = { + "^([ \t\f\v]*```+[ \t\f\v]*)" .. safe_pattern, + "(\n[ \t\f\v]*```+[ \t\f\v]*)" .. safe_pattern + } + + str = str:gsub(patterns[1], "%1" .. replacement):gsub(patterns[2], "%1" .. replacement) + end + return str, replacements +end + +function unescape_invalid_tags(str, tags) + for replacement, k in pairs(tags) do + -- replace all lua special replacement characters with their + -- escaped versions, so that when we restore the behavior, + -- we don't accidentally create a pattern + local result = k:gsub("([$%%])", "%%%1") + str = str:gsub(replacement, result) + end + return str +end local preprocess_qmd_filter = { - + -- search for `import_svelte("X.svelte")` refs in codeblocks and switch them -- to `import("X.js")` CodeBlock = function(block) @@ -114,15 +226,62 @@ end -- transform each input qmd, saving the transformation in .sverto/[path] -- (write the identified .svelte files out to a file too!) for key, qmd_path in ipairs(in_files) do - - local doc = pandoc.read(read_file(qmd_path)) + + -- before we read the file in with pandoc.read, let's read it in as a raw + -- string and do the quarto team's qmd-reader processing on it. THEN we can + -- read it with pandoc.read + + print(">>> PROCESSING " .. qmd_path) + + local raw_doc = read_file(qmd_path) -- store the current qmd_path on disk so the filter can access it write_file(".sverto/.sverto-current-qmd-folder", path_dir(qmd_path)) + + -- escape invalid tags + local txt, tags = escape_invalid_tags(tostring(raw_doc)) + + -- some extension + format stuff that we can maybe ignore? + + -- for k, v in pairs(opts.extensions) do + -- extensions[v] = true + -- end + + -- if param("user-defined-from") then + -- local user_format = _quarto.format.parse_format(param("user-defined-from")) + -- for k, v in pairs(user_format.extensions) do + -- extensions[k] = v + -- end + -- end + + -- -- Format flavor, i.e., which extensions should be enabled/disabled + -- local flavor = { + -- format = "markdown", + -- extensions = extensions, + -- } + + local function restore_invalid_tags(tag) + return tags[tag] or tag + end + + -- NOW we read in with pandoc (hopefully ending up with real code blocks) + -- and restore them + -- local doc = pandoc.read(txt, flavor, opts):walk { + local doc = pandoc.read(txt, "markdown") + + local restored_doc = doc:walk { + CodeBlock = function (cb) + cb.classes = cb.classes:map(restore_invalid_tags) + cb.text = unescape_invalid_tags(cb.text, tags) + return cb + end + } + + -- local doc = pandoc.read(read_file(qmd_path)) -- pre-process the qmd, populating `svelte_files` in the process -- local svelte_files = {} - local transformed_doc = doc:walk(preprocess_qmd_filter) + local transformed_doc = restored_doc:walk(preprocess_qmd_filter) create_dir_recursively(".sverto/" .. path_dir(qmd_path)) write_file(".sverto/" .. qmd_path, pandoc.write(transformed_doc, "markdown")) @@ -132,3 +291,4 @@ end write_file(".sverto/.sverto-outdir", os.getenv("QUARTO_PROJECT_OUTPUT_DIR")) -- TODO - if there's no {{< import .sverto/file.qmd >}} block, add it? + diff --git a/docs/_extensions/sverto/create-imports.lua b/docs/_extensions/sverto/create-imports.lua index 62c714a..4e6d462 100644 --- a/docs/_extensions/sverto/create-imports.lua +++ b/docs/_extensions/sverto/create-imports.lua @@ -1,3 +1,11 @@ +-- create-imports: a project pre-render script that: +-- * replaces svelte_import() ojs statements in qmd files, saving to /.sverto +-- * writes svelte import paths to /.sverto s othey can be compiled + +-- some content from quarto's qmd-reader.lua +-- (c) 2023 rstudio, pbc. licensed under gnu gpl v2: +-- https://github.com/quarto-dev/quarto-cli/blob/main/COPYRIGHT + -- return contents of named file function read_file(name) local file = io.open(name, "r") @@ -53,10 +61,114 @@ function path_dir(path) return path:match("(.*".. get_path_sep() ..")") or "" end --- TODO - +-- content following from quarto's qmd-reader.lua +-- (c) 2023 rstudio, pbc. licensed under gnu gpl v2: +-- https://github.com/quarto-dev/quarto-cli/blob/main/COPYRIGHT + +function random_string(size) + -- we replace invalid tags with random strings of the same size + -- to safely allow code blocks inside pipe tables + -- note that we can't use uppercase letters here + -- because pandoc canonicalizes classes to lowercase. + local chars = "abcdefghijklmnopqrstuvwxyz" + local lst = {} + for _ = 1,size do + local ix = math.random(1, #chars) + table.insert(lst, string.sub(chars, ix, ix)) + end + return table.concat(lst, "") +end + +function find_invalid_tags(str) + -- [^.=\n] + -- we disallow "." to avoid catching {.python} + -- we disallow "=" to avoid catching {foo="bar"} + -- we disallow "\n" to avoid multiple lines + + -- no | in lua patterns... + + -- (c standard, 7.4.1.10, isspace function) + -- %s catches \n and \r, so we must use [ \t\f\v] instead + + local patterns = { + "^[ \t\f\v]*(```+[ \t\f\v]*)(%{+[^.=\n\r]*%}+)", + "\n[ \t\f\v]*(```+[ \t\f\v]*)(%{+[^.=\n\r]+%}+)" + } + local function find_it(init) + for _, pattern in ipairs(patterns) do + local range_start, range_end, ticks, tag = str:find(pattern, init) + if range_start ~= nil then + return range_start, range_end, ticks, tag + end + end + return nil + end + + local init = 1 + local range_start, range_end, ticks, tag = find_it(init) + local tag_set = {} + local tags = {} + while tag ~= nil do + init = range_end + 1 + if not tag_set[tag] then + tag_set[tag] = true + table.insert(tags, tag) + end + range_start, range_end, ticks, tag = find_it(init) + end + return tags +end + +function escape_invalid_tags(str) + local tags = find_invalid_tags(str) + -- we must now replace the tags in a careful order. Specifically, + -- we can't replace a key that's a substring of a larger key without + -- first replacing the larger key. + -- + -- ie. if we replace {python} before {{python}}, Bad Things Happen. + -- so we sort the tags by descending size, which suffices + table.sort(tags, function(a, b) return #b < #a end) + + local replacements = {} + for _, k in ipairs(tags) do + local replacement + local attempts = 1 + repeat + replacement = random_string(#k) + attempts = attempts + 1 + until str:find(replacement, 1, true) == nil or attempts == 100 + if attempts == 100 then + print("Internal error, could not find safe replacement for "..k.." after 100 tries") + print("Please file a bug at https://github.com/quarto-dev/quarto-cli") + os.exit(1) + end + -- replace all lua special pattern characters with their + -- escaped versions + local safe_pattern = k:gsub("([%^%$%(%)%%%.%[%]%*%+%-%?])", "%%%1") + replacements[replacement] = k + local patterns = { + "^([ \t\f\v]*```+[ \t\f\v]*)" .. safe_pattern, + "(\n[ \t\f\v]*```+[ \t\f\v]*)" .. safe_pattern + } + + str = str:gsub(patterns[1], "%1" .. replacement):gsub(patterns[2], "%1" .. replacement) + end + return str, replacements +end + +function unescape_invalid_tags(str, tags) + for replacement, k in pairs(tags) do + -- replace all lua special replacement characters with their + -- escaped versions, so that when we restore the behavior, + -- we don't accidentally create a pattern + local result = k:gsub("([$%%])", "%%%1") + str = str:gsub(replacement, result) + end + return str +end local preprocess_qmd_filter = { - + -- search for `import_svelte("X.svelte")` refs in codeblocks and switch them -- to `import("X.js")` CodeBlock = function(block) @@ -114,15 +226,62 @@ end -- transform each input qmd, saving the transformation in .sverto/[path] -- (write the identified .svelte files out to a file too!) for key, qmd_path in ipairs(in_files) do - - local doc = pandoc.read(read_file(qmd_path)) + + -- before we read the file in with pandoc.read, let's read it in as a raw + -- string and do the quarto team's qmd-reader processing on it. THEN we can + -- read it with pandoc.read + + print(">>> PROCESSING " .. qmd_path) + + local raw_doc = read_file(qmd_path) -- store the current qmd_path on disk so the filter can access it write_file(".sverto/.sverto-current-qmd-folder", path_dir(qmd_path)) + + -- escape invalid tags + local txt, tags = escape_invalid_tags(tostring(raw_doc)) + + -- some extension + format stuff that we can maybe ignore? + + -- for k, v in pairs(opts.extensions) do + -- extensions[v] = true + -- end + + -- if param("user-defined-from") then + -- local user_format = _quarto.format.parse_format(param("user-defined-from")) + -- for k, v in pairs(user_format.extensions) do + -- extensions[k] = v + -- end + -- end + + -- -- Format flavor, i.e., which extensions should be enabled/disabled + -- local flavor = { + -- format = "markdown", + -- extensions = extensions, + -- } + + local function restore_invalid_tags(tag) + return tags[tag] or tag + end + + -- NOW we read in with pandoc (hopefully ending up with real code blocks) + -- and restore them + -- local doc = pandoc.read(txt, flavor, opts):walk { + local doc = pandoc.read(txt, "markdown") + + local restored_doc = doc:walk { + CodeBlock = function (cb) + cb.classes = cb.classes:map(restore_invalid_tags) + cb.text = unescape_invalid_tags(cb.text, tags) + return cb + end + } + + -- local doc = pandoc.read(read_file(qmd_path)) -- pre-process the qmd, populating `svelte_files` in the process -- local svelte_files = {} - local transformed_doc = doc:walk(preprocess_qmd_filter) + local transformed_doc = restored_doc:walk(preprocess_qmd_filter) create_dir_recursively(".sverto/" .. path_dir(qmd_path)) write_file(".sverto/" .. qmd_path, pandoc.write(transformed_doc, "markdown")) @@ -132,3 +291,4 @@ end write_file(".sverto/.sverto-outdir", os.getenv("QUARTO_PROJECT_OUTPUT_DIR")) -- TODO - if there's no {{< import .sverto/file.qmd >}} block, add it? + diff --git a/docs/_quarto.yml b/docs/_quarto.yml index 5e35c3d..5c1a937 100644 --- a/docs/_quarto.yml +++ b/docs/_quarto.yml @@ -17,6 +17,8 @@ website: menu: - text: "Bar chart" file: examples/barchart/index.qmd + - text: Changelog + file: news.qmd # - about.qmd right: - icon: github diff --git a/docs/index.qmd b/docs/index.qmd index 0adff72..149e759 100644 --- a/docs/index.qmd +++ b/docs/index.qmd @@ -68,7 +68,7 @@ Here's the short way to add Svelte component you've written to a Quarto doc: 5. Update the instantiated component with `myVisual.propName` 6. Render your Quarto project as usual with `quarto render` or `quarto preview`. -**To see this all in practice, check out [`example.qmd`](https://github.com/360-info/sverto/blob/firstrelease/example.qmd).** +**To see this all in practice, check out [`example.qmd`](https://github.com/360-info/sverto/blob/main/example.qmd).** :::{.callout-note} The `quarto preview` command won't "live reload" when you modify your Svelte component—but if you modify and save the Quarto doc that imports it, that will trigger a re-render. You may need to hard reload the page in your browser to see the updated Svelte component. diff --git a/docs/news.qmd b/docs/news.qmd new file mode 100644 index 0000000..7828ced --- /dev/null +++ b/docs/news.qmd @@ -0,0 +1,22 @@ +--- +title: News +author: James Goldie, 360info +date: last-modified +format: + html: + title-block-banner: "#e1e4e6" + linkcolor: "#36a7e9" +--- + +## Sverto 0.0.2 + +- Bump minimum Quarto version to 1.3.0. +- Fixes for compatibility with newer Quarto 1.3 pre-releases + - Quarto's switch from Pandoc 2 to Pandoc 3 caused some issues with the way Sverto identifies Svelte import statements. This should no longer be a problem. +- We now take advantage of the improved `.quartoignore` functionality in Quarto 1.3 to: + 1. avoid copying the `docs` folder in with the project template; and + 2. include the `.gitignore` with the template + +## 0.0.1 + +- Initial release