Merge pull request #3828 from Hannah-Sten/tectonic-toml

Add support for Tectonic.toml

Author: PHPirates

CHANGELOG.md

@@ -3,6 +3,9 @@
 ## [Unreleased]
 
 ### Added
+* Support Tectonic V2 CLI in run configuration
+* Add basic support for multiple inputs in Tectonic.toml
+* Improve performance of file set cache used by inspections
 * Support label references to user defined listings environment
 * Add option to disable automatic compilation in power save mode
 * Convert automatic compilation settings to a combobox

Writerside/topics/Run-configuration-settings.md

@@ -92,9 +92,10 @@ _Since b0.6.6_
 
 See [https://tectonic-typesetting.github.io/en-US/](https://tectonic-typesetting.github.io/en-US/) for installation and more info.
 Tectonic has the advantage that it downloads packages automatically, compiles just as much times as needed and handles BibTeX, but it often only works for not too complicated LaTeX documents.
-
 It also has automatic compilation using `tectonic -X watch`.
 
+There is some basic support for a `Tectonic.toml` file, including inspection support (missing imports, for example) for multiple inputs in the toml file (Tectonic 0.15.1 or later).
+
 The documentation can be found at [https://tectonic-typesetting.github.io/book/latest/](https://tectonic-typesetting.github.io/book/latest/)
 
 ## BibTeX compilers

build.gradle.kts

@@ -140,6 +140,7 @@ dependencies {
     // Parsing xml
     implementation("com.fasterxml.jackson.core:jackson-core:2.18.2")
     implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.18.2")
+    implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-toml:2.18.2")
     implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.18.2")
 
     // Http requests

src/nl/hannahsten/texifyidea/run/compiler/LatexCompiler.kt

@@ -12,6 +12,7 @@ import nl.hannahsten.texifyidea.settings.sdk.DockerSdk
 import nl.hannahsten.texifyidea.settings.sdk.DockerSdkAdditionalData
 import nl.hannahsten.texifyidea.settings.sdk.LatexSdkUtil
 import nl.hannahsten.texifyidea.util.LatexmkRcFileFinder
+import nl.hannahsten.texifyidea.util.files.hasTectonicTomlFile
 import nl.hannahsten.texifyidea.util.runCommand
 import java.util.*
 
@@ -231,15 +232,22 @@ enum class LatexCompiler(private val displayName: String, val executableName: St
             moduleRoot: VirtualFile?,
             moduleRoots: Array<VirtualFile>
         ): MutableList<String> {
-            // The available command line arguments can be found at https://github.com/tectonic-typesetting/tectonic/blob/d7a8497c90deb08b5e5792a11d6e8b082f53bbb7/src/bin/tectonic.rs#L158
             val command = mutableListOf(runConfig.compilerPath ?: executableName)
 
-            command.add("--synctex")
+            // The available command line arguments can be found at https://github.com/tectonic-typesetting/tectonic/blob/d7a8497c90deb08b5e5792a11d6e8b082f53bbb7/src/bin/tectonic.rs#L158
+            // The V2 CLI uses a toml file and should not have arguments
+            if (runConfig.mainFile?.hasTectonicTomlFile() != true) {
+                command.add("--synctex")
 
-            command.add("--outfmt=${runConfig.outputFormat.name.lowercase(Locale.getDefault())}")
+                command.add("--outfmt=${runConfig.outputFormat.name.lowercase(Locale.getDefault())}")
 
-            if (outputPath != null) {
-                command.add("--outdir=$outputPath")
+                if (outputPath != null) {
+                    command.add("--outdir=$outputPath")
+                }
+            }
+            else {
+                command.add("-X")
+                command.add("build")
             }
 
             return command
@@ -364,7 +372,7 @@ enum class LatexCompiler(private val displayName: String, val executableName: St
                 command.add(runConfig.beforeRunCommand + " \\input{${mainFile.name}}")
             }
         }
-        else {
+        else if (runConfig.compiler != TECTONIC || runConfig.mainFile?.hasTectonicTomlFile() != true) {
             command.add(mainFile.name)
         }
 

src/nl/hannahsten/texifyidea/run/latex/LatexCommandLineState.kt

@@ -27,6 +27,8 @@ import nl.hannahsten.texifyidea.run.pdfviewer.ExternalPdfViewer
 import nl.hannahsten.texifyidea.run.sumatra.SumatraAvailabilityChecker
 import nl.hannahsten.texifyidea.run.sumatra.SumatraForwardSearchListener
 import nl.hannahsten.texifyidea.util.files.commandsInFileSet
+import nl.hannahsten.texifyidea.util.files.findTectonicTomlFile
+import nl.hannahsten.texifyidea.util.files.hasTectonicTomlFile
 import nl.hannahsten.texifyidea.util.files.psiFile
 import nl.hannahsten.texifyidea.util.includedPackages
 import nl.hannahsten.texifyidea.util.magic.PackageMagic
@@ -92,7 +94,8 @@ open class LatexCommandLineState(environment: ExecutionEnvironment, private val
         val command: List<String> = compiler.getCommand(runConfig, environment.project)
             ?: throw ExecutionException("Compile command could not be created.")
 
-        val commandLine = GeneralCommandLine(command).withWorkDirectory(mainFile.parent.path)
+        val workingDirectory = if (compiler == LatexCompiler.TECTONIC && mainFile.hasTectonicTomlFile()) mainFile.findTectonicTomlFile()!!.parent.path else mainFile.parent.path
+        val commandLine = GeneralCommandLine(command).withWorkDirectory(workingDirectory)
             .withParentEnvironmentType(GeneralCommandLine.ParentEnvironmentType.CONSOLE)
             .withEnvironment(runConfig.environmentVariables.envs)
         val handler = KillableProcessHandler(commandLine)

src/nl/hannahsten/texifyidea/util/files/FileSet.kt

@@ -1,9 +1,12 @@
 package nl.hannahsten.texifyidea.util.files
 
+import com.fasterxml.jackson.dataformat.toml.TomlMapper
 import com.intellij.openapi.application.runReadAction
 import com.intellij.openapi.project.Project
-import com.intellij.openapi.vfs.LocalFileSystem
+import com.intellij.openapi.roots.ProjectFileIndex
 import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.openapi.vfs.findFile
+import com.intellij.openapi.vfs.LocalFileSystem
 import com.intellij.psi.PsiFile
 import com.intellij.psi.search.GlobalSearchScope
 import nl.hannahsten.texifyidea.index.BibtexEntryIndex
@@ -17,51 +20,81 @@ import nl.hannahsten.texifyidea.util.magic.CommandMagic
 import nl.hannahsten.texifyidea.util.magic.cmd
 import nl.hannahsten.texifyidea.util.parser.isDefinition
 import nl.hannahsten.texifyidea.util.parser.requiredParameter
+import java.io.File
 
 /**
  * Finds all the files in the project that are somehow related using includes.
  *
  * When A includes B and B includes C then A, B & C will all return a set containing A, B & C.
+ * There can be multiple root files in one file set.
  *
  * Be careful when using this function directly over something like [ReferencedFileSetService] where the result
  * values are cached.
  *
- * @receiver The file to find the reference set of.
- * @return All the LaTeX and BibTeX files that are cross referenced between each other.
+ * @return Map all root files which include any other file, to the file set containing that root file.
  */
 // Internal because only ReferencedFileSetCache should call this
-internal fun PsiFile.findReferencedFileSetWithoutCache(): Set<PsiFile> {
-    // Setup.
-    val project = this.project
-    val includes = LatexIncludesIndex.Util.getItems(project)
-
+internal fun Project.findReferencedFileSetWithoutCache(): Map<PsiFile, Set<PsiFile>> {
     // Find all root files.
-    val roots = includes.asSequence()
+    return LatexIncludesIndex.Util.getItems(this)
+        .asSequence()
         .map { it.containingFile }
         .distinct()
         .filter { it.isRoot() }
         .toSet()
-
-    // Map root to all directly referenced files.
-    val sets = HashMap<PsiFile, Set<PsiFile>>()
-    for (root in roots) {
-        val referenced = runReadAction { root.referencedFiles(root.virtualFile) } + root
-
-        if (referenced.contains(this)) {
-            return referenced + this
+        .associateWith { root ->
+            // Map root to all directly referenced files.
+            runReadAction { root.referencedFiles(root.virtualFile) } + root
         }
+}
 
-        sets[root] = referenced
+/**
+ * Check for tectonic.toml files in the project.
+ * These files can input multiple tex files, which would then be in the same file set.
+ * Example file: https://github.com/Hannah-Sten/TeXiFy-IDEA/issues/3773#issuecomment-2503221732
+ * @return List of sets of files included by the same toml file.
+ */
+fun findTectonicTomlInclusions(project: Project): List<Set<PsiFile>> {
+    // Actually, according to https://tectonic-typesetting.github.io/book/latest/v2cli/build.html?highlight=tectonic.toml#remarks Tectonic.toml files can appear in any parent directory, but we only search in the project for now
+    val tomlFiles = findTectonicTomlFiles(project)
+    val filesets = tomlFiles.mapNotNull { tomlFile ->
+        val data = TomlMapper().readValue(File(tomlFile.path), Map::class.java)
+        val outputList = data.getOrDefault("output", null) as? List<*> ?: return@mapNotNull null
+        val inputs = (outputList.firstOrNull() as? Map<*, *>)?.getOrDefault("inputs", null) as? List<*> ?: return@mapNotNull null
+        // Inputs can be either a map "inline" -> String or file name
+        // Actually it can also be just a single file name, but then we don't need all this gymnastics
+        inputs.filterIsInstance<String>().mapNotNull {
+            tomlFile.parent.findFile("src/$it")?.psiFile(project)
+        }.toSet()
     }
 
-    // Look for matching root.
-    for (referenced in sets.values) {
-        if (referenced.contains(this)) {
-            return referenced + this
+    return filesets
+}
+
+private fun findTectonicTomlFiles(project: Project): MutableSet<VirtualFile> {
+    val tomlFiles = mutableSetOf<VirtualFile>()
+    ProjectFileIndex.getInstance(project).iterateContent({ tomlFiles.add(it) }, { it.name == "Tectonic.toml" })
+    return tomlFiles
+}
+
+/**
+ * A toml file can be in any parent directory.
+ */
+fun VirtualFile.hasTectonicTomlFile() = findTectonicTomlFile() != null
+
+fun VirtualFile.findTectonicTomlFile(): VirtualFile? {
+    var parent = this
+    for (i in 0..20) {
+        if (parent.parent != null && parent.parent.isDirectory && parent.parent.exists()) {
+            parent = parent.parent
+        }
+        else {
+            break
         }
-    }
 
-    return setOf(this)
+        parent?.findFile("Tectonic.toml")?.let { return it }
+    }
+    return null
 }
 
 /**

src/nl/hannahsten/texifyidea/util/files/ReferencedFileSetCache.kt

@@ -83,14 +83,27 @@ class ReferencedFileSetCache {
      * once and then fill both caches with all the information we have.
      */
     private fun updateCachesFor(requestedFile: PsiFile) {
-        val fileset = requestedFile.findReferencedFileSetWithoutCache()
-        for (file in fileset) {
-            fileSetCache[file.virtualFile] = fileset.map { it.createSmartPointer() }.toSet()
+        val filesets = requestedFile.project.findReferencedFileSetWithoutCache().toMutableMap()
+        val tectonicInclusions = findTectonicTomlInclusions(requestedFile.project)
+
+        // Now we join all the file sets that are in the same file set according to the Tectonic.toml file
+        for (inclusionsSet in tectonicInclusions) {
+            val mappings = filesets.filter { it.value.intersect(inclusionsSet).isNotEmpty() }
+            val newFileSet = mappings.values.flatten().toSet() + inclusionsSet
+            mappings.forEach {
+                filesets[it.key] = newFileSet
+            }
         }
 
-        val rootfiles = requestedFile.findRootFilesWithoutCache(fileset)
-        for (file in fileset) {
-            rootFilesCache[file.virtualFile] = rootfiles.map { it.createSmartPointer() }.toSet()
+        for (fileset in filesets.values) {
+            for (file in fileset) {
+                fileSetCache[file.virtualFile] = fileset.map { it.createSmartPointer() }.toSet()
+            }
+
+            val rootfiles = requestedFile.findRootFilesWithoutCache(fileset)
+            for (file in fileset) {
+                rootFilesCache[file.virtualFile] = rootfiles.map { it.createSmartPointer() }.toSet()
+            }
         }
     }
 
@@ -108,16 +121,11 @@ class ReferencedFileSetCache {
                     // Use the keys of the whole project, because suppose a new include includes the current file, it could be anywhere in the project
                     // Note that LatexIncludesIndex.Util.getItems(file.project) may be a slow operation and should not be run on EDT
                     val includes = LatexIncludesIndex.Util.getItems(file.project)
-                    val numberOfIncludesChanged = if (includes.size != numberOfIncludes[file.project]) {
+
+                    // The cache should be complete once filled, any files not in there are assumed to not be part of a file set that has a valid root file
+                    if (includes.size != numberOfIncludes[file.project]) {
                         numberOfIncludes[file.project] = includes.size
                         dropAllCaches()
-                        true
-                    }
-                    else {
-                        false
-                    }
-
-                    if (!cache.containsKey(file.virtualFile) || numberOfIncludesChanged) {
                         updateCachesFor(file)
                     }
                 }

test/nl/hannahsten/texifyidea/inspections/bibtex/BibtexUnusedEntryInspectionTest.kt

@@ -2,6 +2,7 @@ package nl.hannahsten.texifyidea.inspections.bibtex
 
 import nl.hannahsten.texifyidea.inspections.TexifyInspectionTestBase
 import nl.hannahsten.texifyidea.testutils.writeCommand
+import nl.hannahsten.texifyidea.util.files.ReferencedFileSetService
 
 class BibtexUnusedEntryInspectionTest : TexifyInspectionTestBase(BibtexUnusedEntryInspection()) {
 
@@ -15,7 +16,10 @@ class BibtexUnusedEntryInspectionTest : TexifyInspectionTestBase(BibtexUnusedEnt
     }
 
     fun `test quick fix`() {
-        myFixture.configureByFiles("references-before.bib", "main-quick-fix.tex")
+        myFixture.configureByFiles("references-before.bib", "main-quick-fix.tex").forEach {
+            // Refresh cache
+            ReferencedFileSetService.getInstance().referencedFileSetOf(it)
+        }
         val quickFixes = myFixture.getAllQuickFixes()
         assertEquals("Expected number of quick fixes:", 2, quickFixes.size)
         writeCommand(myFixture.project) {

test/nl/hannahsten/texifyidea/reference/BibtexIdCompletionTest.kt

@@ -3,7 +3,7 @@ package nl.hannahsten.texifyidea.reference
 import com.intellij.codeInsight.completion.CompletionType
 import com.intellij.codeInsight.documentation.DocumentationManager
 import com.intellij.testFramework.fixtures.BasePlatformTestCase
-import org.junit.Test
+import nl.hannahsten.texifyidea.util.files.ReferencedFileSetService
 
 class BibtexIdCompletionTest : BasePlatformTestCase() {
 
@@ -16,7 +16,6 @@ class BibtexIdCompletionTest : BasePlatformTestCase() {
         super.setUp()
     }
 
-    @Test
     fun testCompleteLatexReferences() {
         // when
         runCompletion()
@@ -30,7 +29,6 @@ class BibtexIdCompletionTest : BasePlatformTestCase() {
         assertTrue(entry1.allLookupStrings.contains("{Missing the Point(er): On the Effectiveness of Code Pointer Integrity}"))
     }
 
-    @Test
     fun testCompletionResultsLowerCase() {
         // when
         runCompletion()
@@ -41,7 +39,6 @@ class BibtexIdCompletionTest : BasePlatformTestCase() {
         assertTrue(result?.contains("Muchnick1997") == true)
     }
 
-    @Test
     fun testCompletionResultsSecondEntry() {
         // when
         runCompletion()
@@ -54,14 +51,12 @@ class BibtexIdCompletionTest : BasePlatformTestCase() {
         assertTrue(result?.contains("Burow2016") == true)
     }
 
-    @Test
     fun testCompleteBibtexWithCorrectCase() {
         // Using the following failed sometimes
         val testName = getTestName(false)
         myFixture.testCompletion("${testName}_before.tex", "${testName}_after.tex", "$testName.bib")
     }
 
-    @Test
     fun testBibtexEntryDocumentation() {
         runCompletion()
         val element = DocumentationManager.getInstance(myFixture.project).getElementFromLookup(myFixture.editor, myFixture.file)
@@ -78,7 +73,11 @@ class BibtexIdCompletionTest : BasePlatformTestCase() {
     }
 
     private fun runCompletion() {
-        myFixture.configureByFiles("${getTestName(false)}.tex", "bibtex.bib")
+        val files = myFixture.configureByFiles("${getTestName(false)}.tex", "bibtex.bib")
+        // The first time completion runs, due to caching there may be a race condition
+        for (file in files) {
+            ReferencedFileSetService.getInstance().referencedFileSetOf(file)
+        }
         // when
         myFixture.complete(CompletionType.BASIC)
     }

test/nl/hannahsten/texifyidea/reference/BibtexIdRemoteLibraryCompletionTest.kt

@@ -8,6 +8,7 @@ import nl.hannahsten.texifyidea.remotelibraries.RemoteLibraryManager
 import nl.hannahsten.texifyidea.remotelibraries.state.BibtexEntryListConverter
 import nl.hannahsten.texifyidea.remotelibraries.state.LibraryState
 import nl.hannahsten.texifyidea.remotelibraries.zotero.ZoteroLibrary
+import nl.hannahsten.texifyidea.util.files.ReferencedFileSetService
 
 class BibtexIdRemoteLibraryCompletionTest : BasePlatformTestCase() {
 
@@ -101,7 +102,10 @@ class BibtexIdRemoteLibraryCompletionTest : BasePlatformTestCase() {
         mockkObject(RemoteLibraryManager)
         every { RemoteLibraryManager.getInstance().getLibraries() } returns mutableMapOf("aaa" to LibraryState("mocked", ZoteroLibrary::class.java, BibtexEntryListConverter().fromString(remoteBib), "test url"))
 
-        myFixture.configureByFiles("$path/before.tex", "$path/bibtex_before.bib")
+        myFixture.configureByFiles("$path/before.tex", "$path/bibtex_before.bib").forEach {
+            // Refresh cache
+            ReferencedFileSetService.getInstance().referencedFileSetOf(it)
+        }
 
         myFixture.complete(CompletionType.BASIC)