Properly handle nested directories of Swift Sources in jextract

Author: ktoso

Plugins/JExtractSwiftPlugin/JExtractSwiftPlugin.swift

@@ -93,7 +93,7 @@ struct JExtractSwiftBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin {
       $0.pathExtension == "swift"
     }
 
-    // Output Swift files are just Java filename based converted to Swift files one-to-one
+    // Output files are flattened filenames of the inputs, with the appended +SwiftJava suffix.
     var outputSwiftFiles: [URL] = swiftFiles.compactMap { sourceFileURL in
       guard sourceFileURL.isFileURL else {
         return nil as URL?
@@ -104,7 +104,6 @@ struct JExtractSwiftBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin {
         fatalError("Could not get relative path for source file \(sourceFilePath)")
       }
       let outputURL = outputSwiftDirectory
-        .appending(path: String(sourceFilePath.dropFirst(sourceDir.count).dropLast(sourceFileURL.lastPathComponent.count + 1)))
 
       let inputFileName = sourceFileURL.deletingPathExtension().lastPathComponent
       return outputURL.appending(path: "\(inputFileName)+SwiftJava.swift")

Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift

@@ -175,6 +175,10 @@ struct SwiftJavaBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin {
 
       let displayName = "Wrapping \(classes.count) Java classes in Swift target '\(sourceModule.name)'"
       log("Prepared: \(displayName)")
+
+      for f in outputSwiftFiles {
+        log("Swift output file: \(f)")
+      }
       commands += [
         .buildCommand(
           displayName: displayName,

Samples/SwiftJavaExtractFFMSampleApp/Sources/MySwiftLibrary/LibrarySubDirectory/SwiftTypeInSubDirectory.swift

@@ -0,0 +1,21 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2024 Apple Inc. and the Swift.org project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of Swift.org project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+public final class SwiftTypeInSubDirectory {
+  public init() {}
+
+  public func hello() -> Int {
+    12
+  }
+}

Samples/SwiftJavaExtractFFMSampleApp/src/test/java/com/example/swift/SwiftTypeInSubDirectoryTest.java

@@ -24,47 +24,14 @@
 
 import static org.junit.jupiter.api.Assertions.*;
 
-public class MySwiftClassTest {
-
-    void checkPaths(Throwable throwable) {
-        var paths = SwiftLibraries.getJavaLibraryPath().split(":");
-        for (var path : paths) {
-            Stream.of(new File(path).listFiles())
-                    .filter(file -> !file.isDirectory())
-                    .forEach((file) -> {
-                        System.out.println("  - " + file.getPath());
-                    });
-        }
-
-        throw new RuntimeException(throwable);
-    }
+public class SwiftTypeInSubDirectoryTest {
 
     @Test
     void test_MySwiftClass_voidMethod() {
-        try(var arena = AllocatingSwiftArena.ofConfined()) {
-            MySwiftClass o = MySwiftClass.init(12, 42, arena);
-            o.voidMethod();
-        } catch (Throwable throwable) {
-            checkPaths(throwable);
-        }
-    }
-
-    @Test
-    void test_MySwiftClass_makeIntMethod() {
-        try(var arena = AllocatingSwiftArena.ofConfined()) {
-            MySwiftClass o = MySwiftClass.init(12, 42, arena);
-            var got = o.makeIntMethod();
-            assertEquals(12, got);
-        }
-    }
-
-    @Test
-    @Disabled // TODO: Need var mangled names in interfaces
-    void test_MySwiftClass_property_len() {
-        try(var arena = AllocatingSwiftArena.ofConfined()) {
-            MySwiftClass o = MySwiftClass.init(12, 42, arena);
-            var got = o.getLen();
-            assertEquals(12, got);
+        try (var arena = AllocatingSwiftArena.ofConfined()) {
+            SwiftTypeInSubDirectory o = SwiftTypeInSubDirectory.init(arena);
+            var num = o.hello();
+            assertEquals(12, num);
         }
     }
 

Sources/JExtractSwiftLib/CodePrinter.swift

@@ -14,7 +14,11 @@
 
 import Foundation
 
-let PATH_SEPARATOR = "/"  // TODO: Windows
+#if os(Windows)
+let PATH_SEPARATOR = "\\"
+#else
+let PATH_SEPARATOR = "/"
+#endif
 
 public struct CodePrinter {
   var contents: String = ""

Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator+SwiftThunkPrinting.swift

@@ -22,16 +22,15 @@ extension FFMSwift2JavaGenerator {
   }
 
   package func writeSwiftExpectedEmptySources() throws {
-    let pendingFileCount = self.expectedOutputSwiftFiles.count
+    let pendingFileCount = self.expectedOutputSwiftFileNames.count
     guard pendingFileCount > 0 else {
       return // no need to write any empty files, yay
     }
 
-    print("[swift-java] Write empty [\(self.expectedOutputSwiftFiles.count)] 'expected' files in: \(swiftOutputDirectory)/")
+    log.info("[swift-java] Write empty [\(self.expectedOutputSwiftFileNames.count)] 'expected' files in: \(swiftOutputDirectory)/")
 
-    for expectedFileName in self.expectedOutputSwiftFiles {
-      log.debug("Write SwiftPM-'expected' empty file: \(expectedFileName.bold)")
-
+    for expectedFileName in self.expectedOutputSwiftFileNames {
+      log.info("Write SwiftPM-'expected' empty file: \(expectedFileName.bold)")
 
       var printer = CodePrinter()
       printer.print("// Empty file generated on purpose")
@@ -55,7 +54,7 @@ extension FFMSwift2JavaGenerator {
         javaPackagePath: nil,
         filename: moduleFilename) {
         log.info("Generated: \(moduleFilenameBase.bold).swift (at \(outputFile.absoluteString))")
-        self.expectedOutputSwiftFiles.remove(moduleFilename)
+        self.expectedOutputSwiftFileNames.remove(moduleFilename)
       }
     } catch {
       log.warning("Failed to write to Swift thunks: \(moduleFilename)")
@@ -93,7 +92,9 @@ extension FFMSwift2JavaGenerator {
           javaPackagePath: nil,
           filename: filename) {
           log.info("Done writing Swift thunks to: \(outputFile.absoluteString)")
-          self.expectedOutputSwiftFiles.remove(filename)
+          // log.info("REMOVE FROM: \(expectedOutputSwiftFileNames)")
+          // log.info("REMOVE FROM THE: \(filename)")
+          self.expectedOutputSwiftFileNames.remove(filename)
         }
       } catch {
         log.warning("Failed to write to Swift thunks: \(filename), error: \(error)")

Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator.swift

@@ -39,7 +39,10 @@ package class FFMSwift2JavaGenerator: Swift2JavaGenerator {
 
   /// Because we need to write empty files for SwiftPM, keep track which files we didn't write yet,
   /// and write an empty file for those.
-  var expectedOutputSwiftFiles: Set<String>
+  /// 
+  /// Since Swift files in SwiftPM builds needs to be unique, we use this fact to flatten paths into plain names here.
+  /// For uniqueness checking "did we write this file already", just checking the name should be sufficient.
+  var expectedOutputSwiftFileNames: Set<String>
 
   package init(
     config: Configuration,
@@ -57,19 +60,26 @@ package class FFMSwift2JavaGenerator: Swift2JavaGenerator {
     self.javaOutputDirectory = javaOutputDirectory
     self.lookupContext = translator.lookupContext
 
-    // If we are forced to write empty files, construct the expected outputs
+    // If we are forced to write empty files, construct the expected outputs.
+    // It is sufficient to use file names only, since SwiftPM requires names to be unique within a module anyway.
     if translator.config.writeEmptyFiles ?? false {
-      self.expectedOutputSwiftFiles = Set(translator.inputs.compactMap { (input) -> String? in
-        guard let filePathPart = input.path.split(separator: "/\(translator.swiftModuleName)/").last else {
+      self.expectedOutputSwiftFileNames = Set(translator.inputs.compactMap { (input) -> String? in
+        // guard let filePathPart = input.path.split(separator: "/\(translator.swiftModuleName)/").last else {
+        //   return nil
+        // }
+        // return String(filePathPart.replacing(".swift", with: "+SwiftJava.swift"))
+        guard let fileName = input.path.split(separator: PATH_SEPARATOR).last else {
           return nil
         }
-
-        return String(filePathPart.replacing(".swift", with: "+SwiftJava.swift"))
+        guard fileName.hasSuffix(".swift") else {
+          return nil
+        }
+        return String(fileName.replacing(".swift", with: "+SwiftJava.swift"))
       })
-      self.expectedOutputSwiftFiles.insert("\(translator.swiftModuleName)Module+SwiftJava.swift")
-      self.expectedOutputSwiftFiles.insert("Foundation+SwiftJava.swift")
+      self.expectedOutputSwiftFileNames.insert("\(translator.swiftModuleName)Module+SwiftJava.swift")
+      self.expectedOutputSwiftFileNames.insert("Foundation+SwiftJava.swift")
     } else {
-      self.expectedOutputSwiftFiles = []
+      self.expectedOutputSwiftFileNames = []
     }
   }
 

Sources/JExtractSwiftLib/Swift2Java.swift

@@ -17,6 +17,7 @@ import SwiftSyntax
 import SwiftSyntaxBuilder
 import SwiftJavaShared
 import SwiftJavaConfigurationShared
+import OrderedCollections
 
 public struct SwiftToJava {
   let config: Configuration
@@ -34,6 +35,7 @@ public struct SwiftToJava {
 
     let translator = Swift2JavaTranslator(config: config)
     translator.log.logLevel = config.logLevel ?? .info
+    let log = translator.log
 
     if config.javaPackage == nil || config.javaPackage!.isEmpty {
       translator.log.warning("Configured java package is '', consider specifying concrete package for generated sources.")
@@ -43,28 +45,17 @@ public struct SwiftToJava {
       fatalError("Missing '--swift-input' directory!")
     }
 
-    translator.log.info("Input swift = \(inputSwift)")
+    log.info("Input swift = \(inputSwift)")
     let inputPaths = inputSwift.split(separator: ",").map { URL(string: String($0))! }
-    translator.log.info("Input paths = \(inputPaths)")
-
-    var allFiles: [URL] = []
-    let fileManager = FileManager.default
-    let log = translator.log
+    log.info("Input paths = \(inputPaths)")
 
-    for path in inputPaths {
-      log.info("Input path: \(path)")
-      if isDirectory(url: path) {
-        if let enumerator = fileManager.enumerator(at: path, includingPropertiesForKeys: nil) {
-          for case let fileURL as URL in enumerator {
-            allFiles.append(fileURL)
-          }
-        }
-      } else if path.isFileURL {
-        allFiles.append(path)
-      }
+    let allFiles = collectAllFiles(suffix: ".swift", in: inputPaths, log: translator.log)
+    for f in allFiles {
+      log.warning("INPUT FILE: \(f) ->>>")
     }
 
     // Register files to the translator.
+    let fileManager = FileManager.default
     for file in allFiles {
       guard canExtract(from: file) else {
         continue
@@ -137,8 +128,82 @@ public struct SwiftToJava {
 
 }
 
-func isDirectory(url: URL) -> Bool {
-  var isDirectory: ObjCBool = false
-  _ = FileManager.default.fileExists(atPath: url.path, isDirectory: &isDirectory)
-  return isDirectory.boolValue
+extension URL {
+  var isDirectory: Bool {
+    var isDir: ObjCBool = false
+    _ = FileManager.default.fileExists(atPath: self.path, isDirectory: &isDir)
+    return isDir.boolValue
+  }
+}
+
+/// Collect all files with given 'suffix', will explore directories recursively.
+public func collectAllFiles(suffix: String, in inputPaths: [URL], log: Logger) -> OrderedSet<URL> {
+  guard !inputPaths.isEmpty else {
+    return []
+  }
+
+  let fileManager = FileManager.default
+  var allFiles: OrderedSet<URL> = []
+  allFiles.reserveCapacity(32) // rough guesstimate
+  
+  let resourceKeys: [URLResourceKey] = [
+    .isRegularFileKey,
+    .isDirectoryKey,
+    .nameKey
+  ]
+  
+  for path in inputPaths {
+    do {
+      try collectFilesFromPath(
+        path,
+        suffix: suffix,
+        fileManager: fileManager,
+        resourceKeys: resourceKeys,
+        into: &allFiles,
+        log: log
+      )
+    } catch {
+      log.trace("Failed to collect paths in: \(path), skipping.")
+    }
+  }
+  
+  return allFiles 
+}
+
+private func collectFilesFromPath(
+  _ path: URL,
+  suffix: String,
+  fileManager: FileManager,
+  resourceKeys: [URLResourceKey],
+  into allFiles: inout OrderedSet<URL>,
+  log: Logger
+) throws {
+  guard fileManager.fileExists(atPath: path.path) else {
+    return
+  }
+  
+  if path.isDirectory {
+    let enumerator = fileManager.enumerator(
+      at: path,
+      includingPropertiesForKeys: resourceKeys,
+      options: [.skipsHiddenFiles],
+      errorHandler: { url, error in
+        return true
+      })
+    guard let enumerator else {
+      return
+    }
+    
+    for case let fileURL as URL in enumerator {
+      try? collectFilesFromPath(fileURL, suffix: suffix, fileManager: fileManager, resourceKeys: resourceKeys, into: &allFiles, log: log)
+    }
+  } 
+  
+  guard path.isFileURL else {
+    return
+  }
+  guard path.lastPathComponent.hasSuffix(suffix) else {
+    return
+  }
+  allFiles.append(path)
 }

Sources/JExtractSwiftLib/Swift2JavaTranslator.swift

@@ -78,7 +78,7 @@ extension Swift2JavaTranslator {
   }
 
   package func add(filePath: String, text: String) {
-    log.trace("Adding: \(filePath)")
+    log.info("Adding: \(filePath)")
     let sourceFileSyntax = Parser.parse(source: text)
     self.inputs.append(SwiftJavaInputFile(syntax: sourceFileSyntax, path: filePath))
   }

Sources/SwiftJavaConfigurationShared/Configuration.swift

@@ -309,7 +309,7 @@ public enum LogLevel: String, ExpressibleByStringLiteral, Codable, Hashable {
 
 extension LogLevel {
   public init(from decoder: any Decoder) throws {
-    var container = try decoder.singleValueContainer()
+    let container = try decoder.singleValueContainer()
     let string = try container.decode(String.self)
     switch string {
     case "trace": self = .trace