(UNFINISHED/BROKEN) (don't know if will finish) Somewhat working state

.cargo/config.toml

@@ -8,3 +8,7 @@ rustflags = ["-lz", "-lbz2", "-llzma", "-C", "link-args=-framework AudioUnit"]
 # Windows static build
 [target.stable-x86_64-pc-windows-msvc]
 rustflags = ["-Ctarget-feature=+crt-static"]
+
+# Use ldd for faster compile on Linux
+[target.x86_64-unknown-linux-gnu]
+rustflags = ["-C", "link-arg=-fuse-ld=lld"]

.github/workflows/build.yml

@@ -51,6 +51,7 @@ jobs:
         pnpm run build
         cd ..
         cargo update
+        cargo run --bin onetagger-python-builder --release
         cargo build --release
 
     - name: Bundle
@@ -121,6 +122,7 @@ jobs:
         pnpm run build
         cd ..
         cargo update
+        cargo run --bin onetagger-python-builder --release
         cargo build --release
         
     - name: Bundle
@@ -136,7 +138,9 @@ jobs:
       uses: actions/upload-artifact@v3
       with:
         name: onetagger-win
-        path: dist/OneTagger-windows.exe
+        path: | 
+          dist/OneTagger-windows.exe
+          crates/onetagger-python/pyembedded/python310.dll
     
     - name: Upload Archive
       uses: actions/upload-artifact@v3
@@ -191,6 +195,7 @@ jobs:
         pnpm run build
         cd ..
         cargo update
+        cargo run --bin onetagger-python-builder --release
         cd crates/onetagger
         cargo bundle --release
         cd ../..

assets/installer.nsi

@@ -75,6 +75,7 @@ Section "One Tagger" OneTagger
 	; Copy new
 	SetOutPath $INSTDIR
 	File "..\target\release\onetagger.exe"
+	File "..\crates\onetagger-python\pyembedded\python310.dll"
 	File "..\assets\icon.ico"
 	File "..\vc_redist.x64.exe"
 	File "..\MicrosoftEdgeWebview2Setup.exe"

crates/onetagger-python/Cargo.toml

@@ -9,12 +9,12 @@ edition = "2021"
 dunce = "1.0"
 chrono = "0.4"
 anyhow = "1.0"
+rmp-serde = "1.1"
 serde_json = "1.0"
 
 log = { version = "0.4", features = ["serde"] }
 pyo3 = { version = "0.18", features = ["chrono", "serde"] }
 serde = { version = "1.0", features = ["derive"] }
-bitcode = { version = "0.4", features = ["serde"] }
 pyembed = { git = "https://github.com/indygreg/PyOxidizer.git" }
 
 onetagger-tagger = { path = "../onetagger-tagger", features = ["python"] }

crates/onetagger-python/src/lib.rs

@@ -8,7 +8,6 @@ use std::process::{Command, Stdio};
 use anyhow::Error;
 use onetagger_shared::Settings;
 use onetagger_tagger::{PlatformInfo, AutotaggerSourceBuilder, TaggerConfig, AutotaggerSource, AudioFileInfo, Track, TrackMatch};
-use pyembed::{MainPythonInterpreter, OxidizedPythonInterpreterConfig};
 use serde::{Serialize, Deserialize};
 use subprocess::{SubprocessWrap, PythonResponse, PythonRequest};
 
@@ -45,6 +44,9 @@ pub fn setup() -> Result<(), Error> {
         std::fs::write(dir.join("pip.pyz"), PIP_PYZ)?;
     }
 
+    // Setup pyo3
+    module::setup();
+
     Ok(())
 }
 
@@ -53,31 +55,6 @@ fn stdlib_path() -> Result<PathBuf, Error> {
     Ok(dunce::canonicalize(Settings::get_folder()?.join("python_stdlib.zip"))?)
 }
 
-
-/// pip install packages
-fn pip_install(mut config: OxidizedPythonInterpreterConfig, requirements: &[String]) -> Result<(), Error> {
-    let pip_path = dunce::canonicalize(Settings::get_folder()?.join("pip.pyz"))?;
-    // Params
-    config.interpreter_config.run_filename = Some(pip_path.clone());
-    config.interpreter_config.argv = Some(vec![
-        "pip.pyz".into(),
-        "pip.pyz".into(),
-        "install".into(),
-        "pip".into(),
-        "setuptools".into(),
-        "wheel".into()
-    ]);
-    config.interpreter_config.argv.as_mut().unwrap().extend(requirements.iter().map(|r| r.into()));
-
-    // Run
-    let interpreter = MainPythonInterpreter::new(config)?;
-    let r = interpreter.py_runmain();
-    if r != 0 {
-        return Err(anyhow!("pip install failed with code: {r}"));
-    }
-    Ok(())
-}
-
 /// Platform info for Python
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct PythonPlatformInfo {
@@ -99,6 +76,18 @@ pub fn load_python_platform(path: impl AsRef<Path>) -> Result<PythonPlatformBuil
     Ok(PythonPlatformBuilder { info, path: dunce::canonicalize(path)? })
 }
 
+/// Spawn python child process
+fn spawn_python_child() -> Result<SubprocessWrap, Error> {
+    let child = Command::new(std::env::current_exe()?)
+        .stdout(Stdio::piped())
+        .stdin(Stdio::piped())
+        .stderr(Stdio::inherit())
+        .arg("--python-subprocess")
+        .spawn()?;
+    let wrap = SubprocessWrap::new(child);
+    Ok(wrap)
+}
+
 pub struct PythonPlatformBuilder {
     pub info: PythonPlatformInfo,
     path: PathBuf
@@ -111,27 +100,24 @@ impl AutotaggerSourceBuilder for PythonPlatformBuilder {
 
     fn get_source(&mut self, _config: &TaggerConfig) -> Result<Box<dyn AutotaggerSource>, Error> {
         // Install packages
-        let config = module::pyoxidizer_config(&self.path.join(".python"), stdlib_path()?)?;
         info!("Running pip install {:?}", self.info.requirements);
-        pip_install(config.clone(), &self.info.requirements)?;
+        let mut wrap = spawn_python_child()?;
+        wrap.send(&PythonRequest::PipInstall { 
+            path: self.path.join(".python"), 
+            pip_path: Settings::get_folder()?.join("pip.pyz"), 
+            requirements: self.info.requirements.clone()
+        })?;
+        wrap.recv()?;
+        drop(wrap);
 
         // Load code
         let code = std::fs::read_to_string(self.path.join(&self.info.main))?;
 
         // Spawn subprocess
-        let child = Command::new(std::env::current_exe()?)
-            .stdout(Stdio::piped())
-            .stdin(Stdio::piped())
-            .stderr(Stdio::inherit())
-            .arg("--python-subprocess")
-            .spawn()?;
-        let mut wrap = SubprocessWrap::new(child);
-        debug!("Pre init");
+        let mut wrap = spawn_python_child()?;
         wrap.send(&PythonRequest::Init { path: self.path.join(".python"), code })?;
-        debug!("Post init");
         // Receive init ok or error
         wrap.recv()?;
-        debug!("init ok");
 
         Ok(Box::new(PythonPlatform { subprocess: wrap }))
     }
@@ -150,9 +136,7 @@ pub struct PythonPlatform {
 
 impl AutotaggerSource for PythonPlatform {
     fn match_track(&mut self, info: &AudioFileInfo, config: &TaggerConfig) -> Result<Vec<TrackMatch>, Error> {
-        debug!("Pre match");
         self.subprocess.send(&PythonRequest::MatchTrack { info: info.clone(), config: config.clone() })?;
-        debug!("Post match");
         if let PythonResponse::MatchTrack { result } = self.subprocess.recv()? {
             return result.map_err(|e| anyhow!("{e}"));
         }
@@ -166,4 +150,11 @@ impl AutotaggerSource for PythonPlatform {
         }
         Ok(())
     }
+}
+
+impl Drop for PythonPlatform {
+    fn drop(&mut self) {
+        self.subprocess.send(&PythonRequest::Exit).ok();
+        self.subprocess.recv().ok();
+    }
 }