diff --git a/.cargo/config.toml b/.cargo/config.toml index 317133ce..58e16929 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -8,3 +8,7 @@ rustflags = ["-lz", "-lbz2", "-llzma", "-C", "link-args=-framework AudioUnit"] # Windows static build [target.stable-x86_64-pc-windows-msvc] rustflags = ["-Ctarget-feature=+crt-static"] + +# Use ldd for faster compile on Linux +[target.x86_64-unknown-linux-gnu] +rustflags = ["-C", "link-arg=-fuse-ld=lld"] \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a0806f77..729fc7f4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,6 +51,7 @@ jobs: pnpm run build cd .. cargo update + cargo run --bin onetagger-python-builder --release cargo build --release - name: Bundle @@ -121,6 +122,7 @@ jobs: pnpm run build cd .. cargo update + cargo run --bin onetagger-python-builder --release cargo build --release - name: Bundle @@ -136,7 +138,9 @@ jobs: uses: actions/upload-artifact@v3 with: name: onetagger-win - path: dist/OneTagger-windows.exe + path: | + dist/OneTagger-windows.exe + crates/onetagger-python/pyembedded/python310.dll - name: Upload Archive uses: actions/upload-artifact@v3 @@ -191,6 +195,7 @@ jobs: pnpm run build cd .. cargo update + cargo run --bin onetagger-python-builder --release cd crates/onetagger cargo bundle --release cd ../.. diff --git a/Cargo.lock b/Cargo.lock index 02663167..007b83c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1038,31 +1038,6 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" -[[package]] -name = "bitcode" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fab8eb922a99ab601facde7ae07200e811cdc2ae1df604ffd6c2b7c976dd3fff" -dependencies = [ - "bitcode_derive", - "bytemuck", - "from_bytes_or_zeroed", - "residua-zigzag", - "serde", -] - -[[package]] -name = "bitcode_derive" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2ab471e684e2f6f3c3071361e2c30e41d7543f505658daae3ceb89c202d933e" -dependencies = [ - "packagemerge", - "proc-macro2", - "quote", - "syn 2.0.29", -] - [[package]] name = "bitfield" version = "0.14.0" @@ -2695,12 +2670,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "from_bytes_or_zeroed" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d25934a78435889223e575c7b0fc36a290c5a312e7a7ae901f10587792e142a" - [[package]] name = "fs2" version = "0.4.3" @@ -3168,7 +3137,7 @@ dependencies = [ "fixedbitset", "guppy-workspace-hack", "indexmap 1.9.3", - "itertools 0.10.5", + "itertools", "nested", "once_cell", "pathdiff", @@ -3621,12 +3590,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "itertools" -version = "0.4.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a9b56eb56058f43dc66e58f40a214b2ccbc9f3df51861b63d51dec7b65bc3f" - [[package]] name = "itertools" version = "0.10.5" @@ -3796,7 +3759,7 @@ dependencies = [ "diff", "ena", "is-terminal", - "itertools 0.10.5", + "itertools", "lalrpop-util", "petgraph", "regex", @@ -4868,7 +4831,6 @@ name = "onetagger-python" version = "0.1.0" dependencies = [ "anyhow", - "bitcode", "chrono", "dunce", "log", @@ -4876,6 +4838,7 @@ dependencies = [ "onetagger-tagger", "pyembed", "pyo3", + "rmp-serde", "serde", "serde_json", ] @@ -5040,15 +5003,6 @@ dependencies = [ "elliptic-curve", ] -[[package]] -name = "packagemerge" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0efcf6ee55f8f7a24333bc8d1dd0e541a6cedf903dbc07ae6479d7f8ff32ed08" -dependencies = [ - "itertools 0.4.19", -] - [[package]] name = "pacmog" version = "0.4.1" @@ -5741,7 +5695,7 @@ dependencies = [ "guppy", "handlebars", "hex", - "itertools 0.10.5", + "itertools", "linked-hash-map", "log", "once_cell", @@ -5803,7 +5757,7 @@ dependencies = [ "base64 0.20.0", "byteorder", "encoding_rs", - "itertools 0.10.5", + "itertools", "mailparse", "once_cell", "python-packed-resources 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -5824,7 +5778,7 @@ dependencies = [ "anyhow", "byteorder", "encoding_rs", - "itertools 0.10.5", + "itertools", "mailparse", "once_cell", "python-packed-resources 0.12.0 (git+https://github.com/indygreg/PyOxidizer.git)", @@ -6002,7 +5956,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e1c39006ee96ffe7c83f656a5d20269366098a3feb35ad4a7c022e3784643cb" dependencies = [ - "itertools 0.10.5", + "itertools", "proc-macro2", "quote", "syn 1.0.109", @@ -6193,12 +6147,6 @@ dependencies = [ "winreg 0.50.0", ] -[[package]] -name = "residua-zigzag" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b37805477eee599a61753230f511ae94d737f69b536e468e294723ad5f1b75f" - [[package]] name = "riff" version = "2.0.0" @@ -6229,6 +6177,28 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "rmp" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f9860a6cc38ed1da53456442089b4dfa35e7cedaa326df63017af88385e6b20" +dependencies = [ + "byteorder", + "num-traits", + "paste", +] + +[[package]] +name = "rmp-serde" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bffea85eea980d8a74453e5d02a8d93028f3c34725de143085a844ebe953258a" +dependencies = [ + "byteorder", + "rmp", + "serde", +] + [[package]] name = "rodio" version = "0.16.0" diff --git a/assets/installer.nsi b/assets/installer.nsi index 275178af..40fcb76c 100644 --- a/assets/installer.nsi +++ b/assets/installer.nsi @@ -75,6 +75,7 @@ Section "One Tagger" OneTagger ; Copy new SetOutPath $INSTDIR File "..\target\release\onetagger.exe" + File "..\crates\onetagger-python\pyembedded\python310.dll" File "..\assets\icon.ico" File "..\vc_redist.x64.exe" File "..\MicrosoftEdgeWebview2Setup.exe" diff --git a/crates/onetagger-python/Cargo.toml b/crates/onetagger-python/Cargo.toml index b845f806..a42f2f9a 100644 --- a/crates/onetagger-python/Cargo.toml +++ b/crates/onetagger-python/Cargo.toml @@ -9,12 +9,12 @@ edition = "2021" dunce = "1.0" chrono = "0.4" anyhow = "1.0" +rmp-serde = "1.1" serde_json = "1.0" log = { version = "0.4", features = ["serde"] } pyo3 = { version = "0.18", features = ["chrono", "serde"] } serde = { version = "1.0", features = ["derive"] } -bitcode = { version = "0.4", features = ["serde"] } pyembed = { git = "https://github.com/indygreg/PyOxidizer.git" } onetagger-tagger = { path = "../onetagger-tagger", features = ["python"] } diff --git a/crates/onetagger-python/src/lib.rs b/crates/onetagger-python/src/lib.rs index 200a9d44..d0cf0f45 100644 --- a/crates/onetagger-python/src/lib.rs +++ b/crates/onetagger-python/src/lib.rs @@ -8,7 +8,6 @@ use std::process::{Command, Stdio}; use anyhow::Error; use onetagger_shared::Settings; use onetagger_tagger::{PlatformInfo, AutotaggerSourceBuilder, TaggerConfig, AutotaggerSource, AudioFileInfo, Track, TrackMatch}; -use pyembed::{MainPythonInterpreter, OxidizedPythonInterpreterConfig}; use serde::{Serialize, Deserialize}; use subprocess::{SubprocessWrap, PythonResponse, PythonRequest}; @@ -45,6 +44,9 @@ pub fn setup() -> Result<(), Error> { std::fs::write(dir.join("pip.pyz"), PIP_PYZ)?; } + // Setup pyo3 + module::setup(); + Ok(()) } @@ -53,31 +55,6 @@ fn stdlib_path() -> Result { Ok(dunce::canonicalize(Settings::get_folder()?.join("python_stdlib.zip"))?) } - -/// pip install packages -fn pip_install(mut config: OxidizedPythonInterpreterConfig, requirements: &[String]) -> Result<(), Error> { - let pip_path = dunce::canonicalize(Settings::get_folder()?.join("pip.pyz"))?; - // Params - config.interpreter_config.run_filename = Some(pip_path.clone()); - config.interpreter_config.argv = Some(vec![ - "pip.pyz".into(), - "pip.pyz".into(), - "install".into(), - "pip".into(), - "setuptools".into(), - "wheel".into() - ]); - config.interpreter_config.argv.as_mut().unwrap().extend(requirements.iter().map(|r| r.into())); - - // Run - let interpreter = MainPythonInterpreter::new(config)?; - let r = interpreter.py_runmain(); - if r != 0 { - return Err(anyhow!("pip install failed with code: {r}")); - } - Ok(()) -} - /// Platform info for Python #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PythonPlatformInfo { @@ -99,6 +76,18 @@ pub fn load_python_platform(path: impl AsRef) -> Result Result { + let child = Command::new(std::env::current_exe()?) + .stdout(Stdio::piped()) + .stdin(Stdio::piped()) + .stderr(Stdio::inherit()) + .arg("--python-subprocess") + .spawn()?; + let wrap = SubprocessWrap::new(child); + Ok(wrap) +} + pub struct PythonPlatformBuilder { pub info: PythonPlatformInfo, path: PathBuf @@ -111,27 +100,24 @@ impl AutotaggerSourceBuilder for PythonPlatformBuilder { fn get_source(&mut self, _config: &TaggerConfig) -> Result, Error> { // Install packages - let config = module::pyoxidizer_config(&self.path.join(".python"), stdlib_path()?)?; info!("Running pip install {:?}", self.info.requirements); - pip_install(config.clone(), &self.info.requirements)?; + let mut wrap = spawn_python_child()?; + wrap.send(&PythonRequest::PipInstall { + path: self.path.join(".python"), + pip_path: Settings::get_folder()?.join("pip.pyz"), + requirements: self.info.requirements.clone() + })?; + wrap.recv()?; + drop(wrap); // Load code let code = std::fs::read_to_string(self.path.join(&self.info.main))?; // Spawn subprocess - let child = Command::new(std::env::current_exe()?) - .stdout(Stdio::piped()) - .stdin(Stdio::piped()) - .stderr(Stdio::inherit()) - .arg("--python-subprocess") - .spawn()?; - let mut wrap = SubprocessWrap::new(child); - debug!("Pre init"); + let mut wrap = spawn_python_child()?; wrap.send(&PythonRequest::Init { path: self.path.join(".python"), code })?; - debug!("Post init"); // Receive init ok or error wrap.recv()?; - debug!("init ok"); Ok(Box::new(PythonPlatform { subprocess: wrap })) } @@ -150,9 +136,7 @@ pub struct PythonPlatform { impl AutotaggerSource for PythonPlatform { fn match_track(&mut self, info: &AudioFileInfo, config: &TaggerConfig) -> Result, Error> { - debug!("Pre match"); self.subprocess.send(&PythonRequest::MatchTrack { info: info.clone(), config: config.clone() })?; - debug!("Post match"); if let PythonResponse::MatchTrack { result } = self.subprocess.recv()? { return result.map_err(|e| anyhow!("{e}")); } @@ -166,4 +150,11 @@ impl AutotaggerSource for PythonPlatform { } Ok(()) } +} + +impl Drop for PythonPlatform { + fn drop(&mut self) { + self.subprocess.send(&PythonRequest::Exit).ok(); + self.subprocess.recv().ok(); + } } \ No newline at end of file diff --git a/crates/onetagger-python/src/subprocess.rs b/crates/onetagger-python/src/subprocess.rs index 7350e23a..1edf604f 100644 --- a/crates/onetagger-python/src/subprocess.rs +++ b/crates/onetagger-python/src/subprocess.rs @@ -1,4 +1,4 @@ -use std::io::{Read, Write, BufReader, BufRead}; +use std::io::{Read, Write}; use std::path::PathBuf; use std::process::{ChildStdin, ChildStdout, Child}; use anyhow::Error; @@ -23,6 +23,7 @@ pub enum PythonRequest { /// Exit Exit, + PipInstall { path: PathBuf, pip_path: PathBuf, requirements: Vec }, MatchTrack { info: AudioFileInfo, config: TaggerConfig }, ExtendTrack { track: Track, config: TaggerConfig }, } @@ -33,6 +34,8 @@ pub enum PythonResponse { Log { level: Level, message: String }, Error { error: String }, InitOk, + PipOk, + Exit, MatchTrack { result: Result, String> }, ExtendTrack { result: Result } @@ -50,6 +53,14 @@ pub fn python_process() { match read_stdin() { Ok(PythonRequest::Init { path, code }) => break (path, code), Ok(PythonRequest::Exit) => return, + // Install pip packages and exit + Ok(PythonRequest::PipInstall { path, pip_path, requirements }) => { + match pip_install(path, pip_path, requirements) { + Ok(_) => write_stdout(&PythonResponse::PipOk), + Err(e) => write_stdout(&PythonResponse::Error { error: e.to_string() }), + }.ok(); + return; + }, Err(e) => { error!("{e}"); return }, _ => {} } @@ -66,6 +77,9 @@ pub fn python_process() { write_stdout(&PythonResponse::Error { error: e.to_string() }).ok(); }, } + + // Quit + write_stdout(&PythonResponse::Exit).ok(); } /// Start and run python interpreter @@ -82,14 +96,13 @@ fn python_interpreter(path: PathBuf, code: &str) -> Result<(), Error> { let match_track = module.getattr("match_track")?; let extend_track = module.getattr("extend_track")?; - - // Read loop - while let Ok(request) = read_stdin() { - match request { + loop { + match read_stdin()? { // Ignore - PythonRequest::Init { .. } => {}, + PythonRequest::Init { .. } => unreachable!(), PythonRequest::Exit => return Ok(()), + PythonRequest::PipInstall { .. } => unreachable!(), PythonRequest::MatchTrack { info, config } => { write_stdout(&PythonResponse::MatchTrack { @@ -103,6 +116,36 @@ fn python_interpreter(path: PathBuf, code: &str) -> Result<(), Error> { }, } } + })?; + Ok(()) +} + +/// Install pip packages +fn pip_install(path: PathBuf, pip_path: PathBuf, requirements: Vec) -> Result<(), Error> { + crate::module::setup(); + + // Add pip to path + let mut config = crate::module::pyoxidizer_config(path, stdlib_path()?)?; + let pip_path = dunce::canonicalize(pip_path)?; + config.interpreter_config.module_search_paths.as_mut().unwrap().push(pip_path); + + // Install + let interpreter = MainPythonInterpreter::new(config)?; + interpreter.with_gil(|py| -> Result<(), Error> { + // Load utils + let _util = PyModule::from_code(py, include_str!("util.py"), "", "")?; + + // Package list + let mut params: Vec = vec![ + "install".into(), + "pip".into(), + "setuptools".into(), + "wheel".into() + ]; + params.extend(requirements); + + // Install + py.import("pip")?.call_method1("main", (params,))?; Ok(()) })?; Ok(()) @@ -135,7 +178,6 @@ impl SubprocessWrap { pub fn recv(&mut self) -> Result { loop { let response: PythonResponse = read_message(&mut self.stdout)?; - debug!("{response:?}"); match response { PythonResponse::Log { level, message } => { match level { @@ -147,21 +189,30 @@ impl SubprocessWrap { } }, PythonResponse::Error { error } => return Err(anyhow!("{error}")), + PythonResponse::Exit => { + debug!("Exitting subprocess"); + return Ok(PythonResponse::Exit); + } r => return Ok(r) } } } } +impl Drop for SubprocessWrap { + fn drop(&mut self) { + self.child.kill().ok(); + } +} + /// Read and deserialize message fn read_message(read: &mut R) -> Result { let mut size_buf = [0u8; 4]; read.read_exact(&mut size_buf)?; - debug!("len: {:?}", size_buf); let size = u32::from_be_bytes(size_buf) as usize; let mut buf = vec![0u8; size]; read.read_exact(&mut buf)?; - Ok(bitcode::deserialize(&buf)?) + Ok(rmp_serde::from_slice(&buf)?) } /// Read from stdin @@ -172,7 +223,7 @@ fn read_stdin() -> Result { /// Serialize and write message fn write_message(write: &mut W, msg: &S) -> Result<(), Error> { - let buf = bitcode::serialize(msg)?; + let buf = rmp_serde::to_vec(msg)?; let len = (buf.len() as u32).to_be_bytes(); write.write_all(&len)?; write.write_all(&buf)?; diff --git a/crates/onetagger-python/src/util.py b/crates/onetagger-python/src/util.py index e89f7b61..2abe0c7f 100644 --- a/crates/onetagger-python/src/util.py +++ b/crates/onetagger-python/src/util.py @@ -8,7 +8,7 @@ def __init__(self): pass def write(self, v): if v.strip() != '': - onetagger.info(v) + onetagger.info(v.strip()) sys.stdout = OneTaggerInfoLog() # stderr -> log @@ -17,5 +17,5 @@ def __init__(self): pass def write(self, v): if v.strip() != '': - onetagger.error(v) -sys.stderr = OneTaggerInfoLog() + onetagger.warn(v.strip()) +sys.stderr = OneTaggerErrorLog() \ No newline at end of file