Skip to content

Commit

Permalink
feat: parallelize implementation
Browse files Browse the repository at this point in the history
  • Loading branch information
paolobarbolini committed Apr 20, 2024
1 parent ca391b9 commit 90b214e
Show file tree
Hide file tree
Showing 4 changed files with 99 additions and 27 deletions.
52 changes: 52 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,4 @@ semver = "1"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
anyhow = "1.0.14"
rayon = "1"
2 changes: 1 addition & 1 deletion src/git.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ use crate::package::Package;
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct GitUrl(Url);

#[derive(Debug)]
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct GitRepository {
repo_dir: PathBuf,
}
Expand Down
71 changes: 45 additions & 26 deletions src/main.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use std::{
collections::BTreeMap,
env, fs,
io::{self as std_io, Read},
path::Path,
Expand All @@ -9,6 +10,7 @@ use anyhow::{ensure, Context, Result};
use cargo_lock::{package::SourceKind, Checksum, Lockfile};
use cargo_toml::Manifest;
use git::GitUrl;
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use serde::Deserialize;
use sha2::{Digest as _, Sha256};
use url::Url;
Expand Down Expand Up @@ -76,45 +78,62 @@ fn main() -> Result<()> {

let lock = Lockfile::load(lock).context("decode Cargo.lock")?;

for lock_info in lock.packages {
let name = lock_info.name.clone();
let version = lock_info.version.clone();
let resolved_packages = lock
.packages
.into_par_iter()
.filter_map(|lock_info| {
let name = lock_info.name.clone();
let version = lock_info.version.clone();

let resolved_package = match resolve_package(&http_client, &crates_dir, lock_info) {
Ok(resolved_package) => resolved_package,
Err(err) => {
println!("Couldn't resolve package {name} v{version} err={err:?}");
continue;
match resolve_package(&http_client, &crates_dir, lock_info) {
Ok(resolved_package) => Some(resolved_package),
Err(err) => {
println!("Couldn't resolve package {name} v{version} err={err:?}");
None
}
}
};
})
.collect::<Vec<_>>();

let mut grouped_resolved_packages = BTreeMap::<_, Vec<_>>::new();
for resolved_package in resolved_packages {
grouped_resolved_packages
.entry(resolved_package.repository_url.clone())
.or_default()
.push(resolved_package);
}

let mut git_repository =
match GitRepository::obtain(&repos_dir, resolved_package.repository_url.clone()) {
grouped_resolved_packages
.into_par_iter()
.for_each(|(repository_url, resolved_packages)| {
let mut git_repository = match GitRepository::obtain(&repos_dir, repository_url) {
Ok(git_repository) => git_repository,
Err(err) => {
println!(
"Couldn't obtain git repository for {} v{} err={:?} url={}",
resolved_packages[0].lock_info.name,
resolved_packages[0].lock_info.version,
err,
resolved_packages[0].repository_url
);
return;
}
};

for resolved_package in resolved_packages {
if let Err(err) =
analyze_package(&default_toolchain, &resolved_package, &mut git_repository)
{
println!(
"Couldn't analyze package for {} v{} err={:?} url={}",
resolved_package.lock_info.name,
resolved_package.lock_info.version,
err,
resolved_package.repository_url
);
continue;
}
};

if let Err(err) =
analyze_package(&default_toolchain, &resolved_package, &mut git_repository)
{
println!(
"Couldn't analyze package for {} v{} err={:?} url={}",
resolved_package.lock_info.name,
resolved_package.lock_info.version,
err,
resolved_package.repository_url
);
}
}
}
});

Ok(())
}
Expand Down

0 comments on commit 90b214e

Please sign in to comment.