diff --git a/src/PollinationSDK/Wrapper/JobRunner.cs b/src/PollinationSDK/Wrapper/JobRunner.cs index e1527ec6..404af423 100644 --- a/src/PollinationSDK/Wrapper/JobRunner.cs +++ b/src/PollinationSDK/Wrapper/JobRunner.cs @@ -22,16 +22,16 @@ public JobRunner(JobInfo job) { this.JobInfo = job; } - + public async Task RunOnCloudAsync(Project project, Action progressReporting, System.Threading.CancellationToken token) { - + CloudJob cloudJob = null; try { cloudJob = await ScheduleCloudJobAsync(project, this.Job, progressReporting, token); progressReporting?.Invoke(cloudJob.Status.Status.ToString()); - LogHelper.LogInfo( $"A new cloud job {cloudJob.Id} is started in project {project.Name}"); + LogHelper.LogInfo($"A new cloud job {cloudJob.Id} is started in project {project.Name}"); } catch (Exception e) { @@ -61,7 +61,7 @@ public async Task RunOnCloudAsync(Project project, Action prog { // check if all cloud path artifacts are within the same project - var invalidCloudAssets = job.Arguments.SelectMany(_ => _.OfType()).Where(_ => _.IsAssetUploaded() && _.CloudProjectSlug() != project.Slug).Select(_=> $"{_.ToUserFriendlyString(true)}").Distinct(); + var invalidCloudAssets = job.Arguments.SelectMany(_ => _.OfType()).Where(_ => _.IsAssetUploaded() && _.CloudProjectSlug() != project.Slug).Select(_ => $"{_.ToUserFriendlyString(true)}").Distinct(); if (invalidCloudAssets.Any()) { var error = $"Following cloud assets cannot be uploaded to project {project.Slug}:\n\n{string.Join(Environment.NewLine, invalidCloudAssets)}"; @@ -76,7 +76,8 @@ public async Task RunOnCloudAsync(Project project, Action prog // upload artifacts if (!string.IsNullOrEmpty(tempProjectDir)) { - Action updateMessageProgress = (int p) => { + Action updateMessageProgress = (int p) => + { progressLogAction?.Invoke($"Preparing: [{p}%]"); }; await Helper.UploadDirectoryAsync(project, tempProjectDir, updateMessageProgress, cancellationToken); @@ -90,7 +91,7 @@ public async Task RunOnCloudAsync(Project project, Action prog LogHelper.LogInfo($"Canceled by user"); return null; } - + // update Artifact to cloud's relative path after uploaded. var newJob = UpdateArtifactPath(project.Slug, job, subfolderPath); @@ -124,7 +125,7 @@ public async Task RunOnCloudAsync(Project project, Action prog // Upload artifacts var newJob = await UploadJobAssetsAsync(project, job, this.JobInfo.SubFolderPath, progressLogAction, cancellationToken); - + // create a new Simulation var api = new JobsApi(); progressLogAction?.Invoke($"Start running."); @@ -174,13 +175,23 @@ public string RunOnLocalMachine(string workFolder, int workerNum, bool silentMod var workName = this.Job.Name ?? "Unnamed"; workName = new String(workName.Where(c => char.IsLetterOrDigit(c)).ToArray()); - + var workDir = workFolder; - if (!string.IsNullOrEmpty( this.JobInfo.SubFolderPath)) + if (!string.IsNullOrEmpty(this.JobInfo.SubFolderPath)) workDir = Path.Combine(workDir, this.JobInfo.SubFolderPath); workDir = Path.GetFullPath(workDir); if (Directory.Exists(workDir)) - System.IO.Directory.Delete(workDir, true); + { + try + { + System.IO.Directory.Delete(workDir, true); + } + catch (Exception e) + { + throw new ArgumentException($"Failed to clean up the working folder: {workDir}. Please remove it manually!\n{e}"); + } + } + Directory.CreateDirectory(workDir); var recipeOwner = this.JobInfo.RecipeOwner; @@ -189,7 +200,7 @@ public string RunOnLocalMachine(string workFolder, int workerNum, bool silentMod var localArgs = this.Job.Arguments; var localArg = new LocalRunArguments(localArgs.FirstOrDefault()); //TODO: ignore parametric runs for now - + //localArg.Validate(userRecipe); var inputJson = localArg.SaveToFolder(workDir); //save args to input.json file @@ -236,7 +247,7 @@ public static RunStatusEnum CheckLocalJobStatus(string workDir) { //"C:\Users\mingo\simulation\Unnamed\Unnamed\__logs__\status.json" var logDir = Directory.GetDirectories(workDir, "*__logs__*", SearchOption.AllDirectories).FirstOrDefault(); - + var statusFile = Path.Combine(logDir, "status.json"); if (!File.Exists(statusFile)) return RunStatusEnum.Unknown; @@ -249,7 +260,7 @@ public static RunStatusEnum CheckLocalJobStatus(string workDir) return RunStatusEnum.Unknown; var st = statusToken.ToObject(); - + if (st == RunStatusEnum.Succeeded || st == RunStatusEnum.Failed) return st; else @@ -286,7 +297,7 @@ public static string GetJobErrors(string workDir) private static string CheckRecipeInProject(string recipeSource, Project project) { var found = Helper.GetRecipeFromRecipeSourceURL(recipeSource, out var recOwner, out var recName, out var recVersion); - if (!found) + if (!found) { LogHelper.LogThrowError($"CheckRecipeInProject: invalid recipe source {recipeSource}"); } @@ -448,14 +459,14 @@ private static Job UpdateArtifactPath(string projSlug, Job job, string subFolder } } - + } return newJob; } - +