diff --git a/src/PollinationSDK/Helper/Helper.cs b/src/PollinationSDK/Helper/Helper.cs index c6d2bff6..188152d5 100644 --- a/src/PollinationSDK/Helper/Helper.cs +++ b/src/PollinationSDK/Helper/Helper.cs @@ -1,6 +1,5 @@ using PollinationSDK.Api; using PollinationSDK.Client; -using PollinationSDK.Model; using PollinationSDK.Wrapper; using RestSharp; using System; @@ -96,8 +95,9 @@ public static async Task UploadDirectoryAsync(Project project, string dire var files = Directory.GetFiles(directory, "*", SearchOption.AllDirectories); var api = new ArtifactsApi(); + - var tasks = files.Select(_ => UploadArtifaceAsync(api, project, _, _.Replace(directory, ""))).ToList(); + var tasks = files.Select(_ => UploadArtifactAsync(api, project, _, _.Replace(directory, ""))).ToList(); var total = files.Count(); Helper.Logger.Information($"UploadDirectoryAsync: Uploading {total} assets for project {project.Name}"); @@ -144,7 +144,7 @@ public static async Task UploadDirectoryAsync(Project project, string dire /// /// something like: "C:\Users\mingo\Downloads\Compressed\project_folder\project_folder\model\grid\room.pts" /// "model\grid\room.pts" - public static async Task UploadArtifaceAsync(ArtifactsApi api, Project project, string fullPath, string relativePath) + public static async Task UploadArtifactAsync(ArtifactsApi api, Project project, string fullPath, string relativePath) { var filePath = fullPath; var fileRelativePath = relativePath.Replace('\\', '/'); @@ -188,18 +188,70 @@ public static async Task UploadArtifaceAsync(ArtifactsApi api, Project pro } - public static bool GetRecipeFromRecipeSourceURL(string recipeSource, out RecipeInterface recipe) + + public static IEnumerable CheckCached(IEnumerable assets, string dir) + { + var checkedAssets = new List(); + var inputDir = Path.Combine(dir, "inputs"); + var outputDir = Path.Combine(dir, "outputs"); + foreach (var item in assets) + { + var savedDir = item is RunInputAsset ? inputDir : outputDir; + var dupObj = item.Duplicate() as RunAssetBase; + var isCached = item.CheckIfAssetCached(savedDir); + if (isCached) + { + var cachedPath = item.GetCachedAsset(savedDir); + cachedPath = Helper.CheckPathForDir(cachedPath); + dupObj.LocalPath = cachedPath; + } + + checkedAssets.Add(dupObj); + } + return checkedAssets; + } + + public static IEnumerable CheckCached(IEnumerable assets, string dir) + { + var checkedAssets = new List(); + foreach (var item in assets) + { + var dupObj = item.Duplicate() as CloudReferenceAsset; + var cachedPath = Path.GetFullPath(Path.Combine(dir, dupObj.RelativePath)); + //var isCached = dupObj.CheckIfAssetCached(dir); + if (File.Exists(cachedPath)) + { + dupObj.LocalPath = cachedPath; + } + + checkedAssets.Add(dupObj); + } + return checkedAssets; + } + + + public static string CheckPathForDir(string savedFolderOrFilePath) { - //var recipeSource = this.Run.Job.Source; - var isRecipe = Helper.GetRecipeFromRecipeSourceURL(recipeSource, out var recOwner, out var recName, out var recVersion); - if (!isRecipe) - throw new ArgumentException($"Invalid recipe source URL {recipeSource}.\nThe correct formate should be something like:https://api.staging.pollination.cloud/registries/ladybug-tools/recipe/annual-daylight/0.2.0"); - var recApi = new RecipesApi(); - recipe = recApi.GetRecipeByTag(recOwner, recName, recVersion).Manifest; - return recipe != null; + var p = savedFolderOrFilePath; + // check folder + if (!Path.HasExtension(p)) + { + var tempDir = new DirectoryInfo(p); + var subItems = tempDir.GetFileSystemInfos("*", SearchOption.TopDirectoryOnly); + + if (subItems.Count() == 1) + { + // if there is only one file/folder inside + var f = subItems[0]; + if (f.Exists) p = f.FullName; + } + } + + return p; } + public static bool GetRecipeFromRecipeSourceURL(string recipeSource, out string recipeOwner, out string recipeName, out string recipeVersion) { recipeOwner = null; @@ -227,18 +279,109 @@ public static bool GetRecipeFromRecipeSourceURL(string recipeSource, out string } + public static async Task> DownloadAssetsAsync(string projSlug, IEnumerable assets, string saveAsDir, Action reportProgressAction, bool useCached = false, System.Threading.CancellationToken cancelToken = default) + { + var tasks = new List(); + if (assets == null || !assets.Any()) return tasks; + - public static async Task DownloadArtifact(Project proj, FileMeta file, string saveAsFolder, Action reportProgressAction = default) + var dir = Path.GetFullPath(Path.Combine(saveAsDir, projSlug)); + System.IO.Directory.CreateDirectory(dir); + + var api = new PollinationSDK.Api.ArtifactsApi(); + + var proj = projSlug.Split('/'); + var projOwner = proj[0]; + var projName = proj[1]; + + + // check if cached + if (useCached) + { + assets = CheckCached(assets, dir).ToList(); + } + + var total = assets.Count(); + var completed = 0; + foreach (var asset in assets) + { + try + { + if (asset.IsPathAsset() && !asset.IsSaved()) + { + + Action individualProgress = (percent) => { + reportProgressAction?.Invoke($"[{completed}]: {percent}%"); + }; + + var savedFolderOrFilePath = await DownloadArtifactAsync(api, projOwner, projName, asset.RelativePath, dir, individualProgress, cancelToken); + + // check folder with single file + savedFolderOrFilePath = CheckPathForDir(savedFolderOrFilePath); + + // update saved path + var dup = asset.Duplicate() as CloudReferenceAsset; + dup.LocalPath = savedFolderOrFilePath; + tasks.Add(dup); + } + else + { + tasks.Add(asset); + } + + completed++; + } + catch (Exception e) + { + //canceled by user + if (e is OperationCanceledException) + return null; + + throw new ArgumentException($"Failed to download asset {asset.Name}.\n -{e.Message}"); + } + } + return tasks; + + } + + + /// + /// + /// + /// "model\grid\room.pts" + public static async Task DownloadArtifactAsync(ArtifactsApi api, string projOwner, string projName, string relativePath, string saveAsDir, Action reportProgressAction = default, System.Threading.CancellationToken cancelToken = default) { - var api = new ArtifactsApi(); - var task = api.DownloadArtifactAsync(proj.Owner.Name, proj.Name, file.Key); - var result = await task; + var fileRelativePath = relativePath.Replace('\\', '/'); + if (fileRelativePath.StartsWith("/")) + fileRelativePath = fileRelativePath.Substring(1); + + if (!Path.HasExtension(fileRelativePath)) // dir + { + var msg = $"Cannot download the following folder directly: {fileRelativePath}"; + throw new ArgumentException(msg); + } + + var url = (await api.DownloadArtifactAsync(projOwner, projName, fileRelativePath))?.ToString(); - var downlaodTask = DownloadUrlAsync(result.ToString(), saveAsFolder, reportProgressAction); - var saved = await downlaodTask; + Helper.Logger.Information($"DownloadArtifactAsync: downloading {fileRelativePath} from \n -{url}\n"); + // get relative path correct + saveAsDir = Path.GetDirectoryName(Path.Combine(saveAsDir, relativePath)); + saveAsDir = Path.GetFullPath(saveAsDir); + var path = await Helper.DownloadUrlAsync(url.ToString(), saveAsDir, reportProgressAction, null, cancelToken); + + Helper.Logger.Information($"DownloadArtifactAsync: saved {fileRelativePath} to {path}"); + return path; + } + + + public static async Task DownloadArtifactAsync(Project proj, FileMeta file, string saveAsFolder, Action reportProgressAction = default) + { + var api = new ArtifactsApi(); + var saved = await DownloadArtifactAsync(api, proj.Owner.Name, proj.Name, file.Key, saveAsFolder, reportProgressAction); return saved; } + private static async Task> Download(string url, string dir, Action reportProgressAction) { diff --git a/src/PollinationSDK/ManuallyAdded/Model/JobPathArgument.cs b/src/PollinationSDK/ManuallyAdded/Model/JobPathArgument.cs index 8bd8cf2a..f92c3c6e 100644 --- a/src/PollinationSDK/ManuallyAdded/Model/JobPathArgument.cs +++ b/src/PollinationSDK/ManuallyAdded/Model/JobPathArgument.cs @@ -80,7 +80,7 @@ public string ToUserFriendlyString(bool withProjectSlug) } - var text = $"${hint}/{p}"; + var text = $"{hint}/{p}"; if (!string.IsNullOrEmpty(this.Name)) text = $"#{this.Name}:{text}"; return text; diff --git a/src/PollinationSDK/Wrapper/AssetBase.cs b/src/PollinationSDK/Wrapper/AssetBase.cs new file mode 100644 index 00000000..a9ded99e --- /dev/null +++ b/src/PollinationSDK/Wrapper/AssetBase.cs @@ -0,0 +1,230 @@ + +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace PollinationSDK.Wrapper +{ + public abstract class AssetBase + { + [JsonProperty] + public string Name { get; protected set; } + + [JsonProperty] + public string Description { get; protected set; } + + [JsonProperty] + /// + /// This is valid only if the asset is non-path type: string, int, double, etc + /// + public List Value { get; protected set; } + + [JsonProperty] + /// + /// Original Run source for redownloading the asset. Formated as CLOUD:mingbo/demo/1D725BD1-44E1-4C3C-85D6-4D98F558DE7C + /// + public string RunSource { get; protected set; } + + [JsonProperty] + /// + /// For input asset: a relative path to the cloud project root that this asset belongs to. + /// For output asset: same as asset name + /// + public string RelativePath { get; protected set; } + + [JsonProperty] + public string LocalPath { get; set; } + + /// + /// Load the content of from LacalPath, + /// This property is not serialize-able and will not be duplicated. + /// + public object PreloadedPath { get; set; } + + public bool IsInputAsset => this is RunInputAsset; + + public abstract AssetBase Duplicate(); + + /// + /// Check if a path type asset saved to local drive (LocalPath). + /// + /// + public bool IsSaved() + { + var path = this.LocalPath; + if (string.IsNullOrEmpty(path)) + return false; + + var attr = File.GetAttributes(path); + + if ((attr & FileAttributes.Directory) == FileAttributes.Directory) + return Directory.Exists(path); + else + return File.Exists(path); + } + + /// + /// Check if this asset is value type or path type asset. If it is path type, then check if its CloudRunSource is valid. + /// Lastly, if it is an input asset, check if CloudPath is valid. + /// + /// + public bool IsPathAsset() + { + if (this.Value != null && this.Value.Any()) + return false; + + // CloudRunSource is not required to download the asset since downloading assets also need RunInfo + //if (string.IsNullOrEmpty(CloudRunSource)) + // return false; + + if (this.IsInputAsset) + return !string.IsNullOrEmpty(RelativePath); + else + return true; + + } + + internal virtual bool CheckIfAssetCached(string dir) + { + var assetName = this.Name; + + + var assetDir = Path.Combine(dir, assetName); + if (!Directory.Exists(assetDir)) + return false; + + // check if folder is empty + var cached = Directory.EnumerateFileSystemEntries(assetDir, "*", SearchOption.TopDirectoryOnly).ToList(); + if (!cached.Any()) + return false; + + if (this.IsInputAsset) + { + var assetPath = this.RelativePath; + // folder asset is a zip file, assetDir has all unzipped files + if (assetPath.EndsWith(".zip")) + return true; + else // file asset + return File.Exists(Path.Combine(assetDir, Path.GetFileName(assetPath))); + } + + return true; + + } + + internal string GetCachedAsset(string dir) + { + var assetName = this.Name; + var assetPath = this.RelativePath; + + + var cachedPath = string.Empty; + var assetDir = Path.Combine(dir, assetName); + + if (this.IsInputAsset) + { + // folder asset is a zip file + if (assetPath.EndsWith(".zip")) + { + // assetDir has all unzipped files + cachedPath = assetDir; + } + else // file asset + { + var assetFile = Path.Combine(assetDir, Path.GetFileName(assetPath)); + if (File.Exists(assetFile)) + cachedPath = assetFile; + } + } + else + { + cachedPath = assetDir; + } + + return cachedPath; + } + + + public override string ToString() + { + if (this.Value != null && this.Value.Any()) + return base.ToString(); + + var v = $"{RunSource}/{Name}"; + + if (this is RunOutputAsset outputAsset) + { + v = $"{v}#{outputAsset.AliasName}"; + } + return v; + } + + + + private Run GetRun() + { + CheckRunSource(out var owner, out var proj, out var runId); + + var api = new Api.RunsApi(); + var run = api.GetRun(owner, proj, runId); + return run; + } + + private void CheckRunSource(out string owner, out string proj, out string runId) + { + //CLOUD:mingbo/demo/1D725BD1-44E1-4C3C-85D6-4D98F558DE7C + var runID = this.RunSource; + if (string.IsNullOrEmpty(runID) || runID.StartsWith("LOCAL:")) + throw new ArgumentException($"Invalid cloud run source"); + if (runID.StartsWith("CLOUD:")) + runID = runID.Substring(6); + + var items = runID.Split('/'); + if (items.Length != 3) + throw new ArgumentException($"Invalid cloud run source"); + + owner = items[0]; + proj = items[1]; + runId = items[2]; + + } + + public Project GetRoject() + { + CheckRunSource(out var owner, out var projname, out var runId); + + var api = new Api.ProjectsApi(); + var proj = api.GetProject(owner, projname); + + return proj; + } + + public virtual RunInfo GetRunInfo() + { + if (this.RunSource.StartsWith("LOCAL:")) + { + var folder = this.RunSource.Substring(6); + return new RunInfo(folder); + } + else + { + var run = GetRun(); + var proj = GetRoject(); + + var runInfo = new RunInfo(proj, run); + return runInfo; + } + + } + + + public virtual object CheckOutputWithHandler(object inputData, HandlerChecker handlerChecker) + { + return inputData; + } + + } + +} diff --git a/src/PollinationSDK/Wrapper/CloudReferenceAsset.cs b/src/PollinationSDK/Wrapper/CloudReferenceAsset.cs new file mode 100644 index 00000000..fb237e48 --- /dev/null +++ b/src/PollinationSDK/Wrapper/CloudReferenceAsset.cs @@ -0,0 +1,108 @@ +using Newtonsoft.Json; +using System; +using System.IO; +using System.Linq; + +namespace PollinationSDK.Wrapper +{ + public class CloudReferenceAsset : PollinationSDK.Wrapper.AssetBase + { + public string ProjectSlug { get; set; } + + [JsonConstructorAttribute] + public CloudReferenceAsset() + { + + } + + + public CloudReferenceAsset(string projOwner, string projName, string assetPath) + { + // get name + this.Name = System.IO.Path.GetFileNameWithoutExtension(assetPath); + + + // check path type + this.RelativePath = assetPath; + this.ProjectSlug = $"{projOwner}/{projName}"; + + this.Description = $"CLOUD:{ProjectSlug}/{RelativePath}"; + + + this.RunSource = $"CLOUD:{ProjectSlug}/CloudReferenceAsset"; + } + + + public override string ToString() + { + return this.ToJobPathArgument().ToUserFriendlyString(true); + } + + + public JobPathArgument ToJobPathArgument() + { + var projSlug = this.ProjectSlug; + var assetPath = this.RelativePath; + + var pSource = new ProjectFolder(path: assetPath); + var newPath = new JobPathArgument(string.Empty, pSource); + newPath.IsAssetUploaded(true); + newPath.TagCloudProjectSlug(projSlug); + + return newPath; + } + + public static CloudReferenceAsset FromJobPathArgument(JobPathArgument arg) + { + if(!arg.IsAssetUploaded()) + throw new ArgumentException($"{arg.Name} is not a valid cloud-based argument"); + + + var projSlug = arg.CloudProjectSlug(); + var arr = projSlug.Split('/'); + var projOwner = arr[0]; + var projName = arr[1]; + var relativePath = (arg.Source.Obj as ProjectFolder).Path; + + var asset = new CloudReferenceAsset(projOwner, projName, relativePath); + return asset; + } + + /// + /// + /// + /// CLOUD:owner-name/project-name/asset-path.epw + public static CloudReferenceAsset FromString(string cloudReferenceLink) + { + if (!cloudReferenceLink.StartsWith("CLOUD:")) + throw new ArgumentException($"Invalid Path: {cloudReferenceLink}.\nFormat:\nCLOUD:owner-name/project-name/asset-path.epw"); + + + var text = cloudReferenceLink.Substring(6); + var arrs = text.Split('/'); + + var projOwner = arrs[0]; + var projName = arrs[1]; + var assetPath = string.Join("/", arrs.Skip(2)); + var asset = new CloudReferenceAsset(projOwner, projName, assetPath); + return asset; + } + + public override PollinationSDK.Wrapper.AssetBase Duplicate() + { + var json = Newtonsoft.Json.JsonConvert.SerializeObject(this, Newtonsoft.Json.Formatting.None); + return Newtonsoft.Json.JsonConvert.DeserializeObject(json); + } + + public override RunInfo GetRunInfo() + { + return null; + } + + internal override bool CheckIfAssetCached(string dir) + { + var file = Path.GetFullPath(Path.Combine(dir, Path.GetFileName(this.RelativePath))); + return File.Exists(file); + } + } +} diff --git a/src/PollinationSDK/Wrapper/InputArgumentValidator.cs b/src/PollinationSDK/Wrapper/InputArgumentValidator.cs index b64c4b53..6795f5b0 100644 --- a/src/PollinationSDK/Wrapper/InputArgumentValidator.cs +++ b/src/PollinationSDK/Wrapper/InputArgumentValidator.cs @@ -34,6 +34,10 @@ public static object CheckAndValidate(PollinationSDK.Interface.Io.Inputs.IDag da if (value == null) return null; + // check CloudReferenceAsset + if (value is CloudReferenceAsset cloudRef) + value = cloudRef.ToJobPathArgument(); + // do nothing for the cloud referenced file path if (value is JobPathArgument path && path.IsAssetUploaded()) return path; diff --git a/src/PollinationSDK/Wrapper/RunAssetBase.cs b/src/PollinationSDK/Wrapper/RunAssetBase.cs index f0a79da4..fb88db46 100644 --- a/src/PollinationSDK/Wrapper/RunAssetBase.cs +++ b/src/PollinationSDK/Wrapper/RunAssetBase.cs @@ -1,228 +1,9 @@ - -using Newtonsoft.Json; -using PollinationSDK; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; - -namespace PollinationSDK.Wrapper +namespace PollinationSDK.Wrapper { - public abstract class RunAssetBase + public abstract class RunAssetBase: AssetBase { - [JsonProperty] - public string Name { get; protected set; } - - [JsonProperty] - public string Description { get; protected set; } - - [JsonProperty] - /// - /// This is valid only if the asset is non-path type: string, int, double, etc - /// - public List Value { get; protected set; } - - [JsonProperty] - /// - /// Original Run source for redownloading the asset. Formated as CLOUD:mingbo/demo/1D725BD1-44E1-4C3C-85D6-4D98F558DE7C - /// - public string RunSource { get; protected set; } - - [JsonProperty] - /// - /// For input asset: a relative path to the cloud project root that this asset belongs to. - /// For output asset: same as asset name - /// - public string RelativePath { get; protected set; } - - [JsonProperty] - public string LocalPath { get; set; } - - /// - /// Load the content of from LacalPath, - /// This property is not serialize-able and will not be duplicated. - /// - public object PreloadedPath { get; set; } - - public bool IsInputAsset => this is RunInputAsset; - - public abstract RunAssetBase Duplicate(); - - /// - /// Check if a path type asset saved to local drive (LocalPath). - /// - /// - public bool IsSaved() - { - var path = this.LocalPath; - if (string.IsNullOrEmpty(path)) - return false; - - var attr = File.GetAttributes(path); - - if ((attr & FileAttributes.Directory) == FileAttributes.Directory) - return Directory.Exists(path); - else - return File.Exists(path); - } - - /// - /// Check if this asset is value type or path type asset. If it is path type, then check if its CloudRunSource is valid. - /// Lastly, if it is an input asset, check if CloudPath is valid. - /// - /// - public bool IsPathAsset() - { - if (this.Value != null && this.Value.Any()) - return false; - - // CloudRunSource is not required to download the asset since downloading assets also need RunInfo - //if (string.IsNullOrEmpty(CloudRunSource)) - // return false; - - if (this.IsInputAsset) - return !string.IsNullOrEmpty(RelativePath); - else - return true; - - } - - internal bool CheckIfAssetCached(string dir) - { - var assetName = this.Name; - - - var assetDir = Path.Combine(dir, assetName); - if (!Directory.Exists(assetDir)) - return false; - - // check if folder is empty - var cached = Directory.EnumerateFileSystemEntries(assetDir, "*", SearchOption.TopDirectoryOnly).ToList(); - if (!cached.Any()) - return false; - - if (this.IsInputAsset) - { - var assetPath = this.RelativePath; - // folder asset is a zip file, assetDir has all unzipped files - if (assetPath.EndsWith(".zip")) - return true; - else // file asset - return File.Exists(Path.Combine(assetDir, Path.GetFileName(assetPath))); - } - - return true; - - } - - internal string GetCachedAsset(string dir) - { - var assetName = this.Name; - var assetPath = this.RelativePath; - - - var cachedPath = string.Empty; - var assetDir = Path.Combine(dir, assetName); - - if (this.IsInputAsset) - { - // folder asset is a zip file - if (assetPath.EndsWith(".zip")) - { - // assetDir has all unzipped files - cachedPath = assetDir; - } - else // file asset - { - var assetFile = Path.Combine(assetDir, Path.GetFileName(assetPath)); - if (File.Exists(assetFile)) - cachedPath = assetFile; - } - } - else - { - cachedPath = assetDir; - } - - return cachedPath; - } - - - public override string ToString() - { - if (this.Value != null && this.Value.Any()) - return base.ToString(); - - var v = $"{RunSource}/{Name}"; - - if (this is RunOutputAsset outputAsset) - { - v = $"{v}#{outputAsset.AliasName}"; - } - return v; - } - - - - public Run GetRun() - { - string[] items = CheckRunSource(); - var source = new { owner = items[0], proj = items[1], runId = items[2] }; - - var api = new Api.RunsApi(); - var run = api.GetRun(source.owner, source.proj, source.runId); - //run.Status.JobId - return run; - } - - private string[] CheckRunSource() - { - //CLOUD:mingbo/demo/1D725BD1-44E1-4C3C-85D6-4D98F558DE7C - var runID = this.RunSource; - if (string.IsNullOrEmpty(runID) || runID.StartsWith("LOCAL:")) - throw new ArgumentException($"Not valid cloud run source"); - if (runID.StartsWith("CLOUD:")) - runID = runID.Substring(6); - - var items = runID.Split('/'); - return items; - } - - public Project GetRoject() - { - string[] items = CheckRunSource(); - var source = new { owner = items[0], proj = items[1], runId = items[2] }; - - var api = new Api.ProjectsApi(); - var proj = api.GetProject(source.owner, source.proj); - - return proj; - } - - public RunInfo GetRunInfo() - { - if (this.RunSource.StartsWith("LOCAL:")) - { - var folder = this.RunSource.Substring(6); - return new RunInfo(folder); - } - else - { - var run = GetRun(); - var proj = GetRoject(); - - var runInfo = new RunInfo(proj, run); - return runInfo; - } - } - - - public virtual object CheckOutputWithHandler(object inputData, HandlerChecker handlerChecker) - { - return inputData; - } - } + } diff --git a/src/PollinationSDK/Wrapper/RunInfo.cs b/src/PollinationSDK/Wrapper/RunInfo.cs index d74162fa..2f016efb 100644 --- a/src/PollinationSDK/Wrapper/RunInfo.cs +++ b/src/PollinationSDK/Wrapper/RunInfo.cs @@ -13,6 +13,8 @@ namespace PollinationSDK.Wrapper public class RunInfo { public bool IsLocalRun => !Guid.TryParse(this.RunID, out var res); + public bool IsCloudRunDone => this.Run.Status.FinishedAt >= this.Run.Status.StartedAt; + public string CloudRunStatus => this.Run.Status.Status.ToString(); public string RunID => this.Run.Id; public Run Run { get; private set; } //public Project Project { get; private set; } @@ -346,7 +348,7 @@ public List LoadLocalRunAssets(List runAssets, strin var updatedAssets = new List(); foreach (var item in runAssets) { - var dup = item.Duplicate(); + var dup = item.Duplicate() as RunAssetBase; if (!dup.IsPathAsset()) { updatedAssets.Add(dup); @@ -427,46 +429,16 @@ public async Task> DownloadRunAssetsAsync(List // check if cached if (useCached) { - allAssets = CheckCached(allAssets, dir).ToList(); + allAssets = Helper.CheckCached(allAssets, dir).ToList(); } cancelToken.ThrowIfCancellationRequested(); // assembly download tasks reportingAction?.Invoke("Starting"); var tasks = DownloadAssets(this, allAssets, dir, reportingAction, cancelToken); - await Task.WhenAll(tasks); + downloadedAssets = (await Task.WhenAll(tasks)).ToList(); reportingAction?.Invoke("Downloaded"); - // collect all downloaded assets - var works = allAssets.Zip(tasks, (asset, saved) => new { asset, saved }); - foreach (var item in works) - { - var dup = item.asset.Duplicate(); - - // get saved path type assets - if (item.asset.IsPathAsset()) - { - var savedFolderOrFilePath = await item.saved; - // check folder - if (!Path.HasExtension(savedFolderOrFilePath)) - { - var tempDir = new DirectoryInfo(savedFolderOrFilePath); - var subItems = tempDir.GetFileSystemInfos("*", SearchOption.TopDirectoryOnly); - - if (subItems.Count() == 1) - { - // if there is only one file/folder inside - var f = subItems[0]; - if (f.Exists) savedFolderOrFilePath = f.FullName; - } - } - // update saved path - dup.LocalPath = savedFolderOrFilePath; - } - - downloadedAssets.Add(dup); - } - } catch (Exception e) { @@ -491,26 +463,7 @@ public override string ToString() public bool IsValid => Total > 0; } - private static IEnumerable CheckCached(IEnumerable outputAssets, string dir) - { - var checkedAssets = new List(); - var inputDir = Path.Combine(dir, "inputs"); - var outputDir = Path.Combine(dir, "outputs"); - foreach (var item in outputAssets) - { - var savedDir = item is RunInputAsset ? inputDir : outputDir; - var dupObj = item.Duplicate(); - var isCached = item.CheckIfAssetCached(savedDir); - if (isCached) - { - var cachedPath = item.GetCachedAsset(savedDir); - dupObj.LocalPath = cachedPath; - } - - checkedAssets.Add(dupObj); - } - return checkedAssets; - } + /// @@ -519,9 +472,9 @@ private static IEnumerable CheckCached(IEnumerable o /// /// /// - private static List> DownloadAssets(RunInfo runInfo, IEnumerable assets, string saveAsDir, Action reportProgressAction, System.Threading.CancellationToken cancelToken = default) + private static List> DownloadAssets(RunInfo runInfo, IEnumerable assets, string saveAsDir, Action reportProgressAction, System.Threading.CancellationToken cancelToken = default) { - var tasks = new List>(); + var tasks = new List>(); if (assets == null || !assets.Any()) return tasks; var api = new PollinationSDK.Api.RunsApi(); var inputDir = Path.Combine(saveAsDir, "inputs"); @@ -559,16 +512,23 @@ private static List> DownloadAssets(RunInfo runInfo, IEnumerable asset.LocalPath)); + tasks.Add(Task.Run(() => asset)); completed++; } } diff --git a/src/PollinationSDK/Wrapper/RunInputAsset.cs b/src/PollinationSDK/Wrapper/RunInputAsset.cs index 3e1835f9..41425651 100644 --- a/src/PollinationSDK/Wrapper/RunInputAsset.cs +++ b/src/PollinationSDK/Wrapper/RunInputAsset.cs @@ -31,12 +31,13 @@ public RunInputAsset(Interface.Io.Inputs.IStep dagInput, string runSource = defa this.RunSource = runSource; } - public override RunAssetBase Duplicate() + public override AssetBase Duplicate() { var json = Newtonsoft.Json.JsonConvert.SerializeObject(this, Newtonsoft.Json.Formatting.None); return Newtonsoft.Json.JsonConvert.DeserializeObject(json); } + } } diff --git a/src/PollinationSDK/Wrapper/RunOutputAsset.cs b/src/PollinationSDK/Wrapper/RunOutputAsset.cs index c8ed2319..20151d4c 100644 --- a/src/PollinationSDK/Wrapper/RunOutputAsset.cs +++ b/src/PollinationSDK/Wrapper/RunOutputAsset.cs @@ -61,7 +61,7 @@ public object PreloadLinkedOutputWithHandler(object inputData, HandlerChecker ha return handlerChecker.CheckWithHandlers(inputData, handlerForPreload); } - public override RunAssetBase Duplicate() + public override AssetBase Duplicate() { var json = Newtonsoft.Json.JsonConvert.SerializeObject(this, Newtonsoft.Json.Formatting.None); return Newtonsoft.Json.JsonConvert.DeserializeObject(json);