Skip to content

Commit

Permalink
fix(Helper): download files without extension
Browse files Browse the repository at this point in the history
  • Loading branch information
@MingboPeng
MingboPeng committed Feb 22, 2024
1 parent 7e89158 commit f046c8e
Show file tree
Hide file tree
Showing 5 changed files with 31 additions and 27 deletions.
37 changes: 16 additions & 21 deletions src/PollinationSDK/Helper/Helper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ public static string CheckPathForDir(string savedFolderOrFilePath)
{
var p = savedFolderOrFilePath;
// check folder
if (!Path.HasExtension(p))
if (IsDirectory(p))
{
var tempDir = new DirectoryInfo(p);
var subItems = tempDir.GetFileSystemInfos("*", SearchOption.TopDirectoryOnly);
Expand Down Expand Up @@ -278,6 +278,13 @@ public static bool GetRecipeFromRecipeSourceURL(string recipeSource, out string
return true;
}

public static bool IsDirectory(string path)
{
FileAttributes attr = File.GetAttributes(path);
var isDir = attr.HasFlag(FileAttributes.Directory);
return isDir;
}


/// <summary>
/// Download all cloud reference assets (file or folder) from a project folder
Expand Down Expand Up @@ -306,13 +313,13 @@ public static async Task<List<CloudReferenceAsset>> DownloadAssetsAsync(string p
var projName = proj[1];

// check folder assets
var gp = assets.GroupBy(_ => Path.HasExtension(_.RelativePath));
var fileAssets = gp.FirstOrDefault(_ => _.Key == true)?.Select(_ => _)?.ToList() ?? new List<CloudReferenceAsset>();
var folderAssets = gp.FirstOrDefault(_ => _.Key == false)?.Select(_ => _.RelativePath)?.ToList();
var gp = assets.GroupBy(_ => _.IsFolder);
var fileAssets = gp.FirstOrDefault(_ => _.Key == false)?.Select(_ => _)?.ToList() ?? new List<CloudReferenceAsset>();
var folderAssets = gp.FirstOrDefault(_ => _.Key == true)?.Select(_ => _.RelativePath)?.ToList();
if (folderAssets != null && folderAssets.Any())
{
var allFiles = await GetAllFilesAsync(api, projOwner, projName, folderAssets, saveAsDir);
var cloudRefs = allFiles.Select(_ => new CloudReferenceAsset(projOwner, projName, _.Key));
var cloudRefs = allFiles.Select(_ => new CloudReferenceAsset(projOwner, projName, _.Key, _.FileType));
fileAssets.AddRange(cloudRefs);
}

Expand Down Expand Up @@ -380,15 +387,15 @@ public static async Task<List<CloudReferenceAsset>> DownloadAssetsAsync(string p
var projName = proj[1];

// check folder assets
var gp = assets.GroupBy(_ => Path.HasExtension(_.RelativePath));
var fileAssets = gp.FirstOrDefault(_ => _.Key == true)?.Select(_ => _)?.ToList() ?? new List<CloudReferenceAsset>();
var folderAssets = gp.FirstOrDefault(_ => _.Key == false)?.Select(_ => _.RelativePath)?.ToList();
var gp = assets.GroupBy(_ => _.IsFolder);
var fileAssets = gp.FirstOrDefault(_ => _.Key == false)?.Select(_ => _)?.ToList() ?? new List<CloudReferenceAsset>();
var folderAssets = gp.FirstOrDefault(_ => _.Key == true)?.Select(_ => _.RelativePath)?.ToList();

var api = new PollinationSDK.Api.JobsApi();
if (folderAssets != null && folderAssets.Any())
{
var allFiles = await GetAllFilesAsync(api, projOwner, projName, jobId, folderAssets, saveAsDir);
var cloudRefs = allFiles.Select(_ => new CloudReferenceAsset(projOwner, projName, jobId, _.Key));
var cloudRefs = allFiles.Select(_ => new CloudReferenceAsset(projOwner, projName, jobId, _.Key, _.FileType));
fileAssets.AddRange(cloudRefs);
}

Expand Down Expand Up @@ -495,12 +502,6 @@ public static async Task<string> DownloadArtifactAsync(ArtifactsApi api, string
if (fileRelativePath.StartsWith("/"))
fileRelativePath = fileRelativePath.Substring(1);

if (!Path.HasExtension(fileRelativePath)) // dir
{
var msg = $"Cannot download the following folder directly: {fileRelativePath}";
throw new ArgumentException(msg);
}

var url = (await api.DownloadArtifactAsync(projOwner, projName, fileRelativePath))?.ToString();

Helper.Logger.Information($"DownloadArtifactAsync: downloading {fileRelativePath} from \n -{url}\n");
Expand All @@ -519,12 +520,6 @@ public static async Task<string> DownloadArtifactAsync(JobsApi api, string projO
if (fileRelativePath.StartsWith("/"))
fileRelativePath = fileRelativePath.Substring(1);

if (!Path.HasExtension(fileRelativePath)) // dir
{
var msg = $"Cannot download the following folder directly: {fileRelativePath}";
throw new ArgumentException(msg);
}

var url = (await api.DownloadJobArtifactAsync(projOwner, projName, jobId, fileRelativePath, cancelToken))?.ToString();

Helper.Logger.Information($"DownloadJobArtifactAsync: downloading {fileRelativePath} from \n -{url}\n");
Expand Down
8 changes: 8 additions & 0 deletions src/PollinationSDK/Wrapper/AssetBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,14 @@ public abstract class AssetBase
/// </summary>
public string RelativePath { get; protected set; }

[JsonProperty]
/// <summary>
/// folder or file
/// </summary>
public string PathType { get; protected set; } = "file";
public bool IsFolder => this.PathType == "folder";
public bool IsFile => this.PathType == "file";

[JsonProperty]
public string LocalPath { get; set; }

Expand Down
7 changes: 4 additions & 3 deletions src/PollinationSDK/Wrapper/CloudReferenceAsset.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,13 @@ public CloudReferenceAsset()
}


public CloudReferenceAsset(string projOwner, string projName, string assetPath)
public CloudReferenceAsset(string projOwner, string projName, string assetPath, string pathType = "file")
{
// get name
this.Name = System.IO.Path.GetFileNameWithoutExtension(assetPath);


// check path type
this.PathType = pathType;
this.RelativePath = assetPath;
this.ProjectSlug = $"{projOwner}/{projName}";

Expand All @@ -34,13 +34,14 @@ public CloudReferenceAsset(string projOwner, string projName, string assetPath)
this.RunSource = $"CLOUD:{ProjectSlug}/{_cloudReferenceAssetKey}";
}

public CloudReferenceAsset(string projOwner, string projName, string jobID, string assetPath)
public CloudReferenceAsset(string projOwner, string projName, string jobID, string assetPath, string pathType)
{
// get name
this.Name = System.IO.Path.GetFileNameWithoutExtension(assetPath);


// check path type
this.PathType = pathType;
this.RelativePath = assetPath;
this.ProjectSlug = $"{projOwner}/{projName}";

Expand Down
2 changes: 1 addition & 1 deletion src/PollinationSDK/Wrapper/LocalRunArguments.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public LocalRunArguments(List<AnyOf<JobArgument, JobPathArgument>> arguments)
//copy to folder
var tempPath = (p.Source.Obj as PollinationSDK.ProjectFolder).Path;
var pathName = string.Empty;
if (System.IO.Path.HasExtension(tempPath)) // file type
if (!Helper.IsDirectory(tempPath)) // file type
{
if (!File.Exists(tempPath))
throw new ArgumentException($"Failed to find path argument {tempPath}");
Expand Down
4 changes: 2 additions & 2 deletions src/PollinationSDK/Wrapper/RunInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,7 +363,7 @@ public List<RunAssetBase> LoadLocalRunAssets(List<RunAssetBase> runAssets, strin
}
else if (dup is RunOutputAsset output)
{
var isFile = Path.HasExtension(output.RelativePath);
var isFile = !Helper.IsDirectory(output.RelativePath);
var relativeOutPath = output.RelativePath.Replace("/", @"\");
relativeOutPath = relativeOutPath.StartsWith(@"\") ? relativeOutPath : $@"\{relativeOutPath}";
if (isFile)
Expand Down Expand Up @@ -396,7 +396,7 @@ public List<RunAssetBase> LoadLocalRunAssets(List<RunAssetBase> runAssets, strin
Directory.CreateDirectory(newRoot);

// file type
if (Path.HasExtension(newPath))
if (!Helper.IsDirectory(newPath))
{
File.Copy(dup.LocalPath, newPath, true);
}
Expand Down

0 comments on commit f046c8e

Please sign in to comment.