Skip to content

Commit 839dbc5

Browse files
stevevanhooserstevevanhooser
authored
Merge branch 'main' into feature-NDI-123-update-document-list
2 parents 7fdce69 + b0f407a commit 839dbc5

File tree

3 files changed

+107
-21
lines changed

3 files changed

+107
-21
lines changed

‎.github/badges/code_issues.svg‎

Lines changed: 1 addition & 1 deletion
Loading

‎src/ndi/+ndi/+cloud/+sync/+internal/uploadFilesForDatasetDocuments.m‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,38 @@
11
function [success, message] = uploadFilesForDatasetDocuments(cloudDatasetId, ndiDataset, dataset_documents, options)
2-
% uploadFilesForDatasetDocuments - Upload a set of files belonging to a set of dataset documents
2+
% UPLOADFILESFORDATASETDOCUMENTS - Upload a set of files belonging to a set of dataset documents
3+
%
4+
% [SUCCESS, MESSAGE] = UPLOADFILESFORDATASETDOCUMENTS(CLOUD_DATASET_ID, NDIDATASET, ...
5+
% DATASET_DOCUMENTS, NAME/VALUE PAIRS)
6+
%
7+
% Uploads a set of files that are associated with a given list of NDI_DOCUMENTS.
8+
%
9+
% This function takes a list of NDI_DOCUMENTS, finds all the associated binary data files
10+
% that are stored in the NDIDATASET, and uploads them to the remote dataset identified
11+
% by CLOUD_DATASET_ID.
12+
%
13+
% It can be configured with the following NAME/VALUE pairs:
14+
% | Name | Description |
15+
% |-----------------------|-------------------------------------------|
16+
% | 'Verbose' | (logical) Display verbose output (default true) |
17+
% | 'FileUploadStrategy' | ('serial' or 'batch') Upload strategy (default 'batch') |
18+
% | 'onlyMissing' | (logical) Only upload missing files (default true) |
19+
%
20+
% Outputs:
21+
% | Name | Description |
22+
% |-----------------------|-------------------------------------------|
23+
% | SUCCESS | (logical) True if all files were uploaded successfully. |
24+
% | MESSAGE | (char) An error message if SUCCESS is false. |
25+
%
26+
% See also: ndi.cloud.uploadDataset
27+
%
328
arguments
429
cloudDatasetId (1,1) string
530
ndiDataset (1,1) ndi.dataset
631
dataset_documents (1,:) cell
732
options.Verbose (1,1) logical = true
833
options.FileUploadStrategy (1,1) string ...
934
{mustBeMember(options.FileUploadStrategy, ["serial", "batch"])} = "batch"
10-
options.updateMissing (1,1) logical = true
35+
options.onlyMissing (1,1) logical = true
1136
end
1237

1338
success = true;
@@ -18,7 +43,7 @@
1843
ndiDataset, dataset_documents, options.Verbose);
1944
[file_manifest(:).is_uploaded] = deal(false);
2045

21-
if options.updateMissing
46+
if options.onlyMissing
2247
[b, answer] = ndi.cloud.api.datasets.getDataset(cloudDatasetId);
2348
if b && isfield(answer, 'files')
2449
remote_files = containers.Map();

‎src/ndi/+ndi/+cloud/uploadDataset.m‎

Lines changed: 78 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,87 @@
1-
function cloudDatasetId = uploadDataset(ndiDataset, syncOptions)
1+
function [success, cloudDatasetId, message] = uploadDataset(ndiDataset, syncOptions, options)
22
% UPLOADDATASET - upload a dataset to NDI cloud
33
%
4-
% DATASETID = ndi.cloud.UPLOADDATASET(ndiDataset)
4+
% [SUCCESS, DATASETID, MESSAGE] = ndi.cloud.UPLOADDATASET(ndiDataset, SYNCOPTIONS, ...
5+
% NAME/VALUE PAIRS)
56
%
6-
% Upload an ndi.dataset object to NDI Cloud. The DATASETID on
7-
% NDI Cloud is returned.
7+
% Upload an ndi.dataset object to NDI Cloud.
88
%
9-
% Example:
10-
% ndi.cloud.upload.newDataset(ndiDataset)
9+
% This function uploads all documents and associated data files for a given
10+
% NDIDATASET to the NDI cloud.
11+
%
12+
% Inputs:
13+
% ndiDataset - The ndi.dataset object to be uploaded.
14+
% syncOptions - An ndi.cloud.sync.SyncOptions object for additional configuration.
15+
%
16+
% By default, this function will not re-upload a dataset if it already exists
17+
% on the remote server. See the 'uploadAsNew' option to override this behavior.
18+
%
19+
% The function returns a boolean SUCCESS flag, the CLOUDDATASETID of the
20+
% created remote dataset, and a MESSAGE string that will contain an error
21+
% message if SUCCESS is false.
22+
%
23+
% It can be configured with the following NAME/VALUE pairs:
24+
% | Name | Description |
25+
% |------------------------------|--------------------------------------------------------------------------|
26+
% | 'uploadAsNew' | (logical) If true, any existing remote dataset will be deleted and a new |
27+
% | | one will be created. Default is false. If a remote dataset exists and |
28+
% | | this is false, the function will return an error. |
29+
% | 'skipMetadataEditorMetadata' | (logical) If true, the function will skip generating metadata from the |
30+
% | | dataset. Default is false. If you use this option, you must also provide |
31+
% | | 'remoteDatasetName'. |
32+
% | 'remoteDatasetName' | (char) The name to be assigned to the dataset on the remote server. This |
33+
% | | is *required* if 'skipMetadataEditorMetadata' is true. |
34+
%
35+
% See also: ndi.cloud.sync.SyncOptions, ndi.cloud.downloadDataset
1136
%
1237

1338
arguments
1439
ndiDataset (1,1) ndi.dataset
1540
syncOptions.?ndi.cloud.sync.SyncOptions
41+
options.uploadAsNew (1,1) logical = false
42+
options.skipMetadataEditorMetadata (1,1) logical = false
43+
options.remoteDatasetName (1,:) char = ''
44+
end
45+
46+
success = false;
47+
cloudDatasetId = '';
48+
message = '';
49+
50+
[cloudDatasetId, remote_doc] = ndi.cloud.internal.getCloudDatasetIdForLocalDataset(ndiDataset);
51+
52+
if ~isempty(cloudDatasetId) & ~options.uploadAsNew,
53+
message = ['Dataset has already been uploaded, and "uploadAsNew" is false.'];
54+
return;
55+
elseif ~isempty(cloudDatasetId) & options.uploadAsNew,
56+
[delete_success, delete_message] = ndi.cloud.api.admin.deleteDataset(cloudDatasetId);
57+
if ~delete_success
58+
message = ['Could not delete existing remote dataset: ' delete_message];
59+
return;
60+
end
61+
ndiDataset.database_rm(remote_doc);
62+
cloudDatasetId = '';
1663
end
1764

1865
% Step 1: Create the dataset record on NDI Cloud and insert the metadata
19-
20-
% Step 1a: Retrieve metadata from the dataset
21-
metadata_struct = ndi.database.metadata_ds_core.ndidataset2metadataeditorstruct(ndiDataset);
66+
if options.skipMetadataEditorMetadata
67+
if isempty(options.remoteDatasetName)
68+
message = 'If skipMetadataEditorMetadata is true, remoteDatasetName cannot be empty.';
69+
return;
70+
end
71+
cloud_dataset_info.name = options.remoteDatasetName;
72+
else
73+
% Step 1a: Retrieve metadata from the dataset
74+
metadata_struct = ndi.database.metadata_ds_core.ndidataset2metadataeditorstruct(ndiDataset);
2275

23-
% Step 1b: Convert metadata structure to NDI Cloud Dataset info
24-
cloud_dataset_info = ndi.cloud.utility.createCloudMetadataStruct(metadata_struct);
76+
% Step 1b: Convert metadata structure to NDI Cloud Dataset info
77+
cloud_dataset_info = ndi.cloud.utility.createCloudMetadataStruct(metadata_struct);
78+
end
2579

2680
% Step 1c: Create new NDI Cloud Dataset
27-
[success, answer] = ndi.cloud.api.datasets.createDataset(cloud_dataset_info);
28-
if ~success
29-
error(['Failed to create dataset: ' answer.message]);
81+
[success_create, answer] = ndi.cloud.api.datasets.createDataset(cloud_dataset_info);
82+
if ~success_create
83+
message = ['Failed to create dataset: ' answer.message];
84+
return;
3085
end
3186
cloudDatasetId = answer.dataset_id;
3287

@@ -37,11 +92,17 @@
3792
% Step 2: Upload documents
3893
if syncOptions.Verbose, disp('Uploading dataset documents...'); end
3994
dataset_documents = ndiDataset.database_search( ndi.query('','isa','base') );
40-
ndi.cloud.upload.uploadDocumentCollection(cloudDatasetId, dataset_documents)
95+
ndi.cloud.upload.uploadDocumentCollection(cloudDatasetId, dataset_documents, "onlyUploadMissing", true)
4196

4297
% Step 3: Upload files
43-
ndi.cloud.sync.internal.uploadFilesForDatasetDocuments( ...
98+
[success_upload, message_upload] = ndi.cloud.sync.internal.uploadFilesForDatasetDocuments( ...
4499
cloudDatasetId, ndiDataset, dataset_documents, ...
45100
"Verbose", syncOptions.Verbose, ...
46-
"FileUploadStrategy", syncOptions.FileUploadStrategy)
101+
"FileUploadStrategy", syncOptions.FileUploadStrategy, "onlyMissing", true);
102+
if ~success_upload
103+
message = message_upload;
104+
return;
105+
end
106+
107+
success = true;
47108
end

0 commit comments

Comments
 (0)