feat(rst): expand remote storage target support (#50)

* Add JobRequestCfg defining common job request configuration structure including the ability to stub files.
* Add JobLockInfo for passing local and remote information collected under a BeeGFS file access lock so they can be reused by subsequent steps to optimize job execution and minimize work done by the Remote service.
* Add a BuilderJob type to support walking paths and generating job requests for each file or object. This allows downloading multiple files/objects at once.
* Add a job request GenerationStatus used to handle failed preconditions that prevent a job from being generated, or communicate if there is an existing job already in a terminal or failed state that prevents generating a new job.

Author: swartzn

cpp/beeremote.pb.cc

@@ -37,6 +37,9 @@ message SubmitJobResponse {
         CREATED = 1;
         EXISTING = 2;
         NOT_ALLOWED = 3;
+        ALREADY_COMPLETE = 4;
+        ALREADY_OFFLOADED = 5;
+        FAILED_PRECONDITION = 6;
     }
 }
 
@@ -58,12 +61,40 @@ message JobRequest {
     oneof type {
         flex.SyncJob sync = 10;
         flex.MockJob mock = 11;
+        flex.BuilderJob builder = 12;
     }
     // When force is set this request will create a new job even if there is already a completed
     // job. When forced the request will return an error if there is already a job running. This is
     // part of the JobRequest instead of the SubmitJobRequest message in case it ever is important
     // to know if a particular job request was forced.
-    bool force = 5;    
+    bool force = 5;
+    // When stub_local is set the local file with be a stub file
+    bool stub_local = 7;
+    // generation_status reports the outcome of generating this JobRequest (e.g. already complete,
+    // already offloaded, or an error message). This is used by job manager to determine whether the
+    // job is already in a terminal or failed state.
+    GenerationStatus generation_status = 8;
+    message GenerationStatus {
+        State state = 1;
+        enum State {
+            // This state has no semantic value and should never be UNSPECIFIED unless there is a bug.
+            UNSPECIFIED = 0;
+            // This state indicates the request has already been completed. The generation_status's
+            // message field must be populated with the file's beegfs-mtime in RFC3339 format (i.e.
+            // 2006-01-02T15:04:05Z07:00).
+            ALREADY_COMPLETE = 1;
+            // This state indicates the request has already been offloaded.
+            ALREADY_OFFLOADED = 2;
+            // This state indicates a preconditional failed and will always result in the job being
+            // cancelled. It is imperative that this state is only used when it is safe to submit
+            // the job request again without first calling rst.CompleteWorkRequests().
+            FAILED_PRECONDITION = 3;
+            // This state indicates an error occurred that requires rst.CompleteWorkRequests() to
+            // cleanup.
+            ERROR = 4;
+        }
+        string message = 2;
+    }
 }
 
 // Job contains all the data from the original request plus the job ID and
@@ -146,6 +177,9 @@ message Job {
         // If the job and its WRs completed successfully. This is a terminal state and no further 
         // state changes are possible once a job enters this state.
         COMPLETED = 9;
+        // If the job's WRs completed successfully and a stub has replaced the local file. This is a
+        // terminal state and no further state changes are possible once a job enters this state.
+        OFFLOADED = 10;
     }
     // Used as the upload ID for multipart uploads, or any other time 
     // an external identifer is needed to coordinate a distributed transfer.    

cpp/beeremote.pb.h

@@ -82,21 +82,57 @@ message BulkUpdateWorkResponse {
     string message = 2;
 }
 
+// JobLockedInfo contains require information that must be determined prior to calling a job request for
+// file. This will be produced by sync's job builder but can be produced anywhere it makes sense such as
+// in ctl.
+message JobLockedInfo {
+    // Indicates the file write lock has already been obtained.
+    bool write_locked = 1;
+    // Whether the local file exists.
+    bool exists = 2;
+    // Size of the local file.
+    int64 size = 3;
+    // File mode of the local file
+    uint32 mode = 4;
+    // Last modified time of the local file.
+    google.protobuf.Timestamp mtime = 5;
+    // Size of the remote file or object.
+    int64 remote_size = 6;
+    // Last beegfs_mtime of the remote file or object.
+    google.protobuf.Timestamp remote_mtime = 7;
+    // If stub_url is not empty then path is for a stub file with the provided rst url.
+    uint32 stub_url_rst_id = 8;
+    string stub_url_path = 9;
+    string externalId = 10;
+}
+
+message JobRequestCfg {
+    uint32 remoteStorageTarget = 1;
+    string path = 2;
+    string remotePath = 3;
+    bool download = 4;
+    bool stub_local = 5;
+    bool overwrite = 6;
+    bool flatten = 7;
+    bool force = 8;
+    JobLockedInfo locked_info = 9;
+}
+
 // BeeRemote assigns work for a job to one or more worker nodes.
 message WorkRequest {
     string job_id = 1;
     string request_id = 2;
     // Used as the upload ID for multipart uploads, or any other time 
-    // an external identifer is needed to coordinate a distributed transfer.
+    // an external identifier is needed to coordinate a distributed transfer.
     string external_id = 3;
     string path = 4;
     Segment segment = 5;
     uint32 remote_storage_target = 6;
     oneof Type {
         MockJob mock = 10;
         SyncJob sync = 11;
+        BuilderJob builder = 12;
     }
-
     // A segment indicates what portion of the file a particular worker node should
     // work on. Segments can be divided into one or more parts, which can be used to
     // execute the requested operation in parallel if supported by the RST type.
@@ -109,15 +145,22 @@ message WorkRequest {
         // Inclusive of the last part.
         int32 parts_stop = 4;
     }
+    // When stub_local is set the local file with be a stub file
+    bool stub_local = 8;
+}
 
+// JobBuilderJob is a special type of job that creates job requests of any time.
+message BuilderJob {
+    JobRequestCfg cfg = 1;
 }
 
 message MockJob {
     int32 num_test_segments = 1;
     int64 file_size = 2;
     string external_id = 3;
     bool should_fail = 4;
-    bool can_retry = 5;
+    JobLockedInfo locked_info = 6;
+    JobRequestCfg cfg = 7;
 }
 
 // A SyncJob is WHAT work needs to be done. It is populated based on the
@@ -140,6 +183,10 @@ message SyncJob {
     // for example to restore a file in an RST to a different location in BeeGFS. This currently is
     // ignored for uploads.
     string remote_path = 3;
+    // By default the remote directory structure will be preserved on downloads unless flatten is
+    // set. If the flag is set then the directory delimiter will be replaced with an underscore.
+    bool flatten = 5;
+    JobLockedInfo locked_info = 6;
 }
 
 // Currently while requests types are specific to a particular worker node type,
@@ -226,6 +273,8 @@ message Work {
         string checksum_sha256 = 5;
         bool completed = 6;
     }
+    // Indicates whether the work is a job builder task.
+    bool job_builder = 6;
 }
 
 // We use a common configuration update request/response types for all worker
@@ -256,6 +305,12 @@ message UpdateConfigResponse {
 message BeeRemoteNode {
     string id = 1;
     string address = 2;
+    string mgmtd_address = 3;
+    bytes mgmtd_tls_cert = 4;
+    bool mgmtd_tls_disable_verification = 5;
+    bool mgmtd_tls_disable = 6;
+    bytes auth_secret = 7;
+    bool auth_disable = 8;
 }
 
 // Remote Storage Targets (RSTs) describe where data should be stored or