rsc: Create tables for tracking Blobs

rust/entity/src/blob.rs

@@ -0,0 +1,49 @@
+//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.6
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "blob")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub id: String,
+    pub store: i32,
+    pub workspace_path: String,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {
+    #[sea_orm(
+        belongs_to = "super::blob_store::Entity",
+        from = "Column::Store",
+        to = "super::blob_store::Column::Id",
+        on_update = "NoAction",
+        on_delete = "Restrict"
+    )]
+    BlobStore,
+    #[sea_orm(has_many = "super::job_blob::Entity")]
+    JobBlob,
+}
+
+impl Related<super::blob_store::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::BlobStore.def()
+    }
+}
+
+impl Related<super::job_blob::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::JobBlob.def()
+    }
+}
+
+impl Related<super::job::Entity> for Entity {
+    fn to() -> RelationDef {
+        super::job_blob::Relation::Job.def()
+    }
+    fn via() -> Option<RelationDef> {
+        Some(super::job_blob::Relation::Blob.def().rev())
+    }
+}
+
+impl ActiveModelBehavior for ActiveModel {}

rust/entity/src/blob_store.rs

@@ -0,0 +1,25 @@
+//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.6
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "blob_store")]
+pub struct Model {
+    #[sea_orm(primary_key)]
+    pub id: i32,
+    pub name: String,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {
+    #[sea_orm(has_many = "super::blob::Entity")]
+    Blob,
+}
+
+impl Related<super::blob::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Blob.def()
+    }
+}
+
+impl ActiveModelBehavior for ActiveModel {}

rust/entity/src/job.rs

@@ -34,6 +34,8 @@ pub struct Model {
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
 pub enum Relation {
+    #[sea_orm(has_many = "super::job_blob::Entity")]
+    JobBlob,
     #[sea_orm(has_many = "super::job_use::Entity")]
     JobUse,
     #[sea_orm(has_many = "super::output_dir::Entity")]
@@ -46,6 +48,12 @@ pub enum Relation {
     VisibleFile,
 }
 
+impl Related<super::job_blob::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::JobBlob.def()
+    }
+}
+
 impl Related<super::job_use::Entity> for Entity {
     fn to() -> RelationDef {
         Relation::JobUse.def()
@@ -76,4 +84,13 @@ impl Related<super::visible_file::Entity> for Entity {
     }
 }
 
+impl Related<super::blob::Entity> for Entity {
+    fn to() -> RelationDef {
+        super::job_blob::Relation::Blob.def()
+    }
+    fn via() -> Option<RelationDef> {
+        Some(super::job_blob::Relation::Job.def().rev())
+    }
+}
+
 impl ActiveModelBehavior for ActiveModel {}

rust/entity/src/job_blob.rs

@@ -0,0 +1,46 @@
+//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.6
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "job_blob")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub job_id: i32,
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub blob_id: String,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {
+    #[sea_orm(
+        belongs_to = "super::blob::Entity",
+        from = "Column::BlobId",
+        to = "super::blob::Column::Id",
+        on_update = "NoAction",
+        on_delete = "Restrict"
+    )]
+    Blob,
+    #[sea_orm(
+        belongs_to = "super::job::Entity",
+        from = "Column::JobId",
+        to = "super::job::Column::Id",
+        on_update = "NoAction",
+        on_delete = "Cascade"
+    )]
+    Job,
+}
+
+impl Related<super::blob::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Blob.def()
+    }
+}
+
+impl Related<super::job::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Job.def()
+    }
+}
+
+impl ActiveModelBehavior for ActiveModel {}

rust/entity/src/mod.rs

@@ -3,7 +3,10 @@
 pub mod prelude;
 
 pub mod api_key;
+pub mod blob;
+pub mod blob_store;
 pub mod job;
+pub mod job_blob;
 pub mod job_use;
 pub mod output_dir;
 pub mod output_file;

rust/entity/src/prelude.rs

@@ -1,7 +1,10 @@
 //! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.6
 
 pub use super::api_key::Entity as ApiKey;
+pub use super::blob::Entity as Blob;
+pub use super::blob_store::Entity as BlobStore;
 pub use super::job::Entity as Job;
+pub use super::job_blob::Entity as JobBlob;
 pub use super::job_use::Entity as JobUse;
 pub use super::output_dir::Entity as OutputDir;
 pub use super::output_file::Entity as OutputFile;

rust/migration/src/lib.rs

@@ -6,6 +6,7 @@ mod m20230707_144317_record_uses;
 mod m20231117_162713_add_created_at;
 mod m20231127_232833_drop_use_time;
 mod m20231128_000751_normalize_uses_table;
+mod m20231213_194935_create_blob_table;
 
 pub struct Migrator;
 
@@ -19,6 +20,7 @@ impl MigratorTrait for Migrator {
             Box::new(m20231117_162713_add_created_at::Migration),
             Box::new(m20231127_232833_drop_use_time::Migration),
             Box::new(m20231128_000751_normalize_uses_table::Migration),
+            Box::new(m20231213_194935_create_blob_table::Migration),
         ]
     }
 }

rust/migration/src/m20231213_194935_create_blob_table.rs

@@ -0,0 +1,154 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .create_table(
+                Table::create()
+                    .table(BlobStore::Table)
+                    .col(
+                        ColumnDef::new(BlobStore::Id)
+                            .integer()
+                            .not_null()
+                            .auto_increment()
+                            .primary_key(),
+                    )
+                    .col(ColumnDef::new(BlobStore::Name).string().not_null())
+                    .to_owned(),
+            )
+            .await?;
+
+        // Insert basic backing stores since they must always exist.
+        let insert = Query::insert()
+            .into_table(BlobStore::Table)
+            .columns([BlobStore::Id, BlobStore::Name])
+            .values_panic([1.into(), "local".into()])
+            .values_panic([2.into(), "artifactory".into()])
+            .values_panic([3.into(), "aws".into()])
+            .values_panic([4.into(), "gcs".into()])
+            .values_panic([5.into(), "azure".into()])
+            .to_owned();
+
+        manager.exec_stmt(insert).await?;
+
+        manager
+            .create_table(
+                Table::create()
+                    .table(Blob::Table)
+                    .col(
+                        ColumnDef::new(Blob::Id)
+                            .string()
+                            .not_null()
+                            .primary_key(),
+                    )
+                    .col(ColumnDef::new(Blob::Store).integer().not_null())
+                    .col(ColumnDef::new(Blob::WorkspacePath).string().not_null())
+                    .foreign_key(
+                        ForeignKeyCreateStatement::new()
+                            .name("fk-store-blob_store")
+                            .from_tbl(Blob::Table)
+                            .from_col(Blob::Store)
+                            .to_tbl(BlobStore::Table)
+                            .to_col(BlobStore::Id)
+                            // All blobs for a given store must be manually
+                            // deleted before the blob store may be deleted.
+                            .on_delete(ForeignKeyAction::Restrict),
+                    )
+                    .to_owned(),
+            )
+            .await?;
+
+        manager
+            .create_table(
+                Table::create()
+                    .table(JobBlob::Table)
+                    .col(
+                        ColumnDef::new(JobBlob::JobId)
+                            .integer()
+                            .not_null(),
+                    )
+                    .col(
+                        ColumnDef::new(JobBlob::BlobId)
+                            .string()
+                            .not_null(),
+                    )
+                    .foreign_key(
+                        ForeignKeyCreateStatement::new()
+                            .name("fk-job_id-job")
+                            .from_tbl(JobBlob::Table)
+                            .from_col(JobBlob::JobId)
+                            .to_tbl(Job::Table)
+                            .to_col(Job::Id)
+                            // If the associated job is deleted (likely via eviction)
+                            // then the reference to the blob should be removed so the
+                            // blob may later be cleaned up.
+                            .on_delete(ForeignKeyAction::Cascade),
+                    )
+                    .foreign_key(
+                        ForeignKeyCreateStatement::new()
+                            .name("fk-blob_id-blob")
+                            .from_tbl(JobBlob::Table)
+                            .from_col(JobBlob::BlobId)
+                            .to_tbl(Blob::Table)
+                            .to_col(Blob::Id)
+                            // A blob must not be deleted if it is still referenced by
+                            // a job. Restrict such deletions.
+                            .on_delete(ForeignKeyAction::Restrict),
+                    )
+                    .primary_key(Index::create().col(JobBlob::JobId).col(JobBlob::BlobId))
+                    .to_owned(),
+            )
+            .await?;
+
+        Ok(())
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .drop_table(Table::drop().table(JobBlob::Table).to_owned())
+            .await?;
+
+        manager
+            .drop_table(Table::drop().table(Blob::Table).to_owned())
+            .await?;
+
+        manager
+            .drop_table(Table::drop().table(BlobStore::Table).to_owned())
+            .await?;
+
+        Ok(())
+    }
+}
+
+#[derive(DeriveIden)]
+enum BlobStore {
+    Table,
+    Id,
+    Name,
+}
+
+#[derive(DeriveIden)]
+enum Blob {
+    Table,
+    Id,
+    Store,
+    WorkspacePath,
+    // Job relation
+}
+
+#[derive(DeriveIden)]
+enum Job {
+    Table,
+    Id,
+}
+
+#[derive(DeriveIden)]
+enum JobBlob {
+    Table,
+    JobId,
+    BlobId,
+}

rust/rsc/src/rsc/main.rs

@@ -237,7 +237,7 @@ mod tests {
             server_addr: Some("test:0000".into()),
             database_url: Some("".into()),
             standalone: Some(true),
-            local_store: None,
+            local_store: Some("".into()),
         })?)
     }