feat(platform): igdb categories

This commit adds the support for fetching images from IGDB and saving them to the database, 
to be loaded as categories. it also makes the image-processor to resize and encode the images for us.

common/Cargo.toml

@@ -65,7 +65,7 @@ uuid = { version = "1.6", features = ["v4"], optional = true }
 ulid = { version = "1.1", features = ["uuid"], optional = true}
 
 aws-config = { version = "1.1", optional = true }
-aws-sdk-s3 = { version = "1.12", optional = true }
+aws-sdk-s3 = { version = "1.12", optional = true, features = ["behavior-version-latest"] }
 aws-credential-types = { version = "1.1", optional = true, features = ["hardcoded-credentials"] }
 aws-smithy-types = { version = "1.1", features = ["http-body-1-x"], optional = true }
 http-body = { version = "1.0.0", optional = true }

common/src/config.rs

@@ -195,7 +195,7 @@ pub struct S3BucketConfig {
 impl Default for S3BucketConfig {
 	fn default() -> Self {
 		Self {
-			name: "scuffle-image-processor".to_owned(),
+			name: "scuffle".to_owned(),
 			region: "us-east-1".to_owned(),
 			endpoint: Some("http://localhost:9000".to_string()),
 			credentials: S3CredentialsConfig::default(),

ffmpeg/src/scalar.rs

@@ -57,15 +57,12 @@ impl Scalar {
 			frame_mut.height = height;
 			frame_mut.format = pixel_format as i32;
 
-			// Safety: `av_image_alloc` is safe to call, and the pointer returned is valid.
-			av_image_alloc(
-				frame_mut.data.as_mut_ptr(),
-				frame_mut.linesize.as_mut_ptr(),
-				width,
-				height,
-				pixel_format,
-				32,
-			);
+			// Safety: `av_frame_get_buffer` is safe to call, and the pointer returned is
+			// valid.
+			match av_frame_get_buffer(frame_mut, 32) {
+				0 => {}
+				err => return Err(FfmpegError::Code(err.into())),
+			}
 		}
 
 		Ok(Self {

platform/api/Cargo.toml

@@ -43,14 +43,15 @@ thiserror = "1.0"
 anyhow = "1.0"
 multer = "3.0"
 aws-config = "1.1"
-aws-sdk-s3 = "1.12"
+aws-sdk-s3 = { version = "1.12", features = ["behavior-version-latest"] }
 http-body = "1.0"
 http-body-util = "0.1"
 hyper-util = "0.1"
 pin-project = "1.1"
 base64 = "0.21"
 postgres-from-row = "0.5"
 postgres-types = "0.2"
+fred = { version = "8.0", features = ["enable-rustls", "sentinel-client", "dns"] }
 
 config = { workspace = true }
 pb = { workspace = true }

platform/api/src/api/v1/gql/models/category.rs

@@ -17,7 +17,7 @@ impl From<database::Category> for Category {
 		Self {
 			id: value.id.into(),
 			name: value.name,
-			revision: value.revision,
+			revision: 1,
 			updated_at: value.updated_at.into(),
 		}
 	}

platform/api/src/api/v1/gql/models/image_upload.rs

@@ -6,7 +6,7 @@ use super::ulid::GqlUlid;
 use crate::api::v1::gql::error::{GqlError, Result};
 use crate::api::v1::gql::ext::ContextExt;
 use crate::config::ImageUploaderConfig;
-use crate::database::UploadedFile;
+use crate::database::{UploadedFile, UploadedFileStatus};
 use crate::global::ApiGlobal;
 
 #[derive(SimpleObject, Clone)]
@@ -51,7 +51,7 @@ impl<G: ApiGlobal> ImageUpload<G> {
 
 impl<G: ApiGlobal> ImageUpload<G> {
 	pub fn from_uploaded_file(uploaded_file: UploadedFile) -> Result<Option<Self>> {
-		if uploaded_file.pending {
+		if uploaded_file.status != UploadedFileStatus::Completed {
 			return Ok(None);
 		}
 

platform/api/src/api/v1/gql/subscription/file.rs

@@ -7,6 +7,7 @@ use crate::api::v1::gql::error::ext::{OptionExt, ResultExt};
 use crate::api::v1::gql::error::{GqlError, Result};
 use crate::api::v1::gql::ext::ContextExt;
 use crate::api::v1::gql::models::ulid::GqlUlid;
+use crate::database::UploadedFileStatus;
 use crate::global::ApiGlobal;
 use crate::subscription::SubscriptionTopic;
 
@@ -63,7 +64,7 @@ impl<G: ApiGlobal> FileSubscription<G> {
 			.map_err_gql("failed to subscribe to file status")?;
 
 		Ok(async_stream::stream!({
-			if !file.pending {
+			if file.status != UploadedFileStatus::Queued {
 				// When file isn't pending anymore, just yield once with the status from the db
 				let status = if file.failed.is_some() {
 					FileStatus::Failure

platform/api/src/api/v1/upload/profile_picture.rs

@@ -17,7 +17,7 @@ use crate::api::auth::AuthData;
 use crate::api::error::ApiError;
 use crate::api::Body;
 use crate::config::{ApiConfig, ImageUploaderConfig};
-use crate::database::{FileType, RolePermission};
+use crate::database::{FileType, RolePermission, UploadedFileStatus};
 use crate::global::ApiGlobal;
 
 fn create_task(file_id: Ulid, input_path: &str, config: &ImageUploaderConfig, owner_id: Ulid) -> image_processor::Task {
@@ -33,7 +33,7 @@ fn create_task(file_id: Ulid, input_path: &str, config: &ImageUploaderConfig, ow
 			ImageFormat::Webp as i32,
 			ImageFormat::Avif as i32,
 		],
-		callback_subject: config.profile_picture_callback_subject.clone(),
+		callback_subject: config.callback_subject.clone(),
 		limits: Some(image_processor::task::Limits {
 			max_input_duration_ms: 10 * 1000, // 10 seconds
 			max_input_frame_count: 300,
@@ -197,7 +197,7 @@ impl UploadType for ProfilePicture {
 			.await
 			.map_err_route((StatusCode::INTERNAL_SERVER_ERROR, "failed to insert image job"))?;
 
-		common::database::query("INSERT INTO uploaded_files(id, owner_id, uploader_id, name, type, metadata, total_size, pending, path) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)")
+		common::database::query("INSERT INTO uploaded_files(id, owner_id, uploader_id, name, type, metadata, total_size, path, status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)")
             .bind(file_id) // id
             .bind(auth.session.user_id) // owner_id
             .bind(auth.session.user_id) // uploader_id
@@ -209,8 +209,8 @@ impl UploadType for ProfilePicture {
 				})),
 			})) // metadata
             .bind(file.len() as i64) // total_size
-            .bind(true) // pending
             .bind(&input_path) // path
+			.bind(UploadedFileStatus::Queued) // status
 			.build()
             .execute(&tx)
             .await

platform/api/src/config.rs

@@ -1,5 +1,6 @@
 use std::net::SocketAddr;
 use std::path::PathBuf;
+use std::time::Duration;
 
 use common::config::{S3BucketConfig, TlsConfig};
 
@@ -70,23 +71,27 @@ pub struct ImageUploaderConfig {
 	/// The S3 Bucket which contains the source images
 	pub bucket: S3BucketConfig,
 
+	/// Public endpoint for downloads
+	pub public_endpoint: String,
+
 	/// Profile picture callback subject (can't contain `.`)
-	pub profile_picture_callback_subject: String,
+	pub callback_subject: String,
 
 	/// Profile picture task priority, higher number means higher priority
 	pub profile_picture_task_priority: i64,
 
-	/// Public endpoint for downloads
-	pub public_endpoint: String,
+	/// Igdb image task priority, higher number means higher priority
+	pub igdb_image_task_priority: i32,
 }
 
 impl Default for ImageUploaderConfig {
 	fn default() -> Self {
 		Self {
 			bucket: S3BucketConfig::default(),
-			profile_picture_callback_subject: "scuffle-platform-image_processor-profile_picture".to_string(),
-			profile_picture_task_priority: 2,
+			callback_subject: "scuffle-platform-image_processor-callback".to_string(),
 			public_endpoint: "https://images.scuffle.tv/scuffle-image-processor-public".to_string(),
+			profile_picture_task_priority: 2,
+			igdb_image_task_priority: 1,
 		}
 	}
 }
@@ -138,3 +143,34 @@ pub struct VideoApiPlaybackKeypairConfig {
 	/// Path to the playback private key for the video api
 	pub private_key: PathBuf,
 }
+
+#[derive(Debug, Clone, PartialEq, config::Config, serde::Deserialize)]
+#[serde(default)]
+pub struct IgDbConfig {
+	/// The IGDB Client ID
+	pub client_id: String,
+
+	/// The IGDB Client Secret
+	pub client_secret: String,
+
+	/// Process Images
+	pub process_images: bool,
+
+	/// Refresh Interval
+	pub refresh_interval: Duration,
+
+	/// Igdb Cron Subject
+	pub igdb_cron_subject: String,
+}
+
+impl Default for IgDbConfig {
+	fn default() -> Self {
+		Self {
+			client_id: "igdb_client_id".to_string(),
+			client_secret: "igdb_client_secret".to_string(),
+			process_images: false,
+			refresh_interval: Duration::from_secs(6 * 60 * 60), // 6 hours
+			igdb_cron_subject: "scuffle-platform-igdb_cron".to_string(),
+		}
+	}
+}

platform/api/src/database/category.rs

@@ -4,7 +4,17 @@ use ulid::Ulid;
 #[derive(Debug, Clone, Default, postgres_from_row::FromRow)]
 pub struct Category {
 	pub id: Ulid,
+	pub igdb_id: Option<i32>,
 	pub name: String,
-	pub revision: i32,
+	pub aliases: Vec<String>,
+	pub keywords: Vec<String>,
+	pub storyline: Option<String>,
+	pub summary: Option<String>,
+	pub over_18: bool,
+	pub cover_id: Option<Ulid>,
+	pub rating: f64,
+	pub artwork_ids: Vec<Ulid>,
+	pub igdb_similar_game_ids: Vec<i32>,
+	pub websites: Vec<String>,
 	pub updated_at: DateTime<Utc>,
 }

platform/api/src/database/file_type.rs

@@ -3,14 +3,10 @@ use postgres_types::{FromSql, ToSql};
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, ToSql, FromSql)]
 #[postgres(name = "file_type")]
 pub enum FileType {
-	#[postgres(name = "custom_thumbnail")]
-	CustomThumbnail,
 	#[postgres(name = "profile_picture")]
 	ProfilePicture,
-	#[postgres(name = "offline_banner")]
-	OfflineBanner,
-	#[postgres(name = "role_badge")]
-	RoleBadge,
-	#[postgres(name = "channel_role_badge")]
-	ChannelRoleBadge,
+	#[postgres(name = "category_cover")]
+	CategoryCover,
+	#[postgres(name = "category_artwork")]
+	CategoryArtwork,
 }

platform/api/src/database/mod.rs

@@ -8,6 +8,7 @@ mod search_result;
 mod session;
 mod two_fa_request;
 mod uploaded_file;
+mod uploaded_file_status;
 mod user;
 
 pub use category::*;
@@ -20,4 +21,5 @@ pub use search_result::*;
 pub use session::*;
 pub use two_fa_request::*;
 pub use uploaded_file::*;
+pub use uploaded_file_status::*;
 pub use user::*;

platform/api/src/database/uploaded_file.rs

@@ -1,20 +1,20 @@
 use common::database::protobuf;
 use ulid::Ulid;
 
-use super::FileType;
+use super::{FileType, UploadedFileStatus};
 
 #[derive(Debug, Clone, postgres_from_row::FromRow)]
 pub struct UploadedFile {
 	pub id: Ulid,
-	pub owner_id: Ulid,
-	pub uploader_id: Ulid,
+	pub owner_id: Option<Ulid>,
+	pub uploader_id: Option<Ulid>,
 	pub name: String,
 	#[from_row(rename = "type")]
 	pub ty: FileType,
 	#[from_row(from_fn = "protobuf")]
 	pub metadata: pb::scuffle::platform::internal::types::UploadedFileMetadata,
 	pub total_size: i64,
-	pub pending: bool,
+	pub status: UploadedFileStatus,
 	pub path: String,
 	pub updated_at: chrono::DateTime<chrono::Utc>,
 	pub failed: Option<String>,

platform/api/src/database/uploaded_file_status.rs

@@ -0,0 +1,14 @@
+use postgres_types::{FromSql, ToSql};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, ToSql, FromSql)]
+#[postgres(name = "uploaded_file_status")]
+pub enum UploadedFileStatus {
+	#[postgres(name = "unqueued")]
+	Unqueued,
+	#[postgres(name = "queued")]
+	Queued,
+	#[postgres(name = "failed")]
+	Failed,
+	#[postgres(name = "completed")]
+	Completed,
+}

platform/api/src/global.rs

@@ -1,6 +1,6 @@
 use common::dataloader::DataLoader;
 
-use crate::config::{ApiConfig, ImageUploaderConfig, JwtConfig, TurnstileConfig, VideoApiConfig};
+use crate::config::{ApiConfig, IgDbConfig, ImageUploaderConfig, JwtConfig, TurnstileConfig, VideoApiConfig};
 use crate::dataloader::category::CategoryByIdLoader;
 use crate::dataloader::global_state::GlobalStateLoader;
 use crate::dataloader::role::RoleByIdLoader;
@@ -39,8 +39,10 @@ pub trait ApiGlobal:
 	+ common::global::GlobalConfigProvider<JwtConfig>
 	+ common::global::GlobalConfigProvider<ImageUploaderConfig>
 	+ common::global::GlobalConfigProvider<VideoApiConfig>
+	+ common::global::GlobalConfigProvider<IgDbConfig>
 	+ common::global::GlobalNats
 	+ common::global::GlobalDb
+	+ common::global::GlobalRedis
 	+ common::global::GlobalConfig
 	+ ApiState
 	+ Send
@@ -56,8 +58,10 @@ impl<T> ApiGlobal for T where
 		+ common::global::GlobalConfigProvider<JwtConfig>
 		+ common::global::GlobalConfigProvider<ImageUploaderConfig>
 		+ common::global::GlobalConfigProvider<VideoApiConfig>
+		+ common::global::GlobalConfigProvider<IgDbConfig>
 		+ common::global::GlobalNats
 		+ common::global::GlobalDb
+		+ common::global::GlobalRedis
 		+ common::global::GlobalConfig
 		+ ApiState
 		+ Send