diff --git a/iam-policy-autopilot-policy-generation/src/embedded_data.rs b/iam-policy-autopilot-policy-generation/src/embedded_data.rs index e08b3e3..0670ffd 100644 --- a/iam-policy-autopilot-policy-generation/src/embedded_data.rs +++ b/iam-policy-autopilot-policy-generation/src/embedded_data.rs @@ -5,9 +5,11 @@ //! have been simplified to remove documentation and examples, reducing binary size //! while maintaining all essential functionality. +use std::borrow::Cow; +use std::collections::HashMap; + use crate::errors::{ExtractorError, Result}; use crate::extraction::sdk_model::SdkServiceDefinition; -use crate::providers::JsonProvider; use rust_embed::RustEmbed; /// Embedded AWS service definitions with compression @@ -18,7 +20,7 @@ use rust_embed::RustEmbed; #[derive(RustEmbed)] #[folder = "target/botocore-data-simplified"] #[include = "*.json"] -pub struct Botocore; +struct BotocoreRaw; /// Embedded AWS boto3 resource definitions /// @@ -27,7 +29,7 @@ pub struct Botocore; #[derive(RustEmbed)] #[folder = "target/boto3-data-simplified"] #[include = "*.json"] -pub struct Boto3Resources; +struct Boto3ResourcesRaw; /// Embedded boto3 utilities mapping /// @@ -36,40 +38,23 @@ pub struct Boto3Resources; #[derive(RustEmbed)] #[folder = "resources/config/sdks"] #[include = "boto3_utilities_mapping.json"] -pub struct Boto3Utilities; +struct Boto3UtilitiesRaw; -impl Boto3Utilities { +impl Boto3UtilitiesRaw { /// Get the boto3 utilities mapping configuration - pub fn get_utilities_mapping() -> Option> { + fn get_utilities_mapping() -> Option> { Self::get("boto3_utilities_mapping.json").map(|file| file.data) } } -/// Embedded JavaScript SDK v3 libraries mapping -/// -/// This struct provides access to the JavaScript SDK v3 libraries mapping configuration -/// that defines how lib-* submodule commands map to client-* commands. -#[derive(RustEmbed)] -#[folder = "resources/config/sdks"] -#[include = "js_v3_libraries.json"] -pub(crate) struct JsV3Libraries; - -impl JsV3Libraries { - /// Get the JavaScript SDK v3 libraries mapping configuration - pub fn get_libraries_mapping() -> Option> { - Self::get("js_v3_libraries.json").map(|file| file.data) - } -} - -impl Boto3Resources { +impl Boto3ResourcesRaw { /// Get a boto3 resources definition file by service name and API version - pub fn get_resources_definition(service: &str, api_version: &str) -> Option> { + fn get_resources_definition(service: &str, api_version: &str) -> Option> { let start_time = std::time::Instant::now(); let json_path = format!("{}/{}/resources-1.json", service, api_version); if let Some(file) = Self::get(&json_path) { let file_size = file.data.len(); - let result = Some(file.data.to_vec()); let total_time = start_time.elapsed(); if total_time.as_millis() > 10 { @@ -82,14 +67,14 @@ impl Boto3Resources { ); } - result + Some(file.data) } else { None } } /// Build a complete service-to-versions map for boto3 resources - pub(crate) fn build_service_versions_map() -> std::collections::HashMap> { + fn build_service_versions_map() -> std::collections::HashMap> { log::debug!("Building boto3 service versions map..."); let start_time = std::time::Instant::now(); @@ -99,7 +84,7 @@ impl Boto3Resources { > = std::collections::HashMap::new(); let mut file_count = 0; - for file_path in Boto3Resources::iter() { + for file_path in Self::iter() { file_count += 1; let path_parts: Vec<&str> = file_path.split('/').collect(); if path_parts.len() >= 2 { @@ -130,15 +115,14 @@ impl Boto3Resources { } } -impl Botocore { +impl BotocoreRaw { /// Get a service definition file by service name and API version - pub fn get_service_definition(service: &str, api_version: &str) -> Option> { + fn get_service_definition(service: &str, api_version: &str) -> Option> { let start_time = std::time::Instant::now(); let json_path = format!("{}/{}/service-2.json", service, api_version); if let Some(file) = Self::get(&json_path) { let file_size = file.data.len(); - let result = Some(file.data.to_vec()); let total_time = start_time.elapsed(); if total_time.as_millis() > 10 { @@ -151,32 +135,26 @@ impl Botocore { ); } - result + Some(file.data) } else { None } } /// Get a waiters definition file by service name and API version - pub fn get_waiters( - service: &str, - api_version: &str, - ) -> Option> { + fn get_waiters(service: &str, api_version: &str) -> Option> { let path = format!("{}/{}/waiters-2.json", service, api_version); Self::get(&path).map(|file| file.data) } /// Get a paginators definition file by service name and API version - pub fn get_paginators( - service: &str, - api_version: &str, - ) -> Option> { + fn get_paginators(service: &str, api_version: &str) -> Option> { let path = format!("{}/{}/paginators-1.json", service, api_version); Self::get(&path).map(|file| file.data) } /// Build a complete service-to-versions map in a single iteration - pub(crate) fn build_service_versions_map() -> std::collections::HashMap> { + fn build_service_versions_map() -> std::collections::HashMap> { log::debug!("Building service versions map..."); let start_time = std::time::Instant::now(); @@ -186,7 +164,7 @@ impl Botocore { > = std::collections::HashMap::new(); let mut file_count = 0; - for file_path in Botocore::iter() { + for file_path in BotocoreRaw::iter() { file_count += 1; let path_parts: Vec<&str> = file_path.split('/').collect(); if path_parts.len() >= 2 { @@ -221,9 +199,9 @@ impl Botocore { /// /// Provides convenient access to embedded boto3 resource definitions with /// automatic JSON parsing. -pub(crate) struct EmbeddedBoto3Data; +pub(crate) struct Boto3Data; -impl EmbeddedBoto3Data { +impl Boto3Data { /// Get raw boto3 resources data by service name and API version /// /// # Arguments @@ -232,18 +210,21 @@ impl EmbeddedBoto3Data { /// /// # Returns /// Raw resources JSON data or None if not found - pub fn get_resources_raw(service: &str, api_version: &str) -> Option> { - Boto3Resources::get_resources_definition(service, api_version) + pub(crate) fn get_resources_raw( + service: &str, + api_version: &str, + ) -> Option> { + Boto3ResourcesRaw::get_resources_definition(service, api_version) } /// Build a complete service-to-versions map for boto3 resources pub(crate) fn build_service_versions_map() -> std::collections::HashMap> { - Boto3Resources::build_service_versions_map() + Boto3ResourcesRaw::build_service_versions_map() } /// Get the boto3 utilities mapping configuration from embedded data - pub(crate) fn get_utilities_mapping() -> Option> { - Boto3Utilities::get_utilities_mapping() + pub(crate) fn get_utilities_mapping() -> Option> { + Boto3UtilitiesRaw::get_utilities_mapping() } } @@ -251,9 +232,9 @@ impl EmbeddedBoto3Data { /// /// Provides convenient access to embedded AWS service definitions with /// automatic decompression and JSON parsing. -pub(crate) struct EmbeddedServiceData; +pub(crate) struct BotocoreData; -impl EmbeddedServiceData { +impl BotocoreData { /// Get a parsed service definition by service name and API version /// /// # Arguments @@ -262,40 +243,46 @@ impl EmbeddedServiceData { /// /// # Returns /// Parsed service definition or error if not found or parsing fails - pub(crate) async fn get_service_definition( + pub(crate) fn get_service_definition( service: &str, api_version: &str, ) -> Result { - let data = Botocore::get_service_definition(service, api_version).ok_or_else(|| { + let data = BotocoreRaw::get_service_definition(service, api_version).ok_or_else(|| { ExtractorError::validation(format!( "Service definition not found for {}/{}", service, api_version )) })?; - let json_str = std::str::from_utf8(&data).map_err(|e| { - ExtractorError::validation(format!("Invalid UTF-8 in embedded data: {}", e)) - })?; - - JsonProvider::parse(json_str).await.map_err(|e| { + serde_json::from_slice(&data).map_err(|e| { ExtractorError::sdk_processing_with_source( service, - "Failed to parse embedded service definition", + "Failed to parse service definition", e, ) }) } - /// Get raw waiters data by service name and API version + /// Get waiters data by service name and API version /// /// # Arguments /// * `service` - Service name (e.g., "s3", "ec2", "lambda") /// * `api_version` - API version (e.g., "2006-03-01", "2016-11-15") /// /// # Returns - /// Raw waiters JSON data or None if not found - pub fn get_waiters_raw(service: &str, api_version: &str) -> Option> { - Botocore::get_waiters(service, api_version).map(|data| data.to_vec()) + /// Waiters JSON data or None if not found + pub(crate) fn get_waiters( + service: &str, + api_version: &str, + ) -> Option> { + let waiters_data = BotocoreRaw::get_waiters(service, api_version)?; + + match serde_json::from_slice::( + &waiters_data, + ) { + Ok(waiters_desc) => Some(waiters_desc.waiters), + Err(_) => None, + } } /// Get raw paginators data by service name and API version @@ -307,13 +294,13 @@ impl EmbeddedServiceData { /// # Returns /// Raw paginators JSON data or None if not found #[allow(dead_code)] - pub fn get_paginators_raw(service: &str, api_version: &str) -> Option> { - Botocore::get_paginators(service, api_version).map(|data| data.to_vec()) + pub(crate) fn get_paginators_raw(service: &str, api_version: &str) -> Option> { + BotocoreRaw::get_paginators(service, api_version).map(|data| data.to_vec()) } /// Build a complete service-to-versions map in a single iteration pub(crate) fn build_service_versions_map() -> std::collections::HashMap> { - Botocore::build_service_versions_map() + BotocoreRaw::build_service_versions_map() } } @@ -323,25 +310,25 @@ mod tests { #[test] fn test_botocore_get_service_definition_returns_none_for_invalid_service() { - let result = Botocore::get_service_definition("nonexistent-service", "2023-01-01"); + let result = BotocoreRaw::get_service_definition("nonexistent-service", "2023-01-01"); assert!(result.is_none()); } #[test] fn test_botocore_get_waiters_returns_none_for_invalid_service() { - let result = Botocore::get_waiters("nonexistent-service", "2023-01-01"); + let result = BotocoreRaw::get_waiters("nonexistent-service", "2023-01-01"); assert!(result.is_none()); } #[test] fn test_botocore_get_paginators_returns_none_for_invalid_service() { - let result = Botocore::get_paginators("nonexistent-service", "2023-01-01"); + let result = BotocoreRaw::get_paginators("nonexistent-service", "2023-01-01"); assert!(result.is_none()); } #[test] fn test_build_service_versions_map_returns_hashmap() { - let service_versions = Botocore::build_service_versions_map(); + let service_versions = BotocoreRaw::build_service_versions_map(); // Should return a HashMap assert!(service_versions.is_empty() || !service_versions.is_empty()); @@ -369,8 +356,8 @@ mod tests { #[test] fn test_build_service_versions_map_consistency() { // Call the function twice and ensure results are consistent - let map1 = Botocore::build_service_versions_map(); - let map2 = Botocore::build_service_versions_map(); + let map1 = BotocoreRaw::build_service_versions_map(); + let map2 = BotocoreRaw::build_service_versions_map(); assert_eq!( map1, map2, @@ -380,8 +367,8 @@ mod tests { #[test] fn test_embedded_service_data_build_service_versions_map_delegates() { - let embedded_result = EmbeddedServiceData::build_service_versions_map(); - let botocore_result = Botocore::build_service_versions_map(); + let embedded_result = BotocoreData::build_service_versions_map(); + let botocore_result = BotocoreRaw::build_service_versions_map(); assert_eq!( embedded_result, botocore_result, @@ -389,10 +376,9 @@ mod tests { ); } - #[tokio::test] - async fn test_embedded_service_data_get_service_definition_invalid_service() { - let result = - EmbeddedServiceData::get_service_definition("nonexistent-service", "2023-01-01").await; + #[test] + fn test_embedded_service_data_get_service_definition_invalid_service() { + let result = BotocoreData::get_service_definition("nonexistent-service", "2023-01-01"); assert!( result.is_err(), @@ -411,7 +397,7 @@ mod tests { #[test] fn test_embedded_service_data_get_waiters_raw_invalid_service() { - let result = EmbeddedServiceData::get_waiters_raw("nonexistent-service", "2023-01-01"); + let result = BotocoreData::get_waiters("nonexistent-service", "2023-01-01"); assert!( result.is_none(), "Should return None for nonexistent service" @@ -420,7 +406,7 @@ mod tests { #[test] fn test_embedded_service_data_get_paginators_raw_invalid_service() { - let result = EmbeddedServiceData::get_paginators_raw("nonexistent-service", "2023-01-01"); + let result = BotocoreData::get_paginators_raw("nonexistent-service", "2023-01-01"); assert!( result.is_none(), "Should return None for nonexistent service" @@ -429,7 +415,7 @@ mod tests { #[test] fn test_service_versions_map_structure() { - let service_versions = Botocore::build_service_versions_map(); + let service_versions = BotocoreRaw::build_service_versions_map(); for (service, versions) in &service_versions { // Service names should not contain path separators @@ -486,13 +472,13 @@ mod tests { // This test ensures the timing logic doesn't panic // We can't easily test the actual logging without setting up a logger, // but we can ensure the code path works - let result = Botocore::get_service_definition("nonexistent-service", "2023-01-01"); + let result = BotocoreRaw::get_service_definition("nonexistent-service", "2023-01-01"); assert!(result.is_none()); } #[test] fn test_service_versions_map_no_duplicates() { - let service_versions = Botocore::build_service_versions_map(); + let service_versions = BotocoreRaw::build_service_versions_map(); for (service, versions) in &service_versions { // Check that there are no duplicate versions @@ -512,18 +498,18 @@ mod tests { #[test] fn test_embedded_data_methods_handle_empty_strings() { // Test edge cases with empty strings - let result1 = Botocore::get_service_definition("", ""); - let result2 = Botocore::get_waiters("", ""); - let result3 = Botocore::get_paginators("", ""); + let result1 = BotocoreRaw::get_service_definition("", ""); + let result2 = BotocoreRaw::get_waiters("", ""); + let result3 = BotocoreRaw::get_paginators("", ""); assert!(result1.is_none()); assert!(result2.is_none()); assert!(result3.is_none()); } - #[tokio::test] - async fn test_embedded_service_data_handles_empty_strings() { - let result = EmbeddedServiceData::get_service_definition("", "").await; + #[test] + fn test_embedded_service_data_handles_empty_strings() { + let result = BotocoreData::get_service_definition("", ""); assert!(result.is_err()); } } diff --git a/iam-policy-autopilot-policy-generation/src/extraction/javascript/shared.rs b/iam-policy-autopilot-policy-generation/src/extraction/javascript/shared.rs index 58f0075..ce8e105 100644 --- a/iam-policy-autopilot-policy-generation/src/extraction/javascript/shared.rs +++ b/iam-policy-autopilot-policy-generation/src/extraction/javascript/shared.rs @@ -5,9 +5,26 @@ use crate::extraction::javascript::types::JavaScriptScanResults; use crate::extraction::{Parameter, ParameterValue, SdkMethodCall, SdkMethodCallMetadata}; +use rust_embed::RustEmbed; use serde::Deserialize; use std::collections::HashMap; +/// Embedded JavaScript SDK v3 libraries mapping +/// +/// This struct provides access to the JavaScript SDK v3 libraries mapping configuration +/// that defines how lib-* submodule commands map to client-* commands. +#[derive(RustEmbed)] +#[folder = "resources/config/sdks"] +#[include = "js_v3_libraries.json"] +struct JsV3Libraries; + +impl JsV3Libraries { + /// Get the JavaScript SDK v3 libraries mapping configuration + fn get_libraries_mapping() -> Option> { + Self::get("js_v3_libraries.json").map(|file| file.data) + } +} + /// JSON structure for JS v3 libraries mapping /// /// The AWS SDK fro Javascript defines in aws-sdk-js-v3/lib @@ -24,7 +41,7 @@ struct JsV3LibrariesMapping { /// Load JS v3 libraries mapping from embedded data fn load_libraries_mapping() -> Option { - let content_bytes = crate::embedded_data::JsV3Libraries::get_libraries_mapping()?; + let content_bytes = JsV3Libraries::get_libraries_mapping()?; let content = std::str::from_utf8(&content_bytes).ok()?; diff --git a/iam-policy-autopilot-policy-generation/src/extraction/python/boto3_resources_model.rs b/iam-policy-autopilot-policy-generation/src/extraction/python/boto3_resources_model.rs index 95b57e7..df5789f 100644 --- a/iam-policy-autopilot-policy-generation/src/extraction/python/boto3_resources_model.rs +++ b/iam-policy-autopilot-policy-generation/src/extraction/python/boto3_resources_model.rs @@ -2,7 +2,7 @@ //! //! Parses boto3 resources JSON specifications and utility mappings for resource-based AWS SDK patterns. -use crate::embedded_data::EmbeddedBoto3Data; +use crate::embedded_data::Boto3Data; use convert_case::{Case, Casing}; use serde::Deserialize; use std::collections::HashMap; @@ -22,7 +22,7 @@ pub enum OperationType { /// Extract service names from embedded boto3 utilities mapping fn extract_services_from_embedded_utilities_mapping() -> Result, String> { - let content_bytes = EmbeddedBoto3Data::get_utilities_mapping() + let content_bytes = Boto3Data::get_utilities_mapping() .ok_or_else(|| "Boto3 utilities mapping not found in embedded data".to_string())?; let content = std::str::from_utf8(&content_bytes) @@ -302,7 +302,7 @@ impl Boto3ResourcesModel { /// Loads resource specifications from embedded boto3 data pub fn load_from_embedded(service_name: &str) -> Result { // Get service versions from embedded data - let service_versions = EmbeddedBoto3Data::build_service_versions_map(); + let service_versions = Boto3Data::build_service_versions_map(); // Find the service and get its latest version let versions = service_versions.get(service_name).ok_or_else(|| { @@ -317,7 +317,7 @@ impl Boto3ResourcesModel { .ok_or_else(|| format!("No versions found for service '{}'", service_name))?; // Get the resources data - let resources_data = EmbeddedBoto3Data::get_resources_raw(service_name, latest_version) + let resources_data = Boto3Data::get_resources_raw(service_name, latest_version) .ok_or_else(|| { format!( "Resources data not found for {}/{}", @@ -347,7 +347,7 @@ impl Boto3ResourcesModel { /// Merge utility methods from embedded mapping into model fn merge_utility_methods_from_embedded(model: &mut Boto3ResourcesModel) -> Result<(), String> { - let content_bytes = EmbeddedBoto3Data::get_utilities_mapping() + let content_bytes = Boto3Data::get_utilities_mapping() .ok_or_else(|| "Boto3 utilities mapping not found in embedded data".to_string())?; let content = std::str::from_utf8(&content_bytes) diff --git a/iam-policy-autopilot-policy-generation/src/extraction/sdk_model.rs b/iam-policy-autopilot-policy-generation/src/extraction/sdk_model.rs index 3f3c184..03dab5b 100644 --- a/iam-policy-autopilot-policy-generation/src/extraction/sdk_model.rs +++ b/iam-policy-autopilot-policy-generation/src/extraction/sdk_model.rs @@ -16,7 +16,7 @@ use tokio::task::JoinSet; use convert_case::{Case, Casing}; use serde::{Deserialize, Serialize}; -use crate::embedded_data::EmbeddedServiceData; +use crate::embedded_data::BotocoreData; use crate::errors::{ExtractorError, Result}; use crate::Language; @@ -173,7 +173,7 @@ impl ServiceDiscovery { log::debug!("Starting optimized service discovery..."); // Use the optimized single-iteration approach - let service_versions_map = EmbeddedServiceData::build_service_versions_map(); + let service_versions_map = BotocoreData::build_service_versions_map(); let mut services = Vec::new(); for (service_name, api_versions) in service_versions_map { @@ -326,26 +326,14 @@ impl ServiceDiscovery { })?; // Load service definition from embedded data - let service_definition = EmbeddedServiceData::get_service_definition( + let service_definition = BotocoreData::get_service_definition( &service_info.name, &service_info.api_version, - ) - .await - .map_err(|e| { - ExtractorError::sdk_processing_with_source( - &service_info.name, - "Failed to load embedded service definition", - e, - ) - })?; + )?; // Load waiters from embedded data let waiters = - crate::extraction::waiter_model::WaitersRegistry::load_waiters_from_embedded( - &service_info.name, - &service_info.api_version, - ) - .await; + BotocoreData::get_waiters(&service_info.name, &service_info.api_version); let service_time = service_start.elapsed(); if service_time.as_millis() > 100 { diff --git a/iam-policy-autopilot-policy-generation/src/extraction/waiter_model.rs b/iam-policy-autopilot-policy-generation/src/extraction/waiter_model.rs index d78d6b4..7726051 100644 --- a/iam-policy-autopilot-policy-generation/src/extraction/waiter_model.rs +++ b/iam-policy-autopilot-policy-generation/src/extraction/waiter_model.rs @@ -4,8 +4,6 @@ //! mappings from waiter names to their underlying SDK operations. //! Waiters are available across all AWS SDKs (Python boto3, JavaScript/TypeScript, Go, etc.) -use crate::embedded_data::EmbeddedServiceData; -use crate::providers::JsonProvider; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -26,31 +24,3 @@ pub(crate) struct WaitersDescription { pub(crate) version: u32, pub(crate) waiters: HashMap, } - -/// Registry of all waiters across all AWS services -#[derive(Debug, Clone)] -pub struct WaitersRegistry; - -impl WaitersRegistry { - /// Load waiters from embedded data for a specific service - /// - /// # Arguments - /// * `service_name` - Service name (e.g., "ec2", "s3") - /// * `api_version` - API version (e.g., "2016-11-15", "2006-03-01") - /// - /// # Returns - /// HashMap of waiter names to waiter entries, or None if no waiters found - pub async fn load_waiters_from_embedded( - service_name: &str, - api_version: &str, - ) -> Option> { - let waiters_data = EmbeddedServiceData::get_waiters_raw(service_name, api_version)?; - - let waiters_str = std::str::from_utf8(&waiters_data).ok()?; - - match JsonProvider::parse::(waiters_str).await { - Ok(waiters_desc) => Some(waiters_desc.waiters), - Err(_) => None, // Silently skip on parse error - } - } -}