removes robot tag on every request adds setting for custom response headers

crates/common/src/settings.rs

@@ -5,6 +5,7 @@ use error_stack::{Report, ResultExt};
 use regex::Regex;
 use serde::{de::DeserializeOwned, Deserialize, Deserializer, Serialize};
 use serde_json::Value as JsonValue;
+use std::collections::HashMap;
 use std::sync::OnceLock;
 use url::Url;
 use validator::{Validate, ValidationError};
@@ -134,6 +135,8 @@ pub struct Settings {
     #[serde(default, deserialize_with = "vec_from_seq_or_map")]
     #[validate(nested)]
     pub handlers: Vec<Handler>,
+    #[serde(default)]
+    pub response_headers: HashMap<String, String>,
 }
 
 #[allow(unused)]

crates/fastly/src/main.rs

@@ -84,8 +84,9 @@ async fn route_request(settings: Settings, req: Request) -> Result<Response, Err
     // Convert any errors to HTTP error responses
     let mut response = result.unwrap_or_else(to_error_response);
 
-    // Add X-Robots-Tag header to prevent crawlers and indexers
-    response.set_header("X-Robots-Tag", "noindex, nofollow");
+    for (key, value) in &settings.response_headers {
+        response.set_header(key, value);
+    }
 
     Ok(response)
 }

trusted-server.toml

@@ -28,3 +28,7 @@ secret_key = "trusted-server"
 #   - "publisher_domain"
 #   - "accept_language"
 template = "{{ client_ip }}:{{ user_agent }}:{{ first_party_id }}:{{ auth_user_id }}:{{ publisher_domain }}:{{ accept_language }}"
+
+# Custom headers to be included in every response
+[response_headers]
+X-Custom-Header = "custom header value"