diff --git a/crates/common/README.md b/crates/common/README.md index 35bb785..eb8500f 100644 --- a/crates/common/README.md +++ b/crates/common/README.md @@ -49,10 +49,10 @@ Behavior is covered by an extensive test suite in `crates/common/src/creative.rs ## Synthetic Identifier Propagation -- `synthetic.rs` generates a deterministic synthetic identifier per user request and exposes helpers: - - `generate_synthetic_id` — creates a fresh HMAC-based ID using request signals. - - `get_synthetic_id` — extracts an existing ID from the `x-psid-ts` header or `synthetic_id` cookie. +- `synthetic.rs` generates a synthetic identifier per user request and exposes helpers: + - `generate_synthetic_id` — creates a fresh HMAC-based ID using request signals and appends a short random suffix (format: `64hex.6alnum`). + - `get_synthetic_id` — extracts an existing ID from the `x-synthetic-id` header or `synthetic_id` cookie. - `get_or_generate_synthetic_id` — reuses the existing ID when present, otherwise creates one. -- `publisher.rs::handle_publisher_request` stamps proxied origin responses with `X-Synthetic-Fresh`, `x-psid-ts`, and (when absent) issues the `synthetic_id` cookie so the browser keeps the identifier on subsequent requests. +- `publisher.rs::handle_publisher_request` stamps proxied origin responses with `x-synthetic-id`, and (when absent) issues the `synthetic_id` cookie so the browser keeps the identifier on subsequent requests. - `proxy.rs::handle_first_party_proxy` replays the identifier to third-party creative origins by appending `synthetic_id=` to the reconstructed target URL, follows redirects (301/302/303/307/308) up to four hops, and keeps downstream fetches linked to the same user scope. - `proxy.rs::handle_first_party_click` adds `synthetic_id=` to outbound click redirect URLs so analytics endpoints can associate clicks with impressions without third-party cookies. diff --git a/crates/common/src/constants.rs b/crates/common/src/constants.rs index 775ed11..4ccb77f 100644 --- a/crates/common/src/constants.rs +++ b/crates/common/src/constants.rs @@ -1,9 +1,9 @@ use http::header::HeaderName; -pub const HEADER_SYNTHETIC_FRESH: HeaderName = HeaderName::from_static("x-synthetic-fresh"); -pub const HEADER_SYNTHETIC_PUB_USER_ID: HeaderName = HeaderName::from_static("x-pub-user-id"); +pub const COOKIE_SYNTHETIC_ID: &str = "synthetic_id"; + pub const HEADER_X_PUB_USER_ID: HeaderName = HeaderName::from_static("x-pub-user-id"); -pub const HEADER_SYNTHETIC_TRUSTED_SERVER: HeaderName = HeaderName::from_static("x-psid-ts"); +pub const HEADER_X_SYNTHETIC_ID: HeaderName = HeaderName::from_static("x-synthetic-id"); pub const HEADER_X_CONSENT_ADVERTISING: HeaderName = HeaderName::from_static("x-consent-advertising"); pub const HEADER_X_FORWARDED_FOR: HeaderName = HeaderName::from_static("x-forwarded-for"); diff --git a/crates/common/src/cookies.rs b/crates/common/src/cookies.rs index 48f397b..7511ed2 100644 --- a/crates/common/src/cookies.rs +++ b/crates/common/src/cookies.rs @@ -8,6 +8,7 @@ use error_stack::{Report, ResultExt}; use fastly::http::header; use fastly::Request; +use crate::constants::COOKIE_SYNTHETIC_ID; use crate::error::TrustedServerError; use crate::settings::Settings; @@ -65,8 +66,8 @@ pub fn handle_request_cookies( #[must_use] pub fn create_synthetic_cookie(settings: &Settings, synthetic_id: &str) -> String { format!( - "synthetic_id={}; Domain={}; Path=/; Secure; SameSite=Lax; Max-Age={}", - synthetic_id, settings.publisher.cookie_domain, COOKIE_MAX_AGE, + "{}={}; Domain={}; Path=/; Secure; SameSite=Lax; Max-Age={}", + COOKIE_SYNTHETIC_ID, synthetic_id, settings.publisher.cookie_domain, COOKIE_MAX_AGE, ) } @@ -158,8 +159,8 @@ mod tests { assert_eq!( result, format!( - "synthetic_id=12345; Domain={}; Path=/; Secure; SameSite=Lax; Max-Age={}", - settings.publisher.cookie_domain, COOKIE_MAX_AGE, + "{}=12345; Domain={}; Path=/; Secure; SameSite=Lax; Max-Age={}", + COOKIE_SYNTHETIC_ID, settings.publisher.cookie_domain, COOKIE_MAX_AGE, ) ); } diff --git a/crates/common/src/integrations/prebid.rs b/crates/common/src/integrations/prebid.rs index 7fa6589..c90a561 100644 --- a/crates/common/src/integrations/prebid.rs +++ b/crates/common/src/integrations/prebid.rs @@ -15,7 +15,7 @@ use crate::auction::types::{ AuctionContext, AuctionRequest, AuctionResponse, Bid as AuctionBid, MediaType, }; use crate::backend::ensure_backend_from_url; -use crate::constants::{HEADER_SYNTHETIC_FRESH, HEADER_SYNTHETIC_TRUSTED_SERVER}; +use crate::constants::HEADER_X_SYNTHETIC_ID; use crate::creative; use crate::error::TrustedServerError; use crate::geo::GeoInfo; @@ -29,7 +29,7 @@ use crate::openrtb::{ }; use crate::request_signing::RequestSigner; use crate::settings::{IntegrationConfig, Settings}; -use crate::synthetic::{generate_synthetic_id, get_or_generate_synthetic_id}; +use crate::synthetic::get_or_generate_synthetic_id; const PREBID_INTEGRATION_ID: &str = "prebid"; @@ -384,22 +384,10 @@ async fn handle_prebid_auction( )?; let synthetic_id = get_or_generate_synthetic_id(settings, &req)?; - let fresh_id = generate_synthetic_id(settings, &req)?; - log::info!( - "Using synthetic ID: {}, fresh ID: {}", - synthetic_id, - fresh_id - ); + log::info!("Using synthetic_id: {}", synthetic_id,); - enhance_openrtb_request( - &mut openrtb_request, - &synthetic_id, - &fresh_id, - settings, - &req, - config, - )?; + enhance_openrtb_request(&mut openrtb_request, &synthetic_id, settings, &req, config)?; let mut pbs_req = Request::new( Method::POST, @@ -438,9 +426,7 @@ async fn handle_prebid_auction( Ok(Response::from_status(StatusCode::OK) .with_header(header::CONTENT_TYPE, "application/json") - .with_header("X-Synthetic-ID", &synthetic_id) - .with_header(HEADER_SYNTHETIC_FRESH, &fresh_id) - .with_header(HEADER_SYNTHETIC_TRUSTED_SERVER, &synthetic_id) + .with_header(HEADER_X_SYNTHETIC_ID, &synthetic_id) .with_body(transformed_body)) } Err(_) => Ok(Response::from_status(pbs_response.get_status()) @@ -456,7 +442,6 @@ async fn handle_prebid_auction( fn enhance_openrtb_request( request: &mut Json, synthetic_id: &str, - fresh_id: &str, settings: &Settings, req: &Request, config: &PrebidIntegrationConfig, @@ -469,7 +454,6 @@ fn enhance_openrtb_request( if !request["user"]["ext"].is_object() { request["user"]["ext"] = json!({}); } - request["user"]["ext"]["synthetic_fresh"] = json!(fresh_id); if req.get_header("Sec-GPC").is_some() { if !request["regs"].is_object() { @@ -1262,24 +1246,15 @@ mod tests { }); let synthetic_id = "synthetic-123"; - let fresh_id = "fresh-456"; let mut req = Request::new(Method::POST, "https://edge.example/auction"); req.set_header("Sec-GPC", "1"); let config = base_config(); - enhance_openrtb_request( - &mut request_json, - synthetic_id, - fresh_id, - &settings, - &req, - &config, - ) - .expect("should enhance request"); + enhance_openrtb_request(&mut request_json, synthetic_id, &settings, &req, &config) + .expect("should enhance request"); assert_eq!(request_json["user"]["id"], synthetic_id); - assert_eq!(request_json["user"]["ext"]["synthetic_fresh"], fresh_id); assert_eq!( request_json["regs"]["ext"]["us_privacy"], "1YYN", "GPC header should map to US privacy flag" @@ -1305,21 +1280,13 @@ mod tests { }); let synthetic_id = "synthetic-123"; - let fresh_id = "fresh-456"; let req = Request::new(Method::POST, "https://edge.example/auction"); let mut config = base_config(); config.debug = true; - enhance_openrtb_request( - &mut request_json, - synthetic_id, - fresh_id, - &settings, - &req, - &config, - ) - .expect("should enhance request"); + enhance_openrtb_request(&mut request_json, synthetic_id, &settings, &req, &config) + .expect("should enhance request"); assert_eq!( request_json["ext"]["prebid"]["debug"], true, @@ -1335,21 +1302,13 @@ mod tests { }); let synthetic_id = "synthetic-123"; - let fresh_id = "fresh-456"; let req = Request::new(Method::POST, "https://edge.example/auction"); let mut config = base_config(); config.debug = false; - enhance_openrtb_request( - &mut request_json, - synthetic_id, - fresh_id, - &settings, - &req, - &config, - ) - .expect("should enhance request"); + enhance_openrtb_request(&mut request_json, synthetic_id, &settings, &req, &config) + .expect("should enhance request"); assert!( request_json["ext"]["prebid"]["debug"].is_null(), @@ -1554,7 +1513,6 @@ server_url = "https://prebid.example" let req = Request::new(Method::GET, "https://example.com/test"); let synthetic_id = "test-synthetic-id"; - let fresh_id = "test-fresh-id"; // Test with existing site.page let mut request = json!({ @@ -1565,15 +1523,8 @@ server_url = "https://prebid.example" } }); - enhance_openrtb_request( - &mut request, - synthetic_id, - fresh_id, - &settings, - &req, - &config, - ) - .expect("should enhance request"); + enhance_openrtb_request(&mut request, synthetic_id, &settings, &req, &config) + .expect("should enhance request"); let page = request["site"]["page"].as_str().unwrap(); assert_eq!(page, "https://example.com/page?kargo_debug=true"); @@ -1587,7 +1538,6 @@ server_url = "https://prebid.example" let req = Request::new(Method::GET, "https://example.com/test"); let synthetic_id = "test-synthetic-id"; - let fresh_id = "test-fresh-id"; // Test with existing query params in site.page let mut request = json!({ @@ -1598,15 +1548,8 @@ server_url = "https://prebid.example" } }); - enhance_openrtb_request( - &mut request, - synthetic_id, - fresh_id, - &settings, - &req, - &config, - ) - .expect("should enhance request"); + enhance_openrtb_request(&mut request, synthetic_id, &settings, &req, &config) + .expect("should enhance request"); let page = request["site"]["page"].as_str().unwrap(); assert_eq!( @@ -1624,7 +1567,6 @@ server_url = "https://prebid.example" let req = Request::new(Method::GET, "https://example.com/test"); let synthetic_id = "test-synthetic-id"; - let fresh_id = "test-fresh-id"; // Test with URL that already has the debug params let mut request = json!({ @@ -1635,15 +1577,8 @@ server_url = "https://prebid.example" } }); - enhance_openrtb_request( - &mut request, - synthetic_id, - fresh_id, - &settings, - &req, - &config, - ) - .expect("should enhance request"); + enhance_openrtb_request(&mut request, synthetic_id, &settings, &req, &config) + .expect("should enhance request"); let page = request["site"]["page"].as_str().unwrap(); // Should still only have params once diff --git a/crates/common/src/integrations/prebid.rs.orig b/crates/common/src/integrations/prebid.rs.orig new file mode 100644 index 0000000..18b0890 --- /dev/null +++ b/crates/common/src/integrations/prebid.rs.orig @@ -0,0 +1,1656 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use async_trait::async_trait; +use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; +use error_stack::{Report, ResultExt}; +use fastly::http::{header, Method, StatusCode, Url}; +use fastly::{Request, Response}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value as Json, Value as JsonValue}; +use validator::Validate; + +use crate::auction::provider::AuctionProvider; +use crate::auction::types::{ + AuctionContext, AuctionRequest, AuctionResponse, Bid as AuctionBid, MediaType, +}; +use crate::backend::ensure_backend_from_url; +use crate::constants::HEADER_X_SYNTHETIC_ID; +use crate::creative; +use crate::error::TrustedServerError; +use crate::geo::GeoInfo; +use crate::integrations::{ + AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, + IntegrationEndpoint, IntegrationProxy, IntegrationRegistration, +}; +use crate::openrtb::{ + Banner, Device, Format, Geo, Imp, ImpExt, OpenRtbRequest, PrebidExt, PrebidImpExt, Regs, + RegsExt, RequestExt, Site, TrustedServerExt, User, UserExt, +}; +use crate::request_signing::RequestSigner; +use crate::settings::{IntegrationConfig, Settings}; +use crate::synthetic::get_or_generate_synthetic_id; + +const PREBID_INTEGRATION_ID: &str = "prebid"; + +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +pub struct PrebidIntegrationConfig { + #[serde(default = "default_enabled")] + pub enabled: bool, + pub server_url: String, + #[serde(default = "default_timeout_ms")] + pub timeout_ms: u32, + #[serde( + default = "default_bidders", + deserialize_with = "crate::settings::vec_from_seq_or_map" + )] + pub bidders: Vec, + #[serde(default = "default_auto_configure")] + pub auto_configure: bool, + #[serde(default)] + pub debug: bool, + #[serde(default)] + pub script_handler: Option, + #[serde(default)] + pub debug_query_params: Option, +} + +impl IntegrationConfig for PrebidIntegrationConfig { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +fn default_timeout_ms() -> u32 { + 1000 +} + +fn default_bidders() -> Vec { + vec!["mocktioneer".to_string()] +} + +fn default_auto_configure() -> bool { + true +} + +fn default_enabled() -> bool { + true +} + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct BannerUnit { + sizes: Vec>, +} + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct MediaTypes { + banner: Option, +} + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct AdUnit { + code: String, + media_types: Option, + #[serde(default)] + bids: Option>, +} + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct AdRequest { + ad_units: Vec, + config: Option, +} + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +struct Bid { + bidder: String, + #[serde(default)] + params: JsonValue, +} + +pub struct PrebidIntegration { + config: PrebidIntegrationConfig, +} + +impl PrebidIntegration { + fn new(config: PrebidIntegrationConfig) -> Arc { + Arc::new(Self { config }) + } + + fn error(message: impl Into) -> TrustedServerError { + TrustedServerError::Integration { + integration: PREBID_INTEGRATION_ID.to_string(), + message: message.into(), + } + } + + fn handle_script_handler(&self) -> Result> { + let body = "// Script overridden by Trusted Server\n"; + + Ok(Response::from_status(StatusCode::OK) + .with_header( + header::CONTENT_TYPE, + "application/javascript; charset=utf-8", + ) + .with_header(header::CACHE_CONTROL, "public, max-age=31536000, immutable") + .with_body(body)) + } + + #[allow(dead_code)] + async fn handle_first_party_ad( + &self, + settings: &Settings, + mut req: Request, + ) -> Result> { + let url = req.get_url_str(); + let parsed = Url::parse(url).change_context(TrustedServerError::Prebid { + message: "Invalid first-party serve-ad URL".to_string(), + })?; + let qp = parsed + .query_pairs() + .into_owned() + .collect::>(); + let slot = qp.get("slot").cloned().unwrap_or_default(); + let w = qp + .get("w") + .and_then(|s| s.parse::().ok()) + .unwrap_or(300); + let h = qp + .get("h") + .and_then(|s| s.parse::().ok()) + .unwrap_or(250); + if slot.is_empty() { + return Err(Report::new(TrustedServerError::BadRequest { + message: "missing slot".to_string(), + })); + } + + let ad_req = AdRequest { + ad_units: vec![AdUnit { + code: slot.clone(), + media_types: Some(MediaTypes { + banner: Some(BannerUnit { + sizes: vec![vec![w, h]], + }), + }), + bids: None, + }], + config: None, + }; + + let ortb = build_openrtb_from_ts(&ad_req, settings, &self.config); + req.set_body_json(&ortb) + .change_context(TrustedServerError::Prebid { + message: "Failed to set OpenRTB body".to_string(), + })?; + + let backend_name = ensure_backend_from_url(&self.config.server_url)?; + let mut pbs_resp = req + .send(backend_name) + .change_context(TrustedServerError::Prebid { + message: "Failed to send first-party ad request to Prebid Server".to_string(), + })?; + + let body_bytes = pbs_resp.take_body_bytes(); + let html = match serde_json::from_slice::(&body_bytes) { + Ok(json) => extract_adm_for_slot(&json, &slot) + .unwrap_or_else(|| "".to_string()), + Err(_) => String::from_utf8(body_bytes).unwrap_or_else(|_| String::new()), + }; + + let rewritten = creative::rewrite_creative_html(settings, &html); + + Ok(Response::from_status(StatusCode::OK) + .with_header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .with_body(rewritten)) + } +} + +fn build(settings: &Settings) -> Option> { + let config = settings + .integration_config::(PREBID_INTEGRATION_ID) + .ok() + .flatten()?; + if !config.enabled { + return None; + } + if config.server_url.trim().is_empty() { + log::warn!("Prebid integration disabled: prebid.server_url missing"); + return None; + } + Some(PrebidIntegration::new(config)) +} + +#[must_use] +pub fn register(settings: &Settings) -> Option { + let integration = build(settings)?; + Some( + IntegrationRegistration::builder(PREBID_INTEGRATION_ID) + .with_proxy(integration.clone()) + .with_attribute_rewriter(integration) + .build(), + ) +} + +#[async_trait(?Send)] +impl IntegrationProxy for PrebidIntegration { + fn integration_name(&self) -> &'static str { + PREBID_INTEGRATION_ID + } + + fn routes(&self) -> Vec { + let mut routes = vec![]; + + if let Some(script_path) = &self.config.script_handler { + routes.push(IntegrationEndpoint::get(script_path.clone())); + } + + routes + } + + async fn handle( + &self, + _settings: &Settings, + req: Request, + ) -> Result> { + let path = req.get_path().to_string(); + let method = req.get_method().clone(); + + match method { + Method::GET if self.config.script_handler.as_ref() == Some(&path) => { + self.handle_script_handler() + } + _ => Err(Report::new(Self::error(format!( + "Unsupported Prebid route: {path}" + )))), + } + } +} + +impl IntegrationAttributeRewriter for PrebidIntegration { + fn integration_id(&self) -> &'static str { + PREBID_INTEGRATION_ID + } + + fn handles_attribute(&self, attribute: &str) -> bool { + self.config.auto_configure && matches!(attribute, "src" | "href") + } + + fn rewrite( + &self, + _attr_name: &str, + attr_value: &str, + _ctx: &IntegrationAttributeContext<'_>, + ) -> AttributeRewriteAction { + if self.config.auto_configure && is_prebid_script_url(attr_value) { + AttributeRewriteAction::remove_element() + } else { + AttributeRewriteAction::keep() + } + } +} + +#[allow(dead_code)] +fn build_openrtb_from_ts( + req: &AdRequest, + settings: &Settings, + prebid: &PrebidIntegrationConfig, +) -> OpenRtbRequest { + use uuid::Uuid; + + let imps: Vec = req + .ad_units + .iter() + .map(|unit| { + let formats: Vec = unit + .media_types + .as_ref() + .and_then(|mt| mt.banner.as_ref()) + .map(|b| { + b.sizes + .iter() + .filter(|s| s.len() >= 2) + .map(|s| Format { w: s[0], h: s[1] }) + .collect::>() + }) + .unwrap_or_else(|| vec![Format { w: 300, h: 250 }]); + + let mut bidder: HashMap = HashMap::new(); + if let Some(bids) = &unit.bids { + for bid in bids { + bidder.insert(bid.bidder.clone(), bid.params.clone()); + } + } + if bidder.is_empty() { + for b in &prebid.bidders { + bidder.insert(b.clone(), JsonValue::Object(serde_json::Map::new())); + } + } + + Imp { + id: unit.code.clone(), + banner: Some(Banner { format: formats }), + ext: Some(ImpExt { + prebid: PrebidImpExt { bidder }, + }), + } + }) + .collect(); + + OpenRtbRequest { + id: Uuid::new_v4().to_string(), + imp: imps, + site: Some(Site { + domain: Some(settings.publisher.domain.clone()), + page: Some(format!("https://{}", &settings.publisher.domain)), + }), + user: None, + device: None, + regs: None, + ext: None, + } +} + +fn is_prebid_script_url(url: &str) -> bool { + let lower = url.to_ascii_lowercase(); + let without_query = lower.split('?').next().unwrap_or(""); + let filename = without_query.rsplit('/').next().unwrap_or(""); + matches!( + filename, + "prebid.js" | "prebid.min.js" | "prebidjs.js" | "prebidjs.min.js" + ) +} + +#[allow(dead_code)] +async fn handle_prebid_auction( + settings: &Settings, + mut req: Request, + config: &PrebidIntegrationConfig, +) -> Result> { + log::info!("Handling Prebid auction request"); + let mut openrtb_request: Json = serde_json::from_slice(&req.take_body_bytes()).change_context( + TrustedServerError::Prebid { + message: "Failed to parse OpenRTB request".to_string(), + }, + )?; + + let synthetic_id = get_or_generate_synthetic_id(settings, &req)?; + + log::info!("Using synthetic_id: {}", synthetic_id); + +<<<<<<< HEAD + enhance_openrtb_request(&mut openrtb_request, &synthetic_id, settings, &req)?; +======= + enhance_openrtb_request( + &mut openrtb_request, + &synthetic_id, + &fresh_id, + settings, + &req, + config, + )?; +>>>>>>> main + + let mut pbs_req = Request::new( + Method::POST, + format!("{}/openrtb2/auction", config.server_url), + ); + copy_request_headers(&req, &mut pbs_req); + pbs_req + .set_body_json(&openrtb_request) + .change_context(TrustedServerError::Prebid { + message: "Failed to set request body".to_string(), + })?; + + log::info!("Sending request to Prebid Server"); + + let backend_name = ensure_backend_from_url(&config.server_url)?; + let mut pbs_response = + pbs_req + .send(backend_name) + .change_context(TrustedServerError::Prebid { + message: "Failed to send request to Prebid Server".to_string(), + })?; + + if pbs_response.get_status().is_success() { + let response_body = pbs_response.take_body_bytes(); + match serde_json::from_slice::(&response_body) { + Ok(mut response_json) => { + let request_host = get_request_host(&req); + let request_scheme = get_request_scheme(&req); + transform_prebid_response(&mut response_json, &request_host, &request_scheme)?; + + let transformed_body = serde_json::to_vec(&response_json).change_context( + TrustedServerError::Prebid { + message: "Failed to serialize transformed response".to_string(), + }, + )?; + + Ok(Response::from_status(StatusCode::OK) + .with_header(header::CONTENT_TYPE, "application/json") + .with_header(HEADER_X_SYNTHETIC_ID, &synthetic_id) + .with_body(transformed_body)) + } + Err(_) => Ok(Response::from_status(pbs_response.get_status()) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body(response_body)), + } + } else { + Ok(pbs_response) + } +} + +#[allow(dead_code)] +fn enhance_openrtb_request( + request: &mut Json, + synthetic_id: &str, + settings: &Settings, + req: &Request, + config: &PrebidIntegrationConfig, +) -> Result<(), Report> { + if !request["user"].is_object() { + request["user"] = json!({}); + } + request["user"]["id"] = json!(synthetic_id); + + if !request["user"]["ext"].is_object() { + request["user"]["ext"] = json!({}); + } + + if req.get_header("Sec-GPC").is_some() { + if !request["regs"].is_object() { + request["regs"] = json!({}); + } + if !request["regs"]["ext"].is_object() { + request["regs"]["ext"] = json!({}); + } + request["regs"]["ext"]["us_privacy"] = json!("1YYN"); + } + + if let Some(geo_info) = GeoInfo::from_request(req) { + let geo_obj = json!({ + "type": 2, + "country": geo_info.country, + "city": geo_info.city, + "region": geo_info.region, + }); + + if !request["device"].is_object() { + request["device"] = json!({}); + } + request["device"]["geo"] = geo_obj; + } + + if !request["site"].is_object() { + let mut page_url = format!("https://{}", settings.publisher.domain); + + // Append debug query params if configured + if let Some(ref params) = config.debug_query_params { + page_url = append_query_params(&page_url, params); + } + + request["site"] = json!({ + "domain": settings.publisher.domain, + "page": page_url, + }); + } else if let Some(ref params) = config.debug_query_params { + // If site already exists, append debug params to existing page URL + if let Some(page_url) = request["site"]["page"].as_str() { + let updated_url = append_query_params(page_url, params); + if updated_url != page_url { + request["site"]["page"] = json!(updated_url); + } + } + } + + if let Some(request_signing_config) = &settings.request_signing { + if request_signing_config.enabled && request["id"].is_string() { + if !request["ext"].is_object() { + request["ext"] = json!({}); + } + + let id = request["id"] + .as_str() + .expect("should have string id when is_string checked"); + let signer = RequestSigner::from_config()?; + let signature = signer.sign(id.as_bytes())?; + request["ext"]["trusted_server"] = json!({ + "signature": signature, + "kid": signer.kid + }); + } + } + + if config.debug { + if !request["ext"].is_object() { + request["ext"] = json!({}); + } + if !request["ext"]["prebid"].is_object() { + request["ext"]["prebid"] = json!({}); + } + request["ext"]["prebid"]["debug"] = json!(true); + } + + Ok(()) +} + +fn transform_prebid_response( + response: &mut Json, + request_host: &str, + request_scheme: &str, +) -> Result<(), Report> { + if let Some(seatbids) = response["seatbid"].as_array_mut() { + for seatbid in seatbids { + if let Some(bids) = seatbid["bid"].as_array_mut() { + for bid in bids { + if let Some(adm) = bid["adm"].as_str() { + bid["adm"] = json!(rewrite_ad_markup(adm, request_host, request_scheme)); + } + + if let Some(nurl) = bid["nurl"].as_str() { + bid["nurl"] = json!(make_first_party_proxy_url( + nurl, + request_host, + request_scheme, + "track" + )); + } + + if let Some(burl) = bid["burl"].as_str() { + bid["burl"] = json!(make_first_party_proxy_url( + burl, + request_host, + request_scheme, + "track" + )); + } + } + } + } + } + + Ok(()) +} + +fn rewrite_ad_markup(markup: &str, request_host: &str, request_scheme: &str) -> String { + let mut content = markup.to_string(); + let cdn_patterns = vec![ + ("https://cdn.adsrvr.org", "adsrvr"), + ("https://ib.adnxs.com", "adnxs"), + ("https://rtb.openx.net", "openx"), + ("https://as.casalemedia.com", "casale"), + ("https://eus.rubiconproject.com", "rubicon"), + ]; + + for (cdn_url, cdn_name) in cdn_patterns { + if content.contains(cdn_url) { + let proxy_base = format!( + "{}://{}/ad-proxy/{}", + request_scheme, request_host, cdn_name + ); + content = content.replace(cdn_url, &proxy_base); + } + } + + content = content.replace( + "//cdn.adsrvr.org", + &format!("//{}/ad-proxy/adsrvr", request_host), + ); + content = content.replace( + "//ib.adnxs.com", + &format!("//{}/ad-proxy/adnxs", request_host), + ); + content +} + +fn make_first_party_proxy_url( + third_party_url: &str, + request_host: &str, + request_scheme: &str, + proxy_type: &str, +) -> String { + let encoded = BASE64.encode(third_party_url.as_bytes()); + format!( + "{}://{}/ad-proxy/{}/{}", + request_scheme, request_host, proxy_type, encoded + ) +} + +fn copy_request_headers(from: &Request, to: &mut Request) { + let headers_to_copy = [ + header::COOKIE, + header::USER_AGENT, + header::HeaderName::from_static("x-forwarded-for"), + header::REFERER, + header::ACCEPT_LANGUAGE, + ]; + + for header_name in &headers_to_copy { + if let Some(value) = from.get_header(header_name) { + to.set_header(header_name, value); + } + } +} + +fn get_request_host(req: &Request) -> String { + req.get_header(header::HOST) + .and_then(|h| h.to_str().ok()) + .unwrap_or("") + .to_string() +} + +fn get_request_scheme(req: &Request) -> String { + if req.get_tls_protocol().is_some() || req.get_tls_cipher_openssl_name().is_some() { + return "https".to_string(); + } + + if let Some(proto) = req.get_header("X-Forwarded-Proto") { + if let Ok(proto_str) = proto.to_str() { + return proto_str.to_lowercase(); + } + } + + "https".to_string() +} + +/// Appends query parameters to a URL, handling both URLs with and without existing query strings. +/// Returns the original URL unchanged if params are empty or already present. +fn append_query_params(url: &str, params: &str) -> String { + if params.is_empty() || url.contains(params) { + return url.to_string(); + } + if url.contains('?') { + format!("{}&{}", url, params) + } else { + format!("{}?{}", url, params) + } +} + +/// Extracts the `adm` field from the first bid matching the given slot (by `impid`). +/// +/// Searches through the `OpenRTB` seatbid/bid structure for a bid whose `impid` +/// matches `slot` and returns its `adm` (ad markup) value. +#[allow(dead_code)] +fn extract_adm_for_slot(response: &Json, slot: &str) -> Option { + let seatbids = response.get("seatbid")?.as_array()?; + for seatbid in seatbids { + let bids = seatbid.get("bid")?.as_array()?; + for bid in bids { + if bid.get("impid").and_then(|v| v.as_str()) == Some(slot) { + return bid.get("adm").and_then(|v| v.as_str()).map(String::from); + } + } + } + None +} + +// ============================================================================ +// Prebid Auction Provider +// ============================================================================ + +/// Prebid Server auction provider. +pub struct PrebidAuctionProvider { + config: PrebidIntegrationConfig, +} + +impl PrebidAuctionProvider { + /// Create a new Prebid auction provider. + #[must_use] + pub fn new(config: PrebidIntegrationConfig) -> Self { + Self { config } + } + + /// Convert auction request to `OpenRTB` format with all enrichments. + fn to_openrtb( + &self, + request: &AuctionRequest, + context: &AuctionContext<'_>, + signer: Option<(&RequestSigner, String)>, + ) -> OpenRtbRequest { + let imps: Vec = request + .slots + .iter() + .map(|slot| { + let formats: Vec = slot + .formats + .iter() + .filter(|f| f.media_type == MediaType::Banner) + .map(|f| Format { + w: f.width, + h: f.height, + }) + .collect(); + + // Use bidder params from the slot (passed through from the request) + let mut bidder: HashMap = slot + .bidders + .iter() + .map(|(name, params)| (name.clone(), params.clone())) + .collect(); + + // Fallback to config bidders if none provided + if bidder.is_empty() { + for b in &self.config.bidders { + bidder.insert(b.clone(), Json::Object(serde_json::Map::new())); + } + } + + Imp { + id: slot.id.clone(), + banner: Some(Banner { format: formats }), + ext: Some(ImpExt { + prebid: PrebidImpExt { bidder }, + }), + } + }) + .collect(); + + // Build page URL with debug query params if configured + let page_url = request.publisher.page_url.as_ref().map(|url| { + if let Some(ref params) = self.config.debug_query_params { + append_query_params(url, params) + } else { + url.clone() + } + }); + + // Build user object + let user = Some(User { + id: Some(request.user.id.clone()), + ext: Some(UserExt { + synthetic_fresh: Some(request.user.fresh_id.clone()), + }), + }); + + // Build device object with geo if available + let device = request.device.as_ref().and_then(|d| { + d.geo.as_ref().map(|geo| Device { + geo: Some(Geo { + geo_type: 2, // IP address per OpenRTB spec + country: Some(geo.country.clone()), + city: Some(geo.city.clone()), + region: geo.region.clone(), + }), + }) + }); + + // Build regs object if Sec-GPC header is present + let regs = if context.request.get_header("Sec-GPC").is_some() { + Some(Regs { + ext: Some(RegsExt { + us_privacy: Some("1YYN".to_string()), + }), + }) + } else { + None + }; + + // Build ext object + let request_host = get_request_host(context.request); + let request_scheme = get_request_scheme(context.request); + + let (signature, kid) = signer + .map(|(s, sig)| (Some(sig), Some(s.kid.clone()))) + .unwrap_or((None, None)); + + let ext = Some(RequestExt { + prebid: if self.config.debug { + Some(PrebidExt { debug: Some(true) }) + } else { + None + }, + trusted_server: Some(TrustedServerExt { + signature, + kid, + request_host: Some(request_host), + request_scheme: Some(request_scheme), + }), + }); + + OpenRtbRequest { + id: request.id.clone(), + imp: imps, + site: Some(Site { + domain: Some(request.publisher.domain.clone()), + page: page_url, + }), + user, + device, + regs, + ext, + } + } + + /// Parse `OpenRTB` response into auction response. + fn parse_openrtb_response(&self, json: &Json, response_time_ms: u64) -> AuctionResponse { + let mut bids = Vec::new(); + + if let Some(seatbids) = json.get("seatbid").and_then(|v| v.as_array()) { + for seatbid in seatbids { + let seat = seatbid + .get("seat") + .and_then(|v| v.as_str()) + .unwrap_or("unknown"); + + if let Some(bid_array) = seatbid.get("bid").and_then(|v| v.as_array()) { + for bid_obj in bid_array { + if let Ok(bid) = self.parse_bid(bid_obj, seat) { + bids.push(bid); + } + } + } + } + } + + if bids.is_empty() { + AuctionResponse::no_bid("prebid", response_time_ms) + } else { + AuctionResponse::success("prebid", bids, response_time_ms) + } + } + + /// Parse a single bid from `OpenRTB` response. + fn parse_bid(&self, bid_obj: &Json, seat: &str) -> Result { + let slot_id = bid_obj + .get("impid") + .and_then(|v| v.as_str()) + .ok_or(())? + .to_string(); + + let price = bid_obj + .get("price") + .and_then(serde_json::Value::as_f64) + .ok_or(())?; + + let creative = bid_obj + .get("adm") + .and_then(|v| v.as_str()) + .map(std::string::ToString::to_string); + + let width = bid_obj + .get("w") + .and_then(serde_json::Value::as_u64) + .unwrap_or(300) as u32; + let height = bid_obj + .get("h") + .and_then(serde_json::Value::as_u64) + .unwrap_or(250) as u32; + + let nurl = bid_obj + .get("nurl") + .and_then(|v| v.as_str()) + .map(std::string::ToString::to_string); + + let burl = bid_obj + .get("burl") + .and_then(|v| v.as_str()) + .map(std::string::ToString::to_string); + + let adomain = bid_obj + .get("adomain") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str().map(std::string::ToString::to_string)) + .collect() + }); + + Ok(AuctionBid { + slot_id, + price: Some(price), // Prebid provides decoded prices + currency: "USD".to_string(), + creative, + adomain, + bidder: seat.to_string(), + width, + height, + nurl, + burl, + metadata: std::collections::HashMap::new(), + }) + } +} + +impl AuctionProvider for PrebidAuctionProvider { + fn provider_name(&self) -> &'static str { + "prebid" + } + + fn request_bids( + &self, + request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + log::info!("Prebid: requesting bids for {} slots", request.slots.len()); + + // Create signer and compute signature if request signing is enabled + let signer_with_signature = + if let Some(request_signing_config) = &context.settings.request_signing { + if request_signing_config.enabled { + let signer = RequestSigner::from_config()?; + let signature = signer.sign(request.id.as_bytes())?; + Some((signer, signature)) + } else { + None + } + } else { + None + }; + + // Convert to OpenRTB with all enrichments + let openrtb = self.to_openrtb( + request, + context, + signer_with_signature + .as_ref() + .map(|(s, sig)| (s, sig.clone())), + ); + + // Create HTTP request + let mut pbs_req = Request::new( + Method::POST, + format!("{}/openrtb2/auction", self.config.server_url), + ); + copy_request_headers(context.request, &mut pbs_req); + + pbs_req + .set_body_json(&openrtb) + .change_context(TrustedServerError::Prebid { + message: "Failed to set request body".to_string(), + })?; + + // Send request asynchronously + let backend_name = ensure_backend_from_url(&self.config.server_url)?; + let pending = + pbs_req + .send_async(backend_name) + .change_context(TrustedServerError::Prebid { + message: "Failed to send async request to Prebid Server".to_string(), + })?; + + Ok(pending) + } + + fn parse_response( + &self, + mut response: fastly::Response, + response_time_ms: u64, + ) -> Result> { + // Parse response + if !response.get_status().is_success() { + log::warn!( + "Prebid returned non-success status: {}", + response.get_status() + ); + return Ok(AuctionResponse::error("prebid", response_time_ms)); + } + + let body_bytes = response.take_body_bytes(); + + let mut response_json: Json = + serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Prebid { + message: "Failed to parse Prebid response".to_string(), + })?; + + let request_host = response_json + .get("ext") + .and_then(|ext| ext.get("trusted_server")) + .and_then(|trusted_server| trusted_server.get("request_host")) + .and_then(|value| value.as_str()) + .unwrap_or("") + .to_string(); + let request_scheme = response_json + .get("ext") + .and_then(|ext| ext.get("trusted_server")) + .and_then(|trusted_server| trusted_server.get("request_scheme")) + .and_then(|value| value.as_str()) + .unwrap_or("https") + .to_string(); + + if request_host.is_empty() { + log::warn!("Prebid response missing request host; skipping URL rewrites"); + } else { + transform_prebid_response(&mut response_json, &request_host, &request_scheme)?; + } + + let auction_response = self.parse_openrtb_response(&response_json, response_time_ms); + + log::info!( + "Prebid returned {} bids in {}ms", + auction_response.bids.len(), + response_time_ms + ); + + Ok(auction_response) + } + + fn supports_media_type(&self, media_type: &MediaType) -> bool { + matches!(media_type, MediaType::Banner) + } + + fn timeout_ms(&self) -> u32 { + self.config.timeout_ms + } + + fn is_enabled(&self) -> bool { + self.config.enabled + } + + fn backend_name(&self) -> Option { + ensure_backend_from_url(&self.config.server_url).ok() + } +} + +// ============================================================================ +// Provider Auto-Registration +// ============================================================================ + +/// Auto-register Prebid provider based on settings configuration. +/// +/// This function checks the settings for Prebid configuration and returns +/// the provider if enabled. +#[must_use] +pub fn register_auction_provider(settings: &Settings) -> Vec> { + let mut providers: Vec> = Vec::new(); + + // Prebid provider is always registered if integration is enabled + if let Ok(Some(config)) = settings.integration_config::("prebid") { + log::info!("Registering Prebid auction provider"); + providers.push(Arc::new(PrebidAuctionProvider::new(config))); + } + + providers +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::html_processor::{create_html_processor, HtmlProcessorConfig}; + use crate::integrations::{AttributeRewriteAction, IntegrationRegistry}; + use crate::settings::Settings; + use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; + use crate::test_support::tests::crate_test_settings_str; + use fastly::http::Method; + use serde_json::json; + use std::io::Cursor; + + fn make_settings() -> Settings { + Settings::from_toml(&crate_test_settings_str()).expect("should parse settings") + } + + fn base_config() -> PrebidIntegrationConfig { + PrebidIntegrationConfig { + enabled: true, + server_url: "https://prebid.example".to_string(), + timeout_ms: 1000, + bidders: vec!["exampleBidder".to_string()], + auto_configure: true, + debug: false, + script_handler: None, + debug_query_params: None, + } + } + + fn config_from_settings( + settings: &Settings, + registry: &IntegrationRegistry, + ) -> HtmlProcessorConfig { + HtmlProcessorConfig::from_settings( + settings, + registry, + "origin.example.com", + "test.example.com", + "https", + ) + } + + #[test] + fn attribute_rewriter_removes_prebid_scripts() { + let integration = PrebidIntegration { + config: base_config(), + }; + let ctx = IntegrationAttributeContext { + attribute_name: "src", + request_host: "pub.example", + request_scheme: "https", + origin_host: "origin.example", + }; + + let rewritten = integration.rewrite("src", "https://cdn.prebid.org/prebid.min.js", &ctx); + assert!(matches!(rewritten, AttributeRewriteAction::RemoveElement)); + + let untouched = integration.rewrite("src", "https://cdn.example.com/app.js", &ctx); + assert!(matches!(untouched, AttributeRewriteAction::Keep)); + } + + #[test] + fn attribute_rewriter_handles_query_strings_and_links() { + let integration = PrebidIntegration { + config: base_config(), + }; + let ctx = IntegrationAttributeContext { + attribute_name: "href", + request_host: "pub.example", + request_scheme: "https", + origin_host: "origin.example", + }; + + let rewritten = + integration.rewrite("href", "https://cdn.prebid.org/prebid.js?v=1.2.3", &ctx); + assert!(matches!(rewritten, AttributeRewriteAction::RemoveElement)); + } + + #[test] + fn html_processor_keeps_prebid_scripts_when_auto_config_disabled() { + let html = r#" + + + "#; + + let mut settings = make_settings(); + settings + .integrations + .insert_config( + "prebid", + &json!({ + "enabled": true, + "server_url": "https://test-prebid.com/openrtb2/auction", + "timeout_ms": 1000, + "bidders": ["mocktioneer"], + "auto_configure": false, + "debug": false + }), + ) + .expect("should update prebid config"); + let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let config = config_from_settings(&settings, ®istry); + let processor = create_html_processor(config); + let pipeline_config = PipelineConfig { + input_compression: Compression::None, + output_compression: Compression::None, + chunk_size: 8192, + }; + let mut pipeline = StreamingPipeline::new(pipeline_config, processor); + + let mut output = Vec::new(); + let result = pipeline.process(Cursor::new(html.as_bytes()), &mut output); + assert!(result.is_ok()); + let processed = String::from_utf8_lossy(&output); + assert!( + processed.contains("tsjs-unified"), + "Unified bundle should be injected" + ); + assert!( + processed.contains("prebid.min.js"), + "Prebid script should remain when auto-config is disabled" + ); + assert!( + processed.contains("cdn.prebid.org/prebid.js"), + "Prebid preload should remain when auto-config is disabled" + ); + } + + #[test] + fn html_processor_removes_prebid_scripts_when_auto_config_enabled() { + let html = r#" + + + "#; + + let mut settings = make_settings(); + settings + .integrations + .insert_config( + "prebid", + &json!({ + "enabled": true, + "server_url": "https://test-prebid.com/openrtb2/auction", + "timeout_ms": 1000, + "bidders": ["mocktioneer"], + "auto_configure": true, + "debug": false + }), + ) + .expect("should update prebid config"); + let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let config = config_from_settings(&settings, ®istry); + let processor = create_html_processor(config); + let pipeline_config = PipelineConfig { + input_compression: Compression::None, + output_compression: Compression::None, + chunk_size: 8192, + }; + let mut pipeline = StreamingPipeline::new(pipeline_config, processor); + + let mut output = Vec::new(); + let result = pipeline.process(Cursor::new(html.as_bytes()), &mut output); + assert!(result.is_ok()); + let processed = String::from_utf8_lossy(&output); + assert!( + processed.contains("tsjs-unified"), + "Unified bundle should be injected" + ); + assert!( + !processed.contains("prebid.min.js"), + "Prebid script should be removed when auto-config is enabled" + ); + assert!( + !processed.contains("cdn.prebid.org/prebid.js"), + "Prebid preload should be removed when auto-config is enabled" + ); + } + + #[test] + fn enhance_openrtb_request_adds_ids_and_regs() { + let settings = make_settings(); + let mut request_json = json!({ + "id": "openrtb-request-id" + }); + + let synthetic_id = "synthetic-123"; +<<<<<<< HEAD + let mut req = Request::new(Method::POST, "https://edge.example/third-party/ad"); + req.set_header("Sec-GPC", "1"); + + enhance_openrtb_request(&mut request_json, synthetic_id, &settings, &req) + .expect("should enhance request"); +======= + let fresh_id = "fresh-456"; + let mut req = Request::new(Method::POST, "https://edge.example/auction"); + req.set_header("Sec-GPC", "1"); + + let config = base_config(); + + enhance_openrtb_request( + &mut request_json, + synthetic_id, + fresh_id, + &settings, + &req, + &config, + ) + .expect("should enhance request"); +>>>>>>> main + + assert_eq!(request_json["user"]["id"], synthetic_id); + assert_eq!( + request_json["regs"]["ext"]["us_privacy"], "1YYN", + "GPC header should map to US privacy flag" + ); + assert_eq!( + request_json["site"]["domain"], settings.publisher.domain, + "site domain should match publisher domain" + ); + assert!( + request_json["site"]["page"] + .as_str() + .unwrap() + .starts_with("https://"), + "site page should be populated" + ); + } + + #[test] + fn enhance_openrtb_request_adds_debug_flag_when_enabled() { + let settings = make_settings(); + let mut request_json = json!({ + "id": "openrtb-request-id" + }); + + let synthetic_id = "synthetic-123"; + let fresh_id = "fresh-456"; + let req = Request::new(Method::POST, "https://edge.example/auction"); + + let mut config = base_config(); + config.debug = true; + + enhance_openrtb_request( + &mut request_json, + synthetic_id, + fresh_id, + &settings, + &req, + &config, + ) + .expect("should enhance request"); + + assert_eq!( + request_json["ext"]["prebid"]["debug"], true, + "debug flag should be set to true when config.debug is enabled" + ); + } + + #[test] + fn enhance_openrtb_request_does_not_add_debug_flag_when_disabled() { + let settings = make_settings(); + let mut request_json = json!({ + "id": "openrtb-request-id" + }); + + let synthetic_id = "synthetic-123"; + let fresh_id = "fresh-456"; + let req = Request::new(Method::POST, "https://edge.example/auction"); + + let mut config = base_config(); + config.debug = false; + + enhance_openrtb_request( + &mut request_json, + synthetic_id, + fresh_id, + &settings, + &req, + &config, + ) + .expect("should enhance request"); + + assert!( + request_json["ext"]["prebid"]["debug"].is_null(), + "debug flag should not be set when config.debug is disabled" + ); + } + + #[test] + fn transform_prebid_response_rewrites_creatives_and_tracking() { + let mut response = json!({ + "seatbid": [{ + "bid": [{ + "adm": r#""#, + "nurl": "https://notify.example/win", + "burl": "https://notify.example/bill" + }] + }] + }); + + transform_prebid_response(&mut response, "pub.example", "https") + .expect("should rewrite response"); + + let rewritten_adm = response["seatbid"][0]["bid"][0]["adm"] + .as_str() + .expect("adm should be string"); + assert!( + rewritten_adm.contains("/ad-proxy/adsrvr"), + "creative markup should proxy CDN urls" + ); + + for url_field in ["nurl", "burl"] { + let value = response["seatbid"][0]["bid"][0][url_field] + .as_str() + .unwrap(); + assert!( + value.contains("/ad-proxy/track/"), + "tracking URLs should be proxied" + ); + } + } + + #[test] + fn make_first_party_proxy_url_base64_encodes_target() { + let url = "https://cdn.example/path?x=1"; + let rewritten = make_first_party_proxy_url(url, "pub.example", "https", "track"); + assert!( + rewritten.starts_with("https://pub.example/ad-proxy/track/"), + "proxy prefix should be applied" + ); + + let encoded = rewritten.split("/ad-proxy/track/").nth(1).unwrap(); + let decoded = BASE64 + .decode(encoded.as_bytes()) + .expect("should decode base64 proxy payload"); + assert_eq!(String::from_utf8(decoded).unwrap(), url); + } + + #[test] + fn is_prebid_script_url_matches_common_variants() { + assert!(is_prebid_script_url("https://cdn.com/prebid.js")); + assert!(is_prebid_script_url( + "https://cdn.com/prebid.min.js?version=1" + )); + assert!(!is_prebid_script_url("https://cdn.com/app.js")); + } + + #[test] + fn test_script_handler_config_parsing() { + let toml_str = r#" +[publisher] +domain = "test-publisher.com" +cookie_domain = ".test-publisher.com" +origin_url = "https://origin.test-publisher.com" +proxy_secret = "test-secret" + +[synthetic] +counter_store = "test-counter-store" +opid_store = "test-opid-store" +secret_key = "test-secret-key" +template = "{{client_ip}}:{{user_agent}}" + +[integrations.prebid] +enabled = true +server_url = "https://prebid.example" +script_handler = "/prebid.js" +"#; + + let settings = Settings::from_toml(toml_str).expect("should parse TOML"); + let config = settings + .integration_config::("prebid") + .expect("should get config") + .expect("should be enabled"); + + assert_eq!(config.script_handler, Some("/prebid.js".to_string())); + } + + #[test] + fn test_script_handler_none_by_default() { + let toml_str = r#" +[publisher] +domain = "test-publisher.com" +cookie_domain = ".test-publisher.com" +origin_url = "https://origin.test-publisher.com" +proxy_secret = "test-secret" + +[synthetic] +counter_store = "test-counter-store" +opid_store = "test-opid-store" +secret_key = "test-secret-key" +template = "{{client_ip}}:{{user_agent}}" + +[integrations.prebid] +enabled = true +server_url = "https://prebid.example" +"#; + + let settings = Settings::from_toml(toml_str).expect("should parse TOML"); + let config = settings + .integration_config::("prebid") + .expect("should get config") + .expect("should be enabled"); + + assert_eq!(config.script_handler, None); + } + + #[test] + fn test_script_handler_returns_empty_js() { + let config = PrebidIntegrationConfig { + enabled: true, + server_url: "https://prebid.example".to_string(), + timeout_ms: 1000, + bidders: vec![], + auto_configure: false, + debug: false, + script_handler: Some("/prebid.js".to_string()), + debug_query_params: None, + }; + let integration = PrebidIntegration::new(config); + + let response = integration + .handle_script_handler() + .expect("should return response"); + + assert_eq!(response.get_status(), StatusCode::OK); + + let content_type = response + .get_header_str(header::CONTENT_TYPE) + .expect("should have content-type"); + assert_eq!(content_type, "application/javascript; charset=utf-8"); + + let cache_control = response + .get_header_str(header::CACHE_CONTROL) + .expect("should have cache-control"); + assert!(cache_control.contains("max-age=31536000")); + assert!(cache_control.contains("immutable")); + + let body = response.into_body_str(); + assert!(body.contains("// Script overridden by Trusted Server")); + } + + #[test] + fn test_routes_includes_script_handler() { + let config = PrebidIntegrationConfig { + enabled: true, + server_url: "https://prebid.example".to_string(), + timeout_ms: 1000, + bidders: vec![], + auto_configure: false, + debug: false, + script_handler: Some("/prebid.js".to_string()), + debug_query_params: None, + }; + let integration = PrebidIntegration::new(config); + + let routes = integration.routes(); + + // Should have 1 route: script handler + assert_eq!(routes.len(), 1); + + let has_script_route = routes + .iter() + .any(|r| r.path == "/prebid.js" && r.method == Method::GET); + assert!(has_script_route, "should register script handler route"); + } + + #[test] + fn test_routes_without_script_handler() { + let config = base_config(); // Has script_handler: None + let integration = PrebidIntegration::new(config); + + let routes = integration.routes(); + + // Should have 0 routes when no script handler configured + assert_eq!(routes.len(), 0); + } + + #[test] + fn debug_query_params_appended_to_existing_site_page_in_enhance() { + let settings = make_settings(); + let mut config = base_config(); + config.debug_query_params = Some("kargo_debug=true".to_string()); + + let req = Request::new(Method::GET, "https://example.com/test"); + let synthetic_id = "test-synthetic-id"; + let fresh_id = "test-fresh-id"; + + // Test with existing site.page + let mut request = json!({ + "id": "test-id", + "site": { + "domain": "example.com", + "page": "https://example.com/page" + } + }); + + enhance_openrtb_request( + &mut request, + synthetic_id, + fresh_id, + &settings, + &req, + &config, + ) + .expect("should enhance request"); + + let page = request["site"]["page"].as_str().unwrap(); + assert_eq!(page, "https://example.com/page?kargo_debug=true"); + } + + #[test] + fn debug_query_params_appended_to_url_with_existing_query() { + let settings = make_settings(); + let mut config = base_config(); + config.debug_query_params = Some("kargo_debug=true".to_string()); + + let req = Request::new(Method::GET, "https://example.com/test"); + let synthetic_id = "test-synthetic-id"; + let fresh_id = "test-fresh-id"; + + // Test with existing query params in site.page + let mut request = json!({ + "id": "test-id", + "site": { + "domain": "example.com", + "page": "https://example.com/page?existing=param" + } + }); + + enhance_openrtb_request( + &mut request, + synthetic_id, + fresh_id, + &settings, + &req, + &config, + ) + .expect("should enhance request"); + + let page = request["site"]["page"].as_str().unwrap(); + assert_eq!( + page, + "https://example.com/page?existing=param&kargo_debug=true" + ); + } + + #[test] + fn debug_query_params_not_duplicated() { + // Verify that if params are already in the URL, they aren't added again + let settings = make_settings(); + let mut config = base_config(); + config.debug_query_params = Some("kargo_debug=true".to_string()); + + let req = Request::new(Method::GET, "https://example.com/test"); + let synthetic_id = "test-synthetic-id"; + let fresh_id = "test-fresh-id"; + + // Test with URL that already has the debug params + let mut request = json!({ + "id": "test-id", + "site": { + "domain": "example.com", + "page": "https://example.com/page?kargo_debug=true" + } + }); + + enhance_openrtb_request( + &mut request, + synthetic_id, + fresh_id, + &settings, + &req, + &config, + ) + .expect("should enhance request"); + + let page = request["site"]["page"].as_str().unwrap(); + // Should still only have params once + assert_eq!(page, "https://example.com/page?kargo_debug=true"); + // Verify params appear exactly once + assert_eq!(page.matches("kargo_debug=true").count(), 1); + } +} diff --git a/crates/common/src/integrations/testlight.rs b/crates/common/src/integrations/testlight.rs index d3d48af..b6883ac 100644 --- a/crates/common/src/integrations/testlight.rs +++ b/crates/common/src/integrations/testlight.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use validator::Validate; -use crate::constants::{HEADER_SYNTHETIC_FRESH, HEADER_SYNTHETIC_TRUSTED_SERVER}; +use crate::constants::HEADER_X_SYNTHETIC_ID; use crate::error::TrustedServerError; use crate::integrations::{ AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, @@ -16,7 +16,7 @@ use crate::integrations::{ }; use crate::proxy::{proxy_request, ProxyRequestConfig}; use crate::settings::{IntegrationConfig, Settings}; -use crate::synthetic::{generate_synthetic_id, get_or_generate_synthetic_id}; +use crate::synthetic::get_or_generate_synthetic_id; use crate::tsjs; const TESTLIGHT_INTEGRATION_ID: &str = "testlight"; @@ -142,8 +142,6 @@ impl IntegrationProxy for TestlightIntegration { let synthetic_id = get_or_generate_synthetic_id(settings, &req) .change_context(Self::error("Failed to fetch or mint synthetic ID"))?; - let fresh_id = generate_synthetic_id(settings, &req) - .change_context(Self::error("Failed to mint fresh synthetic ID"))?; payload.user.id = Some(synthetic_id.clone()); @@ -177,8 +175,7 @@ impl IntegrationProxy for TestlightIntegration { } } - response.set_header(HEADER_SYNTHETIC_TRUSTED_SERVER, &synthetic_id); - response.set_header(HEADER_SYNTHETIC_FRESH, &fresh_id); + response.set_header(HEADER_X_SYNTHETIC_ID, &synthetic_id); Ok(response) } } @@ -221,7 +218,7 @@ fn default_shim_src() -> String { } fn default_enabled() -> bool { - true + false } impl Default for TestlightRequestBody { diff --git a/crates/common/src/proxy.rs b/crates/common/src/proxy.rs index e393211..6ae4985 100644 --- a/crates/common/src/proxy.rs +++ b/crates/common/src/proxy.rs @@ -1279,10 +1279,7 @@ mod tests { sig ), ); - req.set_header( - crate::constants::HEADER_SYNTHETIC_TRUSTED_SERVER, - "synthetic-123", - ); + req.set_header(crate::constants::HEADER_X_SYNTHETIC_ID, "synthetic-123"); let resp = handle_first_party_click(&settings, req) .await diff --git a/crates/common/src/publisher.rs b/crates/common/src/publisher.rs index 58907cc..c919750 100644 --- a/crates/common/src/publisher.rs +++ b/crates/common/src/publisher.rs @@ -5,7 +5,7 @@ use fastly::{Body, Request, Response}; use crate::backend::ensure_backend_from_url; use crate::http_util::serve_static_with_etag; -use crate::constants::{HEADER_SYNTHETIC_TRUSTED_SERVER, HEADER_X_COMPRESS_HINT}; +use crate::constants::{COOKIE_SYNTHETIC_ID, HEADER_X_COMPRESS_HINT, HEADER_X_SYNTHETIC_ID}; use crate::cookies::create_synthetic_cookie; use crate::error::TrustedServerError; use crate::integrations::IntegrationRegistry; @@ -270,9 +270,11 @@ pub fn handle_publisher_request( .get_header(header::COOKIE) .and_then(|h| h.to_str().ok()) .map(|cookies| { - cookies - .split(';') - .any(|cookie| cookie.trim_start().starts_with("synthetic_id=")) + cookies.split(';').any(|cookie| { + cookie + .trim_start() + .starts_with(&format!("{}=", COOKIE_SYNTHETIC_ID)) + }) }) .unwrap_or(false); @@ -373,7 +375,7 @@ pub fn handle_publisher_request( ); } - response.set_header(HEADER_SYNTHETIC_TRUSTED_SERVER, synthetic_id.as_str()); + response.set_header(HEADER_X_SYNTHETIC_ID, synthetic_id.as_str()); if !has_synthetic_cookie { response.set_header( header::SET_COOKIE, diff --git a/crates/common/src/synthetic.rs b/crates/common/src/synthetic.rs index 5ae78aa..abbc6b0 100644 --- a/crates/common/src/synthetic.rs +++ b/crates/common/src/synthetic.rs @@ -3,25 +3,62 @@ //! This module provides functionality for generating privacy-preserving synthetic IDs //! based on various request parameters and a secret key. +use std::net::IpAddr; + use error_stack::{Report, ResultExt}; use fastly::http::header; use fastly::Request; use handlebars::Handlebars; use hmac::{Hmac, Mac}; +use rand::Rng; use serde_json::json; use sha2::Sha256; +use uuid::Uuid; -use crate::constants::{HEADER_SYNTHETIC_PUB_USER_ID, HEADER_SYNTHETIC_TRUSTED_SERVER}; +use crate::constants::{COOKIE_SYNTHETIC_ID, HEADER_X_SYNTHETIC_ID}; use crate::cookies::handle_request_cookies; use crate::error::TrustedServerError; use crate::settings::Settings; type HmacSha256 = Hmac; +const ALPHANUMERIC_CHARSET: &[u8] = + b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + +/// Normalizes an IP address for stable synthetic ID generation. +/// +/// For IPv6 addresses, masks to /64 prefix to handle Privacy Extensions +/// where devices rotate their interface identifier (lower 64 bits). +/// IPv4 addresses are returned unchanged. +fn normalize_ip(ip: IpAddr) -> String { + match ip { + IpAddr::V4(ipv4) => ipv4.to_string(), + IpAddr::V6(ipv6) => { + let segments = ipv6.segments(); + // Keep only the first 4 segments (64 bits) for /64 prefix + format!( + "{:x}:{:x}:{:x}:{:x}::", + segments[0], segments[1], segments[2], segments[3] + ) + } + } +} + +/// Generates a random alphanumeric string of the specified length. +fn generate_random_suffix(length: usize) -> String { + let mut rng = rand::thread_rng(); + (0..length) + .map(|_| { + let idx = rng.gen_range(0..ALPHANUMERIC_CHARSET.len()); + ALPHANUMERIC_CHARSET[idx] as char + }) + .collect() +} + /// Generates a fresh synthetic ID based on request parameters. /// -/// Creates a deterministic ID using HMAC-SHA256 with the configured secret key -/// and various request attributes including IP, user agent, cookies, and headers. +/// Creates an HMAC-SHA256-based ID using the configured secret key and request +/// attributes, then appends a random suffix for additional uniqueness. /// /// # Errors /// @@ -31,33 +68,26 @@ pub fn generate_synthetic_id( settings: &Settings, req: &Request, ) -> Result> { + let client_ip = req.get_client_ip_addr().map(normalize_ip); let user_agent = req .get_header(header::USER_AGENT) .map(|h| h.to_str().unwrap_or("unknown")); - let first_party_id = handle_request_cookies(req).ok().flatten().and_then(|jar| { - jar.get("pub_userid") - .map(|cookie| cookie.value().to_string()) - }); - let auth_user_id = req - .get_header(HEADER_SYNTHETIC_PUB_USER_ID) - .map(|h| h.to_str().unwrap_or("anonymous")); - let publisher_domain = req - .get_header(header::HOST) - .map(|h| h.to_str().unwrap_or("unknown")); - let client_ip = req.get_client_ip_addr().map(|ip| ip.to_string()); let accept_language = req .get_header(header::ACCEPT_LANGUAGE) .and_then(|h| h.to_str().ok()) .map(|lang| lang.split(',').next().unwrap_or("unknown")); + let accept_encoding = req + .get_header(header::ACCEPT_ENCODING) + .and_then(|h| h.to_str().ok()); + let random_uuid = Uuid::new_v4().to_string(); let handlebars = Handlebars::new(); let data = &json!({ "client_ip": client_ip.unwrap_or("unknown".to_string()), "user_agent": user_agent.unwrap_or("unknown"), - "first_party_id": first_party_id.unwrap_or("anonymous".to_string()), - "auth_user_id": auth_user_id.unwrap_or("anonymous"), - "publisher_domain": publisher_domain.unwrap_or("unknown.com"), - "accept_language": accept_language.unwrap_or("unknown") + "accept_language": accept_language.unwrap_or("unknown"), + "accept_encoding": accept_encoding.unwrap_or("unknown"), + "random_uuid": random_uuid }); let input_string = handlebars @@ -73,17 +103,21 @@ pub fn generate_synthetic_id( message: "Failed to create HMAC instance".to_string(), })?; mac.update(input_string.as_bytes()); - let fresh_id = hex::encode(mac.finalize().into_bytes()); + let hmac_hash = hex::encode(mac.finalize().into_bytes()); - log::info!("Generated fresh ID: {}", fresh_id); + // Append random 6-character alphanumeric suffix for additional uniqueness + let random_suffix = generate_random_suffix(6); + let synthetic_id = format!("{}.{}", hmac_hash, random_suffix); - Ok(fresh_id) + log::info!("Generated fresh ID: {}", synthetic_id); + + Ok(synthetic_id) } /// Gets or creates a synthetic ID from the request. /// /// Attempts to retrieve an existing synthetic ID from: -/// 1. The `x-psid-ts` header +/// 1. The `x-synthetic-id` header /// 2. The `synthetic_id` cookie /// /// If neither exists, generates a new synthetic ID. @@ -94,7 +128,7 @@ pub fn generate_synthetic_id( /// - [`TrustedServerError::SyntheticId`] if ID generation fails pub fn get_synthetic_id(req: &Request) -> Result, Report> { if let Some(synthetic_id) = req - .get_header(HEADER_SYNTHETIC_TRUSTED_SERVER) + .get_header(HEADER_X_SYNTHETIC_ID) .and_then(|h| h.to_str().ok()) { let id = synthetic_id.to_string(); @@ -104,7 +138,7 @@ pub fn get_synthetic_id(req: &Request) -> Result, Report { - if let Some(cookie) = jar.get("synthetic_id") { + if let Some(cookie) = jar.get(COOKIE_SYNTHETIC_ID) { let id = cookie.value().to_string(); log::info!("Using existing Trusted Server ID from cookie: {}", id); return Ok(Some(id)); @@ -138,22 +172,49 @@ pub fn get_or_generate_synthetic_id( } // If no existing Synthetic ID found, generate a fresh one - let fresh_id = generate_synthetic_id(settings, req)?; - log::info!( - "No existing Synthetic ID found, using fresh ID: {}", - fresh_id - ); - Ok(fresh_id) + let synthetic_id = generate_synthetic_id(settings, req)?; + log::info!("No existing synthetic_id, generated: {}", synthetic_id); + Ok(synthetic_id) } #[cfg(test)] mod tests { use super::*; use fastly::http::{HeaderName, HeaderValue}; + use std::net::{Ipv4Addr, Ipv6Addr}; - use crate::constants::HEADER_X_PUB_USER_ID; use crate::test_support::tests::create_test_settings; + #[test] + fn test_normalize_ip_ipv4_unchanged() { + let ipv4 = IpAddr::V4(Ipv4Addr::new(192, 168, 1, 100)); + assert_eq!(normalize_ip(ipv4), "192.168.1.100"); + } + + #[test] + fn test_normalize_ip_ipv6_masks_to_64() { + // Full IPv6 address with interface identifier + let ipv6 = IpAddr::V6(Ipv6Addr::new( + 0x2001, 0x0db8, 0x85a3, 0x0000, 0x8a2e, 0x0370, 0x7334, 0x1234, + )); + assert_eq!(normalize_ip(ipv6), "2001:db8:85a3:0::"); + } + + #[test] + fn test_normalize_ip_ipv6_different_suffix_same_prefix() { + // Two IPv6 addresses with same /64 prefix but different interface identifiers + // (simulating Privacy Extensions rotation) + let ipv6_a = IpAddr::V6(Ipv6Addr::new( + 0x2001, 0x0db8, 0xabcd, 0x0001, 0x1111, 0x2222, 0x3333, 0x4444, + )); + let ipv6_b = IpAddr::V6(Ipv6Addr::new( + 0x2001, 0x0db8, 0xabcd, 0x0001, 0xaaaa, 0xbbbb, 0xcccc, 0xdddd, + )); + // Both should normalize to the same /64 prefix + assert_eq!(normalize_ip(ipv6_a), normalize_ip(ipv6_b)); + assert_eq!(normalize_ip(ipv6_a), "2001:db8:abcd:1::"); + } + fn create_test_request(headers: Vec<(HeaderName, &str)>) -> Request { let mut req = Request::new("GET", "http://example.com"); for (key, value) in headers { @@ -166,33 +227,89 @@ mod tests { req } + fn is_synthetic_id_format(value: &str) -> bool { + let mut parts = value.split('.'); + let hmac_part = match parts.next() { + Some(part) => part, + None => return false, + }; + let suffix_part = match parts.next() { + Some(part) => part, + None => return false, + }; + if parts.next().is_some() { + return false; + } + if hmac_part.len() != 64 || suffix_part.len() != 6 { + return false; + } + if !hmac_part.chars().all(|c| c.is_ascii_hexdigit()) { + return false; + } + if !suffix_part.chars().all(|c| c.is_ascii_alphanumeric()) { + return false; + } + true + } + #[test] fn test_generate_synthetic_id() { let settings: Settings = create_test_settings(); let req = create_test_request(vec![ (header::USER_AGENT, "Mozilla/5.0"), - (header::COOKIE, "pub_userid=12345"), - (HEADER_X_PUB_USER_ID, "67890"), - (header::HOST, settings.publisher.domain.as_str()), (header::ACCEPT_LANGUAGE, "en-US,en;q=0.9"), + (header::ACCEPT_ENCODING, "gzip, deflate, br"), ]); let synthetic_id = generate_synthetic_id(&settings, &req).expect("should generate synthetic ID"); log::info!("Generated synthetic ID: {}", synthetic_id); - assert_eq!( - synthetic_id, - "a1748067b3908f2c9e0f6ea30a341328ba4b84de45448b13d1007030df14a98e" - ) + assert!( + is_synthetic_id_format(&synthetic_id), + "should match synthetic ID format" + ); + } + + #[test] + fn test_is_synthetic_id_format_accepts_valid_value() { + let value = format!("{}.{}", "a".repeat(64), "Ab12z9"); + assert!( + is_synthetic_id_format(&value), + "should accept a valid synthetic ID format" + ); + } + + #[test] + fn test_is_synthetic_id_format_rejects_invalid_values() { + let missing_suffix = "a".repeat(64); + assert!( + !is_synthetic_id_format(&missing_suffix), + "should reject missing suffix" + ); + + let invalid_hex = format!("{}.{}", "a".repeat(63) + "g", "Ab12z9"); + assert!( + !is_synthetic_id_format(&invalid_hex), + "should reject non-hex HMAC content" + ); + + let invalid_suffix = format!("{}.{}", "a".repeat(64), "ab-129"); + assert!( + !is_synthetic_id_format(&invalid_suffix), + "should reject non-alphanumeric suffix" + ); + + let extra_segment = format!("{}.{}.{}", "a".repeat(64), "Ab12z9", "zz"); + assert!( + !is_synthetic_id_format(&extra_segment), + "should reject extra segments" + ); } #[test] fn test_get_synthetic_id_with_header() { let settings = create_test_settings(); - let req = create_test_request(vec![( - HEADER_SYNTHETIC_TRUSTED_SERVER, - "existing_synthetic_id", - )]); + let req = create_test_request(vec![(HEADER_X_SYNTHETIC_ID, "existing_synthetic_id")]); let synthetic_id = get_synthetic_id(&req).expect("should get synthetic ID"); assert_eq!(synthetic_id, Some("existing_synthetic_id".to_string())); @@ -205,7 +322,10 @@ mod tests { #[test] fn test_get_synthetic_id_with_cookie() { let settings = create_test_settings(); - let req = create_test_request(vec![(header::COOKIE, "synthetic_id=existing_cookie_id")]); + let req = create_test_request(vec![( + header::COOKIE, + &format!("{}=existing_cookie_id", COOKIE_SYNTHETIC_ID), + )]); let synthetic_id = get_synthetic_id(&req).expect("should get synthetic ID"); assert_eq!(synthetic_id, Some("existing_cookie_id".to_string())); diff --git a/trusted-server.toml b/trusted-server.toml index da92cb4..cd51b7b 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -16,11 +16,10 @@ secret_key = "trusted-server" # Possible values # - "client_ip" # - "user_agent" -# - "first_party_id" -# - "auth_user_id" -# - "publisher_domain" # - "accept_language" -template = "{{ client_ip }}:{{ user_agent }}:{{ first_party_id }}:{{ auth_user_id }}:{{ publisher_domain }}:{{ accept_language }}" +# - "accept_encoding" +# - "random_uuid" +template = "{{ client_ip }}:{{ user_agent }}:{{ accept_language }}:{{ accept_encoding }}" # Custom headers to be included in every response [response_headers]