// Copyright (c) Microsoft Corporation. // Licensed under the PostgreSQL License. //! ExecuteFunctionGraph orchestration - the main durable function executor //! //! ⚠️ DETERMINISTIC CODE ONLY in this file! //! - No I/O except through activities //! - No random numbers, current time, or other non-deterministic sources //! - Same input must always produce the same scheduling decisions use std::collections::HashMap; use std::str::FromStr; use std::time::Duration; use chrono::{DateTime, Utc}; use cron::Schedule as CronSchedule; use duroxide::OrchestrationContext; use crate::activities; use crate::types::{ evaluate_condition, string_map_to_json, substitute_all, substitute_all_raw, FunctionGraph, FunctionInput, FunctionNode, SystemVars, }; /// Orchestration name for ExecuteFunctionGraph pub const NAME: &str = "pg_durable::orchestration::execute-function-graph"; /// Orchestration name for ExecuteSubtree (used for parallel JOIN/RACE) pub const SUBTREE_NAME: &str = "pg_durable::orchestration::execute-subtree"; /// Execution context containing vars and metadata #[derive(Clone)] struct ExecutionContext { vars: HashMap, label: Option, /// Loop iteration counter (persisted across continue_as_new generations). loop_iteration: u64, /// Node id at the root of the *current* orchestration's node tree: `graph.root_node_id` /// for `execute`, the branch/loop node id for `execute_subtree`. A loop sitting on this /// node runs inline and drives this orchestration's own `continue_as_new`; any deeper /// loop is spawned as a child so its `continue_as_new` cannot re-execute an upstream /// prefix (#227). subtree_root: String, /// Shape of the input this orchestration re-enters itself with on loop `continue_as_new`. continuation: Continuation, } /// Which input envelope an inline loop must rebuild when it calls `continue_as_new`. /// /// Both orchestrations that can host an inline loop re-enter themselves, but they are /// registered with different input shapes, so the loop node handler picks the right one. #[derive(Clone, Copy)] enum Continuation { /// The root `execute` orchestration, whose input is a `FunctionInput`. Root, /// An `execute_subtree` child, whose input is a `SubtreeInput`. Subtree, } /// Input envelope for `execute_subtree`. /// /// Carries the serialized graph inline. A subtree therefore runs against the same immutable /// snapshot its parent already validated, and an inline loop re-emits that snapshot across /// `continue_as_new` — so `df.nodes` is read exactly once per instance and a post-start /// tamper cannot change the identity a node executes under. Role deletion and privilege /// revocation are still enforced on every node execution (`execute_sql` connects *as* /// `submitted_by`; the HTTP activities re-check `EXECUTE` privilege per request). /// /// `instance_id` is retained alongside the graph so a startup failure can still stamp the /// subtree root even when the graph itself fails to parse. `iteration` is threaded across /// `continue_as_new` when the subtree root is a loop. #[derive(serde::Serialize, serde::Deserialize)] struct SubtreeInput { instance_id: String, node_id: String, /// Serialized `FunctionGraph` snapshot inherited from the parent. graph: String, /// JSON-encoded named-results map inherited from the parent. results: String, /// JSON-encoded workflow vars map. #[serde(default)] vars: Option, #[serde(default)] label: Option, #[serde(default)] iteration: u64, } /// Control-flow-aware error type returned by every node handler. /// /// `Break` is **not** a failure: it unwinds through compound nodes (THEN, IF, JOIN, /// RACE, and the subtree boundary) via the `?` operator until the nearest enclosing /// `execute_loop_node` catches it. `Failure` is a genuine error that propagates to the /// orchestration result. Encoding break this way means forgetting to propagate it is a /// compile error rather than a silently-ignored value (see issue #148 / #132). #[derive(Debug)] enum NodeError { /// A `df.break()` signal carrying its (already-stringified) value, caught by the loop. Break(String), /// A real failure; propagates to the orchestration's `Err` result. Failure(String), } /// All helper functions (`substitute_all`, `evaluate_condition`) and activity scheduling /// return `Result<_, String>`. This conversion lets `?` turn those `String` errors into /// `NodeError::Failure` automatically, so only genuine control flow needs explicit handling. impl From for NodeError { fn from(e: String) -> Self { NodeError::Failure(e) } } /// Mirrors `From` for the many `.ok_or("literal")?` sites that yield `&str` errors, /// preserving the ergonomics those calls had when handlers returned `Result<_, String>`. impl From<&str> for NodeError { fn from(e: &str) -> Self { NodeError::Failure(e.to_string()) } } /// Result type for node handlers: `Ok` value string, or a typed control-flow/failure error. type NodeResult = Result; /// Distinguishes a normal subtree result from one that unwound via `df.break()`. /// /// Stored as `Option` in the envelope (see `SubtreeEnvelope::control`): a /// missing field deserializes to `None`, which unambiguously marks an envelope recorded by a /// pre-#148 binary (`<= v0.2.2`, no control field). A new binary always writes an explicit /// `Some(Normal)` / `Some(Break)`, so the legacy break-sentinel fallback can be gated to /// `None` only — keeping a user payload from impersonating control flow on a fresh envelope. #[derive(serde::Serialize, serde::Deserialize)] enum SubtreeControl { Normal, Break, } /// Envelope returned by `execute_subtree` containing the SQL result and the updated /// named-results map so the parent orchestration can merge any new entries after join/race. /// `control` carries a `df.break()` signal back across the sub-orchestration boundary so the /// parent can re-raise it as `NodeError::Break` rather than smuggling a sentinel in `result`. #[derive(serde::Serialize, serde::Deserialize)] struct SubtreeEnvelope { /// `None` only when deserialized from a pre-#148 envelope that had no `control` field; a /// new binary always serializes `Some(..)`. `parse_subtree_envelope` relies on this to run /// the legacy break-sentinel fallback exclusively on old envelopes. #[serde(default)] control: Option, result: String, #[serde(serialize_with = "crate::types::serialize_string_map")] results: HashMap, } /// Execute a complete function graph — the entry point for a durable function. /// /// # Control flow /// Internally every node handler returns `NodeResult`, where `NodeError::Break` is /// **intentional control flow** (a `df.break()` signal), not a failure. Break unwinds /// through compound nodes via `?` and is caught by the nearest enclosing /// `execute_loop_node`; only `NodeError::Failure` represents a genuine error. This /// boundary collapses the typed result back to `Result`: a `Break` /// that reaches here was used outside `df.loop()`, so it is surfaced as a clear failure /// rather than completing with a control-flow value. Callers should treat the returned /// `Err` strictly as a failure and must not add retry/recovery logic for break. pub async fn execute(ctx: OrchestrationContext, input_json: String) -> Result { let input: FunctionInput = serde_json::from_str(&input_json) .map_err(|e| format!("Invalid orchestration input: {e}"))?; let label_info = input .label .as_ref() .map(|l| format!(" ({l})")) .unwrap_or_default(); ctx.trace_info(format!( "Starting ExecuteFunctionGraph for instance: {}{}", input.instance_id, label_info )); if !input.vars.is_empty() { // Sort keys for deterministic logging let mut keys: Vec<_> = input.vars.keys().collect(); keys.sort(); ctx.trace_info(format!("Workflow vars: {keys:?}")); } // Generation 0 loads the graph from the database; a root loop continuing as new carries // it inline, so an instance reads `df.nodes` exactly once however many iterations it runs. // That load is also the admission check (`submitted_by` resolution plus the superuser // guard), which belongs at instance start rather than on every iteration — re-reading // mid-flight would make a post-start tamper of `df.nodes.submitted_by` take effect // instead of being ignored. let graph_json = match input.graph.clone() { Some(json) => json, None => match ctx .schedule_activity( activities::load_function_graph::NAME, input.instance_id.clone(), ) .await { Ok(json) => json, Err(e) => { // load_function_graph failed (e.g., superuser blocked). // Mark the instance as failed before propagating. let status_input = serde_json::json!({ "instance_id": input.instance_id, "status": "failed" }); let _ = ctx .schedule_activity( activities::update_instance_status::NAME, status_input.to_string(), ) .await; return Err(e); } }, }; let graph: FunctionGraph = serde_json::from_str(&graph_json) .map_err(|e| format!("Failed to parse function graph: {e}"))?; ctx.trace_info(format!( "Executing function with {} nodes, root: {}", graph.nodes.len(), graph.root_node_id )); // Mark the instance as running now that we have loaded the graph and are // about to execute. This call is idempotent: on continue_as_new the // instance is already 'running', so re-issuing the update is harmless. let running_input = serde_json::json!({ "instance_id": input.instance_id, "status": "running" }); let _ = ctx .schedule_activity( activities::update_instance_status::NAME, running_input.to_string(), ) .await; let mut results: HashMap = HashMap::new(); // Create execution context with vars let exec_ctx = ExecutionContext { vars: input.vars.clone(), label: input.label.clone(), loop_iteration: input.loop_iteration, subtree_root: graph.root_node_id.clone(), continuation: Continuation::Root, }; let function_outcome = execute_function_node_with_vars(&ctx, &graph, &graph.root_node_id, &mut results, &exec_ctx) .await; // Normalize the typed node result into the orchestration's String boundary. A `Break` // that reaches this point was never caught by a loop, i.e. `df.break()` was used outside // of `df.loop()` — surface it as a clear, actionable failure rather than completing with a // control-flow value as the function's result. let function_result: Result = match function_outcome { Ok(result) => Ok(result), Err(NodeError::Failure(err)) => Err(err), Err(NodeError::Break(_)) => Err( "df.break() was called outside of a loop. df.break() may only be used inside df.loop()." .to_string(), ), }; match &function_result { Ok(result) => { ctx.trace_info(format!("Function completed with result: {result}")); let status_input = serde_json::json!({ "instance_id": input.instance_id, "status": "completed" }); let _ = ctx .schedule_activity( activities::update_instance_status::NAME, status_input.to_string(), ) .await; } Err(err) => { ctx.trace_info(format!("Function failed with error: {err}")); let status_input = serde_json::json!({ "instance_id": input.instance_id, "status": "failed" }); let _ = ctx .schedule_activity( activities::update_instance_status::NAME, status_input.to_string(), ) .await; } } function_result } /// Execute a subtree of a function graph rooted at `node_id`. /// /// Used for JOIN/RACE branches and for any non-root `df.loop()`. Structurally this mirrors /// `execute`: it roots an `ExecutionContext` at its own node and — when that node is a loop — /// lets the loop drive `continue_as_new` on *this* orchestration. Because the subtree has no /// upstream prefix, re-entering from its root lands back on the same loop node each /// generation, exactly as it does for a root loop in `execute`. /// /// Unlike `execute`, this never loads the graph (the parent passes it inline) and never /// touches instance-level status: the parent owns that. pub async fn execute_subtree( ctx: OrchestrationContext, input_json: String, ) -> Result { let input: SubtreeInput = serde_json::from_str(&input_json) .map_err(|e| format!("Failed to parse ExecuteSubtree input: {e}"))?; // The subtree root is owned by this child instance, so stamp it failed on any startup // error. Without this a child that dies before `execute_function_node_with_vars` runs // would leave its root node stuck in a non-terminal state (the parent does not stamp // branch roots). let fail = |ctx: &OrchestrationContext, e: String| { let stamp = format!("{}::{}", ctx.instance_id(), ctx.execution_id()); let status_input = serde_json::json!({ "node_id": input.node_id, "instance_id": input.instance_id, "status": "failed", "result": e, "execution_id": stamp, }); (status_input.to_string(), e) }; ctx.trace_info(format!( "ExecuteSubtree: executing node {} (iteration {})", input.node_id, input.iteration )); // The graph arrives inline from the parent — no database read here. The subtree runs // against the snapshot the parent already validated, and an inline loop re-emits it on // `continue_as_new`, so nothing re-reads `df.nodes` mid-flight. let parsed = serde_json::from_str::(&input.graph) .map_err(|e| format!("Failed to parse graph in ExecuteSubtree: {e}")) .and_then(|graph| { let results: HashMap = serde_json::from_str(&input.results) .map_err(|e| format!("Failed to parse results in ExecuteSubtree: {e}"))?; let vars: HashMap = match input.vars.as_deref() { Some(vars_json) => serde_json::from_str(vars_json) .map_err(|e| format!("Failed to parse vars in ExecuteSubtree: {e}"))?, None => HashMap::new(), }; Ok((graph, results, vars)) }); let (graph, mut results, vars) = match parsed { Ok(parsed) => parsed, Err(e) => { let (status_input, e) = fail(&ctx, e); let _ = ctx .schedule_activity(activities::update_node_status::NAME, status_input) .await; return Err(e); } }; let exec_ctx = ExecutionContext { vars, label: input.label.clone(), loop_iteration: input.iteration, subtree_root: input.node_id.clone(), continuation: Continuation::Subtree, }; // Build the envelope carrying the result, the updated named-results map, and a typed // control signal. A `Break` inside the subtree is re-encoded as `control: Break` (not a // sentinel smuggled inside `result`) so the parent can re-raise it as `NodeError::Break`. // A genuine `Failure` propagates as `Err` across the sub-orchestration boundary. // // When the root node is a loop that needs another iteration it calls `continue_as_new`, // whose future never resolves — so none of the arms below run for a continuing // generation, and no envelope is produced until the loop actually exits. let envelope = match execute_function_node_with_vars( &ctx, &graph, &input.node_id, &mut results, &exec_ctx, ) .await { Ok(result) => { ctx.trace_info(format!("ExecuteSubtree: node {} completed", input.node_id)); SubtreeEnvelope { control: Some(SubtreeControl::Normal), result, results, } } Err(NodeError::Break(value)) => { ctx.trace_info(format!( "ExecuteSubtree: node {} broke (propagating)", input.node_id )); SubtreeEnvelope { control: Some(SubtreeControl::Break), result: value, results, } } Err(NodeError::Failure(e)) => return Err(e), }; serde_json::to_string(&envelope) .map_err(|e| format!("Failed to serialize subtree envelope: {e}")) } /// Build the `execute_subtree` input for a child rooted at `node_id`. /// /// Shared by JOIN/RACE branch scheduling and by non-root loop spawning — all three are the /// same operation now: run this node in its own durable instance. fn build_subtree_input( graph: &FunctionGraph, node_id: &str, results: &HashMap, exec_ctx: &ExecutionContext, ) -> Result { let input = SubtreeInput { instance_id: graph.instance_id.clone(), node_id: node_id.to_string(), graph: serde_json::to_string(graph) .map_err(|e| format!("Failed to serialize graph: {e}"))?, results: string_map_to_json(results) .map_err(|e| format!("Failed to serialize results: {e}"))?, vars: Some( string_map_to_json(&exec_ctx.vars) .map_err(|e| format!("Failed to serialize vars: {e}"))?, ), label: exec_ctx.label.clone(), iteration: 0, }; serde_json::to_string(&input).map_err(|e| format!("Failed to serialize subtree input: {e}")) } /// Compose the deterministic instance id for a JOIN/RACE branch sub-orchestration. /// /// `schedule_sub_orchestration_with_id` uses this value verbatim (no parent prefix), so we /// build `{parent_instance_id}::{parent_execution_id}::{child_root_node_id}`. This guarantees /// a complete parent-to-child lineage and per-generation uniqueness: the parent execution id /// advances on every loop `continue_as_new`, while the child root node id distinguishes sibling /// branches. df.instance_nodes() and the write fence walk the full composed lineage. fn subtree_instance_id(ctx: &OrchestrationContext, child_root_node_id: &str) -> String { format!( "{}::{}::{}", ctx.instance_id(), ctx.execution_id(), child_root_node_id ) } /// Recursively execute function nodes with vars support async fn execute_function_node_with_vars( ctx: &OrchestrationContext, graph: &FunctionGraph, node_id: &str, results: &mut HashMap, exec_ctx: &ExecutionContext, ) -> NodeResult { let node = graph .nodes .get(node_id) .ok_or_else(|| format!("Node not found: {node_id}"))?; ctx.trace_info(format!( "Executing node {} (type: {})", node_id, node.node_type )); // A loop that is NOT the root of the current orchestration's node tree runs as a child // sub-orchestration, so its `continue_as_new` restarts only the loop body rather than // re-executing the upstream prefix (#227), and a loop nested in a parallel branch gets // its own durable instance (#233). The parent does NOT stamp such a loop node // running/terminal: the child owns the node as its root and stamps it there. Intercept // here, before any status stamping. // // A loop that IS this orchestration's root falls through and runs inline via // `execute_loop_node`, driving `continue_as_new` on this orchestration. That is safe // precisely because there is no upstream prefix to re-execute: re-entering from the root // lands back on this same loop node. This holds identically for the root `execute` // orchestration and for an `execute_subtree` child rooted at a loop. if node.node_type.eq_ignore_ascii_case("loop") && node_id != exec_ctx.subtree_root { return execute_loop_suborchestration(ctx, graph, node, node_id, results, exec_ctx).await; } // Stamp identifying which orchestration generation is transitioning this // node: "{orchestration_instance_id}::{execution_id}". For the root // orchestration this is "{df_instance_id}::{loop_generation}"; for a JOIN/RACE // sub-orchestration the instance id already carries the composed lineage (see // `subtree_instance_id`). df.instance_nodes() and update_node_status walk the // lineage generations to infer superseded nodes and fence stale writes. Both // reads are deterministic (instance_id/execution_id are stable within an execution). let execution_stamp = format!("{}::{}", ctx.instance_id(), ctx.execution_id()); // Mark node as running let running_input = serde_json::json!({ "node_id": node_id, "instance_id": graph.instance_id, "status": "running", "execution_id": execution_stamp, }); let _ = ctx .schedule_activity( activities::update_node_status::NAME, running_input.to_string(), ) .await; let execute_result = execute_node_inner(ctx, graph, node_id, node, results, exec_ctx).await; // Update node with final status and result. A `Break` is control flow rather than a // failure: record the node as completed (carrying the break value) so observability is // unchanged from when break travelled as a normal `Ok` sentinel. Only `Failure` marks // the node failed. All three arms schedule exactly one `update_node_status`, so collapse // them to a single (status, result) pair to keep the recorded history identical. let (status, status_result) = match &execute_result { Ok(result) => ("completed", result.as_str()), Err(NodeError::Break(value)) => ("completed", value.as_str()), Err(NodeError::Failure(err)) => ("failed", err.as_str()), }; let status_input = serde_json::json!({ "node_id": node_id, "instance_id": graph.instance_id, "status": status, "result": status_result, "execution_id": execution_stamp, }); let _ = ctx .schedule_activity( activities::update_node_status::NAME, status_input.to_string(), ) .await; execute_result } /// Inner function that actually executes the node logic async fn execute_node_inner( ctx: &OrchestrationContext, graph: &FunctionGraph, node_id: &str, node: &FunctionNode, results: &mut HashMap, exec_ctx: &ExecutionContext, ) -> NodeResult { // Build system vars let sys_vars = SystemVars { instance_id: graph.instance_id.clone(), label: exec_ctx.label.clone(), }; match node.node_type.to_lowercase().as_str() { "sql" => execute_sql_node(ctx, node, node_id, results, exec_ctx, &sys_vars).await, "then" => execute_then_node(ctx, graph, node, node_id, results, exec_ctx).await, "sleep" => execute_sleep_node(ctx, node, node_id).await, "wait_schedule" => execute_wait_schedule_node(ctx, node, node_id).await, "loop" => execute_loop_node(ctx, graph, node, node_id, results, exec_ctx).await, "if" => execute_if_node(ctx, graph, node, node_id, results, exec_ctx).await, "join" => execute_join_node(ctx, graph, node, node_id, results, exec_ctx).await, "race" => execute_race_node(ctx, graph, node, node_id, results, exec_ctx).await, "http" => execute_http_node(ctx, node, node_id, results, exec_ctx, &sys_vars).await, "http_multipart" => { execute_http_multipart_node(ctx, node, node_id, results, exec_ctx, &sys_vars).await } "signal" => execute_signal_node(ctx, node, node_id, results).await, "break" => execute_break_node(ctx, node, node_id).await, other => Err(NodeError::Failure(format!("Unknown node type: {other}"))), } } // ============================================================================ // Node Type Handlers // ============================================================================ async fn execute_sql_node( ctx: &OrchestrationContext, node: &FunctionNode, node_id: &str, results: &mut HashMap, exec_ctx: &ExecutionContext, sys_vars: &SystemVars, ) -> NodeResult { let query = node .query .as_ref() .ok_or_else(|| format!("SQL node {node_id} has no query"))?; let final_query = substitute_all(query, results, &exec_ctx.vars, sys_vars)?; ctx.trace_info(format!("Executing SQL: {final_query}")); let input = serde_json::json!({ "query": final_query, "submitted_by": node.submitted_by, "database": node.database, }); let result = ctx .schedule_activity(activities::execute_sql::NAME, input.to_string()) .await?; if let Some(name) = &node.result_name { ctx.trace_info(format!("Storing result as ${name}")); results.insert(name.clone(), result.clone()); } Ok(result) } fn store_named_result( ctx: &OrchestrationContext, node: &FunctionNode, result: &str, results: &mut HashMap, node_label: &str, ) { if let Some(name) = &node.result_name { ctx.trace_info(format!("Storing {node_label} result as ${name}")); results.insert(name.clone(), result.to_string()); } } async fn execute_then_node( ctx: &OrchestrationContext, graph: &FunctionGraph, node: &FunctionNode, node_id: &str, results: &mut HashMap, exec_ctx: &ExecutionContext, ) -> NodeResult { let left_id = node .left_node .as_ref() .ok_or_else(|| format!("THEN node {node_id} has no left_node"))?; let right_id = node .right_node .as_ref() .ok_or_else(|| format!("THEN node {node_id} has no right_node"))?; // A `df.break()` anywhere in the left branch propagates automatically via `?` to the // enclosing loop, skipping the right branch — no explicit sentinel check needed. Box::pin(execute_function_node_with_vars( ctx, graph, left_id, results, exec_ctx, )) .await?; let right_result = Box::pin(execute_function_node_with_vars( ctx, graph, right_id, results, exec_ctx, )) .await?; store_named_result(ctx, node, &right_result, results, "THEN"); Ok(right_result) } async fn execute_sleep_node( ctx: &OrchestrationContext, node: &FunctionNode, node_id: &str, ) -> NodeResult { let seconds_str = node .query .as_ref() .ok_or_else(|| format!("SLEEP node {node_id} has no duration"))?; let seconds: u64 = seconds_str .parse() .map_err(|_| format!("Invalid sleep duration: {seconds_str}"))?; ctx.trace_info(format!("Sleeping for {seconds} seconds")); ctx.schedule_timer(Duration::from_secs(seconds)).await; Ok(format!(r#"{{"slept": true, "seconds": {seconds}}}"#)) } async fn execute_wait_schedule_node( ctx: &OrchestrationContext, node: &FunctionNode, node_id: &str, ) -> NodeResult { let config_str = node .query .as_ref() .ok_or_else(|| format!("WAIT_SCHEDULE node {node_id} has no config"))?; let config: serde_json::Value = serde_json::from_str(config_str) .map_err(|e| format!("Invalid WAIT_SCHEDULE config: {e}"))?; let cron_expr = config["cron_expr"] .as_str() .ok_or_else(|| "WAIT_SCHEDULE missing cron_expr".to_string())?; // A cron schedule is a function of "now", so the next tick MUST be computed // when this node actually executes — not at df.start() time — so that any // delay before execution, and every iteration of a recurring `@>` loop, // targets the correct upcoming tick. // // `ctx.utc_now()` is duroxide's deterministic clock (the only sanctioned way // to read wall-clock time in this deterministic file): the value is recorded // in history and replayed verbatim. The cron math below is pure given `now`, // so the whole computation is replay-safe. The "0 " prefix supplies the // seconds field the `cron` crate expects (mirrors df.wait_for_schedule()). let now: DateTime = ctx .utc_now() .await .map_err(|e| format!("WAIT_SCHEDULE failed to read deterministic clock: {e}"))? .into(); let cron_with_seconds = format!("0 {cron_expr}"); let schedule = CronSchedule::from_str(&cron_with_seconds) .map_err(|e| format!("Invalid cron expression '{cron_expr}': {e}"))?; let next = schedule .after(&now) .next() .ok_or_else(|| format!("No upcoming schedule found for '{cron_expr}'"))?; // Clamp to zero if the tick is already in the past by the time we get here. // // NOTE: once duroxide gains an absolute-deadline timer // (https://github.com/microsoft/duroxide/issues/34), this `now`-read + // subtraction can be replaced with `ctx.schedule_timer_until(next)`, which // targets the absolute tick directly and drops the extra utc_now() syscall. let wait = (next - now).to_std().unwrap_or(Duration::ZERO); ctx.trace_info(format!( "Waiting {}s until next schedule tick {next} (cron: {cron_expr})", wait.as_secs() )); ctx.schedule_timer(wait).await; Ok(r#"{"scheduled": true}"#.to_string()) } /// Minimum wall-clock duration that every loop iteration must take before /// `continue_as_new` is called. If the body (plus any while-condition /// evaluation) completes faster than this, a compensating timer makes up the /// deficit so an empty-bodied loop can't busy-spin via continue_as_new. const LOOP_MIN_ITER_DURATION: Duration = Duration::from_secs(1); /// Maximum loop iterations before the orchestration is forcibly terminated. /// This prevents runaway infinite loops from consuming resources indefinitely. /// At the minimum 1-second rate limit, this allows ~27 hours of looping. const MAX_LOOP_ITERATIONS: u64 = 100_000; /// Stamp a loop node's status from its *parent* orchestration. /// /// A non-root loop node is the root of its own child instance, so the child normally owns /// its status transitions (via `execute_function_node_with_vars`). This helper covers the /// cases where the child never got far enough to stamp itself — spawn failure, a failed /// branch future, or a losing RACE branch — and the parent must record a terminal state so /// the node does not linger as running. async fn stamp_loop_node( ctx: &OrchestrationContext, instance_id: &str, loop_node_id: &str, status: &str, result: Option<&str>, stamp: &str, ) { let mut input = serde_json::json!({ "node_id": loop_node_id, "instance_id": instance_id, "status": status, "execution_id": stamp, }); if let Some(r) = result { input["result"] = serde_json::Value::String(r.to_string()); } let _ = ctx .schedule_activity(activities::update_node_status::NAME, input.to_string()) .await; } async fn fail_loop_before_start( ctx: &OrchestrationContext, graph: &FunctionGraph, loop_node_id: &str, error: String, ) -> NodeResult { let execution_stamp = format!("{}::{}", ctx.instance_id(), ctx.execution_id()); stamp_loop_node( ctx, &graph.instance_id, loop_node_id, "failed", Some(&error), &execution_stamp, ) .await; Err(NodeError::Failure(error)) } async fn fail_loop_child_future( ctx: &OrchestrationContext, graph: &FunctionGraph, loop_node_id: &str, error: String, ) -> NodeResult { let child_stamp = format!("{}::1", subtree_instance_id(ctx, loop_node_id)); stamp_loop_node( ctx, &graph.instance_id, loop_node_id, "failed", Some(&error), &child_stamp, ) .await; Err(NodeError::Failure(error)) } /// Run one iteration of a loop body (and its optional while-condition). /// /// Shared by both inline loop paths: a loop at the root of the root orchestration and a /// loop at the root of an `execute_subtree` child. /// /// Returns `Ok(Some(final_result))` when the loop should exit (a `df.break()` in the body, /// or the while-condition evaluating false), `Ok(None)` when another iteration is needed, /// and `Err` when the body or condition fails. async fn run_loop_iteration( ctx: &OrchestrationContext, graph: &FunctionGraph, node: &FunctionNode, loop_node_id: &str, body_id: &str, results: &mut HashMap, exec_ctx: &ExecutionContext, ) -> Result, String> { // The loop is where `NodeError::Break` is caught: a break unwinds through the body via // `?` and is converted here into the loop's normal exit value. A `Failure` propagates // out of the sub-orchestration unchanged. let body_result = match execute_function_node_with_vars(ctx, graph, body_id, results, exec_ctx).await { Ok(v) => v, Err(NodeError::Break(break_value)) => { ctx.trace_info(format!( "Loop terminated by break with value: {break_value}" )); store_named_result(ctx, node, &break_value, results, "LOOP"); return Ok(Some(break_value)); } Err(NodeError::Failure(e)) => return Err(e), }; // While-condition: if present and false, exit the loop. if let Some(ref config_str) = node.query { let config: serde_json::Value = serde_json::from_str(config_str).map_err(|e| { // M8: Malformed condition config should fail the loop rather than // silently creating an infinite loop without exit condition. format!("LOOP node {loop_node_id}: failed to parse condition config: {e}") })?; if let Some(condition_node_id) = config["condition_node"].as_str() { ctx.trace_info("Evaluating loop condition"); let condition_result = match execute_function_node_with_vars( ctx, graph, condition_node_id, results, exec_ctx, ) .await { Ok(v) => v, Err(NodeError::Break(break_value)) => { store_named_result(ctx, node, &break_value, results, "LOOP"); return Ok(Some(break_value)); } Err(NodeError::Failure(e)) => return Err(e), }; // Parse condition result to check truthiness (uses evaluate_condition to extract boolean from SQL result) let should_continue = evaluate_condition(&condition_result).unwrap_or(false); ctx.trace_info(format!( "Loop condition evaluated to: {condition_result} (continue={should_continue})" )); if !should_continue { ctx.trace_info("Loop condition false, exiting loop"); store_named_result(ctx, node, &body_result, results, "LOOP"); return Ok(Some(body_result)); } } } Ok(None) } /// Execute a loop node inline, driving the *current* orchestration's `continue_as_new`. /// /// Only a loop sitting at `exec_ctx.subtree_root` reaches this function; a deeper loop is /// intercepted in `execute_function_node_with_vars` and delegated to a child. Running the /// root loop inline is safe because there is no upstream prefix to re-execute: re-entering /// this orchestration from its root lands back on this same loop node each generation. /// /// Both hosts are handled: the root `execute` orchestration and an `execute_subtree` child /// rooted at the loop node. They differ only in the input envelope they re-enter with, which /// `exec_ctx.continuation` selects. /// /// Note that `ctx.continue_as_new()` returns a future that never resolves, so on a /// continuing generation this function does not return and the caller's terminal node /// stamping never runs — the loop node stays `running` until it actually exits. async fn execute_loop_node( ctx: &OrchestrationContext, graph: &FunctionGraph, node: &FunctionNode, node_id: &str, results: &mut HashMap, exec_ctx: &ExecutionContext, ) -> NodeResult { debug_assert_eq!( node_id, exec_ctx.subtree_root, "inline loop must be the current orchestration's root" ); let body_id = node .left_node .as_ref() .ok_or_else(|| format!("LOOP node {node_id} has no body"))?; // Capture the iteration start time so we can rate-limit `continue_as_new` // below. `utc_now()` is duroxide's deterministic clock (recorded in // history and replayed verbatim), so this remains replay-safe. let iter_started = ctx.utc_now().await.ok(); ctx.trace_info("Executing loop iteration"); if let Some(final_result) = Box::pin(run_loop_iteration( ctx, graph, node, node_id, body_id, results, exec_ctx, )) .await .map_err(NodeError::Failure)? { return Ok(final_result); } ctx.trace_info("Continuing as new for next loop iteration"); // M7: Enforce maximum iteration count to prevent runaway infinite loops let next_iteration = exec_ctx.loop_iteration + 1; if next_iteration >= MAX_LOOP_ITERATIONS { return Err(NodeError::Failure(format!( "Loop exceeded maximum iteration count of {MAX_LOOP_ITERATIONS}. \ Use df.break() to exit the loop or restructure the workflow." ))); } // Enforce a minimum per-iteration wall-clock duration to prevent // busy-looping (e.g. `df.loop(df.sleep(0))`). Compute the elapsed time // from the deterministic clock; if the iteration finished faster than // LOOP_MIN_ITER_DURATION, schedule a timer for the deficit so the next // continue_as_new is gated by at least that much real-clock time. if let Some(started) = iter_started { if let Ok(now) = ctx.utc_now().await { let elapsed = now.duration_since(started).unwrap_or(Duration::ZERO); if elapsed < LOOP_MIN_ITER_DURATION { let deficit = LOOP_MIN_ITER_DURATION - elapsed; ctx.trace_info(format!( "Loop iteration took {elapsed:?} (< {LOOP_MIN_ITER_DURATION:?}); \ adding {deficit:?} rate-limit delay" )); ctx.schedule_timer(deficit).await; } } } // Rebuild this orchestration's own input for the next generation. The root // orchestration re-enters with a `FunctionInput` (named results are rebuilt from // scratch, as they always have been); an `execute_subtree` child re-enters with a // `SubtreeInput` that threads the accumulated named results forward. // Rebuild this orchestration's own input for the next generation. The root // orchestration re-enters with a `FunctionInput` (named results are rebuilt from // scratch, as they always have been); an `execute_subtree` child re-enters with a // `SubtreeInput` that threads the accumulated named results forward. Both carry the // graph snapshot forward so no generation re-reads `df.nodes`. let graph_json = serde_json::to_string(graph).map_err(|e| format!("Failed to serialize graph: {e}"))?; let new_input_json = match exec_ctx.continuation { Continuation::Root => { let new_input = FunctionInput { instance_id: graph.instance_id.clone(), label: exec_ctx.label.clone(), vars: exec_ctx.vars.clone(), loop_iteration: next_iteration, graph: Some(graph_json), }; serde_json::to_string(&new_input) .map_err(|e| format!("Failed to serialize loop input: {e}"))? } Continuation::Subtree => { let new_input = SubtreeInput { instance_id: graph.instance_id.clone(), node_id: node_id.to_string(), graph: graph_json, results: string_map_to_json(results) .map_err(|e| format!("Failed to serialize updated results: {e}"))?, vars: Some( string_map_to_json(&exec_ctx.vars) .map_err(|e| format!("Failed to serialize vars: {e}"))?, ), label: exec_ctx.label.clone(), iteration: next_iteration, }; serde_json::to_string(&new_input) .map_err(|e| format!("Failed to serialize loop input: {e}"))? } }; // duroxide: continue_as_new returns an awaitable future - return it directly ctx.continue_as_new(new_input_json) .await .map(|_| String::new()) .map_err(|e| NodeError::Failure(format!("continue_as_new failed: {e:?}"))) } /// Run a loop that is *not* the current orchestration's root as a child sub-orchestration. /// /// Such a loop must not run inline: its `continue_as_new` would restart the parent /// orchestration and re-execute the upstream prefix (#227), and a loop nested inside a /// parallel branch needs its own durable instance (#233). The child is an ordinary /// `execute_subtree` rooted at the loop node, which then runs the loop inline and advances /// iterations via its own `continue_as_new` — so the parent's prefix is preserved. /// /// The child instance id is built with `subtree_instance_id`, preserving every ancestor /// scope and generation for node-status inference and the write fence. It is unique per /// iteration because the parent execution id advances on each generation, so a loop inside /// a parallel branch never collides across iterations. The loop node itself is the /// root of this child instance and is stamped (running/completed/failed) there — NOT by the /// parent — so the parent invokes this *before* any status stamping (see /// `execute_function_node_with_vars`). async fn execute_loop_suborchestration( ctx: &OrchestrationContext, graph: &FunctionGraph, node: &FunctionNode, node_id: &str, results: &mut HashMap, exec_ctx: &ExecutionContext, ) -> NodeResult { // Validate the loop has a body before spawning the child sub-orchestration. if node.left_node.is_none() { return fail_loop_before_start( ctx, graph, node_id, format!("LOOP node {node_id} has no body"), ) .await; } let loop_input = match build_subtree_input(graph, node_id, results, exec_ctx) { Ok(input) => input, Err(e) => { return fail_loop_before_start(ctx, graph, node_id, e).await; } }; ctx.trace_info(format!( "Spawning loop sub-orchestration for node {node_id}" )); let raw = match ctx .schedule_sub_orchestration_with_id( SUBTREE_NAME, subtree_instance_id(ctx, node_id), loop_input, ) .await { Ok(raw) => raw, Err(e) => { return fail_loop_child_future( ctx, graph, node_id, format!("Loop sub-orchestration failed: {e}"), ) .await; } }; // Merge named results from the loop sub-orchestration back into the parent map and // return the loop's final result. The loop always returns a `Normal` envelope (a break // inside the body is the loop's own terminator), so `parse_subtree_envelope` will not // re-raise a `NodeError::Break` here. parse_subtree_envelope(&raw, "LOOP", results) } async fn execute_break_node( ctx: &OrchestrationContext, node: &FunctionNode, node_id: &str, ) -> NodeResult { let break_value = node .query .as_ref() .and_then(|config_str| serde_json::from_str::(config_str).ok()) .and_then(|config| config.get("break_value").cloned()) .and_then(|v| { if v.is_null() { None } else { v.as_str().map(|s| s.to_string()) } }); ctx.trace_info(format!( "BREAK node {node_id} executed with value: {break_value:?}" )); // Encode the break value as the stringified JSON the loop will surface as its result: // a value that parses as JSON is preserved as-is (e.g. `{"status":"done"}`), a bare // string round-trips as a quoted JSON string, and an absent value becomes `null`. // The signal travels as a typed `NodeError::Break`, so `?` unwinds it to the loop. let value = match break_value { Some(v) => serde_json::from_str::(&v) .unwrap_or(serde_json::Value::String(v)) .to_string(), None => "null".to_string(), }; Err(NodeError::Break(value)) } async fn execute_if_node( ctx: &OrchestrationContext, graph: &FunctionGraph, node: &FunctionNode, node_id: &str, results: &mut HashMap, exec_ctx: &ExecutionContext, ) -> NodeResult { let config_str = node .query .as_ref() .ok_or_else(|| format!("IF node {node_id} has no config"))?; let config: serde_json::Value = serde_json::from_str(config_str).map_err(|e| format!("Invalid IF config: {e}"))?; let then_id = node .left_node .as_ref() .ok_or_else(|| format!("IF node {node_id} has no then branch"))?; let else_id = node .right_node .as_ref() .ok_or_else(|| format!("IF node {node_id} has no else branch"))?; let is_true = if config.get("condition_type").and_then(|ct| ct.as_str()) == Some("result_has_rows") { // df.if_rows: check row_count from in-memory results — no activity needed let result_name = config["result_name"] .as_str() .ok_or_else(|| "df.if_rows: missing result_name".to_string())?; let result_json = results .get(result_name) .ok_or_else(|| format!("df.if_rows: result '{result_name}' not found"))?; let parsed: serde_json::Value = serde_json::from_str(result_json) .map_err(|e| format!("df.if_rows: invalid result JSON: {e}"))?; let row_count = parsed .get("row_count") .and_then(|rc| rc.as_u64()) .ok_or_else(|| { format!( "df.if_rows: result '{result_name}' is not a SQL result (missing row_count)" ) })?; ctx.trace_info(format!("if_rows '{result_name}': {row_count} rows")); row_count > 0 } else { // df.if: execute condition node as SQL let condition_node_id = config["condition_node"] .as_str() .ok_or_else(|| "IF node missing condition_node".to_string())?; ctx.trace_info("Evaluating IF condition"); let condition_result = Box::pin(execute_function_node_with_vars( ctx, graph, condition_node_id, results, exec_ctx, )) .await?; evaluate_condition(&condition_result)? }; ctx.trace_info(format!("Condition evaluated to: {is_true}")); if is_true { let result = Box::pin(execute_function_node_with_vars( ctx, graph, then_id, results, exec_ctx, )) .await?; store_named_result(ctx, node, &result, results, "IF"); Ok(result) } else { let result = Box::pin(execute_function_node_with_vars( ctx, graph, else_id, results, exec_ctx, )) .await?; store_named_result(ctx, node, &result, results, "IF"); Ok(result) } } /// Sentinel key used by pre-#148 binaries (<= v0.2.2) to encode a `df.break()` *inside* the /// subtree envelope's `result` string, as `{"__break__": true, "value": ...}`. This binary no /// longer writes it (break now travels as the typed `control` field), but it is still read on /// the in-flight upgrade path: see `parse_subtree_envelope`. const LEGACY_BREAK_SENTINEL: &str = "__break__"; /// Decode a pre-#148 break sentinel for in-flight upgrade compatibility. /// /// Returns `Some(value)` if `raw` is a legacy `{"__break__": true, "value": ...}` object, /// where `value` is the break value stringified exactly as the old `extract_break_value` /// produced it (the JSON value's `to_string()`, or `"null"` when absent). Returns `None` for /// any normal result. Only envelopes with an absent `control` field (pre-#148 binaries) reach /// this path; anything written by the new binary carries an explicit `control` and skips it. fn parse_legacy_break_sentinel(raw: &str) -> Option { let value = serde_json::from_str::(raw).ok()?; if value.get(LEGACY_BREAK_SENTINEL).and_then(|b| b.as_bool()) != Some(true) { return None; } Some( value .get("value") .cloned() .map(|v| v.to_string()) .unwrap_or_else(|| "null".to_string()), ) } /// Parse the JSON envelope returned by `execute_subtree`, extract the SQL result string, /// and merge the branch's named results into `parent_results`. A branch that broke out via /// `df.break()` carries `control = Break`, which is re-raised here as `NodeError::Break` so /// the enclosing loop catches it. fn parse_subtree_envelope( raw: &str, context: &str, parent_results: &mut HashMap, ) -> NodeResult { let envelope: SubtreeEnvelope = serde_json::from_str(raw).map_err(|e| format!("{context} envelope parse error: {e}"))?; parent_results.extend(envelope.results); match envelope.control { Some(SubtreeControl::Break) => Err(NodeError::Break(envelope.result)), // A new binary always writes an explicit `control`, so `Some(Normal)` is a genuine // normal result and must NOT be run through the legacy sentinel check: otherwise a // branch whose real SQL result happens to be shaped like `{"__break__": true, ...}` // would be falsely re-raised as a `Break` — exactly the payload-impersonates-control // bug class #148 set out to remove. Some(SubtreeControl::Normal) => Ok(envelope.result), // `None` means the envelope was recorded by a pre-#148 binary (`<= v0.2.2`): it had no // `control` field and instead smuggled a break as a `{"__break__": true, ...}` // sentinel inside `result`. Re-raise such a legacy sentinel as a typed `Break` so a // JOIN/RACE-in-loop break still unwinds when an orchestration started under the old // binary resumes under this one, instead of being silently swallowed and treated as a // normal branch result. None => match parse_legacy_break_sentinel(&envelope.result) { Some(value) => Err(NodeError::Break(value)), None => Ok(envelope.result), }, } } async fn cancel_losing_loop_branch( ctx: &OrchestrationContext, graph: &FunctionGraph, branch_node_id: &str, ) { let is_loop = graph .nodes .get(branch_node_id) .map(|node| node.node_type.eq_ignore_ascii_case("loop")) .unwrap_or(false); if !is_loop { return; } let child_instance_id = subtree_instance_id(ctx, branch_node_id); let child_stamp = format!("{child_instance_id}::1"); stamp_loop_node( ctx, &graph.instance_id, branch_node_id, "failed", Some("Loop branch cancelled after losing RACE"), &child_stamp, ) .await; } async fn execute_join_node( ctx: &OrchestrationContext, graph: &FunctionGraph, node: &FunctionNode, node_id: &str, results: &mut HashMap, exec_ctx: &ExecutionContext, ) -> NodeResult { let left_id = node .left_node .as_ref() .ok_or_else(|| format!("JOIN node {node_id} has no left branch"))?; let right_id = node .right_node .as_ref() .ok_or_else(|| format!("JOIN node {node_id} has no right branch"))?; ctx.trace_info("Executing JOIN branches in parallel"); // Collect the branch root node ids (left, right, and any join3 extras). Each branch is // spawned as its own child sub-orchestration with a deterministic, generation-stamped // instance id (see `subtree_instance_id`) so its node stamps carry the root loop // generation and remain unique across loop iterations. let mut branch_ids: Vec = vec![left_id.clone(), right_id.clone()]; // Check for extra nodes (join3) if let Some(config_str) = &node.query { if let Ok(config) = serde_json::from_str::(config_str) { if let Some(extra_nodes) = config["extra_nodes"].as_array() { for extra_node_val in extra_nodes { if let Some(extra_id) = extra_node_val.as_str() { branch_ids.push(extra_id.to_string()); } } } } } // Schedule each branch as an `execute_subtree` child rooted at the branch node. A branch // whose root is a loop needs no special case: `execute_subtree` runs a root loop inline // and drives its own `continue_as_new`, so exactly one child instance is created either // way, and every branch returns a `SubtreeEnvelope`. let mut durable_futures = Vec::new(); for child_root in &branch_ids { let input = build_subtree_input(graph, child_root, results, exec_ctx)?; let fut = ctx.schedule_sub_orchestration_with_id( SUBTREE_NAME, subtree_instance_id(ctx, child_root), input, ); durable_futures.push(fut); } // Use ctx.join() - Duroxide's proper join method for parallel execution let results_vec = ctx.join(durable_futures).await; // Process results - join now returns Vec> directly. // Each Ok value is a JSON envelope {"result": "...", "results": {...}} produced by // execute_subtree; unwrap it and merge the branch's named results into the parent map. let mut join_results: Vec = Vec::new(); for (i, result) in results_vec.into_iter().enumerate() { match result { Ok(r) => { let context = format!("JOIN branch {}", i + 1); // A break in any branch surfaces as `NodeError::Break` from // `parse_subtree_envelope` and unwinds via `?` to the enclosing loop. let branch_result = parse_subtree_envelope(&r, &context, results)?; let parsed = serde_json::from_str::(&branch_result) .map_err(|e| format!("JOIN branch {} result parse error: {}", i + 1, e))?; join_results.push(parsed); } Err(e) => { if graph .nodes .get(&branch_ids[i]) .map(|branch| branch.node_type.eq_ignore_ascii_case("loop")) .unwrap_or(false) { return fail_loop_child_future( ctx, graph, &branch_ids[i], format!("JOIN branch {} failed: {}", i + 1, e), ) .await; } return Err(NodeError::Failure(format!( "JOIN branch {} failed: {}", i + 1, e ))); } } } ctx.trace_info(format!( "JOIN completed with {} results", join_results.len() )); let result = serde_json::to_string(&join_results).unwrap_or_else(|_| "[]".to_string()); // Store result if named if let Some(name) = &node.result_name { ctx.trace_info(format!("Storing JOIN result as ${name}")); results.insert(name.clone(), result.clone()); } Ok(result) } async fn execute_race_node( ctx: &OrchestrationContext, graph: &FunctionGraph, node: &FunctionNode, node_id: &str, results: &mut HashMap, exec_ctx: &ExecutionContext, ) -> NodeResult { let left_id = node .left_node .as_ref() .ok_or_else(|| format!("RACE node {node_id} has no left branch"))?; let right_id = node .right_node .as_ref() .ok_or_else(|| format!("RACE node {node_id} has no right branch"))?; ctx.trace_info("Executing RACE branches in parallel (first wins)"); // Schedule each branch as an `execute_subtree` child with a deterministic, // generation-stamped instance id so its node stamps carry the root loop generation (see // `subtree_instance_id`). A branch whose root is a loop needs no special case: // `execute_subtree` runs a root loop inline and drives its own `continue_as_new`. let left_input = build_subtree_input(graph, left_id, results, exec_ctx)?; let right_input = build_subtree_input(graph, right_id, results, exec_ctx)?; let left_fut = ctx.schedule_sub_orchestration_with_id( SUBTREE_NAME, subtree_instance_id(ctx, left_id), left_input, ); let right_fut = ctx.schedule_sub_orchestration_with_id( SUBTREE_NAME, subtree_instance_id(ctx, right_id), right_input, ); // Use ctx.select2() - first to complete wins // select2 now returns Either2 instead of (winner_idx, DurableOutput) let raw = match ctx.select2(left_fut, right_fut).await { duroxide::Either2::First(Ok(r)) => { ctx.trace_info("RACE completed - left branch won"); cancel_losing_loop_branch(ctx, graph, right_id).await; Ok(r) } duroxide::Either2::First(Err(e)) => { cancel_losing_loop_branch(ctx, graph, right_id).await; if graph .nodes .get(left_id) .map(|branch| branch.node_type.eq_ignore_ascii_case("loop")) .unwrap_or(false) { return fail_loop_child_future( ctx, graph, left_id, format!("RACE left branch failed: {e}"), ) .await; } Err(format!("RACE left branch failed: {e}")) } duroxide::Either2::Second(Ok(r)) => { ctx.trace_info("RACE completed - right branch won"); cancel_losing_loop_branch(ctx, graph, left_id).await; Ok(r) } duroxide::Either2::Second(Err(e)) => { cancel_losing_loop_branch(ctx, graph, left_id).await; if graph .nodes .get(right_id) .map(|branch| branch.node_type.eq_ignore_ascii_case("loop")) .unwrap_or(false) { return fail_loop_child_future( ctx, graph, right_id, format!("RACE right branch failed: {e}"), ) .await; } Err(format!("RACE right branch failed: {e}")) } }?; // Parse the subtree output envelope produced by execute_subtree and merge any named // results from the winning branch into the parent results map. If the winning branch // broke out via `df.break()`, `parse_subtree_envelope` returns `NodeError::Break`, which // unwinds via `?` to the enclosing loop. let result = parse_subtree_envelope(&raw, "RACE branch", results)?; // Store result if named if let Some(name) = &node.result_name { ctx.trace_info(format!("Storing RACE result as ${name}")); results.insert(name.clone(), result.clone()); } Ok(result) } async fn execute_http_node( ctx: &OrchestrationContext, node: &FunctionNode, node_id: &str, results: &mut HashMap, exec_ctx: &ExecutionContext, sys_vars: &SystemVars, ) -> NodeResult { let config_str = node .query .as_ref() .ok_or_else(|| format!("HTTP node {node_id} has no config"))?; // Parse config to substitute variables in body and URL let mut config: serde_json::Value = serde_json::from_str(config_str).map_err(|e| format!("Invalid HTTP config: {e}"))?; // Substitute variables in body if present if let Some(body) = config.get("body").and_then(|b| b.as_str()) { let substituted_body = substitute_all_raw(body, results, &exec_ctx.vars, sys_vars)?; config["body"] = serde_json::Value::String(substituted_body); } // Substitute variables in URL if present if let Some(url) = config.get("url").and_then(|u| u.as_str()) { let substituted_url = substitute_all_raw(url, results, &exec_ctx.vars, sys_vars)?; config["url"] = serde_json::Value::String(substituted_url); } // Substitute variables in headers if present // Sort keys for deterministic iteration order if let Some(headers) = config.get("headers").and_then(|h| h.as_object()) { let mut new_headers = serde_json::Map::new(); let mut sorted_keys: Vec<_> = headers.keys().collect(); sorted_keys.sort(); for key in sorted_keys { if let Some(value) = headers.get(key) { if let Some(v) = value.as_str() { let substituted = substitute_all_raw(v, results, &exec_ctx.vars, sys_vars)?; new_headers.insert(key.clone(), serde_json::Value::String(substituted)); } else { new_headers.insert(key.clone(), value.clone()); } } } config["headers"] = serde_json::Value::Object(new_headers); } // Inject audit context from the function node config["submitted_by"] = serde_json::Value::String(node.submitted_by.clone()); let final_config = config.to_string(); let url = config["url"].as_str().unwrap_or("?"); let method = config["method"].as_str().unwrap_or("POST"); ctx.trace_info(format!("Executing HTTP {method} {url}")); let result = ctx .schedule_activity(activities::execute_http::NAME, final_config) .await?; // Store result if named if let Some(name) = &node.result_name { ctx.trace_info(format!("Storing HTTP result as ${name}")); results.insert(name.clone(), result.clone()); } Ok(result) } async fn execute_http_multipart_node( ctx: &OrchestrationContext, node: &FunctionNode, node_id: &str, results: &mut HashMap, exec_ctx: &ExecutionContext, sys_vars: &SystemVars, ) -> NodeResult { let config_str = node .query .as_ref() .ok_or_else(|| format!("HTTP_MULTIPART node {node_id} has no config"))?; let mut config: serde_json::Value = serde_json::from_str(config_str) .map_err(|e| format!("Invalid multipart HTTP config: {e}"))?; // Substitute variables in URL. if let Some(url) = config.get("url").and_then(|u| u.as_str()) { let substituted_url = substitute_all_raw(url, results, &exec_ctx.vars, sys_vars)?; config["url"] = serde_json::Value::String(substituted_url); } // Substitute variables in headers (same pattern as execute_http_node). if let Some(headers) = config.get("headers").and_then(|h| h.as_object()) { let mut new_headers = serde_json::Map::new(); let mut sorted_keys: Vec<_> = headers.keys().collect(); sorted_keys.sort(); for key in sorted_keys { if let Some(value) = headers.get(key) { if let Some(v) = value.as_str() { let substituted = substitute_all_raw(v, results, &exec_ctx.vars, sys_vars)?; new_headers.insert(key.clone(), serde_json::Value::String(substituted)); } else { new_headers.insert(key.clone(), value.clone()); } } } config["headers"] = serde_json::Value::Object(new_headers); } // Substitute variables in each part's text metadata (name, filename) with // the usual inline rules. // // `data_b64` follows a stricter rule: it is substituted only when its entire // value is a single reference (e.g. `$speech.body` or `{payload}`). That is // what lets one step's output become the next step's upload. Splicing a // substitution into the *middle* of a base64 string, by contrast, can only // corrupt the payload, so partial interpolation is rejected outright rather // than silently producing garbage. The base64 alphabet contains neither `$` // nor `{`, so their presence is unambiguously an attempted reference. if let Some(parts) = config.get_mut("parts").and_then(|p| p.as_array_mut()) { for part in parts.iter_mut() { if let Some(name) = part.get("name").and_then(|n| n.as_str()) { let substituted = substitute_all_raw(name, results, &exec_ctx.vars, sys_vars)?; part["name"] = serde_json::Value::String(substituted); } if let Some(filename) = part.get("filename").and_then(|f| f.as_str()) { let substituted = substitute_all_raw(filename, results, &exec_ctx.vars, sys_vars)?; part["filename"] = serde_json::Value::String(substituted); } if let Some(data_b64) = part.get("data_b64").and_then(|d| d.as_str()) { if crate::types::is_whole_value_reference(data_b64) { let substituted = substitute_all_raw(data_b64, results, &exec_ctx.vars, sys_vars)?; part["data_b64"] = serde_json::Value::String(substituted); } else if data_b64.contains('$') || data_b64.contains('{') { let part_name = part.get("name").and_then(|n| n.as_str()).unwrap_or("?"); return Err(NodeError::Failure(format!( "HTTP_MULTIPART node {node_id}: data_b64 for part '{part_name}' mixes a \ variable reference with other text. Only a whole-value reference is \ supported (e.g. data_b64 => '$result' or '{{myvar}}'), because splicing \ into base64 would corrupt the payload." ))); } } } } // Inject audit context from the function node. config["submitted_by"] = serde_json::Value::String(node.submitted_by.clone()); let final_config = config.to_string(); let url = config["url"].as_str().unwrap_or("?"); let method = config["method"].as_str().unwrap_or("POST"); ctx.trace_info(format!("Executing HTTP_MULTIPART {method} {url}")); let result = ctx .schedule_activity(activities::execute_multipart::NAME, final_config) .await?; // Store result if named if let Some(name) = &node.result_name { ctx.trace_info(format!("Storing HTTP_MULTIPART result as ${name}")); results.insert(name.clone(), result.clone()); } Ok(result) } async fn execute_signal_node( ctx: &OrchestrationContext, node: &FunctionNode, node_id: &str, results: &mut HashMap, ) -> NodeResult { let parse_signal_data = |data_str: &str| { serde_json::from_str::(data_str) .unwrap_or_else(|_| serde_json::Value::String(data_str.to_string())) }; let config_str = node .query .as_ref() .ok_or_else(|| format!("SIGNAL node {node_id} has no config"))?; let config: serde_json::Value = serde_json::from_str(config_str).map_err(|e| format!("Invalid SIGNAL config: {e}"))?; let signal_name = config["signal_name"] .as_str() .ok_or("Missing signal_name in SIGNAL config")?; let timeout_seconds = config["timeout_seconds"].as_i64(); ctx.trace_info(format!( "Waiting for signal: {}{}", signal_name, timeout_seconds .map(|t| format!(" (timeout: {t}s)")) .unwrap_or_default() )); let result = if let Some(timeout_secs) = timeout_seconds { // Race between signal and timeout using select2 let signal_fut = ctx.schedule_wait(signal_name); let timeout_fut = ctx.schedule_timer(Duration::from_secs(timeout_secs as u64)); // select2 now returns Either2 instead of (winner_idx, DurableOutput) match ctx.select2(signal_fut, timeout_fut).await { duroxide::Either2::First(data_str) => { // Signal received - data_str is String directly let data = parse_signal_data(&data_str); serde_json::json!({ "signal_name": signal_name, "timed_out": false, "data": data }) } duroxide::Either2::Second(()) => { // Timeout serde_json::json!({ "signal_name": signal_name, "timed_out": true, "data": null }) } } } else { // Wait forever - schedule_wait returns String directly now let data_str = ctx.schedule_wait(signal_name).await; let data = parse_signal_data(&data_str); serde_json::json!({ "signal_name": signal_name, "timed_out": false, "data": data }) }; let result_str = result.to_string(); // Store result if named if let Some(name) = &node.result_name { ctx.trace_info(format!("Storing signal result as ${name}")); results.insert(name.clone(), result_str.clone()); } Ok(result_str) } #[cfg(test)] mod tests { use super::*; /// Build an envelope JSON string the way `execute_subtree` serializes a `SubtreeEnvelope`. /// When `control` is `None` the field is omitted entirely, reproducing an envelope recorded /// by a pre-#148 binary (<= v0.2.2) that had no `control` field. fn envelope_json(control: Option<&str>, result: &str, results: serde_json::Value) -> String { let mut obj = serde_json::Map::new(); if let Some(c) = control { obj.insert( "control".to_string(), serde_json::Value::String(c.to_string()), ); } obj.insert( "result".to_string(), serde_json::Value::String(result.to_string()), ); obj.insert("results".to_string(), results); serde_json::Value::Object(obj).to_string() } /// Reproduce the exact break sentinel string a pre-#148 binary stored inside the envelope's /// `result` field for a `df.break(value)`. fn legacy_sentinel(value: serde_json::Value) -> String { serde_json::json!({ "__break__": true, "value": value }).to_string() } fn expect_break(result: NodeResult) -> String { match result { Err(NodeError::Break(v)) => v, other => panic!("expected NodeError::Break, got {other:?}"), } } fn expect_ok(result: NodeResult) -> String { match result { Ok(v) => v, other => panic!("expected Ok, got {other:?}"), } } #[test] fn parse_legacy_break_sentinel_decodes_string_value() { // A JSON string value round-trips as the quoted JSON string, matching the old // `extract_break_value` (which called `Value::to_string()` on the `value`). assert_eq!( parse_legacy_break_sentinel(&legacy_sentinel(serde_json::json!("hello"))), Some("\"hello\"".to_string()) ); } #[test] fn parse_legacy_break_sentinel_decodes_object_value() { assert_eq!( parse_legacy_break_sentinel(&legacy_sentinel(serde_json::json!({"status": "done"}))), Some("{\"status\":\"done\"}".to_string()) ); } #[test] fn parse_legacy_break_sentinel_decodes_null_value() { assert_eq!( parse_legacy_break_sentinel(&legacy_sentinel(serde_json::Value::Null)), Some("null".to_string()) ); } #[test] fn parse_legacy_break_sentinel_ignores_non_break_json() { assert_eq!(parse_legacy_break_sentinel(r#"{"status":"done"}"#), None); assert_eq!(parse_legacy_break_sentinel(r#"{"__break__":false}"#), None); assert_eq!(parse_legacy_break_sentinel(r#""just a string""#), None); assert_eq!(parse_legacy_break_sentinel("not json at all"), None); } #[test] fn envelope_new_format_break_is_reraised() { let raw = envelope_json(Some("Break"), "\"done\"", serde_json::json!({})); let mut parent = HashMap::new(); assert_eq!( expect_break(parse_subtree_envelope(&raw, "JOIN", &mut parent)), "\"done\"" ); } #[test] fn envelope_new_format_normal_passes_through() { let raw = envelope_json(Some("Normal"), "42", serde_json::json!({})); let mut parent = HashMap::new(); assert_eq!( expect_ok(parse_subtree_envelope(&raw, "JOIN", &mut parent)), "42" ); } #[test] fn envelope_new_format_normal_with_sentinel_shaped_result_is_not_reraised() { // Regression guard for the #229 review finding: a new-binary `Normal` envelope whose // genuine result happens to be shaped like the legacy break sentinel must pass through // untouched. The legacy fallback now runs only when `control` is absent (`None`), so a // JOIN/RACE branch result can no longer impersonate control flow under the new binary. let payload = legacy_sentinel(serde_json::json!("not-a-break")); let raw = envelope_json(Some("Normal"), &payload, serde_json::json!({})); let mut parent = HashMap::new(); assert_eq!( expect_ok(parse_subtree_envelope(&raw, "JOIN", &mut parent)), payload ); } // --- In-flight upgrade path (pre-#148 envelopes, no `control` field) --- #[test] fn legacy_envelope_break_is_reraised_not_swallowed() { // Regression guard for the v0.2.2 -> 0.2.3 upgrade: an envelope recorded by the old // binary smuggled the break as a sentinel in `result` and had no `control` field. The // new binary must re-raise it as a typed Break instead of returning it as a normal // result (which would silently swallow the break and let the loop keep iterating). let raw = envelope_json( None, &legacy_sentinel(serde_json::json!("v")), serde_json::json!({}), ); let mut parent = HashMap::new(); assert_eq!( expect_break(parse_subtree_envelope(&raw, "JOIN", &mut parent)), "\"v\"" ); } #[test] fn legacy_envelope_null_break_is_reraised() { let raw = envelope_json( None, &legacy_sentinel(serde_json::Value::Null), serde_json::json!({}), ); let mut parent = HashMap::new(); assert_eq!( expect_break(parse_subtree_envelope(&raw, "RACE branch", &mut parent)), "null" ); } #[test] fn legacy_envelope_normal_result_passes_through() { // An old envelope whose result is a real value (not a sentinel) is unaffected. let raw = envelope_json(None, r#"{"rows":1}"#, serde_json::json!({})); let mut parent = HashMap::new(); assert_eq!( expect_ok(parse_subtree_envelope(&raw, "JOIN", &mut parent)), r#"{"rows":1}"# ); } #[test] fn envelope_merges_named_results_even_on_break() { // Named results produced inside the branch must still be merged into the parent map // before the break unwinds, on both the new and legacy paths. let raw = envelope_json( Some("Break"), "\"x\"", serde_json::json!({"branch_result": "stored"}), ); let mut parent = HashMap::new(); let _ = parse_subtree_envelope(&raw, "JOIN", &mut parent); assert_eq!( parent.get("branch_result").map(String::as_str), Some("stored") ); } }