diff --git a/docs/sdk-reference/state/serialization.md b/docs/sdk-reference/state/serialization.md index 259059a..5b8016a 100644 --- a/docs/sdk-reference/state/serialization.md +++ b/docs/sdk-reference/state/serialization.md @@ -1,851 +1,298 @@ # Serialization -Learn how the SDK serializes and deserializes data for durable execution checkpoints. +## Checkpointed data transformation -## Table of Contents +The SDK serializes durable operation results to checkpoint storage. When your durable +function replays the SDK uses those stored results rather than re-running the code +wrapped inside the operation. A SerDes (serializer/deserializer) is the object that +serializes the operation result, and deserializes it during replay. -- [Terminology](#terminology) -- [What is serialization?](#what-is-serialization) -- [Key features](#key-features) -- [Default serialization behavior](#default-serialization-behavior) -- [Supported types](#supported-types) -- [Converting non-serializable types](#converting-non-serializable-types) -- [Custom serialization](#custom-serialization) -- [Serialization in configurations](#serialization-in-configurations) -- [Best practices](#best-practices) -- [Troubleshooting](#troubleshooting) -- [FAQ](#faq) +Each SDK ships a default SerDes that handles the most common types. You only need a +custom SerDes when the default SerDes cannot handle your data types, or when you need +special behavior such as encryption, compression or writing to external storage. -[← Back to main index](../index.md) - -## Terminology - -**Serialization** - Converting Python objects to strings for storage in checkpoints. - -**Deserialization** - Converting checkpoint strings back to Python objects. - -**SerDes** - Short for Serializer/Deserializer, a custom class that handles both serialization and deserialization. - -**Checkpoint** - A saved state of execution that includes serialized operation results. - -**Extended types** - Types beyond basic JSON (datetime, Decimal, UUID, bytes) that the SDK serializes automatically. - -**Envelope format** - The SDK's internal format that wraps complex types with type tags for accurate deserialization. - -[↑ Back to top](#table-of-contents) - -## What is serialization? - -Serialization converts Python objects into strings that can be stored in checkpoints. When your durable function resumes, deserialization converts those strings back into Python objects. The SDK handles this automatically for most types. - -[↑ Back to top](#table-of-contents) - -## Key features - -- Automatic serialization for common Python types -- Extended type support (datetime, Decimal, UUID, bytes) -- Custom serialization for complex objects -- Type preservation during round-trip serialization -- Efficient plain JSON for primitives - -[↑ Back to top](#table-of-contents) - -## Default serialization behavior - -The SDK handles most Python types automatically: +The following example uses no SerDes configuration. The SDK serializes and deserializes +the step result automatically. === "TypeScript" - ``` typescript - --8<-- "examples/typescript/advanced/serialization/default-behavior.ts" + ```typescript + --8<-- "examples/typescript/sdk-reference/serialization/walkthrough.ts" ``` === "Python" - ``` python - --8<-- "examples/python/advanced/serialization/default-behavior.py" + ```python + --8<-- "examples/python/sdk-reference/serialization/walkthrough.py" ``` === "Java" - ``` java - --8<-- "examples/java/advanced/serialization/default-behavior.java" + ```java + --8<-- "examples/java/sdk-reference/serialization/Walkthrough.java" ``` +## Lambda handler serialization -The SDK serializes data automatically when: -- Checkpointing step results -- Storing callback payloads -- Passing data to child contexts -- Returning results from your handler - -[↑ Back to top](#table-of-contents) - -## Supported types - -### Primitive types - -These types serialize as plain JSON for performance: +The Durable Execution SDK SerDes only applies to durable operation results. It does not +affect the final return value of your Lambda handler, which Lambda serializes +separately. === "TypeScript" - ``` typescript - --8<-- "examples/typescript/advanced/serialization/primitive-types.ts" - ``` + Lambda serializes handler return values with `JSON.stringify`. The same types that work + with `defaultSerdes` work as handler return values. === "Python" - ``` python - --8<-- "examples/python/advanced/serialization/primitive-types.py" - ``` + Lambda serializes handler return values with `json.dumps`. Types like `datetime` and + `Decimal` are safe inside steps because `ExtendedTypeSerDes` handles them, but returning + them directly from your handler raises + `TypeError: Object of type X is not JSON serializable`. This is a common error when + working with Amazon DynamoDB results. Convert those values to JSON-safe types before + returning from the handler. === "Java" - ``` java - --8<-- "examples/java/advanced/serialization/primitive-types.java" - ``` + Lambda serializes handler return values with Jackson. The same `JacksonSerDes` + configuration applies to both step results and handler return values when you use + `DurableHandler`. +## Default serialization -**Supported primitive types:** -- `None` -- `str` -- `int` -- `float` -- `bool` -- Lists containing only primitives - -[↑ Back to top](#table-of-contents) - -### Extended types - -The SDK automatically handles these types using envelope format: +Each SDK uses a default SerDes when you do not provide one. === "TypeScript" - ``` typescript - --8<-- "examples/typescript/advanced/serialization/extended-types.ts" - ``` + The default is `defaultSerdes`, which uses `JSON.stringify` to serialize and + `JSON.parse` to deserialize. It handles any value that `JSON.stringify` accepts. === "Python" - ``` python - --8<-- "examples/python/advanced/serialization/extended-types.py" - ``` - -=== "Java" - - ``` java - --8<-- "examples/java/advanced/serialization/extended-types.java" - ``` - - -**Supported extended types:** -- `datetime` - ISO format with timezone -- `date` - ISO date format -- `Decimal` - Precise decimal numbers -- `UUID` - Universally unique identifiers -- `bytes`, `bytearray`, `memoryview` - Binary data (base64 encoded) -- `tuple` - Immutable sequences -- `list` - Mutable sequences (including nested) -- `dict` - Dictionaries (including nested) + The default is `ExtendedTypeSerDes`. It uses plain JSON for primitives (`None`, `str`, + `int`, `float`, `bool`, and lists of primitives) and an envelope format for everything + else. The envelope format preserves the exact Python type through the round-trip. -[↑ Back to top](#table-of-contents) - -### Container types - -Containers can hold any supported type, including nested containers: - -=== "TypeScript" - - ``` typescript - --8<-- "examples/typescript/advanced/serialization/container-types.ts" - ``` - -=== "Python" - - ``` python - --8<-- "examples/python/advanced/serialization/container-types.py" - ``` + Supported types beyond primitives: `datetime`, `date`, `Decimal`, `UUID`, `bytes`, + `bytearray`, `memoryview`, `tuple`, `list`, `dict`, and `BatchResult`. === "Java" - ``` java - --8<-- "examples/java/advanced/serialization/container-types.java" - ``` + The default is `JacksonSerDes`, which uses Jackson's `ObjectMapper`. It supports Java 8 + time types, serializes dates as ISO-8601 strings, and ignores unknown properties during + deserialization. + Pass a custom `ObjectMapper` to the `JacksonSerDes` constructor to override the default + configuration. -[↑ Back to top](#table-of-contents) - -## Converting non-serializable types - -Some Python types aren't serializable by default. Convert them before passing to durable operations. - -### Dataclasses - -Convert dataclasses to dictionaries: +## SerDes interface definition === "TypeScript" - ``` typescript - --8<-- "examples/typescript/advanced/serialization/dataclasses.ts" - ``` - -=== "Python" - - ``` python - --8<-- "examples/python/advanced/serialization/dataclasses.py" - ``` - -=== "Java" - - ``` java - --8<-- "examples/java/advanced/serialization/dataclasses.java" + ```typescript + --8<-- "examples/typescript/sdk-reference/serialization/serdes-interface.ts" ``` + **Parameters:** -### Pydantic models + - `serialize` An async function that receives the value and a `SerdesContext`, and + returns `Promise`. + - `deserialize` An async function that receives the serialized string and a + `SerdesContext`, and returns `Promise`. -Use Pydantic's built-in serialization: + Both methods are async so that implementations can interact with external services such + as S3 or KMS. -=== "TypeScript" + **SerdesContext fields:** - ``` typescript - --8<-- "examples/typescript/advanced/serialization/convert-to-dicts.ts" - ``` + - `entityId` The operation ID for the current step or operation. + - `durableExecutionArn` The ARN of the current durable execution. === "Python" - ``` python - --8<-- "examples/python/advanced/serialization/convert-to-dicts.py" + ```python + --8<-- "examples/python/sdk-reference/serialization/serdes-interface.py" ``` -=== "Java" - - ``` java - --8<-- "examples/java/advanced/serialization/convert-to-dicts.java" - ``` - - -### Custom objects - -Implement `to_dict()` and `from_dict()` methods: - -=== "TypeScript" + **Parameters:** - ``` typescript - --8<-- "examples/typescript/advanced/serialization/custom-objects.ts" - ``` + - `serialize(value, serdes_context)` Converts the value to a string. + - `deserialize(data, serdes_context)` Converts the string back to the original type. -=== "Python" + **SerDesContext fields:** - ``` python - --8<-- "examples/python/advanced/serialization/custom-objects.py" - ``` + - `operation_id` The operation ID for the current step or operation. + - `durable_execution_arn` The ARN of the current durable execution. === "Java" - ``` java - --8<-- "examples/java/advanced/serialization/custom-objects.java" - ``` - - -[↑ Back to top](#table-of-contents) - -## Custom serialization - -Implement custom serialization for specialized needs like encryption or compression. - -### Creating a custom SerDes - -Extend the `SerDes` base class: - -=== "TypeScript" - - ``` typescript - --8<-- "examples/typescript/advanced/serialization/custom-serdes.ts" + ```java + --8<-- "examples/java/sdk-reference/serialization/SerdesInterface.java" ``` -=== "Python" + **Parameters:** - ``` python - --8<-- "examples/python/advanced/serialization/custom-serdes.py" - ``` + - `serialize(Object value)` Converts the value to a JSON string. Returns `null` if + `value` is `null`. + - `deserialize(String data, TypeToken typeToken)` Converts the JSON string back to + type `T`. Returns `null` if `data` is `null`. -=== "Java" + Use `TypeToken` to capture generic type information that Java erases at runtime. For + example: `new TypeToken>() {}`. - ``` java - --8<-- "examples/java/advanced/serialization/custom-serdes.java" - ``` +## Custom SerDes example - -### Using custom SerDes with steps - -Pass your custom SerDes in `StepConfig`: +Implement the SerDes interface when the default cannot handle your types, or when you +need special behavior such as encryption or compression. === "TypeScript" - ``` typescript - --8<-- "examples/typescript/advanced/serialization/use-custom-serdes.ts" + ```typescript + --8<-- "examples/typescript/sdk-reference/serialization/custom-serdes.ts" ``` === "Python" - ``` python - --8<-- "examples/python/advanced/serialization/use-custom-serdes.py" + ```python + --8<-- "examples/python/sdk-reference/serialization/custom-serdes.py" ``` === "Java" - ``` java - --8<-- "examples/java/advanced/serialization/use-custom-serdes.java" + ```java + --8<-- "examples/java/sdk-reference/serialization/OrderSerDes.java" ``` +## Custom SerDes on durable operations -### Encryption example - -Encrypt sensitive data in checkpoints: - -=== "TypeScript" - - ``` typescript - --8<-- "examples/typescript/advanced/serialization/encryption.ts" - ``` - -=== "Python" - - ``` python - --8<-- "examples/python/advanced/serialization/encryption.py" - ``` - -=== "Java" - - ``` java - --8<-- "examples/java/advanced/serialization/encryption.java" - ``` - - -[↑ Back to top](#table-of-contents) - -## Serialization in configurations - -Different operations support custom serialization through their configuration objects. +Pass a SerDes instance in the configuration object for the operation you want to +customize. The SDK uses that SerDes for that operation only. Other operations in the +same handler continue to use the default. ### StepConfig -Control serialization for step results: - === "TypeScript" - ``` typescript - --8<-- "examples/typescript/advanced/serialization/step-config.ts" + ```typescript + --8<-- "examples/typescript/sdk-reference/serialization/step-config.ts" ``` === "Python" - ``` python - --8<-- "examples/python/advanced/serialization/step-config.py" + ```python + --8<-- "examples/python/sdk-reference/serialization/step-config.py" ``` === "Java" - ``` java - --8<-- "examples/java/advanced/serialization/step-config.java" + ```java + --8<-- "examples/java/sdk-reference/serialization/StepConfigExample.java" ``` - ### CallbackConfig -Control serialization for callback payloads: - -=== "TypeScript" - - ``` typescript - --8<-- "examples/typescript/advanced/serialization/callback-config.ts" - ``` - -=== "Python" - - ``` python - --8<-- "examples/python/advanced/serialization/callback-config.py" - ``` - -=== "Java" - - ``` java - --8<-- "examples/java/advanced/serialization/callback-config.java" - ``` - - -### MapConfig and ParallelConfig - -Control serialization for batch results: - -=== "TypeScript" - - ``` typescript - --8<-- "examples/typescript/advanced/serialization/map-parallel-config.ts" - ``` - -=== "Python" - - ``` python - --8<-- "examples/python/advanced/serialization/map-parallel-config.py" - ``` - -=== "Java" - - ``` java - --8<-- "examples/java/advanced/serialization/map-parallel-config.java" - ``` - - -**Note:** When both `serdes` and `item_serdes` are provided: -- `item_serdes` serializes individual item results in child contexts -- `serdes` serializes the entire `BatchResult` at the handler level - -For backward compatibility, if only `serdes` is provided, it's used for both individual items and the `BatchResult`. - -[↑ Back to top](#table-of-contents) - -## Best practices - -### Use default serialization when possible - -The SDK handles most cases efficiently without custom serialization: - -=== "TypeScript" - - ``` typescript - --8<-- "examples/typescript/advanced/serialization/use-default-serialization.ts" - ``` - -=== "Python" - - ``` python - --8<-- "examples/python/advanced/serialization/use-default-serialization.py" - ``` - -=== "Java" - - ``` java - --8<-- "examples/java/advanced/serialization/use-default-serialization.java" - ``` - - -### Convert complex objects to dicts - -Convert custom objects to dictionaries before passing to durable operations: - -=== "TypeScript" - - ``` typescript - --8<-- "examples/typescript/advanced/serialization/convert-to-dicts.ts" - ``` - -=== "Python" - - ``` python - --8<-- "examples/python/advanced/serialization/convert-to-dicts.py" - ``` - -=== "Java" - - ``` java - --8<-- "examples/java/advanced/serialization/convert-to-dicts.java" - ``` - - -### Keep serialized data small - -Large checkpoints might slow down execution. Keep data compact: - -=== "TypeScript" - - ``` typescript - --8<-- "examples/typescript/advanced/serialization/keep-data-small.ts" - ``` - -=== "Python" - - ``` python - --8<-- "examples/python/advanced/serialization/keep-data-small.py" - ``` - -=== "Java" - - ``` java - --8<-- "examples/java/advanced/serialization/keep-data-small.java" - ``` - - -### Use appropriate types - -Choose types that serialize efficiently: +The callback SerDes controls how the SDK deserializes the payload that an external +system sends when it completes the callback. === "TypeScript" - ``` typescript - --8<-- "examples/typescript/advanced/serialization/appropriate-types.ts" - ``` - -=== "Python" + In TypeScript, the callback SerDes only needs `deserialize`. The `serialize` method is + not used because the external system provides the payload directly. - ``` python - --8<-- "examples/python/advanced/serialization/appropriate-types.py" - ``` - -=== "Java" - - ``` java - --8<-- "examples/java/advanced/serialization/appropriate-types.java" - ``` - - -### Test serialization round-trips - -Verify your data survives serialization: - -=== "TypeScript" - - ``` typescript - --8<-- "examples/typescript/advanced/serialization/test-round-trips.ts" + ```typescript + --8<-- "examples/typescript/sdk-reference/serialization/callback-config.ts" ``` === "Python" - ``` python - --8<-- "examples/python/advanced/serialization/test-round-trips.py" + ```python + --8<-- "examples/python/sdk-reference/serialization/callback-config.py" ``` === "Java" - ``` java - --8<-- "examples/java/advanced/serialization/test-round-trips.java" + ```java + --8<-- "examples/java/sdk-reference/serialization/CallbackConfigExample.java" ``` +### MapConfig & ParallelConfig -### Handle serialization errors gracefully - -Catch and handle serialization errors: +Map and parallel perations support two SerDes fields that apply at different levels. === "TypeScript" - ``` typescript - --8<-- "examples/typescript/advanced/serialization/handle-errors-gracefully.ts" + ```typescript + --8<-- "examples/typescript/sdk-reference/serialization/map-config.ts" ``` -=== "Python" - - ``` python - --8<-- "examples/python/advanced/serialization/handle-errors-gracefully.py" - ``` - -=== "Java" - - ``` java - --8<-- "examples/java/advanced/serialization/handle-errors-gracefully.java" - ``` - - -[↑ Back to top](#table-of-contents) - -## Troubleshooting - -### Unsupported type error - -**Problem:** `SerDesError: Unsupported type: ` - -**Solution:** Convert custom objects to supported types: - -=== "TypeScript" - - ``` typescript - --8<-- "examples/typescript/advanced/serialization/unsupported-type-error.ts" - ``` + - `itemSerdes` serializes each item result. + - `serdes` serializes the aggregated `BatchResult`. + - `ParallelConfig` has the same two fields. === "Python" - ``` python - --8<-- "examples/python/advanced/serialization/unsupported-type-error.py" + ```python + --8<-- "examples/python/sdk-reference/serialization/map-config.py" ``` -=== "Java" - - ``` java - --8<-- "examples/java/advanced/serialization/unsupported-type-error.java" - ``` - - -### Serialization failed error - -**Problem:** `ExecutionError: Serialization failed for id: step-123` - -**Cause:** The data contains types that can't be serialized. - -**Solution:** Check for circular references or unsupported types: - -=== "TypeScript" - - ``` typescript - --8<-- "examples/typescript/advanced/serialization/serialization-failed-error.ts" - ``` - -=== "Python" - - ``` python - --8<-- "examples/python/advanced/serialization/serialization-failed-error.py" - ``` + - `item_serdes` serializes each item result. + - `serdes` serializes the aggregated `BatchResult`. + - When you provide only `serdes`, the SDK uses it for both for backward compatibility. + `ParallelConfig` has the same two fields. === "Java" - ``` java - --8<-- "examples/java/advanced/serialization/serialization-failed-error.java" + ```java + --8<-- "examples/java/sdk-reference/serialization/MapConfigExample.java" ``` + - `serDes` applies to each item result. + - Java `ParallelConfig` does not have a `serDes` field. -### Type not preserved after deserialization - -**Problem:** `tuple` becomes `list` or `Decimal` becomes `float` - -**Cause:** Using a custom SerDes that doesn't preserve types. - -**Solution:** Use default serialization which preserves types: +## Built-in SerDes helpers === "TypeScript" - ``` typescript - --8<-- "examples/typescript/advanced/serialization/type-not-preserved.ts" - ``` - -=== "Python" - - ``` python - --8<-- "examples/python/advanced/serialization/type-not-preserved.py" - ``` - -=== "Java" - - ``` java - --8<-- "examples/java/advanced/serialization/type-not-preserved.java" - ``` - - -### Large payload errors - -**Problem:** Checkpoint size exceeds limits - -**Solution:** Reduce data size or use summary generators: - -=== "TypeScript" + `createClassSerdes(cls)` creates a `Serdes` that preserves class instances through + the round-trip. It deserializes by calling `Object.assign(new cls(), JSON.parse(data))`, + so class methods are available on the deserialized value. The constructor must take no + required parameters. Private fields and getters are not preserved. - ``` typescript - --8<-- "examples/typescript/advanced/serialization/large-payload-errors.ts" + ```typescript + --8<-- "examples/typescript/sdk-reference/serialization/builtin-helpers.ts" ``` === "Python" - ``` python - --8<-- "examples/python/advanced/serialization/large-payload-errors.py" - ``` - -=== "Java" - - ``` java - --8<-- "examples/java/advanced/serialization/large-payload-errors.java" - ``` - - -### Datetime timezone issues - -**Problem:** Datetime loses timezone information - -**Solution:** Always use timezone-aware datetime objects: - -=== "TypeScript" + `PassThroughSerDes` stores the value as-is (the value must already be a string). + `JsonSerDes` uses `json.dumps` and `json.loads` without the envelope format that + `ExtendedTypeSerDes` adds for complex types. - ``` typescript - --8<-- "examples/typescript/advanced/serialization/datetime-timezone.ts" - ``` - -=== "Python" - - ``` python - --8<-- "examples/python/advanced/serialization/datetime-timezone.py" + ```python + --8<-- "examples/python/sdk-reference/serialization/builtin-helpers.py" ``` === "Java" - ``` java - --8<-- "examples/java/advanced/serialization/datetime-timezone.java" - ``` + The built-in `JacksonSerDes` handles class instances via Jackson's `ObjectMapper`. You + can create your own passthrough SerDes like this: - -[↑ Back to top](#table-of-contents) - -## FAQ - -### What types can I serialize? - -The SDK supports: -- Primitives: `None`, `str`, `int`, `float`, `bool` -- Extended: `datetime`, `date`, `Decimal`, `UUID`, `bytes`, `tuple` -- Containers: `list`, `dict` (including nested) - -For other types, convert to dictionaries first. - -### Do I need custom serialization? - -Most applications don't need custom serialization. Use it for: -- Encryption of sensitive data -- Compression of large payloads -- Special encoding requirements -- Legacy format compatibility - -### How does serialization affect performance? - -The SDK optimizes for performance: -- Primitives use plain JSON (fast) -- Extended types use envelope format (slightly slower but preserves types) -- Custom SerDes adds overhead based on your implementation - -### Can I serialize Pydantic models? - -Yes, convert them to dictionaries: - -=== "TypeScript" - - ``` typescript - --8<-- "examples/typescript/advanced/serialization/faq-pydantic-models.ts" - ``` - -=== "Python" - - ``` python - --8<-- "examples/python/advanced/serialization/faq-pydantic-models.py" - ``` - -=== "Java" - - ``` java - --8<-- "examples/java/advanced/serialization/faq-pydantic-models.java" - ``` - - -### What's the difference between serdes and item_serdes? - -In `MapConfig` and `ParallelConfig`: -- `item_serdes`: Serializes individual item results in child contexts -- `serdes`: Serializes the entire `BatchResult` at handler level - -If only `serdes` is provided, it's used for both (backward compatibility). - -### How do I handle binary data? - -Use `bytes` type - it's automatically base64 encoded: - -=== "TypeScript" - - ``` typescript - --8<-- "examples/typescript/advanced/serialization/faq-binary-data.ts" - ``` - -=== "Python" - - ``` python - --8<-- "examples/python/advanced/serialization/faq-binary-data.py" - ``` - -=== "Java" - - ``` java - --8<-- "examples/java/advanced/serialization/faq-binary-data.java" - ``` - - -### Can I use JSON strings directly? - -Yes, use `PassThroughSerDes` or `JsonSerDes`: - -=== "TypeScript" - - ``` typescript - --8<-- "examples/typescript/advanced/serialization/faq-json-strings.ts" - ``` - -=== "Python" - - ``` python - --8<-- "examples/python/advanced/serialization/faq-json-strings.py" - ``` - -=== "Java" - - ``` java - --8<-- "examples/java/advanced/serialization/faq-json-strings.java" - ``` - - -### What happens if serialization fails? - -The SDK raises `ExecutionError` with details. Handle it in your code: - -=== "TypeScript" - - ``` typescript - --8<-- "examples/typescript/advanced/serialization/faq-serialization-fails.ts" - ``` - -=== "Python" - - ``` python - --8<-- "examples/python/advanced/serialization/faq-serialization-fails.py" + ```java + --8<-- "examples/java/sdk-reference/serialization/PassThroughSerdesExample.java" ``` -=== "Java" - - ``` java - --8<-- "examples/java/advanced/serialization/faq-serialization-fails.java" - ``` - - -### How do I debug serialization issues? - -Test serialization independently: - -=== "TypeScript" - - ``` typescript - --8<-- "examples/typescript/advanced/serialization/faq-debug-issues.ts" - ``` - -=== "Python" - - ``` python - --8<-- "examples/python/advanced/serialization/faq-debug-issues.py" - ``` +## Serialization errors -=== "Java" - - ``` java - --8<-- "examples/java/advanced/serialization/faq-debug-issues.java" - ``` - - -### Are there size limits for serialized data? - -Yes, checkpoints have size limits (typically 256KB). Keep data compact: -- Only checkpoint necessary data -- Use summary generators for large results -- Store large data externally (S3) and checkpoint references - -[↑ Back to top](#table-of-contents) +When serialization or deserialization fails, each SDK raises or throws a specific +exception type. See +[Serialization errors](../error-handling/errors.md#serialization-errors) for the +exception hierarchy and how to handle serialization failures. ## See also -- [Steps](../core/steps.md) - Using steps with custom serialization -- [Callbacks](../core/callbacks.md) - Serializing callback payloads -- [Map Operations](../core/map.md) - Serialization in map operations -- [Error Handling](error-handling.md) - Handling serialization errors -- [Best Practices](../best-practices.md) - General best practices - -[↑ Back to index](#table-of-contents) +- [StepConfig](../operations/step.md#stepconfig) Step serialization configuration +- [CallbackConfig](../operations/callback.md#callbackconfig) Callback serialization + configuration +- [MapConfig](../operations/map.md#mapconfig) Map serialization configuration +- [Serialization errors](../error-handling/errors.md#serialization-errors) Serialization + error types diff --git a/examples/java/sdk-reference/serialization/CallbackConfigExample.java b/examples/java/sdk-reference/serialization/CallbackConfigExample.java new file mode 100644 index 0000000..842b586 --- /dev/null +++ b/examples/java/sdk-reference/serialization/CallbackConfigExample.java @@ -0,0 +1,21 @@ +import java.util.Map; +import software.amazon.lambda.durable.DurableCallbackFuture; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.config.CallbackConfig; +import software.amazon.lambda.durable.serde.JacksonSerDes; + +public class CallbackConfigExample extends DurableHandler> { + @Override + public Map handleRequest(Object input, DurableContext context) { + CallbackConfig config = CallbackConfig.builder() + .serDes(new JacksonSerDes()) + .build(); + + DurableCallbackFuture> callback = + context.createCallback("await-approval", Map.class, config); + + // Send callback.getCallbackId() to the external system here. + return callback.get(); + } +} diff --git a/examples/java/sdk-reference/serialization/MapConfigExample.java b/examples/java/sdk-reference/serialization/MapConfigExample.java new file mode 100644 index 0000000..248ecd2 --- /dev/null +++ b/examples/java/sdk-reference/serialization/MapConfigExample.java @@ -0,0 +1,26 @@ +import java.util.List; +import java.util.Map; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.config.MapConfig; +import software.amazon.lambda.durable.model.MapResult; +import software.amazon.lambda.durable.serde.JacksonSerDes; + +public class MapConfigExample extends DurableHandler>> { + @Override + public List> handleRequest(Object input, DurableContext context) { + MapConfig config = MapConfig.builder() + .serDes(new JacksonSerDes()) + .build(); + + List items = List.of("a", "b", "c"); + MapResult> result = context.map( + "process-items", + items, + Map.class, + (item, index, ctx) -> Map.of("id", item, "status", "done"), + config + ); + return result.succeeded(); + } +} diff --git a/examples/java/sdk-reference/serialization/OrderSerDes.java b/examples/java/sdk-reference/serialization/OrderSerDes.java new file mode 100644 index 0000000..1fa4f41 --- /dev/null +++ b/examples/java/sdk-reference/serialization/OrderSerDes.java @@ -0,0 +1,26 @@ +import com.fasterxml.jackson.databind.ObjectMapper; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.SerDesException; +import software.amazon.lambda.durable.serde.SerDes; + +public class OrderSerDes implements SerDes { + private final ObjectMapper mapper = new ObjectMapper(); + + @Override + public String serialize(Object value) { + try { + return mapper.writeValueAsString(value); + } catch (Exception e) { + throw new SerDesException("Serialization failed", e); + } + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + try { + return mapper.readValue(data, mapper.getTypeFactory().constructType(typeToken.getType())); + } catch (Exception e) { + throw new SerDesException("Deserialization failed", e); + } + } +} diff --git a/examples/java/sdk-reference/serialization/PassThroughSerdesExample.java b/examples/java/sdk-reference/serialization/PassThroughSerdesExample.java new file mode 100644 index 0000000..60f82da --- /dev/null +++ b/examples/java/sdk-reference/serialization/PassThroughSerdesExample.java @@ -0,0 +1,38 @@ +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.StepContext; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.exception.SerDesException; +import software.amazon.lambda.durable.serde.SerDes; + +// A pass-through SerDes stores the value as-is (already a JSON string). +class PassThroughSerDes implements SerDes { + @Override + public String serialize(Object value) { + return (String) value; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + @SuppressWarnings("unchecked") + T result = (T) data; + return result; + } +} + +public class PassThroughSerdesExample extends DurableHandler { + @Override + public String handleRequest(Object input, DurableContext context) { + StepConfig config = StepConfig.builder() + .serDes(new PassThroughSerDes()) + .build(); + + return context.step( + "fetch-raw", + String.class, + (StepContext ctx) -> "{\"id\":\"order-123\"}", + config + ); + } +} diff --git a/examples/java/sdk-reference/serialization/SerdesInterface.java b/examples/java/sdk-reference/serialization/SerdesInterface.java new file mode 100644 index 0000000..c20cc21 --- /dev/null +++ b/examples/java/sdk-reference/serialization/SerdesInterface.java @@ -0,0 +1,9 @@ +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.serde.SerDes; + +// SerDes interface +interface SerDesInterface { + String serialize(Object value); + + T deserialize(String data, TypeToken typeToken); +} diff --git a/examples/java/sdk-reference/serialization/StepConfigExample.java b/examples/java/sdk-reference/serialization/StepConfigExample.java new file mode 100644 index 0000000..77a55a3 --- /dev/null +++ b/examples/java/sdk-reference/serialization/StepConfigExample.java @@ -0,0 +1,23 @@ +import java.util.Map; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.StepContext; +import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.serde.JacksonSerDes; + +public class StepConfigExample extends DurableHandler> { + @Override + public Map handleRequest(Object input, DurableContext context) { + StepConfig config = StepConfig.builder() + .serDes(new JacksonSerDes()) + .build(); + + Map order = context.step( + "fetch-order", + Map.class, + (StepContext ctx) -> Map.of("id", "order-123", "total", "99.99"), + config + ); + return order; + } +} diff --git a/examples/java/sdk-reference/serialization/Walkthrough.java b/examples/java/sdk-reference/serialization/Walkthrough.java new file mode 100644 index 0000000..29d4148 --- /dev/null +++ b/examples/java/sdk-reference/serialization/Walkthrough.java @@ -0,0 +1,17 @@ +import java.util.Map; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.StepContext; + +public class Walkthrough extends DurableHandler> { + @Override + public Map handleRequest(Object input, DurableContext context) { + // No SerDes config — the SDK serializes and deserializes the result automatically. + Map order = context.step( + "fetch-order", + Map.class, + (StepContext ctx) -> Map.of("id", "order-123", "total", "99.99") + ); + return order; + } +} diff --git a/examples/python/sdk-reference/serialization/builtin-helpers.py b/examples/python/sdk-reference/serialization/builtin-helpers.py new file mode 100644 index 0000000..72158ae --- /dev/null +++ b/examples/python/sdk-reference/serialization/builtin-helpers.py @@ -0,0 +1,25 @@ +from aws_durable_execution_sdk_python import ( + DurableContext, + StepContext, + durable_execution, + durable_step, +) +from aws_durable_execution_sdk_python.config import StepConfig +from aws_durable_execution_sdk_python.serdes import JsonSerDes, PassThroughSerDes + +# PassThroughSerDes — stores the value as-is (must already be a string) +# JsonSerDes — standard json.dumps / json.loads, no envelope format + + +@durable_step +def fetch_raw(ctx: StepContext) -> str: + return '{"id":"order-123"}' + + +@durable_execution +def handler(event: dict, context: DurableContext) -> str: + raw = context.step( + fetch_raw(), + config=StepConfig(serdes=PassThroughSerDes()), + ) + return raw diff --git a/examples/python/sdk-reference/serialization/callback-config.py b/examples/python/sdk-reference/serialization/callback-config.py new file mode 100644 index 0000000..cdc6449 --- /dev/null +++ b/examples/python/sdk-reference/serialization/callback-config.py @@ -0,0 +1,28 @@ +import json + +from aws_durable_execution_sdk_python import ( + DurableContext, + durable_execution, +) +from aws_durable_execution_sdk_python.config import CallbackConfig +from aws_durable_execution_sdk_python.serdes import SerDes, SerDesContext + + +class ApprovalSerDes(SerDes[dict]): + def serialize(self, value: dict, ctx: SerDesContext) -> str: + return json.dumps(value) + + def deserialize(self, data: str, ctx: SerDesContext) -> dict: + return json.loads(data) + + +@durable_execution +def handler(event: dict, context: DurableContext) -> dict: + callback = context.create_callback( + "await-approval", + config=CallbackConfig(serdes=ApprovalSerDes()), + ) + + # Send callback.callback_id to the external system here. + result = callback.result() + return result diff --git a/examples/python/sdk-reference/serialization/custom-serdes.py b/examples/python/sdk-reference/serialization/custom-serdes.py new file mode 100644 index 0000000..66fd5a1 --- /dev/null +++ b/examples/python/sdk-reference/serialization/custom-serdes.py @@ -0,0 +1,11 @@ +import json + +from aws_durable_execution_sdk_python.serdes import SerDes, SerDesContext + + +class OrderSerDes(SerDes[dict]): + def serialize(self, value: dict, ctx: SerDesContext) -> str: + return json.dumps(value) + + def deserialize(self, data: str, ctx: SerDesContext) -> dict: + return json.loads(data) diff --git a/examples/python/sdk-reference/serialization/map-config.py b/examples/python/sdk-reference/serialization/map-config.py new file mode 100644 index 0000000..280018d --- /dev/null +++ b/examples/python/sdk-reference/serialization/map-config.py @@ -0,0 +1,27 @@ +import json + +from aws_durable_execution_sdk_python import ( + DurableContext, + durable_execution, +) +from aws_durable_execution_sdk_python.config import MapConfig +from aws_durable_execution_sdk_python.serdes import SerDes, SerDesContext + + +class ItemSerDes(SerDes[dict]): + def serialize(self, value: dict, ctx: SerDesContext) -> str: + return json.dumps(value) + + def deserialize(self, data: str, ctx: SerDesContext) -> dict: + return json.loads(data) + + +@durable_execution +def handler(event: dict, context: DurableContext) -> list: + items = ["a", "b", "c"] + result = context.map( + items, + lambda ctx, item, idx, arr: {"id": item, "status": "done"}, + config=MapConfig(item_serdes=ItemSerDes()), + ) + return result.get_results() diff --git a/examples/python/sdk-reference/serialization/pass-through-serdes.py b/examples/python/sdk-reference/serialization/pass-through-serdes.py new file mode 100644 index 0000000..faaece6 --- /dev/null +++ b/examples/python/sdk-reference/serialization/pass-through-serdes.py @@ -0,0 +1,22 @@ +from aws_durable_execution_sdk_python import ( + DurableContext, + StepContext, + durable_execution, + durable_step, +) +from aws_durable_execution_sdk_python.config import StepConfig +from aws_durable_execution_sdk_python.serdes import PassThroughSerDes + + +@durable_step +def fetch_raw(ctx: StepContext) -> str: + return '{"id":"order-123"}' + + +@durable_execution +def handler(event: dict, context: DurableContext) -> str: + raw = context.step( + fetch_raw(), + config=StepConfig(serdes=PassThroughSerDes()), + ) + return raw diff --git a/examples/python/sdk-reference/serialization/serdes-interface.py b/examples/python/sdk-reference/serialization/serdes-interface.py new file mode 100644 index 0000000..273182c --- /dev/null +++ b/examples/python/sdk-reference/serialization/serdes-interface.py @@ -0,0 +1,19 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Generic, TypeVar + +T = TypeVar("T") + + +@dataclass(frozen=True) +class SerDesContext: + operation_id: str = "" + durable_execution_arn: str = "" + + +class SerDes(ABC, Generic[T]): + @abstractmethod + def serialize(self, value: T, serdes_context: SerDesContext) -> str: ... + + @abstractmethod + def deserialize(self, data: str, serdes_context: SerDesContext) -> T: ... diff --git a/examples/python/sdk-reference/serialization/step-config.py b/examples/python/sdk-reference/serialization/step-config.py new file mode 100644 index 0000000..04feccc --- /dev/null +++ b/examples/python/sdk-reference/serialization/step-config.py @@ -0,0 +1,32 @@ +import json + +from aws_durable_execution_sdk_python import ( + DurableContext, + StepContext, + durable_execution, + durable_step, +) +from aws_durable_execution_sdk_python.config import StepConfig +from aws_durable_execution_sdk_python.serdes import SerDes, SerDesContext + + +class OrderSerDes(SerDes[dict]): + def serialize(self, value: dict, ctx: SerDesContext) -> str: + return json.dumps(value) + + def deserialize(self, data: str, ctx: SerDesContext) -> dict: + return json.loads(data) + + +@durable_step +def fetch_order(ctx: StepContext) -> dict: + return {"id": "order-123", "total": "99.99"} + + +@durable_execution +def handler(event: dict, context: DurableContext) -> dict: + order = context.step( + fetch_order(), + config=StepConfig(serdes=OrderSerDes()), + ) + return order diff --git a/examples/python/sdk-reference/serialization/walkthrough.py b/examples/python/sdk-reference/serialization/walkthrough.py new file mode 100644 index 0000000..dd301e6 --- /dev/null +++ b/examples/python/sdk-reference/serialization/walkthrough.py @@ -0,0 +1,18 @@ +from aws_durable_execution_sdk_python import ( + DurableContext, + StepContext, + durable_execution, + durable_step, +) + + +@durable_step +def fetch_order(ctx: StepContext) -> dict: + return {"id": "order-123", "total": "99.99"} + + +@durable_execution +def handler(event: dict, context: DurableContext) -> dict: + # No SerDes config — the SDK serializes and deserializes the result automatically. + order = context.step(fetch_order()) + return order diff --git a/examples/typescript/sdk-reference/serialization/builtin-helpers.ts b/examples/typescript/sdk-reference/serialization/builtin-helpers.ts new file mode 100644 index 0000000..9d165e7 --- /dev/null +++ b/examples/typescript/sdk-reference/serialization/builtin-helpers.ts @@ -0,0 +1,28 @@ +import { + DurableContext, + createClassSerdes, + withDurableExecution, +} from "@aws/durable-execution-sdk-js"; + +class Order { + id: string = ""; + total: string = ""; + + label() { + return `Order ${this.id} — ${this.total}`; + } +} + +const orderSerdes = createClassSerdes(Order); + +export const handler = withDurableExecution( + async (event: unknown, context: DurableContext) => { + const order = await context.step( + "fetch-order", + async () => Object.assign(new Order(), { id: "order-123", total: "99.99" }), + { serdes: orderSerdes }, + ); + // order.label() works — class methods are preserved after deserialization + return order.label(); + }, +); diff --git a/examples/typescript/sdk-reference/serialization/callback-config.ts b/examples/typescript/sdk-reference/serialization/callback-config.ts new file mode 100644 index 0000000..2722f5f --- /dev/null +++ b/examples/typescript/sdk-reference/serialization/callback-config.ts @@ -0,0 +1,31 @@ +import { + DurableContext, + Serdes, + SerdesContext, + withDurableExecution, +} from "@aws/durable-execution-sdk-js"; + +interface ApprovalResult { + approved: boolean; + reason: string; +} + +// Callbacks only need deserialization — the external system sends the payload. +const approvalSerdes: Omit, "serialize"> = { + deserialize: async (data: string | undefined, _ctx: SerdesContext) => + data !== undefined ? (JSON.parse(data) as ApprovalResult) : undefined, +}; + +export const handler = withDurableExecution( + async (event: unknown, context: DurableContext) => { + const [approval, callbackId] = context.createCallback("await-approval", { + serdes: approvalSerdes, + }); + + // Send callbackId to the external system here. + console.log("Callback ID:", callbackId); + + const result = await approval; + return result; + }, +); diff --git a/examples/typescript/sdk-reference/serialization/custom-serdes.ts b/examples/typescript/sdk-reference/serialization/custom-serdes.ts new file mode 100644 index 0000000..69483d7 --- /dev/null +++ b/examples/typescript/sdk-reference/serialization/custom-serdes.ts @@ -0,0 +1,17 @@ +import { Serdes, SerdesContext } from "@aws/durable-execution-sdk-js"; + +interface Order { + id: string; + total: string; +} + +const orderSerdes: Serdes = { + serialize: async (value: Order | undefined, _ctx: SerdesContext) => { + if (value === undefined) return undefined; + return JSON.stringify(value); + }, + deserialize: async (data: string | undefined, _ctx: SerdesContext) => { + if (data === undefined) return undefined; + return JSON.parse(data) as Order; + }, +}; diff --git a/examples/typescript/sdk-reference/serialization/map-config.ts b/examples/typescript/sdk-reference/serialization/map-config.ts new file mode 100644 index 0000000..af50afd --- /dev/null +++ b/examples/typescript/sdk-reference/serialization/map-config.ts @@ -0,0 +1,37 @@ +import { + BatchResult, + DurableContext, + MapConfig, + Serdes, + SerdesContext, + withDurableExecution, +} from "@aws/durable-execution-sdk-js"; + +interface ProcessedItem { + id: string; + status: string; +} + +const itemSerdes: Serdes = { + serialize: async (value: ProcessedItem | undefined, _ctx: SerdesContext) => + value !== undefined ? JSON.stringify(value) : undefined, + deserialize: async (data: string | undefined, _ctx: SerdesContext) => + data !== undefined ? (JSON.parse(data) as ProcessedItem) : undefined, +}; + +const mapConfig: MapConfig = { + itemSerdes, +}; + +export const handler = withDurableExecution( + async (event: unknown, context: DurableContext) => { + const items = ["a", "b", "c"]; + const result: BatchResult = await context.map( + "process-items", + items, + async (_ctx, item) => ({ id: item, status: "done" }), + mapConfig, + ); + return result.getResults(); + }, +); diff --git a/examples/typescript/sdk-reference/serialization/pass-through-serdes.ts b/examples/typescript/sdk-reference/serialization/pass-through-serdes.ts new file mode 100644 index 0000000..d827545 --- /dev/null +++ b/examples/typescript/sdk-reference/serialization/pass-through-serdes.ts @@ -0,0 +1,25 @@ +import { + DurableContext, + Serdes, + SerdesContext, + StepConfig, + withDurableExecution, +} from "@aws/durable-execution-sdk-js"; + +// A pass-through serdes returns the value as-is (already a JSON string). +const passThroughSerdes: Serdes = { + serialize: async (value: string | undefined, _ctx: SerdesContext) => value, + deserialize: async (data: string | undefined, _ctx: SerdesContext) => data, +}; + +export const handler = withDurableExecution( + async (event: unknown, context: DurableContext) => { + const config: StepConfig = { serdes: passThroughSerdes }; + const raw = await context.step( + "fetch-raw", + async (): Promise => '{"id":"order-123"}', + config, + ); + return raw; + }, +); diff --git a/examples/typescript/sdk-reference/serialization/serdes-interface.ts b/examples/typescript/sdk-reference/serialization/serdes-interface.ts new file mode 100644 index 0000000..dc6eaee --- /dev/null +++ b/examples/typescript/sdk-reference/serialization/serdes-interface.ts @@ -0,0 +1,13 @@ +import { Serdes, SerdesContext } from "@aws/durable-execution-sdk-js"; + +// Serdes interface +interface SerdesInterface { + serialize: (value: T | undefined, context: SerdesContext) => Promise; + deserialize: (data: string | undefined, context: SerdesContext) => Promise; +} + +// SerdesContext +interface SerdesContextShape { + entityId: string; + durableExecutionArn: string; +} diff --git a/examples/typescript/sdk-reference/serialization/step-config.ts b/examples/typescript/sdk-reference/serialization/step-config.ts new file mode 100644 index 0000000..1676368 --- /dev/null +++ b/examples/typescript/sdk-reference/serialization/step-config.ts @@ -0,0 +1,32 @@ +import { + DurableContext, + Serdes, + SerdesContext, + StepConfig, + withDurableExecution, +} from "@aws/durable-execution-sdk-js"; + +interface Order { + id: string; + total: string; +} + +const orderSerdes: Serdes = { + serialize: async (value: Order | undefined, _ctx: SerdesContext) => + value !== undefined ? JSON.stringify(value) : undefined, + deserialize: async (data: string | undefined, _ctx: SerdesContext) => + data !== undefined ? (JSON.parse(data) as Order) : undefined, +}; + +const stepConfig: StepConfig = { serdes: orderSerdes }; + +export const handler = withDurableExecution( + async (event: unknown, context: DurableContext) => { + const order = await context.step( + "fetch-order", + async (): Promise => ({ id: "order-123", total: "99.99" }), + stepConfig, + ); + return order; + }, +); diff --git a/examples/typescript/sdk-reference/serialization/walkthrough.ts b/examples/typescript/sdk-reference/serialization/walkthrough.ts new file mode 100644 index 0000000..5e525d0 --- /dev/null +++ b/examples/typescript/sdk-reference/serialization/walkthrough.ts @@ -0,0 +1,16 @@ +import { DurableContext, withDurableExecution } from "@aws/durable-execution-sdk-js"; + +interface Order { + id: string; + total: string; +} + +export const handler = withDurableExecution( + async (event: unknown, context: DurableContext) => { + // No SerDes config — the SDK serializes and deserializes the result automatically. + const order = await context.step("fetch-order", async (): Promise => { + return { id: "order-123", total: "99.99" }; + }); + return order; + }, +);