[{"data":1,"prerenderedAt":98},["ShallowReactive",2],{"snippet-integration-kinesis-to-lambda-with-batch-item-handling":3},{"id":4,"title":5,"contributors":6,"description":5,"extension":9,"gitHub":10,"introBox":13,"languages":17,"meta":21,"resources":22,"services":27,"snippets":30,"stem":94,"tags":95,"type":96,"__hash__":97},"snippets\u002Fsnippets\u002Fintegration-kinesis-to-lambda-with-batch-item-handling.json","Using AWS Lambda with Amazon Kinesis with Batch Item Handling.",[7,8],"content\u002Fcontributors\u002Fmohammed-tomeh.json","content\u002Fcontributors\u002Fjan-richter.json","json",{"template":11},{"repoURL":12},"https:\u002F\u002Fgithub.com\u002Faws-samples\u002Fserverless-snippets\u002Ftree\u002Fmain\u002Fintegration-kinesis-to-lambda-with-batch-item-handling",{"headline":14,"text":15},"How it works",[16],"The Lambda function retrieves Kinesis Records from the event parameter that it receives. The function retrieves Kinesis payload, decode from Base64 and log them to cloudwatch. The function reports batchItemFailures with SequenceNumber, in order for this feature to work ReportBatchItemFailures should be enabled in lambda configurations",[18,19,20],"Node.js","TypeScript",".NET",{},{"bullets":23},[24],{"text":25,"link":26},"Using AWS Lambda with Amazon Kinesis","https:\u002F\u002Fdocs.aws.amazon.com\u002Flambda\u002Flatest\u002Fdg\u002Fwith-kinesis.html",[28,29],"lambda-event-functions","kinesis",[31],{"title":32,"codeTabs":33},"Runtimes",[34,41,48,56,64,71,78,86],{"id":18,"title":35,"description":36,"snippets":37},"Usage Example with Node:","Consuming Kinesis event with Lambda using JavaScript with batch item handling.",[38],{"language":39,"code":40},"js","exports.handler = async (event, context) => {\n  for (const record of event.Records) {\n    try {\n      console.log(`Processed Kinesis Event - EventID: ${record.eventID}`);\n      const recordData = await getRecordDataAsync(record.kinesis);\n      console.log(`Record Data: ${recordData}`);\n      \u002F\u002F TODO: Do interesting work based on the new data\n    } catch (err) {\n      console.error(`An error occurred ${err}`);\n      \u002F* Since we are working with streams, we can return the failed item immediately.\n            Lambda will immediately begin to retry processing from this failed item onwards. *\u002F\n      return {\n        batchItemFailures: [{ itemIdentifier: record.kinesis.sequenceNumber }],\n      };\n    }\n  }\n  console.log(`Successfully processed ${event.Records.length} records.`);\n  return { batchItemFailures: [] };\n};\n\nasync function getRecordDataAsync(payload) {\n  var data = Buffer.from(payload.data, \"base64\").toString(\"utf-8\");\n  await Promise.resolve(1); \u002F\u002FPlaceholder for actual async work\n  return data;\n}\n",{"id":19,"title":42,"description":43,"snippets":44},"Usage Example with TypeScript:","Consuming Kinesis event with Lambda using TypeScript with batch item handling.",[45],{"language":46,"code":47},"ts","import {\n  KinesisStreamEvent,\n  Context,\n  KinesisStreamHandler,\n  KinesisStreamRecordPayload,\n  KinesisStreamBatchResponse,\n} from \"aws-lambda\";\nimport { Buffer } from \"buffer\";\nimport { Logger } from \"@aws-lambda-powertools\u002Flogger\";\n\nconst logger = new Logger({\n  logLevel: \"INFO\",\n  serviceName: \"kinesis-stream-handler-sample\",\n});\n\nexport const functionHandler: KinesisStreamHandler = async (\n  event: KinesisStreamEvent,\n  context: Context\n): Promise<KinesisStreamBatchResponse> => {\n  for (const record of event.Records) {\n    try {\n      logger.info(`Processed Kinesis Event - EventID: ${record.eventID}`);\n      const recordData = await getRecordDataAsync(record.kinesis);\n      logger.info(`Record Data: ${recordData}`);\n      \u002F\u002F TODO: Do interesting work based on the new data\n    } catch (err) {\n      logger.error(`An error occurred ${err}`);\n      \u002F* Since we are working with streams, we can return the failed item immediately.\n            Lambda will immediately begin to retry processing from this failed item onwards. *\u002F\n      return {\n        batchItemFailures: [{ itemIdentifier: record.kinesis.sequenceNumber }],\n      };\n    }\n  }\n  logger.info(`Successfully processed ${event.Records.length} records.`);\n  return { batchItemFailures: [] };\n};\n\nasync function getRecordDataAsync(\n  payload: KinesisStreamRecordPayload\n): Promise<string> {\n  var data = Buffer.from(payload.data, \"base64\").toString(\"utf-8\");\n  await Promise.resolve(1); \u002F\u002FPlaceholder for actual async work\n  return data;\n}\n",{"id":49,"title":50,"description":51,"snippets":52},"Java","Usage Example with Java:","Consuming Kinesis event with Lambda using Java with batch item handling.",[53],{"language":54,"code":55},"java","import com.amazonaws.services.lambda.runtime.Context;\nimport com.amazonaws.services.lambda.runtime.RequestHandler;\nimport com.amazonaws.services.lambda.runtime.events.KinesisEvent;\nimport com.amazonaws.services.lambda.runtime.events.StreamsEventResponse;\n\nimport java.io.Serializable;\nimport java.util.ArrayList;\nimport java.util.List;\n\npublic class ProcessKinesisRecords implements RequestHandler<KinesisEvent, StreamsEventResponse> {\n\n    @Override\n    public StreamsEventResponse handleRequest(KinesisEvent input, Context context) {\n\n        List<StreamsEventResponse.BatchItemFailure> batchItemFailures = new ArrayList<>();\n        String curRecordSequenceNumber = \"\";\n\n        for (KinesisEvent.KinesisEventRecord kinesisEventRecord : input.getRecords()) {\n            try {\n                \u002F\u002FProcess your record\n                KinesisEvent.Record kinesisRecord = kinesisEventRecord.getKinesis();\n                curRecordSequenceNumber = kinesisRecord.getSequenceNumber();\n\n            } catch (Exception e) {\n                \u002F* Since we are working with streams, we can return the failed item immediately.\n                   Lambda will immediately begin to retry processing from this failed item onwards. *\u002F\n                batchItemFailures.add(new StreamsEventResponse.BatchItemFailure(curRecordSequenceNumber));\n                return new StreamsEventResponse(batchItemFailures);\n            }\n        }\n       \n       return new StreamsEventResponse(batchItemFailures);   \n    }\n}\n",{"id":57,"title":58,"description":59,"snippets":60},"Python","Usage Example with Python:","Consuming Kinesis event with Lambda using Python with batch item handling.",[61],{"language":62,"code":63},"py","def handler(event, context):\n    records = event.get(\"Records\")\n    curRecordSequenceNumber = \"\"\n    \n    for record in records:\n        try:\n            # Process your record\n            curRecordSequenceNumber = record[\"kinesis\"][\"sequenceNumber\"]\n        except Exception as e:\n            # Return failed record's sequence number\n            return {\"batchItemFailures\":[{\"itemIdentifier\": curRecordSequenceNumber}]}\n\n    return {\"batchItemFailures\":[]}\n",{"id":20,"title":65,"description":66,"snippets":67},"Usage Example with .NET:","Consuming Kinesis event with Lambda using .NET with batch item handling.",[68],{"language":69,"code":70},"dotnet","using System.Text;\nusing System.Text.Json.Serialization;\nusing Amazon.Lambda.Core;\nusing Amazon.Lambda.KinesisEvents;\nusing AWS.Lambda.Powertools.Logging;\n\n\u002F\u002F Assembly attribute to enable the Lambda function's JSON input to be converted into a .NET class.\n[assembly: LambdaSerializer(typeof(Amazon.Lambda.Serialization.SystemTextJson.DefaultLambdaJsonSerializer))]\n\nnamespace KinesisIntegration;\n\npublic class Function\n{\n    \u002F\u002F Powertools Logger requires an environment variables against your function\n    \u002F\u002F POWERTOOLS_SERVICE_NAME\n    [Logging(LogEvent = true)]\n    public async Task<StreamsEventResponse> FunctionHandler(KinesisEvent evnt, ILambdaContext context)\n    {\n        if (evnt.Records.Count == 0)\n        {\n            Logger.LogInformation(\"Empty Kinesis Event received\");\n            return new StreamsEventResponse();\n        }\n\n        foreach (var record in evnt.Records)\n        {\n            try\n            {\n                Logger.LogInformation($\"Processed Event with EventId: {record.EventId}\");\n                string data = await GetRecordDataAsync(record.Kinesis, context);\n                Logger.LogInformation($\"Data: {data}\");\n                \u002F\u002F TODO: Do interesting work based on the new data\n            }\n            catch (Exception ex)\n            {\n                Logger.LogError($\"An error occurred {ex.Message}\");\n                \u002F* Since we are working with streams, we can return the failed item immediately.\n                   Lambda will immediately begin to retry processing from this failed item onwards. *\u002F\n                return new StreamsEventResponse\n                {\n                    BatchItemFailures = new List<StreamsEventResponse.BatchItemFailure>\n                    {\n                        new StreamsEventResponse.BatchItemFailure { ItemIdentifier = record.Kinesis.SequenceNumber }\n                    }\n                };\n            }\n        }\n        Logger.LogInformation($\"Successfully processed {evnt.Records.Count} records.\");\n        return new StreamsEventResponse();\n    }\n\n    private async Task<string> GetRecordDataAsync(KinesisEvent.Record record, ILambdaContext context)\n    {\n        byte[] bytes = record.Data.ToArray();\n        string data = Encoding.UTF8.GetString(bytes);\n        await Task.CompletedTask; \u002F\u002FPlaceholder for actual async work\n        return data;\n    }\n}\n\npublic class StreamsEventResponse\n{\n    [JsonPropertyName(\"batchItemFailures\")]\n    public IList<BatchItemFailure> BatchItemFailures { get; set; }\n    public class BatchItemFailure\n    {\n        [JsonPropertyName(\"itemIdentifier\")]\n        public string ItemIdentifier { get; set; }\n    }\n}",{"id":72,"title":73,"description":74,"snippets":75},"PHP","Usage Example with PHP","Consuming Kinesis event with Lambda using PHP with batch item handling.",[76],{"language":72,"code":77},"<?php\n\n# using bref\u002Fbref and bref\u002Flogger for simplicity\n\nuse Bref\\Context\\Context;\nuse Bref\\Event\\Kinesis\\KinesisEvent;\nuse Bref\\Event\\Handler as StdHandler;\nuse Bref\\Logger\\StderrLogger;\n\nrequire __DIR__ . '\u002Fvendor\u002Fautoload.php';\n\nclass Handler implements StdHandler\n{\n    private StderrLogger $logger;\n    public function __construct(StderrLogger $logger)\n    {\n        $this->logger = $logger;\n    }\n\n    \u002F**\n     * @throws JsonException\n     * @throws \\Bref\\Event\\InvalidLambdaEvent\n     *\u002F\n    public function handle(mixed $event, Context $context): array\n    {\n        $kinesisEvent = new KinesisEvent($event);\n        $this->logger->info(\"Processing records\");\n        $records = $kinesisEvent->getRecords();\n\n        $failedRecords = [];\n        foreach ($records as $record) {\n            try {\n                $data = $record->getData();\n                $this->logger->info(json_encode($data));\n                \u002F\u002F TODO: Do interesting work based on the new data\n            } catch (Exception $e) {\n                $this->logger->error($e->getMessage());\n                \u002F\u002F failed processing the record\n                $failedRecords[] = $record->getSequenceNumber();\n            }\n        }\n        $totalRecords = count($records);\n        $this->logger->info(\"Successfully processed $totalRecords records\");\n\n        \u002F\u002F change format for the response\n        $failures = array_map(\n            fn(string $sequenceNumber) => ['itemIdentifier' => $sequenceNumber],\n            $failedRecords\n        );\n\n        return [\n            'batchItemFailures' => $failures\n        ];\n    }\n}\n\n$logger = new StderrLogger();\nreturn new Handler($logger);\n",{"id":79,"title":80,"description":81,"snippets":82},"Go","Usage Example with Go:","Consuming Kinesis event with Lambda using Go with batch item handling.",[83],{"language":84,"code":85},"go","package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"github.com\u002Faws\u002Faws-lambda-go\u002Fevents\"\n\t\"github.com\u002Faws\u002Faws-lambda-go\u002Flambda\"\n)\n\nfunc handler(ctx context.Context, kinesisEvent events.KinesisEvent) (map[string]interface{}, error) {\n\tbatchItemFailures := []map[string]interface{}{}\n\n\tfor _, record := range kinesisEvent.Records {\n\t\tcurRecordSequenceNumber := \"\"\n\n\t\t\u002F\u002F Process your record\n\t\tif \u002F* Your record processing condition here *\u002F {\n\t\t\tcurRecordSequenceNumber = record.Kinesis.SequenceNumber\n\t\t}\n\n\t\t\u002F\u002F Add a condition to check if the record processing failed\n\t\tif curRecordSequenceNumber != \"\" {\n\t\t\tbatchItemFailures = append(batchItemFailures, map[string]interface{}{\"itemIdentifier\": curRecordSequenceNumber})\n\t\t}\n\t}\n\n\tkinesisBatchResponse := map[string]interface{}{\n\t\t\"batchItemFailures\": batchItemFailures,\n\t}\n\treturn kinesisBatchResponse, nil\n}\n\nfunc main() {\n\tlambda.Start(handler)\n}\n",{"id":87,"title":88,"description":89,"snippets":90},"Rust","Usage Example with Rust:","Consuming Kinesis event with Lambda using Rust with batch item handling.",[91],{"language":92,"code":93},"rust","use aws_lambda_events::{\n    event::kinesis::KinesisEvent,\n    kinesis::KinesisEventRecord,\n    streams::{KinesisBatchItemFailure, KinesisEventResponse},\n};\nuse lambda_runtime::{run, service_fn, Error, LambdaEvent};\n\nasync fn function_handler(event: LambdaEvent<KinesisEvent>) -> Result<KinesisEventResponse, Error> {\n    let mut response = KinesisEventResponse {\n        batch_item_failures: vec![],\n    };\n\n    if event.payload.records.is_empty() {\n        tracing::info!(\"No records found. Exiting.\");\n        return Ok(response);\n    }\n\n    for record in &event.payload.records {\n        tracing::info!(\n            \"EventId: {}\",\n            record.event_id.as_deref().unwrap_or_default()\n        );\n\n        let record_processing_result = process_record(record);\n\n        if record_processing_result.is_err() {\n            response.batch_item_failures.push(KinesisBatchItemFailure {\n                item_identifier: record.kinesis.sequence_number.clone(),\n            });\n            \u002F* Since we are working with streams, we can return the failed item immediately.\n            Lambda will immediately begin to retry processing from this failed item onwards. *\u002F\n            return Ok(response);\n        }\n    }\n\n    tracing::info!(\n        \"Successfully processed {} records\",\n        event.payload.records.len()\n    );\n\n    Ok(response)\n}\n\nfn process_record(record: &KinesisEventRecord) -> Result<(), Error> {\n    let record_data = std::str::from_utf8(record.kinesis.data.as_slice());\n\n    if let Some(err) = record_data.err() {\n        tracing::error!(\"Error: {}\", err);\n        return Err(Error::from(err));\n    }\n\n    let record_data = record_data.unwrap_or_default();\n\n    \u002F\u002F do something interesting with the data\n    tracing::info!(\"Data: {}\", record_data);\n\n    Ok(())\n}\n\n#[tokio::main]\nasync fn main() -> Result<(), Error> {\n    tracing_subscriber::fmt()\n        .with_max_level(tracing::Level::INFO)\n        \u002F\u002F disable printing the name of the module in every log line.\n        .with_target(false)\n        \u002F\u002F disabling time is handy because CloudWatch will add the ingestion time.\n        .without_time()\n        .init();\n\n    run(service_fn(function_handler)).await\n}","snippets\u002Fintegration-kinesis-to-lambda-with-batch-item-handling",[],"Integration","eAgvlAjc1XO7uk5BBfdP9ZNr69VaJIZydnTmYgCdUuY",1792665263054]