[{"data":1,"prerenderedAt":38},["ShallowReactive",2],{"snippet-lambda-function-kafka-consumer-glue-schema-registry":3},{"id":4,"title":5,"contributors":6,"description":8,"extension":9,"gitHub":10,"introBox":13,"languages":17,"meta":18,"resources":19,"services":20,"snippets":22,"stem":34,"tags":35,"type":36,"__hash__":37},"snippets\u002Fsnippets\u002Flambda-function-kafka-consumer-glue-schema-registry.json","Deserializes Kafka events using schema in AWS Glue Schema Registry",[7],"content\u002Fcontributors\u002Fali-alemi.json","AWS Lambda function that deserializes events in a Kafka topic using Avro schema in AWS Glue Schema Registry.","json",{"template":11},{"repoURL":12},"https:\u002F\u002Fgithub.com\u002Faws-samples\u002Fserverless-snippets\u002Ftree\u002Fmain\u002Flambda-function-kafka-consumer-glue-schema-registry",{"headline":14,"text":15},"How it works",[16],"This code example shows how an AWS Lambda function can get invoked by Event Source Mapping to a Kafka topic. It downloads the corresponding schema from AWS Glue Schema Registry by looking up the schema version (by which the record has been serialized). The deserializes record is stored in JSON format in Amazon S3.",[],{},null,[21],"lambda-event-functions",[23],{"title":24,"codeTabs":25},"Runtimes",[26],{"id":27,"title":28,"description":29,"snippets":30},"Python","Copy the code into your lambda function","Follow the AWS Lambda instructions for packaging with dependencies, and uploading in .zip format.",[31],{"language":32,"code":33},"py","import boto3\nimport avro.io\nfrom uuid import UUID\nfrom datetime import datetime\nimport base64\nimport json\nimport io\n\nprint('Loading function')\n\nglue_client = boto3.client('glue')\ns3_client = boto3.client('s3')\n# memory cache for the inline schemas\nschema_map = {}\n\ndef get_schema(schema_version_id):\n    print(\"loading schema for version: \" + schema_version_id)\n\n    response = glue_client.get_schema_version(\n        SchemaVersionId=schema_version_id\n    )\n\n    return response['SchemaDefinition']\n\ndef gsr_value_deserializer(raw_record):\n    # First two control bytes are put by Glue Schema Registry serializer.\n    # control_byte = raw_record[0]\n    # compression_byte = raw_record[1]\n\n    # version to deserialize each record.\n    version_bytes = raw_record[2:18]\n    schema_version = UUID(bytes=version_bytes)\n    schema_key = str(schema_version)\n    # print(schema_version)\n    \n    if schema_key in schema_map:\n        schema_inline = schema_map[schema_key]\n    else:\n        schema_inline = get_schema(schema_key)\n        schema_map[schema_key] = schema_inline\n\n    # raw data record begins after byte #18\n    raw_record = raw_record[18:]\n    schema = avro.schema.parse(schema_inline)\n    reader = avro.io.DatumReader(schema)\n\n    bytes_reader = io.BytesIO(raw_record)\n\n    decoder = avro.io.BinaryDecoder(bytes_reader)\n    msg = reader.read(decoder)\n    return msg\n\n\ndef lambda_handler(event, context):\n    \n    # use time based file partitioning in S3\n    current_month = datetime.now().strftime('%m')\n    current_day = datetime.now().strftime('%d') \n    current_year_full = datetime.now().strftime('%Y')\n    current_hour = datetime.now().strftime('%H')\n    current_minute = datetime.now().strftime('%M')\n    ct = datetime.now()\n\n    records = event['records']\n    messageAr = []\n    for key in records:\n        base64records = event['records'][key]\n        raw_records = [base64.b64decode(x[\"value\"]) for x in base64records]\n        \n        for raw_record in raw_records:\n            record = gsr_value_deserializer(raw_record)\n            messageAr.append(record)\n            \n    s3_client.put_object(Body=json.dumps(messageAr), Bucket='\u003CYour S3 Bucket>', Key='\u003Cyour s3 prefix>\u002Fyear=' + current_year_full + '\u002Fmonth=' + current_month + '\u002Fday=' + current_day + '\u002Fhour=' + current_hour + '\u002Fminute=' + current_minute + '\u002Ffile-' + str(ct) + '.json')\n            \n    return {\n        'statusCode': 200,\n        'body': json.dumps('Finished processing!')\n    }","snippets\u002Flambda-function-kafka-consumer-glue-schema-registry",[],"Integration","vlorbSt9ykCE0Caez1WEq2WTBXqijAGbxA0BsM3ARE0",1792578864438]