Update: There is a library now: https://pypi.org/project/dynamodb-json/
Here is an improved version of indiangolfer's answer. While @indiangolfer's solution works for the question, this improved version might be more useful for others who stumble upon this thread.
def unmarshal_dynamodb_json(node):
data = dict({})
data['M'] = node
return _unmarshal_value(data)
def _unmarshal_value(node):
if type(node) is not dict:
return node
for key, value in node.items():
# S โ String - return string
# N โ Number - return int or float (if includes '.')
# B โ Binary - not handled
# BOOL โ Boolean - return Bool
# NULL โ Null - return None
# M โ Map - return a dict
# L โ List - return a list
# SS โ String Set - not handled
# NN โ Number Set - not handled
# BB โ Binary Set - not handled
key = key.lower()
if key == 'bool':
return value
if key == 'null':
return None
if key == 's':
return value
if key == 'n':
if '.' in str(value):
return float(value)
return int(value)
if key in ['m', 'l']:
if key == 'm':
data = {}
for key1, value1 in value.items():
if key1.lower() == 'l':
data = [_unmarshal_value(n) for n in value1]
else:
if type(value1) is not dict:
return _unmarshal_value(value)
data[key1] = _unmarshal_value(value1)
return data
data = []
for item in value:
data.append(_unmarshal_value(item))
return data
It is improved in the following ways:
handles more data types, including lists, which were not handled correctly previously
handles lowercase and uppercase keys
Edit: fix recursive object bug
Answer from vekerdyb on Stack OverflowPython Lambda Function Parsing DynamoDB's JSON Format - Stack Overflow
Python Lambda: How to convert DynamoDB json from stream into regular json?
python 3.x - How to convert simple JSON to DynamoDB JSON? - Stack Overflow
How to convert a DynamoDB Json in a regular JSON in Python? - Stack Overflow
Update: There is a library now: https://pypi.org/project/dynamodb-json/
Here is an improved version of indiangolfer's answer. While @indiangolfer's solution works for the question, this improved version might be more useful for others who stumble upon this thread.
def unmarshal_dynamodb_json(node):
data = dict({})
data['M'] = node
return _unmarshal_value(data)
def _unmarshal_value(node):
if type(node) is not dict:
return node
for key, value in node.items():
# S โ String - return string
# N โ Number - return int or float (if includes '.')
# B โ Binary - not handled
# BOOL โ Boolean - return Bool
# NULL โ Null - return None
# M โ Map - return a dict
# L โ List - return a list
# SS โ String Set - not handled
# NN โ Number Set - not handled
# BB โ Binary Set - not handled
key = key.lower()
if key == 'bool':
return value
if key == 'null':
return None
if key == 's':
return value
if key == 'n':
if '.' in str(value):
return float(value)
return int(value)
if key in ['m', 'l']:
if key == 'm':
data = {}
for key1, value1 in value.items():
if key1.lower() == 'l':
data = [_unmarshal_value(n) for n in value1]
else:
if type(value1) is not dict:
return _unmarshal_value(value)
data[key1] = _unmarshal_value(value1)
return data
data = []
for item in value:
data.append(_unmarshal_value(item))
return data
It is improved in the following ways:
handles more data types, including lists, which were not handled correctly previously
handles lowercase and uppercase keys
Edit: fix recursive object bug
I couldn't find anything out in the wild. So, I decided to port the PHP implementation of dynamodb json to standard json that was published here. I tested this in a python lambda function processing DynamoDB stream. If there is a better way to do this, please let me know.
(PS: This is not a complete port of PHP Marshaler)
The JSON in the question gets transformed to:
{
"feas":{
"fea":[
{
"pre":"1",
"Mo":"1",
"Ti":"20160618184156529",
"Fa":"0",
"Li":"1",
"Fr":"4088682"
}
]
}
}
def unmarshalJson(node):
data = {}
data["M"] = node
return unmarshalValue(data, True)
def unmarshalValue(node, mapAsObject):
for key, value in node.items():
if(key == "S" or key == "N"):
return value
if(key == "M" or key == "L"):
if(key == "M"):
if(mapAsObject):
data = {}
for key1, value1 in value.items():
data[key1] = unmarshalValue(value1, mapAsObject)
return data
data = []
for item in value:
data.append(unmarshalValue(item, mapAsObject))
return data
Using Python Lambda as a trigger for DynamoDB stream, how do I convert the provided DynamoDB json into regular json?
If you mean JsonString to Dynamodb Map, you can use boto3.
Here is the example.
import boto3
import json
json_string = '{"key1": 1, "key2": "value"}'
json_obj = json.loads(json_string)
dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table('test-table')
table.put_item(Item={'pk': 'pk-value', 'map': json_obj})
If you just want to update the while Map attribute, you can use just JSON format the same as put_item.
json_string = '{"key1": 2, "key2": "value2"}'
json_obj = json.loads(json_string2)
rsp = table.update_item(
Key={'pk': 'pk-value'},
AttributeUpdates={'map': {'Value': json_obj2, 'Action': 'PUT'}}
)
However, If you want to update only specific nested attribute, you need to use UpdateExpression. For example, the below is code to update only key1 attribute to 'value3'.
nested_json_string = '{"nested": "key3"}'
nested_json_obj = json.loads(nested_json_string)
rsp = table.update_item(
Key={'pk': 'pk-value'},
UpdateExpression='SET #map.#key1 = :val3',
ExpressionAttributeNames={'#map': 'map', '#key1': 'key1'},
ExpressionAttributeValues={':val3': nested_json_obj}
)
I know this is an old question, but I came across it and the accepted answer didn't help me (it seems to suggest that you can feed boto3 with plain JSON, but it didn't work for me) and the library mentioned in the comments didn't help me either.
What did work for me was using the serializer/deserializer from boto3.dynamodb.types, basically as suggested by this answer on a very similar topic.
from boto3.dynamodb.types import TypeSerializer, TypeDeserializer
import json
serializer = TypeSerializer()
deserializer = TypeDeserializer()
# for building a DynamoDB JSON from a Python object
def serialize(item):
serialized_item = serializer.serialize(vars(item) if hasattr(item, '__dict__') else item)
return item if 'M' not in serialized_item else serialized_item['M']
# for building a plain JSON from a DynamoDB JSON
def deserialize(dynamodb_json_string):
return deserializer.deserialize({'M': dynamodb_json_string})
class MyItem:
def __init__(self, some_string_value=None, some_numeric_value=None):
self.my_key = some_string_value
self.my_other_key = some_numeric_value
def __str__(self):
return json.dumps(self, default=lambda x: x.__dict__)
if __name__ == '__main__':
my_classy_item = MyItem("my_string_value", 5)
my_string_item = json.loads('{"my_key": "my_string_value", "my_other_key" : 5}')
print(serialize(my_classy_item))
print(serialize(my_string_item))
I authored a library called cerealbox that makes it easier to perform this common conversion as follows.
from cerealbox.dynamo import from_dynamodb_json
# convert the DynamoDB image to a regular python dictionary
result = from_dynamodb_json(payload_stack['Records'][0]['dynamodb']['NewImage'])
# access the body as a regular dictionary
body = result['txt_vlr_parm_requ']['body']
The documentation covers how to perfom the inverse using as_dynamodb_json.
This can also be done using boto3's TypeDeserializer/TypeSerializer - a good example of this can be found here
I was able to develop a little code to convert this DynamoDB json into a regular json. I used the dynamodb_json import.
##that one in the question
payload_stack = {...}
convert_regular_json = json.loads(payload_stack)
print(convert_regular_json)
The output:
{
'Records': [{
'eventID': '123456',
'eventName': 'INSERT',
'eventVersion': '1.1',
'eventSource': 'aws:dynamodb',
'awsRegion': 'sa-east-1',
'dynamodb': {
'ApproximateCreationDateTime': 1644956685.0,
'Keys': {
'body_field': 1931
},
'NewImage': {
'body_field': 1931,
'txt_vlr_parm_requ': {
'headers': {
'Authorization': 'token',
'correlationID': '987654321'
},
'requestContext': {
'requestId': '123'
},
'body': {
'avro_schema': '{"type":"record","namespace":"Tutorialspoint","name":"Employee","fields":[{"name":"Name","type":"string"},{"name":"Age","type":"int"}, {"name":"Address","type":"string"}, {"name":"Role","type":"string"} ]}',
'cluster': 'events',
'sigla': 'ft7',
'subject': 'teste-dynamo',
'branch': 'development',
'id_requisicao': 1818
}
},
'nom_tabe': 'tabela_teste',
'cod_situ_psst_ingo': 'NOVO',
'historic': '{"historico": [{"data/hora": "09-02-22 18:18:41", "status": "NOVO"}]}',
'nom_arqu_bckt': 'arquivo.avro'
},
'SequenceNumber': '87226300000000005691898607',
'SizeBytes': 1672,
'StreamViewType': 'NEW_IMAGE'
},
'eventSourceARN': 'arn:aws'
}]
}
And to catch the 'body' field:
catch_body_payload = var['Records'][0].get('dynamodb').get('NewImage').get('txt_vlr_parm_requ').get('body')
I solved my problem. You can set null as follows.
from __future__ import print_function
import boto3
import codecs
import json
dynamodb = boto3.resource('dynamodb', region_name='ap-northeast-1', endpoint_url="http://localhost:8000")
table = dynamodb.Table('questionListTable')
with open("questionList.json", "r", encoding='utf-8_sig') as json_file:
items = json.load(json_file)
for item in items:
q_id = item['q_id']
q_body = item['q_body']
q_answer = item['q_answer']
image_url = item['image_url'] if item['image_url'] else None
keywords = item['keywords'] if item['keywords'] else None
print("Adding detail:", q_id, q_body)
table.put_item(
Item={
'q_id': q_id,
'q_body': q_body,
'q_answer': q_answer,
'image_url': image_url,
'keywords': keywords,
}
)
In order to check the situation of Dynamodb, use the offline plugin of the serverless framework to run the API Gateway in the local environment. When I actually called the API using Postman, Null was properly inserted in the value.
{
"q_id" : "001",
"q_body" : "Where is the capital of the United States?",
"q_answer" : "Washington, D.C.",
"image_url" : "/Washington.jpg",
"keywords" : [
"UnitedStates",
"Washington"
]
},
{
"q_id" : "002",
"q_body" : "Where is the capital city of the UK?",
"q_answer" : "London",
"image_url" : "null",
"keywords" : [
"UK",
"London"
]
},
@uhiyama The following line of code in your solution can be summarized/simplified using the get() method:
image_url = item['image_url'] if item['image_url'] else None
image_url = item.get("image_url")
The best place to look would be Boto3's readthedocs here: https://boto3.readthedocs.org/en/latest/reference/services/dynamodb.html#DynamoDB.Client.batch_write_item
As long as your JSON was formatted correctly for the request as in the example you could use:
f = open('MyData.json')
request_items = json.loads(f.read())
client = boto3.client('dynamodb')
response = client.batch_write_item(RequestItems=request_items)
I loaded the JSON this way
import boto3
import json
dynamodbclient=boto3.resource('dynamodb')
sample_table = dynamodbclient.Table('ec2metadata')
with open('/samplepath/spotec2interruptionevent.json', 'r') as myfile:
data=myfile.read()
# parse file
obj = json.loads(data)
#instance_id and cluster_id is the Key in dynamodb table
response=sample_table.put_item(
Item={
'instance_id': instanceId,
'cluster_id': clusterId,
'event':obj
}
)
at the end of my code where it says "print(json.dumps(i, cls=DecimalEncoder))" I changed that to "d = ast.literal_eval((json.dumps(i, cls=DecimalEncoder)))" I also added import ast at the top. It worked beautifully.
import ast
table = dynamodb.Table('footable')
response = table.scan(
Select="ALL_ATTRIBUTES",
)
for i in response['Items']:
d = ast.literal_eval((json.dumps(i, cls=DecimalEncoder)))
import boto3
import json
import decimal
from boto3.dynamodb.conditions import Key, Attr
# Helper class to convert a DynamoDB item to JSON.
class DecimalEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, decimal.Decimal):
return str(o)
if isinstance(o, set): #<---resolving sets as lists
return list(o)
return super(DecimalEncoder, self).default(o)
dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table('mytable')
response = table.query(
KeyConditionExpression=Key('object_type').eq("employee")
)
print(json.dumps((response), indent=4, cls=DecimalEncoder))