My use case: removing old data from dynamodb using a date attribute.
Important things to know:
Possible Solutions:
Eventually batch delete the result set.
Notes: Building the entity I was using the amazon dynamo annotations. I was using DynamoDBQueryExpression to query, getting the result page with the defined Class object.
public static void cleanUpOldData(AmazonDynamoDB amazonDynamoDB, String dynamoDBTablesPrefix, String tableName,
String dateRangeField, String dateHashKey, String dateIndex, Class clazz) {
log.info(String.format("Cleaning old data from table: %s", tableName));
long cleanUpDateInMillis = (new Date()).getTime() - CLEAN_UP_TIME_MILLIS;
SimpleDateFormat dateFormatter = new SimpleDateFormat(DYNAMO_DATE_FORMAT);
final TimeZone utcTimeZone = TimeZone.getTimeZone("UTC");
dateFormatter.setTimeZone(utcTimeZone);
String cleanUpDate = dateFormatter.format(cleanUpDateInMillis);
Calendar calendar = Calendar.getInstance(utcTimeZone);
calendar.setTimeInMillis(cleanUpDateInMillis);
final String dailyHashKey = String.format("%s_%s", calendar.get(Calendar.YEAR), calendar.get(Calendar.DAY_OF_YEAR));
final String pastDayHashKey = String.format("%s_%s", calendar.get(Calendar.YEAR), calendar.get(Calendar.DAY_OF_YEAR)-1);
final String fullTableName = dynamoDBTablesPrefix + "_" + tableName;
final DynamoDBMapperConfig dbMapperConfig = new DynamoDBMapperConfig(new DynamoDBMapperConfig.TableNameOverride(fullTableName));
DynamoDBMapper mapper = new DynamoDBMapper(amazonDynamoDB, dbMapperConfig);
DynamoDBTableMapper dbTableMapper = mapper.newTableMapper(clazz);
final QueryResultPage dailyResultPage = getDailyQueryResultPage(dateRangeField, dateHashKey, dateIndex, cleanUpDate, dailyHashKey, dbTableMapper);
final QueryResultPage pastDayResultPage = getDailyQueryResultPage(dateRangeField, dateHashKey, dateIndex, cleanUpDate, pastDayHashKey, dbTableMapper);
deleteOldData(dbTableMapper, dailyResultPage, pastDayResultPage);
log.info(String.format("Completed cleaning old data from table: %s, %s items were deleted", tableName,
dailyResultPage.getCount() + pastDayResultPage.getCount()));
}
private static QueryResultPage getDailyQueryResultPage(String dateRangeField, String dateHashKey, String dateIndex,
String cleanUpDate, String dayHashKey, DynamoDBTableMapper dbTableMapper) {
HashMap<String, String > nameMap = new HashMap<>();
nameMap.put("#date", dateRangeField);
nameMap.put("#day", dateHashKey);
HashMap<String, AttributeValue> valueMap = new HashMap<>();
valueMap.put(":date", new AttributeValue().withS(cleanUpDate)) ;
valueMap.put(":day", new AttributeValue().withS(dayHashKey));
final DynamoDBQueryExpression dbQueryExpression = new DynamoDBQueryExpression()
.withIndexName(dateIndex)
.withConsistentRead(false)
.withKeyConditionExpression("#day = :day and #date < :date")
.withExpressionAttributeNames(nameMap)
.withExpressionAttributeValues(valueMap);
return dbTableMapper.query(dbQueryExpression);
}
private static void deleteOldData(DynamoDBTableMapper dbTableMapper, QueryResultPage dailyResultPage, QueryResultPage pastDayResultPage) {
if (dailyResultPage.getCount() > 0) {
dbTableMapper.batchDelete(dailyResultPage.getResults());
}
if (pastDayResultPage.getCount() > 0) {
dbTableMapper.batchDelete(pastDayResultPage.getResults());
}
}