Hello team,
I am getting "Task was cancelled" exception randomly while accessing AWS DynamoDb.
There is no much data in that Object_lock table. Even this error is random means sometimes it is working fine sometimes giving error. I have noticed it is giving error while multiple requests are there. Please check this .net core code from where I am accessing dynamoDb. My dynamoDb request timeout is 35sec.
----------------- Exception ----------------------------------------
"Exception": "System.Threading.Tasks.TaskCanceledException: A task was canceled.\n at Amazon.Runtime.Internal.RetryHandler.InvokeAsync[T](IExecutionContext executionContext)\n
at Amazon.Runtime.Internal.CallbackHandler.InvokeAsync[T](IExecutionContext executionContext)\n at Amazon.Runtime.Internal.CallbackHandler.InvokeAsync[T](IExecutionContext executionContext)\n
at Amazon.Runtime.Internal.ErrorCallbackHandler.InvokeAsync[T](IExecutionContext executionContext)\n at Amazon.Runtime.Internal.MetricsHandler.InvokeAsync[T](IExecutionContext executionContext)\n
at TrOp.DynamoDbServices.Services.DynamoDbService.GetItemsByPartialValueAsync(String tableName, String fieldName, String fieldValueStartsWith, String customerCodeValue, CancellationToken token)\n
at TrOp.LockingServices.Services.LockingServices.GetLockDetailsByApplicationAsync(String appName, String customerCode)\n
-------------------------------Code-------------------------------------
public async Task<DynamoDbResult<List<Dictionary<string, AttributeValue>>>> GetItemsByPartialSortKeyAsync(string tableName,
string partitionKeyName,
string partitionKeyValue,
string sortKeyName,
string sortKeyStartsWith,
CancellationToken token = default)
{
try
{
DynamoDbResult<List<Dictionary<string, AttributeValue>>> dynamoDbResult = new DynamoDbResult<List<Dictionary<string, AttributeValue>>>() { Value = new List<Dictionary<string, AttributeValue>>() { } };
QueryResponse dbResponse = null;
QueryRequest request = new QueryRequest(tableName)
{
KeyConditionExpression = "#PK = :key AND begins_with(#SK, :sortKey)",
ExpressionAttributeNames = new Dictionary<string, string>() { ["#PK"] = partitionKeyName, ["#SK"] = sortKeyName },
ExpressionAttributeValues = new Dictionary<string, AttributeValue>() { [":key"] = new AttributeValue() { S = partitionKeyValue }, [":sortKey"] = new AttributeValue() { S = sortKeyStartsWith } }
};
do
{
request.ExclusiveStartKey = dbResponse?.LastEvaluatedKey;
using var tokenSource = CreateCancellationTokenSource(token);
dbResponse = await _dynamoDbClient.QueryAsync(request, tokenSource.Token);
if (dbResponse.HttpStatusCode != System.Net.HttpStatusCode.OK) // Queries are batched in 1MB chunks, if we fail at any point then return the error
return new DynamoDbResult<List<Dictionary<string, AttributeValue>>>() { Error = $"Unexpected response returned while querying data : {dbResponse.HttpStatusCode}. Database response object '{JsonSerializer.Serialize(dbResponse)}'." };
if (dbResponse.Items != null)
dynamoDbResult.Value.AddRange(dbResponse.Items);
} while (dbResponse.LastEvaluatedKey.Count > 0);
return dynamoDbResult;
}
catch (Exception ex)
{
Log.Error($"Exception From GetItemsByPartialSortKeyAsync line 259 (tableName,partitionKeyName,partitionKeyValue,sortKeyName,sortKeyStartsWith,token): {ex.StackTrace}");
throw ex;
}
}
private CancellationTokenSource CreateCancellationTokenSource(CancellationToken token = default)
{
try
{
// Get timeout setting in seconds (default 20 seconds)
int dbTimeoutSeconds = int.TryParse(Environment.GetEnvironmentVariable("DYNAMODB_TIMEOUT_SECS"), out int timeOut) ? timeOut : 20;
CancellationTokenSource tokenSource = token == default ? new CancellationTokenSource() : CancellationTokenSource.CreateLinkedTokenSource(token);
tokenSource.CancelAfter(dbTimeoutSeconds * 1000);
return tokenSource;
}
catch (Exception ex)
{
Log.Error($"Exception From CreateCancellationTokenSource: {ex.StackTrace}");
throw ex;
}
}