DynamoDB: Using DELETE fails if attribute does not exist

[MartinAltmayerTMH]

I believe there is a mismatch in behavior between moto and the real DynamoDB:

• On the real DynamoDB, when using the DELETE operator in update_item to remove an item from an attribute that does not exist, it will just be ignored.
• In moto, a cryptic error message is raised instead.

This is important, because it means that deleting the final element of a set is idempotent in AWS, but unfortunately not in moto.
See also #3296

Example program

Requires pip install moto boto3 pytest. The following test should pass. It uses DELETE on an attribute that does not exist. DynamoDB will simply ignore this, but moto raises an exception (see below).

import pytest
import boto3
import moto

@pytest.fixture()
def boto_table():
   # This just sets up a table using "name" as primary key.
    with moto.mock_aws():
        dynamodb_resource = boto3.resource("dynamodb", region_name="eu-west-1")

        yield dynamodb_resource.create_table(
            TableName="test-table",
            AttributeDefinitions=[
                {"AttributeName": "name", "AttributeType": "S"},
            ],
            KeySchema=[{"AttributeName": "name", "KeyType": "HASH"}],
            BillingMode="PAY_PER_REQUEST",
        )

def test_remove_user_that_does_not_exist(boto_table):
    name = "name"
    user_name = "karl"
    initial_item = {"name": name}  # Note: this does not contain a 'users' attribute!

    boto_table.put_item(Item=initial_item)

    # Nevertheless, we now try to remove a user from it.
    response = boto_table.update_item(
        Key={"name": name},
        UpdateExpression="DELETE #users :users",
        ExpressionAttributeValues={":users": {user_name}},
        ExpressionAttributeNames={"#users": "users"},
        ReturnValues="ALL_NEW",
    )

    assert response["Attributes"] == initial_item

Expected Behavior

The test should pass. (It does pass, if you add a users attribute, e.g. initial_item={"name": name, "users":{"otto"}}.)

Actual Behaviour

I receive an error botocore.exceptions.ClientError: An error occurred (ValidationException) when calling the UpdateItem operation: The provided key element does not match the schema.

Full output:

(venv) martin@tatooine ~/Temp/moto-dynamodb » pytest test.py                                                                                            1 ↵ 
=================================================================== test session starts ====================================================================
platform linux -- Python 3.11.8, pytest-8.1.1, pluggy-1.4.0                                                                                                 
rootdir: /home/martin/Temp/moto-dynamodb                                                                                                                    
collected 1 item                                                                                                                                            
                                                                                                                                                            
test.py F                                                                                                                                            [100%] 
                                                                                                                                                            
========================================================================= FAILURES =========================================================================
___________________________________________________________ test_remove_user_that_does_not_exist ___________________________________________________________
                                                                                                                                                            
boto_table = dynamodb.Table(name='test-table')                                                                                                              
                                                                                                                                                            
    def test_remove_user_that_does_not_exist(boto_table):                                                                                                   
        name = "name"                                                                                                                                       
        user_name = "karl"                                                                                                                                  
        initial_item = {"name": name}                                                                                                                       
                                                                                                                                                            
        boto_table.put_item(Item=initial_item)                                                                                                              
                                                                                                                                                            
>       response = boto_table.update_item(                                                                                                                  
            Key={"name": name},                                                                                                                             
            UpdateExpression="DELETE #users :users",                                                                                                        
            ExpressionAttributeValues={":users": {user_name}},                                                                                              
            ExpressionAttributeNames={"#users": "users"},                                                                                                   
            ReturnValues="ALL_NEW",                                                                                                                         
        )                                                                                                                                                   
                                                                                                                                                            
test.py:28:                                                                                                                                                 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
venv/lib/python3.11/site-packages/boto3/resources/factory.py:581: in do_action                                                                              
    response = action(self, *args, **kwargs)                                                                                                                
venv/lib/python3.11/site-packages/boto3/resources/action.py:88: in __call__                                                                                 
    response = getattr(parent.meta.client, operation_name)(*args, **params)                                                                                 
venv/lib/python3.11/site-packages/botocore/client.py:565: in _api_call                                                                                      
    return self._make_api_call(operation_name, kwargs)                                                                                                      
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
                                                                                                                                                            
self = <botocore.client.DynamoDB object at 0x75863d92ad50>, operation_name = 'UpdateItem'                                                                   
api_params = {'ExpressionAttributeNames': {'#users': 'users'}, 'ExpressionAttributeValues': {':users': {'SS': ['karl']}}, 'Key': {'name': {'S': 'name'}}, 'R
eturnValues': 'ALL_NEW', ...}                                                                                                                               
                                                                                                                                                            
    def _make_api_call(self, operation_name, api_params):                                                                                                   
        operation_model = self._service_model.operation_model(operation_name)                                                                               
        service_name = self._service_model.service_name                                                                                                     
        history_recorder.record(                                                                                                                            
            'API_CALL',                                                                                                                                     
            {                                                                                                                                               
                'service': service_name,                                                                                                                    
                'operation': operation_name,                                                                                                                
                'params': api_params,                                                                                                                       
            },                                                                                                                                              
        )                                                                                                                                                   
        if operation_model.deprecated:                                                                                                                      
            logger.debug(                                                                                                                                   
                'Warning: %s.%s() is deprecated', service_name, operation_name                                                                              
            )                                                                                                                                               
        request_context = {                                                                                                                                 
            'client_region': self.meta.region_name,                                                                                                         
            'client_config': self.meta.config,                                                                                                              
            'has_streaming_input': operation_model.has_streaming_input,
            'auth_type': operation_model.auth_type,
        }
        api_params = self._emit_api_params(
            api_params=api_params,
            operation_model=operation_model,
            context=request_context,
        )
        (
            endpoint_url,
            additional_headers,
            properties,
        ) = self._resolve_endpoint_ruleset(
            operation_model, api_params, request_context
        )
        if properties:
            # Pass arbitrary endpoint info with the Request
            # for use during construction.
            request_context['endpoint_properties'] = properties
        request_dict = self._convert_to_request_dict(
            api_params=api_params,
            operation_model=operation_model,
            endpoint_url=endpoint_url,
            context=request_context,
            headers=additional_headers,
        )
        resolve_checksum_context(request_dict, operation_model, api_params)
    
        service_id = self._service_model.service_id.hyphenize()
        handler, event_response = self.meta.events.emit_until_response(
            'before-call.{service_id}.{operation_name}'.format(
                service_id=service_id, operation_name=operation_name
            ),
            model=operation_model,
            params=request_dict,
            request_signer=self._request_signer,
            context=request_context,
        )
    
        if event_response is not None:
            http, parsed_response = event_response
        else:
            maybe_compress_request(
                self.meta.config, request_dict, operation_model
            )
            apply_request_checksum(request_dict)
            http, parsed_response = self._make_request(
                operation_model, request_dict, request_context
            )
    
        self.meta.events.emit(
            'after-call.{service_id}.{operation_name}'.format(
                service_id=service_id, operation_name=operation_name
            ),
            http_response=http,
            parsed=parsed_response,
            model=operation_model,
            context=request_context,
        )
    
        if http.status_code >= 300:
            error_info = parsed_response.get("Error", {})
            error_code = error_info.get("QueryErrorCode") or err.9or_info.get(
                "Code"
            )
            error_class = self.exceptions.from_code(error_code)
>           raise error_class(parsed_response, operation_name)
E           botocore.exceptions.ClientError: An error occurred (ValidationException) when calling the UpdateItem operation: The provided key element does not match the schema

venv/lib/python3.11/site-packages/botocore/client.py:1021: ClientError
================================================================= short test summary info ==================================================================
FAILED test.py::test_remove_user_that_does_not_exist - botocore.exceptions.ClientError: An error occurred (ValidationException) when calling the UpdateItem operation: The provided key element does not match...
==================================================================== 1 failed in 0.22s =====================================================================

Versions

Python 3.11.8
moto 5.0.5