diff --git a/src/pytest_mock/plugin.py b/src/pytest_mock/plugin.py
index fd039cf..5ee170e 100644
--- a/src/pytest_mock/plugin.py
+++ b/src/pytest_mock/plugin.py
@@ -147,6 +147,7 @@ def _start_patch(self, mock_func, *args, **kwargs):
             module, registering the patch to stop it later and returns the
             mock object resulting from the mock call.
             """
+            self._enforce_no_with_context(inspect.stack())
             p = mock_func(*args, **kwargs)
             mocked = p.start()
             self._patches.append(p)
@@ -154,14 +155,9 @@ def _start_patch(self, mock_func, *args, **kwargs):
                 self._mocks.append(mocked)
             return mocked
 
-        def object(self, *args, **kwargs):
-            """API to mock.patch.object"""
-            self._enforce_no_with_context(inspect.stack())
-            return self._start_patch(self.mock_module.patch.object, *args, **kwargs)
-
         def _enforce_no_with_context(self, stack):
             """raises a ValueError if mocker is used in a with context"""
-            caller = stack[1]
+            caller = stack[2]
             frame = caller[0]
             info = inspect.getframeinfo(frame)
             code_context = " ".join(info.code_context).strip()
@@ -172,6 +168,10 @@ def _enforce_no_with_context(self, stack):
                     "https://github.com/pytest-dev/pytest-mock#note-about-usage-as-context-manager"
                 )
 
+        def object(self, *args, **kwargs):
+            """API to mock.patch.object"""
+            return self._start_patch(self.mock_module.patch.object, *args, **kwargs)
+
         def multiple(self, *args, **kwargs):
             """API to mock.patch.multiple"""
             return self._start_patch(self.mock_module.patch.multiple, *args, **kwargs)
diff --git a/tests/test_pytest_mock.py b/tests/test_pytest_mock.py
index 4ff741a..d1d8db9 100644
--- a/tests/test_pytest_mock.py
+++ b/tests/test_pytest_mock.py
@@ -723,7 +723,7 @@ def test_get_random_number(mocker):
     assert "RuntimeError" not in result.stderr.str()
 
 
-def test_abort_context_manager(mocker):
+def test_abort_patch_object_context_manager(mocker):
     class A(object):
         def doIt(self):
             return False
@@ -740,3 +740,16 @@ def doIt(self):
     )
 
     assert str(excinfo.value) == expected_error_msg
+
+
+def test_abort_patch_context_manager(mocker):
+    with pytest.raises(ValueError) as excinfo:
+        with mocker.patch("some_package"):
+            pass
+
+    expected_error_msg = (
+        "Using mocker in a with context is not supported. "
+        "https://github.com/pytest-dev/pytest-mock#note-about-usage-as-context-manager"
+    )
+
+    assert str(excinfo.value) == expected_error_msg