diff --git a/AUTHORS b/AUTHORS index 72391122eb5..20798f3093d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -56,6 +56,7 @@ Charles Cloud Charles Machalow Charnjit SiNGH (CCSJ) Chris Lamb +Chris NeJame Christian Boelsen Christian Fetzer Christian Neumüller diff --git a/doc/en/example/fixtures/fixture_availability.svg b/doc/en/example/fixtures/fixture_availability.svg new file mode 100644 index 00000000000..3ca28447c45 --- /dev/null +++ b/doc/en/example/fixtures/fixture_availability.svg @@ -0,0 +1,132 @@ + + + + + + + + + + + tests + + + + + + + + + + subpackage + + + + + + + + + + test_subpackage.py + + + + + + + + + + + innermost + + test_order + + mid + + + + + + + 1 + + + + + + + 2 + + + + + + + 3 + + + + + + + + + test_top.py + + + + + + + + + innermost + + test_order + + + + + + + 1 + + + 2 + + + top + + order + diff --git a/doc/en/example/fixtures/fixture_availability_plugins.svg b/doc/en/example/fixtures/fixture_availability_plugins.svg new file mode 100644 index 00000000000..88e32d90809 --- /dev/null +++ b/doc/en/example/fixtures/fixture_availability_plugins.svg @@ -0,0 +1,142 @@ + + + + + + + + + + + plugin_a + + + + + + + + 4 + + + + + + + + + plugin_b + + + + + + + + 4 + + + + + + + + + tests + + + + + + + + 3 + + + + + + + + + subpackage + + + + + + + + 2 + + + + + + + + + test_subpackage.py + + + + + + + + 1 + + + + + + + + + + + + + inner + + test_order + + mid + + order + + + a_fix + + b_fix + diff --git a/doc/en/example/fixtures/test_fixtures_order.py b/doc/en/example/fixtures/test_fixtures_order.py deleted file mode 100644 index 97b3e80052b..00000000000 --- a/doc/en/example/fixtures/test_fixtures_order.py +++ /dev/null @@ -1,38 +0,0 @@ -import pytest - -# fixtures documentation order example -order = [] - - -@pytest.fixture(scope="session") -def s1(): - order.append("s1") - - -@pytest.fixture(scope="module") -def m1(): - order.append("m1") - - -@pytest.fixture -def f1(f3): - order.append("f1") - - -@pytest.fixture -def f3(): - order.append("f3") - - -@pytest.fixture(autouse=True) -def a1(): - order.append("a1") - - -@pytest.fixture -def f2(): - order.append("f2") - - -def test_order(f1, m1, f2, s1): - assert order == ["s1", "m1", "a1", "f3", "f1", "f2"] diff --git a/doc/en/example/fixtures/test_fixtures_order_autouse.py b/doc/en/example/fixtures/test_fixtures_order_autouse.py new file mode 100644 index 00000000000..ec282ab4b2b --- /dev/null +++ b/doc/en/example/fixtures/test_fixtures_order_autouse.py @@ -0,0 +1,45 @@ +import pytest + + +@pytest.fixture +def order(): + return [] + + +@pytest.fixture +def a(order): + order.append("a") + + +@pytest.fixture +def b(a, order): + order.append("b") + + +@pytest.fixture(autouse=True) +def c(b, order): + order.append("c") + + +@pytest.fixture +def d(b, order): + order.append("d") + + +@pytest.fixture +def e(d, order): + order.append("e") + + +@pytest.fixture +def f(e, order): + order.append("f") + + +@pytest.fixture +def g(f, c, order): + order.append("g") + + +def test_order_and_g(g, order): + assert order == ["a", "b", "c", "d", "e", "f", "g"] diff --git a/doc/en/example/fixtures/test_fixtures_order_autouse.svg b/doc/en/example/fixtures/test_fixtures_order_autouse.svg new file mode 100644 index 00000000000..36362e4fb00 --- /dev/null +++ b/doc/en/example/fixtures/test_fixtures_order_autouse.svg @@ -0,0 +1,64 @@ + + + + + + + + + + + + + autouse + + order + + a + + b + + c + + d + + e + + f + + g + + test_order + diff --git a/doc/en/example/fixtures/test_fixtures_order_autouse_multiple_scopes.py b/doc/en/example/fixtures/test_fixtures_order_autouse_multiple_scopes.py new file mode 100644 index 00000000000..de0c2642793 --- /dev/null +++ b/doc/en/example/fixtures/test_fixtures_order_autouse_multiple_scopes.py @@ -0,0 +1,31 @@ +import pytest + + +@pytest.fixture(scope="class") +def order(): + return [] + + +@pytest.fixture(scope="class", autouse=True) +def c1(order): + order.append("c1") + + +@pytest.fixture(scope="class") +def c2(order): + order.append("c2") + + +@pytest.fixture(scope="class") +def c3(order, c1): + order.append("c3") + + +class TestClassWithC1Request: + def test_order(self, order, c1, c3): + assert order == ["c1", "c3"] + + +class TestClassWithoutC1Request: + def test_order(self, order, c2): + assert order == ["c1", "c2"] diff --git a/doc/en/example/fixtures/test_fixtures_order_autouse_multiple_scopes.svg b/doc/en/example/fixtures/test_fixtures_order_autouse_multiple_scopes.svg new file mode 100644 index 00000000000..9f2180fe548 --- /dev/null +++ b/doc/en/example/fixtures/test_fixtures_order_autouse_multiple_scopes.svg @@ -0,0 +1,76 @@ + + + + + + + + order + + c1 + + c3 + + test_order + + + + + + TestWithC1Request + + + + + + + order + + c1 + + c2 + + test_order + + + + + + TestWithoutC1Request + + + + + autouse + diff --git a/doc/en/example/fixtures/test_fixtures_order_autouse_temp_effects.py b/doc/en/example/fixtures/test_fixtures_order_autouse_temp_effects.py new file mode 100644 index 00000000000..ba01ad32f57 --- /dev/null +++ b/doc/en/example/fixtures/test_fixtures_order_autouse_temp_effects.py @@ -0,0 +1,36 @@ +import pytest + + +@pytest.fixture +def order(): + return [] + + +@pytest.fixture +def c1(order): + order.append("c1") + + +@pytest.fixture +def c2(order): + order.append("c2") + + +class TestClassWithAutouse: + @pytest.fixture(autouse=True) + def c3(self, order, c2): + order.append("c3") + + def test_req(self, order, c1): + assert order == ["c2", "c3", "c1"] + + def test_no_req(self, order): + assert order == ["c2", "c3"] + + +class TestClassWithoutAutouse: + def test_req(self, order, c1): + assert order == ["c1"] + + def test_no_req(self, order): + assert order == [] diff --git a/doc/en/example/fixtures/test_fixtures_order_autouse_temp_effects.svg b/doc/en/example/fixtures/test_fixtures_order_autouse_temp_effects.svg new file mode 100644 index 00000000000..ac62ae46b40 --- /dev/null +++ b/doc/en/example/fixtures/test_fixtures_order_autouse_temp_effects.svg @@ -0,0 +1,100 @@ + + + + + + + + + + + TestWithAutouse + + + + + + + + + order + + c2 + + c3 + + c1 + + test_req + + + + + order + + c2 + + c3 + + test_no_req + + + autouse + + + + + + + + + TestWithoutAutouse + + + + + + order + + c1 + + test_req + + + + + order + + test_no_req + diff --git a/doc/en/example/fixtures/test_fixtures_order_dependencies.py b/doc/en/example/fixtures/test_fixtures_order_dependencies.py new file mode 100644 index 00000000000..b3512c2a64d --- /dev/null +++ b/doc/en/example/fixtures/test_fixtures_order_dependencies.py @@ -0,0 +1,45 @@ +import pytest + + +@pytest.fixture +def order(): + return [] + + +@pytest.fixture +def a(order): + order.append("a") + + +@pytest.fixture +def b(a, order): + order.append("b") + + +@pytest.fixture +def c(a, b, order): + order.append("c") + + +@pytest.fixture +def d(c, b, order): + order.append("d") + + +@pytest.fixture +def e(d, b, order): + order.append("e") + + +@pytest.fixture +def f(e, order): + order.append("f") + + +@pytest.fixture +def g(f, c, order): + order.append("g") + + +def test_order(g, order): + assert order == ["a", "b", "c", "d", "e", "f", "g"] diff --git a/doc/en/example/fixtures/test_fixtures_order_dependencies.svg b/doc/en/example/fixtures/test_fixtures_order_dependencies.svg new file mode 100644 index 00000000000..24418e63c9d --- /dev/null +++ b/doc/en/example/fixtures/test_fixtures_order_dependencies.svg @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + order + + a + + b + + c + + d + + e + + f + + g + + test_order + diff --git a/doc/en/example/fixtures/test_fixtures_order_dependencies_flat.svg b/doc/en/example/fixtures/test_fixtures_order_dependencies_flat.svg new file mode 100644 index 00000000000..bbe7ad28339 --- /dev/null +++ b/doc/en/example/fixtures/test_fixtures_order_dependencies_flat.svg @@ -0,0 +1,51 @@ + + + + + order + + a + + b + + c + + d + + e + + f + + g + + test_order + diff --git a/doc/en/example/fixtures/test_fixtures_order_dependencies_unclear.svg b/doc/en/example/fixtures/test_fixtures_order_dependencies_unclear.svg new file mode 100644 index 00000000000..150724f80a3 --- /dev/null +++ b/doc/en/example/fixtures/test_fixtures_order_dependencies_unclear.svg @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + order + + a + + b + + c + + d + + e + + f + + g + + test_order + diff --git a/doc/en/example/fixtures/test_fixtures_order_scope.py b/doc/en/example/fixtures/test_fixtures_order_scope.py new file mode 100644 index 00000000000..5d9487cab34 --- /dev/null +++ b/doc/en/example/fixtures/test_fixtures_order_scope.py @@ -0,0 +1,36 @@ +import pytest + + +@pytest.fixture(scope="session") +def order(): + return [] + + +@pytest.fixture +def func(order): + order.append("function") + + +@pytest.fixture(scope="class") +def cls(order): + order.append("class") + + +@pytest.fixture(scope="module") +def mod(order): + order.append("module") + + +@pytest.fixture(scope="package") +def pack(order): + order.append("package") + + +@pytest.fixture(scope="session") +def sess(order): + order.append("session") + + +class TestClass: + def test_order(self, func, cls, mod, pack, sess, order): + assert order == ["session", "package", "module", "class", "function"] diff --git a/doc/en/example/fixtures/test_fixtures_order_scope.svg b/doc/en/example/fixtures/test_fixtures_order_scope.svg new file mode 100644 index 00000000000..ebaf7e4e245 --- /dev/null +++ b/doc/en/example/fixtures/test_fixtures_order_scope.svg @@ -0,0 +1,55 @@ + + + + + + + order + + sess + + pack + + mod + + cls + + func + + test_order + + + + + + TestClass + + diff --git a/doc/en/example/fixtures/test_fixtures_request_different_scope.py b/doc/en/example/fixtures/test_fixtures_request_different_scope.py new file mode 100644 index 00000000000..00e2e46d845 --- /dev/null +++ b/doc/en/example/fixtures/test_fixtures_request_different_scope.py @@ -0,0 +1,29 @@ +import pytest + + +@pytest.fixture +def order(): + return [] + + +@pytest.fixture +def outer(order, inner): + order.append("outer") + + +class TestOne: + @pytest.fixture + def inner(self, order): + order.append("one") + + def test_order(self, order, outer): + assert order == ["one", "outer"] + + +class TestTwo: + @pytest.fixture + def inner(self, order): + order.append("two") + + def test_order(self, order, outer): + assert order == ["two", "outer"] diff --git a/doc/en/example/fixtures/test_fixtures_request_different_scope.svg b/doc/en/example/fixtures/test_fixtures_request_different_scope.svg new file mode 100644 index 00000000000..ad98469ced0 --- /dev/null +++ b/doc/en/example/fixtures/test_fixtures_request_different_scope.svg @@ -0,0 +1,115 @@ + + + + + + + + + + test_fixtures_request_different_scope.py + + + + + + + + + + + + inner + + test_order + + + + + + TestOne + + + + + + + + 1 + + + + + + + 2 + + + + + + + + + + + inner + + test_order + + + + + + TestTwo + + + + + + + + 1 + + + 2 + + + outer + + order + diff --git a/doc/en/fixture.rst b/doc/en/fixture.rst index 963fc32e6b0..c74984563ab 100644 --- a/doc/en/fixture.rst +++ b/doc/en/fixture.rst @@ -12,6 +12,8 @@ pytest fixtures: explicit, modular, scalable .. _`xUnit`: https://en.wikipedia.org/wiki/XUnit .. _`Software test fixtures`: https://en.wikipedia.org/wiki/Test_fixture#Software .. _`Dependency injection`: https://en.wikipedia.org/wiki/Dependency_injection +.. _`Transaction`: https://en.wikipedia.org/wiki/Transaction_processing +.. _`linearizable`: https://en.wikipedia.org/wiki/Linearizability `Software test fixtures`_ initialize test functions. They provide a fixed baseline so that tests execute reliably and produce consistent, @@ -35,6 +37,10 @@ style of setup/teardown functions: to configuration and component options, or to re-use fixtures across function, class, module or whole test session scopes. +* teardown logic can be easily, and safely managed, no matter how many fixtures + are used, without the need to carefully handle errors by hand or micromanage + the order that cleanup steps are added. + In addition, pytest continues to support :ref:`xunitsetup`. You can mix both styles, moving incrementally from classic to new style, as you prefer. You can also start out from existing :ref:`unittest.TestCase @@ -115,32 +121,529 @@ for reference: .. _`@pytest.fixture`: .. _`pytest.fixture`: -Fixtures as Function arguments ------------------------------------------ +What fixtures are +----------------- + +Before we dive into what fixtures are, let's first look at what a test is. + +In the simplest terms, a test is meant to look at the result of a particular +behavior, and make sure that result aligns with what you would expect. +Behavior is not something that can be empirically measured, which is why writing +tests can be challenging. + +"Behavior" is the way in which some system **acts in response** to a particular +situation and/or stimuli. But exactly *how* or *why* something is done is not +quite as important as *what* was done. + +You can think of a test as being broken down into four steps: + +1. **Arrange** +2. **Act** +3. **Assert** +4. **Cleanup** + +**Arrange** is where we prepare everything for our test. This means pretty +much everything except for the "**act**". It's lining up the dominoes so that +the **act** can do its thing in one, state-changing step. This can mean +preparing objects, starting/killing services, entering records into a database, +or even things like defining a URL to query, generating some credentials for a +user that doesn't exist yet, or just waiting for some process to finish. + +**Act** is the singular, state-changing action that kicks off the **behavior** +we want to test. This behavior is what carries out the changing of the state of +the system under test (SUT), and it's the resulting changed state that we can +look at to make a judgement about the behavior. This typically takes the form of +a function/method call. + +**Assert** is where we look at that resulting state and check if it looks how +we'd expect after the dust has settled. It's where we gather evidence to say the +behavior does or does not aligns with what we expect. The ``assert`` in our test +is where we take that measurement/observation and apply our judgement to it. If +something should be green, we'd say ``assert thing == "green"``. + +**Cleanup** is where the test picks up after itself, so other tests aren't being +accidentally influenced by it. + +At it's core, the test is ultimately the **act** and **assert** steps, with the +**arrange** step only providing the context. **Behavior** exists between **act** +and **assert**. + +Back to fixtures +^^^^^^^^^^^^^^^^ + +"Fixtures", in the literal sense, are each of the **arrange** steps and data. They're +everything that test needs to do its thing. + +At a basic level, test functions request fixtures by declaring them as +arguments, as in the ``test_ehlo(smtp_connection):`` in the previous example. -Test functions can receive fixture objects by naming them as an input -argument. For each argument name, a fixture function with that name provides -the fixture object. Fixture functions are registered by marking them with -:py:func:`@pytest.fixture `. Let's look at a simple -self-contained test module containing a fixture and a test function -using it: +In pytest, "fixtures" are functions you define that serve this purpose. But they +don't have to be limited to just the **arrange** steps. They can provide the +**act** step, as well, and this can be a powerful technique for designing more +complex tests, especially given how pytest's fixture system works. But we'll get +into that further down. + +We can tell pytest that a particular function is a fixture by decorating it with +:py:func:`@pytest.fixture `. Here's a simple example of +what a fixture in pytest might look like: .. code-block:: python - # content of ./test_smtpsimple.py import pytest + class Fruit: + def __init__(self, name): + self.name = name + + def __eq__(self, other): + return self.name == other.name + + @pytest.fixture - def smtp_connection(): - import smtplib + def my_fruit(): + return Fruit("apple") + + + @pytest.fixture + def fruit_basket(my_fruit): + return [Fruit("banana"), my_fruit] + + + def test_my_fruit_in_basket(my_fruit, fruit_basket): + assert my_fruit in fruit_basket + + + +Tests don't have to be limited to a single fixture, either. They can depend on +as many fixtures as you want, and fixtures can use other fixtures, as well. This +is where pytest's fixture system really shines. + +Don't be afraid to break things up if it makes things cleaner. + +"Requesting" fixtures +--------------------- + +So fixtures are how we *prepare* for a test, but how do we tell pytest what +tests and fixtures need which fixtures? + +At a basic level, test functions request fixtures by declaring them as +arguments, as in the ``test_my_fruit_in_basket(my_fruit, fruit_basket):`` in the +previous example. + +At a basic level, pytest depends on a test to tell it what fixtures it needs, so +we have to build that information into the test itself. We have to make the test +"**request**" the fixtures it depends on, and to do this, we have to +list those fixtures as parameters in the test function's "signature" (which is +the ``def test_something(blah, stuff, more):`` line). + +When pytest goes to run a test, it looks at the parameters in that test +function's signature, and then searches for fixtures that have the same names as +those parameters. Once pytest finds them, it runs those fixtures, captures what +they returned (if anything), and passes those objects into the test function as +arguments. + +Quick example +^^^^^^^^^^^^^ + +.. code-block:: python + + import pytest + + + class Fruit: + def __init__(self, name): + self.name = name + self.cubed = False + + def cube(self): + self.cubed = True + + + class FruitSalad: + def __init__(self, *fruit_bowl): + self.fruit = fruit_bowl + self._cube_fruit() + + def _cube_fruit(self): + for fruit in self.fruit: + fruit.cube() + + + # Arrange + @pytest.fixture + def fruit_bowl(): + return [Fruit("apple"), Fruit("banana")] + + + def test_fruit_salad(fruit_bowl): + # Act + fruit_salad = FruitSalad(*fruit_bowl) + + # Assert + assert all(fruit.cubed for fruit in fruit_salad.fruit) + +In this example, ``test_fruit_salad`` "**requests**" ``fruit_bowl`` (i.e. +``def test_fruit_salad(fruit_bowl):``), and when pytest sees this, it will +execute the ``fruit_bowl`` fixture function and pass the object it returns into +``test_fruit_salad`` as the ``fruit_bowl`` argument. + +Here's roughly +what's happening if we were to do it by hand: + +.. code-block:: python + + def fruit_bowl(): + return [Fruit("apple"), Fruit("banana")] + + + def test_fruit_salad(fruit_bowl): + # Act + fruit_salad = FruitSalad(*fruit_bowl) + + # Assert + assert all(fruit.cubed for fruit in fruit_salad.fruit) + + + # Arrange + bowl = fruit_bowl() + test_fruit_salad(fruit_bowl=bowl) + +Fixtures can **request** other fixtures +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +One of pytest's greatest strengths is its extremely flexible fixture system. It +allows us to boil down complex requirements for tests into more simple and +organized functions, where we only need to have each one describe the things +they are dependent on. We'll get more into this further down, but for now, +here's a quick example to demonstrate how fixtures can use other fixtures: + +.. code-block:: python + + # contents of test_append.py + import pytest + + + # Arrange + @pytest.fixture + def first_entry(): + return "a" + + + # Arrange + @pytest.fixture + def order(first_entry): + return [first_entry] + + + def test_string(order): + # Act + order.append("b") + + # Assert + assert order == ["a", "b"] + + +Notice that this is the same example from above, but very little changed. The +fixtures in pytest **request** fixtures just like tests. All the same +**requesting** rules apply to fixtures that do for tests. Here's how this +example would work if we did it by hand: + +.. code-block:: python + + def first_entry(): + return "a" + + + def order(first_entry): + return [first_entry] + + + def test_string(order): + # Act + order.append("b") + + # Assert + assert order == ["a", "b"] + + + entry = first_entry() + the_list = order(first_entry=entry) + test_string(order=the_list) + +Fixtures are reusable +^^^^^^^^^^^^^^^^^^^^^ + +One of the things that makes pytest's fixture system so powerful, is that it +gives us the abilty to define a generic setup step that can reused over and +over, just like a normal function would be used. Two different tests can request +the same fixture and have pytest give each test their own result from that +fixture. + +This is extremely useful for making sure tests aren't affected by each other. We +can use this system to make sure each test gets its own fresh batch of data and +is starting from a clean state so it can provide consistent, repeatable results. + +Here's an example of how this can come in handy: + +.. code-block:: python + + # contents of test_append.py + import pytest + + + # Arrange + @pytest.fixture + def first_entry(): + return "a" + + + # Arrange + @pytest.fixture + def order(first_entry): + return [first_entry] + + + def test_string(order): + # Act + order.append("b") + + # Assert + assert order == ["a", "b"] + + + def test_int(order): + # Act + order.append(2) + + # Assert + assert order == ["a", 2] + + +Each test here is being given its own copy of that ``list`` object, +which means the ``order`` fixture is getting executed twice (the same +is true for the ``first_entry`` fixture). If we were to do this by hand as +well, it would look something like this: + +.. code-block:: python + + def first_entry(): + return "a" + + + def order(first_entry): + return [first_entry] + + def test_string(order): + # Act + order.append("b") + + # Assert + assert order == ["a", "b"] + + + def test_int(order): + # Act + order.append(2) + + # Assert + assert order == ["a", 2] + + + entry = first_entry() + the_list = order(first_entry=entry) + test_string(order=the_list) + + entry = first_entry() + the_list = order(first_entry=entry) + test_int(order=the_list) + +A test/fixture can **request** more than one fixture at a time +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Tests and fixtures aren't limited to **requesting** a single fixture at a time. +They can request as many as they like. Here's another quick example to +demonstrate: + +.. code-block:: python + + # contents of test_append.py + import pytest + + + # Arrange + @pytest.fixture + def first_entry(): + return "a" + + + # Arrange + @pytest.fixture + def second_entry(): + return 2 + + + # Arrange + @pytest.fixture + def order(first_entry, second_entry): + return [first_entry, second_entry] + + + # Arrange + @pytest.fixture + def expected_list(): + return ["a", 2, 3.0] + + + def test_string(order, expected_list): + # Act + order.append(3.0) + + # Assert + assert order == expected_list + +Fixtures can be **requested** more than once per test (return values are cached) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Fixtures can also be **requested** more than once during the same test, and +pytest won't execute them again for that test. This means we can **request** +fixtures in multiple fixtures that are dependent on them (and even again in the +test itself) without those fixtures being executed more than once. + +.. code-block:: python + + # contents of test_append.py + import pytest + + + # Arrange + @pytest.fixture + def first_entry(): + return "a" + + + # Arrange + @pytest.fixture + def order(): + return [] + + + # Act + @pytest.fixture + def append_first(order, first_entry): + return order.append(first_entry) + + + def test_string_only(append_first, order, first_entry): + # Assert + assert order == [first_entry] + +If a **requested** fixture was executed once for every time it was **requested** +during a test, then this test would fail because both ``append_first`` and +``test_string_only`` would see ``order`` as an empty list (i.e. ``[]``), but +since the return value of ``order`` was cached (along with any side effects +executing it may have had) after the first time it was called, both the test and +``append_first`` were referencing the same object, and the test saw the effect +``append_first`` had on that object. + +.. _`autouse`: +.. _`autouse fixtures`: + +Autouse fixtures (fixtures you don't have to request) +----------------------------------------------------- + +Sometimes you may want to have a fixture (or even several) that you know all +your tests will depend on. "Autouse" fixtures are a convenient way to make all +tests automatically **request** them. This can cut out a +lot of redundant **requests**, and can even provide more advanced fixture usage +(more on that further down). + +We can make a fixture an autouse fixture by passing in ``autouse=True`` to the +fixture's decorator. Here's a simple example for how they can be used: + +.. code-block:: python + + # contents of test_append.py + import pytest + + + @pytest.fixture + def first_entry(): + return "a" + + + @pytest.fixture + def order(first_entry): + return [] + + + @pytest.fixture(autouse=True) + def append_first(order, first_entry): + return order.append(first_entry) + + + def test_string_only(order, first_entry): + assert order == [first_entry] + + + def test_string_and_int(order, first_entry): + order.append(2) + assert order == [first_entry, 2] + +In this example, the ``append_first`` fixture is an autouse fixture. Because it +happens automatically, both tests are affected by it, even though neither test +**requested** it. That doesn't mean they *can't* be **requested** though; just +that it isn't *necessary*. + +.. _smtpshared: + +Scope: sharing fixtures across classes, modules, packages or session +-------------------------------------------------------------------- + +.. regendoc:wipe + +Fixtures requiring network access depend on connectivity and are +usually time-expensive to create. Extending the previous example, we +can add a ``scope="module"`` parameter to the +:py:func:`@pytest.fixture ` invocation +to cause a ``smtp_connection`` fixture function, responsible to create a connection to a preexisting SMTP server, to only be invoked +once per test *module* (the default is to invoke once per test *function*). +Multiple test functions in a test module will thus +each receive the same ``smtp_connection`` fixture instance, thus saving time. +Possible values for ``scope`` are: ``function``, ``class``, ``module``, ``package`` or ``session``. + +The next example puts the fixture function into a separate ``conftest.py`` file +so that tests from multiple test modules in the directory can +access the fixture function: + +.. code-block:: python + + # content of conftest.py + import pytest + import smtplib + + + @pytest.fixture(scope="module") + def smtp_connection(): return smtplib.SMTP("smtp.gmail.com", 587, timeout=5) +.. code-block:: python + + # content of test_module.py + + def test_ehlo(smtp_connection): response, msg = smtp_connection.ehlo() assert response == 250 + assert b"smtp.gmail.com" in msg + assert 0 # for demo purposes + + + def test_noop(smtp_connection): + response, msg = smtp_connection.noop() + assert response == 250 assert 0 # for demo purposes Here, the ``test_ehlo`` needs the ``smtp_connection`` fixture value. pytest @@ -149,442 +652,967 @@ marked ``smtp_connection`` fixture function. Running the test looks like this: .. code-block:: pytest - $ pytest test_smtpsimple.py + $ pytest test_module.py =========================== test session starts ============================ platform linux -- Python 3.x.y, pytest-6.x.y, py-1.x.y, pluggy-0.x.y cachedir: $PYTHON_PREFIX/.pytest_cache rootdir: $REGENDOC_TMPDIR - collected 1 item + collected 2 items + + test_module.py FF [100%] + + ================================= FAILURES ================================= + ________________________________ test_ehlo _________________________________ + + smtp_connection = + + def test_ehlo(smtp_connection): + response, msg = smtp_connection.ehlo() + assert response == 250 + assert b"smtp.gmail.com" in msg + > assert 0 # for demo purposes + E assert 0 + + test_module.py:7: AssertionError + ________________________________ test_noop _________________________________ + + smtp_connection = + + def test_noop(smtp_connection): + response, msg = smtp_connection.noop() + assert response == 250 + > assert 0 # for demo purposes + E assert 0 + + test_module.py:13: AssertionError + ========================= short test summary info ========================== + FAILED test_module.py::test_ehlo - assert 0 + FAILED test_module.py::test_noop - assert 0 + ============================ 2 failed in 0.12s ============================= + +You see the two ``assert 0`` failing and more importantly you can also see +that the **exactly same** ``smtp_connection`` object was passed into the +two test functions because pytest shows the incoming argument values in the +traceback. As a result, the two test functions using ``smtp_connection`` run +as quick as a single one because they reuse the same instance. + +If you decide that you rather want to have a session-scoped ``smtp_connection`` +instance, you can simply declare it: + +.. code-block:: python + + @pytest.fixture(scope="session") + def smtp_connection(): + # the returned fixture value will be shared for + # all tests requesting it + ... + + +Fixture scopes +^^^^^^^^^^^^^^ + +Fixtures are created when first requested by a test, and are destroyed based on their ``scope``: + +* ``function``: the default scope, the fixture is destroyed at the end of the test. +* ``class``: the fixture is destroyed during teardown of the last test in the class. +* ``module``: the fixture is destroyed during teardown of the last test in the module. +* ``package``: the fixture is destroyed during teardown of the last test in the package. +* ``session``: the fixture is destroyed at the end of the test session. + +.. note:: + + Pytest only caches one instance of a fixture at a time, which + means that when using a parametrized fixture, pytest may invoke a fixture more than once in + the given scope. + +.. _dynamic scope: + +Dynamic scope +^^^^^^^^^^^^^ + +.. versionadded:: 5.2 + +In some cases, you might want to change the scope of the fixture without changing the code. +To do that, pass a callable to ``scope``. The callable must return a string with a valid scope +and will be executed only once - during the fixture definition. It will be called with two +keyword arguments - ``fixture_name`` as a string and ``config`` with a configuration object. + +This can be especially useful when dealing with fixtures that need time for setup, like spawning +a docker container. You can use the command-line argument to control the scope of the spawned +containers for different environments. See the example below. + +.. code-block:: python + + def determine_scope(fixture_name, config): + if config.getoption("--keep-containers", None): + return "session" + return "function" + + + @pytest.fixture(scope=determine_scope) + def docker_container(): + yield spawn_container() + +Fixture errors +-------------- + +pytest does its best to put all the fixtures for a given test in a linear order +so that it can see which fixture happens first, second, third, and so on. If an +earlier fixture has a problem, though, and raises an exception, pytest will stop +executing fixtures for that test and mark the test as having an error. + +When a test is marked as having an error, it doesn't mean the test failed, +though. It just means the test couldn't even be attempted because one of the +things it depends on had a problem. + +This is one reason why it's a good idea to cut out as many unnecessary +dependencies as possible for a given test. That way a problem in something +unrelated isn't causing us to have an incomplete picture of what may or may not +have issues. + +Here's a quick example to help explain: + +.. code-block:: python + + import pytest + + + @pytest.fixture + def order(): + return [] + + + @pytest.fixture + def append_first(order): + order.append(1) + + + @pytest.fixture + def append_second(order, append_first): + order.extend([2]) + + + @pytest.fixture(autouse=True) + def append_third(order, append_second): + order += [3] + + + def test_order(order): + assert order == [1, 2, 3] + + +If, for whatever reason, ``order.append(1)`` had a bug and it raises an exception, +we wouldn't be able to know if ``order.extend([2])`` or ``order += [3]`` would +also have problems. After ``append_first`` throws an exception, pytest won't run +any more fixtures for ``test_order``, and it won't even try to run +``test_order`` itself. The only things that would've run would be ``order`` and +``append_first``. + + + + +.. _`finalization`: - test_smtpsimple.py F [100%] +Teardown/Cleanup (AKA Fixture finalization) +------------------------------------------- + +When we run our tests, we'll want to make sure they clean up after themselves so +they don't mess with any other tests (and also so that we don't leave behind a +mountain of test data to bloat the system). Fixtures in pytest offer a very +useful teardown system, which allows us to define the specific steps necessary +for each fixture to clean up after itself. + +This system can be leveraged in two ways. + +.. _`yield fixtures`: + +1. ``yield`` fixtures (recommended) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +"Yield" fixtures ``yield`` instead of ``return``. With these +fixtures, we can run some code and pass an object back to the requesting +fixture/test, just like with the other fixtures. The only differences are: + +1. ``return`` is swapped out for ``yield``. +2. Any teardown code for that fixture is placed *after* the ``yield``. + +Once pytest figures out a linear order for the fixtures, it will run each one up +until it returns or yields, and then move on to the next fixture in the list to +do the same thing. + +Once the test is finished, pytest will go back down the list of fixtures, but in +the *reverse order*, taking each one that yielded, and running the code inside +it that was *after* the ``yield`` statement. + +As a simple example, let's say we want to test sending email from one user to +another. We'll have to first make each user, then send the email from one user +to the other, and finally assert that the other user received that message in +their inbox. If we want to clean up after the test runs, we'll likely have to +make sure the other user's mailbox is emptied before deleting that user, +otherwise the system may complain. + +Here's what that might look like: + +.. code-block:: python + + import pytest + + from emaillib import Email, MailAdminClient + + + @pytest.fixture + def mail_admin(): + return MailAdminClient() + + + @pytest.fixture + def sending_user(mail_admin): + user = mail_admin.create_user() + yield user + admin_client.delete_user(user) + + + @pytest.fixture + def receiving_user(mail_admin): + user = mail_admin.create_user() + yield user + admin_client.delete_user(user) + + + def test_email_received(receiving_user, email): + email = Email(subject="Hey!", body="How's it going?") + sending_user.send_email(_email, receiving_user) + assert email in receiving_user.inbox + +Because ``receiving_user`` is the last fixture to run during setup, it's the first to run +during teardown. + +There is a risk that even having the order right on the teardown side of things +doesn't guarantee a safe cleanup. That's covered in a bit more detail in +:ref:`safe teardowns`. + +Handling errors for yield fixture +""""""""""""""""""""""""""""""""" + +If a yield fixture raises an exception before yielding, pytest won't try to run +the teardown code after that yield fixture's ``yield`` statement. But, for every +fixture that has already run successfully for that test, pytest will still +attempt to tear them down as it normally would. + +2. Adding finalizers directly +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +While yield fixtures are considered to be the cleaner and more straighforward +option, there is another choice, and that is to add "finalizer" functions +directly to the test's `request-context`_ object. It brings a similar result as +yield fixtures, but requires a bit more verbosity. + +In order to use this approach, we have to request the `request-context`_ object +(just like we would request another fixture) in the fixture we need to add +teardown code for, and then pass a callable, containing that teardown code, to +its ``addfinalizer`` method. + +We have to be careful though, because pytest will run that finalizer once it's +been added, even if that fixture raises an exception after adding the finalizer. +So to make sure we don't run the finalizer code when we wouldn't need to, we +would only add the finalizer once the fixture would have done something that +we'd need to teardown. + +Here's how the previous example would look using the ``addfinalizer`` method: + +.. code-block:: python + + import pytest + + from emaillib import Email, MailAdminClient + + + @pytest.fixture + def mail_admin(): + return MailAdminClient() + + + @pytest.fixture + def sending_user(mail_admin): + user = mail_admin.create_user() + yield user + admin_client.delete_user(user) + + + @pytest.fixture + def receiving_user(mail_admin, request): + user = mail_admin.create_user() + + def delete_user(): + admin_client.delete_user(user) + + request.addfinalizer(delete_user) + return user + + + @pytest.fixture + def email(sending_user, receiving_user, request): + _email = Email(subject="Hey!", body="How's it going?") + sending_user.send_email(_email, receiving_user) + + def empty_mailbox(): + receiving_user.delete_email(_email) + + request.addfinalizer(empty_mailbox) + return _email + + + def test_email_received(receiving_user, email): + assert email in receiving_user.inbox + + +It's a bit longer than yield fixtures and a bit more complex, but it +does offer some nuances for when you're in a pinch. + +.. _`safe teardowns`: + +Safe teardowns +-------------- + +The fixture system of pytest is *very* powerful, but it's still being run by a +computer, so it isn't able to figure out how to safely teardown everything we +throw at it. If we aren't careful, an error in the wrong spot might leave stuff +from our tests behind, and that can cause further issues pretty quickly. + +For example, consider the following tests (based off of the mail example from +above): + +.. code-block:: python + + import pytest + + from emaillib import Email, MailAdminClient + + + @pytest.fixture + def setup(): + mail_admin = MailAdminClient() + sending_user = mail_admin.create_user() + receiving_user = mail_admin.create_user() + email = Email(subject="Hey!", body="How's it going?") + sending_user.send_emai(email, receiving_user) + yield receiving_user, email + receiving_user.delete_email(email) + admin_client.delete_user(sending_user) + admin_client.delete_user(receiving_user) + + + def test_email_received(setup): + receiving_user, email = setup + assert email in receiving_user.inbox + +This version is a lot more compact, but it's also harder to read, doesn't have a +very descriptive fixture name, and none of the fixtures can be reused easily. + +There's also a more serious issue, which is that if any of those steps in the +setup raise an exception, none of the teardown code will run. + +One option might be to go with the ``addfinalizer`` method instead of yield +fixtures, but that might get pretty complex and difficult to maintain (and it +wouldn't be compact anymore). + +.. _`safe fixture structure`: + +Safe fixture structure +^^^^^^^^^^^^^^^^^^^^^^ + +The safest and simplest fixture structure requires limiting fixtures to only +making one state-changing action each, and then bundling them together with +their teardown code, as :ref:`the email examples above ` showed. + +The chance that a state-changing operation can fail but still modify state is +neglibible, as most of these operations tend to be `transaction`_-based (at +least at the level of testing where state could be left behind). So if we make +sure that any successful state-changing action gets torn down by moving it to a +separate fixture function and separating it from other, potentially failing +state-changing actions, then our tests will stand the best chance at leaving the +test environment the way they found it. + +For an example, let's say we have a website with a login page, and we have +access to an admin API where we can generate users. For our test, we want to: + +1. Create a user through that admin API +2. Launch a browser using Selenium +3. Go to the login page of our site +4. Log in as the user we created +5. Assert that their name is in the header of the landing page + +We wouldn't want to leave that user in the system, nor would we want to leave +that browser session running, so we'll want to make sure the fixtures that +create those things clean up after themselves. + +Here's what that might look like: + +.. note:: + + For this example, certain fixtures (i.e. ``base_url`` and + ``admin_credentials``) are implied to exist elsewhere. So for now, let's + assume they exist, and we're just not looking at them. + +.. code-block:: python + + from uuid import uuid4 + from urllib.parse import urljoin + + from selenium.webdriver import Chrome + import pytest + + from src.utils.pages import LoginPage, LandingPage + from src.utils import AdminApiClient + from src.utils.data_types import User + + + @pytest.fixture + def admin_client(base_url, admin_credentials): + return AdminApiClient(base_url, **admin_credentials) + + + @pytest.fixture + def user(admin_client): + _user = User(name="Susan", username=f"testuser-{uuid4()}", password="P4$$word") + admin_client.create_user(_user) + yield _user + admin_client.delete_user(_user) + + + @pytest.fixture + def driver(): + _driver = Chrome() + yield _driver + _driver.quit() + + + @pytest.fixture + def login(driver, base_url, user): + driver.get(urljoin(base_url, "/login")) + page = LoginPage(driver) + page.login(user) + + + @pytest.fixture + def landing_page(driver, login): + return LandingPage(driver) + + + def test_name_on_landing_page_after_login(landing_page, user): + assert landing_page.header == f"Welcome, {user.name}!" + +The way the dependencies are laid out means it's unclear if the ``user`` fixture +would execute before the ``driver`` fixture. But that's ok, because those are +atomic operations, and so it doesn't matter which one runs first because the +sequence of events for the test is still `linearizable`_. But what *does* matter +is that, no matter which one runs first, if the one raises an exception while +the other would not have, neither will have left anything behind. If ``driver`` +executes before ``user``, and ``user`` raises an exception, the driver will +still quit, and the user was never made. And if ``driver`` was the one to raise +the exception, then the driver would never have been started and the user would +never have been made. + +.. note: + + While the ``user`` fixture doesn't *actually* need to happen before the + ``driver`` fixture, if we made ``driver`` request ``user``, it might save + some time in the event that making the user raises an exception, since it + won't bother trying to start the driver, which is a fairly expensive + operation. + +.. _`conftest.py`: +.. _`conftest`: - ================================= FAILURES ================================= - ________________________________ test_ehlo _________________________________ +Fixture availabiility +--------------------- - smtp_connection = +Fixture availability is determined from the perspective of the test. A fixture +is only available for tests to request if they are in the scope that fixture is +defined in. If a fixture is defined inside a class, it can only be requested by +tests inside that class. But if a fixture is defined inside the global scope of +the module, than every test in that module, even if it's defined inside a class, +can request it. - def test_ehlo(smtp_connection): - response, msg = smtp_connection.ehlo() - assert response == 250 - > assert 0 # for demo purposes - E assert 0 +Similarly, a test can also only be affected by an autouse fixture if that test +is in the same scope that autouse fixture is defined in (see +:ref:`autouse order`). - test_smtpsimple.py:14: AssertionError - ========================= short test summary info ========================== - FAILED test_smtpsimple.py::test_ehlo - assert 0 - ============================ 1 failed in 0.12s ============================= +A fixture can also request any other fixture, no matter where it's defined, so +long as the test requesting them can see all fixtures involved. -In the failure traceback we see that the test function was called with a -``smtp_connection`` argument, the ``smtplib.SMTP()`` instance created by the fixture -function. The test function fails on our deliberate ``assert 0``. Here is -the exact protocol used by ``pytest`` to call the test function this way: +For example, here's a test file with a fixture (``outer``) that requests a +fixture (``inner``) from a scope it wasn't defined in: -1. pytest :ref:`finds ` the test ``test_ehlo`` because - of the ``test_`` prefix. The test function needs a function argument - named ``smtp_connection``. A matching fixture function is discovered by - looking for a fixture-marked function named ``smtp_connection``. +.. literalinclude:: example/fixtures/test_fixtures_request_different_scope.py -2. ``smtp_connection()`` is called to create an instance. +From the tests' perspectives, they have no problem seeing each of the fixtures +they're dependent on: -3. ``test_ehlo()`` is called and fails in the last - line of the test function. +.. image:: example/fixtures/test_fixtures_request_different_scope.svg + :align: center -Note that if you misspell a function argument or want -to use one that isn't available, you'll see an error -with a list of available function arguments. +So when they run, ``outer`` will have no problem finding ``inner``, because +pytest searched from the tests' perspectives. .. note:: + The scope a fixture is defined in has no bearing on the order it will be + instantiated in: the order is mandated by the logic described + :ref:`here `. - You can always issue: +``conftest.py``: sharing fixtures across multiple files +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - .. code-block:: bash +The ``conftest.py`` file serves as a means of providing fixtures for an entire +directory. Fixtures defined in a ``conftest.py`` can be used by any test +in that package without needing to import them (pytest will automatically +discover them). - pytest --fixtures test_simplefactory.py +You can have multiple nested directories/packages containing your tests, and +each directory can have its own ``conftest.py`` with its own fixtures, adding on +to the ones provided by the ``conftest.py`` files in parent directories. - to see available fixtures (fixtures with leading ``_`` are only shown if you add the ``-v`` option). +For example, given a test file structure like this: -Fixtures: a prime example of dependency injection ---------------------------------------------------- +:: -Fixtures allow test functions to easily receive and work -against specific pre-initialized application objects without having -to care about import/setup/cleanup details. -It's a prime example of `dependency injection`_ where fixture -functions take the role of the *injector* and test functions are the -*consumers* of fixture objects. + tests/ + __init__.py -.. _`conftest.py`: -.. _`conftest`: + conftest.py + # content of tests/conftest.py + import pytest -``conftest.py``: sharing fixture functions ------------------------------------------- + @pytest.fixture + def order(): + return [] -If during implementing your tests you realize that you -want to use a fixture function from multiple test files you can move it -to a ``conftest.py`` file. -You don't need to import the fixture you want to use in a test, it -automatically gets discovered by pytest. The discovery of -fixture functions starts at test classes, then test modules, then -``conftest.py`` files and finally builtin and third party plugins. + @pytest.fixture + def top(order, innermost): + order.append("top") -You can also use the ``conftest.py`` file to implement -:ref:`local per-directory plugins `. + test_top.py + # content of tests/test_top.py + import pytest -Sharing test data ------------------ + @pytest.fixture + def innermost(order): + order.append("innermost top") -If you want to make test data from files available to your tests, a good way -to do this is by loading these data in a fixture for use by your tests. -This makes use of the automatic caching mechanisms of pytest. + def test_order(order, top): + assert order == ["innermost top", "top"] -Another good approach is by adding the data files in the ``tests`` folder. -There are also community plugins available to help managing this aspect of -testing, e.g. `pytest-datadir `__ -and `pytest-datafiles `__. + subpackage/ + __init__.py -.. _smtpshared: + conftest.py + # content of tests/subpackage/conftest.py + import pytest -Scope: sharing fixtures across classes, modules, packages or session --------------------------------------------------------------------- + @pytest.fixture + def mid(order): + order.append("mid subpackage") -.. regendoc:wipe + test_subpackage.py + # content of tests/subpackage/test_subpackage.py + import pytest -Fixtures requiring network access depend on connectivity and are -usually time-expensive to create. Extending the previous example, we -can add a ``scope="module"`` parameter to the -:py:func:`@pytest.fixture ` invocation -to cause the decorated ``smtp_connection`` fixture function to only be invoked -once per test *module* (the default is to invoke once per test *function*). -Multiple test functions in a test module will thus -each receive the same ``smtp_connection`` fixture instance, thus saving time. -Possible values for ``scope`` are: ``function``, ``class``, ``module``, ``package`` or ``session``. + @pytest.fixture + def innermost(order, mid): + order.append("innermost subpackage") -The next example puts the fixture function into a separate ``conftest.py`` file -so that tests from multiple test modules in the directory can -access the fixture function: + def test_order(order, top): + assert order == ["mid subpackage", "innermost subpackage", "top"] -.. code-block:: python +The boundaries of the scopes can be visualized like this: - # content of conftest.py - import pytest - import smtplib +.. image:: example/fixtures/fixture_availability.svg + :align: center +The directories become their own sort of scope where fixtures that are defined +in a ``conftest.py`` file in that directory become available for that whole +scope. - @pytest.fixture(scope="module") - def smtp_connection(): - return smtplib.SMTP("smtp.gmail.com", 587, timeout=5) +Tests are allowed to search upward (stepping outside a circle) for fixtures, but +can never go down (stepping inside a circle) to continue their search. So +``tests/subpackage/test_subpackage.py::test_order`` would be able to find the +``innermost`` fixture defined in ``tests/subpackage/test_subpackage.py``, but +the one defined in ``tests/test_top.py`` would be unavailable to it because it +would have to step down a level (step inside a circle) to find it. -The name of the fixture again is ``smtp_connection`` and you can access its -result by listing the name ``smtp_connection`` as an input parameter in any -test or fixture function (in or below the directory where ``conftest.py`` is -located): +The first fixture the test finds is the one that will be used, so +:ref:`fixtures can be overriden ` if you need to change or +extend what one does for a particular scope. -.. code-block:: python +You can also use the ``conftest.py`` file to implement +:ref:`local per-directory plugins `. - # content of test_module.py +Fixtures from third-party plugins +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Fixtures don't have to be defined in this structure to be available for tests, +though. They can also be provided by third-party plugins that are installed, and +this is how many pytest plugins operate. As long as those plugins are installed, +the fixtures they provide can be requested from anywhere in your test suite. - def test_ehlo(smtp_connection): - response, msg = smtp_connection.ehlo() - assert response == 250 - assert b"smtp.gmail.com" in msg - assert 0 # for demo purposes +Because they're provided from outside the structure of your test suite, +third-party plugins don't really provide a scope like `conftest.py` files and +the directories in your test suite do. As a result, pytest will search for +fixtures stepping out through scopes as explained previously, only reaching +fixtures defined in plugins *last*. +For example, given the following file structure: - def test_noop(smtp_connection): - response, msg = smtp_connection.noop() - assert response == 250 - assert 0 # for demo purposes +:: -We deliberately insert failing ``assert 0`` statements in order to -inspect what is going on and can now run the tests: + tests/ + __init__.py -.. code-block:: pytest + conftest.py + # content of tests/conftest.py + import pytest - $ pytest test_module.py - =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-6.x.y, py-1.x.y, pluggy-0.x.y - cachedir: $PYTHON_PREFIX/.pytest_cache - rootdir: $REGENDOC_TMPDIR - collected 2 items + @pytest.fixture + def order(): + return [] - test_module.py FF [100%] + subpackage/ + __init__.py - ================================= FAILURES ================================= - ________________________________ test_ehlo _________________________________ + conftest.py + # content of tests/subpackage/conftest.py + import pytest - smtp_connection = + @pytest.fixture(autouse=True) + def mid(order, b_fix): + order.append("mid subpackage") - def test_ehlo(smtp_connection): - response, msg = smtp_connection.ehlo() - assert response == 250 - assert b"smtp.gmail.com" in msg - > assert 0 # for demo purposes - E assert 0 + test_subpackage.py + # content of tests/subpackage/test_subpackage.py + import pytest - test_module.py:7: AssertionError - ________________________________ test_noop _________________________________ + @pytest.fixture + def inner(order, mid, a_fix): + order.append("inner subpackage") - smtp_connection = + def test_order(order, inner): + assert order == ["b_fix", "mid subpackage", "a_fix", "inner subpackage"] - def test_noop(smtp_connection): - response, msg = smtp_connection.noop() - assert response == 250 - > assert 0 # for demo purposes - E assert 0 +If ``plugin_a`` is installed and provides the fixture ``a_fix``, and +``plugin_b`` is installed and provides the fixture ``b_fix``, then this is what +the test's search for fixtures would look like: - test_module.py:13: AssertionError - ========================= short test summary info ========================== - FAILED test_module.py::test_ehlo - assert 0 - FAILED test_module.py::test_noop - assert 0 - ============================ 2 failed in 0.12s ============================= +.. image:: example/fixtures/fixture_availability_plugins.svg + :align: center -You see the two ``assert 0`` failing and more importantly you can also see -that the same (module-scoped) ``smtp_connection`` object was passed into the -two test functions because pytest shows the incoming argument values in the -traceback. As a result, the two test functions using ``smtp_connection`` run -as quick as a single one because they reuse the same instance. +pytest will only search for ``a_fix`` and ``b_fix`` in the plugins after +searching for them first in the scopes inside ``tests/``. -If you decide that you rather want to have a session-scoped ``smtp_connection`` -instance, you can simply declare it: +.. note: -.. code-block:: python + pytest can tell you what fixtures are available for a given test if you call + ``pytests`` along with the test's name (or the scope it's in), and provide + the ``--fixtures`` flag, e.g. ``pytest --fixtures test_something.py`` + (fixtures with names that start with ``_`` will only be shown if you also + provide the ``-v`` flag). - @pytest.fixture(scope="session") - def smtp_connection(): - # the returned fixture value will be shared for - # all tests needing it - ... +Sharing test data +----------------- +If you want to make test data from files available to your tests, a good way +to do this is by loading these data in a fixture for use by your tests. +This makes use of the automatic caching mechanisms of pytest. -Fixture scopes -^^^^^^^^^^^^^^ +Another good approach is by adding the data files in the ``tests`` folder. +There are also community plugins available to help managing this aspect of +testing, e.g. `pytest-datadir `__ +and `pytest-datafiles `__. -Fixtures are created when first requested by a test, and are destroyed based on their ``scope``: +.. _`fixture order`: -* ``function``: the default scope, the fixture is destroyed at the end of the test. -* ``class``: the fixture is destroyed during teardown of the last test in the class. -* ``module``: the fixture is destroyed during teardown of the last test in the module. -* ``package``: the fixture is destroyed during teardown of the last test in the package. -* ``session``: the fixture is destroyed at the end of the test session. +Fixture instantiation order +--------------------------- -.. note:: +When pytest wants to execute a test, once it knows what fixtures will be +executed, it has to figure out the order they'll be executed in. To do this, it +considers 3 factors: - Pytest only caches one instance of a fixture at a time, which - means that when using a parametrized fixture, pytest may invoke a fixture more than once in - the given scope. +1. scope +2. dependencies +3. autouse -.. _dynamic scope: +Names of fixtures or tests, where they're defined, the order they're defined in, +and the order fixtures are requested in have no bearing on execution order +beyond coincidence. While pytest will try to make sure coincidences like these +stay consistent from run to run, it's not something that should be depended on. +If you want to control the order, it's safest to rely on these 3 things and make +sure dependencies are clearly established. -Dynamic scope -^^^^^^^^^^^^^ +Higher-scoped fixtures are executed first +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 5.2 +Within a function request for fixtures, those of higher-scopes (such as +``session``) are executed before lower-scoped fixtures (such as ``function`` or +``class``). -In some cases, you might want to change the scope of the fixture without changing the code. -To do that, pass a callable to ``scope``. The callable must return a string with a valid scope -and will be executed only once - during the fixture definition. It will be called with two -keyword arguments - ``fixture_name`` as a string and ``config`` with a configuration object. +Here's an example: -This can be especially useful when dealing with fixtures that need time for setup, like spawning -a docker container. You can use the command-line argument to control the scope of the spawned -containers for different environments. See the example below. +.. literalinclude:: example/fixtures/test_fixtures_order_scope.py -.. code-block:: python +The test will pass because the larger scoped fixtures are executing first. - def determine_scope(fixture_name, config): - if config.getoption("--keep-containers", None): - return "session" - return "function" +The order breaks down to this: +.. image:: example/fixtures/test_fixtures_order_scope.svg + :align: center - @pytest.fixture(scope=determine_scope) - def docker_container(): - yield spawn_container() +Fixtures of the same order execute based on dependencies +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +When a fixture requests another fixture, the other fixture is executed first. +So if fixture ``a`` requests fixture ``b``, fixture ``b`` will execute first, +because ``a`` depends on ``b`` and can't operate without it. Even if ``a`` +doesn't need the result of ``b``, it can still request ``b`` if it needs to make +sure it is executed after ``b``. +For example: -Order: Higher-scoped fixtures are instantiated first ----------------------------------------------------- +.. literalinclude:: example/fixtures/test_fixtures_order_dependencies.py +If we map out what depends on what, we get something that look like this: +.. image:: example/fixtures/test_fixtures_order_dependencies.svg + :align: center -Within a function request for fixtures, those of higher-scopes (such as ``session``) are instantiated before -lower-scoped fixtures (such as ``function`` or ``class``). The relative order of fixtures of same scope follows -the declared order in the test function and honours dependencies between fixtures. Autouse fixtures will be -instantiated before explicitly used fixtures. +The rules provided by each fixture (as to what fixture(s) each one has to come +after) are comprehensive enough that it can be flattened to this: -Consider the code below: +.. image:: example/fixtures/test_fixtures_order_dependencies_flat.svg + :align: center -.. literalinclude:: example/fixtures/test_fixtures_order.py +Enough information has to be provided through these requests in order for pytest +to be able to figure out a clear, linear chain of dependencies, and as a result, +an order of operations for a given test. If there's any ambiguity, and the order +of operations can be interpreted more than one way, you should assume pytest +could go with any one of those interpretations at any point. -The fixtures requested by ``test_order`` will be instantiated in the following order: +For example, if ``d`` didn't request ``c``, i.e.the graph would look like this: -1. ``s1``: is the highest-scoped fixture (``session``). -2. ``m1``: is the second highest-scoped fixture (``module``). -3. ``a1``: is a ``function``-scoped ``autouse`` fixture: it will be instantiated before other fixtures - within the same scope. -4. ``f3``: is a ``function``-scoped fixture, required by ``f1``: it needs to be instantiated at this point -5. ``f1``: is the first ``function``-scoped fixture in ``test_order`` parameter list. -6. ``f2``: is the last ``function``-scoped fixture in ``test_order`` parameter list. +.. image:: example/fixtures/test_fixtures_order_dependencies_unclear.svg + :align: center +Because nothing requested ``c`` other than ``g``, and ``g`` also requests ``f``, +it's now unclear if ``c`` should go before/after ``f``, ``e``, or ``d``. The +only rules that were set for ``c`` is that it must execute after ``b`` and +before ``g``. -.. _`finalization`: +pytest doesn't know where ``c`` should go in the case, so it should be assumed +that it could go anywhere between ``g`` and ``b``. -Fixture finalization / executing teardown code -------------------------------------------------------------- +This isn't necessarily bad, but it's something to keep in mind. If the order +they execute in could affect the behavior a test is targetting, or could +otherwise influence the result of a test, then the order should be defined +explicitely in a way that allows pytest to linearize/"flatten" that order. -pytest supports execution of fixture specific finalization code -when the fixture goes out of scope. By using a ``yield`` statement instead of ``return``, all -the code after the *yield* statement serves as the teardown code: +.. _`autouse order`: -.. code-block:: python +Autouse fixtures are executed first within their scope +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - # content of conftest.py +Autouse fixtures are assumed to apply to every test that could reference them, +so they are executed before other fixtures in that scope. Fixtures that are +requested by autouse fixtures effectively become autouse fixtures themselves for +the tests that the real autouse fixture applies to. - import smtplib - import pytest +So if fixture ``a`` is autouse and fixture ``b`` is not, but fixture ``a`` +requests fixture ``b``, then fixture ``b`` will effectively be an autouse +fixture as well, but only for the tests that ``a`` applies to. +In the last example, the graph became unclear if ``d`` didn't request ``c``. But +if ``c`` was autouse, then ``b`` and ``a`` would effectively also be autouse +because ``c`` depends on them. As a result, they would all be shifted above +non-autouse fixtures within that scope. - @pytest.fixture(scope="module") - def smtp_connection(): - smtp_connection = smtplib.SMTP("smtp.gmail.com", 587, timeout=5) - yield smtp_connection # provide the fixture value - print("teardown smtp") - smtp_connection.close() +So if the test file looked like this: -The ``print`` and ``smtp.close()`` statements will execute when the last test in -the module has finished execution, regardless of the exception status of the -tests. +.. literalinclude:: example/fixtures/test_fixtures_order_autouse.py -Let's execute it: +the graph would look like this: -.. code-block:: pytest +.. image:: example/fixtures/test_fixtures_order_autouse.svg + :align: center - $ pytest -s -q --tb=no - FFteardown smtp +Because ``c`` can now be put above ``d`` in the graph, pytest can once again +linearize the graph to this: - ========================= short test summary info ========================== - FAILED test_module.py::test_ehlo - assert 0 - FAILED test_module.py::test_noop - assert 0 - 2 failed in 0.12s +In this example, ``c`` makes ``b`` and ``a`` effectively autouse fixtures as +well. -We see that the ``smtp_connection`` instance is finalized after the two -tests finished execution. Note that if we decorated our fixture -function with ``scope='function'`` then fixture setup and cleanup would -occur around each single test. In either case the test -module itself does not need to change or know about these details -of fixture setup. +Be careful with autouse, though, as an autouse fixture will automatically +execute for every test that can reach it, even if they don't request it. For +example, consider this file: -Note that we can also seamlessly use the ``yield`` syntax with ``with`` statements: +.. literalinclude:: example/fixtures/test_fixtures_order_autouse_multiple_scopes.py -.. code-block:: python +Even though nothing in ``TestClassWithC1Request`` is requesting ``c1``, it still +is executed for the tests inside it anyway: - # content of test_yield2.py +.. image:: example/fixtures/test_fixtures_order_autouse_multiple_scopes.svg + :align: center - import smtplib - import pytest +But just because one autouse fixture requested a non-autouse fixture, that +doesn't mean the non-autouse fixture becomes an autouse fixture for all contexts +that it can apply to. It only effectively becomes an auotuse fixture for the +contexts the real autouse fixture (the one that requested the non-autouse +fixture) can apply to. +For example, take a look at this test file: - @pytest.fixture(scope="module") - def smtp_connection(): - with smtplib.SMTP("smtp.gmail.com", 587, timeout=5) as smtp_connection: - yield smtp_connection # provide the fixture value +.. literalinclude:: example/fixtures/test_fixtures_order_autouse_temp_effects.py +It would break down to something like this: -The ``smtp_connection`` connection will be closed after the test finished -execution because the ``smtp_connection`` object automatically closes when -the ``with`` statement ends. +.. image:: example/fixtures/test_fixtures_order_autouse_temp_effects.svg + :align: center -Using the contextlib.ExitStack context manager finalizers will always be called -regardless if the fixture *setup* code raises an exception. This is handy to properly -close all resources created by a fixture even if one of them fails to be created/acquired: +For ``test_req`` and ``test_no_req`` inside ``TestClassWithAutouse``, ``c3`` +effectively makes ``c2`` an autouse fixture, which is why ``c2`` and ``c3`` are +executed for both tests, despite not being requested, and why ``c2`` and ``c3`` +are executed before ``c1`` for ``test_req``. -.. code-block:: python +If this made ``c2`` an *actual* autouse fixture, then ``c2`` would also execute +for the tests inside ``TestClassWithoutAutouse``, since they can reference +``c2`` if they wanted to. But it doesn't, because from the perspective of the +``TestClassWithoutAutouse`` tests, ``c2`` isn't an autouse fixture, since they +can't see ``c3``. - # content of test_yield3.py - import contextlib +.. note: - import pytest + pytest can tell you what order the fixtures will execute in for a given test + if you call ``pytests`` along with the test's name (or the scope it's in), + and provide the ``--setup-plan`` flag, e.g. + ``pytest --setup-plan test_something.py`` (fixtures with names that start + with ``_`` will only be shown if you also provide the ``-v`` flag). - @contextlib.contextmanager - def connect(port): - ... # create connection - yield - ... # close connection +Running multiple ``assert`` statements safely +--------------------------------------------- +Sometimes you may want to run multiple asserts after doing all that setup, which +makes sense as, in more complex systems, a single action can kick off multiple +behaviors. pytest has a convenient way of handling this and it combines a bunch +of what we've gone over so far. - @pytest.fixture - def equipments(): - with contextlib.ExitStack() as stack: - yield [stack.enter_context(connect(port)) for port in ("C1", "C3", "C28")] +All that's needed is stepping up to a larger scope, then having the **act** +step defined as an autouse fixture, and finally, making sure all the fixtures +are targetting that highler level scope. -In the example above, if ``"C28"`` fails with an exception, ``"C1"`` and ``"C3"`` will still -be properly closed. +Let's pull :ref:`an example from above `, and tweak it a +bit. Let's say that in addition to checking for a welcome message in the header, +we also want to check for a sign out button, and a link to the user's profile. -Note that if an exception happens during the *setup* code (before the ``yield`` keyword), the -*teardown* code (after the ``yield``) will not be called. +Let's take a look at how we can structure that so we can run multiple asserts +without having to repeat all those steps again. -An alternative option for executing *teardown* code is to -make use of the ``addfinalizer`` method of the `request-context`_ object to register -finalization functions. +.. note:: -Here's the ``smtp_connection`` fixture changed to use ``addfinalizer`` for cleanup: + For this example, certain fixtures (i.e. ``base_url`` and + ``admin_credentials``) are implied to exist elsewhere. So for now, let's + assume they exist, and we're just not looking at them. .. code-block:: python - # content of conftest.py - import smtplib + # contents of tests/end_to_end/test_login.py + from uuid import uuid4 + from urllib.parse import urljoin + + from selenium.webdriver import Chrome import pytest + from src.utils.pages import LoginPage, LandingPage + from src.utils import AdminApiClient + from src.utils.data_types import User - @pytest.fixture(scope="module") - def smtp_connection(request): - smtp_connection = smtplib.SMTP("smtp.gmail.com", 587, timeout=5) - def fin(): - print("teardown smtp_connection") - smtp_connection.close() + @pytest.fixture(scope="class") + def admin_client(base_url, admin_credentials): + return AdminApiClient(base_url, **admin_credentials) - request.addfinalizer(fin) - return smtp_connection # provide the fixture value + @pytest.fixture(scope="class") + def user(admin_client): + _user = User(name="Susan", username=f"testuser-{uuid4()}", password="P4$$word") + admin_client.create_user(_user) + yield _user + admin_client.delete_user(_user) -Here's the ``equipments`` fixture changed to use ``addfinalizer`` for cleanup: -.. code-block:: python + @pytest.fixture(scope="class") + def driver(): + _driver = Chrome() + yield _driver + _driver.quit() - # content of test_yield3.py - import contextlib - import functools + @pytest.fixture(scope="class") + def landing_page(driver, login): + return LandingPage(driver) - import pytest + class TestLandingPageSuccess: + @pytest.fixture(scope="class", autouse=True) + def login(self, driver, base_url, user): + driver.get(urljoin(base_url, "/login")) + page = LoginPage(driver) + page.login(user) - @contextlib.contextmanager - def connect(port): - ... # create connection - yield - ... # close connection + def test_name_in_header(self, landing_page, user): + assert landing_page.header == f"Welcome, {user.name}!" + def test_sign_out_button(self, landing_page): + assert landing_page.sign_out_button.is_displayed() - @pytest.fixture - def equipments(request): - r = [] - for port in ("C1", "C3", "C28"): - cm = connect(port) - equip = cm.__enter__() - request.addfinalizer(functools.partial(cm.__exit__, None, None, None)) - r.append(equip) - return r + def test_profile_link(self, landing_page, user): + profile_href = urljoin(base_url, f"/profile?id={user.profile_id}") + assert landing_page.profile_link.get_attribute("href") == profile_href +Notice that the methods are only referencing ``self`` in the signature as a +formality. No state is tied to the actual test class as it might be in the +``unittest.TestCase`` framework. Everything is managed by the pytest fixture +system. -Both ``yield`` and ``addfinalizer`` methods work similarly by calling their code after the test -ends. Of course, if an exception happens before the finalize function is registered then it -will not be executed. +Each method only has to request the fixtures that it actually needs without +worrying about order. This is because the **act** fixture is an autouse fixture, +and it made sure all the other fixtures executed before it. There's no more +changes of state that need to take place, so the tests are free to make as many +non-state-changing queries as they want without risking stepping on the toes of +the other tests. + +The ``login`` fixture is defined inside the class as well, because not every one +of the other tests in the module will be expecting a successful login, and the **act** may need to +be handled a little differently for another test class. For example, if we +wanted to write another test scenario around submitting bad credentials, we +could handle it by adding something like this to the test file: + +.. note: + + It's assumed that the page object for this (i.e. ``LoginPage``) raises a + custom exception, ``BadCredentialsException``, when it recognizes text + signifying that on the login form after attempting to log in. + +.. code-block:: python + + class TestLandingPageBadCredentials: + @pytest.fixture(scope="class") + def faux_user(self, user): + _user = deepcopy(user) + _user.password = "badpass" + return _user + + def test_raises_bad_credentials_exception(self, login_page, faux_user): + with pytest.raises(BadCredentialsException): + login_page.login(faux_user) .. _`request-context`: @@ -1239,116 +2267,7 @@ into an ini-file: Currently this will not generate any error or warning, but this is intended to be handled by `#3664 `_. - -.. _`autouse`: -.. _`autouse fixtures`: - -Autouse fixtures (xUnit setup on steroids) ----------------------------------------------------------------------- - -.. regendoc:wipe - -Occasionally, you may want to have fixtures get invoked automatically -without declaring a function argument explicitly or a `usefixtures`_ decorator. -As a practical example, suppose we have a database fixture which has a -begin/rollback/commit architecture and we want to automatically surround -each test method by a transaction and a rollback. Here is a dummy -self-contained implementation of this idea: - -.. code-block:: python - - # content of test_db_transact.py - - import pytest - - - class DB: - def __init__(self): - self.intransaction = [] - - def begin(self, name): - self.intransaction.append(name) - - def rollback(self): - self.intransaction.pop() - - - @pytest.fixture(scope="module") - def db(): - return DB() - - - class TestClass: - @pytest.fixture(autouse=True) - def transact(self, request, db): - db.begin(request.function.__name__) - yield - db.rollback() - - def test_method1(self, db): - assert db.intransaction == ["test_method1"] - - def test_method2(self, db): - assert db.intransaction == ["test_method2"] - -The class-level ``transact`` fixture is marked with *autouse=true* -which implies that all test methods in the class will use this fixture -without a need to state it in the test function signature or with a -class-level ``usefixtures`` decorator. - -If we run it, we get two passing tests: - -.. code-block:: pytest - - $ pytest -q - .. [100%] - 2 passed in 0.12s - -Here is how autouse fixtures work in other scopes: - -- autouse fixtures obey the ``scope=`` keyword-argument: if an autouse fixture - has ``scope='session'`` it will only be run once, no matter where it is - defined. ``scope='class'`` means it will be run once per class, etc. - -- if an autouse fixture is defined in a test module, all its test - functions automatically use it. - -- if an autouse fixture is defined in a conftest.py file then all tests in - all test modules below its directory will invoke the fixture. - -- lastly, and **please use that with care**: if you define an autouse - fixture in a plugin, it will be invoked for all tests in all projects - where the plugin is installed. This can be useful if a fixture only - anyway works in the presence of certain settings e. g. in the ini-file. Such - a global fixture should always quickly determine if it should do - any work and avoid otherwise expensive imports or computation. - -Note that the above ``transact`` fixture may very well be a fixture that -you want to make available in your project without having it generally -active. The canonical way to do that is to put the transact definition -into a conftest.py file **without** using ``autouse``: - -.. code-block:: python - - # content of conftest.py - @pytest.fixture - def transact(request, db): - db.begin() - yield - db.rollback() - -and then e.g. have a TestClass using it by declaring the need: - -.. code-block:: python - - @pytest.mark.usefixtures("transact") - class TestClass: - def test_method1(self): - ... - -All test methods in this TestClass will use the transaction fixture while -other test classes or functions in the module will not use it unless -they also add a ``transact`` reference. +.. _`override fixtures`: Overriding fixtures on various levels -------------------------------------