Python 3.10 support

.github/workflows/tox.yml

@@ -30,7 +30,7 @@ jobs:
     strategy:
       max-parallel: 5
       matrix:
-        python-version: [2.7, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9]
+        python-version: [2.7, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10']
 
     steps:
     - uses: actions/checkout@v2

kolibri/utils/compat.py

@@ -82,3 +82,22 @@ def parse_version(v):
     parsed = _parse_version(v)
 
     return VersionCompat(parsed)
+
+
+def monkey_patch_collections():
+    """
+    Monkey-patching for the collections module is required for Python 3.10
+    and above.
+    Prior to 3.10, the collections module still contained all the entities defined in
+    collections.abc from Python 3.3 onwards. Here we patch those back into main
+    collections module.
+    This can be removed when we upgrade to a version of Django that is Python 3.10 compatible.
+    """
+    if sys.version_info < (3, 10):
+        return
+    import collections
+    from collections import abc
+
+    for name in dir(abc):
+        if not hasattr(collections, name):
+            setattr(collections, name, getattr(abc, name))

kolibri/utils/env.py

@@ -18,6 +18,7 @@
     ColoredFormatter = None
 
 from .logger import LOG_COLORS
+from kolibri.utils.compat import monkey_patch_collections
 
 
 logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO)
@@ -124,6 +125,8 @@ def set_env():
 
     check_python_versions()
 
+    monkey_patch_collections()
+
     sys.path = [os.path.realpath(os.path.dirname(kolibri_dist.__file__))] + sys.path
 
     # Add path for c extensions to sys.path

setup.py

@@ -107,5 +107,5 @@ def run(self):
         "Programming Language :: Python :: Implementation :: PyPy",
     ],
     cmdclass={"install_scripts": gen_windows_batch_files},
-    python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <3.10",
+    python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <3.11",
 )

test/patch_pytest.py

@@ -0,0 +1,38 @@
+"""
+This script backports this Python 3.10 compatibility fix https://github.com/pytest-dev/pytest/pull/8540
+in order to allow pytest to run in Python 3.10 without upgrading to version 6.2.5 which does not support 2.7
+"""
+import os
+import subprocess
+
+import pytest
+
+site_packages_dir = os.path.dirname(pytest.__file__)
+
+patch_file = os.path.join(os.path.dirname(__file__), "pytest_3.10.patch")
+
+print("Applying patch: " + str(patch_file))
+
+# -N: insist this is FORWARD patch, don't reverse apply
+# -p1: strip first path component
+# -t: batch mode, don't ask questions
+patch_command = ["patch", "-N", "-p1", "-d", site_packages_dir, "-t", "-i", patch_file]
+print(" ".join(patch_command))
+try:
+    # Use a dry run to establish whether the patch is already applied.
+    # If we don't check this, the patch may be partially applied (which is bad!)
+    subprocess.check_output(patch_command + ["--dry-run"])
+except subprocess.CalledProcessError as e:
+    if e.returncode == 1:
+        # Return code 1 means not all hunks could be applied, this usually
+        # means the patch is already applied.
+        print(
+            "Warning: failed to apply patch (exit code 1), "
+            "assuming it is already applied: ",
+            str(patch_file),
+        )
+    else:
+        raise e
+else:
+    # The dry run worked, so do the real thing
+    subprocess.check_output(patch_command)

test/pytest_3.10.patch

@@ -0,0 +1,38 @@
+diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
+index 4f96b9e8c..1aa5b12de 100644
+--- a/_pytest/assertion/rewrite.py
++++ b/_pytest/assertion/rewrite.py
+@@ -587,10 +587,6 @@ class AssertionRewriter(ast.NodeVisitor):
+             return
+         # Insert some special imports at the top of the module but after any
+         # docstrings and __future__ imports.
+-        aliases = [
+-            ast.alias(py.builtin.builtins.__name__, "@py_builtins"),
+-            ast.alias("_pytest.assertion.rewrite", "@pytest_ar"),
+-        ]
+         doc = getattr(mod, "docstring", None)
+         expect_docstring = doc is None
+         if doc is not None and self.is_rewrite_disabled(doc):
+@@ -617,6 +613,22 @@ class AssertionRewriter(ast.NodeVisitor):
+             pos += 1
+         else:
+             lineno = item.lineno
++        # Now actually insert the special imports.
++        if sys.version_info >= (3, 10):
++            aliases = [
++                ast.alias("builtins", "@py_builtins", lineno=lineno, col_offset=0),
++                ast.alias(
++                    "_pytest.assertion.rewrite",
++                    "@pytest_ar",
++                    lineno=lineno,
++                    col_offset=0,
++                ),
++            ]
++        else:
++            aliases = [
++                ast.alias("builtins", "@py_builtins"),
++                ast.alias("_pytest.assertion.rewrite", "@pytest_ar"),
++            ]
+         imports = [
+             ast.Import([alias], lineno=lineno, col_offset=0) for alias in aliases
+         ]

test/test_future_and_futures.py

@@ -2,9 +2,10 @@
 import os
 import sys
 
-from django.test import TestCase
+# Import from kolibri first to ensure Kolibri's monkey patches are applied.
+from kolibri import dist as kolibri_dist  # noreorder
 
-from kolibri import dist as kolibri_dist
+from django.test import TestCase  # noreorder
 
 dist_dir = os.path.realpath(os.path.dirname(kolibri_dist.__file__))
 

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py{2.7,3.4,3.5,3.6,3.7,3.8,3.9}, postgres
+envlist = py{2.7,3.4,3.5,3.6,3.7,3.8,3.9,3.10}, postgres
 
 [testenv]
 usedevelop = True
@@ -20,6 +20,7 @@ basepython =
     py3.7: python3.7
     py3.8: python3.8
     py3.9: python3.9
+    py3.10: python3.10
 deps =
     -r{toxinidir}/requirements/test.txt
     -r{toxinidir}/requirements/base.txt
@@ -32,6 +33,17 @@ commands =
     # Fail if the log is longer than 200 lines (something erroring or very noisy got added)
     sh -c "if [ `cat {env:KOLIBRI_HOME}/logs/kolibri.txt | wc -l` -gt 200 ] ; then echo 'Log too long' && echo '' && tail -n 20 {env:KOLIBRI_HOME}/logs/kolibri.txt && exit 1 ; fi"
 
+[testenv:py3.10]
+commands =
+    sh -c 'kolibri manage makemigrations --check'
+    python test/patch_pytest.py
+    # Run the actual tests
+    python -O -m pytest {posargs:--cov=kolibri --cov-report= --cov-append --color=no} kolibri
+    python -O -m pytest {posargs:--cov=kolibri --cov-report= --cov-append --color=no} -p no:django test
+    # Fail if the log is longer than 200 lines (something erroring or very noisy got added)
+    sh -c "if [ `cat {env:KOLIBRI_HOME}/logs/kolibri.txt | wc -l` -gt 200 ] ; then echo 'Log too long' && echo '' && tail -n 20 {env:KOLIBRI_HOME}/logs/kolibri.txt && exit 1 ; fi"
+
+
 [testenv:postgres]
 passenv = TOX_ENV
 setenv =