CI: Get coverage of filecheck tests (#256)

This PR leverages the Coverage package to generate the code coverage of our file checks test suite. 
 Notice that we now lose some coverage as I have specified the source directories, so non-executed files are counted as 0% coverage. Furthermore, coverage is now generated on the MLIR CI.

.coveragerc

@@ -5,6 +5,11 @@ omit =
     xdsl/_version.py
 concurrency = multiprocessing
 parallel = True
+source =
+    xdsl/
+    tests/
+
+
 
 [report]
 # Regexes for lines to exclude from consideration

.github/workflows/ci-core.yml

@@ -32,14 +32,10 @@ jobs:
         pip install --upgrade pip
     - name: Install the package locally
       run: pip install -e .[extras]
-    - name: Test with pytest and check code coverage
+    - name: Test with pytest
       run: |
-        pytest --cov --cov-config=.coveragerc --cov-report=xml tests/
+        pytest
     - name: Execute lit tests
       run: |
         export PYTHONPATH=$(pwd)
         lit -v tests/filecheck/
-    - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v3
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/ci-mlir.yml

@@ -62,8 +62,7 @@ jobs:
         pip install --upgrade pip
 
     - name: Install the package locally
-      run: |
-        pip install -e ${GITHUB_WORKSPACE}/xdsl
+      run: pip install -e .[extras]
 
     - name: MLIR Build Setup
       run: |
@@ -77,10 +76,33 @@ jobs:
         cd llvm-project/build
         cmake --build . --target mlir-opt MLIRPythonModules
 
-    - name: Test
+    - name: Test with pytest and generate code coverage
       run: |
-        # Add the Python Bindings to the pythonpath
+        cd xdsl
+        # Add the MLIR Python bindings to the PYTHONPATH
         export PYTHONPATH=$PYTHONPATH:${GITHUB_WORKSPACE}/llvm-project/build/tools/mlir/python_packages/mlir_core
+        pytest --cov --cov-config=.coveragerc tests
+
+    - name: Execute lit tests
+      run: |
+        cd xdsl
+        export PYTHONPATH=$(pwd)
         # Add mlir-opt to the path
         export PATH=$PATH:${GITHUB_WORKSPACE}/llvm-project/build/bin/
-        lit -v ${GITHUB_WORKSPACE}/xdsl/tests/filecheck/mlir-conversion/with-bindings/
+        # Add the MLIR Python bindings to the PYTHONPATH
+        export PYTHONPATH=$PYTHONPATH:${GITHUB_WORKSPACE}/llvm-project/build/tools/mlir/python_packages/mlir_core
+        lit -v tests/filecheck/ -DCOVERAGE -DCOVERAGE_CONFIG=$(pwd)/.coveragerc
+
+    - name: Combine coverage data
+      run: |
+        coverage combine --append $(find xdsl/tests/filecheck -type d)
+        coverage report --data-file=xdsl/.coverage
+        coverage xml --data-file=xdsl/.coverage
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v3
+      with:
+        fail_ci_if_error: true
+        verbose: true
+        directory: ${GITHUB_WORKSPACE}/../
+        files: coverage.xml

requirements.txt

@@ -2,3 +2,4 @@ pytest<8.0
 filecheck<0.0.23
 lit<16.0.0
 frozenlist<1.4.*
+coverage<7.0.0

tests/filecheck/lit.cfg

@@ -14,4 +14,8 @@ else:
 
 config.environment["PATH"] = config.test_source_root + "/../../xdsl/tools/:" + os.environ["PATH"]
 
-
+if "COVERAGE" in lit_config.params:
+    if "COVERAGE_CONFIG" in lit_config.params:
+        config.substitutions.append(('xdsl-opt', "xdsl-opt --generate-coverage --coverage-config=" + lit_config.params["COVERAGE_CONFIG"]))
+    else:
+        config.substitutions.append(('xdsl-opt', "xdsl-opt --generate-coverage"))

xdsl/xdsl_opt_main.py

@@ -2,6 +2,7 @@
 import sys
 import os
 from io import IOBase, StringIO
+import coverage
 
 from xdsl.ir import MLContext
 from xdsl.parser import Parser
@@ -69,6 +70,14 @@ def run(self):
         """
         Executes the different steps.
         """
+        if self.args.generate_coverage:
+            cov = coverage.Coverage(config_file=self.args.coverage_config,
+                                    auto_data=True,
+                                    data_file='.coverage',
+                                    data_suffix=True)
+
+            cov.start()
+
         module = self.parse_input()
         if not self.args.verify_diagnostics:
             self.apply_passes(module)
@@ -82,6 +91,9 @@ def run(self):
         contents = self.output_resulting_program(module)
         self.print_to_output_stream(contents)
 
+        if self.args.generate_coverage:
+            cov.stop()
+
     def register_all_arguments(self, arg_parser: argparse.ArgumentParser):
         """
         Registers all the command line arguments that are used by this tool.
@@ -155,6 +167,21 @@ def register_all_arguments(self, arg_parser: argparse.ArgumentParser):
             action='store_true',
             help="Allow the parsing of unregistered operations.")
 
+        arg_parser.add_argument(
+            "--generate-coverage",
+            default=False,
+            action='store_true',
+            help="Generate the xDSL code coverage for this run.")
+
+        arg_parser.add_argument(
+            "--coverage-config",
+            type=str,
+            default=False,
+            required=False,
+            help=
+            "Link to the coverage config file. This flag only takes effect if `--generate-config` was specified."
+        )
+
     def register_all_dialects(self):
         """
         Register all dialects that can be used.