CI: Get coverage of filecheck tests

xdslproject · Dec 9, 2022 · 6e4a56b · 6e4a56b
1 parent c864833
commit 6e4a56b
Show file tree

Hide file tree

Showing 28 changed files with 101 additions and 59 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -5,6 +5,11 @@ omit =
     xdsl/_version.py
 concurrency = multiprocessing
 parallel = True
+source =
+    xdsl/
+    tests/
+
+
 
 [report]
 # Regexes for lines to exclude from consideration

diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml
@@ -32,13 +32,18 @@ jobs:
         pip install --upgrade pip
     - name: Install the package locally
       run: pip install -e .[extras]
-    - name: Test with pytest and check code coverage
+    - name: Test with pytest and generate code coverage
       run: |
-        pytest --cov --cov-config=.coveragerc --cov-report=xml tests/
+        pytest --cov --cov-config=.coveragerc
     - name: Execute lit tests
       run: |
         export PYTHONPATH=$(pwd)
-        lit -v tests/filecheck/
+        lit -v tests/filecheck/ -DCOVERAGE -DCOVERAGE_CONFIG=$(pwd)/.coveragerc
+    - name: Combine coverage data
+      run: |
+        coverage combine --append tests/filecheck/ tests/filecheck/mlir-conversion/ tests/filecheck/mlir-conversion/with-bindings/ tests/filecheck/irdl-printer/ tests/filecheck/parser-printer/
+        coverage report
+        coverage xml
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3
       with:

diff --git a/requirements.txt b/requirements.txt
@@ -2,3 +2,4 @@ pytest<8.0
 filecheck<0.0.23
 lit<16.0.0
 frozenlist<1.4.*
+coverage<7.0.0
diff --git a/tests/filecheck/affine_ops.xdsl b/tests/filecheck/affine_ops.xdsl
@@ -1,4 +1,4 @@
-// RUN: xdsl-opt %s | xdsl-opt | filecheck %s
+// RUN: xdsl-opt %s %cov | xdsl-opt %cov | filecheck %s
 
 builtin.module() {
 

diff --git a/tests/filecheck/arith_invalid.xdsl b/tests/filecheck/arith_invalid.xdsl
@@ -1,7 +1,7 @@
-// RUN: xdsl-opt %s --verify-diagnostics | filecheck %s
+// RUN: xdsl-opt %cov %s --verify-diagnostics | filecheck %s
 
 builtin.module () {
-    
+
   func.func() ["sym_name" = "wrong_types", "function_type" = !fun<[], []>, "sym_visibility" = "private"] {
     %0 : !i32 = arith.constant() ["value" = 32 : !i32]
     %1 : !i64 = arith.constant() ["value" = 64 : !i64]

diff --git a/tests/filecheck/arith_ops.xdsl b/tests/filecheck/arith_ops.xdsl
@@ -1,4 +1,4 @@
-// RUN: xdsl-opt %s | xdsl-opt | filecheck %s
+// RUN: xdsl-opt %cov %s | xdsl-opt %cov | filecheck %s
 
 builtin.module() {
 func.func() ["function_type" = !fun<[!i32, !i32], [!i32]>, "sym_name" = "divsi"] {
@@ -267,4 +267,4 @@ func.func() ["function_type" = !fun<[!i32, !i32], [!i32]>, "sym_name" = "shrsi"]
 }
 
 // CHECK:   %{{.*}} !i32 = arith.shrsi(%{{.*}} : !i32, %{{.*}} : !i32)
-}
+}
diff --git a/tests/filecheck/builtin_attrs.xdsl b/tests/filecheck/builtin_attrs.xdsl
@@ -1,4 +1,4 @@
-// RUN: xdsl-opt %s | xdsl-opt | filecheck %s
+// RUN: xdsl-opt %cov %s | xdsl-opt %cov | filecheck %s
 
 
 // CHECK: module
@@ -26,4 +26,4 @@ builtin.module() {
     func.return()
   }
 
-}
+}
diff --git a/tests/filecheck/cf_ops.xdsl b/tests/filecheck/cf_ops.xdsl
@@ -1,14 +1,14 @@
-// RUN: xdsl-opt %s | xdsl-opt | filecheck %s
+// RUN: xdsl-opt %cov %s | xdsl-opt %cov | filecheck %s
 
 builtin.module() {
-  
+
   func.func() ["sym_name" = "unconditional_br", "function_type" = !fun<[], []>, "sym_visibility" = "private"] {
     ^0:
       cf.br() (^1)
     ^1:
       cf.br() (^1)
   }
-  // CHECK: func.func() ["sym_name" = "unconditional_br", 
+  // CHECK: func.func() ["sym_name" = "unconditional_br",
   // CHECK-NEXT:  ^{{.*}}:
   // CHECK-NEXT:    cf.br() (^{{.*}})
   // CHECK-NEXT:  ^{{.*}}:
@@ -39,7 +39,7 @@ builtin.module() {
     func.return(%24 : !i32)
   }
 
-  // CHECK: func.func() ["sym_name" = "cond_br", 
+  // CHECK: func.func() ["sym_name" = "cond_br",
   // CHECK-NEXT: ^{{.*}}(%{{.*}} : !i1, %{{.*}} : !i32):
   // CHECK-NEXT:   cf.br(%{{.*}} : !i1, %{{.*}} : !i32) (^{{.*}})
   // CHECK-NEXT: ^{{.*}}(%{{.*}} : !i1, %{{.*}} : !i32):

diff --git a/tests/filecheck/cmath.xdsl b/tests/filecheck/cmath.xdsl
@@ -1,4 +1,4 @@
-// RUN: xdsl-opt %s | xdsl-opt | filecheck %s
+// RUN: xdsl-opt %cov %s | xdsl-opt %cov | filecheck %s
 
 
 builtin.module() {
@@ -23,7 +23,7 @@ builtin.module() {
         }
 
         // CHECK: irdl.operation() ["name" = "mul"] {
-        // CHECK: irdl.operands() ["params" = [!irdl.named_type_constraint<"lhs" : !irdl.dyn_type_params_constraint<"complex", [!irdl.any_of_type<[!irdl.equality_type_constraint<!i32>, !irdl.equality_type_constraint<!i64>]>]>>]] 
+        // CHECK: irdl.operands() ["params" = [!irdl.named_type_constraint<"lhs" : !irdl.dyn_type_params_constraint<"complex", [!irdl.any_of_type<[!irdl.equality_type_constraint<!i32>, !irdl.equality_type_constraint<!i64>]>]>>]]
         // CHECK: irdl.operands() ["params" = [!irdl.named_type_constraint<"rhs" : !irdl.dyn_type_params_constraint<"complex", [!irdl.any_of_type<[!irdl.equality_type_constraint<!i32>, !irdl.equality_type_constraint<!i64>]>]>>]]
         // CHECK: irdl.results() ["params" = [!irdl.named_type_constraint<"results" : !irdl.dyn_type_params_constraint<"complex", [!irdl.any_of_type<[!irdl.equality_type_constraint<!i32>, !irdl.equality_type_constraint<!i64>]>]>>]]
         // CHECK: }
@@ -34,4 +34,3 @@ builtin.module() {
         }
     }
 }
-
diff --git a/tests/filecheck/cmath_ops.xdsl b/tests/filecheck/cmath_ops.xdsl
@@ -1,4 +1,4 @@
-// RUN: xdsl-opt %s | xdsl-opt | filecheck %s
+// RUN: xdsl-opt %cov %s | xdsl-opt %cov | filecheck %s
 
 builtin.module() {
 
@@ -10,7 +10,7 @@ builtin.module() {
         func.return(%pq : !f32)
     }
 
-    // CHECK: func.func() ["sym_name" = "conorm", 
+    // CHECK: func.func() ["sym_name" = "conorm",
     // CHECK-NEXT: ^{{.*}}({{.*}}: !cmath.complex<!f32>, {{.*}}: !cmath.complex<!f32>):
     // CHECK-NEXT: %{{.*}} : !f32 = cmath.norm(%{{.*}} : !cmath.complex<!f32>)
     // CHECK-NEXT: %{{.*}} : !f32 = cmath.norm(%{{.*}} : !cmath.complex<!f32>)
@@ -24,7 +24,7 @@ builtin.module() {
         %conorm : !f32 = cmath.norm(%ab : !cmath.complex<!f32>)
         func.return(%conorm : !f32)
     }
-    // CHECK: func.func() ["sym_name" = "conorm2", 
+    // CHECK: func.func() ["sym_name" = "conorm2",
     // CHECK-NEXT: ^{{.*}}(%{{.*}}: !cmath.complex<!f32>, %{{.*}}: !cmath.complex<!f32>):
     // CHECK-NEXT: %{{.*}} : !cmath.complex<!f32> = cmath.mul(%{{.*}} : !cmath.complex<!f32>, %{{.*}} : !cmath.complex<!f32>)
     // CHECK-NEXT: %{{.*}} : !f32 = cmath.norm(%{{.*}} : !cmath.complex<!f32>)

diff --git a/tests/filecheck/func_ops.xdsl b/tests/filecheck/func_ops.xdsl
@@ -1,12 +1,12 @@
-// RUN: xdsl-opt %s | xdsl-opt | filecheck %s
+// RUN: xdsl-opt %cov %s | xdsl-opt %cov | filecheck %s
 
 builtin.module() {
-  
+
   func.func() ["sym_name" = "noarg_void", "function_type" = !fun<[], []>, "sym_visibility" = "private"] {
     func.return()
   }
 
-  // CHECK: func.func() ["sym_name" = "noarg_void", 
+  // CHECK: func.func() ["sym_name" = "noarg_void",
   // CHECK-NEXT:  func.return()
   // CHECK-NEXT:}
 
@@ -15,7 +15,7 @@ builtin.module() {
     func.return()
   }
 
-  // CHECK: func.func() ["sym_name" = "call_void", 
+  // CHECK: func.func() ["sym_name" = "call_void",
   // CHECK-NEXT:  func.call() ["callee" = @call_void]
   // CHECK-NEXT:  func.return()
   // CHECK-NEXT:}
@@ -26,7 +26,7 @@ builtin.module() {
     func.return(%1 : !i32)
   }
 
-  // CHECK: func.func() ["sym_name" = "arg_rec", 
+  // CHECK: func.func() ["sym_name" = "arg_rec",
   // CHECK-NEXT: ^{{.*}}(%{{.*}} : !i32):
   // CHECK-NEXT:   %{{.*}} : !i32 = func.call(%{{.*}} : !i32) ["callee" = @arg_rec]
   // CHECK-NEXT:   func.return(%{{.*}} : !i32)

diff --git a/tests/filecheck/irdl-printer/any_and_base_type.xdsl b/tests/filecheck/irdl-printer/any_and_base_type.xdsl
@@ -1,9 +1,9 @@
-// RUN: xdsl-opt %s -t irdl | filecheck %s
+// RUN: xdsl-opt %cov %s -t irdl | filecheck %s
 
 builtin.module() {
 
     irdl.dialect() ["name" = "test_any_dyn"] {
-        
+
         irdl.type() ["name" = "complex"] {
             irdl.parameters() ["params" = [!irdl.named_type_constraint<"elementType" : !irdl.equality_type_constraint<!i32>>]]
         }
@@ -23,4 +23,4 @@ builtin.module() {
 //CHECK:            irdl.operands(any: Any, basetype: complex)
 //CHECK:        }
 //CHECK:    }
-//CHECK: }   
+//CHECK: }
diff --git a/tests/filecheck/irdl-printer/cmath_program.xdsl b/tests/filecheck/irdl-printer/cmath_program.xdsl
@@ -1,4 +1,4 @@
-// RUN: xdsl-opt %s -t irdl | filecheck %s
+// RUN: xdsl-opt %cov %s -t irdl | filecheck %s
 
 
 builtin.module() {

diff --git a/tests/filecheck/irdl_any_and_base_type.xdsl b/tests/filecheck/irdl_any_and_base_type.xdsl
@@ -1,4 +1,4 @@
-// RUN: xdsl-opt %s | xdsl-opt | filecheck %s
+// RUN: xdsl-opt %cov %s | xdsl-opt %cov | filecheck %s
 
 builtin.module() {
 

diff --git a/tests/filecheck/lit.cfg b/tests/filecheck/lit.cfg
@@ -14,4 +14,10 @@ else:
 
 config.environment["PATH"] = config.test_source_root + "/../../xdsl/tools/:" + os.environ["PATH"]
 
-
+if "COVERAGE" in lit_config.params:
+    if "COVERAGE_CONFIG" in lit_config.params:
+        config.substitutions.append(('%cov', "--generate-coverage --coverage-config=" + lit_config.params["COVERAGE_CONFIG"]))
+    else:
+        config.substitutions.append(('%cov', "--generate-coverage"))
+else:
+    config.substitutions.append(('%cov', ""))
diff --git a/tests/filecheck/memref_ops.xdsl b/tests/filecheck/memref_ops.xdsl
@@ -1,7 +1,7 @@
-// RUN: xdsl-opt %s | xdsl-opt | filecheck %s
+// RUN: xdsl-opt %cov %s | xdsl-opt %cov | filecheck %s
 
 builtin.module() {
-  
+
   memref.global() ["sym_name" = "g", "type" = !memref<[1 : !i64], !index>, "initial_value" = !dense<!tensor<[1 : !i64], !index>, [0 : !index]>, "sym_visibility" = "public"]
 
   func.func() ["sym_name" = "memref_test", "function_type" = !fun<[], []>, "sym_visibility" = "private"] {

diff --git a/tests/filecheck/mlir-conversion/builtin_attrs.mlir b/tests/filecheck/mlir-conversion/builtin_attrs.mlir
@@ -1,4 +1,4 @@
-// RUN: xdsl-opt %s -t mlir | xdsl-opt -f mlir -t mlir | filecheck %s
+// RUN: xdsl-opt %cov %s -t mlir | xdsl-opt %cov -f mlir -t mlir | filecheck %s
 
 "builtin.module"() ({
   "func.func"() ({

diff --git a/tests/filecheck/mlir-conversion/llvm_test.xdsl b/tests/filecheck/mlir-conversion/llvm_test.xdsl
@@ -1,4 +1,4 @@
-// RUN: xdsl-opt -t mlir %s | xdsl-opt -t mlir -f mlir | filecheck %s
+// RUN: xdsl-opt %cov -t mlir %s | xdsl-opt %cov -t mlir -f mlir | filecheck %s
 
 
 builtin.module() {

diff --git a/tests/filecheck/mlir-conversion/unregistered_ops.mlir b/tests/filecheck/mlir-conversion/unregistered_ops.mlir
@@ -1,4 +1,4 @@
-// RUN: xdsl-opt %s -t mlir --allow-unregistered-ops | xdsl-opt -f mlir -t mlir --allow-unregistered-ops  | filecheck %s
+// RUN: xdsl-opt %cov %s -t mlir --allow-unregistered-ops | xdsl-opt %cov -f mlir -t mlir --allow-unregistered-ops  | filecheck %s
 
 "builtin.module"() ({
 
@@ -12,4 +12,4 @@
   // CHECK-NEXT:   "op_with_operands"(%{{.*}}, %{{.*}}) : (i32, i32) -> ()
   // CHECK-NEXT:  }) {"testattr" = "foo"} : () -> i32
 
-}) : () -> ()
+}) : () -> ()
diff --git a/tests/filecheck/mlir-conversion/with-bindings/llvm_test.xdsl b/tests/filecheck/mlir-conversion/with-bindings/llvm_test.xdsl
@@ -1,4 +1,4 @@
-// RUN: xdsl-opt -t mlir --use-mlir-bindings %s | filecheck %s
+// RUN: xdsl-opt %cov -t mlir --use-mlir-bindings %s | filecheck %s
 
 
 builtin.module() {

diff --git a/tests/filecheck/mlir-conversion/with-bindings/ops.xdsl b/tests/filecheck/mlir-conversion/with-bindings/ops.xdsl
@@ -1,5 +1,5 @@
-// RUN: xdsl-opt -t mlir --use-mlir-bindings %s | filecheck %s
-// RUN: xdsl-opt -t mlir --use-mlir-bindings %s | mlir-opt --mlir-print-op-generic > %t-1 && xdsl-opt -t mlir %s | mlir-opt --mlir-print-op-generic > %t-2 && diff %t-1 %t-2
+// RUN: xdsl-opt %cov -t mlir --use-mlir-bindings %s | filecheck %s
+// RUN: xdsl-opt %cov -t mlir --use-mlir-bindings %s | mlir-opt --mlir-print-op-generic > %t-1 && xdsl-opt %cov -t mlir %s | mlir-opt --mlir-print-op-generic > %t-2 && diff %t-1 %t-2
 
 // Tests if the non generic form can be printed.
 
@@ -8,11 +8,11 @@ builtin.module() {
   func.func() ["sym_name" = "simple_ops", "function_type" = !fun<[], []>, "sym_visibility" = "private"]{
     %0 : !i32 = arith.constant() ["value" = 42 : !i32]
     %1 : !i32 = arith.constant() ["value" = 42 : !i32]
-    %2 : !i32 = arith.addi(%0 : !i32, %1 : !i32) 
-    %3 : !i32 = arith.subi(%0 : !i32, %1 : !i32) 
-    %4 : !i32 = arith.muli(%0 : !i32, %1 : !i32) 
-    %5 : !i32 = arith.floordivsi(%0 : !i32, %1 : !i32) 
-    %6 : !i32 = arith.remsi(%0 : !i32, %1 : !i32) 
+    %2 : !i32 = arith.addi(%0 : !i32, %1 : !i32)
+    %3 : !i32 = arith.subi(%0 : !i32, %1 : !i32)
+    %4 : !i32 = arith.muli(%0 : !i32, %1 : !i32)
+    %5 : !i32 = arith.floordivsi(%0 : !i32, %1 : !i32)
+    %6 : !i32 = arith.remsi(%0 : !i32, %1 : !i32)
     func.return()
   }
 

diff --git a/tests/filecheck/parser-printer/builtin_types.xdsl b/tests/filecheck/parser-printer/builtin_types.xdsl
@@ -1,4 +1,4 @@
-// RUN: xdsl-opt %s | xdsl-opt | filecheck %s
+// RUN: xdsl-opt %cov %s | xdsl-opt %cov | filecheck %s
 
 
 builtin.module() {

diff --git a/tests/filecheck/parser-printer/escaped_characters.xdsl b/tests/filecheck/parser-printer/escaped_characters.xdsl
@@ -1,4 +1,4 @@
-// RUN: xdsl-opt %s | xdsl-opt | filecheck %s
+// RUN: xdsl-opt %cov %s | xdsl-opt %cov | filecheck %s
 
 builtin.module() {
   func.func() ["sym_name" = "\"", "function_type" = !fun<[], []>, "sym_visibility" = "private"] {}

diff --git a/tests/filecheck/parser-printer/float_parsing.xdsl b/tests/filecheck/parser-printer/float_parsing.xdsl
@@ -1,4 +1,4 @@
-// RUN: xdsl-opt %s | xdsl-opt | filecheck %s
+// RUN: xdsl-opt %cov %s | xdsl-opt %cov | filecheck %s
 
 builtin.module() {
   func.func() ["sym_name" = "main", "function_type" = !fun<[], []>, "sym_visibility" = "private"]{
@@ -23,4 +23,3 @@ builtin.module() {
     func.return()
   }
 }
-
diff --git a/tests/filecheck/parser-printer/region_name_clash.xdsl b/tests/filecheck/parser-printer/region_name_clash.xdsl
@@ -1,4 +1,4 @@
-// RUN: xdsl-opt %s | xdsl-opt | filecheck %s
+// RUN: xdsl-opt %cov %s | xdsl-opt %cov | filecheck %s
 
 // Check that SSA values and blocks can reuse names across regions
 
@@ -20,7 +20,7 @@ builtin.module() {
 
   func.func() ["sym_name" = "id2", "function_type" = !fun<[!i64], [!i64]>, "sym_visibility" = "private"] {
     ^0(%0 : !i64):
-      
+
       func.return(%0 : !i64)
   }
 
@@ -81,4 +81,4 @@ builtin.module() {
 
 }
 
-// CHECK-NEXT: }
+// CHECK-NEXT: }
diff --git a/tests/filecheck/parser-printer/unregistered_ops.xdsl b/tests/filecheck/parser-printer/unregistered_ops.xdsl
@@ -1,4 +1,4 @@
-// RUN: xdsl-opt %s --allow-unregistered-ops | xdsl-opt --allow-unregistered-ops  | filecheck %s
+// RUN: xdsl-opt %cov %s --allow-unregistered-ops | xdsl-opt %cov --allow-unregistered-ops  | filecheck %s
 
 builtin.module() {
 // CHECK: builtin.module() {

diff --git a/tests/filecheck/scf_ops.xdsl b/tests/filecheck/scf_ops.xdsl
@@ -1,22 +1,22 @@
-// RUN: xdsl-opt %s | xdsl-opt | filecheck %s
+// RUN: xdsl-opt %cov %s | xdsl-opt %cov | filecheck %s
 
 builtin.module() {
   func.func() ["sym_name" = "if", "function_type" = !fun<[], []>, "sym_visibility" = "private"] {
     %0 : !i1 = arith.constant() ["value" = true]
-    scf.if(%0 : !i1) 
+    scf.if(%0 : !i1)
     {
         %1 : !i32 = arith.constant() ["value" = 42 : !i32]
         scf.yield()
-    } 
+    }
     {
         %2 : !i32 = arith.constant() ["value" = 24 : !i32]
         scf.yield()
     }
-    %5 : !i32 = scf.if(%0 : !i1) 
+    %5 : !i32 = scf.if(%0 : !i1)
     {
         %3 : !i32 = arith.constant() ["value" = 42 : !i32]
         scf.yield(%3 : !i32)
-    } 
+    }
     {
         %4 : !i32 = arith.constant() ["value" = 24 : !i32]
         scf.yield(%4 : !i32)
@@ -59,7 +59,7 @@ builtin.module() {
     }
     func.return()
   }
- 
+
   // CHECK: func.func() ["sym_name" = "while"
   // CHECK-NEXT:  %{{.*}} : !i32 = arith.constant() ["value" = 0 : !i32]
   // CHECK-NEXT:  %{{.*}} : !i32 = scf.while(%{{.*}} : !i32) {