diff --git a/snakemake/dag.py b/snakemake/dag.py index aa2a7f007..6b35bed31 100755 --- a/snakemake/dag.py +++ b/snakemake/dag.py @@ -1108,11 +1108,23 @@ def update_needrun(job): reason.updated_input.update(updated_input) if not updated_input: # check for other changes like parameters, set of input files, or code + depends_on_checkpoint_target = any( + f.flags.get("checkpoint_target") for f in job.input + ) + if "params" in self.workflow.rerun_triggers: reason.params_changed = any( self.workflow.persistence.params_changed(job) ) - if "input" in self.workflow.rerun_triggers: + if ( + "input" in self.workflow.rerun_triggers + and not depends_on_checkpoint_target + ): + # When the job depends on a checkpoint, it will be revaluated in a second pass + # after the checkpoint output has been determined. + # The first pass (with depends_on_checkpoint_target == True) is not informative + # for determining the input file set, as it will change after evaluating the + # input function of the job in the second pass. reason.input_changed = any( self.workflow.persistence.input_changed(job) ) diff --git a/tests/test_github_issue1818/.snakemake/metadata/YWdncmVnYXRlZC50eHQ= b/tests/test_github_issue1818/.snakemake/metadata/YWdncmVnYXRlZC50eHQ= new file mode 100644 index 000000000..e60f0ee8d --- /dev/null +++ b/tests/test_github_issue1818/.snakemake/metadata/YWdncmVnYXRlZC50eHQ= @@ -0,0 +1 @@ +{"version": null, "code": "gASVpQEAAAAAAAAoQxJ0AGQBfA58EWQCjQMBAGQAUwCUKIwFaW5wdXSUjAZvdXRwdXSUjAZwYXJhbXOUjAl3aWxkY2FyZHOUjAd0aHJlYWRzlIwJcmVzb3VyY2VzlIwDbG9nlIwHdmVyc2lvbpSMBHJ1bGWUjAljb25kYV9lbnaUjA1jb250YWluZXJfaW1nlIwQc2luZ3VsYXJpdHlfYXJnc5SMD3VzZV9zaW5ndWxhcml0eZSMC2Vudl9tb2R1bGVzlIwMYmVuY2hfcmVjb3JklIwFam9iaWSUjAhpc19zaGVsbJSMD2JlbmNoX2l0ZXJhdGlvbpSMD2NsZWFudXBfc2NyaXB0c5SMCnNoYWRvd19kaXKUjA1lZGl0X25vdGVib29rlIwPY29uZGFfYmFzZV9wYXRolIwHYmFzZWRpcpSMGHJ1bnRpbWVfc291cmNlY2FjaGVfcGF0aJSMGF9faXNfc25ha2VtYWtlX3J1bGVfZnVuY5R0lF2UKE6MGmVjaG8gQUdHUkVHQVRFRCA+IHtvdXRwdXR9lGgPaBKGlGWMBXNoZWxslIWUdJQu", "rule": "aggregate", "input": ["my_directory/1.txt", "my_directory/2.txt", "my_directory/3.txt"], "log": [], "params": [], "shellcmd": "echo AGGREGATED > aggregated.txt", "incomplete": false, "starttime": 1665672954.609385, "endtime": 1665672954.619385, "job_hash": 8729438519112, "conda_env": null, "container_img_url": null, "input_checksums": {"my_directory/1.txt": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "my_directory/2.txt": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "my_directory/3.txt": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}} \ No newline at end of file diff --git a/tests/test_github_issue1818/.snakemake/metadata/bXlfZGlyZWN0b3J5 b/tests/test_github_issue1818/.snakemake/metadata/bXlfZGlyZWN0b3J5 new file mode 100644 index 000000000..8bbe66788 --- /dev/null +++ b/tests/test_github_issue1818/.snakemake/metadata/bXlfZGlyZWN0b3J5 @@ -0,0 +1 @@ +{"version": null, "code": "gASV9gEAAAAAAAAoQxJ0AGQBfA58EWQCjQMBAGQAUwCUKIwFaW5wdXSUjAZvdXRwdXSUjAZwYXJhbXOUjAl3aWxkY2FyZHOUjAd0aHJlYWRzlIwJcmVzb3VyY2VzlIwDbG9nlIwHdmVyc2lvbpSMBHJ1bGWUjAljb25kYV9lbnaUjA1jb250YWluZXJfaW1nlIwQc2luZ3VsYXJpdHlfYXJnc5SMD3VzZV9zaW5ndWxhcml0eZSMC2Vudl9tb2R1bGVzlIwMYmVuY2hfcmVjb3JklIwFam9iaWSUjAhpc19zaGVsbJSMD2JlbmNoX2l0ZXJhdGlvbpSMD2NsZWFudXBfc2NyaXB0c5SMCnNoYWRvd19kaXKUjA1lZGl0X25vdGVib29rlIwPY29uZGFfYmFzZV9wYXRolIwHYmFzZWRpcpSMGHJ1bnRpbWVfc291cmNlY2FjaGVfcGF0aJSMGF9faXNfc25ha2VtYWtlX3J1bGVfZnVuY5R0lF2UKE6MawogICAgICAgIG1rZGlyIG15X2RpcmVjdG9yeS8KICAgICAgICBjZCBteV9kaXJlY3RvcnkKICAgICAgICBmb3IgaSBpbiAxIDIgMzsgZG8gdG91Y2ggJGkudHh0OyBkb25lCiAgICAgICAglGgPaBKGlGWMBXNoZWxslIWUdJQu", "rule": "somestep", "input": [], "log": [], "params": [], "shellcmd": "\n mkdir my_directory/\n cd my_directory\n for i in 1 2 3; do touch $i.txt; done\n ", "incomplete": false, "starttime": 1665672954.5860517, "endtime": 1665672954.6027184, "job_hash": 8729438582497, "conda_env": null, "container_img_url": null, "input_checksums": {}} \ No newline at end of file diff --git a/tests/test_github_issue1818/.snakemake/metadata/cHJvY2Vzc2VkLnR4dA== b/tests/test_github_issue1818/.snakemake/metadata/cHJvY2Vzc2VkLnR4dA== new file mode 100644 index 000000000..156790056 --- /dev/null +++ b/tests/test_github_issue1818/.snakemake/metadata/cHJvY2Vzc2VkLnR4dA== @@ -0,0 +1 @@ +{"version": null, "code": "gASVpAEAAAAAAAAoQxJ0AGQBfA58EWQCjQMBAGQAUwCUKIwFaW5wdXSUjAZvdXRwdXSUjAZwYXJhbXOUjAl3aWxkY2FyZHOUjAd0aHJlYWRzlIwJcmVzb3VyY2VzlIwDbG9nlIwHdmVyc2lvbpSMBHJ1bGWUjAljb25kYV9lbnaUjA1jb250YWluZXJfaW1nlIwQc2luZ3VsYXJpdHlfYXJnc5SMD3VzZV9zaW5ndWxhcml0eZSMC2Vudl9tb2R1bGVzlIwMYmVuY2hfcmVjb3JklIwFam9iaWSUjAhpc19zaGVsbJSMD2JlbmNoX2l0ZXJhdGlvbpSMD2NsZWFudXBfc2NyaXB0c5SMCnNoYWRvd19kaXKUjA1lZGl0X25vdGVib29rlIwPY29uZGFfYmFzZV9wYXRolIwHYmFzZWRpcpSMGHJ1bnRpbWVfc291cmNlY2FjaGVfcGF0aJSMGF9faXNfc25ha2VtYWtlX3J1bGVfZnVuY5R0lF2UKE6MGWVjaG8gUFJPQ0VTU0VEID4ge291dHB1dH2UaA9oEoaUZYwFc2hlbGyUhZR0lC4=", "rule": "process", "input": ["aggregated.txt"], "log": [], "params": [], "shellcmd": "echo PROCESSED > processed.txt", "incomplete": false, "starttime": 1665672954.6227183, "endtime": 1665672954.6327186, "job_hash": 8729438582463, "conda_env": null, "container_img_url": null, "input_checksums": {"aggregated.txt": "a144e9df267c10532db93a5e1625d6362d934c9973be60a6c07718a1f1942f12"}} \ No newline at end of file diff --git a/tests/test_github_issue1818/.snakemake/metadata/cHJvY2Vzc2VkMi50eHQ= b/tests/test_github_issue1818/.snakemake/metadata/cHJvY2Vzc2VkMi50eHQ= new file mode 100644 index 000000000..a960cb46e --- /dev/null +++ b/tests/test_github_issue1818/.snakemake/metadata/cHJvY2Vzc2VkMi50eHQ= @@ -0,0 +1 @@ +{"version": null, "code": "gASVpQEAAAAAAAAoQxJ0AGQBfA58EWQCjQMBAGQAUwCUKIwFaW5wdXSUjAZvdXRwdXSUjAZwYXJhbXOUjAl3aWxkY2FyZHOUjAd0aHJlYWRzlIwJcmVzb3VyY2VzlIwDbG9nlIwHdmVyc2lvbpSMBHJ1bGWUjAljb25kYV9lbnaUjA1jb250YWluZXJfaW1nlIwQc2luZ3VsYXJpdHlfYXJnc5SMD3VzZV9zaW5ndWxhcml0eZSMC2Vudl9tb2R1bGVzlIwMYmVuY2hfcmVjb3JklIwFam9iaWSUjAhpc19zaGVsbJSMD2JlbmNoX2l0ZXJhdGlvbpSMD2NsZWFudXBfc2NyaXB0c5SMCnNoYWRvd19kaXKUjA1lZGl0X25vdGVib29rlIwPY29uZGFfYmFzZV9wYXRolIwHYmFzZWRpcpSMGHJ1bnRpbWVfc291cmNlY2FjaGVfcGF0aJSMGF9faXNfc25ha2VtYWtlX3J1bGVfZnVuY5R0lF2UKE6MGmVjaG8gUFJPQ0VTU0VEMiA+IHtvdXRwdXR9lGgPaBKGlGWMBXNoZWxslIWUdJQu", "rule": "process2", "input": ["processed.txt"], "log": [], "params": [], "shellcmd": "echo PROCESSED2 > processed2.txt", "incomplete": false, "starttime": 1665672954.636052, "endtime": 1665672954.6493852, "job_hash": 8729438582446, "conda_env": null, "container_img_url": null, "input_checksums": {"processed.txt": "49f193a0dae7bdb2368a9f113cf2e1f3827a787dde3808a5d7838bc8ff63164c"}} \ No newline at end of file diff --git a/tests/test_github_issue1818/Snakefile b/tests/test_github_issue1818/Snakefile new file mode 100644 index 000000000..e87fbe034 --- /dev/null +++ b/tests/test_github_issue1818/Snakefile @@ -0,0 +1,55 @@ +# a target rule to define the desired final output +rule all: + input: + "processed2.txt", + + +# the checkpoint that shall trigger re-evaluation of the DAG +# an number of file is created in a defined directory +checkpoint somestep: + output: + directory("my_directory/"), + shell: + """ + mkdir my_directory/ + cd my_directory + for i in 1 2 3; do touch $i.txt; done + """ + + +# input function for rule aggregate, return paths to all files produced by the checkpoint 'somestep' +def aggregate_input(wildcards): + checkpoint_output = checkpoints.somestep.get(**wildcards).output[0] + return expand( + "my_directory/{i}.txt", + i=glob_wildcards(os.path.join(checkpoint_output, "{i}.txt")).i, + ) + + +rule aggregate: + input: + aggregate_input, + output: + "aggregated.txt", + shell: + "echo AGGREGATED > {output}" + + +# Fail here if the job runs again, as we want to ensure that snakemake does not false trigger a rerun +# as reported in issue #1818. +rule process: + input: + "aggregated.txt", + output: + "processed.txt", + shell: + "exit 1; echo PROCESSED > {output}" + + +rule process2: + input: + "processed.txt", + output: + "processed2.txt", + shell: + "echo PROCESSED2 > {output}" diff --git a/tests/test_github_issue1818/aggregated.txt b/tests/test_github_issue1818/aggregated.txt new file mode 100644 index 000000000..ab46ad293 --- /dev/null +++ b/tests/test_github_issue1818/aggregated.txt @@ -0,0 +1 @@ +AGGREGATED diff --git a/tests/test_github_issue1818/expected-results/aggregated.txt b/tests/test_github_issue1818/expected-results/aggregated.txt new file mode 100644 index 000000000..ab46ad293 --- /dev/null +++ b/tests/test_github_issue1818/expected-results/aggregated.txt @@ -0,0 +1 @@ +AGGREGATED diff --git a/tests/test_github_issue1818/expected-results/my_directory/.snakemake_timestamp b/tests/test_github_issue1818/expected-results/my_directory/.snakemake_timestamp new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_github_issue1818/expected-results/my_directory/1.txt b/tests/test_github_issue1818/expected-results/my_directory/1.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_github_issue1818/expected-results/my_directory/2.txt b/tests/test_github_issue1818/expected-results/my_directory/2.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_github_issue1818/expected-results/my_directory/3.txt b/tests/test_github_issue1818/expected-results/my_directory/3.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_github_issue1818/expected-results/processed.txt b/tests/test_github_issue1818/expected-results/processed.txt new file mode 100644 index 000000000..5805d7e64 --- /dev/null +++ b/tests/test_github_issue1818/expected-results/processed.txt @@ -0,0 +1 @@ +PROCESSED diff --git a/tests/test_github_issue1818/expected-results/processed2.txt b/tests/test_github_issue1818/expected-results/processed2.txt new file mode 100644 index 000000000..d0fddfb01 --- /dev/null +++ b/tests/test_github_issue1818/expected-results/processed2.txt @@ -0,0 +1 @@ +PROCESSED2 diff --git a/tests/test_github_issue1818/my_directory/.snakemake_timestamp b/tests/test_github_issue1818/my_directory/.snakemake_timestamp new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_github_issue1818/my_directory/1.txt b/tests/test_github_issue1818/my_directory/1.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_github_issue1818/my_directory/2.txt b/tests/test_github_issue1818/my_directory/2.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_github_issue1818/my_directory/3.txt b/tests/test_github_issue1818/my_directory/3.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_github_issue1818/processed.txt b/tests/test_github_issue1818/processed.txt new file mode 100644 index 000000000..5805d7e64 --- /dev/null +++ b/tests/test_github_issue1818/processed.txt @@ -0,0 +1 @@ +PROCESSED diff --git a/tests/test_github_issue1818/processed2.txt b/tests/test_github_issue1818/processed2.txt new file mode 100644 index 000000000..d0fddfb01 --- /dev/null +++ b/tests/test_github_issue1818/processed2.txt @@ -0,0 +1 @@ +PROCESSED2 diff --git a/tests/tests.py b/tests/tests.py index 3bea992d6..650bb7c78 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1982,3 +1982,8 @@ def test_github_issue1618(): def test_conda_python_script(): run(dpath("test_conda_python_script"), use_conda=True) + + +@skip_on_windows +def test_github_issue1818(): + run(dpath("test_github_issue1818"), rerun_triggers="input")