diff --git a/scripts/run-module-bundlers-smoketests.js b/scripts/run-module-bundlers-smoketests.js
index 9a688246..94a3a549 100755
--- a/scripts/run-module-bundlers-smoketests.js
+++ b/scripts/run-module-bundlers-smoketests.js
@@ -8,7 +8,10 @@ Test webextension-polyfill bundled with webpack
 ===============================================`);
 
 process.env.TEST_BUNDLED_POLYFILL = "/tmp/webpack-bundle.js";
-result = shell.exec(`webpack --mode production --entry ./test/fixtures/bundle-entrypoint.js --output ${process.env.TEST_BUNDLED_POLYFILL}`).code ||
+const webpackOutputDir = process.env.TEST_BUNDLED_POLYFILL.replace(/\/[^/]+$/, "");
+const webpackOutputFilename = process.env.TEST_BUNDLED_POLYFILL.replace(/^.*\//, "");
+
+result = shell.exec(`webpack --mode production --entry ./test/fixtures/bundle-entrypoint.js --output-path ${webpackOutputDir} --output-filename ${webpackOutputFilename}`).code ||
   shell.exec("npm run test").code || result;
 
 console.log(`