Fix for issue 6047

scrapy/crawler.py

@@ -138,6 +138,32 @@ def _apply_settings(self) -> None:
             "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}
         )
 
+    # @inlineCallbacks
+    # def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]:
+    #     if self.crawling:
+    #         raise RuntimeError("Crawling already taking place")
+    #     if self._started:
+    #         warnings.warn(
+    #             "Running Crawler.crawl() more than once is deprecated.",
+    #             ScrapyDeprecationWarning,
+    #             stacklevel=2,
+    #         )
+    #     self.crawling = self._started = True
+
+    #     try:
+    #         self.spider = self._create_spider(*args, **kwargs)
+    #         self._apply_settings()
+    #         self._update_root_log_handler()
+    #         self.engine = self._create_engine()
+    #         start_requests = iter(self.spider.start_requests())
+    #         yield self.engine.open_spider(self.spider, start_requests)
+    #         yield maybeDeferred(self.engine.start)
+    #     except Exception:
+    #         self.crawling = False
+    #         if self.engine is not None:
+    #             yield self.engine.close()
+    #         raise
+
     @inlineCallbacks
     def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]:
         if self.crawling:
@@ -150,20 +176,26 @@ def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]:
             )
         self.crawling = self._started = True
 
-        try:
-            self.spider = self._create_spider(*args, **kwargs)
-            self._apply_settings()
-            self._update_root_log_handler()
-            self.engine = self._create_engine()
-            start_requests = iter(self.spider.start_requests())
-            yield self.engine.open_spider(self.spider, start_requests)
-            yield maybeDeferred(self.engine.start)
-        except Exception:
-            self.crawling = False
-            if self.engine is not None:
-                yield self.engine.close()
-            raise
+    def errorHandler(failure, self):
+        failure.trap(Exception)
+        self.crawling = False
+        if self.engine is not None:
+            yield self.engine.close()
+        raise
+
+    def crawlHelper(self, *args: Any, **kwargs: Any):
+        self.spider = self._create_spider(*args, **kwargs)
+        self._apply_settings()
+        self._update_root_log_handler()
+        self.engine = self._create_engine()
+        start_requests = iter(self.spider.start_requests())
+        yield self.engine.open_spider(self.spider, start_requests)
+        yield maybeDeferred(self.engine.start)
+
+    crawl.addCallback(crawlHelper)
+    crawl.addErrback(errorHandler)
 
+
     def _create_spider(self, *args: Any, **kwargs: Any) -> Spider:
         return self.spidercls.from_crawler(self, *args, **kwargs)
 
@@ -243,13 +275,44 @@ def crawl(
 
         :param kwargs: keyword arguments to initialize the spider
         """
+        # if isinstance(crawler_or_spidercls, Spider):
+        #     raise ValueError(
+        #         "The crawler_or_spidercls argument cannot be a spider object, "
+        #         "it must be a spider class (or a Crawler object)"
+        #     )
+        # crawler = self.create_crawler(crawler_or_spidercls)
+        # return self._crawl(crawler, *args, **kwargs)
+
+        # # one method
+        # def createInstance():
+        #     if isinstance(crawler_or_spidercls, Spider):
+        #         raise ValueError(
+        #             "The crawler_or_spidercls argument cannot be a spider object, "
+        #             "it must be a spider class (or a Crawler object)"
+        #         )
+        # def errorHandler(failure):
+        #     failure.trap(ValueError)
+
+        # self.addCallBack(createInstance)
+        # self.addErrback(errorHandler)
+
+        # crawler = self.create_crawler(crawler_or_spidercls)
+
         if isinstance(crawler_or_spidercls, Spider):
             raise ValueError(
                 "The crawler_or_spidercls argument cannot be a spider object, "
                 "it must be a spider class (or a Crawler object)"
             )
-        crawler = self.create_crawler(crawler_or_spidercls)
-        return self._crawl(crawler, *args, **kwargs)
+        def errorHandler(failure):
+            failure.trap(ValueError)
+
+        def crawlHelper(self, crawler_or_spidercls: Union[Type[Spider], str, Crawler], *args: Any, **kwargs: Any):
+            crawler = self.create_crawler(crawler_or_spidercls)
+            return self._crawl(crawler, *args, **kwargs)
+
+        crawl.addCallback(crawlHelper)
+        crawl.addErrback(errorHandler)
+
 
     def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred:
         self.crawlers.add(crawler)