2023-09-05 16:11:55 [scrapy.utils.log] INFO: Scrapy 1.5.0 started (bot: magic_mirror) 2023-09-05 16:11:55 [scrapy.utils.log] INFO: Versions: lxml 4.5.1.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 18.9.0, Python 3.6.7 (default, Oct 22 2018, 11:32:17) - [GCC 8.2.0], pyOpenSSL 19.1.0 (OpenSSL 1.1.1g 21 Apr 2020), cryptography 2.9.2, Platform Linux-4.15.0-29-generic-x86_64-with-Ubuntu-18.04-bionic 2023-09-05 16:11:55 [scrapy.crawler] INFO: Overridden settings: {'BOT_NAME': 'magic_mirror', 'DOWNLOAD_DELAY': 3, 'DOWNLOAD_TIMEOUT': 200, 'DUPEFILTER_CLASS': 'scrapy_redis.dupefilter.RFPDupeFilter', 'LOG_FILE': '/home/ubuntu/project/spider/logs/magic_mirror/jiqizhixin/9bd383ce4bc311ee9fbdfa163e6898ec.log', 'NEWSPIDER_MODULE': 'magic_mirror.spiders', 'RETRY_HTTP_CODES': [500, 503, 504, 400, 403, 404, 408, 302, 304], 'RETRY_TIMES': 5, 'SCHEDULER': 'scrapy_redis.scheduler.Scheduler', 'SPIDER_MODULES': ['magic_mirror.spiders']} 2023-09-05 16:11:55 [scrapy.middleware] INFO: Enabled extensions: ['scrapy.extensions.corestats.CoreStats', 'scrapy.extensions.telnet.TelnetConsole', 'scrapy.extensions.memusage.MemoryUsage', 'scrapy.extensions.logstats.LogStats'] 2023-09-05 16:11:55 [py.warnings] WARNING: /home/ubuntu/.local/lib/python3.6/site-packages/scrapy/utils/deprecate.py:156: ScrapyDeprecationWarning: `scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware` class is deprecated, use `scrapy.downloadermiddlewares.useragent.UserAgentMiddleware` instead ScrapyDeprecationWarning) 2023-09-05 16:11:55 [py.warnings] WARNING: /home/ubuntu/.local/lib/python3.6/site-packages/scrapy/utils/deprecate.py:156: ScrapyDeprecationWarning: `scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware` class is deprecated, use `scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` instead ScrapyDeprecationWarning) 2023-09-05 16:11:55 [py.warnings] WARNING: /home/ubuntu/.local/lib/python3.6/site-packages/scrapy/utils/deprecate.py:156: ScrapyDeprecationWarning: `scrapy.contrib.downloadermiddleware.retry.RetryMiddleware` class is deprecated, use `scrapy.downloadermiddlewares.retry.RetryMiddleware` instead ScrapyDeprecationWarning) 2023-09-05 16:11:55 [py.warnings] WARNING: /home/ubuntu/.local/lib/python3.6/site-packages/scrapy/utils/deprecate.py:156: ScrapyDeprecationWarning: `scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware` class is deprecated, use `scrapy.downloadermiddlewares.cookies.CookiesMiddleware` instead ScrapyDeprecationWarning) 2023-09-05 16:11:55 [scrapy.middleware] INFO: Enabled downloader middlewares: ['scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', 'magic_mirror.spiders.random_proxy.RandomProxy', 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', 'scrapy.downloadermiddlewares.retry.RetryMiddleware', 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', 'magic_mirror.spiders.rotate_useragent.RotateUserAgentMiddleware', 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware', 'scrapy.downloadermiddlewares.stats.DownloaderStats'] 2023-09-05 16:11:55 [scrapy.middleware] INFO: Enabled spider middlewares: ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', 'scrapy.spidermiddlewares.referer.RefererMiddleware', 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', 'scrapy.spidermiddlewares.depth.DepthMiddleware'] 2023-09-05 16:11:55 [scrapy.middleware] INFO: Enabled item pipelines: ['magic_mirror.pipelines.MongoPipeline', 'scrapy_redis.pipelines.RedisPipeline'] 2023-09-05 16:11:55 [scrapy.core.engine] INFO: Spider opened 2023-09-05 16:11:55 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) 2023-09-05 16:11:55 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6032 2023-09-05 16:11:56 [selenium.webdriver.remote.remote_connection] DEBUG: POST http://127.0.0.1:15315/session {"capabilities": {"firstMatch": [{}], "alwaysMatch": {"browserName": "chrome", "platformName": "any", "goog:chromeOptions": {"extensions": [], "args": ["--headless"]}}}, "desiredCapabilities": {"browserName": "chrome", "version": "", "platform": "ANY", "goog:chromeOptions": {"extensions": [], "args": ["--headless"]}}} 2023-09-05 16:11:56 [urllib3.connectionpool] DEBUG: Starting new HTTP connection (1): 127.0.0.1 2023-09-05 16:11:56 [urllib3.connectionpool] DEBUG: http://127.0.0.1:15315 "POST /session HTTP/1.1" 200 679 2023-09-05 16:11:56 [selenium.webdriver.remote.remote_connection] DEBUG: Finished Request 2023-09-05 16:11:56 [selenium.webdriver.remote.remote_connection] DEBUG: POST http://127.0.0.1:15315/session/d7c0c706d94cc6af26b430d66de2bda6/url {"url": "https://www.jiqizhixin.com/industry"} 2023-09-05 16:11:59 [urllib3.connectionpool] DEBUG: http://127.0.0.1:15315 "POST /session/d7c0c706d94cc6af26b430d66de2bda6/url HTTP/1.1" 200 14 2023-09-05 16:11:59 [selenium.webdriver.remote.remote_connection] DEBUG: Finished Request 2023-09-05 16:12:00 [selenium.webdriver.remote.remote_connection] DEBUG: POST http://127.0.0.1:15315/session/d7c0c706d94cc6af26b430d66de2bda6/execute/sync {"script": "window.scrollTo(0,document.body.scrollHeight)", "args": []} 2023-09-05 16:12:00 [urllib3.connectionpool] DEBUG: http://127.0.0.1:15315 "POST /session/d7c0c706d94cc6af26b430d66de2bda6/execute/sync HTTP/1.1" 200 14 2023-09-05 16:12:00 [selenium.webdriver.remote.remote_connection] DEBUG: Finished Request 2023-09-05 16:12:02 [selenium.webdriver.remote.remote_connection] DEBUG: POST http://127.0.0.1:15315/session/d7c0c706d94cc6af26b430d66de2bda6/execute/sync {"script": "window.scrollTo(0,document.body.scrollHeight)", "args": []} 2023-09-05 16:12:02 [urllib3.connectionpool] DEBUG: http://127.0.0.1:15315 "POST /session/d7c0c706d94cc6af26b430d66de2bda6/execute/sync HTTP/1.1" 200 14 2023-09-05 16:12:02 [selenium.webdriver.remote.remote_connection] DEBUG: Finished Request 2023-09-05 16:12:04 [selenium.webdriver.remote.remote_connection] DEBUG: POST http://127.0.0.1:15315/session/d7c0c706d94cc6af26b430d66de2bda6/execute/sync {"script": "window.scrollTo(0,document.body.scrollHeight)", "args": []} 2023-09-05 16:12:04 [urllib3.connectionpool] DEBUG: http://127.0.0.1:15315 "POST /session/d7c0c706d94cc6af26b430d66de2bda6/execute/sync HTTP/1.1" 200 14 2023-09-05 16:12:04 [selenium.webdriver.remote.remote_connection] DEBUG: Finished Request 2023-09-05 16:12:06 [selenium.webdriver.remote.remote_connection] DEBUG: POST http://127.0.0.1:15315/session/d7c0c706d94cc6af26b430d66de2bda6/execute/sync {"script": "window.scrollTo(0,document.body.scrollHeight)", "args": []} 2023-09-05 16:12:06 [urllib3.connectionpool] DEBUG: http://127.0.0.1:15315 "POST /session/d7c0c706d94cc6af26b430d66de2bda6/execute/sync HTTP/1.1" 200 14 2023-09-05 16:12:06 [selenium.webdriver.remote.remote_connection] DEBUG: Finished Request 2023-09-05 16:12:08 [selenium.webdriver.remote.remote_connection] DEBUG: POST http://127.0.0.1:15315/session/d7c0c706d94cc6af26b430d66de2bda6/execute/sync {"script": "window.scrollTo(0,document.body.scrollHeight)", "args": []} 2023-09-05 16:12:08 [urllib3.connectionpool] DEBUG: http://127.0.0.1:15315 "POST /session/d7c0c706d94cc6af26b430d66de2bda6/execute/sync HTTP/1.1" 200 14 2023-09-05 16:12:08 [selenium.webdriver.remote.remote_connection] DEBUG: Finished Request 2023-09-05 16:12:09 [selenium.webdriver.remote.remote_connection] DEBUG: POST http://127.0.0.1:15315/session/d7c0c706d94cc6af26b430d66de2bda6/elements {"using": "xpath", "value": "//article[@class=\"article-item__container\"]"} 2023-09-05 16:12:09 [urllib3.connectionpool] DEBUG: http://127.0.0.1:15315 "POST /session/d7c0c706d94cc6af26b430d66de2bda6/elements HTTP/1.1" 200 3329 2023-09-05 16:12:09 [selenium.webdriver.remote.remote_connection] DEBUG: Finished Request 2023-09-05 16:12:09 [selenium.webdriver.remote.remote_connection] DEBUG: POST http://127.0.0.1:15315/session/d7c0c706d94cc6af26b430d66de2bda6/element/ab9728c1-d091-427b-8682-bac9a1f910fa/element {"using": "xpath", "value": ".//a[@class=\"article-item__title t-strong js-open-modal\"]", "id": "ab9728c1-d091-427b-8682-bac9a1f910fa"} 2023-09-05 16:12:09 [urllib3.connectionpool] DEBUG: http://127.0.0.1:15315 "POST /session/d7c0c706d94cc6af26b430d66de2bda6/element/ab9728c1-d091-427b-8682-bac9a1f910fa/element HTTP/1.1" 200 88 2023-09-05 16:12:09 [selenium.webdriver.remote.remote_connection] DEBUG: Finished Request 2023-09-05 16:12:09 [selenium.webdriver.remote.remote_connection] DEBUG: POST http://127.0.0.1:15315/session/d7c0c706d94cc6af26b430d66de2bda6/execute/sync {"script": "return (function(){return function(){var d=this;function f(a){return\"string\"==typeof a};function h(a,b){this.code=a;this.a=l[a]||m;this.message=b||\"\";a=this.a.replace(/((?:^|\\s+)[a-z])/g,function(a){return a.toUpperCase().replace(/^[\\s\\xa0]+/g,\"\")});b=a.length-5;if(0>b||a.indexOf(\"Error\",b)!=b)a+=\"Error\";this.name=a;a=Error(this.message);a.name=this.name;this.stack=a.stack||\"\"}\n(function(){var a=Error;function b(){}b.prototype=a.prototype;h.b=a.prototype;h.prototype=new b;h.prototype.constructor=h;h.a=function(b,c,g){for(var e=Array(arguments.length-2),k=2;kparseFloat(D)){C=String(F);break a}}C=D}var G;var H=d.document;G=H&&y?B()||(\"CSS1Compat\"==H.compatMode?parseInt(C,10):5):void 0;var ba=r(\"Firefox\"),ca=v()||r(\"iPod\"),da=r(\"iPad\"),I=r(\"Android\")&&!(w()||r(\"Firefox\")||r(\"Opera\")||r(\"Silk\")),ea=w(),J=r(\"Safari\")&&!(w()||r(\"Coast\")||r(\"Opera\")||r(\"Edge\")||r(\"Silk\")||r(\"Android\"))&&!(v()||r(\"iPad\")||r(\"iPod\"));function K(a){return(a=a.exec(n))?a[1]:\"\"}(function(){if(ba)return K(/Firefox\\/([0-9.]+)/);if(y||z||x)return C;if(ea)return v()||r(\"iPad\")||r(\"iPod\")?K(/CriOS\\/([0-9.]+)/):K(/Chrome\\/([0-9.]+)/);if(J&&!(v()||r(\"iPad\")||r(\"iPod\")))return K(/Version\\/([0-9.]+)/);if(ca||da){var a=/Version\\/(\\S+).*Mobile\\/(\\S+)/.exec(n);if(a)return a[1]+\".\"+a[2]}else if(I)return(a=K(/Android\\s+([0-9.]+)/))?a:K(/Version\\/([0-9.]+)/);return\"\"})();var L,M=function(){if(!A)return!1;var a=d.Components;if(!a)return!1;try{if(!a.classes)return!1}catch(g){return!1}var b=a.classes,a=a.interfaces,e=b[\"@mozilla.org/xpcom/version-comparator;1\"].getService(a.nsIVersionComparator),c=b[\"@mozilla.org/xre/app-info;1\"].getService(a.nsIXULAppInfo).version;L=function(a){e.compare(c,\"\"+a)};return!0}(),N=y&&!(8<=Number(G)),fa=y&&!(9<=Number(G));I&&M&&L(2.3);I&&M&&L(4);J&&M&&L(6);var ga={SCRIPT:1,STYLE:1,HEAD:1,IFRAME:1,OBJECT:1},O={IMG:\" \",BR:\"\\n\"};function P(a,b,e){if(!(a.nodeName in ga))if(3==a.nodeType)e?b.push(String(a.nodeValue).replace(/(\\r\\n|\\r|\\n)/g,\"\")):b.push(a.nodeValue);else if(a.nodeName in O)b.push(O[a.nodeName]);else for(a=a.firstChild;a;)P(a,b,e),a=a.nextSibling};function Q(a,b){b=b.toLowerCase();return\"style\"==b?ha(a.style.cssText):N&&\"value\"==b&&R(a,\"INPUT\")?a.value:fa&&!0===a[b]?String(a.getAttribute(b)):(a=a.getAttributeNode(b))&&a.specified?a.value:null}var ia=/[;]+(?=(?:(?:[^\"]*\"){2})*[^\"]*$)(?=(?:(?:[^']*'){2})*[^']*$)(?=(?:[^()]*\\([^()]*\\))*[^()]*$)/;\nfunction ha(a){var b=[];t(a.split(ia),function(a){var c=a.indexOf(\":\");0