使用scrapy调用execjs时会报错?

Question

# -*- coding:utf-8 -*-
import json
import os
from uuid import uuid4
import time, execjs


def get_token(uuid_time):
    t = f"pageEnName=index&moduleInfoList=[object Object],[object Object],[object Object]&uuid={uuid_time}&platform=3&partner=150&optimusCode=10&originUrl=https://m.dianping.com/"
    to = "//m.dianping.com/index/api/module" + '&' + t
    _token = execjs.compile(
        open('/Users/licongjian/Desktop/dazhongdianping/dazhongdianping/utils/c.js',
             encoding='utf-8').read()).call(
        'get_token', to)
    return _token


def get_keyword_url(keyword, start=0):
    print(keyword, start)
    uuid_time = f'{uuid4()}.{str(time.time()).split(".")[0]}'
    _token = get_token(uuid_time)

    data = {"uuid": uuid_time, "platform": 1, "partner": 150, "optimusCode": 10,
            "originUrl": "https://m.dianping.com/shoplist/1/search?from': 'm_search','keyword': %s" % keyword,
            "pageEnName": "shopList", "moduleInfoList": [{"moduleName": "mapiSearch", "query": {
            "search": {"start": start, "categoryId": 0, "parentCategoryId": 0, "locateCityid": 0, "limit": 20,
                       "sortId": 0,
                       "cityId": 1, "keyword": keyword, "regionId": 0, "maptype": 0}}}],
            "_token": _token}
    print(data)
    return json.dumps(data)


headers = {
    'Connection': 'keep-alive',
    'Pragma': 'no-cache',
    'Cache-Control': 'no-cache',
    'sec-ch-ua': '"Google Chrome";v="89", "Chromium";v="89", ";Not A Brand";v="99"',
    'sec-ch-ua-mobile': '?0',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36',
    'Content-Type': 'application/json',
    'Accept': '*/*',
    'Origin': 'https://m.dianping.com',
    'Sec-Fetch-Site': 'same-origin',
    'Sec-Fetch-Mode': 'cors',
    'Sec-Fetch-Dest': 'empty',
    'Referer': 'https://m.dianping.com/shoplist/1/search?from=m_search&keyword=%E6%97%B6%E5%AE%AB%E6%96%B0%E6%9D%91',
    'Accept-Language': 'zh-CN,zh;q=0.9',
}

在本文件调用execjs是没有问题的,在scrapy中调用会报错

2021-04-26 17:46:34 [scrapy.core.engine] ERROR: Error while obtaining start requests
Traceback (most recent call last):
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/scrapy/core/engine.py", line 129, in _next_request
    request = next(slot.start_requests)
  File "/Users/licongjian/Desktop/dazhongdianping/dazhongdianping/spiders/dp.py", line 17, in start_requests
    data = get_keyword_url('天安门')
  File "/Users/licongjian/Desktop/dazhongdianping/dazhongdianping/utils/get_keyword.py", line 21, in get_keyword_url
    _token = get_token(uuid_time)
  File "/Users/licongjian/Desktop/dazhongdianping/dazhongdianping/utils/get_keyword.py", line 14, in get_token
    'get_token', to)
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/execjs/_abstract_runtime_context.py", line 37, in call
    return self._call(name, *args)
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/execjs/_external_runtime.py", line 92, in _call
    return self._eval("{identifier}.apply(this, {args})".format(identifier=identifier, args=args))
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/execjs/_external_runtime.py", line 78, in _eval
    return self.exec_(code)
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/execjs/_abstract_runtime_context.py", line 18, in exec_
    return self._exec_(source)
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/execjs/_external_runtime.py", line 87, in _exec_
    output = self._exec_with_pipe(source)
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/execjs/_external_runtime.py", line 99, in _exec_with_pipe
    p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=self._cwd, universal_newlines=True)
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/subprocess.py", line 729, in __init__
    restore_signals, start_new_session)
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/subprocess.py", line 1288, in _execute_child
    self.pid = _posixsubprocess.fork_exec(
AttributeError: 'NoneType' object has no attribute 'fork_exec'

Answer 1

这是一个Scrapy框架的错误日志。从日志中可以看出，在获取起始请求时发生了错误。具体地，问题出现在get_keyword.py文件的get_token函数中，该函数使用了execjs库来执行JavaScript代码。由于在执行子进程时发生了错误，因此出现了AttributeError: 'NoneType' object has no attribute 'fork_exec'异常。这个错误可能是由于subprocess库没有正确导入，或者是因为您的Python环境没有足够的权限来执行fork_exec操作。建议检查您的环境设置以解决这个问题。
在Unix或类Unix系统中，可以使用以下命令提高Python脚本的执行权限：

chmod +x your_python_script.py

这将使脚本具有可执行权限，从而使您能够在终端中通过以下方式运行脚本：

./your_python_script.py