在crontab中运行mwget多线程下载出错？(操作系统-linux)

Question

请教一下，我有一个脚本调用的mwget多线程下载ECMWF数据的，在shell中运行正常，但是添加到crontab中定时执行的时候总出错，是什么原因？已经在脚本中使用了绝对路径了。

crontab如下：

*/1 * * * * bash /home/METLED/metled-downloader-ec/download_ec_enfo_grib2_0p25.sh

download_ec_enfo_grib2_0p25.sh的内容：

#!/bin/bash
source /root/.bashrc
ps_n=0
ps_n=`ps -ef | grep -v grep | grep download_ec_enfo_grib2_0p25.py | wc -l`

if [ $ps_n -eq 0 ];then
    nohup /usr/bin/python3 /home/METLED/metled-downloader-ec/download_ec_enfo_grib2_0p25.py > /home/METLED/metled-downloader-ec/download_ec_enfo_grib2_0p25.log 2>&1 &
fi

download_ec_enfo_grib2_0p25.py内容如下：

import datetime
import logging.config
import os
import sys
import subprocess
from lock import LockServer

POOL_SIZE = 10
SAVE_DIR_PATH = "/home/METLED_DATA/nwp/ecmwf/grib2"
SCRIPT_PATH = "/home/METLED/metled-downloader-ec"
LOGS_PATH = os.path.join(SCRIPT_PATH, "logs")
if not os.path.exists(LOGS_PATH):
    os.makedirs(LOGS_PATH)

LOGGING_CONF_PATH = os.path.join(SCRIPT_PATH, "conf/logging.conf")
logging.config.fileConfig(LOGGING_CONF_PATH)
logger = logging.getLogger("download_ec_enfo_grib2_0p25.py")

def down(url, dir_path, filename):
    command = "/usr/local/bin/mwget -n {} -d '{}' -f '{}' {}".format(POOL_SIZE, dir_path, filename, url)
    if not os.path.exists(os.path.join(dir_path, filename)):
        try:
            status = os.system(command)
            if status == 0 and os.path.exists(os.path.join(dir_path, filename)):
                logger.info("download complete {}".format(os.path.join(dir_path, filename)))
            else:
                logger.error("download failed. status: {}, command: '{}'".format(status, command))
        except Exception as e:
            logger.error("download error: {}, command: '{}'".format(e, command))

def down_from_http(time_string):
    date = time_string[0:8]
    hour = time_string[8:10]
    url_remote = "https://data.ecmwf.int/forecasts/{}/{}z/ifs/0p25/enfo/".format(date, hour)
    save_path = os.path.join(SAVE_DIR_PATH, date, hour + "z")
    if not os.path.exists(save_path):
        os.makedirs(save_path)
    for i in range(0, 75, 3):
        filename = "{}0000-{}h-enfo-ef.grib2".format(time_string, i)
        tmp_filename = "{}.mg!".format(filename)
        tmp_filepath = os.path.join(save_path, tmp_filename)
        if os.path.exists(tmp_filepath):
            os.remove(tmp_filepath)
            logger.info("remove tmp file: {}", tmp_filepath)
        down(url_remote + filename, save_path, filename)

def process():
    try:
        time_string = sys.argv[1]
    except Exception as e:
        utc_now = datetime.datetime.utcnow()
        # 延迟10小时
        utc_now = utc_now - datetime.timedelta(hours=10)
        utc_now = utc_now - datetime.timedelta(seconds=utc_now.second)
        utc_now = utc_now - datetime.timedelta(minutes=utc_now.minute)
        # 减去对6取的余数，获取00,06,12,18时次
        utc_now = utc_now - datetime.timedelta(hours=utc_now.hour % 6)
        time_string = utc_now.strftime("%Y%m%d%H")
    down_from_http(time_string)

if __name__ == '__main__':
    lock_ser = LockServer("download_ec_enfo_grib2_0p25.lock")
    if lock_ser.check_exists():
        logger.warn("download_ec_enfo_grib2_0p25 program is running")
        exit()
    lock_ser.update_time()
    lock_ser.start_threading()
    try:
        process()
    except Exception as e:
        logger.error(e)
    lock_ser.end_threading()

报错信息：

2024-04-03 22:46:03,873 ERROR 127056699904448 -- [MainThread] /home/METLED/metled-downloader-ec/download_ec_enfo_grib2_0p25.py:33: download failed. status: 35584, command: '/usr/local/bin/mwget -n 10 -d '/home/METLED_DATA/nwp/ecmwf/grib2/20240403/12z' -f '20240403120000-48h-enfo-ef.grib2' https://data.ecmwf.int/forecasts/20240403/12z/ifs/0p25/enfo/20240403120000-48h-enfo-ef.grib2'
2024-04-03 22:46:04,038 ERROR 127056699904448 -- [MainThread] /home/METLED/metled-downloader-ec/download_ec_enfo_grib2_0p25.py:33: download failed. status: 35584, command: '/usr/local/bin/mwget -n 10 -d '/home/METLED_DATA/nwp/ecmwf/grib2/20240403/12z' -f '20240403120000-51h-enfo-ef.grib2' https://data.ecmwf.int/forecasts/20240403/12z/ifs/0p25/enfo/20240403120000-51h-enfo-ef.grib2'
2024-04-03 22:46:04,205 ERROR 127056699904448 -- [MainThread] /home/METLED/metled-downloader-ec/download_ec_enfo_grib2_0p25.py:33: download failed. status: 35584, command: '/usr/local/bin/mwget -n 10 -d '/home/METLED_DATA/nwp/ecmwf/grib2/20240403/12z' -f '20240403120000-54h-enfo-ef.grib2' https://data.ecmwf.int/forecasts/20240403/12z/ifs/0p25/enfo/20240403120000-54h-enfo-ef.grib2'
2024-04-03 22:46:04,368 ERROR 127056699904448 -- [MainThread] /home/METLED/metled-downloader-ec/download_ec_enfo_grib2_0p25.py:33: download failed. status: 35584, command: '/usr/local/bin/mwget -n 10 -d '/home/METLED_DATA/nwp/ecmwf/grib2/20240403/12z' -f '20240403120000-57h-enfo-ef.grib2' https://data.ecmwf.int/forecasts/20240403/12z/ifs/0p25/enfo/20240403120000-57h-enfo-ef.grib2'
2024-04-03 22:46:04,533 ERROR 127056699904448 -- [MainThread] /home/METLED/metled-downloader-ec/download_ec_enfo_grib2_0p25.py:33: download failed. status: 35584, command: '/usr/local/bin/mwget -n 10 -d '/home/METLED_DATA/nwp/ecmwf/grib2/20240403/12z' -f '20240403120000-60h-enfo-ef.grib2' https://data.ecmwf.int/forecasts/20240403/12z/ifs/0p25/enfo/20240403120000-60h-enfo-ef.grib2'
2024-04-03 22:46:04,700 ERROR 127056699904448 -- [MainThread] /home/METLED/metled-downloader-ec/download_ec_enfo_grib2_0p25.py:33: download failed. status: 35584, command: '/usr/local/bin/mwget -n 10 -d '/home/METLED_DATA/nwp/ecmwf/grib2/20240403/12z' -f '20240403120000-63h-enfo-ef.grib2' https://data.ecmwf.int/forecasts/20240403/12z/ifs/0p25/enfo/20240403120000-63h-enfo-ef.grib2'
2024-04-03 22:46:04,862 ERROR 127056699904448 -- [MainThread] /home/METLED/metled-downloader-ec/download_ec_enfo_grib2_0p25.py:33: download failed. status: 35584, command: '/usr/local/bin/mwget -n 10 -d '/home/METLED_DATA/nwp/ecmwf/grib2/20240403/12z' -f '20240403120000-66h-enfo-ef.grib2' https://data.ecmwf.int/forecasts/20240403/12z/ifs/0p25/enfo/20240403120000-66h-enfo-ef.grib2'
2024-04-03 22:46:05,021 ERROR 127056699904448 -- [MainThread] /home/METLED/metled-downloader-ec/download_ec_enfo_grib2_0p25.py:33: download failed. status: 35584, command: '/usr/local/bin/mwget -n 10 -d '/home/METLED_DATA/nwp/ecmwf/grib2/20240403/12z' -f '20240403120000-69h-enfo-ef.grib2' https://data.ecmwf.int/forecasts/20240403/12z/ifs/0p25/enfo/20240403120000-69h-enfo-ef.grib2'
2024-04-03 22:46:05,183 ERROR 127056699904448 -- [MainThread] /home/METLED/metled-downloader-ec/download_ec_enfo_grib2_0p25.py:33: download failed. status: 35584, command: '/usr/local/bin/mwget -n 10 -d '/home/METLED_DATA/nwp/ecmwf/grib2/20240403/12z' -f '20240403120000-72h-enfo-ef.grib2' https://data.ecmwf.int/forecasts/20240403/12z/ifs/0p25/enfo/20240403120000-72h-enfo-ef.grib2'

Answer 1

上午好☀️☀️☀️️
本答案参考ChatGPT-3.5

可能出现错误的原因有很多，以下是一些可能的解决方案：

1. 确保crontab中的脚本路径和文件路径都是正确的。如果你在脚本中使用了绝对路径，确保路径是正确的。

2. 在脚本中使用了source /root/.bashrc，但是crontab运行时可能没有加载该文件。可以尝试将需要设置的环境变量直接写入脚本中，而不是依赖于.bashrc文件。

3. 让脚本输出日志信息可以帮助我们定位问题。可以在crontab的定时执行命令后加上>> /path/to/logfile.log 2>&1，将输出重定向到一个日志文件中，以便于排查错误。

4. 检查脚本中使用的命令是否都是绝对路径。crontab中运行脚本时，可能没有设置相应的环境变量，导致命令无法找到。可以使用绝对路径来执行命令。

5. 检查脚本中与时间相关的部分是否正常运行。比如，检查时间字符串是否正确，检查是否创建了正确的目录。可以将相关部分的日志打印出来，以便于排查问题。

6. 检查脚本中调用的外部命令是否存在问题。比如，mwget命令是否可以在crontab中正常执行。可以尝试手动执行命令，看是否有报错信息。

7. 检查脚本中使用的文件和路径权限是否正确。确保脚本有足够的权限来读取和写入文件。

8. 检查是否有其他进程占用了资源，导致脚本无法正常执行。可以使用ps命令来查看是否有进程正在运行。

修改后的代码：

#!/bin/bash
PS_N=0
PS_N=$(ps -ef | grep -v grep | grep download_ec_enfo_grib2_0p25.py | wc -l)

if [ $PS_N -eq 0 ]; then
    nohup /usr/bin/python3 /home/METLED/metled-downloader-ec/download_ec_enfo_grib2_0p25.py > /home/METLED/metled-downloader-ec/download_ec_enfo_grib2_0p25.log 2>&1 &
fi

import datetime
import logging.config
import os
import sys
import subprocess
from lock import LockServer

POOL_SIZE = 10
SAVE_DIR_PATH = "/home/METLED_DATA/nwp/ecmwf/grib2"
SCRIPT_PATH = "/home/METLED/metled-downloader-ec"
LOGS_PATH = os.path.join(SCRIPT_PATH, "logs")
if not os.path.exists(LOGS_PATH):
    os.makedirs(LOGS_PATH)

LOGGING_CONF_PATH = os.path.join(SCRIPT_PATH, "conf/logging.conf")
logging.config.fileConfig(LOGGING_CONF_PATH)
logger = logging.getLogger("download_ec_enfo_grib2_0p25.py")

def down(url, dir_path, filename):
    command = "/usr/local/bin/mwget -n {} -d '{}' -f '{}' {}".format(POOL_SIZE, dir_path, filename, url)
    if not os.path.exists(os.path.join(dir_path, filename)):
        try:
            status = subprocess.call(command, shell=True)
            if status == 0 and os.path.exists(os.path.join(dir_path, filename)):
                logger.info("download complete {}".format(os.path.join(dir_path, filename)))
            else:
                logger.error("download failed. status: {}, command: '{}'".format(status, command))
        except Exception as e:
            logger.error("download error: {}, command: '{}'".format(e, command))

def down_from_http(time_string):
    date = time_string[0:8]
    hour = time_string[8:10]
    url_remote = "https://data.ecmwf.int/forecasts/{}/{}z/ifs/0p25/enfo/".format(date, hour)
    save_path = os.path.join(SAVE_DIR_PATH, date, hour + "z")
    if not os.path.exists(save_path):
        os.makedirs(save_path)
    for i in range(0, 75, 3):
        filename = "{}0000-{}h-enfo-ef.grib2".format(time_string, i)
        tmp_filename = "{}.mg!".format(filename)
        tmp_filepath = os.path.join(save_path, tmp_filename)
        if os.path.exists(tmp_filepath):
            os.remove(tmp_filepath)
            logger.info("remove tmp file: {}", tmp_filepath)
        down(url_remote + filename, save_path, filename)

def process():
    try:
        time_string = sys.argv[1]
    except Exception as e:
        utc_now = datetime.datetime.utcnow()
        # 延迟10小时
        utc_now = utc_now - datetime.timedelta(hours=10)
        utc_now = utc_now - datetime.timedelta(seconds=utc_now.second)
        utc_now = utc_now - datetime.timedelta(minutes=utc_now.minute)
        # 减去对6取的余数，获取00,06,12,18时次
        utc_now = utc_now - datetime.timedelta(hours=utc_now.hour % 6)
        time_string = utc_now.strftime("%Y%m%d%H")
    down_from_http(time_string)

if __name__ == '__main__':
    lock_ser = LockServer("download_ec_enfo_grib2_0p25.lock")
    if lock_ser.check_exists():
        logger.warn("download_ec_enfo_grib2_0p25 program is running")
        exit()
    lock_ser.update_time()
    lock_ser.start_threading()
    try:
        process()
    except Exception as e:
        logger.error(e)
    lock_ser.end_threading()

希望以上解决方案能帮助您解决问题。如果问题仍然存在，请提供更多的错误信息以便于更好地定位问题。