remote-ffmpeg-h265/rh265.sh

#!/data/data/com.termux/files/usr/bin/bash

# 远程ffmpeg转码
# @todo 断点续传失败，文件自动从0开始，需要仔细研究下rsync的相关参数
# @todo 需要dashboard跟踪每个任务的文件绝对路径、日志、状态。能识别是否因网络不稳定等原因导致某个循环代码僵住无法跳出





# 基础支持
CUR_DIR="$(dirname "$(readlink -f "$0")")"
. "${CUR_DIR}"/lib/get_abs_filename.lib
. "${CUR_DIR}"/lib/echolog.lib





# 参数
#       接受最后一个参数，作为本地视频文件路径，转码完成后的结果文件为 [输入路径再追加"{.SERV}{.CRF}.mkv"]  （此参数必须写在最尾）
#       -c 参数可定制 ffmpeg 的crf参数值 （可选）
#       -s 参数指定配置文件的简称，例如 -s mm 会指定 rh265.mm.conf  （可选）
ARGS=("$@")
if [[ $# = 0 ]]; then less "${CUR_DIR}/readme.md"; exit; fi
VDPATH=${ARGS[$(($#-1))]}   # 输入文件名
CRF=""                      # ffmpeg命令的crf残片
CRF_SUFFIX=""               # 本地生成文件名后缀的crf部分
SERV=""                     # 带有服务器简称的文件名中缀残片
while getopts "c:s:" optname; do
    case "$optname" in
        c)
            CRF="-crf ${OPTARG}"
            CRF_SUFFIX=".crf${OPTARG}"
            ;;
        s)
            SERV=".${OPTARG}"
            ;;
        *)
            echo 'error arg option: -${optname}.'
            exit
            ;;
    esac
done





# 参数拦截
if ! [[ -e "${VDPATH}" ]]; then 
    echo "错误：输入的文件不存在"
    exit
fi





# 根据选择的服务器，装载配置文件
conf="${CUR_DIR}"/conf/rh265${SERV}.conf
if ! [[ -e "$conf" ]]; then 
    echo '配置文件 ${conf} 不存在.'
    exit
fi
. $conf





# 本地生成文件统一用的完整中缀，如 ".mm.crf23"，生成某文件的具体名称为 xxxxxxx.mm.crf23.finished
GENF_SUFFIX=${SERV}${CRF_SUFFIX}    

# 生成PID标记文件，便于跟踪
PID_FILE="${VDPATH}"${GENF_SUFFIX}.pid.$$
touch "$PID_FILE"

# 准备好本地日志文件及归档目录  @todo 改为存储json，方便共享读取
LOG_FILE="${CUR_DIR}"/logs/${REMOTE_TMPFILE}-pid-$$.log
LOG_FILE_END="${CUR_DIR}"/logs/end
mkdir -p "${LOG_FILE_END}"
echo "======== PID=$$ ======== 
    cwd: `ls -l /proc/$$/cwd`
    CMD: $0 $@
    relate pids: `ls /proc/$$/task`
    local: $VDPATH
    local(full): `lib_get_abs_filename "$VDPATH"`
    remote: ${REMOTEDIR}/${REMOTE_TMPFILE}.input
    log-ing: ${CUR_DIR}/logs/${REMOTE_TMPFILE}-pid-$$.log
    log-end: ${CUR_DIR}/logs/end/${REMOTE_TMPFILE}-pid-$$.log
    log-remote: sshpass -p '${PASSWD}' ssh -l $USER -p $PORT $HOST 'tail -f -n 100 ${REMOTEDIR}/${REMOTE_TMPFILE}.nohup'
" >> "${LOG_FILE_END}/log.map"

# 生成便于查看本地日志的脚本文件
LOG_SHOTCUT="${VDPATH}"${GENF_SUFFIX}.locallog.sh
echo "tail -f -n 100 '${LOG_FILE}' " > "$LOG_SHOTCUT"

# 生成便于查看远程日志的脚本文件（等开始转码时再写）
RMLOG_SHOTCUT="${VDPATH}${GENF_SUFFIX}.remotelog.sh"

# 控制错误输出
# exec 2>> "`getLogPPath`"





{ # <<<<<<<<<<<<<<<<< 主逻辑开始 <<<<<<<<<<<<<<<<<





    # 上传 
    echolog "上传中... ${VDPATH}  =>  远程目录${REMOTEDIR}/${REMOTE_TMPFILE}.input"
    rm -f "${VDPATH}${GENF_SUFFIX}.input.md5"
    uploadTo(){
        sshpass -p "${PASSWD}" ssh -l $USER -p $PORT $HOST "mkdir -p ${REMOTEDIR}" # 创建远程目录
        sshpass -p "${PASSWD}" rsync -avP -e "ssh -p ${PORT}" "${VDPATH}" ${USER}@${HOST}:"${REMOTEDIR}/${REMOTE_TMPFILE}.input" # 上传  （注意，多个了P参数，支持断点续传）
        rsyncResult=$?
        #if [[ "no input" = $(sshpass -p "${PASSWD}" ssh -l $USER -p $PORT $HOST "if ! [[ -e ${REMOTEDIR}/${REMOTE_TMPFILE}.input ]]; then echo 'no input'; else echo 'uploaded'; fi") ]]; then
        #    return 0  # 暂时不需要这个判断，有下方while中的input.md5有效性检测足矣
        #fi
        if [[ "0" = "${rsyncResult}" ]]; then
            sshpass -p "${PASSWD}" ssh -l $USER -p $PORT $HOST "md5sum ${REMOTEDIR}/${REMOTE_TMPFILE}.input |awk '{print \$1}' > ${REMOTEDIR}/${REMOTE_TMPFILE}.input.md5" # 上传后写md5
            sshpass -p "${PASSWD}" rsync -av -e "ssh -p ${PORT}" ${USER}@${HOST}:"${REMOTEDIR}/${REMOTE_TMPFILE}.input.md5"  "${VDPATH}${GENF_SUFFIX}.input.md5" # 下载md5到本地用于验证
        fi
        # return 1
    }
    while true; do
        if [[ -e "${VDPATH}${GENF_SUFFIX}.input.md5" ]]; then
        
            if [[ $(cat "${VDPATH}${GENF_SUFFIX}.input.md5") = '' ]]; then
                echolog "检测到远程无效的input.md5文件，可能rsync上传被中断，现在重试）"
                uploadTo
                continue
            fi
            
            if [[ $(cat "${VDPATH}${GENF_SUFFIX}.input.md5") = $(md5sum "${VDPATH}"|awk '{print $1}') ]]; then 
                echolog "已确认完整上传: ${VDPATH}"
                break  # 确保完整上传后，才可跳出重试的循环
            else
                echolog "上传失败，现在重试..."
                uploadTo
            fi
        else
            echolog C
            uploadTo
        fi
        sleep 1
    done





    # 后台远程转码  @todo: 一定要确保网络不稳定时，正确完整执行（观察到的情况：在服务器准备转码时，提示input文件不存在。不知道是怎么到这一步的）
    # @todo: 通过网络检测ffmpeg进程和mkv文件的过程，其实是不可信的，因为会遇到网络波动的情况，造成误判，进而重复提交ffmpeg命令。在遇到确实已转出mkv文件时，会因无法答复系统的[是否覆盖文件]的提问，造成一直提交失败的假象
    echolog '上传完毕，开始转码...'
    RM_COUNT=0
    tracingTranscode(){ # @todo: 已有mkv文件，但是被重复提交，提示是否覆盖 mkv already exists. Overwrite ? [y/N] Not overwriting - exiting
        pnlist=$(sshpass -p "$PASSWD" ssh -l $USER -p $PORT $HOST "ps -ef|grep '${REMOTE_TMPFILE}'|grep ffmpeg|grep -vw grep|awk '{print \$8}'")
        for pn in $pnlist; do
            if [[ "$pn" = "ffmpeg" ]]; then
                return 1
            fi
        done
        #检测不到ffmpeg进程，可能视频文件太小，ffmpeg快速完成了，那么检测mkv文件是否存在
        remoteMkvFileExists=$(sshpass -p "$PASSWD" ssh -l $USER -p $PORT $HOST "if [[ -e ${REMOTEDIR}/${REMOTE_TMPFILE}.mkv ]]; then echo 1; else echo 2; fi")
        if [[ 1 -eq "$remoteOutputFileExists" ]]; then
            return 1
        fi
        #进程和文件都找不到
        return 0
    }
    while [ $RM_COUNT -lt 5 ]; do
        RM_COUNT=$((RM_COUNT+1))
        echolog "第${RM_COUNT}次提交转码命令.."
		# sshpass -p "${PASSWD}" ssh -l $USER -p $PORT $HOST "nohup sh -c 'ffmpeg -i ${REMOTEDIR}/${REMOTE_TMPFILE}.input -c:v libx265 -c:a copy $CRF -movflags +faststart ${REMOTEDIR}/${REMOTE_TMPFILE}.mkv; md5sum ${REMOTEDIR}/${REMOTE_TMPFILE}.mkv|awk \"{print \\\$1}\" > ${REMOTEDIR}/${REMOTE_TMPFILE}.output.md5; touch ${REMOTEDIR}/${REMOTE_TMPFILE}.finished' > ${REMOTEDIR}/${REMOTE_TMPFILE}.nohup 2>&1 &"
        # 上面这条命令太复杂，以后可能还需要添加更多逻辑，有必要拆行，故采用传递远程脚本文件的形式
        echo "
            inputfile=${REMOTEDIR}/${REMOTE_TMPFILE}.input
            outputfile=${REMOTEDIR}/${REMOTE_TMPFILE}.mkv
            pnlist=\`ps -ef|grep \"${REMOTE_TMPFILE}\"|grep ffmpeg|grep -vw grep|awk '{print \$8}'\`
            for pn in \$pnlist; do
                if [[ \"\$pn\" = \"ffmpeg\" ]]; then
                    echo \"已检测到ffmpeg进程，不必再提交\"
                    exit
                fi
            done
            if [[ -e \"\$outputfile\" ]]; then 
                echo \"已有mkv文件，不必重复提交\"
                exit
            fi
            ffmpeg -i \$inputfile -c:v libx265 -c:a copy $CRF -movflags +faststart \$outputfile
            ffmpegResult=\$?
            md5sum \$outputfile|awk \"{print \\\$1}\" > ${REMOTEDIR}/${REMOTE_TMPFILE}.output.md5
            # 必须确保finished文件一定是成功转码完成后写入的
            if [[ -e \"\$outputfile\" ]] && [[ \"0\" -eq \"\$ffmpegResult\" ]]; then
                touch ${REMOTEDIR}/${REMOTE_TMPFILE}.finished 
            fi
        " > "${VDPATH}${GENF_SUFFIX}.ffmpeg"
        sshpass -p "${PASSWD}" rsync -avP -e "ssh -p ${PORT}" "${VDPATH}${GENF_SUFFIX}.ffmpeg" ${USER}@${HOST}:"${REMOTEDIR}/${REMOTE_TMPFILE}.ffmpeg"
        sshpass -p "${PASSWD}" ssh -l $USER -p $PORT $HOST "nohup bash ${REMOTEDIR}/${REMOTE_TMPFILE}.ffmpeg > ${REMOTEDIR}/${REMOTE_TMPFILE}.nohup 2>&1 &"
        rm -f "${VDPATH}${GENF_SUFFIX}.ffmpeg"
        
        TRACE_NUM=0
        while [ $TRACE_NUM -lt 20 ]; do
            TRACE_NUM=$((TRACE_NUM+1))
            tracingTranscode
            if [[ $? -eq 1 ]]; then
                echolog "已确认成功提交转码命令，现进入循环等待阶段..."
                break 2;
            fi
            echolog "无法确认是否完整提交转码命令，可能网络不稳定，现等待${TRACE_NUM}秒后再次检查（第${RM_COUNT}轮/第${TRACE_NUM}次）"
            sleep $TRACE_NUM
        done
    done

    if [ $RM_COUNT -ge 3 ]; then
        echolog "三次机会提交命令结果均失败，程序被迫中止，请自行清理垃圾文件"
        touch "${VDPATH}${GENF_SUFFIX}.ffmpegfail"
        exit
    fi
    echo "sshpass -p '${PASSWD}' ssh -l $USER -p $PORT $HOST 'tail -f -n 100 ${REMOTEDIR}/${REMOTE_TMPFILE}.nohup'; " > "${RMLOG_SHOTCUT}" # 提供一条看远程日志的命令





    # 检查转码是否完成（1、如果完成则在服务端写入标记文件*.finished; 2、下载*.finished标记文件到本地；3、当本地检测到*.finished文件时则确认转码完成）
    WAIT_FF=0
    while true; do
        if [[ $WAIT_FF -lt 30 ]]; then WAIT_FF=$((WAIT_FF+1)); fi
        echolog "waiting for ${WAIT_FF}s ..."
        sleep $WAIT_FF
        # 获取远程结果文件的大小
        sshpass -p "${PASSWD}" ssh -l $USER -p $PORT $HOST "ls -sh ${REMOTEDIR}/${REMOTE_TMPFILE}.mkv|awk '{print \$1}' > ${REMOTEDIR}/${REMOTE_TMPFILE}.output.size"
        sshpass -p "${PASSWD}" rsync -av -e "ssh -p ${PORT}" ${USER}@${HOST}:"${REMOTEDIR}/${REMOTE_TMPFILE}.output.size"  "${VDPATH}${GENF_SUFFIX}.output.size" > /dev/null 2>&1
        if [[ -e "${VDPATH}${GENF_SUFFIX}.output.size" ]]; then
            echolog '进行中, 远程结果文件大小：'`cat "${VDPATH}${GENF_SUFFIX}.output.size"`
        fi
        # 检查是否完成
        sshpass -p "${PASSWD}" rsync -av -e "ssh -p ${PORT}" ${USER}@${HOST}:"${REMOTEDIR}/${REMOTE_TMPFILE}.finished"  "${VDPATH}${GENF_SUFFIX}.finished" > /dev/null 2>&1
        if [[ -e "${VDPATH}${GENF_SUFFIX}.finished" ]]; then
            break
        fi
    done





    # 取回前先改名，减少在审查方面的麻烦
    sshpass -p "${PASSWD}" ssh -l $USER -p $PORT $HOST "mv ${REMOTEDIR}/${REMOTE_TMPFILE}.mkv ${REMOTEDIR}/${REMOTE_TMPFILE}.output"





    # 下载会确认完整性，并最后清理垃圾
    dlPath="${VDPATH}${GENF_SUFFIX}.mkv"
    echolog "转码完毕，开始下载结果...  ${USER}@${HOST}:${REMOTEDIR}/${REMOTE_TMPFILE}.output.md5  => ${dlPath} "
    downloadResult(){
        sshpass -p "${PASSWD}" rsync -avP -e "ssh -p ${PORT}" ${USER}@${HOST}:"${REMOTEDIR}/${REMOTE_TMPFILE}.output"  "${dlPath}"  # 注意，多个了P参数，支持断点续传
        sshpass -p "${PASSWD}" rsync -av -e "ssh -p ${PORT}" ${USER}@${HOST}:"${REMOTEDIR}/${REMOTE_TMPFILE}.output.md5"  "${VDPATH}${GENF_SUFFIX}.output.md5"
    }
    #while true; do
    DL_NUM=0
    while [ $DL_NUM -lt 1000 ]; do
        DL_NUM=$((DL_NUM+1))
        if [[ -e "${dlPath}" ]]; then
            if [[ $(cat "${VDPATH}${GENF_SUFFIX}.output.md5") = $(md5sum "${dlPath}"|awk '{print $1}') ]]; then 
                echolog D
                # 确认已下载，开始清理垃圾
                sshpass -p "${PASSWD}" ssh -l $USER -p $PORT $HOST " rm ${REMOTEDIR}/${REMOTE_TMPFILE}.input  ${REMOTEDIR}/${REMOTE_TMPFILE}.output  ${REMOTEDIR}/${REMOTE_TMPFILE}.input.md5  ${REMOTEDIR}/${REMOTE_TMPFILE}.output.md5  ${REMOTEDIR}/${REMOTE_TMPFILE}.output.size -f"
                rm "${VDPATH}${GENF_SUFFIX}.input.md5" "${VDPATH}${GENF_SUFFIX}.output.md5" "${VDPATH}${GENF_SUFFIX}.output.size" "${VDPATH}${GENF_SUFFIX}.finished" -f
                echolog "取回完毕： ${dlPath}"
                break
            else
                downloadResult
            fi
        else
            downloadResult
        fi
        # @todo: 由于网络环境切换或波动，下载不一定成功，故给定一些下载的机会；每失败一次，等待时间会增加；当次数用完，则终止程序
        echolog "已尝试第${DL_NUM}次下载，下次确认需等待${DL_NUM}秒..."
        sleep $DL_NUM
    done
    # 失败会写标志文件
    if ! [[ -e "${dlPath}" ]]; then
        touch "${VDPATH}${GENF_SUFFIX}.downloadfail"
    fi
    




} 2>&1 | tee -a "${LOG_FILE}" # >>>>>>>>>>>>>>> 主逻辑结束 >>>>>>>>>>>>>>>


# 结束时，清理PID和日志捷径；并将日志归档
rm -f "$PID_FILE"
rm -f "$LOG_SHOTCUT"
rm -f "$RMLOG_SHOTCUT"
mv  "${LOG_FILE}"  "${LOG_FILE_END}"