dwcrawl git init

.gitignore

@@ -0,0 +1,3 @@
+.idea/
+.vscode/launch.json
+protected/node_modules/

protected/admin/restart.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+cd $(dirname $0)/../
+
+forever restart start.js --harmony;

protected/admin/start.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+cd $(dirname $0)/../
+
+forever start start.js --harmony;

protected/admin/stop.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+cd $(dirname $0)/../
+
+forever stop start.js;

protected/app.js

@@ -0,0 +1,113 @@
+'use strict'
+
+let express = require('express');
+let path = require('path');
+let logger = require('morgan');
+let cookieParser = require('cookie-parser');
+let bodyParser = require('body-parser');
+let domain = require('domain');
+require('./common.js');
+
+let app = express();
+
+// view engine setup
+app.set('views', path.join(__dirname, 'views'));
+app.set('view engine', 'ejs');
+
+// uncomment after placing your favicon in /public
+//app.use(favicon(path.join(__dirname, 'public', 'favicon.ico')));
+app.use(logger('dev'));
+app.use(bodyParser.json());
+app.use(bodyParser.urlencoded({
+    extended: false
+}));
+app.use(cookieParser());
+// app.use(express.static(path.join(__dirname, 'public')));
+
+// 加载扩展函数
+require('./extensions/function_extend.js');
+let Router = require('./framework/lib/Router.js');
+let codeMap = require('./conf/code.inc.js');
+
+let Response = require('./framework/lib/Response.js');
+global['AppErrors'] = require('./framework/lib/AppErrors.js');
+
+// catch 404 and forward to error handler
+app.use(function(req, res, next) {
+    let objDomain = domain.create();
+    objDomain.run(function () {
+        main(req, res, next);
+    });
+
+    objDomain.on('error', function (err) {
+        let objResponse = new Response(req, res, codeMap);
+        handlerException(objResponse, err);
+    });
+});
+
+function handlerException(objResponse, err) {
+    try {
+        if (err instanceof AppErrors.Interrupt) {
+            // console.log('catch Interrupt.');
+        } else if (err instanceof AppErrors.DbError) {
+            objResponse.error(CODE_DB_ERROR, null, err.message, null);
+        } else if (err instanceof AppErrors.RedisError) {
+            objResponse.error(CODE_REDIS_ERROR, null, err.message, null);
+        } else {
+            // 对于未知情况，则返回位置错误
+            objResponse.error(CODE_UNKNOW_ERROT, null, err.message, null);
+        }
+    } catch(ex) {
+        console.error('catch(ex)');
+        console.error(ex);
+    }
+
+}
+
+function main(req, res, next) {
+    let objRouter = new Router(req.originalUrl);
+    /**
+     * @type {Doc}
+     */
+    if(req.query['doc']) {
+        objRouter.genDoc(req.query['doc']);
+    } else {
+        /**
+         * @type {Controller}
+         */
+        let SomeController = objRouter.getController();
+        let objResponse = new Response(req, res, codeMap);
+
+        if (SomeController) {
+            let objController = new SomeController(req, res);
+            let action = objRouter.getFullActionName();
+            // 先获取get参数，然后获取post参数
+            let args = req.query || req.body || {};
+            if (objController[action]) {
+                objController[action](args);
+            } else {
+                let msg = 'can not find function ' + objRouter.getActionName();
+                objResponse.error(CODE_NOT_EXIST_INTERFACE, msg);
+            }
+        } else {
+            let msg = 'can not find class ' + objRouter.getControllerName();
+            objResponse.error(CODE_NOT_EXIST_INTERFACE, msg);
+        }
+    }
+}
+
+// 捕捉全局异常
+// uncaughtException 避免程序崩溃
+process.on('uncaughtException', function(err) {
+    let CallLog = require('./framework/lib/CallLog.js');
+    // 记录访问日志
+    let objCallLog = new CallLog(null);
+    let msg = err.message && err.message.message;
+    objCallLog.logSelfCall(CODE_UNKNOW_ERROT, msg);
+    setTimeout(function() {
+        process.exit(1);
+    }, 100);
+});
+
+
+module.exports = app;

protected/bin/NameClient/subNsEvent.js

@@ -0,0 +1,200 @@
+require('../../common.js');
+const TableHelper = require('../../framework/lib/TableHelper.js');
+const php = require('phpjs');
+const Tool = require('../../framework/lib/Tool.js');
+const OujRedis = require('../../framework/lib/OujRedis.js');
+const fs = require('fs');
+const Template = require('ejs');
+
+let env = 1;
+if (ENV === ENV_NEW) {
+    env = 2;
+} else if (ENV === ENV_FORMAL) {
+    env = 4;
+} else {
+    env = 1;
+}
+
+for (let name of GLOBALS['nameServ_php']) {
+    _writeConfigFile(env, name, -1, false);
+}
+
+async function _writeConfigFile(env, name, version, notify = true){
+    path = CONF_PATH;
+    if (!fs.existsSync(path)) {
+        fs.mkdir(path);
+    }
+
+    path += `/config.${name}.inc.php`;
+    Tool.log(`准备写入：${name}`);
+
+    let objRedis = OujRedis.init('name_serv');
+    let nsNodes = await objRedis.get(`pub_data:${env}:${name}`);
+    if (!nsNodes) {
+        return false;
+    }
+
+    nsNodes = JSON.parse(nsNodes);
+    let configs = [];
+    for (let i = 0; i < nsNodes.length; i++) {
+        let info = nsNodes[i];
+        let info_name = info['node_name'];
+        info_name = info_name.split(':');
+        if (info['node_type'] === 'hash_table') {
+            let keyName = getArrayKey(info['node_name']);
+            let hashTables = info['items'];
+            let value = [];
+            for (let table of hashTables) {
+                value[table['key_name']] = table['key_value'];
+            }
+            configs['hash'][keyName] = value;
+        } else {
+            configs['string'][info_name[1]] = info['node_value'];
+        }
+    }
+
+    let tmpl = 'js';
+    Template.render('name_server/js')
+    // let template = Template.init();
+    // $template->assign(compact('configs'));
+    // $strConfig = $template->fetch("name_server/{$tmpl}");
+    // if ($tmpl == 'php') {
+    //     $strConfig = "<?php\n" . $strConfig;
+    // }
+    // $now = date('Y-m-d H:i:s');
+    // $result = 1;
+    // if (file_put_contents($path, $strConfig)) {
+    //     var_dump("[{$now}] path:{$path} 生成配置成功");
+    //     ob_flush();
+    // } else {
+    //     $result = -1;
+    //     var_dump("[{$now}] path:{$path} 生成配置失败");
+    //     ob_flush();
+    // }
+
+}
+
+function getArrayKey(keyName) {
+    keyName = keyName.split(':');
+    keyName.shift();
+    let str =  "['" + keyName.join("']['") + "']";
+
+    return str;
+}
+
+//
+// $ip = $serverNodes['server_ip'];
+//
+// $objRedis = dwRedis::init('name_serv');
+// $keys = $objRedis->Keys('_keys*');
+//
+// if ($keys) { // 初始化文件 生成配置文件
+//     foreach ($keys as $key) {
+//         $key = explode(':', trim($key));
+//         if ($key[1]) {
+//             _writeConfigFile($env, trim($key[1]), -1, false);
+//         }
+//     }
+// }
+
+
+// $GLOBALS['targetEnv'] = $env;
+//
+// $serverNodes = getServerConf($env);
+// $ip = $serverNodes['server_ip'];
+//
+// $objRedis = dwRedis::init('name_serv');
+// $keys = $objRedis->Keys('_keys*');
+//
+// if ($keys) { // 初始化文件 生成配置文件
+//     foreach ($keys as $key) {
+//         $key = explode(':', trim($key));
+//         if ($key[1]) {
+//             _writeConfigFile($env, trim($key[1]), -1, false);
+//         }
+//     }
+// }
+//
+// if ($serverNodes['keys']) {
+//     $nodes = explode(',', $serverNodes['keys']);
+//     foreach ($nodes as $node){
+//         $node = trim($node);
+//         $subscribeDatas[] = "pub_event::{$env}:$node";
+//     }
+// }
+//
+// $restartChannel = "pub_restart:{$env}:{$ip}";
+// $echoChannel = "pub_echo:{$env}:{$ip}";
+//
+// $subscribeDatas[] = $restartChannel;
+// $subscribeDatas[] = $echoChannel;
+//
+// if ($subscribeDatas) {
+//
+//     $objRedis->subscribe($subscribeDatas, function ($instance, $channel, $message) use ($restartChannel, $echoChannel) {
+//         if ($channel == $restartChannel) {
+//             var_dump('nameService restart');
+//             exit;
+//         } else if ($channel == $echoChannel) {
+//             notifyPubResult(1, $message);
+//         }
+//
+//         $channels = explode(':', $channel);
+//         $channelName = $channels[3];
+//         _writeConfigFile($GLOBALS['targetEnv'], $channelName, $message);
+//
+//     });
+// }
+//
+// function _writeConfigFile($env, $name, $version, $notify = true){
+//     $path = CONF_PATH;
+//     if(!file_exists($path)){
+//         mkdir($path);
+//     }
+//     $path .= "/config.{$name}.inc.php";
+//     var_dump("准备写入：{$name}");
+//     $objRedis = dwRedis::init('name_serv');
+//
+//     $nsNodes = $objRedis->get("pub_data:{$env}:$name");
+//     if (!$nsNodes) {
+//         return false;
+//     }
+//     $nsNodes = json_decode($nsNodes, true);
+//     $configs = [];
+//     foreach ($nsNodes as $info){
+//         $tmp = [];
+//         $info_name = $info['node_name'];
+//         $info_name = explode(':', $info_name);
+//         if ($info['node_type'] == 'hash_table') {
+//             $keyName = getArrayKey($info['node_name']);
+//             $hashTables = $info['items'];
+//             $value = [];
+//             foreach ($hashTables as $table) {
+//                 $value[$table['key_name']] = $table['key_value'];
+//             }
+//             $configs['hash'][$keyName] = $value;
+//         }else{
+//             $configs['string'][$info_name[1]] = $info['node_value'];
+//         }
+//     }
+//     $tmpl = 'php';
+//     $template = Template::init();
+//     $template->assign(compact('configs'));
+//     $strConfig = $template->fetch("name_server/{$tmpl}");
+//     if ($tmpl == 'php') {
+//         $strConfig = "<?php\n" . $strConfig;
+//     }
+//     $now = date('Y-m-d H:i:s');
+//     $result = 1;
+//     if (file_put_contents($path, $strConfig)) {
+//         var_dump("[{$now}] path:{$path} 生成配置成功");
+//         ob_flush();
+//     } else {
+//         $result = -1;
+//         var_dump("[{$now}] path:{$path} 生成配置失败");
+//         ob_flush();
+//     }
+//     if ($notify) {
+//         notifyPubResult($result, $version);
+//     }
+// }

protected/bin/checkProxyPool.js

@@ -0,0 +1,115 @@
+/**
+ * 定时检查脚本
+ */
+require('../common.js');
+const OujRedis = require('../framework/lib/OujRedis.js');
+const php = require('phpjs');
+const dwHttp = require('../framework/lib/dwHttp');
+const TOTAL_KEY = 'globals:ip_proxy_pool:total';
+const TOTAL_KEY_CHECK = 'globals:ip_proxy_pool:total_check:';
+const objLogic = OujRedis.init('logic');
+const Tool = require('../framework/lib/Tool');
+
+async function main() {
+    let list = await objLogic.zrange(TOTAL_KEY, 0, -1, 'WITHSCORES');
+    for (let i = 0; i < list.length; i += 2) {
+        let proxy = list[i];
+        let score = list[i + 1];
+
+        if (score > 10) {
+            let proxyKey = TOTAL_KEY_CHECK + proxy;
+            let flag = await objLogic.exists(proxyKey);
+            if (flag) {
+                continue;
+            } else {
+                objLogic.setex(proxyKey, 10, php.date('Y-m-d H:i:s'));
+            }
+        }
+
+        requestByProxy(proxy);
+    }
+
+    setTimeout(main, 5000);
+}
+
+async function requestByProxy(proxy) {
+    let request_time = (new Date).getTime();
+    let objHttp = new dwHttp();
+    objHttp.setProxy(proxy);
+    // let url = `http://${server_ip}:${process.env.PORT}/proxy?proxy=${proxy}&request_time=${request_time}`;
+    // let url = `http://localhost:${process.env.PORT}/proxy?proxy=${proxy}&request_time=${request_time}`;
+    let url = `http://14.17.108.216:${process.env.PORT}/proxy?proxy=${proxy}&request_time=${request_time}`;
+    // let url = `http://ka.duowan.com/user/centerRedPoint?_from=ajax`;
+
+    let startTime = (new Date).getTime();
+    objHttp.get(url, 10).then(async function(json) {
+        let incr = 0;
+        let span = (new Date).getTime() - startTime;
+        try {
+            let result = JSON.parse(json);
+            if (result['result']) {
+                if (span < 1000) {
+                    incr = 10;
+                } else if (span < 2000) {
+                    incr = 5;
+                } else if (span < 3000) {
+                    incr = 1;
+                } else if (span > 8000) {
+                    incr = -5;
+                } else if (span > 6000) {
+                    incr = -3;
+                }
+            } else {
+                incr = -20;
+            }
+        } catch (ex) {
+            Tool.error(ex.message);
+            incr = -20;
+        }
+
+        if (incr > 0) {
+            objLogic.zincrby(TOTAL_KEY, incr, proxy);
+        } else if (incr < 0) {
+            let score = await objLogic.zscore(TOTAL_KEY, proxy);
+            await objLogic.zrem(TOTAL_KEY, proxy);
+            score += incr;
+            if (score > 0) {
+                objLogic.zadd(TOTAL_KEY, score, proxy);
+            }
+        }
+        Tool.log(`proxy:${proxy}, span:${span}, incr:${incr}`);
+
+    }).catch(async function(ex) {
+        Tool.error(ex.message);
+        let errArr = [
+            'Error: connect ECONNREFUSED',
+        ];
+
+        for (let err of errArr) {
+            if (ex.message.indexOf(err) >= 0) {
+                // 代理失效要删除
+                objLogic.zrem(TOTAL_KEY, proxy);
+                Tool.log('remove proxy:' + proxy);
+                return;
+            }
+        }
+
+        // 请求超时
+        // if (ex.message.indexOf('Error: ETIMEDOUT') >= 0) {
+        let score = await objLogic.zscore(TOTAL_KEY, proxy);
+        let desc = Math.ceil(parseInt(score) / 2);
+        score = score - desc;
+        if (score > 10) {
+            score = 10;
+        }
+
+        await objLogic.zrem(TOTAL_KEY, proxy);
+        if (score > 0) {
+            await objLogic.zadd(TOTAL_KEY, score, proxy);
+        }
+        Tool.log('catch proxy:' + proxy + ', desc:' + desc);
+        // }
+    });
+}
+
+main();

protected/bin/checkZombieChrome.js

@@ -0,0 +1,61 @@
+/**
+ * 定时检查脚本
+ */
+require('../common.js');
+
+const exec = require('child_process').exec;
+
+killTimeoutChrome();
+
+function killTimeoutChrome() {
+    const cmdStr = 'ps -A -opid,etime,args | grep "/protected/node_modules/puppeteer"';
+    // const cmdStr = 'ps -A -opid,etime,args | grep Chrome';
+    exec(cmdStr, function(err, stdout, stderr){
+        if (err) {
+            console.log('get weather api error:' + stderr);
+        } else {
+            let rows = stdout.split("\n");
+            let index = 1;
+            for (let row of rows) {
+                let data = getPidTime(row);
+                let cmd = '';
+                if (data.time > 60 * 60) {
+                    // 强制kill
+                    cmd = `kill -9 ${data.pid}`;
+                } else if (data.time > 30 * 60) {
+                    // 普通kill
+                    cmd = `kill ${data.pid}`;
+                }
+
+                if (cmd) {
+                    // console.log(`${index}: ${cmd}, time:${data.time}`);
+                    exec(cmd, function(err, stdout, stderr) {
+                        console.log(`${index}: ${cmd}, time:${data.time}, stdout:${stdout}, stderr:${stderr}`);
+                    });
+                    index++;
+                }
+            }
+        }
+    });
+}
+
+function getPidTime(row) {
+    let pid = 0;
+    let time = 0;
+    let parts = row.trim().split(' ');
+    for (let part of parts) {
+        part = part.trim();
+        if (!part) continue;
+        if (!pid) {
+            pid = part;
+        } else if (!time) {
+            let timeParts = part.split(':');
+            for (let j = timeParts.length - 1; j >= 0; j--) {
+                time += timeParts[j] * Math.pow(60, timeParts.length - 1 - j);
+            }
+            break;
+        }
+    }
+
+    return {pid, time};
+}

protected/bin/checkZombieSpider.js

@@ -0,0 +1,210 @@
+#!/usr/bin/env node --harmony --trace_gc
+require('../common.js');
+const TableHelper = require('../framework/lib/TableHelper.js');
+const exec = require('child_process').exec;
+const Tool = require('../framework/lib/Tool.js');
+const php = require('phpjs');
+
+let server_ip = getServerIp();
+checkZombieSpider();
+// checkOutsideZombieSpider();
+deleteLog();
+
+// 查数据库里面的进程
+async function checkZombieSpider() {
+    await checkOutsideZombieSpider();
+
+    let objProcLog = new TableHelper('proc_log', 'crawl');
+    let where = {server_ip};
+    let keyWord = {'_field' : 'proc_id, update_time'};
+    let datas = await objProcLog.getAll(where, keyWord);
+    let len = datas.length;
+    Tool.log(`找到${len}个进程正在运行.`);
+
+    let now = (new Date).getTime();
+    let cmd = '';
+    let num = len + 1;
+    for (let data of datas) {
+        // 检查进程是否存在
+        let proc_id = data.proc_id;
+        cmd = `ps ${proc_id}`;
+        exec(cmd, function(err, stdout, stderr) {
+            let parts = stdout.trim().split('\n');
+            if (parts.length <= 1) {
+                Tool.log(`server_ip: ${server_ip}, 找不到进程${proc_id}，删除数据库数据`);
+                // 找不到进程，需要删除数据库数据
+                objProcLog.delObject({server_ip, proc_id}).then(function() {
+                    checkExit(--num);
+                });
+            } else {
+                // PID TTY      STAT   TIME COMMAND
+                // 7943 ?        Sl     2:03 /usr/local/bin/node /data/webapps/spider.duowan.com/protected/index.js
+                // 判断是否是爬虫进程，不要误杀。。。
+                if (parts[1].indexOf('node ') === -1 || parts[1].indexOf('spider.duowan.com') === -1) {
+                    // 不是node进程
+                    Tool.log('不知道是什么鬼进程：' + parts[1]);
+                    checkExit(--num);
+                    return;
+                }
+
+                let cmd = false;
+                // 检查是否超时
+                let timespan = now - (new Date(data['update_time']));
+                if (timespan > 180 * 1000) {
+                    // 超时心跳要kill掉
+                    cmd = `kill ${data.proc_id}`;
+                } else if (timespan > 240 * 1000) {
+                    cmd = `kill -9 ${data.proc_id}`;
+                }
+
+                if (cmd) {
+                    Tool.log('timespan:' + timespan);
+                    exec(cmd, function(err, stdout, stderr) {
+                        Tool.log(`${cmd}, stdout:${stdout}, stderr:${stderr}`);
+                        checkExit(--num);
+                    });
+                } else {
+                    checkExit(--num);
+                }
+            }
+        });
+    }
+
+    checkExit(--num);
+}
+
+// 检查外部野生的进程
+async function checkOutsideZombieSpider() {
+    let host = 'spider.duowan.com';
+    if (ENV == ENV_DEV) {
+        host = 'test.spider.duowan.com';
+    }
+
+    let cmd = `ps -A -opid,stime,etime,cmd | grep 'node /data/webapps/${host}/protected/bin/crawlWorker.js'`;
+    // let cmd = "ps -A -opid,stime,etime,command | grep 'Google'";
+    return new Promise(function(resolve, reject) {
+        exec(cmd, async function(err, stdout, stderr) {
+            let parts = stdout.trim().split('\n');
+            if (parts.length <= 1) {
+                Tool.log(`server_ip: ${server_ip}, 找不到crawlWorker进程`);
+                resolve(server_ip);
+            } else {
+                // 58521 11:31       07:33 node /data/webapps/spider.duowan.com/protected/bin/crawlWorker.js 29
+                // 58530 11:31       07:33 node /data/webapps/spider.duowan.com/protected/bin/crawlWorker.js 1
+                // 获取这批进程
+                let pids = {};
+                for (let part of parts) {
+                    if (part.indexOf(' grep ') > 0) {
+                        continue;
+                    }
+                    let parts2 = part.trim().split(' ');
+                    let pid = parts2[0];
+                    pids[pid] = part.trim();
+                }
+
+                let objProcLog = new TableHelper('proc_log', 'crawl');
+                let where = {server_ip};
+                let keyWord = {'_field' : 'proc_id'};
+                let datas = await objProcLog.getCol(where, keyWord);
+                for (let pid of datas) {
+                    delete pids[pid];
+                }
+
+                datas = [];
+                let outside_pids = [];
+                for (let pid in pids) {
+                    let data = {server_ip};
+                    data.proc_id = pid;
+                    data.create_time = data.update_time = php.date('Y-m-d H:i:s');
+                    datas.push(data);
+                    outside_pids.push(pid);
+                    Tool.log('流浪进程：' + pids[pid]);
+                }
+
+                if (datas.length) {
+                    await objProcLog.addObjects2(datas);
+                    Tool.log('checkOutsideZombieSpider, 添加了：' + outside_pids.join(','));
+                }
+
+                resolve(datas);
+            }
+        });
+    });
+}
+
+
+
+async function moveLog(times) {
+    times = times || 0;
+    let time = php.time() - 2 * 86400;
+    let tableName = 'crawl_log_' + php.date('Ymd', time);
+    let objCrawlLog2 = new TableHelper(tableName, 'crawl');
+    let flag = await objCrawlLog2.checkTableExist(tableName);
+    if (!flag) {
+        let sql = `CREATE TABLE ${tableName} LIKE _crawl_log`;
+        await objCrawlLog2.objMySql.update(sql);
+    }
+
+    let objCrawlLog = new TableHelper('crawl_log', 'crawl');
+    let time2  = php.time() - 86400;
+    let datas = await objCrawlLog.getAll({}, {'_where' : "create_time < '" + php.date('Y-m-d', time2) + "'", '_limit' : 100 });
+    if (!datas) return;
+
+    await objCrawlLog2.replaceObjects2(datas);
+
+    let log_id = [];
+    for (let data of datas) {
+        log_id.push(data.log_id);
+    }
+
+    if (log_id.length) {
+        times++;
+        await objCrawlLog.delObject({log_id});
+        if (log_id.length >= 100 && times < 10) {
+            await moveLog(times);
+        }
+    }
+}
+
+
+async function checkExit(num) {
+    // Tool.log('checkExit:' + num);
+    if (num <= 0) {
+        await moveLog();
+        process.exit(0);
+    }
+}
+
+
+async function deleteLog() {
+    let objCrawlLog = new TableHelper('crawl_log', 'crawl');
+    var sql = "show tables like '%crawl_log%'";
+    let tables = await objCrawlLog.objMySql.update(sql);
+    tables.forEach(async table => {
+        var table_name = table["Tables_in_crawl (%crawl_log%)"];
+        var m = table_name.match(/^crawl_log_(\d+)$/);
+        if (m) {
+            //获取日志日期期限
+            var now = new Date();//获取当前时间
+            var nowMs = now.getTime();//获取当前时间的毫秒数 
+            var beforeMs =  nowMs -  1000 * 60 * 60 * 24 * parseInt(7);//前几天，n就取几，整数 
+            var beforeDate = new Date().setTime(beforeMs);
+            beforeDate = new Date(beforeDate);
+            var year = beforeDate.getFullYear();
+            var month = (beforeDate.getMonth() + 1);
+            var day = beforeDate.getDate();
+
+            if (month < 10) {
+                month = '0'+month;
+            }
+            if (day < 10) {
+                day = '0'+day;
+            }
+            beforeDate = year + '' + month + '' + day;
+            if (m[1] && parseInt(beforeDate) > parseInt(m[1])) {
+                var sql = `drop table ${table_name}`;
+                await objCrawlLog.objMySql.update(sql);
+            }   
+        }
+    });
+}

protected/bin/crawl.js

@@ -0,0 +1,126 @@
+/**
+ * Created by ben on 2017/10/24.
+ */
+require('../common.js');
+
+let php = require('phpjs');
+// let cheerio = require('cheerio');
+let Tool = require('../framework/lib/Tool.js');
+let Spider = require('../models/Spider.js');
+
+// const puppeteer = require('puppeteer');
+const TableHelper = require('../framework/lib/TableHelper.js');
+
+// const STATE_EXECING = 0;
+// const STATE_SUCC = 1;
+// const STATE_ERROR = 2;
+// const STATE_TIMEOUT = 3;
+let args = process.argv.splice(2);
+let proc_index = args[0] || 0;
+let proc_num = args[1] || 1;
+let server_ip = getServerIp();
+let proc_id = process.pid;
+
+async function run(cb) {
+    // let startTime = (new Date).getTime();
+    let task = null;
+    if (proc_index === 0) {
+        task = await getTask('headless');
+    }
+
+    if (!task) {
+        task = await getTask('normal');
+    }
+
+    let flag = false;
+    if (task) {
+        let objSpider = new Spider(task);
+        await objSpider.run();
+        flag = true;
+    } else {
+        Tool.log(`no tasks need run.`);
+    }
+
+    logProc(task);
+
+    cb && cb(flag);
+}
+
+async function logProc(task) {
+    task = task || {};
+    let objProcLog = new TableHelper('proc_log', 'crawl');
+    let where = {server_ip, proc_id};
+    let count = await objProcLog.getCount(where);
+    let now = php.date('Y-m-d H:i:s');
+    let data = {
+        task_id : task.task_id || 0,
+        rule_id : task.rule_id || 0,
+        url : task.url || '',
+        update_time : now
+    };
+
+    if (count > 0) {
+        objProcLog.updateObject(data, where);
+    } else {
+        let data2 = {
+            server_ip,
+            proc_id,
+            create_time : now,
+        };
+        objProcLog.addObject(Object.assign(data, data2));
+    }
+}
+
+async function getTask(request_mode) {
+    const objRule = new TableHelper('rule', 'crawl');
+    let _field = 'rule_id';
+    let enable = 1;
+    let rule_ids = await objRule.getCol({enable, request_mode}, {_field});
+
+    let len = rule_ids.length;
+    let tempRule_id = [];
+    if (proc_num > len) {
+        tempRule_id.push(rule_ids[proc_index % len]);
+    } else {
+        for (let i = 0; i < len; i++) {
+            if (i % proc_index) {
+                tempRule_id.push(rule_ids[i]);
+            }
+        }
+    }
+
+    let task = null;
+    if (tempRule_id.length) {
+        task = await tryGetTask(tempRule_id);
+    }
+
+    if (!task) {
+        task = await tryGetTask(rule_ids);
+    }
+
+    return task;
+}
+
+async function tryGetTask(rule_id) {
+    const objTask = new TableHelper('task', 'crawl');
+    let last_crawl_time = php.time();
+    let _where = `next_crawl_time < ${last_crawl_time} `;
+    if (php.empty(rule_id)) {
+        return null;
+    }
+
+    let enable = 1;
+    return await objTask.getRow({enable, rule_id}, {_where});
+}
+
+setTimeout(function() {
+    run(function _callback(flag) {
+        let timeout = flag ? php.rand(100, 500) : 3000;
+        setTimeout(async function() {
+            await run(_callback);
+        }, timeout);
+    });
+}, proc_index * 271);
+
+
+

protected/bin/crawlMaster.js

@@ -0,0 +1,98 @@
+/**
+ * 爬虫Master，定时发布任务
+ */
+require('../common.js');
+
+let php = require('phpjs');
+// let cheerio = require('cheerio');
+let Tool = require('../framework/lib/Tool.js');
+
+// const puppeteer = require('puppeteer');
+const TableHelper = require('../framework/lib/TableHelper.js');
+const OujRedis  = require('../framework/lib/OujRedis');
+
+const objRule = new TableHelper('rule', 'crawl');
+const objTask = new TableHelper('task', 'crawl');
+const objRedis = OujRedis.init('logic');
+const objProcLog = new TableHelper('proc_log', 'crawl');
+const NUM = 30;
+const FIRST_MAX_NUM = 5;
+
+
+async function run(cb) {
+    let flag = false;
+    const key = 'globals:crawl';
+    let len = await objRedis.scard(key);
+    if (len < NUM * 4) {
+        let task_id = await getTasks();
+        if (task_id && task_id.length) {
+            Tool.log(`${key} lpush task_id num:` + task_id.length);
+            await objRedis.sadd(key, task_id);
+            flag = task_id.length >= NUM;
+
+            let next_crawl_time = php.time() + 60;
+            objTask.updateObject({next_crawl_time}, {task_id})
+        }
+    }
+
+
+    cb && cb(flag);
+}
+
+async function getTasks() {
+    let _field = 'rule_id';
+    let enable = 0;
+    let exclude_rule_id = await objRule.getCol({enable}, {_field});
+    exclude_rule_id = exclude_rule_id || [];
+    Tool.log(`exclude_rule_id:` + exclude_rule_id.length);
+
+    // 排除正在执行的任务数>=2的规则
+    let sql = 'SELECT rule_id FROM `proc_log` GROUP BY rule_id HAVING rule_id != "0" AND  COUNT(*) >= ' + FIRST_MAX_NUM;
+    let exclude_rule_id2 = await objProcLog.objMySql.getCol(sql);
+
+    let tasks = null;
+    if (exclude_rule_id2 && exclude_rule_id2.length) {
+        exclude_rule_id2 = exclude_rule_id2.concat(exclude_rule_id);
+        let tasks = await tryGetTasks(exclude_rule_id2);
+        if (tasks && tasks.length > NUM / 2) {
+            return tasks;
+        }
+    }
+
+    let tasks2 = await tryGetTasks(exclude_rule_id);
+    if (tasks && tasks2) {
+        return tasks2.concat(tasks);
+    } else if (tasks) {
+        return tasks;
+    } else {
+        return tasks2;
+    }
+}
+
+async function tryGetTasks(exclude_rule_id) {
+    let last_crawl_time = php.time();
+    let _where = `next_crawl_time < ${last_crawl_time} `;
+    if (exclude_rule_id && exclude_rule_id.length) {
+        _where += "AND rule_id NOT IN('" + exclude_rule_id.join("', '") + "')";
+    }
+
+    let _field = 'task_id';
+    let enable = 1;
+    let _sortKey = 'next_crawl_time ASC';
+    return await objTask.getCol({enable}, {_where, _field, _sortKey, _limit:NUM});
+}
+
+let i = 0;
+run(function _callback(flag) {
+    let timeout = flag ? 1000 : 10000;
+    if (i++ < 600) {
+        setTimeout(async function() {
+            await run(_callback);
+        }, timeout);
+    } else {
+        process.exit();
+    }
+});
+
+
+

protected/bin/crawlWorker.js

@@ -0,0 +1,123 @@
+/**
+ * 爬虫工人
+ */
+require('../common.js');
+
+const php = require('phpjs');
+// let cheerio = require('cheerio');
+const Tool = require('../framework/lib/Tool.js');
+const Spider = require('../models/Spider.js');
+
+const TableHelper = require('../framework/lib/TableHelper.js');
+const OujRedis  = require('../framework/lib/OujRedis');
+
+const objTask = new TableHelper('task', 'crawl');
+const objRedis = OujRedis.init('logic');
+const objProcLog = new TableHelper('proc_log', 'crawl');
+const HOST_MAX_NUM = DEBUG ? 1 : 20;
+const RULE_MAX_NUM = DEBUG ? 1 : 8;
+const { URL } = require('url');
+
+let args = process.argv.splice(2);
+let proc_index = args[0] || 0;
+// let proc_num = args[1] || 1;
+let server_ip = getServerIp();
+let proc_id = process.pid;
+
+async function run(cb) {
+    // let startTime = (new Date).getTime();
+    let task = await getTask();
+    await logProc(task);
+    let flag = false;
+    if (task) {
+        let objSpider = new Spider(task);
+        await objSpider.run();
+        flag = true;
+    } else {
+        Tool.log(`no tasks need run.`);
+    }
+    await logProc();
+
+    cb && cb(flag);
+}
+
+async function logProc(task) {
+    task = task || {};
+    let where = {server_ip, proc_id};
+    let count = await objProcLog.getCount(where);
+    let now = php.date('Y-m-d H:i:s');
+
+    let host = '';
+    if (task.url) {
+        const taskUrl = new URL(task.url);
+        host = taskUrl.host;
+    }
+
+    let data = {
+        task_id : task.task_id || 0,
+        rule_id : task.rule_id || 0,
+        host : host,
+        url : task.url || '',
+        update_time : now
+    };
+
+    if (count > 0) {
+        await objProcLog.updateObject(data, where);
+    } else {
+        let data2 = {
+            server_ip,
+            proc_id,
+            create_time : now,
+        };
+        await objProcLog.addObject(Object.assign(data, data2));
+    }
+}
+
+async function getTask() {
+    const key = 'globals:crawl';
+    let task_id = await objRedis.spop(key);
+    if (!task_id) {
+        return false;
+    }
+
+    let task = await objTask.getRow({task_id});
+
+    let ruleCount = await objProcLog.getCount({rule_id:task.rule_id});
+    let flag = ruleCount > RULE_MAX_NUM;
+    if (!flag) {
+        const taskUrl = new URL(task.url);
+        let hostCount = await objProcLog.getCount({host:taskUrl.host});
+        flag = hostCount > HOST_MAX_NUM;
+        if (flag) {
+            Tool.log(`Host, 当前数量：${hostCount}, 超过${taskUrl.host} 的最大限制：${HOST_MAX_NUM}`);
+        }
+    } else {
+        Tool.log(`Rule, 当前数量：${ruleCount}, 超过${task.rule_id} 的最大限制：${RULE_MAX_NUM}`);
+    }
+
+    if (flag) {
+        // 任务加回去
+        await objRedis.sadd(key, task_id);
+        return false;
+    } else {
+        return task;
+    }
+}
+
+let i = 0;
+setTimeout(async function() {
+    if (i++ < 1000) {
+        run(function _callback(flag) {
+            let timeout = flag ? 100 : php.rand(5000, 10000);
+            setTimeout(async function() {
+                await run(_callback);
+            }, timeout);
+        });
+    } else {
+        process.exit();
+    }
+}, 171 * proc_index);
+
+
+
+

protected/bin/fetchPage.js

@@ -0,0 +1,51 @@
+/**
+ * Created by ben on 2017/10/24.
+ */
+require('../common.js');
+const TableHelper = require('../framework/lib/TableHelper.js');
+const Spider = require('../models/Spider.js');
+
+async function main(rule_id, cb) {
+    const objRule = new TableHelper('rule', 'crawl');
+    let rule = await objRule.getRow({rule_id});
+
+    let task = {};
+    task['task_id'] = -1;
+    task['task_key'] = -1;
+    task['rule_id'] = rule_id;
+    task['url'] = rule['demo_url'];
+    // task['url'] = 'http://apps.game.qq.com/wmp/v3.1/?p0=42&p1=searchKeywordsList&page=1&pagesize=15&order=sIdxTime&r0=s';
+    // task['url'] = 'https://v.qq.com/x/page/a0552az0da5.html';
+    // task['url'] = 'https://steamdb.info/app/762520/';
+    task['url'] = 'https://www.taptap.com/app/94706 ';
+    let objSpider = new Spider(task);
+    // let content = await objSpider.run(true);
+    let content = await objSpider.run();
+
+    cb && cb(content);
+}
+
+
+let args = process.argv.splice(2);
+// let rule_id = args[0] || "steam:game_player_data";
+// let rule_id = args[0] || "steam:discount_info:base";
+// let rule_id = args[0] || "steam:discount_detail";
+// let rule_id = args[0] || "ka:2002363:video_list";
+// let rule_id = args[0] || "ka:2002363:video_detail";
+// let rule_id = args[0] || "steam:discount_detail";
+let rule_id = args[0] || "sy:taptap:new_game_detail";
+// let rule_id = args[0] || "steam:game_dlc";
+// let rule_id = args[0] || "steam:game_data:steamdb";
+// let rule_id = args[0] || "steam:game_data:yesterday_increase";
+// let rule_id = args[0] || "steam:game_detail";
+// let rule_id = args[0] || "steam:xiaoheihe_api";
+
+if (!rule_id) {
+    console.error('rule_id');
+    return;
+}
+
+main(rule_id, function(content) {
+    console.log(content);
+    process.exit(0);
+});

protected/bin/linux_bash/crontab.sh

@@ -0,0 +1,4 @@
+
+*/1 * * * * root echo "[`date +"\%F \%T"`] `node /data/webapps/spider.duowan.com/protected/bin/checkZombieChrome.js`" >> /tmp/checkZombieChrome.log &
+
+*/1 * * * * root echo "[`date +"\%F \%T"`] `node /data/webapps/spider.duowan.com/protected/bin/checkZombieSpider.js`" >> /tmp/checkZombieSpider.log &

protected/bin/linux_bash/supervisor.ini

@@ -0,0 +1,18 @@
+
+[program:node5]
+command=node /data/webapps/spider.duowan.com/protected/bin/checkProxyPool.js
+process_name=checkProxyPool
+directory=/data/webapps/spider.duowan.com/protected/bin/
+numprocs=1
+autostart=true
+autorestart=true
+stdout_logfile=/tmp/checkProxyPool.log
+
+[program:node6]
+command=node /data/webapps/spider.duowan.com/protected/bin/crawl.js
+process_name=spider_crawl
+directory=/data/webapps/spider.duowan.com/protected/
+numprocs=1
+autostart=true
+autorestart=true
+stdout_logfile=/tmp/spider_crawl.log

protected/bin/test.js

@@ -0,0 +1,13 @@
+//test
+
+require('../common.js');
+
+
+new Promise(async () => {
+	const puppeteer = require('puppeteer');
+	const browser = await puppeteer.launch({args: ['--no-sandbox',
+    '--disable-setuid-sandbox']});
+	console.log(browser.process());
+	process.exit();
+});
+

protected/common.js

@@ -0,0 +1,44 @@
+
+require('./extensions/function_extend.js');
+
+global['DEFAULT_CHARSET'] = 'utf-8';
+
+global['BASE_DIR'] = __dirname + '/../';
+global['ROOT_PATH'] = __dirname + '/';
+
+global['ENV_LOCAL'] = 'local';
+global['ENV_DEV'] = 'dev';
+global['ENV_FORMAL'] = 'form';
+global['ENV_NEW'] = 'new';
+
+let fileName = __filename;
+let env = ENV_DEV;
+if (fileName.indexOf('/data/webapps/') === 0) {
+    if (fileName.indexOf('/test.') !== -1 || fileName.indexOf('/test-') !== -1 || fileName.indexOf('_test/') !== -1) {
+        global['CONF_PATH'] = '/data/webapps/conf_v2/test/';
+        global['FRAMEWORK_PATH'] = '/data/webapps/framework/nodebase2_test/';
+        env = ENV_DEV;
+    } else if (fileName.indexOf('/new.') !== -1 || fileName.indexOf('/new-') !== -1 || fileName.indexOf('_new/') !== -1) {
+        global['CONF_PATH'] = '/data/webapps/conf_v2/new/';
+        global['FRAMEWORK_PATH'] = '/data/webapps/framework/nodebase2_new/';
+        env = ENV_NEW;
+    } else {
+        global['CONF_PATH'] = '/data/webapps/conf_v2/form/';
+        global['FRAMEWORK_PATH'] = '/data/webapps/framework/nodebase2/';
+        env = ENV_FORMAL;
+    }
+} else if (fileName.indexOf('/data_dev/') === 0 ) {
+    // 内网开发环境
+    global['CONF_PATH'] = '/data/webapps/conf_v2/test/';
+    global['FRAMEWORK_PATH'] = '/data/webapps/framework/nodebase2_test/';
+    env = ENV_DEV;
+} else {
+    // 本地环境
+    global['CONF_PATH'] = ROOT_PATH + '/conf/conf_ns/';
+    global['FRAMEWORK_PATH'] = ROOT_PATH + '../../framework/nodebase2/';
+    env = ENV_DEV;
+}
+global['ENV'] = env;
+
+// 公共配置文件
+global['GLOBALS'] = require(`${ROOT_PATH}/conf/config.inc.js`);

protected/conf/code.inc.js

@@ -0,0 +1,47 @@
+/**
+ * Created by benzhan on 15/8/15.
+ */
+
+//常规错误码
+global["CODE_SUCCESS"] = 0;
+global["CODE_UNKNOW_ERROT"] = -1;
+global["CODE_NOT_EXIST_INTERFACE"] = -2;
+global["CODE_PARAM_ERROR"] = -3;
+global["CODE_INTER_ERROR"] = -4;
+global["CODE_USER_LOGIN_FAIL"] = -5;
+global["CODE_SIGN_ERROR"] = -6;
+global["CODE_UNLOGIN_ERROR"] = -7;
+global["CODE_NO_PERMITION"] = -8;
+global["CODE_NORMAL_ERROR"] = -9;
+global["CODE_DB_ERROR"] = -10;
+global["CODE_REQUEST_TIMEOUT"] = -11;
+global["CODE_REQUEST_ERROR"] = -12;
+global["CODE_REDIS_ERROR"] = -13;
+global["CODE_UNAUTH_ERROR"] = -14;
+
+
+var code_map = {
+    // 常规错误码
+    CODE_SUCCESS: '成功',
+    CODE_UNKNOW_ERROT: '未知错误', // 这个需要告警的错误码
+    CODE_NOT_EXIST_INTERFACE: '接口不存在',
+    CODE_PARAM_ERROR: '参数错误',
+    CODE_INTER_ERROR: '系统繁忙，请稍后再试', // http请求返回错误
+    CODE_USER_LOGIN_FAIL: '登录态失效，请重新登录',
+    CODE_SIGN_ERROR: '签名错误',
+    CODE_UNLOGIN_ERROR: '没有登录',
+    CODE_NO_PERMITION: '没有权限',
+    CODE_NORMAL_ERROR: '常规错误',
+    CODE_DB_ERROR: '系统繁忙，请稍后再试', // 这个需要告警的错误码
+    CODE_REQUEST_TIMEOUT: '网络请求超时，请重试',
+    CODE_REQUEST_ERROR: '访问外部接口出错，请稍后重试', // 这个需要告警的错误码
+    CODE_REDIS_ERROR: '系统繁忙，缓存出错，请稍后再试', // 这个需要告警的错误码
+    CODE_UNAUTH_ERROR: '未授权',
+};
+
+var map = {};
+for (var str in code_map) {
+    map[global[str]] = code_map[str];
+}
+
+module.exports = map;

protected/conf/conf_ns/config.code.inc.js

@@ -0,0 +1,40 @@
+// 常量
+global["CODE_301_REDIRECT"] = -16;
+global["CODE_DB_ERROR"] = -10;
+global["CODE_INTER_ERROR"] = -4;
+global["CODE_IP_LIMITED"] = -15;
+global["CODE_KA_GOLD_ERROR"] = -8004;
+global["CODE_KA_LINQU_ERROR"] = -8002;
+global["CODE_KA_LOTTERY_ERROR"] = -8005;
+global["CODE_NEED_CAPTCHA"] = -21;
+global["CODE_NEED_CHECK_ANOTHER_PWD"] = -1001;
+global["CODE_NEED_SET_ANOTHER_PWD"] = -1000;
+global["CODE_NEED_VERIFY"] = -1002;
+global["CODE_NORMAL_ERROR"] = -9;
+global["CODE_NOT_ENGOUGH_PRODUCT"] = -4000;
+global["CODE_NOT_EXIST_CART"] = -4004;
+global["CODE_NOT_EXIST_INTERFACE"] = -2;
+global["CODE_NOT_EXIST_NODE"] = -1002;
+global["CODE_NO_PERMITION"] = -8;
+global["CODE_ORDER_ERROR_FEE"] = -5001;
+global["CODE_PARAM_ERROR"] = -3;
+global["CODE_PRODUCT_EMPTY_ERROR"] = -4002;
+global["CODE_PRODUCT_ERROR_FEE"] = -4001;
+global["CODE_QUEUE_REQUEST"] = -18;
+global["CODE_REDIS_ERROR"] = -13;
+global["CODE_REPEAT_REQUEST"] = -17;
+global["CODE_REQUEST_ERROR"] = -12;
+global["CODE_REQUEST_TIMEOUT"] = -11;
+global["CODE_SIGN_ERROR"] = -6;
+global["CODE_SUCCESS"] = 0;
+global["CODE_UNAUTH_ERROR"] = -14;
+global["CODE_UNKNOW_ERROT"] = -1;
+global["CODE_UNLOGIN_ERROR"] = -7;
+global["CODE_USER_LIMITED"] = -19;
+global["CODE_USER_LOGIN_FAIL"] = -5;
+
+// 数组
+ 
+var code_map = {};
+code_map = {"-16":"正在跳转中...","-10":"系统繁忙，请稍后再试","-4":"系统繁忙，请稍后再试","-15":"IP受限","-8004":"钻石不够","-8002":"礼包领取失败","-8005":"抽奖失败，请重试！","-21":"请输入验证码","-1001":"需要验证二次密码","-1000":"需要设置二次密码","-1002":"节点不存在","-9":"常规错误","-4000":"商品库存不足","-4004":"该商品已下架","-2":"接口不存在","-8":"没有权限","-5001":"总价不正确","-3":"参数错误","-4002":"商品为空","-4001":"商品价格变动","-18":"排队中","-13":"系统繁忙，缓存错误，请稍后再试","-17":"重复请求","-12":"访问外部接口出错，请稍后重试","-11":"网络请求超时，请重试","-6":"签名错误","0":"成功","-14":"未授权","-1":"未知错误","-7":"没有登录","-19":"用户受限","-5":"登录态失效，请重新登录"};
+exports["code_map"] = code_map;

protected/conf/conf_ns/config.globals.inc.js

@@ -0,0 +1,255 @@
+// 常量
+global["BAIDU_AK"] = 'E3ZEgAwYfHrQvGjBFFonfG7m';
+global["BS2_ACCESS_KEY"] = 'ak_uiz';
+global["BS2_ACCESS_SECRET"] = 'fd855479b53b6632204c8d0f0b8ed47428265bda';
+global["BS2_AUDIO_BUCKET"] = 'ojiastoreaudio';
+global["BS2_DEL_HOST"] = 'bs2.yy.com';
+global["BS2_DL_HOST"] = 'bs2dl.huanjuyun.com';
+global["BS2_FILE_BUCKET"] = 'ojiastoreimage';
+global["BS2_HOST"] = 'bs2ul.yy.com';
+global["BS2_LARGE_FILE_BUCKET"] = 'ojiaauthorvideos';
+global["BS2_SNS_BUCKET"] = 'ojiasnsimage';
+global["BS2_VIDEO_BUCKET"] = 'ojiastorevideos';
+global["CJMS_HOST"] = 'test.admin.ouj.com';
+global["CJMS_HOST_DW"] = 'test.admin.duowan.com';
+global["CJMS_IP"] = '61.160.36.225';
+global["COOKIE_DOMAIN_DUOWAN"] = 'duowan.com';
+global["COOKIE_DOMAIN_HIYD"] = 'hiyd.com';
+global["COOKIE_DOMAIN_KU_HIYD"] = 'ku.hiyd.com';
+global["COOKIE_DOMAIN_OUJ"] = 'ouj.com';
+global["DOMAIN_ADMIN_OUJ"] = 'test.admin.ouj.com';
+global["EXCHANGE_GOLD_NUM_LIMIT"] = 50;
+global["HIYD_QQ_APP_ID"] = 101218804;
+global["HIYD_QQ_APP_SECRE"] = 'c298fb6dd6857c6541dff61ed7370265';
+global["IP_ADMIN_OUJ"] = '61.160.36.225';
+global["KUAIDI_CALLBACK"] = 'http://test.m.shop.hiyd.com/kuaidi/callback';
+global["KUAIDI_CALLBACK_DW"] = 'http://test.hezi.plus.duowan.com/kuaidi/callback';
+global["MAX_DEVICETOKEN_GEN"] = 1;
+global["NODE_ID_HIYD"] = 118;
+global["REDIS_ACCESSCODE_PREFIX"] = 'ACCESS_CODE:';
+global["REDIS_KEY_ARTICLE_CLICK_RANK"] = 'globals:article_click_rank:';
+global["REDIS_KEY_DEVICETOKEN_ALL"] = 'device_token:all:';
+global["REDIS_KEY_DEVICETOKEN_STORE"] = 'device_token:store:';
+global["REDIS_KEY_DW_BLACK_IPS"] = 'globals:dw_black_ips';
+global["REDIS_KEY_HEALTH_REPORT_DATA"] = 'globals:health_report_data';
+global["REDIS_KEY_HEALTH_REPORT_IMG"] = 'globals:health_report_img';
+global["REDIS_KEY_HEZI_BLACK_IPS"] = 'globals:hezi_black_ips';
+global["REDIS_KEY_HEZI_BLACK_USERS"] = 'globals:hezi_black_users';
+global["REDIS_KEY_HEZI_IPPVS"] = 'globals:hezi_ippvs';
+global["REDIS_KEY_KA_RECEIVE_QUEUE"] = 'globals:ka_receive_queue:';
+global["REDIS_KEY_ORDERID_POOL"] = 'global:orderid_pool';
+global["REDIS_KEY_RECHARGE_SHIPPING"] = 'globals:recharge_shipping';
+global["REDIS_KEY_STATIC_ARTICLE_VISIT"] = 'globals:static_article_visit_count:';
+global["REDIS_KEY_STATIC_IPTASKS"] = 'globals:static_iptasks';
+global["REDIS_KEY_STATIC_IPUSERS"] = 'globals:static_ipusers';
+global["REDIS_KEY_STATIC_TASK_BOOK_GAME"] = 'globals:static_task_book_game';
+global["REDIS_KEY_STATIC_TASK_GIFT_ACTIVE"] = 'globals:static_task_gift_active:';
+global["REDIS_KEY_STATIC_TASK_REPORTS"] = 'globals:static_task_reports:';
+global["REDIS_KEY_STATIC_USERGOLD"] = 'globals:static_usergold:';
+global["REDIS_KEY_TASK_IPLIMITS"] = 200;
+global["REDIS_KEY_TASK_IPUSERLIMITS"] = 2;
+global["REDIS_KEY_USER_ACCESS_MAP"] = 'global:user_access_map';
+global["REDIS_PRE_KEY_DEVICE_SESSION"] = 'Device_Session:';
+global["REDIS_PRE_KEY_GOODS_ACCESS"] = 'goods_access:';
+global["REDIS_PRE_KEY_ORDER_RANK"] = 'ORDERRANKED:';
+global["REDIS_PRE_KEY_SESSION"] = 'User_Session:';
+global["REDIS_PRE_KEY_WXJSSING2"] = 'wx_jssign2_oauth:';
+global["REDIS_PRE_KEY_WX_UNIONID"] = 'globals:wx_unionid:';
+global["REDIS_RRE_KEY_LOGIN_URL"] = 'globals:login_url:';
+global["REDIS_RRE_KEY_LOTTERY_REWARD_STOCK"] = 'globals:lottery_reward_stock:';
+global["SHOP_APP_ID"] = 2;
+global["SHOP_APP_SECRET"] = 'Kvg!F:#X80r5.p9F';
+global["SNS_APP_ID"] = 1;
+global["SNS_APP_SECRET"] = '$DWEeMLJMj9Xl2^4';
+global["SNS_HOST"] = 'test.sns.admin.ouj.com';
+global["SNS_IP"] = '61.160.36.225';
+global["SNS_TEST_TOKEN"] = 'lucifer_test_token';
+global["SNS_TEST_UID"] = 1;
+global["URL_ADMIN_DUOWAN"] = 'http://test.admin.duowan.com/';
+global["URL_ADMIN_OUJ"] = 'http://test.admin.ouj.com/';
+global["URL_DW_RECHARGE_CALLBACK"] = 'http://test.plus.duowan.com/recharge/callback';
+global["URL_FOOD_HIYD"] = 'http://test.food.hiyd.com/';
+global["URL_H5GAME"] = 'http://test.h5game.5253.com/';
+global["URL_HIYD"] = 'http://test.www.hiyd.com/';
+global["URL_KA"] = 'http://test.ka.duowan.com/';
+global["URL_KA_HOST"] = 'test.ka.duowan.com';
+global["URL_KA_HOST_M"] = 'test.mka.duowan.com';
+global["URL_KA_M"] = 'http://test.mka.duowan.com/';
+global["URL_KU_HIYD"] = 'http://test.ku.hiyd.com/';
+global["URL_MOBILE_SHOP"] = 'http://test.m.meals.hiyd.com/';
+global["URL_M_FOOD_HIYD"] = 'http://test.m.food.hiyd.com/';
+global["URL_M_HIYD"] = 'http://test.m.hiyd.com/';
+global["URL_M_OUJ"] = 'http://test.m.ouj.com/';
+global["URL_PAY_API"] = 'http://test.pay.api.oxzj.net/';
+global["URL_SHOP_API"] = 'http://test.m.meals.hiyd.com/';
+global["URL_SHOP_API2"] = 'http://test.m.shop.hiyd.com/';
+global["URL_SHOP_API3"] = 'http://test.m.fit.hiyd.com/';
+global["URL_SHOP_API_DW"] = 'http://test.plus.duowan.com/';
+global["URL_SHOP_API_HZ"] = 'http://test.hezi.plus.duowan.com/';
+global["URL_SHOP_API_KA"] = 'http://test.kaplus.duowan.com/';
+global["URL_SNS_API"] = 'http://test.api.oxzj.net/';
+global["URL_SY"] = 'http://test.sy.duowan.com/';
+global["URL_SY_M"] = 'http://test.sy.duowan.cn/';
+global["URL_USER_API"] = 'http://test.user.api.oxzj.net/';
+global["URL_USER_ORDER"] = 'http://shop.ouj.com/main/#!page=order-review&env=test&order_id=';
+global["URL_WEB_SHOP"] = 'http://test.meals.hiyd.com/';
+global["WX_APP_ID"] = 'wx892910db2cdd2c12';
+global["WX_APP_ID_DW"] = 'wxb25813d7b9e2147b';
+global["WX_APP_ID_FIT"] = 'wxa46e6794bc38e236';
+global["WX_APP_ID_MEAL"] = 'wx489f3e891508cf09';
+global["WX_APP_ID_MEAL_v1"] = 'wxa46e6794bc38e236';
+global["WX_APP_ID_OUJ"] = 'wx892910db2cdd2c12';
+global["WX_APP_SECRE"] = 'eed2932f5a29b4f12d2607aa77548949';
+global["WX_APP_SECRE_DW"] = '51710497377c2bf56e00fe38b161b14d';
+global["WX_APP_SECRE_FIT"] = 'cf876b3bbae4b8b2bdb03570c391964f';
+global["WX_APP_SECRE_MEAL"] = 'fea0ce52a21fb194422277010cb42580';
+global["WX_APP_SECRE_MEAL_v1"] = 'cf876b3bbae4b8b2bdb03570c391964f';
+global["WX_APP_SECRE_OUJ"] = 'eed2932f5a29b4f12d2607aa77548949';
+
+// 数组
+ 
+var REDIS_PRE_KEY_QQ_OPENID = {};
+REDIS_PRE_KEY_QQ_OPENID = []; 
+var URL_SHOP_API_MKA = {};
+URL_SHOP_API_MKA = []; 
+var dbInfo = {};
+dbInfo[" gamecenter_opt"] = {"dbHost":"172.27.203.6","dbName":"gamecenter_opt","dbPass":"yxtq@duowan@$","dbPort":"3306","dbType":"mysqli","dbUser":"yxtq","enable":"true"};
+
+//     host : '10.25.68.139',
+//     user : 'ojuser',
+//     password : 'qs45MWKf',
+//     database : 'Web',
+//     port : 6308,
+//     connectionLimit : 100
+
+dbInfo["5253_tq"] = {"dbHost":"172.27.203.6","dbName":"db_groundhog_new","dbPass":"yxtq@duowan@$","dbPort":"3306","dbType":"mysqli","dbUser":"yxtq","enable":"true"}; 
+dbInfo["adsystem"] = {"dbHost":"61.160.36.225","dbName":"adsystem","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["base_db"] = {"dbHost":"61.160.36.225","dbName":"base_db","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["bbs_db"] = {"dbHost":"61.160.36.225","dbName":"bbs_db","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["book_db"] = {"dbHost":"61.160.36.225","dbName":"book_db","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["crawl"] = {"dbHost":"61.160.36.225","dbName":"crawl","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["crka"] = {"dbHost":"61.160.36.225","dbName":"crka","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["default"] = {"dbHost":"61.160.36.225","dbName":"Web","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["dw_game"] = {"dbHost":"61.160.36.225","dbName":"dw_game","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["dw_game_sync"] = {"dbHost":"172.28.19.250","dbName":"test","dbPass":"boke45731","dbPort":"6306","dbType":"mysqli","dbUser":"boketest","enable":"true"}; 
+dbInfo["dw_ka"] = {"dbHost":"61.160.36.225","dbName":"dw_ka","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["dw_ka_book"] = {"dbHost":"61.160.36.225","dbName":"dw_ka_book","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["dw_ka_book_master2"] = {"dbHost":"61.160.36.225","dbName":"dw_ka_book","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["dw_ka_data"] = {"dbHost":"61.160.36.225","dbName":"dw_ka_data","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["dw_ka_master2"] = {"dbHost":"61.160.36.225","dbName":"dw_ka","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["dw_ka_sn"] = {"dbHost":"61.160.36.225","dbName":"dw_ka_sn","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["dw_ka_sn_master2"] = {"dbHost":"61.160.36.225","dbName":"dw_ka_sn","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["dw_ka_user"] = {"dbHost":"61.160.36.225","dbName":"dw_ka_user","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["dw_ls"] = {"dbHost":"61.160.36.225","dbName":"dw_ls","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["dw_ms"] = {"dbHost":"61.160.36.225","dbName":"dw_ms","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["dw_pc"] = {"dbHost":"61.160.36.225","dbName":"dw_pc","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"1"}; 
+dbInfo["dw_pc_data"] = {"dbHost":"61.160.36.225","dbName":"dw_pc_data","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"1"}; 
+dbInfo["dw_shop"] = {"dbHost":"61.160.36.225","dbName":"dw_shop","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["dw_sy"] = {"dbHost":"61.160.36.225","dbName":"dw_sy","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["dw_sy_data"] = {"dbHost":"61.160.36.225","dbName":"dw_sy_data","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["dw_sy_old"] = {"dbHost":"61.160.36.225","dbName":"dw_sy_old","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["dw_task"] = {"dbHost":"61.160.36.225","dbName":"dw_task","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["dw_video"] = {"dbHost":"115.238.171.70","dbName":"video_dw","dbPass":"38v1axmW5x","dbPort":"6304","dbType":"mysqli","dbUser":"net_fanhe_r","enable":"true"}; 
+dbInfo["fhvideo_data"] = {"dbHost":"61.160.36.225","dbName":"fhvideo_data","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["fhvideo_home"] = {"dbHost":"61.160.36.225","dbName":"fhvideo_home","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["fit_db"] = {"dbHost":"61.160.36.225","dbName":"fit_db","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["fit_report"] = {"dbHost":"61.160.36.225","dbName":"fit_report","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["fit_sns"] = {"dbHost":"61.160.36.225","dbName":"fit_sns","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["fit_web"] = {"dbHost":"61.160.36.225","dbName":"fit_web","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["gamecenter_opt"] = {"dbHost":"172.27.203.6","dbName":"gamecenter_opt","dbPass":"yxtq@duowan@$","dbPort":"3306","dbType":"mysqli","dbUser":"yxtq","enable":"true"}; 
+dbInfo["glance_data"] = {"dbHost":"61.160.36.225","dbName":"glance_data","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["glance_home"] = {"dbHost":"61.160.36.225","dbName":"glance_home","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["hiyd_cms"] = {"dbHost":"61.160.36.225","dbName":"hiyd_cms","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["hiyd_cms_test"] = {"dbHost":"61.160.36.225","dbName":"hiyd_cms","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["hiyd_data"] = {"dbHost":"61.160.36.225","dbName":"hiyd_data","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["hiyd_home"] = {"dbHost":"61.160.36.225","dbName":"hiyd_home","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["hiyd_home_slave"] = {"dbHost":"61.160.36.225","dbName":"hiyd_home","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["hiyd_meal"] = {"dbHost":"61.160.36.225","dbName":"hiyd_meal","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["hiyd_shop"] = {"dbHost":"61.160.36.225","dbName":"hiyd_shop","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["issuecode"] = {"dbHost":"172.27.203.6","dbName":"issuecode","dbPass":"yxtq@duowan@$","dbPort":"3306","dbType":"mysqli","dbUser":"yxtq","enable":"true"}; 
+dbInfo["miniworld_data"] = {"dbHost":"61.160.36.225","dbName":"miniworld_data","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["miniworld_home"] = {"dbHost":"61.160.36.225","dbName":"miniworld_home","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["nation_hiyd_cms"] = {"dbHost":"61.160.36.225","dbName":"nation_hiyd_cms","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["nation_hiyd_home"] = {"dbHost":"61.160.36.225","dbName":"nation_hiyd_home","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["o2o_store"] = {"dbHost":"61.160.36.225","dbName":"o2o_store","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["oujhome"] = {"dbHost":"61.160.36.225","dbName":"ouj_home","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["Report"] = {"dbHost":"61.160.36.225","dbName":"Report","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["resource"] = {"dbHost":"61.160.36.225","dbName":"resource","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["shop_base"] = {"dbHost":"61.160.36.225","dbName":"shop_base","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["svideo_data"] = {"dbHost":"61.160.36.225","dbName":"svideo_data","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["svideo_home"] = {"dbHost":"61.160.36.225","dbName":"svideo_home","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+dbInfo["Web"] = {"dbHost":"61.160.36.225","dbName":"Web","dbPass":"ojia305","dbPort":"3306","dbType":"mysqli","dbUser":"ojiatest","enable":"true"}; 
+var qqInfo = {};
+qqInfo["1"] = []; 
+var redisGroup = {};
+redisGroup["dw_ka"] = {"defalut_ttl":"3600","master":"dw_ka","slave1":"dw_ka-slave1"}; 
+var redisInfo = {};
+redisInfo["adsystem"] = {"connet_timeout":"0","db":"11","host":"61.160.36.225","port":"6407","pwd":"ojia123"}; 
+redisInfo["adsystem_api"] = {"connet_timeout":"0","db":"0","host":"61.160.36.225","port":"6408","pwd":"ojia123"}; 
+redisInfo["data_report"] = {"connet_timeout":"1","host":"61.160.36.225","port":"6403","pwd":"ojia123"}; 
+redisInfo["default"] = []; 
+redisInfo["dw_game"] = {"connet_timeout":"0","db":"12","host":"61.160.36.225","port":"6407","pwd":"ojia123"}; 
+redisInfo["dw_ka"] = {"connet_timeout":"0","db":"11","host":"61.160.36.225","port":"6407","pwd":"ojia123","support_pro":"1"}; 
+redisInfo["dw_ka-slave1"] = {"connet_timeout":"0","db":"11","host":"61.160.36.225","port":"6411","pwd":"ojia123","support_pro":"1"}; 
+redisInfo["dw_ka_r2m"] = {"connet_timeout":"0","db":"11","host":"61.160.36.225","port":"6407","pwd":"ojia123","support_pro":"1"}; 
+redisInfo["dw_ka_user"] = {"connet_timeout":"0","db":"11","host":"61.160.36.225","port":"6407","pwd":"ojia123","support_pro":"1"}; 
+redisInfo["dw_ms"] = {"connet_timeout":"0","db":"11","host":"61.160.36.225","port":"6407","pwd":"ojia123"}; 
+redisInfo["dw_shop"] = {"connet_timeout":"0","db":"9","host":"61.160.36.225","port":"6407","pwd":"ojia123"}; 
+redisInfo["dw_sy"] = {"connet_timeout":"3000","db":"0","host":"61.160.36.225","port":"6379","pwd":"ojia123"}; 
+redisInfo["dw_sy_store"] = {"connet_timeout":"3000","db":"0","host":"61.160.36.225","port":"6379","pwd":"ojia123"}; 
+redisInfo["dw_task"] = {"connet_timeout":"0","db":"8","host":"61.160.36.225","port":"6407","pwd":"ojia123"}; 
+redisInfo["fanhe_home"] = {"connet_timeout":"0","db":"2","host":"61.160.36.225","port":"6398","pwd":"ojia_pay"}; 
+redisInfo["fhvideo_app"] = {"connet_timeout":"0","db":"3","host":"61.160.36.225","port":"6398","pwd":"ojia_pay"}; 
+redisInfo["fit_db"] = {"db":"1","host":"61.160.36.225","port":"6407","pwd":"ojia123"}; 
+redisInfo["fit_rank"] = {"db":"4","host":"61.160.36.225","port":"6407","pwd":"ojia123"}; 
+redisInfo["glance_data"] = {"connet_timeout":"0","db":"11","host":"61.160.36.225","port":"6407","pwd":"ojia123"}; 
+redisInfo["glance_event"] = {"connet_timeout":"0","db":"1","host":"61.160.36.19","port":"6380","pwd":"33fa134e11b"}; 
+redisInfo["glance_home"] = {"connet_timeout":"0","db":"11","host":"61.160.36.225","port":"6407","pwd":"ojia123"}; 
+redisInfo["hiyd_cms"] = {"connect_timeout":"0","connet_timeout":"0","db":"3","host":"61.160.36.225","port":"6407","pwd":"ojia123","support_pro":"1"}; 
+redisInfo["hiyd_home"] = {"connet_timeout":"0","db":"4","host":"61.160.36.225","port":"6407","pwd":"ojia123"}; 
+redisInfo["hiyd_meal"] = {"connet_timeout":"0","db":"5","host":"61.160.36.225","port":"6407","pwd":"ojia123"}; 
+redisInfo["hiyd_shop"] = {"connet_timeout":"0","db":"6","host":"61.160.36.225","port":"6407","pwd":"ojia123"}; 
+redisInfo["hiyd_sns"] = {"connet_timeout":"0","host":"61.160.36.225","port":"6382","pwd":"oujsns"}; 
+redisInfo["logic"] = {"host":"61.160.36.225","port":"6404","pwd":"ojia123"}; 
+redisInfo["logstash_redis"] = {"connet_timeout":"1","host":"61.160.36.225","port":"6403","pwd":"ojia123"}; 
+redisInfo["miniworld_data"] = {"connet_timeout":"0","db":"11","host":"61.160.36.225","port":"6407","pwd":"ojia123"}; 
+redisInfo["miniworld_event"] = {"connet_timeout":"0","host":"61.160.36.19","port":"6380","pwd":"33fa134e11b"}; 
+redisInfo["name_serv"] = {"connect_timeout":"0","connet_timeout":"0","db":"1","host":"61.160.36.225","port":"6405","pwd":"ojia123"}; 
+redisInfo["oujhome"] = {"host":"61.160.36.225","port":"6407","pwd":"ojia123"}; 
+redisInfo["oujhome_history"] = {"host":"61.160.36.225","port":"6407","pwd":"ojia123"}; 
+redisInfo["oujhome_logic"] = {"host":"61.160.36.225","port":"6404","pwd":"ojia123"}; 
+redisInfo["shop_base"] = {"connet_timeout":"0","db":"7","host":"61.160.36.225","port":"6407","pwd":"ojia123"}; 
+redisInfo["sns"] = {"connet_timeout":"0","host":"61.160.36.225","port":"6382","pwd":"oujsns"}; 
+redisInfo["store_redis"] = {"connet_timeout":"0","db":"3","host":"61.160.36.225","port":"6398","pwd":"ojia_pay"}; 
+redisInfo["svideo_data"] = {"connet_timeout":"0","db":"11","host":"61.160.36.225","port":"6407","pwd":"ojia123"}; 
+redisInfo["svideo_home"] = {"connet_timeout":"0","db":"10","host":"61.160.36.225","port":"6407","pwd":"ojia123"}; 
+redisInfo["user_token"] = {"connet_timeout":"0","db":"2","host":"61.160.36.225","port":"6400","pwd":"user_token_3"}; 
+var thriftInfo = {};
+thriftInfo["authServer"] = {"host":"61.160.36.225","port":"9002"}; 
+thriftInfo["cacheServer"] = {"host":"61.160.36.225","port":"9004"}; 
+thriftInfo["centerServer"] = {"host":"61.160.36.225","port":"9006"}; 
+thriftInfo["centerServerHezi"] = {"host":"61.160.36.225","port":"9008"}; 
+thriftInfo["glanceServer"] = []; 
+thriftInfo["payServer"] = {"host":"61.160.36.225","port":"9003"}; 
+thriftInfo["userAccountServer"] = {"host":"61.160.36.225","port":"9005"}; 
+thriftInfo["userServer"] = {"host":"61.160.36.225","port":"9001"}; 
+var wxInfo = {};
+wxInfo["1"] = {"appId":"wx892910db2cdd2c12","appSecret":"eed2932f5a29b4f12d2607aa77548949","remark":"偶家app"}; 
+wxInfo["10"] = {"appId":"wxb25813d7b9e2147b","appSecret":"51710497377c2bf56e00fe38b161b14d","remark":"多玩网(订阅号)"}; 
+wxInfo["11"] = {"appId":"wx15447448f1c813d3","appSecret":"a615b4d0a4491f2f52080821a1f57211","remark":"5253手游礼包(服务号)"}; 
+wxInfo["12"] = {"appId":"wxec4655885769a485","appSecret":"d63005d85623cff2857edeaf1139b3c4","remark":"多玩特权(服务号)"}; 
+wxInfo["13"] = {"appId":"wxccdf3948732e9e41","appSecret":"bad95413a0c4c3c3c58f19fa1b6111db","remark":"多玩特权(开发者平台网站应用)"}; 
+wxInfo["14"] = {"appId":"wxd500ae8e6e63b6c9","appSecret":"56e7424d4f7f77c7c0dccf08890fc332","payAppId":"7","remark":"多玩商城(登陆,支付,红包)"}; 
+wxInfo["18"] = []; 
+wxInfo["19"] = {"appId":"wx9f9096ba2851ca72","appSecret":"8babead348bfc30891ce395ca67d07f0","remark":"特权小程序"}; 
+wxInfo["2"] = {"appId":"wx892910db2cdd2c12","appSecret":"eed2932f5a29b4f12d2607aa77548949","remark":"偶家科技公众账号"}; 
+wxInfo["6"] = {"appId":"wx489f3e891508cf09","appSecret":"fea0ce52a21fb194422277010cb42580","remark":"HiMeals善食(老版)"}; 
+wxInfo["7"] = {"appId":"wxa46e6794bc38e236","appSecret":"cf876b3bbae4b8b2bdb03570c391964f","remark":"善食健康餐(新版)"};
+exports["REDIS_PRE_KEY_QQ_OPENID"] = REDIS_PRE_KEY_QQ_OPENID;
+exports["URL_SHOP_API_MKA"] = URL_SHOP_API_MKA;
+exports["dbInfo"] = dbInfo;
+exports["qqInfo"] = qqInfo;
+exports["redisGroup"] = redisGroup;
+exports["redisInfo"] = redisInfo;
+exports["thriftInfo"] = thriftInfo;
+exports["wxInfo"] = wxInfo;

protected/conf/conf_ns/config.r2m.inc.js

@@ -0,0 +1,293 @@
+// 常量
+global["REDIS_PRE_KEY_RANDOM_ID"] = 'genRandomId:';
+
+// 数组
+ 
+var r2mConf = {};
+r2mConf = {"host":"61.160.36.225","port":"9501","pwd":"test"}; 
+r2mConf["default"] = {"host":"61.160.36.225","port":"9501","pwd":"test"}; 
+r2mConf["wuxiduoxian01_shop"] = {"host":"61.160.36.225","port":"9501","pwd":"test"}; 
+var r2mConf_wuxiduoxian01_shop = {};
+r2mConf_wuxiduoxian01_shop = {"host":"61.160.36.225","port":"9501","pwd":"test"}; 
+var r2mInfo = {};
+r2mInfo["dw_ka"] = {};
+r2mInfo["dw_ka"]["act"] = {"key":"act_id","ttl":"3600"}; 
+r2mInfo["dw_ka"]["blacklist_ip"] = {"key":"app_name,ip","ttl":"300"}; 
+r2mInfo["dw_ka"]["blacklist_ka_user"] = {"key":"app_name,user_id","ttl":"600"}; 
+r2mInfo["dw_ka"]["blacklist_yyuser"] = {"key":"app_name,yyuid","ttl":"3600"}; 
+r2mInfo["dw_ka"]["blacklist_yy_user"] = {"key":"app_name,yyuid","ttl":"600"}; 
+r2mInfo["dw_ka"]["game_info"] = {"key":"game_id","ttl":"600"}; 
+r2mInfo["dw_ka"]["game_type"] = {"key":"game_type_id","ttl":"0"}; 
+r2mInfo["dw_ka"]["gift_info"] = {"key":"gift_id","ttl":"600"}; 
+r2mInfo["dw_ka"]["gift_online"] = {"key":"gift_id","ttl":"86400"}; 
+r2mInfo["dw_ka"]["gift_recommend"] = {"key":"gift_id","ttl":"86400"}; 
+r2mInfo["dw_ka"]["gift_res"] = {"key":"res_id","ttl":"600"}; 
+r2mInfo["dw_ka"]["lottery_reward"] = {"key":"reward_id","ttl":"86400"}; 
+r2mInfo["dw_ka"]["qa"] = {"key":"qa_id","ttl":"86400"}; 
+r2mInfo["dw_ka"]["qa_items"] = {"all_key":"qa_id,item_id","key":"item_id","ttl":"86400"}; 
+r2mInfo["dw_ka"]["user"] = {"key":"user_id","ttl":"86400"}; 
+r2mInfo["dw_ka"]["wx_lucky_money"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["dw_ka"]["wx_message"] = {"key":"msg_id","ttl":"86400"}; 
+r2mInfo["dw_ka"]["zt_call"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["dw_ka"]["zt_lottery"] = {"key":"lottery_id","ttl":"86400"}; 
+r2mInfo["dw_ka"]["zt_r_lottery_gift"] = {"key":"gift_id","ttl":"86400"}; 
+r2mInfo["dw_shop"] = {};
+r2mInfo["dw_shop"]["back_order"] = {"key":"back_id","ttl":"86400"}; 
+r2mInfo["dw_shop"]["cart"] = {"all_key":"user_id","key":"cart_id","ttl":"86400"}; 
+r2mInfo["dw_shop"]["collect_goods"] = {"all_key":"user_id","key":"collect_id","ttl":"86400"}; 
+r2mInfo["dw_shop"]["coupons"] = {"key":"coupon_id","ttl":"86400"}; 
+r2mInfo["dw_shop"]["delivery_order"] = {"key":"delivery_id","ttl":"86400"}; 
+r2mInfo["dw_shop"]["order_action"] = {"key":"action_id","ttl":"86400"}; 
+r2mInfo["dw_shop"]["order_finance"] = {"all_key":"finance_order_sn","key":"id","ttl":"86400"}; 
+r2mInfo["dw_shop"]["order_info"] = {"all_key":"user_id","key":"order_id","ttl":"86400"}; 
+r2mInfo["dw_shop"]["user_coupons"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["dw_shop"]["user_exchange_log"] = {"key":"log_id","ttl":"86400"}; 
+r2mInfo["dw_shop"]["user_message"] = {"all_key":"user_id","key":"msg_id","ttl":"86400"}; 
+r2mInfo["dw_sy"] = {};
+r2mInfo["dw_sy"]["game_company"] = {"key":"company_id","ttl":"3600"}; 
+r2mInfo["dw_sy"]["game_forbid_word"] = {"key":"forbid_word","ttl":"3600"}; 
+r2mInfo["dw_sy"]["game_info"] = {"key":"game_id","ttl":"3600"}; 
+r2mInfo["dw_sy"]["game_package_android"] = {"key":"game_id","ttl":"3600"}; 
+r2mInfo["dw_sy"]["game_package_ios"] = {"key":"game_id","ttl":"3600"}; 
+r2mInfo["dw_sy"]["game_package_operations"] = {"key":"game_id","ttl":"3600"}; 
+r2mInfo["dw_sy"]["game_rank"] = {"key":"rank_id","ttl":"3600"}; 
+r2mInfo["dw_sy"]["game_recommend"] = {"key":"game_id","ttl":"3600"}; 
+r2mInfo["dw_sy"]["game_resource"] = {"all_key":"game_id,source_id","key":"source_id","ttl":"3600"}; 
+r2mInfo["dw_sy"]["game_spider_conf"] = []; 
+r2mInfo["dw_sy"]["game_spider_log"] = []; 
+r2mInfo["dw_sy"]["game_type"] = {"key":"game_type_id","ttl":"3600"}; 
+r2mInfo["dw_sy"]["hot_word"] = {"key":"hot_word","ttl":"3600"}; 
+r2mInfo["dw_sy"]["operations_rank"] = {"key":"game_id","ttl":"3600"}; 
+r2mInfo["dw_sy"]["search_static"] = {"key":"keyword","ttl":"3600"}; 
+r2mInfo["dw_sy_data"] = {};
+r2mInfo["dw_sy_data"]["search_static"] = {"key":"keyword","ttl":"3600"}; 
+r2mInfo["dw_task"] = {};
+r2mInfo["dw_task"]["ad_position"] = {"key":"id","ttl":"0"}; 
+r2mInfo["dw_task"]["game_download"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["dw_task"]["gold_record"] = {"key":"id","ttl":"300"}; 
+r2mInfo["dw_task"]["login_record"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["dw_task"]["login_reward_config"] = {"key":"shop_id,login_days","ttl":"86400"}; 
+r2mInfo["dw_task"]["share_visitor_record"] = {"key":"record_id","ttl":"86400"}; 
+r2mInfo["dw_task"]["short_url"] = {"key":"url_id","ttl":"86400"}; 
+r2mInfo["dw_task"]["sign_record"] = {"key":"id","ttl":"172800"}; 
+r2mInfo["dw_task"]["task"] = {"all_key":"shop_id,task_id","key":"task_id","ttl":"86400"}; 
+r2mInfo["dw_task"]["task_execute_record"] = {"key":"record_id","ttl":"86400"}; 
+r2mInfo["dw_task"]["task_record"] = {"key":"id","ttl":"300"}; 
+r2mInfo["dw_task"]["user"] = {"key":"shop_id, user_id","ttl":"86400"}; 
+r2mInfo["dw_task"]["user_task"] = {"key":"yyuid,task_id,create_date","ttl":"300"}; 
+r2mInfo["fhvideo_home"] = {};
+r2mInfo["fhvideo_home"]["appChannel"] = {"key":"id"}; 
+r2mInfo["fit_db"] = {};
+r2mInfo["fit_db"]["app_map"] = {"key":"app_id","ttl":"86400"}; 
+r2mInfo["fit_db"]["course"] = {"all_key":"gym_id","key":"course_id","ttl":"86400"}; 
+r2mInfo["fit_db"]["course_vote_log"] = {"key":"log_id","ttl":"86400"}; 
+r2mInfo["fit_db"]["group_course"] = {"all_key":"gym_id","key":"group_course_id","ttl":"86400"}; 
+r2mInfo["fit_db"]["gym"] = {"key":"gym_id","ttl":"86400"}; 
+r2mInfo["fit_db"]["gym_ad"] = {"key":"ad_id","ttl":"86400"}; 
+r2mInfo["fit_db"]["gym_card"] = {"key":"card_id","ttl":"86400"}; 
+r2mInfo["fit_db"]["gym_subbranch"] = {"key":"subbranch_id","ttl":"86400"}; 
+r2mInfo["fit_db"]["gym_user"] = {"all_key":"gym_id","key":"user_id","ttl":"86400"}; 
+r2mInfo["fit_db"]["gym_user_info"] = {"key":"user_id,gym_id","ttl":"86400"}; 
+r2mInfo["fit_db"]["music"] = {"key":"qq_songmid","ttl":"86400"}; 
+r2mInfo["fit_db"]["music_share"] = {"key":"music_id","ttl":"86400"}; 
+r2mInfo["fit_db"]["order_finance"] = {"all_key":"finance_order_sn","key":"id","ttl":"86400"}; 
+r2mInfo["fit_db"]["place"] = {"all_key":"gym_id","key":"place_id","ttl":"86400"}; 
+r2mInfo["fit_db"]["post"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["fit_db"]["post_comment"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["fit_db"]["post_media"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["fit_db"]["post_support"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["fit_db"]["post_tag"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["fit_db"]["region"] = {"all_key":"parent_id","key":"region_id","ttl":"86400"}; 
+r2mInfo["fit_db"]["trade_log"] = {"key":"log_id","ttl":"86400"}; 
+r2mInfo["fit_db"]["weixinOpenId"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"] = {};
+r2mInfo["hiyd_cms"]["app_tools"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["articles"] = {"key":"aid","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["articles_keyword"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["articles_tags"] = {"key":"aid,tid","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["audio_comment"] = {"key":"course_id,day,index,gender","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["bb_course"] = {"key":"bb_cid","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["bb_course_day_info"] = {"key":"bb_cid,day","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["bb_course_exercise"] = {"key":"bb_cid,day,workout_id,group_sequence,sort","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["bb_course_group"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["bb_exercise_desc"] = {"key":"desc_id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["bb_exercise_workout"] = {"key":"workout_id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["bb_relational_video"] = {"key":"bb_cid","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["bb_super_group"] = {"key":"group_id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["bb_training_point"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["category"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["com_comments"] = {"key":"app_id,comment_target,comment_id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["com_replies"] = {"key":"comment_id,reply_id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["com_support"] = {"key":"comment_id,uid,support_id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["course"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["day_info"] = {"key":"course_id,day,gender","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["equipment"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["exercise"] = {"key":"id","table":"exercise,v_course_exercises,v_workout_exercises","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["file"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["file_dw_video"] = {"key":"vid","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["muscle"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["plan"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["plan_base"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["plan_body"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["plan_info"] = {"key":"plan_id,plan_day","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["plan_target"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["relational_course"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["r_exercise_group"] = {"key":"exercise_id,group_id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["r_plan"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["sns_post_user"] = {"key":"uid,sub_uid","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["sns_region"] = {"key":"region_id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["tags"] = {"key":"tid","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["template"] = {"key":"tpl_id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["topic_url"] = {"key":"tpl_id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["training_point"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["v_bb_course"] = {"key":"bb_cid","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["v_course"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["v_course_exercises"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["v_workout_exercises"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_cms"]["workout"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_home"] = {};
+r2mInfo["hiyd_home"]["ad"] = {"key":"ad_id","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["badge"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["badgeRecord"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["black_domain_ip"] = {"key":"ad_id","ttl":"3600"}; 
+r2mInfo["hiyd_home"]["food_group"] = {"key":"group_id","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["food_info"] = {"key":"info_id","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["food_restaurant"] = {"key":"restaurant_id","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["health_report"] = {"key":"report_id","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["health_report_app"] = {"key":"report_id","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["mission"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["post"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["postComment"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["postMedia"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["share_visitor_record"] = {"key":"share_uid,device_token","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["user"] = {"key":"user_id","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["userAccount"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["userCustomExerciseRecord"] = {"key":"uid,bid,courseDay,type","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["userFinishedPlanCourse"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["userFinishedStep"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["userFinishedWorkout"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["userFoodRecommend"] = {"key":"uid","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["userJoinBBCourse"] = {"key":"uid,bb_cid","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["userJoinCourse"] = {"key":"uid,courseId","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["userJoinWorkout"] = {"key":"uid,workoutId","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["userSharePlan"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["user_trained_statistics_daily"] = {"key":"uid,trained_date","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["user_trained_statistics_monthly"] = {"key":"uid,trained_year,month_index","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["user_trained_statistics_weekly"] = {"key":"uid,trained_year,week_index","ttl":"86400"}; 
+r2mInfo["hiyd_home"]["v_user"] = {"key":"user_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"] = {};
+r2mInfo["hiyd_meal"]["ad"] = {"key":"ad_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["admin_log"] = {"key":"log_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["attribute"] = {"key":"attr_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["back_order"] = {"key":"back_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["cart"] = {"all_key":"user_id","key":"cart_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["collect_goods"] = {"all_key":"user_id","key":"collect_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["coupons"] = {"key":"coupon_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["delivery_order"] = {"key":"delivery_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["food_recommend"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["gift"] = {"key":"gift_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["goods"] = {"key":"goods_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["goods_gallery"] = {"key":"img_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["meals_daily"] = {"key":"meals_day,goods_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["meals_delivery_log"] = {"key":"log_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["meals_item"] = {"key":"item_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["money_record"] = {"key":"rec_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["order_action"] = {"key":"action_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["order_finance"] = {"all_key":"finance_order_sn","key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["order_info"] = {"all_key":"user_id","key":"order_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["products"] = {"key":"product_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["recharge_order_finance"] = {"all_key":"finance_order_sn","key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["region"] = {"all_key":"parent_id","key":"region_id","ttl":"0"}; 
+r2mInfo["hiyd_meal"]["r_shop_company"] = {"key":"shop_id,company_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["shipping"] = {"key":"shipping_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["shipping_area"] = {"key":"shipping_area_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["shipping_company"] = {"key":"company_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["shop"] = {"key":"shop_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["shop_bulletin"] = {"key":"bulletin_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["shop_minus"] = {"key":"minus_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["star_card"] = {"key":"card_id","ttl":"0"}; 
+r2mInfo["hiyd_meal"]["suppliers"] = {"key":"suppliers_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["user"] = {"key":"user_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["user_coupons"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["user_info"] = {"key":"user_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["user_message"] = {"all_key":"user_id","key":"msg_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["web_recommend_group"] = {"key":"group_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["web_recommend_item"] = {"key":"item_id","ttl":"86400"}; 
+r2mInfo["hiyd_meal"]["web_shop_ad"] = {"key":"ad_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"] = {};
+r2mInfo["hiyd_shop"]["ad"] = {"key":"ad_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["admin_log"] = {"key":"log_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["attribute"] = {"key":"attr_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["back_order"] = {"key":"back_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["cart"] = {"all_key":"user_id","key":"cart_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["collect_goods"] = {"all_key":"user_id","key":"collect_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["coupons"] = {"key":"coupon_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["delivery_order"] = {"key":"delivery_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["food_recommend"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["gift"] = {"key":"gift_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["goods"] = {"key":"goods_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["goods_gallery"] = {"key":"img_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["meals_delivery_log"] = {"key":"log_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["meals_item"] = {"key":"item_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["money_record"] = {"key":"rec_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["order_action"] = {"key":"action_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["order_finance"] = {"all_key":"finance_order_sn","key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["order_info"] = {"all_key":"user_id","key":"order_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["products"] = {"key":"product_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["region"] = {"all_key":"parent_id","key":"region_id","ttl":"0"}; 
+r2mInfo["hiyd_shop"]["r_shop_company"] = {"key":"shop_id,company_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["shipping"] = {"key":"shipping_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["shipping_area"] = {"key":"shipping_area_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["shipping_company"] = {"key":"company_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["shop"] = {"key":"shop_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["shop_bulletin"] = {"key":"bulletin_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["shop_minus"] = {"key":"minus_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["star_card"] = {"key":"card_id","ttl":"0"}; 
+r2mInfo["hiyd_shop"]["suppliers"] = {"key":"suppliers_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["user"] = {"key":"user_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["user_coupons"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["user_exchange_log"] = {"key":"log_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["user_info"] = {"key":"user_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["user_message"] = {"all_key":"user_id","key":"msg_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["web_recommend_group"] = {"key":"group_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["web_recommend_item"] = {"key":"item_id","ttl":"86400"}; 
+r2mInfo["hiyd_shop"]["web_shop_ad"] = {"key":"ad_id","ttl":"86400"}; 
+r2mInfo["o2o_store"] = {};
+r2mInfo["o2o_store"]["bodybuilding"] = {"key":"auto_id","ttl":"86400"}; 
+r2mInfo["oujhome"] = {};
+r2mInfo["oujhome"]["chapterHistory"] = {"all_key":"uid,device_token,bookId","key":"chapter_history_id","ttl":"86400"}; 
+r2mInfo["oujhome"]["news"] = {"key":"news_id","ttl":"86400"}; 
+r2mInfo["oujhome"]["recommendBook"] = {"key":"rec_id","ttl":"86400"}; 
+r2mInfo["shop_base"] = {};
+r2mInfo["shop_base"]["ad"] = {"key":"ad_id","ttl":"86400"}; 
+r2mInfo["shop_base"]["attribute"] = {"key":"attr_id","ttl":"86400"}; 
+r2mInfo["shop_base"]["category"] = {"key":"cat_id","ttl":"86400"}; 
+r2mInfo["shop_base"]["gift"] = {"key":"gift_id","ttl":"86400"}; 
+r2mInfo["shop_base"]["goods"] = {"key":"goods_id","ttl":"86400"}; 
+r2mInfo["shop_base"]["goods_gallery"] = {"key":"img_id","ttl":"86400"}; 
+r2mInfo["shop_base"]["products"] = {"key":"product_id","ttl":"86400"}; 
+r2mInfo["shop_base"]["region"] = {"all_key":"parent_id","key":"region_id","ttl":"0"}; 
+r2mInfo["shop_base"]["r_shop_company"] = {"key":"shop_id,company_id","ttl":"86400"}; 
+r2mInfo["shop_base"]["shipping"] = {"key":"shipping_id","ttl":"86400"}; 
+r2mInfo["shop_base"]["shipping_area"] = {"key":"shipping_area_id","ttl":"86400"}; 
+r2mInfo["shop_base"]["shipping_company"] = {"key":"company_id","ttl":"86400"}; 
+r2mInfo["shop_base"]["shop"] = {"key":"shop_id","ttl":"86400"}; 
+r2mInfo["shop_base"]["shop_bulletin"] = {"key":"bulletin_id","ttl":"86400"}; 
+r2mInfo["shop_base"]["shop_minus"] = {"key":"minus_id","ttl":"86400"}; 
+r2mInfo["shop_base"]["suppliers"] = {"key":"suppliers_id","ttl":"86400"}; 
+r2mInfo["shop_base"]["tpl_group"] = {"key":"tpl_goods_id","ttl":"3600"}; 
+r2mInfo["svideo_home"] = {};
+r2mInfo["svideo_home"]["admin"] = {"key":"uid","ttl":"86400"}; 
+r2mInfo["svideo_home"]["adminMediaAccount"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["svideo_home"]["article"] = {"key":"uid","ttl":"86400"}; 
+r2mInfo["svideo_home"]["articleAuditRecord"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["svideo_home"]["articleTag"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["svideo_home"]["articleTagWait"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["svideo_home"]["articleWait"] = {"key":"uid","ttl":"86400"}; 
+r2mInfo["svideo_home"]["mediaAccount"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["svideo_home"]["notice"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["svideo_home"]["tag"] = {"key":"id","ttl":"86400"}; 
+r2mInfo["svideo_home"]["tagUserMap"] = {"key":"id","ttl":"86400"};
+exports["r2mConf"] = r2mConf;
+exports["r2mConf_wuxiduoxian01_shop"] = r2mConf_wuxiduoxian01_shop;
+exports["r2mInfo"] = r2mInfo;

protected/conf/conf_ns/config.shop.inc.js

@@ -0,0 +1,81 @@
+// 常量
+global["CODE_NOT_ENGOUGH_PRODUCT"] = -4000;
+global["CODE_NOT_EXIST_CART"] = -4004;
+global["CODE_ORDER_ERROR_FEE"] = -5001;
+global["CODE_PRODUCT_EMPTY_ERROR"] = -4002;
+global["CODE_PRODUCT_ERROR_FEE"] = -4001;
+
+// 数组
+ 
+var CODE_ADDR_GETLOCATION_ERROR = {};
+CODE_ADDR_GETLOCATION_ERROR = []; 
+var CODE_ADDR_MATCH_ERROR = {};
+CODE_ADDR_MATCH_ERROR = []; 
+var CODE_ADDR_NUM_LIMIT = {};
+CODE_ADDR_NUM_LIMIT = []; 
+var CODE_CARD_ORDER_EMPTY_ERROR = {};
+CODE_CARD_ORDER_EMPTY_ERROR = []; 
+var CODE_CART_MODIFIED_ERROR = {};
+CODE_CART_MODIFIED_ERROR = []; 
+var CODE_KUAIDI_ERROR = {};
+CODE_KUAIDI_ERROR = []; 
+var CODE_MONEY_NO_ENOUGH = {};
+CODE_MONEY_NO_ENOUGH = []; 
+var CODE_MONEY_RECORD_ERROR = {};
+CODE_MONEY_RECORD_ERROR = []; 
+var CODE_NOT_ENABLED_SHOP = {};
+CODE_NOT_ENABLED_SHOP = []; 
+var CODE_NOT_EXIST_ADDRESS = {};
+CODE_NOT_EXIST_ADDRESS = []; 
+var CODE_NOT_EXIST_BACK_ORDER = {};
+CODE_NOT_EXIST_BACK_ORDER = []; 
+var CODE_NOT_EXIST_SHOP = {};
+CODE_NOT_EXIST_SHOP = []; 
+var CODE_NOT_EXIST_USER = {};
+CODE_NOT_EXIST_USER = []; 
+var CODE_ORDER_CONTENT_LENGTH_ERROR = {};
+CODE_ORDER_CONTENT_LENGTH_ERROR = []; 
+var CODE_ORDER_ERROR_CALLBACK = {};
+CODE_ORDER_ERROR_CALLBACK = []; 
+var CODE_ORDER_ERROR_OPERATION = {};
+CODE_ORDER_ERROR_OPERATION = []; 
+var CODE_ORDER_FEEDBACK_ALREADY = {};
+CODE_ORDER_FEEDBACK_ALREADY = []; 
+var CODE_ORDER_NOT_ALLOW_FEEDBACK = {};
+CODE_ORDER_NOT_ALLOW_FEEDBACK = []; 
+var CODE_ORDER_NOT_ALLOW_RETURN = {};
+CODE_ORDER_NOT_ALLOW_RETURN = []; 
+var CODE_ORDER_NOT_FEEDBACK = {};
+CODE_ORDER_NOT_FEEDBACK = []; 
+var CODE_ORDER_RETURN_ALREADY = {};
+CODE_ORDER_RETURN_ALREADY = []; 
+var CODE_ORDER_RETURN_ERROR = {};
+CODE_ORDER_RETURN_ERROR = []; 
+var CODE_PRODUCT_NOT_SALE_ERROR = {};
+CODE_PRODUCT_NOT_SALE_ERROR = []; 
+var code_map = {};
+code_map = {"-4000":"商品库存不足","-4004":"该商品已下架","-5001":"总价不正确","-4002":"商品为空","-4001":"商品价格变动"};
+exports["CODE_ADDR_GETLOCATION_ERROR"] = CODE_ADDR_GETLOCATION_ERROR;
+exports["CODE_ADDR_MATCH_ERROR"] = CODE_ADDR_MATCH_ERROR;
+exports["CODE_ADDR_NUM_LIMIT"] = CODE_ADDR_NUM_LIMIT;
+exports["CODE_CARD_ORDER_EMPTY_ERROR"] = CODE_CARD_ORDER_EMPTY_ERROR;
+exports["CODE_CART_MODIFIED_ERROR"] = CODE_CART_MODIFIED_ERROR;
+exports["CODE_KUAIDI_ERROR"] = CODE_KUAIDI_ERROR;
+exports["CODE_MONEY_NO_ENOUGH"] = CODE_MONEY_NO_ENOUGH;
+exports["CODE_MONEY_RECORD_ERROR"] = CODE_MONEY_RECORD_ERROR;
+exports["CODE_NOT_ENABLED_SHOP"] = CODE_NOT_ENABLED_SHOP;
+exports["CODE_NOT_EXIST_ADDRESS"] = CODE_NOT_EXIST_ADDRESS;
+exports["CODE_NOT_EXIST_BACK_ORDER"] = CODE_NOT_EXIST_BACK_ORDER;
+exports["CODE_NOT_EXIST_SHOP"] = CODE_NOT_EXIST_SHOP;
+exports["CODE_NOT_EXIST_USER"] = CODE_NOT_EXIST_USER;
+exports["CODE_ORDER_CONTENT_LENGTH_ERROR"] = CODE_ORDER_CONTENT_LENGTH_ERROR;
+exports["CODE_ORDER_ERROR_CALLBACK"] = CODE_ORDER_ERROR_CALLBACK;
+exports["CODE_ORDER_ERROR_OPERATION"] = CODE_ORDER_ERROR_OPERATION;
+exports["CODE_ORDER_FEEDBACK_ALREADY"] = CODE_ORDER_FEEDBACK_ALREADY;
+exports["CODE_ORDER_NOT_ALLOW_FEEDBACK"] = CODE_ORDER_NOT_ALLOW_FEEDBACK;
+exports["CODE_ORDER_NOT_ALLOW_RETURN"] = CODE_ORDER_NOT_ALLOW_RETURN;
+exports["CODE_ORDER_NOT_FEEDBACK"] = CODE_ORDER_NOT_FEEDBACK;
+exports["CODE_ORDER_RETURN_ALREADY"] = CODE_ORDER_RETURN_ALREADY;
+exports["CODE_ORDER_RETURN_ERROR"] = CODE_ORDER_RETURN_ERROR;
+exports["CODE_PRODUCT_NOT_SALE_ERROR"] = CODE_PRODUCT_NOT_SALE_ERROR;
+exports["code_map"] = code_map;

protected/conf/config.dev.inc.js

@@ -0,0 +1,85 @@
+
+global['DEBUG'] = true;
+global['TYPE_SELF_CALL'] = 'test_spider_self_call';
+global['TYPE_MODULE_CALL'] = 'test_spider_self_call';
+
+// let dbInfo = {};
+//
+// dbInfo['Web'] = {
+//     host : '61.160.36.225',
+//     user : 'ojiatest',
+//     password : 'ojia305',
+//     database : 'Web',
+//     port : 3306,
+//     connectionLimit : 100
+// };
+//
+// dbInfo['Report'] = {
+//     host : '61.160.36.225',
+//     user : 'ojiatest',
+//     password : 'ojia305',
+//     database : 'Report',
+//     port : 3306,
+//     connectionLimit : 100
+// };
+//
+// dbInfo['crawl'] = {
+//     host : '61.160.36.225',
+//     user : 'ojiatest',
+//     password : 'ojia305',
+//     database : 'crawl',
+//     port : 3306,
+//     connectionLimit : 100
+// };
+//
+// dbInfo['dw_pc'] = {
+//     host : '61.160.36.225',
+//     user : 'ojiatest',
+//     password : 'ojia305',
+//     database : 'dw_pc',
+//     port : 3306,
+//     connectionLimit : 100
+// };
+//
+// dbInfo['dw_pc_data'] = {
+//     host : '61.160.36.225',
+//     user : 'ojiatest',
+//     password : 'ojia305',
+//     database : 'dw_pc_data',
+//     port : 3306,
+//     connectionLimit : 100
+// };
+//
+// let redisInfo = {};
+// redisInfo['name_serv'] = {
+//     'host' : '61.160.36.225',
+//     'port' : 6405,
+//     'pwd' : 'ojia123',
+//     'db' : 1,
+//     'connet_timeout' : 0
+// };
+//
+// redisInfo['Web'] = {
+//     host : '61.160.36.225',
+//     port : 6407,
+//     pwd : 'ojia123',
+//     db : 1
+// };
+//
+// redisInfo['logstash_redis'] = {
+//     host : '61.160.36.225',
+//     port : 6407,
+//     pwd : 'ojia123',
+//     db : 2
+// };
+//
+// redisInfo['logic'] = {
+//     host : "61.160.36.225",
+//     port : 6404,
+//     pwd : "ojia123"
+// };
+
+process.env.PORT = 9998;
+//
+// exports.dbInfo = dbInfo;
+// exports.redisInfo = redisInfo;

protected/conf/config.form.inc.js

@@ -0,0 +1,61 @@
+
+global['DEBUG'] = false;
+global['TYPE_SELF_CALL'] = 'spider_self_call';
+global['TYPE_MODULE_CALL'] = 'spider_self_call';
+
+// let dbInfo = {};
+//
+// dbInfo['Web'] = {
+//     host : '10.25.68.139',
+//     user : 'ojuser',
+//     password : 'qs45MWKf',
+//     database : 'Web',
+//     port : 6308,
+//     connectionLimit : 100
+// };
+//
+// dbInfo['Report'] = {
+//     host : '10.25.68.139',
+//     user : 'ojuser',
+//     password : 'qs45MWKf',
+//     database : 'Report',
+//     port : 6308,
+//     connectionLimit : 100
+// };
+//
+// dbInfo['crawl'] = {
+//     host : '10.25.68.139',
+//     user : 'ojuser',
+//     password : 'qs45MWKf',
+//     database : 'crawl',
+//     port : 6308,
+//     connectionLimit : 100
+// };
+//
+// dbInfo['dw_pc'] = {
+//     host : '10.20.167.232',
+//     user : 'dw_sy_rw',
+//     password : 'Yn7wx9g5QG',
+//     database : 'dw_pc',
+//     port : 6301,
+//     connectionLimit : 100
+// };
+//
+// dbInfo['dw_pc_data'] = {
+//     host : '10.20.167.232',
+//     user : 'dw_sy_rw',
+//     password : 'Yn7wx9g5QG',
+//     database : 'dw_pc_data',
+//     port : 6301,
+//     connectionLimit : 100
+// };
+
+// let redisInfo = {};
+//
+// redisInfo["logstash_redis"] = {"connet_timeout":"1","host":"10.20.164.64","port":"6399","pwd":"Fh7fVj8fSsKf"};
+// redisInfo["logic"] = {"host":"10.20.164.64","port":"6398","pwd":"Fh7fVj8fSsKf"};
+
+process.env.PORT = 10000;
+//
+// exports.dbInfo = dbInfo;
+// exports.redisInfo = redisInfo;

protected/conf/config.inc.js

@@ -0,0 +1,21 @@
+
+let config = require(`${ROOT_PATH}conf/config.${ENV}.inc.js`);
+
+let keys = [
+    'globals',
+    'code',
+    'r2m',
+];
+
+for (let name of keys) {
+    try {
+        let conf = require(`${CONF_PATH}config.${name}.inc.js`);
+        config = Object.assign(config, conf);
+    } catch(ex) {
+        console.error(ex.message);
+    }
+}
+
+module.exports = config;
+
+// 包含本地配置

protected/conf/r2m_config.inc.js

@@ -0,0 +1,23 @@
+
+/*
+ * 配置说明，注意all_key字段不能更改（不然，deleteListCache会有问题）
+ r2mInfo[dbKey][tableName] = {
+     'key' : 'key',
+     'all_key' : 'all_key',
+     'ttl' : '每条数据的缓存时间，0为永久缓存，其他为秒数',
+ };
+*/
+
+var DEFAULT_R2M_TTL = 3600;
+var r2mInfo = {};
+var Web = {};
+
+Web['cUser'] = {
+    key : 'userId',
+    all_key : 'enable',
+    ttl : DEFAULT_R2M_TTL
+};
+
+r2mInfo['Web'] = Web;
+
+module.exports = r2mInfo;

protected/controllers/DefaultController.js

@@ -0,0 +1,92 @@
+const Controller = require('../framework/Controller.js');
+const TableHelper = require('../framework/lib/TableHelper');
+const Spider = require('../models/Spider');
+const Browser = require('../models/Browser');
+/**
+ * 首页Controller
+ */
+class DefaultController extends Controller {
+    //var i = 0;
+    constructor(req, res) {
+        super(req, res);
+        this.i = 0;
+    }
+
+    actionIndex(args) {
+        let title = args['title'] || 'Hello';
+        this.assign('title', title);
+        this.display('index');
+    }
+
+    /**
+     * 预览规则
+     * @param args
+     * @returns {Promise.<void>}
+     */
+    async actionPreviewRule(args) {
+        // let rule_id = args['rule_id'] || "steam:game_player_data";
+        let rule_id = args['rule_id'] || "steam:game_detail";
+
+        const objRule = new TableHelper('rule', 'crawl');
+        let rule = await objRule.getRow({rule_id});
+
+        let task = {};
+        task['task_id'] = -1;
+        task['rule_id'] = rule_id;
+        task['url'] = args['url'] || rule['demo_url'];
+        let objSpider = new Spider(task);
+        let content = await objSpider.run(true);
+
+        this.exitMsg(content);
+    }
+
+    /**
+     * 执行任务
+     * @param args
+     * @returns {Promise.<void>}
+     */
+    async actionRunTask(args) {
+        let rules = {
+            'task_id' : {'type' : 'int' },
+        };
+        this.checkParam2(rules, args);
+
+        let task_id = args['task_id'];
+        const obTask = new TableHelper('task', 'crawl');
+        let task = await obTask.getRow({task_id});
+
+        let objSpider = new Spider(task);
+        await objSpider.run();
+
+        let logs = objSpider.getLogs();
+        this._res.header('Content-Type', 'application/json;charset=' + DEFAULT_CHARSET);
+
+        // // 格式化样式
+        // let matches = logs.match(/[^\n]+【error】[^\n]+/g);
+        // for (let match of matches) {
+        //     logs = logs.replace(match,`<span style="color:red;">${match}</span>`);
+        // }
+        // logs = logs.replace(/\n/g, '<\br>')
+
+        this.exitMsg(logs);
+    }
+
+    async actionProxy(args) {
+        let rules = {
+            'proxy' : {'type' : 'string' },
+            'request_time' : {'type' : 'int' },
+        };
+        this.checkParam2(rules, args);
+
+        let ip = getClientIp(this._req);
+        let time = (new Date).getTime();
+        let time_span = time - args['request_time'];
+
+
+        let data = {ip, time, time_span};
+        this.success(data);
+    }
+}
+
+module.exports = DefaultController;
+

protected/extensions/function_extend.js

@@ -0,0 +1,56 @@
+/**
+ * Created by benzhan on 15/8/25.
+ */
+let php = require('phpjs');
+
+function getServerIp() {
+    let os = require('os');
+    let ipv4;
+    let net = os.networkInterfaces();
+    let en0 = net.en0 || net.eth0;
+    if (en0) {
+        for (let i = 0; i < en0.length; i++) {
+            if (en0[i].family.toLowerCase() == 'ipv4') {
+                ipv4 = en0[i].address;
+            }
+        }
+    }
+
+    return ipv4 || '127.0.0.1';
+}
+
+function getClientIp(_req) {
+    let ip = _req.headers['x-forwarded-for'] ||
+        _req.connection.remoteAddress ||
+        _req.socket && _req.socket.remoteAddress ||
+        _req.connection.socket && _req.connection.socket.remoteAddress;
+
+    let parts = ip && ip.split(':');
+    if (parts && parts.length) {
+        return parts[parts.length - 1];
+    } else {
+        return '';
+    }
+}
+
+function checkIpRange(remote_ip, ip_array){
+    //判断ip是否在白名单
+    for (let i in ip_array) {
+        let ip = ip_array[i];
+        let ip_info = ip.split('/');
+        let mask = ip_info[1] ? ip_info[1] : 32;
+
+        let ip_mask = php.sprintf("%032b", php.ip2long(ip_info[0])).substr(0, mask);
+        let remote_ip_mask = php.sprintf("%032b", php.ip2long(remote_ip)).substr(0, mask);
+        if (ip_mask === remote_ip_mask) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+global['getServerIp'] = getServerIp;
+global['getClientIp'] = getClientIp;
+global['checkIpRange'] = checkIpRange;
+

protected/framework/Controller.js

@@ -0,0 +1,70 @@
+const Response = require('./lib/Response.js');
+const Param = require('./lib/Param.js');
+
+class Controller extends Response {
+    constructor(req, res) {
+        super(req, res);
+
+        if (res) {
+            res.header("Cache-Control", "private");
+            res.header('Content-Type', 'text/html;charset=' + DEFAULT_CHARSET);
+        }
+
+        /**
+         * 模板数据
+         * @type {{}}
+         * @private
+         */
+        this._tpl_data = {};
+        this._startTime = new Date().getTime();
+        this._req = req;
+        this._res = res;
+        this._objParam = new Param(this);
+    }
+
+    /**
+     * 检查参数
+     * @param rules 检查规则
+     * {
+     *     'appId' : 'int',    //int类型
+     *     'owners' : 'array',     //array类型
+     *     'instanceIds' : 'intArr',   //array类型，元素为int类型
+     *     'instanceTypes' : 'strArr',     //array类型，元素为string类型
+     *     'deviceId' : 'int/array',       //int类型或者array类型，最后转化为元素为idArr类型
+     *     'deviceClass' : 'string/array',     //string类型或者array类型，最后转化为strArr类型
+     *     'blocks' : {type : 'int', range : '(5, 10)'}    //int类型， > 5 , < 10
+     *     'blocks2' : {type : 'int', range : '[5, 10]'}   //int类型， >= 5 , <= 10
+     *     'percent' : {type : 'float', range : '[5.1, 10.9]'}     //float类型，>= 5.1 , <= 10.9
+     *     'appName' : {type : 'string'}       //string类型
+     *     'appName2' : {type : 'string', reg : '[^0-9A-Za-z]', 'len' : '[1, 10]', 'nullable' : true}      //string类型，支持正则
+     * }
+     * @param args 参数合集
+     * @param exitError 遇到错误是否直接exit
+     * @static
+     * @return boolean 是否检查通过
+     */
+    checkParam(rules, args, exitError) {
+        this._objParam.checkParam(rules, args, exitError);
+    }
+
+    checkParam2(rules, args, exitError) {
+        this._objParam.checkParam2(rules, args, exitError);
+    }
+
+    assign(key, value) {
+        if (typeof key === 'object') {
+            for (let k in key) {
+                this._tpl_data[k] = key[k];
+            }
+        } else {
+            this._tpl_data[key] = Object.assign(value, this._tpl_data[key]);
+        }
+    };
+
+    display(tpl) {
+        tpl = tpl || CONTROLLER_NAME + '/' + ACTION_NAME;
+        this._res.render(tpl, this._tpl_data);
+    }
+}
+
+module.exports = Controller;

protected/framework/Doc.js

@@ -0,0 +1,90 @@
+var fs = require('fs');
+var Buffer = require('buffer').Buffer;
+function Doc() {
+
+}
+
+Doc.prototype.getClassName = function(filePath) {
+    fs.open(filePath, 'r', function(err, fd) {
+        var contents = "";
+
+        var raw = new Buffer(1024);
+
+        fs.read(fd, raw, 0, raw.length, null, function(err, bytesRead, buffer) {
+            var buf = buffer.slice(0, bytesRead);
+
+            contents += buf.toString().split('module.exports = ')[1].split(';')[0];
+
+            fs.close(fd, function() {
+                console.log(contents);
+            })
+        })
+    })
+}
+
+Doc.prototype.getClassInfos = function(path) {
+    var fileNames = fs.readdirSync(path);
+    var classInfos = {};
+    var that = this;
+    for(var i in fileNames) {
+        var className = fileNames[i].split('Controller.js')[0];
+        classInfos[className] = {
+            author : "",
+            desc : "",
+            funcs : that.getFunc(className)
+        }
+    }
+    return classInfos;
+}
+
+/**
+ * 获取类的信息
+ * @param className     类名
+ */
+Doc.prototype.getClassInfo = function(className) {
+    return;
+}
+
+/**
+ * 获取接口信息
+ * @param className     类名
+ * @param funcName      接口名
+ */
+Doc.prototype.getFuncInfo = function(className, funcName) {
+    var controllerName = className.charAt(0).toUpperCase() + className.slice(1);
+    var Controller;
+
+    try{
+        //Controller = require('../controllers/' + controllerName + 'Controller.js');
+        //var objFunc = Controller.prototype[funcName];
+        //return
+    } catch(e) {
+        console.error(e);
+    }
+}
+
+Doc.prototype.getFunc = function(className) {
+    var controllerName = className.charAt(0).toUpperCase() + className.slice(1);
+    var Controller;
+
+    try{
+        Controller = require('../controllers/' + controllerName + 'Controller.js');
+        var funcList = {};
+
+        for(var i in Controller.prototype) {
+            if(i.match('action')){
+                funcList[i] = {
+                    author : "",
+                    desc : "",
+                    param : []
+                }
+            }
+        }
+        return funcList;
+    } catch(e) {
+        console.error(e);
+        return null;
+    }
+}
+
+module.exports = Doc;

protected/framework/Model.js

@@ -0,0 +1,41 @@
+'use strict';
+var MySql = require('../framework/lib/OujMySql.js');
+var TableHelper = require('../framework/lib/TableHelper.js');
+
+class Model {
+    constructor(tableName, dbKey) {
+        this.tableName = tableName;
+        this.dbKey = dbKey || 'default';
+
+        if (!tableName) {
+            return;
+        }
+
+        /**
+         * 数据库的表助手
+         * @type {TableHelper}
+         */
+        this.objTable = new TableHelper(tableName, this.dbKey);
+
+        /**
+         * 数据库的操作类
+         * @type {OujMySql}
+         */
+        this.objDb = new MySql(this.dbKey);
+    }
+
+    /**
+     * 返回数据库操作对象
+     * @param dbKey
+     * @returns {OujMySql}
+     */
+    getObjDb(dbKey) {
+        if (dbKey) {
+            return new MySql(dbKey);
+        } else {
+            return this.objDb;
+        }
+    }
+}
+
+module.exports = Model;

protected/framework/R2MModel.js

@@ -0,0 +1,26 @@
+"use strict"
+var Model = require('./Model.js');
+var R2M_Client = require('./lib/r2m/Client.js');
+
+class R2MModel extends Model {
+    
+    constructor(tableName, dbKey, cacheKey) {
+        super(tableName, dbKey);
+        this.cacheKey = cacheKey || 'default';
+        this._getHelper();
+    }
+
+    _getHelper() {
+        if (!this.objR2m) {
+            this.objR2m = new R2M_Client(this.tableName, this.dbKey, this.cacheKey);
+        }
+        
+        return this.objR2m;
+    }
+
+    // getRedis() {
+    //     return dwRedis::init($this->cacheKey);
+    // }    
+
+}
+module.exports = R2MModel;

protected/framework/lib/AppErrors.js

@@ -0,0 +1,63 @@
+"use strict";
+
+class AbstractError extends  Error {
+    constructor(msg) {
+        super(msg);
+        this.name = 'Abstract Error';
+    }
+}
+
+/**
+ * 中断（正常停止）
+ * @param msg
+ * @constructor
+ */
+class Interrupt extends  AbstractError {
+    constructor(msg) {
+        super(msg);
+        this.name = 'Interrupt';
+    }
+}
+
+/**
+ * 数据库错误
+ * @param msg
+ * @constructor
+ */
+class DbError extends  AbstractError {
+    constructor(msg) {
+        super(msg);
+        this.name = 'Database Error';
+    }
+}
+
+/**
+ * Redis错误
+ * @param msg
+ * @constructor
+ */
+class RedisError extends  AbstractError {
+    constructor(msg) {
+        super(msg);
+        this.name = 'Redis Error';
+    }
+}
+
+/**
+ * R2M错误
+ * @param msg
+ * @constructor
+ */
+class R2MError extends  AbstractError {
+    constructor(msg) {
+        super(msg);
+        this.name = 'R2M Error';
+    }
+}
+
+module.exports = {
+    Interrupt,
+    DbError,
+    RedisError,
+    R2MError
+};

protected/framework/lib/CallLog.js

@@ -0,0 +1,139 @@
+"use strict"
+
+let OujRedis = require('./OujRedis.js');
+let php = require('phpjs');
+
+let call_id = null;
+let objRedis = OujRedis.init('logstash_redis');
+
+class CallLog {
+    constructor(objController) {
+        /**
+         * @type {Controller}
+         */
+        this.objController = objController;
+    }
+
+    getCallId() {
+        if (!call_id) {
+            call_id = new Date().getTime() + "" + parseInt(Math.random() * 10000);
+        }
+        return call_id;
+    }
+
+    logSelfCall(code, response) {
+        let _req = this.objController && this.objController._req || {};
+        let _startTime = this.objController && this.objController._startTime;
+
+        let data = {};
+        data['call_id'] = this.getCallId();
+        data['url'] = '/' + CONTROLLER_NAME + '/' + ACTION_NAME;
+        data['method'] = _req.method;
+        let getParam = php.http_build_query(_req.query);
+        let postParam = php.http_build_query(_req.body);
+
+        let param = '';
+        if (getParam && postParam) {
+            param = getParam + '&' + postParam;
+        } else if (getParam) {
+            param = getParam;
+        } else {
+            param = postParam;
+        }
+
+        data['param'] = param;
+        if (_req.headers) {
+            data['cookie'] = _req.headers.cookie;
+            data['useragent'] = _req.headers['user-agent'];
+        }
+
+        if (this.objController && this.objController.debugMsg) {
+            response = '【debugMsg:' + this.objController.debugMsg + "】{response}";
+        }
+
+        data['response'] = response ? response.substr(0, 3000) : '';
+        data['code'] = code;
+        let curTime = new Date().getTime();
+        data['delay'] = (curTime - _startTime) / 1000;
+
+        data['server_ip'] = getServerIp();
+        if (this.objController) {
+            data['client_ip'] = getClientIp(this.objController._req);
+        } else {
+            data['client_ip'] = '0.0.0.0';
+        }
+
+        let pushData = {
+            'message': data,
+            'type': TYPE_SELF_CALL,
+            'time': php.date('Y-m-d H:i:s', curTime / 1000)
+        };
+
+        writeRedis(pushData);
+    }
+
+    logModuleCall(method, toUrl, postData, response, startTime) {
+        let data = {};
+        data['from_call_id'] = this.getCallId();
+        data['from_url'] = '/'.CONTROLLER_NAME + '/' + ACTION_NAME;
+        data['method'] = method;
+
+        let parts = explode('?', toUrl);
+        data['to_url'] = parts[0];
+        let getParam = parts[1];
+
+        let postParam = '';
+        if (php.is_array(postData)) {
+            postParam = php.http_build_query(postData);
+        } else {
+            postParam = postData;
+        }
+
+        let param = '';
+        if (getParam && postParam) {
+            param = getParam + '&' + postParam;
+        } else if (getParam) {
+            param = getParam;
+        } else {
+            param = postParam;
+        }
+
+        data['param'] = param;
+        if (response !== false) {
+            let objResult = JSON.parse(response);
+            data['code'] = objResult['code'];
+            data['response'] = response.substr(0, 3000);
+        } else {
+            data['code'] = CODE_REQUEST_TIMEOUT;
+        }
+
+        data['delay'] = new Date().getTime() - startTime;
+        data['server_ip'] = getServerIp();
+
+        let pushData = {
+            message: data,
+            type: TYPE_MODULE_CALL,
+            time: php.date('Y-m-d H:i:s', startTime / 1000)
+        };
+
+        writeRedis(pushData);
+    }
+
+}
+
+function writeRedis(pushData) {
+    try {
+        let config = GLOBALS['redisInfo']['logstash_redis'];
+        if (config) {
+            objRedis.rpush('logstash:redis', JSON.stringify(pushData));
+        }
+    } catch (ex) {
+        // 这里需要告警
+        if (DEBUG) {
+            throw ex;
+        }
+    }
+}
+
+module.exports = CallLog;
+

protected/framework/lib/OujMySql.js

@@ -0,0 +1,194 @@
+"use strict";
+
+const Mysql = require('mysql');
+const AppErrors = require('./AppErrors.js');
+const Tool = require('./Tool.js');
+
+let objPools = {};
+
+/**
+ * 基本类，提供增删改查
+ * @param {string} dbKey
+ */
+class OujMySql {
+
+    constructor(dbKey) {
+        this.dbKey = dbKey || 'default';
+        this.sqls = [];
+        this.num = 0;
+    }
+
+    getPool() {
+        if (!objPools[this.dbKey]) {
+            let config = GLOBALS.dbInfo[this.dbKey];
+            if (config) {
+                config['host'] = config['host'] || config['dbHost'];
+                config['user'] = config['user'] || config['dbUser'];
+                config['password'] = config['password'] || config['dbPass'];
+                config['database'] = config['database'] || config['dbName'];
+                config['port'] = config['port'] || config['dbPort'];
+                config['connectionLimit'] = config['connectionLimit'] || 100;
+                config['dateStrings'] = true;
+                objPools[this.dbKey] = Mysql.createPool(config);
+            } else {
+                throw new AppErrors.DbError("数据库:" + this.dbKey + "没有配置");
+            }
+        }
+
+        return objPools[this.dbKey];
+    }
+
+    query(sql, values) {
+        let that = this;
+        return new Promise(function(resolve, reject) {
+            let index = that.num++;
+            let time = (new Date).getTime();
+            that.getPool().query(sql, values, function(err, result) {
+                let timespan = ((new Date).getTime() - time) / 1000;
+                sql = sql.substr(0, 8000);
+                // 只记录前10000条
+                // if (index < 10000) {
+                    // that.sqls[index] = `[${that.dbKey}] [${index}] [exec:${timespan}] ${sql}`;
+                // }
+                Tool.log(`[${that.dbKey}] [exec:${timespan}] ${sql}`);
+
+                if (err) {
+                    reject(err);
+                    Tool.error(`DB ERROR. dbKey:${that.dbKey}, sql:${sql}`);
+                    throw new AppErrors.DbError(err);
+                } else {
+                    resolve(result);
+                }
+            });
+        });
+    };
+
+    /**
+     * 更新数据
+     * @param {string} sql
+     */
+    update(sql) {
+        return this.query(sql, []);
+    };
+
+    /**
+     * 获取一个值
+     * @param {string}  sql
+     * @return Promise
+     */
+    getOne(sql) {
+        let that = this;
+        return new Promise(function(resolve, reject) {
+            let p1 = that.getAll(sql, 1);
+            p1.then(function(rows) {
+                let row = rows[0];
+                let val = null;
+                if (row) {
+                    let keys = Object.keys(row);
+                    val = keys && row[keys[0]];
+                }
+                resolve(val);
+            }, reject);
+        });
+    };
+
+    /**
+     * 获取一列数据
+     * @param sql
+     * @param limit 行数
+     */
+    getCol(sql, limit) {
+        let that = this;
+        return new Promise(function(resolve, reject) {
+            let p1 = that.getAll(sql, limit);
+            p1.then(function(rows) {
+                let cols = [];
+                if (rows[0] && rows.length) {
+                    let keys = Object.keys(rows[0]);
+                    let key = keys[0];
+
+                    for (let i = 0; i < rows.length; i++) {
+                        cols.push(rows[i][key]);
+                    }
+                }
+
+                resolve(cols);
+            }, reject);
+        });
+    };
+
+    /**
+     * 获取一行数据
+     * @param sql
+     */
+    getRow(sql) {
+        let that = this;
+        return new Promise(function(resolve, reject) {
+            let p1 = that.getAll(sql, 1);
+
+            p1.then(function(rows) {
+                resolve(rows[0] || null);
+            }, reject);
+        });
+    };
+
+    /**
+     * 获取所有行数据
+     * @param sql
+     * @param limit 行数
+     */
+    getAll(sql, limit) {
+        if (typeof limit == 'number' && limit > 0) {
+            sql += " LIMIT " + limit;
+        }
+
+        return this.query(sql, []);
+    };
+
+    /**
+     * 关闭连接池
+     */
+    close() {
+        return new Promise(function(resolve, reject) {
+            this.getPool().end(function(err) {
+                if (err) {
+                    reject(err);
+                } else {
+                    resolve();
+                }
+            });
+        });
+    };
+
+    /**
+     * 转义需要插入或者更新的字段值
+     *
+     * 在所有查询和更新的字段变量都需要调用此方法处理数据
+     *
+     * @param mixed $str 需要处理的变量
+     * @return mixed 返回转义后的结果
+     */
+    escape(str) {
+        if (typeof str === 'object') {
+            for (let key in str) {
+                str[key] = this.escape(str[key]);
+            }
+        } else {
+            return this.getPool().escape(str);
+        }
+
+        return str;
+    }
+
+    getSql() {
+        return this.sqls;
+    };
+
+    clearSql() {
+        this.sqls = [];
+        this.num = 0;
+    };
+
+}
+
+module.exports = OujMySql;

protected/framework/lib/OujRedis.js

@@ -0,0 +1,37 @@
+"use strict";
+
+let Redis = require('ioredis');
+let map = {};
+
+class OujRedis {
+    /**
+     * 初始化Redis对象
+     * @param cacheKey
+     * @return {Redis}
+     */
+    static init(cacheKey) {
+        cacheKey = cacheKey || 'default';
+        if (!map[cacheKey]) {
+            let redisInfo = GLOBALS.redisInfo;
+            if (!redisInfo[cacheKey]) {
+                console.error('OujRedis can not find ' + cacheKey)
+            }
+            redisInfo[cacheKey]['lazyConnect'] = true;
+            redisInfo[cacheKey]['password'] = redisInfo[cacheKey]['password'] || redisInfo[cacheKey]['pwd'];
+            map[cacheKey] = new Redis(redisInfo[cacheKey]);
+        }
+
+        return map[cacheKey];
+    }
+
+    static async endAll() {
+        for (let cacheKey in map) {
+            let objRedis = map[cacheKey];
+            if (objRedis.status !== 'end') {
+                await objRedis.end();
+            }
+        }
+    }
+}
+
+module.exports = OujRedis;

protected/framework/lib/Param.js

@@ -0,0 +1,468 @@
+"use strict";
+
+let validator = require('validator');
+let AppErrors = require('./AppErrors.js');
+// let OujRedis = require('./OujRedis.js');
+let _ = require('underscore');
+
+class Param {
+
+    constructor(objResponse) {
+        this.objResponse = objResponse;
+    }
+
+    /**
+     * 检查参数
+     * @param rules 检查规则
+     * {
+     *  'appId' : 'int',    //int类型
+     *  'owners' : 'array',     //array类型
+     *  'instanceIds' : 'intArr',   //array类型，元素为int类型
+     *  'instanceTypes' : 'strArr',     //array类型，元素为string类型
+     *  'deviceId' : 'int/array',       //int类型或者array类型，最后转化为元素为idArr类型
+     *  'deviceClass' : 'string/array',     //string类型或者array类型，最后转化为strArr类型
+     *  'blocks' : {type : 'int', range : '(5, 10)'}    //int类型， > 5 , < 10
+     *  'blocks2' : {type : 'int', range : '[5, 10]'}   //int类型， >= 5 , <= 10
+     *  'percent' : {type : 'float', range : '[5.1, 10.9]'}     //float类型，>= 5.1 , <= 10.9
+     *  'appName' : {type : 'string'}       //string类型
+     *  'appName2' : {type : 'string', reg : '[^0-9A-Za-z]', 'len' : '[1, 10]', 'nullable' : true}      //string类型，支持正则
+     * }
+     * @param args 参数合集
+     * @param exitError 遇到错误是否直接exit
+     * @static
+     * @return boolean 是否检查通过
+     */
+    checkParam(rules, args, exitError) {
+        exitError = exitError === false ? exitError : true;
+
+        //_private.getRule(rules, args);
+
+        for (let i in rules) {
+            let result = _private.checkRule(rules[i], args, i);
+
+            if (!result['result']) {
+                if (exitError) {
+                    this.objResponse.error(CODE_PARAM_ERROR, null, result['msg']);
+
+                    // 仅仅是中断后续运行
+                    throw new AppErrors.Interrupt(result['msg']);
+                } else {
+                    return result;
+                }
+            }
+        }
+
+        return _private.succ;
+    }
+
+    checkParam2(rules, args, exitError) {
+        let flag = this.checkParam(rules, args, exitError);
+
+        for (let i in args) {
+            if (!_.has(rules, i)) {
+                delete (args[i]);
+            }
+        }
+
+        return flag;
+    }
+
+}
+
+
+let _private = {
+    succ: {
+        result: true
+    },
+
+    makeRules: function (rules, key) {
+
+    },
+
+    getRule: function (rules, args) {
+        if (args['__getRules']) {
+            let params = args['__params'];
+            params['rules'] = rules;
+
+            for (let i in rules) {
+                let value = params['params'][i];
+                if (Array.isArray(rules[i])) {
+                    value['type'] = rules[i]['type'] || rules[i][0];
+                    value['rule'] = _private.genDoc(rules[i]);
+                } else if (value) {
+                    value['type'] = rules[i];
+                }
+                params['params'][i] = value;
+            }
+
+            // let config = GLOBALS.redisInfo.logstash_redis;
+            // let objRedis = OujRedis.init('logstash_redis');
+            //if (config && objRedis) {
+            //    Response.exitMsg(objRedis);
+            //}
+        }
+    },
+
+    genDoc: function (rules) {
+        let str = '';
+        if (rules['nullable']) {
+            str += '【可为null】';
+        }
+
+        if (rules['emptyable']) {
+            str += '【可为空值】';
+        }
+
+        if (rules['len']) {
+            str += "【长度范围：{$rule['len']}】";
+        }
+
+        if (rules['range']) {
+            str += "【取值范围：{$rule['range']}】";
+        }
+
+        if (rules['reg']) {
+            str += "【正则：{$rule['reg']}】";
+        }
+
+        if (rules['enum']) {
+            str += '【值枚举：' + JSON.stringify(rules['enum']) + '】';
+        }
+
+        return str;
+    },
+
+    checkRule: function (rules, args, key) {
+        let type = rules['type'] || rules,
+            that = _private,
+            result;
+
+        if (type == "int/array") {
+            result = that.checkIntArr(rules, args, key);
+        } else if (type == "string/array") {
+            result = that.checkStrArr(rules, args, key);
+        } else if (type) {
+            let funcName = "check" + type.charAt(0).toUpperCase() + type.slice(1);
+            result = that[funcName].apply(that, [rules, args, key]);
+        } else {
+            return that.error(key + ': type must not be empty!');
+        }
+        return result;
+    },
+
+    error: function (msg) {
+        return {result: false, msg: msg};
+    },
+
+    checkBase: function (rules, args, key) {
+        let value = args[key],
+            that = _private;
+
+        //判断是否可为空
+        if (rules['nullable'] && value == null) {
+            return _.extend({
+                nullable: true
+            }, that.succ);
+        }
+
+        //判断是否可为0或空字符串
+        if ((rules['nullable'] || rules['emptyable']) && !value && value != null) {
+            return _.extend({
+                emptyable: true
+            }, that.succ);
+        }
+
+        //判断是否在enum中
+        if (rules['enum'] && rules['enum'].indexOf(value) == -1) {
+            return that.error(key + ':' + args[key] + ' is not in ' + rules['enum']);
+        }
+
+        //判断是否为空
+        if (!value) {
+            return that.error(key + ' is null or empty!');
+        }
+
+        return that.succ;
+    },
+
+    checkRange: function (rules, args, key) {
+        let range = rules['range'],
+            that = _private;
+
+        if (range) {
+            range = range.trim();
+            let ranges = range.split(',');
+            let errMsg = key + ' is not in range ' + range;
+            let from = parseFloat(ranges[0].substring(1).trim());
+
+            if (from !== '-' && from !== '~') {
+                let flag = ranges[0].charAt(0);
+                if (flag === '[' && args[key] < from) {
+                    return that.error(errMsg);
+                } else if (flag === '(' && args[key] <= from) {
+                    return that.error(errMsg);
+                }
+            }
+
+            let to = parseFloat(ranges[1].slice(0, -1));
+            if (to != '+' && to != '~') {
+                let flag = ranges[1].slice(-1);
+                if (flag === ']' && args[key] > to) {
+                    return that.error(errMsg);
+                } else if (flag === ')' && args[key] >= to) {
+                    return that.error(errMsg);
+                }
+            }
+        }
+
+        return that.succ;
+    },
+
+    checkLen: function (rules, args, key) {
+        let len = rules['len'],
+            that = _private;
+
+        if (len) {
+            len = len.trim();
+            let ranges = len.split(',');
+            let errMsg = key + ' is not valid. len must in' + len;
+            let from = parseFloat(ranges[0].substring(1).trim());
+            let strLength = args[key].length;
+
+            if (from != '-' && from != '~') {
+                let flag = ranges[0].charAt(0);
+                if (flag === '[' && strLength < from) {
+                    return that.error(errMsg);
+                } else if (flag === '(' && strLength <= from) {
+                    return that.error(errMsg);
+                }
+            }
+
+            let to = parseFloat(ranges[1].slice(0, -1));
+            if (to != '+' && to != '~') {
+                let flag = ranges[1].slice(-1);
+                if (flag === ']' && strLength > to) {
+                    return that.error(errMsg);
+                } else if (flag === ')' && strLength >= to) {
+                    return that.error(errMsg);
+                }
+            }
+        }
+
+        return that.succ;
+    },
+
+    checkDefault: function (rules, args, key) {
+        let that = _private;
+
+        if (rules['emptyable']) {
+            rules['emptyable'] = true;
+        }
+
+        let result = that.checkBase(rules, args, key);
+
+        if (!result['result'] || result['nullable']) {
+            return result;
+        }
+
+        result = that.checkRange(rules, args, key);
+
+        if (!result['result']) {
+            return result;
+        }
+
+        return that.succ;
+    },
+
+    checkInt: function (rules, args, key) {
+        let that = _private;
+
+        if (rules['emptyable']) {
+            rules['emptyable'] = true;
+        }
+
+        let result = that.checkBase(rules, args, key);
+
+        if (!result['result'] || result['nullable']) {
+            return result;
+        }
+
+        if (!validator.isInt(args[key])) {
+            return that.error(key + ': ' + args[key] + ' is not int!');
+        }
+
+        result = that.checkRange(rules, args, key);
+
+        if (!result['result']) {
+            return result;
+        }
+
+        return that.succ;
+    },
+
+    checkIp: function (rules, args, key) {
+        let that = _private;
+
+        let result = that.checkBase(rules, args, key);
+
+        if (!result['result'] || result['nullable']) {
+            return result;
+        }
+
+        if (!validator.isIP(args[key], 4)) {
+            return that.error(key + ': ' + args[key] + ' is not valid ip format!');
+        }
+
+        return that.succ;
+    },
+
+    checkFloat: function (rules, args, key) {
+        let that = _private;
+
+        let result = that.checkBase(rules, args, key);
+
+        if (!result['result'] || result['nullable']) {
+            return result;
+        }
+
+        if (!validator.isFloat(args[key])) {
+            return that.error(key + ': ' + args[key] + ' is not float!');
+        }
+
+        result = that.checkRange(rules, args, key);
+
+        if (!result['result']) {
+            return result;
+        }
+
+        return that.succ;
+    },
+
+    checkString: function (rules, args, key) {
+        let that = _private;
+
+        if (args[key] !== null && args[key]) {
+            args[key] = args[key].toString().trim();
+        }
+
+        let result = that.checkBase(rules, args, key);
+
+        if (!result['result'] || result['nullable']) {
+            return result;
+        }
+
+        result = that.checkLen(rules, args, key);
+
+        if (!result['result']) {
+            return result;
+        }
+
+        if (rules['reg']) {
+            let reg = new RegExp(rules['reg']);
+            if (reg.test(args[key])) {
+                return that.error(key + ' preg_match error! The reg rule is:' + rules['reg']);
+            }
+        }
+
+        return that.succ;
+    },
+
+    checkArray: function (rules, args, key) {
+        let that = _private;
+
+        let result = that.checkBase(rules, args, key);
+
+        if (!result['result'] || result['nullable']) {
+            return result;
+        }
+
+        if (!Array.isArray(args[key])) {
+            return that.error(key + ' is not array');
+        }
+
+        return that.succ;
+    },
+
+    checkJson: function (rules, args, key) {
+        let that = _private;
+
+        let result = that.checkBase(rules, args, key);
+
+        if (!result['result'] || result['nullable']) {
+            return result;
+        }
+
+        if (!validator.isJSON(args[key])) {
+            return that.error(key + ' is not JSON');
+        }
+
+        return that.succ;
+    },
+
+    checkObject: function (rules, args, key) {
+        let that = _private;
+
+        let result = that.checkBase(rules, args, key);
+
+        if (!result['result'] || result['nullable']) {
+            return result;
+        }
+
+        if (!rules['items']) {
+            return that.succ;
+        }
+
+        for (let i in rules['items']) {
+            result = that.checkRule(rules['items'][i], args[key], i);
+            if (!result['result']) {
+                result['msg'] = {
+                    parent: key
+                }
+                return result;
+            }
+        }
+
+        return that.succ;
+    },
+
+    checkIntArr: function (rules, args, key) {
+        let that = _private;
+        return that._checkRuleArr(rules, args, key, 'int');
+    },
+
+    checkStrArr: function (rules, args, key) {
+        let that = _private;
+        return that._checkRuleArr(rules, args, key, 'string');
+    },
+
+    checkIpArr: function (rules, args, key) {
+        let that = _private;
+        return that._checkRuleArr(rules, args, key, 'ip');
+    },
+
+    _checkRuleArr: function (rules, args, key, type) {
+        let that = _private;
+
+        let result = that.checkArray(rules, args, key);
+
+        if (!result['result'] || result['nullable']) {
+            return result;
+        }
+
+        for (let i in args[key]) {
+            result = that.checkRule(type, args[key], i);
+            if (!result['result']) {
+                result['msg'] = {
+                    parent: key
+                }
+                return result;
+            }
+        }
+
+        return that.succ;
+    }
+};
+
+
+module.exports = Param;
+

protected/framework/lib/Redis2MySql.js

@@ -0,0 +1,375 @@
+"use strict";
+
+var OujRedis = require('./OujRedis.js');
+var TableHelper = require('./TableHelper.js');
+var r2m_configs = require('../../conf/r2m_config.inc.js');
+var php = require('phpjs');
+var co = require('co');
+
+/**
+ * 基本类，提供增删改查
+ * @param {string} dbKey
+ * @author benzhan
+ */
+class Redis2MySql {
+
+    constructor(tableName, dbKey, cacheKey) {
+        this.tableName = tableName;
+        this.dbKey = dbKey || 'default';
+        this.cacheKey = cacheKey || 'default';
+        this.cacheInfo = r2m_configs[this.dbKey][tableName];
+        if (!this.cacheInfo) {
+            throw new Error("redis没配置table name:" + tableName, CODE_REDIS_ERROR);
+        }
+
+        /**
+         * @type {TableHelper}
+         */
+        this.objTable = new TableHelper(tableName, this.dbKey);
+        /**
+         * @type {Redis}
+         */
+        this.objRedis = OujRedis.init(this.cacheKey);
+    }
+
+
+    /**
+     * 获取行的key
+     * @param args
+     * @returns {string}
+     * @private
+     */
+    _getRowKey(args) {
+        var keys = this.cacheInfo['key'].split(',');
+        var cacheKeys = [];
+        keys.map(function(key) {
+            key = key.trim();
+            if (args[key]) {
+                if (Array.isArray(args[key])) {
+                    cacheKeys.push(key + "=" + args[key].join('|'));
+                } else {
+                    cacheKeys.push(key + "=" + args[key]);
+                }
+            }
+        });
+
+        var cacheKey = this.tableName + ":row";
+        if (cacheKey) {
+            return cacheKey + ':' + cacheKeys.join(':');
+        } else {
+            return cacheKey;
+        }
+    }
+
+    /**
+     * 获取getAll的key
+     * @param args
+     * @returns {string}
+     * @private
+     */
+    _getAllKey(args) {
+        var cacheKey = this.tableName + ":all";
+        var otherCacheKey = cacheKey + ":others";
+
+        var key = this.cacheInfo['all_key'];
+        if (key) {
+            var keys = key.split(',');
+            var cacheKeys = [];
+            keys.map(function(key) {
+                key = key.trim();
+                if (Array.isArray(args[key]) || typeof args[key] == 'undefined' || args[key] === null) {
+                    // 如果有数组，或者有key不存在，则归类到others
+                    return otherCacheKey;
+                } else {
+                    cacheKeys.push(key + "=" + args[key]);
+                }
+            });
+
+            if (cacheKeys) {
+                return cacheKey + ':' + cacheKeys.join(':');
+            } else {
+                return otherCacheKey;
+            }
+        } else {
+            return otherCacheKey;
+        }
+    }
+
+    /**
+     * 获取一个key的数据
+     * @param where
+     * @return <NULL, array>
+     * @public
+     */
+    getRow(where) {
+        var cacheKey = this._getRowKey(where);
+        var that = this;
+        return co(function*() {
+            var data = yield that.objRedis.hgetall(cacheKey);
+            if (php.empty(data)) {
+                // 从数据库重建
+                var row = yield that.objTable.getRow(where);
+                if (!php.empty(row)) {
+                    // 设置缓存，无需等待成功就能返回
+                    that._setRowCache(row);
+                }
+                return row;
+            } else {
+                return data;
+            }
+        });
+    }
+
+    /**
+     * 读取多行数据
+     * @param where
+     * @param keyWord 查询关键字, array('_field', '_where', '_limit', '_sortKey', '_sortDir', '_lockRow', '_tableName')
+     * @param updateList 是否强制更新缓存
+     * @return array:
+     * @public
+     */
+    getAll(where, keyWord, updateList) {
+        var that = this;
+        var args = Object.assign({}, where, keyWord);
+        var key = php.http_build_query(args);
+        if (key.length > 32) {
+            key = php.md5(key);
+        } else {
+            key = php.http_build_query(args);
+        }
+
+        return co(function*() {
+            var cacheKey = that._getAllKey(where) + ":" + key;
+            var data = null;
+            if (!updateList) {
+                data = yield that.objRedis.get(cacheKey);
+                if (data) {
+                    return JSON.parse(data);
+                }
+            }
+
+            data = yield that.objTable.getAll(where, keyWord);
+
+            // 以下内容只是用来同步redis用的，可以异步跳过
+            var pipeline = that.objRedis.pipeline();
+            pipeline.set(cacheKey, JSON.stringify(data));
+            if (that.cacheInfo['ttl'] > 0) {
+                pipeline.expire(cacheKey, that.cacheInfo['ttl']);
+            }
+            pipeline.exec();
+
+            return data;
+        });
+    }
+
+
+    /**
+     * 增加一行数据
+     * @param args
+     * @param updateList
+     * @return int
+     * @public
+     */
+    addObject(args, updateList) {
+        updateList = updateList === false ? updateList : true;
+        var that = this;
+
+        return co(function*() {
+            var ret = yield that.objTable.addObject(args);
+            if (updateList) {
+                that.delListCache(args);
+            }
+
+            return ret;
+        });
+    }
+
+    /**
+     * 删除列表的缓存
+     * @param where
+     * @public
+     */
+    delListCache(where) {
+        var that = this;
+        var cacheKey = that._getAllKey(where);
+        var otherKey = that.tableName + ":all:others";
+        return co(function*() {
+            var keys = [];
+            if (cacheKey === otherKey) {
+                // 需要清除所有key删除
+                cacheKey = that.tableName + ":all";
+                keys = yield that.objRedis.keys(cacheKey + '*');
+            } else {
+                // 除了删除当前keys，还需要删除others
+                keys = yield that.objRedis.keys(cacheKey + '*');
+                var keys1 = yield that.objRedis.keys(otherKey + '*');
+                keys = Object.assign({}, keys, keys1);
+            }
+
+            if (keys.length > 0) {
+                that.objRedis.del(keys);
+            }
+
+            return 1;
+        });
+    }
+
+    /**
+     * 设置行的缓存
+     * @param args
+     * @returns {Array|{index: number, input: string}|*|{arity, flags, keyStart, keyStop, step}}
+     * @private
+     */
+    _setRowCache(args) {
+        var cacheKey = this._getRowKey(args);
+        if (!cacheKey) {
+            var msg = "没设置key:cacheKey,".JSON.stringify(this.cacheInfo);
+            throw new Error(msg, CODE_REDIS_ERROR);
+        }
+
+        var pipeline = this.objRedis.pipeline();
+        pipeline.hmset(cacheKey, args);
+        if (this.cacheInfo['ttl'] > 0) {
+            pipeline.expire(cacheKey, this.cacheInfo['ttl']);
+        }
+
+        return pipeline.exec();
+    }
+
+    /**
+     * 更新行缓存
+     * @param args
+     * @returns {*}
+     * @private
+     */
+    _updateRowCache(args) {
+        var cacheKey = this._getRowKey(args);
+        if (!cacheKey) {
+            var msg = "没设置key:cacheKey,".JSON.stringify(this.cacheInfo);
+            throw new Error(msg, CODE_REDIS_ERROR);
+        }
+
+        var that = this;
+        return co(function *() {
+            var flag = yield that.objRedis.exists(cacheKey);
+            if (flag == 1) {
+                var pipeline = that.objRedis.pipeline();
+                pipeline.hmset(cacheKey, args);
+
+                if (that.cacheInfo['ttl'] > 0) {
+                    pipeline.expire(cacheKey, that.cacheInfo['ttl']);
+                }
+
+                yield pipeline.exec();
+
+                return 1;
+            }
+        });
+    }
+
+
+    /**
+     * 修改一个key的数据
+     * @param args 更新的内容
+     * @param where 更新的条件
+     * @param updateList
+     * @return int 影响行数
+     * @public
+     */
+    updateObject(args, where, updateList) {
+        var that = this;
+        updateList = updateList === false ? updateList : true;
+
+        return co(function*() {
+            var result = yield that.objTable.updateObject(args, where);
+            args = php.array_merge(args, where);
+            if (result.affectedRows > 0) {
+                yield that._updateRowCache(args);
+
+                if (updateList) {
+                    that.delListCache(where);
+                }
+            }
+
+            return result;
+        });
+    }
+
+
+    /**
+     * 设置一个key的数据
+     * @param args
+     * @param updateList
+     * @return int 影响行数
+     * @public
+     */
+    replaceObject(args, updateList) {
+        var that = this;
+        return co(function*() {
+            var result = yield that.objTable.replaceObject(args);
+            yield that._setRowCache(args);
+
+            if (updateList) {
+                that.delListCache(args);
+            }
+
+            return result;
+        });
+    }
+
+    /**
+     * 删除数据
+     * @param where
+     * @param updateList
+     * @throws RedisException
+     * @return unknown
+     * @public
+     */
+    delObject(where, updateList) {
+        var that = this;
+        updateList = updateList === false ? updateList : true;
+
+        return co(function*() {
+            var result = yield that.objTable.delObject(where);
+
+            that._delRowCache(where);
+            if (updateList) {
+                that.delListCache(where);
+            }
+
+            return result;
+        });
+    }
+
+    /**
+     * 删除行的缓存
+     * @param where
+     * @throws RedisException
+     * @private
+     */
+    _delRowCache(where) {
+        var cacheKey = this._getRowKey(where);
+        if (!cacheKey) {
+            var msg = "没设置key:cacheKey,".JSON.stringify(this.cacheInfo);
+            throw new Error(msg, CODE_REDIS_ERROR);
+        }
+
+        return this.objRedis.del(cacheKey);
+    }
+
+    /**
+     * 关闭连接
+     * @public
+     */
+    close() {
+        if (this.objTable) {
+            this.objTable.close();
+        }
+
+        if (this.objRedis) {
+            this.objRedis.disconnect();
+        }
+    }
+}
+
+module.exports = Redis2MySql;

protected/framework/lib/Response.js

@@ -0,0 +1,106 @@
+"use strict";
+
+let CallLog = require('./CallLog.js');
+
+class Response {
+
+    constructor(req, res) {
+        this._req = req;
+        this._res = res;
+        this.codeMap = require('../../conf/code.inc.js');
+    }
+
+    /**
+     * sucess
+     * @param {Array|String|Number} data 返回的数据
+     * @param {String} msg 返回的msg
+     * @param {String} debugMsg 调试的msg
+     * @author benzhan
+     */
+    success(data, msg, debugMsg) {
+        let code = CODE_SUCCESS;
+        this.debugMsg = debugMsg;
+
+        msg = msg || this.codeMap[code];
+        if (global['DEBUG'] && debugMsg) {
+            msg = msg + " 【调试信息:" + debugMsg + "】";
+        }
+
+        let ret = {
+            result: 1,
+            code: code,
+            msg: msg,
+            data: data
+        };
+
+        this.exitData(ret);
+    }
+
+    /**
+     * error with code
+     * @param {Number} code
+     * @param {String} msg
+     * @param {String} debugMsg
+     * @param {String} extData
+     * @author benzhan
+     */
+    error(code, msg, debugMsg, extData) {
+        this.debugMsg = debugMsg;
+
+        msg = msg || this.codeMap[code];
+        if (global['DEBUG'] && debugMsg) {
+            msg = msg + " 【调试信息:" + debugMsg + "】";
+        }
+
+        let ret = {
+            result: 0,
+            code: code,
+            msg: msg
+        };
+
+        if (extData) {
+            ret['data'] = extData;
+        }
+
+        this.exitData(ret);
+    }
+
+    exitData(ret) {
+        let json = JSON.stringify(ret);
+        if (!this._res._headerSent) {
+            this._res.header('Content-Type', 'application/json;charset=' + DEFAULT_CHARSET);
+        }
+        this.exitMsg(json, ret['code']);
+    }
+
+    exitMsg(content, code) {
+        let res = this._res;
+        let req = this._req;
+        code = code || CODE_SUCCESS;
+
+        //必须是字符串
+        if (typeof content !== "string") {
+            content = JSON.stringify(content);
+        }
+
+        //jquery jsonp callback处理
+        if (req.query && req.query.callback) {
+            if (/^jQuery(\d+)_(\d+)$/.test(req.query.callback)) {
+                content = req.query.callback + '(' + content + ');';
+            }
+        }
+
+        // 记录访问日志
+        let objCallLog = new CallLog(this);
+        objCallLog.logSelfCall(code, content);
+
+        if (!res.finished) {
+            res.write(content);
+            res.end();
+        }
+    }
+
+}
+
+
+module.exports = Response;

protected/framework/lib/Router.js

@@ -0,0 +1,105 @@
+"use strict";
+
+var php = require('phpjs');
+let Response = require('./Response.js');
+
+class Router {
+    constructor(url) {
+        //var parts = url.match("http[s]*://[^/]+/([^/]+)/([^/?]+)");
+        let parts = url.match(/(http[s]*:\/\/[^/]+)?\/([^/]+)\/([^/?]+)/);
+        if (parts && parts.length >= 4) {
+            this.controllerName = parts[2];
+            this.actionName = parts[3];
+        } else {
+            this.controllerName = 'default';
+            parts = url.match(/(http[s]*:\/\/[^/]+)?\/([^/?]+)/);
+            if (parts && parts.length >= 3) {
+                this.actionName = parts[2];
+            } else {
+                this.actionName = 'index';
+            }
+        }
+
+        global['CONTROLLER_NAME'] = this.controllerName;
+        global['ACTION_NAME'] = this.actionName;
+    }
+
+
+    getControllerName() {
+        return this.controllerName;
+    };
+
+    getActionName() {
+        return this.actionName;
+    }
+
+    getController() {
+        var controllerName = php.ucfirst(this.controllerName);
+        var objController;
+        try {
+            objController = require('../../controllers/' + controllerName + 'Controller.js');
+            return objController;
+        } catch (ex) {
+            console.error(ex);
+            return null;
+        }
+    }
+
+    getFullActionName() {
+        return 'action' + php.ucfirst(this.actionName);
+    }
+
+    /**
+     * 文档处理方法
+     * @param type     查询对象类型【module, class, func】
+     */
+    genDoc(type) {
+        var className = this.controllerName,
+            funcName = this.getFullActionName();
+
+        className = className.charAt(0).toUpperCase() + className.slice(1);
+        var Doc = require('../Doc.js');
+        var Controller = require('../Controller.js');
+
+        var doc = new Doc();
+        var docController = new Controller;
+
+
+        switch (type) {
+            case "module" :
+                var classInfos = doc.getClassInfos(__dirname + '/../../controllers');
+                docController.assign({classInfos: classInfos});
+                docController.display('doc');
+                break;
+
+            case "class" :
+                var classInfo = doc.getClassInfo(className);
+                docController.assign({classInfos: classInfo});
+                docController.display('doc');
+                break;
+
+            case "func" :
+                var params = doc.getFuncInfo(className, funcName) || {
+                        params: {},
+                        rules: {}
+                    };
+                var args = {
+                    "__getRules": true,
+                    "__params": params
+                };
+
+                var Controller2 = require('../../controllers/' + className + 'Controller.js');
+                var _controller = new Controller2;
+
+                _controller[funcName].apply(this, [args]);
+                break;
+
+            default :
+                var msg = 'can not find doc type ' + type;
+                Response.error(CODE_NOT_EXIST_INTERFACE, msg);
+        }
+    }
+}
+
+
+module.exports = Router;

protected/framework/lib/TableHelper.js

@@ -0,0 +1,298 @@
+"use strict";
+
+let MySql = require('./OujMySql');
+let php = require('phpjs');
+
+class TableHelper {
+    /**
+     * 基本类，提供增删改查
+     * @param {string} tableName 表名
+     * @param {string} dbKey
+     */
+    constructor(tableName, dbKey) {
+        this.tableName = tableName;
+        this.dbKey = dbKey || 'default';
+        this.objMySql = new MySql(this.dbKey);
+    }
+
+    setTableName(tableName) {
+        this.tableName = tableName;
+    }
+
+    getCount(where, keyWord) {
+        keyWord = keyWord || {};
+        keyWord['_field'] = 'COUNT(*)';
+        return this.getOne(where, keyWord);
+    }
+
+    /**
+     * 读取数据
+     * @param  {object} where   参数列表，特殊参数前缀
+     * @param  {object} keyWord 查询关键字['_field', '_where', '_limit', '_sortKey', '_sortDir', '_lockRow', '_tableName', '_groupby']
+     * @return {object}
+     */
+    getOne(where, keyWord) {
+        let sql = this.buildSelectSql(where, keyWord);
+        return this.objMySql.getOne(sql);
+    }
+
+    getCol(where, keyWord) {
+        let sql = this.buildSelectSql(where, keyWord);
+        return this.objMySql.getCol(sql);
+    }
+
+    getRow(where, keyWord) {
+        let sql = this.buildSelectSql(where, keyWord);
+        return this.objMySql.getRow(sql);
+    }
+
+    getAll(where, keyWord) {
+        let sql = this.buildSelectSql(where, keyWord);
+        return this.objMySql.getAll(sql, 0);
+    }
+
+    updateObject(newData, where) {
+        if (!where) {
+            throw new SQLException('updateObject没有传入where');
+        }
+
+        let _where = this.buildWhereSql(where);
+        let sql = 'UPDATE `' + this.tableName + '` SET ';
+        sql += this.buildValueSql(newData) + ' WHERE ' + _where;
+
+        return this.objMySql.update(sql);
+    }
+
+    delObject(where) {
+        if (!where) {
+            throw new SQLException('updateObject没有传入where');
+        }
+
+        let _where = this.buildWhereSql(where);
+        let sql = 'DELETE FROM `' + this.tableName + '` WHERE ' + _where;
+
+        return this.objMySql.update(sql);
+    }
+
+    /**
+     * 检查表是否存在
+     * @param string tableName 表名
+     * @param string dbName 【可选】数据库名
+     * @return bool true/false
+     */
+    async checkTableExist(tableName, dbName) {
+        if (tableName.indexOf(".") > 0) {
+            let parts = explode(".", tableName);
+            dbName = parts[0];
+            tableName = parts[1];
+        }
+
+        let sql, result;
+        if (dbName) {
+            //------------检查数据库是否存在
+            sql = `SELECT 1 FROM information_schema.SCHEMATA WHERE schema_name = '${dbName}'`;
+            result = await this.objMySql.getOne(sql);
+            if (!result) {
+                return false;
+            }
+
+            //------------检查数据表是否存在
+            sql = `SHOW TABLES FROM dbName LIKE '${tableName}'`;
+            let datas = await this.objMySql.getAll(sql);
+            result = datas[0];
+        } else {
+            //------------检查数据表是否存在
+            sql = `SHOW TABLES LIKE '${tableName}'`;
+            let datas = await this.objMySql.getAll(sql);
+            result = datas[0];
+        }
+
+        return !!result;
+    }
+
+    replaceObject(newData) {
+        let sql = 'REPLACE INTO `' + this.tableName + '` SET ';
+        sql += this.buildValueSql(newData);
+        return this.objMySql.update(sql);
+    }
+
+    addObject(data) {
+        let sql = 'INSERT `' + this.tableName + '` SET ';
+        sql += this.buildValueSql(data);
+
+        return this.objMySql.update(sql);
+    }
+
+    getArrayCol(args) {
+        if (!args || !args.length) { return false; }
+        let value = args[0];
+        return php.array_keys(value);
+    }
+
+    /**
+     * INSERT多行数据
+     * @author benzhan
+     * @param  args array(array(key => $value, ...))
+     */
+    async addObjects2(args) {
+        let cols = this.getArrayCol(args);
+        if (!cols) { return false; }
+
+        return await this.addObjects(cols, args);
+    }
+
+    /**
+     * INSERT多行数据(避免重复插入记录)
+     * @author hawklim
+     * @param array $args array(array(key => $value, ...))
+     */
+    async addObjectsIfNotExist(args) {
+        let cols = this.getArrayCol(args);
+        if (!cols) { return false; }
+
+        return await this._addObjects(cols, args, 'addIfNotExist');
+    }
+
+    /**
+     * INSERT多行数据
+     * @author benzhan
+     * @param cols 列名
+     * @param args 参数列表
+     */
+    async addObjects(col, args) {
+        return this._addObjects(col, args, 'add');
+    }
+
+    async _addObjects(cols, args, type) {
+        type = type || 'add';
+        args = php.array_chunk(args, 3000);
+        for (let datas of args) {
+            await this._addObjects2(cols, datas, type);
+        }
+    }
+
+    getTableName(args) {
+        args = args || {};
+        let tableName = this.tableName;
+        if (args['_tableName']) {
+            tableName = args['_tableName'];
+            delete args['_tableName'];
+        }
+        return tableName;
+    }
+
+    async _addObjects2(cols, args, type) {
+        type = type || 'add';
+            // $sql = ($type == 'add' ? 'INSERT ' : 'REPLACE ');
+        let sql = '';
+        if (type === 'add') {
+            sql = 'INSERT ';
+        } else if (type === 'addIfNotExist') {
+            sql = 'INSERT IGNORE ';
+        } else {
+            sql = 'REPLACE ';
+        }
+
+        args = this.objMySql.escape(args);
+        sql += this.tableName + " (`" + cols.join("`,`") + "`) VALUES ";
+        for (let value of args) {
+            sql += "(" + php.join(", ", value) + "),";
+        }
+        sql = sql.substr(0, sql.length - 1);
+
+        return await this.objMySql.update(sql);
+    }
+
+    /**
+     * REPLACE多行数据
+     * @author benzhan
+     * @param  args array(array(key => $value, ...))
+     */
+    replaceObjects2(args) {
+        let cols = this.getArrayCol(args);
+        if (!cols) { return false; }
+
+        return this.replaceObjects(cols, args);
+    }
+
+    /**
+     * REPLACE多行数据
+     * @author benzhan
+     * @param cols 列名
+     * @param args 参数列表
+     */
+    replaceObjects(cols, args) {
+        return this._addObjects(cols, args, 'replace');
+    }
+
+    async addObjectNx(args, where) {
+        let count = await this.getCount(where);
+        if (count) { return true; }
+
+        return this.addObject(args);
+    }
+
+    close() {
+        return this.objMySql.close();
+    }
+
+    buildValueSql(data, separator) {
+        separator = separator || ',';
+        let pool = this.objMySql.getPool();
+        let values = [];
+        for (let key in data) {
+            values.push(pool.escapeId(key) + '=' + pool.escape(data[key]));
+        }
+
+        return values.join(separator);
+    }
+
+    buildWhereSql(where, keyWord) {
+        keyWord = keyWord || {};
+        let _sql = keyWord['_where'] || '1';
+        let pool = this.objMySql.getPool();
+
+        where = where || {};
+        for (let key in where) {
+            if (php.is_array(where[key])) {
+                _sql += " AND " + pool.escapeId(key) + "IN (" + pool.escape(where[key]) + ")";
+            } else {
+                _sql += " AND " + pool.escapeId(key) + "=" + pool.escape(where[key]);
+            }
+        }
+
+        if (keyWord['_groupby']) {
+            _sql += " GROUP BY " + keyWord['_groupby'];
+        }
+
+        if (keyWord['_sortKey']) {
+            _sql += " ORDER BY " + keyWord['_sortKey'];
+            if (keyWord['_sortDir']) {
+                _sql += " " + keyWord['_sortDir'];
+            }
+        }
+
+        return _sql;
+    }
+
+    buildSelectSql(where, keyWord) {
+        keyWord = keyWord || {};
+        let _field = keyWord['_field'] || '*';
+        let pool = this.objMySql.getPool();
+
+        let tableName = keyWord['_tableName'] || this.tableName;
+        let _where = this.buildWhereSql(where, keyWord);
+        tableName = pool.escapeId(tableName);
+
+        let sql = `SELECT ${_field} FROM ${tableName} WHERE ${_where} `;
+
+        if (keyWord['_limit']) {
+            sql += ' LIMIT ' + keyWord['_limit'];
+        }
+
+        return sql;
+    }
+
+}
+
+module.exports = TableHelper;

protected/framework/lib/Tool.js

@@ -0,0 +1,100 @@
+"use strict";
+
+const php = require('phpjs');
+
+/**
+ * 调试日志(只有DEBUG为true时, 才记录)
+ * @author benzhan
+ * @param content 调试内容
+ * @param callLevel 调用深度
+ */
+function debug(content, callLevel) {
+    callLevel = callLevel || 1;
+    if (DEBUG) {
+        writeLog(content, 'debug', callLevel);
+    }
+}
+
+/**
+ * 记录流水日志
+ * @author benzhan
+ * @param content 流水日志内容
+ * @param callLevel 调用深度
+ */
+function log(content, callLevel = 1) {
+    callLevel = callLevel || 1;
+    writeLog(content, 'log', callLevel);
+}
+
+/**
+ * 记录警告日志
+ * @author benzhan
+ * @param content 流水日志内容
+ * @param callLevel 调用深度
+ */
+function warning(content, callLevel = 1) {
+    callLevel = callLevel || 1;
+    writeLog(content, 'warning', callLevel);
+}
+
+
+/**
+ * 记录错误日志
+ * @author benzhan
+ * @param content 流水日志内容
+ * @param callLevel 调用深度
+ */
+function err(content, callLevel = 1) {
+    callLevel = callLevel || 1;
+    writeLog(content, 'error', callLevel);
+}
+
+
+let logs = [];
+let recordLog = false;
+
+function startRecordLog() {
+    recordLog = true;
+    logs = [];
+}
+
+function writeLog(content, label = 'log', callLevel = 0, file = null, line = null) {
+
+    if (php.is_array(content)) {
+        content = JSON.stringify(content);
+    }
+
+    let time = php.date('Y-m-d H:i:s');
+    if (content && content.length > 9000) {
+        content = content.substr(0, 9000) + '...';
+    }
+
+    if (label !== 'log') {
+        content = `【${time}】【${label}】 ${content} `;
+    } else {
+        content = `【${time}】 ${content} `;
+    }
+
+
+    let func = console[label] || console.log;
+    func(content);
+
+    if (recordLog) {
+        logs.push(content);
+    }
+}
+
+function stopRecordLog() {
+    recordLog = false;
+    return logs;
+}
+
+module.exports = {
+    debug,
+    log,
+    warning,
+    err,
+    error : err,
+    startRecordLog,
+    stopRecordLog
+};

protected/framework/lib/dwHttp.js

@@ -0,0 +1,185 @@
+const request = require('request-promise');
+const php = require('phpjs');
+const CallLog = require('./CallLog');
+
+class dwHttp {
+
+    constructor() {
+        this.addCallId = true;
+        this.proxy = null;
+    }
+
+    /**
+     * 是否需要添加callId
+     * @param flag
+     */
+    enableCallId(flag) {
+        this.addCallId = flag;
+    }
+
+    setProxy(proxy) {
+        this.proxy = proxy;
+    }
+
+    /**
+     * 通过get方式获取数据
+     * @author ben
+     * @param url 请求地址
+     * @param timeout 超时时间
+     * @param header 请求头部
+     * @param ttl 缓存时间(秒)，默认为0，不缓存
+     * @return string 响应数据
+     */
+    async get(url, timeout = 5, header = "", ttl = 0) {
+        timeout = timeout || 5;
+        header = header || "";
+        ttl = ttl || 0;
+        if (php.empty(url)||php.empty(timeout)) return false;
+
+        let key = '';
+        // if (ttl > 0) {
+        //     key = url;
+        //     let response = this.getCache(key);
+        //     if (response) {
+        //         return response;
+        //     }
+        // }
+
+        let startTime = (new Date).getTime();
+        // if (this.addCallId && php.strpos(url, '_call_id=') === false && php.strpos(url, '&sign=') === false) {
+        //     let and = '&';
+        //     if (php.strpos(url, '?') === false) {
+        //         and = '?';
+        //     }
+        //
+        //     url += and + '_call_id=' . CallLog.getCallId();
+        // }
+
+        let r = request;
+        let options = {
+            url : url,
+            timeout: timeout * 1000,
+            headers : this.getHeaders(header),
+        };
+
+        if (this.proxy) {
+            options['proxy'] = `http://${this.proxy}`;
+        }
+
+        let response = await r(options);
+
+        // if (ttl > 0 && response) {
+        //     this.setCache(key, ttl, response);
+        // }
+
+        // let objCallLog = new CallLog();
+        // objCallLog.logModuleCall("GET", url, null, response, startTime);
+        return response;
+    }
+
+    getHeaders(header) {
+        let headers = {};
+        if (typeof header === 'string') {
+            if (header && header.trim()) {
+                let parts = header.trim().split("\n");
+                for (let part of parts) {
+                    let val = part.split(":");
+                    headers[val[0].trim()] = val[1].trim();
+                }
+            }
+        } else {
+            headers = header;
+        }
+
+        headers['Accept-Language'] = headers['Accept-Language'] || 'en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4';
+        headers['User-Agent'] = headers['User-Agent'] || 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36';
+        return headers;
+    }
+
+    /**
+     * 重试n次，通过get方式获取数据
+     * @author ben
+     * @param url 请求地址
+     * @param times 重试次数，默认为3次
+     * @param firstTimeout 第一次超时时间，后续超时间为：$i * $firstTimeout
+     * @param header 头部信息
+     * @param ttl 缓存时间(秒)，默认为0，不缓存
+     * @return string 响应数据
+     */
+    async get2(url, times = 3, firstTimeout, header, ttl) {
+        times = times || 3;
+        firstTimeout = firstTimeout || 3;
+        header = header || "";
+        ttl = ttl || 0;
+        for (let i = 1; i <= times; i++) {
+            let response = await this.get(url, firstTimeout * i, header, ttl);
+            if (response !== false) {
+                return response;
+            }
+        }
+
+        return false;
+    }
+
+
+    async post2(url, data, times, firstTimeout, header) {
+        times = times || 3;
+        firstTimeout = firstTimeout || 3;
+        header = header || "";
+        for (let i = 1; i <= times; i++) {
+            let response = await this.post(url, data, firstTimeout * i, header);
+            if (response !== false) {
+                return response;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * post
+     * @author ben
+     * @param url 请求地址
+     * @param data 请求数据
+     * @param timeout 超时时间
+     * @param header 请求头部
+     * @return string 响应数据
+     */
+    async post(url, data, timeout, header) {
+        timeout = timeout || 5;
+        header = header || "";
+        if (php.empty(url) || php.empty(timeout)) return false;
+
+        let startTime = (new Date).getTime();
+        // if (this.addCallId && php.strpos(url, '_call_id=') === false && php.strpos(url, '&sign=') === false) {
+        //     let and = '&';
+        //     if (php.strpos(url, '?') === false) {
+        //         and = '?';
+        //     }
+        //
+        //     url += and + '_call_id=' . CallLog.getCallId();
+        // }
+
+        let r = request;
+        let options = {
+            method:'post',
+            url : url,
+            form : data,
+            timeout: timeout * 1000,
+            headers : this.getHeaders(header),
+        };
+
+        if (this.proxy) {
+            options['proxy'] = `http://${this.proxy}`;
+        }
+
+        let response = await r(options);
+
+        // let objCallLog = new CallLog();
+        // objCallLog.logModuleCall("GET", url, null, response, startTime);
+        return response;
+    }
+
+}
+
+module.exports = dwHttp;

protected/index.js

@@ -0,0 +1,30 @@
+#!/usr/bin/env node --harmony --trace_gc
+let cluster = require('cluster');
+let http = require('http');
+
+if (cluster.isMaster) {
+    let num = require('os').cpus().length;
+    num = num / 4;
+    for (let i = 0; i < num; i++) {
+        cluster.fork();
+    }
+
+    cluster.on('exit', function(worker, code, signal) {
+        console.error('worker ' + worker.process.pid + ' died');
+        cluster.fork();
+    });
+
+    cluster.on('listening', function(worker, address) {
+        console.log("A worker with #" + worker.id + " is now connected to "
+            + address.address + ":" + address.port);
+    });
+
+} else {
+    let app = require('./app.js');
+
+    let port = process.env.PORT || '3000';
+    app.set('port', port);
+    let server = http.createServer(app);
+
+    server.listen(port);
+}

protected/models/AmcMsg.js

@@ -0,0 +1,69 @@
+/**
+ * Created by ben on 2017/10/24.
+ */
+const php = require('phpjs');
+const Tool = require('../framework/lib/Tool.js');
+const dwHttp = require('../framework/lib/dwHttp.js');
+const TableHelper = require('../framework/lib/TableHelper.js');
+
+class AmcMsg {
+	getAppInfo() {
+		let appInfo = {
+			"app_id": "crawl",
+			"app_key": "d5d32a173ad01f7edd9b26248bbbcbd3",
+			"server_ip": "14.17.108.216",
+			"alerm_type": "ERROR",
+		}
+		return appInfo;
+	}
+
+	async recordMsg(code, msg_content) {
+		var msg_content = encodeURI(msg_content);
+        let objHttp = new dwHttp();
+        let appInfo = this.getAppInfo();
+        let url = 'http://61.160.36.226/default/recordMsg';
+
+        var data = {
+        	"code": code,
+        	"code_msg": msg_content,
+        	"app_id": appInfo.app_id,
+			"server_ip": appInfo.server_ip,
+			"alerm_type": appInfo.alerm_type,
+			"time": php.time()
+        };
+        var sign = await this.getSign(data, appInfo.app_key);
+        data['sign'] = sign;
+
+        let headers = {};
+        headers['HOST'] = 'amc-admin.duowan.com';
+        let rep = await objHttp.post2(url, data, 3, 3, headers);
+
+        if (rep) {
+            let result = JSON.parse(rep);
+            return result;
+        } else {
+            return [];
+        }	
+	}
+
+	async getSign(args, app_key) {
+		var tmpArgs = {
+			"code": args.code,
+        	"code_msg": args.code_msg,
+        	"app_id": args.app_id,
+			"server_ip": args.server_ip,
+			"alerm_type": args.alerm_type,
+			"time": args.time,
+			"app_key": app_key
+		};
+		var tmpArr = [];
+		for (let key of Object.keys(tmpArgs).sort()) {
+		  tmpArr.push(tmpArgs[key]);
+		}
+        var str = tmpArr.sort().join('');
+        var sign = php.sha1(str);
+        return sign;
+	}
+}
+
+module.exports = AmcMsg;

protected/models/Browser.js

@@ -0,0 +1,172 @@
+/**
+ * 浏览器类代理池
+ */
+const puppeteer = require('puppeteer');
+const exec = require('child_process').exec;
+const Tool = require('../framework/lib/Tool.js');
+const TableHelper = require('../framework/lib/TableHelper.js');
+const URL = require('url');
+const ProxyPool = require('../models/ProxyPool');
+
+let map = {};
+let page = null;
+let browser_pid = null;
+class Browser {
+	static async init(proxy = false) {
+		//如果 map 是空对象数组，那么说明是程序退出然后守护进程重启了，需要先杀死所有的 chrome 
+		//因为 browser.process() 拿不到程序id，所以只能 ps 查看所有的 chrome 然后杀掉进程
+		if (Object.keys(map).length == 0) {
+			//await this.killAllChrome();	
+		}
+		
+		await this.checkBrowser();
+		if (!map.browser) {
+			try {
+				Tool.log("hava no browser");
+	        	let args = ['--no-sandbox','--disable-setuid-sandbox'];
+
+		        if (proxy) {args.push(`--proxy-server=${proxy}`);}
+
+		        let browser = await puppeteer.launch({
+			        args: args,
+			        handleSIGINT: false
+			    });
+			    process.on('SIGINT', () => { browser.close(); process.exit(130); })
+
+		        var date = new Date();
+				var now = Math.floor(date.getTime()/1000); 
+				map = {browser: browser, expire: now + 15 * 60, score: 0};
+	    	} catch (ex) {
+	    		await this.close();
+	    	}
+    	} else {
+			Tool.log("hava browser");
+			return map.browser;
+		}
+    }
+
+    static async checkBrowser() {
+    	if (Object.keys(map).length > 0) {
+    		var date = new Date();
+			var now = Math.floor(date.getTime()/1000); 
+    		
+
+    		if (map.expire < now || map.score < 0) {
+				await this.close();
+			}
+        }
+    }
+
+    static async close() {
+    	if (map.browser) {
+    		await map.browser.close();
+    		page = null;
+    	}
+    	map = {};
+    }
+
+
+    static getBrowserMap() {
+    	return map;
+    }
+
+    //杀死可能是上一个错误使得 chrome 没有杀死的所有浏览器进程
+    static async killAllChrome() {
+    	const cmdStr = 'ps -A -opid,etime,args | grep "/protected/node_modules/puppeteer"';
+    	let that = this;
+	    // const cmdStr = 'ps -A -opid,etime,args | grep Chrome';
+	    exec(cmdStr, function(err, stdout, stderr){
+	        if (err) {
+	            Tool.log('get weather api error:' + stderr);
+	        } else {
+	            let rows = stdout.split("\n");
+	            let index = 1;
+	            for (let row of rows) {
+	                let data = that.getPidTime(row);
+
+	                let cmd = '';
+	                if (data.time > 30 * 60) {
+                    	// 强制kill
+	                    cmd = `kill -9 ${data.pid}`;
+	                } else if (data.time > 15 * 60) {
+	                    // 普通kill
+	                    cmd = `kill ${data.pid}`;
+	                }
+
+	                if (cmd) {
+	                    // Tool.log(`${index}: ${cmd}, time:${data.time}`);
+	                    exec(cmd, function(err, stdout, stderr) {
+	                        Tool.log(`${index}: ${cmd}, time:${data.time}, stdout:${stdout}, stderr:${stderr}`);
+	                    });
+	                    index++;
+	                }
+	            }
+	        }
+	    });
+    }
+
+    static getPidTime(row) {
+	    let pid = 0;
+	    let time = 0;
+	    let parts = row.trim().split(' ');
+	    for (let part of parts) {
+	        part = part.trim();
+	        if (!part) continue;
+	        if (!pid) {
+	            pid = part;
+	        } else if (!time) {
+	            let timeParts = part.split(':');
+	            for (let j = timeParts.length - 1; j >= 0; j--) {
+	                time += timeParts[j] * Math.pow(60, timeParts.length - 1 - j);
+	            }
+	            break;
+	        }
+	    }
+
+	    return {pid, time};
+	}
+
+	static incScore() {
+		map.score++;
+	}
+
+	static reduceScore() {
+		map.score = map.score - 10;
+	}
+
+	static async newPage() {
+		if (!map.browser) {
+			return null;
+		} else {
+			if (page) {
+				Tool.log("hava page");
+				return page;
+			} else {
+				Tool.log("hava no page");
+				page = await map.browser.newPage();
+				return page;
+			}	
+		}
+	}
+}
+
+module.exports = Browser;
+
+
+/*** Usage:
+*
+const Browser = require('../models/Browser');
+let browser = await Browser.init();
+var map = await Browser.getBrowserMap();
+let page = await browser.newPage();
+await page.goto("http://www.baidu.com");
+new Promise(async () => {
+    let content = await page.content();
+    console.log(content);
+    if (!content) {
+        await Browser.close();
+    }
+});
+Browser.closAll();
+*
+***/

protected/models/JTool.js

@@ -0,0 +1,166 @@
+"use strict";
+
+let exportObj = {
+    url : '',
+    protocol : '',
+    origin : '',
+    url_relative : '',
+    $ : null,
+    /**
+     * 展开td的colspan
+     * @author benzhan
+     */
+    fixColspan($table) {
+        let $ = this.$;
+        let $trs = $table.find('tr');
+        for (let i = 0; i < $trs.length; i++) {
+            let $tr = $($trs[i]);
+            let $tds = $tr.find('td[colspan]');
+            for (let j = 0; j < $tds.length; j++) {
+                let $td = $($tds[j]);
+                let span = $td.attr('colspan');
+                for (let k = 1; k < span; k++) {
+                    $td.after('<td></td>');
+                }
+                $td.removeAttr('colspan');
+            }
+        }
+    },
+    formatDate: function(date) {
+        let y, m, d = '01';
+        if (typeof date === 'string') {
+            let parts = date.match(/[\d]{4}-[\d]{1,2}-[\d]{1,2}/);
+            if (parts && parts.length) {
+                return parts[0];
+            }
+
+            let parts1 = date.match(/([\d]{4})[\s]*年/);
+            let parts2 = date.match(/([\d]{1,2})[\s]*月/);
+            let parts3 = date.match(/([\d]{1,2})[\s]*日/);
+            if ((parts1 && parts1.length || parts3 && parts3.length) && parts2.length ) {
+                y = parseInt(parts1[1]);
+                m = parseInt(parts2[1]);
+                d = parseInt(parts3 && parts3[1]) || 1;
+            } else {
+                return '';
+            }
+        } else {
+            if (typeof date === 'number') {
+                date = new Date(date);
+            }
+
+            y = date.getFullYear();
+            m = date.getMonth() + 1;
+            d = date.getDate();
+        }
+
+        m = m < 10 ? '0' + m : m;
+        d = d < 10 ? ('0' + d) : d;
+        return y + '-' + m + '-' + d;
+    },
+    formatDateTime: function(date) {
+        let dateStr = this.formatDate(date);
+
+        let h, minute, second = '00';
+        if (typeof date === 'string') {
+            let parts = date.match(/[\d]{1,2}:[\d]{1,2}(:[\d]{1,2})?/);
+            if (parts && parts.length) {
+                return dateStr + ' ' + parts[0];
+            }
+
+            let parts1 = date.match(/([\d]{1,2})[\s]*时/);
+            let parts2 = date.match(/([\d]{1,2})[\s]*分/);
+            let parts3 = date.match(/([\d]{1,2})[\s]*时/);
+            if (parts1 && parts1.length && parts2 && parts2.length) {
+                h = parseInt(parts1[1]);
+                minute = parseInt(parts2[1]);
+                second = parseInt(parts3 && parts3[1]) || 0;
+            } else {
+                return dateStr;
+            }
+        } else  {
+            if (typeof date === 'number') {
+                date = new Date(date);
+            }
+
+            h = date.getHours();
+            second = date.getSeconds();
+            minute = date.getMinutes();
+        }
+
+        h = h < 10 ? ('0' + h) : h;
+        minute = minute < 10 ? ('0' + minute) : minute;
+        second = second < 10 ? ('0' + second) : second;
+        return dateStr + ' ' + h + ':' + minute + ':' + second;
+    },
+    initUrl(url) {
+        this.url = url;
+        let parts = url.match(/(http[s]?:)\/\/[^\/]+[\/]?/);
+        this.origin = parts[0];
+        this.protocol = parts[1];
+        this.url_relative = url.substr(0, url.lastIndexOf('/') + 1);
+    },
+    initJquery($) {
+        this.$ = $;
+    },
+    formatUrl(href) {
+        href = href.trim();
+        let origin = this.origin;
+        if (/http[s]?:\/\//.test(href)) {
+            return href;
+        } else if (href.substr(0, 2) === '//') {
+            return this.protocol + href;
+        } else if (href[0] === '/') {
+            if (origin[origin.length - 1] === '/') {
+                return origin + href.substr(1);
+            } else {
+                return origin + href;
+            }
+        } else {
+            return this.url_relative + href;
+        }
+    },
+    formatRichText(content) {
+        let $ = this.$;
+        let $content = $('<div>' + content.trim() + '</div>');
+        $content.find('script').remove();
+        $content.find('img').each(function() {
+            let formatSrc = exportObj.formatUrl($(this).attr('src'));
+            $(this).attr('src', formatSrc);
+        });
+
+        $content.find('a').each(function() {
+            let href = $(this).attr('href');
+            if (href) {
+                let formatHref = exportObj.formatUrl(href);
+                $(this).attr('href', formatHref);
+            }
+
+            $(this).attr('rel', 'nofollow');
+        });
+
+        return $content.html();
+    },
+    md5(text) {
+        let crypto = require('crypto');
+        return crypto.createHash('md5').update(text).digest('hex');
+    },
+    setNextTime(task_id, interval) {
+        const php = require('phpjs');
+        const TableHelper = require('../framework/lib/TableHelper.js');
+
+        // next_crawl_time = 0;
+        const where = {task_id};
+        // 修改最后更新时间
+        let last_crawl_time = php.date('Y-m-d H:i:s');
+        let next_crawl_time = php.time() + interval;
+
+        const objTask = new TableHelper('task', 'crawl');
+        objTask.updateObject({last_crawl_time, next_crawl_time}, where);
+    }
+}
+
+// 兼容老版本
+exportObj.formaRichText = exportObj.formatRichText;
+
+module.exports = exportObj;

protected/models/MapData.js

@@ -0,0 +1,69 @@
+/**
+ * Created by ben on 2017/11/07.
+ */
+require('../extensions/function_extend.js');
+// let php = require('phpjs');
+// let Tool = require('../framework/lib/Tool.js');
+
+const TableHelper = require('../framework/lib/TableHelper.js');
+const OujMySql = require('../framework/lib/OujMySql.js');
+
+let mapCache = {};
+async function getData(mapKey, value) {
+    let cacheKey = `${mapKey}_` + JSON.stringify(value);
+    if (!mapCache[cacheKey]) {
+        const objMap = new TableHelper('Cmdb3Map', 'Report');
+
+        let mapInfo = await objMap.getRow({mapKey});
+        // const objTable = new TableHelper(mapInfo.sourceTable, mapInfo.nameDb);
+
+        let keyField = mapInfo['keyName'].trim();
+        let valField = mapInfo['valueName'].trim();
+        let tableName = `\`${mapInfo['sourceTable']}\``;
+
+        if (keyField.indexOf('_') === -1 && keyField.indexOf(' ') === -1) {
+            keyField = `\`${keyField}\``;
+        }
+
+        if (valField.indexOf('_') === -1 && valField.indexOf(' ') === -1) {
+            valField = `\`${valField}\``;
+        }
+
+        let sql = `SELECT ${keyField}, ${valField} FROM ${tableName} WHERE 1 `;
+        if (value) {
+            if (Array.isArray(value)) {
+                let str = value.join("', '");
+                sql += `AND ${keyField} IN ('${str}') `;
+            } else {
+                sql += `AND ${keyField} = '${value}' `;
+            }
+        }
+
+        let filter = mapInfo['mapFilter'];
+        if (filter) {
+            sql += `AND ${filter} `;
+        }
+
+        const objMySql = new OujMySql(mapInfo.nameDb);
+        let datas = await objMySql.getAll(sql);
+
+        let map = {};
+        keyField = mapInfo['keyName'].trim();
+        valField = mapInfo['valueName'].trim();
+        for (let data of datas) {
+            if (parseInt(mapInfo.mapType) === 1) {
+                map[data[keyField]] = map[data[keyField]] = {};
+                map[data[keyField]] = data[valField];
+            } else {
+                map[data[keyField]] = map[data[keyField]] = [];
+                map[data[keyField]].push(data[valField]);
+            }
+        }
+
+        mapCache[cacheKey] = map;
+    }
+
+    return mapCache[cacheKey];
+}
+
+exports.getData = getData;

protected/models/ProxyPool.js

@@ -0,0 +1,144 @@
+/**
+ * Created by ben on 2017/10/24.
+ */
+const php = require('phpjs');
+const Tool = require('../framework/lib/Tool.js');
+const dwHttp = require('../framework/lib/dwHttp.js');
+
+const TableHelper = require('../framework/lib/TableHelper.js');
+
+class ProxyPool {
+
+    getYYProxyList() {
+        let proxyList = [
+            "10.20.60.190:8118",
+            "10.20.60.225:8118",
+            "10.20.60.243:8118",
+            "10.20.60.241:8118",
+            "10.20.60.170:8118",
+            "10.20.60.162:8118",
+            "10.20.60.155:8118",
+            "10.20.60.87:8118",
+            "10.20.60.168:8118",
+            "10.20.60.137:8118",
+            "10.20.60.136:8118",
+            "10.20.60.132:8118",
+            "10.20.60.129:8118",
+            "10.20.60.128:8118",
+            "10.20.60.127:8118",
+            "10.20.60.126:8118",
+            "10.20.61.250:8118",
+            "10.20.60.236:8118",
+            "10.20.60.233:8118",
+            "10.20.60.232:8118",
+            "10.20.60.230:8118",
+            "10.20.60.228:8118",
+            "10.20.60.227:8118",
+            "10.20.60.224:8118",
+            "10.20.61.118:8118",
+            "10.20.61.98:8118",
+            "10.20.60.186:8118",
+            "10.20.60.172:8118",
+            "10.20.60.171:8118",
+            "10.20.61.216:8118",
+            "10.20.61.215:8118",
+            "10.20.49.190:8118",
+            "10.20.49.189:8118",
+            "10.20.60.23:8118",
+            "10.20.96.171:8118",
+            "10.20.96.170:8118",
+            "10.20.96.169:8118",
+            "10.20.60.250:8118",
+            "10.20.60.153:8118",
+            "10.20.60.108:8118",
+            "10.20.61.117:8118",
+            "10.20.61.116:8118",
+            "10.20.61.84:8118",
+            "10.20.61.83:8118",
+        ];
+
+        return proxyList;
+    }
+
+    async getFreeProxyList() {
+        return [];
+
+        let objHttp = new dwHttp();
+
+        let url = 'http://61.160.36.225:8000/?count=10';
+        let rep = await objHttp.get2(url);
+
+        if (rep) {
+            let list = JSON.parse(rep);
+            for (let i = 0; i < list.length && i < 100; i++) {
+                list[i] = `${list[i][0]}:${list[i][1]}`;
+            }
+
+            return list;
+        } else {
+            return [];
+        }
+    }
+
+    async getXProxyList() {
+        let objHttp = new dwHttp();
+
+        let url = 'http://61.160.36.226/spider/ipList';
+        let rep = await objHttp.get2(url, 3, 3, "HOST:admin.duowan.com");
+        console.log("XProxyList:" + rep);
+        if (rep) {
+            let result = JSON.parse(rep);
+            if (result.result) {
+                return result.data.proxyList;
+            }
+        }
+
+        return [];
+    }
+
+    async getXProxyBest(domain) {
+        let objHttp = new dwHttp();
+
+        let num = 100;
+        let url = `http://61.160.36.226/spider/bestProxy?num=${num}&domain=${domain}`;
+        let rep = await objHttp.get2(url, 3, 3, "HOST:admin.duowan.com");
+        if (rep) {
+            let result = JSON.parse(rep);
+            if (result.result) {
+                return result.data.proxyList;
+            }
+        }
+
+        return [];
+    }
+
+    async reportProxy(domain, proxy, score) {
+        let objHttp = new dwHttp();
+
+        let url = `http://61.160.36.226/spider/reportProxy?proxy=${proxy}&domain=${domain}&score=${score}`;
+        let rep = await objHttp.get2(url, 3, 3, "HOST:admin.duowan.com");
+        if (rep) {
+            let result = JSON.parse(rep);
+            if (result.result) {
+                return result.data;
+            }
+        }
+
+        return 0;
+    }
+
+    async getProxyList() {
+        let freeProxyList = await this.getFreeProxyList();
+        let xProxyList = await this.getXProxyList();
+        freeProxyList = freeProxyList.concat(xProxyList);
+        if (ENV === ENV_DEV) {
+            return freeProxyList;
+        } else {
+            let yyProxyList = await this.getYYProxyList();
+            return yyProxyList.concat(freeProxyList);
+        }
+    }
+
+}
+
+module.exports = ProxyPool;

protected/models/Spider.js

@@ -0,0 +1,1576 @@
+/**
+ * Created by ben on 2017/10/24.
+ */
+const php = require('phpjs');
+let cheerio = require('cheerio');
+const request = require('request-promise');
+const Tool = require('../framework/lib/Tool.js');
+const iconv = require('iconv-lite');
+const Http = require("http");
+const zlib = require('zlib');
+const Browser = require('../models/Browser');
+const AmcMsg = require('../models/AmcMsg');
+
+// 回调里面可能回用到
+const JTool = require('./JTool');
+const OujRedis = require('../framework/lib/OujRedis');
+const dwHttp = require('../framework/lib/dwHttp.js');
+
+const URL = require('url');
+const puppeteer = require('puppeteer');
+const TableHelper = require('../framework/lib/TableHelper.js');
+
+const MapData = require('./MapData.js');
+const ProxyPool = require('./ProxyPool');
+let pagePool = {};
+
+let uaList = [
+    'Mozilla/4.0 (compatible; MSIE 9.0; Windows NT 6.1; 125LA; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022)',
+    'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)',
+    'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.108 Safari/537.36 2345Explorer/8.8.3.16721',
+    'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 BIDUBrowser/8.7 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063',
+    'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.3427.400 QQBrowser/9.6.12513.400',
+    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0',
+    'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092416 Firefox/3.0.3',
+];
+
+class Spider {
+
+    /**
+     * 基本类，提供增删改查
+     * @param {string} task 表名
+     */
+    constructor(task) {
+        this.task = task;
+        // this.dbKey = dbKey || 'default';
+        // this.objMySql = new MySql(this.dbKey);
+
+        this.STATE_EXECING = 0;
+        this.STATE_SUCC = 1;
+        this.STATE_ERROR = 2;
+        this.STATE_TIMEOUT = 3;
+        this.STATE_PART_SUCC = 4;
+        this.STATE_PROXY_ERROR = 5;
+
+        this.INSERT_TYPE_ONLY_INSERT = 'only_insert';
+        this.INSERT_TYPE_ONLY_UPDATE = 'only_update';
+        this.INSERT_TYPE_UPDATE = 'update';
+
+        this.state = this.STATE_SUCC;
+        this.skip = false;
+        this.http_code = 0;
+        Tool.startRecordLog();
+    }
+
+    async run(preview) {
+        const objTask = new TableHelper('task', 'crawl');
+        const objRule = new TableHelper('rule', 'crawl');
+        this.rule = await objRule.getRow({'rule_id' : this.task['rule_id']});
+        if (!this.rule) {
+            await objTask.updateObject({'enable' : 0}, {'task_id' : this.task.task_id});
+            Tool.error(`error task have no rule. task_id:${this.task.task_id}, rule_id:${this.task.rule_id}`);
+            return false;
+        }
+        await this._mergeRule();
+        await this._delayRun();
+
+        let startTime = (new Date).getTime();
+
+        let result = {};
+        if (!preview) {
+            result = await this._logStart();
+        }
+
+        // 找这个任务的规则
+        const objItem = new TableHelper('item', 'crawl');
+        let items = await objItem.getAll({'rule_id' : this.task.rule_id, 'enable' : 1});
+        await this._mergeItems(items);
+
+        let content = '';
+        let data = {};
+
+        try {
+            Tool.log(`爬虫：${this.task.url}`);
+            Tool.log(`请求模式：${this.rule.request_mode}，更新模式：${this.rule.update_mode}`);
+            JTool.initUrl(this.task.url);
+
+            //this.rule.request_mode = 'headless';
+            if (this.rule.request_mode === 'headless') {
+                content = await this._headless();
+            } else {
+                content = await this._request();
+            }
+            
+
+            // let objRedis = OujRedis.init('logic');
+            // content = await objRedis.get('globals:url_map:http://14.17.108.216:9998/previewRule?rule_id=steam:game_data:steamdb&url=');
+            Tool.log('http code:' + this.http_code);
+            if (preview) {
+                return content;
+            }
+
+            if (this.skip) {
+                Tool.log('预处理返回了false，当作成功，并跳过处理');
+                this.state = this.STATE_SUCC;
+                this.http_code = 200;
+                if (this.rule.request_mode === 'headless') {
+                    await this.scoreBrowser(this.http_code);
+                }
+            } else if (!content || content.length <= this.rule.min_length) {
+                // 代理出问题，当作超时处理
+                Tool.log('内容过短，代理出问题');
+                if (this.rule.request_mode === 'headless') {
+                    await this.scoreBrowser(0);
+                }
+                this.state = this.STATE_TIMEOUT;
+            } else {
+                if (this.rule.request_mode === 'headless') {
+                    await this.scoreBrowser(200);
+                }
+                
+                Tool.log('开始分析页面');
+                let $ = null;
+                let $el = null;
+                if (this.rule['data_type'] === 'html') {
+                    $ = cheerio.load(content, { decodeEntities: false });
+                } else if (this.rule['data_type'] === 'json') {
+                    $el = JSON.parse(content);
+                }
+
+                JTool.initJquery($);
+                for (let item of items) {
+                    try {
+                        let ret = this._handle(item, $, $el);
+                        if (!ret) {
+                            if (item.require) {
+                                this.state = this.STATE_ERROR;
+                                Tool.error('lack of required field:' + item.field_name);
+                                break;
+                            } else {
+                                Tool.log('skip not found field:' + item.field_name);
+                                continue;
+                            }
+                        }
+
+                        let {value, task_key} = ret;
+                        data[item.field_name] = value;
+                        if (item.next_rule_id) {
+                            await this._addNewTask(item, value, task_key);
+                        }
+                    } catch (ex) {
+                        this.state = this.STATE_PART_SUCC;
+                        let errorMsg = `error item:${item.field_name}\nselector:${item.selector}\nfetch_value:\n${item.fetch_value}`;
+                        if (item.new_task_key) {
+                            errorMsg += '\n\nnew_task_key:\n' + item.new_task_key;
+                        }
+                        Tool.log(errorMsg);
+                        Tool.error(ex.stack);
+                    }
+
+                    if (item.require && !data[item.field_name]) {
+                        this.state = this.STATE_ERROR;
+                        Tool.error(`lack of require item:${item.field_name}\nselector:${item.selector}\nfetch_value:\n${item.fetch_value}`);
+                        break;
+                    }
+                }
+
+                if (this.state !== this.STATE_ERROR) {
+                    await this._insertData(result.insertId, items, data);
+                }
+            }
+
+            await this._logEnd(result.insertId, content, data, startTime);
+        } catch (ex) {
+            if (this.rule.request_mode === 'headless') {
+                await this.closeBrowser();
+            }
+            Tool.err(ex.message);
+            Tool.err(ex.stack);
+
+            if (preview) {
+                return ex.stack;
+            }
+
+            this._preprocess(ex.message);
+            // 如果需要跳过，则跳过
+            let flag2 = false;
+            if (!this.skip) {
+                // 代理出问题的情况
+                let flag = ex.message.indexOf("Error: ") >= 0;
+                flag = flag || ex.message.indexOf("net::") >= 0;
+
+                // 代理访问慢的情况
+                flag2 = ex.message.indexOf('Navigation Timeout Exceeded') >= 0;
+                flag2 = flag2 || ex.message.indexOf("Most likely the page has been closed") >= 0;
+                if (flag) {
+                    this.state = this.STATE_PROXY_ERROR;
+                } else if (flag2) {
+                    this.state = this.STATE_TIMEOUT;
+                } else {
+                    this.state = this.STATE_ERROR;
+                }
+
+                if (!this.http_code) {
+                    this.http_code = parseInt(ex.message);
+                    Tool.err('http_code:' + this.http_code);
+                }
+
+                let error_codes = [403, 429, 502, 503, 504];
+                if (error_codes.indexOf(this.http_code) >= 0) {
+                    this.state = this.STATE_PROXY_ERROR;
+                    // 请求太频繁了
+                    if (this.http_code === 429 || this.http_code === 502) {
+                        let next_crawl_time = php.time() + 300 * php.rand(1, 6);
+                        this._setNextTime(next_crawl_time);
+                    }
+                }
+            }
+
+            await this._logEnd(result.insertId, content, data, startTime);
+
+            if (flag2) {
+                // 异常情况，要重启进程
+                
+                process.exit(0);
+            }
+        }
+
+        return content;
+    }
+
+    async _mergeRule() {
+        // 继承父规则的数据
+        if (this.rule['parent_rule_id']) {
+            const objRule = new TableHelper('rule', 'crawl');
+            let parent_rule = await objRule.getRow({'rule_id': this.rule['parent_rule_id']});
+            Tool.log('old rule:' + JSON.stringify(this.rule));
+            for (let key in parent_rule) {
+                // enable 不继承
+                if (key === 'enable') continue;
+                // 填充空白数据
+                let flag = !this.rule[key] || this.rule[key] === 'undefined';
+                if (flag && parent_rule[key]) {
+                    this.rule[key] = parent_rule[key];
+                }
+            }
+
+            Tool.log('new rule:' + JSON.stringify(this.rule));
+        }
+    }
+
+    async _mergeItems(items) {
+        // 继承父规则的数据
+        if (this.rule['parent_rule_id']) {
+            const objItem = new TableHelper('item', 'crawl');
+            let parent_items = await objItem.getAll({'rule_id' : this.rule['parent_rule_id'], 'enable' : 1});
+            let items_map = {};
+            for (let item of items) {
+                items_map[item['field_name']] = 1;
+            }
+
+            // 合并规则详情
+            for (let parent_item of parent_items) {
+                if (!items_map[parent_item['field_name']]) {
+                    // 要修改rule_id
+                    parent_item.rule_id = this.rule.rule_id;
+                    items.push(parent_item);
+                }
+            }
+        }
+    }
+
+    async _delayRun() {
+        // 先延后，怕并行运行
+        let next_crawl_time = php.time() + 30;
+        await this._setNextTime(next_crawl_time);
+
+        let objTask = new TableHelper('task', 'crawl');
+        const objCrawlLog = await this._getLogObject();
+        let create_time = php.date('Y-m-d H:i:s', php.time() - 3600);
+        let _where = `create_time > '${create_time}'`;
+        let where = {'task_id' : this.task.task_id, 'state' : [this.STATE_ERROR, this.STATE_TIMEOUT]};
+        let errorNum = await objCrawlLog.getCount(where, {_where});
+
+        let max_exception_count = this.task.max_exception_count || this.rule.max_exception_count;
+        if (errorNum > max_exception_count - 1) {
+            // 大于最大异常数，要停止任务
+            let newData = {'enable' : 0, 'state' : 2};
+            await objTask.updateObject(newData, {'task_id' : this.task.task_id});
+            Tool.log("大于最大异常数，要停止任务");
+        } else if (errorNum > 0) {
+            // 延后重试
+            let next_crawl_time = php.time() + (errorNum + 1) * 30;
+            await this._setNextTime(next_crawl_time);
+        }
+    }
+
+    async _request() {
+        await this._resetProxy();
+
+        let r = request;
+        if (this.proxy && this.rule.need_proxy) {
+            r = request.defaults({'proxy':`http://${this.proxy}`});
+        } else if (!this.proxy && this.rule.need_proxy) {
+            return '';
+        }
+
+        // let j = request.jar();
+        // this.task.url = 'http://ka.duowan.com';
+
+        // 先decode，再encode，可以把字符给encode，又不会引起多次encode
+        let url = encodeURI(decodeURI(this.task.url));
+        let response = await r({
+            url : url,
+            // jar:  j,
+            headers : this.getHeaders(this.rule.header),
+            timeout: 60000,
+            gzip: true,
+            encoding: null,
+            resolveWithFullResponse: true,
+            rejectUnauthorized: false
+        });
+
+        this.http_code = response.statusCode;
+        let content = response.body.toString('utf8');
+        let $ = cheerio.load(content, { decodeEntities: false });
+
+        let head = $('head').html() || '';
+        let matches = head.match(/[;\s]charset=['"]?(\w+)['"]?/);
+        if (matches && matches[1].match(/gb/ig)) {
+            content = iconv.decode(response.body, 'gbk');
+        }
+
+        return this._preprocess(content);
+    }
+
+    async scoreBrowser(http_code = '') {
+        if (http_code == 200) {
+            Browser.incScore();
+        } else {
+            Browser.reduceScore();
+        }
+    }
+
+    async closeBrowser() {
+        await Browser.close();
+    }
+
+    async _getPage() {
+        var url = this.task.url;
+        var p = URL.parse(this.task.url);
+        var taget_host = p.host;
+        var proxy = this.proxy;
+        if (!this.rule.need_proxy) {
+            proxy = false;
+        }
+        let browser = await Browser.init(proxy);
+        
+        const page = await Browser.newPage();
+        const viewport = {
+            width : 1440,
+            height: 706
+        };
+        await page.setViewport(viewport);
+
+        return page;
+    }
+
+    async _headlessCookie(page, headers) {
+        if (headers['Cookie']) {
+            let parts = headers['Cookie'].split(';');
+            let url = this.task.url;
+            let urlInfo = URL.parse(url);
+            for (let part of parts) {
+                part = part.trim();
+                if (!part) continue;
+                let index = part.indexOf('=');
+                if (index < 0) continue;
+                let name = part.substr(0, index);
+                let value = part.substr(index + 1);
+
+                let domain = urlInfo['host'];
+                let path = '/';
+                let expires = php.time() + 86400;
+                await page.setCookie({name, value, url, domain, path, expires});
+            }
+            delete headers['Cookie'];
+        }
+    }
+
+    async _headless() {
+        // 设置头部
+        let headers = this.getHeaders(this.rule.header);
+        await this._resetProxy();
+        if (!this.proxy && this.rule.need_proxy) {
+            return '';
+        }
+
+        let page = await this._getPage();
+        await this._headlessCookie(page, headers);
+        page.setExtraHTTPHeaders(headers);
+
+        let cookies = await page.cookies(this.task.url);
+        Tool.log(cookies);
+
+
+        // 开始爬虫
+        if (!headers['User-Agent']) {
+            //因为在请求中有些api会针对 user-agent 做限制，所以动态做一下调整
+            let ua = uaList[Math.floor((Math.random()*uaList.length))];
+            page.setUserAgent(ua);
+        }
+
+        if (this.rule.wait_request_url) {
+            page.on('request', request => {
+                var request_url = request.url();
+                if (request_url.indexOf(this.rule.wait_request_url) != -1) {
+                    Tool.log('请求api链接：' + request.url());
+                }
+            });
+            page.on('requestfinished', async request => {
+                
+                if (request.url().indexOf(this.rule.wait_request_url) != -1) {
+                    Tool.log('【success】请求api成功：' + this.rule.wait_request_url);
+                    let api_response = await request.response().text();
+                    Tool.log('请求api内容：' + api_response);
+                }
+            });
+            page.on('requestfailed', async request => {
+                if (request.url().indexOf(this.rule.wait_request_url) != -1) {
+                    Tool.log('【error】请求api失败：' + this.rule.wait_request_url);
+                }
+            });
+        }
+        
+        //开始打开页面
+        let response = await page.goto(this.task.url, {
+            waitUntil : 'domcontentloaded'
+
+            // waitUntil : 'load'
+        });
+        
+        await this._autoScroll(page);
+
+        await this._waitRequired(page);
+
+        this.http_code = response.status();
+        let content = await page.content();
+
+        return this._preprocess(content, page);
+    }
+
+    //页面懒加载-滚动
+    async _autoScroll(page) {
+        return page.evaluate(() => {
+            return new Promise((resolve, reject) => {
+                var scroll_count = 0;
+                var totalHeight = 0;
+                var distance = 100;
+                var timer = setInterval(() => {
+                    var scrollHeight = document.body.scrollHeight;
+                    window.scrollBy(0, distance);
+                    totalHeight += distance;
+
+                    if(totalHeight >= scrollHeight || scroll_count > 30){
+                        clearInterval(timer);
+                        resolve();
+                    }
+                    scroll_count++;
+                }, 100);
+            })
+        });
+    }
+
+
+    async _setNextTime(next_crawl_time) {
+        // next_crawl_time = 0;
+        const where = {'task_id': this.task['task_id']};
+        // 修改最后更新时间
+        let last_crawl_time = php.date('Y-m-d H:i:s');
+
+        const objTask = new TableHelper('task', 'crawl');
+        await objTask.updateObject({last_crawl_time, next_crawl_time}, where);
+    }
+
+    async _getLogObject() {
+        if (!this.objCrawlLog) {
+            // let tableName = 'crawl_log_' + php.date('Ymd');
+            // let objCrawlLog = new TableHelper(tableName, 'crawl');
+            // let flag = await objCrawlLog.checkTableExist(tableName);
+            // if (!flag) {
+            //     let sql = `CREATE TABLE ${tableName} LIKE _crawl_log`;
+            //     await objCrawlLog.objMySql.update(sql);
+            // }
+            // this.objCrawlLog = objCrawlLog;
+
+            this.objCrawlLog = new TableHelper('crawl_log', 'crawl');
+        }
+
+        return this.objCrawlLog;
+    }
+
+    async _logStart() {
+        const objCrawlLog = await this._getLogObject();
+        let log = await objCrawlLog.getRow({"task_id" : this.task['task_id'], "state" : this.STATE_EXECING});
+        if (log) {
+            await objCrawlLog.updateObject({'state' : this.STATE_TIMEOUT}, {"log_id" : log.log_id});
+        }
+
+        let logData = {
+            'task_id' : this.task['task_id'],
+            'task_key' : this.task['task_key'],
+            'rule_id' : this.task['rule_id'],
+            'url' : this.task['url'],
+            'state' : this.STATE_EXECING,
+            'rule_name' : this.rule['rule_name'],
+            'create_time' : php.date('Y-m-d H:i:s'),
+        };
+
+        return await objCrawlLog.addObject(logData);
+    }
+
+    async _logEnd(log_id, raw_content, data, startTime) {
+        if (this.state === this.STATE_EXECING) {
+            this.state = this.STATE_SUCC;
+        }
+
+        Tool.log(`log id: ${log_id}, this.state: ${this.state}`);
+        Tool.log('Raw Content:\n' + raw_content);
+
+        let exec_timespan = ((new Date).getTime() - startTime) / 1000;
+        this.reportProxy(exec_timespan);
+
+        let logData = {
+            proxy : this.proxy,
+            state : this.state,
+            content : JSON.stringify(data),
+            'exec_end_time' : php.date('Y-m-d H:i:s'),
+            'exec_timespan' : exec_timespan,
+            'exec_log' : this.getLogs().substr(0, 500000)
+        };
+
+        if(log_id) {
+            const objCrawlLog = await this._getLogObject();
+            let where = {log_id};
+            await objCrawlLog.updateObject(logData, where);
+
+            //通用告警
+            var errMsgList = this.getLogs().substr(0, 500000).match(/【error】([^\n]+)\n/);
+            if (errMsgList) {
+                var errMsg = '';
+                if (errMsgList && errMsgList[1]) {
+                    errMsg = errMsgList[1];
+                    errMsg = errMsg.substr(0, 50);
+                    var result = await this.recordAmcMsg(errMsg);
+                    Tool.log(`【AMC_Msg】` + JSON.stringify(result));
+                }
+            }
+            
+
+            // 成功后，才修改下一阶段的时间
+            if (this.state === this.STATE_SUCC || this.state === this.STATE_PART_SUCC) {
+                let interval = this.task['interval'] || this.rule['interval'];
+                let next_crawl_time = php.time() + interval;
+                await this._setNextTime(next_crawl_time);
+            }
+        }
+    }
+
+    //通用告警
+    async recordAmcMsg(errMsg) {
+        if (errMsg !== '' && errMsg.indexOf('waiting for selector')  !== -1) {
+            return;
+        }
+        //过滤掉 400,401,404
+        if (errMsg.match(/4[\d]{2}\s-/)) {
+            return;
+        }
+        //过滤掉 5xx
+        if (errMsg.match(/5[\d]{2}\s-/)) {
+            return;
+        }
+        if (this.http_code == 400 || this.http_code == 401 || this.http_code == 404) {
+            return;
+        }
+        //过滤掉超时，代理错误，成功的状态
+        if (this.state == this.STATE_TIMEOUT || this.state == this.STATE_PROXY_ERROR || this.state == this.STATE_SUCC || this.state == this.STATE_EXECING || this.state == this.STATE_PART_SUCC) {
+            return;
+        }
+
+        let objAmcMsg = new AmcMsg;
+
+        var msg_code = -1;
+        if (!msg_code) {
+            msg_code = -1;
+        }
+        var creator = this.rule.creator;
+        if (!creator) {
+            creator = 'spider';
+        }
+        var rule_id = this.rule.rule_id;
+        var rule_name = this.rule.rule_name;
+        var msg_content = `【${creator}】的《${rule_name}》（${rule_id}）有异常： ${errMsg}`;
+        let ret = await objAmcMsg.recordMsg(msg_code, msg_content);
+        if (ret.length == 0) {
+            return false;
+        }
+        return ret;
+    }
+
+    reportProxy(exec_timespan) {
+        let score = 0;
+        if (this.state === this.STATE_SUCC || this.state === this.STATE_PART_SUCC) {
+            if (exec_timespan < 3) {
+                score = 2;
+            } else if (exec_timespan < 8) {
+                score = 1;
+            } else if (exec_timespan > 30) {
+                score = -1;
+            } else if (exec_timespan > 50) {
+                score = -2;
+            }
+        } else if (this.state === this.STATE_PROXY_ERROR) {
+            score = -20;
+        }
+
+        if (score && this.proxy) {
+            const objProxyPool = new ProxyPool();
+            let p = URL.parse(this.task.url);
+            Tool.log(`reportProxy(p.host:${p.host}, this.proxy:${this.proxy}, score:${score})`);
+            objProxyPool.reportProxy(p.host, this.proxy, score);
+        }
+    }
+
+    getLogs() {
+        return Tool.stopRecordLog().join('\n');
+    }
+
+    getHeaders(header) {
+        Tool.log(header);
+
+        let headers = {};
+        if (header && header.trim()) {
+            let parts = header.trim().split("\n");
+            for (let part of parts) {
+                let index = part.indexOf(':');
+                let key = part.substr(0, index);
+                let val = part.substr(index + 1);
+                headers[key.trim()] = val.trim();
+            }
+        }
+
+        headers['Accept'] = headers['Accept'] || 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8';
+        headers['Accept-Encoding'] = headers['Accept-Encoding'] || 'gzip, deflate';
+        headers['Accept-Language'] = headers['Accept-Language'] || 'en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4';
+
+        let index = php.rand(0, uaList.length - 1);
+        let ua = uaList[index];
+        if (this.rule.request_mode != 'headless' || !this.rule.wait_request_url) {
+            headers['User-Agent'] = headers['User-Agent'] || ua;
+        }
+        
+        return headers;
+    }
+
+    async _resetProxy() {
+        let p = URL.parse(this.task.url);
+        const objProxyPool = new ProxyPool();
+        let proxyList = await objProxyPool.getXProxyBest(p.host);
+
+        this.proxy = '';
+        if (!php.empty(proxyList)) {
+            this.proxy = proxyList[php.rand(0, proxyList.length - 1)];
+            Tool.log('select proxy:' + this.proxy);
+        } else {
+            Tool.log('没有代理，停止爬虫');
+            this.state = this.STATE_PROXY_ERROR;
+        }
+    }
+
+    async _getArgs() {
+        let args = [
+            '--no-sandbox',
+            '--disable-setuid-sandbox',
+            // '--proxy-server=211.138.60.25:80'
+        ];
+
+        if (this.proxy && this.rule.need_proxy) {
+            args.push(`--proxy-server=${this.proxy}`);
+        }
+
+        return args;
+    }
+
+    async _waitRequired(page) {
+        let waitFor = this.rule['wait_for'];
+        if (waitFor) {
+            Tool.log('等待页面准备好. waitFor:' + waitFor);
+            let waitForInt = parseInt(waitFor);
+            if (waitForInt) {
+                waitFor = waitForInt;
+            }
+            await page.waitFor(waitFor);
+        } else {
+            //如果没有等待条件 wait_for 取 item 内一条必填的选择器当做等待
+            const objItem = new TableHelper('item', 'crawl');
+            let require_item = await objItem.getRow({'rule_id' : this.task.rule_id, 'enable' : 1, 'require' : 1});
+            if (require_item) {
+                Tool.log(`【timeout】wait for item selector(item.field_name=${require_item.field_name}):`+ require_item.selector);
+                await page.waitForSelector(require_item.selector);
+            }
+        }
+    }
+
+    async _preprocess(content, page) {
+        if (this.rule.data_type === 'json') {
+            content = content.trim();
+            let lastChar = php.substr(content, -1);
+            if (lastChar === ')') {
+                let pos = content.indexOf('(');
+                content = content.substr(pos + 1, content.length - pos - 2);
+            }
+        }
+
+        let preprocess = this.rule.preprocess && this.rule.preprocess.trim();
+        if (preprocess) {
+            let func = php.create_function('$html, $, page, _task, JTool, Tool', preprocess);
+
+            let $ = null;
+            let $html = null;
+            if (this.rule.data_type === 'html') {
+                $ = cheerio.load(content, { decodeEntities: false });
+                JTool.initJquery($);
+
+                $html = $('html');
+                let flag = func($html, $, page, this.task, JTool, Tool);
+                if (flag === false) {
+                    this.skip = true;
+                }
+                return $('<div></div>').html($html).html();
+            } else if (this.rule.data_type === 'json') {
+                $html = content;
+                return func($html, $, page, JTool, Tool);
+            }
+        }
+        
+        return content;
+    }
+
+    _getElemOnce(key, val, selector) {
+        let sections = [];
+        let parts = selector.split(key);
+        if (parts.length === 1) return [selector];
+
+        for (let part of parts) {
+            if (part.trim()) {
+                sections.push(part);
+                sections.push(key);
+            }
+        }
+
+        if (val[1] === false) {
+            sections.pop();
+        }
+
+        return sections;
+    }
+
+    _getElem($, selector) {
+        // 蛋疼，cheerio不支持:first, :last, :nth(xx), :eq(xx)的选择器，需要自己实现
+        let map = {
+            ':first' : ['eq', 0],
+            ':last' : ['eq', -1],
+            ':nth(' : ['eq', false],
+            ':eq(' : ['eq', false]
+        };
+
+        let sections = [selector];
+        for (let key in map) {
+            for (let i = 0; i < sections.length; i++) {
+                let tempSections = this._getElemOnce(key, map[key], sections[i]);
+                if (tempSections.length > 1) {
+                    sections = sections.slice(0, i).concat(tempSections).concat(sections.splice(i + 1));
+                    // i += tempSections.length - 1;
+                    i--;
+                }
+            }
+
+        }
+
+        let $el = null;
+        for (let i = 0; i < sections.length; i++) {
+            let sl = sections[i];
+            if (!sl.trim()) continue;
+
+            let val = map[sl];
+            if (val) {
+                // 用函数代替选择器
+                if (val[1] === false) {
+                    let sl2 = sections[i + 1];
+                    let num = parseInt(sl2.trim());
+                    sections[i + 1] = sl2.replace(/[\s]*[-\d]+[\s]*\)/, '');
+
+                    $el = $el[val[0]](num);
+                } else {
+                    $el = $el[val[0]](val[1]);
+                }
+            } else {
+                if ($el === null) {
+                    $el = $(sl);
+                } else if (sl[0] === ' ') {
+                    $el = $el.find(sl.trim());
+                } else {
+                    $el = $el.filter(sl.trim());
+                }
+            }
+        }
+
+        return $el;
+    }
+
+    _handle(item, $, $el) {
+        if ($) {
+            // html模式才需要获取元素
+            $el = this._getElem($, item.selector);
+            if ($el.length === 0) {
+                return false;
+            }
+        }
+
+        let value = this._fetchVal2($el, item, false, $);
+        let task_key = this._fetchVal2($el, item, true, $);
+
+        if (!task_key) {
+            task_key = value;
+        }
+
+        return {value, task_key};
+    }
+
+    _fetchVal2($el, item, iskey, $) {
+        // var key = iskey ? item.field_name + '-new_task_key' : item.field_name;
+        // var func = __crawPage[key];
+        let str = iskey ? item.new_task_key : item.fetch_value;
+        if (!str) {
+            return null;
+        }
+
+        let func = php.create_function('$el, $, _task, JTool, Tool', str);
+        if (!func) {
+            console.error('can not find:' + key);
+            return null;
+        }
+
+        if (item.is_multi && this.rule.data_type === 'html') {
+            let value = [];
+            for (let i = 0; i < $el.length; i++) {
+                let val = func($($el[i]), $, this.task, JTool, Tool);
+                if (Array.isArray(val)) {
+                    return val;
+                } else {
+                    value[i] = val;
+                }
+            }
+            return value;
+        } else {
+            return func($el, $, this.task, JTool, Tool);
+        }
+    }
+
+
+    _predoRuleId(item) {
+        if (!this.rule.parent_rule_id) {
+            return;
+        }
+
+        let next_id = item.next_rule_id;
+        Tool.log('next_rule_id：' + next_id + '，rule_id：' + item.rule_id);
+        // 这里特殊处理，支持专区爬虫...
+        let matches = next_id.match(/ka:(\d+):/);
+        if (matches) {
+            let next_game_id = matches[1];
+            if (next_game_id === '0') {
+                matches = item.rule_id.match(/ka:(\d+):/);
+                if (matches && matches[1]) {
+                    Tool.log('next_game_id：' + next_game_id + ' => game_id：' + matches[1]);
+                    next_id = next_id.replace('ka:0:', `ka:${matches[1]}:`);
+                    if (next_id.endsWith('_tpl')) {
+                        next_id = next_id.substr(0, next_id.length - 4);
+                    }
+                    item.next_rule_id = next_id;
+                }
+            }
+        }
+    }
+
+    async _addNewTask(item, value, task_key) {
+        let objTask = new TableHelper('task', 'crawl');
+        if (!Array.isArray(value)) {
+            value = [value];
+            task_key = [task_key];
+        }
+
+        // 预处理下一个规则id
+        this._predoRuleId(item);
+
+        let where = {
+            'rule_id' : item.next_rule_id,
+        };
+
+        let demo_url = null;
+        let batchNum = 1000;
+        value = php.array_chunk(value, batchNum);
+        task_key = php.array_chunk(task_key, batchNum);
+        for (let batchIndex in value) {
+            let key = task_key[batchIndex];
+            where['task_key'] = key;
+
+            let task_keys = await objTask.getCol(where, {_field : 'task_key'});
+            let map = {};
+            for (let k of task_keys) {
+                map[k] = 1;
+            }
+
+            let datas = [];
+            let now = php.date('Y-m-d H:i:s');
+            for (let i in key) {
+                let task_key = key[i];
+                let url = value[batchIndex][i];
+                if (task_key && url && !map[task_key]) {
+                    map[task_key] = 1; // 防止自身就有重复链接
+                    datas.push({
+                        parent_task_id : this.task.task_id,
+                        rule_id : item.next_rule_id,
+                        url : url,
+                        task_key : task_key,
+                        create_time : now,
+                        update_time : now,
+                    });
+
+                    demo_url = demo_url || value[batchIndex][i];
+                }
+            }
+
+            if (datas.length) {
+                await objTask.addObjectsIfNotExist(datas);
+            }
+
+            const objRule = new TableHelper('rule', 'crawl');
+            let where2 = {
+                'rule_id' : item.next_rule_id,
+            };
+            let nextRule = await objRule.getRow(where2);
+            if (demo_url && !nextRule['demo_url']) {
+                objRule.updateObject({demo_url}, where2);
+            }
+        }
+    }
+
+    _formatData(data) {
+        for (let tableName in data) {
+            let arrLen = 0;
+            let hasArr = false;
+            for (let key in data[tableName]) {
+                let value = data[tableName][key];
+                if (Array.isArray(value)) {
+                    arrLen = Math.max(arrLen, value.length);
+                    hasArr = true;
+                }
+            }
+
+            if (hasArr) {
+                let newArr = [];
+                for (let i = 0; i < arrLen; i++) {
+                    let item = {};
+                    let value = data[tableName];
+                    for (let key in value) {
+                        if (Array.isArray(value[key])) {
+                            item[key] = value[key][i];
+                        } else {
+                            item[key] = value[key];
+                        }
+                    }
+
+                    newArr[i] = item;
+                }
+                data[tableName] = newArr;
+            } else {
+                // 统一转化为数组
+                data[tableName] = [data[tableName]];
+            }
+        }
+
+        return data;
+    }
+
+    async _queryTables(rule_id) {
+        const objTable = new TableHelper('data_db', 'crawl');
+        let sql = 'SELECT data_db.db_id, db_name, table_name, pri_key, notice_url, rule_id, is_default, update_mode ' +
+            'FROM data_db JOIN rule_db_conf ON data_db.db_id = rule_db_conf.db_id ' +
+            `WHERE rule_id = '${rule_id}'`;
+
+        return await objTable.objMySql.getAll(sql);
+    }
+
+    async _getTables() {
+        let tables = await this._queryTables(this.rule.rule_id);
+        if (this.rule.parent_rule_id) {
+            let parentTables = await this._queryTables(this.rule.parent_rule_id);
+            let map = {};
+            for (let table of tables) {
+                let key = table.table_name + ':' + table.table_name;
+                map[key] = true;
+            }
+
+            for (let table of parentTables) {
+                let key = table.table_name + ':' + table.table_name;
+                if (!map[key]) {
+                    tables.push(table);
+                }
+            }
+        }
+
+        return tables;
+    }
+
+    async _insertData(log_id, items, data) {
+        let tables = await this._getTables();
+        if (tables.length === 0) {
+            return;
+        }
+
+        let defaultTable = tables[0].table_name;
+        for (let table of tables) {
+            if (table.is_default) {
+                defaultTable = table.table_name;
+                break;
+            }
+        }
+
+        let onlyInsertData = {};
+        let onlyUpdateData = {};
+        let updateData = {};
+        // let data = {};
+        let items2 = {};
+        let saveasItems = {};
+        for (let item of items) {
+            let value = data[item.field_name];
+            if (value === null || value === undefined || item.next_rule_id){
+                continue;
+            }
+
+            // 判断是否要翻译
+            if (item.map_key) {
+                let map = await MapData.getData(item.map_key, value);
+                if (Array.isArray(value)) {
+                    for (let k in value) {
+                        value[k] = map[value[k]];
+                    }
+                } else {
+                    value = map[value];
+                }
+            }
+
+            let table_name = defaultTable;
+            let field_names = item.field_name.split(',');
+
+            for (let field_name of field_names) {
+                field_name = field_name.trim();
+                let parts = field_name.split('.');
+                if (parts.length > 1) {
+                    table_name = parts[0];
+                    field_name = parts[1];
+                }
+
+                items2[table_name] = items2[table_name] || {};
+                if (!item.next_rule_id) {
+                    if (item.insert_type === this.INSERT_TYPE_ONLY_INSERT) {
+                        onlyInsertData[table_name] = onlyInsertData[table_name] || {};
+                        onlyInsertData[table_name][field_name] = value;
+                    } else if (item.insert_type === this.INSERT_TYPE_ONLY_UPDATE) {
+                        onlyUpdateData[table_name] = onlyUpdateData[table_name] || {};
+                        onlyUpdateData[table_name][field_name] = value;
+                    } else if (item.insert_type === this.INSERT_TYPE_UPDATE) {
+                        updateData[table_name] = updateData[table_name] || {};
+                        updateData[table_name][field_name] = value;
+                    } else {
+                        Tool.error('unknown insert type.');
+                    }
+                }
+
+                items2[table_name][field_name] = item;
+                if (item.save_as > 0) {
+                    saveasItems[table_name] = saveasItems[table_name] || [];
+                    saveasItems[table_name].push(field_name);
+                }
+            }
+        }
+
+        this._formatData(onlyInsertData);
+        this._formatData(onlyUpdateData);
+        this._formatData(updateData);
+
+        for (let table of tables) {
+            let tableName = table.table_name;
+            let priKey = table.pri_key && table.pri_key.trim();
+            if (!priKey || php.empty(onlyInsertData[tableName]) && php.empty(onlyUpdateData[tableName]) && php.empty(updateData[tableName])) {
+                continue;
+            }
+
+            let onlyInsertArr = onlyInsertData[tableName] || [];
+            let onlyUpdateArr = onlyUpdateData[tableName] || [];
+            let updateArr = updateData[tableName] || [];
+
+            let result = null;
+            let update_mode = table.update_mode ? table.update_mode : this.rule.update_mode;
+            if (update_mode === 'replace') {
+                result = await this._replaceBatch(table, items2, onlyInsertArr, updateArr, onlyUpdateArr);
+            } else {
+                result = await this._updateOneByOne(table, items2, onlyInsertArr, updateArr, onlyUpdateArr);
+            }
+
+            await this._notifyUrl(log_id, table.notice_url, result);
+
+            // 图片/视频 转存信息
+            await this._saveasData(table, items2, saveasItems, result.datas, result.temp_saveas_fields);
+        }
+    }
+
+    async _notifyUrl(log_id, notice_url, data) {
+        if (!notice_url) {
+            return false;
+        }
+
+        if (php.empty(data.datas)) {
+            Tool.log('没发生数据变化，不发送通知');
+            return false;
+        }
+
+        let objHttp = new dwHttp();
+        data.log_id = log_id;
+        data.oldDatas = JSON.stringify(data.oldDatas);
+        data.datas = JSON.stringify(data.datas);
+        let json = await objHttp.post2(notice_url, data);
+
+        Tool.log('发送数据变更通知：' + notice_url + '，返回：' + json);
+    }
+
+    async _replaceBatch(table, items, onlyInsertArr, updateArr, onlyUpdateArr) {
+        let objTable = new TableHelper(table.table_name, table.db_name);
+        let priKey = table.pri_key && table.pri_key.trim();
+
+        let len = onlyInsertArr.length || updateArr.length || onlyUpdateArr.length;
+        let allArr = [];
+        let oldDatas = {};
+        let datas = {};
+
+        Tool.log(`_replaceBatch, onlyInsertArr:${onlyInsertArr.length}, updateArr:${updateArr.length}, onlyUpdateArr:${onlyUpdateArr.length}, getTempData:false`);
+        let {allOldDatas} = await this._getOldDatas(table, onlyInsertArr, updateArr, onlyUpdateArr, false);
+
+        // let objTempData = new TableHelper('temp_data', 'crawl');
+        // let has_temp_data = await objTempData.getCount({db_id:table.db_id});
+
+        for (let i = 0; i < len; i++) {
+            onlyInsertArr[i] = onlyInsertArr[i] || {};
+            updateArr[i] = updateArr[i] || {};
+            onlyUpdateArr[i] = onlyUpdateArr[i] || {};
+
+            let allData = Object.assign({}, onlyInsertArr[i], updateArr[i], onlyUpdateArr[i]);
+            let where = this._getWhere(allData, priKey);
+            if (where === false) {
+                Tool.error('Replace的数据必须要含有主键');
+                this.state = this.STATE_PART_SUCC;
+            } else {
+                // let oldData = await objTable.getRow(where);
+                let whereStr = JSON.stringify(where);
+                let oldData = allOldDatas[whereStr];
+                if (oldData) {
+                    // 存在不同的数据才更新
+                    for (let key in allData) {
+                        if (allData[key] != oldData[key]) {
+                            allArr.push(allData);
+                            oldDatas[whereStr] = oldData;
+                            datas[whereStr] = allData;
+                            break;
+                        }
+                    }
+                } else {
+                    // replace方式，暂时不支持暂存数据
+                    // if (tempDatas) {
+                    //     let whereStr2 = table.db_id + php.md5(whereStr);
+                    //     if (tempDatas[whereStr2]) {
+                    //         let temp_value = JSON.parse(tempDatas[whereStr2]);
+                    //         allData = Object.assign(temp_value, allData);
+                    //     }
+                    // }
+                    allArr.push(allData);
+                    datas[whereStr] = allData;
+                }
+            }
+        }
+
+        try {
+            if (allArr.length) {
+                await objTable.replaceObjects2(allArr);
+            } else {
+                Tool.log('没有新数据，不需要replaceObjects2');
+            }
+        } catch (ex) {
+            this.state = this.STATE_ERROR;
+            Tool.err(ex.stack);
+        }
+
+        return {oldDatas, datas};
+    }
+
+    async _saveasData(table, items2, saveasItems, datas, temp_saveas_fields) {
+        let table_name = table.table_name;
+        if ((!saveasItems[table_name] && php.empty(temp_saveas_fields)) || php.empty(datas)) {
+            return false;
+        }
+
+        let saveasArr = [];
+        let insert_saveas_data_ids = [];
+        for (let whereStr in datas) {
+            let allData = datas[whereStr];
+            // 判断是否存在转存字段
+            if (saveasItems[table_name]) {
+                for (let field_name of saveasItems[table_name]) {
+                    if (!allData[field_name]) {
+                        continue;
+                    }
+
+                    let item = items2[table_name][field_name];
+                    if (insert_saveas_data_ids.indexOf(table.db_id + '|' + whereStr + '|' + field_name) == '-1') {
+                        saveasArr.push({
+                            saveas_data_id : php.md5(table.db_id + '|' + whereStr + '|' + field_name),
+                            db_id : table.db_id,
+                            key_value : whereStr,
+                            save_as : item['save_as'],
+                            save_as_referer : item['save_as_referer'],
+                            field_name
+                        });
+
+                        insert_saveas_data_ids.push(table.db_id + '|' + whereStr + '|' + field_name);
+                    }
+                }
+            }
+            
+
+            //临时表合并过来的数据需要检查是否插入转存
+            if (temp_saveas_fields[whereStr]) {
+                let temp_data = temp_saveas_fields[whereStr];
+                for (let temp_field in temp_data) {
+                    if (!allData[temp_field]) {
+                        continue;
+                    }
+
+                    if (insert_saveas_data_ids.indexOf(table.db_id + '|' + whereStr + '|' + temp_field) == '-1') {
+                        let saveas_db_id = table.db_id;
+                        if (temp_data[temp_field].db_id) {
+                            saveas_db_id = temp_data[temp_field].db_id;
+                        }
+                        saveasArr.push({
+                            saveas_data_id : php.md5(table.db_id + '|' + whereStr + '|' + temp_field),
+                            db_id : saveas_db_id,
+                            key_value : whereStr,
+                            save_as : temp_data[temp_field].save_as,
+                            save_as_referer : temp_data[temp_field].save_as_referer,
+                            field_name : temp_field
+                        });
+
+                        insert_saveas_data_ids.push(table.db_id + '|' + whereStr + '|' + temp_field);
+                    }
+                }
+            }
+            
+        }
+
+        try {
+            if (saveasArr.length) {
+                let objTable = new TableHelper('saveas_data', 'crawl');
+                await objTable.replaceObjects2(saveasArr);
+            }
+        } catch (ex) {
+            this.state = this.STATE_ERROR;
+            Tool.err(ex.stack);
+            return false;
+        }
+
+        return true;
+    }
+
+    async _getOldDatas(table, onlyInsertArr, updateArr, onlyUpdateArr, getTempData) {
+        let objTable = new TableHelper(table.table_name, table.db_name);
+        let priKey = table.pri_key && table.pri_key.trim();
+
+        let mergeWhere = this._getMergeWhere(priKey, onlyInsertArr, updateArr, onlyUpdateArr);
+        let allOldDatas = {}, tempDatas = {};
+        let len = onlyInsertArr.length || updateArr.length || onlyUpdateArr.length;
+        if (len <= 0) {
+            return {allOldDatas, tempDatas};
+        }
+
+        let row = Object.assign({}, onlyInsertArr[0], updateArr[0], onlyUpdateArr[0]);
+        let keys = Object.keys(row);
+
+        let has_temp_data = false;
+        let objTempData = new TableHelper('temp_data', 'crawl');
+        if (getTempData) {
+            let dateStr = php.date('Y-m-d', php.time() - 86400 * 3);
+            let _where = `create_time > '${dateStr}'`;
+            // 判断是否有临时数据
+            has_temp_data = await objTempData.getCount({db_id: table.db_id}, {_where});
+        }
+
+        let _field = '`' + keys.join('`,`') + '`';
+        let temp_keys = [];
+        if (!mergeWhere) {
+            for (let i = 0; i < len; i++) {
+                let allData = Object.assign({}, onlyInsertArr[i], updateArr[i], onlyUpdateArr[i]);
+                let where = this._getWhere(allData, priKey);
+                if (where === false) {
+                    continue;
+                }
+
+                let key = JSON.stringify(where);
+                has_temp_data && temp_keys.push(php.md5(key));
+                allOldDatas[key] = await objTable.getRow(where, {_field});
+            }
+        } else {
+            let datas = await objTable.getAll(mergeWhere, {_field});
+            for (let data of datas) {
+                let where = this._getWhere(data, priKey);
+                let key = JSON.stringify(where);
+                allOldDatas[key] = data;
+            }
+
+            if (has_temp_data) {
+                // 构造临时数据的条件
+                for (let i = 0; i < len; i++) {
+                    let allData = Object.assign({}, onlyInsertArr[i], updateArr[i], onlyUpdateArr[i]);
+                    let where = this._getWhere(allData, priKey);
+                    if (where === false) {
+                        continue;
+                    }
+
+                    let key = JSON.stringify(where);
+                    temp_keys.push(php.md5(key));
+                }
+            }
+        }
+
+        if (has_temp_data && temp_keys.length) {
+            let where = {
+                db_id: table.db_id,
+                temp_key: temp_keys,
+            };
+
+            let datas = await objTempData.getAll(where);
+            for (let data of datas) {
+                tempDatas[data.db_id + data.temp_key] = {"temp_value": data.temp_value, "temp_saveas_data": data.temp_saveas_data, "db_id": table.db_id};
+            }
+
+            objTempData.updateObject({'create_time': '2000-01-01'}, where);
+        }
+
+        return {allOldDatas, tempDatas};
+    }
+
+    _getMergeWhere(priKey, onlyInsertArr, updateArr, onlyUpdateArr) {
+        let mergeWhere = {};
+        let len = onlyInsertArr.length || updateArr.length || onlyUpdateArr.length;
+        for (let i = 0; i < len; i++) {
+            let allData = Object.assign({}, onlyInsertArr[i], updateArr[i], onlyUpdateArr[i]);
+            let where = this._getWhere(allData, priKey);
+            if (where === false) {
+                continue;
+            }
+
+            let arrNum = 0;
+            for (let key in where) {
+                let val = where[key];
+                if (!mergeWhere[key]) {
+                    // 第一次赋值
+                    mergeWhere[key] = val;
+                } else if (mergeWhere[key] !== val) {
+                    if (typeof mergeWhere[key] === 'object') {
+                        mergeWhere[key].push(val);
+                    } else {
+                        // 转化为数组
+                        arrNum++;
+                        if (arrNum > 1) {
+                            return false;
+                        } else {
+                            let preValue = mergeWhere[key];
+                            mergeWhere[key] = [];
+                            mergeWhere[key].push(preValue);
+                            mergeWhere[key].push(val);
+                        }
+                    }
+                }
+            }
+        }
+
+        return mergeWhere;
+    }
+
+    async _updateOneByOne(table, items, onlyInsertArr, updateArr, onlyUpdateArr) {
+        let len = onlyInsertArr.length || updateArr.length || onlyUpdateArr.length;
+        let priKey = table.pri_key && table.pri_key.trim();
+        let oldDatas = {};
+
+        let datas = {};
+        let getTempData = onlyInsertArr.length || updateArr.length;
+        Tool.log(`_updateOneByOne, onlyInsertArr:${onlyInsertArr.length}, updateArr:${updateArr.length}, onlyUpdateArr:${onlyUpdateArr.length}, getTempData:${getTempData}`);
+        let {allOldDatas, tempDatas} = await this._getOldDatas(table, onlyInsertArr, updateArr, onlyUpdateArr, getTempData);
+        let prefix = `db_name:${table.db_name}, table:${table.table_name}: `;
+
+        let objTempData = new TableHelper('temp_data', 'crawl');
+        let need_saveas_data = {};
+        for (let key in items[table.table_name]) {
+            if (items[table.table_name][key]['save_as'] > 0) {
+                need_saveas_data[key] = {'save_as': items[table.table_name][key]['save_as'], 'save_as_referer': items[table.table_name][key]['save_as_referer']};
+            }
+        }
+
+
+        let onlyUpdataDatas = [];
+        let temp_saveas_fields = {};
+        try {
+            for (let i = 0; i < len; i++) {
+                onlyInsertArr[i] = onlyInsertArr[i] || {};
+                updateArr[i] = updateArr[i] || {};
+                onlyUpdateArr[i] = onlyUpdateArr[i] || {};
+
+                let allData = Object.assign({}, onlyInsertArr[i], updateArr[i], onlyUpdateArr[i]);
+
+                let where = this._getWhere(allData, priKey);
+                if (where === false) {
+                    Tool.err('数据没包含主键：' + priKey);
+                    continue;
+                }
+
+                let whereStr = JSON.stringify(where);
+                let objTable = new TableHelper(table.table_name, table.db_name);
+                // let oldData = await objTable.getRow(where);
+                let oldData = allOldDatas[whereStr];
+                if (!oldData) {
+                    allData = Object.assign({}, onlyInsertArr[i], updateArr[i]);
+                    let newData = {};
+                    // 去掉null的数据
+                    for (let key in allData) {
+                        if (allData[key] !== null && allData[key] !== undefined) {
+                            newData[key] = allData[key];
+                        }
+                    }
+
+                    if (!php.empty(newData)) {
+                        // 判断是否有暂存数据，如果有则合并
+                        if (tempDatas) {
+                            if (tempDatas[table.db_id + php.md5(whereStr)] != undefined) {
+                                let temp_value = tempDatas[table.db_id + php.md5(whereStr)].temp_value;
+                                temp_value = JSON.parse(temp_value);
+                                newData = Object.assign(temp_value, newData);
+                            }
+                        }
+
+                        await objTable.addObject(newData);
+                        if (tempDatas[table.db_id + php.md5(whereStr)] != undefined) {
+                            let temp_saveas_data = tempDatas[table.db_id + php.md5(whereStr)].temp_saveas_data;
+                            temp_saveas_data = JSON.parse(temp_saveas_data);
+                            temp_saveas_fields[whereStr] = temp_saveas_data;
+                        }
+                        
+                        datas[whereStr] = newData;
+                    } else if (!php.empty(onlyUpdateArr)) {
+                        // 处理暂存的数据
+                        let db_id = table.db_id;
+                        let temp_key = php.md5(whereStr);
+                        let temp_value = JSON.stringify(onlyUpdateArr[i]);
+                        let temp_saveas_data = {}
+                        for (let field in onlyUpdateArr[i]) {
+                            if (need_saveas_data[field]) {
+                                temp_saveas_data[field] = need_saveas_data[field];
+                            }
+                        }
+
+                        temp_saveas_data = JSON.stringify(temp_saveas_data);
+                        onlyUpdataDatas.push({db_id, temp_key, temp_value, temp_saveas_data});
+                    } else {
+                        Tool.log(prefix + "没有'只插入'或'更新'的数据");
+                    }
+                } else {
+                    allData = Object.assign({}, updateArr[i], onlyUpdateArr[i]);
+                    if (!php.empty(allData)) {
+                        let newData = {};
+                        // 存在老数据，只更新updateData
+                        for (let key in allData) {
+                            if (allData[key] === null && allData[key] === undefined) {
+                                continue;
+                            }
+
+                            // 只填充的字段，只在更新模式有限
+                            if (items[table.table_name][key]['only_fill']) {
+                                // 老数据不存在时才填充
+                                if (!oldData[key] && allData[key])  {
+                                    newData[key] = allData[key];
+                                } else {
+                                    // Tool.log(`只填充字段：${key}, 已存在老数据:${oldData[key]}, 新数据:${allData[key]}`);
+                                    Tool.log(`只填充字段：${key}, 已存在老数据，不需要填充`);
+                                }
+                            } else if (allData[key] != oldData[key]) {
+                                newData[key] = allData[key];
+                            }
+                        }
+
+                        if (!php.empty(newData)) {
+                            await objTable.updateObject(newData, where);
+                            oldDatas[whereStr] = oldData;
+                            datas[whereStr] = newData;
+                        } else {
+                            Tool.log(prefix + "newData和oldData数据一样，不需要更新");
+                        }
+                    } else {
+                        Tool.log(prefix + "没有'只更新'或'更新'的数据");
+                    }
+                }
+            }
+
+            // 暂存的数据
+            if (onlyUpdataDatas.length) {
+                await objTempData.replaceObjects2(onlyUpdataDatas);
+            }
+
+        } catch (ex) {
+            this.state = this.STATE_ERROR;
+            Tool.err(ex.stack);
+        }
+
+        return {oldDatas, datas, temp_saveas_fields};
+    }
+
+    _getWhere(allData, priKey) {
+        let where = {};
+        let parts = priKey.split(',');
+        for (let part of parts) {
+            priKey = part.trim();
+            if (allData[priKey] === undefined) {
+                return false;
+            }
+
+            where[priKey] = allData[priKey] + '';
+        }
+
+        return where;
+    }
+
+}
+
+module.exports = Spider;

protected/package-lock.json

@@ -0,0 +1,1567 @@
+{
+  "name": "crawl",
+  "version": "1.0.0",
+  "lockfileVersion": 1,
+  "requires": true,
+  "dependencies": {
+    "@types/node": {
+      "version": "10.3.2",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-10.3.2.tgz",
+      "integrity": "sha512-9NfEUDp3tgRhmoxzTpTo+lq+KIVFxZahuRX0LHF/9IzKHaWuoWsIrrJ61zw5cnnlGINX8lqJzXYfQTOICS5Q+A=="
+    },
+    "abbrev": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
+      "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
+      "dev": true
+    },
+    "accepts": {
+      "version": "1.2.13",
+      "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.2.13.tgz",
+      "integrity": "sha1-5fHzkoxtlf2WVYw27D2dDeSm7Oo=",
+      "requires": {
+        "mime-types": "2.1.18",
+        "negotiator": "0.5.3"
+      }
+    },
+    "agent-base": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.0.tgz",
+      "integrity": "sha512-c+R/U5X+2zz2+UCrCFv6odQzJdoqI+YecuhnAJLa1zYaMc13zPfwMwZrr91Pd1DYNo/yPRbiM4WVf9whgwFsIg==",
+      "requires": {
+        "es6-promisify": "5.0.0"
+      }
+    },
+    "ajv": {
+      "version": "5.5.2",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz",
+      "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=",
+      "requires": {
+        "co": "4.6.0",
+        "fast-deep-equal": "1.1.0",
+        "fast-json-stable-stringify": "2.0.0",
+        "json-schema-traverse": "0.3.1"
+      }
+    },
+    "array-flatten": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
+      "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
+    },
+    "asn1": {
+      "version": "0.2.3",
+      "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz",
+      "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y="
+    },
+    "assert-plus": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
+      "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
+    },
+    "async-limiter": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz",
+      "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg=="
+    },
+    "asynckit": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+      "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
+    },
+    "aws-sign2": {
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
+      "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg="
+    },
+    "aws4": {
+      "version": "1.7.0",
+      "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.7.0.tgz",
+      "integrity": "sha512-32NDda82rhwD9/JBCCkB+MRYDp0oSvlo2IL6rQWA10PQi7tDUM3eqMSltXmY+Oyl/7N3P3qNtAlv7X0d9bI28w=="
+    },
+    "balanced-match": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
+      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
+    },
+    "basic-auth": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-1.0.4.tgz",
+      "integrity": "sha1-Awk1sB3nyblKgksp8/zLdQ06UpA="
+    },
+    "bcrypt-pbkdf": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz",
+      "integrity": "sha1-Y7xdy2EzG5K8Bf1SiVPDNGKgb40=",
+      "optional": true,
+      "requires": {
+        "tweetnacl": "0.14.5"
+      }
+    },
+    "bignumber.js": {
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-4.0.4.tgz",
+      "integrity": "sha512-LDXpJKVzEx2/OqNbG9mXBNvHuiRL4PzHCGfnANHMJ+fv68Ads3exDVJeGDJws+AoNEuca93bU3q+S0woeUaCdg==",
+      "dev": true
+    },
+    "bluebird": {
+      "version": "3.5.1",
+      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz",
+      "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA=="
+    },
+    "body-parser": {
+      "version": "1.13.3",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.13.3.tgz",
+      "integrity": "sha1-wIzzMMM1jhUQFqBXRvE/ApyX+pc=",
+      "requires": {
+        "bytes": "2.1.0",
+        "content-type": "1.0.4",
+        "debug": "2.2.0",
+        "depd": "1.0.1",
+        "http-errors": "1.3.1",
+        "iconv-lite": "0.4.11",
+        "on-finished": "2.3.0",
+        "qs": "4.0.0",
+        "raw-body": "2.1.7",
+        "type-is": "1.6.16"
+      }
+    },
+    "boolbase": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+      "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
+    },
+    "brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "requires": {
+        "balanced-match": "1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "buffer-from": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.0.tgz",
+      "integrity": "sha512-c5mRlguI/Pe2dSZmpER62rSCu0ryKmWddzRYsuXc50U2/g8jMOulc31VZMa4mYx31U5xsmSOpDCgH88Vl9cDGQ=="
+    },
+    "bytes": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/bytes/-/bytes-2.1.0.tgz",
+      "integrity": "sha1-rJPEEOL/ycx89LRks4KJBn9eR7Q="
+    },
+    "caseless": {
+      "version": "0.12.0",
+      "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
+      "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
+    },
+    "cheerio": {
+      "version": "1.0.0-rc.2",
+      "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.2.tgz",
+      "integrity": "sha1-S59TqBsn5NXawxwP/Qz6A8xoMNs=",
+      "requires": {
+        "css-select": "1.2.0",
+        "dom-serializer": "0.1.0",
+        "entities": "1.1.1",
+        "htmlparser2": "3.9.2",
+        "lodash": "4.17.10",
+        "parse5": "3.0.3"
+      }
+    },
+    "cli": {
+      "version": "0.4.4-2",
+      "resolved": "https://registry.npmjs.org/cli/-/cli-0.4.4-2.tgz",
+      "integrity": "sha1-o4yPmR3yLuoBeewW3ZD6PzyF+ko=",
+      "dev": true,
+      "requires": {
+        "glob": "7.1.2"
+      }
+    },
+    "co": {
+      "version": "4.6.0",
+      "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
+      "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ="
+    },
+    "combined-stream": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.6.tgz",
+      "integrity": "sha1-cj599ugBrFYTETp+RFqbactjKBg=",
+      "requires": {
+        "delayed-stream": "1.0.0"
+      }
+    },
+    "commander": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-2.0.0.tgz",
+      "integrity": "sha1-0bhvkB+LZL2UG96tr5JFMDk76Sg=",
+      "dev": true
+    },
+    "concat-map": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
+    },
+    "concat-stream": {
+      "version": "1.6.2",
+      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
+      "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
+      "requires": {
+        "buffer-from": "1.1.0",
+        "inherits": "2.0.3",
+        "readable-stream": "2.3.6",
+        "typedarray": "0.0.6"
+      }
+    },
+    "config-chain": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.11.tgz",
+      "integrity": "sha1-q6CXR9++TD5w52am5BWG4YWfxvI=",
+      "dev": true,
+      "requires": {
+        "ini": "1.3.5",
+        "proto-list": "1.2.4"
+      }
+    },
+    "content-disposition": {
+      "version": "0.5.1",
+      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.1.tgz",
+      "integrity": "sha1-h0dsamfI2qh+Muh2Ft+IO6f7Bxs="
+    },
+    "content-type": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz",
+      "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA=="
+    },
+    "cookie": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.1.3.tgz",
+      "integrity": "sha1-5zSlwUF/zkctWu+Cw4HKu2TRpDU="
+    },
+    "cookie-parser": {
+      "version": "1.3.5",
+      "resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.3.5.tgz",
+      "integrity": "sha1-nXVVcPtdF4kHcSJ6AjFNm+fPg1Y=",
+      "requires": {
+        "cookie": "0.1.3",
+        "cookie-signature": "1.0.6"
+      }
+    },
+    "cookie-signature": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
+      "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw="
+    },
+    "core-util-is": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
+      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
+    },
+    "css-select": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
+      "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
+      "requires": {
+        "boolbase": "1.0.0",
+        "css-what": "2.1.0",
+        "domutils": "1.5.1",
+        "nth-check": "1.0.1"
+      }
+    },
+    "css-what": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.0.tgz",
+      "integrity": "sha1-lGfQMsOM+u+58teVASUwYvh/ob0="
+    },
+    "dashdash": {
+      "version": "1.14.1",
+      "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
+      "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
+      "requires": {
+        "assert-plus": "1.0.0"
+      }
+    },
+    "debug": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz",
+      "integrity": "sha1-+HBX6ZWxofauaklgZkE3vFbwOdo=",
+      "requires": {
+        "ms": "0.7.1"
+      }
+    },
+    "deep-equal": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-0.1.2.tgz",
+      "integrity": "sha1-skbCuApXCkfBG+HZvRBw7IeLh84=",
+      "dev": true
+    },
+    "delayed-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+      "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
+    },
+    "depd": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/depd/-/depd-1.0.1.tgz",
+      "integrity": "sha1-gK7GTJ1tl+ZcwqnKqTwKpqv3Oqo="
+    },
+    "destroy": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
+      "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA="
+    },
+    "diff": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/diff/-/diff-1.0.7.tgz",
+      "integrity": "sha1-JLuwAcSn1VIhaefKvbLCgU7ZHPQ=",
+      "dev": true
+    },
+    "dom-serializer": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.0.tgz",
+      "integrity": "sha1-BzxpdUbOB4DOI75KKOKT5AvDDII=",
+      "requires": {
+        "domelementtype": "1.1.3",
+        "entities": "1.1.1"
+      },
+      "dependencies": {
+        "domelementtype": {
+          "version": "1.1.3",
+          "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz",
+          "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs="
+        }
+      }
+    },
+    "domelementtype": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.0.tgz",
+      "integrity": "sha1-sXrtguirWeUt2cGbF1bg/BhyBMI="
+    },
+    "domhandler": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz",
+      "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==",
+      "requires": {
+        "domelementtype": "1.3.0"
+      }
+    },
+    "domutils": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz",
+      "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=",
+      "requires": {
+        "dom-serializer": "0.1.0",
+        "domelementtype": "1.3.0"
+      }
+    },
+    "double-ended-queue": {
+      "version": "2.1.0-0",
+      "resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz",
+      "integrity": "sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw=",
+      "dev": true
+    },
+    "ecc-jsbn": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz",
+      "integrity": "sha1-D8c6ntXw1Tw4GTOYUj735UN3dQU=",
+      "optional": true,
+      "requires": {
+        "jsbn": "0.1.1"
+      }
+    },
+    "ee-first": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
+      "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
+    },
+    "ejs": {
+      "version": "2.3.4",
+      "resolved": "https://registry.npmjs.org/ejs/-/ejs-2.3.4.tgz",
+      "integrity": "sha1-PHbKoJZks1g7ADevncE2557Gi5g="
+    },
+    "entities": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz",
+      "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA="
+    },
+    "es6-promise": {
+      "version": "4.2.4",
+      "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.4.tgz",
+      "integrity": "sha512-/NdNZVJg+uZgtm9eS3O6lrOLYmQag2DjdEXuPaHlZ6RuVqgqaVZfgYCepEIKsLqwdQArOPtC3XzRLqGGfT8KQQ=="
+    },
+    "es6-promisify": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz",
+      "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=",
+      "requires": {
+        "es6-promise": "4.2.4"
+      }
+    },
+    "escape-html": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
+      "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg="
+    },
+    "etag": {
+      "version": "1.7.0",
+      "resolved": "https://registry.npmjs.org/etag/-/etag-1.7.0.tgz",
+      "integrity": "sha1-A9MLX2fdbmMtKUXTDWZScxo01dg="
+    },
+    "express": {
+      "version": "4.13.4",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.13.4.tgz",
+      "integrity": "sha1-PAt288d1kMg0VzkGHsC9O6Bn7CQ=",
+      "requires": {
+        "accepts": "1.2.13",
+        "array-flatten": "1.1.1",
+        "content-disposition": "0.5.1",
+        "content-type": "1.0.4",
+        "cookie": "0.1.5",
+        "cookie-signature": "1.0.6",
+        "debug": "2.2.0",
+        "depd": "1.1.2",
+        "escape-html": "1.0.3",
+        "etag": "1.7.0",
+        "finalhandler": "0.4.1",
+        "fresh": "0.3.0",
+        "merge-descriptors": "1.0.1",
+        "methods": "1.1.2",
+        "on-finished": "2.3.0",
+        "parseurl": "1.3.2",
+        "path-to-regexp": "0.1.7",
+        "proxy-addr": "1.0.10",
+        "qs": "4.0.0",
+        "range-parser": "1.0.3",
+        "send": "0.13.1",
+        "serve-static": "1.10.3",
+        "type-is": "1.6.16",
+        "utils-merge": "1.0.0",
+        "vary": "1.0.1"
+      },
+      "dependencies": {
+        "cookie": {
+          "version": "0.1.5",
+          "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.1.5.tgz",
+          "integrity": "sha1-armUiksa4hlSzSWIUwpHItQETXw="
+        },
+        "depd": {
+          "version": "1.1.2",
+          "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
+          "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak="
+        }
+      }
+    },
+    "extend": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz",
+      "integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ="
+    },
+    "extract-zip": {
+      "version": "1.6.7",
+      "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.7.tgz",
+      "integrity": "sha1-qEC0uK9kAyZMjbV/Txp0Mz74H+k=",
+      "requires": {
+        "concat-stream": "1.6.2",
+        "debug": "2.6.9",
+        "mkdirp": "0.5.1",
+        "yauzl": "2.4.1"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+          "requires": {
+            "ms": "2.0.0"
+          }
+        },
+        "ms": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+          "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+        }
+      }
+    },
+    "extsprintf": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
+      "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU="
+    },
+    "fast-deep-equal": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz",
+      "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ="
+    },
+    "fast-json-stable-stringify": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz",
+      "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I="
+    },
+    "fd-slicer": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz",
+      "integrity": "sha1-i1vL2ewyfFBBv5qwI/1nUPEXfmU=",
+      "requires": {
+        "pend": "1.2.0"
+      }
+    },
+    "finalhandler": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-0.4.1.tgz",
+      "integrity": "sha1-haF8bFmpRxfSYtYSMNSw6+PUoU0=",
+      "requires": {
+        "debug": "2.2.0",
+        "escape-html": "1.0.3",
+        "on-finished": "2.3.0",
+        "unpipe": "1.0.0"
+      }
+    },
+    "flexbuffer": {
+      "version": "0.0.6",
+      "resolved": "https://registry.npmjs.org/flexbuffer/-/flexbuffer-0.0.6.tgz",
+      "integrity": "sha1-A5/fI/iCPkQMOPMnfm/vEXQhWzA=",
+      "dev": true
+    },
+    "forever-agent": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
+      "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
+    },
+    "form-data": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.2.tgz",
+      "integrity": "sha1-SXBJi+YEwgwAXU9cI67NIda0kJk=",
+      "requires": {
+        "asynckit": "0.4.0",
+        "combined-stream": "1.0.6",
+        "mime-types": "2.1.18"
+      }
+    },
+    "forwarded": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
+      "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ="
+    },
+    "fresh": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.3.0.tgz",
+      "integrity": "sha1-ZR+DjiJCTnVm3hYdg1jKoZn4PU8="
+    },
+    "fs.realpath": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
+    },
+    "getpass": {
+      "version": "0.1.7",
+      "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
+      "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
+      "requires": {
+        "assert-plus": "1.0.0"
+      }
+    },
+    "glob": {
+      "version": "7.1.2",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz",
+      "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==",
+      "requires": {
+        "fs.realpath": "1.0.0",
+        "inflight": "1.0.6",
+        "inherits": "2.0.3",
+        "minimatch": "3.0.4",
+        "once": "1.4.0",
+        "path-is-absolute": "1.0.1"
+      }
+    },
+    "graceful-fs": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-1.2.3.tgz",
+      "integrity": "sha1-FaSAaldUfLLS2/J/QuiajDRRs2Q=",
+      "dev": true
+    },
+    "growl": {
+      "version": "1.7.0",
+      "resolved": "https://registry.npmjs.org/growl/-/growl-1.7.0.tgz",
+      "integrity": "sha1-3i1mE20ALhErpw8/EMMc98NQsto=",
+      "dev": true
+    },
+    "har-schema": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
+      "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI="
+    },
+    "har-validator": {
+      "version": "5.0.3",
+      "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.0.3.tgz",
+      "integrity": "sha1-ukAsJmGU8VlW7xXg/PJCmT9qff0=",
+      "requires": {
+        "ajv": "5.5.2",
+        "har-schema": "2.0.0"
+      }
+    },
+    "htmlparser2": {
+      "version": "3.9.2",
+      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.9.2.tgz",
+      "integrity": "sha1-G9+HrMoPP55T+k/M6w9LTLsAszg=",
+      "requires": {
+        "domelementtype": "1.3.0",
+        "domhandler": "2.4.2",
+        "domutils": "1.5.1",
+        "entities": "1.1.1",
+        "inherits": "2.0.3",
+        "readable-stream": "2.3.6"
+      }
+    },
+    "http-errors": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
+      "integrity": "sha1-GX4izevUGYWF6GlO9nhhl7ke2UI=",
+      "requires": {
+        "inherits": "2.0.3",
+        "statuses": "1.5.0"
+      }
+    },
+    "http-signature": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
+      "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=",
+      "requires": {
+        "assert-plus": "1.0.0",
+        "jsprim": "1.4.1",
+        "sshpk": "1.14.2"
+      }
+    },
+    "https-proxy-agent": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz",
+      "integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==",
+      "requires": {
+        "agent-base": "4.2.0",
+        "debug": "3.1.0"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+          "requires": {
+            "ms": "2.0.0"
+          }
+        },
+        "ms": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+          "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+        }
+      }
+    },
+    "iconv-lite": {
+      "version": "0.4.11",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.11.tgz",
+      "integrity": "sha1-LstC/SlHRJIiCaLnxATayHk9it4="
+    },
+    "inflight": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+      "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
+      "requires": {
+        "once": "1.4.0",
+        "wrappy": "1.0.2"
+      }
+    },
+    "inherits": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
+      "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
+    },
+    "ini": {
+      "version": "1.3.5",
+      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
+      "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==",
+      "dev": true
+    },
+    "ioredis": {
+      "version": "1.15.1",
+      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-1.15.1.tgz",
+      "integrity": "sha1-UlJVzM1Ve904oO00ZhmfWesLnRw=",
+      "dev": true,
+      "requires": {
+        "bluebird": "2.11.0",
+        "debug": "2.2.0",
+        "double-ended-queue": "2.1.0-0",
+        "flexbuffer": "0.0.6",
+        "lodash": "3.10.1"
+      },
+      "dependencies": {
+        "bluebird": {
+          "version": "2.11.0",
+          "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.11.0.tgz",
+          "integrity": "sha1-U0uQM8AiyVecVro7Plpcqvu2UOE=",
+          "dev": true
+        },
+        "lodash": {
+          "version": "3.10.1",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz",
+          "integrity": "sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y=",
+          "dev": true
+        }
+      }
+    },
+    "ipaddr.js": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.0.5.tgz",
+      "integrity": "sha1-X6eM8wG4JceKvDBC2BJyMEnqI8c="
+    },
+    "is-typedarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
+      "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo="
+    },
+    "isarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
+    },
+    "isstream": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
+      "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
+    },
+    "jade": {
+      "version": "0.26.3",
+      "resolved": "https://registry.npmjs.org/jade/-/jade-0.26.3.tgz",
+      "integrity": "sha1-jxDXl32NefL2/4YqgbBRPMslaGw=",
+      "dev": true,
+      "requires": {
+        "commander": "0.6.1",
+        "mkdirp": "0.3.0"
+      },
+      "dependencies": {
+        "commander": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/commander/-/commander-0.6.1.tgz",
+          "integrity": "sha1-+mihT2qUXVTbvlDYzbMyDp47GgY=",
+          "dev": true
+        },
+        "mkdirp": {
+          "version": "0.3.0",
+          "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.0.tgz",
+          "integrity": "sha1-G79asbqCevI1dRQ0kEJkVfSB/h4=",
+          "dev": true
+        }
+      }
+    },
+    "js-beautify": {
+      "version": "1.4.2",
+      "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.4.2.tgz",
+      "integrity": "sha1-iILfRQqejs910ifxzOtajslolrc=",
+      "dev": true,
+      "requires": {
+        "config-chain": "1.1.11",
+        "mkdirp": "0.3.5",
+        "nopt": "2.1.2"
+      },
+      "dependencies": {
+        "mkdirp": {
+          "version": "0.3.5",
+          "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz",
+          "integrity": "sha1-3j5fiWHIjHh+4TaN+EmsRBPsqNc=",
+          "dev": true
+        }
+      }
+    },
+    "jsbn": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
+      "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=",
+      "optional": true
+    },
+    "json-schema": {
+      "version": "0.2.3",
+      "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
+      "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM="
+    },
+    "json-schema-traverse": {
+      "version": "0.3.1",
+      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz",
+      "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A="
+    },
+    "json-stringify-safe": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
+      "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
+    },
+    "jsprim": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
+      "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=",
+      "requires": {
+        "assert-plus": "1.0.0",
+        "extsprintf": "1.3.0",
+        "json-schema": "0.2.3",
+        "verror": "1.10.0"
+      }
+    },
+    "lodash": {
+      "version": "4.17.10",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.10.tgz",
+      "integrity": "sha512-UejweD1pDoXu+AD825lWwp4ZGtSwgnpZxb3JDViD7StjQz+Nb/6l093lx4OQ0foGWNRoc19mWy7BzL+UAK2iVg=="
+    },
+    "lru-cache": {
+      "version": "2.7.3",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.7.3.tgz",
+      "integrity": "sha1-bUUk6LlV+V1PW1iFHOId1y+06VI=",
+      "dev": true
+    },
+    "media-typer": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
+      "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g="
+    },
+    "merge-descriptors": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
+      "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E="
+    },
+    "methods": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
+      "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4="
+    },
+    "mime": {
+      "version": "1.3.4",
+      "resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz",
+      "integrity": "sha1-EV+eO2s9rylZmDyzjxSaLUDrXVM="
+    },
+    "mime-db": {
+      "version": "1.33.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz",
+      "integrity": "sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ=="
+    },
+    "mime-types": {
+      "version": "2.1.18",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz",
+      "integrity": "sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==",
+      "requires": {
+        "mime-db": "1.33.0"
+      }
+    },
+    "minimatch": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
+      "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+      "requires": {
+        "brace-expansion": "1.1.11"
+      }
+    },
+    "minimist": {
+      "version": "0.0.8",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
+      "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
+    },
+    "mkdirp": {
+      "version": "0.5.1",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
+      "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
+      "requires": {
+        "minimist": "0.0.8"
+      }
+    },
+    "mocha": {
+      "version": "1.17.0",
+      "resolved": "https://registry.npmjs.org/mocha/-/mocha-1.17.0.tgz",
+      "integrity": "sha1-WD0SqfIXdbbTfRBKx9sF1Vf6XVA=",
+      "dev": true,
+      "requires": {
+        "commander": "2.0.0",
+        "debug": "2.2.0",
+        "diff": "1.0.7",
+        "glob": "3.2.3",
+        "growl": "1.7.0",
+        "jade": "0.26.3",
+        "mkdirp": "0.3.5"
+      },
+      "dependencies": {
+        "glob": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/glob/-/glob-3.2.3.tgz",
+          "integrity": "sha1-4xPusknHr/qlxHUoaw4RW1mDlGc=",
+          "dev": true,
+          "requires": {
+            "graceful-fs": "2.0.3",
+            "inherits": "2.0.3",
+            "minimatch": "0.2.14"
+          }
+        },
+        "graceful-fs": {
+          "version": "2.0.3",
+          "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-2.0.3.tgz",
+          "integrity": "sha1-fNLNsiiko/Nule+mzBQt59GhNtA=",
+          "dev": true
+        },
+        "minimatch": {
+          "version": "0.2.14",
+          "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-0.2.14.tgz",
+          "integrity": "sha1-x054BXT2PG+aCQ6Q775u9TpqdWo=",
+          "dev": true,
+          "requires": {
+            "lru-cache": "2.7.3",
+            "sigmund": "1.0.1"
+          }
+        },
+        "mkdirp": {
+          "version": "0.3.5",
+          "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz",
+          "integrity": "sha1-3j5fiWHIjHh+4TaN+EmsRBPsqNc=",
+          "dev": true
+        }
+      }
+    },
+    "morgan": {
+      "version": "1.6.1",
+      "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.6.1.tgz",
+      "integrity": "sha1-X9gYOYxoGcuiinzWZk8pL+HAu/I=",
+      "requires": {
+        "basic-auth": "1.0.4",
+        "debug": "2.2.0",
+        "depd": "1.0.1",
+        "on-finished": "2.3.0",
+        "on-headers": "1.0.1"
+      }
+    },
+    "ms": {
+      "version": "0.7.1",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz",
+      "integrity": "sha1-nNE8A62/8ltl7/3nzoZO6VIBcJg="
+    },
+    "mysql": {
+      "version": "2.15.0",
+      "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.15.0.tgz",
+      "integrity": "sha512-C7tjzWtbN5nzkLIV+E8Crnl9bFyc7d3XJcBAvHKEVkjrYjogz3llo22q6s/hw+UcsE4/844pDob9ac+3dVjQSA==",
+      "dev": true,
+      "requires": {
+        "bignumber.js": "4.0.4",
+        "readable-stream": "2.3.3",
+        "safe-buffer": "5.1.1",
+        "sqlstring": "2.3.0"
+      },
+      "dependencies": {
+        "process-nextick-args": {
+          "version": "1.0.7",
+          "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz",
+          "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=",
+          "dev": true
+        },
+        "readable-stream": {
+          "version": "2.3.3",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.3.tgz",
+          "integrity": "sha512-m+qzzcn7KUxEmd1gMbchF+Y2eIUbieUaxkWtptyHywrX0rE8QEYqPC07Vuy4Wm32/xE16NcdBctb8S0Xe/5IeQ==",
+          "dev": true,
+          "requires": {
+            "core-util-is": "1.0.2",
+            "inherits": "2.0.3",
+            "isarray": "1.0.0",
+            "process-nextick-args": "1.0.7",
+            "safe-buffer": "5.1.1",
+            "string_decoder": "1.0.3",
+            "util-deprecate": "1.0.2"
+          }
+        },
+        "safe-buffer": {
+          "version": "5.1.1",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
+          "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==",
+          "dev": true
+        },
+        "string_decoder": {
+          "version": "1.0.3",
+          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz",
+          "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==",
+          "dev": true,
+          "requires": {
+            "safe-buffer": "5.1.1"
+          }
+        }
+      }
+    },
+    "negotiator": {
+      "version": "0.5.3",
+      "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.5.3.tgz",
+      "integrity": "sha1-Jp1cR2gQ7JLtvntsLygxY4T5p+g="
+    },
+    "nopt": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/nopt/-/nopt-2.1.2.tgz",
+      "integrity": "sha1-bMzZd7gBMqB3MdbozljCyDA8+a8=",
+      "dev": true,
+      "requires": {
+        "abbrev": "1.1.1"
+      }
+    },
+    "nth-check": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.1.tgz",
+      "integrity": "sha1-mSms32KPwsQQmN6rgqxYDPFJquQ=",
+      "requires": {
+        "boolbase": "1.0.0"
+      }
+    },
+    "oauth-sign": {
+      "version": "0.8.2",
+      "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz",
+      "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM="
+    },
+    "on-finished": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
+      "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=",
+      "requires": {
+        "ee-first": "1.1.1"
+      }
+    },
+    "on-headers": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.1.tgz",
+      "integrity": "sha1-ko9dD0cNSTQmUepnlLCFfBAGk/c="
+    },
+    "once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+      "requires": {
+        "wrappy": "1.0.2"
+      }
+    },
+    "parse5": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz",
+      "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==",
+      "requires": {
+        "@types/node": "10.3.2"
+      }
+    },
+    "parseurl": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz",
+      "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M="
+    },
+    "path-is-absolute": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+      "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
+    },
+    "path-to-regexp": {
+      "version": "0.1.7",
+      "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
+      "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
+    },
+    "pend": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
+      "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA="
+    },
+    "performance-now": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
+      "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns="
+    },
+    "phpjs": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/phpjs/-/phpjs-1.3.2.tgz",
+      "integrity": "sha1-XHusVdHeCegFQgIlkxYb2YA0irs=",
+      "dev": true,
+      "requires": {
+        "cli": "0.4.4-2",
+        "deep-equal": "0.1.2",
+        "glob": "3.2.1",
+        "js-beautify": "1.4.2",
+        "mocha": "1.17.0",
+        "send": "0.1.0",
+        "underscore": "1.5.2"
+      },
+      "dependencies": {
+        "fresh": {
+          "version": "0.1.0",
+          "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.1.0.tgz",
+          "integrity": "sha1-A+SwF4Qk5MLV0ZpU2IFM3JeTSFA=",
+          "dev": true
+        },
+        "glob": {
+          "version": "3.2.1",
+          "resolved": "https://registry.npmjs.org/glob/-/glob-3.2.1.tgz",
+          "integrity": "sha1-V69w7HO6IyO/4/KaBndl22TF11g=",
+          "dev": true,
+          "requires": {
+            "graceful-fs": "1.2.3",
+            "inherits": "1.0.2",
+            "minimatch": "0.2.14"
+          }
+        },
+        "inherits": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/inherits/-/inherits-1.0.2.tgz",
+          "integrity": "sha1-ykMJ2t7mtUzAuNJH6NfHoJdb3Js=",
+          "dev": true
+        },
+        "mime": {
+          "version": "1.2.6",
+          "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.6.tgz",
+          "integrity": "sha1-sfhsdowCX6h7SAdfFwnyiuryA2U=",
+          "dev": true
+        },
+        "minimatch": {
+          "version": "0.2.14",
+          "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-0.2.14.tgz",
+          "integrity": "sha1-x054BXT2PG+aCQ6Q775u9TpqdWo=",
+          "dev": true,
+          "requires": {
+            "lru-cache": "2.7.3",
+            "sigmund": "1.0.1"
+          }
+        },
+        "range-parser": {
+          "version": "0.0.4",
+          "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-0.0.4.tgz",
+          "integrity": "sha1-wEJ//vUcEKy6B4KkbJYC50T/Ygs=",
+          "dev": true
+        },
+        "send": {
+          "version": "0.1.0",
+          "resolved": "https://registry.npmjs.org/send/-/send-0.1.0.tgz",
+          "integrity": "sha1-z7COvTzsm3/Bo32f+eh1qXHPRkA=",
+          "dev": true,
+          "requires": {
+            "debug": "2.2.0",
+            "fresh": "0.1.0",
+            "mime": "1.2.6",
+            "range-parser": "0.0.4"
+          }
+        },
+        "underscore": {
+          "version": "1.5.2",
+          "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.5.2.tgz",
+          "integrity": "sha1-EzXF5PXm0zu7SwBrqMhqAPVW3gg=",
+          "dev": true
+        }
+      }
+    },
+    "process-nextick-args": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
+      "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw=="
+    },
+    "progress": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.0.tgz",
+      "integrity": "sha1-ihvjZr+Pwj2yvSPxDG/pILQ4nR8="
+    },
+    "proto-list": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz",
+      "integrity": "sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk=",
+      "dev": true
+    },
+    "proxy-addr": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-1.0.10.tgz",
+      "integrity": "sha1-DUCoL4Afw1VWfS7LZe/j8HfxIcU=",
+      "requires": {
+        "forwarded": "0.1.2",
+        "ipaddr.js": "1.0.5"
+      }
+    },
+    "proxy-from-env": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz",
+      "integrity": "sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4="
+    },
+    "punycode": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
+      "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
+    },
+    "puppeteer": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.5.0.tgz",
+      "integrity": "sha512-eELwFtFxL+uhmg4jPZOZXzSrPEYy4CaYQNbcchBbfxY+KjMpnv6XGf/aYWaQG49OTpfi2/DMziXtDM8XuJgoUA==",
+      "requires": {
+        "debug": "3.1.0",
+        "extract-zip": "1.6.7",
+        "https-proxy-agent": "2.2.1",
+        "mime": "2.3.1",
+        "progress": "2.0.0",
+        "proxy-from-env": "1.0.0",
+        "rimraf": "2.6.2",
+        "ws": "5.2.0"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+          "requires": {
+            "ms": "2.0.0"
+          }
+        },
+        "mime": {
+          "version": "2.3.1",
+          "resolved": "https://registry.npmjs.org/mime/-/mime-2.3.1.tgz",
+          "integrity": "sha512-OEUllcVoydBHGN1z84yfQDimn58pZNNNXgZlHXSboxMlFvgI6MXSWpWKpFRra7H1HxpVhHTkrghfRW49k6yjeg=="
+        },
+        "ms": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+          "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+        }
+      }
+    },
+    "qs": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-4.0.0.tgz",
+      "integrity": "sha1-wx2bdOwn33XlQ6hseHKO2NRiNgc="
+    },
+    "range-parser": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.0.3.tgz",
+      "integrity": "sha1-aHKCNTXGkuLCoBA4Jq/YLC4P8XU="
+    },
+    "raw-body": {
+      "version": "2.1.7",
+      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.1.7.tgz",
+      "integrity": "sha1-rf6s4uT7MJgFgBTQjActzFl1h3Q=",
+      "requires": {
+        "bytes": "2.4.0",
+        "iconv-lite": "0.4.13",
+        "unpipe": "1.0.0"
+      },
+      "dependencies": {
+        "bytes": {
+          "version": "2.4.0",
+          "resolved": "https://registry.npmjs.org/bytes/-/bytes-2.4.0.tgz",
+          "integrity": "sha1-fZcZb51br39pNeJZhVSe3SpsIzk="
+        },
+        "iconv-lite": {
+          "version": "0.4.13",
+          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.13.tgz",
+          "integrity": "sha1-H4irpKsLFQjoMSrMOTRfNumS4vI="
+        }
+      }
+    },
+    "readable-stream": {
+      "version": "2.3.6",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
+      "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
+      "requires": {
+        "core-util-is": "1.0.2",
+        "inherits": "2.0.3",
+        "isarray": "1.0.0",
+        "process-nextick-args": "2.0.0",
+        "safe-buffer": "5.1.2",
+        "string_decoder": "1.1.1",
+        "util-deprecate": "1.0.2"
+      }
+    },
+    "request": {
+      "version": "2.87.0",
+      "resolved": "https://registry.npmjs.org/request/-/request-2.87.0.tgz",
+      "integrity": "sha512-fcogkm7Az5bsS6Sl0sibkbhcKsnyon/jV1kF3ajGmF0c8HrttdKTPRT9hieOaQHA5HEq6r8OyWOo/o781C1tNw==",
+      "requires": {
+        "aws-sign2": "0.7.0",
+        "aws4": "1.7.0",
+        "caseless": "0.12.0",
+        "combined-stream": "1.0.6",
+        "extend": "3.0.1",
+        "forever-agent": "0.6.1",
+        "form-data": "2.3.2",
+        "har-validator": "5.0.3",
+        "http-signature": "1.2.0",
+        "is-typedarray": "1.0.0",
+        "isstream": "0.1.2",
+        "json-stringify-safe": "5.0.1",
+        "mime-types": "2.1.18",
+        "oauth-sign": "0.8.2",
+        "performance-now": "2.1.0",
+        "qs": "6.5.2",
+        "safe-buffer": "5.1.2",
+        "tough-cookie": "2.3.4",
+        "tunnel-agent": "0.6.0",
+        "uuid": "3.2.1"
+      },
+      "dependencies": {
+        "qs": {
+          "version": "6.5.2",
+          "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
+          "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA=="
+        }
+      }
+    },
+    "request-promise": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/request-promise/-/request-promise-4.2.2.tgz",
+      "integrity": "sha1-0epG1lSm7k+O5qT+oQGMIpEZBLQ=",
+      "requires": {
+        "bluebird": "3.5.1",
+        "request-promise-core": "1.1.1",
+        "stealthy-require": "1.1.1",
+        "tough-cookie": "2.3.4"
+      }
+    },
+    "request-promise-core": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.1.tgz",
+      "integrity": "sha1-Pu4AssWqgyOc+wTFcA2jb4HNCLY=",
+      "requires": {
+        "lodash": "4.17.10"
+      }
+    },
+    "rimraf": {
+      "version": "2.6.2",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz",
+      "integrity": "sha512-lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w==",
+      "requires": {
+        "glob": "7.1.2"
+      }
+    },
+    "safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
+    },
+    "safer-buffer": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
+      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
+    },
+    "send": {
+      "version": "0.13.1",
+      "resolved": "https://registry.npmjs.org/send/-/send-0.13.1.tgz",
+      "integrity": "sha1-ow1fTILIqbrprQCh2bG9vm8Zntc=",
+      "requires": {
+        "debug": "2.2.0",
+        "depd": "1.1.2",
+        "destroy": "1.0.4",
+        "escape-html": "1.0.3",
+        "etag": "1.7.0",
+        "fresh": "0.3.0",
+        "http-errors": "1.3.1",
+        "mime": "1.3.4",
+        "ms": "0.7.1",
+        "on-finished": "2.3.0",
+        "range-parser": "1.0.3",
+        "statuses": "1.2.1"
+      },
+      "dependencies": {
+        "depd": {
+          "version": "1.1.2",
+          "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
+          "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak="
+        },
+        "statuses": {
+          "version": "1.2.1",
+          "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz",
+          "integrity": "sha1-3e1FzBglbVHtQK7BQkidXGECbSg="
+        }
+      }
+    },
+    "serve-static": {
+      "version": "1.10.3",
+      "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.10.3.tgz",
+      "integrity": "sha1-zlpuzTEB/tXsCYJ9rCKpwpv7BTU=",
+      "requires": {
+        "escape-html": "1.0.3",
+        "parseurl": "1.3.2",
+        "send": "0.13.2"
+      },
+      "dependencies": {
+        "depd": {
+          "version": "1.1.2",
+          "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
+          "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak="
+        },
+        "send": {
+          "version": "0.13.2",
+          "resolved": "https://registry.npmjs.org/send/-/send-0.13.2.tgz",
+          "integrity": "sha1-dl52B8gFVFK7pvCwUllTUJhgNt4=",
+          "requires": {
+            "debug": "2.2.0",
+            "depd": "1.1.2",
+            "destroy": "1.0.4",
+            "escape-html": "1.0.3",
+            "etag": "1.7.0",
+            "fresh": "0.3.0",
+            "http-errors": "1.3.1",
+            "mime": "1.3.4",
+            "ms": "0.7.1",
+            "on-finished": "2.3.0",
+            "range-parser": "1.0.3",
+            "statuses": "1.2.1"
+          }
+        },
+        "statuses": {
+          "version": "1.2.1",
+          "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz",
+          "integrity": "sha1-3e1FzBglbVHtQK7BQkidXGECbSg="
+        }
+      }
+    },
+    "sigmund": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz",
+      "integrity": "sha1-P/IfGYytIXX587eBhT/ZTQ0ZtZA=",
+      "dev": true
+    },
+    "sqlstring": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.0.tgz",
+      "integrity": "sha1-UluKT9Jtb3GqYegipsr5dtMa0qg=",
+      "dev": true
+    },
+    "sshpk": {
+      "version": "1.14.2",
+      "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.14.2.tgz",
+      "integrity": "sha1-xvxhZIo9nE52T9P8306hBeSSupg=",
+      "requires": {
+        "asn1": "0.2.3",
+        "assert-plus": "1.0.0",
+        "bcrypt-pbkdf": "1.0.1",
+        "dashdash": "1.14.1",
+        "ecc-jsbn": "0.1.1",
+        "getpass": "0.1.7",
+        "jsbn": "0.1.1",
+        "safer-buffer": "2.1.2",
+        "tweetnacl": "0.14.5"
+      }
+    },
+    "statuses": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
+      "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow="
+    },
+    "stealthy-require": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
+      "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks="
+    },
+    "string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+      "requires": {
+        "safe-buffer": "5.1.2"
+      }
+    },
+    "tough-cookie": {
+      "version": "2.3.4",
+      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz",
+      "integrity": "sha512-TZ6TTfI5NtZnuyy/Kecv+CnoROnyXn2DN97LontgQpCwsX2XyLYCC0ENhYkehSOwAp8rTQKc/NUIF7BkQ5rKLA==",
+      "requires": {
+        "punycode": "1.4.1"
+      }
+    },
+    "tunnel-agent": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
+      "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
+      "requires": {
+        "safe-buffer": "5.1.2"
+      }
+    },
+    "tweetnacl": {
+      "version": "0.14.5",
+      "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
+      "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=",
+      "optional": true
+    },
+    "type-is": {
+      "version": "1.6.16",
+      "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz",
+      "integrity": "sha512-HRkVv/5qY2G6I8iab9cI7v1bOIdhm94dVjQCPFElW9W+3GeDOSHmy2EBYe4VTApuzolPcmgFTN3ftVJRKR2J9Q==",
+      "requires": {
+        "media-typer": "0.3.0",
+        "mime-types": "2.1.18"
+      }
+    },
+    "typedarray": {
+      "version": "0.0.6",
+      "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
+      "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
+    },
+    "underscore": {
+      "version": "1.9.1",
+      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.9.1.tgz",
+      "integrity": "sha512-5/4etnCkd9c8gwgowi5/om/mYO5ajCaOgdzj/oW+0eQV9WxKBDZw5+ycmKmeaTXjInS/W0BzpGLo2xR2aBwZdg==",
+      "dev": true
+    },
+    "unpipe": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
+      "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
+    },
+    "util-deprecate": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
+      "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
+    },
+    "utils-merge": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz",
+      "integrity": "sha1-ApT7kiu5N1FTVBxPcJYjHyh8ivg="
+    },
+    "uuid": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.2.1.tgz",
+      "integrity": "sha512-jZnMwlb9Iku/O3smGWvZhauCf6cvvpKi4BKRiliS3cxnI+Gz9j5MEpTz2UFuXiKPJocb7gnsLHwiS05ige5BEA=="
+    },
+    "validator": {
+      "version": "4.9.0",
+      "resolved": "https://registry.npmjs.org/validator/-/validator-4.9.0.tgz",
+      "integrity": "sha1-CC/84qdhSP8HqOienCukOq8S7Ew=",
+      "dev": true,
+      "requires": {
+        "depd": "1.1.0"
+      },
+      "dependencies": {
+        "depd": {
+          "version": "1.1.0",
+          "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz",
+          "integrity": "sha1-4b2Cxqq2ztlluXuIsX7T5SjKGMM=",
+          "dev": true
+        }
+      }
+    },
+    "vary": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/vary/-/vary-1.0.1.tgz",
+      "integrity": "sha1-meSYFWaihhGN+yuBc1ffeZM3bRA="
+    },
+    "verror": {
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
+      "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=",
+      "requires": {
+        "assert-plus": "1.0.0",
+        "core-util-is": "1.0.2",
+        "extsprintf": "1.3.0"
+      }
+    },
+    "wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
+    },
+    "ws": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-5.2.0.tgz",
+      "integrity": "sha512-c18dMeW+PEQdDFzkhDsnBAlS4Z8KGStBQQUcQ5mf7Nf689jyGk0594L+i9RaQuf4gog6SvWLJorz2NfSaqxZ7w==",
+      "requires": {
+        "async-limiter": "1.0.0"
+      }
+    },
+    "yauzl": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz",
+      "integrity": "sha1-lSj0QtqxsihOWLQ3m7GU4i4MQAU=",
+      "requires": {
+        "fd-slicer": "1.0.1"
+      }
+    }
+  }
+}

protected/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "crawl",
+  "version": "1.0.0",
+  "description": "headless crawl platform",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "ben",
+  "license": "ISC",
+  "dependencies": {
+    "body-parser": "~1.13.2",
+    "cheerio": "^1.0.0-rc.2",
+    "cookie-parser": "~1.3.5",
+    "debug": "~2.2.0",
+    "ejs": "~2.3.3",
+    "express": "~4.13.1",
+    "morgan": "~1.6.1",
+    "puppeteer": "^1.5.0",
+    "request": "^2.83.0",
+    "request-promise": "^4.2.2"
+  },
+  "devDependencies": {
+    "ioredis": "^1.7.3",
+    "mysql": "^2.8.0",
+    "phpjs": "^1.3.2",
+    "underscore": "^1.8.3",
+    "validator": "^4.0.5"
+  }
+}

protected/test/coTest.js

@@ -0,0 +1,33 @@
+/**
+ * Created by benzhan on 15/8/20.
+ */
+var co = require('co');
+var fs = require('fs');
+var Q = require('q');
+
+//var readdir = Q.nbind(fs.readdir)
+//co(function* () {
+//    console.log(1);
+//    var files = yield readdir('./');
+//    console.log(files);
+//    console.log(2);
+//    return files;
+//}).then(console.log, console.error);
+
+var compute = function* (a, b) {
+    var foo = a + b;
+    yield console.log(foo);
+
+    foo = foo + 5;
+    yield console.log(foo);
+
+    console.log(23);
+};
+
+var generator = compute(4, 2);
+generator.next(3, 8);
+generator.next("Hello world!"); // Hello world!
+generator.next(3, 8);
+
+var i = 1;
+console.log('i:' + i);

protected/test/profilerTest.js

@@ -0,0 +1,18 @@
+#!/usr/bin/env node --debug
+
+require('v8-profiler');
+
+// It is important to use named constructors (like the one below), otherwise
+// the heap snapshots will not produce useful outputs for you.
+function LeakingClass() {
+}
+
+var leaks = [];
+setInterval(function() {
+    for (var i = 0; i < 100; i++) {
+        leaks.push(new LeakingClass);
+    }
+
+    console.error('Leaks: %d', leaks.length);
+}, 1000);
+

protected/test/testOujMySql.js

@@ -0,0 +1,52 @@
+"use strict";
+
+require('../extensions/function_extend.js');
+let OujMySql = require('../framework/lib/OujMySql.js');
+let objMysql = new OujMySql('Web');
+
+var sql = "SELECT * FROM  `cUser`";
+var allDeferred = objMysql.getAll(sql, 5);
+allDeferred.then(function(rows) {
+    console.log('getAll');
+    console.log(rows);
+
+    objMysql.close();
+});
+
+//var sql = "SELECT userName FROM  `cUser`";
+//var oneDeferred = objMysql.getOne(sql);
+//oneDeferred.then(function(one) {
+//    console.log('getOne');
+//    console.log(one);
+//}, function(err) {
+//    console.log(err);
+//});
+//
+//
+//var colDeferred = objMysql.getCol(sql, 3);
+//colDeferred.then(function(col) {
+//    console.log('getCol');
+//    console.log(col);
+//});
+//
+//var colDeferred = objMysql.getRow(sql);
+//colDeferred.then(function(row) {
+//    console.log('getRow');
+//    console.log(row);
+//});
+
+//
+//var updateSql = "";
+//
+//objMysql.update(sql, function(err, rows) {
+//    console.log('getAll');
+//    console.log(rows);
+//}, 5);
+
+//setTimeout(function() {
+//    objMysql.close(function() {
+//        console.log('close');
+//    });
+//}, 1000);
+
+

protected/test/testParam.js

@@ -0,0 +1,42 @@
+var Param = require('../framework/lib/Param.js');
+
+var rules = {
+    'checkIntArr' : {
+        type : 'intArr'
+    },
+    'checkObj' : {
+        type : 'object'
+    },
+    'checkJson' : {
+        type : 'json'
+    },
+    'appId' : {
+        type : 'string',
+        range : '(5, 10)'
+    },
+    'str' : {
+        type : 'float',
+        len : '[0, 10]'
+    },
+    'nullable' : {
+        type : 'string',
+        nullable : true
+    }
+}
+
+var args = {
+    'checkIntArr' : [111,222,333],
+    'checkObj' : {
+        qwe : 111,
+        fff : 322
+    },
+    'checkJson' : '{"111":123,"qqq":333}',
+    'appId' : 6,
+    'str' : 123,
+    'nullable' : '',
+    'fake' : '123'
+}
+
+var testResult = Param.checkParam2(rules, args);
+console.log(args);
+

protected/test/testR2mClient.js

@@ -0,0 +1,66 @@
+'use strict'
+
+/**
+ * @author hawklim
+ */
+
+require('../conf/config.dev.inc.js');
+
+var r2m_client = require('../framework/lib/r2m/Client.js');
+var co = require('co');
+
+
+ //co(function*() {
+	// let objCate = new r2m_client('category', 'hiyd_cms', 'hiyd_cms');
+ //
+	// let where = {'name':'上肢'};
+ //	let val1 = yield objCate.getRow(where);
+ //	console.log(val1);
+ //
+ //	let val2 = yield objCate.getAll();
+ //	console.log(val2);
+ //
+ //	let where2 = {id:3};
+ //	let val3 = yield objCate.getRow(where2);
+ //	console.log(val3);
+ //
+	// let objUser = new r2m_client('user', 'hiyd_home', 'hiyd_home')
+	// let val4 = yield objUser.getAll({user_id:20});
+	// console.log(val4);
+ //
+	// let objTags = new r2m_client('tags', 'hiyd_cms', 'hiyd_cms');
+	// let val5 = yield objTags.getAll();
+	// console.log(val5);
+ //});
+
+let objCate = new r2m_client('category', 'hiyd_cms', 'hiyd_cms');
+let where = {};
+objCate.getAll().then((val3) => {
+	console.log(val3);
+}).catch((err) => {
+	console.log(`err ${err}`);
+});
+objCate.getAll().then((val3) => {
+	console.log(val3);
+}).catch((err) => {
+	console.log(`err ${err}`);
+});
+where['name'] = '上肢';
+objCate.getRow(where).then((val) => {
+
+	console.log(val);
+
+}).catch((err) => {
+   console.log(`err ${err}`);
+});
+objCate.getAll().then((val) => {
+
+	console.log(val);
+
+}).catch((err) => {
+	console.log(`err ${err}`);
+});
+ let objTags = new r2m_client('tags', 'hiyd_cms', 'hiyd_cms');
+ objTags.getAll().then(function(val5) {
+	 console.log(val5);
+ });

protected/test/testRedis.js

@@ -0,0 +1,44 @@
+/**
+ * Created by benzhan on 15/8/10.
+ */
+
+var OujRedis = require('../framework/lib/OujRedis.js');
+var objRedis = OujRedis.init('Web');
+
+var test = objRedis.get('test');
+test.then(function(data) {
+    console.log(data);
+    objRedis.set('test', data + "1").then(function(data) {
+        console.log(data);
+        test = objRedis.get('test');
+        test.then(function(data) {
+            console.log(data);
+            objRedis.disconnect();
+        }, function(err) {
+            console.log(err);
+            objRedis.disconnect();
+        });
+    });
+}, function(err) {
+    console.log(err);
+});
+
+var pipeline = objRedis.pipeline();
+pipeline.exists('foo');
+pipeline.set('foo', 'bar');
+pipeline.exists('foo');
+pipeline.get('foo');
+pipeline.del('foo');
+pipeline.exists('foo');
+test = pipeline.exec();
+test.then(function (results) {
+    // `err` is always null, and `results` is an array of responses
+    // corresponding to the sequence of queued commands.
+    // Each response follows the format `[err, result]`.
+    console.log('pipeline');
+    console.log(results);
+}, function(err) {
+    console.error(err);
+});
+
+

protected/test/testRedis2MySql.js

@@ -0,0 +1,66 @@
+'use strict';
+
+require('../extensions/function_extend.js');
+var co = require('co');
+var R2M = require('../framework/lib/Redis2Mysql.js');
+var objR2m = new R2M('cUser', 'Web', 'Web');
+
+co(function*() {
+    //var where = {userId:13501489295};
+    //var row = objR2m.getRow(where);
+    //console.log(row);
+
+
+    //var where = {enable:1};
+    //// var where = {userId:13501489295};
+    //var data = yield objR2m.getAll(where);
+    //console.log(data);
+
+
+    var userId = 12345;
+    where = {userId:userId};
+    var row = yield objR2m.getRow(where);
+    if (!row) {
+        console.log('addObject');
+        var data = {userId:userId, userName:'MyTest', enable:1, anotherPwd:'another'};
+        var result = yield objR2m.addObject(data);
+    } else {
+        console.log('updateObject');
+        var data = {userName:'edddrwe11223', enable:1, anotherPwd:'another'};
+        var where = {userId:userId};
+        var result = yield objR2m.updateObject(data, where);
+        console.log(result);
+
+        console.log('replaceObject');
+        var args = {userId:userId, userName:'edddrwe11', enable:1, anotherPwd:'another'};
+        var result = yield objR2m.replaceObject(args);
+        console.log(result);
+
+        if (result.changedRows == 0) {
+            console.log('delObject');
+            var result = yield objR2m.delObject(where);
+            console.log(result);
+        }
+
+    }
+
+
+    setTimeout(function() {
+        objR2m.close();
+    }, 1000);
+    console.log('end');
+
+    return 2333;
+
+}).then(console.log, console.error);
+
+
+process.on('exit', function(code) {
+    console.log('About to exit with code:', code);
+});
+
+
+
+
+
+

protected/test/testTableHelper.js

@@ -0,0 +1,66 @@
+/**
+ * Created by benzhan on 15/8/10.
+ */
+require('../extensions/function_extend.js');
+var TableHelper = require('../framework/lib/TableHelper.js');
+var objHelper = new TableHelper('cUser', 'Web');
+
+var allDeferred = objHelper.getAll();
+allDeferred.then(function(rows) {
+    console.log('getAll');
+    console.log(rows);
+}, console.error);
+
+var keyWord = {_field:'userName'};
+var colDeferred = objHelper.getCol({}, keyWord);
+colDeferred.then(function(col) {
+    console.log('getCol');
+    console.log(col);
+}, console.error);
+
+var where = {userId:13501489295};
+var colDeferred = objHelper.getRow(where);
+colDeferred.then(function(row) {
+    console.log('getRow');
+    console.log(row);
+}, console.error);
+
+var data = {userName:'123456', userId:123456, anotherPwd:123};
+var addDeferred = objHelper.addObject(data);
+addDeferred.then(function(result) {
+    console.log('addObject');
+    console.log(result);
+}, console.error);
+
+var where = {userId:123456};
+var newData = {userName:'123456798'};
+var updateDeferred = objHelper.updateObject(newData, where);
+updateDeferred.then(function(result) {
+    console.log('updateObject');
+    console.log(result);
+});
+
+var where = {userId:123456};
+var keyWord = {_field:'userName'};
+var oneDeferred = objHelper.getOne(where, keyWord);
+oneDeferred.then(function(one) {
+    console.log('getOne');
+    console.log(one);
+}, console.error);
+
+var data = {userName:'1234526', userId:123456, anotherPwd:123};
+var replaceDeferred = objHelper.replaceObject(data);
+replaceDeferred.then(function(result) {
+    console.log('replaceObject');
+    console.log(result);
+}, console.error);
+
+var where = {userId:123456};
+var delDeferred = objHelper.delObject(where);
+delDeferred.then(function(result) {
+    console.log('delObject');
+    console.log(result);
+}, console.error);
+
+
+

protected/toutiao.html

@@ -0,0 +1,12 @@
+【2018-08-14 17:37:36】 [crawl] [exec:0.049] SELECT * FROM `rule` WHERE 1 AND `rule_id`='zixun_video:lol:video_url'  LIMIT 1 
+【2018-08-14 17:37:36】 [crawl] [exec:0.049] UPDATE `task` SET `last_crawl_time`='2018-08-14 17:37:36',`next_crawl_time`=1534239486 WHERE 1 AND `task_id`=2359018 
+【2018-08-14 17:37:36】 [crawl] [exec:0.05] SELECT COUNT(*) FROM `crawl_log` WHERE create_time > '2018-08-14 16:37:36' AND `task_id`=2359018 AND `state`IN (2, 3)  LIMIT 1 
+【2018-08-14 17:37:36】 [crawl] [exec:0.049] SELECT * FROM `crawl_log` WHERE 1 AND `task_id`=2359018 AND `state`=0  LIMIT 1 
+【2018-08-14 17:37:36】 [crawl] [exec:0.049] INSERT `crawl_log` SET `task_id`=2359018,`task_key`='http://qt.qq.com/v/hero/pc_index.html?file_uuid=fd4e02b63f644bc9b03857b0b3a6441e&from=manage',`rule_id`='zixun_video:lol:video_url',`url`='http://qt.qq.com/php_cgi/cod_video/php/get_video_url.php?json=1&multirate=1&game_id=2103041&vid=fd4e02b63f644bc9b03857b0b3a6441e',`state`=0,`rule_name`='LOL-视频抓取：视频地址',`create_time`='2018-08-14 17:37:36' 
+【2018-08-14 17:37:36】 [crawl] [exec:0.05] SELECT * FROM `item` WHERE 1 AND `rule_id`='zixun_video:lol:video_url' AND `enable`=1  
+【2018-08-14 17:37:36】 爬虫：http://qt.qq.com/php_cgi/cod_video/php/get_video_url.php?json=1&multirate=1&game_id=2103041&vid=fd4e02b63f644bc9b03857b0b3a6441e 
+【2018-08-14 17:37:36】 请求模式：normal，更新模式：update 
+【2018-08-14 17:37:36】 select proxy:46.250.120.187:8080 
+【2018-08-14 17:37:36】 null 
+【2018-08-14 17:37:37】【error】 500 - {"type":"Buffer","data":[]}
+【2018-08-14 17:37:37】【error】 StatusCodeError: 500 - {"type":"Buffer","data":[]}

protected/views/doc.ejs

@@ -0,0 +1,53 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>文档中心</title>
+<link rel="stylesheet" href="/public/css/bootstrap.min.css">
+<link rel="stylesheet" href="/public/js/treeTable/vsStyle/jquery.treeTable.css">
+<style>
+ .red {color:red;}
+ .result div { display:none; }
+ .result div:nth-child(1) { display:block; }
+</style>
+
+</head>
+<body>
+<div class="container">
+    <table id="class_list" class='table'>
+        <thead>
+            <tr>
+                <th class="th">接口名</th>
+                <th class="th">作者</th>
+                <th class="th">注释</th>
+            </tr>
+        </thead>
+        <tbody>
+            <% for(var i in classInfos) { %>
+            <tr id="<%= i %>">
+                <th><span controller="true" style="font-size:16px;"><%= i %></span></th>
+                <td><%= classInfos[i]['author'] %></td>
+                <td class="td"><%= classInfos[i]['desc'] %></td>
+            </tr>
+            <% for(var o in classInfos[i]['funcs']) { %>
+            <% var func = classInfos[i]['funcs'][o] %>
+            <tr id="<%= i + '_' + o %>" pId="<%= o %>" hasChild="<%= func['param'].length > 0 ? 'true' : '' %>">
+                <th>
+                    <span controller="true" style="font-size:16px;"><%= i + '/' + o%></span>
+                </th>
+                <td class="td"><%= func['author'] %></td>
+                <td class="td"><%= func['desc'] %></td>
+            </tr>
+            <% } %>
+            <% } %>
+        </tbody>
+    </table>
+</div>
+
+<script src="/public/js/jquery-1.10.2.js"></script>
+<script src="/public/js/bootstrap.min.js"></script>
+<script src="/public/js/treeTable/jquery.treeTable.js"></script>
+<script src="/public/js/doc.js"></script>
+
+</body>
+</html>

protected/views/error.ejs

@@ -0,0 +1,3 @@
+<h1><%= message %></h1>
+<h2><%= error.status %></h2>
+<pre><%= error.stack %></pre>

protected/views/index.ejs

@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title><%= title %></title>
+  </head>
+  <body>
+    <h1><%= title %></h1>
+    <p>Welcome to <%= title %></p>
+  </body>
+</html>

protected/views/name_server/js.ejs

@@ -0,0 +1,45 @@
+// 常量
+<% foreach ($configs['string'] as $k => $v):%>
+<% !is_numeric($v) && $v = "'{$v}'"; %>
+global["<%=$k;%>"] = <%=$v;%>;
+<% endforeach;%>
+
+// 数组
+<% 
+$_hash = []; 
+$_subHash = [];
+%>
+<% foreach ($configs['hash'] as $k => $v):%>
+<%
+$k = explode('][', $k);
+$subKey = '';
+$subKey2 = '';
+$m = [];
+
+if (count($k) > 1 && preg_match("/\'(.+)\'/i", $k[1], $m)) {
+    $subKey = $m[1];
+}
+
+if (count($k) > 2 && preg_match("/\'(.+)\'/i", $k[2], $m)) {
+    $subKey2 = $m[1];
+}
+
+$k = $k[0];
+if (!preg_match("/\'(.+)\'/i", $k, $m)) {
+    continue;
+}
+$k = $m[1];
+%> 
+<% if (!$_hash[$k]): %>
+<% $_hash[$k] = 1; %>
+var <%=$k%> = {};
+<% endif;%>
+<% if ($subKey2 && !$_subHash["{$k}_{$subKey}"]): %>
+<% $_subHash["{$k}_{$subKey}"] = 1; %>
+<%=$k%>["<%=$subKey%>"] = {};
+<% endif;%>
+<%=$k%><% if ($subKey): %>["<%=$subKey%>"]<% endif; %><% if ($subKey2): %>["<%=$subKey2%>"]<% endif; %> = <%= JSON.stringify($v, JSON_UNESCAPED_UNICODE); %>;<% endforeach;%>
+
+<% foreach ($_hash as $k => $v): %>
+exports["<%=$k%>"] = <%=$k%>;
+<% endforeach %>