update: 优化任务数据导入导出，重试10次失败后退出，优化命令详细错误信息，输出到日志

README.md

@@ -1,2 +1,12 @@
 # ColdVerify_local
 
+### mysqldump常见报错：
+##### 1、mysqldump: Couldn't execute 'SELECT COLUMN_NAME, JSON_EXTRACT(HISTOGRAM, '$."number-of-buckets-specified"')
+
+可在命令中添加 column-statistics=0 参数。因 MySQL 数据库早期版本 information_schema 数据库中没有名为 COLUMN_STATISTICS 的数据表，新版 mysqldump 默认启用，我们可以通过此命令禁用它。
+
+##### 2、ERROR 1273 (HY000) at line 25: Unknown collation: 'utf8mb4_0900_ai_ci'
+ - 原sql文件是mysql（8.0版本），高级往低级（5.6版本）导入时出现版本不兼容的情况。
+ - 2个数据库排序规则不同 utf8mb4_general_ci utf8mb4_0900_ai_ci
+
+mysqldump 命令参数大全 https://blog.csdn.net/piaoranyuji/article/details/116193100

controllers/TaskData.go

@@ -3,6 +3,8 @@ package controllers
 import (
 	"bzd_server/conf"
 	"bzd_server/lib"
+	"bzd_server/logs"
+	"bzd_server/models/System"
 	"bzd_server/models/Task"
 	"fmt"
 	beego "github.com/beego/beego/v2/server/web"
@@ -348,26 +350,40 @@ func (c *TaskDataController) TaskData_Import_TaskData() {
 		return
 	}
 	sql_file := fmt.Sprintf("%sZ_TaskData_%s.sql", conf.Sql_Temp_Path, T_task_id)
-	for {
-		err := Task.Dump_TaskData(T_task_id, conf.MysqlServer2_Username, conf.MysqlServer2_Password, conf.MysqlServer2_UrlPort, conf.MysqlServer2_Database, sql_file)
-		if err == nil {
-			break
-		}
+	err := Task.Dump_TaskData(T_task_id, conf.MysqlServer2_Username, conf.MysqlServer2_Password, conf.MysqlServer2_UrlPort, conf.MysqlServer2_Database, sql_file)
+	if err != nil {
+		System.Add_Logs("任务数据-打包本地数据", "导出线上数据Z_TaskData_"+T_task_id, err.Error())
+		c.Data["json"] = lib.JSONS{Code: 202, Msg: "Z_TaskData_" + T_task_id + "线上数据导出失败!"}
+		c.ServeJSON()
+		return
 	}
 
-	for {
+	i := 0
+	flag := false
+	Task.CREATE_TaskData(conf.Local_AliasName, T_task_id)
+	for i < 10 {
 		Task.Truncate_TaskData(conf.Local_AliasName, T_task_id)
-
-		err := Task.Insert_TaskData(conf.MysqlServer_Username, conf.MysqlServer_Password, conf.MysqlServer_UrlPort, conf.MysqlServer_Database, sql_file)
-		if err == nil {
+		err = Task.Insert_TaskData(conf.MysqlServer_Username, conf.MysqlServer_Password, conf.MysqlServer_UrlPort, conf.MysqlServer_Database, sql_file)
+		if err != nil {
+			logs.Println("任务数据-打包本地数据", "线下导入数据Z_TaskData_"+T_task_id, err.Error())
+		} else {
 			if Task.Check_TaskData_Num(T_task_id) {
+				flag = true
 				break
 			}
 		}
+		i++
+	}
+	// 重试10次后仍然没有成功导入数据
+	if !flag {
+		System.Add_Logs("任务数据-打包本地数据", "线下导入数据Z_TaskData_"+T_task_id, err.Error())
+		c.Data["json"] = lib.JSONS{Code: 202, Msg: "Z_TaskData_" + T_task_id + "线下导入数据失败!"}
+		c.ServeJSON()
+		return
 	}
 
 	//删除导出的sql文件
-	//_ = os.Remove(sql_file)
+	_ = os.Remove(sql_file)
 
 	c.Data["json"] = lib.JSONS{Code: 200, Msg: "ok!"}
 	c.ServeJSON()
@@ -380,27 +396,40 @@ func (c *TaskDataController) TaskData_Up_TaskData() {
 
 	Task_r, is := Task.Read_Task(T_task_id)
 	if !is {
-		c.Data["json"] = lib.JSONS{Code: 201, Msg: "T_task_id 错误!"}
+		c.Data["json"] = lib.JSONS{Code: 202, Msg: "T_task_id 错误!"}
 		c.ServeJSON()
 		return
 	}
 
 	sql_file := fmt.Sprintf("%sZ_TaskData_%s.sql", conf.Sql_Temp_Path, T_task_id)
-	for {
-		err := Task.Dump_TaskData(T_task_id, conf.MysqlServer_Username, conf.MysqlServer_Password, conf.MysqlServer_UrlPort, conf.MysqlServer_Database, sql_file)
-		if err == nil {
-			break
-		}
+	err := Task.Dump_TaskData(T_task_id, conf.MysqlServer_Username, conf.MysqlServer_Password, conf.MysqlServer_UrlPort, conf.MysqlServer_Database, sql_file)
+	if err != nil {
+		System.Add_Logs("任务数据-更新线上数据", "导出线下数据Z_TaskData_"+T_task_id, err.Error())
+		c.Data["json"] = lib.JSONS{Code: 202, Msg: "Z_TaskData_" + T_task_id + "线下数据导出失败!"}
+		c.ServeJSON()
+		return
 	}
 
-	for {
+	i := 0
+	flag := false
+	for i < 10 {
 		Task.Truncate_TaskData(conf.Server_AliasName, T_task_id)
-		err := Task.Insert_TaskData(conf.MysqlServer2_Username, conf.MysqlServer2_Password, conf.MysqlServer2_UrlPort, conf.MysqlServer2_Database, sql_file)
-		if err == nil {
+		err = Task.Insert_TaskData(conf.MysqlServer2_Username, conf.MysqlServer2_Password, conf.MysqlServer2_UrlPort, conf.MysqlServer2_Database, sql_file)
+		if err != nil {
+			System.Add_Logs("任务数据-更新线上数据", "线上导入数据Z_TaskData_"+T_task_id, err.Error())
+		} else {
 			if Task.Check_TaskData_Num(T_task_id) {
+				flag = true
 				break
 			}
 		}
+		i++
+	}
+	// 重试10次后仍然没有成功导入数据
+	if !flag {
+		c.Data["json"] = lib.JSONS{Code: 202, Msg: "Z_TaskData_" + T_task_id + "线上导入数据失败!"}
+		c.ServeJSON()
+		return
 	}
 
 	//删除导出的sql文件
@@ -408,7 +437,12 @@ func (c *TaskDataController) TaskData_Up_TaskData() {
 
 	// 提交后将当前任务 报告编写 标志为 1
 	Task_r.T_delivery_state = 1
-	Task.Update_Task(Task_r, "T_delivery_state")
+	if !Task.Update_Task(Task_r, "T_delivery_state") {
+		System.Add_Logs("任务数据-更新线上数据", "修改任务报告编写状态"+Task_r.T_name, Task_r.T_task_id)
+		c.Data["json"] = lib.JSONS{Code: 202, Msg: "修改失败!"}
+		c.ServeJSON()
+		return
+	}
 
 	c.Data["json"] = lib.JSONS{Code: 200, Msg: "ok!"}
 	c.ServeJSON()

lib/lib.go

@@ -1,11 +1,13 @@
 package lib
 
 import (
+	"bytes"
+	"bzd_server/logs"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"github.com/signintech/gopdf"
 	"io/ioutil"
-	"log"
 	"math/rand"
 	"os/exec"
 	"runtime"
@@ -411,32 +413,36 @@ func Command(arg ...string) (result string, err error) {
 	}
 	arg = append([]string{c}, arg...)
 	cmd := exec.Command(name, arg...)
-
+	//var out bytes.Buffer
+	//cmd.Stdout = &out
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
 	//创建获取命令输出管道
 	stdout, err := cmd.StdoutPipe()
 	if err != nil {
-		log.Println("Error:can not obtain stdout pipe for command:%s\n", err)
+		logs.Println("Error:can not obtain stdout pipe for command:%s\n", err)
 		return result, err
 	}
 
 	//执行命令
-	if err := cmd.Start(); err != nil {
-		log.Println("Error:The command is err,", err)
+	if err = cmd.Start(); err != nil {
+		logs.Println("Error:The command is err,", err)
 		return result, err
 	}
 
 	//读取所有输出
-	bytes, err := ioutil.ReadAll(stdout)
+	res, err := ioutil.ReadAll(stdout)
 	if err != nil {
-		log.Println("ReadAll Stdout:", err.Error())
+		logs.Println("ReadAll Stdout:", err.Error())
 		return result, err
 	}
 
-	if err := cmd.Wait(); err != nil {
-		log.Println("wait:", err.Error())
-		return result, err
+	if err = cmd.Wait(); err != nil {
+		e := errors.New(fmt.Sprintf("wait:%s:%s", err.Error(), stderr.String()))
+		logs.Println(e.Error())
+		return result, e
 	}
 
-	result = string(bytes)
+	result = string(res)
 	return result, nil
 }

models/Task/TaskData.go

@@ -36,10 +36,10 @@ func init() {
 }
 
 //创建数据库  Device.CREATE_TaskData("")
-func CREATE_TaskData(T_task_id string) bool {
-	o := orm.NewOrm()
+func CREATE_TaskData(alias_name, T_task_id string) bool {
+	o := orm2.NewOrmUsingDB(alias_name)
 
-	sql := "DROP TABLE `Z_TaskData_" + T_task_id + "`"
+	sql := "DROP TABLE IF EXISTS `Z_TaskData_" + T_task_id + "`"
 	o.Raw(sql).Exec()
 
 	sql = "CREATE TABLE IF NOT EXISTS `Z_TaskData_" + T_task_id + "` ( " +
@@ -416,10 +416,11 @@ func Dump_TaskData(T_task_id, root, password, url_port, database, sql_file strin
 	if v >= 8 {
 		org += "--column-statistics=0 "
 	}
-	org = org + fmt.Sprintf("-u%s -p%s -h%s -P%s %s %s> %s",
+	//--no-create-info 只导出数据，而不添加 CREATE TABLE 语句。
+	org = org + fmt.Sprintf("-u%s -p%s -h%s -P%s %s %s> %s --no-create-info",
 		root, password, host_port[0], host_port[1], database, table_name, sql_file)
 
-	println(org)
+	logs.Println(org)
 	_, err := lib.Command(org)
 	return err
 }
@@ -430,6 +431,7 @@ func Insert_TaskData(root, password, url_port, database, sql_file string) error
 	host_port := strings.Split(url_port, ":")
 	org := fmt.Sprintf("mysql -u%s -p%s -h%s -P%s %s < %s",
 		root, password, host_port[0], host_port[1], database, sql_file)
+	logs.Println(org)
 	_, err := lib.Command(org)
 	return err
 }