ADD：数据去重

Nats/Nats.go

@@ -163,12 +163,20 @@ func NatsInit() {
 		//删除导出的sql文件
 		_ = os.Remove(sql_file)
 
+		// 删除重复数据
+		_, err = Task.DeleteDeduplicate(T_task_id)
+		if err != nil {
+			logs.Error(lib.FuncName(), err)
+			return
+		}
+
 		Task_r.T_collection_state = 1
 		err = NatsServer.Update_Task(Task_r)
 		if err != nil {
 			logs.Error(lib.FuncName(), err)
 			return
 		}
+
 		System.Add_UserLogs_T(T_uuid, "任务数据-打包本地数据", "z_task_data_"+T_task_id, "成功")
 		logs.Info("--------导入数据到本地【成功】！---------")
 

controllers/TaskData.go

@@ -731,13 +731,13 @@ func (c *TaskDataController) TaskData_AddS_Excel() {
 	}
 
 	// 删除重复数据
-	err = Task.DeleteDeduplicate(T_task_id)
+	_, err = Task.DeleteDeduplicate(T_task_id)
 	if err != nil {
 		c.Data["json"] = lib.JSONS{Code: 202, Msg: "导入失败"}
 		c.ServeJSON()
 		return
 	}
-	// 创建唯一缩影
+	// 创建唯一索引
 	err = Task.Create_Unique_Index(T_task_id)
 	if err != nil {
 		c.Data["json"] = lib.JSONS{Code: 202, Msg: "导入失败"}
@@ -2070,6 +2070,38 @@ func (c *TaskDataController) CopyFromPositionSN() {
 	return
 }
 
+// DataRemoveDuplicates 数据去重
+func (c *TaskDataController) DataRemoveDuplicates() {
+	T_uuid, _ := lib.GetAdminT_Uuid(c.Ctx.GetCookie("User_tokey"), c.GetString("User_tokey"))
+
+	T_task_id := c.GetString("T_task_id")
+	Task_r, err := NatsServer.Read_Task(T_task_id)
+	if err != nil {
+		c.Data["json"] = lib.JSONS{Code: 202, Msg: "T_task_id 错误!"}
+		c.ServeJSON()
+		return
+	}
+	if Task_r.T_collection_state == 2 {
+		c.Data["json"] = lib.JSONS{Code: 202, Msg: "数据采集中，请稍后!"}
+		c.ServeJSON()
+		return
+	}
+
+	// 删除重复数据
+	var count int64
+	count, err = Task.DeleteDeduplicate(T_task_id)
+	if err != nil {
+		c.Data["json"] = lib.JSONS{Code: 202, Msg: "导入失败"}
+		c.ServeJSON()
+		return
+	}
+
+	System.Add_UserLogs_T(T_uuid, "数据去重", fmt.Sprintf("数据平滑(%s)[%d]条", T_task_id, count), nil)
+
+	c.Data["json"] = lib.JSONS{200, "操作成功处理" + fmt.Sprint(count) + "条数据", nil}
+	c.ServeJSON()
+}
+
 // 数据偏移到区间内
 func (c *TaskDataController) DataSkewingInterval() {
 	T_uuid, _ := lib.GetAdminT_Uuid(c.Ctx.GetCookie("User_tokey"), c.GetString("User_tokey"))

models/Task/TaskData.go

@@ -485,7 +485,7 @@ func Up_TaskData(T_task_id string, Id string, T_t string, T_rh string, T_time st
 	return true
 }
 
-// 添加
+// 删除
 func Del_TaskData(T_task_id string, Id string) bool {
 
 	o := orm.NewOrm()
@@ -972,28 +972,29 @@ func GroupDataByTID(data []TaskData) map[string][]TaskData {
 }
 
 // 删除的重复数据
-func DeleteDeduplicate(T_task_id string) error {
+func DeleteDeduplicate(T_task_id string) (int64, error) {
 	localOrm := orm2.NewOrmUsingDB(conf.Local_AliasName)
 	tb_name := "z_task_data_" + T_task_id
-
+	var cnt int64
 	// 创建临时表
-	sqlCreate := "CREATE TABLE `tmp_table` AS (SELECT MAX(`ID`) AS `max_id` FROM " + tb_name + " GROUP BY `t_sn`,`t_id`,`t_time`);"
+	sqlCreate := "CREATE TABLE `tmp_table` AS (SELECT MIN(`ID`) AS `min_id` FROM " + tb_name + " GROUP BY `t_sn`,`t_id`,`t_time`);"
 	_, err := localOrm.Raw(sqlCreate).Exec()
 	if err != nil {
 		logs.Error(lib.FuncName(), err)
-		return err
+		return cnt, err
 	}
-	sqlDelete := "DELETE FROM " + tb_name + "  WHERE `ID` NOT IN (SELECT `max_id` FROM `tmp_table`);"
-	_, err = localOrm.Raw(sqlDelete).Exec()
+	sqlDelete := "DELETE FROM " + tb_name + "  WHERE `ID` NOT IN (SELECT `min_id` FROM `tmp_table`);"
+	res, err := localOrm.Raw(sqlDelete).Exec()
 	if err != nil {
 		logs.Error(lib.FuncName(), err)
-		return err
+		return cnt, err
 	}
+	cnt, _ = res.RowsAffected()
 	sqlDrop := "DROP TABLE `tmp_table`;"
 	_, err = localOrm.Raw(sqlDrop).Exec()
 	if err != nil {
 		logs.Error(lib.FuncName(), err)
-		return err
+		return cnt, err
 	}
-	return nil
+	return cnt, nil
 }

routers/TaskData.go

@@ -42,6 +42,7 @@ func init() {
 	beego.Router("/TaskData/CopyFromPositionSN", &controllers.TaskDataController{}, "*:CopyFromPositionSN")               // 增加复制到某测点
 	beego.Router("/TaskData/DataSkewingInterval", &controllers.TaskDataController{}, "*:DataSkewingInterval")             // 数据偏移到给定区间内
 	beego.Router("/TaskData/DataGeometricScale", &controllers.TaskDataController{}, "*:DataGeometricScale")               // 数据等比咧缩放
+	beego.Router("/TaskData/DataRemoveDuplicates", &controllers.TaskDataController{}, "*:DataRemoveDuplicates")           // 数据去重
 
 	beego.Router("/TaskDataCopy/List", &controllers.TaskDataController{}, "*:TaskDataCopy_List")       // 数据存档列表
 	beego.Router("/TaskDataCopy/Add", &controllers.TaskDataController{}, "*:TaskDataCopy_Add")         // 添加数据存档