|
|
@@ -374,7 +374,8 @@ func Add_DeviceData(SN string, v DeviceData_T) bool {
|
|
|
// return true
|
|
|
//}
|
|
|
|
|
|
- key_time := SN + "|" + strconv.Itoa(v.T_id) + "|" + v.T_time.Format("2006-01-02 15:04:05")
|
|
|
+ // 使用分钟级别的key进行去重判断,忽略秒数差异
|
|
|
+ key_time := SN + "|" + strconv.Itoa(v.T_id) + "|" + v.T_time.Format("2006-01-02 15:04")
|
|
|
logs.Debug(key_time)
|
|
|
if redis_DeviceData.IsExist(key_time) {
|
|
|
//println("找到key:",key)
|
|
|
@@ -386,9 +387,10 @@ func Add_DeviceData(SN string, v DeviceData_T) bool {
|
|
|
// 检查 超过时间,查询 数据库
|
|
|
logs.Info("Add_DeviceData 时间差s:", time.Now().Unix()-v.T_time.Unix())
|
|
|
if time.Now().Unix()-v.T_time.Unix() >= 60*40 {
|
|
|
- // 查看是否 有记录
|
|
|
+ // 查看是否 有记录 - 按分钟查询,忽略秒数差异
|
|
|
var maps_z []orm2.ParamsList
|
|
|
- sql_c := "SELECT COUNT(ID) FROM z_device_data_" + SN + " WHERE t_time = '" + v.T_time.Format("2006-01-02 15:04:05") + "' AND t_id = " + strconv.Itoa(v.T_id)
|
|
|
+ // 使用 DATE_FORMAT 截断到分钟进行比较
|
|
|
+ sql_c := "SELECT COUNT(*) FROM z_device_data_" + SN + " WHERE DATE_FORMAT(t_time,'%Y-%m-%d %H:%i') = '" + v.T_time.Format("2006-01-02 15:04") + "' AND t_id = " + strconv.Itoa(v.T_id)
|
|
|
logs.Info("检查 超过时间,查询 数据库 SQL:", sql_c)
|
|
|
_, err := o.Raw(sql_c).ValuesList(&maps_z)
|
|
|
if err != nil {
|
|
|
@@ -403,7 +405,7 @@ func Add_DeviceData(SN string, v DeviceData_T) bool {
|
|
|
|
|
|
// 开始插入数据
|
|
|
sql := "INSERT INTO z_device_data_" + SN + " ( `t_id`, `t_t`, `t_rh`, `t_site`, `t_time`) " +
|
|
|
- "VALUES (" + strconv.Itoa(v.T_id) + ", " + lib.To_string(v.T_t) + ", " + lib.To_string(v.T_rh) + ", '" + v.T_Site + "'," + v.T_time.Format("2006-01-02 15:04:05") + "')"
|
|
|
+ "VALUES (" + strconv.Itoa(v.T_id) + ", " + lib.To_string(v.T_t) + ", " + lib.To_string(v.T_rh) + ", '" + v.T_Site + "', '" + v.T_time.Format("2006-01-02 15:04:05") + "')"
|
|
|
// 这里有时间优化 用于一次 prepare 多次 exec,以提高批量执行的速度
|
|
|
logs.Debug(sql)
|
|
|
res, err := o.Raw(sql).Exec()
|
|
|
@@ -1288,12 +1290,16 @@ func GetNewLocus(T_sn string, T_id int, list *[]DeviceData_) error {
|
|
|
|
|
|
// 去重并追加设备数据
|
|
|
func appendUniqueDeviceData(list []DeviceData_, newData []DeviceData_) []DeviceData_ {
|
|
|
- uniqueMap := make(map[time.Time]bool)
|
|
|
+ // 使用分钟级别的时间戳进行去重,忽略秒数差异
|
|
|
+ uniqueMap := make(map[string]bool)
|
|
|
for _, d := range list {
|
|
|
- uniqueMap[d.T_time] = true
|
|
|
+ // 截断到分钟级别
|
|
|
+ timeKey := d.T_time.Format("2006-01-02 15:04")
|
|
|
+ uniqueMap[timeKey] = true
|
|
|
}
|
|
|
for _, d := range newData {
|
|
|
- if !uniqueMap[d.T_time] {
|
|
|
+ timeKey := d.T_time.Format("2006-01-02 15:04")
|
|
|
+ if !uniqueMap[timeKey] {
|
|
|
if len(d.T_site) > 0 && d.T_site != "0,0" {
|
|
|
split := strings.Split(d.T_site, ",")
|
|
|
defer func() {
|
|
|
@@ -1308,6 +1314,7 @@ func appendUniqueDeviceData(list []DeviceData_, newData []DeviceData_) []DeviceD
|
|
|
mLng, mLat := lib.Wgs84ToGcj02(Lngs, Lats)
|
|
|
d.T_site = fmt.Sprintf("%v,%v", mLng, mLat)
|
|
|
list = append(list, d)
|
|
|
+ uniqueMap[timeKey] = true // 标记该分钟的数据已添加
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
@@ -1317,3 +1324,206 @@ func appendUniqueDeviceData(list []DeviceData_, newData []DeviceData_) []DeviceD
|
|
|
})
|
|
|
return list
|
|
|
}
|
|
|
+
|
|
|
+// 数据缺失检测结果结构体
|
|
|
+type DeviceDataMissingInfo struct {
|
|
|
+ T_sn string // 设备序列号
|
|
|
+ T_id int // 传感器ID
|
|
|
+ Time_start string // 开始时间
|
|
|
+ Time_end string // 结束时间
|
|
|
+ T_saveT int // 数据保存间隔(秒)
|
|
|
+ ExpectedCount int // 期望的数据条数
|
|
|
+ ActualCount int // 实际的数据条数
|
|
|
+ MissingMinutes []string // 缺失的分钟点(忽略秒)
|
|
|
+ MissingPeriods []MissingPeriod // 缺失时间段列表
|
|
|
+}
|
|
|
+
|
|
|
+// 缺失时间段
|
|
|
+type MissingPeriod struct {
|
|
|
+ StartTime string // 缺失开始时间
|
|
|
+ EndTime string // 缺失结束时间
|
|
|
+ Duration int // 缺失时长(秒)
|
|
|
+ MissingCount int // 该时间段缺失的数据条数
|
|
|
+}
|
|
|
+
|
|
|
+// Read_DeviceData_Missing_Check 检测设备数据缺失情况
|
|
|
+func Read_DeviceData_Missing_Check(T_sn string, T_id int, Time_start_ string, Time_end_ string) (DeviceDataMissingInfo, error) {
|
|
|
+ var result DeviceDataMissingInfo
|
|
|
+ result.T_sn = T_sn
|
|
|
+ result.T_id = T_id
|
|
|
+ result.Time_start = Time_start_
|
|
|
+ result.Time_end = Time_end_
|
|
|
+
|
|
|
+ // 1. 获取设备参数,获取 T_saveT
|
|
|
+ deviceParams := Read_DeviceParameter_SN(T_sn)
|
|
|
+ if len(deviceParams) == 0 {
|
|
|
+ return result, errors.New("未找到设备参数")
|
|
|
+ }
|
|
|
+ result.T_saveT = deviceParams[0].T_saveT
|
|
|
+ if result.T_saveT <= 0 {
|
|
|
+ result.T_saveT = 60 // 默认60秒
|
|
|
+ }
|
|
|
+
|
|
|
+ // 2. 解析时间
|
|
|
+ startTime, err := time.Parse("2006-01-02 15:04:05", Time_start_)
|
|
|
+ if err != nil {
|
|
|
+ return result, errors.New("开始时间格式错误")
|
|
|
+ }
|
|
|
+ endTime, err := time.Parse("2006-01-02 15:04:05", Time_end_)
|
|
|
+ if err != nil {
|
|
|
+ return result, errors.New("结束时间格式错误")
|
|
|
+ }
|
|
|
+ if startTime.After(endTime) {
|
|
|
+ return result, errors.New("开始时间不能晚于结束时间")
|
|
|
+ }
|
|
|
+
|
|
|
+ // 3. 计算期望的数据条数
|
|
|
+ duration := int(endTime.Sub(startTime).Seconds())
|
|
|
+ //按区间步长计算并包含起点(例如:0、5、10 分钟应为 3 条)
|
|
|
+ result.ExpectedCount = duration/result.T_saveT + 1
|
|
|
+
|
|
|
+ // 4. 查询实际数据,获取所有时间点
|
|
|
+ o := orm.NewOrm()
|
|
|
+ var timePoints []orm2.ParamsList
|
|
|
+ sql := "SELECT DISTINCT DATE_FORMAT(t_time,'%Y-%m-%d %H:%i:%s') AS t_time FROM z_device_data_" + T_sn +
|
|
|
+ " WHERE t_id = " + strconv.Itoa(T_id) +
|
|
|
+ " AND t_time >= '" + Time_start_ + "'" +
|
|
|
+ " AND t_time <= '" + Time_end_ + "'" +
|
|
|
+ " ORDER BY t_time ASC"
|
|
|
+
|
|
|
+ logs.Debug(sql)
|
|
|
+ _, err = o.Raw(sql).ValuesList(&timePoints)
|
|
|
+ if err != nil {
|
|
|
+ logs.Error(lib.FuncName(), err)
|
|
|
+ return result, errors.New("查询数据失败")
|
|
|
+ }
|
|
|
+
|
|
|
+ result.ActualCount = len(timePoints)
|
|
|
+
|
|
|
+ // 4.1 计算缺失的分钟点(逐分钟对齐,忽略秒)
|
|
|
+ // 构建已存在的分钟集合
|
|
|
+ presentMinutes := make(map[string]struct{})
|
|
|
+ for _, tp := range timePoints {
|
|
|
+ if len(tp) > 0 {
|
|
|
+ if timeStr, ok := tp[0].(string); ok {
|
|
|
+ if t, err := time.Parse("2006-01-02 15:04:05", timeStr); err == nil {
|
|
|
+ minute := time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), 0, 0, t.Location())
|
|
|
+ presentMinutes[minute.Format("2006-01-02 15:04")] = struct{}{}
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ // 构建 presentMinutes 已去重,不再需要对 actualMinuteTimes 进行去重收集
|
|
|
+
|
|
|
+ // 基于保存间隔(分钟)生成应有的时间点,再找缺失
|
|
|
+ startMinute := time.Date(startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(), 0, 0, startTime.Location())
|
|
|
+ endMinute := time.Date(endTime.Year(), endTime.Month(), endTime.Day(), endTime.Hour(), endTime.Minute(), 0, 0, endTime.Location())
|
|
|
+ intervalMin := result.T_saveT / 60
|
|
|
+ if intervalMin <= 0 {
|
|
|
+ intervalMin = 1
|
|
|
+ }
|
|
|
+ // 将起点对齐到间隔边界(例如 5 分钟对齐到 00/05/10/...)
|
|
|
+ firstExpected := startMinute
|
|
|
+ if intervalMin > 1 {
|
|
|
+ rem := firstExpected.Minute() % intervalMin
|
|
|
+ if rem != 0 {
|
|
|
+ firstExpected = firstExpected.Add(time.Duration(intervalMin-rem) * time.Minute)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ var missingMinutes []string
|
|
|
+ for t := firstExpected; !t.After(endMinute); t = t.Add(time.Duration(intervalMin) * time.Minute) {
|
|
|
+ mk := t.Format("2006-01-02 15:04")
|
|
|
+ if _, ok := presentMinutes[mk]; !ok {
|
|
|
+ missingMinutes = append(missingMinutes, mk)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ result.MissingMinutes = missingMinutes
|
|
|
+
|
|
|
+ // 5. 计算缺失时间段
|
|
|
+ result.MissingPeriods = calculateMissingPeriods(startTime, endTime, timePoints, result.T_saveT)
|
|
|
+
|
|
|
+ return result, nil
|
|
|
+}
|
|
|
+
|
|
|
+// calculateMissingPeriods 计算缺失时间段
|
|
|
+func calculateMissingPeriods(startTime, endTime time.Time, timePoints []orm2.ParamsList, saveInterval int) []MissingPeriod {
|
|
|
+ var missingPeriods []MissingPeriod
|
|
|
+
|
|
|
+ if len(timePoints) == 0 {
|
|
|
+ // 完全没有数据
|
|
|
+ duration := int(endTime.Sub(startTime).Seconds())
|
|
|
+ missingPeriods = append(missingPeriods, MissingPeriod{
|
|
|
+ StartTime: startTime.Format("2006-01-02 15:04:05"),
|
|
|
+ EndTime: endTime.Format("2006-01-02 15:04:05"),
|
|
|
+ Duration: duration,
|
|
|
+ MissingCount: duration / saveInterval,
|
|
|
+ })
|
|
|
+ return missingPeriods
|
|
|
+ }
|
|
|
+
|
|
|
+ // 解析实际数据时间点
|
|
|
+ var actualTimes []time.Time
|
|
|
+ for _, tp := range timePoints {
|
|
|
+ if len(tp) > 0 {
|
|
|
+ if timeStr, ok := tp[0].(string); ok {
|
|
|
+ if t, err := time.Parse("2006-01-02 15:04:05", timeStr); err == nil {
|
|
|
+ actualTimes = append(actualTimes, t)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 检查开始时间之前是否有缺失
|
|
|
+ if len(actualTimes) > 0 && actualTimes[0].After(startTime) {
|
|
|
+ duration := int(actualTimes[0].Sub(startTime).Seconds())
|
|
|
+ if duration >= saveInterval {
|
|
|
+ missingPeriods = append(missingPeriods, MissingPeriod{
|
|
|
+ StartTime: startTime.Format("2006-01-02 15:04:05"),
|
|
|
+ EndTime: actualTimes[0].Format("2006-01-02 15:04:05"),
|
|
|
+ Duration: duration,
|
|
|
+ MissingCount: duration / saveInterval,
|
|
|
+ })
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 检查中间缺失的时间段
|
|
|
+ for i := 0; i < len(actualTimes)-1; i++ {
|
|
|
+ currentTime := actualTimes[i]
|
|
|
+ nextTime := actualTimes[i+1]
|
|
|
+
|
|
|
+ // 截断到分钟级别进行比较,忽略秒数差异
|
|
|
+ currentMinute := time.Date(currentTime.Year(), currentTime.Month(), currentTime.Day(),
|
|
|
+ currentTime.Hour(), currentTime.Minute(), 0, 0, currentTime.Location())
|
|
|
+ nextMinute := time.Date(nextTime.Year(), nextTime.Month(), nextTime.Day(),
|
|
|
+ nextTime.Hour(), nextTime.Minute(), 0, 0, nextTime.Location())
|
|
|
+
|
|
|
+ gap := int(nextMinute.Sub(currentMinute).Seconds())
|
|
|
+
|
|
|
+ // 如果时间间隔大于保存间隔的2倍,认为有缺失
|
|
|
+ if gap >= saveInterval*2 {
|
|
|
+ duration := gap - saveInterval // 减去正常间隔
|
|
|
+ missingPeriods = append(missingPeriods, MissingPeriod{
|
|
|
+ StartTime: currentMinute.Add(time.Duration(saveInterval) * time.Second).Format("2006-01-02 15:04:05"),
|
|
|
+ EndTime: nextMinute.Format("2006-01-02 15:04:05"),
|
|
|
+ Duration: duration,
|
|
|
+ MissingCount: duration / saveInterval,
|
|
|
+ })
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 检查结束时间之前是否有缺失
|
|
|
+ if len(actualTimes) > 0 && actualTimes[len(actualTimes)-1].Before(endTime) {
|
|
|
+ lastTime := actualTimes[len(actualTimes)-1]
|
|
|
+ duration := int(endTime.Sub(lastTime).Seconds())
|
|
|
+ if duration >= saveInterval {
|
|
|
+ missingPeriods = append(missingPeriods, MissingPeriod{
|
|
|
+ StartTime: lastTime.Add(time.Duration(saveInterval) * time.Second).Format("2006-01-02 15:04:05"),
|
|
|
+ EndTime: endTime.Format("2006-01-02 15:04:05"),
|
|
|
+ Duration: duration,
|
|
|
+ MissingCount: duration / saveInterval,
|
|
|
+ })
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return missingPeriods
|
|
|
+}
|