# 优化定时任务

apps/mosquito/migrations/0015_devicepoststatistic_increment.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2020-06-03 07:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('mosquito', '0014_auto_20200601_1646'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='devicepoststatistic',
+            name='increment',
+            field=models.IntegerField(blank=True, null=True, verbose_name='增量'),
+        ),
+    ]

apps/mosquito/models.py

@@ -61,6 +61,7 @@ class DevicePostStatistic(models.Model):
     device_id = models.CharField(max_length=100, verbose_name='设备ID')
     org = models.ForeignKey('Org', verbose_name='所在组织', on_delete='PROTECT')
     total = models.PositiveIntegerField(verbose_name='总数')
+    increment = models.IntegerField(blank=True, null=True, verbose_name='增量')
     date = models.DateField(auto_now_add=False, verbose_name='日期', unique=False)
     create_time = models.DateTimeField(auto_now_add=True, blank=True, null=True, verbose_name='创建时间')
     update_time = models.DateTimeField(auto_now=True, blank=True, null=True, verbose_name='更新时间')

apps/mosquito/tasks.py

@@ -37,19 +37,20 @@ def fill_date(ret=None, tz=None):
 def get_daily_statistic(enable_container=False):
     device_container = {}
     daily_ret = {}
-    queryset = DeviceCount.objects.raw('select id, date(data_time) as date from device_count group by date')
+    qs = DeviceCount.objects.raw('select id, date(data_time) as date from device_count group by date')
-    if queryset:
+    if qs:
         tz = pytz.timezone("UTC")
         device_list = {item.device_id for item in DeviceCount.objects.distinct()}
         for item in device_list:
             device_container[item] = set()
-        for q in queryset:
+        for q in qs:
             # 按照 device_id 分组查询最新时间 max_time，然后通过id + max_time 获取实例，用来得到count
             midnight = tz.localize(datetime.combine(q.date, time(23, 59)), is_dst=None)
-            daily_queryset = [DeviceCount.objects.filter(device_id=x['device_id'], data_time=x['max_time'])
+            dc = DeviceCount.objects.filter(data_time__lte=midnight).values('device_id').\
-                              for x in DeviceCount.objects.filter(data_time__lte=midnight).values(
+                annotate(max_time=Max('data_time'))
-                    'device_id').annotate(max_time=Max('data_time'))]
+            daily_queryset = [DeviceCount.objects.filter(device_id=x['device_id'], data_time=x['max_time']) for x in dc]
             if daily_queryset:
+                # 有些相同时间的记录，选取最大值的记录
                 daily_queryset = [reduce(max_count, entry) if len(entry) > 1 else entry[0] for entry in daily_queryset]
                 if enable_container:
                     for device_id in device_list:
@@ -64,6 +65,10 @@ def get_daily_statistic(enable_container=False):
 
 @task()
 def update_daily_statistic():
+    """
+    daily_ret: {datetime.date(2018, 8, 10): 124, datetime.date(2018, 8, 11): 137, ... }
+    device_container: {'869300034538870': {(datetime.date(2019, 5, 25), '7178'), (datetime.date(2019, 7, 11), '9293') ... }
+    """
     full_ret = None
     # 计算每天聚合值
     daily_ret, device_container = get_daily_statistic(enable_container=True)
@@ -81,17 +86,24 @@ def update_daily_statistic():
 
             _date_list = [k for k in full_daily_ret]
             _date_list.sort()
+            is_first = True
             for d in _date_list:
+                if is_first:
+                    increment = full_daily_ret[d]
+                else:
+                    increment = int(full_daily_ret[d]) - int(full_daily_ret[d - timedelta(days=1)])
+
                 try:
                     obj = DevicePostStatistic.objects.get(device_id=device_id, date=d)
                 except DevicePostStatistic.DoesNotExist:
-                    obj = None
+                    DevicePostStatistic.objects.create(
-                if obj:
+                        device_id=device_id, total=full_daily_ret[d], increment=increment, org=org, date=d)
+                else:
                     obj.total = full_daily_ret[d]
+                    obj.increment = increment
                     obj.org = org
                     obj.save()
-                else:
+                is_first = False
-                    DevicePostStatistic.objects.create(device_id=device_id, total=full_daily_ret[d], org=org, date=d)
 
     # 写入数据库
     if full_ret: