337 lines
14 KiB
Python
Executable File
337 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import time
|
|
import json
|
|
import logging
|
|
from datetime import datetime, timedelta
|
|
from typing import Dict, List, Any
|
|
import threading
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import signal
|
|
import sys
|
|
from pathlib import Path
|
|
from api_tester import APITester
|
|
from db_config import load_db_config
|
|
import random
|
|
import queue
|
|
|
|
class FastUploader:
|
|
def __init__(self, base_url: str = "http://192.168.1.83:5001"):
|
|
self.base_url = base_url
|
|
self.db_config = load_db_config()
|
|
self.running = False
|
|
|
|
# 高并发设置
|
|
self.max_workers = 20 # 线程池大小
|
|
self.session_pool = self._create_api_testers(self.max_workers)
|
|
|
|
# 统计信息
|
|
self.stats = {
|
|
'total_uploads': 0,
|
|
'successful_uploads': 0,
|
|
'failed_uploads': 0,
|
|
'start_time': None,
|
|
'devices_count': 0,
|
|
'time_points_completed': 0,
|
|
'total_time_points': 0
|
|
}
|
|
|
|
# 线程安全的锁
|
|
self.stats_lock = threading.Lock()
|
|
|
|
# 设置日志
|
|
self.setup_logging()
|
|
|
|
# 信号处理
|
|
signal.signal(signal.SIGINT, self.signal_handler)
|
|
signal.signal(signal.SIGTERM, self.signal_handler)
|
|
|
|
def _create_api_testers(self, count: int) -> queue.Queue:
|
|
"""创建API测试器池"""
|
|
tester_queue = queue.Queue()
|
|
for _ in range(count):
|
|
tester_queue.put(APITester(self.base_url))
|
|
return tester_queue
|
|
|
|
def setup_logging(self):
|
|
log_dir = Path("logs")
|
|
log_dir.mkdir(exist_ok=True)
|
|
|
|
log_file = log_dir / f"fast_upload_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log"
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s - %(levelname)s - %(message)s',
|
|
handlers=[
|
|
logging.FileHandler(log_file, encoding='utf-8'),
|
|
logging.StreamHandler(sys.stdout)
|
|
]
|
|
)
|
|
self.logger = logging.getLogger(__name__)
|
|
self.logger.info(f"日志文件: {log_file}")
|
|
|
|
def signal_handler(self, signum, frame):
|
|
self.logger.info(f"接收到信号 {signum}，停止上传...")
|
|
self.running = False
|
|
sys.exit(0)
|
|
|
|
def generate_data_for_time(self, device_sn: str, target_time: datetime) -> Dict[str, Any]:
|
|
"""为指定时间生成数据"""
|
|
timestamp_str = target_time.strftime("%Y-%m-%d %H:%M:%S")
|
|
|
|
# 健康数据
|
|
health_data = {
|
|
"data": {
|
|
"deviceSn": device_sn,
|
|
"heart_rate": random.randint(60, 120),
|
|
"blood_oxygen": random.randint(95, 100) if random.random() > 0.3 else 0,
|
|
"body_temperature": f"{random.uniform(36.0, 37.5):.1f}",
|
|
"step": random.randint(0, 15000),
|
|
"distance": f"{random.uniform(0, 10):.1f}",
|
|
"calorie": f"{random.uniform(0, 500):.1f}",
|
|
"latitude": f"{random.uniform(22.5, 22.6):.6f}",
|
|
"longitude": f"{random.uniform(114.0, 114.1):.6f}",
|
|
"altitude": f"{random.uniform(0, 100):.1f}",
|
|
"stress": random.randint(0, 100),
|
|
"upload_method": random.choice(["wifi", "4g", "bluetooth"]),
|
|
"blood_pressure_systolic": random.randint(110, 140),
|
|
"blood_pressure_diastolic": random.randint(70, 90),
|
|
"sleepData": "null",
|
|
"exerciseDailyData": "null",
|
|
"exerciseWeekData": "null",
|
|
"scientificSleepData": "null",
|
|
"workoutData": "null",
|
|
"timestamp": timestamp_str
|
|
}
|
|
}
|
|
|
|
# 设备信息
|
|
device_info = {
|
|
"System Software Version": f"GLL-AL30BCN {random.randint(3,5)}.0.0.{random.randint(800,999)}",
|
|
"Wifi Address": ":".join([f"{random.randint(0,255):02x}" for _ in range(6)]),
|
|
"Bluetooth Address": ":".join([f"{random.randint(0,255):02X}" for _ in range(6)]),
|
|
"IP Address": f"192.168.1.{random.randint(100, 254)}",
|
|
"Network Access Mode": random.choice([1, 2, 3]),
|
|
"SerialNumber": device_sn,
|
|
"Device Name": f"HUAWEI WATCH B7-{random.randint(500,600)}-BF{random.randint(0,9)}",
|
|
"IMEI": f"86615206{random.randint(10000000, 99999999)}",
|
|
"batteryLevel": random.randint(10, 100),
|
|
"voltage": random.randint(3500, 4500),
|
|
"chargingStatus": random.choice(["NONE", "CHARGING", "FULL"]),
|
|
"status": random.choice(["ACTIVE", "INACTIVE", "SLEEP"]),
|
|
"timestamp": timestamp_str,
|
|
"wearState": random.choice([0, 1])
|
|
}
|
|
|
|
# 通用事件
|
|
common_event = {
|
|
'eventType': 'com.tdtech.ohos.action.WEAR_STATUS_CHANGED',
|
|
'eventValue': str(random.choice([0, 1])),
|
|
'deviceSn': device_sn,
|
|
'latitude': round(random.uniform(22.5, 22.6), 6),
|
|
'longitude': round(random.uniform(114.0, 114.1), 6),
|
|
'altitude': random.randint(0, 100),
|
|
'timestamp': timestamp_str,
|
|
'healthData': health_data
|
|
}
|
|
|
|
return {
|
|
'upload_health_data': health_data,
|
|
'upload_device_info': device_info,
|
|
'upload_common_event': common_event
|
|
}
|
|
|
|
def upload_device_data(self, device_sn: str, target_time: datetime) -> List[Dict[str, Any]]:
|
|
"""上传单个设备在指定时间的所有数据"""
|
|
# 获取API测试器
|
|
api_tester = self.session_pool.get()
|
|
|
|
try:
|
|
data_set = self.generate_data_for_time(device_sn, target_time)
|
|
results = []
|
|
|
|
for endpoint, data in data_set.items():
|
|
try:
|
|
result = api_tester.make_request(endpoint, data, timeout=30)
|
|
result['device_sn'] = device_sn
|
|
result['endpoint'] = endpoint
|
|
results.append(result)
|
|
|
|
# 更新统计
|
|
with self.stats_lock:
|
|
self.stats['total_uploads'] += 1
|
|
if result.get('success', False):
|
|
self.stats['successful_uploads'] += 1
|
|
else:
|
|
self.stats['failed_uploads'] += 1
|
|
|
|
except Exception as e:
|
|
error_result = {
|
|
'device_sn': device_sn,
|
|
'endpoint': endpoint,
|
|
'success': False,
|
|
'error': str(e),
|
|
'timestamp': datetime.now().isoformat()
|
|
}
|
|
results.append(error_result)
|
|
|
|
with self.stats_lock:
|
|
self.stats['total_uploads'] += 1
|
|
self.stats['failed_uploads'] += 1
|
|
|
|
return results
|
|
|
|
finally:
|
|
# 归还API测试器到池中
|
|
self.session_pool.put(api_tester)
|
|
|
|
def upload_historical_data_fast(self, days: int = 30):
|
|
"""高速上传历史数据"""
|
|
if not self.db_config.connect():
|
|
self.logger.error("数据库连接失败")
|
|
return
|
|
|
|
# 获取设备数据
|
|
user_devices = self.db_config.get_user_devices(100)
|
|
if not user_devices:
|
|
devices = self.db_config.get_devices(100)
|
|
device_list = [d['device_sn'] for d in devices if d.get('device_sn')]
|
|
else:
|
|
device_list = [ud['device_sn'] for ud in user_devices]
|
|
|
|
if not device_list:
|
|
self.logger.error("未找到设备数据")
|
|
return
|
|
|
|
self.stats['devices_count'] = len(device_list)
|
|
|
|
# 计算时间范围 - 每分钟一次
|
|
end_time = datetime.now()
|
|
start_time = end_time - timedelta(days=days)
|
|
|
|
# 生成时间点（每分钟）
|
|
time_points = []
|
|
current_time = start_time
|
|
while current_time <= end_time:
|
|
time_points.append(current_time)
|
|
current_time += timedelta(minutes=1)
|
|
|
|
self.stats['total_time_points'] = len(time_points)
|
|
total_operations = len(device_list) * len(time_points) * 3
|
|
|
|
self.logger.info("🚀 高速数据上传开始")
|
|
self.logger.info(f"设备数量: {len(device_list)}")
|
|
self.logger.info(f"时间范围: {start_time.strftime('%Y-%m-%d %H:%M')} 到 {end_time.strftime('%Y-%m-%d %H:%M')}")
|
|
self.logger.info(f"时间点数量: {len(time_points)} (每分钟)")
|
|
self.logger.info(f"总操作数: {total_operations}")
|
|
self.logger.info(f"线程池大小: {self.max_workers}")
|
|
self.logger.info(f"预计耗时: {total_operations/100/60:.1f} 分钟 (假设100次/分钟)")
|
|
|
|
self.running = True
|
|
self.stats['start_time'] = datetime.now()
|
|
|
|
# 使用线程池并行处理
|
|
with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
|
|
# 创建所有任务
|
|
futures = []
|
|
for time_idx, time_point in enumerate(time_points):
|
|
if not self.running:
|
|
break
|
|
|
|
for device_sn in device_list:
|
|
if not self.running:
|
|
break
|
|
|
|
future = executor.submit(self.upload_device_data, device_sn, time_point)
|
|
futures.append((future, time_point, device_sn, time_idx))
|
|
|
|
self.logger.info(f"已创建 {len(futures)} 个上传任务")
|
|
|
|
# 等待所有任务完成并显示进度
|
|
completed_count = 0
|
|
for future, time_point, device_sn, time_idx in futures:
|
|
if not self.running:
|
|
break
|
|
|
|
try:
|
|
results = future.result(timeout=60)
|
|
completed_count += 1
|
|
|
|
# 每完成10个任务显示进度
|
|
if completed_count % 10 == 0 or completed_count == len(futures):
|
|
elapsed = datetime.now() - self.stats['start_time']
|
|
with self.stats_lock:
|
|
success_rate = (self.stats['successful_uploads'] / self.stats['total_uploads'] * 100) if self.stats['total_uploads'] > 0 else 0
|
|
upload_speed = self.stats['total_uploads'] / elapsed.total_seconds() if elapsed.total_seconds() > 0 else 0
|
|
|
|
progress = completed_count / len(futures) * 100
|
|
time_str = time_point.strftime('%H:%M')
|
|
self.logger.info(f"📊 进度: {progress:.1f}% | {completed_count}/{len(futures)} | "
|
|
f"时间: {time_str} | 设备: {device_sn[:15]} | "
|
|
f"成功率: {success_rate:.1f}% | 速度: {upload_speed:.1f} 次/秒")
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"任务处理错误 - 设备: {device_sn}, 时间: {time_point}: {e}")
|
|
completed_count += 1
|
|
|
|
self.db_config.disconnect()
|
|
self.print_final_stats()
|
|
|
|
def print_final_stats(self):
|
|
"""打印最终统计"""
|
|
self.logger.info("=" * 80)
|
|
self.logger.info("📊 高速上传完成统计")
|
|
self.logger.info("=" * 80)
|
|
|
|
if self.stats['start_time']:
|
|
duration = datetime.now() - self.stats['start_time']
|
|
self.logger.info(f"总耗时: {duration}")
|
|
|
|
# 计算速度
|
|
if self.stats['total_uploads'] > 0 and duration.total_seconds() > 0:
|
|
upload_speed = self.stats['total_uploads'] / duration.total_seconds()
|
|
self.logger.info(f"平均上传速度: {upload_speed:.2f} 次/秒")
|
|
self.logger.info(f"峰值处理能力: {upload_speed * 60:.0f} 次/分钟")
|
|
|
|
self.logger.info(f"设备数量: {self.stats['devices_count']}")
|
|
self.logger.info(f"总操作数: {self.stats['total_uploads']}")
|
|
self.logger.info(f"成功次数: {self.stats['successful_uploads']}")
|
|
self.logger.info(f"失败次数: {self.stats['failed_uploads']}")
|
|
|
|
if self.stats['total_uploads'] > 0:
|
|
success_rate = (self.stats['successful_uploads'] / self.stats['total_uploads']) * 100
|
|
self.logger.info(f"成功率: {success_rate:.2f}%")
|
|
|
|
def main():
|
|
print("🚀 高速数据上传工具 - 模拟客户手表每分钟上传")
|
|
print("=" * 60)
|
|
|
|
days = 30
|
|
if len(sys.argv) > 1:
|
|
try:
|
|
days = float(sys.argv[1])
|
|
except ValueError:
|
|
print("❌ 无效的天数参数")
|
|
return
|
|
|
|
print(f"📅 准备上传过去 {days} 天的数据")
|
|
print("⚡ 使用高并发模式，最大化上传速度")
|
|
print("💡 按 Ctrl+C 可以安全停止")
|
|
print()
|
|
|
|
confirm = input("确认开始上传? (y/N): ").strip().lower()
|
|
if confirm != 'y':
|
|
print("已取消")
|
|
return
|
|
|
|
uploader = FastUploader()
|
|
|
|
try:
|
|
uploader.upload_historical_data_fast(days)
|
|
except KeyboardInterrupt:
|
|
print("\n\n⏸️ 用户中断操作")
|
|
uploader.running = False
|
|
except Exception as e:
|
|
print(f"\n❌ 程序错误: {e}")
|
|
|
|
if __name__ == "__main__":
|
|
main() |