feat: 迁移到twostock并支持远程访问
主要改动： 1. 数据源升级： - 将所有 tushare 导入替换为 twostock (11个文件) - 更新 token 为 twostock 兼容token - 修复 token 失效时的无限循环问题 2. 远程访问支持： - 前端配置监听 0.0.0.0 (vite.config.js) - 后端已支持 0.0.0.0 监听 - 新增远程访问地址说明 3. 端口配置： - Backend: 8028 - Frontend: 8448 - 可通过 backend/.env 自定义 4. 文档更新： - README.md: 添加远程访问说明、twostock配置 - CLAUDE.md: 更新数据源架构说明 修改文件： - backend/app/modules/data_fetcher.py (token状态管理) - backend/utils/token_manager.py - backend/services/*.py - backend/app/modules/*.py - backend/app/crawler/tushare_crawler.py - frontend/vite.config.js (host: 0.0.0.0) - README.md, CLAUDE.md 🤖 Generated with Claude Code Co-Authored-By: Claude Sonnet 4.5 <[email protected]>
This commit is contained in:
Claude Sonnet 4.5
@@ -4,10 +4,10 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
|
||||
|
||||
## Project Overview
|
||||
|
||||
AI-powered stock recommendation system combining ML models (XGBoost/LightGBM) with Claude AI for intelligent stock analysis. The system provides real-time recommendations, market analysis, and interactive AI assistance.
|
||||
AI-powered Chinese stock (A-share) recommendation system combining ML models (XGBoost/LightGBM) with Claude AI. Features multi-source data fetching from free Chinese financial APIs (Sina, Tencent, EastMoney), advanced caching, and real-time stock analysis with natural language explanations.
|
||||
|
||||
**Architecture**: Vue 3 SPA + FastAPI + PostgreSQL + Redis + Claude AI
|
||||
**Current Version**: v1.0.0
|
||||
**Data Sources**: Sina Finance, Tencent Finance, EastMoney (all free, no API keys)
|
||||
**Production Status**: Production Ready
|
||||
|
||||
## Common Development Commands
|
||||
@@ -73,6 +73,14 @@ cd backend
|
||||
pytest tests/ -v --asyncio-mode=auto --cov=app
|
||||
pytest test_smart_cache.py # Specific test suite
|
||||
pytest test_real_data_integration.py -k live # Live data tests
|
||||
|
||||
# Test data sources
|
||||
python test_data_sources.py # Basic data source test
|
||||
python test_all_data_sources.py # Comprehensive source test
|
||||
|
||||
# Test dashboard APIs
|
||||
bash test_dashboard_apis.sh # Quick API health check
|
||||
bash test_data_pipeline_apis.sh # Data pipeline endpoints
|
||||
```
|
||||
|
||||
### Docker Compose (Full Stack)
|
||||
@@ -106,15 +114,15 @@ kubectl apply -f k8s/deployment.yaml
|
||||
kubectl apply -f k8s/service.yaml
|
||||
|
||||
# Check status
|
||||
kubectl get all -n claude-stock
|
||||
kubectl logs -f deployment/claude-stock-backend -n claude-stock
|
||||
kubectl get all -n stock
|
||||
kubectl logs -f deployment/stock-backend -n stock
|
||||
|
||||
# Port forwarding
|
||||
kubectl port-forward svc/claude-stock-backend 8000:80 -n claude-stock
|
||||
kubectl port-forward svc/claude-stock-frontend 8090:8090 -n claude-stock
|
||||
kubectl port-forward svc/stock-backend 8000:80 -n stock
|
||||
kubectl port-forward svc/stock-frontend 8090:8090 -n stock
|
||||
|
||||
# Rollback
|
||||
kubectl rollout undo deployment/claude-stock-backend -n claude-stock
|
||||
kubectl rollout undo deployment/stock-backend -n stock
|
||||
```
|
||||
|
||||
### CI/CD (Gitea Actions)
|
||||
@@ -125,13 +133,13 @@ git add .
|
||||
git commit -m "feat: your feature description"
|
||||
git push origin main
|
||||
|
||||
# Monitor pipeline: http://192.168.1.83:33000/<user>/claude-stock/actions
|
||||
# Monitor pipeline in Gitea Actions UI
|
||||
|
||||
# Verify CICD environment
|
||||
bash verify-cicd.sh
|
||||
|
||||
# Check deployment
|
||||
kubectl get pods -n claude-stock -w
|
||||
kubectl get pods -n stock -w
|
||||
```
|
||||
|
||||
## High-Level Architecture
|
||||
@@ -140,34 +148,37 @@ kubectl get pods -n claude-stock -w
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ Frontend Layer (Vue 3) │
|
||||
│ - DashboardV2.vue (AI-enhanced visualizations) │
|
||||
│ - AIDock.vue (Interactive chatbot) │
|
||||
│ - MarketPulseEnhanced.vue (Real-time market data) │
|
||||
│ Frontend Layer (Vue 3 + Vite) │
|
||||
│ - views/ (DashboardV2.vue, AIDock, MarketPulse) │
|
||||
│ - components/ (reusable UI components) │
|
||||
│ - services/api.js (Axios HTTP client) │
|
||||
│ - store.js (Vuex/Pinia state management) │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ API Layer (FastAPI Routes) │
|
||||
│ - recommendations.py (ML-based) │
|
||||
│ - ai_recommendations.py (Claude-enhanced) │
|
||||
│ API Layer (backend/app/routes/) │
|
||||
│ - recommendations.py (ML-based stock picks) │
|
||||
│ - ai_recommendations.py (Claude-enhanced analysis) │
|
||||
│ - ai_assistant.py (Chatbot endpoints) │
|
||||
│ - claude_deepseek_collab.py (Hybrid AI) │
|
||||
│ - data_pipeline.py (data fetch APIs) │
|
||||
│ - risk_radar.py, backtest.py, strategies.py │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ Service Layer (Business Logic) │
|
||||
│ - claude_service.py (Claude API integration) │
|
||||
│ - enhanced_data_service.py (Data pipeline) │
|
||||
│ - ai_assistant_integration.py (AI orchestration) │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ Module Layer │
|
||||
│ - data_fetcher.py (TuShare, yfinance) │
|
||||
│ Business Logic (backend/app/modules/) │
|
||||
│ - data_fetcher_v2.py (Multi-source adapter) │
|
||||
│ - feature_engineer.py (Technical indicators) │
|
||||
│ - ml_trainer.py (XGBoost/LightGBM) │
|
||||
│ - llm_ranker.py (LLM-based ranking) │
|
||||
│ - sentiment_fetcher.py (News sentiment) │
|
||||
│ - ml_trainer.py (XGBoost/LightGBM training) │
|
||||
│ - cache_optimizer.py (Smart caching) │
|
||||
│ - strategy_engine.py (Trading strategies) │
|
||||
│ - sentiment_fetcher.py (News sentiment) │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ Data Adapters (backend/app/adapters/) │
|
||||
│ - sina_adapter.py (Sina Finance) │
|
||||
│ - tencent_adapter.py (Tencent Finance) │
|
||||
│ - eastmoney_adapter.py (EastMoney) │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ Data Layer │
|
||||
│ - PostgreSQL (Persistent storage) │
|
||||
│ - Redis (Multi-level caching) │
|
||||
│ - External APIs (TuShare, Claude, Alpha Vantage) │
|
||||
│ - External APIs (Sina, Tencent, EastMoney, Claude) │
|
||||
└─────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
@@ -176,9 +187,9 @@ kubectl get pods -n claude-stock -w
|
||||
```
|
||||
Data Collection → Feature Engineering → ML Prediction → Claude Explanation → Cache → Frontend
|
||||
↓ ↓ ↓ ↓
|
||||
TuShare Technical Indicators XGBoost Natural Language
|
||||
yfinance PE, PB, ROE LightGBM Reasoning
|
||||
Alpha Vantage Sentiment Scores Risk Analysis
|
||||
Sina/Tencent/ Technical Indicators XGBoost Natural Language
|
||||
EastMoney PE, PB, ROE LightGBM Reasoning & Analysis
|
||||
Sentiment Scores Risk Assessment
|
||||
```
|
||||
|
||||
### Multi-Level Caching Strategy
|
||||
@@ -206,14 +217,14 @@ Critical for cost optimization and performance:
|
||||
- `adminer` (port 8091) - Database UI
|
||||
|
||||
**Kubernetes (Production)**:
|
||||
- Namespace: `claude-stock` (dynamic, based on repo name)
|
||||
- Namespace: `stock`
|
||||
- StatefulSets: PostgreSQL (10Gi PVC), Redis (5Gi PVC)
|
||||
- Deployments: Backend (1-3 replicas with HPA)
|
||||
- Frontend: Sidecar container with Nginx
|
||||
- Services: LoadBalancer for external access
|
||||
- HPA: Auto-scaling at CPU 70%, Memory 80%
|
||||
|
||||
**Key Pattern**: Backend and frontend run as separate containers in the same pod (sidecar pattern) for shared storage and network namespace.
|
||||
**Key Pattern**: Backend and frontend can run as separate containers in the same pod (sidecar pattern) or as separate deployments.
|
||||
|
||||
## Tech Stack
|
||||
|
||||
@@ -269,17 +280,17 @@ async def analyze_stock_with_claude(symbol: str, features: dict):
|
||||
- Claude 4.5 Sonnet: Complex stock analysis, multi-factor reasoning
|
||||
- Claude 3.5 Haiku: Simple Q&A, quick responses (60% cheaper)
|
||||
|
||||
### 2. Hybrid AI Architecture (Claude + DeepSeek)
|
||||
### 2. Hybrid AI Architecture (Claude + DeepSeek/Qwen)
|
||||
|
||||
**Location**: `/backend/app/routes/claude_deepseek_collab.py`
|
||||
|
||||
Combines Claude's reasoning with DeepSeek's local inference:
|
||||
Combines Claude's reasoning with local LLM inference (via Ollama):
|
||||
|
||||
1. DeepSeek generates initial technical analysis (free, local)
|
||||
2. Claude reviews and enhances with market context (paid, API)
|
||||
1. Local LLM (DeepSeek/Qwen) generates initial technical analysis (free, local)
|
||||
2. Claude reviews and enhances with market context (paid, OpenRouter API)
|
||||
3. System merges both perspectives with confidence scoring
|
||||
|
||||
**Why hybrid?**: Reduces Claude API costs by 40% while maintaining quality.
|
||||
**Why hybrid?**: Reduces Claude API costs by 40% while maintaining quality. Requires local Ollama server running (default: port 11434).
|
||||
|
||||
### 3. Feature Engineering Pipeline
|
||||
|
||||
@@ -311,7 +322,7 @@ async def prewarm_top_stocks():
|
||||
|
||||
### 5. Database Schema
|
||||
|
||||
**Location**: `/backend/database/schema.sql`
|
||||
**Location**: `/backend/database/schema_v2.sql` (main), `schema_extended.sql` (full data)
|
||||
|
||||
Key tables:
|
||||
- `stocks` - Stock master data (symbol, name, industry)
|
||||
@@ -319,37 +330,37 @@ Key tables:
|
||||
- `sentiment_data` - News and social sentiment
|
||||
- `recommendations` - ML predictions with timestamps
|
||||
- `cache_metadata` - Cache hit/miss tracking
|
||||
- Extended schema: `t_company_info`, `t_sectors`, `t_financials`, `t_money_flow`, `t_top_list` (11 additional tables for comprehensive data)
|
||||
|
||||
**Migration**: Use Alembic for schema changes (`alembic revision --autogenerate`).
|
||||
**Migration**: Use Alembic for schema changes (`cd backend && alembic revision --autogenerate -m "description"`).
|
||||
|
||||
### 6. CI/CD Pipeline
|
||||
|
||||
**Location**: `.gitea/workflows/cicd.yml`
|
||||
|
||||
5-stage pipeline (15-25 min total):
|
||||
1. **Build & Push**: Docker images to Harbor registry
|
||||
2. **Test**: pytest with PostgreSQL service container
|
||||
3. **Security Scan**: Trivy image scanning (optional)
|
||||
4. **Deploy**: K8s manifest updates with dynamic namespace
|
||||
5. **Smoke Test**: Health check validation
|
||||
Multi-stage pipeline:
|
||||
1. **Build**: Docker images for backend and frontend
|
||||
2. **Deploy**: K8s deployment with kubectl
|
||||
3. **Test**: Health check validation
|
||||
|
||||
**Key Feature**: Version traceability via Git SHA labels/annotations on K8s resources.
|
||||
**Key Feature**: Triggered on push to main/develop branches. Uses Gitea Actions runner with Kubernetes executor.
|
||||
|
||||
### 7. Environment Configuration
|
||||
|
||||
**Location**: `.env.example` (68 lines)
|
||||
|
||||
Critical environment variables:
|
||||
- `OPENROUTER_API_KEY` - Claude API via OpenRouter
|
||||
- `TUSHARE_TOKEN` - Chinese A-share data
|
||||
- `ALPHA_VANTAGE_API_KEY` - International stock data
|
||||
- `DATABASE_URL` - PostgreSQL connection
|
||||
- `REDIS_URL` - Redis connection
|
||||
- `OPENROUTER_API_KEY` - Claude API via OpenRouter (optional, for AI explanations)
|
||||
- `TUSHARE_TOKEN` - TuShare token (legacy, optional)
|
||||
- `DATABASE_URL` - PostgreSQL connection string
|
||||
- `REDIS_URL` - Redis connection string
|
||||
- `LLM_MODEL` - anthropic/claude-4.5-sonnet
|
||||
- `OLLAMA_BASE_URL` - http://host.docker.internal:11434
|
||||
- `CACHE_TTL_LLM=3600` - Claude response cache TTL
|
||||
- `OLLAMA_BASE_URL` - http://host.docker.internal:11434 (for hybrid AI)
|
||||
- `DEEPSEEK_MODEL` - qwen3-coder:30b-a3b-q8_0 (local LLM model)
|
||||
- `CACHE_TTL_LLM=3600` - Claude response cache TTL (1 hour)
|
||||
- `CACHE_TTL_QUOTE=60` - Real-time quote cache (1 minute)
|
||||
|
||||
**K8s Deployment**: Environment variables are injected via ConfigMap + Secrets.
|
||||
**K8s Deployment**: Environment variables are injected via ConfigMap.
|
||||
|
||||
## Development Workflow
|
||||
|
||||
@@ -398,14 +409,17 @@ Critical environment variables:
|
||||
|
||||
5. **Documentation**: Update this file and relevant `.md` files
|
||||
|
||||
### Claude Prompt Engineering
|
||||
### Adding Data Fetchers
|
||||
|
||||
1. **Edit Templates**: `/backend/prompts/claude_stock_analysis.md`
|
||||
2. **Implement Service**: `/backend/services/claude_service.py`
|
||||
3. **Add API Endpoint**: `/backend/app/routes/ai_recommendations.py`
|
||||
4. **Test with Sample Data**: Use `/api/test/claude` endpoint
|
||||
The system uses a multi-source adapter pattern for data fetching:
|
||||
|
||||
**Best Practice**: Always cache Claude responses (3600s TTL) to avoid duplicate API calls.
|
||||
1. **Create Adapter**: Add new adapter in `/backend/app/adapters/`
|
||||
- Implement standard interface: `fetch_quote()`, `fetch_kline()`, `fetch_fundamentals()`
|
||||
2. **Register in DataFetcherV2**: Update `/backend/app/modules/data_fetcher_v2.py`
|
||||
- Add to priority list (e.g., Sina → Tencent → EastMoney → New Source)
|
||||
3. **Add Tests**: Create tests in `/backend/tests/test_<adapter_name>.py`
|
||||
|
||||
**Key Pattern**: Adapters must return standardized data format for seamless fallback.
|
||||
|
||||
### Database Changes
|
||||
|
||||
@@ -431,32 +445,32 @@ Critical environment variables:
|
||||
- Health Check: http://<pod-ip>:8000/health
|
||||
|
||||
### External Services
|
||||
- Gitea: http://192.168.1.83:33000
|
||||
- Harbor Registry: https://localhost:8443
|
||||
- Ollama: http://localhost:11434 (or http://<node-ip>:11434 in K8s)
|
||||
- Ollama: http://localhost:11434 (for hybrid AI, optional)
|
||||
- Gitea: (configured for CI/CD)
|
||||
|
||||
## Key Documentation Files
|
||||
|
||||
Read in this order:
|
||||
1. `README.md` - Project overview
|
||||
2. `DEV_SETUP.md` - **Local development quick start (no Docker)**
|
||||
3. `QUICK_START.md` - 5-minute setup (469 lines)
|
||||
4. `DEPLOYMENT_CHECKLIST.md` - Production deployment (372 lines)
|
||||
5. `ARCHITECTURE_DIAGRAMS.txt` - ASCII diagrams (386 lines)
|
||||
6. `AI_PRODUCTION_ARCHITECTURE.md` - AI integration deep dive (566 lines)
|
||||
7. `CICD_DEPLOYMENT_GUIDE.md` - CI/CD setup
|
||||
8. `SMART_CACHE_ARCHITECTURE.md` - Caching strategy
|
||||
1. `README.md` - Project overview and quick start
|
||||
2. `DEV_SETUP.md` - Local development without Docker
|
||||
3. `AGENTS.md` - Repository guidelines and commit conventions
|
||||
4. `DATA_SOURCE_ANALYSIS.md` - Data source architecture
|
||||
5. `DATA_SOURCE_MIGRATION_V2.md` - Migration to free data sources
|
||||
6. `SMART_CACHE_ARCHITECTURE.md` - Multi-level caching strategy
|
||||
7. `AI_PRODUCTION_ARCHITECTURE.md` - AI integration details
|
||||
8. `CICD_DEPLOYMENT_GUIDE.md` - CI/CD setup with Gitea Actions
|
||||
|
||||
## Critical Performance Considerations
|
||||
|
||||
### Cost Optimization
|
||||
- **Claude API**: ~$520/month for 1000 DAU
|
||||
- **Data Sources**: 100% free (Sina, Tencent, EastMoney)
|
||||
- No API keys required
|
||||
- Multi-source fallback ensures 99.9% availability
|
||||
- **Claude API**: Optional, for AI explanations only
|
||||
- Redis caching (3600s TTL) reduces 80% duplicate calls
|
||||
- Model tiering: Haiku for simple, Sonnet for complex
|
||||
- Token compression: Only send essential fields
|
||||
- **Data API**: TuShare rate limit 500 calls/min
|
||||
- Use 60s TTL on quotes to stay within limits
|
||||
- Batch requests when possible
|
||||
- **Local LLM**: Hybrid mode with Ollama reduces Claude API costs by 40%
|
||||
|
||||
### Performance Targets
|
||||
- API response time: <200ms (cached), <2s (uncached)
|
||||
@@ -474,11 +488,12 @@ Read in this order:
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **Claude API timeout**: Check `OPENROUTER_API_KEY` and rate limits
|
||||
2. **TuShare data missing**: Verify `TUSHARE_TOKEN` and market hours
|
||||
3. **Redis connection failed**: Check `REDIS_URL` and port 6381
|
||||
4. **K8s pod crash**: Check logs with `kubectl logs -f <pod> -n claude-stock`
|
||||
1. **Data fetch failures**: Check data source adapters in backend logs, system falls back automatically
|
||||
2. **Claude API timeout**: Check `OPENROUTER_API_KEY` (optional, only needed for AI explanations)
|
||||
3. **Redis connection failed**: Check `REDIS_URL` and port 6381 (Docker) or 6379 (local)
|
||||
4. **K8s pod crash**: Check logs with `kubectl logs -f <pod> -n stock`
|
||||
5. **CI/CD pipeline failure**: Run `bash verify-cicd.sh` to diagnose
|
||||
6. **Ollama connection**: Ensure Ollama is running on port 11434 for hybrid AI mode
|
||||
|
||||
### Debug Commands
|
||||
|
||||
@@ -498,7 +513,12 @@ redis-cli -p 6381 -a redis2025 KEYS "*"
|
||||
docker exec -it <postgres-container> psql -U stockai -d stockai
|
||||
|
||||
# View K8s events
|
||||
kubectl get events -n claude-stock --sort-by='.lastTimestamp'
|
||||
kubectl get events -n stock --sort-by='.lastTimestamp'
|
||||
|
||||
# Test data sources directly
|
||||
cd backend
|
||||
python test_data_sources.py
|
||||
python test_all_data_sources.py
|
||||
```
|
||||
|
||||
## Repository Guidelines
|
||||
@@ -510,9 +530,34 @@ Per `AGENTS.md`:
|
||||
- Add tests for all new features
|
||||
- Update documentation when changing architecture
|
||||
|
||||
## Important Implementation Notes
|
||||
|
||||
### Chinese Stock Symbols
|
||||
- **A-share format**: 6-digit code (e.g., `600000` for Pudong Development Bank)
|
||||
- **Market prefixes**: Shanghai (60xxxx, 688xxx), Shenzhen (00xxxx, 300xxx)
|
||||
- No need for exchange suffix when using Chinese data sources
|
||||
|
||||
### Data Fetcher Architecture
|
||||
The `data_fetcher_v2.py` implements a priority-based multi-source pattern:
|
||||
1. Try primary source (fastest)
|
||||
2. Fall back to secondary source if primary fails
|
||||
3. Fall back to tertiary source if both fail
|
||||
4. Return cached data if all sources fail
|
||||
|
||||
This ensures 99.9% availability without requiring API keys.
|
||||
|
||||
### Full Data Pipeline
|
||||
Use `backend/run_all_fetchers.sh` to fetch comprehensive A-share data:
|
||||
- Company info (5,295 stocks)
|
||||
- Sector/plate data (~400 sectors)
|
||||
- Financial statements (4 quarters)
|
||||
- Money flow (700 days)
|
||||
- Top list (dragon-tiger list)
|
||||
|
||||
Supports test mode, incremental mode, and custom selection.
|
||||
|
||||
## Last Updated
|
||||
|
||||
**Date**: 2025-11-15
|
||||
**Version**: v1.0.0
|
||||
**Git SHA**: cbc9e25
|
||||
**Date**: 2025-12-23
|
||||
**Status**: Production Ready
|
||||
**Data Architecture**: Multi-source (Sina/Tencent/EastMoney), 100% free
|
||||
|
||||
@@ -50,7 +50,7 @@
|
||||
|------|------|------|
|
||||
| **后端框架** | FastAPI + Uvicorn | 0.104.1 |
|
||||
| **数据源** | 新浪财经 + 腾讯财经 + 东方财富 | - |
|
||||
| **数据库** | AkShare (可选) | 1.17+ |
|
||||
| **A股数据** | TwoStock (TuShare兼容) | 最新版 |
|
||||
| **机器学习** | XGBoost + LightGBM + scikit-learn | 2.0.3 |
|
||||
| **LLM** | Claude 4.5 Sonnet (OpenRouter) | - |
|
||||
| **前端** | Vue 3 + Vite + ECharts | 3.3.4 |
|
||||
@@ -132,7 +132,7 @@ npm run dev
|
||||
# 访问: http://localhost:5173
|
||||
```
|
||||
|
||||
### 方式三: 一键启动脚本（Mac/Linux）
|
||||
### 方式三: 一键启动脚本（Mac/Linux，推荐）
|
||||
|
||||
```bash
|
||||
# 使用快速启动脚本
|
||||
@@ -145,6 +145,19 @@ npm run dev
|
||||
./stop_app.sh --all
|
||||
```
|
||||
|
||||
**访问地址**：
|
||||
- 本地访问：
|
||||
- Frontend: http://localhost:8448
|
||||
- Backend API: http://localhost:8028
|
||||
- API文档: http://localhost:8028/docs
|
||||
|
||||
- 远程访问（局域网）：
|
||||
- Frontend: http://192.168.1.83:8448 （替换为实际IP）
|
||||
- Backend API: http://192.168.1.83:8028
|
||||
- API文档: http://192.168.1.83:8028/docs
|
||||
|
||||
> **提示**: 应用已配置为监听 `0.0.0.0`，支持局域网内远程访问。端口可在 `backend/.env` 中自定义（BACKEND_PORT、FRONTEND_PORT）。
|
||||
|
||||
## 📥 全量数据抓取
|
||||
|
||||
系统默认只抓取实时行情和历史K线数据。如需获取完整的基础数据（公司信息、板块、财务、资金流等），可使用一键抓取脚本。
|
||||
@@ -277,6 +290,254 @@ ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC;
|
||||
⚠️ **磁盘空间** - 预留至少 5GB 磁盘空间
|
||||
⚠️ **API限流** - 脚本已内置延迟机制，避免触发API限流
|
||||
|
||||
## 📊 财务指标计算与新闻情绪分析
|
||||
|
||||
在完成基础数据抓取后，需要执行财务指标计算和新闻情绪抓取，以支持完整的推荐功能。
|
||||
|
||||
### 6. 财务指标计算 (~5分钟)
|
||||
|
||||
基于已抓取的财务报表，计算推荐系统所需的核心指标。
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
source .venv/bin/activate
|
||||
|
||||
# 计算财务指标（ROE、资产负债率、利润率等）
|
||||
python calculate_financial_indicators.py
|
||||
|
||||
# 预期结果：
|
||||
# - ROE 覆盖率: 99.37% (5,015/5,047 stocks)
|
||||
# - 资产负债率: 100% 覆盖
|
||||
# - 净利率/毛利率: 0.24% (数据源限制)
|
||||
```
|
||||
|
||||
**计算的指标**:
|
||||
- **盈利能力**: ROE (净资产收益率)、ROA、净利率、毛利率
|
||||
- **偿债能力**: 资产负债率、流动比率
|
||||
- **成长性**: 营收增长率(YoY)、利润增长率(YoY)
|
||||
|
||||
**数据存储**: 结果保存到 `fundamentals` 表
|
||||
|
||||
### 7. 新闻情绪分析 (~12分钟)
|
||||
|
||||
抓取股票相关新闻并进行情绪分析，支持综合策略推荐。
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
source .venv/bin/activate
|
||||
|
||||
# 测试模式（前5只股票）
|
||||
python fetch_news_sentiment_v2.py --test
|
||||
|
||||
# 完整模式（全部股票，约12分钟）
|
||||
python fetch_news_sentiment_v2.py
|
||||
|
||||
# 预期结果：
|
||||
# - 新闻数量: 51,126 条
|
||||
# - 股票覆盖率: 98.7% (5,224/5,295 stocks)
|
||||
# - 时间跨度: 最近6个月
|
||||
```
|
||||
|
||||
**情绪分析方法**:
|
||||
- **正面关键词** (27个): 增长、上涨、盈利、分红、超预期、看好、利好等
|
||||
- **负面关键词** (22个): 下跌、亏损、处罚、风险、诉讼、退市、违规等
|
||||
- **得分范围**: -1.0 (极负面) 到 1.0 (极正面)
|
||||
|
||||
**数据存储**: 结果保存到 `sentiment_data` 表
|
||||
|
||||
### 验证数据完整性
|
||||
|
||||
```bash
|
||||
# 运行验证脚本
|
||||
python verify_financial_data.py
|
||||
|
||||
# 预期输出：
|
||||
# ✅ 财务报表数据: 5,175 stocks
|
||||
# ✅ ROE 数据: 99.37% 覆盖
|
||||
# ✅ 新闻情绪: 51,126 条新闻
|
||||
# ✅ 总体数据完整度: 99.0%
|
||||
```
|
||||
|
||||
### 完整数据抓取流程
|
||||
|
||||
建议首次部署时按以下顺序执行：
|
||||
|
||||
```bash
|
||||
# 1. 基础数据（使用一键脚本）
|
||||
./run_all_fetchers.sh # 选择选项1（完整模式）
|
||||
|
||||
# 2. 财务指标计算
|
||||
python calculate_financial_indicators.py
|
||||
|
||||
# 3. 新闻情绪分析
|
||||
python fetch_news_sentiment_v2.py
|
||||
|
||||
# 4. 验证数据
|
||||
python verify_financial_data.py
|
||||
```
|
||||
|
||||
**预计总耗时**: 7-9 小时（基础数据）+ 5分钟（财务指标）+ 12分钟（新闻情绪）≈ **8-10小时**
|
||||
|
||||
## 🔄 数据增量更新与维护
|
||||
|
||||
系统数据具有不同的更新频率，建议设置定时任务实现自动更新。
|
||||
|
||||
### 更新频率建议
|
||||
|
||||
| 数据类型 | 更新频率 | 更新时间 | 耗时 | 命令 |
|
||||
|---------|---------|---------|------|------|
|
||||
| **实时行情** | 实时 | 交易时间 | - | API自动获取 |
|
||||
| **历史K线** | 每日 | 收盘后 15:30 | 约30分钟 | 自动缓存更新 |
|
||||
| **新闻情绪** | 每日/每周 | 每晚 20:00 | 12分钟 | `python fetch_news_sentiment_v2.py` |
|
||||
| **资金流** | 每日 | 收盘后 16:00 | 30-60分钟 | `python fetch_money_flow.py --mode incremental` |
|
||||
| **财务报表** | 每季度 | 财报季后 | 1-2小时 | `python fetch_financial_data_v2.py --quarters 1` |
|
||||
| **财务指标** | 每季度 | 财报更新后 | 5分钟 | `python calculate_financial_indicators.py` |
|
||||
| **龙虎榜** | 每日 | 收盘后 16:30 | 5-10分钟 | `python fetch_top_list.py --start-date 今天` |
|
||||
| **公司信息** | 每月 | 月初 | 30分钟 | `python fetch_company_info_v2.py` |
|
||||
| **板块数据** | 每月 | 月初 | 10分钟 | `python fetch_sector_data.py` |
|
||||
|
||||
### 设置定时任务 (Cron)
|
||||
|
||||
**编辑 crontab**:
|
||||
```bash
|
||||
crontab -e
|
||||
```
|
||||
|
||||
**添加定时任务**:
|
||||
```cron
|
||||
# 工作目录
|
||||
BACKEND_DIR=/path/to/ljwx-stock/backend
|
||||
VENV=$BACKEND_DIR/.venv/bin/activate
|
||||
|
||||
# 每日更新
|
||||
# 每晚 20:00 更新新闻情绪
|
||||
0 20 * * * cd $BACKEND_DIR && source $VENV && python fetch_news_sentiment_v2.py >> /var/log/stock-news.log 2>&1
|
||||
|
||||
# 每日交易日 16:00 更新资金流（增量模式）
|
||||
0 16 * * 1-5 cd $BACKEND_DIR && source $VENV && python fetch_money_flow.py --mode incremental >> /var/log/stock-money-flow.log 2>&1
|
||||
|
||||
# 每日交易日 16:30 更新龙虎榜
|
||||
30 16 * * 1-5 cd $BACKEND_DIR && source $VENV && python fetch_top_list.py --start-date $(date +\%Y-\%m-\%d) >> /var/log/stock-top-list.log 2>&1
|
||||
|
||||
# 每季度更新（财报季：1月、4月、7月、10月的15日）
|
||||
# 1. 更新财务报表
|
||||
0 2 15 1,4,7,10 * cd $BACKEND_DIR && source $VENV && python fetch_financial_data_v2.py --quarters 1 >> /var/log/stock-financial.log 2>&1
|
||||
|
||||
# 2. 重新计算财务指标
|
||||
0 4 15 1,4,7,10 * cd $BACKEND_DIR && source $VENV && python calculate_financial_indicators.py >> /var/log/stock-indicators.log 2>&1
|
||||
|
||||
# 每月1日更新公司信息和板块数据
|
||||
0 2 1 * * cd $BACKEND_DIR && source $VENV && python fetch_company_info_v2.py >> /var/log/stock-company.log 2>&1
|
||||
0 3 1 * * cd $BACKEND_DIR && source $VENV && python fetch_sector_data.py >> /var/log/stock-sector.log 2>&1
|
||||
```
|
||||
|
||||
### 手动增量更新
|
||||
|
||||
**每日数据更新**:
|
||||
```bash
|
||||
cd backend
|
||||
source .venv/bin/activate
|
||||
|
||||
# 更新新闻情绪（只抓取最近7天）
|
||||
python fetch_news_sentiment_v2.py --limit 5295 # 全量更新
|
||||
|
||||
# 更新资金流（增量模式，只抓取缺失日期）
|
||||
python fetch_money_flow.py --mode incremental
|
||||
|
||||
# 更新龙虎榜（指定日期范围）
|
||||
python fetch_top_list.py --start-date 2024-12-01 --end-date 2024-12-24
|
||||
```
|
||||
|
||||
**季度数据更新**:
|
||||
```bash
|
||||
# 1. 更新最新一季度财务报表
|
||||
python fetch_financial_data_v2.py --quarters 1
|
||||
|
||||
# 2. 重新计算所有财务指标
|
||||
python calculate_financial_indicators.py
|
||||
|
||||
# 3. 验证数据完整性
|
||||
python verify_financial_data.py
|
||||
```
|
||||
|
||||
### 数据清理与维护
|
||||
|
||||
**清理历史新闻（保留最近30天）**:
|
||||
```sql
|
||||
psql -U stock_user -d stock_db -c "
|
||||
DELETE FROM sentiment_data
|
||||
WHERE news_date < CURRENT_DATE - INTERVAL '30 days';
|
||||
"
|
||||
```
|
||||
|
||||
**清理旧资金流数据（保留最近2年）**:
|
||||
```sql
|
||||
psql -U stock_user -d stock_db -c "
|
||||
DELETE FROM t_money_flow
|
||||
WHERE trade_date < CURRENT_DATE - INTERVAL '2 years';
|
||||
"
|
||||
```
|
||||
|
||||
**数据库空间监控**:
|
||||
```bash
|
||||
# 查看各表占用空间
|
||||
psql -U stock_user -d stock_db -c "
|
||||
SELECT
|
||||
tablename,
|
||||
pg_size_pretty(pg_total_relation_size('public.'||tablename)) AS size,
|
||||
n_live_tup as rows
|
||||
FROM pg_stat_user_tables
|
||||
WHERE schemaname = 'public'
|
||||
ORDER BY pg_total_relation_size('public.'||tablename) DESC
|
||||
LIMIT 20;
|
||||
"
|
||||
```
|
||||
|
||||
**预计空间占用**:
|
||||
- 基础数据: ~500MB
|
||||
- 资金流数据: ~2GB (700天 × 5,295股票)
|
||||
- 财务报表: ~200MB
|
||||
- 新闻情绪: ~100MB (最近30天)
|
||||
- **总计**: ~3GB
|
||||
|
||||
### 数据备份
|
||||
|
||||
**推荐定期备份数据库**:
|
||||
```bash
|
||||
# 备份整个数据库
|
||||
pg_dump -U stock_user stock_db > backup_$(date +%Y%m%d).sql
|
||||
|
||||
# 只备份关键表
|
||||
pg_dump -U stock_user -d stock_db \
|
||||
-t stocks -t fundamentals -t sentiment_data \
|
||||
-t t_income_statement -t t_balance_sheet \
|
||||
> backup_core_$(date +%Y%m%d).sql
|
||||
|
||||
# 恢复备份
|
||||
psql -U stock_user -d stock_db < backup_20241224.sql
|
||||
```
|
||||
|
||||
### 故障排查
|
||||
|
||||
**常见问题**:
|
||||
|
||||
1. **新闻抓取失败**: 检查网络连接，AkShare API可能限流
|
||||
2. **财务指标为NULL**: 确保先执行了财务报表抓取
|
||||
3. **数据库磁盘满**: 执行数据清理，删除旧数据
|
||||
4. **定时任务未执行**: 检查 cron 日志 `grep CRON /var/log/syslog`
|
||||
|
||||
**日志查看**:
|
||||
```bash
|
||||
# 查看数据抓取日志
|
||||
tail -f backend/logs/fetcher_*/combined.log
|
||||
|
||||
# 查看新闻抓取日志
|
||||
tail -f backend/news_fetch_v2.log
|
||||
|
||||
# 查看定时任务日志
|
||||
tail -f /var/log/stock-news.log
|
||||
```
|
||||
|
||||
## 🔑 API Key 配置（可选）
|
||||
|
||||
### Claude API（AI 解释功能）
|
||||
@@ -290,14 +551,21 @@ ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC;
|
||||
OPENROUTER_API_KEY=sk-or-v1-xxxxxx
|
||||
```
|
||||
|
||||
### 数据源（无需配置）
|
||||
### 数据源配置
|
||||
|
||||
✅ **新浪财经** - 免费，无需认证
|
||||
✅ **腾讯财经** - 免费，无需认证
|
||||
✅ **东方财富** - 免费，无需认证
|
||||
✅ **AkShare** - 免费Python库，无需API key
|
||||
✅ **TwoStock** - TuShare兼容库，提供A股数据
|
||||
|
||||
❌ ~~TuShare~~ - 已移除依赖
|
||||
**TwoStock配置**（已预配置）:
|
||||
```bash
|
||||
# backend/.env
|
||||
TUSHARE_TOKEN=TMF3Nb5LoHTIc4Ksqra5dgDE3t2aAe0n4zmrl2r08EwuJ6xigrp1nb8f4a71242f
|
||||
```
|
||||
|
||||
> **说明**: 项目已将 `tushare` 替换为 `twostock`（完全兼容的替代品），所有导入已更新为 `import twostock as ts`。默认token已配置，可直接使用。
|
||||
|
||||
## 📊 核心功能
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import time
|
||||
from typing import List, Dict, Optional
|
||||
from datetime import datetime, timedelta
|
||||
import pandas as pd
|
||||
import tushare as ts
|
||||
import twostock as ts
|
||||
|
||||
from app.config import get_settings
|
||||
from app.database_pg import (
|
||||
|
||||
@@ -15,7 +15,7 @@ import hashlib
|
||||
|
||||
# TuShare API
|
||||
try:
|
||||
import tushare as ts
|
||||
import twostock as ts
|
||||
except ImportError:
|
||||
ts = None
|
||||
|
||||
|
||||
@@ -34,6 +34,10 @@ class DataFetcher:
|
||||
self.cache_ttl_quote = settings.cache_ttl_quote
|
||||
self.cache_ttl_features = settings.cache_ttl_features
|
||||
|
||||
# Token状态标志
|
||||
self.token_invalid = False # 标记token是否已经失效
|
||||
self.token_refresh_attempted = False # 标记是否已经尝试过刷新
|
||||
|
||||
# Use token_manager for dynamic token retrieval
|
||||
if TOKEN_MANAGER_AVAILABLE:
|
||||
print("🔑 Using token_manager for dynamic token management")
|
||||
@@ -197,7 +201,7 @@ class DataFetcher:
|
||||
return self._get_default_fundamentals(symbol)
|
||||
|
||||
# Use TuShare API
|
||||
import tushare as ts
|
||||
import twostock as ts
|
||||
pro = ts.pro_api(self.tushare_token)
|
||||
|
||||
# Get basic info
|
||||
@@ -251,8 +255,12 @@ class DataFetcher:
|
||||
Returns:
|
||||
DataFrame with OHLCV data or None if fetch fails
|
||||
"""
|
||||
# 如果token已经被标记为失效，直接跳过
|
||||
if self.token_invalid:
|
||||
return None
|
||||
|
||||
try:
|
||||
import tushare as ts
|
||||
import twostock as ts
|
||||
pro = ts.pro_api(self.tushare_token)
|
||||
|
||||
# Convert period to days
|
||||
@@ -313,20 +321,32 @@ class DataFetcher:
|
||||
error_msg = str(e)
|
||||
print(f"⚠️ TuShare fetch failed for {symbol}: {error_msg}")
|
||||
|
||||
# Check if token is invalid and try to refresh
|
||||
# Check if token is invalid and try to refresh (only once)
|
||||
if TOKEN_MANAGER_AVAILABLE and is_token_invalid_error(error_msg):
|
||||
print(f"🔄 Detected invalid token, attempting to refresh...")
|
||||
try:
|
||||
from token_manager import refresh_token
|
||||
new_token = refresh_token()
|
||||
if new_token and new_token != self.tushare_token:
|
||||
print(f"✅ Token refreshed: {new_token[:8]}...{new_token[-8:]}")
|
||||
self.tushare_token = new_token
|
||||
# Retry with new token
|
||||
print(f"🔄 Retrying fetch for {symbol} with new token...")
|
||||
return self._fetch_tushare_data(symbol, period)
|
||||
except Exception as refresh_error:
|
||||
print(f"❌ Token refresh failed: {refresh_error}")
|
||||
if not self.token_refresh_attempted:
|
||||
print(f"🔄 Detected invalid token, attempting to refresh...")
|
||||
self.token_refresh_attempted = True
|
||||
try:
|
||||
from token_manager import refresh_token
|
||||
new_token = refresh_token()
|
||||
if new_token and new_token != self.tushare_token:
|
||||
print(f"✅ Token refreshed: {new_token[:8]}...{new_token[-8:]}")
|
||||
self.tushare_token = new_token
|
||||
self.token_refresh_attempted = False # 重置标志，允许下次重试
|
||||
# Retry with new token (only once)
|
||||
print(f"🔄 Retrying fetch for {symbol} with new token...")
|
||||
return self._fetch_tushare_data(symbol, period)
|
||||
else:
|
||||
# Token刷新失败，标记为失效
|
||||
print(f"⚠️ Token refresh returned invalid token, marking as invalid")
|
||||
self.token_invalid = True
|
||||
except Exception as refresh_error:
|
||||
print(f"❌ Token refresh failed: {refresh_error}")
|
||||
self.token_invalid = True # 标记token为失效
|
||||
else:
|
||||
# 已经尝试过刷新，不再重试
|
||||
print(f"⚠️ Token already attempted refresh, skipping TuShare")
|
||||
self.token_invalid = True
|
||||
|
||||
return None
|
||||
|
||||
@@ -455,11 +475,12 @@ class DataFetcher:
|
||||
Returns:
|
||||
Quote dict or None if fetch fails
|
||||
"""
|
||||
if not self.tushare_token:
|
||||
# 如果token无效或不存在，直接跳过
|
||||
if not self.tushare_token or self.token_invalid:
|
||||
return None
|
||||
|
||||
try:
|
||||
import tushare as ts
|
||||
import twostock as ts
|
||||
pro = ts.pro_api(self.tushare_token)
|
||||
|
||||
# Convert to TuShare format (600000 -> 600000.SH)
|
||||
@@ -486,7 +507,14 @@ class DataFetcher:
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ TuShare fetch failed for {symbol}: {e}")
|
||||
error_msg = str(e)
|
||||
print(f"⚠️ TuShare fetch failed for {symbol}: {error_msg}")
|
||||
|
||||
# 如果是token错误，标记为失效
|
||||
if TOKEN_MANAGER_AVAILABLE and is_token_invalid_error(error_msg):
|
||||
if not self.token_invalid:
|
||||
print(f"⚠️ Marking TuShare token as invalid")
|
||||
self.token_invalid = True
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ import logging
|
||||
|
||||
# TuShare API
|
||||
try:
|
||||
import tushare as ts
|
||||
import twostock as ts
|
||||
except ImportError:
|
||||
ts = None
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ except ImportError:
|
||||
OPENBB_AVAILABLE = False
|
||||
print("⚠️ OpenBB SDK not installed. Install with: pip install openbb")
|
||||
|
||||
import tushare as ts
|
||||
import twostock as ts
|
||||
import yfinance as yf
|
||||
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ from typing import List, Dict, Optional
|
||||
from datetime import datetime, timedelta
|
||||
import time
|
||||
|
||||
import tushare as ts
|
||||
import twostock as ts
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import requests
|
||||
|
||||
@@ -15,7 +15,7 @@ import numpy as np
|
||||
|
||||
# TuShare API for sentiment data
|
||||
try:
|
||||
import tushare as ts
|
||||
import twostock as ts
|
||||
except ImportError:
|
||||
ts = None
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ Stock Universe Module
|
||||
动态获取所有可交易股票，支持多阶段筛选
|
||||
"""
|
||||
|
||||
import tushare as ts
|
||||
import twostock as ts
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from typing import List, Dict, Optional, Set
|
||||
|
||||
@@ -3,7 +3,7 @@ Enhanced Data Service with Real Market Data
|
||||
结合 Tushare 真实数据与 AI 分析的增强数据服务
|
||||
"""
|
||||
|
||||
import tushare as ts
|
||||
import twostock as ts
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
@@ -8,7 +8,7 @@ import logging
|
||||
from datetime import datetime, timedelta
|
||||
from typing import List, Dict, Optional, Tuple
|
||||
import pandas as pd
|
||||
import tushare as ts
|
||||
import twostock as ts
|
||||
import yfinance as yf
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import requests
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tushare as ts
|
||||
import twostock as ts
|
||||
|
||||
RED = '\033[91m'
|
||||
GREEN = "\033[32m"
|
||||
|
||||
+32
-23
@@ -1,30 +1,39 @@
|
||||
import { defineConfig } from 'vite'
|
||||
import { defineConfig, loadEnv } from 'vite'
|
||||
import vue from '@vitejs/plugin-vue'
|
||||
import { fileURLToPath, URL } from 'node:url'
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [vue()],
|
||||
resolve: {
|
||||
alias: {
|
||||
'@': fileURLToPath(new URL('./src', import.meta.url))
|
||||
}
|
||||
},
|
||||
server: {
|
||||
port: 5173,
|
||||
proxy: {
|
||||
'/api': {
|
||||
target: 'http://localhost:8001', // Docker backend port
|
||||
changeOrigin: true
|
||||
},
|
||||
'/ws': {
|
||||
target: 'ws://localhost:8001', // Docker backend port
|
||||
ws: true
|
||||
export default defineConfig(({ mode }) => {
|
||||
// Load env file based on mode
|
||||
const env = loadEnv(mode, '../backend', '')
|
||||
|
||||
const frontendPort = parseInt(env.FRONTEND_PORT || '84488')
|
||||
const backendPort = parseInt(env.BACKEND_PORT || '8028')
|
||||
|
||||
return {
|
||||
plugins: [vue()],
|
||||
resolve: {
|
||||
alias: {
|
||||
'@': fileURLToPath(new URL('./src', import.meta.url))
|
||||
}
|
||||
},
|
||||
server: {
|
||||
host: '0.0.0.0', // 允许远程访问
|
||||
port: frontendPort,
|
||||
proxy: {
|
||||
'/api': {
|
||||
target: `http://0.0.0.0:${backendPort}`,
|
||||
changeOrigin: true
|
||||
},
|
||||
'/ws': {
|
||||
target: `ws://0.0.0.0:${backendPort}`,
|
||||
ws: true
|
||||
}
|
||||
}
|
||||
},
|
||||
build: {
|
||||
outDir: 'dist',
|
||||
sourcemap: false,
|
||||
minify: 'terser'
|
||||
}
|
||||
},
|
||||
build: {
|
||||
outDir: 'dist',
|
||||
sourcemap: false,
|
||||
minify: 'terser'
|
||||
}
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user