chore: Update company information and add documentation

- Update footer with Shenzhen company details - Add NEWS_CRAWLER_SETUP.md documentation
2025-10-02 21:48:54 +08:00
parent f2d4f25bed
commit c88f438cf7
2 changed files with 464 additions and 3 deletions
@@ -0,0 +1,461 @@
+# 业界动态新闻爬虫系统设置指南
+
+## 系统架构
+
+### 前端展示
+- ✅ 业界动态页面 (`/industry-news`)
+- ✅ 分类筛选和搜索功能
+- ✅ 响应式设计和实时更新
+- ✅ 模拟数据展示
+
+### 后端数据源（需要实现）
+可以选择以下任一方案：
+
+## 方案一：自建爬虫系统 (推荐)
+
+### 技术栈
+```
+Node.js + Puppeteer + Cheerio + MongoDB/PostgreSQL + Redis
+```
+
+### 目录结构
+```
+news-crawler/
+├── src/
+│   ├── crawlers/          # 爬虫实现
+│   │   ├── base-crawler.js
+│   │   ├── policy-crawler.js
+│   │   ├── robotics-crawler.js
+│   │   └── investment-crawler.js
+│   ├── processors/        # 内容处理
+│   │   ├── text-processor.js
+│   │   ├── ai-summarizer.js
+│   │   └── category-classifier.js
+│   ├── database/          # 数据库操作
+│   ├── api/              # API接口
+│   └── schedulers/       # 定时任务
+├── config/               # 配置文件
+└── docker-compose.yml    # 容器化部署
+```
+
+### 核心爬虫代码示例
+
+```javascript
+// src/crawlers/base-crawler.js
+class BaseCrawler {
+  constructor(config) {
+    this.sources = config.sources;
+    this.keywords = config.keywords;
+    this.db = config.database;
+  }
+
+  async crawlSite(sourceConfig) {
+    const browser = await puppeteer.launch();
+    const page = await browser.newPage();
+    
+    try {
+      await page.goto(sourceConfig.url);
+      
+      const articles = await page.evaluate(() => {
+        return Array.from(document.querySelectorAll('.article-item')).map(item => ({
+          title: item.querySelector('.title')?.textContent,
+          summary: item.querySelector('.summary')?.textContent,
+          link: item.querySelector('a')?.href,
+          publishedAt: item.querySelector('.date')?.textContent
+        }));
+      });
+
+      for (const article of articles) {
+        if (this.isRelevant(article.title, article.summary)) {
+          await this.processArticle(article, sourceConfig);
+        }
+      }
+    } finally {
+      await browser.close();
+    }
+  }
+
+  isRelevant(title, summary) {
+    const text = (title + ' ' + summary).toLowerCase();
+    return this.keywords.some(keyword => text.includes(keyword));
+  }
+
+  async processArticle(article, source) {
+    // AI处理：分类、摘要、关键词提取
+    const processed = await this.aiProcessor.process(article);
+    
+    // 保存到数据库
+    await this.db.saveNews({
+      ...article,
+      ...processed,
+      source: source.name,
+      crawledAt: new Date(),
+      category: this.classifyCategory(processed.keywords)
+    });
+  }
+}
+
+// src/schedulers/news-scheduler.js
+const cron = require('node-cron');
+
+class NewsScheduler {
+  constructor(crawler) {
+    this.crawler = crawler;
+  }
+
+  start() {
+    // 每30分钟运行一次
+    cron.schedule('*/30 * * * *', async () => {
+      console.log('Starting news crawl...');
+      await this.crawler.crawlAll();
+      console.log('News crawl completed');
+    });
+
+    // 每天凌晨2点清理旧数据
+    cron.schedule('0 2 * * *', async () => {
+      await this.crawler.cleanOldNews();
+    });
+  }
+}
+```
+
+### 主要爬取网站
+
+#### 康养政策类
+- 国家卫健委官网
+- 民政部官网
+- 各省市卫健委
+- 政策解读网站
+
+#### 医疗机器人类
+- 动脉网 (vcbeat.net)
+- 医学界 (yxj.org.cn)
+- 健康界 (cn-healthcare.com)
+- 器械之家
+
+#### 投融资类
+- 36氪 (36kr.com)
+- 投中网 (chinaventure.com.cn)
+- 亿欧 (iyiou.com)
+- IT橘子
+
+#### 技术前沿类
+- 雷锋网 (leiphone.com)
+- 机器之心 (jiqizhixin.com)
+- AI科技大本营
+
+### 部署配置
+
+```dockerfile
+# Dockerfile
+FROM node:18-alpine
+
+WORKDIR /app
+
+COPY package*.json ./
+RUN npm install
+
+COPY . .
+
+EXPOSE 3001
+
+CMD ["npm", "start"]
+```
+
+```yaml
+# docker-compose.yml
+version: '3.8'
+services:
+  news-crawler:
+    build: .
+    ports:
+      - "3001:3001"
+    environment:
+      - DB_URL=mongodb://mongo:27017/news
+      - REDIS_URL=redis://redis:6379
+    depends_on:
+      - mongo
+      - redis
+
+  mongo:
+    image: mongo:5
+    volumes:
+      - mongo_data:/data/db
+
+  redis:
+    image: redis:7-alpine
+
+volumes:
+  mongo_data:
+```
+
+## 方案二：第三方新闻API
+
+### NewsAPI.org
+```javascript
+const newsapi = require('newsapi');
+const api = new newsapi('YOUR_API_KEY');
+
+// 搜索特定关键词
+api.v2.everything({
+  q: '医疗机器人 OR 康养 OR 智慧医疗',
+  sources: 'techcrunch,reuters',
+  language: 'zh',
+  sortBy: 'publishedAt'
+}).then(response => {
+  console.log(response.articles);
+});
+```
+
+### 聚合数据API
+```javascript
+const axios = require('axios');
+
+const getJuheNews = async () => {
+  const response = await axios.get('http://v.juhe.cn/toutiao/index', {
+    params: {
+      type: 'keji',
+      key: 'YOUR_JUHE_KEY'
+    }
+  });
+  return response.data.result.data;
+};
+```
+
+## 方案三：RSS聚合
+
+### RSS源配置
+```javascript
+const rssSources = [
+  {
+    name: '动脉网',
+    url: 'https://vcbeat.net/rss',
+    category: 'medical-robotics'
+  },
+  {
+    name: '36氪',
+    url: 'https://36kr.com/rss',
+    category: 'investment'
+  },
+  {
+    name: '亿欧',
+    url: 'https://www.iyiou.com/rss',
+    category: 'industry-trends'
+  }
+];
+
+// RSS解析器
+const Parser = require('rss-parser');
+const parser = new Parser();
+
+async function crawlRSS() {
+  for (const source of rssSources) {
+    try {
+      const feed = await parser.parseURL(source.url);
+      
+      feed.items.forEach(item => {
+        if (isHealthcareRelated(item.title, item.contentSnippet)) {
+          saveNewsItem({
+            title: item.title,
+            summary: item.contentSnippet,
+            sourceUrl: item.link,
+            publishedAt: item.pubDate,
+            source: source.name,
+            category: source.category
+          });
+        }
+      });
+    } catch (error) {
+      console.error(`Failed to crawl ${source.name}:`, error);
+    }
+  }
+}
+```
+
+## AI增强功能
+
+### 内容摘要生成
+```javascript
+const OpenAI = require('openai');
+const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
+
+async function generateSummary(content) {
+  const response = await openai.chat.completions.create({
+    model: "gpt-3.5-turbo",
+    messages: [{
+      role: "user",
+      content: `请为以下医疗健康新闻生成简洁的摘要（100字以内）：\n\n${content}`
+    }],
+    max_tokens: 200
+  });
+  
+  return response.choices[0].message.content;
+}
+```
+
+### 自动分类
+```javascript
+async function classifyNews(title, content) {
+  const response = await openai.chat.completions.create({
+    model: "gpt-3.5-turbo",
+    messages: [{
+      role: "user",
+      content: `请将以下新闻分类到这些类别之一：
+      - healthcare-policy (康养政策)
+      - medical-robotics (医疗机器人)
+      - tech-innovation (技术前沿)
+      - investment (投融资)
+      - industry-trends (行业动态)
+      
+      新闻标题：${title}
+      新闻内容：${content.substring(0, 500)}...
+      
+      只回答类别名称，不要其他内容。`
+    }]
+  });
+  
+  return response.choices[0].message.content.trim();
+}
+```
+
+## 环境变量配置
+
+### 后端爬虫服务 (.env)
+```bash
+NODE_ENV=production
+PORT=3001
+
+# 数据库
+DB_URL=mongodb://localhost:27017/news
+REDIS_URL=redis://localhost:6379
+
+# API密钥
+NEWSAPI_KEY=your_newsapi_key
+JUHE_KEY=your_juhe_key
+OPENAI_API_KEY=your_openai_key
+
+# 爬虫配置
+CRAWL_INTERVAL=30
+MAX_NEWS_AGE_DAYS=30
+ENABLE_AI_PROCESSING=true
+```
+
+### 前端Vite项目 (.env.local)
+```bash
+# News API Configuration
+VITE_NEWS_API_URL=http://localhost:3001/api
+VITE_NEWS_API_KEY=your_api_key
+
+# Third-party APIs (if using)
+VITE_NEWSAPI_KEY=your_newsapi_key
+VITE_JUHE_KEY=your_juhe_key
+
+# Development
+VITE_ENABLE_MOCK_DATA=false
+```
+
+## API接口设计
+
+```javascript
+// src/api/routes.js
+app.get('/api/news', async (req, res) => {
+  const {
+    category,
+    page = 1,
+    limit = 10,
+    query,
+    sortBy = 'publishedAt'
+  } = req.query;
+
+  const filters = {};
+  if (category && category !== 'all') {
+    filters.category = category;
+  }
+  if (query) {
+    filters.$or = [
+      { title: { $regex: query, $options: 'i' } },
+      { summary: { $regex: query, $options: 'i' } },
+      { tags: { $in: [new RegExp(query, 'i')] } }
+    ];
+  }
+
+  const total = await News.countDocuments(filters);
+  const items = await News.find(filters)
+    .sort({ [sortBy]: -1 })
+    .skip((page - 1) * limit)
+    .limit(limit);
+
+  res.json({
+    items,
+    total,
+    page: parseInt(page),
+    totalPages: Math.ceil(total / limit),
+    hasNext: page * limit < total,
+    hasPrev: page > 1
+  });
+});
+
+app.get('/api/news/stats', async (req, res) => {
+  const stats = await News.aggregate([
+    {
+      $group: {
+        _id: null,
+        totalNews: { $sum: 1 },
+        categories: { $push: '$category' },
+        sources: { $push: '$source' }
+      }
+    }
+  ]);
+
+  res.json(stats[0]);
+});
+```
+
+## 监控和维护
+
+### 日志记录
+```javascript
+const winston = require('winston');
+
+const logger = winston.createLogger({
+  level: 'info',
+  format: winston.format.json(),
+  transports: [
+    new winston.transports.File({ filename: 'logs/error.log', level: 'error' }),
+    new winston.transports.File({ filename: 'logs/combined.log' })
+  ]
+});
+
+// 爬虫监控
+logger.info('Crawl started', { source: 'vcbeat.net', timestamp: new Date() });
+logger.error('Crawl failed', { source: 'vcbeat.net', error: error.message });
+```
+
+### 健康检查
+```javascript
+app.get('/health', (req, res) => {
+  res.json({
+    status: 'healthy',
+    timestamp: new Date(),
+    uptime: process.uptime(),
+    memory: process.memoryUsage(),
+    lastCrawl: await getLastCrawlTime()
+  });
+});
+```
+
+## 部署建议
+
+1. **开发阶段**：使用模拟数据和RSS聚合
+2. **测试阶段**：部署基础爬虫 + 第三方API
+3. **生产阶段**：自建完整爬虫系统 + AI增强
+
+## 法律合规
+
+- 遵守robots.txt规范
+- 设置合理的爬虫间隔
+- 尊重网站服务条款
+- 标注新闻来源和链接
+- 不存储完整原文，仅保存摘要和链接
+
+现在前端已经完全就绪，您可以选择任一后端方案来实现数据抓取功能！
@@ -72,7 +72,7 @@ const Footer = () => {
            <div className="space-y-3 text-gray-400">
              <div className="flex items-center gap-3">
                <MapPin size={18} className="text-tech-blue" />
-                <span>北京市海淀区中关村科技园</span>
+                <span>深圳-福田-香蜜湖</span>
              </div>
              <div className="flex items-center gap-3">
                <Phone size={18} className="text-tech-blue" />
@@ -179,7 +179,7 @@ const Footer = () => {
            transition={{ duration: 0.6, delay: 0.4 }}
            viewport={{ once: true }}
          >
-            <p>© {currentYear} 灵境万象科技有限公司. All rights reserved.</p>
+            <p>© {currentYear} 深圳灵境万象智能科技有限公司. All rights reserved.</p>
          </motion.div>
          
          <motion.div
@@ -193,7 +193,7 @@ const Footer = () => {
            <a href="#" className="hover:text-tech-blue transition-colors">服务条款</a>
            <a href="#" className="hover:text-tech-blue transition-colors">Cookie设置</a>
            <span className="flex items-center gap-1">
-              Made with <Heart size={14} className="text-red-500 fill-red-500" /> in Beijing
+              Made with <Heart size={14} className="text-red-500 fill-red-500" /> in shenzhen
            </span>
          </motion.div>
        </div>