website/NEWS_CRAWLER_SETUP.md

业界动态新闻爬虫系统设置指南

系统架构

前端展示

• ✅ 业界动态页面 (/industry-news)
• ✅ 分类筛选和搜索功能
• ✅ 响应式设计和实时更新
• ✅ 模拟数据展示

后端数据源（需要实现）

可以选择以下任一方案：

方案一：自建爬虫系统 (推荐)

技术栈

Node.js + Puppeteer + Cheerio + MongoDB/PostgreSQL + Redis

目录结构

news-crawler/
├── src/
│   ├── crawlers/          # 爬虫实现
│   │   ├── base-crawler.js
│   │   ├── policy-crawler.js
│   │   ├── robotics-crawler.js
│   │   └── investment-crawler.js
│   ├── processors/        # 内容处理
│   │   ├── text-processor.js
│   │   ├── ai-summarizer.js
│   │   └── category-classifier.js
│   ├── database/          # 数据库操作
│   ├── api/              # API接口
│   └── schedulers/       # 定时任务
├── config/               # 配置文件
└── docker-compose.yml    # 容器化部署

核心爬虫代码示例

// src/crawlers/base-crawler.js
class BaseCrawler {
  constructor(config) {
    this.sources = config.sources;
    this.keywords = config.keywords;
    this.db = config.database;
  }

  async crawlSite(sourceConfig) {
    const browser = await puppeteer.launch();
    const page = await browser.newPage();
    
    try {
      await page.goto(sourceConfig.url);
      
      const articles = await page.evaluate(() => {
        return Array.from(document.querySelectorAll('.article-item')).map(item => ({
          title: item.querySelector('.title')?.textContent,
          summary: item.querySelector('.summary')?.textContent,
          link: item.querySelector('a')?.href,
          publishedAt: item.querySelector('.date')?.textContent
        }));
      });

      for (const article of articles) {
        if (this.isRelevant(article.title, article.summary)) {
          await this.processArticle(article, sourceConfig);
        }
      }
    } finally {
      await browser.close();
    }
  }

  isRelevant(title, summary) {
    const text = (title + ' ' + summary).toLowerCase();
    return this.keywords.some(keyword => text.includes(keyword));
  }

  async processArticle(article, source) {
    // AI处理：分类、摘要、关键词提取
    const processed = await this.aiProcessor.process(article);
    
    // 保存到数据库
    await this.db.saveNews({
      ...article,
      ...processed,
      source: source.name,
      crawledAt: new Date(),
      category: this.classifyCategory(processed.keywords)
    });
  }
}

// src/schedulers/news-scheduler.js
const cron = require('node-cron');

class NewsScheduler {
  constructor(crawler) {
    this.crawler = crawler;
  }

  start() {
    // 每30分钟运行一次
    cron.schedule('*/30 * * * *', async () => {
      console.log('Starting news crawl...');
      await this.crawler.crawlAll();
      console.log('News crawl completed');
    });

    // 每天凌晨2点清理旧数据
    cron.schedule('0 2 * * *', async () => {
      await this.crawler.cleanOldNews();
    });
  }
}

主要爬取网站

康养政策类

• 国家卫健委官网
• 民政部官网
• 各省市卫健委
• 政策解读网站

医疗机器人类

• 动脉网 (vcbeat.net)
• 医学界 (yxj.org.cn)
• 健康界 (cn-healthcare.com)
• 器械之家

投融资类

• 36氪 (36kr.com)
• 投中网 (chinaventure.com.cn)
• 亿欧 (iyiou.com)
• IT橘子

技术前沿类

• 雷锋网 (leiphone.com)
• 机器之心 (jiqizhixin.com)
• AI科技大本营

部署配置

# Dockerfile
FROM node:18-alpine

WORKDIR /app

COPY package*.json ./
RUN npm install

COPY . .

EXPOSE 3001

CMD ["npm", "start"]

# docker-compose.yml
version: '3.8'
services:
  news-crawler:
    build: .
    ports:
      - "3001:3001"
    environment:
      - DB_URL=mongodb://mongo:27017/news
      - REDIS_URL=redis://redis:6379
    depends_on:
      - mongo
      - redis

  mongo:
    image: mongo:5
    volumes:
      - mongo_data:/data/db

  redis:
    image: redis:7-alpine

volumes:
  mongo_data:

方案二：第三方新闻API

NewsAPI.org

const newsapi = require('newsapi');
const api = new newsapi('YOUR_API_KEY');

// 搜索特定关键词
api.v2.everything({
  q: '医疗机器人 OR 康养 OR 智慧医疗',
  sources: 'techcrunch,reuters',
  language: 'zh',
  sortBy: 'publishedAt'
}).then(response => {
  console.log(response.articles);
});

聚合数据API

const axios = require('axios');

const getJuheNews = async () => {
  const response = await axios.get('http://v.juhe.cn/toutiao/index', {
    params: {
      type: 'keji',
      key: 'YOUR_JUHE_KEY'
    }
  });
  return response.data.result.data;
};

方案三：RSS聚合

RSS源配置

const rssSources = [
  {
    name: '动脉网',
    url: 'https://vcbeat.net/rss',
    category: 'medical-robotics'
  },
  {
    name: '36氪',
    url: 'https://36kr.com/rss',
    category: 'investment'
  },
  {
    name: '亿欧',
    url: 'https://www.iyiou.com/rss',
    category: 'industry-trends'
  }
];

// RSS解析器
const Parser = require('rss-parser');
const parser = new Parser();

async function crawlRSS() {
  for (const source of rssSources) {
    try {
      const feed = await parser.parseURL(source.url);
      
      feed.items.forEach(item => {
        if (isHealthcareRelated(item.title, item.contentSnippet)) {
          saveNewsItem({
            title: item.title,
            summary: item.contentSnippet,
            sourceUrl: item.link,
            publishedAt: item.pubDate,
            source: source.name,
            category: source.category
          });
        }
      });
    } catch (error) {
      console.error(`Failed to crawl ${source.name}:`, error);
    }
  }
}

AI增强功能

内容摘要生成

const OpenAI = require('openai');
const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });

async function generateSummary(content) {
  const response = await openai.chat.completions.create({
    model: "gpt-3.5-turbo",
    messages: [{
      role: "user",
      content: `请为以下医疗健康新闻生成简洁的摘要（100字以内）：\n\n${content}`
    }],
    max_tokens: 200
  });
  
  return response.choices[0].message.content;
}

自动分类

async function classifyNews(title, content) {
  const response = await openai.chat.completions.create({
    model: "gpt-3.5-turbo",
    messages: [{
      role: "user",
      content: `请将以下新闻分类到这些类别之一：
      - healthcare-policy (康养政策)
      - medical-robotics (医疗机器人)
      - tech-innovation (技术前沿)
      - investment (投融资)
      - industry-trends (行业动态)
      
      新闻标题：${title}
      新闻内容：${content.substring(0, 500)}...
      
      只回答类别名称，不要其他内容。`
    }]
  });
  
  return response.choices[0].message.content.trim();
}

环境变量配置

后端爬虫服务 (.env)

NODE_ENV=production
PORT=3001

# 数据库
DB_URL=mongodb://localhost:27017/news
REDIS_URL=redis://localhost:6379

# API密钥
NEWSAPI_KEY=your_newsapi_key
JUHE_KEY=your_juhe_key
OPENAI_API_KEY=your_openai_key

# 爬虫配置
CRAWL_INTERVAL=30
MAX_NEWS_AGE_DAYS=30
ENABLE_AI_PROCESSING=true

前端Vite项目 (.env.local)

# News API Configuration
VITE_NEWS_API_URL=http://localhost:3001/api
VITE_NEWS_API_KEY=your_api_key

# Third-party APIs (if using)
VITE_NEWSAPI_KEY=your_newsapi_key
VITE_JUHE_KEY=your_juhe_key

# Development
VITE_ENABLE_MOCK_DATA=false

API接口设计

// src/api/routes.js
app.get('/api/news', async (req, res) => {
  const {
    category,
    page = 1,
    limit = 10,
    query,
    sortBy = 'publishedAt'
  } = req.query;

  const filters = {};
  if (category && category !== 'all') {
    filters.category = category;
  }
  if (query) {
    filters.$or = [
      { title: { $regex: query, $options: 'i' } },
      { summary: { $regex: query, $options: 'i' } },
      { tags: { $in: [new RegExp(query, 'i')] } }
    ];
  }

  const total = await News.countDocuments(filters);
  const items = await News.find(filters)
    .sort({ [sortBy]: -1 })
    .skip((page - 1) * limit)
    .limit(limit);

  res.json({
    items,
    total,
    page: parseInt(page),
    totalPages: Math.ceil(total / limit),
    hasNext: page * limit < total,
    hasPrev: page > 1
  });
});

app.get('/api/news/stats', async (req, res) => {
  const stats = await News.aggregate([
    {
      $group: {
        _id: null,
        totalNews: { $sum: 1 },
        categories: { $push: '$category' },
        sources: { $push: '$source' }
      }
    }
  ]);

  res.json(stats[0]);
});

监控和维护

日志记录

const winston = require('winston');

const logger = winston.createLogger({
  level: 'info',
  format: winston.format.json(),
  transports: [
    new winston.transports.File({ filename: 'logs/error.log', level: 'error' }),
    new winston.transports.File({ filename: 'logs/combined.log' })
  ]
});

// 爬虫监控
logger.info('Crawl started', { source: 'vcbeat.net', timestamp: new Date() });
logger.error('Crawl failed', { source: 'vcbeat.net', error: error.message });

健康检查

app.get('/health', (req, res) => {
  res.json({
    status: 'healthy',
    timestamp: new Date(),
    uptime: process.uptime(),
    memory: process.memoryUsage(),
    lastCrawl: await getLastCrawlTime()
  });
});

部署建议

1. 开发阶段：使用模拟数据和RSS聚合
2. 测试阶段：部署基础爬虫 + 第三方API
3. 生产阶段：自建完整爬虫系统 + AI增强

法律合规

• 遵守robots.txt规范
• 设置合理的爬虫间隔
• 尊重网站服务条款
• 标注新闻来源和链接
• 不存储完整原文，仅保存摘要和链接

现在前端已经完全就绪，您可以选择任一后端方案来实现数据抓取功能！