init

app/core/cache.py

@@ -0,0 +1,150 @@
+import redis
+from pydantic import BaseModel
+import json
+from typing import Any, Optional, Dict, List, Union
+import time
+
+from app.db.redis import get_redis_client
+from app.utils.logger import log
+
+# 默认缓存过期时间（1小时）
+DEFAULT_EXPIRE = 3600
+
+def init_cache():
+    """初始化缓存连接"""
+    try:
+        redis = get_redis_client()
+        redis.ping()
+        log.info("Cache connection established")
+    except Exception as e:
+        log.error(f"Failed to connect to cache: {e}")
+
+def close_cache():
+    """关闭缓存连接"""
+    try:
+        redis = get_redis_client()
+        redis.connection_pool.disconnect()
+        log.info("Cache connection closed")
+    except Exception as e:
+        log.error(f"Error closing cache connection: {e}")
+
+def set_cache(key: str, value: Any, expire: int = DEFAULT_EXPIRE) -> bool:
+    """设置缓存，支持自动序列化复杂对象"""
+    try:
+        redis = get_redis_client()
+        if isinstance(value, (dict, list, tuple)):
+            value = json.dumps(value)
+        elif isinstance(value, bool):
+            value = "1" if value else "0"
+
+        if expire > 0:
+            redis.setex(key, expire, value)
+        else:
+            redis.set(key, value)
+        return True
+    except Exception as e:
+        log.error(f"Error setting cache for key111111 {key}: {e}")
+        return False
+
+def get_cache(key: str) -> Optional[Any]:
+    try:
+        redis = get_redis_client()
+        value = redis.get(key)
+        
+        if value is None:
+            return None
+        
+        if isinstance(value, bytes):
+            value = value.decode('utf-8')
+            
+        try:
+            return json.loads(value)
+        except (json.JSONDecodeError, TypeError):
+            return value
+    except Exception as e:
+        log.error(f"Error getting cache for key {key}: {e}")
+        return None
+
+def delete_cache(key: str) -> bool:
+    try:
+        redis = get_redis_client()
+        redis.delete(key)
+        return True
+    except Exception as e:
+        log.error(f"Error deleting cache for key {key}: {e}")
+        return False
+
+def clear_cache_pattern(pattern: str) -> int:
+    try:
+        redis = get_redis_client()
+        keys = redis.keys(pattern)
+        if keys:
+            return redis.delete(*keys)
+        return 0
+    except Exception as e:
+        log.error(f"Error clearing cache pattern {pattern}: {e}")
+        return 0
+
+
+def get(key):
+    try:
+        redis_client = get_redis_client()
+    except Exception as e:
+        log.error(f"Error getting redis client: {e}")
+        return None
+
+    value = redis_client.get(key)
+    if value is None:
+        return None
+
+    return value.decode("utf-8")
+
+
+def set(key, value, ex=None):
+    try:
+        redis_client = get_redis_client()
+    except Exception as e:
+        log.error(f"Error getting redis client: {e}")
+        return None
+
+    return redis_client.set(key, value, ex=ex)
+
+
+def delete(key):
+
+    try:
+        redis_client = get_redis_client()
+    except Exception as e:
+        log.error(f"Error getting redis client: {e}")
+        return None
+
+    return redis_client.delete(key)
+
+
+def hset(name, key, value):
+
+    try:
+        redis_client = get_redis_client()
+    except Exception as e:
+        log.error(f"Error getting redis client: {e}")
+        return None
+
+    return redis_client.hset(name, key, value)
+
+
+def hget(name, key):
+
+    try:
+        redis_client = get_redis_client()
+    except Exception as e:
+        log.error(f"Error getting redis client: {e}")
+        return None
+
+    return redis_client.hget(name, key)
+
+
+class CacheNews(BaseModel):
+    title: str
+    url: str
+    score: int
+    desc: str

app/core/config.py

@@ -0,0 +1,121 @@
+import os
+import yaml
+from typing import Dict, Any, List, Optional
+from pydantic import BaseModel, Field
+
+# 配置文件路径
+CONFIG_PATH = os.environ.get("CONFIG_PATH", "config/config.yaml")
+
+class AppConfig(BaseModel):
+    title: str
+    description: str
+    version: str
+    host: str
+    port: int
+    debug: bool = True
+    cors: Dict[str, Any]
+
+class DatabaseConfig(BaseModel):
+    host: str
+    user: str
+    password: str
+    db: str
+    charset: str
+    autocommit: bool = True
+
+class RedisConfig(BaseModel):
+    host: str
+    port: int
+    db: int
+    password: str = ""
+    decode_responses: bool = False
+    socket_timeout: int = 5
+    socket_connect_timeout: int = 5
+    health_check_interval: int = 30
+
+class CrawlerConfig(BaseModel):
+    interval: int
+    timeout: int
+    max_retry_count: int
+    max_instances: int
+    misfire_grace_time: int
+
+class LoggingConfig(BaseModel):
+    level: str
+    format: str
+    dir: str
+    file: str
+    max_size: int
+    backup_count: int
+    daily_backup_count: int
+    timezone: str
+
+class SchedulerConfig(BaseModel):
+    thread_pool_size: int
+    process_pool_size: int
+    coalesce: bool
+    max_instances: int
+    misfire_grace_time: int
+    timezone: str
+
+class NotificationConfig(BaseModel):
+    dingtalk: Dict[str, Any] = Field(default_factory=dict)
+    # 可以添加其他通知方式的配置
+    # wechat: Dict[str, Any] = Field(default_factory=dict)
+    # email: Dict[str, Any] = Field(default_factory=dict)
+
+class Config(BaseModel):
+    app: AppConfig
+    database: DatabaseConfig
+    redis: RedisConfig
+    crawler: CrawlerConfig
+    logging: LoggingConfig
+    scheduler: SchedulerConfig
+    notification: Optional[NotificationConfig] = None
+
+# 全局配置对象
+_config: Optional[Config] = None
+
+def load_config() -> Config:
+    """加载配置文件"""
+    global _config
+    if _config is None:
+        try:
+            with open(CONFIG_PATH, 'r') as f:
+                config_data = yaml.safe_load(f)
+                _config = Config(**config_data)
+        except Exception as e:
+            raise RuntimeError(f"Failed to load configuration: {e}")
+    return _config
+
+def get_config() -> Config:
+    """获取配置对象"""
+    if _config is None:
+        return load_config()
+    return _config
+
+# 便捷访问函数
+def get_app_config() -> AppConfig:
+    return get_config().app
+
+def get_db_config() -> DatabaseConfig:
+    return get_config().database
+
+def get_redis_config() -> RedisConfig:
+    return get_config().redis
+
+def get_crawler_config() -> CrawlerConfig:
+    return get_config().crawler
+
+def get_logging_config() -> LoggingConfig:
+    return get_config().logging
+
+def get_scheduler_config() -> SchedulerConfig:
+    return get_config().scheduler
+
+def get_notification_config() -> Dict[str, Any]:
+    """获取通知配置"""
+    config = get_config()
+    if config.notification:
+        return config.notification.dict()
+    return {}

app/core/db.py

@@ -0,0 +1,131 @@
+import time
+from typing import List, Dict, Any, Optional
+from contextlib import contextmanager
+import traceback
+
+import pymysql
+from pymysql.cursors import DictCursor
+
+from app.utils.logger import log
+from app.core.config import get_db_config
+
+# 连接池
+_connection = None
+
+def init_db():
+    """初始化数据库连接"""
+    global _connection
+    try:
+        db_config = get_db_config()
+        _connection = pymysql.connect(
+            host=db_config.host,
+            user=db_config.user,
+            password=db_config.password,
+            db=db_config.db,
+            charset=db_config.charset,
+            cursorclass=DictCursor,
+            autocommit=db_config.autocommit
+        )
+        log.info("Database connection established")
+    except Exception as e:
+        log.error(f"Failed to connect to database: {e}")
+        raise
+
+def close_db():
+    """关闭数据库连接"""
+    global _connection
+    if _connection:
+        _connection.close()
+        _connection = None
+        log.info("Database connection closed")
+
+@contextmanager
+def get_cursor():
+    """获取数据库游标的上下文管理器"""
+    global _connection
+    
+    # 如果连接不存在或已关闭，重新连接
+    if _connection is None or not _connection.open:
+        init_db()
+    
+    cursor = None
+    try:
+        cursor = _connection.cursor()
+        yield cursor
+    except pymysql.OperationalError as e:
+        # 处理连接断开的情况
+        if e.args[0] in (2006, 2013):  # MySQL server has gone away, Lost connection
+            log.warning("Database connection lost, reconnecting...")
+            init_db()
+            cursor = _connection.cursor()
+            yield cursor
+        else:
+            raise
+    except Exception as e:
+        log.error(f"Database error: {e}")
+        raise
+    finally:
+        if cursor:
+            cursor.close()
+
+def insert_news(news_list: List[Dict[str, Any]]) -> int:
+    """插入新闻数据，返回成功插入的数量"""
+    if not news_list:
+        return 0
+    
+    inserted_count = 0
+    start_time = time.time()
+    
+    try:
+        with get_cursor() as cursor:
+            for news in news_list:
+                # 检查是否已存在
+                cursor.execute(
+                    "SELECT id FROM news WHERE url = %s LIMIT 1",
+                    (news.get('url', ''),)
+                )
+                if cursor.fetchone():
+                    continue
+                
+                # 插入新数据
+                cursor.execute(
+                    """
+                    INSERT INTO news (title, content, url, source, publish_time, created_at)
+                    VALUES (%s, %s, %s, %s, %s, NOW())
+                    """,
+                    (
+                        news.get('title', ''),
+                        news.get('content', ''),
+                        news.get('url', ''),
+                        news.get('source', ''),
+                        news.get('publish_time', None),
+                    )
+                )
+                inserted_count += 1
+        
+        duration = time.time() - start_time
+        log.info(f"Inserted {inserted_count}/{len(news_list)} news items in {duration:.2f}s")
+        return inserted_count
+    
+    except Exception as e:
+        log.error(f"Error inserting news: {e}")
+        log.error(traceback.format_exc())
+        return 0
+
+def get_news_by_date(date_str: str, limit: int = 100) -> List[Dict[str, Any]]:
+    """获取指定日期的新闻"""
+    try:
+        with get_cursor() as cursor:
+            cursor.execute(
+                """
+                SELECT * FROM news 
+                WHERE DATE(publish_time) = %s
+                ORDER BY publish_time DESC
+                LIMIT %s
+                """,
+                (date_str, limit)
+            )
+            return cursor.fetchall()
+    except Exception as e:
+        log.error(f"Error getting news by date: {e}")
+        return []