diff --git a/XianyuAutoAsync.py b/XianyuAutoAsync.py index 1950121..796b10c 100644 --- a/XianyuAutoAsync.py +++ b/XianyuAutoAsync.py @@ -118,6 +118,10 @@ class XianyuLive: _order_detail_locks = defaultdict(lambda: asyncio.Lock()) # 记录订单详情锁的使用时间 _order_detail_lock_times = {} + + # 商品详情缓存（24小时有效） + _item_detail_cache = {} # {item_id: {'detail': str, 'timestamp': float}} + _item_detail_cache_lock = asyncio.Lock() def _safe_str(self, e): """安全地将异常转换为字符串""" @@ -864,7 +868,7 @@ class XianyuLive: return False async def fetch_item_detail_from_api(self, item_id: str) -> str: - """从外部API获取商品详情 + """获取商品详情（优先使用浏览器，备用外部API，支持24小时缓存） Args: item_id: 商品ID @@ -881,6 +885,181 @@ class XianyuLive: logger.debug(f"自动获取商品详情功能已禁用: {item_id}") return "" + # 1. 首先检查缓存（24小时有效） + async with self._item_detail_cache_lock: + if item_id in self._item_detail_cache: + cache_data = self._item_detail_cache[item_id] + cache_time = cache_data['timestamp'] + current_time = time.time() + + # 检查缓存是否在24小时内 + if current_time - cache_time < 24 * 60 * 60: # 24小时 + logger.info(f"从缓存获取商品详情: {item_id}") + return cache_data['detail'] + else: + # 缓存过期，删除 + del self._item_detail_cache[item_id] + logger.debug(f"缓存已过期，删除: {item_id}") + + # 2. 尝试使用浏览器获取商品详情 + detail_from_browser = await self._fetch_item_detail_from_browser(item_id) + if detail_from_browser: + # 保存到缓存 + async with self._item_detail_cache_lock: + self._item_detail_cache[item_id] = { + 'detail': detail_from_browser, + 'timestamp': time.time() + } + logger.info(f"成功通过浏览器获取商品详情: {item_id}, 长度: {len(detail_from_browser)}") + return detail_from_browser + + # 3. 浏览器获取失败，使用外部API作为备用 + logger.warning(f"浏览器获取商品详情失败，尝试外部API: {item_id}") + detail_from_api = await self._fetch_item_detail_from_external_api(item_id) + if detail_from_api: + # 保存到缓存 + async with self._item_detail_cache_lock: + self._item_detail_cache[item_id] = { + 'detail': detail_from_api, + 'timestamp': time.time() + } + logger.info(f"成功通过外部API获取商品详情: {item_id}, 长度: {len(detail_from_api)}") + return detail_from_api + + logger.warning(f"所有方式都无法获取商品详情: {item_id}") + return "" + + except Exception as e: + logger.error(f"获取商品详情异常: {item_id}, 错误: {self._safe_str(e)}") + return "" + + async def _fetch_item_detail_from_browser(self, item_id: str) -> str: + """使用浏览器获取商品详情""" + try: + from playwright.async_api import async_playwright + + logger.info(f"开始使用浏览器获取商品详情: {item_id}") + + playwright = await async_playwright().start() + + # 启动浏览器（参照order_detail_fetcher的配置） + browser_args = [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + '--disable-accelerated-2d-canvas', + '--no-first-run', + '--no-zygote', + '--disable-gpu', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-renderer-backgrounding', + '--disable-features=TranslateUI', + '--disable-ipc-flooding-protection', + '--disable-extensions', + '--disable-default-apps', + '--disable-sync', + '--disable-translate', + '--hide-scrollbars', + '--mute-audio', + '--no-default-browser-check', + '--no-pings' + ] + + # 在Docker环境中添加额外参数 + if os.getenv('DOCKER_ENV'): + browser_args.extend([ + '--single-process', + '--disable-background-networking', + '--disable-client-side-phishing-detection', + '--disable-hang-monitor', + '--disable-popup-blocking', + '--disable-prompt-on-repost', + '--disable-web-resources', + '--metrics-recording-only', + '--safebrowsing-disable-auto-update', + '--enable-automation', + '--password-store=basic', + '--use-mock-keychain' + ]) + + browser = await playwright.chromium.launch( + headless=True, + args=browser_args + ) + + # 创建浏览器上下文 + context = await browser.new_context( + viewport={'width': 1920, 'height': 1080}, + user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36' + ) + + # 设置Cookie + cookies = [] + for cookie_pair in self.cookies_str.split('; '): + if '=' in cookie_pair: + name, value = cookie_pair.split('=', 1) + cookies.append({ + 'name': name.strip(), + 'value': value.strip(), + 'domain': '.goofish.com', + 'path': '/' + }) + + await context.add_cookies(cookies) + logger.debug(f"已设置 {len(cookies)} 个Cookie") + + # 创建页面 + page = await context.new_page() + + # 构造商品详情页面URL + item_url = f"https://www.goofish.com/item?id={item_id}" + logger.info(f"访问商品页面: {item_url}") + + # 访问页面 + await page.goto(item_url, wait_until='networkidle', timeout=30000) + + # 等待页面完全加载 + await asyncio.sleep(3) + + # 获取商品详情内容 + try: + # 等待目标元素出现 + await page.wait_for_selector('.desc--GaIUKUQY', timeout=10000) + + # 获取商品详情文本 + detail_element = await page.query_selector('.desc--GaIUKUQY') + if detail_element: + detail_text = await detail_element.inner_text() + logger.info(f"成功获取商品详情: {item_id}, 长度: {len(detail_text)}") + + # 清理资源 + await browser.close() + await playwright.stop() + + return detail_text.strip() + else: + logger.warning(f"未找到商品详情元素: {item_id}") + + except Exception as e: + logger.warning(f"获取商品详情元素失败: {item_id}, 错误: {self._safe_str(e)}") + + # 清理资源 + await browser.close() + await playwright.stop() + + return "" + + except Exception as e: + logger.error(f"浏览器获取商品详情异常: {item_id}, 错误: {self._safe_str(e)}") + return "" + + async def _fetch_item_detail_from_external_api(self, item_id: str) -> str: + """从外部API获取商品详情（备用方案）""" + try: + from config import config + auto_fetch_config = config.get('ITEM_DETAIL', {}).get('auto_fetch', {}) + # 从配置获取API地址和超时时间 api_base_url = auto_fetch_config.get('api_url', 'https://selfapi.zhinianboke.com/api/getItemDetail') timeout_seconds = auto_fetch_config.get('timeout', 10) @@ -891,7 +1070,6 @@ class XianyuLive: # 使用aiohttp发送异步请求 import aiohttp - import asyncio timeout = aiohttp.ClientTimeout(total=timeout_seconds) @@ -903,21 +1081,20 @@ class XianyuLive: # 检查返回状态 if result.get('status') == '200' and result.get('data'): item_detail = result['data'] - logger.info(f"成功获取商品详情: {item_id}, 长度: {len(item_detail)}") - logger.debug(f"商品详情内容: {item_detail[:200]}...") + logger.info(f"外部API成功获取商品详情: {item_id}, 长度: {len(item_detail)}") return item_detail else: - logger.warning(f"API返回状态异常: {result.get('status')}, message: {result.get('message')}") + logger.warning(f"外部API返回状态异常: {result.get('status')}, message: {result.get('message')}") return "" else: - logger.warning(f"API请求失败: HTTP {response.status}") + logger.warning(f"外部API请求失败: HTTP {response.status}") return "" except asyncio.TimeoutError: - logger.warning(f"获取商品详情超时: {item_id}") + logger.warning(f"外部API获取商品详情超时: {item_id}") return "" except Exception as e: - logger.error(f"获取商品详情异常: {item_id}, 错误: {self._safe_str(e)}") + logger.error(f"外部API获取商品详情异常: {item_id}, 错误: {self._safe_str(e)}") return "" async def save_items_list_to_db(self, items_list): diff --git a/reply_server.py b/reply_server.py index 5ebc527..1c9df63 100644 --- a/reply_server.py +++ b/reply_server.py @@ -444,51 +444,18 @@ async def admin_page(): return HTMLResponse(f.read()) -# 用户管理页面路由 -@app.get('/user_management.html', response_class=HTMLResponse) -async def user_management_page(): - page_path = os.path.join(static_dir, 'user_management.html') - if os.path.exists(page_path): - with open(page_path, 'r', encoding='utf-8') as f: - return HTMLResponse(f.read()) - else: - return HTMLResponse('

User management page not found

') - - -# 日志管理页面路由 -@app.get('/log_management.html', response_class=HTMLResponse) -async def log_management_page(): - page_path = os.path.join(static_dir, 'log_management.html') - if os.path.exists(page_path): - with open(page_path, 'r', encoding='utf-8') as f: - return HTMLResponse(f.read()) - else: - return HTMLResponse('

Log management page not found

') - - -# 数据管理页面路由 -@app.get('/data_management.html', response_class=HTMLResponse) -async def data_management_page(): - page_path = os.path.join(static_dir, 'data_management.html') - if os.path.exists(page_path): - with open(page_path, 'r', encoding='utf-8') as f: - return HTMLResponse(f.read()) - else: - return HTMLResponse('

Data management page not found

') -# 商品搜索页面路由 -@app.get('/item_search.html', response_class=HTMLResponse) -async def item_search_page(): - page_path = os.path.join(static_dir, 'item_search.html') - if os.path.exists(page_path): - with open(page_path, 'r', encoding='utf-8') as f: - return HTMLResponse(f.read()) - else: - return HTMLResponse('

Item search page not found

') + + + + + + + # 登录接口 @@ -2884,32 +2851,6 @@ async def search_multiple_pages( raise HTTPException(status_code=500, detail=f"多页商品搜索失败: {error_msg}") -@app.get("/items/detail/{item_id}") -async def get_public_item_detail( - item_id: str, - current_user: Optional[Dict[str, Any]] = Depends(get_current_user_optional) -): - """获取公开商品详情（通过外部API）""" - try: - from utils.item_search import get_item_detail_from_api - - # 从外部API获取商品详情 - detail = await get_item_detail_from_api(item_id) - - if detail: - return { - "success": True, - "data": detail - } - else: - raise HTTPException(status_code=404, detail="商品详情获取失败") - - except HTTPException: - raise - except Exception as e: - logger.error(f"获取商品详情失败: {str(e)}") - raise HTTPException(status_code=500, detail=f"获取商品详情失败: {str(e)}") - @app.get("/items/cookie/{cookie_id}") def get_items_by_cookie(cookie_id: str, current_user: Dict[str, Any] = Depends(get_current_user)): diff --git a/static/data_management.html b/static/data_management.html deleted file mode 100644 index a931dcf..0000000 --- a/static/data_management.html +++ /dev/null @@ -1,556 +0,0 @@ - - - - - - 数据管理 - 闲鱼自动回复系统 - - - - - - - - - -
-
-

- 数据管理 -

-

查看和管理数据库中的所有表数据

-
-
- -
- -
-
-
- 数据表选择 -
-
-
-
-
- - -
-
- -
-
-
-
- 条记录 -
-
-
-
- -
- -
-
-
-
-
- - - - - -
-
-
- 数据内容 -
-
- -
-
-
- -
- -

请选择要查看的数据表

-
- - -
-
-
- - - - - - - - - - - diff --git a/static/item_search.html b/static/item_search.html deleted file mode 100644 index d137867..0000000 --- a/static/item_search.html +++ /dev/null @@ -1,954 +0,0 @@ - - - - - - 商品搜索 - 闲鱼管理系统 - - - - - - - - -
- -
- -

正在验证登录状态...

-
- - - - - - -
- - - - - diff --git a/static/log_management.html b/static/log_management.html deleted file mode 100644 index 7060b15..0000000 --- a/static/log_management.html +++ /dev/null @@ -1,457 +0,0 @@ - - - - - - 日志管理 - 闲鱼自动回复系统 - - - - - - - - - -
-
-

- 日志管理 -

-

查看和监控系统运行日志

-
-
- -
- -
-
-
- 日志控制 -
-
-
-
-
- - -
-
- -
- - 全部 - - - INFO - - - WARNING - - - ERROR - - - DEBUG - -
-
-
- -
- - -
-
-
- -
- -
-
-
-
-
- - -
-
-
- 日志信息 -
- 最后更新: - -
-
-
-
- 日志文件: - - -
-
- 显示行数: - - -
-
- 当前过滤: - 全部 -
-
-
-
- - -
-
-
- 日志内容 -
-
- - -
-
-
-
- -

正在加载日志...

-
- - -
-
-
- - - - - diff --git a/static/user_management.html b/static/user_management.html deleted file mode 100644 index 4be17a8..0000000 --- a/static/user_management.html +++ /dev/null @@ -1,400 +0,0 @@ - - - - - - 用户管理 - 闲鱼自动回复系统 - - - - - - - - - -
-
-

- 用户管理 -

-

管理系统中的所有用户账号

-
-
- -
- -
-
-
-
-

-

-

总用户数

-
-
-
-
-
-
-

-

-

总Cookie数

-
-
-
-
-
-
-

-

-

总卡券数

-
-
-
-
-
-
-

运行中

-

系统状态

-
-
-
-
- - -
-
-
- 用户列表 -
- -
-
-
- -

正在加载用户信息...

-
- - -
-
-
- - - - - - - - diff --git a/utils/item_search.py b/utils/item_search.py index d9cb54c..dc1e622 100644 --- a/utils/item_search.py +++ b/utils/item_search.py @@ -57,23 +57,6 @@ class XianyuSearcher: return default return data - async def test_browser_launch(self): - """测试浏览器是否能正常启动""" - try: - if not PLAYWRIGHT_AVAILABLE: - return False, "Playwright 未安装" - - playwright = await async_playwright().start() - browser = await playwright.chromium.launch(headless=True) - context = await browser.new_context() - page = await context.new_page() - await page.goto("https://www.baidu.com") - await asyncio.sleep(2) - await browser.close() - return True, "浏览器测试成功" - except Exception as e: - return False, f"浏览器测试失败: {str(e)}" - async def get_first_valid_cookie(self): """获取第一个有效的cookie""" try: @@ -889,123 +872,6 @@ class XianyuSearcher: } - - - async def _parse_item_old(self, item_data: Dict[str, Any]) -> Optional[Dict[str, Any]]: - """ - 解析单个商品数据 - - Args: - item_data: 商品原始数据 - - Returns: - 解析后的商品信息 - """ - try: - # 基本信息 - 适配多种可能的字段名 - item_id = (item_data.get('id') or - item_data.get('itemId') or - item_data.get('item_id') or - item_data.get('spuId') or '') - - title = (item_data.get('title') or - item_data.get('name') or - item_data.get('itemTitle') or - item_data.get('subject') or '') - - # 价格处理 - price_raw = (item_data.get('price') or - item_data.get('priceText') or - item_data.get('currentPrice') or - item_data.get('realPrice') or '') - - # 清理价格格式 - if isinstance(price_raw, (int, float)): - price = str(price_raw) - elif isinstance(price_raw, str): - price = price_raw.replace('¥', '').replace('元', '').strip() - else: - price = '价格面议' - - # 卖家信息 - seller_info = (item_data.get('seller') or - item_data.get('user') or - item_data.get('owner') or {}) - - seller_name = '' - if seller_info: - seller_name = (seller_info.get('nick') or - seller_info.get('nickname') or - seller_info.get('name') or - seller_info.get('userName') or '匿名用户') - - # 商品链接 - if item_id: - item_url = f"https://www.goofish.com/item?id={item_id}" - else: - item_url = item_data.get('url', item_data.get('link', '')) - - # 发布时间 - publish_time = (item_data.get('publishTime') or - item_data.get('createTime') or - item_data.get('gmtCreate') or - item_data.get('time') or '') - - # 格式化时间 - if publish_time and isinstance(publish_time, (int, float)): - import datetime - publish_time = datetime.datetime.fromtimestamp(publish_time / 1000).strftime('%Y-%m-%d') - - # 商品标签 - tags = item_data.get('tags', item_data.get('labels', [])) - tag_names = [] - if isinstance(tags, list): - for tag in tags: - if isinstance(tag, dict): - tag_name = (tag.get('name') or - tag.get('text') or - tag.get('label') or '') - if tag_name: - tag_names.append(tag_name) - elif isinstance(tag, str): - tag_names.append(tag) - - # 图片 - images = (item_data.get('images') or - item_data.get('pics') or - item_data.get('pictures') or - item_data.get('imgList') or []) - - main_image = '' - if images and len(images) > 0: - if isinstance(images[0], str): - main_image = images[0] - elif isinstance(images[0], dict): - main_image = (images[0].get('url') or - images[0].get('src') or - images[0].get('image') or '') - - # 如果没有图片，使用默认占位图 - if not main_image: - main_image = f'https://via.placeholder.com/200x200?text={title[:10] if title else "商品"}...' - - return { - 'item_id': item_id, - 'title': title, - 'price': price, - 'seller_name': seller_name, - 'item_url': item_url, - 'publish_time': publish_time, - 'tags': tag_names, - 'main_image': main_image, - 'raw_data': item_data # 保留原始数据以备后用 - } - - except Exception as e: - logger.warning(f"解析商品数据失败: {str(e)}") - return None - - # 搜索器工具函数 async def search_xianyu_items(keyword: str, page: int = 1, page_size: int = 20) -> Dict[str, Any]: @@ -1130,50 +996,3 @@ async def search_multiple_pages_xianyu(keyword: str, total_pages: int = 1) -> Di - -async def get_item_detail_from_api(item_id: str) -> Optional[str]: - """ - 从外部API获取商品详情 - - Args: - item_id: 商品ID - - Returns: - 商品详情文本，获取失败返回None - """ - try: - # 使用默认的API配置 - api_base_url = 'https://selfapi.zhinianboke.com/api/getItemDetail' - timeout_seconds = 10 - - api_url = f"{api_base_url}/{item_id}" - - logger.info(f"正在从外部API获取商品详情: {item_id}") - - # 使用简单的HTTP请求 - import aiohttp - timeout = aiohttp.ClientTimeout(total=timeout_seconds) - - async with aiohttp.ClientSession(timeout=timeout) as session: - async with session.get(api_url) as response: - if response.status == 200: - result = await response.json() - - # 检查返回状态 - if result.get('status') == '200' and result.get('data'): - item_detail = result['data'] - logger.info(f"成功获取商品详情: {item_id}, 长度: {len(item_detail)}") - return item_detail - else: - logger.warning(f"API返回状态异常: {result.get('status')}, message: {result.get('message')}") - return None - else: - logger.warning(f"API请求失败: HTTP {response.status}") - return None - - except asyncio.TimeoutError: - logger.warning(f"获取商品详情超时: {item_id}") - return None - except Exception as e: - logger.error(f"获取商品详情异常: {item_id}, 错误: {str(e)}") - return None