From cddb0ab0dda11f0c484c71561c3c728cabdce003 Mon Sep 17 00:00:00 2001 From: xingc Date: Mon, 18 Aug 2025 01:59:01 +0800 Subject: [PATCH] =?UTF-8?q?fix(card):=20=E4=BF=AE=E5=A4=8D`cgc`=E6=90=9C?= =?UTF-8?q?=E5=8D=A15s=E7=9B=BE=E9=AA=8C=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 7 +++++-- config.py | 3 +++ core/routers/card/schema.py | 4 ++++ core/routers/card/scraper.py | 31 ++++++++++++++++++++----------- 4 files changed, 32 insertions(+), 13 deletions(-) diff --git a/.env.example b/.env.example index 24736ca..2e19cba 100644 --- a/.env.example +++ b/.env.example @@ -17,7 +17,7 @@ REDIS_PORT=6379 REDIS_DB=0 # 文件上传配置 -ASSETS_BASE_URL =http://127.0.0.1:8001 +ASSETS_BASE_URL=http://127.0.0.1:8001 # 代理 OVERSEAS_PROXY=http://127.0.0.1:7890 @@ -31,4 +31,7 @@ PAY_NOTIFY_URL= # 会员配置 USER_MAX_CREATED_CARD_NUM=18 -UPLOAD_REWARD_POINTS_GIFT_ID=-1 \ No newline at end of file +UPLOAD_REWARD_POINTS_GIFT_ID=-1 + +# cgc通信密钥 +CGC_HTTP_PASSWORD= \ No newline at end of file diff --git a/config.py b/config.py index 1b56e72..18d3d0f 100644 --- a/config.py +++ b/config.py @@ -70,6 +70,9 @@ class Settings(BaseSettings): USER_MAX_CREATED_CARD_NUM: int = 18 UPLOAD_REWARD_POINTS_GIFT_ID: int = -1 + # cgc通信密钥 + CGC_HTTP_PASSWORD: str = '' + model_config = SettingsConfigDict(env_file=f'.env.{ENVIRONMENT}', env_file_encoding='utf-8') diff --git a/core/routers/card/schema.py b/core/routers/card/schema.py index f4d22f0..bbf8ec4 100644 --- a/core/routers/card/schema.py +++ b/core/routers/card/schema.py @@ -58,6 +58,10 @@ class UpdateCard(BaseModel): display_price: condecimal(max_digits=10, decimal_places=2) +class PlatformSessionsIn(BaseModel): + data: dict + + class SearchCardOut(Card): pass diff --git a/core/routers/card/scraper.py b/core/routers/card/scraper.py index 38c3971..833a9ac 100644 --- a/core/routers/card/scraper.py +++ b/core/routers/card/scraper.py @@ -15,9 +15,9 @@ from parsel import Selector from simple_spider_tool import jsonpath, regx_match, format_json from config import settings +from core.db.engine import get_cache from core.base.exceptions import SearchCardRetry, SearchCardRetryFail, SearchCardHandlerFail -from core.utils.helper import async_retry -from core.utils.helper import decrypt_aes_cbc +from core.utils.helper import async_retry, decrypt_aes_cbc, generate_keyname logger = logging.getLogger(__name__) @@ -49,14 +49,17 @@ class SiteApi: data=None, json=None, use_abroad_proxy: bool = False, + use_random_ua: bool = True, retry_num: int = 3, **kwargs ): headers = { **self.headers, - 'User-Agent': self.user_agent.random, **(headers or {}), } + if use_random_ua: + headers['User-Agent'] = self.user_agent.random + # 境外代理 if use_abroad_proxy: kwargs['proxies'] = { @@ -69,7 +72,11 @@ class SiteApi: r = await self.session.request( method, url, headers=headers, params=params, data=data, json=json, **kwargs ) - logger.debug(f'耗时:{url} {r.elapsed}') + + chck_string = 'Just a moment...' + if chck_string in r.text: + raise SearchCardRetry(chck_string) + return r except CurlError as e: @@ -95,9 +102,6 @@ class SiteApi: 'Referer': f'https://www.psacard.com/cert/{cert_number}', } r = await self.get(url, headers=headers, impersonate="chrome110") - chck_string = 'Just a moment...' - if chck_string in r.text: - raise SearchCardRetry(chck_string) return r async def psa_images(self, cert_number: str | int, images_id: str | int): @@ -126,10 +130,15 @@ class SiteApi: async def cgc(self, cert_number: str | int): """CGC""" url = f'https://www.cgccards.com/certlookup/{cert_number}/' - headers = { - 'referer': 'https://www.cgccards.com/' - } - return await self.get(url, headers=headers, impersonate="chrome110") + redis_conn = await get_cache() + async with redis_conn: + cache_session_str = await redis_conn.get(generate_keyname('session:cgc')) + if isinstance(cache_session_str, str): + headers = ujson.decode(cache_session_str) + else: + headers = {} + + return await self.get(url, headers=headers, use_random_ua=False, use_abroad_proxy=True) async def gbtc(self, cert_number: str | int): """北京公博"""