191 lines
8.5 KiB
Python
191 lines
8.5 KiB
Python
from typing import Dict, Optional
|
|
from app.services.util._docker import DockerUtil
|
|
from enum import Enum
|
|
import browser_cookie3
|
|
import os
|
|
import json
|
|
import requests
|
|
import time
|
|
from datetime import datetime
|
|
import logging
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class Browser(Enum):
|
|
"""Supported browser types for cookie extraction"""
|
|
BRAVE = "brave"
|
|
CHROME = "chrome"
|
|
FIREFOX = "firefox"
|
|
|
|
class Method(Enum):
|
|
"""Supported HTTP methods"""
|
|
GET = "GET"
|
|
POST = "POST"
|
|
|
|
class TCGPlayerEndpoints(Enum):
|
|
"""Supported TCGPlayer API endpoints"""
|
|
ORDERS = "https://order-management-api.tcgplayer.com/orders"
|
|
|
|
class Headers:
|
|
ACCEPT = 'application/json, text/plain, */*'
|
|
ACCEPT_ENCODING = 'gzip, deflate, br, zstd'
|
|
ACCEPT_LANGUAGE = 'en-US,en;q=0.8'
|
|
PRIORITY = 'u=1, i'
|
|
SECCHUA = '"Not(A:Brand";v="99", "Brave";v="133", "Chromium";v="133"'
|
|
SECCHUA_MOBILE = '?0'
|
|
SECCHUA_PLATFORM = '"macOS"'
|
|
SEC_FETCH_DEST = 'empty'
|
|
SEC_FETCH_MODE = 'cors'
|
|
SEC_FETCH_SITE = 'same-site'
|
|
SEC_GPC = '1'
|
|
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36'
|
|
SELLER_ORIGIN = 'https://sellerportal.tcgplayer.com'
|
|
SELLER_REFERER = 'https://sellerportal.tcgplayer.com/'
|
|
|
|
class RequestHeaders:
|
|
BASE_HEADERS = {
|
|
'accept': Headers.ACCEPT,
|
|
'accept-encoding': Headers.ACCEPT_ENCODING,
|
|
'accept-language': Headers.ACCEPT_LANGUAGE,
|
|
'priority': Headers.PRIORITY,
|
|
'sec-ch-ua': Headers.SECCHUA,
|
|
'sec-ch-ua-mobile': Headers.SECCHUA_MOBILE,
|
|
'sec-ch-ua-platform': Headers.SECCHUA_PLATFORM,
|
|
'sec-fetch-dest': Headers.SEC_FETCH_DEST,
|
|
'sec-fetch-mode': Headers.SEC_FETCH_MODE,
|
|
'sec-fetch-site': Headers.SEC_FETCH_SITE,
|
|
'sec-gpc': Headers.SEC_GPC,
|
|
'user-agent': Headers.USER_AGENT
|
|
}
|
|
SELLER_HEADERS = {
|
|
'origin': Headers.SELLER_ORIGIN,
|
|
'referer': Headers.SELLER_REFERER
|
|
}
|
|
|
|
class URLHeaders:
|
|
# combine base and seller headers
|
|
ORDER_HEADERS = {**RequestHeaders.BASE_HEADERS, **RequestHeaders.SELLER_HEADERS}
|
|
|
|
class RequestsUtil:
|
|
def __init__(self, browser_type: Browser = Browser.BRAVE):
|
|
self.browser_type = browser_type
|
|
self.docker_util = DockerUtil()
|
|
self.previous_request_time = datetime.now()
|
|
|
|
def get_session(self, cookies: Dict = None) -> requests.Session:
|
|
"""Create a session with the specified cookies"""
|
|
session = requests.Session()
|
|
if cookies:
|
|
session.cookies.update(cookies)
|
|
return session
|
|
|
|
def bare_request(self, url: str, method: str, cookies: dict, data=None) -> requests.Response:
|
|
"""Send a request without any additional processing"""
|
|
try:
|
|
response = requests.request(method, url, cookies=cookies, data=data)
|
|
response.raise_for_status()
|
|
return response
|
|
except requests.RequestException as e:
|
|
logger.error(f"Request failed: {str(e)}")
|
|
return None
|
|
|
|
def get_tcgplayer_cookies_from_file(self) -> Dict:
|
|
# check if cookies file exists
|
|
if not os.path.exists('cookies/tcg_cookies.json'):
|
|
raise ValueError("Cookies file not found")
|
|
with open('cookies/tcg_cookies.json', 'r') as f:
|
|
logger.debug("Loading cookies from file")
|
|
cookies = json.load(f)
|
|
return cookies
|
|
|
|
def get_tcgplayer_browser_cookies(self) -> Optional[Dict]:
|
|
"""Retrieve cookies from the specified browser"""
|
|
try:
|
|
cookie_getter = getattr(browser_cookie3, self.browser_type.value, None)
|
|
if not cookie_getter:
|
|
raise ValueError(f"Unsupported browser type: {self.browser_type.value}")
|
|
return cookie_getter()
|
|
except Exception as e:
|
|
logger.error(f"Failed to get browser cookies: {str(e)}")
|
|
return None
|
|
|
|
def rate_limit(self, time_between_requests: int = 10):
|
|
"""Rate limit requests by waiting for a specified time between requests"""
|
|
time_diff = (datetime.now() - self.previous_request_time).total_seconds()
|
|
if time_diff < time_between_requests:
|
|
# logger.info(f"Waiting {time_between_requests - time_diff} seconds before next request...")
|
|
time.sleep(time_between_requests - time_diff)
|
|
|
|
def send_request(self, url: str, method: str, cookies: dict, data=None) -> requests.Response:
|
|
"""Send a request with the specified cookies"""
|
|
|
|
headers = self.set_headers(url)
|
|
if not headers:
|
|
raise ValueError("Headers not set")
|
|
|
|
try:
|
|
self.rate_limit()
|
|
response = requests.request(method, url, headers=headers, cookies=cookies, data=data)
|
|
response.raise_for_status()
|
|
self.previous_request_time = datetime.now()
|
|
|
|
return response
|
|
|
|
except requests.RequestException as e:
|
|
logger.error(f"Request failed: {str(e)}")
|
|
return None
|
|
|
|
def set_headers(self, url: str):
|
|
# use tcgplayerendpoints enum to set headers where url partially matches enum value
|
|
for endpoint in TCGPlayerEndpoints:
|
|
if endpoint.value in url:
|
|
return URLHeaders.ORDER_HEADERS
|
|
else:
|
|
raise ValueError(f"Endpoint not found in TCGPlayerEndpoints: {url}")
|
|
|
|
def old_set_headers(self, method: str) -> Dict:
|
|
base_headers = {
|
|
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
|
|
'accept-language': 'en-US,en;q=0.8',
|
|
'priority': 'u=0, i',
|
|
'referer': 'https://store.tcgplayer.com/admin/pricing',
|
|
'sec-ch-ua': '"Not A(Brand";v="8", "Chromium";v="132", "Brave";v="132"',
|
|
'sec-ch-ua-mobile': '?0',
|
|
'sec-ch-ua-platform': '"macOS"',
|
|
'sec-fetch-dest': 'document',
|
|
'sec-fetch-mode': 'navigate',
|
|
'sec-fetch-site': 'same-origin',
|
|
'sec-fetch-user': '?1',
|
|
'sec-gpc': '1',
|
|
'upgrade-insecure-requests': '1',
|
|
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36'
|
|
}
|
|
|
|
if method == 'POST':
|
|
post_headers = {
|
|
'cache-control': 'max-age=0',
|
|
'content-type': 'application/x-www-form-urlencoded',
|
|
'origin': 'https://store.tcgplayer.com'
|
|
}
|
|
base_headers.update(post_headers)
|
|
|
|
return base_headers
|
|
|
|
|
|
"""
|
|
curl 'https://order-management-api.tcgplayer.com/orders/E576ED4C-38871F-B0277?api-version=2.0' \
|
|
-H 'accept: application/json, text/plain, */*' \
|
|
-H 'accept-language: en-US,en;q=0.8' \
|
|
-b 'tcgpartner=PK=TRADECARDS&M=1; valid=set=true; product-display-settings=sort=price+shipping&size=10; TCG_Data=M=1&SearchGameNameID=magic; tcg-uuid=ab16b5f8-dd66-446d-b217-d394328a5cf1; setting=CD=US&M=1; tracking-preferences={%22version%22:1%2C%22destinations%22:{%22Actions%20Amplitude%22:false%2C%22AdWords%22:false%2C%22Google%20AdWords%20New%22:false%2C%22Google%20Enhanced%20Conversions%22:false%2C%22Google%20Tag%20Manager%22:false%2C%22Impact%20Partnership%20Cloud%22:false%2C%22Optimizely%22:false}%2C%22custom%22:{%22advertising%22:false%2C%22functional%22:false%2C%22marketingAndAnalytics%22:false}}; OAuthLoginSessionId=63b1a89d-1ac2-43f7-8e79-55a9ca5e761d; TCGAuthTicket_Production=F453BFF2FA3FAA3D1ACA23F319314F8273713DECEB06C69BB7C208A77C81559B46E1AA22A7E70FABC8D7F681A423C86870FAE318B76048CDE7BF6D73D220631B899BEBA86C422E1EBBF2ACD1921E0846F708AFE203C844031364E13B047465E7B41CB6460E4F4AAB278B614445B93E722E976688; BuyerRevalidationKey=; ASP.NET_SessionId=oouwzrh3jkhdrmaioooqhr4k; TCG_VisitorKey=431efcca-2d5b-404d-a04f-3ae979696051; __RequestVerificationToken_L2FkbWlu0=Lw1sfWh823UeJ7zRux0b1ZTI4Vg4i_dFt97a55aQpf-qBURVuwWDCJyuCxSwgLNLe9nPlfDSc1AMV5nyqhY4Q4jurxs1; fileDownloadToken=1740145585435; spDisabledUIFeatures=' \
|
|
-H 'origin: https://sellerportal.tcgplayer.com' \
|
|
-H 'priority: u=1, i' \
|
|
-H 'referer: https://sellerportal.tcgplayer.com/' \
|
|
-H 'sec-ch-ua: "Not(A:Brand";v="99", "Brave";v="133", "Chromium";v="133"' \
|
|
-H 'sec-ch-ua-mobile: ?0' \
|
|
-H 'sec-ch-ua-platform: "macOS"' \
|
|
-H 'sec-fetch-dest: empty' \
|
|
-H 'sec-fetch-mode: cors' \
|
|
-H 'sec-fetch-site: same-site' \
|
|
-H 'sec-gpc: 1' \
|
|
-H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36'
|
|
""" |