zman27/ai_giga_tcg
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kind of a mess lol but file caching and front end

Browse Source
This commit is contained in:
zman
2025-04-17 13:28:49 -04:00
parent 21408af48c
commit 8f35cedb4a
45 changed files with 1435 additions and 1316 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
app/services/external_api/base_external_service.py
View File

@ -2,6 +2,11 @@ from typing import Any, Dict, Optional, Union
import aiohttp
import logging
import json
import csv
import io
from app.services.service_manager import ServiceManager
from app.schemas.file import FileInDB
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
@ -10,6 +15,8 @@ class BaseExternalService:
self.base_url = base_url
self.api_key = api_key
self.session = None
self.service_manager = ServiceManager()
self._services = {}
async def _get_session(self) -> aiohttp.ClientSession:
if self.session is None or self.session.closed:
@ -73,4 +80,36 @@ class BaseExternalService:
if self.session and not self.session.closed:
await self.session.close()
self.session = None
logger.info(f"Closed session for {self.__class__.__name__}")
logger.info(f"Closed session for {self.__class__.__name__}")
def get_service(self, name: str) -> Any:
"""Get a service by name with lazy loading"""
if name not in self._services:
self._services[name] = self.service_manager.get_service(name)
return self._services[name]
@property
def file_service(self):
"""Convenience property for file service"""
return self.get_service('file')
async def save_file(self, db: Session, file_data: Union[bytes, list[dict]], file_name: str, subdir: str, file_type: Optional[str] = None) -> FileInDB:
"""Save a file using the FileService"""
if isinstance(file_data, list):
# Convert list of dictionaries to CSV bytes
output = io.StringIO()
writer = csv.DictWriter(output, fieldnames=file_data[0].keys())
writer.writeheader()
writer.writerows(file_data)
file_data = output.getvalue().encode('utf-8')
file_type = file_type or 'text/csv'
# Use FileService to save the file
file_service = self.get_service('file')
return await file_service.save_file(
db=db,
file_data=file_data,
filename=file_name,
subdir=subdir,
file_type=file_type
)

66
app/services/external_api/mtgjson/mtgjson_service.py
View File

@ -11,9 +11,12 @@ from datetime import datetime
from app.models.mtgjson_card import MTGJSONCard
from app.models.mtgjson_sku import MTGJSONSKU
from app.db.database import get_db, transaction
from app.services.external_api.base_external_service import BaseExternalService
from app.schemas.file import FileInDB
class MTGJSONService:
class MTGJSONService(BaseExternalService):
def __init__(self, cache_dir: str = "app/data/cache/mtgjson", batch_size: int = 1000):
super().__init__(base_url="https://mtgjson.com/api/v5/")
self.cache_dir = cache_dir
self.identifiers_dir = os.path.join(cache_dir, "identifiers")
self.skus_dir = os.path.join(cache_dir, "skus")
@ -38,27 +41,22 @@ class MTGJSONService:
"""Print progress message with flush"""
print(message, end=end, flush=True)
async def _download_file(self, url: str, output_path: str) -> None:
"""Download a file from the given URL to the specified path using streaming"""
async def _download_file(self, db: Session, url: str, filename: str, subdir: str) -> FileInDB:
"""Download a file from the given URL and save it using FileService"""
print(f"Downloading {url}...")
start_time = time.time()
total_size = 0
async with aiohttp.ClientSession() as session:
async with session.get(url) as response:
if response.status == 200:
total_size = int(response.headers.get('content-length', 0))
with open(output_path, 'wb') as f:
downloaded = 0
async for chunk in response.content.iter_chunked(8192):
f.write(chunk)
downloaded += len(chunk)
if total_size > 0:
percent = (downloaded / total_size) * 100
elapsed = time.time() - start_time
speed = downloaded / elapsed / 1024 / 1024 # MB/s
print(f"\rDownloading: {percent:.1f}% ({downloaded/1024/1024:.1f}MB/{total_size/1024/1024:.1f}MB) at {speed:.1f}MB/s", end="")
print("\nDownload complete!")
file_data = await response.read()
return await self.save_file(
db=db,
file_data=file_data,
file_name=filename,
subdir=f"mtgjson/{subdir}",
file_type=response.headers.get('content-type', 'application/octet-stream')
)
else:
raise Exception(f"Failed to download file from {url}. Status: {response.status}")
@ -153,14 +151,16 @@ class MTGJSONService:
self._print_progress("Starting MTGJSON identifiers processing...")
start_time = time.time()
zip_path = os.path.join(self.identifiers_dir, "AllIdentifiers.json.zip")
await self._download_file(
"https://mtgjson.com/api/v5/AllIdentifiers.json.zip",
zip_path
# Download the file using FileService
file_record = await self._download_file(
db=db,
url="https://mtgjson.com/api/v5/AllIdentifiers.json.zip",
filename="AllIdentifiers.json.zip",
subdir="identifiers"
)
self._print_progress("Unzipping file...")
json_path = await self._unzip_file(zip_path, self.identifiers_dir)
# Get the file path from the database record
zip_path = file_record.path
cards_processed = 0
current_batch = []
@ -169,7 +169,7 @@ class MTGJSONService:
self._print_progress("Processing cards...")
try:
for item in self._stream_json_file(json_path):
for item in self._stream_json_file(zip_path):
if item["type"] == "meta":
self._print_progress(f"Processing MTGJSON data version {item['data'].get('version')} from {item['data'].get('date')}")
continue
@ -239,14 +239,16 @@ class MTGJSONService:
self._print_progress("Starting MTGJSON SKUs processing...")
start_time = time.time()
zip_path = os.path.join(self.skus_dir, "TcgplayerSkus.json.zip")
await self._download_file(
"https://mtgjson.com/api/v5/TcgplayerSkus.json.zip",
zip_path
# Download the file using FileService
file_record = await self._download_file(
db=db,
url="https://mtgjson.com/api/v5/TcgplayerSkus.json.zip",
filename="TcgplayerSkus.json.zip",
subdir="skus"
)
self._print_progress("Unzipping file...")
json_path = await self._unzip_file(zip_path, self.skus_dir)
# Get the file path from the database record
zip_path = file_record.path
skus_processed = 0
current_batch = []
@ -255,7 +257,7 @@ class MTGJSONService:
self._print_progress("Processing SKUs...")
try:
for item in self._stream_json_file(json_path):
for item in self._stream_json_file(zip_path):
if item["type"] == "meta":
self._print_progress(f"Processing MTGJSON SKUs version {item['data'].get('version')} from {item['data'].get('date')}")
continue

29
app/services/external_api/tcgcsv/tcgcsv_service.py
View File

@ -10,6 +10,7 @@ from app.db.database import get_db, transaction
from sqlalchemy.orm import Session
import py7zr
import os
from app.schemas.file import FileInDB
class TCGCSVService(BaseExternalService):
def __init__(self):
@ -37,32 +38,28 @@ class TCGCSVService(BaseExternalService):
endpoint = "tcgplayer/categories"
return await self._make_request("GET", endpoint)
async def get_archived_prices_for_date(self, date_str: str):
async def get_archived_prices_for_date(self, db: Session, date_str: str) -> str:
"""Fetch archived prices from TCGCSV API"""
# Check if the date directory already exists
extract_path = f"app/data/cache/tcgcsv/prices/{date_str}"
if os.path.exists(extract_path):
print(f"Prices for date {date_str} already exist, skipping download")
return date_str
# Download the archive file
endpoint = f"archive/tcgplayer/prices-{date_str}.ppmd.7z"
response = await self._make_request("GET", endpoint, binary=True)
# Save the archive file
archive_path = f"app/data/cache/tcgcsv/prices/zip/prices-{date_str}.ppmd.7z"
os.makedirs(os.path.dirname(archive_path), exist_ok=True)
with open(archive_path, "wb") as f:
f.write(response)
# Save the archive file using FileService
file_record = await self.save_file(
db=db,
file_data=response,
file_name=f"prices-{date_str}.ppmd.7z",
subdir=f"tcgcsv/prices/zip",
file_type="application/x-7z-compressed"
)
# Extract the 7z file
with py7zr.SevenZipFile(archive_path, 'r') as archive:
with py7zr.SevenZipFile(file_record.path, 'r') as archive:
# Extract to a directory named after the date
extract_path = f"app/data/cache/tcgcsv/prices/{date_str}"
os.makedirs(extract_path, exist_ok=True)
archive.extractall(path=extract_path)
# The extracted files will be in a directory structure like:
# {date_str}/{game_id}/{group_id}/prices
return date_str
async def get_archived_prices_for_date_range(self, start_date: str, end_date: str):

69
app/services/external_api/tcgplayer/order_management_service.py
View File

@ -1,4 +1,4 @@
from typing import Any, Dict, Optional, Union
from typing import Any, Dict, Optional, Union, Literal
import logging
from app.services.external_api.tcgplayer.base_tcgplayer_service import BaseTCGPlayerService
from app.schemas.tcgplayer import (
@ -21,7 +21,8 @@ from app.db.database import transaction
import os
import csv
import io
from app.schemas.file import FileInDB
from datetime import datetime
logger = logging.getLogger(__name__)
class OrderManagementService(BaseTCGPlayerService):
@ -87,7 +88,12 @@ class OrderManagementService(BaseTCGPlayerService):
response = await self._make_request("GET", f"/{order_id}{self.API_VERSION}")
return response
async def get_packing_slip(self, order_ids: list[str]):
async def get_or_create_packing_slip(self, db: Session, order_ids: list[str]) -> FileInDB:
# check if the file already exists
file_service = self.get_service('file')
file = await file_service.get_file_by_metadata(db, "order_ids", order_ids, "packing_slip", "application/pdf")
if file:
return file
payload = {
"sortingType": "byRelease",
"format": "default",
@ -95,40 +101,53 @@ class OrderManagementService(BaseTCGPlayerService):
"orderNumbers": order_ids
}
response = await self._make_request("POST", self.packing_slip_endpoint, data=payload, headers=self._get_headers("POST", "application/json"), auth_required=True, download_file=True)
return response
return await file_service.save_file(
db=db,
file_data=response,
filename=f"packing_slip_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.pdf",
subdir='tcgplayer/packing_slips/pdf',
file_type='packing_slip',
content_type='application/pdf',
metadata={"order_ids": order_ids}
)
async def get_pull_sheet(self, order_ids: list[str]):
async def get_pull_sheet(self, db: Session, order_ids: list[str]) -> FileInDB:
payload = {
"orderNumbers": order_ids,
"timezoneOffset": -4
}
response = await self._make_request("POST", self.pull_sheet_endpoint, data=payload, headers=self._get_headers("POST", "application/json"), auth_required=True, download_file=True)
return response
# get file service
file_service = self.get_service('file')
# save file
return await file_service.save_file(
db=db,
file_data=response,
filename=f"pull_sheet_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.csv",
subdir='tcgplayer/pull_sheets/csv',
file_type='pull_sheet',
content_type='text/csv',
metadata={"order_ids": order_ids}
)
async def get_shipping_csv(self, order_ids: list[str]):
async def get_shipping_csv(self, db: Session, order_ids: list[str]) -> FileInDB:
payload = {
"orderNumbers": order_ids,
"timezoneOffset": -4
}
response = await self._make_request("POST", self.shipping_endpoint, data=payload, headers=self._get_headers("POST", "application/json"), auth_required=True, download_file=True)
return response
async def save_file(self, file_data: Union[bytes, list[dict]], file_name: str) -> str:
if not os.path.exists("app/data/cache/tcgplayer/orders"):
os.makedirs("app/data/cache/tcgplayer/orders")
file_path = f"app/data/cache/tcgplayer/orders/{file_name}"
if isinstance(file_data, list):
# Convert list of dictionaries to CSV bytes
output = io.StringIO()
writer = csv.DictWriter(output, fieldnames=file_data[0].keys())
writer.writeheader()
writer.writerows(file_data)
file_data = output.getvalue().encode('utf-8')
with open(file_path, "wb") as f:
f.write(file_data)
return file_path
# get file service
file_service = self.get_service('file')
# save file
return await file_service.save_file(
db=db,
file_data=response,
filename=f"shipping_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.csv",
subdir='tcgplayer/shipping/csv',
file_type='shipping_csv',
content_type='text/csv',
metadata={"order_ids": order_ids}
)
async def save_order_to_db(self, order: dict, db: Session):
# Parse API response using our API schema

16
app/services/external_api/tcgplayer/tcgplayer_inventory_service.py
View File

@ -1,21 +1,33 @@
from typing import Dict, List, Optional
from app.services.external_api.tcgplayer.base_tcgplayer_service import BaseTCGPlayerService
from sqlalchemy.orm import Session
from app.schemas.file import FileInDB
class TCGPlayerInventoryService(BaseTCGPlayerService):
def __init__(self):
super().__init__()
async def get_tcgplayer_export(self, export_type: str):
async def get_tcgplayer_export(self, db: Session, export_type: str) -> FileInDB:
"""
Get a TCGPlayer Staged Inventory Export, Live Inventory Export, or Pricing Export
"""
if export_type == "staged":
endpoint = self.staged_inventory_endpoint
file_type = "text/csv"
elif export_type == "live":
endpoint = self.live_inventory_endpoint
file_type = "text/csv"
elif export_type == "pricing":
endpoint = self.pricing_export_endpoint
file_type = "text/csv"
else:
raise ValueError(f"Invalid export type: {export_type}, must be 'staged', 'live', or 'pricing'")
file_bytes = await self._make_request("GET", endpoint, download_file=True)
return file_bytes
return await self.save_file(
db=db,
file_data=file_bytes,
file_name=f"tcgplayer_{export_type}_export.csv",
subdir="tcgplayer/inventory",
file_type=file_type
)