133 lines
6.4 KiB
Python
133 lines
6.4 KiB
Python
from app.services.base_service import BaseService
|
|
from sqlalchemy.orm import Session
|
|
from app.db.database import transaction
|
|
from app.schemas.file import FileInDB
|
|
from app.models.tcgplayer_products import TCGPlayerProduct, MTGJSONCard, MTGJSONSKU
|
|
from app.models.critical_error_log import CriticalErrorLog
|
|
from app.models.manabox_import_staging import ManaboxImportStaging
|
|
from typing import Dict, Any, Union, List
|
|
import csv
|
|
import logging
|
|
from datetime import datetime
|
|
import asyncio
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class ManaboxService(BaseService):
|
|
def __init__(self):
|
|
super().__init__(None)
|
|
|
|
async def process_manabox_csv(self, db: Session, bytes: bytes, metadata: Dict[str, Any], wait: bool = False) -> Union[bool, List[FileInDB]]:
|
|
# save file
|
|
file = await self.file_service.save_file(
|
|
db=db,
|
|
file_data=bytes,
|
|
filename=f"manabox_{datetime.now().strftime('%Y%m%d%H%M%S')}.csv",
|
|
subdir="manabox",
|
|
file_type="manabox",
|
|
content_type="text/csv",
|
|
metadata=metadata
|
|
)
|
|
|
|
# Create the background task
|
|
task = asyncio.create_task(self._process_file_background(db, file))
|
|
|
|
# If wait is True, wait for the task to complete and return the file
|
|
if wait:
|
|
await task
|
|
return_value = await self.file_service.get_file(db, file.id)
|
|
return [return_value] if return_value else []
|
|
|
|
return True
|
|
|
|
async def _process_file_background(self, db: Session, file: FileInDB):
|
|
try:
|
|
# Read the CSV file
|
|
with open(file.path, 'r') as csv_file:
|
|
reader = csv.DictReader(csv_file)
|
|
# Pre-fetch all MTGJSONCards for Scryfall IDs in the file
|
|
scryfall_ids = {row['Scryfall ID'] for row in reader}
|
|
mtg_json_map = {card.scryfall_id: card for card in db.query(MTGJSONCard).filter(MTGJSONCard.scryfall_id.in_(scryfall_ids)).all()}
|
|
|
|
# Re-read the file to process the rows
|
|
csv_file.seek(0)
|
|
next(reader) # Skip the header row
|
|
|
|
staging_entries = [] # To collect all staging entries for batch insert
|
|
critical_errors = [] # To collect errors for logging
|
|
|
|
for row in reader:
|
|
mtg_json = mtg_json_map.get(row['Scryfall ID'])
|
|
|
|
if not mtg_json:
|
|
error_message = f"Error: No MTGJSONCard found for scryfall id: {row['Scryfall ID']}"
|
|
critical_errors.append(error_message)
|
|
continue # Skip this row
|
|
|
|
language = 'ENGLISH' if row['Language'] == 'en' else 'JAPANESE' if row['Language'] == 'ja' else None # manabox only needs en and jp for now
|
|
printing = 'foil' if 'foil' in row['Foil'].lower() or 'etched' in row['Foil'].lower() else 'normal'
|
|
condition = row['Condition'].replace('_', ' ').upper()
|
|
|
|
# Query the correct TCGPlayer SKU
|
|
sku_query = db.query(MTGJSONSKU).filter(
|
|
MTGJSONSKU.tcgplayer_product_id == (mtg_json.tcgplayer_etched_product_id if row['Foil'].lower() == 'etched' else mtg_json.tcgplayer_product_id)
|
|
).filter(
|
|
MTGJSONSKU.condition == condition,
|
|
MTGJSONSKU.normalized_printing == printing,
|
|
MTGJSONSKU.language == language
|
|
).distinct()
|
|
|
|
if sku_query.count() != 1:
|
|
error_message = f"Error: Multiple TCGplayer SKUs found for mtgjson name: {mtg_json.name} condition: {row['Condition']} language: {language} printing: {printing}"
|
|
critical_errors.append(error_message)
|
|
continue # Skip this row
|
|
|
|
tcgplayer_sku = sku_query.first()
|
|
|
|
if not tcgplayer_sku:
|
|
error_message = f"Error: No TCGplayer SKU found for mtgjson name: {mtg_json.name} condition: {row['Condition']} language: {language} printing: {printing}"
|
|
critical_errors.append(error_message)
|
|
continue # Skip this row
|
|
|
|
# Query TCGPlayer product data
|
|
tcgplayer_product = db.query(TCGPlayerProduct).filter(
|
|
TCGPlayerProduct.tcgplayer_product_id == tcgplayer_sku.tcgplayer_product_id,
|
|
TCGPlayerProduct.normalized_sub_type_name == tcgplayer_sku.normalized_printing
|
|
).distinct()
|
|
|
|
if tcgplayer_product.count() != 1:
|
|
error_message = f"Error: Multiple TCGPlayer products found for SKU {tcgplayer_sku.tcgplayer_sku_id}"
|
|
critical_errors.append(error_message)
|
|
continue # Skip this row
|
|
|
|
tcgplayer_product = tcgplayer_product.first()
|
|
|
|
if not tcgplayer_product:
|
|
error_message = f"Error: No TCGPlayer product found for SKU {tcgplayer_sku.tcgplayer_sku_id}"
|
|
critical_errors.append(error_message)
|
|
continue # Skip this row
|
|
|
|
# Prepare the staging entry
|
|
quantity = int(row['Quantity'])
|
|
staging_entries.append(ManaboxImportStaging(
|
|
file_id=file.id,
|
|
tcgplayer_product_id=tcgplayer_product.tcgplayer_product_id,
|
|
tcgplayer_sku_id=tcgplayer_sku.tcgplayer_sku_id,
|
|
quantity=quantity
|
|
))
|
|
|
|
# Bulk insert all valid ManaboxImportStaging entries
|
|
if staging_entries:
|
|
db.bulk_save_objects(staging_entries)
|
|
|
|
# Log any critical errors that occurred
|
|
for error_message in critical_errors:
|
|
with transaction(db):
|
|
critical_error_log = CriticalErrorLog(error_message=error_message)
|
|
db.add(critical_error_log)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error processing file: {str(e)}")
|
|
with transaction(db):
|
|
critical_error_log = CriticalErrorLog(error_message=f"Error processing file: {str(e)}")
|
|
db.add(critical_error_log) |