from app.services.base_service import BaseService from sqlalchemy.orm import Session from app.db.database import transaction from app.schemas.file import FileInDB from app.models.tcgplayer_products import TCGPlayerProduct, MTGJSONCard, MTGJSONSKU from app.models.critical_error_log import CriticalErrorLog from app.models.manabox_import_staging import ManaboxImportStaging from typing import Dict, Any, Union, List import csv import logging from datetime import datetime import asyncio logger = logging.getLogger(__name__) class ManaboxService(BaseService): def __init__(self): super().__init__(None) async def process_manabox_csv(self, db: Session, bytes: bytes, metadata: Dict[str, Any], wait: bool = False) -> Union[bool, List[FileInDB]]: # save file file = await self.file_service.save_file( db=db, file_data=bytes, filename=f"manabox_{datetime.now().strftime('%Y%m%d%H%M%S')}.csv", subdir="manabox", file_type="manabox", content_type="text/csv", metadata=metadata ) # Create the background task task = asyncio.create_task(self._process_file_background(db, file)) # If wait is True, wait for the task to complete and return the file if wait: await task return_value = await self.file_service.get_file(db, file.id) return [return_value] if return_value else [] return True async def _process_file_background(self, db: Session, file: FileInDB): try: # Read the CSV file with open(file.path, 'r') as csv_file: reader = csv.DictReader(csv_file) # Pre-fetch all MTGJSONCards for Scryfall IDs in the file scryfall_ids = {row['Scryfall ID'] for row in reader} mtg_json_map = {card.scryfall_id: card for card in db.query(MTGJSONCard).filter(MTGJSONCard.scryfall_id.in_(scryfall_ids)).all()} # Re-read the file to process the rows csv_file.seek(0) next(reader) # Skip the header row staging_entries = [] # To collect all staging entries for batch insert critical_errors = [] # To collect errors for logging for row in reader: mtg_json = mtg_json_map.get(row['Scryfall ID']) if not mtg_json: error_message = f"Error: No MTGJSONCard found for scryfall id: {row['Scryfall ID']}" critical_errors.append(error_message) continue # Skip this row language = 'ENGLISH' if row['Language'] == 'en' else 'JAPANESE' if row['Language'] == 'ja' else None # manabox only needs en and jp for now printing = 'foil' if 'foil' in row['Foil'].lower() or 'etched' in row['Foil'].lower() else 'normal' condition = row['Condition'].replace('_', ' ').upper() # Query the correct TCGPlayer SKU sku_query = db.query(MTGJSONSKU).filter( MTGJSONSKU.tcgplayer_product_id == (mtg_json.tcgplayer_etched_product_id if row['Foil'].lower() == 'etched' else mtg_json.tcgplayer_product_id) ).filter( MTGJSONSKU.condition == condition, MTGJSONSKU.normalized_printing == printing, MTGJSONSKU.language == language ).distinct() if sku_query.count() != 1: error_message = f"Error: Multiple TCGplayer SKUs found for mtgjson name: {mtg_json.name} condition: {row['Condition']} language: {language} printing: {printing}" critical_errors.append(error_message) continue # Skip this row tcgplayer_sku = sku_query.first() if not tcgplayer_sku: error_message = f"Error: No TCGplayer SKU found for mtgjson name: {mtg_json.name} condition: {row['Condition']} language: {language} printing: {printing}" critical_errors.append(error_message) continue # Skip this row # Query TCGPlayer product data tcgplayer_product = db.query(TCGPlayerProduct).filter( TCGPlayerProduct.tcgplayer_product_id == tcgplayer_sku.tcgplayer_product_id, TCGPlayerProduct.normalized_sub_type_name == tcgplayer_sku.normalized_printing ).distinct() if tcgplayer_product.count() != 1: error_message = f"Error: Multiple TCGPlayer products found for SKU {tcgplayer_sku.tcgplayer_sku_id}" critical_errors.append(error_message) continue # Skip this row tcgplayer_product = tcgplayer_product.first() if not tcgplayer_product: error_message = f"Error: No TCGPlayer product found for SKU {tcgplayer_sku.tcgplayer_sku_id}" critical_errors.append(error_message) continue # Skip this row # Prepare the staging entry quantity = int(row['Quantity']) staging_entries.append(ManaboxImportStaging( file_id=file.id, tcgplayer_product_id=tcgplayer_product.tcgplayer_product_id, tcgplayer_sku_id=tcgplayer_sku.tcgplayer_sku_id, quantity=quantity )) # Bulk insert all valid ManaboxImportStaging entries if staging_entries: db.bulk_save_objects(staging_entries) # Log any critical errors that occurred for error_message in critical_errors: with transaction(db): critical_error_log = CriticalErrorLog(error_message=error_message) db.add(critical_error_log) except Exception as e: logger.error(f"Error processing file: {str(e)}") with transaction(db): critical_error_log = CriticalErrorLog(error_message=f"Error processing file: {str(e)}") db.add(critical_error_log)