i think most of this works lole
This commit is contained in:
@ -2,23 +2,22 @@ from app.services.base_service import BaseService
|
||||
from sqlalchemy.orm import Session
|
||||
from app.db.database import transaction
|
||||
from app.schemas.file import FileInDB
|
||||
from app.models.mtgjson_card import MTGJSONCard
|
||||
from app.models.mtgjson_sku import MTGJSONSKU
|
||||
from app.models.tcgplayer_inventory import TCGPlayerInventory
|
||||
from app.models.tcgplayer_product import TCGPlayerProduct
|
||||
from app.models.tcgplayer_category import TCGPlayerCategory
|
||||
from app.models.tcgplayer_group import TCGPlayerGroup
|
||||
from typing import Dict, Any
|
||||
from app.models.tcgplayer_products import TCGPlayerProduct, MTGJSONCard, MTGJSONSKU
|
||||
from app.models.critical_error_log import CriticalErrorLog
|
||||
from app.models.manabox_import_staging import ManaboxImportStaging
|
||||
from typing import Dict, Any, Union, List
|
||||
import csv
|
||||
import logging
|
||||
from datetime import datetime
|
||||
import asyncio
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class ManaboxService(BaseService):
|
||||
def __init__(self):
|
||||
super().__init__(None)
|
||||
|
||||
async def process_manabox_csv(self, db: Session, bytes: bytes) -> bool:
|
||||
async def process_manabox_csv(self, db: Session, bytes: bytes, metadata: Dict[str, Any], wait: bool = False) -> Union[bool, List[FileInDB]]:
|
||||
# save file
|
||||
file = await self.file_service.save_file(
|
||||
db=db,
|
||||
@ -26,77 +25,88 @@ class ManaboxService(BaseService):
|
||||
filename=f"manabox_{datetime.now().strftime('%Y%m%d%H%M%S')}.csv",
|
||||
subdir="manabox",
|
||||
file_type="manabox",
|
||||
content_type="text/csv"
|
||||
content_type="text/csv",
|
||||
metadata=metadata
|
||||
)
|
||||
# Read the CSV file
|
||||
with open(file.path, 'r') as file:
|
||||
reader = csv.DictReader(file)
|
||||
# validate headers
|
||||
if reader.fieldnames != ['Name', 'Set code', 'Set name', 'Collector number', 'Foil', 'Rarity', 'Quantity', 'ManaBox ID', 'Scryfall ID', 'Purchase price', 'Misprint', 'Altered', 'Condition', 'Language', 'Purchase price currency']:
|
||||
logger.error("Invalid headers")
|
||||
return False
|
||||
# skip header row
|
||||
next(reader)
|
||||
for row in reader:
|
||||
# match scryfall id to mtgjson scryfall id, make sure only one distinct tcgplayer id
|
||||
mtg_json = db.query(MTGJSONCard).filter(MTGJSONCard.scryfall_id == row['Scryfall ID']).all()
|
||||
# count distinct tcgplayer ids
|
||||
cd_tcgplayer_ids = db.query(MTGJSONCard.tcgplayer_product_id).filter(MTGJSONCard.scryfall_id == row['Scryfall ID']).distinct().count()
|
||||
if cd_tcgplayer_ids != 1:
|
||||
logger.error(f"Error: {cd_tcgplayer_ids} TCGplayer IDs found for {row['Scryfall ID']}")
|
||||
return False
|
||||
else:
|
||||
mtg_json = mtg_json[0]
|
||||
# get tcgplayer sku id from mtgjson skus
|
||||
language = 'ENGLISH' if row['Language'] == 'en' else 'JAPANESE' if row['Language'] == 'ja' else None
|
||||
if row['Foil'].lower() == 'etched':
|
||||
printing = 'FOIL'
|
||||
tcgplayer_sku = db.query(MTGJSONSKU.sku_id).filter(MTGJSONSKU.product_id == mtg_json.tcgplayer_etched_product_id).filter(MTGJSONSKU.condition == row['Condition'].replace('_', ' ').upper()).filter(MTGJSONSKU.printing == printing).filter(MTGJSONSKU.language == language).all().distinct()
|
||||
else:
|
||||
printing = 'FOIL' if row['Foil'].lower() == 'foil' else 'NON FOIL'
|
||||
tcgplayer_sku = db.query(MTGJSONSKU.sku_id).filter(MTGJSONSKU.product_id == mtg_json.tcgplayer_product_id).filter(MTGJSONSKU.condition == row['Condition'].replace('_', ' ').upper()).filter(MTGJSONSKU.printing == printing).filter(MTGJSONSKU.language == language).all().distinct()
|
||||
# count distinct tcgplayer skus
|
||||
if len(tcgplayer_sku) == 0:
|
||||
logger.error(f"Error: No TCGplayer SKU found for {mtg_json.name} {row['Condition']}")
|
||||
logger.debug(row)
|
||||
logger.debug(language)
|
||||
logger.debug(row['Condition'].replace('_', ' ').upper())
|
||||
logger.debug(mtg_json.tcgplayer_product_id)
|
||||
logger.debug(printing)
|
||||
return False
|
||||
elif len(tcgplayer_sku) > 1:
|
||||
logger.error(f"Error: {len(tcgplayer_sku)} TCGplayer SKUs found for {mtg_json.name} {row['Condition']}")
|
||||
return False
|
||||
else:
|
||||
tcgplayer_sku = tcgplayer_sku[0]
|
||||
# look up tcgplayer product data for sku
|
||||
tcgplayer_product = db.query(TCGPlayerProduct).filter(TCGPlayerProduct.product_id == tcgplayer_sku.product_id).first()
|
||||
# temp just dump into tcgplayer inventory
|
||||
condition = f'{tcgplayer_sku.condition.title()} Foil' if 'Foil' in tcgplayer_product.sub_type_name else f'{tcgplayer_sku.condition.title()}'
|
||||
# join tcgplaeyer product on tcgplayer category on category_id and get name
|
||||
product_line = db.query(TCGPlayerCategory).filter(TCGPlayerCategory.category_id == tcgplayer_product.category_id).first().name
|
||||
# join tcgplaeyer product on tcgplayer group on group_id and get name
|
||||
set_name = db.query(TCGPlayerGroup).filter(TCGPlayerGroup.group_id == tcgplayer_product.group_id).first().name
|
||||
with transaction(db):
|
||||
tcgplayer_inventory = TCGPlayerInventory(
|
||||
tcgplayer_id=tcgplayer_sku.sku_id,
|
||||
product_line=product_line,
|
||||
set_name=set_name,
|
||||
product_name=tcgplayer_product.name,
|
||||
title=None,
|
||||
number=tcgplayer_product.ext_number,
|
||||
rarity=tcgplayer_product.ext_rarity,
|
||||
condition=condition,
|
||||
tcg_market_price=tcgplayer_product.market_price,
|
||||
tcg_direct_low=tcgplayer_product.direct_low_price,
|
||||
tcg_low_price_with_shipping=tcgplayer_product.low_price,
|
||||
tcg_low_price=tcgplayer_product.low_price,
|
||||
total_quantity=row['Quantity'],
|
||||
add_to_quantity=row['Quantity']
|
||||
)
|
||||
db.add(tcgplayer_inventory)
|
||||
|
||||
|
||||
|
||||
# Create the background task
|
||||
task = asyncio.create_task(self._process_file_background(db, file))
|
||||
|
||||
# If wait is True, wait for the task to complete and return the file
|
||||
if wait:
|
||||
await task
|
||||
return_value = await self.file_service.get_file(db, file.id)
|
||||
return [return_value] if return_value else []
|
||||
|
||||
return True
|
||||
|
||||
# Name,Set code,Set name,Collector number,Foil,Rarity,Quantity,ManaBox ID,Scryfall ID,Purchase price,Misprint,Altered,Condition,Language,Purchase price currency
|
||||
async def _process_file_background(self, db: Session, file: FileInDB):
|
||||
try:
|
||||
# Read the CSV file
|
||||
with open(file.path, 'r') as csv_file:
|
||||
reader = csv.DictReader(csv_file)
|
||||
# skip header row
|
||||
next(reader)
|
||||
for row in reader:
|
||||
# match scryfall id to mtgjson scryfall id, make sure only one distinct tcgplayer id
|
||||
mtg_json = db.query(MTGJSONCard).filter(MTGJSONCard.scryfall_id == row['Scryfall ID']).all()
|
||||
# count distinct tcgplayer ids
|
||||
cd_tcgplayer_ids = db.query(MTGJSONCard.tcgplayer_sku_id).filter(MTGJSONCard.scryfall_id == row['Scryfall ID']).distinct().count()
|
||||
if cd_tcgplayer_ids != 1:
|
||||
logger.error(f"Error: multiple TCGplayer IDs found for scryfall id: {row['Scryfall ID']} found {cd_tcgplayer_ids} ids expected 1")
|
||||
with transaction(db):
|
||||
critical_error_log = CriticalErrorLog(
|
||||
error_message=f"Error: multiple TCGplayer IDs found for scryfall id: {row['Scryfall ID']} found {cd_tcgplayer_ids} ids expected 1"
|
||||
)
|
||||
db.add(critical_error_log)
|
||||
continue
|
||||
else:
|
||||
mtg_json = mtg_json[0]
|
||||
# get tcgplayer sku id from mtgjson skus
|
||||
language = 'ENGLISH' if row['Language'] == 'en' else 'JAPANESE' if row['Language'] == 'ja' else None
|
||||
if row['Foil'].lower() == 'etched':
|
||||
printing = 'FOIL'
|
||||
tcgplayer_sku = db.query(MTGJSONSKU).filter(MTGJSONSKU.tcgplayer_sku_id == mtg_json.tcgplayer_etched_sku_id).filter(MTGJSONSKU.condition == row['Condition'].replace('_', ' ').upper()).filter(MTGJSONSKU.printing == printing).filter(MTGJSONSKU.language == language).distinct().all()
|
||||
else:
|
||||
printing = 'FOIL' if row['Foil'].lower() == 'foil' else 'NON FOIL'
|
||||
tcgplayer_sku = db.query(MTGJSONSKU).filter(MTGJSONSKU.tcgplayer_sku_id == mtg_json.tcgplayer_sku_id).filter(MTGJSONSKU.condition == row['Condition'].replace('_', ' ').upper()).filter(MTGJSONSKU.printing == printing).filter(MTGJSONSKU.language == language).distinct().all()
|
||||
# count distinct tcgplayer skus
|
||||
if len(tcgplayer_sku) == 0:
|
||||
logger.error(f"Error: No TCGplayer SKU found for mtgjson name: {mtg_json.name} condition: {row['Condition']} language: {language} printing: {printing}")
|
||||
with transaction(db):
|
||||
critical_error_log = CriticalErrorLog(
|
||||
error_message=f"Error: No TCGplayer SKU found for mtgjson name: {mtg_json.name} condition: {row['Condition']} language: {language} printing: {printing}"
|
||||
)
|
||||
db.add(critical_error_log)
|
||||
continue
|
||||
elif len(tcgplayer_sku) > 1:
|
||||
logger.error(f"Error: {len(tcgplayer_sku)} TCGplayer SKUs found for mtgjson name: {mtg_json.name} condition: {row['Condition']} language: {language} printing: {printing}")
|
||||
with transaction(db):
|
||||
critical_error_log = CriticalErrorLog(
|
||||
error_message=f"Error: {len(tcgplayer_sku)} TCGplayer SKUs found for mtgjson name: {mtg_json.name} condition: {row['Condition']} language: {language} printing: {printing}"
|
||||
)
|
||||
db.add(critical_error_log)
|
||||
continue
|
||||
else:
|
||||
tcgplayer_sku = tcgplayer_sku[0]
|
||||
# look up tcgplayer product data for sku
|
||||
tcgplayer_product = db.query(TCGPlayerProduct).filter(TCGPlayerProduct.tcgplayer_product_id == tcgplayer_sku.tcgplayer_product_id).filter(TCGPlayerProduct.condition == row['Condition'].replace('_', ' ').upper()).filter(TCGPlayerProduct.language == language).filter(TCGPlayerProduct.printing == printing).first()
|
||||
|
||||
quantity = int(row['Quantity'])
|
||||
|
||||
with transaction(db):
|
||||
manabox_import_staging = ManaboxImportStaging(
|
||||
file_id=file.id,
|
||||
product_id=tcgplayer_product.id,
|
||||
quantity=quantity,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now()
|
||||
)
|
||||
db.add(manabox_import_staging)
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing file: {str(e)}")
|
||||
with transaction(db):
|
||||
critical_error_log = CriticalErrorLog(
|
||||
error_message=f"Error processing file: {str(e)}"
|
||||
)
|
||||
db.add(critical_error_log)
|
||||
Reference in New Issue
Block a user