zman27/giga_tcg
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

GIGA FIXED EVERYTHING OMG

Browse Source
This commit is contained in:
zman
2025-02-04 22:30:33 -05:00
parent 85510a4671
commit bd9cfca7a9
14 changed files with 1182 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
services/file.py
View File

@@ -1,75 +1,127 @@
from sqlalchemy.orm import Session
from db.utils import db_transaction
from db.models import File, StagedFileProduct
from schemas.file import FileMetadata, FileUploadResponse, GetPreparedFilesResponse, FileDeleteResponse
from schemas.file import CreateFileRequest
import os
from uuid import uuid4 as uuid
import logging
import csv
from io import StringIO
from typing import Optional, List
logger = logging.getLogger(__name__)
# Name,Set code,Set name,Collector number,Foil,Rarity,Quantity,ManaBox ID,Scryfall ID,Purchase price,Misprint,Altered,Condition,Language,Purchase price currency
MANABOX_REQUIRED_FILE_HEADERS = ['Name', 'Set code', 'Set name', 'Collector number', 'Foil', 'Rarity', 'Quantity', 'ManaBox ID', 'Scryfall ID', 'Purchase price', 'Misprint', 'Altered', 'Condition', 'Language', 'Purchase price currency']
MANABOX_ALLOWED_FILE_EXTENSIONS = ['.csv']
MANABOX_ALLOWED_FILE_TYPES = ['scan_export']
MANABOX_CONFIG = {
"required_headers": MANABOX_REQUIRED_FILE_HEADERS,
"allowed_extensions": MANABOX_ALLOWED_FILE_EXTENSIONS,
"allowed_types": MANABOX_ALLOWED_FILE_TYPES
}
SOURCES = {
"manabox": MANABOX_CONFIG
}
TEMP_DIR = os.getcwd() + '/temp/'
class FileService:
def __init__(self, db: Session):
self.db = db
# CONFIG
def get_config(self, source: str) -> dict:
return SOURCES.get(source)
def _format_response(self, file: File) -> FileUploadResponse:
response = FileUploadResponse(
id = file.id,
filename = file.filename,
type = file.type,
source = file.source,
status = file.status,
service = file.service,
file_size_kb = file.filesize_kb,
date_created=file.date_created,
date_modified=file.date_modified
)
return response
# VALIDATION
def validate_file_extension(self, filename: str, config: dict) -> bool:
return filename.endswith(tuple(config.get("allowed_extensions")))
def upload_file(self, content: bytes, filename: str, metadata: FileMetadata) -> FileUploadResponse:
# Save file to database
def validate_file_type(self, metadata: CreateFileRequest, config: dict) -> bool:
return metadata.type in config.get("allowed_types")
def validate_csv(self, content: bytes, required_headers: Optional[List[str]] = None) -> bool:
try:
# Try to decode and parse as CSV
csv_text = content.decode('utf-8')
csv_file = StringIO(csv_text)
csv_reader = csv.reader(csv_file)
# Check headers if specified
headers = next(csv_reader, None)
if required_headers and not all(header in headers for header in required_headers):
return False
return True
except (UnicodeDecodeError, csv.Error):
return False
def validate_file_content(self, content: bytes, metadata: CreateFileRequest, config: dict) -> bool:
extension = metadata.filename.split('.')[-1]
if extension == 'csv':
return self.validate_csv(content, config.get("required_headers"))
return False
def validate_file(self, content: bytes, metadata: CreateFileRequest) -> bool:
# 1. Get config
config = self.get_config(metadata.source)
# 2. Validate file extension
if not self.validate_file_extension(metadata.filename, config):
raise Exception("Invalid file extension")
# 2. validate file type
if not self.validate_file_type(metadata, config):
raise Exception("Invalid file type")
# 3. Validate file content
if not self.validate_file_content(content, metadata, config):
raise Exception("Invalid file content")
return True
# CRUD
# CREATE
def create_file(self, content: bytes, metadata: CreateFileRequest) -> File:
with db_transaction(self.db):
file = File(
id = str(uuid()),
filename = filename,
filepath = os.getcwd() + '/temp/' + filename, # TODO: config variable
filename = metadata.filename,
filepath = TEMP_DIR + metadata.filename, # TODO config variable
type = metadata.type,
source = metadata.source,
filesize_kb = round(len(content) / 1024,2),
filesize_kb = round(len(content) / 1024, 2),
status = 'pending',
service = metadata.service
)
self.db.add(file)
# save file
with open(file.filepath, 'wb') as f:
f.write(content)
response = self._format_response(file)
return response
return file
def get_file(self, file_id: str) -> File:
return self.db.query(File).filter(File.id == file_id).first()
# GET
def get_file(self, file_id: str) -> List[File]:
file = self.db.query(File).filter(File.id == file_id).first()
if not file:
raise Exception(f"File with id {file_id} not found")
return file
def get_prepared_files(self) -> list[FileUploadResponse]:
files = self.db.query(File).filter(File.status == 'prepared').all()
if len(files) == 0:
raise Exception("No prepared files found")
result = [self._format_response(file) for file in files]
logger.debug(f"Prepared files: {result}")
response = GetPreparedFilesResponse(files=result)
return response
def get_files(self, status: Optional[str] = None) -> List[File]:
if status:
return self.db.query(File).filter(File.status == status).all()
return self.db.query(File).all()
def get_staged_products(self, file_id: str) -> list[StagedFileProduct]:
# DELETE
def get_staged_products(self, file_id: str) -> List[StagedFileProduct]:
return self.db.query(StagedFileProduct).filter(StagedFileProduct.file_id == file_id).all()
def delete_file(self, file_id: str) -> FileDeleteResponse:
def delete_file(self, file_id: str) -> List[File]:
file = self.get_file(file_id)
if not file:
raise Exception(f"File with id {file_id} not found")
staged_products = self.get_staged_products(file_id)
if file:
with db_transaction(self.db):
self.db.delete(file)
for staged_product in staged_products:
self.db.delete(staged_product)
return {"id": file_id, "status": "deleted"}
else:
raise Exception(f"File with id {file_id} not found")
with db_transaction(self.db):
file.status = 'deleted'
for staged_product in staged_products:
self.db.delete(staged_product)
return file

0
services/order.py Normal file
View File

25
services/product.py
View File

@@ -1,9 +1,10 @@
from sqlalchemy.orm import Session
from db.utils import db_transaction
from db.models import Product, File, CardManabox, Card, StagedFileProduct
from db.models import Product, File, CardManabox, Card, StagedFileProduct, CardTCGPlayer
from io import StringIO
import pandas as pd
from services.file import FileService
from services.tcgplayer import TCGPlayerService
from uuid import uuid4 as uuid
import logging
@@ -25,9 +26,10 @@ class ManaboxRow:
self.quantity = row['quantity']
class ProductService:
def __init__(self, db: Session, file_service: FileService):
def __init__(self, db: Session, file_service: FileService, tcgplayer_service: TCGPlayerService):
self.db = db
self.file_service = file_service
self.tcgplayer_service = tcgplayer_service
def _format_manabox_df(self, df: pd.DataFrame) -> pd.DataFrame:
# format columns
@@ -81,16 +83,27 @@ class ProductService:
)
return product
def get_tcgplayer_card(self, card_manabox: CardManabox) -> CardTCGPlayer:
# check if tcgplayer_id exists for product_id in CardTCGPlayer
tcgplayer_card = self.db.query(CardTCGPlayer).filter(CardTCGPlayer.product_id == card_manabox.product_id).first()
if tcgplayer_card:
return tcgplayer_card
# if not, get tcgplayer_id from tcgplayer_service
tcgplayer_card = self.tcgplayer_service.get_tcgplayer_card(card_manabox)
return tcgplayer_card
def create_card(self, card_manabox: CardManabox) -> Card:
tcgplayer_card = self.get_tcgplayer_card(card_manabox)
card = Card(
product_id = card_manabox.product_id,
product_id = tcgplayer_card.product_id if tcgplayer_card else card_manabox.product_id,
number = card_manabox.collector_number,
foil = card_manabox.foil,
rarity = card_manabox.rarity,
condition = card_manabox.condition,
language = card_manabox.language,
scryfall_id = card_manabox.scryfall_id,
manabox_id = card_manabox.manabox_id
manabox_id = card_manabox.manabox_id,
tcgplayer_id = tcgplayer_card.tcgplayer_id if tcgplayer_card else None
)
return card
@@ -114,6 +127,8 @@ class ProductService:
card_manabox = self.create_card_manabox(manabox_row)
product = self.create_product(card_manabox)
card = self.create_card(card_manabox)
card_manabox.product_id = card.product_id
product.id = card.product_id
self.db.add(card_manabox)
self.db.add(product)
self.db.add(card)
@@ -129,7 +144,7 @@ class ProductService:
staged_product = self.create_staged_product(file, card_manabox, row)
# update file status
with db_transaction(self.db):
file.status = 'prepared'
file.status = 'completed'
except Exception as e:
with db_transaction(self.db):
file.status = 'error'

40
services/task.py Normal file
View File

@@ -0,0 +1,40 @@
from apscheduler.schedulers.background import BackgroundScheduler
import logging
from typing import Dict, Callable
from sqlalchemy.orm import Session
from services.product import ProductService
from db.models import File
class TaskService:
def __init__(self, db: Session, product_service: ProductService):
self.scheduler = BackgroundScheduler()
self.logger = logging.getLogger(__name__)
self.tasks: Dict[str, Callable] = {}
self.db = db
self.product_service = product_service
async def start(self):
self.scheduler.start()
self.logger.info("Task scheduler started.")
self.register_scheduled_tasks()
def register_scheduled_tasks(self):
self.scheduler.add_job(
self.daily_report,
'cron',
hour=0,
minute=0,
id='daily_report'
)
# Tasks that should be scheduled
async def daily_report(self):
self.logger.info("Generating daily report")
# Daily report logic
async def process_manabox_file(self, file: File):
self.logger.info("Processing ManaBox file")
self.product_service.bg_process_manabox_file(file.id)
self.logger.info("Finished processing ManaBox file")

132
services/tcgplayer.py
View File

@@ -1,4 +1,4 @@
from db.models import ManaboxExportData, Box, TCGPlayerGroups, TCGPlayerInventory, TCGPlayerExportHistory, TCGPlayerPricingHistory, TCGPlayerProduct, ManaboxTCGPlayerMapping
from db.models import ManaboxExportData, Box, TCGPlayerGroups, TCGPlayerInventory, TCGPlayerExportHistory, TCGPlayerPricingHistory, TCGPlayerProduct, ManaboxTCGPlayerMapping, CardManabox, CardTCGPlayer
import requests
from sqlalchemy.orm import Session
from db.utils import db_transaction
@@ -39,8 +39,7 @@ class TCGPlayerConfig:
max_retries: int = 1
class TCGPlayerService:
def __init__(self, db: Session,
pricing_service: PricingService,
def __init__(self, db: Session,
config: TCGPlayerConfig=TCGPlayerConfig(),
browser_type: Browser=Browser.BRAVE):
self.db = db
@@ -48,7 +47,6 @@ class TCGPlayerService:
self.browser_type = browser_type
self.cookies = None
self.previous_request_time = None
self.pricing_service = pricing_service
def _insert_groups(self, groups):
for group in groups:
@@ -282,16 +280,20 @@ class TCGPlayerService:
return {"message": "Inventory updated successfully", "export_id": export_id}
def _get_export_csv(self, set_name_ids: List[str]) -> bytes:
def _get_export_csv(self, set_name_ids: List[str], convert=True) -> bytes:
"""
Download export CSV and save to specified path
Returns True if successful, False otherwise
"""
logger.info(f"Downloading pricing export from tcgplayer with ids {set_name_ids}")
payload = self._set_pricing_export_payload(set_name_ids)
export_csv_download_url = f"{self.config.tcgplayer_base_url}{self.config.pricing_export_path}"
response = self._send_request(export_csv_download_url, method='POST', data=payload)
csv = self._process_content(response.content)
return csv
if convert:
csv = self._process_content(response.content)
return csv
else:
return response.content
def _update_tcgplayer_products(self):
pass
@@ -414,7 +416,8 @@ class TCGPlayerService:
export_id = self.update_inventory("live")['export_id']
self.tcg_set_tcg_inventory_product_relationship(export_id)
self.update_pricing_for_existing_product_groups()
update_csv = self.pricing_service.create_live_inventory_pricing_update_csv()
# update_csv = self.pricing_service.create_live_inventory_pricing_update_csv()
update_csv = None
return update_csv
def get_group_ids_for_box(self, box_id: str) -> List[str]:
@@ -448,5 +451,114 @@ class TCGPlayerService:
else:
raise ValueError("Must provide either box_id or upload_id")
self.update_pricing({'set_name_ids': group_ids})
add_csv = self.pricing_service.create_add_to_tcgplayer_csv(box_id)
return add_csv
# add_csv = self.pricing_service.create_add_to_tcgplayer_csv(box_id)
add_csv = None
return add_csv
def load_export_csv_to_card_tcgplayer(self, export_csv: bytes, group_id: int) -> None:
if not export_csv:
raise ValueError("No export CSV provided")
# Convert bytes to string first
text_content = export_csv.decode('utf-8')
csv_file = StringIO(text_content)
try:
reader = csv.DictReader(csv_file)
for row in reader:
product = CardTCGPlayer(
product_id=str(uuid.uuid4()),
tcgplayer_id=row['TCGplayer Id'],
group_id=group_id,
product_line=row['Product Line'],
set_name=row['Set Name'],
product_name=row['Product Name'],
title=row['Title'],
number=row['Number'],
rarity=row['Rarity'],
condition=row['Condition']
)
with db_transaction(self.db):
self.db.add(product)
finally:
csv_file.close()
def match_card_tcgplayer_to_manabox(self, card: CardManabox, group_id: int) -> CardTCGPlayer:
# Expanded rarity mapping
mb_to_tcg_rarity_mapping = {
"common": "C",
"uncommon": "U",
"rare": "R",
"mythic": "M",
"special": "S"
}
# Mapping from Manabox condition+foil to TCGPlayer condition
mb_to_tcg_condition_mapping = {
("near_mint", "foil"): "Near Mint Foil",
("near_mint", "normal"): "Near Mint",
("near_mint", "etched"): "Near Mint Foil"
}
# Get TCGPlayer condition from Manabox condition+foil combination
tcg_condition = mb_to_tcg_condition_mapping.get((card.condition, card.foil))
if tcg_condition is None:
logger.error(f"Unsupported condition/foil combination: {card.condition}, {card.foil}")
logger.error(f"Card details: name={card.name}, set_name={card.set_name}, collector_number={card.collector_number}")
return None
# Get TCGPlayer rarity from Manabox rarity
tcg_rarity = mb_to_tcg_rarity_mapping.get(card.rarity)
if tcg_rarity is None:
logger.error(f"Unsupported rarity: {card.rarity}")
logger.error(f"Card details: name={card.name}, set_name={card.set_name}, collector_number={card.collector_number}")
return None
# First query for matching products without rarity filter
base_query = self.db.query(CardTCGPlayer).filter(
CardTCGPlayer.number == card.collector_number,
CardTCGPlayer.condition == tcg_condition,
CardTCGPlayer.group_id == group_id
)
# Get all potential matches
products = base_query.all()
# If no products found, return None
if not products:
logger.error(f"No matching TCGPlayer product found for card {card.name} ({card.set_code} {card.collector_number})")
return None
# Look for an exact match including rarity, unless the TCGPlayer product is a land
for product in products:
if product.rarity == "L" or product.rarity == tcg_rarity:
return product
# If we got here, we found products but none matched our rarity criteria
logger.error(f"No matching TCGPlayer product with correct rarity found for card {card.name} ({card.set_name} {card.collector_number})")
return None
def get_tcgplayer_card(self, card: CardManabox) -> CardTCGPlayer:
# find tcgplayer group id for set code
group_id = self.db.query(TCGPlayerGroups.group_id).filter(
TCGPlayerGroups.abbreviation == card.set_code
).first()
if not group_id:
logger.error(f"Group ID not found for set code {card.set_code}")
logger.error(f"Card details: name={card.name}, set_name={card.set_name}, collector_number={card.collector_number}")
return None
group_id = group_id[0]
# check for group_id in CardTCGPlayer
group_id_exists = self.db.query(CardTCGPlayer).filter(
CardTCGPlayer.group_id == group_id).first()
if not group_id_exists:
export_csv = self._get_export_csv([str(group_id)], convert=False) # TODO should be file service
self.load_export_csv_to_card_tcgplayer(export_csv, group_id)
# match card to tcgplayer product
matching_product = self.match_card_tcgplayer_to_manabox(card, group_id)
if not matching_product:
return None
return matching_product