from sqlalchemy.orm import Session from db.models import File, CardTCGPlayer, Price from services.util._dataframe import TCGPlayerPricingRow, DataframeUtil from services.file import FileService from services.tcgplayer import TCGPlayerService from uuid import uuid4 from db.utils import db_transaction class PricingService: def __init__(self, db: Session, file_service: FileService, tcgplayer_service: TCGPlayerService): self.db = db self.file_service = file_service self.tcgplayer_service = tcgplayer_service self.df_util = DataframeUtil() # function for taking a tcgplayer pricing export with all set ids and loading it into the price table # can be run as needed or scheduled def get_pricing_export_content(self, file: File = None) -> bytes: if file: file_content = self.file_service.get_file_content(file.id) else: file = self.tcgplayer_service.get_pricing_export_for_all_products() file_content = self.file_service.get_file_content(file.id) return file_content def load_pricing_csv_content_to_db(self, file_content: bytes): try: if not file_content: raise ValueError("No file content provided") price_types = { "tcg_market_price": "tcg_market_price", "tcg_direct_low": "tcg_direct_low", "tcg_low_price_with_shipping": "tcg_low_price_with_shipping", "tcg_low_price": "tcg_low_price", "tcg_marketplace_price": "listed_price" } required_columns = ["tcgplayer_id"] + list(price_types.keys()) df = self.df_util.csv_bytes_to_df(file_content) # Validate columns missing_columns = set(required_columns) - set(df.columns) if missing_columns: raise ValueError(f"Missing required columns: {missing_columns}") # Process in true batches for i in range(0, len(df), 1000): batch = df.iloc[i:i+1000] pricing_rows = [TCGPlayerPricingRow(row) for _, row in batch.iterrows()] # Query cards for this batch only tcgplayer_ids = [row.tcgplayer_id for row in pricing_rows] batch_cards = self.db.query(CardTCGPlayer).filter( CardTCGPlayer.tcgplayer_id.in_(tcgplayer_ids) ).all() existing_cards = {card.tcgplayer_id: card for card in batch_cards} new_prices = [] for row in pricing_rows: if row.tcgplayer_id not in existing_cards: continue card = existing_cards[row.tcgplayer_id] row_prices = [ Price( id=str(uuid4()), product_id=card.product_id, marketplace_id=None, type=price_type, # Added missing price_type price=getattr(row, col_name) ) for col_name, price_type in price_types.items() if getattr(row, col_name, None) is not None and getattr(row, col_name) > 0 ] new_prices.extend(row_prices) # Save each batch separately if new_prices: with db_transaction(self.db): self.db.bulk_save_objects(new_prices) except Exception as e: raise e # Consider adding logging here def cron_load_prices(self, file: File = None): file_content = self.get_pricing_export_content(file) self.load_pricing_csv_content_to_db(file_content)