From a78c3bcba303c2605b6277c1db33b155abe4db1b Mon Sep 17 00:00:00 2001 From: zman Date: Wed, 5 Feb 2025 21:51:22 -0500 Subject: [PATCH] more stuff yay --- db/models.py | 21 +++- dependencies.py | 43 ++++++++- main.py | 3 +- routes/routes.py | 76 +++++++++++++-- schemas/box.py | 83 ++++++++++++---- services/box.py | 93 +++++++++++++++++- services/file.py | 2 +- services/pricing.py | 199 +------------------------------------- services/pricing_old.py | 205 ++++++++++++++++++++++++++++++++++++++++ services/product.py | 14 ++- services/storage.py | 176 ++++++++++++++++++++++++++++++++++ services/task.py | 19 +++- services/tcgplayer.py | 117 ++++++++++++++++++++++- tests/box_test.py | 154 ++++++++++++++++++++++++++++++ tests/file_test.py | 2 +- 15 files changed, 958 insertions(+), 249 deletions(-) create mode 100644 services/pricing_old.py create mode 100644 services/storage.py create mode 100644 tests/box_test.py diff --git a/db/models.py b/db/models.py index c504986..c0294b3 100644 --- a/db/models.py +++ b/db/models.py @@ -113,7 +113,8 @@ class Box(Base): product_id = Column(String, ForeignKey("products.id"), primary_key=True) type = Column(String) # collector box, play box, etc. - sku = Column(String) + set_code = Column(String) + sku = Column(String, nullable=True) num_cards_expected = Column(Integer, nullable=True) date_created = Column(DateTime, default=datetime.now) date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now) @@ -208,7 +209,7 @@ class Warehouse(Base): __tablename__ = "warehouse" id = Column(String, primary_key=True) - user_id = Column(String, ForeignKey("users.id"), default="admin") + user_id = Column(String, ForeignKey("users.id")) date_created = Column(DateTime, default=datetime.now) date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now) @@ -246,6 +247,7 @@ class User(Base): __tablename__ = "users" id = Column(String, primary_key=True) + username = Column(String) date_created = Column(DateTime, default=datetime.now) date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now) @@ -283,6 +285,7 @@ class File(Base): filepath = Column(String) # backup location filesize_kb = Column(Float) status = Column(String) + box_id = Column(String, ForeignKey("boxes.product_id"), nullable=True) date_created = Column(DateTime, default=datetime.now) date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now) @@ -303,9 +306,16 @@ class StorageBlock(Base): """ __tablename__ = "storage_blocks" + @validates("type") + def validate_type(self, key, type: str): + if type not in StorageBlockTypeEnum or type.lower() not in StorageBlockTypeEnum: + raise ValueError(f"Invalid storage block type: {type}") + return type + id = Column(String, primary_key=True) warehouse_id = Column(String, ForeignKey("warehouse.id")) name = Column(String) + type = Column(String) # rare or common date_created = Column(DateTime, default=datetime.now) date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now) @@ -530,6 +540,7 @@ class BoxTypeEnum(str, Enum): PLAY = "play" DRAFT = "draft" COMMANDER = "commander" + SET = "set" class ProductLineEnum(str, Enum): MTG = "mtg" @@ -537,4 +548,8 @@ class ProductLineEnum(str, Enum): class ProductTypeEnum(str, Enum): BOX = "box" - CARD = "card" \ No newline at end of file + CARD = "card" + +class StorageBlockTypeEnum(str, Enum): + RARE = "rare" + COMMON = "common" \ No newline at end of file diff --git a/dependencies.py b/dependencies.py index 9d874d9..10ff10e 100644 --- a/dependencies.py +++ b/dependencies.py @@ -8,9 +8,11 @@ from services.file import FileService from services.product import ProductService from services.inventory import InventoryService from services.task import TaskService +from services.storage import StorageService from fastapi import Depends, Form from db.database import get_db from schemas.file import CreateFileRequest +from schemas.box import CreateBoxRequest, UpdateBoxRequest, CreateOpenBoxRequest ## file @@ -29,21 +31,54 @@ def get_create_file_metadata( """Dependency injection for FileMetadata""" return CreateFileRequest(type=type, source=source, service=service, filename=filename) +def get_box_data( + type: str = Form(...), + sku: str = Form(None), + set_code: str = Form(...), + num_cards_expected: int = Form(None) + ) -> CreateBoxRequest: + """Dependency injection for BoxData""" + return CreateBoxRequest(type=type, sku=sku, set_code=set_code, num_cards_expected=num_cards_expected) + +def get_box_update_data( + type: str = Form(None), + sku: str = Form(None), + set_code: str = Form(None), + num_cards_expected: int = Form(None) + ) -> UpdateBoxRequest: + """Dependency injection for BoxUpdateData""" + return UpdateBoxRequest(type=type, sku=sku, set_code=set_code, num_cards_expected=num_cards_expected) + +def get_open_box_data( + product_id: str = Form(...), + file_ids: list[str] = Form(None), + num_cards_actual: int = Form(None), + date_opened: str = Form(None) + ) -> CreateOpenBoxRequest: + """Dependency injection for OpenBoxData""" + return CreateOpenBoxRequest(product_id=product_id, file_ids=file_ids, num_cards_actual=num_cards_actual, date_opened=date_opened) + def get_tcgplayer_service( db: Session = Depends(get_db) ) -> TCGPlayerService: """Dependency injection for TCGPlayerService""" return TCGPlayerService(db) +# storage + +def get_storage_service(db: Session = Depends(get_db)) -> StorageService: + """Dependency injection for StorageService""" + return StorageService(db) + # product -def get_product_service(db: Session = Depends(get_db), file_service: FileService = Depends(get_file_service), tcgplayer_service: TCGPlayerService = Depends(get_tcgplayer_service)) -> ProductService: +def get_product_service(db: Session = Depends(get_db), file_service: FileService = Depends(get_file_service), tcgplayer_service: TCGPlayerService = Depends(get_tcgplayer_service), storage_service: StorageService = Depends(get_storage_service)) -> ProductService: """Dependency injection for ProductService""" - return ProductService(db, file_service, tcgplayer_service) + return ProductService(db, file_service, tcgplayer_service, storage_service) # task -def get_task_service(db: Session = Depends(get_db), product_service: ProductService = Depends(get_product_service)) -> TaskService: +def get_task_service(db: Session = Depends(get_db), product_service: ProductService = Depends(get_product_service), tcgplayer_service: TCGPlayerService = Depends(get_tcgplayer_service)) -> TaskService: """Dependency injection for TaskService""" - return TaskService(db, product_service) + return TaskService(db, product_service, tcgplayer_service) ## Inventory def get_inventory_service(db: Session = Depends(get_db)) -> InventoryService: diff --git a/main.py b/main.py index 563a674..2b70d3c 100644 --- a/main.py +++ b/main.py @@ -10,6 +10,7 @@ import sys from services.tcgplayer import TCGPlayerService, PricingService from services.product import ProductService from services.file import FileService +from services.storage import StorageService from db.models import TCGPlayerGroups @@ -62,7 +63,7 @@ async def startup_event(): tcgplayer_service = TCGPlayerService(db) tcgplayer_service.populate_tcgplayer_groups() # Start task service - task_service = TaskService(db, ProductService(db, FileService(db), TCGPlayerService(db))) + task_service = TaskService(db, ProductService(db, FileService(db), TCGPlayerService(db), StorageService(db)), TCGPlayerService(db)) await task_service.start() diff --git a/routes/routes.py b/routes/routes.py index f592996..b745d58 100644 --- a/routes/routes.py +++ b/routes/routes.py @@ -11,8 +11,8 @@ from services.file import FileService from services.product import ProductService from services.task import TaskService from schemas.file import FileSchema, CreateFileRequest, CreateFileResponse, GetFileResponse, DeleteFileResponse, GetFileQueryParams -from schemas.box import CreateBoxResponse, CreateBoxRequestData -from dependencies import get_data_service, get_upload_service, get_tcgplayer_service, get_box_service, get_create_file_metadata, get_file_service, get_product_service, get_task_service +from schemas.box import CreateBoxResponse, CreateBoxRequest, BoxSchema, UpdateBoxRequest, CreateOpenBoxRequest, CreateOpenBoxResponse, OpenBoxSchema +from dependencies import get_data_service, get_upload_service, get_tcgplayer_service, get_box_service, get_create_file_metadata, get_file_service, get_product_service, get_task_service, get_box_data, get_box_update_data, get_open_box_data import logging @@ -31,11 +31,6 @@ MAX_FILE_SIZE = 1024 * 1024 * 100 # 100 MB response_model=CreateFileResponse, status_code=201 ) -@router.post( - "/files", - response_model=CreateFileResponse, - status_code=201 -) async def create_file( background_tasks: BackgroundTasks, file: UploadFile = File(...), @@ -144,9 +139,76 @@ async def delete_file( raise HTTPException(status_code=400, detail=str(e)) +## BOX +## CREATE +@router.post("/boxes", response_model=CreateBoxResponse, status_code=201) +async def create_box( + box_data: CreateBoxRequest = Depends(get_box_data), + box_service: BoxService = Depends(get_box_service) +): + try: + result = box_service.create_box(box_data) + return CreateBoxResponse( + status_code=201, + success=True, + box=[BoxSchema.from_orm(result)] + ) + except Exception as e: + logger.error(f"Create box failed: {str(e)}") + raise HTTPException(status_code=400, detail=str(e)) +## UPDATE +@router.put("/boxes/{box_id}", response_model=CreateBoxResponse) +async def update_box( + box_id: str, + box_data: UpdateBoxRequest = Depends(get_box_update_data), + box_service: BoxService = Depends(get_box_service) +): + try: + result = box_service.update_box(box_id, box_data) + return CreateBoxResponse( + status_code=200, + success=True, + box=[BoxSchema.from_orm(result)] + ) + except Exception as e: + logger.error(f"Update box failed: {str(e)}") + raise HTTPException(status_code=400, detail=str(e)) +## DELETE +@router.delete("/boxes/{box_id}", response_model=CreateBoxResponse) +async def delete_box( + box_id: str, + box_service: BoxService = Depends(get_box_service) +): + try: + result = box_service.delete_box(box_id) + return CreateBoxResponse( + status_code=200, + success=True, + box=[BoxSchema.from_orm(result)] + ) + except Exception as e: + logger.error(f"Delete box failed: {str(e)}") + raise HTTPException(status_code=400, detail=str(e)) +# BOX OPEN +@router.post("/boxes/{box_id}/open", response_model=CreateOpenBoxResponse, status_code=201) +async def open_box( + box_id: str, + box_data: CreateOpenBoxRequest = Depends(get_open_box_data), + box_service: BoxService = Depends(get_box_service) +): + try: + result = box_service.open_box(box_id, box_data) + return CreateOpenBoxResponse( + status_code=201, + success=True, + open_box=[OpenBoxSchema.from_orm(result)] + ) + except Exception as e: + logger.error(f"Open box failed: {str(e)}") + raise HTTPException(status_code=400, detail=str(e)) diff --git a/schemas/box.py b/schemas/box.py index aa9da13..865600c 100644 --- a/schemas/box.py +++ b/schemas/box.py @@ -1,25 +1,66 @@ -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, ConfigDict from schemas.base import BaseSchema +from typing import Optional +from datetime import datetime -class CreateBoxResponse(BaseModel): - success: bool = Field(..., title="Success") - -class CreateOpenBoxResponse(CreateBoxResponse): - open_box_id: str - -class CreateSealedBoxResponse(CreateBoxResponse): - product_id: str - quantity: int - -class CreateBoxRequestData(BaseModel): +#BOX +class BoxSchema(BaseSchema): + product_id: str = Field(..., title="Product ID") type: str = Field(..., title="Box Type (collector, play, draft)") - name: str = Field(..., title="Name") - product_line: str = Field(..., title="Product Line (MTG, Pokemon, etc)") - set_name: str = Field(..., title="Set Name") set_code: str = Field(..., title="Set Code") - sealed: bool = Field(..., title="Sealed: Boolean") - sku: str = Field(..., title="SKU") - num_cards_expected: int = Field(..., title="Number of cards expected") - num_cards_actual: int = Field(None, title="Number of cards actual") - date_purchased: str = Field(..., title="Date purchased") - date_opened: str = Field(None, title="Date opened") \ No newline at end of file + sku: Optional[str] = Field(None, title="SKU") + num_cards_expected: Optional[int] = Field(None, title="Number of cards expected") + + model_config = ConfigDict(from_attributes=True) + +# CREATE +# REQUEST +class CreateBoxRequest(BaseModel): + type: str = Field(..., title="Box Type (collector, play, draft)") + set_code: str = Field(..., title="Set Code") + sku: Optional[str] = Field(None, title="SKU") + num_cards_expected: Optional[int] = Field(None, title="Number of cards expected") + +# RESPONSE +class CreateBoxResponse(BaseModel): + status_code: int = Field(..., title="status_code") + success: bool = Field(..., title="success") + box: list[BoxSchema] = Field(..., title="box") + +# UPDATE +# REQUEST +class UpdateBoxRequest(BaseModel): + type: Optional[str] = Field(None, title="Box Type (collector, play, draft)") + set_code: Optional[str] = Field(None, title="Set Code") + sku: Optional[str] = Field(None, title="SKU") + num_cards_expected: Optional[int] = Field(None, title="Number of cards expected") + +# GET +# RESPONSE +class GetBoxResponse(BaseModel): + status_code: int = Field(..., title="status_code") + success: bool = Field(..., title="success") + boxes: list[BoxSchema] = Field(..., title="boxes") + + +# OPEN BOX +class OpenBoxSchema(BaseModel): + id: str = Field(..., title="id") + num_cards_actual: Optional[int] = Field(None, title="Number of cards actual") + date_opened: Optional[datetime] = Field(None, title="Date Opened") + + model_config = ConfigDict(from_attributes=True) + +# CREATE +# REQUEST +class CreateOpenBoxRequest(BaseModel): + product_id: str = Field(..., title="Product ID") + file_ids: list[str] = Field(None, title="File IDs") + num_cards_actual: Optional[int] = Field(None, title="Number of cards actual") + date_opened: Optional [str] = Field(None, title="Date Opened") + +# RESPONSE +class CreateOpenBoxResponse(BaseModel): + status_code: int = Field(..., title="status_code") + success: bool = Field(..., title="success") + open_box: list[OpenBoxSchema] = Field(..., title="open_box") diff --git a/services/box.py b/services/box.py index 77344d0..a52a48e 100644 --- a/services/box.py +++ b/services/box.py @@ -1,10 +1,11 @@ -from db.models import Box, File, StagedFileProduct, Product, OpenBoxCard, OpenBox, Inventory +from db.models import Box, File, StagedFileProduct, Product, OpenBoxCard, OpenBox, Inventory, TCGPlayerGroups from db.utils import db_transaction from uuid import uuid4 as uuid from datetime import datetime from sqlalchemy.orm import Session from sqlalchemy.engine.result import Row -from schemas.box import CreateOpenBoxResponse, CreateSealedBoxResponse, CreateBoxResponse +from sqlalchemy import or_ +from schemas.box import CreateBoxRequest, CreateBoxResponse, UpdateBoxRequest, CreateOpenBoxRequest import logging from typing import Any from db.utils import db_transaction @@ -92,7 +93,7 @@ class BoxService: response = CreateBoxResponse(success=True) return response - def create_box(self, create_box_data: dict[str, Any], file_ids: list[str] = None) -> CreateBoxResponse: + def _create_box(self, create_box_data: dict[str, Any], file_ids: list[str] = None) -> CreateBoxResponse: sealed = create_box_data["sealed"] assert isinstance(sealed, bool) if file_ids and not sealed: @@ -132,4 +133,88 @@ class BoxService: except Exception as e: logger.error(f"Error creating box: {str(e)}") - raise e \ No newline at end of file + raise e + + def validate_box_type(self, box_type: str) -> bool: + return box_type in ["collector", "play", "draft", "set", "commander"] + + def validate_set_code(self, set_code: str) -> bool: + exists = self.db.query(TCGPlayerGroups).filter( + TCGPlayerGroups.abbreviation == set_code + ).first() is not None + return exists + + def create_box(self, create_box_data: CreateBoxRequest) -> Box: + # validate box data + if not self.validate_box_type(create_box_data.type): + raise Exception("Invalid box type") + if not self.validate_set_code(create_box_data.set_code): + raise Exception("Invalid set code") + # check if box exists by type and set code or sku + existing_box = self.db.query(Box).filter( + or_( + Box.type == create_box_data.type, + Box.sku == create_box_data.sku + ), + Box.set_code == create_box_data.set_code + ).first() + if existing_box: + raise Exception("Box already exists") + # create box + with db_transaction(self.db): + box = Box( + product_id=str(uuid()), + type=create_box_data.type, + set_code=create_box_data.set_code, + sku=create_box_data.sku, + num_cards_expected=create_box_data.num_cards_expected + ) + self.db.add(box) + + return box + + def update_box(self, box_id: str, update_box_data: UpdateBoxRequest) -> Box: + box = self.db.query(Box).filter(Box.product_id == box_id).first() + if not box: + raise Exception("Box not found") + with db_transaction(self.db): + if update_box_data.type: + box.type = update_box_data.type + if update_box_data.set_code: + box.set_code = update_box_data.set_code + if update_box_data.sku: + box.sku = update_box_data.sku + if update_box_data.num_cards_expected: + box.num_cards_expected = update_box_data.num_cards_expected + return box + + def delete_box(self, box_id: str) -> Box: + box = self.db.query(Box).filter(Box.product_id == box_id).first() + if not box: + raise Exception("Box not found") + with db_transaction(self.db): + self.db.delete(box) + return box + + def open_box(self, box_id: str, box_data: CreateOpenBoxRequest): + box = self.db.query(Box).filter(Box.product_id == box_id).first() + if not box: + raise Exception("Box not found") + with db_transaction(self.db): + open_box = OpenBox( + id=str(uuid()), + product_id=box_id, + num_cards_actual=box_data.num_cards_actual, + date_opened=datetime.strptime(box_data.date_opened, "%Y-%m-%d") if box_data.date_opened else datetime.now() + ) + self.db.add(open_box) + staged_product_data = self.get_staged_product_data(box_data.file_ids) + product_data = self.aggregate_staged_product_data(staged_product_data) + self.inventory_service.process_staged_products(product_data) + self.add_products_to_open_box(open_box, product_data) + # update box_id for files + for file_id in box_data.file_ids: + file = self.db.query(File).filter(File.id == file_id).first() + file.box_id = open_box.id + self.db.add(file) + return open_box \ No newline at end of file diff --git a/services/file.py b/services/file.py index 058f5b7..2adda7e 100644 --- a/services/file.py +++ b/services/file.py @@ -14,7 +14,7 @@ logger = logging.getLogger(__name__) # Name,Set code,Set name,Collector number,Foil,Rarity,Quantity,ManaBox ID,Scryfall ID,Purchase price,Misprint,Altered,Condition,Language,Purchase price currency MANABOX_REQUIRED_FILE_HEADERS = ['Name', 'Set code', 'Set name', 'Collector number', 'Foil', 'Rarity', 'Quantity', 'ManaBox ID', 'Scryfall ID', 'Purchase price', 'Misprint', 'Altered', 'Condition', 'Language', 'Purchase price currency'] MANABOX_ALLOWED_FILE_EXTENSIONS = ['.csv'] -MANABOX_ALLOWED_FILE_TYPES = ['scan_export'] +MANABOX_ALLOWED_FILE_TYPES = ['scan_export_common', 'scan_export_rare'] MANABOX_CONFIG = { "required_headers": MANABOX_REQUIRED_FILE_HEADERS, "allowed_extensions": MANABOX_ALLOWED_FILE_EXTENSIONS, diff --git a/services/pricing.py b/services/pricing.py index c41997d..9837ae3 100644 --- a/services/pricing.py +++ b/services/pricing.py @@ -1,205 +1,8 @@ -import logging -from typing import Callable -from db.models import TCGPlayerInventory, TCGPlayerExportHistory, TCGPlayerPricingHistory, ManaboxExportData, ManaboxTCGPlayerMapping, TCGPlayerProduct from sqlalchemy.orm import Session -import pandas as pd -from db.utils import db_transaction -from sqlalchemy import func, and_, exists -logger = logging.getLogger(__name__) - class PricingService: def __init__(self, db: Session): self.db = db - - def get_box_with_most_recent_prices(self, box_id: str) -> pd.DataFrame: - latest_prices = ( - self.db.query( - TCGPlayerPricingHistory.tcgplayer_product_id, - func.max(TCGPlayerPricingHistory.date_created).label('max_date') - ) - .group_by(TCGPlayerPricingHistory.tcgplayer_product_id) - .subquery('latest') # Added name to subquery - ) - - result = ( - self.db.query(ManaboxExportData, TCGPlayerPricingHistory, TCGPlayerProduct) - .join(ManaboxTCGPlayerMapping, ManaboxExportData.id == ManaboxTCGPlayerMapping.manabox_id) - .join(TCGPlayerProduct, ManaboxTCGPlayerMapping.tcgplayer_id == TCGPlayerProduct.id) - .join(TCGPlayerPricingHistory, TCGPlayerProduct.id == TCGPlayerPricingHistory.tcgplayer_product_id) - .join( - latest_prices, - and_( - TCGPlayerPricingHistory.tcgplayer_product_id == latest_prices.c.tcgplayer_product_id, - TCGPlayerPricingHistory.date_created == latest_prices.c.max_date - ) - ) - .filter(ManaboxExportData.box_id == box_id) # Removed str() conversion - .all() - ) - - logger.debug(f"Found {len(result)} rows") - - df = pd.DataFrame([{ - **{f"manabox_{k}": v for k, v in row[0].__dict__.items() if not k.startswith('_')}, - **{f"pricing_{k}": v for k, v in row[1].__dict__.items() if not k.startswith('_')}, - **{f"tcgproduct_{k}": v for k, v in row[2].__dict__.items() if not k.startswith('_')} - } for row in result]) - - return df - def get_live_inventory_with_most_recent_prices(self) -> pd.DataFrame: - # Get latest export IDs using subqueries - latest_inventory_export = ( - self.db.query(TCGPlayerExportHistory.inventory_export_id) - .filter(TCGPlayerExportHistory.type == "live_inventory") - .order_by(TCGPlayerExportHistory.date_created.desc()) - .limit(1) - .scalar_subquery() - ) - # this is bad because latest pricing export is not guaranteed to be related to the latest inventory export - latest_pricing_export = ( - self.db.query(TCGPlayerExportHistory.pricing_export_id) - .filter(TCGPlayerExportHistory.type == "pricing") - .order_by(TCGPlayerExportHistory.date_created.desc()) - .limit(1) - .scalar_subquery() - ) - - # Join inventory and pricing data in a single query - inventory_with_pricing = ( - self.db.query(TCGPlayerInventory, TCGPlayerPricingHistory) - .join( - TCGPlayerPricingHistory, - TCGPlayerInventory.tcgplayer_product_id == TCGPlayerPricingHistory.tcgplayer_product_id - ) - .filter( - TCGPlayerInventory.export_id == latest_inventory_export, - TCGPlayerPricingHistory.export_id == latest_pricing_export - ) - .all() - ) - - # Convert to pandas DataFrame - df = pd.DataFrame([{ - # Inventory columns - **{f"inventory_{k}": v - for k, v in row[0].__dict__.items() - if not k.startswith('_')}, - # Pricing columns - **{f"pricing_{k}": v - for k, v in row[1].__dict__.items() - if not k.startswith('_')} - } for row in inventory_with_pricing]) - - return df - - def default_pricing_algo(self, df: pd.DataFrame = None): - if df is None: - logger.debug("No DataFrame provided, fetching live inventory with most recent prices") - df = self.get_live_inventory_with_most_recent_prices() - # if tcg low price is < 0.35, set my_price to 0.35 - # if either tcg low price or tcg low price with shipping is under 5, set my_price to tcg low price * 1.25 - # if tcg low price with shipping is > 25 set price to tcg low price with shipping * 1.025 - # otherwise, set price to tcg low price with shipping * 1.10 - # also round to 2 decimal places - df['my_price'] = df.apply(lambda row: round( - 0.35 if row['pricing_tcg_low_price'] < 0.35 else - row['pricing_tcg_low_price'] * 1.25 if row['pricing_tcg_low_price'] < 5 or row['pricing_tcg_low_price_with_shipping'] < 5 else - row['pricing_tcg_low_price_with_shipping'] * 1.025 if row['pricing_tcg_low_price_with_shipping'] > 25 else - row['pricing_tcg_low_price_with_shipping'] * 1.10, 2), axis=1) - # log rows with no price - no_price = df[df['my_price'].isnull()] - if len(no_price) > 0: - logger.warning(f"Found {len(no_price)} rows with no price") - logger.warning(no_price) - # remove rows with no price - df = df.dropna(subset=['my_price']) - return df - - def convert_df_to_csv(self, df: pd.DataFrame): - # Flip the mapping to be from current names TO desired names - column_mapping = { - 'inventory_tcgplayer_id': 'TCGplayer Id', - 'inventory_product_line': 'Product Line', - 'inventory_set_name': 'Set Name', - 'inventory_product_name': 'Product Name', - 'inventory_title': 'Title', - 'inventory_number': 'Number', - 'inventory_rarity': 'Rarity', - 'inventory_condition': 'Condition', - 'pricing_tcg_market_price': 'TCG Market Price', - 'pricing_tcg_direct_low': 'TCG Direct Low', - 'pricing_tcg_low_price_with_shipping': 'TCG Low Price With Shipping', - 'pricing_tcg_low_price': 'TCG Low Price', - 'inventory_total_quantity': 'Total Quantity', - 'inventory_add_to_quantity': 'Add to Quantity', - 'my_price': 'TCG Marketplace Price', - 'inventory_photo_url': 'Photo URL' - } - - df['pricing_tcg_market_price'] = "" - df['pricing_tcg_direct_low'] = "" - df['pricing_tcg_low_price_with_shipping'] = "" - df['pricing_tcg_low_price'] = "" - df['inventory_total_quantity'] = "" - df['inventory_add_to_quantity'] = 0 - df['inventory_photo_url'] = "" - - # First select the columns we want (using the keys of our mapping) - # Then rename them to the desired names (the values in our mapping) - df = df[column_mapping.keys()].rename(columns=column_mapping) - - return df.to_csv(index=False, quoting=1, quotechar='"') - - def convert_add_df_to_csv(self, df: pd.DataFrame): - column_mapping = { - 'tcgproduct_tcgplayer_id': 'TCGplayer Id', - 'tcgproduct_product_line': 'Product Line', - 'tcgproduct_set_name': 'Set Name', - 'tcgproduct_product_name': 'Product Name', - 'tcgproduct_title': 'Title', - 'tcgproduct_number': 'Number', - 'tcgproduct_rarity': 'Rarity', - 'tcgproduct_condition': 'Condition', - 'pricing_tcg_market_price': 'TCG Market Price', - 'pricing_tcg_direct_low': 'TCG Direct Low', - 'pricing_tcg_low_price_with_shipping': 'TCG Low Price With Shipping', - 'pricing_tcg_low_price': 'TCG Low Price', - 'tcgproduct_group_id': 'Total Quantity', - 'manabox_quantity': 'Add to Quantity', - 'my_price': 'TCG Marketplace Price', - 'tcgproduct_photo_url': 'Photo URL' - } - df['tcgproduct_group_id'] = "" - df['pricing_tcg_market_price'] = "" - df['pricing_tcg_direct_low'] = "" - df['pricing_tcg_low_price_with_shipping'] = "" - df['pricing_tcg_low_price'] = "" - df['tcgproduct_photo_url'] = "" - - df = df[column_mapping.keys()].rename(columns=column_mapping) - - return df.to_csv(index=False, quoting=1, quotechar='"') - - def create_live_inventory_pricing_update_csv(self, algo: Callable = None) -> str: - actual_algo = algo if algo is not None else self.default_pricing_algo - df = actual_algo() - csv = self.convert_df_to_csv(df) - return csv - - def create_add_to_tcgplayer_csv(self, box_id: str = None, upload_id: str = None, algo: Callable = None) -> str: - actual_algo = algo if algo is not None else self.default_pricing_algo - if box_id and upload_id: - raise ValueError("Cannot specify both box_id and upload_id") - elif not box_id and not upload_id: - raise ValueError("Must specify either box_id or upload_id") - elif box_id: - logger.debug("creating df") - df = self.get_box_with_most_recent_prices(box_id) - elif upload_id: - raise NotImplementedError("Not yet implemented") - df = actual_algo(df) - csv = self.convert_add_df_to_csv(df) - return csv \ No newline at end of file + \ No newline at end of file diff --git a/services/pricing_old.py b/services/pricing_old.py new file mode 100644 index 0000000..c41997d --- /dev/null +++ b/services/pricing_old.py @@ -0,0 +1,205 @@ +import logging +from typing import Callable +from db.models import TCGPlayerInventory, TCGPlayerExportHistory, TCGPlayerPricingHistory, ManaboxExportData, ManaboxTCGPlayerMapping, TCGPlayerProduct +from sqlalchemy.orm import Session +import pandas as pd +from db.utils import db_transaction +from sqlalchemy import func, and_, exists + + +logger = logging.getLogger(__name__) + +class PricingService: + def __init__(self, db: Session): + self.db = db + + def get_box_with_most_recent_prices(self, box_id: str) -> pd.DataFrame: + latest_prices = ( + self.db.query( + TCGPlayerPricingHistory.tcgplayer_product_id, + func.max(TCGPlayerPricingHistory.date_created).label('max_date') + ) + .group_by(TCGPlayerPricingHistory.tcgplayer_product_id) + .subquery('latest') # Added name to subquery + ) + + result = ( + self.db.query(ManaboxExportData, TCGPlayerPricingHistory, TCGPlayerProduct) + .join(ManaboxTCGPlayerMapping, ManaboxExportData.id == ManaboxTCGPlayerMapping.manabox_id) + .join(TCGPlayerProduct, ManaboxTCGPlayerMapping.tcgplayer_id == TCGPlayerProduct.id) + .join(TCGPlayerPricingHistory, TCGPlayerProduct.id == TCGPlayerPricingHistory.tcgplayer_product_id) + .join( + latest_prices, + and_( + TCGPlayerPricingHistory.tcgplayer_product_id == latest_prices.c.tcgplayer_product_id, + TCGPlayerPricingHistory.date_created == latest_prices.c.max_date + ) + ) + .filter(ManaboxExportData.box_id == box_id) # Removed str() conversion + .all() + ) + + logger.debug(f"Found {len(result)} rows") + + df = pd.DataFrame([{ + **{f"manabox_{k}": v for k, v in row[0].__dict__.items() if not k.startswith('_')}, + **{f"pricing_{k}": v for k, v in row[1].__dict__.items() if not k.startswith('_')}, + **{f"tcgproduct_{k}": v for k, v in row[2].__dict__.items() if not k.startswith('_')} + } for row in result]) + + return df + + def get_live_inventory_with_most_recent_prices(self) -> pd.DataFrame: + # Get latest export IDs using subqueries + latest_inventory_export = ( + self.db.query(TCGPlayerExportHistory.inventory_export_id) + .filter(TCGPlayerExportHistory.type == "live_inventory") + .order_by(TCGPlayerExportHistory.date_created.desc()) + .limit(1) + .scalar_subquery() + ) + # this is bad because latest pricing export is not guaranteed to be related to the latest inventory export + latest_pricing_export = ( + self.db.query(TCGPlayerExportHistory.pricing_export_id) + .filter(TCGPlayerExportHistory.type == "pricing") + .order_by(TCGPlayerExportHistory.date_created.desc()) + .limit(1) + .scalar_subquery() + ) + + # Join inventory and pricing data in a single query + inventory_with_pricing = ( + self.db.query(TCGPlayerInventory, TCGPlayerPricingHistory) + .join( + TCGPlayerPricingHistory, + TCGPlayerInventory.tcgplayer_product_id == TCGPlayerPricingHistory.tcgplayer_product_id + ) + .filter( + TCGPlayerInventory.export_id == latest_inventory_export, + TCGPlayerPricingHistory.export_id == latest_pricing_export + ) + .all() + ) + + # Convert to pandas DataFrame + df = pd.DataFrame([{ + # Inventory columns + **{f"inventory_{k}": v + for k, v in row[0].__dict__.items() + if not k.startswith('_')}, + # Pricing columns + **{f"pricing_{k}": v + for k, v in row[1].__dict__.items() + if not k.startswith('_')} + } for row in inventory_with_pricing]) + + return df + + def default_pricing_algo(self, df: pd.DataFrame = None): + if df is None: + logger.debug("No DataFrame provided, fetching live inventory with most recent prices") + df = self.get_live_inventory_with_most_recent_prices() + # if tcg low price is < 0.35, set my_price to 0.35 + # if either tcg low price or tcg low price with shipping is under 5, set my_price to tcg low price * 1.25 + # if tcg low price with shipping is > 25 set price to tcg low price with shipping * 1.025 + # otherwise, set price to tcg low price with shipping * 1.10 + # also round to 2 decimal places + df['my_price'] = df.apply(lambda row: round( + 0.35 if row['pricing_tcg_low_price'] < 0.35 else + row['pricing_tcg_low_price'] * 1.25 if row['pricing_tcg_low_price'] < 5 or row['pricing_tcg_low_price_with_shipping'] < 5 else + row['pricing_tcg_low_price_with_shipping'] * 1.025 if row['pricing_tcg_low_price_with_shipping'] > 25 else + row['pricing_tcg_low_price_with_shipping'] * 1.10, 2), axis=1) + # log rows with no price + no_price = df[df['my_price'].isnull()] + if len(no_price) > 0: + logger.warning(f"Found {len(no_price)} rows with no price") + logger.warning(no_price) + # remove rows with no price + df = df.dropna(subset=['my_price']) + return df + + def convert_df_to_csv(self, df: pd.DataFrame): + # Flip the mapping to be from current names TO desired names + column_mapping = { + 'inventory_tcgplayer_id': 'TCGplayer Id', + 'inventory_product_line': 'Product Line', + 'inventory_set_name': 'Set Name', + 'inventory_product_name': 'Product Name', + 'inventory_title': 'Title', + 'inventory_number': 'Number', + 'inventory_rarity': 'Rarity', + 'inventory_condition': 'Condition', + 'pricing_tcg_market_price': 'TCG Market Price', + 'pricing_tcg_direct_low': 'TCG Direct Low', + 'pricing_tcg_low_price_with_shipping': 'TCG Low Price With Shipping', + 'pricing_tcg_low_price': 'TCG Low Price', + 'inventory_total_quantity': 'Total Quantity', + 'inventory_add_to_quantity': 'Add to Quantity', + 'my_price': 'TCG Marketplace Price', + 'inventory_photo_url': 'Photo URL' + } + + df['pricing_tcg_market_price'] = "" + df['pricing_tcg_direct_low'] = "" + df['pricing_tcg_low_price_with_shipping'] = "" + df['pricing_tcg_low_price'] = "" + df['inventory_total_quantity'] = "" + df['inventory_add_to_quantity'] = 0 + df['inventory_photo_url'] = "" + + # First select the columns we want (using the keys of our mapping) + # Then rename them to the desired names (the values in our mapping) + df = df[column_mapping.keys()].rename(columns=column_mapping) + + return df.to_csv(index=False, quoting=1, quotechar='"') + + def convert_add_df_to_csv(self, df: pd.DataFrame): + column_mapping = { + 'tcgproduct_tcgplayer_id': 'TCGplayer Id', + 'tcgproduct_product_line': 'Product Line', + 'tcgproduct_set_name': 'Set Name', + 'tcgproduct_product_name': 'Product Name', + 'tcgproduct_title': 'Title', + 'tcgproduct_number': 'Number', + 'tcgproduct_rarity': 'Rarity', + 'tcgproduct_condition': 'Condition', + 'pricing_tcg_market_price': 'TCG Market Price', + 'pricing_tcg_direct_low': 'TCG Direct Low', + 'pricing_tcg_low_price_with_shipping': 'TCG Low Price With Shipping', + 'pricing_tcg_low_price': 'TCG Low Price', + 'tcgproduct_group_id': 'Total Quantity', + 'manabox_quantity': 'Add to Quantity', + 'my_price': 'TCG Marketplace Price', + 'tcgproduct_photo_url': 'Photo URL' + } + df['tcgproduct_group_id'] = "" + df['pricing_tcg_market_price'] = "" + df['pricing_tcg_direct_low'] = "" + df['pricing_tcg_low_price_with_shipping'] = "" + df['pricing_tcg_low_price'] = "" + df['tcgproduct_photo_url'] = "" + + df = df[column_mapping.keys()].rename(columns=column_mapping) + + return df.to_csv(index=False, quoting=1, quotechar='"') + + def create_live_inventory_pricing_update_csv(self, algo: Callable = None) -> str: + actual_algo = algo if algo is not None else self.default_pricing_algo + df = actual_algo() + csv = self.convert_df_to_csv(df) + return csv + + def create_add_to_tcgplayer_csv(self, box_id: str = None, upload_id: str = None, algo: Callable = None) -> str: + actual_algo = algo if algo is not None else self.default_pricing_algo + if box_id and upload_id: + raise ValueError("Cannot specify both box_id and upload_id") + elif not box_id and not upload_id: + raise ValueError("Must specify either box_id or upload_id") + elif box_id: + logger.debug("creating df") + df = self.get_box_with_most_recent_prices(box_id) + elif upload_id: + raise NotImplementedError("Not yet implemented") + df = actual_algo(df) + csv = self.convert_add_df_to_csv(df) + return csv \ No newline at end of file diff --git a/services/product.py b/services/product.py index 9d09ebc..aa9f803 100644 --- a/services/product.py +++ b/services/product.py @@ -5,6 +5,7 @@ from io import StringIO import pandas as pd from services.file import FileService from services.tcgplayer import TCGPlayerService +from services.storage import StorageService from uuid import uuid4 as uuid import logging @@ -26,10 +27,11 @@ class ManaboxRow: self.quantity = row['quantity'] class ProductService: - def __init__(self, db: Session, file_service: FileService, tcgplayer_service: TCGPlayerService): + def __init__(self, db: Session, file_service: FileService, tcgplayer_service: TCGPlayerService, storage_service: StorageService): self.db = db self.file_service = file_service self.tcgplayer_service = tcgplayer_service + self.storage_service = storage_service def _format_manabox_df(self, df: pd.DataFrame) -> pd.DataFrame: # format columns @@ -140,7 +142,9 @@ class ProductService: df = self._manabox_file_to_df(file) for index, row in df.iterrows(): manabox_row = ManaboxRow(row) + # create card concepts - manabox, tcgplayer, card, product card_manabox = self.card_manabox_lookup_create_if_not_exist(manabox_row) + # create staged inventory with quantity for processing down the marketplace pipeline staged_product = self.create_staged_product(file, card_manabox, row) # update file status with db_transaction(self.db): @@ -148,4 +152,10 @@ class ProductService: except Exception as e: with db_transaction(self.db): file.status = 'error' - raise e \ No newline at end of file + raise e + try: + # create storage records for physically storing individual cards + self.storage_service.store_staged_products_for_file(file.id) + except Exception as e: + logger.error(f"Error creating storage records: {str(e)}") + raise e diff --git a/services/storage.py b/services/storage.py new file mode 100644 index 0000000..d4ea90f --- /dev/null +++ b/services/storage.py @@ -0,0 +1,176 @@ +from sqlalchemy.orm import Session +from db.utils import db_transaction +from db.models import Warehouse, User, StagedFileProduct, StorageBlock, ProductBlock, File, Card +from uuid import uuid4 as uuid +from typing import List, TypedDict + +class ProductAttributes(TypedDict): + product_id: str + card_number: str + +class StorageService: + def __init__(self, db: Session): + self.db = db + + def get_or_create_user(self, username: str) -> User: + user = self.db.query(User).filter(User.username == username).first() + if user is None: + user = User( + id = str(uuid()), + username = username + ) + with db_transaction(self.db): + self.db.add(user) + return user + + def get_or_create_warehouse(self) -> Warehouse: + warehouse = self.db.query(Warehouse).first() + user = self.get_or_create_user('admin') + if warehouse is None: + warehouse = Warehouse( + id = str(uuid()), + user_id = user.id + ) + with db_transaction(self.db): + self.db.add(warehouse) + return warehouse + + def get_staged_product(self, file_id: str) -> List[StagedFileProduct]: + staged_product = self.db.query(StagedFileProduct).filter(StagedFileProduct.file_id == file_id).all() + return staged_product + + def get_storage_block_name(self, warehouse: Warehouse, file_id: str) -> str: + # Get file type from id + current_file = self.db.query(File).filter(File.id == file_id).first() + if not current_file: + raise ValueError(f"No file found with id {file_id}") + + # Determine storage block type + storage_block_type = 'rare' if 'rare' in current_file.type else 'common' + prefix = storage_block_type[0] + + # Get most recent storage block with same type and warehouse id + latest_block = ( + self.db.query(StorageBlock) + .filter( + StorageBlock.warehouse_id == warehouse.id, + StorageBlock.type == storage_block_type + ) + .order_by(StorageBlock.date_created.desc()) + .first() + ) + + # If no existing block, start with number 1 + if not latest_block: + return f"{prefix}1" + + # Start with the next number after the latest block + number = int(latest_block.name[1:]) + + while True: + number += 1 + new_name = f"{prefix}{number}" + + # Check if the new name already exists + exists = ( + self.db.query(StorageBlock) + .filter( + StorageBlock.warehouse_id == warehouse.id, + StorageBlock.name == new_name + ) + .first() + ) + + if not exists: + return new_name + + def create_storage_block(self, warehouse: Warehouse, file_id: str) -> StorageBlock: + current_file = self.db.query(File).filter(File.id == file_id).first() + if not current_file: + raise ValueError(f"No file found with id {file_id}") + + storage_block_type = 'rare' if 'rare' in current_file.type else 'common' + + storage_block = StorageBlock( + id = str(uuid()), + warehouse_id = warehouse.id, + name = self.get_storage_block_name(warehouse, file_id), + type = storage_block_type + ) + with db_transaction(self.db): + self.db.add(storage_block) + return storage_block + + def add_staged_product_to_product_block( + self, + staged_product: StagedFileProduct, + storage_block: StorageBlock, + product_attributes: ProductAttributes, + block_index: int + ) -> ProductBlock: + """Create a new ProductBlock for a single unit of a staged product.""" + product_block = ProductBlock( + id=str(uuid()), + product_id=staged_product.product_id, + block_id=storage_block.id, + block_index=block_index + ) + + with db_transaction(self.db): + self.db.add(product_block) + + return product_block + + def get_staged_product_attributes_for_storage( + self, + staged_product: StagedFileProduct + ) -> List[ProductAttributes]: + """Get attributes for each unit of a staged product.""" + result = ( + self.db.query( + StagedFileProduct.product_id, + StagedFileProduct.quantity, + Card.number + ) + .join(Card, Card.product_id == StagedFileProduct.product_id) + .filter(StagedFileProduct.id == staged_product.id) + .first() + ) + + if not result: + return [] + + return [ + ProductAttributes( + product_id=result.product_id, + card_number=result.number + ) + for i in range(result.quantity) + ] + + def store_staged_products_for_file(self, file_id: str) -> StorageBlock: + """Store all staged products for a file in a new storage block.""" + warehouse = self.get_or_create_warehouse() + storage_block = self.create_storage_block(warehouse, file_id) + staged_products = self.get_staged_product(file_id) + + # Collect all product attributes first + all_product_attributes = [] + for staged_product in staged_products: + product_attributes_list = self.get_staged_product_attributes_for_storage(staged_product) + for attrs in product_attributes_list: + all_product_attributes.append((staged_product, attrs)) + + # Sort by card number as integer to determine block indices + sorted_attributes = sorted(all_product_attributes, key=lambda x: int(x[1]['card_number'])) + + # Add products with correct block indices + for block_index, (staged_product, product_attributes) in enumerate(sorted_attributes, 1): + self.add_staged_product_to_product_block( + staged_product=staged_product, + storage_block=storage_block, + product_attributes=product_attributes, + block_index=block_index + ) + + return storage_block \ No newline at end of file diff --git a/services/task.py b/services/task.py index 226ef68..d5e71d8 100644 --- a/services/task.py +++ b/services/task.py @@ -3,16 +3,18 @@ import logging from typing import Dict, Callable from sqlalchemy.orm import Session from services.product import ProductService +from services.tcgplayer import TCGPlayerService from db.models import File class TaskService: - def __init__(self, db: Session, product_service: ProductService): + def __init__(self, db: Session, product_service: ProductService, tcgplayer_service: TCGPlayerService): self.scheduler = BackgroundScheduler() self.logger = logging.getLogger(__name__) self.tasks: Dict[str, Callable] = {} self.db = db self.product_service = product_service + self.tcgplayer_service = tcgplayer_service async def start(self): self.scheduler.start() @@ -27,12 +29,21 @@ class TaskService: minute=0, id='daily_report' ) + + self.scheduler.add_job( + self.pricing_update, + 'cron', + minute=28, + id='pricing_update' + ) - # Tasks that should be scheduled - async def daily_report(self): + def daily_report(self): # Removed async self.logger.info("Generating daily report") # Daily report logic - + + def pricing_update(self): # Removed async + self.logger.info("Hourly pricing update") + self.tcgplayer_service.cron_load_prices() async def process_manabox_file(self, file: File): self.logger.info("Processing ManaBox file") diff --git a/services/tcgplayer.py b/services/tcgplayer.py index d08575c..e1b8ac6 100644 --- a/services/tcgplayer.py +++ b/services/tcgplayer.py @@ -1,4 +1,4 @@ -from db.models import ManaboxExportData, Box, TCGPlayerGroups, TCGPlayerInventory, TCGPlayerExportHistory, TCGPlayerPricingHistory, TCGPlayerProduct, ManaboxTCGPlayerMapping, CardManabox, CardTCGPlayer +from db.models import ManaboxExportData, Box, TCGPlayerGroups, TCGPlayerInventory, TCGPlayerExportHistory, TCGPlayerPricingHistory, TCGPlayerProduct, ManaboxTCGPlayerMapping, CardManabox, CardTCGPlayer, Price import requests from sqlalchemy.orm import Session from db.utils import db_transaction @@ -18,6 +18,8 @@ from typing import List, Dict, Optional from io import StringIO, BytesIO from services.pricing import PricingService from sqlalchemy.sql import exists +import pandas as pd +from sqlalchemy.exc import SQLAlchemyError logger = logging.getLogger(__name__) @@ -559,6 +561,115 @@ class TCGPlayerService: return None return matching_product - - + def get_pricing_export_for_all_products(self) -> bytes: + """ + Retrieves pricing export data for all products in TCGPlayer format. + Returns: + bytes: Raw CSV data containing pricing information + """ + try: + all_group_ids = self.db.query(TCGPlayerGroups.group_id).all() + all_group_ids = [str(group_id) for group_id, in all_group_ids] + export_csv = self._get_export_csv(all_group_ids, convert=False) + return export_csv + except SQLAlchemyError as e: + raise RuntimeError(f"Failed to retrieve group IDs: {str(e)}") + + def pricing_export_to_df(self, export_csv: bytes) -> pd.DataFrame: + """ + Converts raw CSV pricing data to a pandas DataFrame. + + Args: + export_csv (bytes): Raw CSV data in bytes format + + Returns: + pd.DataFrame: Processed pricing data + + Raises: + ValueError: If no CSV data is provided or if CSV parsing fails + """ + if not export_csv: + raise ValueError("No export CSV provided") + + csv_file = None + try: + text_content = export_csv.decode('utf-8') + csv_file = StringIO(text_content) + df = pd.read_csv(csv_file) + + if df.empty: + raise ValueError("CSV data is empty") + + return df + except UnicodeDecodeError as e: + raise ValueError(f"Failed to decode CSV data: {str(e)}") + except pd.errors.EmptyDataError: + raise ValueError("CSV file is empty or malformed") + finally: + if csv_file: + csv_file.close() + + def cron_load_prices(self) -> None: + """ + Scheduled task to load and update product prices. + Fetches current prices, processes them, and saves new price records to the database. + """ + try: + # Get and process price data + price_csv = self.get_pricing_export_for_all_products() + price_df = self.pricing_export_to_df(price_csv) + + # Clean column names + price_df.columns = price_df.columns.str.lower().str.replace(' ', '_') + + # Get all products efficiently + products_query = self.db.query( + CardTCGPlayer.tcgplayer_id, + CardTCGPlayer.product_id + ) + product_df = pd.read_sql(products_query.statement, self.db.bind) + + # Merge dataframes + merged_df = pd.merge( + price_df, + product_df, + on='tcgplayer_id', + how='inner' + ) + + # Define price columns to process + price_columns = [ + 'tcg_market_price', + 'tcg_direct_low', + 'tcg_low_price_with_shipping', + 'tcg_low_price', + 'tcg_marketplace_price' + ] + + # Process in batches to avoid memory issues + BATCH_SIZE = 1000 + for price_column in price_columns: + records = [] + + for start_idx in range(0, len(merged_df), BATCH_SIZE): + batch_df = merged_df.iloc[start_idx:start_idx + BATCH_SIZE] + + batch_records = [ + Price( + id=str(uuid.uuid4()), + product_id=row['product_id'], + type=price_column, + price=row[price_column] + ) + for _, row in batch_df.iterrows() + if pd.notna(row[price_column]) # Skip rows with NaN prices + ] + + with db_transaction(self.db): + self.db.bulk_save_objects(batch_records) + self.db.flush() + + except Exception as e: + logger.error(f"Failed to load prices: {str(e)}") + raise \ No newline at end of file diff --git a/tests/box_test.py b/tests/box_test.py new file mode 100644 index 0000000..442e116 --- /dev/null +++ b/tests/box_test.py @@ -0,0 +1,154 @@ +from fastapi.testclient import TestClient +from fastapi import BackgroundTasks +import pytest +from unittest.mock import Mock, patch +import asyncio +import os +from main import app +from services.file import FileService +from services.task import TaskService + + + +client = TestClient(app) + +test_boxes = [] + +def test_create_box(): + # Send as form data, not JSON + response = client.post("/api/boxes", + data={ + "type": "play", + "set_code": "BLB", + "sku": "1234", + "num_cards_expected": 504 + } + ) + test_boxes.append(response.json()["box"][0]["product_id"]) + + assert response.status_code == 201 + assert response.json()["success"] == True + assert response.json()["box"][0]["type"] == "play" + assert response.json()["box"][0]["set_code"] == "BLB" + assert response.json()["box"][0]["sku"] == "1234" + assert response.json()["box"][0]["num_cards_expected"] == 504 + +def test_update_box(): + # Create a box first + create_response = client.post("/api/boxes", + data={ + "type": "collector", + "set_code": "MKM", + "sku": "3456", + "num_cards_expected": 504 + } + ) + box_id = create_response.json()["box"][0]["product_id"] + test_boxes.append(box_id) + + # Update the box + response = client.put(f"/api/boxes/{box_id}", + data={ + "num_cards_expected": 500 + } + ) + + assert response.status_code == 200 + assert response.json()["success"] == True + assert response.json()["box"][0]["type"] == "collector" + assert response.json()["box"][0]["set_code"] == "MKM" + assert response.json()["box"][0]["sku"] == "3456" + assert response.json()["box"][0]["num_cards_expected"] == 500 + +def test_delete_box(): + # Create a box first + create_response = client.post("/api/boxes", + data={ + "type": "set", + "set_code": "LCI", + "sku": "7890", + "num_cards_expected": 504 + } + ) + box_id = create_response.json()["box"][0]["product_id"] + + # Delete the box + response = client.delete(f"/api/boxes/{box_id}") + + assert response.status_code == 200 + assert response.json()["success"] == True + assert response.json()["box"][0]["type"] == "set" + assert response.json()["box"][0]["set_code"] == "LCI" + assert response.json()["box"][0]["sku"] == "7890" + assert response.json()["box"][0]["num_cards_expected"] == 504 + +# Constants for reused values +TEST_FILE_PATH = os.path.join(os.getcwd(), "tests/test_files", "manabox_test_file.csv") +DEFAULT_METADATA = { + "source": "manabox", + "type": "scan_export_common" +} + +def get_file_size_kb(file_path): + """Helper to consistently calculate file size in KB""" + with open(file_path, "rb") as f: + return round(len(f.read()) / 1024, 2) + +@pytest.mark.asyncio +async def test_open_box(): + """Test creating a new manabox file""" + # Open file within the test scope + with open(TEST_FILE_PATH, "rb") as test_file: + files = {"file": test_file} + + # Make request + response = client.post("/api/files", data=DEFAULT_METADATA, files=files) + + # Check response + assert response.status_code == 201 + assert response.json()["success"] == True + + file_data = response.json()["files"][0] + assert file_data["source"] == DEFAULT_METADATA["source"] + assert file_data["type"] == DEFAULT_METADATA["type"] + assert file_data["status"] == "pending" + assert file_data["service"] == None + assert file_data["filename"] == "manabox_test_file.csv" + assert file_data["filesize_kb"] == get_file_size_kb(TEST_FILE_PATH) + assert file_data["id"] is not None + + # Execute background tasks if they were added + background_tasks = BackgroundTasks() + for task in background_tasks.tasks: + await task() + + # Create a box first + create_response = client.post("/api/boxes", + data={ + "type": "play", + "set_code": "OTJ", + "sku": "2314", + "num_cards_expected": 504 + } + ) + box_id = create_response.json()["box"][0]["product_id"] + test_boxes.append(box_id) + + # Open the box + response = client.post(f"/api/boxes/{box_id}/open", + data={ + "product_id": box_id, + "file_ids": [file_data["id"]], + "num_cards_actual": 500 + } + ) + + assert response.status_code == 201 + assert response.json()["success"] == True + + +def test_cleanup(): + # Delete all boxes created during testing + for box_id in test_boxes: + client.delete(f"/api/boxes/{box_id}") + \ No newline at end of file diff --git a/tests/file_test.py b/tests/file_test.py index 9a8d282..2378d52 100644 --- a/tests/file_test.py +++ b/tests/file_test.py @@ -14,7 +14,7 @@ client = TestClient(app) TEST_FILE_PATH = os.path.join(os.getcwd(), "tests/test_files", "manabox_test_file.csv") DEFAULT_METADATA = { "source": "manabox", - "type": "scan_export" + "type": "scan_export_rare" } def get_file_size_kb(file_path):