prep for pricing service work

.gitignore

@@ -171,4 +171,5 @@ cython_debug/
 # my stuff 
 *.db
 temp/
-.DS_Store
+.DS_Store
+*.db-journal

db/database.py

@@ -3,6 +3,10 @@ from sqlalchemy.orm import sessionmaker, Session
 from contextlib import contextmanager
 from typing import Generator
 import os
+from sqlalchemy import inspect
+from services.tcgplayer import TCGPlayerService
+#from services.pricing import PricingService
+from services.file import FileService
 
 
 import logging
@@ -45,11 +49,33 @@ def get_db() -> Generator[Session, None, None]:
         yield session
 
 def init_db() -> None:
-    """Initialize database tables"""
+    """Initialize database tables and run first-time setup if needed"""
     from .models import Base
     try:
+        inspector = inspect(engine)
+        tables_exist = all(
+            table in inspector.get_table_names()
+            for table in Base.metadata.tables.keys()
+        )
+        # if tables_exist:
+            # drop all tables except file
+            # for table in inspector.get_table_names():
+                # if table != 'files':
+                    # Base.metadata.drop_all(bind=engine, tables=[Base.metadata.tables[table]])
+                    # logger.info(f"Dropped table: {table}")
+        
+        # Create tables if they don't exist
         Base.metadata.create_all(bind=engine)
-        logger.info("Database tables created successfully")
+        
+        # Run first-time setup only if tables were just created
+        if not tables_exist:
+#            with get_db_session() as session:
+#                tcgplayer_service = TCGPlayerService(session, PricingService(session), FileService(session))
+#                tcgplayer_service.populate_tcgplayer_groups()
+#                tcgplayer_service.cron_load_prices()
+            logger.info("First-time database setup completed")
+        
+        logger.info("Database initialization completed")
     except Exception as e:
         logger.error(f"Failed to initialize database: {str(e)}")
         raise

db/models.py

@@ -31,11 +31,8 @@ class Product(Base):
         return product_line
 
     id = Column(String, primary_key=True)
-    name = Column(String)
     type = Column(String) # box or card
     product_line = Column(String) # pokemon, mtg, etc.
-    set_name = Column(String)
-    set_code = Column(String)
     date_created = Column(DateTime, default=datetime.now)
     date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
 
@@ -133,43 +130,10 @@ class Card(Base):
     """
     Card represents the concept of a distinct card
     Cards have metadata from different sources
-    internal: box, inventory, upload
-    external: price, attributes - scryfall, tcgplayer, manabox
     """
     __tablename__ = "cards"
 
-    @validates("rarity")
-    def validate_rarity(self, key, rarity: str):
-        single_character_rarity = {'m': 'mythic', 'r': 'rare', 'u': 'uncommon', 'c': 'common', 'l': 'land', 'p': 'promo', 's': 'special'}
-        if rarity not in RarityEnum:
-            if rarity.lower() in RarityEnum:
-                rarity = rarity.lower()
-            elif rarity in single_character_rarity:
-                rarity = single_character_rarity[rarity]
-            else:
-                raise ValueError(f"Invalid rarity: {rarity}")
-        return rarity
-    
-    @validates("condition")
-    def validate_condition(self, key, condition: str):
-        if condition not in ConditionEnum:
-            if condition.lower() in ConditionEnum:
-                condition = condition.lower()
-            elif condition.lower().strip().replace(' ', '_') in ConditionEnum:
-                condition = condition.lower().strip().replace(' ', '_')
-            else:
-                raise ValueError(f"Invalid condition: {condition}")
-        return condition
-
     product_id = Column(String, ForeignKey("products.id"), primary_key=True)
-    number = Column(String)
-    foil = Column(String)
-    rarity = Column(String)
-    condition = Column(String)
-    language = Column(String)
-    scryfall_id = Column(String)
-    manabox_id = Column(String)
-    tcgplayer_id = Column(Integer)
     date_created = Column(DateTime, default=datetime.now)
     date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
 
@@ -183,7 +147,7 @@ class CardManabox(Base):
     collector_number = Column(String)
     foil = Column(String)
     rarity = Column(String)
-    manabox_id = Column(String)
+    manabox_id = Column(Integer)
     scryfall_id = Column(String)
     condition = Column(String)
     language = Column(String)

dependencies.py

@@ -1,6 +1,7 @@
+from typing import Annotated
 from sqlalchemy.orm import Session
-from services.data import DataService
+from fastapi import Depends, Form
-from services.upload import UploadService
+
 from services.box import BoxService
 from services.tcgplayer import TCGPlayerService
 from services.pricing import PricingService
@@ -9,105 +10,116 @@ from services.product import ProductService
 from services.inventory import InventoryService
 from services.task import TaskService
 from services.storage import StorageService
-from fastapi import Depends, Form
 from db.database import get_db
 from schemas.file import CreateFileRequest
 from schemas.box import CreateBoxRequest, UpdateBoxRequest, CreateOpenBoxRequest
 
+# Common type annotation for database dependency
+DB = Annotated[Session, Depends(get_db)]
 
-## file 
+# Base Services (no dependencies besides DB)
-# file
+def get_file_service(db: DB) -> FileService:
-def get_file_service(db: Session = Depends(get_db)) -> FileService:
+    """FileService with only database dependency"""
-    """Dependency injection for FileService"""
     return FileService(db)
 
-# metadata
+def get_pricing_service(db: DB) -> PricingService:
-def get_create_file_metadata(
+    """PricingService with only database dependency"""
-        type: str = Form(...),
-        source: str = Form(...),
-        service: str = Form(None),
-        filename: str = Form(None)
-    ) -> CreateFileRequest:
-    """Dependency injection for FileMetadata"""
-    return CreateFileRequest(type=type, source=source, service=service, filename=filename)
-
-def get_box_data(
-        type: str = Form(...),
-        sku: str = Form(None),
-        set_code: str = Form(...),
-        num_cards_expected: int = Form(None)
-    ) -> CreateBoxRequest:
-    """Dependency injection for BoxData"""
-    return CreateBoxRequest(type=type, sku=sku, set_code=set_code, num_cards_expected=num_cards_expected)
-
-def get_box_update_data(
-        type: str = Form(None),
-        sku: str = Form(None),
-        set_code: str = Form(None),
-        num_cards_expected: int = Form(None)
-    ) -> UpdateBoxRequest:
-    """Dependency injection for BoxUpdateData"""
-    return UpdateBoxRequest(type=type, sku=sku, set_code=set_code, num_cards_expected=num_cards_expected)
-
-def get_open_box_data(
-        product_id: str = Form(...),
-        file_ids: list[str] = Form(None),
-        num_cards_actual: int = Form(None),
-        date_opened: str = Form(None)
-    ) -> CreateOpenBoxRequest:
-    """Dependency injection for OpenBoxData"""
-    return CreateOpenBoxRequest(product_id=product_id, file_ids=file_ids, num_cards_actual=num_cards_actual, date_opened=date_opened)
-
-def get_tcgplayer_service(
-        db: Session = Depends(get_db)
-        ) -> TCGPlayerService:
-    """Dependency injection for TCGPlayerService"""
-    return TCGPlayerService(db)
-
-# storage
-
-def get_storage_service(db: Session = Depends(get_db)) -> StorageService:
-    """Dependency injection for StorageService"""
-    return StorageService(db)
-
-# product
-def get_product_service(db: Session = Depends(get_db), file_service: FileService = Depends(get_file_service), tcgplayer_service: TCGPlayerService = Depends(get_tcgplayer_service), storage_service: StorageService = Depends(get_storage_service)) -> ProductService:
-    """Dependency injection for ProductService"""
-    return ProductService(db, file_service, tcgplayer_service, storage_service)
-
-# task
-def get_task_service(db: Session = Depends(get_db), product_service: ProductService = Depends(get_product_service), tcgplayer_service: TCGPlayerService = Depends(get_tcgplayer_service)) -> TaskService:
-    """Dependency injection for TaskService"""
-    return TaskService(db, product_service, tcgplayer_service)
-
-## Inventory
-def get_inventory_service(db: Session = Depends(get_db)) -> InventoryService:
-    """Dependency injection for InventoryService"""
-    return InventoryService(db)
-
-## Upload
-
-def get_upload_service(db: Session = Depends(get_db)) -> UploadService:
-    """Dependency injection for UploadService"""
-    return UploadService(db)
-
-## box
-
-def get_box_service(db: Session = Depends(get_db), inventory_service: InventoryService = Depends(get_inventory_service)) -> BoxService:
-    """Dependency injection for BoxService"""
-    return BoxService(db, inventory_service)
-
-## Pricing
-
-def get_pricing_service(db: Session = Depends(get_db)) -> PricingService:
-    """Dependency injection for PricingService"""
     return PricingService(db)
 
-## Data
+def get_storage_service(db: DB) -> StorageService:
-def get_data_service(
+    """StorageService with only database dependency"""
-        db: Session = Depends(get_db), 
+    return StorageService(db)
-        tcgplayer_service: TCGPlayerService = Depends(get_tcgplayer_service)
-        ) -> DataService:
-    """Dependency injection for DataService"""
-    return DataService(db, tcgplayer_service)
 
+def get_inventory_service(db: DB) -> InventoryService:
+    """InventoryService with only database dependency"""
+    return InventoryService(db)
+
+# Services with dependencies on other services
+def get_tcgplayer_service(
+    db: DB,
+    pricing_service: Annotated[PricingService, Depends(get_pricing_service)],
+    file_service: Annotated[FileService, Depends(get_file_service)]
+) -> TCGPlayerService:
+    """TCGPlayerService depends on PricingService"""
+    return TCGPlayerService(db, pricing_service, file_service)
+
+def get_product_service(
+    db: DB,
+    file_service: Annotated[FileService, Depends(get_file_service)],
+    tcgplayer_service: Annotated[TCGPlayerService, Depends(get_tcgplayer_service)],
+    storage_service: Annotated[StorageService, Depends(get_storage_service)]
+) -> ProductService:
+    """ProductService with multiple service dependencies"""
+    return ProductService(db, file_service, tcgplayer_service, storage_service)
+
+def get_box_service(
+    db: DB,
+    inventory_service: Annotated[InventoryService, Depends(get_inventory_service)]
+) -> BoxService:
+    """BoxService depends on InventoryService"""
+    return BoxService(db, inventory_service)
+
+def get_task_service(
+    db: DB,
+    product_service: Annotated[ProductService, Depends(get_product_service)],
+    tcgplayer_service: Annotated[TCGPlayerService, Depends(get_tcgplayer_service)]
+) -> TaskService:
+    """TaskService depends on ProductService and TCGPlayerService"""
+    return TaskService(db, product_service, tcgplayer_service)
+
+# Form data dependencies
+def get_create_file_metadata(
+    type: str = Form(...),
+    source: str = Form(...),
+    service: str = Form(None),
+    filename: str = Form(None)
+) -> CreateFileRequest:
+    """Form dependency for file creation"""
+    return CreateFileRequest(
+        type=type,
+        source=source,
+        service=service,
+        filename=filename
+    )
+
+def get_box_data(
+    type: str = Form(...),
+    sku: str = Form(None),
+    set_code: str = Form(...),
+    num_cards_expected: int = Form(None)
+) -> CreateBoxRequest:
+    """Form dependency for box creation"""
+    return CreateBoxRequest(
+        type=type,
+        sku=sku,
+        set_code=set_code,
+        num_cards_expected=num_cards_expected
+    )
+
+def get_box_update_data(
+    type: str = Form(None),
+    sku: str = Form(None),
+    set_code: str = Form(None),
+    num_cards_expected: int = Form(None)
+) -> UpdateBoxRequest:
+    """Form dependency for box updates"""
+    return UpdateBoxRequest(
+        type=type,
+        sku=sku,
+        set_code=set_code,
+        num_cards_expected=num_cards_expected
+    )
+
+def get_open_box_data(
+    product_id: str = Form(...),
+    file_ids: list[str] = Form(None),
+    num_cards_actual: int = Form(None),
+    date_opened: str = Form(None)
+) -> CreateOpenBoxRequest:
+    """Form dependency for opening boxes"""
+    return CreateOpenBoxRequest(
+        product_id=product_id,
+        file_ids=file_ids,
+        num_cards_actual=num_cards_actual,
+        date_opened=date_opened
+    )

main.py

@@ -1,25 +1,30 @@
-from fastapi import FastAPI
+from fastapi import FastAPI, Depends
 from fastapi.middleware.cors import CORSMiddleware
 import uvicorn
 from routes.routes import router
 from db.database import init_db, check_db_connection, destroy_db, get_db
 from db.utils import db_transaction
-from services.task import TaskService
 import logging
 import sys
-from services.tcgplayer import TCGPlayerService, PricingService
-from services.product import ProductService
-from services.file import FileService
-from services.storage import StorageService
 from db.models import TCGPlayerGroups
+from sqlalchemy.orm import Session
 
+# Import your dependency functions
+from dependencies import (
+    get_task_service,
+    get_tcgplayer_service,
+    get_pricing_service,
+    get_file_service,
+    get_product_service,
+    get_storage_service,
+)
 
 logging.basicConfig(
     level=logging.DEBUG,
     format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
     handlers=[
         logging.StreamHandler(sys.stdout),
-        logging.FileHandler('app.log')  # Added this line
+        logging.FileHandler('app.log')
     ]
 )
 
@@ -45,38 +50,54 @@ app.add_middleware(
 # Include routers
 app.include_router(router)
 
-# Optional: Add startup and shutdown events
 @app.on_event("startup")
 async def startup_event():
-    # Check database connection
+    try:
-    if not check_db_connection():
+        # Check database connection
-        raise Exception("Database connection failed")
+        if not check_db_connection():
-        # destroy db
+            logger.error("Database connection failed")
-    #destroy_db()
+            raise Exception("Database connection failed")
-    # Initialize database
-    init_db()
-    # get db session
-    db = next(get_db())
-    # populate tcgplayer groups
-    if db.query(TCGPlayerGroups).count() == 0:
-        with db_transaction(db):
-            tcgplayer_service = TCGPlayerService(db)
-            tcgplayer_service.populate_tcgplayer_groups()
-    # Start task service
-    task_service = TaskService(db, ProductService(db, FileService(db), TCGPlayerService(db), StorageService(db)), TCGPlayerService(db))
-    await task_service.start()
 
+        # Initialize database
+        init_db()
+        
+        # Get database session
+        db = next(get_db())
+        
+        # Use dependency injection to get services
+
+        pricing_service = get_pricing_service(db)
+        file_service = get_file_service(db)
+        storage_service = get_storage_service(db)
+        tcgplayer_service = get_tcgplayer_service(db, pricing_service, file_service)
+        product_service = get_product_service(db, file_service, tcgplayer_service, storage_service)
+        task_service = get_task_service(db, product_service, tcgplayer_service)
+        
+        # Initialize TCGPlayer groups if needed
+        if db.query(TCGPlayerGroups).count() == 0:
+            with db_transaction(db):
+                tcgplayer_service.populate_tcgplayer_groups()
+        
+        # DEBUG
+        tcgplayer_service.cron_load_prices()
+        
+        # Start task service
+        await task_service.start()
+        
+        logger.info("Application started successfully")
+        
+    except Exception as e:
+        logger.error(f"Startup failed: {str(e)}")
+        raise
 
 @app.on_event("shutdown")
 async def shutdown_event():
-    # Clean up any connections or resources
+    logger.info("Application shutting down")
     pass
 
-# Root endpoint
 @app.get("/")
 async def root():
     return {"message": "Card Management API"}
 
-# Run the application
 if __name__ == "__main__":
     uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True)

routes/routes.py

@@ -1,99 +1,102 @@
-from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Request, BackgroundTasks
+from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, BackgroundTasks
 from fastapi.responses import StreamingResponse
-from sqlalchemy.orm import Session
+from typing import Optional
-from typing import Dict, Any, List, Optional
-from db.database import get_db
-from services.upload import UploadService
-from services.box import BoxService
-from services.tcgplayer import TCGPlayerService
-from services.data import DataService
-from services.file import FileService
-from services.product import ProductService
-from services.task import TaskService
-from schemas.file import FileSchema, CreateFileRequest, CreateFileResponse, GetFileResponse, DeleteFileResponse, GetFileQueryParams
-from schemas.box import CreateBoxResponse, CreateBoxRequest, BoxSchema, UpdateBoxRequest, CreateOpenBoxRequest, CreateOpenBoxResponse, OpenBoxSchema
-from dependencies import get_data_service, get_upload_service, get_tcgplayer_service, get_box_service, get_create_file_metadata, get_file_service, get_product_service, get_task_service, get_box_data, get_box_update_data, get_open_box_data
-
-
 import logging
+
+from schemas.file import (
+    FileSchema,
+    CreateFileRequest,
+    CreateFileResponse,
+    GetFileResponse,
+    DeleteFileResponse,
+    GetFileQueryParams
+)
+from schemas.box import (
+    CreateBoxResponse,
+    CreateBoxRequest,
+    BoxSchema,
+    UpdateBoxRequest,
+    CreateOpenBoxRequest,
+    CreateOpenBoxResponse,
+    OpenBoxSchema
+)
+from services.file import FileService
+from services.box import BoxService
+from services.task import TaskService
+from dependencies import (
+    get_file_service,
+    get_box_service,
+    get_task_service,
+    get_create_file_metadata,
+    get_box_data,
+    get_box_update_data,
+    get_open_box_data
+)
+
 logger = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/api", tags=["cards"])
-MAX_FILE_SIZE = 1024 * 1024 * 100 # 100 MB
+MAX_FILE_SIZE = 100 * 1024 * 1024  # 100 MB
 
+async def validate_file_upload(file: UploadFile) -> bytes:
+    """Validate uploaded file and return its contents."""
+    if not file.filename:
+        raise HTTPException(status_code=400, detail="No filename provided")
+    
+    content = await file.read()
+    if len(content) > MAX_FILE_SIZE:
+        raise HTTPException(status_code=413, detail="File too large")
+    
+    return content
 
-
+@router.post("/files", response_model=CreateFileResponse, status_code=201)
-## GIGA FOR REAL
-## FILE
-## CREATE
-@router.post(
-    "/files",
-    response_model=CreateFileResponse,
-    status_code=201
-)
 async def create_file(
     background_tasks: BackgroundTasks,
     file: UploadFile = File(...),
     metadata: CreateFileRequest = Depends(get_create_file_metadata),
     file_service: FileService = Depends(get_file_service),
     task_service: TaskService = Depends(get_task_service)
-):
+) -> CreateFileResponse:
+    """Create a new file entry with the uploaded file."""
     try:
-        # Validate file size before reading
+        content = await validate_file_upload(file)
-        if not file.filename:
-            raise HTTPException(status_code=400, detail="No filename provided")
-            
-        # File size check
-        content = await file.read()
-        if len(content) > MAX_FILE_SIZE:
-            raise HTTPException(status_code=413, detail="File too large")
- 
         logger.debug(f"File received: {file.filename}")
         logger.debug(f"Metadata: {metadata}")
-        
+
-        # ADD FILENAME TO METADATA
+        metadata.filename = metadata.filename or file.filename
-        if not metadata.filename:
+
-            metadata.filename = file.filename
-        
-        # VALIDATE FILE
         if not file_service.validate_file(content, metadata):
             raise HTTPException(status_code=400, detail="Invalid file content")
-        
-        # STORE FILE
-        created_file = file_service.create_file(content, metadata)
-        
-        # Close file after processing
-        await file.close()
 
-        # handle manabox file background task
+        created_file = file_service.create_file(content, metadata)
+
         if metadata.source == 'manabox':
             background_tasks.add_task(task_service.process_manabox_file, created_file)
-        
+
         return CreateFileResponse(
             status_code=201,
             success=True,
-            files=[FileSchema.from_orm(created_file)]  # Changed to return list
+            files=[FileSchema.from_orm(created_file)]
         )
+
     except HTTPException as http_ex:
-        await file.close()
         raise http_ex
     except Exception as e:
-        await file.close()
         logger.error(f"File upload failed: {str(e)}")
         raise HTTPException(
             status_code=500,
             detail="Internal server error occurred during file upload"
         )
+    finally:
+        await file.close()
 
-## FILE
-## GET
 @router.get("/files/{file_id:path}", response_model=GetFileResponse)
 @router.get("/files", response_model=GetFileResponse)
 async def get_file(
     file_id: Optional[str] = None,
     query: GetFileQueryParams = Depends(),
     file_service: FileService = Depends(get_file_service)
-):
+) -> GetFileResponse:
     """
     Get file(s) by optional ID and/or status.
     If file_id is provided, returns that specific file.
@@ -102,31 +105,26 @@ async def get_file(
     """
     try:
         if file_id:
-            # Get specific file by ID
             file = file_service.get_file(file_id)
-            return GetFileResponse(
+            files = [file]
-                status_code=200,
-                success=True,
-                files=[FileSchema.from_orm(file)]
-            )
         else:
-            # Get multiple files with optional status filter
             files = file_service.get_files(status=query.status)
-            return GetFileResponse(
+
-                status_code=200,
+        return GetFileResponse(
-                success=True,
+            status_code=200,
-                files=[FileSchema.from_orm(f) for f in files]
+            success=True,
-            )
+            files=[FileSchema.from_orm(f) for f in files]
+        )
     except Exception as e:
         logger.error(f"Get file(s) failed: {str(e)}")
         raise HTTPException(status_code=400, detail=str(e))
 
-## DELETE
 @router.delete("/files/{file_id}", response_model=DeleteFileResponse)
 async def delete_file(
     file_id: str,
     file_service: FileService = Depends(get_file_service)
-):
+) -> DeleteFileResponse:
+    """Delete a file by ID."""
     try:
         file = file_service.delete_file(file_id)
         return DeleteFileResponse(
@@ -138,14 +136,12 @@ async def delete_file(
         logger.error(f"Delete file failed: {str(e)}")
         raise HTTPException(status_code=400, detail=str(e))
 
-
-## BOX
-## CREATE
 @router.post("/boxes", response_model=CreateBoxResponse, status_code=201)
 async def create_box(
     box_data: CreateBoxRequest = Depends(get_box_data),
     box_service: BoxService = Depends(get_box_service)
-):
+) -> CreateBoxResponse:
+    """Create a new box."""
     try:
         result = box_service.create_box(box_data)
         return CreateBoxResponse(
@@ -157,13 +153,13 @@ async def create_box(
         logger.error(f"Create box failed: {str(e)}")
         raise HTTPException(status_code=400, detail=str(e))
 
-## UPDATE
 @router.put("/boxes/{box_id}", response_model=CreateBoxResponse)
 async def update_box(
     box_id: str,
     box_data: UpdateBoxRequest = Depends(get_box_update_data),
     box_service: BoxService = Depends(get_box_service)
-):
+) -> CreateBoxResponse:
+    """Update an existing box."""
     try:
         result = box_service.update_box(box_id, box_data)
         return CreateBoxResponse(
@@ -175,12 +171,12 @@ async def update_box(
         logger.error(f"Update box failed: {str(e)}")
         raise HTTPException(status_code=400, detail=str(e))
 
-## DELETE
 @router.delete("/boxes/{box_id}", response_model=CreateBoxResponse)
 async def delete_box(
     box_id: str,
     box_service: BoxService = Depends(get_box_service)
-):
+) -> CreateBoxResponse:
+    """Delete a box by ID."""
     try:
         result = box_service.delete_box(box_id)
         return CreateBoxResponse(
@@ -192,13 +188,13 @@ async def delete_box(
         logger.error(f"Delete box failed: {str(e)}")
         raise HTTPException(status_code=400, detail=str(e))
 
-# BOX OPEN
 @router.post("/boxes/{box_id}/open", response_model=CreateOpenBoxResponse, status_code=201)
 async def open_box(
     box_id: str,
     box_data: CreateOpenBoxRequest = Depends(get_open_box_data),
     box_service: BoxService = Depends(get_box_service)
-):
+) -> CreateOpenBoxResponse:
+    """Open a box by ID."""
     try:
         result = box_service.open_box(box_id, box_data)
         return CreateOpenBoxResponse(
@@ -208,215 +204,4 @@ async def open_box(
         )
     except Exception as e:
         logger.error(f"Open box failed: {str(e)}")
-        raise HTTPException(status_code=400, detail=str(e))
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-# FILE
-
-
-
-
-
-"""
-@router.post("/file/uploadManabox", response_model=FileUploadResponse)
-async def upload_file(
-    background_tasks: BackgroundTasks,
-    file: UploadFile = File(...),
-    file_service: FileService = Depends(get_file_service),
-    product_service: ProductService = Depends(get_product_service),
-    metadata: FileMetadata = Depends(get_file_metadata)) -> FileUploadResponse:
-    try:
-        content = await file.read()
-        metadata.service = 'product'
-        result = file_service.upload_file(content, file.filename, metadata)
-        background_tasks.add_task(product_service.bg_process_manabox_file, result.id)
-        return result
-    except Exception as e:
-        logger.error(f"File upload failed: {str(e)}")
-        raise HTTPException(status_code=400, detail=str(e))
-
-        
-@router.get("/file/getPreparedFiles", response_model=GetPreparedFilesResponse)
-async def get_prepared_files(file_service: FileService = Depends(get_file_service)) -> GetPreparedFilesResponse:
-    try:
-        response = file_service.get_prepared_files()
-        return response
-    except Exception as e:
-        logger.error(f"Get prepared files failed: {str(e)}")
-        raise HTTPException(status_code=400, detail=str(e))
-
-@router.post("/file/deleteFile", response_model=FileDeleteResponse)
-async def delete_file(file_id: str, file_service: FileService = Depends(get_file_service)) -> FileDeleteResponse:
-    try:
-        response = file_service.delete_file(file_id)
-        return response
-    except Exception as e:
-        logger.error(f"Delete file failed: {str(e)}")
-        raise HTTPException(status_code=400, detail=str(e))
-
-@router.post("/box/createBox", response_model=CreateBoxResponse)
-async def create_box(file_ids: list[str], create_box_data: CreateBoxRequestData, box_service: BoxService = Depends(get_box_service)) -> CreateBoxResponse:
-    try:
-        create_box_data = create_box_data.dict()
-        response = box_service.create_box(create_box_data, file_ids)
-        return response
-    except Exception as e:
-        logger.error(f"Create box failed: {str(e)}")
-        raise HTTPException(status_code=400, detail=str(e))
-"""
-## all old below
-
-@router.post("/upload/manabox", response_model=dict)
-async def upload_manabox(
-    background_tasks: BackgroundTasks,
-    upload_service: UploadService = Depends(get_upload_service),
-    data_service: DataService = Depends(get_data_service),
-    file: UploadFile = File(...)
-) -> dict:
-    """
-    Upload endpoint for Manabox CSV files
-    """
-    try:
-        logger.info(f"file received: {file.filename}")
-        # Read the file content
-        content = await file.read()
-        filename = file.filename
-        file_size = len(content)
-        file_size_kb = file_size / 1024
-        if not content:
-            logger.error("Empty file content")
-            raise HTTPException(status_code=400, detail="Empty file content")
-            
-        # You might want to validate it's a CSV file
-        if not file.filename.endswith('.csv'):
-            logger.error("File must be a CSV")
-            raise HTTPException(status_code=400, detail="File must be a CSV")
-
-        result = upload_service.process_manabox_upload(content, filename, file_size_kb)
-        background_tasks.add_task(data_service.bg_set_manabox_tcg_relationship, upload_id=result[1])
-        return result[0]
-    except Exception as e:
-        logger.error(f"Manabox upload failed: {str(e)}")
-        raise HTTPException(status_code=400, detail=str(e))
-
-
-
-@router.post("/createBox", response_model=dict)
-async def create_box(
-    upload_id: str,
-    box_service: BoxService = Depends(get_box_service)
-) -> dict:
-    try:
-        result = box_service.convert_upload_to_boxes(upload_id)
-    except Exception as e:
-        logger.error(f"Box creation failed: {str(e)}")
-        raise HTTPException(status_code=400, detail=str(e))
-    return result
-
-@router.post("/deleteBox", response_model=dict)
-async def delete_box(
-    box_id: str,
-    box_service: BoxService = Depends(get_box_service)
-) -> dict:
-    try:
-        result = box_service.delete_box(box_id)
-    except Exception as e:
-        logger.error(f"Box deletion failed: {str(e)}")
-        raise HTTPException(status_code=400, detail=str(e))
-    return result
-
-
-
-@router.post("/tcgplayer/add/box/{box_id}", response_model=dict)
-async def add_box(box_id: str = None, tcgplayer_service: TCGPlayerService = Depends(get_tcgplayer_service)):
-    try:
-        csv_content = tcgplayer_service.add_to_tcgplayer(box_id)
-        return StreamingResponse(
-            iter([csv_content]),
-            media_type="text/csv",
-            headers={"Content-Disposition": "attachment; filename=add_to_tcgplayer.csv"}
-        )
-    except Exception as e:
-        logger.error(f"Box add failed: {str(e)}")
-        raise HTTPException(status_code=400, detail=str(e))
-
-@router.post("/tcgplayer/update/box/{box_id}", response_model=dict)
-async def update_box(box_id: int = None):
-    """asdf"""
-    pass
-
-@router.post("/tcgplayer/updateInventory", response_model=dict)
-async def update_inventory(
-    background_tasks: BackgroundTasks,
-    tcgplayer_service: TCGPlayerService = Depends(get_tcgplayer_service),
-    data_service: DataService = Depends(get_data_service)):
-    try:
-        result = tcgplayer_service.update_inventory('live')
-        export_id = result['export_id']
-        background_tasks.add_task(data_service.bg_set_tcg_inventory_product_relationship, export_id)
-        return result
-    except Exception as e:
-        logger.error(f"Inventory update failed: {str(e)}")
-        raise HTTPException(status_code=400, detail=str(e))
-    
-@router.post("/tcgplayer/updatePricing", response_model=dict)
-async def update_inventory(
-    tcgplayer_service: TCGPlayerService = Depends(get_tcgplayer_service),
-    group_ids: Dict = None):
-    try:
-        result = tcgplayer_service.update_pricing(group_ids)
-        return result
-    except Exception as e:
-        logger.error(f"Pricing update failed: {str(e)}")
-        raise HTTPException(status_code=400, detail=str(e))
-    
-@router.post("/tcgplayer/updatePricingAll", response_model=dict)
-async def update_inventory(tcgplayer_service: TCGPlayerService = Depends(get_tcgplayer_service)):
-    try:
-        result = tcgplayer_service.update_pricing_all()
-        return result
-    except Exception as e:
-        logger.error(f"Pricing update failed: {str(e)}")
-        raise HTTPException(status_code=400, detail=str(e))
-
-@router.get("/tcgplayer/createLiveInventoryPricingUpdateFile")
-async def create_inventory_import(
-    tcgplayer_service: TCGPlayerService = Depends(get_tcgplayer_service)
-):
-    try:
-        csv_content = tcgplayer_service.get_live_inventory_pricing_update_csv()
-        return StreamingResponse(
-            iter([csv_content]),
-            media_type="text/csv",
-            headers={"Content-Disposition": "attachment; filename=inventory_pricing_update.csv"}
-        )
-    except Exception as e:
-        logger.error(f"Inventory import creation failed: {str(e)}")
         raise HTTPException(status_code=400, detail=str(e))

services/box.py

@@ -1,55 +1,71 @@
-from db.models import Box, File, StagedFileProduct, Product, OpenBoxCard, OpenBox, Inventory, TCGPlayerGroups
-from db.utils import db_transaction
-from uuid import uuid4 as uuid
 from datetime import datetime
-from sqlalchemy.orm import Session
+from typing import Any, Dict, List, Optional
-from sqlalchemy.engine.result import Row
+from uuid import uuid4
 from sqlalchemy import or_
-from schemas.box import CreateBoxRequest, CreateBoxResponse, UpdateBoxRequest, CreateOpenBoxRequest
+from sqlalchemy.orm import Session
 import logging
-from typing import Any
+
+from db.models import (
+    Box,
+    File,
+    StagedFileProduct,
+    Product,
+    OpenBoxCard,
+    OpenBox,
+    Inventory,
+    TCGPlayerGroups
+)
 from db.utils import db_transaction
+from schemas.box import CreateBoxRequest, CreateBoxResponse, UpdateBoxRequest, CreateOpenBoxRequest
 from services.inventory import InventoryService
 
-
 logger = logging.getLogger(__name__)
 
+VALID_BOX_TYPES = {"collector", "play", "draft", "set", "commander"}
+
 class BoxService:
     def __init__(self, db: Session, inventory_service: InventoryService):
         self.db = db
         self.inventory_service = inventory_service
 
-    def validate_file_ids(self, file_ids: list[str]):
+    def validate_file_ids(self, file_ids: List[str]) -> None:
-        # check if all file_ids are valid
+        """Validate that all provided file IDs exist in the database."""
-        for file_id in file_ids:
+        invalid_files = [
-            if self.db.query(File).filter(File.id == file_id).first() is None:
+            file_id for file_id in file_ids
-                raise Exception(f"File ID {file_id} not found")
+            if not self.db.query(File).filter(File.id == file_id).first()
-    
+        ]
-    def get_staged_product_data(self, file_ids: list[str]) -> StagedFileProduct:
+        if invalid_files:
-        staged_product_data = self.db.query(StagedFileProduct).filter(StagedFileProduct.file_id.in_(file_ids)).all()
+            raise ValueError(f"File IDs not found: {', '.join(invalid_files)}")
-        return staged_product_data
+
-    
+    def get_staged_product_data(self, file_ids: List[str]) -> List[StagedFileProduct]:
-    def aggregate_staged_product_data(self, staged_product_data: list[Row]) -> dict[Product, int]:
+        """Retrieve staged product data for given file IDs."""
+        return self.db.query(StagedFileProduct).filter(
+            StagedFileProduct.file_id.in_(file_ids)
+        ).all()
+
+    def aggregate_staged_product_data(self, staged_product_data: List[StagedFileProduct]) -> Dict[Product, int]:
+        """Aggregate staged product data by product and quantity."""
         product_data = {}
         for row in staged_product_data:
             product = self.db.query(Product).filter(Product.id == row.product_id).first()
-            if product not in product_data:
+            if product:
-                product_data[product] = 0
+                product_data[product] = product_data.get(product, 0) + row.quantity
-            product_data[product] += row.quantity
         return product_data
-    
+
-    def find_product_for_box_data(self, create_box_data: dict[str, Any]) -> Product:
+    def find_product_for_box_data(self, create_box_data: Dict[str, Any]) -> Optional[Product]:
-        existing_product = self.db.query(Product).filter(
+        """Find existing product matching box data."""
-            Product.name == create_box_data["name"], # TODO: needs complex enum
+        return self.db.query(Product).filter(
+            Product.name == create_box_data["name"],
             Product.type == "box",
-            Product.set_code == create_box_data["set_code"], # TODO: needs complex enum
+            Product.set_code == create_box_data["set_code"],
-            Product.set_name == create_box_data["set_name"], # TODO: needs complex enum
+            Product.set_name == create_box_data["set_name"],
-            Product.product_line == create_box_data["product_line"]).first()
+            Product.product_line == create_box_data["product_line"]
-        return existing_product
+        ).first()
-    
+
-    def create_product_for_box(self, create_box_data: dict[str, Any]) -> Product:
+    def create_product_for_box(self, create_box_data: Dict[str, Any]) -> Product:
+        """Create a new product for a box."""
         product = Product(
-            id=str(uuid()),
+            id=str(uuid4()),
             name=create_box_data["name"],
             type="box",
             set_code=create_box_data["set_code"],
@@ -58,8 +74,9 @@ class BoxService:
         )
         self.db.add(product)
         return product
-    
+
-    def create_box_db(self, product: Product, create_box_data: dict[str, Any]) -> Box:
+    def create_box_db(self, product: Product, create_box_data: Dict[str, Any]) -> Box:
+        """Create a new box record in the database."""
         box = Box(
             product_id=product.id,
             type=create_box_data["type"],
@@ -68,89 +85,91 @@ class BoxService:
         )
         self.db.add(box)
         return box
-    
+
-    def create_open_box(self, product: Product, create_box_data: dict[str, Any]) -> OpenBox:
+    def create_open_box(self, product: Product, create_box_data: Dict[str, Any]) -> OpenBox:
+        """Create a new open box record."""
         open_box = OpenBox(
-            id = str(uuid()),
+            id=str(uuid4()),
             product_id=product.id,
             num_cards_actual=create_box_data["num_cards_actual"],
             date_opened=datetime.strptime(create_box_data["date_opened"], "%Y-%m-%d")
         )
         self.db.add(open_box)
         return open_box
-    
-    def add_products_to_open_box(self, open_box: OpenBox, product_data: dict[Product, int]) -> None:
-            for product, quantity in product_data.items():
-                open_box_card = OpenBoxCard(
-                    id=str(uuid()),
-                    open_box_id=open_box.id,
-                    card_id=product.id,
-                    quantity=quantity
-                )
-                self.db.add(open_box_card)
-    
-    def format_response(self, open_box: OpenBox = None, inventory: Inventory = None) -> CreateBoxResponse:
-        response = CreateBoxResponse(success=True)
-        return response
 
-    def _create_box(self, create_box_data: dict[str, Any], file_ids: list[str] = None) -> CreateBoxResponse:
+    def add_products_to_open_box(self, open_box: OpenBox, product_data: Dict[Product, int]) -> None:
+        """Add products to an open box."""
+        for product, quantity in product_data.items():
+            open_box_card = OpenBoxCard(
+                id=str(uuid4()),
+                open_box_id=open_box.id,
+                card_id=product.id,
+                quantity=quantity
+            )
+            self.db.add(open_box_card)
+
+    def format_response(self, open_box: Optional[OpenBox] = None, inventory: Optional[Inventory] = None) -> CreateBoxResponse:
+        """Format the response for box creation."""
+        return CreateBoxResponse(success=True)
+
+    def _create_box(self, create_box_data: Dict[str, Any], file_ids: Optional[List[str]] = None) -> CreateBoxResponse:
+        """Internal method to handle box creation logic."""
         sealed = create_box_data["sealed"]
-        assert isinstance(sealed, bool)
+        
+        if file_ids and sealed:
+            raise ValueError("Cannot add cards with a sealed box")
+        
         if file_ids and not sealed:
             self.validate_file_ids(file_ids)
             staged_product_data = self.get_staged_product_data(file_ids)
             product_data = self.aggregate_staged_product_data(staged_product_data)
-        elif file_ids and sealed:
+
-            raise Exception("Cannot add cards with a sealed box")
+        box_product = self.find_product_for_box_data(create_box_data)
-        
+
-        # find product with all same box data
-        existing_product = self.find_product_for_box_data(create_box_data)
-        
-        if existing_product:
-            box_product = existing_product
-        
         try:
             with db_transaction(self.db):
-                if not existing_product:
+                if not box_product:
                     box_product = self.create_product_for_box(create_box_data)
+                
                 box = self.create_box_db(box_product, create_box_data)
+                
                 if not sealed:
                     open_box = self.create_open_box(box_product, create_box_data)
                     if file_ids:
-                        process_staged_products = self.inventory_service.process_staged_products(product_data)
+                        self.inventory_service.process_staged_products(product_data)
                         self.add_products_to_open_box(open_box, product_data)
-                        # should be the file service handling this but im about to die irl
+                        
-                        # update file id status to processed
+                        # Update file statuses to processed
-                        for file_id in file_ids:
+                        self.db.query(File).filter(File.id.in_(file_ids)).update(
-                            file = self.db.query(File).filter(File.id == file_id).first()
+                            {"status": "processed"}, synchronize_session=False
-                            file.status = "processed"
+                        )
-                            self.db.add(file)
+                        
                         return self.format_response(open_box=open_box)
-                elif not file_ids and sealed:
+                elif sealed:
-                    # add sealed box to inventory
                     inventory = self.inventory_service.add_sealed_box_to_inventory(box_product, 1)
                     return self.format_response(inventory=inventory)
 
         except Exception as e:
             logger.error(f"Error creating box: {str(e)}")
-            raise e
+            raise
-    
+
     def validate_box_type(self, box_type: str) -> bool:
-        return box_type in ["collector", "play", "draft", "set", "commander"]
+        """Validate if the box type is supported."""
-    
+        return box_type in VALID_BOX_TYPES
+
     def validate_set_code(self, set_code: str) -> bool:
-        exists = self.db.query(TCGPlayerGroups).filter(
+        """Validate if the set code exists in TCGPlayer groups."""
+        return self.db.query(TCGPlayerGroups).filter(
             TCGPlayerGroups.abbreviation == set_code
         ).first() is not None
-        return exists
+
-        
     def create_box(self, create_box_data: CreateBoxRequest) -> Box:
-        # validate box data
+        """Create a new box."""
         if not self.validate_box_type(create_box_data.type):
-            raise Exception("Invalid box type")
+            raise ValueError("Invalid box type")
         if not self.validate_set_code(create_box_data.set_code):
-            raise Exception("Invalid set code")
+            raise ValueError("Invalid set code")
-        # check if box exists by type and set code or sku
+
         existing_box = self.db.query(Box).filter(
             or_(
                 Box.type == create_box_data.type,
@@ -158,12 +177,13 @@ class BoxService:
             ),
             Box.set_code == create_box_data.set_code
         ).first()
+
         if existing_box:
-            raise Exception("Box already exists")
+            raise ValueError("Box already exists")
-        # create box
+
         with db_transaction(self.db):
             box = Box(
-                product_id=str(uuid()),
+                product_id=str(uuid4()),
                 type=create_box_data.type,
                 set_code=create_box_data.set_code,
                 sku=create_box_data.sku,
@@ -172,49 +192,65 @@ class BoxService:
             self.db.add(box)
 
         return box
-    
+
     def update_box(self, box_id: str, update_box_data: UpdateBoxRequest) -> Box:
+        """Update an existing box."""
         box = self.db.query(Box).filter(Box.product_id == box_id).first()
         if not box:
-            raise Exception("Box not found")
+            raise ValueError("Box not found")
+
+        update_data = update_box_data.dict(exclude_unset=True)
+        
+        # Validate box type if it's being updated
+        if "type" in update_data and update_data["type"] is not None:
+            if not self.validate_box_type(update_data["type"]):
+                raise ValueError(f"Invalid box type: {update_data['type']}")
+
+        # Validate set code if it's being updated
+        if "set_code" in update_data and update_data["set_code"] is not None:
+            if not self.validate_set_code(update_data["set_code"]):
+                raise ValueError(f"Invalid set code: {update_data['set_code']}")
+
         with db_transaction(self.db):
-            if update_box_data.type:
+            for field, value in update_data.items():
-                box.type = update_box_data.type
+                if value is not None:  # Only update non-None values
-            if update_box_data.set_code:
+                    setattr(box, field, value)
-                box.set_code = update_box_data.set_code
+
-            if update_box_data.sku:
-                box.sku = update_box_data.sku
-            if update_box_data.num_cards_expected:
-                box.num_cards_expected = update_box_data.num_cards_expected
         return box
-    
+
     def delete_box(self, box_id: str) -> Box:
+        """Delete a box."""
         box = self.db.query(Box).filter(Box.product_id == box_id).first()
         if not box:
-            raise Exception("Box not found")
+            raise ValueError("Box not found")
+
         with db_transaction(self.db):
             self.db.delete(box)
         return box
-    
+
-    def open_box(self, box_id: str, box_data: CreateOpenBoxRequest):
+    def open_box(self, box_id: str, box_data: CreateOpenBoxRequest) -> OpenBox:
+        """Open a box and process its contents."""
         box = self.db.query(Box).filter(Box.product_id == box_id).first()
         if not box:
-            raise Exception("Box not found")
+            raise ValueError("Box not found")
+
         with db_transaction(self.db):
             open_box = OpenBox(
-                id=str(uuid()),
+                id=str(uuid4()),
                 product_id=box_id,
                 num_cards_actual=box_data.num_cards_actual,
                 date_opened=datetime.strptime(box_data.date_opened, "%Y-%m-%d") if box_data.date_opened else datetime.now()
             )
             self.db.add(open_box)
+
             staged_product_data = self.get_staged_product_data(box_data.file_ids)
             product_data = self.aggregate_staged_product_data(staged_product_data)
             self.inventory_service.process_staged_products(product_data)
             self.add_products_to_open_box(open_box, product_data)
-            # update box_id for files
+
-            for file_id in box_data.file_ids:
+            # Update file box IDs
-                file = self.db.query(File).filter(File.id == file_id).first()
+            self.db.query(File).filter(File.id.in_(box_data.file_ids)).update(
-                file.box_id = open_box.id
+                {"box_id": open_box.id}, synchronize_session=False
-                self.db.add(file)
+            )
+
         return open_box

services/data.py

@@ -1,149 +0,0 @@
-from sqlalchemy.orm import Session
-import logging
-from fastapi import BackgroundTasks
-from db.models import TCGPlayerGroups, SetCodeGroupIdMapping, ManaboxExportData, TCGPlayerProduct, ManaboxTCGPlayerMapping, UnmatchedManaboxData, TCGPlayerInventory
-from db.utils import db_transaction
-import uuid
-from services.tcgplayer import TCGPlayerService
-from sqlalchemy.sql import exists
-
-
-logger = logging.getLogger(__name__)
-
-class DataService:
-    def __init__(self, db: Session, tcgplayer_service: TCGPlayerService):
-        self.db = db
-        self.tcgplayer_service = tcgplayer_service
-    
-    def _normalize_rarity(self, rarity: str) -> str:
-        if rarity.lower() == "rare":
-            return "R"
-        elif rarity.lower() == "mythic":
-            return "M"
-        elif rarity.lower() == "uncommon":
-            return "U"
-        elif rarity.lower() == "common":
-            return "C"
-        elif rarity.lower() in ["R", "M", "U", "C"]:
-            return rarity.upper()
-        else:
-            raise ValueError(f"Invalid rarity: {rarity}")
-    
-    def _normalize_condition(self, condition: str, foil: str) -> str:
-        if condition.lower() == "near_mint":
-            condition1 = "Near Mint"
-        else:
-            raise ValueError(f"Invalid condition: {condition}")
-        if foil.lower() == "foil":
-            condition2 = " Foil"
-        elif foil.lower() == "normal":
-            condition2 = ""
-        else:
-            raise ValueError(f"Invalid foil: {foil}")
-        return condition1 + condition2
-    
-    def _normalize_number(self, number: str) -> str:
-        return str(number.split(".")[0])
-    
-    def _convert_set_code_to_group_id(self, set_code: str) -> str:
-        group = self.db.query(TCGPlayerGroups).filter(TCGPlayerGroups.abbreviation == set_code).first()
-        return group.group_id
-
-    def _add_set_group_mapping(self, set_code: str, group_id: str) -> None:
-        with db_transaction(self.db):
-            self.db.add(SetCodeGroupIdMapping(id=str(uuid.uuid4()), set_code=set_code, group_id=group_id))
-
-    def _get_set_codes(self, **filters) -> list:
-        query = self.db.query(ManaboxExportData.set_code).distinct()
-        for field, value in filters.items():
-            if value is not None:
-                query = query.filter(getattr(ManaboxExportData, field) == value)
-        return [code[0] for code in query.all()]
-
-    async def bg_set_manabox_tcg_relationship(self, box_id: str = None, upload_id: str = None) -> None:
-        if not bool(box_id) ^ bool(upload_id):
-            raise ValueError("Must provide exactly one of box_id or upload_id")
-
-        filters = {"box_id": box_id} if box_id else {"upload_id": upload_id}
-        set_codes = self._get_set_codes(**filters)
-
-        for set_code in set_codes:
-            try:
-                group_id = self._convert_set_code_to_group_id(set_code)
-            except AttributeError:
-                logger.warning(f"No group found for set code {set_code}")
-                continue
-            self._add_set_group_mapping(set_code, group_id)
-            # update pricing for groups
-            if self.db.query(TCGPlayerProduct).filter(TCGPlayerProduct.group_id == group_id).count() == 0:
-                self.tcgplayer_service.update_pricing(set_name_ids={"set_name_ids":[group_id]})
-        
-        # match manabox data to tcgplayer pricing data
-        # match on manabox - set_code (through group_id), collector_number, foil, rarity, condition
-        # match on tcgplayer - group_id, number, rarity, condition (condition + foil)
-        # use normalizing functions
-        matched_records = self.db.query(ManaboxExportData).filter(ManaboxExportData.set_code.in_(set_codes)).all()
-        for record in matched_records:
-            rarity = self._normalize_rarity(record.rarity)
-            condition = self._normalize_condition(record.condition, record.foil)
-            number = self._normalize_number(record.collector_number)
-            group_id = self._convert_set_code_to_group_id(record.set_code)
-            tcg_record = self.db.query(TCGPlayerProduct).filter(
-                TCGPlayerProduct.group_id == group_id,
-                TCGPlayerProduct.number == number,
-                TCGPlayerProduct.rarity == rarity,
-                TCGPlayerProduct.condition == condition
-            ).all()
-            if len(tcg_record) == 0:
-                logger.warning(f"No match found for {record.name}")
-                if self.db.query(UnmatchedManaboxData).filter(UnmatchedManaboxData.manabox_id == record.id).count() == 0:
-                    with db_transaction(self.db):
-                        self.db.add(UnmatchedManaboxData(id=str(uuid.uuid4()), manabox_id=record.id, reason="No match found"))
-            elif len(tcg_record) > 1:
-                logger.warning(f"Multiple matches found for {record.name}")
-                if self.db.query(UnmatchedManaboxData).filter(UnmatchedManaboxData.manabox_id == record.id).count() == 0:
-                    with db_transaction(self.db):
-                        self.db.add(UnmatchedManaboxData(id=str(uuid.uuid4()), manabox_id=record.id, reason="Multiple matches found"))
-            else:
-                with db_transaction(self.db):
-                    self.db.add(ManaboxTCGPlayerMapping(id=str(uuid.uuid4()), manabox_id=record.id, tcgplayer_id=tcg_record[0].id))
-    
-    async def bg_set_tcg_inventory_product_relationship(self, export_id: str) -> None:
-        inventory_without_product = (
-            self.db.query(TCGPlayerInventory.tcgplayer_id, TCGPlayerInventory.set_name)
-            .filter(TCGPlayerInventory.total_quantity > 0)
-            .filter(TCGPlayerInventory.product_line == "Magic")
-            .filter(TCGPlayerInventory.export_id == export_id)
-            .filter(TCGPlayerInventory.tcgplayer_product_id.is_(None))
-            .filter(~exists().where(
-                TCGPlayerProduct.id == TCGPlayerInventory.tcgplayer_product_id
-            ))
-            .all()
-        )
-
-        set_names = list(set(inv.set_name for inv in inventory_without_product 
-                            if inv.set_name is not None and isinstance(inv.set_name, str)))
-        
-        group_ids = self.db.query(TCGPlayerGroups.group_id).filter(
-            TCGPlayerGroups.name.in_(set_names)
-        ).all()
-        
-        group_ids = [str(group_id[0]) for group_id in group_ids]
-        
-        self.tcgplayer_service.update_pricing(set_name_ids={"set_name_ids": group_ids})
-        
-        for inventory in inventory_without_product:
-            product = self.db.query(TCGPlayerProduct).filter(
-                TCGPlayerProduct.tcgplayer_id == inventory.tcgplayer_id
-            ).first()
-            
-            if product:
-                with db_transaction(self.db):
-                    inventory_record = self.db.query(TCGPlayerInventory).filter(
-                        TCGPlayerInventory.tcgplayer_id == inventory.tcgplayer_id,
-                        TCGPlayerInventory.export_id == export_id
-                    ).first()
-                    
-                    if inventory_record:
-                        inventory_record.tcgplayer_product_id = product.id
-                        self.db.add(inventory_record)

services/file.py

@@ -1,127 +1,156 @@
 from sqlalchemy.orm import Session
+from typing import Optional, List, Dict, Any
+from uuid import uuid4
+import csv
+import logging
+import os
+from io import StringIO
+
 from db.utils import db_transaction
 from db.models import File, StagedFileProduct
 from schemas.file import CreateFileRequest
-import os
-from uuid import uuid4 as uuid
-import logging
-import csv
-from io import StringIO
-from typing import Optional, List
 
 logger = logging.getLogger(__name__)
 
-# Name,Set code,Set name,Collector number,Foil,Rarity,Quantity,ManaBox ID,Scryfall ID,Purchase price,Misprint,Altered,Condition,Language,Purchase price currency
+class FileConfig:
-MANABOX_REQUIRED_FILE_HEADERS = ['Name', 'Set code', 'Set name', 'Collector number', 'Foil', 'Rarity', 'Quantity', 'ManaBox ID', 'Scryfall ID', 'Purchase price', 'Misprint', 'Altered', 'Condition', 'Language', 'Purchase price currency']
+    """Configuration constants for file processing"""
-MANABOX_ALLOWED_FILE_EXTENSIONS = ['.csv']
+    TEMP_DIR = os.path.join(os.getcwd(), 'temp')
-MANABOX_ALLOWED_FILE_TYPES = ['scan_export_common', 'scan_export_rare']
+    
-MANABOX_CONFIG = {
+    MANABOX_HEADERS = [
-    "required_headers": MANABOX_REQUIRED_FILE_HEADERS,
+        'Name', 'Set code', 'Set name', 'Collector number', 'Foil', 
-    "allowed_extensions": MANABOX_ALLOWED_FILE_EXTENSIONS,
+        'Rarity', 'Quantity', 'ManaBox ID', 'Scryfall ID', 'Purchase price',
-    "allowed_types": MANABOX_ALLOWED_FILE_TYPES
+        'Misprint', 'Altered', 'Condition', 'Language', 'Purchase price currency'
-}
+    ]
-SOURCES = {
+    
-    "manabox": MANABOX_CONFIG
+    SOURCES = {
-}
+        "manabox": {
-TEMP_DIR = os.getcwd() + '/temp/'
+            "required_headers": MANABOX_HEADERS,
+            "allowed_extensions": ['.csv'],
+            "allowed_types": ['scan_export_common', 'scan_export_rare']
+        }
+    }
+
+class FileValidationError(Exception):
+    """Custom exception for file validation errors"""
+    pass
 
 class FileService:
     def __init__(self, db: Session):
         self.db = db
 
-    # CONFIG
+    def get_config(self, source: str) -> Dict[str, Any]:
-    def get_config(self, source: str) -> dict:
+        """Get configuration for a specific source"""
-        return SOURCES.get(source)
+        config = FileConfig.SOURCES.get(source)
-    
+        if not config:
-    # VALIDATION
+            raise FileValidationError(f"Unsupported source: {source}")
-    def validate_file_extension(self, filename: str, config: dict) -> bool:
+        return config
-        return filename.endswith(tuple(config.get("allowed_extensions")))
+
-    
+    def validate_file_extension(self, filename: str, config: Dict[str, Any]) -> bool:
-    def validate_file_type(self, metadata: CreateFileRequest, config: dict) -> bool:
+        """Validate file extension against allowed extensions"""
-        return metadata.type in config.get("allowed_types")
+        return any(filename.endswith(ext) for ext in config["allowed_extensions"])
+
+    def validate_file_type(self, metadata: CreateFileRequest, config: Dict[str, Any]) -> bool:
+        """Validate file type against allowed types"""
+        return metadata.type in config["allowed_types"]
 
     def validate_csv(self, content: bytes, required_headers: Optional[List[str]] = None) -> bool:
+        """Validate CSV content and headers"""
         try:
-            # Try to decode and parse as CSV
             csv_text = content.decode('utf-8')
             csv_file = StringIO(csv_text)
             csv_reader = csv.reader(csv_file)
             
-            # Check headers if specified
+            if required_headers:
-            headers = next(csv_reader, None)
+                headers = next(csv_reader, None)
-            if required_headers and not all(header in headers for header in required_headers):
+                if not headers or not all(header in headers for header in required_headers):
-                return False
+                    return False
-                
             return True
             
-        except (UnicodeDecodeError, csv.Error):
+        except (UnicodeDecodeError, csv.Error) as e:
+            logger.error(f"CSV validation error: {str(e)}")
             return False
-    
+
-    def validate_file_content(self, content: bytes, metadata: CreateFileRequest, config: dict) -> bool:
+    def validate_file_content(self, content: bytes, metadata: CreateFileRequest, config: Dict[str, Any]) -> bool:
-        extension = metadata.filename.split('.')[-1]
+        """Validate file content based on file type"""
-        if extension == 'csv':
+        extension = os.path.splitext(metadata.filename)[1].lower()
+        if extension == '.csv':
             return self.validate_csv(content, config.get("required_headers"))
         return False
 
     def validate_file(self, content: bytes, metadata: CreateFileRequest) -> bool:
-        # 1. Get config
+        """Validate file against all criteria"""
         config = self.get_config(metadata.source)
-        # 2. Validate file extension
+        
         if not self.validate_file_extension(metadata.filename, config):
-            raise Exception("Invalid file extension")
+            raise FileValidationError("Invalid file extension")
-        # 2. validate file type
+        
         if not self.validate_file_type(metadata, config):
-            raise Exception("Invalid file type")
+            raise FileValidationError("Invalid file type")
-        # 3. Validate file content
+        
         if not self.validate_file_content(content, metadata, config):
-            raise Exception("Invalid file content")
+            raise FileValidationError("Invalid file content or headers")
+            
         return True
-    
+
-    # CRUD
-    # CREATE
     def create_file(self, content: bytes, metadata: CreateFileRequest) -> File:
+        """Create a new file record and save the file"""
         with db_transaction(self.db):
             file = File(
-                id = str(uuid()),
+                id=str(uuid4()),
-                filename = metadata.filename,
+                filename=metadata.filename,
-                filepath = TEMP_DIR + metadata.filename, # TODO config variable
+                filepath=os.path.join(FileConfig.TEMP_DIR, metadata.filename),
-                type = metadata.type,
+                type=metadata.type,
-                source = metadata.source,
+                source=metadata.source,
-                filesize_kb = round(len(content) / 1024, 2),
+                filesize_kb=round(len(content) / 1024, 2),
-                status = 'pending',
+                status='pending',
-                service = metadata.service
+                service=metadata.service
             )
             self.db.add(file)
+            
+        os.makedirs(FileConfig.TEMP_DIR, exist_ok=True)
         with open(file.filepath, 'wb') as f:
             f.write(content)
+            
         return file
-    
+
-    # GET
+    def get_file(self, file_id: str) -> File:
-    def get_file(self, file_id: str) -> List[File]:
+        """Get a file by ID"""
         file = self.db.query(File).filter(File.id == file_id).first()
         if not file:
-            raise Exception(f"File with id {file_id} not found")
+            raise FileValidationError(f"File with id {file_id} not found")
         return file
-    
+
     def get_files(self, status: Optional[str] = None) -> List[File]:
+        """Get all files, optionally filtered by status"""
+        query = self.db.query(File)
         if status:
-            return self.db.query(File).filter(File.status == status).all()
+            query = query.filter(File.status == status)
-        return self.db.query(File).all()
+        return query.all()
-    
+
-    # DELETE
     def get_staged_products(self, file_id: str) -> List[StagedFileProduct]:
-        return self.db.query(StagedFileProduct).filter(StagedFileProduct.file_id == file_id).all()
+        """Get staged products for a file"""
-    
+        return self.db.query(StagedFileProduct).filter(
-    def delete_file(self, file_id: str) -> List[File]:
+            StagedFileProduct.file_id == file_id
+        ).all()
+
+    def delete_file(self, file_id: str) -> File:
+        """Mark a file as deleted and remove associated staged products"""
         file = self.get_file(file_id)
-        if not file:  
-            raise Exception(f"File with id {file_id} not found")
-            
         staged_products = self.get_staged_products(file_id)
         
         with db_transaction(self.db):
-            file.status = 'deleted' 
+            file.status = 'deleted'
             for staged_product in staged_products:
                 self.db.delete(staged_product)
                 
         return file
+
+    def get_file_content(self, file_id: str) -> bytes:
+        """Get the content of a file"""
+        file = self.get_file(file_id)
+        try:
+            with open(file.filepath, 'rb') as f:
+                return f.read()
+        except IOError as e:
+            logger.error(f"Error reading file {file_id}: {str(e)}")
+            raise FileValidationError(f"Could not read file content for {file_id}")

services/inventory.py

@@ -1,28 +1,90 @@
+from typing import Dict
+
 from sqlalchemy.orm import Session
+from sqlalchemy.exc import SQLAlchemyError
+
 from db.models import Product, Inventory
 from schemas.inventory import UpdateInventoryResponse
 from db.utils import db_transaction
 
+
 class InventoryService:
-    def __init__(self, db: Session):
+    """Service class for managing product inventory operations."""
+
+    def __init__(self, db: Session) -> None:
+        """
+        Initialize the InventoryService.
+
+        Args:
+            db: SQLAlchemy database session
+        """
         self.db = db
-    
+
     def add_inventory(self, product: Product, quantity: int) -> Inventory:
-        inventory = self.db.query(Inventory).filter(Inventory.product_id == product.id).first()
+        """
+        Add or update inventory for a product.
+
+        Args:
+            product: Product model instance
+            quantity: Quantity to add to inventory
+
+        Returns:
+            Updated Inventory model instance
+        """
+        inventory = self.db.query(Inventory).filter(
+            Inventory.product_id == product.id
+        ).first()
+
         if inventory is None:
-            inventory = Inventory(product_id=product.id, quantity=quantity)
+            inventory = Inventory(
+                product_id=product.id,
+                quantity=quantity
+            )
             self.db.add(inventory)
         else:
             inventory.quantity += quantity
+
         return inventory
 
-    def process_staged_products(self, product_data: dict[Product, int]) -> UpdateInventoryResponse:
+    def process_staged_products(
-        with db_transaction(self.db):
+        self,
-            for product, quantity in product_data.items():
+        product_data: Dict[Product, int]
+    ) -> UpdateInventoryResponse:
+        """
+        Process multiple products and update their inventory.
+
+        Args:
+            product_data: Dictionary mapping Products to their quantities
+
+        Returns:
+            Response indicating success status
+        """
+        try:
+            with db_transaction(self.db):
+                for product, quantity in product_data.items():
+                    self.add_inventory(product, quantity)
+            return UpdateInventoryResponse(success=True)
+        except SQLAlchemyError:
+            return UpdateInventoryResponse(success=False)
+
+    def add_sealed_box_to_inventory(
+        self,
+        product: Product,
+        quantity: int
+    ) -> UpdateInventoryResponse:
+        """
+        Add sealed box inventory for a single product.
+
+        Args:
+            product: Product model instance
+            quantity: Quantity to add to inventory
+
+        Returns:
+            Response indicating success status
+        """
+        try:
+            with db_transaction(self.db):
                 self.add_inventory(product, quantity)
-        return UpdateInventoryResponse(success=True)
+            return UpdateInventoryResponse(success=True)
-    
+        except SQLAlchemyError:
-    def add_sealed_box_to_inventory(self, product: Product, quantity: int) -> UpdateInventoryResponse:
+            return UpdateInventoryResponse(success=False)
-        with db_transaction(self.db):
-            inventory = self.add_inventory(product, quantity)
-        return UpdateInventoryResponse(success=True)

services/old_box.py

@@ -1,100 +0,0 @@
-from db.models import ManaboxExportData, Box, UploadHistory
-from db.utils import db_transaction
-import uuid
-from datetime import datetime
-from sqlalchemy.orm import Session
-from sqlalchemy.engine.result import Row
-
-
-
-import logging
-logger = logging.getLogger(__name__)
-
-class BoxObject:
-    def __init__(
-            self, upload_id: str, set_name: str, 
-            set_code: str, cost: float = None, date_purchased: datetime = None, 
-            date_opened: datetime = None, box_id: str = None):
-        self.upload_id = upload_id
-        self.box_id = box_id if box_id else str(uuid.uuid4())
-        self.set_name = set_name
-        self.set_code = set_code
-        self.cost = cost
-        self.date_purchased = date_purchased
-        self.date_opened = date_opened
-
-class BoxService:
-    def __init__(self, db: Session):
-        self.db = db
-    
-    def _validate_upload_id(self, upload_id: str):
-        # check if upload_history status = 'success'
-        if self.db.query(UploadHistory).filter(UploadHistory.upload_id == upload_id).first() is None:
-            raise Exception(f"Upload ID {upload_id} not found")
-        if self.db.query(UploadHistory).filter(UploadHistory.upload_id == upload_id).first().status != 'success':
-            raise Exception(f"Upload ID {upload_id} not successful")
-        # check if at least 1 row in manabox_export_data with upload_id
-        if self.db.query(ManaboxExportData).filter(ManaboxExportData.upload_id == upload_id).first() is None:
-            raise Exception(f"Upload ID {upload_id} has no data")
-    
-    def _get_set_info(self, upload_id: str) -> list[Row[tuple[str, str]]]:
-        # get distinct set_name, set_code from manabox_export_data for upload_id
-        boxes = self.db.query(ManaboxExportData.set_name, ManaboxExportData.set_code).filter(ManaboxExportData.upload_id == upload_id).distinct().all()
-        if not boxes or len(boxes) == 0:
-            raise Exception(f"Upload ID {upload_id} has no data")
-        return boxes
-    
-    def _update_manabox_export_data_box_id(self, box: Box):
-        # based on upload_id, set_name, set_code, update box_id in manabox_export_data for all rows where box id is null
-        with db_transaction(self.db):
-            self.db.query(ManaboxExportData).filter(
-                ManaboxExportData.upload_id == box.upload_id).filter(
-                    ManaboxExportData.set_name == box.set_name).filter(
-                        ManaboxExportData.set_code == box.set_code).filter(
-                            ManaboxExportData.box_id == None).update({ManaboxExportData.box_id: box.id})
-
-    def convert_upload_to_boxes(self, upload_id: str):
-        self._validate_upload_id(upload_id)
-        # get distinct set_name, set_code from manabox_export_data for upload_id
-        box_set_info = self._get_set_info(upload_id)
-        created_boxes = []
-        # create boxes
-        for box in box_set_info:
-            box_obj = BoxObject(upload_id, set_name = box.set_name, set_code = box.set_code)
-            new_box = self.create_box(box_obj)
-            logger.info(f"Created box {new_box.id} for upload {upload_id}")
-            self._update_manabox_export_data_box_id(new_box)
-            created_boxes.append(new_box)
-        
-        return {"status": "success", "boxes": f"{[box.id for box in created_boxes]}"}
-
-    
-    def create_box(self, box: BoxObject):
-        with db_transaction(self.db):
-            box_record = Box(
-                id = box.box_id,
-                upload_id = box.upload_id,
-                set_name = box.set_name,
-                set_code = box.set_code,
-                cost = box.cost,
-                date_purchased = box.date_purchased,
-                date_opened = box.date_opened
-            )
-            self.db.add(box_record)
-            return box_record
-
-    def get_box(self):
-        pass
-
-    def delete_box(self, box_id: str):
-        # delete box
-        with db_transaction(self.db):
-            self.db.query(Box).filter(Box.id == box_id).delete()
-        # update manabox_export_data box_id to null
-        with db_transaction(self.db):
-            self.db.query(ManaboxExportData).filter(ManaboxExportData.box_id == box_id).update({ManaboxExportData.box_id: None})
-        return {"status": "success", "box_id": box_id}
-
-    def update_box(self):
-        pass
-        

services/pricing.py

@@ -4,5 +4,3 @@ from sqlalchemy.orm import Session
 class PricingService:
     def __init__(self, db: Session):
         self.db = db
-    
-    

services/pricing_old.py

@@ -1,205 +0,0 @@
-import logging
-from typing import Callable
-from db.models import TCGPlayerInventory, TCGPlayerExportHistory, TCGPlayerPricingHistory, ManaboxExportData, ManaboxTCGPlayerMapping, TCGPlayerProduct
-from sqlalchemy.orm import Session
-import pandas as pd
-from db.utils import db_transaction
-from sqlalchemy import func, and_, exists
-
-
-logger = logging.getLogger(__name__)
-
-class PricingService:
-    def __init__(self, db: Session):
-        self.db = db
-
-    def get_box_with_most_recent_prices(self, box_id: str) -> pd.DataFrame:
-        latest_prices = (
-            self.db.query(
-                TCGPlayerPricingHistory.tcgplayer_product_id,
-                func.max(TCGPlayerPricingHistory.date_created).label('max_date')
-            )
-            .group_by(TCGPlayerPricingHistory.tcgplayer_product_id)
-            .subquery('latest')  # Added name to subquery
-        )
-
-        result = (
-            self.db.query(ManaboxExportData, TCGPlayerPricingHistory, TCGPlayerProduct)
-            .join(ManaboxTCGPlayerMapping, ManaboxExportData.id == ManaboxTCGPlayerMapping.manabox_id)
-            .join(TCGPlayerProduct, ManaboxTCGPlayerMapping.tcgplayer_id == TCGPlayerProduct.id)
-            .join(TCGPlayerPricingHistory, TCGPlayerProduct.id == TCGPlayerPricingHistory.tcgplayer_product_id)
-            .join(
-                latest_prices,
-                and_(
-                    TCGPlayerPricingHistory.tcgplayer_product_id == latest_prices.c.tcgplayer_product_id,
-                    TCGPlayerPricingHistory.date_created == latest_prices.c.max_date
-                )
-            )
-            .filter(ManaboxExportData.box_id == box_id)  # Removed str() conversion
-            .all()
-        )
-
-        logger.debug(f"Found {len(result)} rows")
-
-        df = pd.DataFrame([{
-            **{f"manabox_{k}": v for k, v in row[0].__dict__.items() if not k.startswith('_')},
-            **{f"pricing_{k}": v for k, v in row[1].__dict__.items() if not k.startswith('_')},
-            **{f"tcgproduct_{k}": v for k, v in row[2].__dict__.items() if not k.startswith('_')}
-        } for row in result])
-        
-        return df
-    
-    def get_live_inventory_with_most_recent_prices(self) -> pd.DataFrame:
-            # Get latest export IDs using subqueries
-            latest_inventory_export = (
-                self.db.query(TCGPlayerExportHistory.inventory_export_id)
-                .filter(TCGPlayerExportHistory.type == "live_inventory")
-                .order_by(TCGPlayerExportHistory.date_created.desc())
-                .limit(1)
-                .scalar_subquery()
-            )
-            # this is bad because latest pricing export is not guaranteed to be related to the latest inventory export
-            latest_pricing_export = (
-                self.db.query(TCGPlayerExportHistory.pricing_export_id)
-                .filter(TCGPlayerExportHistory.type == "pricing")
-                .order_by(TCGPlayerExportHistory.date_created.desc())
-                .limit(1)
-                .scalar_subquery()
-            )
-            
-            # Join inventory and pricing data in a single query
-            inventory_with_pricing = (
-                self.db.query(TCGPlayerInventory, TCGPlayerPricingHistory)
-                .join(
-                    TCGPlayerPricingHistory,
-                    TCGPlayerInventory.tcgplayer_product_id == TCGPlayerPricingHistory.tcgplayer_product_id
-                )
-                .filter(
-                    TCGPlayerInventory.export_id == latest_inventory_export,
-                    TCGPlayerPricingHistory.export_id == latest_pricing_export
-                )
-                .all()
-            )
-            
-            # Convert to pandas DataFrame
-            df = pd.DataFrame([{
-                # Inventory columns
-                **{f"inventory_{k}": v 
-                for k, v in row[0].__dict__.items() 
-                if not k.startswith('_')},
-                # Pricing columns
-                **{f"pricing_{k}": v 
-                for k, v in row[1].__dict__.items() 
-                if not k.startswith('_')}
-            } for row in inventory_with_pricing])
-            
-            return df
-
-    def default_pricing_algo(self, df: pd.DataFrame = None):
-        if df is None:
-            logger.debug("No DataFrame provided, fetching live inventory with most recent prices")
-            df = self.get_live_inventory_with_most_recent_prices()
-        # if tcg low price is < 0.35, set my_price to 0.35
-        # if either tcg low price or tcg low price with shipping is under 5, set my_price to tcg low price * 1.25
-        # if tcg low price with shipping is > 25 set price to tcg low price with shipping * 1.025
-        # otherwise, set price to tcg low price with shipping * 1.10
-        # also round to 2 decimal places
-        df['my_price'] = df.apply(lambda row: round(
-            0.35 if row['pricing_tcg_low_price'] < 0.35 else
-            row['pricing_tcg_low_price'] * 1.25 if row['pricing_tcg_low_price'] < 5 or row['pricing_tcg_low_price_with_shipping'] < 5 else
-            row['pricing_tcg_low_price_with_shipping'] * 1.025 if row['pricing_tcg_low_price_with_shipping'] > 25 else
-            row['pricing_tcg_low_price_with_shipping'] * 1.10, 2), axis=1)
-        # log rows with no price
-        no_price = df[df['my_price'].isnull()]
-        if len(no_price) > 0:
-            logger.warning(f"Found {len(no_price)} rows with no price")
-            logger.warning(no_price)
-            # remove rows with no price
-            df = df.dropna(subset=['my_price'])
-        return df
-    
-    def convert_df_to_csv(self, df: pd.DataFrame):
-        # Flip the mapping to be from current names TO desired names
-        column_mapping = {
-            'inventory_tcgplayer_id': 'TCGplayer Id',
-            'inventory_product_line': 'Product Line',
-            'inventory_set_name': 'Set Name',
-            'inventory_product_name': 'Product Name',
-            'inventory_title': 'Title',
-            'inventory_number': 'Number',
-            'inventory_rarity': 'Rarity',
-            'inventory_condition': 'Condition',
-            'pricing_tcg_market_price': 'TCG Market Price',
-            'pricing_tcg_direct_low': 'TCG Direct Low',
-            'pricing_tcg_low_price_with_shipping': 'TCG Low Price With Shipping',
-            'pricing_tcg_low_price': 'TCG Low Price',
-            'inventory_total_quantity': 'Total Quantity',
-            'inventory_add_to_quantity': 'Add to Quantity',
-            'my_price': 'TCG Marketplace Price',
-            'inventory_photo_url': 'Photo URL'
-        }
-
-        df['pricing_tcg_market_price'] = ""
-        df['pricing_tcg_direct_low'] = ""
-        df['pricing_tcg_low_price_with_shipping'] = ""
-        df['pricing_tcg_low_price'] = ""
-        df['inventory_total_quantity'] = ""
-        df['inventory_add_to_quantity'] = 0
-        df['inventory_photo_url'] = ""
-
-        # First select the columns we want (using the keys of our mapping)
-        # Then rename them to the desired names (the values in our mapping)
-        df = df[column_mapping.keys()].rename(columns=column_mapping)
-        
-        return df.to_csv(index=False, quoting=1, quotechar='"')
-    
-    def convert_add_df_to_csv(self, df: pd.DataFrame):
-        column_mapping = {
-            'tcgproduct_tcgplayer_id': 'TCGplayer Id',
-            'tcgproduct_product_line': 'Product Line',
-            'tcgproduct_set_name': 'Set Name',
-            'tcgproduct_product_name': 'Product Name',
-            'tcgproduct_title': 'Title',
-            'tcgproduct_number': 'Number',
-            'tcgproduct_rarity': 'Rarity',
-            'tcgproduct_condition': 'Condition',
-            'pricing_tcg_market_price': 'TCG Market Price',
-            'pricing_tcg_direct_low': 'TCG Direct Low',
-            'pricing_tcg_low_price_with_shipping': 'TCG Low Price With Shipping',
-            'pricing_tcg_low_price': 'TCG Low Price',
-            'tcgproduct_group_id': 'Total Quantity',
-            'manabox_quantity': 'Add to Quantity',
-            'my_price': 'TCG Marketplace Price',
-            'tcgproduct_photo_url': 'Photo URL'
-        }
-        df['tcgproduct_group_id'] = ""
-        df['pricing_tcg_market_price'] = ""
-        df['pricing_tcg_direct_low'] = ""
-        df['pricing_tcg_low_price_with_shipping'] = ""
-        df['pricing_tcg_low_price'] = ""
-        df['tcgproduct_photo_url'] = ""
-
-        df = df[column_mapping.keys()].rename(columns=column_mapping)
-
-        return df.to_csv(index=False, quoting=1, quotechar='"')
-
-    def create_live_inventory_pricing_update_csv(self, algo: Callable = None) -> str:
-        actual_algo = algo if algo is not None else self.default_pricing_algo
-        df = actual_algo()
-        csv = self.convert_df_to_csv(df)
-        return csv
-    
-    def create_add_to_tcgplayer_csv(self, box_id: str = None, upload_id: str = None, algo: Callable = None) -> str:
-        actual_algo = algo if algo is not None else self.default_pricing_algo
-        if box_id and upload_id:
-            raise ValueError("Cannot specify both box_id and upload_id")
-        elif not box_id and not upload_id:
-            raise ValueError("Must specify either box_id or upload_id")
-        elif box_id:
-            logger.debug("creating df")
-            df = self.get_box_with_most_recent_prices(box_id)
-        elif upload_id:
-            raise NotImplementedError("Not yet implemented")
-        df = actual_algo(df)
-        csv = self.convert_add_df_to_csv(df)
-        return csv

services/product.py

@@ -1,161 +1,183 @@
+from logging import getLogger
+from uuid import uuid4
+from pandas import DataFrame
 from sqlalchemy.orm import Session
+
 from db.utils import db_transaction
-from db.models import Product, File, CardManabox, Card, StagedFileProduct, CardTCGPlayer
+from db.models import CardManabox, CardTCGPlayer, StagedFileProduct, TCGPlayerGroups
-from io import StringIO
+from services.util._dataframe import ManaboxRow, DataframeUtil
-import pandas as pd
 from services.file import FileService
 from services.tcgplayer import TCGPlayerService
 from services.storage import StorageService
-from uuid import uuid4 as uuid
-import logging
 
-logger = logging.getLogger(__name__)
+logger = getLogger(__name__)
 
 
-class ManaboxRow:
-    def __init__(self, row: pd.Series):
-        self.name = row['name']
-        self.set_code = row['set_code']
-        self.set_name = row['set_name']
-        self.collector_number = row['collector_number']
-        self.foil = row['foil']
-        self.rarity = row['rarity']
-        self.manabox_id = row['manabox_id']
-        self.scryfall_id = row['scryfall_id']
-        self.condition = row['condition']
-        self.language = row['language']
-        self.quantity = row['quantity']
-
 class ProductService:
-    def __init__(self, db: Session, file_service: FileService, tcgplayer_service: TCGPlayerService, storage_service: StorageService):
+    def __init__(
+        self,
+        db: Session,
+        file_service: FileService,
+        tcgplayer_service: TCGPlayerService,
+        storage_service: StorageService,
+    ):
         self.db = db
         self.file_service = file_service
         self.tcgplayer_service = tcgplayer_service
         self.storage_service = storage_service
+        self.df_util = DataframeUtil()
 
-    def _format_manabox_df(self, df: pd.DataFrame) -> pd.DataFrame:
+    def create_staged_file_product(
-        # format columns
+        self, file_id: str, card_manabox: CardManabox, row: ManaboxRow
-        df.columns = df.columns.str.lower()
+    ) -> StagedFileProduct:
-        df.columns = df.columns.str.replace(' ', '_')
+        """Create a staged file product entry.
-        return df
+
-    
+        Args:
-    def _manabox_file_to_df(self, file: File) -> pd.DataFrame:
+            file_id: The ID of the file being processed
-        with open(file.filepath, 'rb') as f:
+            card_manabox: The Manabox card details
-            content = f.read()
+            row: The row data from the Manabox file
-        content = content.decode('utf-8')
+
-        df = pd.read_csv(StringIO(content))
+        Returns:
-        df = self._format_manabox_df(df)
+            The created staged file product
-        return df
+        """
-    
-    def create_staged_product(self, file: File, card_manabox:CardManabox, row: ManaboxRow) -> StagedFileProduct:
         staged_product = StagedFileProduct(
-            id = str(uuid()),
+            id=str(uuid4()),
-            file_id = file.id,
+            file_id=file_id,
-            product_id = card_manabox.product_id,
+            product_id=card_manabox.product_id,
-            quantity = row.quantity
+            quantity=row.quantity,
         )
         with db_transaction(self.db):
             self.db.add(staged_product)
         return staged_product
-    
+
-    def create_card_manabox(self, manabox_row: ManaboxRow) -> CardManabox:
+    def create_card_manabox(
+        self, manabox_row: ManaboxRow, card_tcgplayer: CardTCGPlayer
+    ) -> CardManabox:
+        """Create a Manabox card entry.
+
+        Args:
+            manabox_row: The row data from the Manabox file
+            card_tcgplayer: The TCGPlayer card details
+
+        Returns:
+            The created Manabox card
+        """
+        if not card_tcgplayer:
+            group = (
+                self.db.query(TCGPlayerGroups)
+                .filter(TCGPlayerGroups.abbreviation == manabox_row.set_code)
+                .first()
+            )
+            card_tcgplayer = self.tcgplayer_service.get_card_tcgplayer_from_manabox_row(
+                manabox_row, group.group_id
+            )
+
         card_manabox = CardManabox(
-            product_id = str(uuid()),
+            product_id=card_tcgplayer.product_id,
-            name = manabox_row.name,
+            name=manabox_row.name,
-            set_code = manabox_row.set_code,
+            set_code=manabox_row.set_code,
-            set_name = manabox_row.set_name,
+            set_name=manabox_row.set_name,
-            collector_number = manabox_row.collector_number,
+            collector_number=manabox_row.collector_number,
-            foil = manabox_row.foil,
+            foil=manabox_row.foil,
-            rarity = manabox_row.rarity,
+            rarity=manabox_row.rarity,
-            manabox_id = manabox_row.manabox_id,
+            manabox_id=manabox_row.manabox_id,
-            scryfall_id = manabox_row.scryfall_id,
+            scryfall_id=manabox_row.scryfall_id,
-            condition = manabox_row.condition,
+            condition=manabox_row.condition,
-            language = manabox_row.language
+            language=manabox_row.language,
         )
-        return card_manabox
+
-    
+        with db_transaction(self.db):
-    def create_product(self, card_manabox: CardManabox) -> Product:
+            self.db.add(card_manabox)
-        product = Product(
+            return card_manabox
-            id = card_manabox.product_id,
+
-            name = card_manabox.name,
+    def card_manabox_lookup_create_if_not_exist(
-            set_code = card_manabox.set_code,
+        self, manabox_row: ManaboxRow
-            set_name = card_manabox.set_name,
+    ) -> CardManabox:
-            type = 'card',
+        """Lookup a Manabox card or create it if it doesn't exist.
-            product_line = 'mtg'
+
+        Args:
+            manabox_row: The row data from the Manabox file
+
+        Returns:
+            The existing or newly created Manabox card
+        """
+        card_manabox = (
+            self.db.query(CardManabox)
+            .filter(
+                CardManabox.name == manabox_row.name,
+                CardManabox.set_code == manabox_row.set_code,
+                CardManabox.set_name == manabox_row.set_name,
+                CardManabox.collector_number == manabox_row.collector_number,
+                CardManabox.foil == manabox_row.foil,
+                CardManabox.rarity == manabox_row.rarity,
+                CardManabox.manabox_id == manabox_row.manabox_id,
+                CardManabox.scryfall_id == manabox_row.scryfall_id,
+                CardManabox.condition == manabox_row.condition,
+                CardManabox.language == manabox_row.language,
+            )
+            .first()
         )
-        return product
+
-    
-    def get_tcgplayer_card(self, card_manabox: CardManabox) -> CardTCGPlayer:
-        # check if tcgplayer_id exists for product_id in CardTCGPlayer
-        tcgplayer_card = self.db.query(CardTCGPlayer).filter(CardTCGPlayer.product_id == card_manabox.product_id).first()
-        if tcgplayer_card:
-            return tcgplayer_card
-        # if not, get tcgplayer_id from tcgplayer_service
-        tcgplayer_card = self.tcgplayer_service.get_tcgplayer_card(card_manabox)
-        return tcgplayer_card
-    
-    def create_card(self, card_manabox: CardManabox) -> Card:
-        tcgplayer_card = self.get_tcgplayer_card(card_manabox)
-        card = Card(
-            product_id = tcgplayer_card.product_id if tcgplayer_card else card_manabox.product_id,
-            number = card_manabox.collector_number,
-            foil = card_manabox.foil,
-            rarity = card_manabox.rarity,
-            condition = card_manabox.condition,
-            language = card_manabox.language,
-            scryfall_id = card_manabox.scryfall_id,
-            manabox_id = card_manabox.manabox_id,
-            tcgplayer_id = tcgplayer_card.tcgplayer_id if tcgplayer_card else None
-        )
-        return card
-    
-    def card_manabox_lookup_create_if_not_exist(self, manabox_row: ManaboxRow) -> CardManabox:
-        # query based on all fields in manabox_row
-        card_manabox = self.db.query(CardManabox).filter(
-            CardManabox.name == manabox_row.name,
-            CardManabox.set_code == manabox_row.set_code,
-            CardManabox.set_name == manabox_row.set_name,
-            CardManabox.collector_number == manabox_row.collector_number,
-            CardManabox.foil == manabox_row.foil,
-            CardManabox.rarity == manabox_row.rarity,
-            CardManabox.manabox_id == manabox_row.manabox_id,
-            CardManabox.scryfall_id == manabox_row.scryfall_id,
-            CardManabox.condition == manabox_row.condition,
-            CardManabox.language == manabox_row.language
-        ).first()
         if not card_manabox:
-            # create new card_manabox, card, and product
+            logger.debug(f"card_manabox not found for row: {manabox_row.__dict__}")
-            with db_transaction(self.db):
+            group = (
-                card_manabox = self.create_card_manabox(manabox_row)
+                self.db.query(TCGPlayerGroups)
-                product = self.create_product(card_manabox)
+                .filter(TCGPlayerGroups.abbreviation == manabox_row.set_code)
-                card = self.create_card(card_manabox)
+                .first()
-                card_manabox.product_id = card.product_id
+            )
-                product.id = card.product_id
+            if not group:
-                self.db.add(card_manabox)
+                logger.error(f"Group not found for set code: {manabox_row.set_code}")
-                self.db.add(product)
+                logger.error(f"Row data: {manabox_row.__dict__}")
-                self.db.add(card)
+                return None
+
+            card_tcgplayer = self.tcgplayer_service.get_card_tcgplayer_from_manabox_row(
+                manabox_row, group.group_id
+            )
+            if not card_tcgplayer:
+                logger.error(f"Card not found for row: {manabox_row.__dict__}")
+                return None
+            card_manabox = self.create_card_manabox(manabox_row, card_tcgplayer)
+
         return card_manabox
 
-    def bg_process_manabox_file(self, file_id: str):
+    def process_manabox_df(self, df: DataFrame, file_id: str) -> None:
+        """Process a Manabox dataframe.
+
+        Args:
+            df: The Manabox dataframe to process
+            file_id: The ID of the file being processed
+        """
+        for _, row in df.iterrows():
+            manabox_row = ManaboxRow(row)
+            card_manabox = self.card_manabox_lookup_create_if_not_exist(manabox_row)
+            if not card_manabox:
+                continue
+            self.create_staged_file_product(file_id, card_manabox, row)
+
+    def bg_process_manabox_file(self, file_id: str) -> None:
+        """Process a Manabox file in the background.
+
+        Args:
+            file_id: The ID of the file to process
+
+        Raises:
+            Exception: If there's an error during processing
+        """
         try:
-            file = self.file_service.get_file(file_id)
+            manabox_file = self.file_service.get_file(file_id)
-            df = self._manabox_file_to_df(file)
+            manabox_df = self.df_util.file_to_df(manabox_file)
-            for index, row in df.iterrows():
+            self.process_manabox_df(manabox_df, file_id)
-                manabox_row = ManaboxRow(row)
+
-                # create card concepts - manabox, tcgplayer, card, product
-                card_manabox = self.card_manabox_lookup_create_if_not_exist(manabox_row)
-                # create staged inventory with quantity for processing down the marketplace pipeline
-                staged_product = self.create_staged_product(file, card_manabox, row)
-            # update file status
             with db_transaction(self.db):
-                file.status = 'completed'
+                manabox_file.status = "completed"
+
         except Exception as e:
             with db_transaction(self.db):
-                file.status = 'error'
+                manabox_file.status = "error"
             raise e
+
         try:
-            # create storage records for physically storing individual cards
+            self.storage_service.store_staged_products_for_file(file_id)
-            self.storage_service.store_staged_products_for_file(file.id)
         except Exception as e:
             logger.error(f"Error creating storage records: {str(e)}")
             raise e

services/storage.py

@@ -1,55 +1,103 @@
+from uuid import uuid4
+from typing import List, TypedDict, Optional
 from sqlalchemy.orm import Session
+
 from db.utils import db_transaction
-from db.models import Warehouse, User, StagedFileProduct, StorageBlock, ProductBlock, File, Card
+from db.models import (
-from uuid import uuid4 as uuid
+    Warehouse,
-from typing import List, TypedDict
+    User,
+    StagedFileProduct,
+    StorageBlock,
+    ProductBlock,
+    File,
+    CardTCGPlayer
+)
 
 class ProductAttributes(TypedDict):
+    """Attributes for a product to be stored."""
     product_id: str
     card_number: str
 
 class StorageService:
-    def __init__(self, db: Session):
+    """Service for managing product storage and warehouse operations."""
+    
+    def __init__(self, db: Session) -> None:
+        """Initialize the storage service.
+        
+        Args:
+            db: SQLAlchemy database session
+        """
         self.db = db
     
     def get_or_create_user(self, username: str) -> User:
+        """Get an existing user or create a new one if not found.
+        
+        Args:
+            username: Username to look up or create
+            
+        Returns:
+            The existing or newly created User
+        """
         user = self.db.query(User).filter(User.username == username).first()
         if user is None:
             user = User(
-                id = str(uuid()),
+                id=str(uuid4()),
-                username = username
+                username=username
             )
             with db_transaction(self.db):
                 self.db.add(user)
         return user
     
     def get_or_create_warehouse(self) -> Warehouse:
+        """Get the default warehouse or create it if it doesn't exist.
+        
+        Returns:
+            The existing or newly created Warehouse
+        """
         warehouse = self.db.query(Warehouse).first()
         user = self.get_or_create_user('admin')
         if warehouse is None:
             warehouse = Warehouse(
-                id = str(uuid()),
+                id=str(uuid4()),
-                user_id = user.id
+                user_id=user.id
             )
             with db_transaction(self.db):
                 self.db.add(warehouse)
         return warehouse
     
     def get_staged_product(self, file_id: str) -> List[StagedFileProduct]:
-        staged_product = self.db.query(StagedFileProduct).filter(StagedFileProduct.file_id == file_id).all()
+        """Get all staged products for a given file.
-        return staged_product
+        
+        Args:
+            file_id: ID of the file to get staged products for
+            
+        Returns:
+            List of staged products
+        """
+        return self.db.query(StagedFileProduct).filter(
+            StagedFileProduct.file_id == file_id
+        ).all()
     
     def get_storage_block_name(self, warehouse: Warehouse, file_id: str) -> str:
-        # Get file type from id
+        """Generate a unique name for a new storage block.
+        
+        Args:
+            warehouse: Warehouse the block belongs to
+            file_id: ID of the file being processed
+            
+        Returns:
+            Unique storage block name
+            
+        Raises:
+            ValueError: If no file is found with the given ID
+        """
         current_file = self.db.query(File).filter(File.id == file_id).first()
         if not current_file:
             raise ValueError(f"No file found with id {file_id}")
 
-        # Determine storage block type
         storage_block_type = 'rare' if 'rare' in current_file.type else 'common'
         prefix = storage_block_type[0]
 
-        # Get most recent storage block with same type and warehouse id
         latest_block = (
             self.db.query(StorageBlock)
             .filter(
@@ -60,18 +108,10 @@ class StorageService:
             .first()
         )
 
-        # If no existing block, start with number 1
+        start_number = 1 if not latest_block else int(latest_block.name[1:]) + 1
-        if not latest_block:
-            return f"{prefix}1"
-
-        # Start with the next number after the latest block
-        number = int(latest_block.name[1:])
         
         while True:
-            number += 1
+            new_name = f"{prefix}{start_number}"
-            new_name = f"{prefix}{number}"
-            
-            # Check if the new name already exists
             exists = (
                 self.db.query(StorageBlock)
                 .filter(
@@ -83,8 +123,21 @@ class StorageService:
             
             if not exists:
                 return new_name
+            start_number += 1
     
     def create_storage_block(self, warehouse: Warehouse, file_id: str) -> StorageBlock:
+        """Create a new storage block for the given warehouse and file.
+        
+        Args:
+            warehouse: Warehouse to create the block in
+            file_id: ID of the file being processed
+            
+        Returns:
+            Newly created StorageBlock
+            
+        Raises:
+            ValueError: If no file is found with the given ID
+        """
         current_file = self.db.query(File).filter(File.id == file_id).first()
         if not current_file:
             raise ValueError(f"No file found with id {file_id}")
@@ -92,10 +145,10 @@ class StorageService:
         storage_block_type = 'rare' if 'rare' in current_file.type else 'common'
         
         storage_block = StorageBlock(
-            id = str(uuid()),
+            id=str(uuid4()),
-            warehouse_id = warehouse.id,
+            warehouse_id=warehouse.id,
-            name = self.get_storage_block_name(warehouse, file_id),
+            name=self.get_storage_block_name(warehouse, file_id),
-            type = storage_block_type
+            type=storage_block_type
         )
         with db_transaction(self.db):
             self.db.add(storage_block)
@@ -108,9 +161,19 @@ class StorageService:
         product_attributes: ProductAttributes,
         block_index: int
     ) -> ProductBlock:
-        """Create a new ProductBlock for a single unit of a staged product."""
+        """Create a new ProductBlock for a single unit of a staged product.
+        
+        Args:
+            staged_product: The staged product to store
+            storage_block: The block to store the product in
+            product_attributes: Additional product attributes
+            block_index: Index within the storage block
+            
+        Returns:
+            Newly created ProductBlock
+        """
         product_block = ProductBlock(
-            id=str(uuid()),
+            id=str(uuid4()),
             product_id=staged_product.product_id,
             block_id=storage_block.id,
             block_index=block_index
@@ -125,14 +188,21 @@ class StorageService:
         self,
         staged_product: StagedFileProduct
     ) -> List[ProductAttributes]:
-        """Get attributes for each unit of a staged product."""
+        """Get attributes for each unit of a staged product.
+        
+        Args:
+            staged_product: The staged product to get attributes for
+            
+        Returns:
+            List of attributes for each unit of the product
+        """
         result = (
             self.db.query(
                 StagedFileProduct.product_id,
                 StagedFileProduct.quantity,
-                Card.number
+                CardTCGPlayer.number
             )
-            .join(Card, Card.product_id == StagedFileProduct.product_id)
+            .join(CardTCGPlayer, CardTCGPlayer.product_id == StagedFileProduct.product_id)
             .filter(StagedFileProduct.id == staged_product.id)
             .first()
         )
@@ -145,11 +215,18 @@ class StorageService:
                 product_id=result.product_id,
                 card_number=result.number
             )
-            for i in range(result.quantity)
+            for _ in range(result.quantity)
         ]
 
     def store_staged_products_for_file(self, file_id: str) -> StorageBlock:
-        """Store all staged products for a file in a new storage block."""
+        """Store all staged products for a file in a new storage block.
+        
+        Args:
+            file_id: ID of the file containing staged products
+            
+        Returns:
+            The newly created StorageBlock containing all products
+        """
         warehouse = self.get_or_create_warehouse()
         storage_block = self.create_storage_block(warehouse, file_id)
         staged_products = self.get_staged_product(file_id)
@@ -162,7 +239,10 @@ class StorageService:
                 all_product_attributes.append((staged_product, attrs))
         
         # Sort by card number as integer to determine block indices
-        sorted_attributes = sorted(all_product_attributes, key=lambda x: int(x[1]['card_number']))
+        sorted_attributes = sorted(
+            all_product_attributes,
+            key=lambda x: int(''.join(filter(str.isdigit, x[1]['card_number'])))
+        )
         
         # Add products with correct block indices
         for block_index, (staged_product, product_attributes) in enumerate(sorted_attributes, 1):

services/task.py

@@ -30,12 +30,12 @@ class TaskService:
             id='daily_report'
         )
             
-        self.scheduler.add_job(
+#        self.scheduler.add_job(
-            self.pricing_update,
+#            self.pricing_update,
-            'cron',
+#            'cron',
-            minute=28,
+#            minute=41,
-            id='pricing_update'
+#            id='pricing_update'
-        )
+#        )
     
     def daily_report(self):  # Removed async
         self.logger.info("Generating daily report")

services/tcgplayer.py

@@ -1,8 +1,10 @@
-from db.models import ManaboxExportData, Box, TCGPlayerGroups, TCGPlayerInventory, TCGPlayerExportHistory, TCGPlayerPricingHistory, TCGPlayerProduct, ManaboxTCGPlayerMapping, CardManabox, CardTCGPlayer, Price
+from db.models import ManaboxExportData, Box, TCGPlayerGroups, TCGPlayerInventory, TCGPlayerExportHistory, TCGPlayerPricingHistory, TCGPlayerProduct, ManaboxTCGPlayerMapping, CardManabox, CardTCGPlayer, Price, Product, Card, File
 import requests
+from services.util._dataframe import TCGPlayerPricingRow, DataframeUtil, ManaboxRow
+from services.file import FileService
 from sqlalchemy.orm import Session
 from db.utils import db_transaction
-import uuid
+from uuid import uuid4 as uuid
 import browser_cookie3
 import webbrowser
 from typing import Optional, Dict ,List
@@ -20,6 +22,7 @@ from services.pricing import PricingService
 from sqlalchemy.sql import exists
 import pandas as pd
 from sqlalchemy.exc import SQLAlchemyError
+from schemas.file import CreateFileRequest
 
 
 logger = logging.getLogger(__name__)
@@ -42,6 +45,8 @@ class TCGPlayerConfig:
 
 class TCGPlayerService:
     def __init__(self, db: Session,
+                 pricing_service: PricingService,
+                 file_service: FileService,
                  config: TCGPlayerConfig=TCGPlayerConfig(), 
                  browser_type: Browser=Browser.BRAVE):
         self.db = db
@@ -49,11 +54,14 @@ class TCGPlayerService:
         self.browser_type = browser_type
         self.cookies = None
         self.previous_request_time = None
+        self.pricing_service = pricing_service
+        self.df_util = DataframeUtil()
+        self.file_service = file_service
     
     def _insert_groups(self, groups):
         for group in groups:
             db_group = TCGPlayerGroups(
-                id=str(uuid.uuid4()),
+                id=str(uuid()),
                 group_id=group['groupId'],
                 name=group['name'],
                 abbreviation=group['abbreviation'],
@@ -70,15 +78,27 @@ class TCGPlayerService:
         response.raise_for_status()
         groups = response.json()['results']
         # manually add broken groups
-        groups.append({
+        manual_groups = [
-            "groupId": 2422,
+                        {
-            "name": "Modern Horizons 2 Timeshifts",
+                "groupId": 2422,
-            "abbreviation": "H2R",
+                "name": "Modern Horizons 2 Timeshifts",
-            "isSupplemental": "false",
+                "abbreviation": "H2R",
-            "publishedOn": "2018-11-08T00:00:00",
+                "isSupplemental": "0",
-            "modifiedOn": "2018-11-08T00:00:00",
+                "publishedOn": "2018-11-08T00:00:00",
-            "categoryId": 1
+                "modifiedOn": "2018-11-08T00:00:00",
-        })
+                "categoryId": 1
+            },
+            {
+                "groupId": 52,
+                "name": "Store Championships",
+                "abbreviation": "SCH",
+                "isSupplemental": "1",
+                "publishedOn": "2007-07-14T00:00:00",
+                "modifiedOn": "2007-07-14T00:00:00",
+                "categoryId": 1
+            }
+        ]
+        groups.extend(manual_groups)
         # Insert groups into db
         with db_transaction(self.db):
             self._insert_groups(groups)
@@ -230,7 +250,7 @@ class TCGPlayerService:
     def update_inventory(self, version: str) -> Dict:
         if version not in ['staged', 'live']:
             raise ValueError("Invalid inventory version")
-        export_id = str(uuid.uuid4())
+        export_id = str(uuid())
         inventory = self._get_inventory(version)
         if not inventory:
             return {"message": "No inventory to update"}
@@ -267,14 +287,14 @@ class TCGPlayerService:
 
         with db_transaction(self.db):
             export_history = TCGPlayerExportHistory(
-                id=str(uuid.uuid4()),
+                id=str(uuid()),
                 type=version + '_inventory',
                 inventory_export_id=export_id
             )
             self.db.add(export_history)
             for item in inventory:
                 db_item = TCGPlayerInventory(
-                    id=str(uuid.uuid4()),
+                    id=str(uuid()),
                     **{db_field: item.get(csv_field) 
                     for csv_field, db_field in inventory_fields.items()}
                 )
@@ -301,7 +321,7 @@ class TCGPlayerService:
         pass
     
     def update_pricing(self, set_name_ids: Dict[str, List[str]]) -> Dict:
-        export_id = str(uuid.uuid4())
+        export_id = str(uuid())
         product_fields = {
             'TCGplayer Id': 'tcgplayer_id',
             'group_id': 'group_id',
@@ -337,7 +357,7 @@ class TCGPlayerService:
                 else:
                     with db_transaction(self.db):
                         product = TCGPlayerProduct(
-                            id=str(uuid.uuid4()),
+                            id=str(uuid()),
                             **{db_field: item.get(csv_field)
                             for csv_field, db_field in product_fields.items()}
                         )
@@ -346,7 +366,7 @@ class TCGPlayerService:
 
                 with db_transaction(self.db):
                     ph_item = TCGPlayerPricingHistory(
-                        id=str(uuid.uuid4()),
+                        id=str(uuid()),
                         **{db_field: item.get(csv_field) 
                         for csv_field, db_field in pricing_fields.items()}
                     )
@@ -355,7 +375,7 @@ class TCGPlayerService:
 
         with db_transaction(self.db):
             export_history = TCGPlayerExportHistory(
-                id=str(uuid.uuid4()),
+                id=str(uuid()),
                 type='pricing',
                 pricing_export_id=export_id
             )
@@ -457,34 +477,173 @@ class TCGPlayerService:
         add_csv = None
         return add_csv
     
-    def load_export_csv_to_card_tcgplayer(self, export_csv: bytes, group_id: int) -> None:
+    def create_tcgplayer_card(self, row: TCGPlayerPricingRow, group_id: int):
-        if not export_csv:
+        # if card already exists, return none
-            raise ValueError("No export CSV provided")
+        card_exists = self.db.query(CardTCGPlayer).filter(
-        
+            CardTCGPlayer.tcgplayer_id == row.tcgplayer_id,
-        # Convert bytes to string first
+            CardTCGPlayer.group_id == group_id
-        text_content = export_csv.decode('utf-8')
+        ).first()
-        csv_file = StringIO(text_content)
+        if card_exists:
-        try:
+            return card_exists
-            reader = csv.DictReader(csv_file)
+        # create product
-            for row in reader:
+        product = Product(
-                product = CardTCGPlayer(
+            id=str(uuid()),
-                    product_id=str(uuid.uuid4()),
+            type = 'card',
-                    tcgplayer_id=row['TCGplayer Id'],
+            product_line = 'mtg'
-                    group_id=group_id,
+        )
-                    product_line=row['Product Line'],
+        # create card
-                    set_name=row['Set Name'],
+        card = Card(
-                    product_name=row['Product Name'],
+            product_id=product.id,
-                    title=row['Title'],
+        )
-                    number=row['Number'],
+        # create Cardtcgplayer
-                    rarity=row['Rarity'],
+        tcgcard = CardTCGPlayer(
-                    condition=row['Condition']
+            product_id=product.id,
-                )
+            group_id=group_id,
-                with db_transaction(self.db):
+            tcgplayer_id=row.tcgplayer_id,
-                    self.db.add(product)
+            product_line=row.product_line,
-        finally:
+            set_name=row.set_name,
-            csv_file.close()
+            product_name=row.product_name,
+            title=row.title,
+            number=row.number,
+            rarity=row.rarity,
+            condition=row.condition
+        )
+        with db_transaction(self.db):
+            self.db.add(product)
+            self.db.add(card)
+            self.db.add(tcgcard)
+        return tcgcard
     
-    def match_card_tcgplayer_to_manabox(self, card: CardManabox, group_id: int) -> CardTCGPlayer:
+    def create_tcgplayer_cards_batch(self, rows: list[TCGPlayerPricingRow], set_to_group: dict) -> list[CardTCGPlayer]:
+        # Get existing cards in a single query
+        existing_cards = {
+            (card.tcgplayer_id, card.group_id): card 
+            for card in self.db.query(CardTCGPlayer).filter(
+                CardTCGPlayer.tcgplayer_id.in_([row.tcgplayer_id for row in rows]),
+                CardTCGPlayer.group_id.in_([set_to_group[row.set_name] for row in rows])
+            ).all()
+        }
+        
+        # Pre-allocate lists for better memory efficiency
+        new_products = []
+        new_cards = []
+        new_tcgcards = []
+        # new_prices = []
+        
+        # price_types = {
+        #    'tcg_market_price': 'tcg_market_price',
+        #    'tcg_direct_low': 'tcg_direct_low',
+        #    'tcg_low_price_with_shipping': 'tcg_low_price_with_shipping',
+        #    'tcg_low_price': 'tcg_low_price',
+        #    'tcg_marketplace_price': 'tcg_marketplace_price'
+        #}
+        
+        for row in rows:
+            # Get the correct group_id for this row's set
+            group_id = set_to_group[row.set_name]
+            
+            if (row.tcgplayer_id, group_id) in existing_cards:
+                continue
+                
+            product_id = str(uuid())
+            
+            new_products.append(Product(
+                id=product_id,
+                type='card',
+                product_line='mtg'
+            ))
+            
+            new_cards.append(Card(
+                product_id=product_id,
+            ))
+            
+            new_tcgcards.append(CardTCGPlayer(
+                product_id=product_id,
+                group_id=group_id,  # Use the correct group_id for this specific row
+                tcgplayer_id=row.tcgplayer_id,
+                product_line=row.product_line,
+                set_name=row.set_name,
+                product_name=row.product_name,
+                title=row.title,
+                number=row.number,
+                rarity=row.rarity,
+                condition=row.condition
+            ))
+
+            # Batch create price objects
+            # row_prices = [
+            #     Price(
+            #         id=str(uuid()),
+            #         product_id=product_id,
+            #         marketplace_id=None,
+            #         type=price_type,
+            #         price=getattr(row, col_name)
+            #     )
+            #     for col_name, price_type in price_types.items()
+            #     if getattr(row, col_name, None) is not None and getattr(row, col_name) > 0
+            # ]
+            # new_prices.extend(row_prices)
+        
+        if new_products:
+            with db_transaction(self.db):
+                self.db.bulk_save_objects(new_products)
+                self.db.bulk_save_objects(new_cards)
+                self.db.bulk_save_objects(new_tcgcards)
+                # if new_prices:
+                #    self.db.bulk_save_objects(new_prices)
+        
+        return new_tcgcards
+
+    def load_export_csv_to_card_tcgplayer(self, export_csv: bytes, file_id: str = None, batch_size: int = 1000) -> None:
+        try:
+            if not export_csv:
+                raise ValueError("No export CSV provided")
+            
+            df = self.df_util.csv_bytes_to_df(export_csv)
+            
+            logger.debug(f"Loaded {len(df)} rows from export CSV")
+
+            # Get all group_ids upfront in a single query
+            set_to_group = dict(
+                self.db.query(TCGPlayerGroups.name, TCGPlayerGroups.group_id).all()
+            )
+            
+            # Process in batches
+            for i in range(0, len(df), batch_size):
+                batch_df = df.iloc[i:i + batch_size]
+                batch_rows = [TCGPlayerPricingRow(row) for _, row in batch_df.iterrows()]
+                
+                # Filter rows with valid group_ids
+                valid_rows = [
+                    row for row in batch_rows 
+                    if row.set_name in set_to_group
+                ]
+
+                # logger.debug(f"Processing batch {i // batch_size + 1}: {len(valid_rows)} valid rows")
+                
+                if valid_rows:
+                    # Pass the entire set_to_group mapping
+                    self.create_tcgplayer_cards_batch(valid_rows, set_to_group)
+        except Exception as e:
+            logger.error(f"Failed to load export CSV: {e}")
+            # set file upload to failed
+            if file_id:
+                with db_transaction(self.db):
+                    file = self.db.query(File).filter(File.id == file_id).first()
+                    if file:
+                        file.status = 'failed'
+                        self.db.add(file)
+            raise
+        finally:
+            if file_id:
+                with db_transaction(self.db):
+                    file = self.db.query(File).filter(File.id == file_id).first()
+                    if file:
+                        file.status = 'completed'
+                        self.db.add(file)
+
+
+    def get_card_tcgplayer_from_manabox_row(self, card: ManaboxRow, group_id: int) -> CardTCGPlayer:
         # Expanded rarity mapping
         mb_to_tcg_rarity_mapping = {
             "common": "C",
@@ -516,12 +675,20 @@ class TCGPlayerService:
             return None
 
         # First query for matching products without rarity filter
+        # debug
+        # log everything in this query
+        # remove letters from card.collector_number FOR JOIN ONLY
+        join_collector_number = ''.join(filter(str.isdigit, card.collector_number))
+        # logger.debug(f"Querying for card: {card.name}, {card.set_code}, {card.collector_number}, {tcg_condition}, {group_id}")
         base_query = self.db.query(CardTCGPlayer).filter(
-            CardTCGPlayer.number == card.collector_number,
+            CardTCGPlayer.number == join_collector_number,
             CardTCGPlayer.condition == tcg_condition,
-            CardTCGPlayer.group_id == group_id
+            CardTCGPlayer.group_id == group_id,
+            CardTCGPlayer.rarity != "T" # TOKENS ARE NOT SUPPORTED CUZ BROKE LOL
         )
 
+        # logger.debug(f"Base query: {base_query.statement.compile(compile_kwargs={'literal_binds': True})}")
+
         # Get all potential matches
         products = base_query.all()
         
@@ -535,44 +702,42 @@ class TCGPlayerService:
             if product.rarity == "L" or product.rarity == tcg_rarity:
                 return product
 
-        # If we got here, we found products but none matched our rarity criteria
+        # ignore rarity, just make sure only one product is returned
-        logger.error(f"No matching TCGPlayer product with correct rarity found for card {card.name} ({card.set_name} {card.collector_number})")
+        if len(products) > 1:
-        return None  
+            # try to match on name before failing
-    
+            for product in products:
-    def get_tcgplayer_card(self, card: CardManabox) -> CardTCGPlayer:
+                if product.product_name == card.name:
-        # find tcgplayer group id for set code
+                    return product
-        group_id = self.db.query(TCGPlayerGroups.group_id).filter(
+        elif len(products) == 1:
-            TCGPlayerGroups.abbreviation == card.set_code
+            return products[0]
-        ).first()
-        if not group_id:
-            logger.error(f"Group ID not found for set code {card.set_code}")
-            logger.error(f"Card details: name={card.name}, set_name={card.set_name}, collector_number={card.collector_number}")
-            return None
-        group_id = group_id[0]
-        # check for group_id in CardTCGPlayer
-        group_id_exists = self.db.query(CardTCGPlayer).filter(
-            CardTCGPlayer.group_id == group_id).first()
-        if not group_id_exists:
-            export_csv = self._get_export_csv([str(group_id)], convert=False) # TODO should be file service
-            self.load_export_csv_to_card_tcgplayer(export_csv, group_id)
-        # match card to tcgplayer product
-        matching_product = self.match_card_tcgplayer_to_manabox(card, group_id)
-        if not matching_product:
-            return None
-        return matching_product
-
-    def get_pricing_export_for_all_products(self) -> bytes:
-        """
-        Retrieves pricing export data for all products in TCGPlayer format.
         
-        Returns:
+        logger.error(f"Multiple matching TCGPlayer products found for card {card.name} ({card.set_code} {card.collector_number})")
-            bytes: Raw CSV data containing pricing information
+        return None
+
+        # If we got here, we found products but none matched our rarity criteria
+        # logger.error(f"No matching TCGPlayer product with correct rarity found for card {card.name} {card.rarity} {group_id} ({card.set_name} {card.collector_number})")
+        # return None
+
+
+    def get_pricing_export_for_all_products(self) -> File:
         """
+        """
+        DEBUG = True
+        if DEBUG:
+            logger.debug("DEBUG: Using existing pricing export file")
+            file = self.db.query(File).filter(File.type == 'tcgplayer_pricing_export').first()
+            if file:
+                return file
         try:
             all_group_ids = self.db.query(TCGPlayerGroups.group_id).all()
             all_group_ids = [str(group_id) for group_id, in all_group_ids]
             export_csv = self._get_export_csv(all_group_ids, convert=False)
-            return export_csv
+            export_csv_file = self.file_service.create_file(export_csv, CreateFileRequest(
+                source="tcgplayer",
+                type="tcgplayer_pricing_export",
+                filename="tcgplayer_pricing_export.csv"
+            ))
+            return export_csv_file
         except SQLAlchemyError as e:
             raise RuntimeError(f"Failed to retrieve group IDs: {str(e)}")
 
@@ -613,63 +778,21 @@ class TCGPlayerService:
     def cron_load_prices(self) -> None:
         """
         Scheduled task to load and update product prices.
-        Fetches current prices, processes them, and saves new price records to the database.
+        Uses optimized bulk processing for better performance.
         """
+        logger.debug("Running cron_load_prices...")
         try:
-            # Get and process price data
+            # Get pricing export
-            price_csv = self.get_pricing_export_for_all_products()
+            export_csv_file = self.get_pricing_export_for_all_products()
-            price_df = self.pricing_export_to_df(price_csv)
+            export_csv = self.file_service.get_file_content(export_csv_file.id)
+
+            # load to card tcgplayer
+            self.load_export_csv_to_card_tcgplayer(export_csv, export_csv_file.id)
             
-            # Clean column names
+            # Process the export with optimized bulk operations
-            price_df.columns = price_df.columns.str.lower().str.replace(' ', '_')
+            # the pricing service proves that there is no god
+            # self.pricing_service.process_pricing_export(export_csv)
             
-            # Get all products efficiently
-            products_query = self.db.query(
-                CardTCGPlayer.tcgplayer_id,
-                CardTCGPlayer.product_id
-            )
-            product_df = pd.read_sql(products_query.statement, self.db.bind)
-            
-            # Merge dataframes
-            merged_df = pd.merge(
-                price_df,
-                product_df,
-                on='tcgplayer_id',
-                how='inner'
-            )
-            
-            # Define price columns to process
-            price_columns = [
-                'tcg_market_price',
-                'tcg_direct_low',
-                'tcg_low_price_with_shipping',
-                'tcg_low_price',
-                'tcg_marketplace_price'
-            ]
-            
-            # Process in batches to avoid memory issues
-            BATCH_SIZE = 1000
-            for price_column in price_columns:
-                records = []
-                
-                for start_idx in range(0, len(merged_df), BATCH_SIZE):
-                    batch_df = merged_df.iloc[start_idx:start_idx + BATCH_SIZE]
-                    
-                    batch_records = [
-                        Price(
-                            id=str(uuid.uuid4()),
-                            product_id=row['product_id'],
-                            type=price_column,
-                            price=row[price_column]
-                        )
-                        for _, row in batch_df.iterrows()
-                        if pd.notna(row[price_column])  # Skip rows with NaN prices
-                    ]
-                    
-                    with db_transaction(self.db):
-                        self.db.bulk_save_objects(batch_records)
-                        self.db.flush()
-                    
         except Exception as e:
-            logger.error(f"Failed to load prices: {str(e)}")
+            logger.error(f"Failed to load prices: {e}")
             raise

services/unholy_pricing.py

@@ -0,0 +1,500 @@
+from dataclasses import dataclass
+from decimal import Decimal, ROUND_HALF_UP
+from enum import Enum
+from typing import Optional, Dict, List, Any
+import pandas as pd
+import logging
+from db.models import Product, Price
+from sqlalchemy.orm import Session
+from uuid import uuid4 as uuid
+from datetime import datetime
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from sqlalchemy import text
+from services.util._dataframe import DataframeUtil
+
+
+logger = logging.getLogger(__name__)
+
+class PriceType(str, Enum):
+    TCG_MARKET = 'tcg_market_price'
+    TCG_DIRECT_LOW = 'tcg_direct_low'
+    TCG_LOW_WITH_SHIPPING = 'tcg_low_price_with_shipping'
+    TCG_LOW = 'tcg_low_price'
+    TCG_MARKETPLACE = 'tcg_marketplace_price'
+    MY_PRICE = 'my_price'
+
+class PricingStrategy(str, Enum):
+    DEFAULT = 'default'
+    AGGRESSIVE = 'aggressive'
+    CONSERVATIVE = 'conservative'
+
+@dataclass
+class PriceRange:
+    min_price: Decimal
+    max_price: Decimal
+    multiplier: Decimal
+    ceiling_price: Optional[Decimal] = None
+    include_shipping: bool = False
+
+    def __post_init__(self):
+        # Convert all values to Decimal for precise calculations
+        self.min_price = Decimal(str(self.min_price))
+        self.max_price = Decimal(str(self.max_price))
+        self.multiplier = Decimal(str(self.multiplier))
+        if self.ceiling_price is not None:
+            self.ceiling_price = Decimal(str(self.ceiling_price))
+
+    def contains_price(self, price: Decimal) -> bool:
+        """Check if a price falls within this range, inclusive of min, exclusive of max."""
+        return self.min_price <= price < self.max_price
+
+    def calculate_price(self, base_price: Decimal) -> Decimal:
+        """Calculate the final price for this range, respecting ceiling."""
+        calculated = base_price * self.multiplier
+        if self.ceiling_price is not None:
+            calculated = min(calculated, self.ceiling_price)
+        return calculated.quantize(Decimal('0.01'), rounding=ROUND_HALF_UP)
+
+class PricingConfiguration:
+    """Centralized configuration for pricing rules and thresholds."""
+    
+    # Price thresholds
+    FLOOR_PRICE = Decimal('0.35')
+    MAX_PRICE = Decimal('100000.00')  # Safety cap for maximum price
+    SHIPPING_THRESHOLD = Decimal('5.00')
+    
+    # Multipliers
+    FLOOR_MULT = Decimal('1.25')
+    NEAR_FLOOR_MULT = Decimal('1.25')
+    UNDER_FIVE_MULT = Decimal('1.25')
+    FIVE_TO_TEN_MULT = Decimal('1.15')
+    TEN_TO_TWENTYFIVE_MULT = Decimal('1.10')
+    TWENTYFIVE_TO_FIFTY_MULT = Decimal('1.05')
+    FIFTY_PLUS_MULT = Decimal('1.025')
+    
+    # Price variance thresholds
+    MAX_PRICE_VARIANCE = Decimal('0.50')  # Maximum allowed variance between prices as a ratio
+    
+    @classmethod
+    def get_price_ranges(cls) -> list[PriceRange]:
+        """Get the list of price ranges with their respective rules."""
+        return [
+            PriceRange(
+                min_price=Decimal('0'),
+                max_price=cls.FLOOR_PRICE,
+                multiplier=cls.FLOOR_MULT,
+                include_shipping=False
+            ),
+            PriceRange(
+                min_price=cls.FLOOR_PRICE,
+                max_price=Decimal('5'),
+                multiplier=cls.UNDER_FIVE_MULT,
+                ceiling_price=Decimal('4.99'),
+                include_shipping=False
+            ),
+            PriceRange(
+                min_price=Decimal('5'),
+                max_price=Decimal('10'),
+                multiplier=cls.FIVE_TO_TEN_MULT,
+                ceiling_price=Decimal('9.99'),
+                include_shipping=True
+            ),
+            PriceRange(
+                min_price=Decimal('10'),
+                max_price=Decimal('25'),
+                multiplier=cls.TEN_TO_TWENTYFIVE_MULT,
+                ceiling_price=Decimal('24.99'),
+                include_shipping=True
+            ),
+            PriceRange(
+                min_price=Decimal('25'),
+                max_price=Decimal('50'),
+                multiplier=cls.TWENTYFIVE_TO_FIFTY_MULT,
+                ceiling_price=Decimal('49.99'),
+                include_shipping=True
+            ),
+            PriceRange(
+                min_price=Decimal('50'),
+                max_price=cls.MAX_PRICE,
+                multiplier=cls.FIFTY_PLUS_MULT,
+                include_shipping=True
+            )
+        ]
+
+class PriceCalculationResult:
+    """Represents the result of a price calculation."""
+    
+    def __init__(
+        self,
+        product: Product,
+        calculated_price: Optional[Decimal],
+        base_prices: Dict[str, Decimal],
+        error: Optional[str] = None
+    ):
+        self.product = product
+        self.calculated_price = calculated_price
+        self.base_prices = base_prices
+        self.error = error
+        
+    @property
+    def success(self) -> bool:
+        return self.calculated_price is not None and self.error is None
+
+    @property
+    def max_base_price(self) -> Optional[Decimal]:
+        """Returns the highest base price."""
+        return max(self.base_prices.values()) if self.base_prices else None
+
+
+class PricingService:
+    CHUNK_SIZE = 5000  # Configurable batch size
+    MAX_WORKERS = 4    # Configurable worker count
+
+    def __init__(self, db: Session):
+        self.db = db
+        self.df_util = DataframeUtil()
+        self.config = PricingConfiguration
+        self.price_ranges = self.config.get_price_ranges()
+
+    def get_product_by_id(self, product_id: str) -> Optional[Product]:
+        """Get a product by its ID."""
+        return self.db.query(Product)\
+            .filter(Product.id == str(product_id))\
+            .all()[0] if len(self.db.query(Product)\
+            .filter(Product.id == str(product_id))\
+            .all()) > 0 else None
+
+    def get_latest_price_for_product(self, product: Product, price_type: PriceType) -> Optional[Price]:
+        """Get the most recent price of a specific type for a product."""
+        prices = self.db.query(Price)\
+            .filter(
+                Price.product_id == str(product.id),
+                Price.type == price_type.value
+            )\
+            .order_by(Price.date_created.desc())\
+            .all()
+        return prices[0] if prices else None
+
+    def get_historical_prices_for_product(
+        self, product: Product, price_type: Optional[PriceType] = None
+    ) -> dict[PriceType, list[Price]]:
+        """Get historical prices for a product, optionally filtered by type."""
+        query = self.db.query(Price).filter(Price.product_id == str(product.id))
+        
+        if price_type:
+            query = query.filter(Price.type == price_type.value)  # Fixed: Use enum value
+            
+        prices = query.order_by(Price.date_created.desc()).all()
+        
+        if price_type:
+            return {price_type: prices}
+            
+        # Group prices by type
+        result = {t: [] for t in PriceType}
+        for price in prices:
+            result[PriceType(price.type)].append(price)  # Fixed: Convert string to enum
+        return result
+
+    def _validate_price_data(self, prices: dict[str, Optional[Price]]) -> Optional[str]:
+        """Validate price data and return error message if invalid."""
+        # Filter out None values and get valid prices
+        valid_prices = {k: v for k, v in prices.items() if v is not None}
+        
+        if not valid_prices:
+            return "No valid price data available"
+            
+        for price in valid_prices.values():
+            if price.price < 0:
+                return f"Negative price found: {price.price}"
+            if price.price > self.config.MAX_PRICE:
+                return f"Price exceeds maximum allowed: {price.price}"
+                
+        return None
+
+    def _check_price_variance(self, prices: Dict[str, Decimal]) -> bool:
+        """Check if the variance between prices is within acceptable limits."""
+        if not prices:
+            return True
+            
+        min_price = min(prices.values())
+        max_price = max(prices.values())
+        
+        if min_price == 0:
+            return False
+            
+        variance_ratio = max_price / min_price
+        return variance_ratio <= (1 + self.config.MAX_PRICE_VARIANCE)
+
+    def _get_relevant_prices(self, product: Product) -> dict[str, Optional[Price]]:
+        """Get all relevant prices for a product."""
+        return {
+            PriceType.TCG_LOW.value: self.get_latest_price_for_product(product, PriceType.TCG_LOW),
+            PriceType.TCG_DIRECT_LOW.value: self.get_latest_price_for_product(product, PriceType.TCG_DIRECT_LOW),
+            PriceType.TCG_MARKET.value: self.get_latest_price_for_product(product, PriceType.TCG_MARKET),
+            PriceType.TCG_LOW_WITH_SHIPPING.value: self.get_latest_price_for_product(product, PriceType.TCG_LOW_WITH_SHIPPING)
+        }
+
+    def _get_base_prices(
+        self, prices: dict[str, Price], include_shipping: bool = False
+    ) -> Dict[str, Decimal]:
+        """Get base prices, excluding None values."""
+        base_prices = {}
+        
+        # Add core prices if they exist
+        if tcg_low := prices.get(PriceType.TCG_LOW.value):
+            base_prices[PriceType.TCG_LOW.value] = Decimal(str(tcg_low.price))
+        if tcg_direct := prices.get(PriceType.TCG_DIRECT_LOW.value):
+            base_prices[PriceType.TCG_DIRECT_LOW.value] = Decimal(str(tcg_direct.price))
+        if tcg_market := prices.get(PriceType.TCG_MARKET.value):
+            base_prices[PriceType.TCG_MARKET.value] = Decimal(str(tcg_market.price))
+        
+        # Add shipping price if requested and available
+        if include_shipping:
+            if tcg_shipping := prices.get(PriceType.TCG_LOW_WITH_SHIPPING.value):
+                base_prices[PriceType.TCG_LOW_WITH_SHIPPING.value] = Decimal(str(tcg_shipping.price))
+                
+        return base_prices
+
+    def _get_price_range(self, price: Decimal) -> Optional[PriceRange]:
+        """Get the appropriate price range for a given price."""
+        for price_range in self.price_ranges:
+            if price_range.contains_price(price):
+                return price_range
+        return None
+
+    def _handle_floor_price_cases(
+        self, base_prices: Dict[str, Decimal]
+    ) -> Optional[Decimal]:
+        """Handle special cases for prices near or below floor price."""
+        if all(price < self.config.FLOOR_PRICE for price in base_prices.values()):
+            return self.config.FLOOR_PRICE
+            
+        if any(price < self.config.FLOOR_PRICE for price in base_prices.values()):
+            max_price = max(base_prices.values())
+            return max_price * self.config.NEAR_FLOOR_MULT
+            
+        return None
+
+    def calculate_price(
+        self, product_id: str, strategy: PricingStrategy = PricingStrategy.DEFAULT
+    ) -> PriceCalculationResult:
+        """Calculate the final price for a product using the specified pricing strategy."""
+        # get product 
+        product = self.get_product_by_id(str(product_id))  # Fixed: Ensure string UUID
+        if not product:
+            logger.error(f"Product not found: {product_id}")
+            return PriceCalculationResult(product, None, {}, "Product not found")
+
+        # Get all relevant prices
+        prices = self._get_relevant_prices(product)
+        
+        # Validate price data
+        if error := self._validate_price_data(prices):
+            logger.error(f"Invalid price data: {error}")
+            logger.error(f"product: {product.id}")
+            return PriceCalculationResult(product, None, {}, error)
+        
+        # Get initial base prices without shipping
+        base_prices = self._get_base_prices(prices, include_shipping=False)
+        
+        # Check price variance
+        if not self._check_price_variance(base_prices):
+            logger.error(f"Price variance exceeds acceptable threshold")
+            logger.error(f"Base prices: {base_prices}")
+            logger.error(f"product: {product.id}")
+            return PriceCalculationResult(
+                product, None, base_prices,
+                "Price variance exceeds acceptable threshold"
+            )
+        
+        # Handle floor price cases
+        if floor_price := self._handle_floor_price_cases(base_prices):
+            return PriceCalculationResult(product, floor_price, base_prices)
+        
+        # Get max base price and its range
+        max_base_price = max(base_prices.values())
+        price_range = self._get_price_range(max_base_price)
+        
+        if not price_range:
+            logger.error(f"No valid price range found for price")
+            logger.error(f"Base prices: {base_prices}, max_base_price: {max_base_price}")
+            logger.error(f"product: {product.id}")
+            return PriceCalculationResult(
+                product, None, base_prices,
+                f"No valid price range found for price: {max_base_price}"
+            )
+        
+        # Include shipping prices if necessary
+        if price_range.include_shipping:
+            base_prices = self._get_base_prices(prices, include_shipping=True)
+            max_base_price = max(base_prices.values())
+            
+            # Recheck price range with shipping
+            price_range = self._get_price_range(max_base_price)
+            
+            if not price_range:
+                logger.error(f"No valid price range found for price with shipping")
+                logger.error(f"Base prices: {base_prices}, max_base_price: {max_base_price}")
+                logger.error(f"product: {product.id}")
+                return PriceCalculationResult(
+                    product, None, base_prices,
+                    f"No valid price range found for price with shipping: {max_base_price}"
+                )
+        
+        # Calculate final price using the price range
+        calculated_price = price_range.calculate_price(max_base_price)
+        
+        # Apply strategy-specific adjustments
+        if strategy == PricingStrategy.AGGRESSIVE:
+            calculated_price *= Decimal('0.95')
+        elif strategy == PricingStrategy.CONSERVATIVE:
+            calculated_price *= Decimal('1.05')
+
+        debug_base_prices_with_name_string = ", ".join([f"{k}: {v}" for k, v in base_prices.items()])
+        
+        logger.debug(f"Set price for to {calculated_price.quantize(Decimal('0.01'), rounding=ROUND_HALF_UP)} based on {debug_base_prices_with_name_string}")
+        
+        return PriceCalculationResult(
+            product,
+            calculated_price.quantize(Decimal('0.01'), rounding=ROUND_HALF_UP),
+            base_prices
+        )
+    
+    def _bulk_generate_uuids(self, size: int) -> List[str]:
+        """Generate UUIDs in bulk for better performance."""
+        return [str(uuid()) for _ in range(size)]
+
+    def _prepare_price_records(self, df: pd.DataFrame, price_type: str, uuids: List[str]) -> List[Dict]:
+        """Prepare price records in bulk using vectorized operations."""
+        records = []
+        df['price_id'] = uuids[:len(df)]
+        df['type'] = price_type  # price_type should already be a string value
+        df['date_created'] = datetime.utcnow()
+
+        return df[['price_id', 'product_id', 'type', 'price', 'date_created']].to_dict('records')
+
+    def _calculate_suggested_prices_batch(self, product_ids: List[str]) -> Dict[str, float]:
+        """Calculate suggested prices in parallel for a batch of products."""
+        with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            future_to_id = {
+                executor.submit(self.calculate_price, str(pid)): pid  # Fixed: Ensure string UUID
+                for pid in product_ids
+            }
+            
+            results = {}
+            for future in as_completed(future_to_id):
+                product_id = future_to_id[future]
+                try:
+                    result = future.result()
+                    if result.success:
+                        results[str(product_id)] = float(result.calculated_price)  # Fixed: Ensure string UUID
+                except Exception as e:
+                    logger.error(f"Failed to calculate price for product {product_id}: {e}")
+                    
+            return results
+
+    def _bulk_insert_prices(self, records: List[Dict]) -> None:
+        """Efficiently insert price records in bulk."""
+        if not records:
+            return
+
+        try:
+            df = pd.DataFrame(records)
+            df.to_sql('prices', self.db.bind, 
+                     if_exists='append', 
+                     index=False,
+                     method='multi',
+                     chunksize=self.CHUNK_SIZE)
+            
+        except Exception as e:
+            logger.error(f"Failed to bulk insert prices: {e}")
+            raise
+
+    def process_pricing_export(self, export_csv: bytes) -> None:
+        """Process pricing export with optimized bulk operations."""
+        try:
+            # Convert CSV to DataFrame
+            df = self.df_util.csv_bytes_to_df(export_csv)
+            df.columns = df.columns.str.lower().str.replace(' ', '_')
+
+            # Get product mappings efficiently - SQLite compatible with chunking
+            SQLITE_MAX_VARS = 999  # SQLite parameter limit
+            tcgplayer_ids = df['tcgplayer_id'].tolist()
+            all_product_dfs = []
+            
+            for i in range(0, len(tcgplayer_ids), SQLITE_MAX_VARS):
+                chunk_ids = tcgplayer_ids[i:i + SQLITE_MAX_VARS]
+                placeholders = ','.join([':id_' + str(j) for j in range(len(chunk_ids))])
+                product_query = f"""
+                    SELECT tcgplayer_id, product_id 
+                    FROM card_tcgplayer
+                    WHERE tcgplayer_id IN ({placeholders})
+                """
+                
+                # Create a dictionary of parameters
+                params = {f'id_{j}': id_val for j, id_val in enumerate(chunk_ids)}
+                
+                chunk_df = pd.read_sql(
+                    text(product_query),
+                    self.db.bind,
+                    params=params
+                )
+                all_product_dfs.append(chunk_df)
+            
+            # Combine all chunks
+            product_df = pd.concat(all_product_dfs) if all_product_dfs else pd.DataFrame()
+
+            # Merge dataframes efficiently
+            merged_df = pd.merge(
+                df,
+                product_df,
+                on='tcgplayer_id',
+                how='inner'
+            )
+
+            # Define price columns mapping - using enum values directly
+            price_columns = {
+                'tcg_market_price': PriceType.TCG_MARKET.value,
+                'tcg_direct_low': PriceType.TCG_DIRECT_LOW.value,
+                'tcg_low_price_with_shipping': PriceType.TCG_LOW_WITH_SHIPPING.value,
+                'tcg_low_price': PriceType.TCG_LOW.value,
+                'tcg_marketplace_price': PriceType.TCG_MARKETPLACE.value
+            }
+
+            # Process each price type in chunks
+            for price_col, price_type in price_columns.items():
+                valid_prices_df = merged_df[merged_df[price_col].notna()].copy()
+                
+                for chunk_start in range(0, len(valid_prices_df), self.CHUNK_SIZE):
+                    chunk_df = valid_prices_df.iloc[chunk_start:chunk_start + self.CHUNK_SIZE].copy()
+                    uuids = self._bulk_generate_uuids(len(chunk_df))
+                    
+                    chunk_df['price'] = chunk_df[price_col]
+                    chunk_df['product_id'] = chunk_df['product_id'].astype(str)  # Fixed: Ensure string UUIDs
+                    records = self._prepare_price_records(chunk_df, price_type, uuids)
+                    self._bulk_insert_prices(records)
+
+            # Handle suggested prices separately with parallel processing
+            product_ids = merged_df['product_id'].unique()
+            suggested_prices = {}
+            
+            for chunk_start in range(0, len(product_ids), self.CHUNK_SIZE):
+                chunk_ids = product_ids[chunk_start:chunk_start + self.CHUNK_SIZE]
+                chunk_prices = self._calculate_suggested_prices_batch(chunk_ids)
+                suggested_prices.update(chunk_prices)
+
+            # Create suggested price records
+            if suggested_prices:
+                suggested_df = pd.DataFrame([
+                    {'product_id': str(pid), 'price': price}  # Fixed: Ensure string UUIDs
+                    for pid, price in suggested_prices.items()
+                ])
+                
+                uuids = self._bulk_generate_uuids(len(suggested_df))
+                records = self._prepare_price_records(suggested_df, 'suggested_price', uuids)
+                self._bulk_insert_prices(records)
+
+        except Exception as e:
+            logger.error(f"Failed to process pricing export: {e}")
+            logger.error(f"Error occurred during price processing: {str(e)}")
+            raise

services/upload.py

@@ -1,100 +0,0 @@
-from db.models import ManaboxExportData, UploadHistory
-import pandas as pd
-from io import StringIO
-import uuid
-from sqlalchemy.orm import Session
-from db.utils import db_transaction
-from exceptions import FailedUploadException
-
-
-import logging
-logger = logging.getLogger(__name__)
-
-class UploadObject:
-    def __init__(self, 
-                 content: bytes = None, 
-                 upload_id: str = None, 
-                 filename: str = None, 
-                 df: pd.DataFrame = None):
-        self.content = content
-        self.upload_id = upload_id
-        self.filename = filename
-        self.df = df
-
-
-class UploadService:
-    def __init__(self, db: Session):
-        self.db = db
-    
-    def _content_to_df(self, content: bytes) -> pd.DataFrame:
-        df = pd.read_csv(StringIO(content.decode('utf-8')))
-        df.columns = df.columns.str.lower().str.replace(' ', '_')
-        return df
-    
-    def _create_upload_id(self) -> str:
-        return str(uuid.uuid4())
-    
-    def _prepare_manabox_df(self, content: bytes, upload_id: str) -> pd.DataFrame:
-        df = self._content_to_df(content)
-        df['upload_id'] = upload_id
-        df['box_id'] = None
-
-        return df
-    
-    def _create_file_upload_record(self, upload_id: str, filename: str, file_size_kb: float, num_rows: int) -> UploadHistory:
-        file_upload_record = UploadHistory(
-            id = str(uuid.uuid4()),
-            upload_id = upload_id,
-            filename = filename,
-            status = "pending",
-            file_size_kb = file_size_kb,
-            num_rows = num_rows
-        )
-        self.db.add(file_upload_record)
-        return file_upload_record
-    
-    def _update_manabox_data(self, df: pd.DataFrame) -> bool:
-        for index, row in df.iterrows():
-            try:
-                add_row = ManaboxExportData(
-                    id = str(uuid.uuid4()),
-                    upload_id = row['upload_id'],
-                    box_id = row['box_id'],
-                    name = row['name'],
-                    set_code = row['set_code'],
-                    set_name = row['set_name'],
-                    collector_number = row['collector_number'],
-                    foil = row['foil'],
-                    rarity = row['rarity'],
-                    quantity = row['quantity'],
-                    manabox_id = row['manabox_id'],
-                    scryfall_id = row['scryfall_id'],
-                    purchase_price = row['purchase_price'],
-                    misprint = row['misprint'],
-                    altered = row['altered'],
-                    condition = row['condition'],
-                    language = row['language'],
-                    purchase_price_currency = row['purchase_price_currency']
-                )
-                self.db.add(add_row)
-            except Exception as e:
-                logger.error(f"Error adding row to ManaboxExportData")
-                return False
-        return True
-
-    def process_manabox_upload(self, content: bytes, filename: str, file_size_kb: float) -> dict:
-        upload = UploadObject(content=content, filename=filename)
-        upload.upload_id = self._create_upload_id()
-        upload.df = self._prepare_manabox_df(upload.content, upload.upload_id)
-        num_rows = len(upload.df)
-
-        with db_transaction(self.db):
-            file_upload_record = self._create_file_upload_record(upload.upload_id, upload.filename, file_size_kb, num_rows)
-            if not self._update_manabox_data(upload.df):
-                # set upload to failed
-                file_upload_record.status = "failed"
-                raise FailedUploadException(file_upload_record)
-            else:
-                # set upload_history status to success
-                file_upload_record.status = "success"
-            return {"message": f"Manabox upload successful. Upload ID: {upload.upload_id}"}, upload.upload_id

services/util/_dataframe.py

@@ -0,0 +1,68 @@
+import pandas as pd
+from io import StringIO
+from db.models import File
+
+
+class ManaboxRow:
+    def __init__(self, row: pd.Series):
+        # Integer field
+        try:
+            self.manabox_id = int(row['manabox_id'])
+        except (ValueError, TypeError):
+            raise ValueError(f"manabox_id must be convertible to integer, got: {row['manabox_id']}")
+
+        # String fields with None/NaN handling
+        self.name = str(row['name']) if pd.notna(row['name']) else ''
+        self.set_code = str(row['set_code']) if pd.notna(row['set_code']) else ''
+        self.set_name = str(row['set_name']) if pd.notna(row['set_name']) else ''
+        self.collector_number = str(row['collector_number']) if pd.notna(row['collector_number']) else ''
+        self.foil = str(row['foil']) if pd.notna(row['foil']) else ''
+        self.rarity = str(row['rarity']) if pd.notna(row['rarity']) else ''
+        self.scryfall_id = str(row['scryfall_id']) if pd.notna(row['scryfall_id']) else ''
+        self.condition = str(row['condition']) if pd.notna(row['condition']) else ''
+        self.language = str(row['language']) if pd.notna(row['language']) else ''
+        self.quantity = str(row['quantity']) if pd.notna(row['quantity']) else ''
+
+
+class TCGPlayerPricingRow:
+    def __init__(self, row: pd.Series):
+        self.tcgplayer_id = row['tcgplayer_id']
+        self.product_line = row['product_line']
+        self.set_name = row['set_name']
+        self.product_name = row['product_name']
+        self.title = row['title']
+        self.number = row['number']
+        self.rarity = row['rarity']
+        self.condition = row['condition']
+        self.tcg_market_price = row['tcg_market_price']
+        self.tcg_direct_low = row['tcg_direct_low']
+        self.tcg_low_price_with_shipping = row['tcg_low_price_with_shipping']
+        self.tcg_low_price = row['tcg_low_price']
+        self.total_quantity = row['total_quantity']
+        self.add_to_quantity = row['add_to_quantity']
+        self.tcg_marketplace_price = row['tcg_marketplace_price']
+        self.photo_url = row['photo_url']
+
+
+class DataframeUtil:
+    def __init__(self):
+        pass
+
+    def format_df_columns(self, df: pd.DataFrame) -> pd.DataFrame:
+        df.columns = df.columns.str.lower()
+        df.columns = df.columns.str.replace(' ', '_')
+        return df
+    
+    def file_to_df(self, file: File) -> pd.DataFrame:
+        with open(file.filepath, 'rb') as f:
+            content = f.read()
+        content = content.decode('utf-8')
+        df = pd.read_csv(StringIO(content))
+        df = self.format_df_columns(df)
+        return df
+    
+    def csv_bytes_to_df(self, content: bytes) -> pd.DataFrame:
+        content = content.decode('utf-8')
+        df = pd.read_csv(StringIO(content))
+        df = self.format_df_columns(df)
+        return df