more stuff yay

db/models.py

@@ -113,7 +113,8 @@ class Box(Base):
 
     product_id = Column(String, ForeignKey("products.id"), primary_key=True)
     type = Column(String) # collector box, play box, etc.
-    sku = Column(String)
+    set_code = Column(String)
+    sku = Column(String, nullable=True)
     num_cards_expected = Column(Integer, nullable=True)
     date_created = Column(DateTime, default=datetime.now)
     date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
@@ -208,7 +209,7 @@ class Warehouse(Base):
     __tablename__ = "warehouse"
 
     id = Column(String, primary_key=True)
-    user_id = Column(String, ForeignKey("users.id"), default="admin")
+    user_id = Column(String, ForeignKey("users.id"))
     date_created = Column(DateTime, default=datetime.now)
     date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
 
@@ -246,6 +247,7 @@ class User(Base):
     __tablename__ = "users"
 
     id = Column(String, primary_key=True)
+    username = Column(String)
     date_created = Column(DateTime, default=datetime.now)
     date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
 
@@ -283,6 +285,7 @@ class File(Base):
     filepath = Column(String) # backup location
     filesize_kb = Column(Float)
     status = Column(String)
+    box_id = Column(String, ForeignKey("boxes.product_id"), nullable=True)
     date_created = Column(DateTime, default=datetime.now)
     date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
 
@@ -303,9 +306,16 @@ class StorageBlock(Base):
     """
     __tablename__ = "storage_blocks"
 
+    @validates("type")
+    def validate_type(self, key, type: str):
+        if type not in StorageBlockTypeEnum or type.lower() not in StorageBlockTypeEnum:
+            raise ValueError(f"Invalid storage block type: {type}")
+        return type
+
     id = Column(String, primary_key=True)
     warehouse_id = Column(String, ForeignKey("warehouse.id"))
     name = Column(String)
+    type = Column(String) # rare or common
     date_created = Column(DateTime, default=datetime.now)
     date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
 
@@ -530,6 +540,7 @@ class BoxTypeEnum(str, Enum):
     PLAY = "play"
     DRAFT = "draft"
     COMMANDER = "commander"
+    SET = "set"
 
 class ProductLineEnum(str, Enum):
     MTG = "mtg"
@@ -537,4 +548,8 @@ class ProductLineEnum(str, Enum):
 
 class ProductTypeEnum(str, Enum):
     BOX = "box"
-    CARD = "card"
+    CARD = "card"
+
+class StorageBlockTypeEnum(str, Enum):
+    RARE = "rare"
+    COMMON = "common"

dependencies.py

@@ -8,9 +8,11 @@ from services.file import FileService
 from services.product import ProductService
 from services.inventory import InventoryService
 from services.task import TaskService
+from services.storage import StorageService
 from fastapi import Depends, Form
 from db.database import get_db
 from schemas.file import CreateFileRequest
+from schemas.box import CreateBoxRequest, UpdateBoxRequest, CreateOpenBoxRequest
 
 
 ## file 
@@ -29,21 +31,54 @@ def get_create_file_metadata(
     """Dependency injection for FileMetadata"""
     return CreateFileRequest(type=type, source=source, service=service, filename=filename)
 
+def get_box_data(
+        type: str = Form(...),
+        sku: str = Form(None),
+        set_code: str = Form(...),
+        num_cards_expected: int = Form(None)
+    ) -> CreateBoxRequest:
+    """Dependency injection for BoxData"""
+    return CreateBoxRequest(type=type, sku=sku, set_code=set_code, num_cards_expected=num_cards_expected)
+
+def get_box_update_data(
+        type: str = Form(None),
+        sku: str = Form(None),
+        set_code: str = Form(None),
+        num_cards_expected: int = Form(None)
+    ) -> UpdateBoxRequest:
+    """Dependency injection for BoxUpdateData"""
+    return UpdateBoxRequest(type=type, sku=sku, set_code=set_code, num_cards_expected=num_cards_expected)
+
+def get_open_box_data(
+        product_id: str = Form(...),
+        file_ids: list[str] = Form(None),
+        num_cards_actual: int = Form(None),
+        date_opened: str = Form(None)
+    ) -> CreateOpenBoxRequest:
+    """Dependency injection for OpenBoxData"""
+    return CreateOpenBoxRequest(product_id=product_id, file_ids=file_ids, num_cards_actual=num_cards_actual, date_opened=date_opened)
+
 def get_tcgplayer_service(
         db: Session = Depends(get_db)
         ) -> TCGPlayerService:
     """Dependency injection for TCGPlayerService"""
     return TCGPlayerService(db)
 
+# storage
+
+def get_storage_service(db: Session = Depends(get_db)) -> StorageService:
+    """Dependency injection for StorageService"""
+    return StorageService(db)
+
 # product
-def get_product_service(db: Session = Depends(get_db), file_service: FileService = Depends(get_file_service), tcgplayer_service: TCGPlayerService = Depends(get_tcgplayer_service)) -> ProductService:
+def get_product_service(db: Session = Depends(get_db), file_service: FileService = Depends(get_file_service), tcgplayer_service: TCGPlayerService = Depends(get_tcgplayer_service), storage_service: StorageService = Depends(get_storage_service)) -> ProductService:
     """Dependency injection for ProductService"""
-    return ProductService(db, file_service, tcgplayer_service)
+    return ProductService(db, file_service, tcgplayer_service, storage_service)
 
 # task
-def get_task_service(db: Session = Depends(get_db), product_service: ProductService = Depends(get_product_service)) -> TaskService:
+def get_task_service(db: Session = Depends(get_db), product_service: ProductService = Depends(get_product_service), tcgplayer_service: TCGPlayerService = Depends(get_tcgplayer_service)) -> TaskService:
     """Dependency injection for TaskService"""
-    return TaskService(db, product_service)
+    return TaskService(db, product_service, tcgplayer_service)
 
 ## Inventory
 def get_inventory_service(db: Session = Depends(get_db)) -> InventoryService:

main.py

@@ -10,6 +10,7 @@ import sys
 from services.tcgplayer import TCGPlayerService, PricingService
 from services.product import ProductService
 from services.file import FileService
+from services.storage import StorageService
 from db.models import TCGPlayerGroups
 
 
@@ -62,7 +63,7 @@ async def startup_event():
             tcgplayer_service = TCGPlayerService(db)
             tcgplayer_service.populate_tcgplayer_groups()
     # Start task service
-    task_service = TaskService(db, ProductService(db, FileService(db), TCGPlayerService(db)))
+    task_service = TaskService(db, ProductService(db, FileService(db), TCGPlayerService(db), StorageService(db)), TCGPlayerService(db))
     await task_service.start()
 
 

routes/routes.py

@@ -11,8 +11,8 @@ from services.file import FileService
 from services.product import ProductService
 from services.task import TaskService
 from schemas.file import FileSchema, CreateFileRequest, CreateFileResponse, GetFileResponse, DeleteFileResponse, GetFileQueryParams
-from schemas.box import CreateBoxResponse, CreateBoxRequestData
+from schemas.box import CreateBoxResponse, CreateBoxRequest, BoxSchema, UpdateBoxRequest, CreateOpenBoxRequest, CreateOpenBoxResponse, OpenBoxSchema
-from dependencies import get_data_service, get_upload_service, get_tcgplayer_service, get_box_service, get_create_file_metadata, get_file_service, get_product_service, get_task_service
+from dependencies import get_data_service, get_upload_service, get_tcgplayer_service, get_box_service, get_create_file_metadata, get_file_service, get_product_service, get_task_service, get_box_data, get_box_update_data, get_open_box_data
 
 
 import logging
@@ -31,11 +31,6 @@ MAX_FILE_SIZE = 1024 * 1024 * 100 # 100 MB
     response_model=CreateFileResponse,
     status_code=201
 )
-@router.post(
-    "/files",
-    response_model=CreateFileResponse,
-    status_code=201
-)
 async def create_file(
     background_tasks: BackgroundTasks,
     file: UploadFile = File(...),
@@ -144,9 +139,76 @@ async def delete_file(
         raise HTTPException(status_code=400, detail=str(e))
 
 
+## BOX
+## CREATE
+@router.post("/boxes", response_model=CreateBoxResponse, status_code=201)
+async def create_box(
+    box_data: CreateBoxRequest = Depends(get_box_data),
+    box_service: BoxService = Depends(get_box_service)
+):
+    try:
+        result = box_service.create_box(box_data)
+        return CreateBoxResponse(
+            status_code=201,
+            success=True,
+            box=[BoxSchema.from_orm(result)]
+        )
+    except Exception as e:
+        logger.error(f"Create box failed: {str(e)}")
+        raise HTTPException(status_code=400, detail=str(e))
 
+## UPDATE
+@router.put("/boxes/{box_id}", response_model=CreateBoxResponse)
+async def update_box(
+    box_id: str,
+    box_data: UpdateBoxRequest = Depends(get_box_update_data),
+    box_service: BoxService = Depends(get_box_service)
+):
+    try:
+        result = box_service.update_box(box_id, box_data)
+        return CreateBoxResponse(
+            status_code=200,
+            success=True,
+            box=[BoxSchema.from_orm(result)]
+        )
+    except Exception as e:
+        logger.error(f"Update box failed: {str(e)}")
+        raise HTTPException(status_code=400, detail=str(e))
 
+## DELETE
+@router.delete("/boxes/{box_id}", response_model=CreateBoxResponse)
+async def delete_box(
+    box_id: str,
+    box_service: BoxService = Depends(get_box_service)
+):
+    try:
+        result = box_service.delete_box(box_id)
+        return CreateBoxResponse(
+            status_code=200,
+            success=True,
+            box=[BoxSchema.from_orm(result)]
+        )
+    except Exception as e:
+        logger.error(f"Delete box failed: {str(e)}")
+        raise HTTPException(status_code=400, detail=str(e))
 
+# BOX OPEN
+@router.post("/boxes/{box_id}/open", response_model=CreateOpenBoxResponse, status_code=201)
+async def open_box(
+    box_id: str,
+    box_data: CreateOpenBoxRequest = Depends(get_open_box_data),
+    box_service: BoxService = Depends(get_box_service)
+):
+    try:
+        result = box_service.open_box(box_id, box_data)
+        return CreateOpenBoxResponse(
+            status_code=201,
+            success=True,
+            open_box=[OpenBoxSchema.from_orm(result)]
+        )
+    except Exception as e:
+        logger.error(f"Open box failed: {str(e)}")
+        raise HTTPException(status_code=400, detail=str(e))
 
 
 

schemas/box.py

@@ -1,25 +1,66 @@
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, ConfigDict
 from schemas.base import BaseSchema
+from typing import Optional
+from datetime import datetime
 
-class CreateBoxResponse(BaseModel):
+#BOX
-    success: bool = Field(..., title="Success")
+class BoxSchema(BaseSchema):
-
+    product_id: str = Field(..., title="Product ID")
-class CreateOpenBoxResponse(CreateBoxResponse):
-    open_box_id: str
-
-class CreateSealedBoxResponse(CreateBoxResponse):
-    product_id: str
-    quantity: int
-
-class CreateBoxRequestData(BaseModel):
     type: str = Field(..., title="Box Type (collector, play, draft)")
-    name: str = Field(..., title="Name")
-    product_line: str = Field(..., title="Product Line (MTG, Pokemon, etc)")
-    set_name: str = Field(..., title="Set Name")
     set_code: str = Field(..., title="Set Code")
-    sealed: bool = Field(..., title="Sealed: Boolean")
+    sku: Optional[str] = Field(None, title="SKU")
-    sku: str = Field(..., title="SKU")
+    num_cards_expected: Optional[int] = Field(None, title="Number of cards expected")
-    num_cards_expected: int = Field(..., title="Number of cards expected")
+
-    num_cards_actual: int = Field(None, title="Number of cards actual")
+    model_config = ConfigDict(from_attributes=True)
-    date_purchased: str = Field(..., title="Date purchased")
+
-    date_opened: str = Field(None, title="Date opened")
+# CREATE
+# REQUEST
+class CreateBoxRequest(BaseModel):
+    type: str = Field(..., title="Box Type (collector, play, draft)")
+    set_code: str = Field(..., title="Set Code")
+    sku: Optional[str] = Field(None, title="SKU")
+    num_cards_expected: Optional[int] = Field(None, title="Number of cards expected")
+
+# RESPONSE
+class CreateBoxResponse(BaseModel):
+    status_code: int = Field(..., title="status_code")
+    success: bool = Field(..., title="success")
+    box: list[BoxSchema] = Field(..., title="box")
+
+# UPDATE
+# REQUEST
+class UpdateBoxRequest(BaseModel):
+    type: Optional[str] = Field(None, title="Box Type (collector, play, draft)")
+    set_code: Optional[str] = Field(None, title="Set Code")
+    sku: Optional[str] = Field(None, title="SKU")
+    num_cards_expected: Optional[int] = Field(None, title="Number of cards expected")
+
+# GET
+# RESPONSE
+class GetBoxResponse(BaseModel):
+    status_code: int = Field(..., title="status_code")
+    success: bool = Field(..., title="success")
+    boxes: list[BoxSchema] = Field(..., title="boxes")
+
+
+# OPEN BOX
+class OpenBoxSchema(BaseModel):
+    id: str = Field(..., title="id")
+    num_cards_actual: Optional[int] = Field(None, title="Number of cards actual")
+    date_opened: Optional[datetime] = Field(None, title="Date Opened")
+
+    model_config = ConfigDict(from_attributes=True)
+
+# CREATE
+# REQUEST
+class CreateOpenBoxRequest(BaseModel):
+    product_id: str = Field(..., title="Product ID")
+    file_ids: list[str] = Field(None, title="File IDs")
+    num_cards_actual: Optional[int] = Field(None, title="Number of cards actual")
+    date_opened: Optional [str] = Field(None, title="Date Opened")
+
+# RESPONSE
+class CreateOpenBoxResponse(BaseModel):
+    status_code: int = Field(..., title="status_code")
+    success: bool = Field(..., title="success")
+    open_box: list[OpenBoxSchema] = Field(..., title="open_box")

services/box.py

@@ -1,10 +1,11 @@
-from db.models import Box, File, StagedFileProduct, Product, OpenBoxCard, OpenBox, Inventory
+from db.models import Box, File, StagedFileProduct, Product, OpenBoxCard, OpenBox, Inventory, TCGPlayerGroups
 from db.utils import db_transaction
 from uuid import uuid4 as uuid
 from datetime import datetime
 from sqlalchemy.orm import Session
 from sqlalchemy.engine.result import Row
-from schemas.box import CreateOpenBoxResponse, CreateSealedBoxResponse, CreateBoxResponse
+from sqlalchemy import or_
+from schemas.box import CreateBoxRequest, CreateBoxResponse, UpdateBoxRequest, CreateOpenBoxRequest
 import logging
 from typing import Any
 from db.utils import db_transaction
@@ -92,7 +93,7 @@ class BoxService:
         response = CreateBoxResponse(success=True)
         return response
 
-    def create_box(self, create_box_data: dict[str, Any], file_ids: list[str] = None) -> CreateBoxResponse:
+    def _create_box(self, create_box_data: dict[str, Any], file_ids: list[str] = None) -> CreateBoxResponse:
         sealed = create_box_data["sealed"]
         assert isinstance(sealed, bool)
         if file_ids and not sealed:
@@ -132,4 +133,88 @@ class BoxService:
 
         except Exception as e:
             logger.error(f"Error creating box: {str(e)}")
-            raise e
+            raise e
+    
+    def validate_box_type(self, box_type: str) -> bool:
+        return box_type in ["collector", "play", "draft", "set", "commander"]
+    
+    def validate_set_code(self, set_code: str) -> bool:
+        exists = self.db.query(TCGPlayerGroups).filter(
+            TCGPlayerGroups.abbreviation == set_code
+        ).first() is not None
+        return exists
+        
+    def create_box(self, create_box_data: CreateBoxRequest) -> Box:
+        # validate box data
+        if not self.validate_box_type(create_box_data.type):
+            raise Exception("Invalid box type")
+        if not self.validate_set_code(create_box_data.set_code):
+            raise Exception("Invalid set code")
+        # check if box exists by type and set code or sku
+        existing_box = self.db.query(Box).filter(
+            or_(
+                Box.type == create_box_data.type,
+                Box.sku == create_box_data.sku
+            ),
+            Box.set_code == create_box_data.set_code
+        ).first()
+        if existing_box:
+            raise Exception("Box already exists")
+        # create box
+        with db_transaction(self.db):
+            box = Box(
+                product_id=str(uuid()),
+                type=create_box_data.type,
+                set_code=create_box_data.set_code,
+                sku=create_box_data.sku,
+                num_cards_expected=create_box_data.num_cards_expected
+            )
+            self.db.add(box)
+
+        return box
+    
+    def update_box(self, box_id: str, update_box_data: UpdateBoxRequest) -> Box:
+        box = self.db.query(Box).filter(Box.product_id == box_id).first()
+        if not box:
+            raise Exception("Box not found")
+        with db_transaction(self.db):
+            if update_box_data.type:
+                box.type = update_box_data.type
+            if update_box_data.set_code:
+                box.set_code = update_box_data.set_code
+            if update_box_data.sku:
+                box.sku = update_box_data.sku
+            if update_box_data.num_cards_expected:
+                box.num_cards_expected = update_box_data.num_cards_expected
+        return box
+    
+    def delete_box(self, box_id: str) -> Box:
+        box = self.db.query(Box).filter(Box.product_id == box_id).first()
+        if not box:
+            raise Exception("Box not found")
+        with db_transaction(self.db):
+            self.db.delete(box)
+        return box
+    
+    def open_box(self, box_id: str, box_data: CreateOpenBoxRequest):
+        box = self.db.query(Box).filter(Box.product_id == box_id).first()
+        if not box:
+            raise Exception("Box not found")
+        with db_transaction(self.db):
+            open_box = OpenBox(
+                id=str(uuid()),
+                product_id=box_id,
+                num_cards_actual=box_data.num_cards_actual,
+                date_opened=datetime.strptime(box_data.date_opened, "%Y-%m-%d") if box_data.date_opened else datetime.now()
+            )
+            self.db.add(open_box)
+            staged_product_data = self.get_staged_product_data(box_data.file_ids)
+            product_data = self.aggregate_staged_product_data(staged_product_data)
+            self.inventory_service.process_staged_products(product_data)
+            self.add_products_to_open_box(open_box, product_data)
+            # update box_id for files
+            for file_id in box_data.file_ids:
+                file = self.db.query(File).filter(File.id == file_id).first()
+                file.box_id = open_box.id
+                self.db.add(file)
+        return open_box

services/file.py

@@ -14,7 +14,7 @@ logger = logging.getLogger(__name__)
 # Name,Set code,Set name,Collector number,Foil,Rarity,Quantity,ManaBox ID,Scryfall ID,Purchase price,Misprint,Altered,Condition,Language,Purchase price currency
 MANABOX_REQUIRED_FILE_HEADERS = ['Name', 'Set code', 'Set name', 'Collector number', 'Foil', 'Rarity', 'Quantity', 'ManaBox ID', 'Scryfall ID', 'Purchase price', 'Misprint', 'Altered', 'Condition', 'Language', 'Purchase price currency']
 MANABOX_ALLOWED_FILE_EXTENSIONS = ['.csv']
-MANABOX_ALLOWED_FILE_TYPES = ['scan_export']
+MANABOX_ALLOWED_FILE_TYPES = ['scan_export_common', 'scan_export_rare']
 MANABOX_CONFIG = {
     "required_headers": MANABOX_REQUIRED_FILE_HEADERS,
     "allowed_extensions": MANABOX_ALLOWED_FILE_EXTENSIONS,

services/pricing.py

@@ -1,205 +1,8 @@
-import logging
-from typing import Callable
-from db.models import TCGPlayerInventory, TCGPlayerExportHistory, TCGPlayerPricingHistory, ManaboxExportData, ManaboxTCGPlayerMapping, TCGPlayerProduct
 from sqlalchemy.orm import Session
-import pandas as pd
-from db.utils import db_transaction
-from sqlalchemy import func, and_, exists
 
 
-logger = logging.getLogger(__name__)
-
 class PricingService:
     def __init__(self, db: Session):
         self.db = db
-
-    def get_box_with_most_recent_prices(self, box_id: str) -> pd.DataFrame:
-        latest_prices = (
-            self.db.query(
-                TCGPlayerPricingHistory.tcgplayer_product_id,
-                func.max(TCGPlayerPricingHistory.date_created).label('max_date')
-            )
-            .group_by(TCGPlayerPricingHistory.tcgplayer_product_id)
-            .subquery('latest')  # Added name to subquery
-        )
-
-        result = (
-            self.db.query(ManaboxExportData, TCGPlayerPricingHistory, TCGPlayerProduct)
-            .join(ManaboxTCGPlayerMapping, ManaboxExportData.id == ManaboxTCGPlayerMapping.manabox_id)
-            .join(TCGPlayerProduct, ManaboxTCGPlayerMapping.tcgplayer_id == TCGPlayerProduct.id)
-            .join(TCGPlayerPricingHistory, TCGPlayerProduct.id == TCGPlayerPricingHistory.tcgplayer_product_id)
-            .join(
-                latest_prices,
-                and_(
-                    TCGPlayerPricingHistory.tcgplayer_product_id == latest_prices.c.tcgplayer_product_id,
-                    TCGPlayerPricingHistory.date_created == latest_prices.c.max_date
-                )
-            )
-            .filter(ManaboxExportData.box_id == box_id)  # Removed str() conversion
-            .all()
-        )
-
-        logger.debug(f"Found {len(result)} rows")
-
-        df = pd.DataFrame([{
-            **{f"manabox_{k}": v for k, v in row[0].__dict__.items() if not k.startswith('_')},
-            **{f"pricing_{k}": v for k, v in row[1].__dict__.items() if not k.startswith('_')},
-            **{f"tcgproduct_{k}": v for k, v in row[2].__dict__.items() if not k.startswith('_')}
-        } for row in result])
-        
-        return df
     
-    def get_live_inventory_with_most_recent_prices(self) -> pd.DataFrame:
+    
-            # Get latest export IDs using subqueries
-            latest_inventory_export = (
-                self.db.query(TCGPlayerExportHistory.inventory_export_id)
-                .filter(TCGPlayerExportHistory.type == "live_inventory")
-                .order_by(TCGPlayerExportHistory.date_created.desc())
-                .limit(1)
-                .scalar_subquery()
-            )
-            # this is bad because latest pricing export is not guaranteed to be related to the latest inventory export
-            latest_pricing_export = (
-                self.db.query(TCGPlayerExportHistory.pricing_export_id)
-                .filter(TCGPlayerExportHistory.type == "pricing")
-                .order_by(TCGPlayerExportHistory.date_created.desc())
-                .limit(1)
-                .scalar_subquery()
-            )
-            
-            # Join inventory and pricing data in a single query
-            inventory_with_pricing = (
-                self.db.query(TCGPlayerInventory, TCGPlayerPricingHistory)
-                .join(
-                    TCGPlayerPricingHistory,
-                    TCGPlayerInventory.tcgplayer_product_id == TCGPlayerPricingHistory.tcgplayer_product_id
-                )
-                .filter(
-                    TCGPlayerInventory.export_id == latest_inventory_export,
-                    TCGPlayerPricingHistory.export_id == latest_pricing_export
-                )
-                .all()
-            )
-            
-            # Convert to pandas DataFrame
-            df = pd.DataFrame([{
-                # Inventory columns
-                **{f"inventory_{k}": v 
-                for k, v in row[0].__dict__.items() 
-                if not k.startswith('_')},
-                # Pricing columns
-                **{f"pricing_{k}": v 
-                for k, v in row[1].__dict__.items() 
-                if not k.startswith('_')}
-            } for row in inventory_with_pricing])
-            
-            return df
-
-    def default_pricing_algo(self, df: pd.DataFrame = None):
-        if df is None:
-            logger.debug("No DataFrame provided, fetching live inventory with most recent prices")
-            df = self.get_live_inventory_with_most_recent_prices()
-        # if tcg low price is < 0.35, set my_price to 0.35
-        # if either tcg low price or tcg low price with shipping is under 5, set my_price to tcg low price * 1.25
-        # if tcg low price with shipping is > 25 set price to tcg low price with shipping * 1.025
-        # otherwise, set price to tcg low price with shipping * 1.10
-        # also round to 2 decimal places
-        df['my_price'] = df.apply(lambda row: round(
-            0.35 if row['pricing_tcg_low_price'] < 0.35 else
-            row['pricing_tcg_low_price'] * 1.25 if row['pricing_tcg_low_price'] < 5 or row['pricing_tcg_low_price_with_shipping'] < 5 else
-            row['pricing_tcg_low_price_with_shipping'] * 1.025 if row['pricing_tcg_low_price_with_shipping'] > 25 else
-            row['pricing_tcg_low_price_with_shipping'] * 1.10, 2), axis=1)
-        # log rows with no price
-        no_price = df[df['my_price'].isnull()]
-        if len(no_price) > 0:
-            logger.warning(f"Found {len(no_price)} rows with no price")
-            logger.warning(no_price)
-            # remove rows with no price
-            df = df.dropna(subset=['my_price'])
-        return df
-    
-    def convert_df_to_csv(self, df: pd.DataFrame):
-        # Flip the mapping to be from current names TO desired names
-        column_mapping = {
-            'inventory_tcgplayer_id': 'TCGplayer Id',
-            'inventory_product_line': 'Product Line',
-            'inventory_set_name': 'Set Name',
-            'inventory_product_name': 'Product Name',
-            'inventory_title': 'Title',
-            'inventory_number': 'Number',
-            'inventory_rarity': 'Rarity',
-            'inventory_condition': 'Condition',
-            'pricing_tcg_market_price': 'TCG Market Price',
-            'pricing_tcg_direct_low': 'TCG Direct Low',
-            'pricing_tcg_low_price_with_shipping': 'TCG Low Price With Shipping',
-            'pricing_tcg_low_price': 'TCG Low Price',
-            'inventory_total_quantity': 'Total Quantity',
-            'inventory_add_to_quantity': 'Add to Quantity',
-            'my_price': 'TCG Marketplace Price',
-            'inventory_photo_url': 'Photo URL'
-        }
-
-        df['pricing_tcg_market_price'] = ""
-        df['pricing_tcg_direct_low'] = ""
-        df['pricing_tcg_low_price_with_shipping'] = ""
-        df['pricing_tcg_low_price'] = ""
-        df['inventory_total_quantity'] = ""
-        df['inventory_add_to_quantity'] = 0
-        df['inventory_photo_url'] = ""
-
-        # First select the columns we want (using the keys of our mapping)
-        # Then rename them to the desired names (the values in our mapping)
-        df = df[column_mapping.keys()].rename(columns=column_mapping)
-        
-        return df.to_csv(index=False, quoting=1, quotechar='"')
-    
-    def convert_add_df_to_csv(self, df: pd.DataFrame):
-        column_mapping = {
-            'tcgproduct_tcgplayer_id': 'TCGplayer Id',
-            'tcgproduct_product_line': 'Product Line',
-            'tcgproduct_set_name': 'Set Name',
-            'tcgproduct_product_name': 'Product Name',
-            'tcgproduct_title': 'Title',
-            'tcgproduct_number': 'Number',
-            'tcgproduct_rarity': 'Rarity',
-            'tcgproduct_condition': 'Condition',
-            'pricing_tcg_market_price': 'TCG Market Price',
-            'pricing_tcg_direct_low': 'TCG Direct Low',
-            'pricing_tcg_low_price_with_shipping': 'TCG Low Price With Shipping',
-            'pricing_tcg_low_price': 'TCG Low Price',
-            'tcgproduct_group_id': 'Total Quantity',
-            'manabox_quantity': 'Add to Quantity',
-            'my_price': 'TCG Marketplace Price',
-            'tcgproduct_photo_url': 'Photo URL'
-        }
-        df['tcgproduct_group_id'] = ""
-        df['pricing_tcg_market_price'] = ""
-        df['pricing_tcg_direct_low'] = ""
-        df['pricing_tcg_low_price_with_shipping'] = ""
-        df['pricing_tcg_low_price'] = ""
-        df['tcgproduct_photo_url'] = ""
-
-        df = df[column_mapping.keys()].rename(columns=column_mapping)
-
-        return df.to_csv(index=False, quoting=1, quotechar='"')
-
-    def create_live_inventory_pricing_update_csv(self, algo: Callable = None) -> str:
-        actual_algo = algo if algo is not None else self.default_pricing_algo
-        df = actual_algo()
-        csv = self.convert_df_to_csv(df)
-        return csv
-    
-    def create_add_to_tcgplayer_csv(self, box_id: str = None, upload_id: str = None, algo: Callable = None) -> str:
-        actual_algo = algo if algo is not None else self.default_pricing_algo
-        if box_id and upload_id:
-            raise ValueError("Cannot specify both box_id and upload_id")
-        elif not box_id and not upload_id:
-            raise ValueError("Must specify either box_id or upload_id")
-        elif box_id:
-            logger.debug("creating df")
-            df = self.get_box_with_most_recent_prices(box_id)
-        elif upload_id:
-            raise NotImplementedError("Not yet implemented")
-        df = actual_algo(df)
-        csv = self.convert_add_df_to_csv(df)
-        return csv

services/pricing_old.py

@@ -0,0 +1,205 @@
+import logging
+from typing import Callable
+from db.models import TCGPlayerInventory, TCGPlayerExportHistory, TCGPlayerPricingHistory, ManaboxExportData, ManaboxTCGPlayerMapping, TCGPlayerProduct
+from sqlalchemy.orm import Session
+import pandas as pd
+from db.utils import db_transaction
+from sqlalchemy import func, and_, exists
+
+
+logger = logging.getLogger(__name__)
+
+class PricingService:
+    def __init__(self, db: Session):
+        self.db = db
+
+    def get_box_with_most_recent_prices(self, box_id: str) -> pd.DataFrame:
+        latest_prices = (
+            self.db.query(
+                TCGPlayerPricingHistory.tcgplayer_product_id,
+                func.max(TCGPlayerPricingHistory.date_created).label('max_date')
+            )
+            .group_by(TCGPlayerPricingHistory.tcgplayer_product_id)
+            .subquery('latest')  # Added name to subquery
+        )
+
+        result = (
+            self.db.query(ManaboxExportData, TCGPlayerPricingHistory, TCGPlayerProduct)
+            .join(ManaboxTCGPlayerMapping, ManaboxExportData.id == ManaboxTCGPlayerMapping.manabox_id)
+            .join(TCGPlayerProduct, ManaboxTCGPlayerMapping.tcgplayer_id == TCGPlayerProduct.id)
+            .join(TCGPlayerPricingHistory, TCGPlayerProduct.id == TCGPlayerPricingHistory.tcgplayer_product_id)
+            .join(
+                latest_prices,
+                and_(
+                    TCGPlayerPricingHistory.tcgplayer_product_id == latest_prices.c.tcgplayer_product_id,
+                    TCGPlayerPricingHistory.date_created == latest_prices.c.max_date
+                )
+            )
+            .filter(ManaboxExportData.box_id == box_id)  # Removed str() conversion
+            .all()
+        )
+
+        logger.debug(f"Found {len(result)} rows")
+
+        df = pd.DataFrame([{
+            **{f"manabox_{k}": v for k, v in row[0].__dict__.items() if not k.startswith('_')},
+            **{f"pricing_{k}": v for k, v in row[1].__dict__.items() if not k.startswith('_')},
+            **{f"tcgproduct_{k}": v for k, v in row[2].__dict__.items() if not k.startswith('_')}
+        } for row in result])
+        
+        return df
+    
+    def get_live_inventory_with_most_recent_prices(self) -> pd.DataFrame:
+            # Get latest export IDs using subqueries
+            latest_inventory_export = (
+                self.db.query(TCGPlayerExportHistory.inventory_export_id)
+                .filter(TCGPlayerExportHistory.type == "live_inventory")
+                .order_by(TCGPlayerExportHistory.date_created.desc())
+                .limit(1)
+                .scalar_subquery()
+            )
+            # this is bad because latest pricing export is not guaranteed to be related to the latest inventory export
+            latest_pricing_export = (
+                self.db.query(TCGPlayerExportHistory.pricing_export_id)
+                .filter(TCGPlayerExportHistory.type == "pricing")
+                .order_by(TCGPlayerExportHistory.date_created.desc())
+                .limit(1)
+                .scalar_subquery()
+            )
+            
+            # Join inventory and pricing data in a single query
+            inventory_with_pricing = (
+                self.db.query(TCGPlayerInventory, TCGPlayerPricingHistory)
+                .join(
+                    TCGPlayerPricingHistory,
+                    TCGPlayerInventory.tcgplayer_product_id == TCGPlayerPricingHistory.tcgplayer_product_id
+                )
+                .filter(
+                    TCGPlayerInventory.export_id == latest_inventory_export,
+                    TCGPlayerPricingHistory.export_id == latest_pricing_export
+                )
+                .all()
+            )
+            
+            # Convert to pandas DataFrame
+            df = pd.DataFrame([{
+                # Inventory columns
+                **{f"inventory_{k}": v 
+                for k, v in row[0].__dict__.items() 
+                if not k.startswith('_')},
+                # Pricing columns
+                **{f"pricing_{k}": v 
+                for k, v in row[1].__dict__.items() 
+                if not k.startswith('_')}
+            } for row in inventory_with_pricing])
+            
+            return df
+
+    def default_pricing_algo(self, df: pd.DataFrame = None):
+        if df is None:
+            logger.debug("No DataFrame provided, fetching live inventory with most recent prices")
+            df = self.get_live_inventory_with_most_recent_prices()
+        # if tcg low price is < 0.35, set my_price to 0.35
+        # if either tcg low price or tcg low price with shipping is under 5, set my_price to tcg low price * 1.25
+        # if tcg low price with shipping is > 25 set price to tcg low price with shipping * 1.025
+        # otherwise, set price to tcg low price with shipping * 1.10
+        # also round to 2 decimal places
+        df['my_price'] = df.apply(lambda row: round(
+            0.35 if row['pricing_tcg_low_price'] < 0.35 else
+            row['pricing_tcg_low_price'] * 1.25 if row['pricing_tcg_low_price'] < 5 or row['pricing_tcg_low_price_with_shipping'] < 5 else
+            row['pricing_tcg_low_price_with_shipping'] * 1.025 if row['pricing_tcg_low_price_with_shipping'] > 25 else
+            row['pricing_tcg_low_price_with_shipping'] * 1.10, 2), axis=1)
+        # log rows with no price
+        no_price = df[df['my_price'].isnull()]
+        if len(no_price) > 0:
+            logger.warning(f"Found {len(no_price)} rows with no price")
+            logger.warning(no_price)
+            # remove rows with no price
+            df = df.dropna(subset=['my_price'])
+        return df
+    
+    def convert_df_to_csv(self, df: pd.DataFrame):
+        # Flip the mapping to be from current names TO desired names
+        column_mapping = {
+            'inventory_tcgplayer_id': 'TCGplayer Id',
+            'inventory_product_line': 'Product Line',
+            'inventory_set_name': 'Set Name',
+            'inventory_product_name': 'Product Name',
+            'inventory_title': 'Title',
+            'inventory_number': 'Number',
+            'inventory_rarity': 'Rarity',
+            'inventory_condition': 'Condition',
+            'pricing_tcg_market_price': 'TCG Market Price',
+            'pricing_tcg_direct_low': 'TCG Direct Low',
+            'pricing_tcg_low_price_with_shipping': 'TCG Low Price With Shipping',
+            'pricing_tcg_low_price': 'TCG Low Price',
+            'inventory_total_quantity': 'Total Quantity',
+            'inventory_add_to_quantity': 'Add to Quantity',
+            'my_price': 'TCG Marketplace Price',
+            'inventory_photo_url': 'Photo URL'
+        }
+
+        df['pricing_tcg_market_price'] = ""
+        df['pricing_tcg_direct_low'] = ""
+        df['pricing_tcg_low_price_with_shipping'] = ""
+        df['pricing_tcg_low_price'] = ""
+        df['inventory_total_quantity'] = ""
+        df['inventory_add_to_quantity'] = 0
+        df['inventory_photo_url'] = ""
+
+        # First select the columns we want (using the keys of our mapping)
+        # Then rename them to the desired names (the values in our mapping)
+        df = df[column_mapping.keys()].rename(columns=column_mapping)
+        
+        return df.to_csv(index=False, quoting=1, quotechar='"')
+    
+    def convert_add_df_to_csv(self, df: pd.DataFrame):
+        column_mapping = {
+            'tcgproduct_tcgplayer_id': 'TCGplayer Id',
+            'tcgproduct_product_line': 'Product Line',
+            'tcgproduct_set_name': 'Set Name',
+            'tcgproduct_product_name': 'Product Name',
+            'tcgproduct_title': 'Title',
+            'tcgproduct_number': 'Number',
+            'tcgproduct_rarity': 'Rarity',
+            'tcgproduct_condition': 'Condition',
+            'pricing_tcg_market_price': 'TCG Market Price',
+            'pricing_tcg_direct_low': 'TCG Direct Low',
+            'pricing_tcg_low_price_with_shipping': 'TCG Low Price With Shipping',
+            'pricing_tcg_low_price': 'TCG Low Price',
+            'tcgproduct_group_id': 'Total Quantity',
+            'manabox_quantity': 'Add to Quantity',
+            'my_price': 'TCG Marketplace Price',
+            'tcgproduct_photo_url': 'Photo URL'
+        }
+        df['tcgproduct_group_id'] = ""
+        df['pricing_tcg_market_price'] = ""
+        df['pricing_tcg_direct_low'] = ""
+        df['pricing_tcg_low_price_with_shipping'] = ""
+        df['pricing_tcg_low_price'] = ""
+        df['tcgproduct_photo_url'] = ""
+
+        df = df[column_mapping.keys()].rename(columns=column_mapping)
+
+        return df.to_csv(index=False, quoting=1, quotechar='"')
+
+    def create_live_inventory_pricing_update_csv(self, algo: Callable = None) -> str:
+        actual_algo = algo if algo is not None else self.default_pricing_algo
+        df = actual_algo()
+        csv = self.convert_df_to_csv(df)
+        return csv
+    
+    def create_add_to_tcgplayer_csv(self, box_id: str = None, upload_id: str = None, algo: Callable = None) -> str:
+        actual_algo = algo if algo is not None else self.default_pricing_algo
+        if box_id and upload_id:
+            raise ValueError("Cannot specify both box_id and upload_id")
+        elif not box_id and not upload_id:
+            raise ValueError("Must specify either box_id or upload_id")
+        elif box_id:
+            logger.debug("creating df")
+            df = self.get_box_with_most_recent_prices(box_id)
+        elif upload_id:
+            raise NotImplementedError("Not yet implemented")
+        df = actual_algo(df)
+        csv = self.convert_add_df_to_csv(df)
+        return csv

services/product.py

@@ -5,6 +5,7 @@ from io import StringIO
 import pandas as pd
 from services.file import FileService
 from services.tcgplayer import TCGPlayerService
+from services.storage import StorageService
 from uuid import uuid4 as uuid
 import logging
 
@@ -26,10 +27,11 @@ class ManaboxRow:
         self.quantity = row['quantity']
 
 class ProductService:
-    def __init__(self, db: Session, file_service: FileService, tcgplayer_service: TCGPlayerService):
+    def __init__(self, db: Session, file_service: FileService, tcgplayer_service: TCGPlayerService, storage_service: StorageService):
         self.db = db
         self.file_service = file_service
         self.tcgplayer_service = tcgplayer_service
+        self.storage_service = storage_service
 
     def _format_manabox_df(self, df: pd.DataFrame) -> pd.DataFrame:
         # format columns
@@ -140,7 +142,9 @@ class ProductService:
             df = self._manabox_file_to_df(file)
             for index, row in df.iterrows():
                 manabox_row = ManaboxRow(row)
+                # create card concepts - manabox, tcgplayer, card, product
                 card_manabox = self.card_manabox_lookup_create_if_not_exist(manabox_row)
+                # create staged inventory with quantity for processing down the marketplace pipeline
                 staged_product = self.create_staged_product(file, card_manabox, row)
             # update file status
             with db_transaction(self.db):
@@ -148,4 +152,10 @@ class ProductService:
         except Exception as e:
             with db_transaction(self.db):
                 file.status = 'error'
-            raise e
+            raise e
+        try:
+            # create storage records for physically storing individual cards
+            self.storage_service.store_staged_products_for_file(file.id)
+        except Exception as e:
+            logger.error(f"Error creating storage records: {str(e)}")
+            raise e

services/storage.py

@@ -0,0 +1,176 @@
+from sqlalchemy.orm import Session
+from db.utils import db_transaction
+from db.models import Warehouse, User, StagedFileProduct, StorageBlock, ProductBlock, File, Card
+from uuid import uuid4 as uuid
+from typing import List, TypedDict
+
+class ProductAttributes(TypedDict):
+    product_id: str
+    card_number: str
+
+class StorageService:
+    def __init__(self, db: Session):
+        self.db = db
+    
+    def get_or_create_user(self, username: str) -> User:
+        user = self.db.query(User).filter(User.username == username).first()
+        if user is None:
+            user = User(
+                id = str(uuid()),
+                username = username
+            )
+            with db_transaction(self.db):
+                self.db.add(user)
+        return user
+    
+    def get_or_create_warehouse(self) -> Warehouse:
+        warehouse = self.db.query(Warehouse).first()
+        user = self.get_or_create_user('admin')
+        if warehouse is None:
+            warehouse = Warehouse(
+                id = str(uuid()),
+                user_id = user.id
+            )
+            with db_transaction(self.db):
+                self.db.add(warehouse)
+        return warehouse
+    
+    def get_staged_product(self, file_id: str) -> List[StagedFileProduct]:
+        staged_product = self.db.query(StagedFileProduct).filter(StagedFileProduct.file_id == file_id).all()
+        return staged_product
+    
+    def get_storage_block_name(self, warehouse: Warehouse, file_id: str) -> str:
+        # Get file type from id
+        current_file = self.db.query(File).filter(File.id == file_id).first()
+        if not current_file:
+            raise ValueError(f"No file found with id {file_id}")
+
+        # Determine storage block type
+        storage_block_type = 'rare' if 'rare' in current_file.type else 'common'
+        prefix = storage_block_type[0]
+
+        # Get most recent storage block with same type and warehouse id
+        latest_block = (
+            self.db.query(StorageBlock)
+            .filter(
+                StorageBlock.warehouse_id == warehouse.id,
+                StorageBlock.type == storage_block_type
+            )
+            .order_by(StorageBlock.date_created.desc())
+            .first()
+        )
+
+        # If no existing block, start with number 1
+        if not latest_block:
+            return f"{prefix}1"
+
+        # Start with the next number after the latest block
+        number = int(latest_block.name[1:])
+        
+        while True:
+            number += 1
+            new_name = f"{prefix}{number}"
+            
+            # Check if the new name already exists
+            exists = (
+                self.db.query(StorageBlock)
+                .filter(
+                    StorageBlock.warehouse_id == warehouse.id,
+                    StorageBlock.name == new_name
+                )
+                .first()
+            )
+            
+            if not exists:
+                return new_name
+    
+    def create_storage_block(self, warehouse: Warehouse, file_id: str) -> StorageBlock:
+        current_file = self.db.query(File).filter(File.id == file_id).first()
+        if not current_file:
+            raise ValueError(f"No file found with id {file_id}")
+            
+        storage_block_type = 'rare' if 'rare' in current_file.type else 'common'
+        
+        storage_block = StorageBlock(
+            id = str(uuid()),
+            warehouse_id = warehouse.id,
+            name = self.get_storage_block_name(warehouse, file_id),
+            type = storage_block_type
+        )
+        with db_transaction(self.db):
+            self.db.add(storage_block)
+        return storage_block
+
+    def add_staged_product_to_product_block(
+        self,
+        staged_product: StagedFileProduct,
+        storage_block: StorageBlock,
+        product_attributes: ProductAttributes,
+        block_index: int
+    ) -> ProductBlock:
+        """Create a new ProductBlock for a single unit of a staged product."""
+        product_block = ProductBlock(
+            id=str(uuid()),
+            product_id=staged_product.product_id,
+            block_id=storage_block.id,
+            block_index=block_index
+        )
+        
+        with db_transaction(self.db):
+            self.db.add(product_block)
+        
+        return product_block
+
+    def get_staged_product_attributes_for_storage(
+        self,
+        staged_product: StagedFileProduct
+    ) -> List[ProductAttributes]:
+        """Get attributes for each unit of a staged product."""
+        result = (
+            self.db.query(
+                StagedFileProduct.product_id,
+                StagedFileProduct.quantity,
+                Card.number
+            )
+            .join(Card, Card.product_id == StagedFileProduct.product_id)
+            .filter(StagedFileProduct.id == staged_product.id)
+            .first()
+        )
+        
+        if not result:
+            return []
+        
+        return [
+            ProductAttributes(
+                product_id=result.product_id,
+                card_number=result.number
+            )
+            for i in range(result.quantity)
+        ]
+
+    def store_staged_products_for_file(self, file_id: str) -> StorageBlock:
+        """Store all staged products for a file in a new storage block."""
+        warehouse = self.get_or_create_warehouse()
+        storage_block = self.create_storage_block(warehouse, file_id)
+        staged_products = self.get_staged_product(file_id)
+        
+        # Collect all product attributes first
+        all_product_attributes = []
+        for staged_product in staged_products:
+            product_attributes_list = self.get_staged_product_attributes_for_storage(staged_product)
+            for attrs in product_attributes_list:
+                all_product_attributes.append((staged_product, attrs))
+        
+        # Sort by card number as integer to determine block indices
+        sorted_attributes = sorted(all_product_attributes, key=lambda x: int(x[1]['card_number']))
+        
+        # Add products with correct block indices
+        for block_index, (staged_product, product_attributes) in enumerate(sorted_attributes, 1):
+            self.add_staged_product_to_product_block(
+                staged_product=staged_product,
+                storage_block=storage_block,
+                product_attributes=product_attributes,
+                block_index=block_index
+            )
+        
+        return storage_block

services/task.py

@@ -3,16 +3,18 @@ import logging
 from typing import Dict, Callable
 from sqlalchemy.orm import Session
 from services.product import ProductService
+from services.tcgplayer import TCGPlayerService
 from db.models import File
 
 
 class TaskService:
-    def __init__(self, db: Session, product_service: ProductService):
+    def __init__(self, db: Session, product_service: ProductService, tcgplayer_service: TCGPlayerService):
         self.scheduler = BackgroundScheduler()
         self.logger = logging.getLogger(__name__)
         self.tasks: Dict[str, Callable] = {}
         self.db = db
         self.product_service = product_service
+        self.tcgplayer_service = tcgplayer_service
     
     async def start(self):
         self.scheduler.start()
@@ -27,12 +29,21 @@ class TaskService:
             minute=0,
             id='daily_report'
         )
+            
+        self.scheduler.add_job(
+            self.pricing_update,
+            'cron',
+            minute=28,
+            id='pricing_update'
+        )
     
-    # Tasks that should be scheduled
+    def daily_report(self):  # Removed async
-    async def daily_report(self):
         self.logger.info("Generating daily report")
         # Daily report logic
-    
+
+    def pricing_update(self):  # Removed async
+        self.logger.info("Hourly pricing update")
+        self.tcgplayer_service.cron_load_prices()
 
     async def process_manabox_file(self, file: File):
         self.logger.info("Processing ManaBox file")

services/tcgplayer.py

@@ -1,4 +1,4 @@
-from db.models import ManaboxExportData, Box, TCGPlayerGroups, TCGPlayerInventory, TCGPlayerExportHistory, TCGPlayerPricingHistory, TCGPlayerProduct, ManaboxTCGPlayerMapping, CardManabox, CardTCGPlayer
+from db.models import ManaboxExportData, Box, TCGPlayerGroups, TCGPlayerInventory, TCGPlayerExportHistory, TCGPlayerPricingHistory, TCGPlayerProduct, ManaboxTCGPlayerMapping, CardManabox, CardTCGPlayer, Price
 import requests
 from sqlalchemy.orm import Session
 from db.utils import db_transaction
@@ -18,6 +18,8 @@ from typing import List, Dict, Optional
 from io import StringIO, BytesIO
 from services.pricing import PricingService
 from sqlalchemy.sql import exists
+import pandas as pd
+from sqlalchemy.exc import SQLAlchemyError
 
 
 logger = logging.getLogger(__name__)
@@ -559,6 +561,115 @@ class TCGPlayerService:
             return None
         return matching_product
 
-
+    def get_pricing_export_for_all_products(self) -> bytes:
-
+        """
+        Retrieves pricing export data for all products in TCGPlayer format.
         
+        Returns:
+            bytes: Raw CSV data containing pricing information
+        """
+        try:
+            all_group_ids = self.db.query(TCGPlayerGroups.group_id).all()
+            all_group_ids = [str(group_id) for group_id, in all_group_ids]
+            export_csv = self._get_export_csv(all_group_ids, convert=False)
+            return export_csv
+        except SQLAlchemyError as e:
+            raise RuntimeError(f"Failed to retrieve group IDs: {str(e)}")
+
+    def pricing_export_to_df(self, export_csv: bytes) -> pd.DataFrame:
+        """
+        Converts raw CSV pricing data to a pandas DataFrame.
+        
+        Args:
+            export_csv (bytes): Raw CSV data in bytes format
+        
+        Returns:
+            pd.DataFrame: Processed pricing data
+            
+        Raises:
+            ValueError: If no CSV data is provided or if CSV parsing fails
+        """
+        if not export_csv:
+            raise ValueError("No export CSV provided")
+        
+        csv_file = None
+        try:
+            text_content = export_csv.decode('utf-8')
+            csv_file = StringIO(text_content)
+            df = pd.read_csv(csv_file)
+            
+            if df.empty:
+                raise ValueError("CSV data is empty")
+                
+            return df
+        except UnicodeDecodeError as e:
+            raise ValueError(f"Failed to decode CSV data: {str(e)}")
+        except pd.errors.EmptyDataError:
+            raise ValueError("CSV file is empty or malformed")
+        finally:
+            if csv_file:
+                csv_file.close()
+
+    def cron_load_prices(self) -> None:
+        """
+        Scheduled task to load and update product prices.
+        Fetches current prices, processes them, and saves new price records to the database.
+        """
+        try:
+            # Get and process price data
+            price_csv = self.get_pricing_export_for_all_products()
+            price_df = self.pricing_export_to_df(price_csv)
+            
+            # Clean column names
+            price_df.columns = price_df.columns.str.lower().str.replace(' ', '_')
+            
+            # Get all products efficiently
+            products_query = self.db.query(
+                CardTCGPlayer.tcgplayer_id,
+                CardTCGPlayer.product_id
+            )
+            product_df = pd.read_sql(products_query.statement, self.db.bind)
+            
+            # Merge dataframes
+            merged_df = pd.merge(
+                price_df,
+                product_df,
+                on='tcgplayer_id',
+                how='inner'
+            )
+            
+            # Define price columns to process
+            price_columns = [
+                'tcg_market_price',
+                'tcg_direct_low',
+                'tcg_low_price_with_shipping',
+                'tcg_low_price',
+                'tcg_marketplace_price'
+            ]
+            
+            # Process in batches to avoid memory issues
+            BATCH_SIZE = 1000
+            for price_column in price_columns:
+                records = []
+                
+                for start_idx in range(0, len(merged_df), BATCH_SIZE):
+                    batch_df = merged_df.iloc[start_idx:start_idx + BATCH_SIZE]
+                    
+                    batch_records = [
+                        Price(
+                            id=str(uuid.uuid4()),
+                            product_id=row['product_id'],
+                            type=price_column,
+                            price=row[price_column]
+                        )
+                        for _, row in batch_df.iterrows()
+                        if pd.notna(row[price_column])  # Skip rows with NaN prices
+                    ]
+                    
+                    with db_transaction(self.db):
+                        self.db.bulk_save_objects(batch_records)
+                        self.db.flush()
+                    
+        except Exception as e:
+            logger.error(f"Failed to load prices: {str(e)}")
+            raise

tests/box_test.py

@@ -0,0 +1,154 @@
+from fastapi.testclient import TestClient
+from fastapi import BackgroundTasks
+import pytest
+from unittest.mock import Mock, patch
+import asyncio
+import os
+from main import app
+from services.file import FileService
+from services.task import TaskService
+
+
+
+client = TestClient(app)
+
+test_boxes = []
+
+def test_create_box():
+    # Send as form data, not JSON
+    response = client.post("/api/boxes", 
+        data={
+            "type": "play",
+            "set_code": "BLB",
+            "sku": "1234",
+            "num_cards_expected": 504
+        }
+    )
+    test_boxes.append(response.json()["box"][0]["product_id"])
+    
+    assert response.status_code == 201
+    assert response.json()["success"] == True
+    assert response.json()["box"][0]["type"] == "play"
+    assert response.json()["box"][0]["set_code"] == "BLB"
+    assert response.json()["box"][0]["sku"] == "1234"
+    assert response.json()["box"][0]["num_cards_expected"] == 504
+
+def test_update_box():
+    # Create a box first
+    create_response = client.post("/api/boxes", 
+        data={
+            "type": "collector",
+            "set_code": "MKM",
+            "sku": "3456",
+            "num_cards_expected": 504
+        }
+    )
+    box_id = create_response.json()["box"][0]["product_id"]
+    test_boxes.append(box_id)
+    
+    # Update the box
+    response = client.put(f"/api/boxes/{box_id}", 
+        data={
+            "num_cards_expected": 500
+        }
+    )
+    
+    assert response.status_code == 200
+    assert response.json()["success"] == True
+    assert response.json()["box"][0]["type"] == "collector"
+    assert response.json()["box"][0]["set_code"] == "MKM"
+    assert response.json()["box"][0]["sku"] == "3456"
+    assert response.json()["box"][0]["num_cards_expected"] == 500
+
+def test_delete_box():
+    # Create a box first
+    create_response = client.post("/api/boxes", 
+        data={
+            "type": "set",
+            "set_code": "LCI",
+            "sku": "7890",
+            "num_cards_expected": 504
+        }
+    )
+    box_id = create_response.json()["box"][0]["product_id"]
+    
+    # Delete the box
+    response = client.delete(f"/api/boxes/{box_id}")
+    
+    assert response.status_code == 200
+    assert response.json()["success"] == True
+    assert response.json()["box"][0]["type"] == "set"
+    assert response.json()["box"][0]["set_code"] == "LCI"
+    assert response.json()["box"][0]["sku"] == "7890"
+    assert response.json()["box"][0]["num_cards_expected"] == 504
+
+# Constants for reused values
+TEST_FILE_PATH = os.path.join(os.getcwd(), "tests/test_files", "manabox_test_file.csv")
+DEFAULT_METADATA = {
+    "source": "manabox",
+    "type": "scan_export_common"
+}
+
+def get_file_size_kb(file_path):
+    """Helper to consistently calculate file size in KB"""
+    with open(file_path, "rb") as f:
+        return round(len(f.read()) / 1024, 2)
+
+@pytest.mark.asyncio
+async def test_open_box():
+    """Test creating a new manabox file"""
+    # Open file within the test scope
+    with open(TEST_FILE_PATH, "rb") as test_file:
+        files = {"file": test_file}
+        
+        # Make request
+        response = client.post("/api/files", data=DEFAULT_METADATA, files=files)
+        
+        # Check response
+        assert response.status_code == 201
+        assert response.json()["success"] == True
+        
+        file_data = response.json()["files"][0]
+        assert file_data["source"] == DEFAULT_METADATA["source"]
+        assert file_data["type"] == DEFAULT_METADATA["type"]
+        assert file_data["status"] == "pending"
+        assert file_data["service"] == None
+        assert file_data["filename"] == "manabox_test_file.csv"
+        assert file_data["filesize_kb"] == get_file_size_kb(TEST_FILE_PATH)
+        assert file_data["id"] is not None
+
+        # Execute background tasks if they were added
+        background_tasks = BackgroundTasks()
+        for task in background_tasks.tasks:
+            await task()
+        
+        # Create a box first
+        create_response = client.post("/api/boxes", 
+            data={
+                "type": "play",
+                "set_code": "OTJ",
+                "sku": "2314",
+                "num_cards_expected": 504
+            }
+        )
+        box_id = create_response.json()["box"][0]["product_id"]
+        test_boxes.append(box_id)
+
+        # Open the box
+        response = client.post(f"/api/boxes/{box_id}/open", 
+            data={
+                "product_id": box_id,
+                "file_ids": [file_data["id"]],
+                "num_cards_actual": 500
+            }
+        )
+
+        assert response.status_code == 201
+        assert response.json()["success"] == True
+
+
+def test_cleanup():
+    # Delete all boxes created during testing
+    for box_id in test_boxes:
+        client.delete(f"/api/boxes/{box_id}")
+    

tests/file_test.py

@@ -14,7 +14,7 @@ client = TestClient(app)
 TEST_FILE_PATH = os.path.join(os.getcwd(), "tests/test_files", "manabox_test_file.csv")
 DEFAULT_METADATA = {
     "source": "manabox",
-    "type": "scan_export"
+    "type": "scan_export_rare"
 }
 
 def get_file_size_kb(file_path):