more stuff yay
This commit is contained in:
parent
bd9cfca7a9
commit
a78c3bcba3
19
db/models.py
19
db/models.py
@ -113,7 +113,8 @@ class Box(Base):
|
||||
|
||||
product_id = Column(String, ForeignKey("products.id"), primary_key=True)
|
||||
type = Column(String) # collector box, play box, etc.
|
||||
sku = Column(String)
|
||||
set_code = Column(String)
|
||||
sku = Column(String, nullable=True)
|
||||
num_cards_expected = Column(Integer, nullable=True)
|
||||
date_created = Column(DateTime, default=datetime.now)
|
||||
date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
|
||||
@ -208,7 +209,7 @@ class Warehouse(Base):
|
||||
__tablename__ = "warehouse"
|
||||
|
||||
id = Column(String, primary_key=True)
|
||||
user_id = Column(String, ForeignKey("users.id"), default="admin")
|
||||
user_id = Column(String, ForeignKey("users.id"))
|
||||
date_created = Column(DateTime, default=datetime.now)
|
||||
date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
|
||||
|
||||
@ -246,6 +247,7 @@ class User(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
id = Column(String, primary_key=True)
|
||||
username = Column(String)
|
||||
date_created = Column(DateTime, default=datetime.now)
|
||||
date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
|
||||
|
||||
@ -283,6 +285,7 @@ class File(Base):
|
||||
filepath = Column(String) # backup location
|
||||
filesize_kb = Column(Float)
|
||||
status = Column(String)
|
||||
box_id = Column(String, ForeignKey("boxes.product_id"), nullable=True)
|
||||
date_created = Column(DateTime, default=datetime.now)
|
||||
date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
|
||||
|
||||
@ -303,9 +306,16 @@ class StorageBlock(Base):
|
||||
"""
|
||||
__tablename__ = "storage_blocks"
|
||||
|
||||
@validates("type")
|
||||
def validate_type(self, key, type: str):
|
||||
if type not in StorageBlockTypeEnum or type.lower() not in StorageBlockTypeEnum:
|
||||
raise ValueError(f"Invalid storage block type: {type}")
|
||||
return type
|
||||
|
||||
id = Column(String, primary_key=True)
|
||||
warehouse_id = Column(String, ForeignKey("warehouse.id"))
|
||||
name = Column(String)
|
||||
type = Column(String) # rare or common
|
||||
date_created = Column(DateTime, default=datetime.now)
|
||||
date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
|
||||
|
||||
@ -530,6 +540,7 @@ class BoxTypeEnum(str, Enum):
|
||||
PLAY = "play"
|
||||
DRAFT = "draft"
|
||||
COMMANDER = "commander"
|
||||
SET = "set"
|
||||
|
||||
class ProductLineEnum(str, Enum):
|
||||
MTG = "mtg"
|
||||
@ -538,3 +549,7 @@ class ProductLineEnum(str, Enum):
|
||||
class ProductTypeEnum(str, Enum):
|
||||
BOX = "box"
|
||||
CARD = "card"
|
||||
|
||||
class StorageBlockTypeEnum(str, Enum):
|
||||
RARE = "rare"
|
||||
COMMON = "common"
|
||||
@ -8,9 +8,11 @@ from services.file import FileService
|
||||
from services.product import ProductService
|
||||
from services.inventory import InventoryService
|
||||
from services.task import TaskService
|
||||
from services.storage import StorageService
|
||||
from fastapi import Depends, Form
|
||||
from db.database import get_db
|
||||
from schemas.file import CreateFileRequest
|
||||
from schemas.box import CreateBoxRequest, UpdateBoxRequest, CreateOpenBoxRequest
|
||||
|
||||
|
||||
## file
|
||||
@ -29,21 +31,54 @@ def get_create_file_metadata(
|
||||
"""Dependency injection for FileMetadata"""
|
||||
return CreateFileRequest(type=type, source=source, service=service, filename=filename)
|
||||
|
||||
def get_box_data(
|
||||
type: str = Form(...),
|
||||
sku: str = Form(None),
|
||||
set_code: str = Form(...),
|
||||
num_cards_expected: int = Form(None)
|
||||
) -> CreateBoxRequest:
|
||||
"""Dependency injection for BoxData"""
|
||||
return CreateBoxRequest(type=type, sku=sku, set_code=set_code, num_cards_expected=num_cards_expected)
|
||||
|
||||
def get_box_update_data(
|
||||
type: str = Form(None),
|
||||
sku: str = Form(None),
|
||||
set_code: str = Form(None),
|
||||
num_cards_expected: int = Form(None)
|
||||
) -> UpdateBoxRequest:
|
||||
"""Dependency injection for BoxUpdateData"""
|
||||
return UpdateBoxRequest(type=type, sku=sku, set_code=set_code, num_cards_expected=num_cards_expected)
|
||||
|
||||
def get_open_box_data(
|
||||
product_id: str = Form(...),
|
||||
file_ids: list[str] = Form(None),
|
||||
num_cards_actual: int = Form(None),
|
||||
date_opened: str = Form(None)
|
||||
) -> CreateOpenBoxRequest:
|
||||
"""Dependency injection for OpenBoxData"""
|
||||
return CreateOpenBoxRequest(product_id=product_id, file_ids=file_ids, num_cards_actual=num_cards_actual, date_opened=date_opened)
|
||||
|
||||
def get_tcgplayer_service(
|
||||
db: Session = Depends(get_db)
|
||||
) -> TCGPlayerService:
|
||||
"""Dependency injection for TCGPlayerService"""
|
||||
return TCGPlayerService(db)
|
||||
|
||||
# storage
|
||||
|
||||
def get_storage_service(db: Session = Depends(get_db)) -> StorageService:
|
||||
"""Dependency injection for StorageService"""
|
||||
return StorageService(db)
|
||||
|
||||
# product
|
||||
def get_product_service(db: Session = Depends(get_db), file_service: FileService = Depends(get_file_service), tcgplayer_service: TCGPlayerService = Depends(get_tcgplayer_service)) -> ProductService:
|
||||
def get_product_service(db: Session = Depends(get_db), file_service: FileService = Depends(get_file_service), tcgplayer_service: TCGPlayerService = Depends(get_tcgplayer_service), storage_service: StorageService = Depends(get_storage_service)) -> ProductService:
|
||||
"""Dependency injection for ProductService"""
|
||||
return ProductService(db, file_service, tcgplayer_service)
|
||||
return ProductService(db, file_service, tcgplayer_service, storage_service)
|
||||
|
||||
# task
|
||||
def get_task_service(db: Session = Depends(get_db), product_service: ProductService = Depends(get_product_service)) -> TaskService:
|
||||
def get_task_service(db: Session = Depends(get_db), product_service: ProductService = Depends(get_product_service), tcgplayer_service: TCGPlayerService = Depends(get_tcgplayer_service)) -> TaskService:
|
||||
"""Dependency injection for TaskService"""
|
||||
return TaskService(db, product_service)
|
||||
return TaskService(db, product_service, tcgplayer_service)
|
||||
|
||||
## Inventory
|
||||
def get_inventory_service(db: Session = Depends(get_db)) -> InventoryService:
|
||||
|
||||
3
main.py
3
main.py
@ -10,6 +10,7 @@ import sys
|
||||
from services.tcgplayer import TCGPlayerService, PricingService
|
||||
from services.product import ProductService
|
||||
from services.file import FileService
|
||||
from services.storage import StorageService
|
||||
from db.models import TCGPlayerGroups
|
||||
|
||||
|
||||
@ -62,7 +63,7 @@ async def startup_event():
|
||||
tcgplayer_service = TCGPlayerService(db)
|
||||
tcgplayer_service.populate_tcgplayer_groups()
|
||||
# Start task service
|
||||
task_service = TaskService(db, ProductService(db, FileService(db), TCGPlayerService(db)))
|
||||
task_service = TaskService(db, ProductService(db, FileService(db), TCGPlayerService(db), StorageService(db)), TCGPlayerService(db))
|
||||
await task_service.start()
|
||||
|
||||
|
||||
|
||||
@ -11,8 +11,8 @@ from services.file import FileService
|
||||
from services.product import ProductService
|
||||
from services.task import TaskService
|
||||
from schemas.file import FileSchema, CreateFileRequest, CreateFileResponse, GetFileResponse, DeleteFileResponse, GetFileQueryParams
|
||||
from schemas.box import CreateBoxResponse, CreateBoxRequestData
|
||||
from dependencies import get_data_service, get_upload_service, get_tcgplayer_service, get_box_service, get_create_file_metadata, get_file_service, get_product_service, get_task_service
|
||||
from schemas.box import CreateBoxResponse, CreateBoxRequest, BoxSchema, UpdateBoxRequest, CreateOpenBoxRequest, CreateOpenBoxResponse, OpenBoxSchema
|
||||
from dependencies import get_data_service, get_upload_service, get_tcgplayer_service, get_box_service, get_create_file_metadata, get_file_service, get_product_service, get_task_service, get_box_data, get_box_update_data, get_open_box_data
|
||||
|
||||
|
||||
import logging
|
||||
@ -31,11 +31,6 @@ MAX_FILE_SIZE = 1024 * 1024 * 100 # 100 MB
|
||||
response_model=CreateFileResponse,
|
||||
status_code=201
|
||||
)
|
||||
@router.post(
|
||||
"/files",
|
||||
response_model=CreateFileResponse,
|
||||
status_code=201
|
||||
)
|
||||
async def create_file(
|
||||
background_tasks: BackgroundTasks,
|
||||
file: UploadFile = File(...),
|
||||
@ -144,9 +139,76 @@ async def delete_file(
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
## BOX
|
||||
## CREATE
|
||||
@router.post("/boxes", response_model=CreateBoxResponse, status_code=201)
|
||||
async def create_box(
|
||||
box_data: CreateBoxRequest = Depends(get_box_data),
|
||||
box_service: BoxService = Depends(get_box_service)
|
||||
):
|
||||
try:
|
||||
result = box_service.create_box(box_data)
|
||||
return CreateBoxResponse(
|
||||
status_code=201,
|
||||
success=True,
|
||||
box=[BoxSchema.from_orm(result)]
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Create box failed: {str(e)}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
## UPDATE
|
||||
@router.put("/boxes/{box_id}", response_model=CreateBoxResponse)
|
||||
async def update_box(
|
||||
box_id: str,
|
||||
box_data: UpdateBoxRequest = Depends(get_box_update_data),
|
||||
box_service: BoxService = Depends(get_box_service)
|
||||
):
|
||||
try:
|
||||
result = box_service.update_box(box_id, box_data)
|
||||
return CreateBoxResponse(
|
||||
status_code=200,
|
||||
success=True,
|
||||
box=[BoxSchema.from_orm(result)]
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Update box failed: {str(e)}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
## DELETE
|
||||
@router.delete("/boxes/{box_id}", response_model=CreateBoxResponse)
|
||||
async def delete_box(
|
||||
box_id: str,
|
||||
box_service: BoxService = Depends(get_box_service)
|
||||
):
|
||||
try:
|
||||
result = box_service.delete_box(box_id)
|
||||
return CreateBoxResponse(
|
||||
status_code=200,
|
||||
success=True,
|
||||
box=[BoxSchema.from_orm(result)]
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Delete box failed: {str(e)}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
# BOX OPEN
|
||||
@router.post("/boxes/{box_id}/open", response_model=CreateOpenBoxResponse, status_code=201)
|
||||
async def open_box(
|
||||
box_id: str,
|
||||
box_data: CreateOpenBoxRequest = Depends(get_open_box_data),
|
||||
box_service: BoxService = Depends(get_box_service)
|
||||
):
|
||||
try:
|
||||
result = box_service.open_box(box_id, box_data)
|
||||
return CreateOpenBoxResponse(
|
||||
status_code=201,
|
||||
success=True,
|
||||
open_box=[OpenBoxSchema.from_orm(result)]
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Open box failed: {str(e)}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
|
||||
|
||||
@ -1,25 +1,66 @@
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, Field, ConfigDict
|
||||
from schemas.base import BaseSchema
|
||||
from typing import Optional
|
||||
from datetime import datetime
|
||||
|
||||
class CreateBoxResponse(BaseModel):
|
||||
success: bool = Field(..., title="Success")
|
||||
|
||||
class CreateOpenBoxResponse(CreateBoxResponse):
|
||||
open_box_id: str
|
||||
|
||||
class CreateSealedBoxResponse(CreateBoxResponse):
|
||||
product_id: str
|
||||
quantity: int
|
||||
|
||||
class CreateBoxRequestData(BaseModel):
|
||||
#BOX
|
||||
class BoxSchema(BaseSchema):
|
||||
product_id: str = Field(..., title="Product ID")
|
||||
type: str = Field(..., title="Box Type (collector, play, draft)")
|
||||
name: str = Field(..., title="Name")
|
||||
product_line: str = Field(..., title="Product Line (MTG, Pokemon, etc)")
|
||||
set_name: str = Field(..., title="Set Name")
|
||||
set_code: str = Field(..., title="Set Code")
|
||||
sealed: bool = Field(..., title="Sealed: Boolean")
|
||||
sku: str = Field(..., title="SKU")
|
||||
num_cards_expected: int = Field(..., title="Number of cards expected")
|
||||
num_cards_actual: int = Field(None, title="Number of cards actual")
|
||||
date_purchased: str = Field(..., title="Date purchased")
|
||||
date_opened: str = Field(None, title="Date opened")
|
||||
sku: Optional[str] = Field(None, title="SKU")
|
||||
num_cards_expected: Optional[int] = Field(None, title="Number of cards expected")
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
# CREATE
|
||||
# REQUEST
|
||||
class CreateBoxRequest(BaseModel):
|
||||
type: str = Field(..., title="Box Type (collector, play, draft)")
|
||||
set_code: str = Field(..., title="Set Code")
|
||||
sku: Optional[str] = Field(None, title="SKU")
|
||||
num_cards_expected: Optional[int] = Field(None, title="Number of cards expected")
|
||||
|
||||
# RESPONSE
|
||||
class CreateBoxResponse(BaseModel):
|
||||
status_code: int = Field(..., title="status_code")
|
||||
success: bool = Field(..., title="success")
|
||||
box: list[BoxSchema] = Field(..., title="box")
|
||||
|
||||
# UPDATE
|
||||
# REQUEST
|
||||
class UpdateBoxRequest(BaseModel):
|
||||
type: Optional[str] = Field(None, title="Box Type (collector, play, draft)")
|
||||
set_code: Optional[str] = Field(None, title="Set Code")
|
||||
sku: Optional[str] = Field(None, title="SKU")
|
||||
num_cards_expected: Optional[int] = Field(None, title="Number of cards expected")
|
||||
|
||||
# GET
|
||||
# RESPONSE
|
||||
class GetBoxResponse(BaseModel):
|
||||
status_code: int = Field(..., title="status_code")
|
||||
success: bool = Field(..., title="success")
|
||||
boxes: list[BoxSchema] = Field(..., title="boxes")
|
||||
|
||||
|
||||
# OPEN BOX
|
||||
class OpenBoxSchema(BaseModel):
|
||||
id: str = Field(..., title="id")
|
||||
num_cards_actual: Optional[int] = Field(None, title="Number of cards actual")
|
||||
date_opened: Optional[datetime] = Field(None, title="Date Opened")
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
# CREATE
|
||||
# REQUEST
|
||||
class CreateOpenBoxRequest(BaseModel):
|
||||
product_id: str = Field(..., title="Product ID")
|
||||
file_ids: list[str] = Field(None, title="File IDs")
|
||||
num_cards_actual: Optional[int] = Field(None, title="Number of cards actual")
|
||||
date_opened: Optional [str] = Field(None, title="Date Opened")
|
||||
|
||||
# RESPONSE
|
||||
class CreateOpenBoxResponse(BaseModel):
|
||||
status_code: int = Field(..., title="status_code")
|
||||
success: bool = Field(..., title="success")
|
||||
open_box: list[OpenBoxSchema] = Field(..., title="open_box")
|
||||
|
||||
@ -1,10 +1,11 @@
|
||||
from db.models import Box, File, StagedFileProduct, Product, OpenBoxCard, OpenBox, Inventory
|
||||
from db.models import Box, File, StagedFileProduct, Product, OpenBoxCard, OpenBox, Inventory, TCGPlayerGroups
|
||||
from db.utils import db_transaction
|
||||
from uuid import uuid4 as uuid
|
||||
from datetime import datetime
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.engine.result import Row
|
||||
from schemas.box import CreateOpenBoxResponse, CreateSealedBoxResponse, CreateBoxResponse
|
||||
from sqlalchemy import or_
|
||||
from schemas.box import CreateBoxRequest, CreateBoxResponse, UpdateBoxRequest, CreateOpenBoxRequest
|
||||
import logging
|
||||
from typing import Any
|
||||
from db.utils import db_transaction
|
||||
@ -92,7 +93,7 @@ class BoxService:
|
||||
response = CreateBoxResponse(success=True)
|
||||
return response
|
||||
|
||||
def create_box(self, create_box_data: dict[str, Any], file_ids: list[str] = None) -> CreateBoxResponse:
|
||||
def _create_box(self, create_box_data: dict[str, Any], file_ids: list[str] = None) -> CreateBoxResponse:
|
||||
sealed = create_box_data["sealed"]
|
||||
assert isinstance(sealed, bool)
|
||||
if file_ids and not sealed:
|
||||
@ -133,3 +134,87 @@ class BoxService:
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating box: {str(e)}")
|
||||
raise e
|
||||
|
||||
def validate_box_type(self, box_type: str) -> bool:
|
||||
return box_type in ["collector", "play", "draft", "set", "commander"]
|
||||
|
||||
def validate_set_code(self, set_code: str) -> bool:
|
||||
exists = self.db.query(TCGPlayerGroups).filter(
|
||||
TCGPlayerGroups.abbreviation == set_code
|
||||
).first() is not None
|
||||
return exists
|
||||
|
||||
def create_box(self, create_box_data: CreateBoxRequest) -> Box:
|
||||
# validate box data
|
||||
if not self.validate_box_type(create_box_data.type):
|
||||
raise Exception("Invalid box type")
|
||||
if not self.validate_set_code(create_box_data.set_code):
|
||||
raise Exception("Invalid set code")
|
||||
# check if box exists by type and set code or sku
|
||||
existing_box = self.db.query(Box).filter(
|
||||
or_(
|
||||
Box.type == create_box_data.type,
|
||||
Box.sku == create_box_data.sku
|
||||
),
|
||||
Box.set_code == create_box_data.set_code
|
||||
).first()
|
||||
if existing_box:
|
||||
raise Exception("Box already exists")
|
||||
# create box
|
||||
with db_transaction(self.db):
|
||||
box = Box(
|
||||
product_id=str(uuid()),
|
||||
type=create_box_data.type,
|
||||
set_code=create_box_data.set_code,
|
||||
sku=create_box_data.sku,
|
||||
num_cards_expected=create_box_data.num_cards_expected
|
||||
)
|
||||
self.db.add(box)
|
||||
|
||||
return box
|
||||
|
||||
def update_box(self, box_id: str, update_box_data: UpdateBoxRequest) -> Box:
|
||||
box = self.db.query(Box).filter(Box.product_id == box_id).first()
|
||||
if not box:
|
||||
raise Exception("Box not found")
|
||||
with db_transaction(self.db):
|
||||
if update_box_data.type:
|
||||
box.type = update_box_data.type
|
||||
if update_box_data.set_code:
|
||||
box.set_code = update_box_data.set_code
|
||||
if update_box_data.sku:
|
||||
box.sku = update_box_data.sku
|
||||
if update_box_data.num_cards_expected:
|
||||
box.num_cards_expected = update_box_data.num_cards_expected
|
||||
return box
|
||||
|
||||
def delete_box(self, box_id: str) -> Box:
|
||||
box = self.db.query(Box).filter(Box.product_id == box_id).first()
|
||||
if not box:
|
||||
raise Exception("Box not found")
|
||||
with db_transaction(self.db):
|
||||
self.db.delete(box)
|
||||
return box
|
||||
|
||||
def open_box(self, box_id: str, box_data: CreateOpenBoxRequest):
|
||||
box = self.db.query(Box).filter(Box.product_id == box_id).first()
|
||||
if not box:
|
||||
raise Exception("Box not found")
|
||||
with db_transaction(self.db):
|
||||
open_box = OpenBox(
|
||||
id=str(uuid()),
|
||||
product_id=box_id,
|
||||
num_cards_actual=box_data.num_cards_actual,
|
||||
date_opened=datetime.strptime(box_data.date_opened, "%Y-%m-%d") if box_data.date_opened else datetime.now()
|
||||
)
|
||||
self.db.add(open_box)
|
||||
staged_product_data = self.get_staged_product_data(box_data.file_ids)
|
||||
product_data = self.aggregate_staged_product_data(staged_product_data)
|
||||
self.inventory_service.process_staged_products(product_data)
|
||||
self.add_products_to_open_box(open_box, product_data)
|
||||
# update box_id for files
|
||||
for file_id in box_data.file_ids:
|
||||
file = self.db.query(File).filter(File.id == file_id).first()
|
||||
file.box_id = open_box.id
|
||||
self.db.add(file)
|
||||
return open_box
|
||||
@ -14,7 +14,7 @@ logger = logging.getLogger(__name__)
|
||||
# Name,Set code,Set name,Collector number,Foil,Rarity,Quantity,ManaBox ID,Scryfall ID,Purchase price,Misprint,Altered,Condition,Language,Purchase price currency
|
||||
MANABOX_REQUIRED_FILE_HEADERS = ['Name', 'Set code', 'Set name', 'Collector number', 'Foil', 'Rarity', 'Quantity', 'ManaBox ID', 'Scryfall ID', 'Purchase price', 'Misprint', 'Altered', 'Condition', 'Language', 'Purchase price currency']
|
||||
MANABOX_ALLOWED_FILE_EXTENSIONS = ['.csv']
|
||||
MANABOX_ALLOWED_FILE_TYPES = ['scan_export']
|
||||
MANABOX_ALLOWED_FILE_TYPES = ['scan_export_common', 'scan_export_rare']
|
||||
MANABOX_CONFIG = {
|
||||
"required_headers": MANABOX_REQUIRED_FILE_HEADERS,
|
||||
"allowed_extensions": MANABOX_ALLOWED_FILE_EXTENSIONS,
|
||||
|
||||
@ -1,205 +1,8 @@
|
||||
import logging
|
||||
from typing import Callable
|
||||
from db.models import TCGPlayerInventory, TCGPlayerExportHistory, TCGPlayerPricingHistory, ManaboxExportData, ManaboxTCGPlayerMapping, TCGPlayerProduct
|
||||
from sqlalchemy.orm import Session
|
||||
import pandas as pd
|
||||
from db.utils import db_transaction
|
||||
from sqlalchemy import func, and_, exists
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class PricingService:
|
||||
def __init__(self, db: Session):
|
||||
self.db = db
|
||||
|
||||
def get_box_with_most_recent_prices(self, box_id: str) -> pd.DataFrame:
|
||||
latest_prices = (
|
||||
self.db.query(
|
||||
TCGPlayerPricingHistory.tcgplayer_product_id,
|
||||
func.max(TCGPlayerPricingHistory.date_created).label('max_date')
|
||||
)
|
||||
.group_by(TCGPlayerPricingHistory.tcgplayer_product_id)
|
||||
.subquery('latest') # Added name to subquery
|
||||
)
|
||||
|
||||
result = (
|
||||
self.db.query(ManaboxExportData, TCGPlayerPricingHistory, TCGPlayerProduct)
|
||||
.join(ManaboxTCGPlayerMapping, ManaboxExportData.id == ManaboxTCGPlayerMapping.manabox_id)
|
||||
.join(TCGPlayerProduct, ManaboxTCGPlayerMapping.tcgplayer_id == TCGPlayerProduct.id)
|
||||
.join(TCGPlayerPricingHistory, TCGPlayerProduct.id == TCGPlayerPricingHistory.tcgplayer_product_id)
|
||||
.join(
|
||||
latest_prices,
|
||||
and_(
|
||||
TCGPlayerPricingHistory.tcgplayer_product_id == latest_prices.c.tcgplayer_product_id,
|
||||
TCGPlayerPricingHistory.date_created == latest_prices.c.max_date
|
||||
)
|
||||
)
|
||||
.filter(ManaboxExportData.box_id == box_id) # Removed str() conversion
|
||||
.all()
|
||||
)
|
||||
|
||||
logger.debug(f"Found {len(result)} rows")
|
||||
|
||||
df = pd.DataFrame([{
|
||||
**{f"manabox_{k}": v for k, v in row[0].__dict__.items() if not k.startswith('_')},
|
||||
**{f"pricing_{k}": v for k, v in row[1].__dict__.items() if not k.startswith('_')},
|
||||
**{f"tcgproduct_{k}": v for k, v in row[2].__dict__.items() if not k.startswith('_')}
|
||||
} for row in result])
|
||||
|
||||
return df
|
||||
|
||||
def get_live_inventory_with_most_recent_prices(self) -> pd.DataFrame:
|
||||
# Get latest export IDs using subqueries
|
||||
latest_inventory_export = (
|
||||
self.db.query(TCGPlayerExportHistory.inventory_export_id)
|
||||
.filter(TCGPlayerExportHistory.type == "live_inventory")
|
||||
.order_by(TCGPlayerExportHistory.date_created.desc())
|
||||
.limit(1)
|
||||
.scalar_subquery()
|
||||
)
|
||||
# this is bad because latest pricing export is not guaranteed to be related to the latest inventory export
|
||||
latest_pricing_export = (
|
||||
self.db.query(TCGPlayerExportHistory.pricing_export_id)
|
||||
.filter(TCGPlayerExportHistory.type == "pricing")
|
||||
.order_by(TCGPlayerExportHistory.date_created.desc())
|
||||
.limit(1)
|
||||
.scalar_subquery()
|
||||
)
|
||||
|
||||
# Join inventory and pricing data in a single query
|
||||
inventory_with_pricing = (
|
||||
self.db.query(TCGPlayerInventory, TCGPlayerPricingHistory)
|
||||
.join(
|
||||
TCGPlayerPricingHistory,
|
||||
TCGPlayerInventory.tcgplayer_product_id == TCGPlayerPricingHistory.tcgplayer_product_id
|
||||
)
|
||||
.filter(
|
||||
TCGPlayerInventory.export_id == latest_inventory_export,
|
||||
TCGPlayerPricingHistory.export_id == latest_pricing_export
|
||||
)
|
||||
.all()
|
||||
)
|
||||
|
||||
# Convert to pandas DataFrame
|
||||
df = pd.DataFrame([{
|
||||
# Inventory columns
|
||||
**{f"inventory_{k}": v
|
||||
for k, v in row[0].__dict__.items()
|
||||
if not k.startswith('_')},
|
||||
# Pricing columns
|
||||
**{f"pricing_{k}": v
|
||||
for k, v in row[1].__dict__.items()
|
||||
if not k.startswith('_')}
|
||||
} for row in inventory_with_pricing])
|
||||
|
||||
return df
|
||||
|
||||
def default_pricing_algo(self, df: pd.DataFrame = None):
|
||||
if df is None:
|
||||
logger.debug("No DataFrame provided, fetching live inventory with most recent prices")
|
||||
df = self.get_live_inventory_with_most_recent_prices()
|
||||
# if tcg low price is < 0.35, set my_price to 0.35
|
||||
# if either tcg low price or tcg low price with shipping is under 5, set my_price to tcg low price * 1.25
|
||||
# if tcg low price with shipping is > 25 set price to tcg low price with shipping * 1.025
|
||||
# otherwise, set price to tcg low price with shipping * 1.10
|
||||
# also round to 2 decimal places
|
||||
df['my_price'] = df.apply(lambda row: round(
|
||||
0.35 if row['pricing_tcg_low_price'] < 0.35 else
|
||||
row['pricing_tcg_low_price'] * 1.25 if row['pricing_tcg_low_price'] < 5 or row['pricing_tcg_low_price_with_shipping'] < 5 else
|
||||
row['pricing_tcg_low_price_with_shipping'] * 1.025 if row['pricing_tcg_low_price_with_shipping'] > 25 else
|
||||
row['pricing_tcg_low_price_with_shipping'] * 1.10, 2), axis=1)
|
||||
# log rows with no price
|
||||
no_price = df[df['my_price'].isnull()]
|
||||
if len(no_price) > 0:
|
||||
logger.warning(f"Found {len(no_price)} rows with no price")
|
||||
logger.warning(no_price)
|
||||
# remove rows with no price
|
||||
df = df.dropna(subset=['my_price'])
|
||||
return df
|
||||
|
||||
def convert_df_to_csv(self, df: pd.DataFrame):
|
||||
# Flip the mapping to be from current names TO desired names
|
||||
column_mapping = {
|
||||
'inventory_tcgplayer_id': 'TCGplayer Id',
|
||||
'inventory_product_line': 'Product Line',
|
||||
'inventory_set_name': 'Set Name',
|
||||
'inventory_product_name': 'Product Name',
|
||||
'inventory_title': 'Title',
|
||||
'inventory_number': 'Number',
|
||||
'inventory_rarity': 'Rarity',
|
||||
'inventory_condition': 'Condition',
|
||||
'pricing_tcg_market_price': 'TCG Market Price',
|
||||
'pricing_tcg_direct_low': 'TCG Direct Low',
|
||||
'pricing_tcg_low_price_with_shipping': 'TCG Low Price With Shipping',
|
||||
'pricing_tcg_low_price': 'TCG Low Price',
|
||||
'inventory_total_quantity': 'Total Quantity',
|
||||
'inventory_add_to_quantity': 'Add to Quantity',
|
||||
'my_price': 'TCG Marketplace Price',
|
||||
'inventory_photo_url': 'Photo URL'
|
||||
}
|
||||
|
||||
df['pricing_tcg_market_price'] = ""
|
||||
df['pricing_tcg_direct_low'] = ""
|
||||
df['pricing_tcg_low_price_with_shipping'] = ""
|
||||
df['pricing_tcg_low_price'] = ""
|
||||
df['inventory_total_quantity'] = ""
|
||||
df['inventory_add_to_quantity'] = 0
|
||||
df['inventory_photo_url'] = ""
|
||||
|
||||
# First select the columns we want (using the keys of our mapping)
|
||||
# Then rename them to the desired names (the values in our mapping)
|
||||
df = df[column_mapping.keys()].rename(columns=column_mapping)
|
||||
|
||||
return df.to_csv(index=False, quoting=1, quotechar='"')
|
||||
|
||||
def convert_add_df_to_csv(self, df: pd.DataFrame):
|
||||
column_mapping = {
|
||||
'tcgproduct_tcgplayer_id': 'TCGplayer Id',
|
||||
'tcgproduct_product_line': 'Product Line',
|
||||
'tcgproduct_set_name': 'Set Name',
|
||||
'tcgproduct_product_name': 'Product Name',
|
||||
'tcgproduct_title': 'Title',
|
||||
'tcgproduct_number': 'Number',
|
||||
'tcgproduct_rarity': 'Rarity',
|
||||
'tcgproduct_condition': 'Condition',
|
||||
'pricing_tcg_market_price': 'TCG Market Price',
|
||||
'pricing_tcg_direct_low': 'TCG Direct Low',
|
||||
'pricing_tcg_low_price_with_shipping': 'TCG Low Price With Shipping',
|
||||
'pricing_tcg_low_price': 'TCG Low Price',
|
||||
'tcgproduct_group_id': 'Total Quantity',
|
||||
'manabox_quantity': 'Add to Quantity',
|
||||
'my_price': 'TCG Marketplace Price',
|
||||
'tcgproduct_photo_url': 'Photo URL'
|
||||
}
|
||||
df['tcgproduct_group_id'] = ""
|
||||
df['pricing_tcg_market_price'] = ""
|
||||
df['pricing_tcg_direct_low'] = ""
|
||||
df['pricing_tcg_low_price_with_shipping'] = ""
|
||||
df['pricing_tcg_low_price'] = ""
|
||||
df['tcgproduct_photo_url'] = ""
|
||||
|
||||
df = df[column_mapping.keys()].rename(columns=column_mapping)
|
||||
|
||||
return df.to_csv(index=False, quoting=1, quotechar='"')
|
||||
|
||||
def create_live_inventory_pricing_update_csv(self, algo: Callable = None) -> str:
|
||||
actual_algo = algo if algo is not None else self.default_pricing_algo
|
||||
df = actual_algo()
|
||||
csv = self.convert_df_to_csv(df)
|
||||
return csv
|
||||
|
||||
def create_add_to_tcgplayer_csv(self, box_id: str = None, upload_id: str = None, algo: Callable = None) -> str:
|
||||
actual_algo = algo if algo is not None else self.default_pricing_algo
|
||||
if box_id and upload_id:
|
||||
raise ValueError("Cannot specify both box_id and upload_id")
|
||||
elif not box_id and not upload_id:
|
||||
raise ValueError("Must specify either box_id or upload_id")
|
||||
elif box_id:
|
||||
logger.debug("creating df")
|
||||
df = self.get_box_with_most_recent_prices(box_id)
|
||||
elif upload_id:
|
||||
raise NotImplementedError("Not yet implemented")
|
||||
df = actual_algo(df)
|
||||
csv = self.convert_add_df_to_csv(df)
|
||||
return csv
|
||||
205
services/pricing_old.py
Normal file
205
services/pricing_old.py
Normal file
@ -0,0 +1,205 @@
|
||||
import logging
|
||||
from typing import Callable
|
||||
from db.models import TCGPlayerInventory, TCGPlayerExportHistory, TCGPlayerPricingHistory, ManaboxExportData, ManaboxTCGPlayerMapping, TCGPlayerProduct
|
||||
from sqlalchemy.orm import Session
|
||||
import pandas as pd
|
||||
from db.utils import db_transaction
|
||||
from sqlalchemy import func, and_, exists
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class PricingService:
|
||||
def __init__(self, db: Session):
|
||||
self.db = db
|
||||
|
||||
def get_box_with_most_recent_prices(self, box_id: str) -> pd.DataFrame:
|
||||
latest_prices = (
|
||||
self.db.query(
|
||||
TCGPlayerPricingHistory.tcgplayer_product_id,
|
||||
func.max(TCGPlayerPricingHistory.date_created).label('max_date')
|
||||
)
|
||||
.group_by(TCGPlayerPricingHistory.tcgplayer_product_id)
|
||||
.subquery('latest') # Added name to subquery
|
||||
)
|
||||
|
||||
result = (
|
||||
self.db.query(ManaboxExportData, TCGPlayerPricingHistory, TCGPlayerProduct)
|
||||
.join(ManaboxTCGPlayerMapping, ManaboxExportData.id == ManaboxTCGPlayerMapping.manabox_id)
|
||||
.join(TCGPlayerProduct, ManaboxTCGPlayerMapping.tcgplayer_id == TCGPlayerProduct.id)
|
||||
.join(TCGPlayerPricingHistory, TCGPlayerProduct.id == TCGPlayerPricingHistory.tcgplayer_product_id)
|
||||
.join(
|
||||
latest_prices,
|
||||
and_(
|
||||
TCGPlayerPricingHistory.tcgplayer_product_id == latest_prices.c.tcgplayer_product_id,
|
||||
TCGPlayerPricingHistory.date_created == latest_prices.c.max_date
|
||||
)
|
||||
)
|
||||
.filter(ManaboxExportData.box_id == box_id) # Removed str() conversion
|
||||
.all()
|
||||
)
|
||||
|
||||
logger.debug(f"Found {len(result)} rows")
|
||||
|
||||
df = pd.DataFrame([{
|
||||
**{f"manabox_{k}": v for k, v in row[0].__dict__.items() if not k.startswith('_')},
|
||||
**{f"pricing_{k}": v for k, v in row[1].__dict__.items() if not k.startswith('_')},
|
||||
**{f"tcgproduct_{k}": v for k, v in row[2].__dict__.items() if not k.startswith('_')}
|
||||
} for row in result])
|
||||
|
||||
return df
|
||||
|
||||
def get_live_inventory_with_most_recent_prices(self) -> pd.DataFrame:
|
||||
# Get latest export IDs using subqueries
|
||||
latest_inventory_export = (
|
||||
self.db.query(TCGPlayerExportHistory.inventory_export_id)
|
||||
.filter(TCGPlayerExportHistory.type == "live_inventory")
|
||||
.order_by(TCGPlayerExportHistory.date_created.desc())
|
||||
.limit(1)
|
||||
.scalar_subquery()
|
||||
)
|
||||
# this is bad because latest pricing export is not guaranteed to be related to the latest inventory export
|
||||
latest_pricing_export = (
|
||||
self.db.query(TCGPlayerExportHistory.pricing_export_id)
|
||||
.filter(TCGPlayerExportHistory.type == "pricing")
|
||||
.order_by(TCGPlayerExportHistory.date_created.desc())
|
||||
.limit(1)
|
||||
.scalar_subquery()
|
||||
)
|
||||
|
||||
# Join inventory and pricing data in a single query
|
||||
inventory_with_pricing = (
|
||||
self.db.query(TCGPlayerInventory, TCGPlayerPricingHistory)
|
||||
.join(
|
||||
TCGPlayerPricingHistory,
|
||||
TCGPlayerInventory.tcgplayer_product_id == TCGPlayerPricingHistory.tcgplayer_product_id
|
||||
)
|
||||
.filter(
|
||||
TCGPlayerInventory.export_id == latest_inventory_export,
|
||||
TCGPlayerPricingHistory.export_id == latest_pricing_export
|
||||
)
|
||||
.all()
|
||||
)
|
||||
|
||||
# Convert to pandas DataFrame
|
||||
df = pd.DataFrame([{
|
||||
# Inventory columns
|
||||
**{f"inventory_{k}": v
|
||||
for k, v in row[0].__dict__.items()
|
||||
if not k.startswith('_')},
|
||||
# Pricing columns
|
||||
**{f"pricing_{k}": v
|
||||
for k, v in row[1].__dict__.items()
|
||||
if not k.startswith('_')}
|
||||
} for row in inventory_with_pricing])
|
||||
|
||||
return df
|
||||
|
||||
def default_pricing_algo(self, df: pd.DataFrame = None):
|
||||
if df is None:
|
||||
logger.debug("No DataFrame provided, fetching live inventory with most recent prices")
|
||||
df = self.get_live_inventory_with_most_recent_prices()
|
||||
# if tcg low price is < 0.35, set my_price to 0.35
|
||||
# if either tcg low price or tcg low price with shipping is under 5, set my_price to tcg low price * 1.25
|
||||
# if tcg low price with shipping is > 25 set price to tcg low price with shipping * 1.025
|
||||
# otherwise, set price to tcg low price with shipping * 1.10
|
||||
# also round to 2 decimal places
|
||||
df['my_price'] = df.apply(lambda row: round(
|
||||
0.35 if row['pricing_tcg_low_price'] < 0.35 else
|
||||
row['pricing_tcg_low_price'] * 1.25 if row['pricing_tcg_low_price'] < 5 or row['pricing_tcg_low_price_with_shipping'] < 5 else
|
||||
row['pricing_tcg_low_price_with_shipping'] * 1.025 if row['pricing_tcg_low_price_with_shipping'] > 25 else
|
||||
row['pricing_tcg_low_price_with_shipping'] * 1.10, 2), axis=1)
|
||||
# log rows with no price
|
||||
no_price = df[df['my_price'].isnull()]
|
||||
if len(no_price) > 0:
|
||||
logger.warning(f"Found {len(no_price)} rows with no price")
|
||||
logger.warning(no_price)
|
||||
# remove rows with no price
|
||||
df = df.dropna(subset=['my_price'])
|
||||
return df
|
||||
|
||||
def convert_df_to_csv(self, df: pd.DataFrame):
|
||||
# Flip the mapping to be from current names TO desired names
|
||||
column_mapping = {
|
||||
'inventory_tcgplayer_id': 'TCGplayer Id',
|
||||
'inventory_product_line': 'Product Line',
|
||||
'inventory_set_name': 'Set Name',
|
||||
'inventory_product_name': 'Product Name',
|
||||
'inventory_title': 'Title',
|
||||
'inventory_number': 'Number',
|
||||
'inventory_rarity': 'Rarity',
|
||||
'inventory_condition': 'Condition',
|
||||
'pricing_tcg_market_price': 'TCG Market Price',
|
||||
'pricing_tcg_direct_low': 'TCG Direct Low',
|
||||
'pricing_tcg_low_price_with_shipping': 'TCG Low Price With Shipping',
|
||||
'pricing_tcg_low_price': 'TCG Low Price',
|
||||
'inventory_total_quantity': 'Total Quantity',
|
||||
'inventory_add_to_quantity': 'Add to Quantity',
|
||||
'my_price': 'TCG Marketplace Price',
|
||||
'inventory_photo_url': 'Photo URL'
|
||||
}
|
||||
|
||||
df['pricing_tcg_market_price'] = ""
|
||||
df['pricing_tcg_direct_low'] = ""
|
||||
df['pricing_tcg_low_price_with_shipping'] = ""
|
||||
df['pricing_tcg_low_price'] = ""
|
||||
df['inventory_total_quantity'] = ""
|
||||
df['inventory_add_to_quantity'] = 0
|
||||
df['inventory_photo_url'] = ""
|
||||
|
||||
# First select the columns we want (using the keys of our mapping)
|
||||
# Then rename them to the desired names (the values in our mapping)
|
||||
df = df[column_mapping.keys()].rename(columns=column_mapping)
|
||||
|
||||
return df.to_csv(index=False, quoting=1, quotechar='"')
|
||||
|
||||
def convert_add_df_to_csv(self, df: pd.DataFrame):
|
||||
column_mapping = {
|
||||
'tcgproduct_tcgplayer_id': 'TCGplayer Id',
|
||||
'tcgproduct_product_line': 'Product Line',
|
||||
'tcgproduct_set_name': 'Set Name',
|
||||
'tcgproduct_product_name': 'Product Name',
|
||||
'tcgproduct_title': 'Title',
|
||||
'tcgproduct_number': 'Number',
|
||||
'tcgproduct_rarity': 'Rarity',
|
||||
'tcgproduct_condition': 'Condition',
|
||||
'pricing_tcg_market_price': 'TCG Market Price',
|
||||
'pricing_tcg_direct_low': 'TCG Direct Low',
|
||||
'pricing_tcg_low_price_with_shipping': 'TCG Low Price With Shipping',
|
||||
'pricing_tcg_low_price': 'TCG Low Price',
|
||||
'tcgproduct_group_id': 'Total Quantity',
|
||||
'manabox_quantity': 'Add to Quantity',
|
||||
'my_price': 'TCG Marketplace Price',
|
||||
'tcgproduct_photo_url': 'Photo URL'
|
||||
}
|
||||
df['tcgproduct_group_id'] = ""
|
||||
df['pricing_tcg_market_price'] = ""
|
||||
df['pricing_tcg_direct_low'] = ""
|
||||
df['pricing_tcg_low_price_with_shipping'] = ""
|
||||
df['pricing_tcg_low_price'] = ""
|
||||
df['tcgproduct_photo_url'] = ""
|
||||
|
||||
df = df[column_mapping.keys()].rename(columns=column_mapping)
|
||||
|
||||
return df.to_csv(index=False, quoting=1, quotechar='"')
|
||||
|
||||
def create_live_inventory_pricing_update_csv(self, algo: Callable = None) -> str:
|
||||
actual_algo = algo if algo is not None else self.default_pricing_algo
|
||||
df = actual_algo()
|
||||
csv = self.convert_df_to_csv(df)
|
||||
return csv
|
||||
|
||||
def create_add_to_tcgplayer_csv(self, box_id: str = None, upload_id: str = None, algo: Callable = None) -> str:
|
||||
actual_algo = algo if algo is not None else self.default_pricing_algo
|
||||
if box_id and upload_id:
|
||||
raise ValueError("Cannot specify both box_id and upload_id")
|
||||
elif not box_id and not upload_id:
|
||||
raise ValueError("Must specify either box_id or upload_id")
|
||||
elif box_id:
|
||||
logger.debug("creating df")
|
||||
df = self.get_box_with_most_recent_prices(box_id)
|
||||
elif upload_id:
|
||||
raise NotImplementedError("Not yet implemented")
|
||||
df = actual_algo(df)
|
||||
csv = self.convert_add_df_to_csv(df)
|
||||
return csv
|
||||
@ -5,6 +5,7 @@ from io import StringIO
|
||||
import pandas as pd
|
||||
from services.file import FileService
|
||||
from services.tcgplayer import TCGPlayerService
|
||||
from services.storage import StorageService
|
||||
from uuid import uuid4 as uuid
|
||||
import logging
|
||||
|
||||
@ -26,10 +27,11 @@ class ManaboxRow:
|
||||
self.quantity = row['quantity']
|
||||
|
||||
class ProductService:
|
||||
def __init__(self, db: Session, file_service: FileService, tcgplayer_service: TCGPlayerService):
|
||||
def __init__(self, db: Session, file_service: FileService, tcgplayer_service: TCGPlayerService, storage_service: StorageService):
|
||||
self.db = db
|
||||
self.file_service = file_service
|
||||
self.tcgplayer_service = tcgplayer_service
|
||||
self.storage_service = storage_service
|
||||
|
||||
def _format_manabox_df(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
# format columns
|
||||
@ -140,7 +142,9 @@ class ProductService:
|
||||
df = self._manabox_file_to_df(file)
|
||||
for index, row in df.iterrows():
|
||||
manabox_row = ManaboxRow(row)
|
||||
# create card concepts - manabox, tcgplayer, card, product
|
||||
card_manabox = self.card_manabox_lookup_create_if_not_exist(manabox_row)
|
||||
# create staged inventory with quantity for processing down the marketplace pipeline
|
||||
staged_product = self.create_staged_product(file, card_manabox, row)
|
||||
# update file status
|
||||
with db_transaction(self.db):
|
||||
@ -149,3 +153,9 @@ class ProductService:
|
||||
with db_transaction(self.db):
|
||||
file.status = 'error'
|
||||
raise e
|
||||
try:
|
||||
# create storage records for physically storing individual cards
|
||||
self.storage_service.store_staged_products_for_file(file.id)
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating storage records: {str(e)}")
|
||||
raise e
|
||||
|
||||
176
services/storage.py
Normal file
176
services/storage.py
Normal file
@ -0,0 +1,176 @@
|
||||
from sqlalchemy.orm import Session
|
||||
from db.utils import db_transaction
|
||||
from db.models import Warehouse, User, StagedFileProduct, StorageBlock, ProductBlock, File, Card
|
||||
from uuid import uuid4 as uuid
|
||||
from typing import List, TypedDict
|
||||
|
||||
class ProductAttributes(TypedDict):
|
||||
product_id: str
|
||||
card_number: str
|
||||
|
||||
class StorageService:
|
||||
def __init__(self, db: Session):
|
||||
self.db = db
|
||||
|
||||
def get_or_create_user(self, username: str) -> User:
|
||||
user = self.db.query(User).filter(User.username == username).first()
|
||||
if user is None:
|
||||
user = User(
|
||||
id = str(uuid()),
|
||||
username = username
|
||||
)
|
||||
with db_transaction(self.db):
|
||||
self.db.add(user)
|
||||
return user
|
||||
|
||||
def get_or_create_warehouse(self) -> Warehouse:
|
||||
warehouse = self.db.query(Warehouse).first()
|
||||
user = self.get_or_create_user('admin')
|
||||
if warehouse is None:
|
||||
warehouse = Warehouse(
|
||||
id = str(uuid()),
|
||||
user_id = user.id
|
||||
)
|
||||
with db_transaction(self.db):
|
||||
self.db.add(warehouse)
|
||||
return warehouse
|
||||
|
||||
def get_staged_product(self, file_id: str) -> List[StagedFileProduct]:
|
||||
staged_product = self.db.query(StagedFileProduct).filter(StagedFileProduct.file_id == file_id).all()
|
||||
return staged_product
|
||||
|
||||
def get_storage_block_name(self, warehouse: Warehouse, file_id: str) -> str:
|
||||
# Get file type from id
|
||||
current_file = self.db.query(File).filter(File.id == file_id).first()
|
||||
if not current_file:
|
||||
raise ValueError(f"No file found with id {file_id}")
|
||||
|
||||
# Determine storage block type
|
||||
storage_block_type = 'rare' if 'rare' in current_file.type else 'common'
|
||||
prefix = storage_block_type[0]
|
||||
|
||||
# Get most recent storage block with same type and warehouse id
|
||||
latest_block = (
|
||||
self.db.query(StorageBlock)
|
||||
.filter(
|
||||
StorageBlock.warehouse_id == warehouse.id,
|
||||
StorageBlock.type == storage_block_type
|
||||
)
|
||||
.order_by(StorageBlock.date_created.desc())
|
||||
.first()
|
||||
)
|
||||
|
||||
# If no existing block, start with number 1
|
||||
if not latest_block:
|
||||
return f"{prefix}1"
|
||||
|
||||
# Start with the next number after the latest block
|
||||
number = int(latest_block.name[1:])
|
||||
|
||||
while True:
|
||||
number += 1
|
||||
new_name = f"{prefix}{number}"
|
||||
|
||||
# Check if the new name already exists
|
||||
exists = (
|
||||
self.db.query(StorageBlock)
|
||||
.filter(
|
||||
StorageBlock.warehouse_id == warehouse.id,
|
||||
StorageBlock.name == new_name
|
||||
)
|
||||
.first()
|
||||
)
|
||||
|
||||
if not exists:
|
||||
return new_name
|
||||
|
||||
def create_storage_block(self, warehouse: Warehouse, file_id: str) -> StorageBlock:
|
||||
current_file = self.db.query(File).filter(File.id == file_id).first()
|
||||
if not current_file:
|
||||
raise ValueError(f"No file found with id {file_id}")
|
||||
|
||||
storage_block_type = 'rare' if 'rare' in current_file.type else 'common'
|
||||
|
||||
storage_block = StorageBlock(
|
||||
id = str(uuid()),
|
||||
warehouse_id = warehouse.id,
|
||||
name = self.get_storage_block_name(warehouse, file_id),
|
||||
type = storage_block_type
|
||||
)
|
||||
with db_transaction(self.db):
|
||||
self.db.add(storage_block)
|
||||
return storage_block
|
||||
|
||||
def add_staged_product_to_product_block(
|
||||
self,
|
||||
staged_product: StagedFileProduct,
|
||||
storage_block: StorageBlock,
|
||||
product_attributes: ProductAttributes,
|
||||
block_index: int
|
||||
) -> ProductBlock:
|
||||
"""Create a new ProductBlock for a single unit of a staged product."""
|
||||
product_block = ProductBlock(
|
||||
id=str(uuid()),
|
||||
product_id=staged_product.product_id,
|
||||
block_id=storage_block.id,
|
||||
block_index=block_index
|
||||
)
|
||||
|
||||
with db_transaction(self.db):
|
||||
self.db.add(product_block)
|
||||
|
||||
return product_block
|
||||
|
||||
def get_staged_product_attributes_for_storage(
|
||||
self,
|
||||
staged_product: StagedFileProduct
|
||||
) -> List[ProductAttributes]:
|
||||
"""Get attributes for each unit of a staged product."""
|
||||
result = (
|
||||
self.db.query(
|
||||
StagedFileProduct.product_id,
|
||||
StagedFileProduct.quantity,
|
||||
Card.number
|
||||
)
|
||||
.join(Card, Card.product_id == StagedFileProduct.product_id)
|
||||
.filter(StagedFileProduct.id == staged_product.id)
|
||||
.first()
|
||||
)
|
||||
|
||||
if not result:
|
||||
return []
|
||||
|
||||
return [
|
||||
ProductAttributes(
|
||||
product_id=result.product_id,
|
||||
card_number=result.number
|
||||
)
|
||||
for i in range(result.quantity)
|
||||
]
|
||||
|
||||
def store_staged_products_for_file(self, file_id: str) -> StorageBlock:
|
||||
"""Store all staged products for a file in a new storage block."""
|
||||
warehouse = self.get_or_create_warehouse()
|
||||
storage_block = self.create_storage_block(warehouse, file_id)
|
||||
staged_products = self.get_staged_product(file_id)
|
||||
|
||||
# Collect all product attributes first
|
||||
all_product_attributes = []
|
||||
for staged_product in staged_products:
|
||||
product_attributes_list = self.get_staged_product_attributes_for_storage(staged_product)
|
||||
for attrs in product_attributes_list:
|
||||
all_product_attributes.append((staged_product, attrs))
|
||||
|
||||
# Sort by card number as integer to determine block indices
|
||||
sorted_attributes = sorted(all_product_attributes, key=lambda x: int(x[1]['card_number']))
|
||||
|
||||
# Add products with correct block indices
|
||||
for block_index, (staged_product, product_attributes) in enumerate(sorted_attributes, 1):
|
||||
self.add_staged_product_to_product_block(
|
||||
staged_product=staged_product,
|
||||
storage_block=storage_block,
|
||||
product_attributes=product_attributes,
|
||||
block_index=block_index
|
||||
)
|
||||
|
||||
return storage_block
|
||||
@ -3,16 +3,18 @@ import logging
|
||||
from typing import Dict, Callable
|
||||
from sqlalchemy.orm import Session
|
||||
from services.product import ProductService
|
||||
from services.tcgplayer import TCGPlayerService
|
||||
from db.models import File
|
||||
|
||||
|
||||
class TaskService:
|
||||
def __init__(self, db: Session, product_service: ProductService):
|
||||
def __init__(self, db: Session, product_service: ProductService, tcgplayer_service: TCGPlayerService):
|
||||
self.scheduler = BackgroundScheduler()
|
||||
self.logger = logging.getLogger(__name__)
|
||||
self.tasks: Dict[str, Callable] = {}
|
||||
self.db = db
|
||||
self.product_service = product_service
|
||||
self.tcgplayer_service = tcgplayer_service
|
||||
|
||||
async def start(self):
|
||||
self.scheduler.start()
|
||||
@ -28,11 +30,20 @@ class TaskService:
|
||||
id='daily_report'
|
||||
)
|
||||
|
||||
# Tasks that should be scheduled
|
||||
async def daily_report(self):
|
||||
self.scheduler.add_job(
|
||||
self.pricing_update,
|
||||
'cron',
|
||||
minute=28,
|
||||
id='pricing_update'
|
||||
)
|
||||
|
||||
def daily_report(self): # Removed async
|
||||
self.logger.info("Generating daily report")
|
||||
# Daily report logic
|
||||
|
||||
def pricing_update(self): # Removed async
|
||||
self.logger.info("Hourly pricing update")
|
||||
self.tcgplayer_service.cron_load_prices()
|
||||
|
||||
async def process_manabox_file(self, file: File):
|
||||
self.logger.info("Processing ManaBox file")
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
from db.models import ManaboxExportData, Box, TCGPlayerGroups, TCGPlayerInventory, TCGPlayerExportHistory, TCGPlayerPricingHistory, TCGPlayerProduct, ManaboxTCGPlayerMapping, CardManabox, CardTCGPlayer
|
||||
from db.models import ManaboxExportData, Box, TCGPlayerGroups, TCGPlayerInventory, TCGPlayerExportHistory, TCGPlayerPricingHistory, TCGPlayerProduct, ManaboxTCGPlayerMapping, CardManabox, CardTCGPlayer, Price
|
||||
import requests
|
||||
from sqlalchemy.orm import Session
|
||||
from db.utils import db_transaction
|
||||
@ -18,6 +18,8 @@ from typing import List, Dict, Optional
|
||||
from io import StringIO, BytesIO
|
||||
from services.pricing import PricingService
|
||||
from sqlalchemy.sql import exists
|
||||
import pandas as pd
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@ -559,6 +561,115 @@ class TCGPlayerService:
|
||||
return None
|
||||
return matching_product
|
||||
|
||||
def get_pricing_export_for_all_products(self) -> bytes:
|
||||
"""
|
||||
Retrieves pricing export data for all products in TCGPlayer format.
|
||||
|
||||
Returns:
|
||||
bytes: Raw CSV data containing pricing information
|
||||
"""
|
||||
try:
|
||||
all_group_ids = self.db.query(TCGPlayerGroups.group_id).all()
|
||||
all_group_ids = [str(group_id) for group_id, in all_group_ids]
|
||||
export_csv = self._get_export_csv(all_group_ids, convert=False)
|
||||
return export_csv
|
||||
except SQLAlchemyError as e:
|
||||
raise RuntimeError(f"Failed to retrieve group IDs: {str(e)}")
|
||||
|
||||
def pricing_export_to_df(self, export_csv: bytes) -> pd.DataFrame:
|
||||
"""
|
||||
Converts raw CSV pricing data to a pandas DataFrame.
|
||||
|
||||
Args:
|
||||
export_csv (bytes): Raw CSV data in bytes format
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: Processed pricing data
|
||||
|
||||
Raises:
|
||||
ValueError: If no CSV data is provided or if CSV parsing fails
|
||||
"""
|
||||
if not export_csv:
|
||||
raise ValueError("No export CSV provided")
|
||||
|
||||
csv_file = None
|
||||
try:
|
||||
text_content = export_csv.decode('utf-8')
|
||||
csv_file = StringIO(text_content)
|
||||
df = pd.read_csv(csv_file)
|
||||
|
||||
if df.empty:
|
||||
raise ValueError("CSV data is empty")
|
||||
|
||||
return df
|
||||
except UnicodeDecodeError as e:
|
||||
raise ValueError(f"Failed to decode CSV data: {str(e)}")
|
||||
except pd.errors.EmptyDataError:
|
||||
raise ValueError("CSV file is empty or malformed")
|
||||
finally:
|
||||
if csv_file:
|
||||
csv_file.close()
|
||||
|
||||
def cron_load_prices(self) -> None:
|
||||
"""
|
||||
Scheduled task to load and update product prices.
|
||||
Fetches current prices, processes them, and saves new price records to the database.
|
||||
"""
|
||||
try:
|
||||
# Get and process price data
|
||||
price_csv = self.get_pricing_export_for_all_products()
|
||||
price_df = self.pricing_export_to_df(price_csv)
|
||||
|
||||
# Clean column names
|
||||
price_df.columns = price_df.columns.str.lower().str.replace(' ', '_')
|
||||
|
||||
# Get all products efficiently
|
||||
products_query = self.db.query(
|
||||
CardTCGPlayer.tcgplayer_id,
|
||||
CardTCGPlayer.product_id
|
||||
)
|
||||
product_df = pd.read_sql(products_query.statement, self.db.bind)
|
||||
|
||||
# Merge dataframes
|
||||
merged_df = pd.merge(
|
||||
price_df,
|
||||
product_df,
|
||||
on='tcgplayer_id',
|
||||
how='inner'
|
||||
)
|
||||
|
||||
# Define price columns to process
|
||||
price_columns = [
|
||||
'tcg_market_price',
|
||||
'tcg_direct_low',
|
||||
'tcg_low_price_with_shipping',
|
||||
'tcg_low_price',
|
||||
'tcg_marketplace_price'
|
||||
]
|
||||
|
||||
# Process in batches to avoid memory issues
|
||||
BATCH_SIZE = 1000
|
||||
for price_column in price_columns:
|
||||
records = []
|
||||
|
||||
for start_idx in range(0, len(merged_df), BATCH_SIZE):
|
||||
batch_df = merged_df.iloc[start_idx:start_idx + BATCH_SIZE]
|
||||
|
||||
batch_records = [
|
||||
Price(
|
||||
id=str(uuid.uuid4()),
|
||||
product_id=row['product_id'],
|
||||
type=price_column,
|
||||
price=row[price_column]
|
||||
)
|
||||
for _, row in batch_df.iterrows()
|
||||
if pd.notna(row[price_column]) # Skip rows with NaN prices
|
||||
]
|
||||
|
||||
with db_transaction(self.db):
|
||||
self.db.bulk_save_objects(batch_records)
|
||||
self.db.flush()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load prices: {str(e)}")
|
||||
raise
|
||||
154
tests/box_test.py
Normal file
154
tests/box_test.py
Normal file
@ -0,0 +1,154 @@
|
||||
from fastapi.testclient import TestClient
|
||||
from fastapi import BackgroundTasks
|
||||
import pytest
|
||||
from unittest.mock import Mock, patch
|
||||
import asyncio
|
||||
import os
|
||||
from main import app
|
||||
from services.file import FileService
|
||||
from services.task import TaskService
|
||||
|
||||
|
||||
|
||||
client = TestClient(app)
|
||||
|
||||
test_boxes = []
|
||||
|
||||
def test_create_box():
|
||||
# Send as form data, not JSON
|
||||
response = client.post("/api/boxes",
|
||||
data={
|
||||
"type": "play",
|
||||
"set_code": "BLB",
|
||||
"sku": "1234",
|
||||
"num_cards_expected": 504
|
||||
}
|
||||
)
|
||||
test_boxes.append(response.json()["box"][0]["product_id"])
|
||||
|
||||
assert response.status_code == 201
|
||||
assert response.json()["success"] == True
|
||||
assert response.json()["box"][0]["type"] == "play"
|
||||
assert response.json()["box"][0]["set_code"] == "BLB"
|
||||
assert response.json()["box"][0]["sku"] == "1234"
|
||||
assert response.json()["box"][0]["num_cards_expected"] == 504
|
||||
|
||||
def test_update_box():
|
||||
# Create a box first
|
||||
create_response = client.post("/api/boxes",
|
||||
data={
|
||||
"type": "collector",
|
||||
"set_code": "MKM",
|
||||
"sku": "3456",
|
||||
"num_cards_expected": 504
|
||||
}
|
||||
)
|
||||
box_id = create_response.json()["box"][0]["product_id"]
|
||||
test_boxes.append(box_id)
|
||||
|
||||
# Update the box
|
||||
response = client.put(f"/api/boxes/{box_id}",
|
||||
data={
|
||||
"num_cards_expected": 500
|
||||
}
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["success"] == True
|
||||
assert response.json()["box"][0]["type"] == "collector"
|
||||
assert response.json()["box"][0]["set_code"] == "MKM"
|
||||
assert response.json()["box"][0]["sku"] == "3456"
|
||||
assert response.json()["box"][0]["num_cards_expected"] == 500
|
||||
|
||||
def test_delete_box():
|
||||
# Create a box first
|
||||
create_response = client.post("/api/boxes",
|
||||
data={
|
||||
"type": "set",
|
||||
"set_code": "LCI",
|
||||
"sku": "7890",
|
||||
"num_cards_expected": 504
|
||||
}
|
||||
)
|
||||
box_id = create_response.json()["box"][0]["product_id"]
|
||||
|
||||
# Delete the box
|
||||
response = client.delete(f"/api/boxes/{box_id}")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["success"] == True
|
||||
assert response.json()["box"][0]["type"] == "set"
|
||||
assert response.json()["box"][0]["set_code"] == "LCI"
|
||||
assert response.json()["box"][0]["sku"] == "7890"
|
||||
assert response.json()["box"][0]["num_cards_expected"] == 504
|
||||
|
||||
# Constants for reused values
|
||||
TEST_FILE_PATH = os.path.join(os.getcwd(), "tests/test_files", "manabox_test_file.csv")
|
||||
DEFAULT_METADATA = {
|
||||
"source": "manabox",
|
||||
"type": "scan_export_common"
|
||||
}
|
||||
|
||||
def get_file_size_kb(file_path):
|
||||
"""Helper to consistently calculate file size in KB"""
|
||||
with open(file_path, "rb") as f:
|
||||
return round(len(f.read()) / 1024, 2)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_box():
|
||||
"""Test creating a new manabox file"""
|
||||
# Open file within the test scope
|
||||
with open(TEST_FILE_PATH, "rb") as test_file:
|
||||
files = {"file": test_file}
|
||||
|
||||
# Make request
|
||||
response = client.post("/api/files", data=DEFAULT_METADATA, files=files)
|
||||
|
||||
# Check response
|
||||
assert response.status_code == 201
|
||||
assert response.json()["success"] == True
|
||||
|
||||
file_data = response.json()["files"][0]
|
||||
assert file_data["source"] == DEFAULT_METADATA["source"]
|
||||
assert file_data["type"] == DEFAULT_METADATA["type"]
|
||||
assert file_data["status"] == "pending"
|
||||
assert file_data["service"] == None
|
||||
assert file_data["filename"] == "manabox_test_file.csv"
|
||||
assert file_data["filesize_kb"] == get_file_size_kb(TEST_FILE_PATH)
|
||||
assert file_data["id"] is not None
|
||||
|
||||
# Execute background tasks if they were added
|
||||
background_tasks = BackgroundTasks()
|
||||
for task in background_tasks.tasks:
|
||||
await task()
|
||||
|
||||
# Create a box first
|
||||
create_response = client.post("/api/boxes",
|
||||
data={
|
||||
"type": "play",
|
||||
"set_code": "OTJ",
|
||||
"sku": "2314",
|
||||
"num_cards_expected": 504
|
||||
}
|
||||
)
|
||||
box_id = create_response.json()["box"][0]["product_id"]
|
||||
test_boxes.append(box_id)
|
||||
|
||||
# Open the box
|
||||
response = client.post(f"/api/boxes/{box_id}/open",
|
||||
data={
|
||||
"product_id": box_id,
|
||||
"file_ids": [file_data["id"]],
|
||||
"num_cards_actual": 500
|
||||
}
|
||||
)
|
||||
|
||||
assert response.status_code == 201
|
||||
assert response.json()["success"] == True
|
||||
|
||||
|
||||
def test_cleanup():
|
||||
# Delete all boxes created during testing
|
||||
for box_id in test_boxes:
|
||||
client.delete(f"/api/boxes/{box_id}")
|
||||
|
||||
@ -14,7 +14,7 @@ client = TestClient(app)
|
||||
TEST_FILE_PATH = os.path.join(os.getcwd(), "tests/test_files", "manabox_test_file.csv")
|
||||
DEFAULT_METADATA = {
|
||||
"source": "manabox",
|
||||
"type": "scan_export"
|
||||
"type": "scan_export_rare"
|
||||
}
|
||||
|
||||
def get_file_size_kb(file_path):
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user