q

app/db/database.py

@@ -0,0 +1,116 @@
+from sqlalchemy import create_engine, text
+from sqlalchemy.orm import sessionmaker, Session
+from contextlib import contextmanager
+from typing import Generator
+import os
+from sqlalchemy import inspect
+from services.tcgplayer import TCGPlayerService
+from services.pricing import PricingService
+from services.file import FileService
+from db.models import Price
+from datetime import datetime
+
+
+import logging
+logger = logging.getLogger(__name__)
+
+# Get database URL from environment variable with fallback
+DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///omegacard.db")
+
+# Create engine with proper configuration
+engine = create_engine(
+    DATABASE_URL,
+    pool_pre_ping=True,  # Enable connection health checks
+    pool_size=5,         # Set reasonable pool size
+    max_overflow=10      # Allow some overflow connections
+)
+
+# Create session factory
+SessionLocal = sessionmaker(
+    bind=engine,
+    autocommit=False,
+    autoflush=False
+)
+
+@contextmanager
+def get_db_session() -> Generator[Session, None, None]:
+    """Context manager for database sessions"""
+    session = SessionLocal()
+    try:
+        yield session
+    except Exception as e:
+        logger.error(f"Database session error: {str(e)}")
+        session.rollback()
+        raise
+    finally:
+        session.close()
+
+def get_db() -> Generator[Session, None, None]:
+    """Dependency for FastAPI to get database sessions"""
+    with get_db_session() as session:
+        yield session
+
+def prepopulate_data(db: Session, db_exist: bool = False) -> None:
+    file_service = FileService(db)
+    tcgplayer_service = TCGPlayerService(db, file_service)
+    pricing_service = PricingService(db, file_service, tcgplayer_service)
+    if not db_exist:
+        tcgplayer_service.populate_tcgplayer_groups()
+        file = tcgplayer_service.load_tcgplayer_cards()
+        pricing_service.cron_load_prices(file)
+    else:
+        pricing_service.cron_load_prices()
+
+def init_db() -> None:
+    """Initialize database tables and run first-time setup if needed"""
+    from .models import Base
+    try:
+        inspector = inspect(engine)
+        tables_exist = all(
+            table in inspector.get_table_names()
+            for table in Base.metadata.tables.keys()
+        )
+        if tables_exist:
+            with get_db_session() as db:
+                # get date created of latest pricing record
+                latest_price = db.query(Price).order_by(Price.date_created.desc()).first()
+                if latest_price:
+                    # check if it is greater than 1.5 hours old
+                    if (datetime.now() - latest_price.date_created).total_seconds() > 5400:
+                        prepopulate_data(db, db_exist=True)
+                else:
+                    prepopulate_data(db, db_exist=True)
+        
+        # Create tables if they don't exist
+        Base.metadata.create_all(bind=engine)
+        
+        # Run first-time setup only if tables were just created
+        if not tables_exist:
+            with get_db_session() as db:
+                prepopulate_data(db)
+            logger.info("First-time database setup completed")
+        
+        logger.info("Database initialization completed")
+    except Exception as e:
+        logger.error(f"Failed to initialize database: {str(e)}")
+        raise
+
+def check_db_connection() -> bool:
+    """Check if database connection is working"""
+    try:
+        with get_db_session() as session:
+            session.execute(text("SELECT 1"))
+        return True
+    except Exception as e:
+        logger.error(f"Database connection check failed: {str(e)}")
+        return False
+
+def destroy_db() -> None:
+    """Destroy all database tables"""
+    from .models import Base
+    try:
+        Base.metadata.drop_all(bind=engine)
+        logger.info("Database tables dropped successfully")
+    except Exception as e:
+        logger.error(f"Failed to destroy database: {str(e)}")
+        raise

app/db/models.py

@@ -0,0 +1,367 @@
+from sqlalchemy import Column, Integer, String, Float, Boolean, DateTime, ForeignKey
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import relationship, validates
+from datetime import datetime
+from enum import Enum
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+Base = declarative_base()
+
+## Core Models
+
+class Product(Base):
+    """
+    product is the concept of a physical item that can be sold
+    """
+    __tablename__ = "products"
+
+    @validates("type")
+    def validate_type(self, key, type: str):
+        if type not in ProductTypeEnum or type.lower() not in ProductTypeEnum:
+            raise ValueError(f"Invalid product type: {type}")
+        return type
+    
+    @validates("product_line")
+    def validate_product_line(self, key, product_line: str):
+        if product_line not in ProductLineEnum or product_line.lower() not in ProductLineEnum:
+            raise ValueError(f"Invalid product line: {product_line}")
+        return product_line
+
+    id = Column(String, primary_key=True)
+    type = Column(String) # box or card
+    product_line = Column(String) # pokemon, mtg, etc.
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class Sale(Base):
+    """
+    sale represents a transaction where a product was sold to a customer on a marketplace
+    """
+    __tablename__ = "sales"
+
+    id = Column(String, primary_key=True)
+    ledger_id = Column(String, ForeignKey("ledgers.id"))
+    customer_id = Column(String, ForeignKey("customers.id"))
+    marketplace_id = Column(String, ForeignKey("marketplaces.id"))
+    amount = Column(Float)
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class Ledger(Base):
+    """
+    ledger associates financial transactions with a user
+    """
+    __tablename__ = "ledgers"
+
+    id = Column(String, primary_key=True)
+    user_id = Column(String, ForeignKey("users.id"))
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class Expense(Base):
+    """
+    expense is any cash outflow associated with moving a product
+    can be optionally associated with a sale or a product
+    """
+    __tablename__ = "expenses"
+
+    id = Column(String, primary_key=True)
+    ledger_id = Column(String, ForeignKey("ledgers.id"))
+    product_id = Column(String, ForeignKey("products.id"), nullable=True)
+    sale_id = Column(String, ForeignKey("sales.id"), nullable=True)
+    cost = Column(Float)
+    type = Column(String) # price paid, cogs, shipping, refund, supplies, subscription, fee, etc.
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class Marketplace(Base):
+    """
+    Marketplace represents a marketplace where products can be sold
+    """
+    __tablename__ = "marketplaces"
+
+    id = Column(String, primary_key=True)
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class Box(Base):
+    """
+    Box Represents a physical product with a sku that contains trading cards
+    Boxes can be sealed or opened
+    Opened boxes have cards associated with them
+    A box contains cards regardless of the inventory status of those cards
+    """
+    __tablename__ = "boxes"
+
+    @validates("type")
+    def validate_type(self, key, type: str):
+        if type not in BoxTypeEnum or type.lower() not in BoxTypeEnum:
+            raise ValueError(f"Invalid box type: {type}")
+        return type
+
+    product_id = Column(String, ForeignKey("products.id"), primary_key=True)
+    type = Column(String) # collector box, play box, etc.
+    set_code = Column(String)
+    sku = Column(String, nullable=True)
+    num_cards_expected = Column(Integer, nullable=True)
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class OpenBox(Base):
+    __tablename__ = "open_boxes"
+
+    id = Column(String, primary_key=True)
+    product_id = Column(String, ForeignKey("products.id"))
+    num_cards_actual = Column(Integer)
+    date_opened = Column(DateTime, default=datetime.now)
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class Card(Base):
+    """
+    Card represents the concept of a distinct card
+    Cards have metadata from different sources
+    """
+    __tablename__ = "cards"
+
+    product_id = Column(String, ForeignKey("products.id"), primary_key=True)
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class CardManabox(Base):
+    __tablename__ = "manabox_cards"
+
+    product_id = Column(String, ForeignKey("cards.product_id"), primary_key=True)
+    name = Column(String)
+    set_code = Column(String)
+    set_name = Column(String)
+    collector_number = Column(String)
+    foil = Column(String)
+    rarity = Column(String)
+    manabox_id = Column(Integer)
+    scryfall_id = Column(String)
+    condition = Column(String)
+    language = Column(String)
+
+class CardTCGPlayer(Base):
+    __tablename__ = "tcgplayer_cards"
+
+    product_id = Column(String, ForeignKey("cards.product_id"), primary_key=True)
+    group_id = Column(Integer, ForeignKey("tcgplayer_groups.group_id"))
+    tcgplayer_id = Column(Integer)
+    product_line = Column(String)
+    set_name = Column(String)
+    product_name = Column(String)
+    title = Column(String)
+    number = Column(String)
+    rarity = Column(String)
+    condition = Column(String)
+
+class Warehouse(Base):
+    """
+    container that is associated with a user and contains inventory and stock
+    """
+    __tablename__ = "warehouses"
+
+    id = Column(String, primary_key=True)
+    user_id = Column(String, ForeignKey("users.id"))
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class Stock(Base):
+    """
+    contains products that are listed for sale
+    """
+    __tablename__ = "stocks"
+
+    product_id = Column(String, ForeignKey("products.id"), primary_key=True)
+    warehouse_id = Column(String, ForeignKey("warehouses.id"), default="default")
+    marketplace_id = Column(String, ForeignKey("marketplaces.id"))
+    quantity = Column(Integer)
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class Inventory(Base):
+    """
+    contains products in inventory (not necessarily listed for sale)
+    sealed product in breakdown queue, held sealed product, speculatively held singles, etc.
+    inventory can contain products across multiple marketplaces
+    """
+    __tablename__ = "inventories"
+
+    product_id = Column(String, ForeignKey("products.id"), primary_key=True)
+    warehouse_id = Column(String, ForeignKey("warehouses.id"), default="default")
+    quantity = Column(Integer)
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class User(Base):
+    """
+    User represents a user in the system
+    """
+    __tablename__ = "users"
+
+    id = Column(String, primary_key=True)
+    username = Column(String)
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class Customer(Base):
+    """
+    Customer represents a customer that has purchased at least 1 product
+    """
+    __tablename__ = "customers"
+
+    id = Column(String, primary_key=True)
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class StagedFileProduct(Base):
+    __tablename__ = "staged_file_products"
+
+    id = Column(String, primary_key=True)
+    product_id = Column(String, ForeignKey("products.id"))
+    file_id = Column(String, ForeignKey("files.id"))
+    quantity = Column(Integer)
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class File(Base):
+    """
+    File represents a file that has been uploaded to or retrieved by the system
+    """
+    __tablename__ = "files"
+
+    id = Column(String, primary_key=True)
+    type = Column(String) # upload, export, etc.
+    source = Column(String) # manabox, tcgplayer, etc.
+    service = Column(String) # pricing, data, etc.
+    filename = Column(String)
+    filepath = Column(String) # backup location
+    filesize_kb = Column(Float)
+    status = Column(String)
+    box_id = Column(String, ForeignKey("boxes.product_id"), nullable=True)
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class Price(Base):
+    __tablename__ = "prices"
+
+    id = Column(String, primary_key=True)
+    product_id = Column(String, ForeignKey("products.id"))
+    marketplace_id = Column(String, ForeignKey("marketplaces.id"))
+    type = Column(String) # market, direct, low, low_with_shipping, marketplace
+    price = Column(Float)
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class StorageBlock(Base):
+    """
+    StorageBlock represents a physical storage location for products (50 card indexed block in a box)
+    """
+    __tablename__ = "storage_blocks"
+
+    @validates("type")
+    def validate_type(self, key, type: str):
+        if type not in StorageBlockTypeEnum or type.lower() not in StorageBlockTypeEnum:
+            raise ValueError(f"Invalid storage block type: {type}")
+        return type
+
+    id = Column(String, primary_key=True)
+    warehouse_id = Column(String, ForeignKey("warehouses.id"))
+    name = Column(String)
+    type = Column(String) # rare or common
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class ProductBlock(Base):
+    """
+    ProductBlock represents the relationship between a product and a storage block
+    which products are in a block and at what index
+    """
+    __tablename__ = "product_blocks"
+
+    id = Column(String, primary_key=True)
+    product_id = Column(String, ForeignKey("products.id"))
+    block_id = Column(String, ForeignKey("storage_blocks.id"))
+    block_index = Column(Integer)
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class OpenBoxCard(Base):
+    """
+    OpenedBoxCard represents the relationship between an opened box and the cards it contains
+    """
+    __tablename__ = "open_box_cards"
+
+    id = Column(String, primary_key=True)
+    open_box_id = Column(String, ForeignKey("open_boxes.id"))
+    card_id = Column(String, ForeignKey("cards.product_id"))
+    quantity = Column(Integer)
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class ProductSale(Base):
+    """
+    ProductSale represents the relationship between products and sales
+    """
+    __tablename__ = "product_sales"
+
+    id = Column(String, primary_key=True)
+    product_id = Column(String, ForeignKey("products.id"))
+    sale_id = Column(String, ForeignKey("sales.id"))
+    date_created = Column(DateTime, default=datetime.now)
+    date_modified = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+class TCGPlayerGroups(Base):
+    __tablename__ = 'tcgplayer_groups'
+
+    id = Column(String, primary_key=True)
+    group_id = Column(Integer)
+    name = Column(String)
+    abbreviation = Column(String)
+    is_supplemental = Column(String)
+    published_on = Column(String)
+    modified_on = Column(String)
+    category_id = Column(Integer)
+
+# enums
+
+class RarityEnum(str, Enum):
+    COMMON = "common"
+    UNCOMMON = "uncommon"
+    RARE = "rare"
+    MYTHIC = "mythic"
+    LAND = "land"
+    PROMO = "promo"
+    SPECIAL = "special"
+
+class ConditionEnum(str, Enum):
+    MINT = "mint"
+    NEAR_MINT = "near_mint"
+    LIGHTLY_PLAYED = "lightly_played"
+    MODERATELY_PLAYED = "moderately_played"
+    HEAVILY_PLAYED = "heavily_played"
+    DAMAGED = "damaged"
+
+class BoxTypeEnum(str, Enum):
+    COLLECTOR = "collector"
+    PLAY = "play"
+    DRAFT = "draft"
+    COMMANDER = "commander"
+    SET = "set"
+
+class ProductLineEnum(str, Enum):
+    MTG = "mtg"
+    POKEMON = "pokemon"
+
+class ProductTypeEnum(str, Enum):
+    BOX = "box"
+    CARD = "card"
+
+class StorageBlockTypeEnum(str, Enum):
+    RARE = "rare"
+    COMMON = "common"

app/db/utils.py

@@ -0,0 +1,23 @@
+from contextlib import contextmanager
+from sqlalchemy.orm import Session
+from exceptions import FailedUploadException
+import logging
+
+logger = logging.getLogger(__name__)
+
+@contextmanager
+def db_transaction(db: Session):
+    """Simple context manager for database transactions"""
+    try:
+        yield
+        db.commit()
+    except FailedUploadException as failed_upload:
+        logger.error(f"Failed upload: {str(failed_upload.message)}")
+        db.rollback()
+        db.add(failed_upload.file_upload_record)
+        db.commit()
+        raise
+    except Exception as e:
+        logger.error(f"Database error: {str(e)}")
+        db.rollback()
+        raise