146 lines
6.3 KiB
Python
146 lines
6.3 KiB
Python
from typing import Optional, List, Dict
|
|
import csv
|
|
import io
|
|
import os
|
|
import json
|
|
from datetime import datetime
|
|
from sqlalchemy.orm import Session
|
|
from app.db.database import transaction
|
|
from app.models.inventory import Inventory
|
|
from app.models.tcgplayer_product import TCGPlayerProduct
|
|
from app.services.inventory_service import InventoryService
|
|
|
|
class FileProcessingService:
|
|
def __init__(self, cache_dir: str = "app/data/cache/tcgplayer"):
|
|
self.cache_dir = cache_dir
|
|
self.inventory_service = InventoryService()
|
|
os.makedirs(cache_dir, exist_ok=True)
|
|
|
|
def _get_cache_path(self, filename: str) -> str:
|
|
return os.path.join(self.cache_dir, filename)
|
|
|
|
async def _cache_export(self, file_bytes: bytes, export_type: str):
|
|
cache_path = self._get_cache_path(f"{export_type}_export.csv")
|
|
with open(cache_path, 'wb') as f:
|
|
f.write(file_bytes)
|
|
|
|
async def _load_cached_export(self, export_type: str) -> Optional[bytes]:
|
|
cache_path = self._get_cache_path(f"{export_type}_export.csv")
|
|
if os.path.exists(cache_path):
|
|
with open(cache_path, 'rb') as f:
|
|
return f.read()
|
|
return None
|
|
|
|
async def process_tcgplayer_export(self, db: Session, file_bytes: bytes, export_type: str = "live", use_cache: bool = False) -> dict:
|
|
"""
|
|
Process a TCGPlayer export file and load it into the inventory table.
|
|
|
|
Args:
|
|
db: Database session
|
|
file_bytes: The downloaded file content as bytes
|
|
export_type: Type of export (staged, live, pricing)
|
|
use_cache: Whether to use cached export file for development
|
|
|
|
Returns:
|
|
dict: Processing statistics
|
|
"""
|
|
stats = {
|
|
"total_rows": 0,
|
|
"processed_rows": 0,
|
|
"errors": 0,
|
|
"error_messages": []
|
|
}
|
|
|
|
try:
|
|
# For development, use cached file if available
|
|
if use_cache:
|
|
cached_bytes = await self._load_cached_export(export_type)
|
|
if cached_bytes:
|
|
file_bytes = cached_bytes
|
|
else:
|
|
await self._cache_export(file_bytes, export_type)
|
|
|
|
# Convert bytes to string and create a file-like object
|
|
file_content = file_bytes.decode('utf-8')
|
|
file_like = io.StringIO(file_content)
|
|
|
|
# Read CSV file
|
|
csv_reader = csv.DictReader(file_like)
|
|
|
|
with transaction(db):
|
|
for row in csv_reader:
|
|
stats["total_rows"] += 1
|
|
try:
|
|
# Process each row and create/update inventory item in database
|
|
inventory_data = self._map_tcgplayer_row_to_inventory(row)
|
|
tcgplayer_id = inventory_data["tcgplayer_id"]
|
|
|
|
# Check if inventory item already exists
|
|
existing_item = self.inventory_service.get_by_tcgplayer_id(db, tcgplayer_id)
|
|
|
|
# Find matching TCGPlayer product
|
|
product_id = int(tcgplayer_id) if tcgplayer_id.isdigit() else None
|
|
if product_id:
|
|
tcg_product = db.query(TCGPlayerProduct).filter(TCGPlayerProduct.product_id == product_id).first()
|
|
if tcg_product:
|
|
# Update inventory data with product information if available
|
|
inventory_data.update({
|
|
"product_name": tcg_product.name,
|
|
"photo_url": tcg_product.image_url,
|
|
"rarity": tcg_product.ext_rarity,
|
|
"number": tcg_product.ext_number
|
|
})
|
|
|
|
if existing_item:
|
|
# Update existing item
|
|
self.inventory_service.update(db, existing_item, inventory_data)
|
|
else:
|
|
# Create new item
|
|
self.inventory_service.create(db, inventory_data)
|
|
|
|
stats["processed_rows"] += 1
|
|
except Exception as e:
|
|
stats["errors"] += 1
|
|
stats["error_messages"].append(f"Error processing row {stats['total_rows']}: {str(e)}")
|
|
|
|
return stats
|
|
|
|
except Exception as e:
|
|
raise Exception(f"Failed to process TCGPlayer export: {str(e)}")
|
|
|
|
def _map_tcgplayer_row_to_inventory(self, row: dict) -> dict:
|
|
"""
|
|
Map TCGPlayer export row to inventory model fields.
|
|
"""
|
|
def safe_float(value: str) -> float:
|
|
"""Convert string to float, returning 0.0 for empty strings or invalid values"""
|
|
try:
|
|
return float(value) if value else 0.0
|
|
except ValueError:
|
|
return 0.0
|
|
|
|
def safe_int(value: str) -> int:
|
|
"""Convert string to int, returning 0 for empty strings or invalid values"""
|
|
try:
|
|
return int(value) if value else 0
|
|
except ValueError:
|
|
return 0
|
|
|
|
return {
|
|
"tcgplayer_id": row.get("TCGplayer Id", ""),
|
|
"product_line": row.get("Product Line", ""),
|
|
"set_name": row.get("Set Name", ""),
|
|
"product_name": row.get("Product Name", ""),
|
|
"title": row.get("Title", ""),
|
|
"number": row.get("Number", ""),
|
|
"rarity": row.get("Rarity", ""),
|
|
"condition": row.get("Condition", ""),
|
|
"tcg_market_price": safe_float(row.get("TCG Market Price", "")),
|
|
"tcg_direct_low": safe_float(row.get("TCG Direct Low", "")),
|
|
"tcg_low_price_with_shipping": safe_float(row.get("TCG Low Price With Shipping", "")),
|
|
"tcg_low_price": safe_float(row.get("TCG Low Price", "")),
|
|
"total_quantity": safe_int(row.get("Total Quantity", "")),
|
|
"add_to_quantity": safe_int(row.get("Add to Quantity", "")),
|
|
"tcg_marketplace_price": safe_float(row.get("TCG Marketplace Price", "")),
|
|
"photo_url": row.get("Photo URL", "")
|
|
} |