This commit is contained in:
zman 2025-04-12 23:51:17 -04:00
parent df6490cab0
commit 56c2d1de47
2870 changed files with 3700 additions and 206 deletions

3
.env.example Normal file
View File

@ -0,0 +1,3 @@
# PostgreSQL connection string
# Format: postgresql://username:password@host:port/database
DATABASE_URL=postgresql://asdf:qwer@192.168.1.1:5432/ai_giga_tcg

6
.gitignore vendored
View File

@ -9,4 +9,8 @@
__pycache__
_cookie.py
app/data/cache/*
app/data/cache
*.log
.env

View File

@ -0,0 +1,96 @@
"""create mtgjson tables
Revision ID: 2025_04_13_create_mtgjson_tables
Revises: 2025_04_09_create_tcgplayer_categories_table
Create Date: 2025-04-13 00:00:00.000000
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '2025_04_13_create_mtgjson_tables'
down_revision = '2025_04_09_create_tcgplayer_categories_table'
branch_labels = None
depends_on = None
def upgrade() -> None:
# Create mtgjson_cards table
op.create_table(
'mtgjson_cards',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('card_id', sa.String(), nullable=False),
sa.Column('name', sa.String(), nullable=True),
sa.Column('set_code', sa.String(), nullable=True),
sa.Column('uuid', sa.String(), nullable=True),
# Identifiers
sa.Column('abu_id', sa.String(), nullable=True),
sa.Column('card_kingdom_etched_id', sa.String(), nullable=True),
sa.Column('card_kingdom_foil_id', sa.String(), nullable=True),
sa.Column('card_kingdom_id', sa.String(), nullable=True),
sa.Column('cardsphere_id', sa.String(), nullable=True),
sa.Column('cardsphere_foil_id', sa.String(), nullable=True),
sa.Column('cardtrader_id', sa.String(), nullable=True),
sa.Column('csi_id', sa.String(), nullable=True),
sa.Column('mcm_id', sa.String(), nullable=True),
sa.Column('mcm_meta_id', sa.String(), nullable=True),
sa.Column('miniaturemarket_id', sa.String(), nullable=True),
sa.Column('mtg_arena_id', sa.String(), nullable=True),
sa.Column('mtgjson_foil_version_id', sa.String(), nullable=True),
sa.Column('mtgjson_non_foil_version_id', sa.String(), nullable=True),
sa.Column('mtgjson_v4_id', sa.String(), nullable=True),
sa.Column('mtgo_foil_id', sa.String(), nullable=True),
sa.Column('mtgo_id', sa.String(), nullable=True),
sa.Column('multiverse_id', sa.String(), nullable=True),
sa.Column('scg_id', sa.String(), nullable=True),
sa.Column('scryfall_id', sa.String(), nullable=True),
sa.Column('scryfall_card_back_id', sa.String(), nullable=True),
sa.Column('scryfall_oracle_id', sa.String(), nullable=True),
sa.Column('scryfall_illustration_id', sa.String(), nullable=True),
sa.Column('tcgplayer_product_id', sa.String(), nullable=True),
sa.Column('tcgplayer_etched_product_id', sa.String(), nullable=True),
sa.Column('tnt_id', sa.String(), nullable=True),
sa.Column('data', sa.JSON(), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_mtgjson_cards_card_id'), 'mtgjson_cards', ['card_id'], unique=True)
op.create_index(op.f('ix_mtgjson_cards_id'), 'mtgjson_cards', ['id'], unique=False)
# Create mtgjson_skus table
op.create_table(
'mtgjson_skus',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('sku_id', sa.String(), nullable=False),
sa.Column('product_id', sa.String(), nullable=True),
sa.Column('condition', sa.String(), nullable=True),
sa.Column('finish', sa.String(), nullable=True),
sa.Column('language', sa.String(), nullable=True),
sa.Column('printing', sa.String(), nullable=True),
sa.Column('card_id', sa.String(), nullable=True),
sa.Column('data', sa.JSON(), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True),
sa.ForeignKeyConstraint(['card_id'], ['mtgjson_cards.card_id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_mtgjson_skus_card_id'), 'mtgjson_skus', ['card_id'], unique=False)
op.create_index(op.f('ix_mtgjson_skus_id'), 'mtgjson_skus', ['id'], unique=False)
op.create_index(op.f('ix_mtgjson_skus_product_id'), 'mtgjson_skus', ['product_id'], unique=False)
op.create_index(op.f('ix_mtgjson_skus_sku_id'), 'mtgjson_skus', ['sku_id'], unique=False)
def downgrade() -> None:
# Drop mtgjson_skus table first due to foreign key constraint
op.drop_index(op.f('ix_mtgjson_skus_sku_id'), table_name='mtgjson_skus')
op.drop_index(op.f('ix_mtgjson_skus_product_id'), table_name='mtgjson_skus')
op.drop_index(op.f('ix_mtgjson_skus_id'), table_name='mtgjson_skus')
op.drop_index(op.f('ix_mtgjson_skus_card_id'), table_name='mtgjson_skus')
op.drop_table('mtgjson_skus')
# Drop mtgjson_cards table
op.drop_index(op.f('ix_mtgjson_cards_id'), table_name='mtgjson_cards')
op.drop_index(op.f('ix_mtgjson_cards_card_id'), table_name='mtgjson_cards')
op.drop_table('mtgjson_cards')

View File

@ -0,0 +1,35 @@
"""remove mtgjson data columns
Revision ID: 2025_04_14_remove_mtgjson_data_columns
Revises: 2025_04_13_create_mtgjson_tables
Create Date: 2025-04-14 00:00:00.000000
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '2025_04_14_remove_mtgjson_data_columns'
down_revision = '2025_04_13_create_mtgjson_tables'
branch_labels = None
depends_on = None
def upgrade() -> None:
# Remove data column from mtgjson_skus table
op.drop_column('mtgjson_skus', 'data')
# Remove data column from mtgjson_cards table
op.drop_column('mtgjson_cards', 'data')
def downgrade() -> None:
# Add data column back to mtgjson_cards table
op.add_column('mtgjson_cards',
sa.Column('data', sa.JSON(), nullable=True)
)
# Add data column back to mtgjson_skus table
op.add_column('mtgjson_skus',
sa.Column('data', sa.JSON(), nullable=True)
)

View File

@ -0,0 +1,26 @@
"""merge heads
Revision ID: 4ad81b486caf
Revises: 2025_04_14_remove_mtgjson_data_columns, 8764850e4e35
Create Date: 2025-04-12 23:38:27.257987
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '4ad81b486caf'
down_revision: Union[str, None] = ('2025_04_14_remove_mtgjson_data_columns', '8764850e4e35')
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass

View File

@ -0,0 +1,26 @@
"""merge heads
Revision ID: 8764850e4e35
Revises: 2025_04_13_create_mtgjson_tables, 2025_04_09_create_tcgplayer_categories_table
Create Date: 2025-04-12 23:16:47.846723
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '8764850e4e35'
down_revision: Union[str, None] = ('2025_04_13_create_mtgjson_tables', '2025_04_09_create_tcgplayer_categories_table')
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass

View File

@ -1,49 +0,0 @@
"""create tcgplayer categories table
Revision ID: create_tcgplayer_categories_table
Revises: remove_product_id_unique_constraint
Create Date: 2025-04-09 23:20:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = 'create_tcgplayer_categories_table'
down_revision: str = 'remove_product_id_unique_constraint'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table('tcgplayer_categories',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('category_id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(), nullable=False),
sa.Column('display_name', sa.String(), nullable=True),
sa.Column('seo_category_name', sa.String(), nullable=True),
sa.Column('category_description', sa.String(), nullable=True),
sa.Column('category_page_title', sa.String(), nullable=True),
sa.Column('sealed_label', sa.String(), nullable=True),
sa.Column('non_sealed_label', sa.String(), nullable=True),
sa.Column('condition_guide_url', sa.String(), nullable=True),
sa.Column('is_scannable', sa.Boolean(), nullable=True, default=False),
sa.Column('popularity', sa.Integer(), nullable=True, default=0),
sa.Column('is_direct', sa.Boolean(), nullable=True, default=False),
sa.Column('modified_on', sa.DateTime(), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=True),
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('category_id')
)
op.create_index('ix_tcgplayer_categories_id', 'tcgplayer_categories', ['id'], unique=False)
op.create_index('ix_tcgplayer_categories_category_id', 'tcgplayer_categories', ['category_id'], unique=True)
def downgrade() -> None:
op.drop_index('ix_tcgplayer_categories_category_id', table_name='tcgplayer_categories')
op.drop_index('ix_tcgplayer_categories_id', table_name='tcgplayer_categories')
op.drop_table('tcgplayer_categories')

22
app.log
View File

@ -1,20 +1,2 @@
2025-04-09 23:51:18,179 - INFO - app.main - Application starting up...
2025-04-09 23:51:18,188 - INFO - app.main - Database initialized successfully
2025-04-09 23:51:22,506 - INFO - app.services.external_api.base_external_service - Making request to https://tcgcsv.com/tcgplayer/1/23167/ProductsAndPrices.csv
2025-04-09 23:51:23,897 - INFO - app.services.external_api.base_external_service - Making request to https://tcgcsv.com/tcgplayer/1/3100/ProductsAndPrices.csv
2025-04-09 23:51:29,067 - INFO - app.services.external_api.base_external_service - Making request to https://tcgcsv.com/tcgplayer/1/2572/ProductsAndPrices.csv
2025-04-09 23:51:37,721 - INFO - app.services.external_api.base_external_service - Making request to https://tcgcsv.com/tcgplayer/1/14/ProductsAndPrices.csv
2025-04-09 23:51:38,716 - INFO - app.services.external_api.base_external_service - Making request to https://tcgcsv.com/tcgplayer/1/29/ProductsAndPrices.csv
2025-04-09 23:51:38,737 - INFO - app.services.external_api.base_external_service - Making request to https://tcgcsv.com/tcgplayer/1/64/ProductsAndPrices.csv
2025-04-09 23:51:44,117 - INFO - app.services.external_api.base_external_service - Making request to https://tcgcsv.com/tcgplayer/3/609/ProductsAndPrices.csv
2025-04-09 23:51:44,140 - INFO - app.services.external_api.base_external_service - Making request to https://tcgcsv.com/tcgplayer/3/610/ProductsAndPrices.csv
2025-04-09 23:51:45,264 - INFO - app.main - TCGPlayer data initialized successfully
2025-04-09 23:51:45,265 - INFO - apscheduler.scheduler - Adding job tentatively -- it will be properly scheduled when the scheduler starts
2025-04-09 23:51:45,265 - INFO - app.services.scheduler.base_scheduler - Scheduled task process_tcgplayer_export to run every 86400 seconds
2025-04-09 23:51:45,265 - INFO - apscheduler.scheduler - Added job "SchedulerService.process_tcgplayer_export" to job store "default"
2025-04-09 23:51:45,265 - INFO - apscheduler.scheduler - Scheduler started
2025-04-09 23:51:45,265 - INFO - app.services.scheduler.base_scheduler - Scheduler started
2025-04-09 23:51:45,265 - INFO - app.services.scheduler.scheduler_service - All scheduled tasks started
2025-04-09 23:51:45,266 - INFO - app.services.scheduler.scheduler_service - Starting scheduled TCGPlayer export processing for live
2025-04-09 23:51:51,510 - INFO - app.services.scheduler.scheduler_service - Completed TCGPlayer export processing: {'total_rows': 5793, 'processed_rows': 5793, 'errors': 0, 'error_messages': []}
2025-04-09 23:51:51,510 - INFO - app.main - Scheduler started successfully
2025-04-12 23:48:13,221 - INFO - app.main - Application starting up...
2025-04-12 23:48:16,568 - INFO - app.main - Database initialized successfully

View File

@ -5,16 +5,27 @@ from sqlalchemy.orm import Session
import logging
import os
from contextlib import contextmanager
from dotenv import load_dotenv
# Load environment variables
load_dotenv()
# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
# Database configuration
DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///database.db")
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@localhost:5432/ai_giga_tcg")
# Create engine with connection pooling and other optimizations for PostgreSQL
engine = create_engine(
DATABASE_URL,
pool_size=5,
max_overflow=10,
pool_timeout=30,
pool_recycle=1800
)
engine = create_engine(DATABASE_URL)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
Base = declarative_base()

View File

@ -8,7 +8,7 @@ from app.routes import routes
from app.db.database import init_db, SessionLocal
from app.services.scheduler.scheduler_service import SchedulerService
from app.services.data_initialization import DataInitializationService
from datetime import datetime
# Configure logging
log_file = "app.log"
if os.path.exists(log_file):
@ -47,7 +47,7 @@ async def lifespan(app: FastAPI):
# Initialize TCGPlayer data
db = SessionLocal()
try:
await data_init_service.initialize_data(db, game_ids=[1, 3]) # 1 = Magic, 3 = Pokemon
await data_init_service.initialize_data(db, game_ids=[1, 3], init_archived_prices=False, archived_prices_start_date="2025-04-01", archived_prices_end_date=datetime.now().strftime("%Y-%m-%d"), init_categories=False, init_groups=False, init_products=False) # 1 = Magic, 3 = Pokemon
logger.info("TCGPlayer data initialized successfully")
except Exception as e:
logger.error(f"Failed to initialize TCGPlayer data: {str(e)}")

View File

@ -0,0 +1,47 @@
from sqlalchemy import Column, Integer, String, DateTime
from sqlalchemy.sql import func
from sqlalchemy.orm import relationship
from app.db.database import Base
class MTGJSONCard(Base):
__tablename__ = "mtgjson_cards"
id = Column(Integer, primary_key=True, index=True)
card_id = Column(String, unique=True, index=True)
name = Column(String)
set_code = Column(String)
uuid = Column(String)
# Identifiers
abu_id = Column(String, nullable=True)
card_kingdom_etched_id = Column(String, nullable=True)
card_kingdom_foil_id = Column(String, nullable=True)
card_kingdom_id = Column(String, nullable=True)
cardsphere_id = Column(String, nullable=True)
cardsphere_foil_id = Column(String, nullable=True)
cardtrader_id = Column(String, nullable=True)
csi_id = Column(String, nullable=True)
mcm_id = Column(String, nullable=True)
mcm_meta_id = Column(String, nullable=True)
miniaturemarket_id = Column(String, nullable=True)
mtg_arena_id = Column(String, nullable=True)
mtgjson_foil_version_id = Column(String, nullable=True)
mtgjson_non_foil_version_id = Column(String, nullable=True)
mtgjson_v4_id = Column(String, nullable=True)
mtgo_foil_id = Column(String, nullable=True)
mtgo_id = Column(String, nullable=True)
multiverse_id = Column(String, nullable=True)
scg_id = Column(String, nullable=True)
scryfall_id = Column(String, nullable=True)
scryfall_card_back_id = Column(String, nullable=True)
scryfall_oracle_id = Column(String, nullable=True)
scryfall_illustration_id = Column(String, nullable=True)
tcgplayer_product_id = Column(String, nullable=True)
tcgplayer_etched_product_id = Column(String, nullable=True)
tnt_id = Column(String, nullable=True)
created_at = Column(DateTime(timezone=True), server_default=func.current_timestamp())
updated_at = Column(DateTime(timezone=True), onupdate=func.current_timestamp())
# Relationships
skus = relationship("MTGJSONSKU", back_populates="card")

21
app/models/mtgjson_sku.py Normal file
View File

@ -0,0 +1,21 @@
from sqlalchemy import Column, Integer, String, DateTime, ForeignKey
from sqlalchemy.sql import func
from sqlalchemy.orm import relationship
from app.db.database import Base
class MTGJSONSKU(Base):
__tablename__ = "mtgjson_skus"
id = Column(Integer, primary_key=True, index=True)
sku_id = Column(String, index=True)
product_id = Column(String, index=True)
condition = Column(String)
finish = Column(String)
language = Column(String)
printing = Column(String)
card_id = Column(String, ForeignKey("mtgjson_cards.card_id"))
created_at = Column(DateTime(timezone=True), server_default=func.current_timestamp())
updated_at = Column(DateTime(timezone=True), onupdate=func.current_timestamp())
# Relationships
card = relationship("MTGJSONCard", back_populates="skus")

View File

@ -1,9 +1,10 @@
import os
import json
from datetime import datetime
from typing import Optional, List
from datetime import datetime, timedelta
from typing import Optional, List, Dict, Any
from sqlalchemy.orm import Session
from app.services.external_api.tcgcsv.tcgcsv_service import TCGCSVService
from app.services.external_api.mtgjson.mtgjson_service import MTGJSONService
from app.models.tcgplayer_group import TCGPlayerGroup
from app.models.tcgplayer_product import TCGPlayerProduct
from app.models.tcgplayer_category import TCGPlayerCategory
@ -11,32 +12,42 @@ from app.models.tcgplayer_category import TCGPlayerCategory
class DataInitializationService:
def __init__(self, cache_dir: str = "app/data/cache/tcgcsv"):
self.cache_dir = cache_dir
self.categories_dir = os.path.join(cache_dir, "categories")
self.groups_dir = os.path.join(cache_dir, "groups")
self.products_dir = os.path.join(cache_dir, "products")
self.tcgcsv_service = TCGCSVService()
self.mtgjson_service = MTGJSONService()
# Create all necessary directories
os.makedirs(cache_dir, exist_ok=True)
os.makedirs(self.categories_dir, exist_ok=True)
os.makedirs(self.groups_dir, exist_ok=True)
os.makedirs(self.products_dir, exist_ok=True)
def _get_cache_path(self, filename: str) -> str:
return os.path.join(self.cache_dir, filename)
def _get_cache_path(self, filename: str, subdir: str) -> str:
"""Get the full path for a cached file in the specified subdirectory"""
return os.path.join(self.cache_dir, subdir, filename)
async def _cache_categories(self, categories_data: dict):
"""Cache categories data to a JSON file"""
cache_path = self._get_cache_path("categories.json")
cache_path = self._get_cache_path("categories.json", "categories")
with open(cache_path, 'w') as f:
json.dump(categories_data, f, indent=2)
async def _cache_groups(self, game_ids: List[int], groups_data: dict):
for game_id in game_ids:
cache_path = self._get_cache_path(f"groups_{game_id}.json")
cache_path = self._get_cache_path(f"groups_{game_id}.json", "groups")
with open(cache_path, 'w') as f:
json.dump(groups_data, f, default=str)
async def _cache_products(self, game_ids: List[int], group_id: int, products_data: list):
for game_id in game_ids:
cache_path = self._get_cache_path(f"products_{game_id}_{group_id}.json")
cache_path = self._get_cache_path(f"products_{game_id}_{group_id}.json", "products")
with open(cache_path, 'w') as f:
json.dump(products_data, f, default=str)
async def _load_cached_categories(self) -> Optional[dict]:
cache_path = self._get_cache_path("categories.json")
cache_path = self._get_cache_path("categories.json", "categories")
if os.path.exists(cache_path):
with open(cache_path, 'r') as f:
return json.load(f)
@ -45,7 +56,7 @@ class DataInitializationService:
async def _load_cached_groups(self, game_ids: List[int]) -> Optional[dict]:
# Try to load cached data for any of the game IDs
for game_id in game_ids:
cache_path = self._get_cache_path(f"groups_{game_id}.json")
cache_path = self._get_cache_path(f"groups_{game_id}.json", "groups")
if os.path.exists(cache_path):
with open(cache_path, 'r') as f:
return json.load(f)
@ -54,147 +65,199 @@ class DataInitializationService:
async def _load_cached_products(self, game_ids: List[int], group_id: int) -> Optional[list]:
# Try to load cached data for any of the game IDs
for game_id in game_ids:
cache_path = self._get_cache_path(f"products_{game_id}_{group_id}.json")
cache_path = self._get_cache_path(f"products_{game_id}_{group_id}.json", "products")
if os.path.exists(cache_path):
with open(cache_path, 'r') as f:
return json.load(f)
return None
async def initialize_data(self, db: Session, game_ids: List[int], use_cache: bool = True) -> None:
"""Initialize TCGPlayer data, using cache if available and requested"""
async def initialize_data(
self,
db: Session,
game_ids: List[int],
use_cache: bool = True,
init_categories: bool = True,
init_groups: bool = True,
init_products: bool = True,
init_archived_prices: bool = False,
archived_prices_start_date: Optional[str] = None,
archived_prices_end_date: Optional[str] = None,
init_mtgjson: bool = True
) -> Dict[str, Any]:
"""Initialize TCGPlayer data with configurable steps"""
print("Initializing TCGPlayer data...")
results = {
"categories": 0,
"groups": {},
"products": {},
"archived_prices": False,
"mtgjson": {}
}
# Handle categories
categories_data = None
if use_cache:
categories_data = await self._load_cached_categories()
if not categories_data:
print("Fetching categories from API...")
categories_data = await self.tcgcsv_service.get_categories()
if init_categories:
print("\nInitializing categories...")
categories_data = None
if use_cache:
await self._cache_categories(categories_data)
categories_data = await self._load_cached_categories()
if not categories_data.get("success"):
raise Exception(f"Failed to fetch categories: {categories_data.get('errors')}")
if not categories_data:
print("Fetching categories from API...")
categories_data = await self.tcgcsv_service.get_categories()
if use_cache:
await self._cache_categories(categories_data)
# Sync categories to database
categories = categories_data.get("results", [])
synced_categories = []
for category_data in categories:
existing_category = db.query(TCGPlayerCategory).filter(TCGPlayerCategory.category_id == category_data["categoryId"]).first()
if existing_category:
synced_categories.append(existing_category)
else:
new_category = TCGPlayerCategory(
category_id=category_data["categoryId"],
name=category_data["name"],
display_name=category_data.get("displayName"),
seo_category_name=category_data.get("seoCategoryName"),
category_description=category_data.get("categoryDescription"),
category_page_title=category_data.get("categoryPageTitle"),
sealed_label=category_data.get("sealedLabel"),
non_sealed_label=category_data.get("nonSealedLabel"),
condition_guide_url=category_data.get("conditionGuideUrl"),
is_scannable=category_data.get("isScannable", False),
popularity=category_data.get("popularity", 0),
is_direct=category_data.get("isDirect", False),
modified_on=datetime.fromisoformat(category_data["modifiedOn"].replace("Z", "+00:00")) if category_data.get("modifiedOn") else None
)
db.add(new_category)
synced_categories.append(new_category)
db.commit()
print(f"Synced {len(synced_categories)} categories")
if not categories_data.get("success"):
raise Exception(f"Failed to fetch categories: {categories_data.get('errors')}")
# Sync categories to database
categories = categories_data.get("results", [])
synced_categories = []
for category_data in categories:
existing_category = db.query(TCGPlayerCategory).filter(TCGPlayerCategory.category_id == category_data["categoryId"]).first()
if existing_category:
synced_categories.append(existing_category)
else:
new_category = TCGPlayerCategory(
category_id=category_data["categoryId"],
name=category_data["name"],
display_name=category_data.get("displayName"),
seo_category_name=category_data.get("seoCategoryName"),
category_description=category_data.get("categoryDescription"),
category_page_title=category_data.get("categoryPageTitle"),
sealed_label=category_data.get("sealedLabel"),
non_sealed_label=category_data.get("nonSealedLabel"),
condition_guide_url=category_data.get("conditionGuideUrl"),
is_scannable=category_data.get("isScannable", False),
popularity=category_data.get("popularity", 0),
is_direct=category_data.get("isDirect", False),
modified_on=datetime.fromisoformat(category_data["modifiedOn"].replace("Z", "+00:00")) if category_data.get("modifiedOn") else None
)
db.add(new_category)
synced_categories.append(new_category)
db.commit()
results["categories"] = len(synced_categories)
print(f"Synced {len(synced_categories)} categories")
# Process each game ID separately
for game_id in game_ids:
print(f"\nProcessing game ID: {game_id}")
results["groups"][game_id] = 0
results["products"][game_id] = {}
# Handle groups for this game ID
groups_data = None
if use_cache:
groups_data = await self._load_cached_groups([game_id])
if not groups_data:
print(f"Fetching groups for game ID {game_id} from API...")
groups_data = await self.tcgcsv_service.get_groups([game_id])
if init_groups:
print(f"Initializing groups for game ID {game_id}...")
groups_data = None
if use_cache:
await self._cache_groups([game_id], groups_data)
groups_data = await self._load_cached_groups([game_id])
if not groups_data.get("success"):
raise Exception(f"Failed to fetch groups for game ID {game_id}: {groups_data.get('errors')}")
# Sync groups to database
groups = groups_data.get("results", [])
synced_groups = []
for group_data in groups:
existing_group = db.query(TCGPlayerGroup).filter(TCGPlayerGroup.group_id == group_data["groupId"]).first()
if existing_group:
synced_groups.append(existing_group)
else:
new_group = TCGPlayerGroup(
group_id=group_data["groupId"],
name=group_data["name"],
abbreviation=group_data.get("abbreviation"),
is_supplemental=group_data.get("isSupplemental", False),
published_on=datetime.fromisoformat(group_data["publishedOn"].replace("Z", "+00:00")) if group_data.get("publishedOn") else None,
modified_on=datetime.fromisoformat(group_data["modifiedOn"].replace("Z", "+00:00")) if group_data.get("modifiedOn") else None,
category_id=group_data.get("categoryId")
)
db.add(new_group)
synced_groups.append(new_group)
db.commit()
print(f"Synced {len(synced_groups)} groups for game ID {game_id}")
# Handle products for each group in this game ID
for group in synced_groups:
products_data = None
if use_cache:
products_data = await self._load_cached_products([game_id], group.group_id)
if not products_data:
print(f"Fetching products for group {group.name} (game ID {game_id}) from API...")
products_data = await self.tcgcsv_service.get_products_and_prices([game_id], group.group_id)
if not groups_data:
print(f"Fetching groups for game ID {game_id} from API...")
groups_data = await self.tcgcsv_service.get_groups([game_id])
if use_cache:
await self._cache_products([game_id], group.group_id, products_data)
await self._cache_groups([game_id], groups_data)
# Sync products to database
synced_products = []
for product_data in products_data:
existing_product = db.query(TCGPlayerProduct).filter(TCGPlayerProduct.product_id == int(product_data["productId"])).first()
if existing_product:
synced_products.append(existing_product)
if not groups_data.get("success"):
raise Exception(f"Failed to fetch groups for game ID {game_id}: {groups_data.get('errors')}")
# Sync groups to database
groups = groups_data.get("results", [])
synced_groups = []
for group_data in groups:
existing_group = db.query(TCGPlayerGroup).filter(TCGPlayerGroup.group_id == group_data["groupId"]).first()
if existing_group:
synced_groups.append(existing_group)
else:
new_product = TCGPlayerProduct(
product_id=int(product_data["productId"]),
name=product_data["name"],
clean_name=product_data.get("cleanName"),
image_url=product_data.get("imageUrl"),
category_id=int(product_data["categoryId"]),
group_id=int(product_data["groupId"]),
url=product_data.get("url"),
modified_on=datetime.fromisoformat(product_data["modifiedOn"].replace("Z", "+00:00")) if product_data.get("modifiedOn") else None,
image_count=int(product_data.get("imageCount", 0)),
ext_rarity=product_data.get("extRarity"),
ext_number=product_data.get("extNumber"),
low_price=float(product_data.get("lowPrice")) if product_data.get("lowPrice") else None,
mid_price=float(product_data.get("midPrice")) if product_data.get("midPrice") else None,
high_price=float(product_data.get("highPrice")) if product_data.get("highPrice") else None,
market_price=float(product_data.get("marketPrice")) if product_data.get("marketPrice") else None,
direct_low_price=float(product_data.get("directLowPrice")) if product_data.get("directLowPrice") else None,
sub_type_name=product_data.get("subTypeName")
new_group = TCGPlayerGroup(
group_id=group_data["groupId"],
name=group_data["name"],
abbreviation=group_data.get("abbreviation"),
is_supplemental=group_data.get("isSupplemental", False),
published_on=datetime.fromisoformat(group_data["publishedOn"].replace("Z", "+00:00")) if group_data.get("publishedOn") else None,
modified_on=datetime.fromisoformat(group_data["modifiedOn"].replace("Z", "+00:00")) if group_data.get("modifiedOn") else None,
category_id=group_data.get("categoryId")
)
db.add(new_product)
synced_products.append(new_product)
db.add(new_group)
synced_groups.append(new_group)
db.commit()
print(f"Synced {len(synced_products)} products for group {group.name} (game ID {game_id})")
results["groups"][game_id] = len(synced_groups)
print(f"Synced {len(synced_groups)} groups for game ID {game_id}")
if init_products:
# Handle products for each group in this game ID
for group in synced_groups:
print(f"Initializing products for group {group.name} (game ID {game_id})...")
products_data = None
if use_cache:
products_data = await self._load_cached_products([game_id], group.group_id)
if not products_data:
print(f"Fetching products for group {group.name} (game ID {game_id}) from API...")
products_data = await self.tcgcsv_service.get_products_and_prices([game_id], group.group_id)
if use_cache:
await self._cache_products([game_id], group.group_id, products_data)
# Sync products to database
synced_products = []
for product_data in products_data:
existing_product = db.query(TCGPlayerProduct).filter(TCGPlayerProduct.product_id == int(product_data["productId"])).first()
if existing_product:
synced_products.append(existing_product)
else:
new_product = TCGPlayerProduct(
product_id=int(product_data["productId"]),
name=product_data["name"],
clean_name=product_data.get("cleanName"),
image_url=product_data.get("imageUrl"),
category_id=int(product_data["categoryId"]),
group_id=int(product_data["groupId"]),
url=product_data.get("url"),
modified_on=datetime.fromisoformat(product_data["modifiedOn"].replace("Z", "+00:00")) if product_data.get("modifiedOn") else None,
image_count=int(product_data.get("imageCount", 0)),
ext_rarity=product_data.get("extRarity"),
ext_number=product_data.get("extNumber"),
low_price=float(product_data.get("lowPrice")) if product_data.get("lowPrice") else None,
mid_price=float(product_data.get("midPrice")) if product_data.get("midPrice") else None,
high_price=float(product_data.get("highPrice")) if product_data.get("highPrice") else None,
market_price=float(product_data.get("marketPrice")) if product_data.get("marketPrice") else None,
direct_low_price=float(product_data.get("directLowPrice")) if product_data.get("directLowPrice") else None,
sub_type_name=product_data.get("subTypeName")
)
db.add(new_product)
synced_products.append(new_product)
db.commit()
results["products"][game_id][group.group_id] = len(synced_products)
print(f"Synced {len(synced_products)} products for group {group.name} (game ID {game_id})")
if init_archived_prices:
if not archived_prices_start_date or not archived_prices_end_date:
raise ValueError("Both start_date and end_date are required for archived prices initialization")
print(f"\nInitializing archived prices from {archived_prices_start_date} to {archived_prices_end_date}...")
await self.tcgcsv_service.get_archived_prices_for_date_range(archived_prices_start_date, archived_prices_end_date)
results["archived_prices"] = True
print("Archived prices initialization completed")
if init_mtgjson:
print("\nInitializing MTGJSON data...")
identifiers_result = await self.mtgjson_service.download_and_process_identifiers(db)
skus_result = await self.mtgjson_service.download_and_process_skus(db)
results["mtgjson"] = {
"cards_processed": identifiers_result["cards_processed"],
"skus_processed": skus_result["skus_processed"]
}
return results
async def clear_cache(self) -> None:
"""Clear all cached data"""
for filename in os.listdir(self.cache_dir):
file_path = os.path.join(self.cache_dir, filename)
if os.path.isfile(file_path):
os.unlink(file_path)
for subdir in ["categories", "groups", "products"]:
dir_path = os.path.join(self.cache_dir, subdir)
if os.path.exists(dir_path):
for filename in os.listdir(dir_path):
file_path = os.path.join(dir_path, filename)
if os.path.isfile(file_path):
os.unlink(file_path)
await self.mtgjson_service.clear_cache()
print("Cache cleared")
async def close(self):

View File

@ -26,8 +26,9 @@ class BaseExternalService:
params: Optional[Dict[str, Any]] = None,
headers: Optional[Dict[str, str]] = None,
data: Optional[Dict[str, Any]] = None,
content_type: str = "application/json"
) -> Union[Dict[str, Any], str]:
content_type: str = "application/json",
binary: bool = False
) -> Union[Dict[str, Any], str, bytes]:
session = await self._get_session()
url = f"{self.base_url}{endpoint}"
@ -43,6 +44,9 @@ class BaseExternalService:
response_content_type = response.headers.get('content-type', '').lower()
logger.info(f"Making request to {url}")
if binary:
return await response.read()
# Get the raw response text first
raw_response = await response.text()

View File

@ -0,0 +1,312 @@
import os
import json
import zipfile
import aiohttp
import asyncio
import time
import sys
from typing import Dict, Any, Optional, Generator
from sqlalchemy.orm import Session
from datetime import datetime
from app.models.mtgjson_card import MTGJSONCard
from app.models.mtgjson_sku import MTGJSONSKU
class MTGJSONService:
def __init__(self, cache_dir: str = "app/data/cache/mtgjson", batch_size: int = 1000):
self.cache_dir = cache_dir
self.identifiers_dir = os.path.join(cache_dir, "identifiers")
self.skus_dir = os.path.join(cache_dir, "skus")
self.batch_size = batch_size
# Create necessary directories
os.makedirs(cache_dir, exist_ok=True)
os.makedirs(self.identifiers_dir, exist_ok=True)
os.makedirs(self.skus_dir, exist_ok=True)
def _format_progress(self, current: int, total: int, start_time: float) -> str:
"""Format a progress message with percentage and timing information"""
elapsed = time.time() - start_time
if total > 0:
percent = (current / total) * 100
items_per_second = current / elapsed if elapsed > 0 else 0
eta = (total - current) / items_per_second if items_per_second > 0 else 0
return f"[{current}/{total} ({percent:.1f}%)] {items_per_second:.1f} items/sec, ETA: {eta:.1f}s"
return f"[{current} items] {current/elapsed:.1f} items/sec"
def _print_progress(self, message: str, end: str = "\n") -> None:
"""Print progress message with flush"""
print(message, end=end, flush=True)
async def _download_file(self, url: str, output_path: str) -> None:
"""Download a file from the given URL to the specified path using streaming"""
print(f"Downloading {url}...")
start_time = time.time()
total_size = 0
async with aiohttp.ClientSession() as session:
async with session.get(url) as response:
if response.status == 200:
total_size = int(response.headers.get('content-length', 0))
with open(output_path, 'wb') as f:
downloaded = 0
async for chunk in response.content.iter_chunked(8192):
f.write(chunk)
downloaded += len(chunk)
if total_size > 0:
percent = (downloaded / total_size) * 100
elapsed = time.time() - start_time
speed = downloaded / elapsed / 1024 / 1024 # MB/s
print(f"\rDownloading: {percent:.1f}% ({downloaded/1024/1024:.1f}MB/{total_size/1024/1024:.1f}MB) at {speed:.1f}MB/s", end="")
print("\nDownload complete!")
else:
raise Exception(f"Failed to download file from {url}. Status: {response.status}")
async def _unzip_file(self, zip_path: str, extract_dir: str) -> str:
"""Unzip a file to the specified directory and return the path to the extracted JSON file"""
with zipfile.ZipFile(zip_path, 'r') as zip_ref:
json_filename = zip_ref.namelist()[0]
zip_ref.extractall(extract_dir)
return os.path.join(extract_dir, json_filename)
def _stream_json_file(self, file_path: str) -> Generator[Dict[str, Any], None, None]:
"""Stream a JSON file and yield items one at a time"""
print(f"Starting to stream JSON file: {file_path}")
with open(file_path, 'r') as f:
# Load the entire file since MTGJSON uses a specific format
data = json.load(f)
# First yield the meta data
if "meta" in data:
yield {"type": "meta", "data": data["meta"]}
# Then yield each item in the data section
if "data" in data:
for key, value in data["data"].items():
yield {"type": "item", "data": {key: value}}
async def _process_batch(self, db: Session, items: list, model_class, commit: bool = True) -> int:
"""Process a batch of items and add them to the database"""
processed = 0
for item in items:
if model_class == MTGJSONCard:
# Check if card already exists
existing_card = db.query(MTGJSONCard).filter(MTGJSONCard.card_id == item["card_id"]).first()
if existing_card:
continue
new_item = MTGJSONCard(
card_id=item["card_id"],
name=item["name"],
set_code=item["set_code"],
uuid=item["uuid"],
abu_id=item.get("abu_id"),
card_kingdom_etched_id=item.get("card_kingdom_etched_id"),
card_kingdom_foil_id=item.get("card_kingdom_foil_id"),
card_kingdom_id=item.get("card_kingdom_id"),
cardsphere_id=item.get("cardsphere_id"),
cardsphere_foil_id=item.get("cardsphere_foil_id"),
cardtrader_id=item.get("cardtrader_id"),
csi_id=item.get("csi_id"),
mcm_id=item.get("mcm_id"),
mcm_meta_id=item.get("mcm_meta_id"),
miniaturemarket_id=item.get("miniaturemarket_id"),
mtg_arena_id=item.get("mtg_arena_id"),
mtgjson_foil_version_id=item.get("mtgjson_foil_version_id"),
mtgjson_non_foil_version_id=item.get("mtgjson_non_foil_version_id"),
mtgjson_v4_id=item.get("mtgjson_v4_id"),
mtgo_foil_id=item.get("mtgo_foil_id"),
mtgo_id=item.get("mtgo_id"),
multiverse_id=item.get("multiverse_id"),
scg_id=item.get("scg_id"),
scryfall_id=item.get("scryfall_id"),
scryfall_card_back_id=item.get("scryfall_card_back_id"),
scryfall_oracle_id=item.get("scryfall_oracle_id"),
scryfall_illustration_id=item.get("scryfall_illustration_id"),
tcgplayer_product_id=item.get("tcgplayer_product_id"),
tcgplayer_etched_product_id=item.get("tcgplayer_etched_product_id"),
tnt_id=item.get("tnt_id")
)
else: # MTGJSONSKU
# Check if SKU already exists
existing_sku = db.query(MTGJSONSKU).filter(MTGJSONSKU.sku_id == item["sku_id"]).first()
if existing_sku:
continue
new_item = MTGJSONSKU(
sku_id=str(item["sku_id"]),
product_id=str(item["product_id"]),
condition=item["condition"],
finish=item["finish"],
language=item["language"],
printing=item["printing"],
card_id=item["card_id"]
)
db.add(new_item)
processed += 1
if commit:
try:
db.commit()
except Exception as e:
db.rollback()
raise e
return processed
async def download_and_process_identifiers(self, db: Session) -> Dict[str, int]:
"""Download, unzip and process AllIdentifiers.json.zip using streaming"""
self._print_progress("Starting MTGJSON identifiers processing...")
start_time = time.time()
zip_path = os.path.join(self.identifiers_dir, "AllIdentifiers.json.zip")
await self._download_file(
"https://mtgjson.com/api/v5/AllIdentifiers.json.zip",
zip_path
)
self._print_progress("Unzipping file...")
json_path = await self._unzip_file(zip_path, self.identifiers_dir)
cards_processed = 0
current_batch = []
total_cards = 0
last_progress_time = time.time()
self._print_progress("Processing cards...")
try:
for item in self._stream_json_file(json_path):
if item["type"] == "meta":
self._print_progress(f"Processing MTGJSON data version {item['data'].get('version')} from {item['data'].get('date')}")
continue
card_data = item["data"]
card_id = list(card_data.keys())[0]
card_info = card_data[card_id]
total_cards += 1
current_batch.append({
"card_id": card_id,
"name": card_info.get("name"),
"set_code": card_info.get("setCode"),
"uuid": card_info.get("uuid"),
"abu_id": card_info.get("identifiers", {}).get("abuId"),
"card_kingdom_etched_id": card_info.get("identifiers", {}).get("cardKingdomEtchedId"),
"card_kingdom_foil_id": card_info.get("identifiers", {}).get("cardKingdomFoilId"),
"card_kingdom_id": card_info.get("identifiers", {}).get("cardKingdomId"),
"cardsphere_id": card_info.get("identifiers", {}).get("cardsphereId"),
"cardsphere_foil_id": card_info.get("identifiers", {}).get("cardsphereFoilId"),
"cardtrader_id": card_info.get("identifiers", {}).get("cardtraderId"),
"csi_id": card_info.get("identifiers", {}).get("csiId"),
"mcm_id": card_info.get("identifiers", {}).get("mcmId"),
"mcm_meta_id": card_info.get("identifiers", {}).get("mcmMetaId"),
"miniaturemarket_id": card_info.get("identifiers", {}).get("miniaturemarketId"),
"mtg_arena_id": card_info.get("identifiers", {}).get("mtgArenaId"),
"mtgjson_foil_version_id": card_info.get("identifiers", {}).get("mtgjsonFoilVersionId"),
"mtgjson_non_foil_version_id": card_info.get("identifiers", {}).get("mtgjsonNonFoilVersionId"),
"mtgjson_v4_id": card_info.get("identifiers", {}).get("mtgjsonV4Id"),
"mtgo_foil_id": card_info.get("identifiers", {}).get("mtgoFoilId"),
"mtgo_id": card_info.get("identifiers", {}).get("mtgoId"),
"multiverse_id": card_info.get("identifiers", {}).get("multiverseId"),
"scg_id": card_info.get("identifiers", {}).get("scgId"),
"scryfall_id": card_info.get("identifiers", {}).get("scryfallId"),
"scryfall_card_back_id": card_info.get("identifiers", {}).get("scryfallCardBackId"),
"scryfall_oracle_id": card_info.get("identifiers", {}).get("scryfallOracleId"),
"scryfall_illustration_id": card_info.get("identifiers", {}).get("scryfallIllustrationId"),
"tcgplayer_product_id": card_info.get("identifiers", {}).get("tcgplayerProductId"),
"tcgplayer_etched_product_id": card_info.get("identifiers", {}).get("tcgplayerEtchedProductId"),
"tnt_id": card_info.get("identifiers", {}).get("tntId"),
"data": card_info
})
if len(current_batch) >= self.batch_size:
batch_processed = await self._process_batch(db, current_batch, MTGJSONCard)
cards_processed += batch_processed
current_batch = []
current_time = time.time()
if current_time - last_progress_time >= 1.0: # Update progress every second
self._print_progress(f"\r{self._format_progress(cards_processed, total_cards, start_time)}", end="")
last_progress_time = current_time
except Exception as e:
self._print_progress(f"\nError during processing: {str(e)}")
raise
# Process remaining items
if current_batch:
batch_processed = await self._process_batch(db, current_batch, MTGJSONCard)
cards_processed += batch_processed
total_time = time.time() - start_time
self._print_progress(f"\nProcessing complete! Processed {cards_processed} cards in {total_time:.1f} seconds")
return {"cards_processed": cards_processed}
async def download_and_process_skus(self, db: Session) -> Dict[str, int]:
"""Download, unzip and process TcgplayerSkus.json.zip using streaming"""
self._print_progress("Starting MTGJSON SKUs processing...")
start_time = time.time()
zip_path = os.path.join(self.skus_dir, "TcgplayerSkus.json.zip")
await self._download_file(
"https://mtgjson.com/api/v5/TcgplayerSkus.json.zip",
zip_path
)
self._print_progress("Unzipping file...")
json_path = await self._unzip_file(zip_path, self.skus_dir)
skus_processed = 0
current_batch = []
total_skus = 0
last_progress_time = time.time()
self._print_progress("Processing SKUs...")
try:
for item in self._stream_json_file(json_path):
if item["type"] == "meta":
self._print_progress(f"Processing MTGJSON SKUs version {item['data'].get('version')} from {item['data'].get('date')}")
continue
# The data structure is {card_uuid: [sku1, sku2, ...]}
for card_uuid, sku_list in item["data"].items():
for sku in sku_list:
total_skus += 1
current_batch.append({
"sku_id": str(sku.get("skuId")),
"product_id": str(sku.get("productId")),
"condition": sku.get("condition"),
"finish": sku.get("finish", "NORMAL"), # Default to NORMAL if not specified
"language": sku.get("language"),
"printing": sku.get("printing"),
"card_id": card_uuid,
"data": sku
})
if len(current_batch) >= self.batch_size:
batch_processed = await self._process_batch(db, current_batch, MTGJSONSKU)
skus_processed += batch_processed
current_batch = []
current_time = time.time()
if current_time - last_progress_time >= 1.0: # Update progress every second
self._print_progress(f"\r{self._format_progress(skus_processed, total_skus, start_time)}", end="")
last_progress_time = current_time
except Exception as e:
self._print_progress(f"\nError during processing: {str(e)}")
raise
# Process remaining items
if current_batch:
batch_processed = await self._process_batch(db, current_batch, MTGJSONSKU)
skus_processed += batch_processed
total_time = time.time() - start_time
self._print_progress(f"\nProcessing complete! Processed {skus_processed} SKUs in {total_time:.1f} seconds")
return {"skus_processed": skus_processed}
async def clear_cache(self) -> None:
"""Clear all cached data"""
for subdir in ["identifiers", "skus"]:
dir_path = os.path.join(self.cache_dir, subdir)
if os.path.exists(dir_path):
for filename in os.listdir(dir_path):
file_path = os.path.join(dir_path, filename)
if os.path.isfile(file_path):
os.unlink(file_path)
print("MTGJSON cache cleared")

View File

@ -1,5 +1,5 @@
from typing import List, Dict, Any
from datetime import datetime
from datetime import datetime, timedelta
import csv
import io
from app.services.external_api.base_external_service import BaseExternalService
@ -7,21 +7,23 @@ from app.models.tcgplayer_group import TCGPlayerGroup
from app.models.tcgplayer_product import TCGPlayerProduct
from app.models.tcgplayer_category import TCGPlayerCategory
from sqlalchemy.orm import Session
import py7zr
import os
class TCGCSVService(BaseExternalService):
def __init__(self):
super().__init__(base_url="https://tcgcsv.com/tcgplayer/")
super().__init__(base_url="https://tcgcsv.com/")
async def get_groups(self, game_ids: List[int]) -> Dict[str, Any]:
"""Fetch groups for specific game IDs from TCGCSV API"""
game_ids_str = ",".join(map(str, game_ids))
endpoint = f"{game_ids_str}/groups"
endpoint = f"tcgplayer/{game_ids_str}/groups"
return await self._make_request("GET", endpoint)
async def get_products_and_prices(self, game_ids: List[int], group_id: int) -> List[Dict[str, Any]]:
"""Fetch products and prices for a specific group from TCGCSV API"""
game_ids_str = ",".join(map(str, game_ids))
endpoint = f"{game_ids_str}/{group_id}/ProductsAndPrices.csv"
endpoint = f"tcgplayer/{game_ids_str}/{group_id}/ProductsAndPrices.csv"
response = await self._make_request("GET", endpoint, headers={"Accept": "text/csv"})
# Parse CSV response
@ -31,8 +33,63 @@ class TCGCSVService(BaseExternalService):
async def get_categories(self) -> Dict[str, Any]:
"""Fetch all categories from TCGCSV API"""
endpoint = "categories"
endpoint = "tcgplayer/categories"
return await self._make_request("GET", endpoint)
async def get_archived_prices_for_date(self, date_str: str):
"""Fetch archived prices from TCGCSV API"""
# Check if the date directory already exists
extract_path = f"app/data/cache/tcgcsv/prices/{date_str}"
if os.path.exists(extract_path):
print(f"Prices for date {date_str} already exist, skipping download")
return date_str
# Download the archive file
endpoint = f"archive/tcgplayer/prices-{date_str}.ppmd.7z"
response = await self._make_request("GET", endpoint, binary=True)
# Save the archive file
archive_path = f"app/data/cache/tcgcsv/prices/zip/prices-{date_str}.ppmd.7z"
os.makedirs(os.path.dirname(archive_path), exist_ok=True)
with open(archive_path, "wb") as f:
f.write(response)
# Extract the 7z file
with py7zr.SevenZipFile(archive_path, 'r') as archive:
# Extract to a directory named after the date
os.makedirs(extract_path, exist_ok=True)
archive.extractall(path=extract_path)
# The extracted files will be in a directory structure like:
# {date_str}/{game_id}/{group_id}/prices
return date_str
async def get_archived_prices_for_date_range(self, start_date: str, end_date: str):
"""Fetch archived prices for a date range from TCGCSV API"""
# Convert string dates to datetime objects
start_dt = datetime.strptime(start_date, "%Y-%m-%d")
end_dt = datetime.strptime(end_date, "%Y-%m-%d")
# Set minimum start date
min_start_date = datetime.strptime("2025-02-08", "%Y-%m-%d")
if start_dt < min_start_date:
start_dt = min_start_date
# Set maximum end date to today
today = datetime.now()
if end_dt > today:
end_dt = today
# Generate date range
date_range = []
current_dt = start_dt
while current_dt <= end_dt:
date_range.append(current_dt.strftime("%Y-%m-%d"))
current_dt += timedelta(days=1)
# Process each date
for date_str in date_range:
await self.get_archived_prices_for_date(date_str)
async def sync_groups_to_db(self, db: Session, game_ids: List[int]) -> List[TCGPlayerGroup]:
"""Fetch groups from API and sync them to the database"""

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1 @@
{"success": true, "errors": [], "results": [{"productId": 38444, "lowPrice": 154.95, "midPrice": 223.55, "highPrice": 275.98, "marketPrice": 218.59, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 38445, "lowPrice": 349.49, "midPrice": 374.75, "highPrice": 400.0, "marketPrice": 385.0, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 38446, "lowPrice": 97.71, "midPrice": 120.99, "highPrice": 255.61, "marketPrice": 258.98, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 38447, "lowPrice": 80.0, "midPrice": 109.99, "highPrice": 214.45, "marketPrice": 99.92, "directLowPrice": 94.98, "subTypeName": "Foil"}, {"productId": 57653, "lowPrice": 71.99, "midPrice": 75.99, "highPrice": 100.0, "marketPrice": 69.44, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 67401, "lowPrice": 28.01, "midPrice": 78.72, "highPrice": 119.99, "marketPrice": 33.73, "directLowPrice": 114.99, "subTypeName": "Foil"}, {"productId": 71898, "lowPrice": 167.25, "midPrice": 183.34, "highPrice": 223.35, "marketPrice": 167.25, "directLowPrice": 168.28, "subTypeName": "Foil"}, {"productId": 78237, "lowPrice": 36.0, "midPrice": 89.24, "highPrice": 159.91, "marketPrice": 47.42, "directLowPrice": 59.99, "subTypeName": "Foil"}, {"productId": 95046, "lowPrice": 43.0, "midPrice": 62.92, "highPrice": 97.57, "marketPrice": 64.98, "directLowPrice": 44.0, "subTypeName": "Foil"}, {"productId": 110267, "lowPrice": 23.75, "midPrice": 31.21, "highPrice": 48.4, "marketPrice": 27.89, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 125324, "lowPrice": 18.0, "midPrice": 25.99, "highPrice": 53.45, "marketPrice": 25.86, "directLowPrice": 28.58, "subTypeName": "Foil"}, {"productId": 154792, "lowPrice": 16.73, "midPrice": 25.43, "highPrice": 49.0, "marketPrice": 17.47, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 181788, "lowPrice": 23.57, "midPrice": 27.83, "highPrice": 51.0, "marketPrice": 23.83, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 205180, "lowPrice": 16.99, "midPrice": 29.99, "highPrice": 70.2, "marketPrice": 18.3, "directLowPrice": 70.58, "subTypeName": "Foil"}, {"productId": 228752, "lowPrice": 24.99, "midPrice": 29.98, "highPrice": 224.99, "marketPrice": 29.43, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 257182, "lowPrice": 30.6, "midPrice": 75.0, "highPrice": 141.0, "marketPrice": 30.6, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 455848, "lowPrice": 15.0, "midPrice": 27.98, "highPrice": 134.0, "marketPrice": 20.73, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 527738, "lowPrice": 14.69, "midPrice": 19.31, "highPrice": 80.0, "marketPrice": 16.28, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 600517, "lowPrice": 47.97, "midPrice": 61.22, "highPrice": 75.0, "marketPrice": 61.22, "directLowPrice": null, "subTypeName": "Foil"}]}

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1 @@
{"success": true, "errors": [], "results": [{"productId": 21705, "lowPrice": 0.18, "midPrice": 0.57, "highPrice": 29.99, "marketPrice": 0.56, "directLowPrice": 0.43, "subTypeName": "Foil"}, {"productId": 37765, "lowPrice": 0.1, "midPrice": 0.25, "highPrice": 1.6, "marketPrice": 0.14, "directLowPrice": 0.18, "subTypeName": "Normal"}, {"productId": 37767, "lowPrice": 0.18, "midPrice": 0.41, "highPrice": 4.99, "marketPrice": 0.25, "directLowPrice": 0.6, "subTypeName": "Normal"}, {"productId": 37772, "lowPrice": 0.14, "midPrice": 0.39, "highPrice": 4.99, "marketPrice": 0.15, "directLowPrice": 0.17, "subTypeName": "Normal"}, {"productId": 37773, "lowPrice": 0.05, "midPrice": 0.24, "highPrice": 3.99, "marketPrice": 0.12, "directLowPrice": 0.07, "subTypeName": "Normal"}, {"productId": 37778, "lowPrice": 0.09, "midPrice": 0.25, "highPrice": 3.13, "marketPrice": 0.22, "directLowPrice": 0.18, "subTypeName": "Normal"}, {"productId": 37780, "lowPrice": 0.04, "midPrice": 0.37, "highPrice": 4.99, "marketPrice": 0.13, "directLowPrice": 0.19, "subTypeName": "Normal"}, {"productId": 37784, "lowPrice": 0.15, "midPrice": 0.3, "highPrice": 4.99, "marketPrice": 0.22, "directLowPrice": 0.2, "subTypeName": "Normal"}, {"productId": 37785, "lowPrice": 0.15, "midPrice": 0.37, "highPrice": 3.99, "marketPrice": 0.19, "directLowPrice": 0.09, "subTypeName": "Normal"}, {"productId": 37788, "lowPrice": 0.14, "midPrice": 0.3, "highPrice": 3.0, "marketPrice": 0.31, "directLowPrice": 0.2, "subTypeName": "Normal"}, {"productId": 37789, "lowPrice": 0.15, "midPrice": 0.35, "highPrice": 4.99, "marketPrice": 0.31, "directLowPrice": 0.15, "subTypeName": "Normal"}, {"productId": 37790, "lowPrice": 0.14, "midPrice": 0.39, "highPrice": 3.99, "marketPrice": 0.22, "directLowPrice": 0.14, "subTypeName": "Normal"}, {"productId": 37793, "lowPrice": 0.05, "midPrice": 0.25, "highPrice": 3.99, "marketPrice": 0.08, "directLowPrice": 0.08, "subTypeName": "Normal"}, {"productId": 37799, "lowPrice": 0.1, "midPrice": 0.25, "highPrice": 3.0, "marketPrice": 0.2, "directLowPrice": 0.19, "subTypeName": "Normal"}, {"productId": 37802, "lowPrice": 0.1, "midPrice": 0.25, "highPrice": 5.0, "marketPrice": 0.17, "directLowPrice": 0.1, "subTypeName": "Normal"}, {"productId": 37809, "lowPrice": 0.1, "midPrice": 0.25, "highPrice": 1.67, "marketPrice": 0.19, "directLowPrice": 0.07, "subTypeName": "Normal"}, {"productId": 37810, "lowPrice": 0.09, "midPrice": 0.35, "highPrice": 5.25, "marketPrice": 0.28, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 37812, "lowPrice": 0.18, "midPrice": 0.51, "highPrice": 4.99, "marketPrice": 0.83, "directLowPrice": 0.18, "subTypeName": "Normal"}, {"productId": 37813, "lowPrice": 0.15, "midPrice": 0.25, "highPrice": 2.99, "marketPrice": 0.22, "directLowPrice": 0.14, "subTypeName": "Normal"}, {"productId": 37814, "lowPrice": 0.14, "midPrice": 0.28, "highPrice": 1.76, "marketPrice": 0.18, "directLowPrice": null, "subTypeName": "Normal"}]}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1 @@
{"success": true, "errors": [], "results": [{"productId": 70757, "lowPrice": 0.8, "midPrice": 2.48, "highPrice": 54.0, "marketPrice": 2.61, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 70758, "lowPrice": 1.14, "midPrice": 1.74, "highPrice": 12.88, "marketPrice": 1.37, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 70759, "lowPrice": 0.75, "midPrice": 2.05, "highPrice": 8.99, "marketPrice": 2.51, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 70760, "lowPrice": 0.79, "midPrice": 1.61, "highPrice": 6.0, "marketPrice": 1.17, "directLowPrice": 1.6, "subTypeName": "Foil"}, {"productId": 70761, "lowPrice": 8.0, "midPrice": 11.12, "highPrice": 49.95, "marketPrice": 11.01, "directLowPrice": 11.96, "subTypeName": "Foil"}, {"productId": 70762, "lowPrice": 0.66, "midPrice": 1.34, "highPrice": 3.99, "marketPrice": 1.22, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 70763, "lowPrice": 4.1, "midPrice": 6.0, "highPrice": 19.99, "marketPrice": 5.09, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 70764, "lowPrice": 15.53, "midPrice": 19.96, "highPrice": 159.99, "marketPrice": 19.05, "directLowPrice": 21.69, "subTypeName": "Foil"}, {"productId": 70765, "lowPrice": 100.0, "midPrice": 119.99, "highPrice": 400.0, "marketPrice": 85.88, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 70782, "lowPrice": 7.26, "midPrice": 13.24, "highPrice": 20.57, "marketPrice": 13.16, "directLowPrice": 17.05, "subTypeName": "Foil"}, {"productId": 70783, "lowPrice": 4.0, "midPrice": 6.15, "highPrice": 11.84, "marketPrice": 5.69, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 70784, "lowPrice": 3.86, "midPrice": 5.41, "highPrice": 19.55, "marketPrice": 4.66, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 70785, "lowPrice": 1.25, "midPrice": 1.9, "highPrice": 7.4, "marketPrice": 1.6, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 70786, "lowPrice": 3.0, "midPrice": 4.4, "highPrice": 10.0, "marketPrice": 3.42, "directLowPrice": 4.39, "subTypeName": "Foil"}, {"productId": 70787, "lowPrice": 9.25, "midPrice": 11.47, "highPrice": 49.88, "marketPrice": 11.19, "directLowPrice": 19.37, "subTypeName": "Foil"}, {"productId": 70789, "lowPrice": 0.5, "midPrice": 1.04, "highPrice": 16.99, "marketPrice": 0.8, "directLowPrice": 1.07, "subTypeName": "Foil"}, {"productId": 70790, "lowPrice": 0.39, "midPrice": 0.99, "highPrice": 16.45, "marketPrice": 0.68, "directLowPrice": 0.6, "subTypeName": "Foil"}, {"productId": 70791, "lowPrice": 0.23, "midPrice": 0.59, "highPrice": 3.99, "marketPrice": 0.45, "directLowPrice": 0.49, "subTypeName": "Foil"}, {"productId": 70792, "lowPrice": 1.5, "midPrice": 2.49, "highPrice": 9.99, "marketPrice": 2.26, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 70793, "lowPrice": 49.0, "midPrice": 59.02, "highPrice": 200.0, "marketPrice": 59.02, "directLowPrice": 51.49, "subTypeName": "Foil"}, {"productId": 70794, "lowPrice": 1.8, "midPrice": 3.91, "highPrice": 88.0, "marketPrice": 3.69, "directLowPrice": 2.94, "subTypeName": "Foil"}]}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1 @@
{"success": true, "errors": [], "results": [{"productId": 21603, "lowPrice": 11.0, "midPrice": 16.55, "highPrice": 97.99, "marketPrice": 17.1, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21604, "lowPrice": 4.95, "midPrice": 10.97, "highPrice": 14.97, "marketPrice": 6.99, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21605, "lowPrice": 10.0, "midPrice": 13.95, "highPrice": 79.45, "marketPrice": 14.97, "directLowPrice": 5.01, "subTypeName": "Normal"}, {"productId": 21606, "lowPrice": 18.95, "midPrice": 21.88, "highPrice": 86.37, "marketPrice": 17.0, "directLowPrice": 43.94, "subTypeName": "Normal"}, {"productId": 21607, "lowPrice": 0.38, "midPrice": 2.46, "highPrice": 5.7, "marketPrice": 1.5, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21608, "lowPrice": 1.55, "midPrice": 2.47, "highPrice": 4.95, "marketPrice": 2.0, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21609, "lowPrice": 98.95, "midPrice": 103.94, "highPrice": 108.93, "marketPrice": 57.85, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21610, "lowPrice": 4.73, "midPrice": 7.87, "highPrice": 11.0, "marketPrice": 7.61, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21611, "lowPrice": 9.0, "midPrice": 16.58, "highPrice": 22.17, "marketPrice": 14.16, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21612, "lowPrice": 5.99, "midPrice": 6.91, "highPrice": 15.74, "marketPrice": 6.29, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21613, "lowPrice": 9.55, "midPrice": 14.61, "highPrice": 18.95, "marketPrice": 13.98, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21614, "lowPrice": 1.7, "midPrice": 2.5, "highPrice": 4.95, "marketPrice": 1.4, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21615, "lowPrice": 3.86, "midPrice": 5.5, "highPrice": 8.0, "marketPrice": 6.07, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21616, "lowPrice": 8.84, "midPrice": 19.97, "highPrice": 24.95, "marketPrice": 8.5, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21617, "lowPrice": 6.92, "midPrice": 8.0, "highPrice": 13.99, "marketPrice": 7.23, "directLowPrice": 5.23, "subTypeName": "Normal"}, {"productId": 21618, "lowPrice": 25.0, "midPrice": 29.91, "highPrice": 50.0, "marketPrice": 27.0, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21619, "lowPrice": 8.0, "midPrice": 11.41, "highPrice": 18.61, "marketPrice": 9.89, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21620, "lowPrice": 15.0, "midPrice": 18.95, "highPrice": 20.59, "marketPrice": 8.76, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21621, "lowPrice": 18.5, "midPrice": 22.49, "highPrice": 49.95, "marketPrice": 25.0, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21622, "lowPrice": 7.99, "midPrice": 16.72, "highPrice": 19.95, "marketPrice": 11.91, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21623, "lowPrice": 12.0, "midPrice": 15.0, "highPrice": 16.99, "marketPrice": 15.95, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21624, "lowPrice": 0.59, "midPrice": 1.87, "highPrice": 9.95, "marketPrice": 2.5, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21625, "lowPrice": 49.95, "midPrice": 71.67, "highPrice": 99.95, "marketPrice": 66.45, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21626, "lowPrice": 3.99, "midPrice": 4.87, "highPrice": 8.12, "marketPrice": 5.5, "directLowPrice": 3.0, "subTypeName": "Normal"}, {"productId": 21627, "lowPrice": 6.77, "midPrice": 9.03, "highPrice": 24.99, "marketPrice": 5.16, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21628, "lowPrice": 1.1, "midPrice": 2.25, "highPrice": 4.23, "marketPrice": 1.5, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21629, "lowPrice": 5.0, "midPrice": 5.22, "highPrice": 13.97, "marketPrice": 5.0, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21630, "lowPrice": 16.8, "midPrice": 27.54, "highPrice": 45.0, "marketPrice": 15.95, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21631, "lowPrice": 114.95, "midPrice": 150.0, "highPrice": 154.97, "marketPrice": 42.5, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21632, "lowPrice": 16.99, "midPrice": 55.98, "highPrice": 150.0, "marketPrice": 14.79, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 21633, "lowPrice": 10.99, "midPrice": 13.84, "highPrice": 23.95, "marketPrice": 15.49, "directLowPrice": 19.88, "subTypeName": "Normal"}, {"productId": 21634, "lowPrice": 49.97, "midPrice": 74.36, "highPrice": 98.75, "marketPrice": 49.97, "directLowPrice": null, "subTypeName": "Normal"}]}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1 @@
{"success": true, "errors": [], "results": [{"productId": 82198, "lowPrice": 79.26, "midPrice": 94.13, "highPrice": 249.99, "marketPrice": 89.08, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 82732, "lowPrice": 0.74, "midPrice": 1.03, "highPrice": 3.01, "marketPrice": 0.98, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 82733, "lowPrice": 13.4, "midPrice": 18.14, "highPrice": 38.5, "marketPrice": 14.09, "directLowPrice": 35.1, "subTypeName": "Normal"}, {"productId": 82734, "lowPrice": 0.99, "midPrice": 1.62, "highPrice": 9.9, "marketPrice": 1.36, "directLowPrice": 1.58, "subTypeName": "Normal"}, {"productId": 82735, "lowPrice": 0.46, "midPrice": 0.85, "highPrice": 4.91, "marketPrice": 0.69, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 82736, "lowPrice": 0.54, "midPrice": 0.82, "highPrice": 3.88, "marketPrice": 0.83, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 82737, "lowPrice": 0.99, "midPrice": 2.25, "highPrice": 19.99, "marketPrice": 2.15, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 82738, "lowPrice": 0.15, "midPrice": 0.5, "highPrice": 2.0, "marketPrice": 0.39, "directLowPrice": 0.25, "subTypeName": "Normal"}, {"productId": 82739, "lowPrice": 1.5, "midPrice": 2.96, "highPrice": 8.87, "marketPrice": 2.32, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 82740, "lowPrice": 0.15, "midPrice": 0.47, "highPrice": 2.99, "marketPrice": 0.34, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 82741, "lowPrice": 0.45, "midPrice": 0.9, "highPrice": 8.53, "marketPrice": 0.56, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 82742, "lowPrice": 0.25, "midPrice": 0.75, "highPrice": 27.99, "marketPrice": 0.5, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 82743, "lowPrice": 0.09, "midPrice": 0.33, "highPrice": 1.99, "marketPrice": 0.21, "directLowPrice": 0.25, "subTypeName": "Normal"}, {"productId": 82744, "lowPrice": 0.18, "midPrice": 0.48, "highPrice": 1.83, "marketPrice": 0.45, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 82745, "lowPrice": 0.77, "midPrice": 1.25, "highPrice": 18.99, "marketPrice": 0.95, "directLowPrice": 1.71, "subTypeName": "Normal"}, {"productId": 82746, "lowPrice": 0.03, "midPrice": 0.25, "highPrice": 1.5, "marketPrice": 0.11, "directLowPrice": 0.1, "subTypeName": "Normal"}, {"productId": 82747, "lowPrice": 0.2, "midPrice": 0.38, "highPrice": 1.99, "marketPrice": 0.26, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 82748, "lowPrice": 0.22, "midPrice": 0.39, "highPrice": 1.52, "marketPrice": 0.29, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 82749, "lowPrice": 20.14, "midPrice": 27.69, "highPrice": 79.99, "marketPrice": 22.14, "directLowPrice": 25.63, "subTypeName": "Normal"}, {"productId": 82750, "lowPrice": 0.06, "midPrice": 0.3, "highPrice": 2.99, "marketPrice": 0.07, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 82751, "lowPrice": 4.97, "midPrice": 7.93, "highPrice": 17.38, "marketPrice": 7.63, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 82752, "lowPrice": 0.12, "midPrice": 0.4, "highPrice": 2.99, "marketPrice": 0.16, "directLowPrice": 0.15, "subTypeName": "Normal"}, {"productId": 82753, "lowPrice": 2.09, "midPrice": 3.5, "highPrice": 11.99, "marketPrice": 2.76, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 82754, "lowPrice": 0.05, "midPrice": 0.26, "highPrice": 1.5, "marketPrice": 0.17, "directLowPrice": 0.14, "subTypeName": "Normal"}, {"productId": 82755, "lowPrice": 0.25, "midPrice": 0.75, "highPrice": 3.59, "marketPrice": 0.58, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 82756, "lowPrice": 0.9, "midPrice": 1.75, "highPrice": 11.97, "marketPrice": 1.36, "directLowPrice": 1.5, "subTypeName": "Normal"}, {"productId": 82757, "lowPrice": 3.97, "midPrice": 6.25, "highPrice": 13.5, "marketPrice": 3.97, "directLowPrice": 7.99, "subTypeName": "Normal"}, {"productId": 83374, "lowPrice": 8.96, "midPrice": 13.5, "highPrice": 19.99, "marketPrice": 11.77, "directLowPrice": 19.97, "subTypeName": "Normal"}, {"productId": 83376, "lowPrice": 0.19, "midPrice": 0.5, "highPrice": 5.0, "marketPrice": 0.7, "directLowPrice": 0.49, "subTypeName": "Normal"}, {"productId": 83377, "lowPrice": 0.05, "midPrice": 0.25, "highPrice": 1.5, "marketPrice": 0.17, "directLowPrice": 0.5, "subTypeName": "Normal"}]}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1 @@
{"success": true, "errors": [], "results": []}

View File

@ -0,0 +1 @@
{"success": true, "errors": [], "results": [{"productId": 91636, "lowPrice": 29.85, "midPrice": 45.99, "highPrice": 495.95, "marketPrice": 30.89, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 91637, "lowPrice": 14.19, "midPrice": 14.94, "highPrice": 34.8, "marketPrice": 14.73, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 91638, "lowPrice": 3.29, "midPrice": 4.86, "highPrice": 16.78, "marketPrice": 4.65, "directLowPrice": 4.0, "subTypeName": "Foil"}, {"productId": 91639, "lowPrice": 3.67, "midPrice": 5.55, "highPrice": 15.99, "marketPrice": 5.57, "directLowPrice": 5.99, "subTypeName": "Foil"}, {"productId": 91640, "lowPrice": 2.92, "midPrice": 4.0, "highPrice": 19.99, "marketPrice": 3.4, "directLowPrice": 3.99, "subTypeName": "Foil"}, {"productId": 91641, "lowPrice": 2.04, "midPrice": 3.0, "highPrice": 9.99, "marketPrice": 2.47, "directLowPrice": 2.64, "subTypeName": "Foil"}, {"productId": 92296, "lowPrice": 0.4, "midPrice": 0.94, "highPrice": 4.78, "marketPrice": 0.53, "directLowPrice": 0.69, "subTypeName": "Foil"}, {"productId": 92297, "lowPrice": 0.82, "midPrice": 1.43, "highPrice": 29.98, "marketPrice": 1.19, "directLowPrice": 1.29, "subTypeName": "Foil"}, {"productId": 92298, "lowPrice": 1.63, "midPrice": 4.48, "highPrice": 15.0, "marketPrice": 5.31, "directLowPrice": 3.29, "subTypeName": "Foil"}, {"productId": 92299, "lowPrice": 0.63, "midPrice": 1.0, "highPrice": 3.99, "marketPrice": 0.83, "directLowPrice": 0.96, "subTypeName": "Foil"}, {"productId": 92300, "lowPrice": 0.72, "midPrice": 2.06, "highPrice": 19.99, "marketPrice": 2.5, "directLowPrice": 2.5, "subTypeName": "Foil"}, {"productId": 92301, "lowPrice": 0.96, "midPrice": 1.65, "highPrice": 19.99, "marketPrice": 1.16, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 92302, "lowPrice": 2.85, "midPrice": 4.29, "highPrice": 39.99, "marketPrice": 4.29, "directLowPrice": 3.99, "subTypeName": "Foil"}, {"productId": 92303, "lowPrice": 2.49, "midPrice": 3.61, "highPrice": 16.99, "marketPrice": 2.86, "directLowPrice": 3.08, "subTypeName": "Foil"}, {"productId": 92304, "lowPrice": 0.25, "midPrice": 0.93, "highPrice": 19.99, "marketPrice": 0.74, "directLowPrice": 1.53, "subTypeName": "Foil"}, {"productId": 92305, "lowPrice": 0.38, "midPrice": 0.81, "highPrice": 6.85, "marketPrice": 0.63, "directLowPrice": null, "subTypeName": "Foil"}]}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1 @@
{"success": true, "errors": [], "results": [{"productId": 95578, "lowPrice": 40.0, "midPrice": 48.63, "highPrice": 150.0, "marketPrice": 40.66, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95579, "lowPrice": 23.58, "midPrice": 31.72, "highPrice": 55.21, "marketPrice": 31.84, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95580, "lowPrice": 0.69, "midPrice": 1.38, "highPrice": 6.0, "marketPrice": 1.06, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95581, "lowPrice": 1.0, "midPrice": 2.79, "highPrice": 6.95, "marketPrice": 1.91, "directLowPrice": 2.35, "subTypeName": "Normal"}, {"productId": 95582, "lowPrice": 7.59, "midPrice": 12.19, "highPrice": 24.66, "marketPrice": 10.49, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95583, "lowPrice": 0.25, "midPrice": 1.16, "highPrice": 4.0, "marketPrice": 0.84, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95584, "lowPrice": 0.45, "midPrice": 1.5, "highPrice": 6.31, "marketPrice": 0.8, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95585, "lowPrice": 0.8, "midPrice": 1.55, "highPrice": 4.79, "marketPrice": 0.85, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95586, "lowPrice": 0.1, "midPrice": 0.44, "highPrice": 1.5, "marketPrice": 0.23, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95587, "lowPrice": 0.15, "midPrice": 0.5, "highPrice": 1.5, "marketPrice": 0.49, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95588, "lowPrice": 0.25, "midPrice": 0.54, "highPrice": 2.43, "marketPrice": 0.3, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95589, "lowPrice": 0.25, "midPrice": 0.75, "highPrice": 4.25, "marketPrice": 0.82, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95590, "lowPrice": 0.1, "midPrice": 0.37, "highPrice": 11.5, "marketPrice": 0.25, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95591, "lowPrice": 0.18, "midPrice": 0.39, "highPrice": 2.0, "marketPrice": 0.19, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95592, "lowPrice": 0.14, "midPrice": 0.49, "highPrice": 1.77, "marketPrice": 0.22, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95593, "lowPrice": 0.15, "midPrice": 0.4, "highPrice": 7.0, "marketPrice": 0.23, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95594, "lowPrice": 0.07, "midPrice": 0.39, "highPrice": 3.0, "marketPrice": 0.22, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95595, "lowPrice": 0.2, "midPrice": 0.48, "highPrice": 2.0, "marketPrice": 0.22, "directLowPrice": 0.25, "subTypeName": "Normal"}, {"productId": 95596, "lowPrice": 1.18, "midPrice": 2.62, "highPrice": 10.49, "marketPrice": 2.36, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95597, "lowPrice": 1.93, "midPrice": 2.96, "highPrice": 6.0, "marketPrice": 2.96, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95598, "lowPrice": 0.19, "midPrice": 0.44, "highPrice": 2.91, "marketPrice": 0.21, "directLowPrice": 0.42, "subTypeName": "Normal"}, {"productId": 95599, "lowPrice": 0.15, "midPrice": 0.5, "highPrice": 2.0, "marketPrice": 0.23, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95600, "lowPrice": 0.24, "midPrice": 0.89, "highPrice": 3.0, "marketPrice": 0.65, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95601, "lowPrice": 0.14, "midPrice": 0.5, "highPrice": 2.91, "marketPrice": 0.2, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 95602, "lowPrice": 0.14, "midPrice": 0.5, "highPrice": 2.5, "marketPrice": 0.21, "directLowPrice": 0.35, "subTypeName": "Normal"}, {"productId": 95603, "lowPrice": 0.15, "midPrice": 0.49, "highPrice": 3.0, "marketPrice": 0.43, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 99943, "lowPrice": 14.32, "midPrice": 20.0, "highPrice": 39.99, "marketPrice": 13.81, "directLowPrice": null, "subTypeName": "Normal"}]}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1 @@
{"success": true, "errors": [], "results": [{"productId": 96436, "lowPrice": 0.41, "midPrice": 0.93, "highPrice": 20.0, "marketPrice": 0.84, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 96437, "lowPrice": 0.39, "midPrice": 0.62, "highPrice": 5.0, "marketPrice": 0.57, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 96438, "lowPrice": 0.1, "midPrice": 0.33, "highPrice": 3.0, "marketPrice": 0.34, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 96439, "lowPrice": 0.05, "midPrice": 0.26, "highPrice": 5.0, "marketPrice": 0.26, "directLowPrice": null, "subTypeName": "Normal"}]}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1 @@
{"success": true, "errors": [], "results": [{"productId": 100611, "lowPrice": 4.33, "midPrice": 6.37, "highPrice": 123.0, "marketPrice": 4.76, "directLowPrice": 6.4, "subTypeName": "Foil"}, {"productId": 100612, "lowPrice": 3.0, "midPrice": 4.17, "highPrice": 15.99, "marketPrice": 3.88, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 100667, "lowPrice": 119.99, "midPrice": 142.49, "highPrice": 999.95, "marketPrice": 124.0, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 101468, "lowPrice": 35.0, "midPrice": 41.65, "highPrice": 75.68, "marketPrice": 43.6, "directLowPrice": 37.15, "subTypeName": "Foil"}, {"productId": 101469, "lowPrice": 5.0, "midPrice": 7.25, "highPrice": 19.99, "marketPrice": 7.0, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 102777, "lowPrice": 0.51, "midPrice": 1.73, "highPrice": 20.03, "marketPrice": 1.7, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 102778, "lowPrice": 7.51, "midPrice": 10.11, "highPrice": 22.72, "marketPrice": 9.96, "directLowPrice": 14.28, "subTypeName": "Foil"}, {"productId": 102779, "lowPrice": 2.54, "midPrice": 3.43, "highPrice": 8.25, "marketPrice": 2.94, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 102780, "lowPrice": 1.99, "midPrice": 3.42, "highPrice": 14.99, "marketPrice": 3.28, "directLowPrice": 2.0, "subTypeName": "Foil"}, {"productId": 102781, "lowPrice": 1.03, "midPrice": 2.0, "highPrice": 8.0, "marketPrice": 1.75, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 102782, "lowPrice": 0.7, "midPrice": 1.07, "highPrice": 3.41, "marketPrice": 0.99, "directLowPrice": 1.02, "subTypeName": "Foil"}, {"productId": 102783, "lowPrice": 0.26, "midPrice": 0.99, "highPrice": 5.0, "marketPrice": 0.82, "directLowPrice": null, "subTypeName": "Foil"}, {"productId": 102784, "lowPrice": 0.21, "midPrice": 0.82, "highPrice": 2.59, "marketPrice": 0.66, "directLowPrice": 0.44, "subTypeName": "Foil"}, {"productId": 102785, "lowPrice": 6.02, "midPrice": 7.52, "highPrice": 19.0, "marketPrice": 7.17, "directLowPrice": 9.99, "subTypeName": "Foil"}, {"productId": 102786, "lowPrice": 0.74, "midPrice": 2.74, "highPrice": 49.99, "marketPrice": 2.61, "directLowPrice": 2.48, "subTypeName": "Foil"}, {"productId": 102787, "lowPrice": 0.98, "midPrice": 2.49, "highPrice": 12.01, "marketPrice": 2.58, "directLowPrice": null, "subTypeName": "Foil"}]}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1 @@
{"success": true, "errors": [], "results": []}

View File

@ -0,0 +1 @@
{"success": true, "errors": [], "results": []}

View File

@ -0,0 +1 @@
{"success": true, "errors": [], "results": []}

View File

@ -0,0 +1 @@
{"success": true, "errors": [], "results": []}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1 @@
{"success": true, "errors": [], "results": [{"productId": 116780, "lowPrice": 0.25, "midPrice": 0.5, "highPrice": 6.99, "marketPrice": 0.29, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 116781, "lowPrice": 0.01, "midPrice": 0.19, "highPrice": 1.32, "marketPrice": 0.07, "directLowPrice": 0.03, "subTypeName": "Normal"}, {"productId": 116782, "lowPrice": 0.01, "midPrice": 0.15, "highPrice": 5.0, "marketPrice": 0.02, "directLowPrice": 0.02, "subTypeName": "Normal"}, {"productId": 116783, "lowPrice": 0.01, "midPrice": 0.2, "highPrice": 16.0, "marketPrice": 0.04, "directLowPrice": 0.14, "subTypeName": "Normal"}, {"productId": 116784, "lowPrice": 0.01, "midPrice": 0.19, "highPrice": 1.32, "marketPrice": 0.04, "directLowPrice": 0.1, "subTypeName": "Normal"}, {"productId": 116785, "lowPrice": 0.01, "midPrice": 0.2, "highPrice": 1.32, "marketPrice": 0.08, "directLowPrice": 0.1, "subTypeName": "Normal"}, {"productId": 116786, "lowPrice": 0.01, "midPrice": 0.2, "highPrice": 1.32, "marketPrice": 0.04, "directLowPrice": 0.25, "subTypeName": "Normal"}, {"productId": 116787, "lowPrice": 0.01, "midPrice": 0.18, "highPrice": 5.0, "marketPrice": 0.13, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 116788, "lowPrice": 0.04, "midPrice": 0.25, "highPrice": 4.99, "marketPrice": 0.13, "directLowPrice": 0.13, "subTypeName": "Normal"}, {"productId": 116789, "lowPrice": 0.01, "midPrice": 0.15, "highPrice": 2.0, "marketPrice": 0.03, "directLowPrice": 0.16, "subTypeName": "Normal"}, {"productId": 116790, "lowPrice": 0.01, "midPrice": 0.2, "highPrice": 1.35, "marketPrice": 0.1, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 116791, "lowPrice": 0.03, "midPrice": 0.2, "highPrice": 1.53, "marketPrice": 0.12, "directLowPrice": 0.1, "subTypeName": "Normal"}, {"productId": 116792, "lowPrice": 0.02, "midPrice": 0.25, "highPrice": 5.06, "marketPrice": 0.12, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 116793, "lowPrice": 0.98, "midPrice": 1.2, "highPrice": 4.99, "marketPrice": 1.03, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 116794, "lowPrice": 0.01, "midPrice": 0.25, "highPrice": 4.99, "marketPrice": 0.16, "directLowPrice": null, "subTypeName": "Normal"}, {"productId": 116795, "lowPrice": 0.01, "midPrice": 0.15, "highPrice": 5.0, "marketPrice": 0.02, "directLowPrice": 0.06, "subTypeName": "Normal"}]}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show More