134 lines
5.4 KiB
Python
134 lines
5.4 KiB
Python
import os
|
|
import json
|
|
import zipfile
|
|
import time
|
|
import shutil
|
|
from typing import Dict, Any, Optional
|
|
from sqlalchemy.orm import Session
|
|
from app.services.external_api.base_external_service import BaseExternalService
|
|
from app.schemas.file import FileInDB
|
|
import logging
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class MTGJSONService(BaseExternalService):
|
|
def __init__(self):
|
|
super().__init__(base_url="https://mtgjson.com/api/v5/")
|
|
|
|
async def _download_and_unzip_file(self, db: Session, url: str, filename: str, subdir: str) -> FileInDB:
|
|
"""Download a file from the given URL and save it using FileService"""
|
|
print(f"Downloading {url}...")
|
|
start_time = time.time()
|
|
|
|
# Use the base external service's _make_request method
|
|
file_data = await self._make_request(
|
|
method="GET",
|
|
endpoint=url.replace(self.base_url, ""),
|
|
binary=True
|
|
)
|
|
|
|
# Save the file using the file service
|
|
file_record = await self.file_service.save_file(
|
|
db=db,
|
|
file_data=file_data,
|
|
filename=filename,
|
|
subdir=f"mtgjson/{subdir}",
|
|
file_type="application/zip",
|
|
content_type="application/zip"
|
|
)
|
|
|
|
# Unzip the file
|
|
await self._unzip_file(file_record, subdir, db)
|
|
|
|
return file_record
|
|
|
|
|
|
async def _unzip_file(self, file_record: FileInDB, subdir: str, db: Session) -> FileInDB:
|
|
"""Unzip a file to the specified subdirectory and return the path to the extracted JSON file"""
|
|
try:
|
|
file_service = self.get_service('file')
|
|
cache_dir = file_service.base_cache_dir
|
|
temp_dir = os.path.join(cache_dir,'mtgjson', subdir, 'temp')
|
|
os.makedirs(temp_dir, exist_ok=True)
|
|
with zipfile.ZipFile(file_record.path, 'r') as zip_ref:
|
|
json_filename = zip_ref.namelist()[0]
|
|
zip_ref.extractall(temp_dir)
|
|
json_path = os.path.join(temp_dir, json_filename)
|
|
|
|
# Create a file record for the extracted JSON file
|
|
with open(json_path, 'r') as f:
|
|
json_data = f.read()
|
|
json_file_record = await self.file_service.save_file(
|
|
db=db,
|
|
file_data=json_data,
|
|
filename=json_filename,
|
|
subdir=f"mtgjson/{subdir}",
|
|
file_type="application/json",
|
|
content_type="application/json"
|
|
)
|
|
|
|
# remove the temp directory
|
|
shutil.rmtree(temp_dir)
|
|
|
|
return json_file_record
|
|
except Exception as e:
|
|
logger.error(f"Error unzipping file: {e}")
|
|
raise
|
|
|
|
async def get_identifiers(self, db: Session, use_cache: bool = True) -> Dict[str, Any]:
|
|
"""Download and get MTGJSON identifiers data"""
|
|
# Check if we have a cached version
|
|
cached_file = await self.file_service.get_file_by_filename(db, "AllIdentifiers.json")
|
|
if cached_file and os.path.exists(cached_file.path) and use_cache:
|
|
with open(cached_file.path, 'r') as f:
|
|
logger.debug(f"Loaded identifiers from cache: {cached_file.path}")
|
|
return json.load(f)
|
|
else:
|
|
# Download and process the file
|
|
logger.debug(f"Downloading identifiers from MTGJSON")
|
|
file_record = await self._download_and_unzip_file(
|
|
db=db,
|
|
url="https://mtgjson.com/api/v5/AllIdentifiers.json.zip",
|
|
filename="AllIdentifiers.json.zip",
|
|
subdir="identifiers"
|
|
)
|
|
|
|
with open(file_record.path, 'r') as f:
|
|
logger.debug(f"Loaded identifiers from MTGJSON: {file_record.path}")
|
|
return json.load(f)
|
|
|
|
async def get_skus(self, db: Session, use_cache: bool = True) -> Dict[str, Any]:
|
|
"""Download and get MTGJSON SKUs data"""
|
|
# Check if we have a cached version
|
|
cached_file = await self.file_service.get_file_by_filename(db, "TcgplayerSkus.json")
|
|
if cached_file and os.path.exists(cached_file.path) and use_cache:
|
|
with open(cached_file.path, 'r') as f:
|
|
logger.debug(f"Loaded SKUs from cache: {cached_file.path}")
|
|
return json.load(f)
|
|
else:
|
|
# Download and process the file
|
|
logger.debug(f"Downloading SKUs from MTGJSON")
|
|
file_record = await self._download_and_unzip_file(
|
|
db=db,
|
|
url="https://mtgjson.com/api/v5/TcgplayerSkus.json.zip",
|
|
filename="TcgplayerSkus.json.zip",
|
|
subdir="skus"
|
|
)
|
|
|
|
with open(file_record.path, 'r') as f:
|
|
logger.debug(f"Loaded SKUs from MTGJSON: {file_record.path}")
|
|
return json.load(f)
|
|
|
|
async def clear_cache(self, db: Session) -> None:
|
|
"""Clear all cached data"""
|
|
try:
|
|
# Delete all files in the mtgjson subdirectory
|
|
files = await self.file_service.list_files(db, file_type=["json", "zip"])
|
|
for file in files:
|
|
if file.path.startswith("mtgjson/"):
|
|
await self.file_service.delete_file(db, file.id)
|
|
logger.info("MTGJSON cache cleared")
|
|
except Exception as e:
|
|
logger.error(f"Error clearing cache: {e}")
|
|
raise
|