kind of a mess lol but file caching and front end

app/services/external_api/mtgjson/mtgjson_service.py

@@ -11,9 +11,12 @@ from datetime import datetime
 from app.models.mtgjson_card import MTGJSONCard
 from app.models.mtgjson_sku import MTGJSONSKU
 from app.db.database import get_db, transaction
+from app.services.external_api.base_external_service import BaseExternalService
+from app.schemas.file import FileInDB
 
-class MTGJSONService:
+class MTGJSONService(BaseExternalService):
     def __init__(self, cache_dir: str = "app/data/cache/mtgjson", batch_size: int = 1000):
+        super().__init__(base_url="https://mtgjson.com/api/v5/")
         self.cache_dir = cache_dir
         self.identifiers_dir = os.path.join(cache_dir, "identifiers")
         self.skus_dir = os.path.join(cache_dir, "skus")
@@ -38,27 +41,22 @@ class MTGJSONService:
         """Print progress message with flush"""
         print(message, end=end, flush=True)
 
-    async def _download_file(self, url: str, output_path: str) -> None:
-        """Download a file from the given URL to the specified path using streaming"""
+    async def _download_file(self, db: Session, url: str, filename: str, subdir: str) -> FileInDB:
+        """Download a file from the given URL and save it using FileService"""
         print(f"Downloading {url}...")
         start_time = time.time()
-        total_size = 0
         
         async with aiohttp.ClientSession() as session:
             async with session.get(url) as response:
                 if response.status == 200:
-                    total_size = int(response.headers.get('content-length', 0))
-                    with open(output_path, 'wb') as f:
-                        downloaded = 0
-                        async for chunk in response.content.iter_chunked(8192):
-                            f.write(chunk)
-                            downloaded += len(chunk)
-                            if total_size > 0:
-                                percent = (downloaded / total_size) * 100
-                                elapsed = time.time() - start_time
-                                speed = downloaded / elapsed / 1024 / 1024  # MB/s
-                                print(f"\rDownloading: {percent:.1f}% ({downloaded/1024/1024:.1f}MB/{total_size/1024/1024:.1f}MB) at {speed:.1f}MB/s", end="")
-                    print("\nDownload complete!")
+                    file_data = await response.read()
+                    return await self.save_file(
+                        db=db,
+                        file_data=file_data,
+                        file_name=filename,
+                        subdir=f"mtgjson/{subdir}",
+                        file_type=response.headers.get('content-type', 'application/octet-stream')
+                    )
                 else:
                     raise Exception(f"Failed to download file from {url}. Status: {response.status}")
 
@@ -153,14 +151,16 @@ class MTGJSONService:
         self._print_progress("Starting MTGJSON identifiers processing...")
         start_time = time.time()
         
-        zip_path = os.path.join(self.identifiers_dir, "AllIdentifiers.json.zip")
-        await self._download_file(
-            "https://mtgjson.com/api/v5/AllIdentifiers.json.zip",
-            zip_path
+        # Download the file using FileService
+        file_record = await self._download_file(
+            db=db,
+            url="https://mtgjson.com/api/v5/AllIdentifiers.json.zip",
+            filename="AllIdentifiers.json.zip",
+            subdir="identifiers"
         )
-
-        self._print_progress("Unzipping file...")
-        json_path = await self._unzip_file(zip_path, self.identifiers_dir)
+        
+        # Get the file path from the database record
+        zip_path = file_record.path
         
         cards_processed = 0
         current_batch = []
@@ -169,7 +169,7 @@ class MTGJSONService:
         
         self._print_progress("Processing cards...")
         try:
-            for item in self._stream_json_file(json_path):
+            for item in self._stream_json_file(zip_path):
                 if item["type"] == "meta":
                     self._print_progress(f"Processing MTGJSON data version {item['data'].get('version')} from {item['data'].get('date')}")
                     continue
@@ -239,14 +239,16 @@ class MTGJSONService:
         self._print_progress("Starting MTGJSON SKUs processing...")
         start_time = time.time()
         
-        zip_path = os.path.join(self.skus_dir, "TcgplayerSkus.json.zip")
-        await self._download_file(
-            "https://mtgjson.com/api/v5/TcgplayerSkus.json.zip",
-            zip_path
+        # Download the file using FileService
+        file_record = await self._download_file(
+            db=db,
+            url="https://mtgjson.com/api/v5/TcgplayerSkus.json.zip",
+            filename="TcgplayerSkus.json.zip",
+            subdir="skus"
         )
-
-        self._print_progress("Unzipping file...")
-        json_path = await self._unzip_file(zip_path, self.skus_dir)
+        
+        # Get the file path from the database record
+        zip_path = file_record.path
         
         skus_processed = 0
         current_batch = []
@@ -255,7 +257,7 @@ class MTGJSONService:
         
         self._print_progress("Processing SKUs...")
         try:
-            for item in self._stream_json_file(json_path):
+            for item in self._stream_json_file(zip_path):
                 if item["type"] == "meta":
                     self._print_progress(f"Processing MTGJSON SKUs version {item['data'].get('version')} from {item['data'].get('date')}")
                     continue