From a60ce0db17b10eda223eb8a4b9ea7977cab75a66 Mon Sep 17 00:00:00 2001 From: zman Date: Tue, 5 Mar 2024 22:22:44 -0500 Subject: [PATCH] lint --- scraper/api.py | 40 ++++++++++++++++++++++++---------------- scraper/app.py | 6 +++--- scraper/app_log.py | 11 ++++++++++- scraper/costco.py | 7 ++++--- scraper/main.py | 12 ++++++------ scraper/reddit.py | 11 ++++++----- scraper/threads.py | 1 - scraper/webhook.py | 10 +++++----- 8 files changed, 58 insertions(+), 40 deletions(-) diff --git a/scraper/api.py b/scraper/api.py index 13c523c..a21c3d4 100644 --- a/scraper/api.py +++ b/scraper/api.py @@ -1,9 +1,14 @@ -import requests +""" +Interacts with the API to handle requests for post and product data. +Utilizes the `requests` library to send requests +""" + from datetime import datetime, timedelta from zoneinfo import ZoneInfo -from exceptions import APIRequestError, InvalidDataTypeError, InvalidMethodError -from app_log import LoggingManager -from models import Post +import requests +from .exceptions import APIRequestError, InvalidDataTypeError, InvalidMethodError +from .app_log import LoggingManager +from .models import Post class ApiRequestHandler: @@ -50,16 +55,17 @@ class ApiRequestHandler: f"Invalid data type: {type(params)} expected dict" ) try: - response = requests.request(method, api_url, data=data, params=params) + response = requests.request( + method, api_url, data=data, params=params, timeout=10 + ) except requests.RequestException as e: self.log_manager.error(f"API request failed: {e}") - raise APIRequestError(0, str(e)) - success_codes = [200, 201, 204] - if response.status_code not in success_codes: - self.log_manager.error( - f"API request failed: {response.status_code} - {response.text}" - ) - raise APIRequestError(response.status_code, response.text) + raise APIRequestError(0, str(e)) from e + try: + response.raise_for_status() + except requests.HTTPError as e: + self.log_manager.error(f"API request failed: {e}") + raise APIRequestError(response.status_code, response.text) from e return response.json() @@ -251,7 +257,7 @@ class CostcoProductManager: api_request_handler (ApiRequestHandler): Handles the API requests for interacting with Costco product data. log_manager (LoggingManager): Manages logging for operations performed by CostcoProductManager. """ - + def __init__(self, api_request_handler: ApiRequestHandler): """ Initializes the CostcoProductManager with an API request handler for making API calls and a logging manager @@ -275,7 +281,7 @@ class CostcoProductManager: "GET", f"{self.api_request_handler.api_url}costco_products/" ) return all_products - + def insert_costco_product(self, product) -> dict: """ Inserts a new Costco product into the database through an API call. @@ -320,7 +326,9 @@ class CostcoProductManager: "active": product.active, } response = self.api_request_handler.send_api_request( - "PUT", f"{self.api_request_handler.api_url}costco_products/{product.sku}/", data=data + "PUT", + f"{self.api_request_handler.api_url}costco_products/{product.sku}/", + data=data, ) return response @@ -338,4 +346,4 @@ class CostcoProductManager: response = self.api_request_handler.send_api_request( "GET", f"{self.api_request_handler.api_url}costco_products/?sku={sku}" ) - return response \ No newline at end of file + return response diff --git a/scraper/app.py b/scraper/app.py index f59caaf..de4d2c5 100644 --- a/scraper/app.py +++ b/scraper/app.py @@ -1,6 +1,6 @@ -from app_log import LoggingManager -from threads import Scheduler -from costco import CostcoMonitor +from .app_log import LoggingManager +from .threads import Scheduler +from .costco import CostcoMonitor class Application: diff --git a/scraper/app_log.py b/scraper/app_log.py index 59e4820..8c99539 100644 --- a/scraper/app_log.py +++ b/scraper/app_log.py @@ -1,9 +1,15 @@ -import logging +""" + +""" from logging.handlers import RotatingFileHandler import sys +import logging class SingletonMeta(type): + """ + A metaclass that creates a Singleton base class when called + """ _instances = {} def __call__(cls, *args, **kwargs): @@ -13,6 +19,9 @@ class SingletonMeta(type): class LoggingManager(metaclass=SingletonMeta): + """ + A class that creates a logger object and sets up the logger with file and stream handlers + """ def __init__(self, log_file): if not hasattr(self, "logger"): self.log_file = log_file diff --git a/scraper/costco.py b/scraper/costco.py index 4a03267..b8fb9dc 100644 --- a/scraper/costco.py +++ b/scraper/costco.py @@ -1,12 +1,13 @@ +import os from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.chrome.options import Options from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.common.exceptions import TimeoutException -from app_log import LoggingManager -from models import Product -import os +from .app_log import LoggingManager +from .models import Product + class CostcoMonitor: diff --git a/scraper/main.py b/scraper/main.py index 26c74b4..82f4beb 100644 --- a/scraper/main.py +++ b/scraper/main.py @@ -1,9 +1,9 @@ -from webhook import WebhookNotifier -from app import Application -from api import ApiRequestHandler, PostManager, PostAnalyticsManager, CostcoProductManager -from reddit import RedditMonitor, SubmissionManager -from config import Config -from app_log import LoggingManager +from .webhook import WebhookNotifier +from .app import Application +from .api import ApiRequestHandler, PostManager, PostAnalyticsManager, CostcoProductManager +from .reddit import RedditMonitor, SubmissionManager +from .config import Config +from .app_log import LoggingManager if __name__ == "__main__": diff --git a/scraper/reddit.py b/scraper/reddit.py index 04f3819..41e81d6 100644 --- a/scraper/reddit.py +++ b/scraper/reddit.py @@ -1,7 +1,8 @@ import praw -from app_log import LoggingManager -from models import Post -from api import PostManager, PostAnalyticsManager +from .app_log import LoggingManager +from .models import Post +from .api import PostManager, PostAnalyticsManager +from .webhook import WebhookNotifier class RedditMonitor: @@ -87,7 +88,7 @@ class SubmissionManager: reddit_monitor: RedditMonitor, post_manager: PostManager, post_analytics_manager: PostAnalyticsManager, - WebhookNotifier, + webhook_notifier: WebhookNotifier, ): """ Initializes the SubmissionManager with necessary components for processing submissions. @@ -101,7 +102,7 @@ class SubmissionManager: self.reddit_monitor = reddit_monitor self.post_manager = post_manager self.post_analytics_manager = post_analytics_manager - self.webhook_notifier = WebhookNotifier + self.webhook_notifier = webhook_notifier self.log_manager = LoggingManager("scraper.log") def convert_submission_to_post(self, submission): diff --git a/scraper/threads.py b/scraper/threads.py index e123d2c..cee4559 100644 --- a/scraper/threads.py +++ b/scraper/threads.py @@ -1,5 +1,4 @@ import threading -import time class Scheduler: def __init__(self): diff --git a/scraper/webhook.py b/scraper/webhook.py index 79c944c..0614a0d 100644 --- a/scraper/webhook.py +++ b/scraper/webhook.py @@ -1,6 +1,6 @@ import requests -from app_log import LoggingManager -from models import Product, Post +from .app_log import LoggingManager +from .models import Product, Post class WebhookNotifier: @@ -9,7 +9,7 @@ class WebhookNotifier: self.disable_webhook = disable_webhook self.log_manager = LoggingManager("scraper.log") - def send_notification(self, post): + def send_notification(self, post: Post): title = post.title url = post.url permalink = post.permalink @@ -23,7 +23,7 @@ class WebhookNotifier: if not self.disable_webhook: self.log_manager.log(f"Sending notification to {self.webhook_url}") try: - requests.post(self.webhook_url, data={"content": content}) + requests.post(self.webhook_url, data={"content": content}, timeout=5) except Exception as e: self.log_manager.error(f"Failed to send notification: {e}") @@ -42,6 +42,6 @@ class WebhookNotifier: if not self.disable_webhook: self.log_manager.log(f"Sending notification to {self.webhook_url}") try: - requests.post(self.webhook_url, data={"content": content}) + requests.post(self.webhook_url, data={"content": content}, timeout=5) except Exception as e: self.log_manager.error(f"Failed to send notification: {e}") \ No newline at end of file