From 91fce540d4752d1f545c716d27ddba108f362229 Mon Sep 17 00:00:00 2001 From: zman Date: Tue, 5 Mar 2024 09:21:57 -0500 Subject: [PATCH] format --- scraper/app.py | 23 +++++++++++++++-------- scraper/config.py | 2 +- scraper/main.py | 6 ++---- scraper/webhook.py | 2 +- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/scraper/app.py b/scraper/app.py index 0d2cdb6..f0cbe0b 100644 --- a/scraper/app.py +++ b/scraper/app.py @@ -74,6 +74,7 @@ class PostManager: api_request_handler (ApiRequestHandler): Handles the API requests for interacting with post data. log_manager (LoggingManager): Manages logging for operations performed by PostManager. """ + def __init__(self, api_request_handler: ApiRequestHandler): """ Initializes the PostManager with an API request handler for making API calls and a logging manager for logging. @@ -166,6 +167,7 @@ class PostAnalyticsManager: post_manager (PostManager): Manages post retrieval and existence checks. log_manager (LoggingManager): Manages logging for analytics operations. """ + def __init__( self, api_request_handler: ApiRequestHandler, post_manager: PostManager ): @@ -202,8 +204,8 @@ class PostAnalyticsManager: now = datetime.now(timezone) # Format datetime objects for the API request - time_begin_str = time_start.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' - time_end_str = now.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + time_begin_str = time_start.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + time_end_str = now.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" post_id = self.post_manager.get_post_by_reddit_id(reddit_id) post_id = post_id[0]["id"] @@ -259,6 +261,7 @@ class RedditMonitor: subreddit (praw.models.Subreddit): The subreddit object for the specified subreddit. log_manager (LoggingManager): Manages logging for Reddit monitoring operations. """ + def __init__( self, client_id, client_secret, user_agent, username, password, subreddit_name ): @@ -325,6 +328,7 @@ class SubmissionManager: webhook_notifier (WebhookNotifier): Handles notifications for new or updated posts. log_manager (LoggingManager): Manages logging for submission processing operations. """ + def __init__( self, reddit_monitor: RedditMonitor, @@ -386,7 +390,9 @@ class SubmissionManager: if self.post_manager.post_exists(submission.id): self.log_manager.log("Post exists") self.log_manager.log(f"post id: {submission.id}") - if self.post_analytics_manager.check_update_requirements(submission.id, update_frequency): + if self.post_analytics_manager.check_update_requirements( + submission.id, update_frequency + ): self.log_manager.log("Update requirements met") post = self.convert_submission_to_post(submission) self.post_analytics_manager.update_post_analytics(post) @@ -415,6 +421,7 @@ class Application: thread_manager: Manages threading for asynchronous operations. update_frequency (int): The frequency, in seconds, at which post analytics should be updated. """ + def __init__( self, reddit_monitor, @@ -423,7 +430,7 @@ class Application: post_manager, post_analytics_manager, submission_manager, - update_frequency + update_frequency, ): """ Initializes the application with all necessary components. @@ -465,7 +472,9 @@ class Application: """ Initializes and runs the scheduler for periodic updates. """ - self.scheduler = Scheduler(self.update_frequency, lambda: self.periodic_update(self.update_frequency)) + self.scheduler = Scheduler( + self.update_frequency, lambda: self.periodic_update(self.update_frequency) + ) self.scheduler.run() def run(self): @@ -474,9 +483,7 @@ class Application: and processing submissions. """ self.log_manager.info("Application started") - self.thread_manager = ThreadManager( - target=self.run_periodic_update, args=() - ) + self.thread_manager = ThreadManager(target=self.run_periodic_update, args=()) self.thread_manager.run() submissions = self.reddit_monitor.stream_submissions() self.submission_manager.process_submissions(submissions, self.update_frequency) diff --git a/scraper/config.py b/scraper/config.py index 3c19cfe..edf951f 100644 --- a/scraper/config.py +++ b/scraper/config.py @@ -12,4 +12,4 @@ class Config: USER_AGENT = "praw:zman.video_repost_bot:v0.1.0 (by u/jzman21)" DISABLE_WEBHOOK = False API_URL = "http://server:8000/api/" - UPDATE_FREQUENCY = 60 * 15 # 15 minutes + UPDATE_FREQUENCY = 60 * 15 # 15 minutes diff --git a/scraper/main.py b/scraper/main.py index 244a215..5c2b0cb 100644 --- a/scraper/main.py +++ b/scraper/main.py @@ -42,17 +42,14 @@ if __name__ == "__main__": post_manager, post_analytics_manager, submission_manager, - update_frequency + update_frequency, ) app.run() """ TODO: -- pull upvote ration into analytics? -- sqlite vs postgres figure out - basic front end (react) - tests -- logging - Filter out canadian/uk deals - track score and number of comments over time in db - try to identify product, number of cards, price per card, etc @@ -60,4 +57,5 @@ TODO: - try to identify platform ie. costco for gift card, tiktok for coupons, etc. - support for craigslist, ebay, etc. - front end - vizualization, classification, lookup, etc. +- postgres """ diff --git a/scraper/webhook.py b/scraper/webhook.py index 1767e3b..359f2e9 100644 --- a/scraper/webhook.py +++ b/scraper/webhook.py @@ -24,4 +24,4 @@ class WebhookNotifier: try: requests.post(self.webhook_url, data={"content": content}) except Exception as e: - self.log_manager.error(f"Failed to send notification: {e}") \ No newline at end of file + self.log_manager.error(f"Failed to send notification: {e}")