From 7035fe69dc3f17f94b95bae9ce22d78ed244be86 Mon Sep 17 00:00:00 2001 From: moshferatu Date: Wed, 25 Oct 2023 06:31:23 -0700 Subject: [PATCH] Backfill script for news events --- news/backfill_news.py | 9 +++++++++ news/requirements.txt | 5 +++++ 2 files changed, 14 insertions(+) create mode 100644 news/backfill_news.py create mode 100644 news/requirements.txt diff --git a/news/backfill_news.py b/news/backfill_news.py new file mode 100644 index 0000000..3d76eb3 --- /dev/null +++ b/news/backfill_news.py @@ -0,0 +1,9 @@ +from database.news import insert +from datetime import datetime +from news import scrape_news_in_range + +start_date = datetime(2023, 10, 25) +end_date = datetime(2023, 10, 25) +scrape_news_in_range(start_date, end_date) +for news_data in scrape_news_in_range(start_date, end_date): + insert(news_data) \ No newline at end of file diff --git a/news/requirements.txt b/news/requirements.txt new file mode 100644 index 0000000..00e235a --- /dev/null +++ b/news/requirements.txt @@ -0,0 +1,5 @@ +pandas +psycopg2 +python-dotenv +schedule +sqlalchemy \ No newline at end of file