Organize data_collector to be similar to data_analyser

2 files changed, 23 insertions, 0 deletions

diff --git a/data_collector/__main__.py b/data_collector/__main__.py
new file mode 100755
index 0000000..11791c2
--- /dev/null
+++ b/data_collector/__main__.py
@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+from time import sleep
+from components.database import subscriptions
+
+from .utils import collect_data
+
+while True:
+    collect_data(subscriptions)
+    sleep(60)
diff --git a/data_collector/utils.py b/data_collector/utils.py
new file mode 100644
index 0000000..6ca5c69
--- /dev/null
+++ b/data_collector/utils.py
@@ -0,0 +1,13 @@
+from datetime import datetime, timedelta, UTC
+
+from pymongo.collection import Collection
+
+from components.subscriptions.main import Subscription
+from components.subscriptions.typing import SubsDict
+
+def collect_data(subs_collection: Collection[SubsDict]) -> None:
+    for sub_dict in subs_collection.find():
+        sub = Subscription(**sub_dict)
+        sub._collection = subs_collection
+        if datetime.now(tz=UTC) - sub.last_fetch > timedelta(seconds=sub.time_between_fetches):
+            sub.fetch()