Coded a petition watcher that saves stuff to json

.gitignore

@@ -0,0 +1,4 @@
+*.log
+*.json
+*.zip
+__pycache__/

LICENSE.md

@@ -0,0 +1,13 @@
+Copyright © 2022 <Ash Entwisle>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+> The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+> You give credit to the creators of the project by providing a link to this repository
+
+
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+

README.md

@@ -0,0 +1,20 @@
+# Depencencies
+
+> requests `pip install requests`
+
+# setup guide
+
+ 1. Open a terminal and cd into your project directory
+ 2. Run `git clone [url] .`
+ 3. Run `py -3 -m launcher.py`
+ 4. Enter the URL of the petition you want to monitor
+
+
+# Disclaimer 
+
+I have not commented the code as of yet. I may in future, but icba atm. 
+`src.lib.rq` is a basic requests wrapper for what I need for this project. 
+`src.lib.petitionwatcher` handles all the data gathering from the response. 
+`src.lib.util` has all the utility functions and classes in that dont need there own module (such as log). 
+That should be it. If anything is wrong, open a pull req or @ me on twitter 
+

launcher.py

@@ -0,0 +1,9 @@
+from src.main import main
+from src.lib.util import getInput, log
+
+url = getInput("Enter URL")
+if "https://petition.parliament.uk/archived/petitions/" in url or "https://petition.parliament.uk/petitions/" in url:
+ log("Valid URL")
+ main(url)
+else:
+ log("Invalid URL")

src/lib/petitionwatcher/__init__.py

@@ -0,0 +1,34 @@
+from src.lib.util import log
+from src.lib.rq import GetReq
+from datetime import datetime
+import json 
+
+
+class PetitionWatcher(GetReq):
+ def __init__(self, url: str):
+ self.url = url
+ super().__init__(self.checkifJSON())
+ if self.isok:
+ #log("getting data...")
+ self._data = self._json["data"]
+ self._attributes = self._data["attributes"]
+ self.id = self._data["id"]
+ self.title = self._attributes["action"]
+ self.state = self._attributes["state"]
+ self.signatures = self._attributes["signature_count"]
+ self.creator = self._attributes["creator_name"]
+
+
+ def checkifJSON(self):
+ # checks if url is a json file
+ if self.url.endswith(".json"):
+ return self.url
+ else:
+ return f"{self.url}.json"
+
+ def dumpJSON(self):
+ log("Dumping JSON")
+ timestamp = datetime.now().strftime("%H%M%S%Y%m%d")
+ with open(f"./src/data/json/{timestamp}.json", "w") as f:
+ json.dump(self._data, f)
+

src/lib/rq/__init__.py

@@ -0,0 +1,38 @@
+import requests as rq
+from src.lib.util import log
+
+
+class GetReq():
+ def __init__(self, url):
+ #log("initialising connection...")
+ # gets url to be used later
+ self.url = url
+ # makes HTTP request to url
+ self.__data = rq.get(url)
+ # checks if data is ok
+ self.isok = self.__data.ok
+
+ if self.isok:
+ #log("connection established")
+ # extracts status code from http req
+ self.code = self.__data.status_code
+ # checks if code == 404
+ self.is404 = self.__isNotFound()
+ # extracts html data from http req
+ self._body = self.__data.text
+ # extracts json data from http req
+ self._json = self.__data.json()
+
+ # makes sure connection is closed
+ self.__data.close()
+
+ # checks if status code is 404
+ def __isNotFound(self):
+ if self.code == 404:
+ self.__data.close()
+ return True
+
+ def close(self):
+ self.__data.close()
+
+

src/lib/util/__init__.py

@@ -0,0 +1,33 @@
+from datetime import datetime
+
+
+def log(info, silent: bool = False):
+ # gets current date and time
+ timenow = datetime.now().strftime("%H:%M:%S")
+ datenow = datetime.now().strftime("%Y-%m-%d")
+ # opens log file for the day
+ with open(f"./src/data/logs/{datenow}.log", "a") as logfile:
+ # formats the input data
+ data = f"[{timenow}] {info}"
+ # writes data to file
+ logfile.write(f"\n {data}")
+ # outputs data to screen
+ if not silent:
+ print(data)
+
+
+# input formatting
+def getInput(txt: str):
+ data = input(f"[--:--:--] {txt} >> ")
+ log(f"{txt} >> {data}", silent=True)
+ return data
+
+
+# turns y/n into bool
+def ynToTF(inp: str):
+ if inp.upper() == "Y":
+ return True
+ else:
+ return False
+
+

src/main.py

@@ -0,0 +1,19 @@
+from src.lib.util import log
+from src.lib.petitionwatcher import PetitionWatcher as pw
+import time
+
+def main(url: str):
+ req = pw(url)
+ log(f"Petition ID: {req.id} by: {req.creator} status: {req.state}")
+ log(f"Title: {req.title}")
+ req.close()
+ count = 0
+ while True:
+ if time.localtime().tm_sec %12 == 0:
+ count += 1
+ req = pw(url)
+ log(f"Signatures: {req.signatures} ({req.signatures*100//100000}%) status: {req.state}")
+ if count % 10 == 0:
+ req.dumpJSON()
+ req.close()
+ time.sleep(11)

src/secrets/__init__.py

@@ -0,0 +1,14 @@
+from src.lib.util import log
+
+keycode = [
+ "TEST1",
+ "TEST2"
+]
+
+def getsecret(code):
+ log(f"fetching key ({code})")
+ code = int(keycode.index(code))
+ with open("./src/secrets/secrets.tkn", "r") as scr:
+ return scr.readlines()[code]
+
+

src/secrets/secrets.tkn

@@ -0,0 +1,2 @@
+This is a secret!
+Dont tell anyone!