diff --git a/api/config.py b/api/config.py index 62ad827..f457bf8 100644 --- a/api/config.py +++ b/api/config.py @@ -63,6 +63,8 @@ class Config: CVTRACK_STATES_URL = "https://covidtracking.com/api/states" TMP_URL = "https://coronavirus-19-api.herokuapp.com/countries/USA" COUNTY_URL = config("COUNTY_URL") + STATE_CONFIRMED = config("STATE_CONFIRMED") + STATE_DEATH = config("STATE_DEATH") class ProductionConfig(Config): diff --git a/api/endpoints.py b/api/endpoints.py index 31e3ca6..d7c14ab 100644 --- a/api/endpoints.py +++ b/api/endpoints.py @@ -16,6 +16,7 @@ from api.utils import read_county_data from api.utils import read_country_data from api.utils import read_county_stats +from api.utils import read_states from cachetools import cached, TTLCache @@ -82,6 +83,7 @@ def get_gnews() -> JSONResponse: try: data = get_us_news() json_data = {"success": True, "message": data} + del data except Exception as ex: return JSONResponse( status_code=404, content={"message": f"[Error] get /News API: {ex}"} @@ -102,6 +104,7 @@ def post_gnews(news: NewsInput) -> JSONResponse: state = reverse_states_map[news.state] data = get_state_topic_google_news(state, news.topic) json_data = {"success": True, "message": data} + del data except Exception as ex: return JSONResponse( status_code=404, content={"message": f"[Error] post /News API: {ex}"} @@ -110,6 +113,91 @@ def post_gnews(news: NewsInput) -> JSONResponse: return json_data +############################################################################### +# +# Twitter Feed Endpoints +# +################################################################################ +class TwitterInput(BaseModel): + state: str = "CA" + + +class Tweets(BaseModel): + tweet_id: int + full_text: str + created_at: datetime + + +class UserTweets(BaseModel): + username: str + full_name: str + tweets: List[Tweets] = None + + +class TwitterOutput(BaseModel): + success: bool + message: UserTweets + + +@router.get( + "/twitter", response_model=TwitterOutput, responses={404: {"model": Message}} +) +def get_twitter() -> JSONResponse: + """Fetch and return Twitter data from MongoDB connection. + + :param: none + :return: str + """ + try: + doc = tm.get_tweet_by_state("US") + username = doc["username"] + full_name = doc["full_name"] + tweets = doc["tweets"] + + # 2020-03-19 triage. lots of empty list at the end of tweets, filtering them out + tweets = [*filter(None, tweets)] + tweets = sorted(tweets, key=lambda i: i["created_at"], reverse=True) + + json_data = { + "success": True, + "message": {"username": username, "full_name": full_name, "tweets": tweets}, + } + del tweets + except Exception as ex: + raise HTTPException(status_code=404, detail=f"[Error] get /twitter API: {ex}") + + return json_data + + +@router.post( + "/twitter", response_model=TwitterOutput, responses={404: {"model": Message}} +) +def post_twitter(twyuser: TwitterInput) -> JSONResponse: + """Fetch and return Twitter data from MongoDB connection. + + :param: none. Two letter state abbreviation. + :return: str + """ + try: + doc = tm.get_tweet_by_state(twyuser.state) + username = doc["username"] + full_name = doc["full_name"] + tweets = doc["tweets"] + # 2020-03-19 triage. lots of empty list at the end of tweets, filtering them out + tweets = [*filter(None, tweets)] + tweets = sorted(tweets, key=lambda i: i["created_at"], reverse=True) + json_data = { + "success": True, + "message": {"username": username, "full_name": full_name, "tweets": tweets}, + } + + del tweets + except Exception as ex: + raise HTTPException(status_code=404, detail=f"[Error] post /twitter API: {ex}") + + return json_data + + ############################################################################### # # County Endpoints @@ -152,6 +240,7 @@ def get_county_data() -> JSONResponse: try: data = read_county_data() json_data = {"success": True, "message": data} + del data except Exception as ex: raise HTTPException(status_code=404, detail=f"[Error] get '/county' API: {ex}") @@ -171,6 +260,7 @@ def post_county(county: CountyInput) -> JSONResponse: try: data = read_county_stats(county.state, county.county) json_data = {"success": True, "message": data} + del data except Exception as ex: raise HTTPException(status_code=404, detail=f"[Error] get '/county' API: {ex}") @@ -179,137 +269,41 @@ def post_county(county: CountyInput) -> JSONResponse: ############################################################################### # -# Stats Endpoints +# State Endpoint # ################################################################################ -class StatsInput(BaseModel): - state: str = "CA" - - -class Stats(BaseModel): - tested: int - todays_tested: int - confirmed: int - todays_confirmed: int - deaths: int - todays_deaths: int - - -class StatsOutput(BaseModel): - success: bool - message: Stats - - -@router.get("/stats", response_model=StatsOutput, responses={404: {"model": Message}}) -def get_stats() -> JSONResponse: - """Get overall tested, confirmed, and deaths stats from the database - and return it as a json string. For the top bar. - - :param: none. - :return: JSONResponse - """ - try: - data = get_daily_stats() - json_data = {"success": True, "message": data} - except Exception as ex: - raise HTTPException(status_code=404, detail=f"[Error] get /stats API: {ex}") - return json_data - - -@router.post("/stats", response_model=StatsOutput, responses={404: {"model": Message}}) -def post_stats(stats: StatsInput) -> JSONResponse: - """Get overall tested, confirmed, and deaths stats from the database - and return it as a json string. For the top bar. - - :param: Stats - :return: JSONResponse - """ - try: - data = get_daily_state_stats(stats.state) - json_data = {"success": True, "message": data} - except Exception as ex: - raise HTTPException(status_code=404, detail=f"[Error] post /stats API: {ex}") - return json_data +class StateInput(BaseModel): + stateAbbr: str -############################################################################### -# -# Twitter Feed Endpoints -# -################################################################################ -class TwitterInput(BaseModel): - state: str = "CA" - - -class Tweets(BaseModel): - tweet_id: int - full_text: str - created_at: datetime - - -class UserTweets(BaseModel): - username: str - full_name: str - tweets: List[Tweets] = None +class State(BaseModel): + Date: str + Confirmed: int + Deaths: int -class TwitterOutput(BaseModel): +class StateOutput(BaseModel): success: bool - message: UserTweets - - -@router.get( - "/twitter", response_model=TwitterOutput, responses={404: {"model": Message}} -) -def get_twitter() -> JSONResponse: - """Fetch and return Twitter data from MongoDB connection. - - :param: none - :return: str - """ - try: - doc = tm.get_tweet_by_state("US") - username = doc["username"] - full_name = doc["full_name"] - tweets = doc["tweets"] - - # 2020-03-19 triage. lots of empty list at the end of tweets, filtering them out - tweets = [*filter(None, tweets)] - tweets = sorted(tweets, key=lambda i: i["created_at"], reverse=True) - - json_data = { - "success": True, - "message": {"username": username, "full_name": full_name, "tweets": tweets}, - } - except Exception as ex: - raise HTTPException(status_code=404, detail=f"[Error] get /twitter API: {ex}") - - return json_data + message: List[State] +@cached(cache=TTLCache(maxsize=3, ttl=3600)) @router.post( - "/twitter", response_model=TwitterOutput, responses={404: {"model": Message}} + "/state", response_model=StateOutput, responses={404: {"model": Message}} ) -def post_twitter(twyuser: TwitterInput) -> JSONResponse: - """Fetch and return Twitter data from MongoDB connection. +def post_state(state: StateInput) -> JSONResponse: + """Fetch state level data time series for a single state, ignoring the + unattributed and out of state cases. - :param: none. Two letter state abbreviation. - :return: str + Input: two letter states code """ + try: - doc = tm.get_tweet_by_state(twyuser.state) - username = doc["username"] - full_name = doc["full_name"] - tweets = doc["tweets"] - # 2020-03-19 triage. lots of empty list at the end of tweets, filtering them out - tweets = [*filter(None, tweets)] - tweets = sorted(tweets, key=lambda i: i["created_at"], reverse=True) - json_data = { - "success": True, - "message": {"username": username, "full_name": full_name, "tweets": tweets}, - } + data = read_states(state.stateAbbr) + json_data = {"success": True, "message": data} + del data except Exception as ex: - raise HTTPException(status_code=404, detail=f"[Error] post /twitter API: {ex}") + raise HTTPException(status_code=404, detail=f"[Error] get /country API: {ex}") return json_data @@ -352,6 +346,63 @@ def get_country(country: CountryInput) -> JSONResponse: return json_data +############################################################################### +# +# Stats Endpoints +# +################################################################################ +class StatsInput(BaseModel): + state: str = "CA" + + +class Stats(BaseModel): + tested: int + todays_tested: int + confirmed: int + todays_confirmed: int + deaths: int + todays_deaths: int + + +class StatsOutput(BaseModel): + success: bool + message: Stats + + +@router.get("/stats", response_model=StatsOutput, responses={404: {"model": Message}}) +def get_stats() -> JSONResponse: + """Get overall tested, confirmed, and deaths stats from the database + and return it as a json string. For the top bar. + + :param: none. + :return: JSONResponse + """ + try: + data = get_daily_stats() + json_data = {"success": True, "message": data} + except Exception as ex: + raise HTTPException(status_code=404, detail=f"[Error] get /stats API: {ex}") + return json_data + + +@router.post("/stats", response_model=StatsOutput, responses={404: {"model": Message}}) +def post_stats(stats: StatsInput) -> JSONResponse: + """Get overall tested, confirmed, and deaths stats from the database + and return it as a json string. For the top bar. + + :param: Stats + :return: JSONResponse + """ + try: + data = get_daily_state_stats(stats.state) + json_data = {"success": True, "message": data} + except Exception as ex: + raise HTTPException(status_code=404, detail=f"[Error] post /stats API: {ex}") + return json_data + + + + # get: diff --git a/api/utils/__init__.py b/api/utils/__init__.py index e8be5f4..f53fb57 100644 --- a/api/utils/__init__.py +++ b/api/utils/__init__.py @@ -1,9 +1,10 @@ -from .states import reverse_states_map +from .dicts import reverse_states_map from .gnews import get_state_topic_google_news, get_us_news from .stats import get_daily_stats, get_daily_state_stats -from .states import reverse_states_map +from .dicts import reverse_states_map from .twitter_mongo import TwitterMongo from .county import read_county_data from .country import read_country_data from .county_mongo import StateMongo from .county import read_county_stats +from .state import read_states diff --git a/api/utils/states.py b/api/utils/dicts.py similarity index 100% rename from api/utils/states.py rename to api/utils/dicts.py diff --git a/api/utils/state.py b/api/utils/state.py new file mode 100644 index 0000000..887f287 --- /dev/null +++ b/api/utils/state.py @@ -0,0 +1,35 @@ +from typing import Dict +import pandas as pd +from api.config import Config +from api.utils import reverse_states_map + + +def read_states(state:str) -> pd.DataFrame: + """read date, confirmed, and death info of a state and return it as + a dictionary + """ + + state = reverse_states_map[state] + + data = pd.read_csv(Config.STATE_CONFIRMED) + data = data[data['FIPS'] < 79999] + deaths = pd.read_csv(Config.STATE_DEATH) + deaths = deaths[deaths['FIPS'] < 79999] + + data = data[data['Province_State'] == state] + data = pd.DataFrame(data.aggregate('sum')[11:], columns=['Confirmed']) + + deaths = deaths[deaths['Province_State'] == state] + deaths = pd.DataFrame(deaths.aggregate('sum')[12:]) + + data['Deaths'] = deaths + del deaths + + data = data.reset_index() + data.columns = ['Date', 'Confirmed', 'Deaths'] + data = data.fillna(0) + # print(data.head()) + # print(data.tail()) + data = pd.DataFrame.to_dict(data, orient="records") + + return data \ No newline at end of file diff --git a/api/utils/stats.py b/api/utils/stats.py index c7ce3e3..800d6e5 100644 --- a/api/utils/stats.py +++ b/api/utils/stats.py @@ -61,6 +61,8 @@ def get_daily_stats() -> Dict: "todays_deaths": todays_deaths, } + del data, data2 + return stats @@ -117,4 +119,6 @@ def get_daily_state_stats(state: str) -> Dict: "todays_deaths": todays_deaths, } + del df, data + return stats