diff options
author | James Maslek <jmaslek11@gmail.com> | 2023-08-18 14:07:48 -0400 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-08-18 18:07:48 +0000 |
commit | 3fcb64d5f2ce26f4f652feccbe81218c9bb89d57 (patch) | |
tree | ee6649498d79e6a98b1792c0cd57c759d8d162f4 | |
parent | 258ed9a11c7fe586f0f3be3e502c14b6a538df77 (diff) |
Fix the CPI CSV and remove twitter functionalities (#5350)
* remove twitter functions
* remove twitter functions and all references to it
* remove twitter functions
* remove twitter functions and all references to it
* Fix units on cpi
* crlf - lf
* missed one
* update ba menu integration test
---------
Co-authored-by: Danglewood <85772166+deeleeramone@users.noreply.github.com>
22 files changed, 259 insertions, 1025 deletions
diff --git a/openbb_terminal/common/behavioural_analysis/twitter_model.py b/openbb_terminal/common/behavioural_analysis/twitter_model.py deleted file mode 100644 index 034cc834113..00000000000 --- a/openbb_terminal/common/behavioural_analysis/twitter_model.py +++ /dev/null @@ -1,212 +0,0 @@ -"""Twitter Model""" -__docformat__ = "numpy" - -import logging -from datetime import datetime, timedelta -from typing import Optional - -import pandas as pd -from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer - -from openbb_terminal.core.session.current_user import get_current_user -from openbb_terminal.decorators import check_api_key, log_start_end -from openbb_terminal.helper_funcs import clean_tweet, get_data, request -from openbb_terminal.rich_config import console - -logger = logging.getLogger(__name__) - -analyzer = SentimentIntensityAnalyzer() - - -@log_start_end(log=logger) -@check_api_key(["API_TWITTER_BEARER_TOKEN"]) -def load_analyze_tweets( - symbol: str, - limit: int = 100, - start_date: Optional[str] = "", - end_date: Optional[str] = "", -) -> pd.DataFrame: - """Load tweets from twitter API and analyzes using VADER. - - Parameters - ---------- - symbol: str - Ticker symbol to search twitter for - limit: int - Number of tweets to analyze - start_date: Optional[str] - If given, the start time to get tweets from - end_date: Optional[str] - If given, the end time to get tweets from - - Returns - ------- - df_tweet: pd.DataFrame - Dataframe of tweets and sentiment - """ - params = { - "query": rf"(\${symbol}) (lang:en)", - "max_results": str(limit), - "tweet.fields": "created_at,lang", - } - - if start_date: - # Assign from and to datetime parameters for the API - params["start_time"] = start_date - if end_date: - params["end_time"] = end_date - - # Request Twitter API - response = request( - "https://api.twitter.com/2/tweets/search/recent", - params=params, # type: ignore - headers={ - "authorization": "Bearer " - + get_current_user().credentials.API_TWITTER_BEARER_TOKEN - }, - ) - - # Create dataframe - df_tweets = pd.DataFrame() - - # Check that the API response was successful - if response.status_code == 200: - tweets = [] - for tweet in response.json()["data"]: - row = get_data(tweet) - tweets.append(row) - df_tweets = pd.DataFrame(tweets) - elif response.status_code == 401: - console.print("Twitter API Key provided is incorrect\n") - return pd.DataFrame() - elif response.status_code == 400: - console.print( - """ - Status Code 400. - This means you are requesting data from beyond the API's 7 day limit""" - ) - return pd.DataFrame() - elif response.status_code == 403: - console.print( - f""" - Status code 403. - It seems you're twitter credentials are invalid - {response.text} - """ - ) - return pd.DataFrame() - else: - console.print( - f""" - Status code {response.status_code}. - Something went wrong - {response.text} - """ - ) - return pd.DataFrame() - - sentiments = [] - pos = [] - neg = [] - neu = [] - - for s_tweet in df_tweets["text"].to_list(): - tweet = clean_tweet(s_tweet, symbol) - sentiments.append(analyzer.polarity_scores(tweet)["compound"]) - pos.append(analyzer.polarity_scores(tweet)["pos"]) - neg.append(analyzer.polarity_scores(tweet)["neg"]) - neu.append(analyzer.polarity_scores(tweet)["neu"]) - # Add sentiments to tweets dataframe - df_tweets["sentiment"] = sentiments - df_tweets["positive"] = pos - df_tweets["negative"] = neg - df_tweets["neutral"] = neu - - return df_tweets - - -@log_start_end(log=logger) -def get_sentiment( - symbol: str, - n_tweets: int = 15, - n_days_past: int = 2, -) -> pd.DataFrame: - """Get sentiments from symbol. - - Parameters - ---------- - symbol: str - Stock ticker symbol to get sentiment for - n_tweets: int - Number of tweets to get per hour - n_days_past: int - Number of days to extract tweets for - - Returns - ------- - df_sentiment: pd.DataFrame - Dataframe of sentiment - """ - # Date format string required by twitter - dt_format = "%Y-%m-%dT%H:%M:%SZ" - - # Algorithm to extract - dt_recent = datetime.utcnow() - timedelta(seconds=20) - dt_old = dt_recent - timedelta(days=n_days_past) - console.print( - f"From {dt_recent.date()} retrieving {n_tweets*24} tweets ({n_tweets} tweets/hour)" - ) - - df_tweets = pd.DataFrame( - columns=[ - "created_at", - "text", - "sentiment", - "positive", - "negative", - "neutral", - ] - ) - while True: - # Iterate until we haven't passed the old number of days - if dt_recent < dt_old: - break - # Update past datetime - dt_past = dt_recent - timedelta(minutes=60) - - temp = load_analyze_tweets( - symbol, - n_tweets, - start_date=dt_past.strftime(dt_format), - end_date=dt_recent.strftime(dt_format), - ) - - if (isinstance(temp, pd.DataFrame) and temp.empty) or ( - not isinstance(temp, pd.DataFrame) and not temp - ): - return pd.DataFrame() - - df_tweets = pd.concat([df_tweets, temp]) - - if dt_past.day < dt_recent.day: - console.print( - f"From {dt_past.date()} retrieving {n_tweets*24} tweets ({n_tweets} tweets/hour)" - ) - - # Update recent datetime - dt_recent = dt_past - - # Sort tweets per date - df_tweets.sort_index(ascending=False, inplace=True) - df_tweets["cumulative_compound"] = df_tweets["sentiment"].cumsum() - df_tweets["prob_sen"] = 1 - - # df_tweets.to_csv(r'notebooks/tweets.csv', index=False) - df_tweets.reset_index(inplace=True) - df_tweets["Month"] = pd.to_datetime(df_tweets["created_at"]).apply( - lambda x: x.month - ) - df_tweets["Day"] = pd.to_datetime(df_tweets["created_at"]).apply(lambda x: x.day) - df_tweets["date"] = pd.to_datetime(df_tweets["created_at"]) - df_tweets = df_tweets.sort_values(by="date") - df_tweets["cumulative_compound"] = df_tweets["sentiment"].cumsum() - - return df_tweets diff --git a/openbb_terminal/common/behavioural_analysis/twitter_view.py b/openbb_terminal/common/behavioural_analysis/twitter_view.py deleted file mode 100644 index fba72770628..00000000000 --- a/openbb_terminal/common/behavioural_analysis/twitter_view.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Twitter view.""" -__docformat__ = "numpy" - -import logging -import os -from typing import Optional, Union - -import numpy as np -import pandas as pd -from dateutil import parser as dparse - -from openbb_terminal import OpenBBFigure, theme -from openbb_terminal.common.behavioural_analysis import twitter_model -from openbb_terminal.decorators import log_start_end -from openbb_terminal.helper_funcs import export_data, get_closing_price -from openbb_terminal.rich_config import console - -logger = logging.getLogger(__name__) - - -@log_start_end(log=logger) -def display_inference( - symbol: str, limit: int = 100, export: str = "", sheet_name: Optional[str] = None -): - """Prints Inference sentiment from past n tweets. - - Parameters - ---------- - symbol: str - Stock ticker symbol - limit: int - Number of tweets to analyze - sheet_name: str - Optionally specify the name of the sheet the data is exported to. - export: str - Format to export tweet dataframe - """ - df_tweets = twitter_model.load_analyze_tweets(symbol, limit) - - if (isinstance(df_tweets, pd.DataFrame) and df_tweets.empty) or ( - not isinstance(df_tweets, pd.DataFrame) and not df_tweets - ): - return - - # Parse tweets - dt_from = dparse.parse(df_tweets["created_at"].values[-1]) - dt_to = dparse.parse(df_tweets["created_at"].values[0]) - console.print(f"From: {dt_from.strftime('%Y-%m-%d %H:%M:%S')}") - console.print(f"To: {dt_to.strftime('%Y-%m-%d %H:%M:%S')}") - - console.print(f"{len(df_tweets)} tweets were analyzed.") - dt_delta = dt_to - dt_from - n_freq = dt_delta.total_seconds() / len(df_tweets) - console.print(f"Frequency of approx 1 tweet every {round(n_freq)} seconds.") - - pos = df_tweets["positive"] - neg = df_tweets["negative"] - - percent_pos = len(np.where(pos > neg)[0]) / len(df_tweets) - percent_neg = len(np.where(pos < neg)[0]) / len(df_tweets) - total_sent = np.round(np.sum(df_tweets["sentiment"]), 2) - mean_sent = np.round(np.mean(df_tweets["sentiment"]), 2) - console.print(f"The summed compound sentiment of {symbol} is: {total_sent}") - console.print(f"The average compound sentiment of {symbol} is: {mean_sent}") - console.print( - f"Of the last {len(df_tweets)} tweets, {100*percent_pos:.2f} % had a higher positive sentiment" - ) - console.print( - f"Of the last {len(df_tweets)} tweets, {100*percent_neg:.2f} % had a higher negative sentiment" - ) - - export_data( - export, - os.path.dirname(os.path.abspath(__file__)), - "infer", - df_tweets, - sheet_name, - ) - - -@log_start_end(log=logger) -def display_sentiment( - symbol: str, - n_tweets: int = 15, - n_days_past: int = 2, - compare: bool = False, - export: str = "", - sheet_name: Optional[str] = None, - external_axes: bool = False, -) -> Union[OpenBBFigure, None]: - """Plots sentiments from symbol - - Parameters - ---------- - symbol: str - Stock ticker symbol to get sentiment for - n_tweets: int - Number of tweets to get per hour - n_days_past: int - Number of days to extract tweets for - compare: bool - Show corresponding change in stock price - sheet_name: str - Optionally specify the name of the sheet the data is exported to. - export: str - Format to export tweet dataframe - external_axes: bool, optional - Whether to return the figure object or not, by default False - """ - - df_tweets = twitter_model.get_sentiment(symbol, n_tweets, n_days_past) - - if df_tweets.empty: - return None - - if compare: - plots_kwargs = dict( - rows=3, - cols=1, - shared_xaxes=True, - vertical_spacing=0.05, - row_heights=[0.2, 0.6, 0.2], - ) - else: - plots_kwargs = dict( - rows=2, - cols=1, - shared_xaxes=True, - vertical_spacing=0.05, - row_heights=[0.6, 0.4], - ) - - fig = OpenBBFigure.create_subplots(**plots_kwargs) # type: ignore - - fig.add_scatter( - x=pd.to_datetime(df_tweets["created_at"]), - y=df_tweets["cumulative_compound"].values, - row=1, - col=1, - ) - - fig.set_yaxis_title("<br>Cumulative<br>VADER Sentiment", row=1, col=1) - - for _, day_df in df_tweets.groupby(by="Day"): - day_df["time"] = pd.to_datetime(day_df["created_at"]) - day_df = day_df.sort_values(by="time") - fig.add_scatter( - x=day_df["time"], - y=day_df["sentiment"].cumsum(), - row=1, - col=1, - ) - fig.add_bar( - x=df_tweets["date"], - y=df_tweets["positive"], - row=2, - col=1, - marker_color=theme.up_color, - ) - - fig.add_bar( - x=df_tweets["date"], - y=-1 * df_tweets["negative"], - row=2, - col=1, - marker_color=theme.down_color, - ) - fig.set_yaxis_title("VADER Polarity Scores", row=2, col=1) - - if compare: - # get stock end price for each corresponding day if compare == True - closing_price_df = get_closing_price(symbol, n_days_past) - fig.add_scatter( - x=closing_price_df["Date"], - y=closing_price_df["Close"], - name=pd.to_datetime(closing_price_df["Date"]).iloc[0].strftime("%Y-%m-%d"), - row=3, - col=1, - ) - fig.set_yaxis_title("Stock Price", row=3, col=1) - - fig.update_layout( - title=f"Twitter's {symbol} total compound sentiment over time is {round(np.sum(df_tweets['sentiment']), 2)}", - xaxis=dict(type="date"), - showlegend=False, - ) - - export_data( - export, - os.path.dirname(os.path.abspath(__file__)), - "sentiment", - df_tweets, - sheet_name, - fig, - ) - - return fig.show(external=external_axes) diff --git a/openbb_terminal/core/sdk/controllers/stocks_sdk_controller.py b/openbb_terminal/core/sdk/controllers/stocks_sdk_controller.py index 676362d2e6e..4ddbb03f5a6 100644 --- a/openbb_terminal/core/sdk/controllers/stocks_sdk_controller.py +++ b/openbb_terminal/core/sdk/controllers/stocks_sdk_controller.py @@ -42,8 +42,6 @@ class StocksController(model.StocksRoot): `getdd`: Get due diligence posts from list of subreddits [Source: reddit].\n `headlines`: Gets Sentiment analysis provided by FinBrain's API [Source: finbrain].\n `headlines_chart`: Plots Sentiment analysis from FinBrain. Prints table if raw is True. [Source: FinBrain]\n - `infer`: Load tweets from twitter API and analyzes using VADER.\n - `infer_chart`: Prints Inference sentiment from past n tweets.\n `mentions`: Get interest over time from google api [Source: google].\n `mentions_chart`: Plots weekly bars of stock's interest over time. other users watchlist. [Source: Google].\n `messages`: Get last messages for a given ticker [Source: stocktwits].\n @@ -55,8 +53,6 @@ class StocksController(model.StocksRoot): `regions`: Get interest by region from google api [Source: google].\n `regions_chart`: Plots bars of regions based on stock's interest. [Source: Google].\n `rise`: Get top rising related queries with this stock's query [Source: google].\n - `sentiment`: Get sentiments from symbol.\n - `sentiment_chart`: Plots sentiments from symbol\n `snews`: Get headlines sentiment using VADER model over time. [Source: Finnhub]\n `snews_chart`: Display stock price and headlines sentiment using VADER model over time. [Source: Finnhub]\n `stalker`: Gets messages from given user [Source: stocktwits].\n diff --git a/openbb_terminal/core/sdk/models/keys_sdk_model.py b/openbb_terminal/core/sdk/models/keys_sdk_model.py index 0572e6d3ded..0ccee257bd4 100644 --- a/openbb_terminal/core/sdk/models/keys_sdk_model.py +++ b/openbb_terminal/core/sdk/models/keys_sdk_model.py @@ -42,7 +42,6 @@ class KeysRoot(Category): `stocksera`: Set Stocksera key.\n `tokenterminal`: Set Token Terminal key.\n `tradier`: Set Tradier key\n - `twitter`: Set Twitter key\n `ultima`: Set Ultima Insights key\n `walert`: Set Walert key\n """ @@ -84,6 +83,5 @@ class KeysRoot(Category): self.stocksera = lib.keys_model.set_stocksera_key self.tokenterminal = lib.keys_model.set_tokenterminal_key self.tradier = lib.keys_model.set_tradier_key - self.twitter = lib.keys_model.set_twitter_key self.ultima = lib.keys_model.set_ultima_key self.walert = lib.keys_model.set_walert_key diff --git a/openbb_terminal/core/sdk/models/stocks_sdk_model.py b/openbb_terminal/core/sdk/models/stocks_sdk_model.py index f0e4c74c4ac..193caf164e9 100644 --- a/openbb_terminal/core/sdk/models/stocks_sdk_model.py +++ b/openbb_terminal/core/sdk/models/stocks_sdk_model.py @@ -42,8 +42,6 @@ class StocksBehavioralAnalysis(Category): `getdd`: Get due diligence posts from list of subreddits [Source: reddit].\n `headlines`: Gets Sentiment analysis provided by FinBrain's API [Source: finbrain].\n `headlines_chart`: Plots Sentiment analysis from FinBrain. Prints table if raw is True. [Source: FinBrain]\n - `infer`: Load tweets from twitter API and analyzes using VADER.\n - `infer_chart`: Prints Inference sentiment from past n tweets.\n `mentions`: Get interest over time from google api [Source: google].\n `mentions_chart`: Plots weekly bars of stock's interest over time. other users watchlist. [Source: Google].\n `messages`: Get last messages for a given ticker [Source: stocktwits].\n @@ -55,8 +53,6 @@ class StocksBehavioralAnalysis(Category): `regions`: Get interest by region from google api [Source: google].\n `regions_chart`: Plots bars of regions based on stock's interest. [Source: Google].\n `rise`: Get top rising related queries with this stock's query [Source: google].\n - `sentiment`: Get sentiments from symbol.\n - `sentiment_chart`: Plots sentiments from symbol\n `snews`: Get headlines sentiment using VADER model over time. [Source: Finnhub]\n `snews_chart`: Display stock price and headlines sentiment using VADER model over time. [Source: Finnhub]\n `stalker`: Gets messages from given user [Source: stocktwits].\n @@ -74,8 +70,6 @@ class StocksBehavioralAnalysis(Category): self.getdd = lib.stocks_ba_reddit_model.get_due_dilligence self.headlines = lib.stocks_ba_finbrain_model.get_sentiment self.headlines_chart = lib.stocks_ba_finbrain_view.display_sentiment_analysis - self.infer = lib.stocks_ba_twitter_model.load_analyze_tweets - self.infer_chart = lib.stocks_ba_twitter_view.display_inference self.mentions = lib.stocks_ba_google_model.get_mentions self.mentions_chart = lib.stocks_ba_google_view.display_mentions self.messages = lib.stocks_ba_stocktwits_model.get_messages @@ -87,8 +81,6 @@ class StocksBehavioralAnalysis(Category): self.regions = lib.stocks_ba_google_model.get_regions self.regions_chart = lib.stocks_ba_google_view.display_regions self.rise = lib.stocks_ba_google_model.get_rise - self.sentiment = lib.stocks_ba_twitter_model.get_sentiment - self.sentiment_chart = lib.stocks_ba_twitter_view.display_sentiment self.snews = lib.stocks_ba_finnhub_model.get_headlines_sentiment self.snews_chart = ( lib.stocks_ba_finnhub_view.display_stock_price_headlines_sentiment diff --git a/openbb_terminal/core/sdk/sdk_init.py b/openbb_terminal/core/sdk/sdk_init.py index 0ec4b436d87..66050033423 100644 --- a/openbb_terminal/core/sdk/sdk_init.py +++ b/openbb_terminal/core/sdk/sdk_init.py @@ -121,8 +121,6 @@ from openbb_terminal.common.behavioural_analysis import ( reddit_view as stocks_ba_reddit_view, stocktwits_model as stocks_ba_stocktwits_model, stocktwits_view as stocks_ba_stocktwits_view, - twitter_model as stocks_ba_twitter_model, - twitter_view as stocks_ba_twitter_view, ) diff --git a/openbb_terminal/core/sdk/trail_map.csv b/openbb_terminal/core/sdk/trail_map.csv index 33ff8830bff..1f62a953e00 100644 --- a/openbb_terminal/core/sdk/trail_map.csv +++ b/openbb_terminal/core/sdk/trail_map.csv @@ -312,7 +312,6 @@ keys.smartstake,keys_model.set_smartstake_key, keys.stocksera,keys_model.set_stocksera_key, keys.tokenterminal,keys_model.set_tokenterminal_key, keys.tradier,keys_model.set_tradier_key, -keys.twitter,keys_model.set_twitter_key, keys.ultima,keys_model.set_ultima_key, keys.walert,keys_model.set_walert_key, login,sdk_session.login, @@ -384,7 +383,6 @@ stocks.ba.bullbear,stocks_ba_stocktwits_model.get_bullbear, stocks.ba.cnews,stocks_ba_finnhub_model.get_company_news, stocks.ba.getdd,stocks_ba_reddit_model.get_due_dilligence, stocks.ba.headlines,stocks_ba_finbrain_model.get_sentiment,stocks_ba_finbrain_view.display_sentiment_analysis -stocks.ba.infer,stocks_ba_twitter_model.load_analyze_tweets,stocks_ba_twitter_view.display_inference stocks.ba.mentions,stocks_ba_google_model.get_mentions,stocks_ba_google_view.display_mentions stocks.ba.messages,stocks_ba_stocktwits_model.get_messages, stocks.ba.ns,stocks_ba_news_sentiment_model.get_data,stocks_ba_news_sentiment_view.display_articles_data @@ -393,7 +391,6 @@ stocks.ba.queries,stocks_ba_google_model.get_queries, stocks.ba.redditsent,stocks_ba_reddit_model.get_posts_about, stocks.ba.regions,stocks_ba_google_model.get_regions,stocks_ba_google_view.display_regions stocks.ba.rise,stocks_ba_google_model.get_rise, -stocks.ba.sentiment,stocks_ba_twitter_model.get_sentiment,stocks_ba_twitter_view.display_sentiment stocks.ba.snews,stocks_ba_finnhub_model.get_headlines_sentiment,stocks_ba_finnhub_view.display_stock_price_headlines_sentiment stocks.ba.stalker,stocks_ba_stocktwits_model.get_stalker, stocks.ba.text_sent,stocks_ba_reddit_model.get_sentiment, diff --git a/openbb_terminal/economy/datasets/cpi.csv b/openbb_terminal/economy/datasets/cpi.csv index a209ce5869e..7671215ca5a 100644 --- a/openbb_terminal/economy/datasets/cpi.csv +++ b/openbb_terminal/economy/datasets/cpi.csv @@ -1,256 +1,256 @@ -title,country,frequency,units,series_id
-Consumer Price Index for Australia,australia,annual,growth_previous,CPALTT01AUA657N
-Consumer Price Index for Australia,australia,annual,growth_same,CPALTT01AUA659N
-Consumer Price Index for Australia,australia,quarterly,growth_previous,CPALTT01AUQ657N
-Consumer Price Index for Australia,australia,quarterly,growth_same,CPALTT01AUQ659N
-Consumer Price Index for Austria,austria,annual,growth_previous,CPALTT01ATA657N
-Consumer Price Index for Austria,austria,annual,growth_same,CPALTT01ATA659N
-Consumer Price Index for Austria,austria,monthly,growth_previous,CPALTT01ATM657N
-Consumer Price Index for Austria,austria,monthly,growth_same,CPALTT01ATM659N
-Consumer Price Index for Austria,austria,quarterly,growth_previous,CPALTT01ATQ657N
-Consumer Price Index for Austria,austria,quarterly,growth_same,CPALTT01ATQ659N
-Consumer Price Index for Belgium,belgium,annual,growth_previous,CPALTT01BEA657N
-Consumer Price Index for Belgium,belgium,annual,growth_same,CPALTT01BEA659N
-Consumer Price Index for Belgium,belgium,monthly,growth_previous,CPALTT01BEM657N
-Consumer Price Index for Belgium,belgium,monthly,growth_same,CPALTT01BEM659N
-Consumer Price Index for Belgium,belgium,quarterly,growth_previous,CPALTT01BEQ657N
-Consumer Price Index for Belgium,belgium,quarterly,growth_same,CPALTT01BEQ659N
-Consumer Price Index for Brazil,brazil,annual,growth_previous,CPALTT01BRA657N
-Consumer Price Index for Brazil,brazil,annual,growth_same,CPALTT01BRA659N
-Consumer Price Index for Brazil,brazil,monthly,growth_previous,CPALTT01BRM657N
-Consumer Price Index for Brazil,brazil,monthly,growth_same,CPALTT01BRM659N
-Consumer Price Index for Brazil,brazil,quarterly,growth_previous,CPALTT01BRQ6 |