import pandas import urllib import time import sys import socket import json import csv import os import numpy import scipy from sklearn.linear_model import LogisticRegression class FRED_API: ##Initialize with FRED Key## def __init__(self, series_id): ##Unique FRED Access Key - You will need a key inorder to run the code self.FRED_API_Key = ??? self.series_id = series_id if type(self.series_id) == list: self.time_type = 4 for x in self.series_id: self.file_type = 'json' self.url = 'https://api.stlouisfed.org/fred/series?series_id=' \ + x + '&api_key=' + self.FRED_API_Key + '&file_type=' \ + self.file_type ##Get the Data failloop = 1 while True: try: req = urllib.request.Request(self.url) url_open = urllib.request.urlopen(req) data = url_open.read().decode('utf-8', errors='ignore') break except Exception as e: print(e) time.sleep(60) failloop += 1 if failloop > 5: print(Exception) sys.exit() ##Put Data into Pandas DataFrame data = json.loads(data) df = pandas.io.json.json_normalize(data['seriess']) df = df.convert_objects(convert_dates=True, convert_numeric=True) temp_time_type = df.frequency_short.values[0] if temp_time_type == 'A': temp_time_type = 0 elif temp_time_type == 'Q': temp_time_type = 1 elif temp_time_type == 'M': temp_time_type = 2 elif temp_time_type == 'W': temp_time_type = 3 elif temp_time_type == 'D': temp_time_type = 4 else: temp_time_type = 'UNKNOWN' if temp_time_type < self.time_type: self.time_type = temp_time_type ##Get Series Observations## def observations(self): if type(self.series_id) == list: total_df = pandas.DataFrame() if self.time_type == 0: for x in self.series_id: self.file_type = 'json' self.url = 'https://api.stlouisfed.org/fred/series/observations?series_id=' \ + x + '&api_key=' + self.FRED_API_Key + '&file_type=' \ + self.file_type ##Get the Data failloop = 1 while True: try: req = urllib.request.Request(self.url) url_open = urllib.request.urlopen(req) data = url_open.read().decode('utf-8', errors='ignore') break except Exception as e: print(e) time.sleep(60) failloop += 1 if failloop > 5: print(Exception) sys.exit() ##Put Data into Pandas DataFrame data = json.loads(data) df = pandas.io.json.json_normalize(data['observations']) df = df.convert_objects(convert_dates=True, convert_numeric=True) df['date'] = pandas.to_datetime(df['date']) df.index = df.date df = df.drop(['realtime_end', 'realtime_start', 'date'], axis=1) df.columns = [x] df = df.dropna() ##Bring up data to the desired time periods df = df.resample('A', how='mean') if total_df.empty: total_df = df else: total_df = total_df.merge(df,left_index=True,right_index=True) return total_df elif self.time_type == 1: for x in self.series_id: self.file_type = 'json' self.url = 'https://api.stlouisfed.org/fred/series/observations?series_id=' \ + x + '&api_key=' + self.FRED_API_Key + '&file_type=' \ + self.file_type ##Get the Data failloop = 1 while True: try: req = urllib.request.Request(self.url) url_open = urllib.request.urlopen(req) data = url_open.read().decode('utf-8', errors='ignore') break except Exception as e: print(e) time.sleep(60) failloop += 1 if failloop > 5: print(Exception) sys.exit() ##Put Data into Pandas DataFrame data = json.loads(data) df = pandas.io.json.json_normalize(data['observations']) df = df.convert_objects(convert_dates=True, convert_numeric=True) df['date'] = pandas.to_datetime(df['date']) df.index = df.date df = df.drop(['realtime_end', 'realtime_start', 'date'], axis=1) df.columns = [x] df = df.dropna() ##Bring up data to the desired time periods df = df.resample('QS', how='mean') if total_df.empty: total_df = df else: total_df = total_df.merge(df,left_index=True,right_index=True) return total_df elif self.time_type == 2: for x in self.series_id: self.file_type = 'json' self.url = 'https://api.stlouisfed.org/fred/series/observations?series_id=' \ + x + '&api_key=' + self.FRED_API_Key + '&file_type=' \ + self.file_type ##Get the Data failloop = 1 while True: try: req = urllib.request.Request(self.url) url_open = urllib.request.urlopen(req) data = url_open.read().decode('utf-8', errors='ignore') break except Exception as e: print(e) time.sleep(60) failloop += 1 if failloop > 5: print(Exception) sys.exit() ##Put Data into Pandas DataFrame data = json.loads(data) df = pandas.io.json.json_normalize(data['observations']) df = df.convert_objects(convert_dates=True, convert_numeric=True) df['date'] = pandas.to_datetime(df['date']) df.index = df.date df = df.drop(['realtime_end', 'realtime_start', 'date'], axis=1) df.columns = [x] df = df.dropna() ##Bring up data to the desired time periods df = df.resample('M', how='mean') if total_df.empty: total_df = df else: total_df = total_df.merge(df,left_index=True,right_index=True) return total_df elif self.time_type == 3: for x in self.series_id: self.file_type = 'json' self.url = 'https://api.stlouisfed.org/fred/series/observations?series_id=' \ + x + '&api_key=' + self.FRED_API_Key + '&file_type=' \ + self.file_type ##Get the Data failloop = 1 while True: try: req = urllib.request.Request(self.url) url_open = urllib.request.urlopen(req) data = url_open.read().decode('utf-8', errors='ignore') break except Exception as e: print(e) time.sleep(60) failloop += 1 if failloop > 5: print(Exception) sys.exit() ##Put Data into Pandas DataFrame data = json.loads(data) df = pandas.io.json.json_normalize(data['observations']) df = df.convert_objects(convert_dates=True, convert_numeric=True) df['date'] = pandas.to_datetime(df['date']) df.index = df.date df = df.drop(['realtime_end', 'realtime_start', 'date'], axis=1) df.columns = [x] df = df.dropna() ##Bring up data to the desired time periods df = df.resample('W', how='mean') if total_df.empty: total_df = df else: total_df = total_df.merge(df,left_index=True,right_index=True) return total_df elif self.time_type == 4: for x in self.series_id: self.file_type = 'json' self.url = 'https://api.stlouisfed.org/fred/series/observations?series_id=' \ + x + '&api_key=' + self.FRED_API_Key + '&file_type=' \ + self.file_type ##Get the Data failloop = 1 while True: try: req = urllib.request.Request(self.url) url_open = urllib.request.urlopen(req) data = url_open.read().decode('utf-8', errors='ignore') break except Exception as e: print(e) time.sleep(60) failloop += 1 if failloop > 5: print(Exception) sys.exit() ##Put Data into Pandas DataFrame data = json.loads(data) df = pandas.io.json.json_normalize(data['observations']) df = df.convert_objects(convert_dates=True, convert_numeric=True) df['date'] = pandas.to_datetime(df['date']) df.index = df.date df = df.drop(['realtime_end', 'realtime_start', 'date'], axis=1) df.columns = [x] df = df.dropna() if total_df.empty: total_df = df else: total_df = total_df.merge(df,left_index=True,right_index=True) return total_df else: return 'Unknown Time Type' else: self.file_type = 'json' self.url = 'https://api.stlouisfed.org/fred/series/observations?series_id=' \ + self.series_id + '&api_key=' + self.FRED_API_Key + '&file_type=' \ + self.file_type ##Get the Data failloop = 1 while True: try: req = urllib.request.Request(self.url) url_open = urllib.request.urlopen(req) data = url_open.read().decode('utf-8', errors='ignore') break except Exception as e: print(e) time.sleep(60) failloop += 1 if failloop > 5: print(Exception) sys.exit() ##Put Data into Pandas DataFrame data = json.loads(data) df = pandas.io.json.json_normalize(data['observations']) df = df.convert_objects(convert_dates=True, convert_numeric=True) df['date'] = pandas.to_datetime(df['date']) df.index = df.date df = df.drop(['realtime_end', 'realtime_start', 'date'], axis=1) df.columns = [self.series_id] df = df.dropna() return df def logit(x): return 1/(1 + numpy.exp(-x)) def get_CAPE(): location = 'http://www.econ.yale.edu/~shiller/data/ie_data.xls' df = pandas.read_excel(location, sheetname='Data', header=7) df = df[numpy.isfinite(df.CAPE)] df = df.dropna(axis=1, how='all') index = pandas.date_range('1881-1-1', periods=df.CAPE.shape[0], freq='M') df.index = index return df ##Get Employment and SP500 Data employ = FRED_API(['ICSA']) employ = employ.observations() employ = employ.resample('M', how='mean') CAPE = get_CAPE() sp500 = pandas.DataFrame(CAPE.Price) employ = pandas.merge(employ, sp500, how='inner', left_index=True, right_index=True) ##Get Jobless_Claims-to-Jobless_Claim_Average Ratio employ['claim_ratio'] = employ.ICSA / numpy.mean(employ.ICSA) - 1 ##Get the min and max spreads and create a list of values between them temp_column_drop = ['ICSA','Price'] ma_only_df = employ.drop(temp_column_drop, axis=1) max_ratio = ma_only_df.values.max() min_ratio = ma_only_df.values.min() number_intervals = 10 ratio_range = numpy.array([-.5,-.35,-.20,-.05,.1,.25,.4,.55,.70,.85])[:,numpy.newaxis] ##Loop through a range of months and find recession probabilities monthly_periods = [6,12,18,24,30,36,42,48] column_names = [] for loop_index,loop in enumerate(monthly_periods): ##Get Binary Response for Wilshire Index temp_emp = employ.copy() temp_emp['wilson_return'] = temp_emp['Price'].shift(-loop) / temp_emp['Price'] temp_emp = temp_emp.head(temp_emp.shape[0]-loop) temp_series = pandas.Series(numpy.where(temp_emp.wilson_return < .9,1,0), name='wilson_return_binary') temp_series.index = temp_emp.index temp_emp = pandas.concat([temp_emp,temp_series], axis=1) ##Get data to Regress x_data = numpy.array(temp_emp['claim_ratio'])[:,numpy.newaxis] y_data = numpy.array(temp_emp['wilson_return_binary']) ##Regression log_mod = LogisticRegression() log_mod_fit = log_mod.fit(x_data,y_data) ##Coefficients beta = float(log_mod.coef_) intercept = float(log_mod.intercept_) ##Find the predicted probabilities for each value in the spread range pred_values = logit(beta * ratio_range.ravel() + intercept) ##Create an array of the probabilities if loop_index == 0: recession_prob_list = numpy.array([pred_values]) else: recession_prob_list = numpy.append(recession_prob_list,[pred_values], axis=0) column_names.append(str(loop) + " Months") recession_prob_df = pandas.DataFrame(recession_prob_list.T) recession_prob_df.index = ratio_range recession_prob_df.columns = column_names