I developed a script and encountering error. Need inputs to finalize script. I am using .csv file in script to analysize future price of #bitcoin
-------------
-------------
# Import Libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import joblib
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from statsmodels.tsa.arima.model import ARIMA
from tensorflow.keras.models import Sequential, load_model
from tensorflow.keras.layers import LSTM, Dense
from prophet import Prophet
import xgboost as xgb
import torch
import torch.nn as nn
import torch.optim as optim
import pytorch_lightning as pl
from torch.utils.data import DataLoader, Dataset, random_split
import pytorch_forecasting
from pytorch_forecasting.data import TimeSeriesDataSet, GroupNormalizer
from pytorch_forecasting.models import TemporalFusionTransformer
import optuna
import shap
# Load Dataset
file_path = '/content/drive/MyDrive/Colab Notebooks/bitcoin_prices' # Update this path
df = pd.read_excel(file_path)
# Data Validation: Handle missing values, outliers, or anomalies
df = df.dropna() # Drop missing values for simplicity
# Feature Engineering
df['Daily_Return'] = df['Close'].pct_change()
df['Log_Return'] = np.log(df['Close'] / df['Close'].shift(1))
df['MA_5'] = df['Close'].rolling(window=5).mean()
df['MA_10'] = df['Close'].rolling(window=10).mean()
df['EMA_5'] = df['Close'].ewm(span=5, adjust=False).mean()
df['EMA_10'] = df['Close'].ewm(span=10, adjust=False).mean()
df['BB_Middle'] = df['Close'].rolling(window=20).mean()
df['BB_Upper'] = df['BB_Middle'] + 2*df['Close'].rolling(window=20).std()
df['BB_Lower'] = df['BB_Middle'] - 2*df['Close'].rolling(window=20).std()
df['Volatility'] = df['Daily_Return'].rolling(window=10).std()
df['Momentum'] = df['Close'] - df['Close'].shift(10)
df['VWAP'] = (df['Volume'] * (df['High'] + df['Low'] + df['Close']) / 3).cumsum() / df['Volume'].cumsum()
delta = df['Close'].diff(1)
gain = delta.where(delta > 0, 0)
loss = -delta.where(delta < 0, 0)
avg_gain = gain.rolling(window=14).mean()
avg_loss = loss.rolling(window=14).mean()
rs = avg_gain / avg_loss
df['RSI'] = 100 - (100 / (1 + rs))
ema_12 = df['Close'].ewm(span=12, adjust=False).mean()
ema_26 = df['Close'].ewm(span=26, adjust=False).mean()
df['MACD'] = ema_12 - ema_26
df['MACD_Signal'] = df['MACD'].ewm(span=9, adjust=False).mean()
# Dynamic Time Features
df['day_of_week'] = df['Timestamp'].dt.dayofweek
df['month'] = df['Timestamp'].dt.month
# Drop rows with NaN values after feature engineering
df = df.dropna()
# ARIMA Model
arima_df = df.set_index('Timestamp')['Close']
arima_model = ARIMA(arima_df, order=(5, 1, 0))
arima_model_fit = arima_model.fit()
arima_forecast = arima_model_fit.forecast(steps=30)
arima_model_fit.save('arima_model.pkl')
# Prophet Model
prophet_df = df[['Timestamp', 'Close']].rename(columns={'Timestamp': 'ds', 'Close': 'y'})
prophet_model = Prophet()
prophet_model.fit(prophet_df)
future = prophet_model.make_future_dataframe(periods=30)
prophet_forecast = prophet_model.predict(future)
# Save the Prophet model using joblib
joblib.dump(prophet_model, 'prophet_model.pkl')
# XGBoost Model
df['Lag_1'] = df['Close'].shift(1)
df['Lag_2'] = df['Close'].shift(2)
df['Lag_3'] = df['Close'].shift(3)
df = df.dropna()
X = df[['Lag_1', 'Lag_2', 'Lag_3', 'day_of_week', 'month']]
y = df['Close']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
xgb_model = xgb.XGBRegressor(objective='reg:squarederror', n_estimators=100, random_state=42)
xgb_model.fit(X_train, y_train)
xgb_forecast = xgb_model.predict(X_test)
joblib.dump(xgb_model, 'xgb_model.pkl')
# SHAP for XGBoost Explainability
shap.initjs()
explainer = shap.Explainer(xgb_model)
shap_values = explainer(X_test)
shap.summary_plot(shap_values, X_test)
# LSTM Model
features = ['Open', 'High', 'Low', 'Volume', 'Daily_Return', 'Log_Return', 'MA_5', 'MA_10', 'EMA_5', 'EMA_10',
'BB_Middle', 'BB_Upper', 'BB_Lower', 'Volatility', 'Momentum', 'VWAP', 'RSI', 'MACD', 'MACD_Signal']
scaler = MinMaxScaler()
df[features] = scaler.fit_transform(df[features])
df['Close'] = scaler.fit_transform(df[['Close']])
sequence_length = 50
X = []
y = []
for i in range(sequence_length, len(df)):
X.append(df[features].iloc[i-sequence_length:i].values)
y.append(df['Close'].iloc[i])
X = np.array(X)
y = np.array(y)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
def create_lstm_model():
model = Sequential([
LSTM(50, activation='relu', return_sequences=True, input_shape=(X_train.shape[1], X_train.shape[2])),
LSTM(50, activation='relu'),
Dense(1)
])
model.compile(optimizer='adam', loss='mse')
return model
# Hyperparameter Tuning for LSTM using Optuna
def objective(trial):
lstm_units = trial.suggest_int('lstm_units', 10, 100)
batch_size = trial.suggest_int('batch_size', 16, 128)
epochs = trial.suggest_int('epochs', 10, 50)
model = Sequential([
LSTM(lstm_units, activation='relu', return_sequences=True, input_shape=(X_train.shape[1], X_train.shape[2])),
LSTM(lstm_units, activation='relu'),
Dense(1)
])
model.compile(optimizer='adam', loss='mse')
model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, validation_split=0.2, verbose=0)
loss = model.evaluate(X_test, y_test, verbose=0)
return loss
study = optuna.create_study(direction='minimize')
study.optimize(objective, n_trials=50)
best_params = study.best_params
print("Best hyperparameters: ", best_params)
lstm_model = Sequential([
LSTM(best_params['lstm_units'], activation='relu', return_sequences=True, input_shape=(X_train.shape[1], X_train.shape[2])),
LSTM(best_params['lstm_units'], activation='relu'),
Dense(1)
])
lstm_model.compile(optimizer='adam', loss='mse')
lstm_model.fit(X_train, y_train, epochs=best_params['epochs'], batch_size=best_params['batch_size'], validation_split=0.2)
lstm_model.save('lstm_model.h5')
y_pred_lstm = lstm_model.predict(X_test)
last_sequence = df[features].iloc[-sequence_length:].values
future_predictions = []
for _ in range(30):
last_sequence = last_sequence.reshape((1, sequence_length, len(features)))
next_pred = lstm_model.predict(last_sequence)
future_predictions.append(next_pred[0, 0])
next_pred_repeated = np.repeat(next_pred, len(features)).reshape(1, 1, len(features))
new_sequence = np.append(last_sequence[:, 1:, :], next_pred_repeated, axis=1)
last_sequence = new_sequence
future_predictions = scaler.inverse_transform(np.array(future_predictions).reshape(-1, 1))
# Transformer Model
class TimeSeriesDataset(Dataset):
def __init__(self, data, seq_length):
self.data = data
self.seq_length = seq_length
def __len__(self):
return len(self.data) - self.seq_length
def __getitem__(self, idx):
x = self.data[idx:idx+self.seq_length]
y = self.data[idx+self.seq_length]
return torch.tensor(x, dtype=torch.float32).unsqueeze(-1), torch.tensor(y, dtype=torch.float32).unsqueeze(-1) # Ensure 3D shape
class TransformerModel(pl.LightningModule):
def __init__(self, input_dim, seq_length, num_heads, num_layers, dim_feedforward, output_dim):
super(TransformerModel, self).__init__()
self.seq_length = seq_length
self.input_dim = input_dim
self.transformer = nn.Transformer(input_dim, num_heads, num_layers, dim_feedforward)
self.fc = nn.Linear(input_dim * seq_length, output_dim)
def forward(self, x):
print(x.shape) # Debug statement to check shape
if x.ndim == 2: # Add a feature dimension if missing
x = x.unsqueeze(-1)
x = x.permute(1, 0, 2) # [sequence_length, batch_size, input_dim]
x = self.transformer(x)
x = x.permute(1, 0, 2) # [batch_size, sequence_length, output_dim]
x = x.reshape(x.size(0), -1)
x = self.fc(x)
return x
def training_step(self, batch, batch_idx):
x, y = batch
y_hat = self(x)
loss = nn.MSELoss()(y_hat, y)
return loss
def configure_optimizers(self):
return optim.Adam(self.parameters(), lr=0.001)
data = df['Close'].values
dataset = TimeSeriesDataset(data, sequence_length)
train_size = int(0.8 * len(dataset))
val_size = len(dataset) - train_size
train_dataset, val_dataset = random_split(dataset, [train_size, val_size])
train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=32)
input_dim = 1 # Use 1 for single feature input
output_dim = 1
num_heads = 1 # Use 1 since input_dim is 1
num_layers = 4
dim_feedforward = 128
transformer_model = TransformerModel(input_dim, sequence_length, num_heads, num_layers, dim_feedforward, output_dim)
trainer = pl.Trainer(max_epochs=10)
trainer.fit(transformer_model, train_loader, val_loader)
torch.save(transformer_model.state_dict(), 'transformer_model.pth')
transformer_model.eval()
test_data = data[-sequence_length:].reshape(1, sequence_length, 1)
test_data = torch.tensor(test_data, dtype=torch.float32)
with torch.no_grad():
transformer_predictions = transformer_model(test_data).numpy()
transformer_predictions = scaler.inverse_transform(transformer_predictions)
# Temporal Fusion Transformer (TFT)
df['time_idx'] = df.index
df['group'] = 0
max_encoder_length = 60
max_prediction_length = 30
training_cutoff = df["time_idx"].max() - max_prediction_length
training = TimeSeriesDataSet(
df[lambda x: x.time_idx <= training_cutoff],
time_idx="time_idx",
target="Close",
group_ids=["group"],
min_encoder_length=max_encoder_length // 2,
max_encoder_length=max_encoder_length,
min_prediction_length=1,
max_prediction_length=max_prediction_length,
static_categoricals=["group"],
time_varying_known_reals=["time_idx"],
time_varying_unknown_reals=["Close"],
target_normalizer=GroupNormalizer(groups=["group"]),
)
batch_size = 64
train_dataloader = training.to_dataloader(train=True, batch_size=batch_size, num_workers=0)
# Hyperparameter Tuning for TFT using Optuna
def tft_objective(trial):
hidden_size = trial.suggest_int('hidden_size', 8, 64)
attention_head_size = trial.suggest_int('attention_head_size', 1, 4)
dropout = trial.suggest_float('dropout', 0.1, 0.5)
hidden_continuous_size = trial.suggest_int('hidden_continuous_size', 8, 64)
tft = TemporalFusionTransformer.from_dataset(
training,
learning_rate=0.03,
hidden_size=hidden_size,
attention_head_size=attention_head_size,
dropout=dropout,
hidden_continuous_size=hidden_continuous_size,
output_size=7,
loss=pytorch_forecasting.metrics.QuantileLoss(),
log_interval=10,
reduce_on_plateau_patience=4,
)
trainer = pl.Trainer(
max_epochs=10,
gpus=0,
gradient_clip_val=0.1,
)
trainer.fit(tft, train_dataloader)
val_loss = trainer.callback_metrics['val_loss'].item()
return val_loss
tft_study = optuna.create_study(direction='minimize')
tft_study.optimize(tft_objective, n_trials=50)
best_tft_params = tft_study.best_params
print("Best TFT hyperparameters: ", best_tft_params)
tft = TemporalFusionTransformer.from_dataset(
training,
learning_rate=0.03,
hidden_size=best_tft_params['hidden_size'],
attention_head_size=best_tft_params['attention_head_size'],
dropout=best_tft_params['dropout'],
hidden_continuous_size=best_tft_params['hidden_continuous_size'],
output_size=7,
loss=pytorch_forecasting.metrics.QuantileLoss(),
log_interval=10,
reduce_on_plateau_patience=4,
)
trainer = pl.Trainer(
max_epochs=30,
gpus=0,
gradient_clip_val=0.1,
)
trainer.fit(tft, train_dataloader)
torch.save(tft.state_dict(), 'tft_model.pth')
best_model_path = trainer.checkpoint_callback.best_model_path
best_tft = TemporalFusionTransformer.load_from_checkpoint(best_model_path)
tft_predictions = best_tft.predict(training, mode="raw")
# Plot the predictions
plt.figure(figsize=(12, 6))
plt.plot(df['Close'].values, label='Actual Prices')
plt.plot(range(len(df), len(df) + len(arima_forecast)), arima_forecast, label='ARIMA Forecast')
plt.plot(prophet_forecast['ds'].values[-30:], prophet_forecast['yhat'].values[-30:], label='Prophet Forecast')
plt.plot(range(len(df), len(df) + len(xgb_forecast)), xgb_forecast, label='XGBoost Forecast')
plt.plot(range(len(df), len(df) + len(future_predictions)), future_predictions, label='LSTM Forecast')
plt.plot(range(len(df), len(df) + len(transformer_predictions)), transformer_predictions, label='Transformer Forecast')
plt.plot(df['time_idx'].values[-len(tft_predictions):], tft_predictions, label='TFT Forecast')
plt.title('Bitcoin Price Prediction using Various Models')
plt.legend()
plt.show()