import pandas as pdimport numpy as npimport matplotlib.pyplot as pltimport tensorflow as tffrom datetime import datetimefrom sklearn.preprocessing import MinMaxScalerfrom keras.preprocessing.sequence import TimeseriesGeneratorfrom keras.models import Sequentialfrom keras.layers import Dense, LSTM, Dropout, Activation, GlobalMaxPooling1Dfrom keras.optimizers import Adam%matplotlib inline

df_confirmed = pd.read_csv("../input/covid-19/time_series_covid19_confirmed_global.csv")df_deaths = pd.read_csv("../input/covid-19/time_series_covid19_deaths_global.csv")df_reco = pd.read_csv("../input/covid-19/time_series_covid19_recovered_global.csv")

df_confirmed.head()

my_confirmed = df_confirmed[df_confirmed["Country/Region"] == "Malaysia"]my_deaths = df_deaths[df_deaths["Country/Region"] == "Malaysia"]my_reco = df_reco[df_reco["Country/Region"] == "Malaysia"]

my_confirmed

# convert passed dataframe to a timeseries (a format easy to graph and use for training models)def confirmed_timeseries(df):    df_series = pd.DataFrame(df[df.columns[4:]].sum(),columns=["confirmed"])    df_series.index = pd.to_datetime(df_series.index,format = '%m/%d/%y')    return df_seriesdef deaths_timeseries(df):    df_series = pd.DataFrame(df[df.columns[4:]].sum(),columns=["deaths"])    df_series.index = pd.to_datetime(df_series.index,format = '%m/%d/%y')    return df_seriesdef reco_timeseries(df):    # no index to timeseries conversion needed (all is joined later)    df_series = pd.DataFrame(df[df.columns[4:]].sum(),columns=["recovered"])    return df_series

# convert each dataframe to a timeseriesmy_con_series = confirmed_timeseries(my_confirmed)my_dea_series = deaths_timeseries(my_deaths)my_reco_series = reco_timeseries(my_reco)

my_con_series

# join all 3 data framesmy_df = my_con_series.join(my_dea_series, how = "inner")my_df = my_df.join(my_reco_series, how = "inner")

my_df

my_df.plot(figsize=(14,7),title="Malysia confirmed, deaths, and recoverd cases")

<matplotlib.axes._subplots.AxesSubplot at 0x7f2ab8657a90>

my_cases_outcome = (my_df.tail(1)['deaths'] + my_df.tail(1)['recovered'])[0]my_outcome_perc = (my_cases_outcome / my_df.tail(1)['confirmed'] * 100)[0]my_death_perc = (my_df.tail(1)['deaths'] / my_cases_outcome * 100)[0]my_reco_perc = (my_df.tail(1)['recovered'] / my_cases_outcome * 100)[0]my_active = (my_df.tail(1)['confirmed'] - my_cases_outcome)[0]print(f"Number of cases which had an outcome: {my_cases_outcome}")print(f"percentage of cases that had an outcome: {round(my_outcome_perc, 2)}%")print(f"Deaths rate: {round(my_death_perc, 2)}%")print(f"Recovery rate: {round(my_reco_perc, 2)}%")print(f"Currently Active cases: {my_active}")

Number of cases which had an outcome: 67575percentage of cases that had an outcome: 84.14%Deaths rate: 0.59%Recovery rate: 99.41%Currently Active cases: 12734

n_input = 20  # number of steps (days to predict)n_features = 1 # number of y (model outputs)# preporcess a dataframe and create required vairables for training the LSTM# train: the data used to make the training generator# test: the data used to make the test generator# scaler: data scaler to normalize the data (easier for the model)# scaled_train: train dataset scaled down to the largest value in the dataset# scaled test: train dataset scaled down to the largest value in the dataset# generator: train data generator, used to train the model by feeding it batches of the train data# validation_generator: validation data generator, used to validate the model during training#                       by feeding it batches of a randomly selected points from the train datasetdef prepare_data(df):    # drop rows with zeros    df = df[(df.T != 0).any()]    num_days = len(df) - n_input    train = df.iloc[:num_days]    test = df.iloc[num_days:]    # normalize the data according to largest value    scaler = MinMaxScaler()    scaler.fit(train) # find max value    scaled_train = scaler.transform(train) # divide every point by max value    scaled_test = scaler.transform(test)    # feed in batches [t1,t2,t3] --> t4    generator = TimeseriesGenerator(scaled_train,scaled_train,length = n_input,batch_size = 1)    validation_set = np.append(scaled_train[55],scaled_test) # random tbh    validation_set = validation_set.reshape(n_input + 1,1)    validation_gen = TimeseriesGenerator(validation_set,validation_set,length = n_input,batch_size = 1)    return scaler, train, test, scaled_train, scaled_test, generator, validation_gen

# create, train and return LSTM modeldef train_lstm_model():    model = Sequential()    model.add(LSTM(84, recurrent_dropout = 0, return_sequences = True, input_shape = (n_input,n_features)))    model.add(LSTM(84, recurrent_dropout = 0.1, use_bias = True, return_sequences = True,))    model.add(GlobalMaxPooling1D())    model.add(Dense(84, activation = "relu"))    model.add(Dense(1))    # compile model    model.compile(loss = 'mse', optimizer = Adam(1e-5))    # finally train the model using generators    model.fit_generator(generator,                        validation_data = validation_gen,                        epochs = 300,                        steps_per_epoch = round(len(train) / n_input),                        verbose = 0                       )    return model

# predict, rescale and append needed columns to final data framedef lstm_predict(model):    # holding predictions    test_prediction = []    # last n points from training set    first_eval_batch = scaled_train[-n_input:]    current_batch = first_eval_batch.reshape(1,n_input,n_features)    # predict first x days from testing data    for i in range(len(test) + n_input):        current_pred = model.predict(current_batch)[0]        test_prediction.append(current_pred)        current_batch = np.append(current_batch[:,1:,:],[[current_pred]],axis=1)    # inverse scaled data    true_prediction = scaler.inverse_transform(test_prediction)    MAPE, accuracy, sum_errs, interval, stdev, df_forecast = gen_metrics(true_prediction)    return MAPE, accuracy, sum_errs, interval, stdev, df_forecast# plotting model lossesdef plot_lstm_losses(model):    pd.DataFrame(model.history.history).plot(figsize = (14,7), title = "loss vs epochs curve")

# generate metrics and final dfdef gen_metrics(pred):    # create time series    time_series_array = test.index    for k in range(0, n_input):        time_series_array = time_series_array.append(time_series_array[-1:] + pd.DateOffset(1))    # create time series data frame    df_forecast = pd.DataFrame(columns = ["confirmed","confirmed_predicted"],index = time_series_array)    # append confirmed and predicted confirmed    df_forecast.loc[:,"confirmed_predicted"] = pred    df_forecast.loc[:,"confirmed"] = test["confirmed"]    # create and append daily cases (for both actual and predicted)    daily_act = []    daily_pred = []    #actual    daily_act.append(abs(df_forecast["confirmed"].iloc[1] - train["confirmed"].iloc[-1]))    for num in range((n_input * 2) - 1):        daily_act.append(df_forecast["confirmed"].iloc[num + 1] - df_forecast["confirmed"].iloc[num])    # predicted    daily_pred.append(df_forecast["confirmed_predicted"].iloc[1] - train["confirmed"].iloc[-1])    for num in range((n_input * 2) - 1):        daily_pred.append(df_forecast["confirmed_predicted"].iloc[num + 1] - df_forecast["confirmed_predicted"].iloc[num])    # calculate mean absolute percentage error    MAPE = np.mean(np.abs(np.array(df_forecast["confirmed"][:n_input]) -                          np.array(df_forecast["confirmed_predicted"][:n_input])) /                   np.array(df_forecast["confirmed"][:n_input]))    accuracy = round((1 - MAPE) * 100, 2)    # the error rate    sum_errs = np.sum((np.array(df_forecast["confirmed"][:n_input]) - np.array(df_forecast["confirmed_predicted"][:n_input])) ** 2)    # error standard deviation    stdev = np.sqrt(1 / (n_input - 2) * sum_errs)    # calculate prediction interval    interval = 1.96 * stdev    # append the min and max cases to final df    df_forecast["confirm_min"] = df_forecast["confirmed_predicted"] - interval    df_forecast["confirm_max"] = df_forecast["confirmed_predicted"] + interval    # appened daily data    df_forecast["daily"] = daily_act    df_forecast["daily_predicted"] = daily_pred    daily_err = np.sum((np.array(df_forecast["daily"][:n_input]) - np.array(df_forecast["daily_predicted"][:n_input])) ** 2)    daily_stdev = np.sqrt(1 / (n_input - 2) * daily_err)    daily_interval = 1.96 * daily_stdev    df_forecast["daily_min"] = df_forecast["daily_predicted"] - daily_interval    df_forecast["daily_max"] = df_forecast["daily_predicted"] + daily_interval    # round all df values to 0 decimal points    df_forecast = df_forecast.round()    return MAPE, accuracy, sum_errs, interval, stdev, df_forecast

# print metrics for given countydef print_metrics(mape, accuracy, errs, interval, std, model_type):    m_str = "LSTM" if model_type == 0 else "ARIMA" if model_type == 1 else "HES"    print(f"{m_str} MAPE: {round(mape * 100, 2)}%")    print(f"{m_str} accuracy: {accuracy}%")    print(f"{m_str} sum of errors: {round(errs)}")    print(f"{m_str} prediction interval: {round(interval)}")    print(f"{m_str} standard deviation: {std}")

# for plotting the range of predicetionsdef plot_results(df, country, algo):    fig, (ax1, ax2) = plt.subplots(2, figsize = (14,20))    ax1.set_title(f"{country} {algo} confirmed predictions")    ax1.plot(df.index,df["confirmed"], label = "confirmed")    ax1.plot(df.index,df["confirmed_predicted"], label = "confirmed_predicted")    ax1.fill_between(df.index,df["confirm_min"], df["confirm_max"], color = "indigo",alpha = 0.09,label = "Confidence Interval")    ax1.legend(loc = 2)    ax2.set_title(f"{country} {algo} confirmed daily predictions")    ax2.plot(df.index, df["daily"], label = "daily")    ax2.plot(df.index, df["daily_predicted"], label = "daily_predicted")    ax2.legend()    import matplotlib.dates as mdates    ax1.xaxis.set_major_formatter(mdates.DateFormatter('%b %-d'))    ax2.xaxis.set_major_formatter(mdates.DateFormatter('%b %-d'))    fig.show()

# prepare the data (using confirmed cases dataset)scaler, train, test, scaled_train, scaled_test, generator, validation_gen = prepare_data(my_con_series)

# train lstm modelmy_lstm_model = train_lstm_model()# plot lstm lossesplot_lstm_losses(my_lstm_model)

my_mape, my_accuracy, my_errs, my_interval, my_std, my_lstm_df = lstm_predict(my_lstm_model)print_metrics(my_mape, my_accuracy, my_errs, my_interval, my_std, 0) # 0 here means LSTMmy_lstm_df

LSTM MAPE: 2.01%LSTM accuracy: 97.99%LSTM sum of errors: 43483777.0LSTM prediction interval: 3046.0LSTM standard deviation: 1554.2732613061567

plot_results(my_lstm_df, "Malaysia", "LSTM")

# generate a dataframe with given rangedef get_range_df(start: str, end: str, df):    target_df = df.loc[pd.to_datetime(start, format='%Y-%m-%d'):pd.to_datetime(end, format='%Y-%m-%d')]    return target_df

my_lockdown = get_range_df('2020-03-16', '2020-05-04', my_con_series)my_pre = get_range_df('2020-01-25', '2020-03-16', my_con_series)my_after = get_range_df('2020-05-04', '2020-12-11', my_con_series)

plt.figure(figsize=(18,9))plt.plot(my_lockdown, color="C7", label="during lockdown", linewidth=5)plt.plot(my_pre, color="C8", label="before lockdown")plt.plot(my_after, color="C3", label="after lockdown")plt.legend()plt.show()