# Machine learning for econometricians
# Course for the ECB
# by Ewen Gallic
# December 2021
# Hands-on Session 2
# Trees


# Support Vector Machines -------------------------------------------------




# 1. Maximal Margin Classifier --------------------------------------------

library(tidyverse)
x <- seq(0,1, by = .1)
y <- 2*x+1
df <- tibble(x = x, y = y)
ggplot(data = df, aes(x = x, y = y)) +
  geom_line()+
  xlab(expression(x[1])) + ylab(expression(x[2])) +
  ggtitle("In two dimensions (p=2), a hyperplane is a one-dimensional subspace, a line.") +
  theme(text = element_text(size=6), legend.position = "right",
        plot.title.position = "plot")

# In three dimension, a hyperplane is a flat two-dimensional subspace, i.e., a plane.
x <- seq(-10, 10, length= 30)
y <- x
f <- function(x, y){
  2*x+4*y
}
z <- outer(x, y, f)
z[is.na(z)] <- 1
persp(x, y, z, theta = 30, phi = 30, expand = 0.5, col = "lightblue",
      ltheta = 120, shade = 0.15, ticktype = "detailed",
      xlab = "x_1", ylab = "x_2", zlab = "x_3=2x+4y"
)


# Hyperplane $x_1 -2 x_2 + 0.1$. 
# The blue region corresponds to the set of points for which $x_1 -2 x_2 + 0.1>0$, 
# the red region corresponds to the set of points for which $x_1 -2 x_2 + 0.1<0$.
x_1 <- seq(-2,2, by = .1)
f <- function(x) (1*x+.1)*0.5
x_2 <- f(x_1)
grid <- expand.grid(x_1=seq(-1.05,1.05, by = .1), 
                    x_2=seq(-1.05,1.05, by = .1)) %>% 
  tbl_df() %>% 
  mutate(colour = ifelse(x_1 + 0.1 - 2*x_2 > 0, "magenta", "blue"))
df <- tibble(x = x_1, y = x_2)

ggplot(data = df, aes(x = x_1, y = x_2)) +
  geom_polygon(data = tibble(x = c(-2, 2, 2), y = c(f(-2), f(2), f(-2))),
               aes(x=x, y = y), fill = "magenta", alpha = .2) +
  geom_polygon(data = tibble(x = c(-2, -2, 2), y = c(f(-2), f(2), f(2))),
               aes(x=x, y = y), fill = "blue", alpha = .2) +
  geom_point(data = grid, aes(x = x_1, y = x_2, colour = colour)) +
  geom_line() +
  theme(legend.position = "none") +
  coord_equal(xlim = c(0, 1), ylim = c(0,1)) +
  xlab(expression(x[1])) + ylab(expression(x[2])) +
  scale_colour_manual(NULL, values = c("magenta" = "#D41159", "blue"="#1A85FF"))


# simulated data in two dimensions, that are perfectly separable by a line
df <- 
  tibble(x_1 = c(5,7,9,12, 13,1,2,4,6,8),
         x_2 = c(1,1,4,7,6,3,1,5,9,7),
         y = c(rep(-1, 5), rep(1, 5))) %>% 
  mutate(colour = ifelse(y == -1, yes ="blue", no = "magenta"))



ggplot() +
  geom_point(data = df, 
             mapping = aes(x = x_1, 
                           y = x_2, colour = colour)) +
  scale_colour_manual(
    NULL, 
    values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  labs(title = "Data in two dimensions with two classes, where a separating line can perfectly separate the data.") +
  theme(plot.title.position = "plot")



slope <- 1.4 ; intercept <- -5



ggplot() +
  geom_point(data = df, 
             mapping = aes(x = x_1, 
                           y = x_2, colour = colour)) +
  scale_colour_manual(
    NULL, 
    values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  geom_abline(slope = slope, intercept = intercept) +
  coord_equal(xlim = c(0, 15), ylim = c(0,10)) + 
  labs(title = "A first line of equation x_2 = 1.4 x_1 - 5 that perfectly separates the data.") +
  theme(legend.position = "none", plot.title.position = "plot")


# Other values for the separating line
slope_2 <- 1 ; intercept_2 <- -2.5



ggplot() +
  geom_point(data = df, 
             mapping = aes(x = x_1, 
                           y = x_2, colour = colour)) +
  scale_colour_manual(NULL, 
                      values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  geom_abline(slope = slope, intercept = intercept) +
  geom_abline(slope = slope_2, intercept = intercept_2, colour = "#FE6100") +
  geom_label(
    data = tibble(x_1 =8, x_2=9, lab = "x_2 = 1.4x_1 - 5"),
    mapping = aes(x = x_1, y = x_2, label = lab), fill = "black", colour = "white") +
  geom_label(
    data = tibble(x_1 = 10, x_2=6, lab = "x_2 = x_1 - 2.5"),
    mapping = aes(x = x_1, y = x_2, label = lab), fill = "#FE6100", colour = "black") +
  labs(title = "Two lines of equations x_2 = 1.4 x_1 - 5 and x_2 = x_1 - 2.5 that perfectly separate the data.") +
  coord_equal(xlim = c(0, 15), ylim = c(0,10)) +
  theme(plot.title.position = "plot")



# 1.2. Margin -------------------------------------------------------------



df_tmp <- 
  df %>% 
  mutate(distance_margin = 
           abs(slope*x_1 - x_2 + intercept) / 
           sqrt(slope^2+(-1)^2))
df_tmp


min_margin <- 
  df_tmp %>% 
  arrange(distance_margin) %>% 
  slice(1) %>% 
  magrittr::extract2("distance_margin")
min_margin


dist_intercept <- min_margin/sin(pi/2-atan(slope))


ggplot() +
  geom_point(data = df, 
             mapping = aes(x = x_1, y = x_2, colour=colour)) +
  scale_colour_manual(NULL, 
                      values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  geom_abline(slope = slope, intercept = intercept, colour = "#FE6100") +
  coord_equal(xlim = c(0, 15), ylim = c(0,10)) +
  geom_abline(slope = slope, intercept = intercept-dist_intercept, 
              colour = "black", linetype = "dashed") +
  geom_abline(slope = slope, intercept = intercept+dist_intercept, 
              colour = "black", linetype = "dashed") +
  labs(title = "Margin obtained using the following separating hyperplane: x_2 = 1.4x_1 -5.") +
  theme(legend.position = "none", plot.title.position = "plot")


# let us get the margin with a slope of 1 and an intercept of -2.5:
df_tmp <- 
  df %>% 
  mutate(distance_margin = 
           abs(slope_2*x_1 - x_2 + intercept_2) / 
           sqrt(slope_2^2+(-1)^2))
min_margin <- 
  df_tmp %>% 
  arrange(distance_margin) %>% 
  slice(1) %>% 
  magrittr::extract2("distance_margin")
min_margin



dist_intercept <- min_margin/sin(pi/2-atan(slope_2))
ggplot() +
  geom_point(data = df, 
             mapping = aes(x = x_1, y = x_2, colour=colour)) +
  scale_colour_manual(NULL, 
                      values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  geom_abline(slope = slope_2, intercept = intercept_2, colour = "black") +
  coord_equal(xlim = c(0, 15), ylim = c(0,10)) +
  geom_abline(slope = slope_2, intercept = intercept_2-dist_intercept, 
              colour = "black", linetype = "dashed") +
  geom_abline(slope = slope_2, intercept = intercept_2+dist_intercept, 
              colour = "black", linetype = "dashed") +
  labs(title = "Margin obtained using the following separating hyperplane: x_2 = x_1 -2.5.") +
  theme(legend.position = "none", plot.title.position = "plot")


# let us admit that the maximum margin is the one obtained with the values
# 1 for the slope and -2.5 for the intercept.

ggplot() +
  geom_point(data = df, 
             mapping = aes(x = x_1, y = x_2, colour=colour)) +
  geom_point(data = 
               tibble(
                 x_1 = c(5,2,8),
                 x_2 = c(1,1,7),
                 colour = c("blue", "magenta", "magenta")),
             mapping = aes(x = x_1, y = x_2, fill = colour),
             colour="black", size = 4, shape = 21) +
  scale_colour_manual(NULL, 
                      values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  scale_fill_manual(NULL,
                    values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  geom_abline(slope = slope_2, intercept = intercept_2, colour = "black") +
  geom_abline(slope = slope_2, intercept = intercept_2-dist_intercept, 
              colour = "black", linetype = "dashed") +
  geom_abline(slope = slope_2, intercept = intercept_2+dist_intercept, 
              colour = "black", linetype = "dashed") +
  coord_equal(xlim = c(0, 15), ylim = c(0,10)) +
  labs(title = "Maximum margin classifier for a perfectly separable binary outcome variable.") +
  theme(legend.position = "none", plot.title.position = "plot")



# 2. Support Vector Classifiers -------------------------------------------

# Let us add some data (four observations) to the synthetic dataset at hand:
new_data <- 
  tibble(x_1 = c(7, 10, 5, 8.5),
         x_2 = c(3.5, 9.5, 2.3, 3.5),
         y = c(1, -1, -1, 1),
         colour = c("magenta", "blue", "blue", "magenta"))

df_2 <- 
  df %>% 
  bind_rows(new_data) %>% 
  mutate(class = factor(colour, levels = c("blue", "magenta")))

ggplot() +
  geom_point(data = df_2, 
             mapping = aes(x = x_1, y = x_2, colour=colour)) +
  geom_point(data = new_data,
             mapping = aes(x = x_1, y = x_2, fill = colour), 
             colour="black", size = 4, shape = 21) +
  scale_colour_manual(NULL, 
                      values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  scale_fill_manual(NULL, 
                    values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  geom_abline(slope = slope_2, intercept = intercept_2, colour = "black") +
  geom_abline(slope = slope_2, intercept = intercept_2-dist_intercept, 
              colour = "black", linetype = "dashed") +
  geom_abline(slope = slope_2, intercept = intercept_2+dist_intercept, 
              colour = "black", linetype = "dashed") +
  coord_equal(xlim = c(0, 15), ylim = c(0,10)) +
  labs(title = "he new observations violate the previous margin.") +
  theme(legend.position = "none", plot.title.position = "plot")


# factor variable in our initial dataset:
df <- df %>% mutate(class=factor(colour))

# Using {e1071}
library(e1071)
svm_fit <- svm(
  class~x_1+x_2,
  data = df,
  cost = 1000,
  # Using a linear kernel
  kernel = "linear",
  # Not scaling the variables
  # Scaling: transforming y and X to get 0 mean and unit variance
  scale = FALSE)
svm_fit

# index of the obs. that define the support
svm_fit$index


df_support_vector <- 
  tibble(df %>% slice(svm_fit$index))
df_support_vector

# The negative intercept is in the element `rho`
beta_0 <- -svm_fit$rho
beta_0

beta_1 <- sum(svm_fit$coefs * df$x_1[svm_fit$index])
beta_1

beta_2 <- sum(svm_fit$coefs * df$x_2[svm_fit$index])
beta_2

intercept <- -beta_0/beta_2
intercept

slope <- -beta_1/beta_2
slope


ggplot() +
  geom_point(data = df, 
             mapping = aes(x = x_1, y = x_2, colour=colour)) +
  geom_point(data = df_support_vector,
             mapping = aes(x = x_1, y = x_2, fill = colour), 
             colour="black", size = 4, shape = 21) +
  scale_colour_manual(
    NULL, values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  geom_abline(slope = slope, intercept = intercept, colour = "black") +
  coord_equal(xlim = c(0, 15), ylim = c(0,10)) +
  geom_abline(intercept = -beta_0/beta_2, slope = -beta_1/beta_2,
              colour = "black") +
  geom_abline(slope = slope, intercept = (-beta_0-1)/beta_2, 
              colour = "black", linetype = "dashed") +
  geom_abline(slope = slope, intercept = (-beta_0+1)/beta_2, 
              colour = "black", linetype = "dashed") +
  labs(title = "Maximal margin classifier for the initial dataset with linearly separable observation.") +
  theme(legend.position = "none", plot.title.position = "plot")


# let us simulate some new data that are not perfectly linearly separable

set.seed(1234)
x_1 <- rbeta(n=5000, shape1 = 1, shape2 = 1)
x_2 <- rbeta(n=5000, shape1 = 1, shape2 = 1)
df <- tibble(x_1 = x_1, x_2 = x_2)

df <- 
  df %>% 
  mutate(
    flag = (0.5* x_1 + x_2 - 0.6 > 0 & 3*x_1 + x_2 - 1.5 < 0) |
      (3*x_1 + x_2 - 1.5 > 0 & 0.5* x_1 + x_2 - 0.6 < 0) |
      (3*x_1 + x_2 - 1.5 > 0 & 1.5* x_1 + x_2 - 1 < 0)) %>% 
  filter(!flag) %>% 
  mutate(
    class = ifelse(3*x_1 + x_2 - 1.5 > 0 & 1.5*x_1 + x_2 - 1 > 0 &
                     0.5*x_1 + x_2 - 0.6 > 0, 
                   yes = "blue", no = "magenta")) %>%
  group_by(class) %>%
  sample_n(200) %>%
  ungroup()

# Adding noise
df_new <- 
  df %>% 
  mutate(
    flag = (0.5* x_1 + x_2 - 0.6 > 0 & 3*x_1 + x_2 - 1.5 < 0) |
      (3*x_1 + x_2 - 1.5 > 0 & 0.5* x_1 + x_2 - 0.6 < 0) |
      (3*x_1 + x_2 - 1.5 > 0 & 1.5* x_1 + x_2 - 1 < 0)) %>% 
  filter(!flag) %>% 
  mutate(
    class = ifelse(3*x_1 + x_2 - 1.5 > 0 & 1.5*x_1 + x_2 - 1 > 0 &
                     0.5*x_1 + x_2 - 0.6 > 0,
                   yes = "magenta", no = "blue")) %>%
  group_by(class) %>%
  sample_n(20) %>%
  ungroup()


df_2 <- 
  df %>% 
  bind_rows(df_new) %>% 
  mutate(class = factor(class)) %>% 
  sample_frac(1)


ggplot() +
  geom_point(data = df_2, 
             mapping = aes(x = x_1, y = x_2, colour = class)) +
  scale_colour_manual(
    NULL, 
    values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  labs(title = "Simulated almost linearly-separable data.") +
  theme(plot.title.position = "plot")


# The cost parameter can however be used to allow more or less missclassified training observations
svm_fit_2 <- svm(
  formula = class~x_1+x_2,
  data = df_2,
  cost = 100,
  # Using a linear kernel
  kernel = "linear",
  # Not scaling the variables
  # Scaling: transforming y and X to get 0 mean and unit variance
  scale = FALSE)
svm_fit_2



plot_results <- function(svm_result){
  df_support_vector <- 
    tibble(df_2 %>% slice(svm_result$index))
  beta_0 <- -svm_result$rho
  beta_1 <- sum(svm_result$coefs * df_2$x_1[svm_result$index])
  beta_2 <- sum(svm_result$coefs * df_2$x_2[svm_result$index])
  intercept <- -beta_0/beta_2
  slope <- -beta_1/beta_2
  f <- function(x) x*slope+intercept
  ggplot() +
    geom_polygon(
      data = tibble(x = c(-2, 2, 2), y = c(f(-2), f(2), f(-2))),
      aes(x=x, y = y), fill = "blue", alpha = .2) +
    geom_polygon(
      data = tibble(x = c(-2, -2, 2), y = c(f(-2), f(2), f(2))),
      aes(x=x, y = y), fill = "magenta", alpha = .2) +
    geom_point(data = df_2, aes(x = x_1, y = x_2, colour=class)) +
    scale_colour_manual(
      NULL, values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
    scale_fill_manual(
      NULL, values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
    geom_abline(intercept = -beta_0/beta_2, slope = -beta_1/beta_2, 
                colour = "black") +
    geom_abline(slope = slope, intercept = (-beta_0-1)/beta_2, 
                colour = "black", linetype = "dashed") +
    geom_abline(slope = slope, intercept = (-beta_0+1)/beta_2,
                colour = "black", linetype = "dashed") +
    coord_equal(xlim = c(0, 1), ylim = c(0,1)) +
    theme(legend.position = "none", plot.title.position = "plot")
}



plot_results(svm_fit_2) +
  labs(title = "Support vector classifier with c=100.")


# confusion table
confusion_t <- 
  table(observed = df_2$class, predicted = predict(svm_fit_2, df_2))
confusion_t


# Pct of missclassified obs:
1 - sum(diag(confusion_t)) / sum(confusion_t)



svm_fit_3 <- svm(
  formula = class~x_1+x_2,
  data = df_2,
  cost = .1,
  # Using a linear kernel
  kernel = "linear",
  # Not scaling the variables
  # Scaling: transforming y and X to get 0 mean and unit variance
  scale = FALSE)
svm_fit_3



plot_results(svm_fit_3) +
  labs(title = "Support vector classifier with c=.1.")


confusion_t <- 
  table(observed = df_2$class, predicted = predict(svm_fit_3, df_2))
confusion_t

# Pct missclassified obs
1 - sum(diag(confusion_t)) / sum(confusion_t)


# Grid of cost values:
values_cost <- 10^seq(3, -2, length = 50)


# 10-fold CV to find the cost parameter that min. the error
tune_out <- tune(svm,
                 train.x = class~x_1+x_2,
                 data=df_2,
                 kernel="linear",
                 ranges=list(cost=values_cost))

tune_out

tune_out$best.parameters


# The best model is saved, no need to estimate it again
tune_out$best.model


confusion_t <- 
  table(observed = df_2$class, predicted = predict(tune_out$best.model, df_2))
confusion_t


# Pct missclassified obs
1 - sum(diag(confusion_t)) / sum(confusion_t)



plot_results(tune_out$best.model) +
  labs(title = "Support vector classifier with the cost parameter obtained by cross-validation.")


# 3. Support Vector Machines ----------------------------------------------


# Let us generate some data
set.seed(1234)
x_1 <- runif(n = 90, 0, 10)
x_2 <- 2*x_1+3 + rnorm(n = length(x_1))

df <- tibble(x_1 = x_1, x_2 = x_2) %>% 
  arrange(x_1)


# assign a class to each observation
df <- 
  df %>% 
  slice(16:30) %>% 
  mutate(class = "blue") %>% 
  bind_rows(
    df %>% 
      slice(31:60) %>% 
      mutate(class = "magenta")
  ) %>% 
  bind_rows(
    df %>% 
      slice(61:75) %>% 
      mutate(class = "blue")
  ) %>% 
  mutate(class = factor(class))



ggplot(data = df, aes(x = x_1, y = x_2, colour = class)) +
  geom_point() +
  scale_colour_manual(
    NULL, values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  coord_equal(xlim = c(1, 10), ylim = c(5,20)) +
  labs(title = "Non linearly-separable data.") +
  theme(legend.position = "none", plot.title.position = "plot")


# Let us try to fit a support vector classifier:
svm_fit_lin <- 
  svm(class~x_1+x_2, data = df, kernel = "linear", scale = FALSE)



beta_0 <- -svm_fit_lin$rho
beta_1 <- sum(svm_fit_lin$coefs * df$x_1[svm_fit_lin$index])
beta_2 <- sum(svm_fit_lin$coefs * df$x_2[svm_fit_lin$index])

intercept <- -beta_0/beta_2
slope = -beta_1/beta_2

f <- function(x) slope*x+intercept


ggplot() +
  geom_polygon(
    data = tibble(x = c(-2, 12, 12), y = c(f(-2), f(12), f(-2))),
    aes(x=x, y = y), fill = "magenta", alpha = .2) +
  geom_polygon(
    data = tibble(x = c(-2, -2, 12), y = c(f(-2), f(12), f(12))),
    aes(x=x, y = y), fill = "blue", alpha = .2) +
  geom_point(data = df, aes(x = x_1, y = x_2, colour=class)) +
  scale_colour_manual(
    NULL, values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  scale_fill_manual(
    NULL, values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  geom_abline(intercept = -beta_0/beta_2, slope = -beta_1/beta_2, 
              colour = "black") +
  geom_abline(slope = slope, intercept = (-beta_0-1)/beta_2, 
              colour = "black", linetype = "dashed") +
  geom_abline(slope = slope, intercept = (-beta_0+1)/beta_2, 
              colour = "black", linetype = "dashed") +
  coord_equal(xlim = c(1, 10), ylim = c(5,20)) +
  labs(title = "Decision boundary of the Support Vector Classifier trained on non linearly-separable data.") +
  theme(legend.position = "none", plot.title.position = "plot")


# 3.1. Polynomial Kernel --------------------------------------------------


degree <- 10

svm_fit_poly <- 
  svm(class~x_1+x_2, data = df, kernel = "polynomial", 
      degree = degree,
      scale = FALSE, cost=1000, coef0=1)

# Grid to display the decision boundaries on a graph
grid <- 
  expand_grid(x_1=seq(0, 11, length.out = 300),
              x_2=seq(4, 21, length.out = 300)) %>% 
  tbl_df()


# Making predictions on each point of the grid
grid$pred <- predict(svm_fit_poly, newdata = grid)
head(grid)


## ---- fig.cap="", fig.height=4--------------------------------------------------------
ggplot() +
  geom_tile(data = grid, aes(x = x_1, y = x_2, fill = pred), alpha = .2) +
  geom_point(data = df, aes(x = x_1, y = x_2, colour=class)) +
  scale_colour_manual(
    NULL, values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  scale_fill_manual(
    NULL, values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  coord_equal(xlim = c(1, 10), ylim = c(5,20)) +
  geom_contour(data = grid %>% 
                 mutate(pre_val = as.numeric(pred)), 
               mapping = aes(x = x_1, y = x_2, z=pre_val),
               breaks = 1.5, colour = "black", linetype = "solid") +
  labs(title = "Polynomial Kernel, degree 10.") +
  theme(legend.position = "none", plot.title.position = "plot")


# Grid search for the cost parameter and the polynomial degree
hyperparam_grid <- list(
  cost = 10^seq(3, -2, length = 50),
  degree = seq(2, 10, by = 2)
)

# 10-fold CV
tune_out <- tune(svm,
                 train.x = class~x_1+x_2,
                 data=df,
                 kernel="polynomial",
                 ranges=hyperparam_grid)


tune_out
tune_out$best.parameters

confusion_t <- 
  table(observed = df$class, predicted = predict(tune_out$best.model, df))
confusion_t


# Pct missclassified
1 - sum(diag(confusion_t)) / sum(confusion_t)


# Predictions on the gris to show decision boundaries
grid$pred <- predict(tune_out$best.model, newdata = grid)


ggplot() +
  geom_tile(data = grid, aes(x = x_1, y = x_2, fill = pred), alpha = .2) +
  geom_point(data = df, aes(x = x_1, y = x_2, colour=class)) +
  scale_colour_manual(
    NULL, values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  scale_fill_manual(
    NULL, values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  coord_equal(xlim = c(1, 10), ylim = c(5,20)) +
  geom_contour(data = grid %>% 
                 mutate(pre_val = as.numeric(pred)), 
               mapping = aes(x = x_1, y = x_2, z=pre_val),
               breaks = 1.5, colour = "black", linetype = "solid") +
  labs(title = "Polynomial Kernel, degree 2.") +
  theme(legend.position = "none", plot.title.position = "plot")



# 3.2. Radial kernel ------------------------------------------------------

gamma <- .1
svm_fit_poly <- 
  svm(class~x_1+x_2, data = df, kernel = "radial", gamma = .1,
      scale = FALSE, cost=1000)

# Grid for decisions boundaries
grid <- 
  expand_grid(
    x_1=seq(0, 11, length.out = 300),
    x_2=seq(4, 21, length.out = 300)) %>% 
  tbl_df()

# Predictions for each point of the grid
grid$pred <- predict(svm_fit_poly, newdata = grid)


ggplot() +
  geom_tile(data = grid,
            mapping = aes(x = x_1, y = x_2, fill = pred), 
            alpha = .2) +
  geom_point(data = df, 
             mapping = aes(x = x_1, y = x_2, colour=class)) +
  scale_colour_manual(
    NULL, values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  scale_fill_manual(
    NULL, values = c("blue" = "#1A85FF", "magenta" = "#D41159")) +
  coord_equal(xlim = c(1, 10), ylim = c(5,20)) +
  geom_contour(data = grid %>% 
                 mutate(pre_val = as.numeric(pred)), 
               mapping = aes(x = x_1, y = x_2, z=pre_val),
               breaks = 1.5, colour = "black", linetype = "solid") +
  labs(title = "Radial Kernel, \\gamma=.1.") +
  theme(legend.position = "none", plot.title.position = "plot")