library(pacman)
p_load(titanic, tidyverse, janitor, naniar, DataExplorer, tidymodels, discrim)

Load the data from the titanic R package. Note that the titanic_train dataset contains the labels for Survived aND THE titanic_test dataset does not contain the labels. So we will build our machine learning model using the titanic_train dataset and then make a final classification for the titanic_test dataset. This is how kaggle competitions are done.

I like to clean names so the variables all have names with lowercase letters and underscores.

titanic_train <- titanic_train %>% clean_names()

head(titanic_train)

It is always a good idea to check for duplicate records/examples/rows in your dataset.

get_dupes(titanic_train)
No variable names specified - using all columns.

No duplicate combinations found of: passenger_id, survived, pclass, name, sex, age, sib_sp, parch, ticket, ... and 3 other variables

Drop the unique identifiers: passenger_id, name, and ticket. Also drop cabin because it has a high missing rate.

titanic_train2 <- titanic_train %>% select(-passenger_id, -name, -ticket, -cabin) %>%
  mutate(
    survived = as_factor(survived),
    pclass = as_factor(pclass),
    sex = as_factor(sex),
    embarked = as_factor(embarked)
  )

head(titanic_train2)
titanic_test <- titanic_test %>% clean_names()

head(titanic_test)

It is always a good idea to check for duplicate records/examples/rows in your dataset.

get_dupes(titanic_test)
No variable names specified - using all columns.

No duplicate combinations found of: passenger_id, pclass, name, sex, age, sib_sp, parch, ticket, fare, ... and 2 other variables
titanic_test2 <- titanic_test %>% select(-passenger_id, -name, -ticket, -cabin) %>%
  mutate(
    pclass = as_factor(pclass),
    sex = as_factor(sex),
    embarked = as_factor(embarked)
  )

head(titanic_test2)
NA

Start by investigating the missing values and completeness of the features in the data. Note that the age variable contains some missing values.

vis_miss(titanic_train2)

gg_miss_var(titanic_train2)

gg_miss_var(titanic_train2, show_pct = TRUE)

create_report(titanic_train2, y = "survived", output_file = "report.html", output_dir = getwd())

Now try the ML algorithms.

Model 0:

Summarize the y-variable. Null Model.

titanic_train2 %>% group_by(survived) %>%
  summarize(n = n()) %>%
  mutate(freq = n / sum(n))

Make the first split with 80% of the data being in the training data set.

titanic_train2_split <- initial_split(titanic_train2, prop = 0.8)
titanic_train2_split
<Analysis/Assess/Total>
<713/178/891>

Training data.

titanic_train2_split %>%
  training() 

Create the recipe for applying the preprocessing. Note the use of step_nzv(), which removes any columns that have very low variability, and the use of the step_meanimpute() function, which fills in the cells that are missing with the mean of the column.

Apply the recipe, so the age variable should be complete after the imputation.

titanic_train2_testing <- titanic_train2_recipe %>%
  bake(testing(titanic_train2_split)) 

titanic_train2_testing
titanic_train2_training <- juice(titanic_train2_recipe)

titanic_train2_training

Model 0: null

null_model(mode = "classification")
Model Specification (classification)
titanic_train2_null <- null_model() %>%
  set_mode("classification") %>%
  fit(survived ~ ., data = titanic_train2_training)
Engine set to `parsnip`.
predict(titanic_train2_null, titanic_train2_training)
titanic_train2_null %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) 
titanic_train2_null %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) %>%
  metrics(truth = survived, estimate = .pred_class)
titanic_train2_null %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) %>%
  conf_mat(truth = survived, estimate = .pred_class)
          Truth
Prediction   0   1
         0 109  69
         1   0   0
titanic_train2_null %>%
  predict(titanic_train2_testing, type = "prob") %>%
  bind_cols(titanic_train2_testing) %>%
  roc_curve(survived, .pred_0) %>%
  autoplot() 

Model 4: GLM

Setup the model.

titanic_train2_glm <- logistic_reg(penalty = 0.001, mixture = 0.5) %>% 
  set_engine("glmnet") %>%
  set_mode("classification") %>%
  fit(survived ~ ., data = titanic_train2_training)
predict(titanic_train2_glm, titanic_train2_training)
titanic_train2_glm %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) 
titanic_train2_glm %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) %>%
  metrics(truth = survived, estimate = .pred_class)
titanic_train2_glm %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) %>%
  conf_mat(truth = survived, estimate = .pred_class)
          Truth
Prediction   0   1
         0 104  48
         1   5  21
titanic_train2_glm %>%
  predict(titanic_train2_testing, type = "prob") %>%
  bind_cols(titanic_train2_testing) %>%
  roc_curve(survived, .pred_0) %>%
  autoplot()

titanic_train2_glm %>%
  predict(titanic_train2_testing, type = "prob") %>%
  bind_cols(titanic_train2_testing) %>%
  roc_auc(survived, .pred_0) 

Model 5: Naive Bayes

Setup the model.

titanic_train2_nb <- naive_Bayes(Laplace = 1) %>% 
  set_engine("klaR") %>%
  set_mode("classification") %>%
  fit(survived ~ ., data = titanic_train2_training)
predict(titanic_train2_nb, titanic_train2_training)
Numerical 0 probability for all classes with observation 348
titanic_train2_nb %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) 
titanic_train2_nb %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) %>%
  metrics(truth = survived, estimate = .pred_class)
titanic_train2_nb %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) %>%
  conf_mat(truth = survived, estimate = .pred_class)
          Truth
Prediction  0  1
         0 81 40
         1 28 29

titanic_train2_nb %>%
  predict(titanic_train2_testing, type = "prob") %>%
  bind_cols(titanic_train2_testing) %>%
  roc_auc(survived, .pred_0) 
titanic_train2_nb %>%
  predict(titanic_train2_testing, type = "prob") %>%
  bind_cols(titanic_train2_testing) %>%
  ggplot() +
  geom_density(aes(x = .pred_1, fill = survived), 
               alpha = 0.5)
LS0tCnRpdGxlOiAiU3RhdCA2NTI6IE1pZHRlcm06IE1vZGVsIDQsIE1vZGVsIDUiCmF1dGhvcjogIlByb2YuIEVyaWMgQS4gU3Vlc3MiCm91dHB1dDoKICBodG1sX25vdGVib29rOiBkZWZhdWx0CiAgd29yZF9kb2N1bWVudDogZGVmYXVsdAogIHBkZl9kb2N1bWVudDogZGVmYXVsdAotLS0KCgoKYGBge3J9CmxpYnJhcnkocGFjbWFuKQpwX2xvYWQodGl0YW5pYywgdGlkeXZlcnNlLCBqYW5pdG9yLCBuYW5pYXIsIERhdGFFeHBsb3JlciwgdGlkeW1vZGVscywgZGlzY3JpbSkKYGBgCgpMb2FkIHRoZSBkYXRhIGZyb20gdGhlIHRpdGFuaWMgUiBwYWNrYWdlLiAgTm90ZSB0aGF0IHRoZSAqdGl0YW5pY190cmFpbiogZGF0YXNldCBjb250YWlucyB0aGUgbGFiZWxzIGZvciBTdXJ2aXZlZCBhTkQgVEhFICp0aXRhbmljX3Rlc3QqIGRhdGFzZXQgZG9lcyBub3QgY29udGFpbiB0aGUgbGFiZWxzLiAgU28gd2Ugd2lsbCBidWlsZCBvdXIgbWFjaGluZSBsZWFybmluZyBtb2RlbCB1c2luZyB0aGUgKnRpdGFuaWNfdHJhaW4qIGRhdGFzZXQgYW5kIHRoZW4gbWFrZSBhIGZpbmFsIGNsYXNzaWZpY2F0aW9uIGZvciB0aGUgKnRpdGFuaWNfdGVzdCogZGF0YXNldC4gIFRoaXMgaXMgaG93IGthZ2dsZSBjb21wZXRpdGlvbnMgYXJlIGRvbmUuCgpJIGxpa2UgdG8gY2xlYW4gbmFtZXMgc28gdGhlIHZhcmlhYmxlcyBhbGwgaGF2ZSBuYW1lcyB3aXRoIGxvd2VyY2FzZSBsZXR0ZXJzIGFuZCB1bmRlcnNjb3Jlcy4KCmBgYHtyfQp0aXRhbmljX3RyYWluIDwtIHRpdGFuaWNfdHJhaW4gJT4lIGNsZWFuX25hbWVzKCkKCmhlYWQodGl0YW5pY190cmFpbikKYGBgCgpJdCBpcyBhbHdheXMgYSBnb29kIGlkZWEgdG8gY2hlY2sgZm9yIGR1cGxpY2F0ZSByZWNvcmRzL2V4YW1wbGVzL3Jvd3MgaW4geW91ciBkYXRhc2V0LgoKYGBge3J9CmdldF9kdXBlcyh0aXRhbmljX3RyYWluKQpgYGAKCkRyb3AgdGhlIHVuaXF1ZSBpZGVudGlmaWVyczogKnBhc3Nlbmdlcl9pZCosICpuYW1lKiwgYW5kICp0aWNrZXQqLiAgQWxzbyBkcm9wICpjYWJpbiogYmVjYXVzZSBpdCBoYXMgYSBoaWdoIG1pc3NpbmcgcmF0ZS4KCmBgYHtyfQp0aXRhbmljX3RyYWluMiA8LSB0aXRhbmljX3RyYWluICU+JSBzZWxlY3QoLXBhc3Nlbmdlcl9pZCwgLW5hbWUsIC10aWNrZXQsIC1jYWJpbikgJT4lCiAgbXV0YXRlKAogICAgc3Vydml2ZWQgPSBhc19mYWN0b3Ioc3Vydml2ZWQpLAogICAgcGNsYXNzID0gYXNfZmFjdG9yKHBjbGFzcyksCiAgICBzZXggPSBhc19mYWN0b3Ioc2V4KSwKICAgIGVtYmFya2VkID0gYXNfZmFjdG9yKGVtYmFya2VkKQogICkKCmhlYWQodGl0YW5pY190cmFpbjIpCmBgYAoKYGBge3J9CnRpdGFuaWNfdGVzdCA8LSB0aXRhbmljX3Rlc3QgJT4lIGNsZWFuX25hbWVzKCkKCmhlYWQodGl0YW5pY190ZXN0KQpgYGAKCkl0IGlzIGFsd2F5cyBhIGdvb2QgaWRlYSB0byBjaGVjayBmb3IgZHVwbGljYXRlIHJlY29yZHMvZXhhbXBsZXMvcm93cyBpbiB5b3VyIGRhdGFzZXQuCgpgYGB7cn0KZ2V0X2R1cGVzKHRpdGFuaWNfdGVzdCkKYGBgCgpgYGB7cn0KdGl0YW5pY190ZXN0MiA8LSB0aXRhbmljX3Rlc3QgJT4lIHNlbGVjdCgtcGFzc2VuZ2VyX2lkLCAtbmFtZSwgLXRpY2tldCwgLWNhYmluKSAlPiUKICBtdXRhdGUoCiAgICBwY2xhc3MgPSBhc19mYWN0b3IocGNsYXNzKSwKICAgIHNleCA9IGFzX2ZhY3RvcihzZXgpLAogICAgZW1iYXJrZWQgPSBhc19mYWN0b3IoZW1iYXJrZWQpCiAgKQoKaGVhZCh0aXRhbmljX3Rlc3QyKQoKYGBgCgoKU3RhcnQgYnkgaW52ZXN0aWdhdGluZyB0aGUgbWlzc2luZyB2YWx1ZXMgYW5kIGNvbXBsZXRlbmVzcyBvZiB0aGUgZmVhdHVyZXMgaW4gdGhlIGRhdGEuICBOb3RlIHRoYXQgdGhlICphZ2UqIHZhcmlhYmxlIGNvbnRhaW5zIHNvbWUgbWlzc2luZyB2YWx1ZXMuCgpgYGB7cn0KdmlzX21pc3ModGl0YW5pY190cmFpbjIpCmdnX21pc3NfdmFyKHRpdGFuaWNfdHJhaW4yKQpnZ19taXNzX3Zhcih0aXRhbmljX3RyYWluMiwgc2hvd19wY3QgPSBUUlVFKQpgYGAKCmBgYHtyIGV2YWwgPSBGQUxTRX0KY3JlYXRlX3JlcG9ydCh0aXRhbmljX3RyYWluMiwgeSA9ICJzdXJ2aXZlZCIsIG91dHB1dF9maWxlID0gInJlcG9ydC5odG1sIiwgb3V0cHV0X2RpciA9IGdldHdkKCkpCmBgYAoKTm93IHRyeSB0aGUgTUwgYWxnb3JpdGhtcy4KCgojIyBNb2RlbCAwOgoKU3VtbWFyaXplIHRoZSB5LXZhcmlhYmxlLiAgTnVsbCBNb2RlbC4KCmBgYHtyfQp0aXRhbmljX3RyYWluMiAlPiUgZ3JvdXBfYnkoc3Vydml2ZWQpICU+JQogIHN1bW1hcml6ZShuID0gbigpKSAlPiUKICBtdXRhdGUoZnJlcSA9IG4gLyBzdW0obikpCmBgYAoKTWFrZSB0aGUgZmlyc3Qgc3BsaXQgd2l0aCA4MCUgb2YgdGhlIGRhdGEgYmVpbmcgaW4gdGhlIHRyYWluaW5nIGRhdGEgc2V0LgoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX3NwbGl0IDwtIGluaXRpYWxfc3BsaXQodGl0YW5pY190cmFpbjIsIHByb3AgPSAwLjgpCnRpdGFuaWNfdHJhaW4yX3NwbGl0CmBgYAoKVHJhaW5pbmcgZGF0YS4KCmBgYHtyfQp0aXRhbmljX3RyYWluMl9zcGxpdCAlPiUKICB0cmFpbmluZygpIApgYGAKCkNyZWF0ZSB0aGUgcmVjaXBlIGZvciBhcHBseWluZyB0aGUgcHJlcHJvY2Vzc2luZy4gIE5vdGUgdGhlIHVzZSBvZiBzdGVwX256digpLCB3aGljaCByZW1vdmVzIGFueSBjb2x1bW5zIHRoYXQgaGF2ZSB2ZXJ5IGxvdyB2YXJpYWJpbGl0eSwgYW5kIHRoZSB1c2Ugb2YgdGhlIHN0ZXBfbWVhbmltcHV0ZSgpIGZ1bmN0aW9uLCB3aGljaCBmaWxscyBpbiB0aGUgY2VsbHMgdGhhdCBhcmUgbWlzc2luZyB3aXRoIHRoZSBtZWFuIG9mIHRoZSBjb2x1bW4uCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfcmVjaXBlIDwtIHRyYWluaW5nKHRpdGFuaWNfdHJhaW4yX3NwbGl0KSAlPiUKICByZWNpcGUoc3Vydml2ZWQgfiAuKSAlPiUKICBzdGVwX3JtKHBjbGFzcywgc2V4LCBlbWJhcmtlZCkgJT4lIAogIHN0ZXBfbnp2KGFsbF9wcmVkaWN0b3JzKCkpICU+JQogIHN0ZXBfbWVhbmltcHV0ZShhZ2UpICU+JQogIHByZXAoKQoKc3VtbWFyeSh0aXRhbmljX3RyYWluMl9yZWNpcGUpCgp0aWR5KHRpdGFuaWNfdHJhaW4yX3JlY2lwZSkKYGBgCgpBcHBseSB0aGUgcmVjaXBlLCBzbyB0aGUgKmFnZSogdmFyaWFibGUgc2hvdWxkIGJlIGNvbXBsZXRlIGFmdGVyIHRoZSBpbXB1dGF0aW9uLgoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcgPC0gdGl0YW5pY190cmFpbjJfcmVjaXBlICU+JQogIGJha2UodGVzdGluZyh0aXRhbmljX3RyYWluMl9zcGxpdCkpIAoKdGl0YW5pY190cmFpbjJfdGVzdGluZwpgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl90cmFpbmluZyA8LSBqdWljZSh0aXRhbmljX3RyYWluMl9yZWNpcGUpCgp0aXRhbmljX3RyYWluMl90cmFpbmluZwpgYGAKCiMjIyBNb2RlbCAwOiBudWxsCgoKYGBge3J9Cm51bGxfbW9kZWwobW9kZSA9ICJjbGFzc2lmaWNhdGlvbiIpCgp0aXRhbmljX3RyYWluMl9udWxsIDwtIG51bGxfbW9kZWwoKSAlPiUKICBzZXRfbW9kZSgiY2xhc3NpZmljYXRpb24iKSAlPiUKICBmaXQoc3Vydml2ZWQgfiAuLCBkYXRhID0gdGl0YW5pY190cmFpbjJfdHJhaW5pbmcpCgpgYGAKCmBgYHtyfQpwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX251bGwsIHRpdGFuaWNfdHJhaW4yX3RyYWluaW5nKQpgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9udWxsICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpIApgYGAKCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfbnVsbCAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBtZXRyaWNzKHRydXRoID0gc3Vydml2ZWQsIGVzdGltYXRlID0gLnByZWRfY2xhc3MpCmBgYAoKCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfbnVsbCAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBjb25mX21hdCh0cnV0aCA9IHN1cnZpdmVkLCBlc3RpbWF0ZSA9IC5wcmVkX2NsYXNzKQpgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9udWxsICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZywgdHlwZSA9ICJwcm9iIikgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIHJvY19jdXJ2ZShzdXJ2aXZlZCwgLnByZWRfMCkgJT4lCiAgYXV0b3Bsb3QoKSAKYGBgCgoKCiMjIyBNb2RlbCA0OiBHTE0KClNldHVwIHRoZSBtb2RlbC4KCmBgYHtyfQp0aXRhbmljX3RyYWluMl9nbG0gPC0gbG9naXN0aWNfcmVnKHBlbmFsdHkgPSAwLjAwMSwgbWl4dHVyZSA9IDAuNSkgJT4lIAogIHNldF9lbmdpbmUoImdsbW5ldCIpICU+JQogIHNldF9tb2RlKCJjbGFzc2lmaWNhdGlvbiIpICU+JQogIGZpdChzdXJ2aXZlZCB+IC4sIGRhdGEgPSB0aXRhbmljX3RyYWluMl90cmFpbmluZykKCmBgYAoKCgpgYGB7cn0KcHJlZGljdCh0aXRhbmljX3RyYWluMl9nbG0sIHRpdGFuaWNfdHJhaW4yX3RyYWluaW5nKQpgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9nbG0gJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgCmBgYAoKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9nbG0gJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgbWV0cmljcyh0cnV0aCA9IHN1cnZpdmVkLCBlc3RpbWF0ZSA9IC5wcmVkX2NsYXNzKQpgYGAKCgoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX2dsbSAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBjb25mX21hdCh0cnV0aCA9IHN1cnZpdmVkLCBlc3RpbWF0ZSA9IC5wcmVkX2NsYXNzKQpgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9nbG0gJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nLCB0eXBlID0gInByb2IiKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgcm9jX2N1cnZlKHN1cnZpdmVkLCAucHJlZF8wKSAlPiUKICBhdXRvcGxvdCgpCmBgYAoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX2dsbSAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcsIHR5cGUgPSAicHJvYiIpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICByb2NfYXVjKHN1cnZpdmVkLCAucHJlZF8wKSAKYGBgCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfZ2xtICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZywgdHlwZSA9ICJwcm9iIikgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIGdncGxvdCgpICsKICBnZW9tX2RlbnNpdHkoYWVzKHggPSAucHJlZF8xLCBmaWxsID0gc3Vydml2ZWQpLCAKICAgICAgICAgICAgICAgYWxwaGEgPSAwLjUpCmBgYAoKIyMjIE1vZGVsIDU6IE5haXZlIEJheWVzCgpTZXR1cCB0aGUgbW9kZWwuCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfbmIgPC0gbmFpdmVfQmF5ZXMoTGFwbGFjZSA9IDEpICU+JSAKICBzZXRfZW5naW5lKCJrbGFSIikgJT4lCiAgc2V0X21vZGUoImNsYXNzaWZpY2F0aW9uIikgJT4lCiAgZml0KHN1cnZpdmVkIH4gLiwgZGF0YSA9IHRpdGFuaWNfdHJhaW4yX3RyYWluaW5nKQoKYGBgCgoKCmBgYHtyfQpwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX25iLCB0aXRhbmljX3RyYWluMl90cmFpbmluZykKYGBgCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfbmIgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgCmBgYAoKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9uYiAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBtZXRyaWNzKHRydXRoID0gc3Vydml2ZWQsIGVzdGltYXRlID0gLnByZWRfY2xhc3MpCmBgYAoKCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfbmIgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgY29uZl9tYXQodHJ1dGggPSBzdXJ2aXZlZCwgZXN0aW1hdGUgPSAucHJlZF9jbGFzcykKYGBgCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfbmIgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nLCB0eXBlID0gInByb2IiKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgcm9jX2N1cnZlKHN1cnZpdmVkLCAucHJlZF8wKSAlPiUKICBhdXRvcGxvdCgpIApgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9uYiAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcsIHR5cGUgPSAicHJvYiIpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICByb2NfYXVjKHN1cnZpdmVkLCAucHJlZF8wKSAKYGBgCgoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX25iICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZywgdHlwZSA9ICJwcm9iIikgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIGdncGxvdCgpICsKICBnZW9tX2RlbnNpdHkoYWVzKHggPSAucHJlZF8xLCBmaWxsID0gc3Vydml2ZWQpLCAKICAgICAgICAgICAgICAgYWxwaGEgPSAwLjUpCmBgYAoKCgo=