library(pacman)
p_load(titanic, tidyverse, janitor, naniar, DataExplorer, tidymodels, discrim)
Load the data from the titanic R package. Note that the titanic_train dataset contains the labels for Survived aND THE titanic_test dataset does not contain the labels. So we will build our machine learning model using the titanic_train dataset and then make a final classification for the titanic_test dataset. This is how kaggle competitions are done.
I like to clean names so the variables all have names with lowercase letters and underscores.
titanic_train <- titanic_train %>% clean_names()
head(titanic_train)
It is always a good idea to check for duplicate records/examples/rows in your dataset.
get_dupes(titanic_train)
No variable names specified - using all columns.
No duplicate combinations found of: passenger_id, survived, pclass, name, sex, age, sib_sp, parch, ticket, ... and 3 other variables
Drop the unique identifiers: passenger_id, name, and ticket. Also drop cabin because it has a high missing rate.
titanic_train2 <- titanic_train %>% select(-passenger_id, -name, -ticket, -cabin) %>%
mutate(
survived = as_factor(survived),
pclass = as_factor(pclass),
sex = as_factor(sex),
embarked = as_factor(embarked)
)
head(titanic_train2)
titanic_test <- titanic_test %>% clean_names()
head(titanic_test)
It is always a good idea to check for duplicate records/examples/rows in your dataset.
get_dupes(titanic_test)
No variable names specified - using all columns.
No duplicate combinations found of: passenger_id, pclass, name, sex, age, sib_sp, parch, ticket, fare, ... and 2 other variables
titanic_test2 <- titanic_test %>% select(-passenger_id, -name, -ticket, -cabin) %>%
mutate(
pclass = as_factor(pclass),
sex = as_factor(sex),
embarked = as_factor(embarked)
)
head(titanic_test2)
NA
Start by investigating the missing values and completeness of the features in the data. Note that the age variable contains some missing values.
vis_miss(titanic_train2)

gg_miss_var(titanic_train2)

gg_miss_var(titanic_train2, show_pct = TRUE)

create_report(titanic_train2, y = "survived", output_file = "report.html", output_dir = getwd())
Now try the ML algorithms.
Model 0:
Summarize the y-variable. Null Model.
titanic_train2 %>% group_by(survived) %>%
summarize(n = n()) %>%
mutate(freq = n / sum(n))
Make the first split with 80% of the data being in the training data set.
titanic_train2_split <- initial_split(titanic_train2, prop = 0.8)
titanic_train2_split
<Analysis/Assess/Total>
<713/178/891>
Training data.
titanic_train2_split %>%
training()
Create the recipe for applying the preprocessing. Note the use of step_nzv(), which removes any columns that have very low variability, and the use of the step_meanimpute() function, which fills in the cells that are missing with the mean of the column.
Apply the recipe, so the age variable should be complete after the imputation.
titanic_train2_testing <- titanic_train2_recipe %>%
bake(testing(titanic_train2_split))
titanic_train2_testing
titanic_train2_training <- juice(titanic_train2_recipe)
titanic_train2_training
Model 0: null
null_model(mode = "classification")
Model Specification (classification)
titanic_train2_null <- null_model() %>%
set_mode("classification") %>%
fit(survived ~ ., data = titanic_train2_training)
Engine set to `parsnip`.
predict(titanic_train2_null, titanic_train2_training)
titanic_train2_null %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing)
titanic_train2_null %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing) %>%
metrics(truth = survived, estimate = .pred_class)
titanic_train2_null %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing) %>%
conf_mat(truth = survived, estimate = .pred_class)
Truth
Prediction 0 1
0 109 69
1 0 0
titanic_train2_null %>%
predict(titanic_train2_testing, type = "prob") %>%
bind_cols(titanic_train2_testing) %>%
roc_curve(survived, .pred_0) %>%
autoplot()

Model 4: GLM
Setup the model.
titanic_train2_glm <- logistic_reg(penalty = 0.001, mixture = 0.5) %>%
set_engine("glmnet") %>%
set_mode("classification") %>%
fit(survived ~ ., data = titanic_train2_training)
predict(titanic_train2_glm, titanic_train2_training)
titanic_train2_glm %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing)
titanic_train2_glm %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing) %>%
metrics(truth = survived, estimate = .pred_class)
titanic_train2_glm %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing) %>%
conf_mat(truth = survived, estimate = .pred_class)
Truth
Prediction 0 1
0 104 48
1 5 21
titanic_train2_glm %>%
predict(titanic_train2_testing, type = "prob") %>%
bind_cols(titanic_train2_testing) %>%
roc_curve(survived, .pred_0) %>%
autoplot()

titanic_train2_glm %>%
predict(titanic_train2_testing, type = "prob") %>%
bind_cols(titanic_train2_testing) %>%
roc_auc(survived, .pred_0)

Model 5: Naive Bayes
Setup the model.
titanic_train2_nb <- naive_Bayes(Laplace = 1) %>%
set_engine("klaR") %>%
set_mode("classification") %>%
fit(survived ~ ., data = titanic_train2_training)
predict(titanic_train2_nb, titanic_train2_training)
Numerical 0 probability for all classes with observation 348
titanic_train2_nb %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing)
titanic_train2_nb %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing) %>%
metrics(truth = survived, estimate = .pred_class)
titanic_train2_nb %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing) %>%
conf_mat(truth = survived, estimate = .pred_class)
Truth
Prediction 0 1
0 81 40
1 28 29

titanic_train2_nb %>%
predict(titanic_train2_testing, type = "prob") %>%
bind_cols(titanic_train2_testing) %>%
roc_auc(survived, .pred_0)
titanic_train2_nb %>%
predict(titanic_train2_testing, type = "prob") %>%
bind_cols(titanic_train2_testing) %>%
ggplot() +
geom_density(aes(x = .pred_1, fill = survived),
alpha = 0.5)
LS0tCnRpdGxlOiAiU3RhdCA2NTI6IE1pZHRlcm06IE1vZGVsIDQsIE1vZGVsIDUiCmF1dGhvcjogIlByb2YuIEVyaWMgQS4gU3Vlc3MiCm91dHB1dDoKICBodG1sX25vdGVib29rOiBkZWZhdWx0CiAgd29yZF9kb2N1bWVudDogZGVmYXVsdAogIHBkZl9kb2N1bWVudDogZGVmYXVsdAotLS0KCgoKYGBge3J9CmxpYnJhcnkocGFjbWFuKQpwX2xvYWQodGl0YW5pYywgdGlkeXZlcnNlLCBqYW5pdG9yLCBuYW5pYXIsIERhdGFFeHBsb3JlciwgdGlkeW1vZGVscywgZGlzY3JpbSkKYGBgCgpMb2FkIHRoZSBkYXRhIGZyb20gdGhlIHRpdGFuaWMgUiBwYWNrYWdlLiAgTm90ZSB0aGF0IHRoZSAqdGl0YW5pY190cmFpbiogZGF0YXNldCBjb250YWlucyB0aGUgbGFiZWxzIGZvciBTdXJ2aXZlZCBhTkQgVEhFICp0aXRhbmljX3Rlc3QqIGRhdGFzZXQgZG9lcyBub3QgY29udGFpbiB0aGUgbGFiZWxzLiAgU28gd2Ugd2lsbCBidWlsZCBvdXIgbWFjaGluZSBsZWFybmluZyBtb2RlbCB1c2luZyB0aGUgKnRpdGFuaWNfdHJhaW4qIGRhdGFzZXQgYW5kIHRoZW4gbWFrZSBhIGZpbmFsIGNsYXNzaWZpY2F0aW9uIGZvciB0aGUgKnRpdGFuaWNfdGVzdCogZGF0YXNldC4gIFRoaXMgaXMgaG93IGthZ2dsZSBjb21wZXRpdGlvbnMgYXJlIGRvbmUuCgpJIGxpa2UgdG8gY2xlYW4gbmFtZXMgc28gdGhlIHZhcmlhYmxlcyBhbGwgaGF2ZSBuYW1lcyB3aXRoIGxvd2VyY2FzZSBsZXR0ZXJzIGFuZCB1bmRlcnNjb3Jlcy4KCmBgYHtyfQp0aXRhbmljX3RyYWluIDwtIHRpdGFuaWNfdHJhaW4gJT4lIGNsZWFuX25hbWVzKCkKCmhlYWQodGl0YW5pY190cmFpbikKYGBgCgpJdCBpcyBhbHdheXMgYSBnb29kIGlkZWEgdG8gY2hlY2sgZm9yIGR1cGxpY2F0ZSByZWNvcmRzL2V4YW1wbGVzL3Jvd3MgaW4geW91ciBkYXRhc2V0LgoKYGBge3J9CmdldF9kdXBlcyh0aXRhbmljX3RyYWluKQpgYGAKCkRyb3AgdGhlIHVuaXF1ZSBpZGVudGlmaWVyczogKnBhc3Nlbmdlcl9pZCosICpuYW1lKiwgYW5kICp0aWNrZXQqLiAgQWxzbyBkcm9wICpjYWJpbiogYmVjYXVzZSBpdCBoYXMgYSBoaWdoIG1pc3NpbmcgcmF0ZS4KCmBgYHtyfQp0aXRhbmljX3RyYWluMiA8LSB0aXRhbmljX3RyYWluICU+JSBzZWxlY3QoLXBhc3Nlbmdlcl9pZCwgLW5hbWUsIC10aWNrZXQsIC1jYWJpbikgJT4lCiAgbXV0YXRlKAogICAgc3Vydml2ZWQgPSBhc19mYWN0b3Ioc3Vydml2ZWQpLAogICAgcGNsYXNzID0gYXNfZmFjdG9yKHBjbGFzcyksCiAgICBzZXggPSBhc19mYWN0b3Ioc2V4KSwKICAgIGVtYmFya2VkID0gYXNfZmFjdG9yKGVtYmFya2VkKQogICkKCmhlYWQodGl0YW5pY190cmFpbjIpCmBgYAoKYGBge3J9CnRpdGFuaWNfdGVzdCA8LSB0aXRhbmljX3Rlc3QgJT4lIGNsZWFuX25hbWVzKCkKCmhlYWQodGl0YW5pY190ZXN0KQpgYGAKCkl0IGlzIGFsd2F5cyBhIGdvb2QgaWRlYSB0byBjaGVjayBmb3IgZHVwbGljYXRlIHJlY29yZHMvZXhhbXBsZXMvcm93cyBpbiB5b3VyIGRhdGFzZXQuCgpgYGB7cn0KZ2V0X2R1cGVzKHRpdGFuaWNfdGVzdCkKYGBgCgpgYGB7cn0KdGl0YW5pY190ZXN0MiA8LSB0aXRhbmljX3Rlc3QgJT4lIHNlbGVjdCgtcGFzc2VuZ2VyX2lkLCAtbmFtZSwgLXRpY2tldCwgLWNhYmluKSAlPiUKICBtdXRhdGUoCiAgICBwY2xhc3MgPSBhc19mYWN0b3IocGNsYXNzKSwKICAgIHNleCA9IGFzX2ZhY3RvcihzZXgpLAogICAgZW1iYXJrZWQgPSBhc19mYWN0b3IoZW1iYXJrZWQpCiAgKQoKaGVhZCh0aXRhbmljX3Rlc3QyKQoKYGBgCgoKU3RhcnQgYnkgaW52ZXN0aWdhdGluZyB0aGUgbWlzc2luZyB2YWx1ZXMgYW5kIGNvbXBsZXRlbmVzcyBvZiB0aGUgZmVhdHVyZXMgaW4gdGhlIGRhdGEuICBOb3RlIHRoYXQgdGhlICphZ2UqIHZhcmlhYmxlIGNvbnRhaW5zIHNvbWUgbWlzc2luZyB2YWx1ZXMuCgpgYGB7cn0KdmlzX21pc3ModGl0YW5pY190cmFpbjIpCmdnX21pc3NfdmFyKHRpdGFuaWNfdHJhaW4yKQpnZ19taXNzX3Zhcih0aXRhbmljX3RyYWluMiwgc2hvd19wY3QgPSBUUlVFKQpgYGAKCmBgYHtyIGV2YWwgPSBGQUxTRX0KY3JlYXRlX3JlcG9ydCh0aXRhbmljX3RyYWluMiwgeSA9ICJzdXJ2aXZlZCIsIG91dHB1dF9maWxlID0gInJlcG9ydC5odG1sIiwgb3V0cHV0X2RpciA9IGdldHdkKCkpCmBgYAoKTm93IHRyeSB0aGUgTUwgYWxnb3JpdGhtcy4KCgojIyBNb2RlbCAwOgoKU3VtbWFyaXplIHRoZSB5LXZhcmlhYmxlLiAgTnVsbCBNb2RlbC4KCmBgYHtyfQp0aXRhbmljX3RyYWluMiAlPiUgZ3JvdXBfYnkoc3Vydml2ZWQpICU+JQogIHN1bW1hcml6ZShuID0gbigpKSAlPiUKICBtdXRhdGUoZnJlcSA9IG4gLyBzdW0obikpCmBgYAoKTWFrZSB0aGUgZmlyc3Qgc3BsaXQgd2l0aCA4MCUgb2YgdGhlIGRhdGEgYmVpbmcgaW4gdGhlIHRyYWluaW5nIGRhdGEgc2V0LgoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX3NwbGl0IDwtIGluaXRpYWxfc3BsaXQodGl0YW5pY190cmFpbjIsIHByb3AgPSAwLjgpCnRpdGFuaWNfdHJhaW4yX3NwbGl0CmBgYAoKVHJhaW5pbmcgZGF0YS4KCmBgYHtyfQp0aXRhbmljX3RyYWluMl9zcGxpdCAlPiUKICB0cmFpbmluZygpIApgYGAKCkNyZWF0ZSB0aGUgcmVjaXBlIGZvciBhcHBseWluZyB0aGUgcHJlcHJvY2Vzc2luZy4gIE5vdGUgdGhlIHVzZSBvZiBzdGVwX256digpLCB3aGljaCByZW1vdmVzIGFueSBjb2x1bW5zIHRoYXQgaGF2ZSB2ZXJ5IGxvdyB2YXJpYWJpbGl0eSwgYW5kIHRoZSB1c2Ugb2YgdGhlIHN0ZXBfbWVhbmltcHV0ZSgpIGZ1bmN0aW9uLCB3aGljaCBmaWxscyBpbiB0aGUgY2VsbHMgdGhhdCBhcmUgbWlzc2luZyB3aXRoIHRoZSBtZWFuIG9mIHRoZSBjb2x1bW4uCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfcmVjaXBlIDwtIHRyYWluaW5nKHRpdGFuaWNfdHJhaW4yX3NwbGl0KSAlPiUKICByZWNpcGUoc3Vydml2ZWQgfiAuKSAlPiUKICBzdGVwX3JtKHBjbGFzcywgc2V4LCBlbWJhcmtlZCkgJT4lIAogIHN0ZXBfbnp2KGFsbF9wcmVkaWN0b3JzKCkpICU+JQogIHN0ZXBfbWVhbmltcHV0ZShhZ2UpICU+JQogIHByZXAoKQoKc3VtbWFyeSh0aXRhbmljX3RyYWluMl9yZWNpcGUpCgp0aWR5KHRpdGFuaWNfdHJhaW4yX3JlY2lwZSkKYGBgCgpBcHBseSB0aGUgcmVjaXBlLCBzbyB0aGUgKmFnZSogdmFyaWFibGUgc2hvdWxkIGJlIGNvbXBsZXRlIGFmdGVyIHRoZSBpbXB1dGF0aW9uLgoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcgPC0gdGl0YW5pY190cmFpbjJfcmVjaXBlICU+JQogIGJha2UodGVzdGluZyh0aXRhbmljX3RyYWluMl9zcGxpdCkpIAoKdGl0YW5pY190cmFpbjJfdGVzdGluZwpgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl90cmFpbmluZyA8LSBqdWljZSh0aXRhbmljX3RyYWluMl9yZWNpcGUpCgp0aXRhbmljX3RyYWluMl90cmFpbmluZwpgYGAKCiMjIyBNb2RlbCAwOiBudWxsCgoKYGBge3J9Cm51bGxfbW9kZWwobW9kZSA9ICJjbGFzc2lmaWNhdGlvbiIpCgp0aXRhbmljX3RyYWluMl9udWxsIDwtIG51bGxfbW9kZWwoKSAlPiUKICBzZXRfbW9kZSgiY2xhc3NpZmljYXRpb24iKSAlPiUKICBmaXQoc3Vydml2ZWQgfiAuLCBkYXRhID0gdGl0YW5pY190cmFpbjJfdHJhaW5pbmcpCgpgYGAKCmBgYHtyfQpwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX251bGwsIHRpdGFuaWNfdHJhaW4yX3RyYWluaW5nKQpgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9udWxsICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpIApgYGAKCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfbnVsbCAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBtZXRyaWNzKHRydXRoID0gc3Vydml2ZWQsIGVzdGltYXRlID0gLnByZWRfY2xhc3MpCmBgYAoKCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfbnVsbCAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBjb25mX21hdCh0cnV0aCA9IHN1cnZpdmVkLCBlc3RpbWF0ZSA9IC5wcmVkX2NsYXNzKQpgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9udWxsICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZywgdHlwZSA9ICJwcm9iIikgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIHJvY19jdXJ2ZShzdXJ2aXZlZCwgLnByZWRfMCkgJT4lCiAgYXV0b3Bsb3QoKSAKYGBgCgoKCiMjIyBNb2RlbCA0OiBHTE0KClNldHVwIHRoZSBtb2RlbC4KCmBgYHtyfQp0aXRhbmljX3RyYWluMl9nbG0gPC0gbG9naXN0aWNfcmVnKHBlbmFsdHkgPSAwLjAwMSwgbWl4dHVyZSA9IDAuNSkgJT4lIAogIHNldF9lbmdpbmUoImdsbW5ldCIpICU+JQogIHNldF9tb2RlKCJjbGFzc2lmaWNhdGlvbiIpICU+JQogIGZpdChzdXJ2aXZlZCB+IC4sIGRhdGEgPSB0aXRhbmljX3RyYWluMl90cmFpbmluZykKCmBgYAoKCgpgYGB7cn0KcHJlZGljdCh0aXRhbmljX3RyYWluMl9nbG0sIHRpdGFuaWNfdHJhaW4yX3RyYWluaW5nKQpgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9nbG0gJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgCmBgYAoKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9nbG0gJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgbWV0cmljcyh0cnV0aCA9IHN1cnZpdmVkLCBlc3RpbWF0ZSA9IC5wcmVkX2NsYXNzKQpgYGAKCgoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX2dsbSAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBjb25mX21hdCh0cnV0aCA9IHN1cnZpdmVkLCBlc3RpbWF0ZSA9IC5wcmVkX2NsYXNzKQpgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9nbG0gJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nLCB0eXBlID0gInByb2IiKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgcm9jX2N1cnZlKHN1cnZpdmVkLCAucHJlZF8wKSAlPiUKICBhdXRvcGxvdCgpCmBgYAoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX2dsbSAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcsIHR5cGUgPSAicHJvYiIpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICByb2NfYXVjKHN1cnZpdmVkLCAucHJlZF8wKSAKYGBgCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfZ2xtICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZywgdHlwZSA9ICJwcm9iIikgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIGdncGxvdCgpICsKICBnZW9tX2RlbnNpdHkoYWVzKHggPSAucHJlZF8xLCBmaWxsID0gc3Vydml2ZWQpLCAKICAgICAgICAgICAgICAgYWxwaGEgPSAwLjUpCmBgYAoKIyMjIE1vZGVsIDU6IE5haXZlIEJheWVzCgpTZXR1cCB0aGUgbW9kZWwuCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfbmIgPC0gbmFpdmVfQmF5ZXMoTGFwbGFjZSA9IDEpICU+JSAKICBzZXRfZW5naW5lKCJrbGFSIikgJT4lCiAgc2V0X21vZGUoImNsYXNzaWZpY2F0aW9uIikgJT4lCiAgZml0KHN1cnZpdmVkIH4gLiwgZGF0YSA9IHRpdGFuaWNfdHJhaW4yX3RyYWluaW5nKQoKYGBgCgoKCmBgYHtyfQpwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX25iLCB0aXRhbmljX3RyYWluMl90cmFpbmluZykKYGBgCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfbmIgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgCmBgYAoKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9uYiAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBtZXRyaWNzKHRydXRoID0gc3Vydml2ZWQsIGVzdGltYXRlID0gLnByZWRfY2xhc3MpCmBgYAoKCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfbmIgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgY29uZl9tYXQodHJ1dGggPSBzdXJ2aXZlZCwgZXN0aW1hdGUgPSAucHJlZF9jbGFzcykKYGBgCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfbmIgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nLCB0eXBlID0gInByb2IiKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgcm9jX2N1cnZlKHN1cnZpdmVkLCAucHJlZF8wKSAlPiUKICBhdXRvcGxvdCgpIApgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9uYiAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcsIHR5cGUgPSAicHJvYiIpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICByb2NfYXVjKHN1cnZpdmVkLCAucHJlZF8wKSAKYGBgCgoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX25iICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZywgdHlwZSA9ICJwcm9iIikgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIGdncGxvdCgpICsKICBnZW9tX2RlbnNpdHkoYWVzKHggPSAucHJlZF8xLCBmaWxsID0gc3Vydml2ZWQpLCAKICAgICAgICAgICAgICAgYWxwaGEgPSAwLjUpCmBgYAoKCgo=