library(pacman)
p_load(titanic, tidyverse, janitor, naniar, DataExplorer, tidymodels)
Load the data from the titanic R package. Note that the titanic_train dataset contains the labels for Survived aND THE titanic_test dataset does not contain the labels. So we will build our machine learning model using the titanic_train dataset and then make a final classification for the titanic_test dataset. This is how kaggle competitions are done.
I like to clean names so the variables all have names with lowercase letters and underscores.
titanic_train <- titanic_train %>% clean_names()
head(titanic_train)
It is always a good idea to check for duplicate records/examples/rows in your dataset.
get_dupes(titanic_train)
No variable names specified - using all columns.
No duplicate combinations found of: passenger_id, survived, pclass, name, sex, age, sib_sp, parch, ticket, ... and 3 other variables
Drop the unique identifiers: passenger_id, name, and ticket. Also drop cabin because it has a high missing rate.
titanic_test <- titanic_test %>% clean_names()
head(titanic_test)
It is always a good idea to check for duplicate records/examples/rows in your dataset.
get_dupes(titanic_test)
No variable names specified - using all columns.
No duplicate combinations found of: passenger_id, pclass, name, sex, age, sib_sp, parch, ticket, fare, ... and 2 other variables
titanic_test2 <- titanic_test %>% select(-passenger_id, -name, -ticket, -cabin) %>%
mutate(
pclass = as_factor(pclass),
sex = as_factor(sex),
embarked = as_factor(embarked)
)
head(titanic_test2)
NA
Start by investigating the missing values and completeness of the features in the data. Note that the age variable contains some missing values.
vis_miss(titanic_train2)

gg_miss_var(titanic_train2)

gg_miss_var(titanic_train2, show_pct = TRUE)

create_report(titanic_train2, y = "survived", output_file = "report.html", output_dir = getwd())
Now try the ML algorithms.
Model 0:
Summarize the y-variable. Null Model.
titanic_train2 %>% group_by(survived) %>%
summarize(n = n()) %>%
mutate(freq = n / sum(n))
Make the first split with 80% of the data being in the training data set.
titanic_train2_split <- initial_split(titanic_train2, prop = 0.8)
titanic_train2_split
<Analysis/Assess/Total>
<713/178/891>
Training data.
titanic_train2_split %>%
training()
Create the recipe for applying the preprocessing. Note the use of step_nzv(), which removes any columns that have very low variability, and the use of the step_meanimpute() function, which fills in the cells that are missing with the mean of the column.
titanic_train2_recipe <- training(titanic_train2_split) %>%
recipe(survived ~ .) %>%
step_rm(pclass, sex, embarked) %>%
step_nzv(all_predictors()) %>%
step_meanimpute(age) %>%
prep()
summary(titanic_train2_recipe)
tidy(titanic_train2_recipe)
Apply the receipe, so the age variable should be complete after the imputation.
titanic_train2_testing <- titanic_train2_recipe %>%
bake(testing(titanic_train2_split))
titanic_train2_testing
titanic_train2_training <- juice(titanic_train2_recipe)
titanic_train2_training
Model 0: null
titanic_train2_null <- null_model() %>%
set_mode("classification") %>%
fit(survived ~ ., data = titanic_train2_training)
Engine set to `parsnip`.
predict(titanic_train2_null, titanic_train2_training)
titanic_train2_null %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing)
titanic_train2_null %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing) %>%
metrics(truth = survived, estimate = .pred_class)
titanic_train2_null %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing) %>%
conf_mat(truth = survived, estimate = .pred_class)
Truth
Prediction 0 1
0 109 69
1 0 0

Model 2: C5.0
Setup the model.
titanic_train2_C50 <- boost_tree(trees = 20) %>%
set_engine("C5.0") %>%
set_mode("classification") %>%
fit(survived ~ ., data = titanic_train2_training)
predict(titanic_train2_C50, titanic_train2_training)
titanic_train2_C50 %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing)
titanic_train2_C50 %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing) %>%
metrics(truth = survived, estimate = .pred_class)
titanic_train2_C50 %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing) %>%
conf_mat(truth = survived, estimate = .pred_class)
Truth
Prediction 0 1
0 92 39
1 17 30
titanic_train2_C50 %>%
predict(titanic_train2_testing, type = "prob") %>%
bind_cols(titanic_train2_testing) %>%
roc_curve(survived, .pred_0) %>%
autoplot()

titanic_train2_C50 %>%
predict(titanic_train2_testing, type = "prob") %>%
bind_cols(titanic_train2_testing) %>%
roc_auc(survived, .pred_0)
titanic_train2_C50 %>%
predict(titanic_train2_testing, type = "prob") %>%
bind_cols(titanic_train2_testing) %>%
ggplot() +
geom_density(aes(x = .pred_1, fill = survived),
alpha = 0.5)

Model 2: XGBoost
Setup the model.
titanic_train2_xgb <- boost_tree(trees = 20) %>%
set_engine("xgboost") %>%
set_mode("classification") %>%
fit(survived ~ ., data = titanic_train2_training)
[09:57:43] WARNING: amalgamation/../src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
predict(titanic_train2_xgb, titanic_train2_training)
titanic_train2_xgb %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing)
titanic_train2_xgb %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing) %>%
metrics(truth = survived, estimate = .pred_class)
titanic_train2_xgb %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing) %>%
conf_mat(truth = survived, estimate = .pred_class)
Truth
Prediction 0 1
0 90 38
1 19 31
titanic_train2_xgb %>%
predict(titanic_train2_testing, type = "prob") %>%
bind_cols(titanic_train2_testing) %>%
roc_curve(survived, .pred_0) %>%
autoplot()

titanic_train2_xgb %>%
predict(titanic_train2_testing, type = "prob") %>%
bind_cols(titanic_train2_testing) %>%
roc_auc(survived, .pred_0)
titanic_train2_xgb %>%
predict(titanic_train2_testing, type = "prob") %>%
bind_cols(titanic_train2_testing) %>%
ggplot() +
geom_density(aes(x = .pred_1, fill = survived),
alpha = 0.5)

Model 3: Random Forest
Setup the model.
titanic_train2_ranger <- rand_forest(trees = 100) %>%
set_engine("ranger") %>%
set_mode("classification") %>%
fit(survived ~ ., data = titanic_train2_training)
predict(titanic_train2_ranger, titanic_train2_training)
titanic_train2_ranger %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing)
titanic_train2_ranger %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing) %>%
metrics(truth = survived, estimate = .pred_class)
titanic_train2_ranger %>%
predict(titanic_train2_testing) %>%
bind_cols(titanic_train2_testing) %>%
conf_mat(truth = survived, estimate = .pred_class)
Truth
Prediction 0 1
0 89 33
1 20 36
titanic_train2_ranger %>%
predict(titanic_train2_testing, type = "prob") %>%
bind_cols(titanic_train2_testing) %>%
roc_curve(survived, .pred_1) %>%
ggplot(aes(x = 1 - specificity, y = sensitivity)) +
geom_path() +
geom_abline(lty = 3) +
coord_equal()

titanic_train2_ranger %>%
predict(titanic_train2_testing, type = "prob") %>%
bind_cols(titanic_train2_testing) %>%
roc_curve(survived, event_level = "first") %>%
autoplot()
Error: No valid variables provided to `...`.
Run `rlang::last_error()` to see where the error occurred.
titanic_train2_ranger %>%
predict(titanic_train2_testing, type = "prob") %>%
bind_cols(titanic_train2_testing) %>%
roc_auc(survived, .pred_0)
titanic_train2_ranger %>%
predict(titanic_train2_testing, type = "prob") %>%
bind_cols(titanic_train2_testing) %>%
ggplot() +
geom_density(aes(x = .pred_1, fill = survived),
alpha = 0.5)

LS0tCnRpdGxlOiAiU3RhdCA2NTI6IE1pZHRlcm06IE1vZGVsIDIsIE1vZGVsIDMiCmF1dGhvcjogIlByb2YuIEVyaWMgQS4gU3Vlc3MiCm91dHB1dDoKICBodG1sX25vdGVib29rOiBkZWZhdWx0CiAgd29yZF9kb2N1bWVudDogZGVmYXVsdAogIHBkZl9kb2N1bWVudDogZGVmYXVsdAotLS0KCgoKYGBge3J9CmxpYnJhcnkocGFjbWFuKQpwX2xvYWQodGl0YW5pYywgdGlkeXZlcnNlLCBqYW5pdG9yLCBuYW5pYXIsIERhdGFFeHBsb3JlciwgdGlkeW1vZGVscykKYGBgCgpMb2FkIHRoZSBkYXRhIGZyb20gdGhlIHRpdGFuaWMgUiBwYWNrYWdlLiAgTm90ZSB0aGF0IHRoZSAqdGl0YW5pY190cmFpbiogZGF0YXNldCBjb250YWlucyB0aGUgbGFiZWxzIGZvciBTdXJ2aXZlZCBhTkQgVEhFICp0aXRhbmljX3Rlc3QqIGRhdGFzZXQgZG9lcyBub3QgY29udGFpbiB0aGUgbGFiZWxzLiAgU28gd2Ugd2lsbCBidWlsZCBvdXIgbWFjaGluZSBsZWFybmluZyBtb2RlbCB1c2luZyB0aGUgKnRpdGFuaWNfdHJhaW4qIGRhdGFzZXQgYW5kIHRoZW4gbWFrZSBhIGZpbmFsIGNsYXNzaWZpY2F0aW9uIGZvciB0aGUgKnRpdGFuaWNfdGVzdCogZGF0YXNldC4gIFRoaXMgaXMgaG93IGthZ2dsZSBjb21wZXRpdGlvbnMgYXJlIGRvbmUuCgpJIGxpa2UgdG8gY2xlYW4gbmFtZXMgc28gdGhlIHZhcmlhYmxlcyBhbGwgaGF2ZSBuYW1lcyB3aXRoIGxvd2VyY2FzZSBsZXR0ZXJzIGFuZCB1bmRlcnNjb3Jlcy4KCmBgYHtyfQp0aXRhbmljX3RyYWluIDwtIHRpdGFuaWNfdHJhaW4gJT4lIGNsZWFuX25hbWVzKCkKCmhlYWQodGl0YW5pY190cmFpbikKYGBgCgpJdCBpcyBhbHdheXMgYSBnb29kIGlkZWEgdG8gY2hlY2sgZm9yIGR1cGxpY2F0ZSByZWNvcmRzL2V4YW1wbGVzL3Jvd3MgaW4geW91ciBkYXRhc2V0LgoKYGBge3J9CmdldF9kdXBlcyh0aXRhbmljX3RyYWluKQpgYGAKCkRyb3AgdGhlIHVuaXF1ZSBpZGVudGlmaWVyczogKnBhc3Nlbmdlcl9pZCosICpuYW1lKiwgYW5kICp0aWNrZXQqLiAgQWxzbyBkcm9wICpjYWJpbiogYmVjYXVzZSBpdCBoYXMgYSBoaWdoIG1pc3NpbmcgcmF0ZS4KCmBgYHtyfQp0aXRhbmljX3RyYWluMiA8LSB0aXRhbmljX3RyYWluICU+JSBzZWxlY3QoLXBhc3Nlbmdlcl9pZCwgLW5hbWUsIC10aWNrZXQsIC1jYWJpbikgJT4lCiAgbXV0YXRlKAogICAgc3Vydml2ZWQgPSBhc19mYWN0b3Ioc3Vydml2ZWQpICwKICAgIHBjbGFzcyA9IGFzX2ZhY3RvcihwY2xhc3MpLAogICAgc2V4ID0gYXNfZmFjdG9yKHNleCksCiAgICBlbWJhcmtlZCA9IGFzX2ZhY3RvcihlbWJhcmtlZCkKICApCgpoZWFkKHRpdGFuaWNfdHJhaW4yKQpgYGAKCmBgYHtyfQp0aXRhbmljX3Rlc3QgPC0gdGl0YW5pY190ZXN0ICU+JSBjbGVhbl9uYW1lcygpCgpoZWFkKHRpdGFuaWNfdGVzdCkKYGBgCgpJdCBpcyBhbHdheXMgYSBnb29kIGlkZWEgdG8gY2hlY2sgZm9yIGR1cGxpY2F0ZSByZWNvcmRzL2V4YW1wbGVzL3Jvd3MgaW4geW91ciBkYXRhc2V0LgoKYGBge3J9CmdldF9kdXBlcyh0aXRhbmljX3Rlc3QpCmBgYAoKYGBge3J9CnRpdGFuaWNfdGVzdDIgPC0gdGl0YW5pY190ZXN0ICU+JSBzZWxlY3QoLXBhc3Nlbmdlcl9pZCwgLW5hbWUsIC10aWNrZXQsIC1jYWJpbikgJT4lCiAgbXV0YXRlKAogICAgcGNsYXNzID0gYXNfZmFjdG9yKHBjbGFzcyksCiAgICBzZXggPSBhc19mYWN0b3Ioc2V4KSwKICAgIGVtYmFya2VkID0gYXNfZmFjdG9yKGVtYmFya2VkKQogICkKCmhlYWQodGl0YW5pY190ZXN0MikKCmBgYAoKClN0YXJ0IGJ5IGludmVzdGlnYXRpbmcgdGhlIG1pc3NpbmcgdmFsdWVzIGFuZCBjb21wbGV0ZW5lc3Mgb2YgdGhlIGZlYXR1cmVzIGluIHRoZSBkYXRhLiAgTm90ZSB0aGF0IHRoZSAqYWdlKiB2YXJpYWJsZSBjb250YWlucyBzb21lIG1pc3NpbmcgdmFsdWVzLgoKYGBge3J9CnZpc19taXNzKHRpdGFuaWNfdHJhaW4yKQpnZ19taXNzX3Zhcih0aXRhbmljX3RyYWluMikKZ2dfbWlzc192YXIodGl0YW5pY190cmFpbjIsIHNob3dfcGN0ID0gVFJVRSkKYGBgCgpgYGB7ciBldmFsID0gRkFMU0V9CmNyZWF0ZV9yZXBvcnQodGl0YW5pY190cmFpbjIsIHkgPSAic3Vydml2ZWQiLCBvdXRwdXRfZmlsZSA9ICJyZXBvcnQuaHRtbCIsIG91dHB1dF9kaXIgPSBnZXR3ZCgpKQpgYGAKCk5vdyB0cnkgdGhlIE1MIGFsZ29yaXRobXMuCgoKIyMgTW9kZWwgMDoKClN1bW1hcml6ZSB0aGUgeS12YXJpYWJsZS4gIE51bGwgTW9kZWwuCgpgYGB7cn0KdGl0YW5pY190cmFpbjIgJT4lIGdyb3VwX2J5KHN1cnZpdmVkKSAlPiUKICBzdW1tYXJpemUobiA9IG4oKSkgJT4lCiAgbXV0YXRlKGZyZXEgPSBuIC8gc3VtKG4pKQpgYGAKCk1ha2UgdGhlIGZpcnN0IHNwbGl0IHdpdGggODAlIG9mIHRoZSBkYXRhIGJlaW5nIGluIHRoZSB0cmFpbmluZyBkYXRhIHNldC4KCmBgYHtyfQp0aXRhbmljX3RyYWluMl9zcGxpdCA8LSBpbml0aWFsX3NwbGl0KHRpdGFuaWNfdHJhaW4yLCBwcm9wID0gMC44KQp0aXRhbmljX3RyYWluMl9zcGxpdApgYGAKClRyYWluaW5nIGRhdGEuCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfc3BsaXQgJT4lCiAgdHJhaW5pbmcoKSAKYGBgCgpDcmVhdGUgdGhlIHJlY2lwZSBmb3IgYXBwbHlpbmcgdGhlIHByZXByb2Nlc3NpbmcuICBOb3RlIHRoZSB1c2Ugb2Ygc3RlcF9uenYoKSwgd2hpY2ggcmVtb3ZlcyBhbnkgY29sdW1ucyB0aGF0IGhhdmUgdmVyeSBsb3cgdmFyaWFiaWxpdHksIGFuZCB0aGUgdXNlIG9mIHRoZSBzdGVwX21lYW5pbXB1dGUoKSBmdW5jdGlvbiwgd2hpY2ggZmlsbHMgaW4gdGhlIGNlbGxzIHRoYXQgYXJlIG1pc3Npbmcgd2l0aCB0aGUgbWVhbiBvZiB0aGUgY29sdW1uLgoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX3JlY2lwZSA8LSB0cmFpbmluZyh0aXRhbmljX3RyYWluMl9zcGxpdCkgJT4lCiAgcmVjaXBlKHN1cnZpdmVkIH4gLikgJT4lCiAgc3RlcF9ybShwY2xhc3MsIHNleCwgZW1iYXJrZWQpICU+JSAKICBzdGVwX256dihhbGxfcHJlZGljdG9ycygpKSAlPiUKICBzdGVwX21lYW5pbXB1dGUoYWdlKSAlPiUKICBwcmVwKCkKCnN1bW1hcnkodGl0YW5pY190cmFpbjJfcmVjaXBlKQoKdGlkeSh0aXRhbmljX3RyYWluMl9yZWNpcGUpCmBgYAoKQXBwbHkgdGhlIHJlY2VpcGUsIHNvIHRoZSAqYWdlKiB2YXJpYWJsZSBzaG91bGQgYmUgY29tcGxldGUgYWZ0ZXIgdGhlIGltcHV0YXRpb24uCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfdGVzdGluZyA8LSB0aXRhbmljX3RyYWluMl9yZWNpcGUgJT4lCiAgYmFrZSh0ZXN0aW5nKHRpdGFuaWNfdHJhaW4yX3NwbGl0KSkgCgp0aXRhbmljX3RyYWluMl90ZXN0aW5nCmBgYAoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX3RyYWluaW5nIDwtIGp1aWNlKHRpdGFuaWNfdHJhaW4yX3JlY2lwZSkKCnRpdGFuaWNfdHJhaW4yX3RyYWluaW5nCmBgYAoKIyMjIE1vZGVsIDA6IG51bGwKCgpgYGB7cn0KCnRpdGFuaWNfdHJhaW4yX251bGwgPC0gbnVsbF9tb2RlbCgpICU+JQogIHNldF9tb2RlKCJjbGFzc2lmaWNhdGlvbiIpICU+JQogIGZpdChzdXJ2aXZlZCB+IC4sIGRhdGEgPSB0aXRhbmljX3RyYWluMl90cmFpbmluZykKCmBgYAoKYGBge3J9CnByZWRpY3QodGl0YW5pY190cmFpbjJfbnVsbCwgdGl0YW5pY190cmFpbjJfdHJhaW5pbmcpCmBgYAoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX251bGwgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgCmBgYAoKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9udWxsICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIG1ldHJpY3ModHJ1dGggPSBzdXJ2aXZlZCwgZXN0aW1hdGUgPSAucHJlZF9jbGFzcykKYGBgCgoKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9udWxsICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIGNvbmZfbWF0KHRydXRoID0gc3Vydml2ZWQsIGVzdGltYXRlID0gLnByZWRfY2xhc3MpCmBgYAoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX251bGwgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nLCB0eXBlID0gInByb2IiKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgcm9jX2N1cnZlKHN1cnZpdmVkLCAucHJlZF8xKSAlPiUKICBhdXRvcGxvdCgpCmBgYAoKIyMjIE1vZGVsIDI6IEM1LjAKClNldHVwIHRoZSBtb2RlbC4KCmBgYHtyfQp0aXRhbmljX3RyYWluMl9DNTAgPC0gYm9vc3RfdHJlZSh0cmVlcyA9IDIwKSAlPiUgCiAgc2V0X2VuZ2luZSgiQzUuMCIpICU+JQogIHNldF9tb2RlKCJjbGFzc2lmaWNhdGlvbiIpICU+JQogIGZpdChzdXJ2aXZlZCB+IC4sIGRhdGEgPSB0aXRhbmljX3RyYWluMl90cmFpbmluZykKCmBgYAoKCgpgYGB7cn0KcHJlZGljdCh0aXRhbmljX3RyYWluMl9DNTAsIHRpdGFuaWNfdHJhaW4yX3RyYWluaW5nKQpgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9DNTAgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgCmBgYAoKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9DNTAgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgbWV0cmljcyh0cnV0aCA9IHN1cnZpdmVkLCBlc3RpbWF0ZSA9IC5wcmVkX2NsYXNzKQpgYGAKCgoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX0M1MCAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBjb25mX21hdCh0cnV0aCA9IHN1cnZpdmVkLCBlc3RpbWF0ZSA9IC5wcmVkX2NsYXNzKQpgYGAKCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfQzUwICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZywgdHlwZSA9ICJwcm9iIikgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIHJvY19jdXJ2ZShzdXJ2aXZlZCwgLnByZWRfMCkgJT4lCiAgYXV0b3Bsb3QoKQpgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9DNTAgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nLCB0eXBlID0gInByb2IiKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgcm9jX2F1YyhzdXJ2aXZlZCwgLnByZWRfMCkgCmBgYAoKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9DNTAgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nLCB0eXBlID0gInByb2IiKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgZ2dwbG90KCkgKwogIGdlb21fZGVuc2l0eShhZXMoeCA9IC5wcmVkXzEsIGZpbGwgPSBzdXJ2aXZlZCksIAogICAgICAgICAgICAgICBhbHBoYSA9IDAuNSkKYGBgCgoKCiMjIyBNb2RlbCAyOiBYR0Jvb3N0CgpTZXR1cCB0aGUgbW9kZWwuCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfeGdiIDwtIGJvb3N0X3RyZWUodHJlZXMgPSAyMCkgJT4lIAogIHNldF9lbmdpbmUoInhnYm9vc3QiKSAlPiUKICBzZXRfbW9kZSgiY2xhc3NpZmljYXRpb24iKSAlPiUKICBmaXQoc3Vydml2ZWQgfiAuLCBkYXRhID0gdGl0YW5pY190cmFpbjJfdHJhaW5pbmcpCgpgYGAKCgoKYGBge3J9CnByZWRpY3QodGl0YW5pY190cmFpbjJfeGdiLCB0aXRhbmljX3RyYWluMl90cmFpbmluZykKYGBgCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfeGdiICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpIApgYGAKCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfeGdiICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIG1ldHJpY3ModHJ1dGggPSBzdXJ2aXZlZCwgZXN0aW1hdGUgPSAucHJlZF9jbGFzcykKYGBgCgoKCmBgYHtyfQp0aXRhbmljX3RyYWluMl94Z2IgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgY29uZl9tYXQodHJ1dGggPSBzdXJ2aXZlZCwgZXN0aW1hdGUgPSAucHJlZF9jbGFzcykKYGBgCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfeGdiICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZywgdHlwZSA9ICJwcm9iIikgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIHJvY19jdXJ2ZShzdXJ2aXZlZCwgLnByZWRfMCkgJT4lCiAgYXV0b3Bsb3QoKSAKYGBgCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfeGdiICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZywgdHlwZSA9ICJwcm9iIikgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIHJvY19hdWMoc3Vydml2ZWQsIC5wcmVkXzApIApgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl94Z2IgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nLCB0eXBlID0gInByb2IiKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgZ2dwbG90KCkgKwogIGdlb21fZGVuc2l0eShhZXMoeCA9IC5wcmVkXzEsIGZpbGwgPSBzdXJ2aXZlZCksIAogICAgICAgICAgICAgICBhbHBoYSA9IDAuNSkKYGBgCgojIyMgTW9kZWwgMzogUmFuZG9tIEZvcmVzdAoKU2V0dXAgdGhlIG1vZGVsLgoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX3JhbmdlciA8LSByYW5kX2ZvcmVzdCh0cmVlcyA9IDEwMCkgJT4lIAogIHNldF9lbmdpbmUoInJhbmdlciIpICU+JQogIHNldF9tb2RlKCJjbGFzc2lmaWNhdGlvbiIpICU+JQogIGZpdChzdXJ2aXZlZCB+IC4sIGRhdGEgPSB0aXRhbmljX3RyYWluMl90cmFpbmluZykKCmBgYAoKCgpgYGB7cn0KcHJlZGljdCh0aXRhbmljX3RyYWluMl9yYW5nZXIsIHRpdGFuaWNfdHJhaW4yX3RyYWluaW5nKQpgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9yYW5nZXIgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgCmBgYAoKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9yYW5nZXIgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgbWV0cmljcyh0cnV0aCA9IHN1cnZpdmVkLCBlc3RpbWF0ZSA9IC5wcmVkX2NsYXNzKQpgYGAKCgoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX3JhbmdlciAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBjb25mX21hdCh0cnV0aCA9IHN1cnZpdmVkLCBlc3RpbWF0ZSA9IC5wcmVkX2NsYXNzKQpgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9yYW5nZXIgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nLCB0eXBlID0gInByb2IiKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgcm9jX2N1cnZlKHN1cnZpdmVkLCAucHJlZF8xKSAlPiUKICBnZ3Bsb3QoYWVzKHggPSAxIC0gc3BlY2lmaWNpdHksIHkgPSBzZW5zaXRpdml0eSkpICsKICBnZW9tX3BhdGgoKSArCiAgZ2VvbV9hYmxpbmUobHR5ID0gMykgKwogIGNvb3JkX2VxdWFsKCkgCgp0aXRhbmljX3RyYWluMl9yYW5nZXIgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nLCB0eXBlID0gInByb2IiKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgcm9jX2N1cnZlKHN1cnZpdmVkLCAucHJlZF8wKSAlPiUKICBhdXRvcGxvdCgpCgoKCmBgYAoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX3JhbmdlciAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcsIHR5cGUgPSAicHJvYiIpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICByb2NfYXVjKHN1cnZpdmVkLCAucHJlZF8wKSAKYGBgCgoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX3JhbmdlciAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcsIHR5cGUgPSAicHJvYiIpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBnZ3Bsb3QoKSArCiAgZ2VvbV9kZW5zaXR5KGFlcyh4ID0gLnByZWRfMSwgZmlsbCA9IHN1cnZpdmVkKSwgCiAgICAgICAgICAgICAgIGFscGhhID0gMC41KQpgYGA=