library(pacman)
p_load(titanic, tidyverse, janitor, naniar, DataExplorer, tidymodels)

Load the data from the titanic R package. Note that the titanic_train dataset contains the labels for Survived aND THE titanic_test dataset does not contain the labels. So we will build our machine learning model using the titanic_train dataset and then make a final classification for the titanic_test dataset. This is how kaggle competitions are done.

I like to clean names so the variables all have names with lowercase letters and underscores.

titanic_train <- titanic_train %>% clean_names()

head(titanic_train)

It is always a good idea to check for duplicate records/examples/rows in your dataset.

get_dupes(titanic_train)
No variable names specified - using all columns.

No duplicate combinations found of: passenger_id, survived, pclass, name, sex, age, sib_sp, parch, ticket, ... and 3 other variables

Drop the unique identifiers: passenger_id, name, and ticket. Also drop cabin because it has a high missing rate.

titanic_test <- titanic_test %>% clean_names()

head(titanic_test)

It is always a good idea to check for duplicate records/examples/rows in your dataset.

get_dupes(titanic_test)
No variable names specified - using all columns.

No duplicate combinations found of: passenger_id, pclass, name, sex, age, sib_sp, parch, ticket, fare, ... and 2 other variables
titanic_test2 <- titanic_test %>% select(-passenger_id, -name, -ticket, -cabin) %>%
  mutate(
    pclass = as_factor(pclass),
    sex = as_factor(sex),
    embarked = as_factor(embarked)
  )

head(titanic_test2)
NA

Start by investigating the missing values and completeness of the features in the data. Note that the age variable contains some missing values.

vis_miss(titanic_train2)

gg_miss_var(titanic_train2)

gg_miss_var(titanic_train2, show_pct = TRUE)

create_report(titanic_train2, y = "survived", output_file = "report.html", output_dir = getwd())

Now try the ML algorithms.

Model 0:

Summarize the y-variable. Null Model.

titanic_train2 %>% group_by(survived) %>%
  summarize(n = n()) %>%
  mutate(freq = n / sum(n))

Make the first split with 80% of the data being in the training data set.

titanic_train2_split <- initial_split(titanic_train2, prop = 0.8)
titanic_train2_split
<Analysis/Assess/Total>
<713/178/891>

Training data.

titanic_train2_split %>%
  training() 

Create the recipe for applying the preprocessing. Note the use of step_nzv(), which removes any columns that have very low variability, and the use of the step_meanimpute() function, which fills in the cells that are missing with the mean of the column.

titanic_train2_recipe <- training(titanic_train2_split) %>%
  recipe(survived ~ .) %>%
  step_rm(pclass, sex, embarked) %>% 
  step_nzv(all_predictors()) %>%
  step_meanimpute(age) %>%
  prep()

summary(titanic_train2_recipe)

tidy(titanic_train2_recipe)

Apply the receipe, so the age variable should be complete after the imputation.

titanic_train2_testing <- titanic_train2_recipe %>%
  bake(testing(titanic_train2_split)) 

titanic_train2_testing
titanic_train2_training <- juice(titanic_train2_recipe)

titanic_train2_training

Model 0: null


titanic_train2_null <- null_model() %>%
  set_mode("classification") %>%
  fit(survived ~ ., data = titanic_train2_training)
Engine set to `parsnip`.
predict(titanic_train2_null, titanic_train2_training)
titanic_train2_null %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) 
titanic_train2_null %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) %>%
  metrics(truth = survived, estimate = .pred_class)
titanic_train2_null %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) %>%
  conf_mat(truth = survived, estimate = .pred_class)
          Truth
Prediction   0   1
         0 109  69
         1   0   0

Model 2: C5.0

Setup the model.

titanic_train2_C50 <- boost_tree(trees = 20) %>% 
  set_engine("C5.0") %>%
  set_mode("classification") %>%
  fit(survived ~ ., data = titanic_train2_training)
predict(titanic_train2_C50, titanic_train2_training)
titanic_train2_C50 %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) 
titanic_train2_C50 %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) %>%
  metrics(truth = survived, estimate = .pred_class)
titanic_train2_C50 %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) %>%
  conf_mat(truth = survived, estimate = .pred_class)
          Truth
Prediction  0  1
         0 92 39
         1 17 30
titanic_train2_C50 %>%
  predict(titanic_train2_testing, type = "prob") %>%
  bind_cols(titanic_train2_testing) %>%
  roc_curve(survived, .pred_0) %>%
  autoplot()

titanic_train2_C50 %>%
  predict(titanic_train2_testing, type = "prob") %>%
  bind_cols(titanic_train2_testing) %>%
  roc_auc(survived, .pred_0) 
titanic_train2_C50 %>%
  predict(titanic_train2_testing, type = "prob") %>%
  bind_cols(titanic_train2_testing) %>%
  ggplot() +
  geom_density(aes(x = .pred_1, fill = survived), 
               alpha = 0.5)

Model 2: XGBoost

Setup the model.

titanic_train2_xgb <- boost_tree(trees = 20) %>% 
  set_engine("xgboost") %>%
  set_mode("classification") %>%
  fit(survived ~ ., data = titanic_train2_training)
[09:57:43] WARNING: amalgamation/../src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
predict(titanic_train2_xgb, titanic_train2_training)
titanic_train2_xgb %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) 
titanic_train2_xgb %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) %>%
  metrics(truth = survived, estimate = .pred_class)
titanic_train2_xgb %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) %>%
  conf_mat(truth = survived, estimate = .pred_class)
          Truth
Prediction  0  1
         0 90 38
         1 19 31
titanic_train2_xgb %>%
  predict(titanic_train2_testing, type = "prob") %>%
  bind_cols(titanic_train2_testing) %>%
  roc_curve(survived, .pred_0) %>%
  autoplot() 

titanic_train2_xgb %>%
  predict(titanic_train2_testing, type = "prob") %>%
  bind_cols(titanic_train2_testing) %>%
  roc_auc(survived, .pred_0) 
titanic_train2_xgb %>%
  predict(titanic_train2_testing, type = "prob") %>%
  bind_cols(titanic_train2_testing) %>%
  ggplot() +
  geom_density(aes(x = .pred_1, fill = survived), 
               alpha = 0.5)

Model 3: Random Forest

Setup the model.

titanic_train2_ranger <- rand_forest(trees = 100) %>% 
  set_engine("ranger") %>%
  set_mode("classification") %>%
  fit(survived ~ ., data = titanic_train2_training)
predict(titanic_train2_ranger, titanic_train2_training)
titanic_train2_ranger %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) 
titanic_train2_ranger %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) %>%
  metrics(truth = survived, estimate = .pred_class)
titanic_train2_ranger %>%
  predict(titanic_train2_testing) %>%
  bind_cols(titanic_train2_testing) %>%
  conf_mat(truth = survived, estimate = .pred_class)
          Truth
Prediction  0  1
         0 89 33
         1 20 36
titanic_train2_ranger %>%
  predict(titanic_train2_testing, type = "prob") %>%
  bind_cols(titanic_train2_testing) %>%
  roc_curve(survived, .pred_1) %>%
  ggplot(aes(x = 1 - specificity, y = sensitivity)) +
  geom_path() +
  geom_abline(lty = 3) +
  coord_equal() 


titanic_train2_ranger %>%
  predict(titanic_train2_testing, type = "prob") %>%
  bind_cols(titanic_train2_testing) %>%
  roc_curve(survived, event_level = "first") %>%
  autoplot()
Error: No valid variables provided to `...`.
Run `rlang::last_error()` to see where the error occurred.
titanic_train2_ranger %>%
  predict(titanic_train2_testing, type = "prob") %>%
  bind_cols(titanic_train2_testing) %>%
  roc_auc(survived, .pred_0) 
titanic_train2_ranger %>%
  predict(titanic_train2_testing, type = "prob") %>%
  bind_cols(titanic_train2_testing) %>%
  ggplot() +
  geom_density(aes(x = .pred_1, fill = survived), 
               alpha = 0.5)

LS0tCnRpdGxlOiAiU3RhdCA2NTI6IE1pZHRlcm06IE1vZGVsIDIsIE1vZGVsIDMiCmF1dGhvcjogIlByb2YuIEVyaWMgQS4gU3Vlc3MiCm91dHB1dDoKICBodG1sX25vdGVib29rOiBkZWZhdWx0CiAgd29yZF9kb2N1bWVudDogZGVmYXVsdAogIHBkZl9kb2N1bWVudDogZGVmYXVsdAotLS0KCgoKYGBge3J9CmxpYnJhcnkocGFjbWFuKQpwX2xvYWQodGl0YW5pYywgdGlkeXZlcnNlLCBqYW5pdG9yLCBuYW5pYXIsIERhdGFFeHBsb3JlciwgdGlkeW1vZGVscykKYGBgCgpMb2FkIHRoZSBkYXRhIGZyb20gdGhlIHRpdGFuaWMgUiBwYWNrYWdlLiAgTm90ZSB0aGF0IHRoZSAqdGl0YW5pY190cmFpbiogZGF0YXNldCBjb250YWlucyB0aGUgbGFiZWxzIGZvciBTdXJ2aXZlZCBhTkQgVEhFICp0aXRhbmljX3Rlc3QqIGRhdGFzZXQgZG9lcyBub3QgY29udGFpbiB0aGUgbGFiZWxzLiAgU28gd2Ugd2lsbCBidWlsZCBvdXIgbWFjaGluZSBsZWFybmluZyBtb2RlbCB1c2luZyB0aGUgKnRpdGFuaWNfdHJhaW4qIGRhdGFzZXQgYW5kIHRoZW4gbWFrZSBhIGZpbmFsIGNsYXNzaWZpY2F0aW9uIGZvciB0aGUgKnRpdGFuaWNfdGVzdCogZGF0YXNldC4gIFRoaXMgaXMgaG93IGthZ2dsZSBjb21wZXRpdGlvbnMgYXJlIGRvbmUuCgpJIGxpa2UgdG8gY2xlYW4gbmFtZXMgc28gdGhlIHZhcmlhYmxlcyBhbGwgaGF2ZSBuYW1lcyB3aXRoIGxvd2VyY2FzZSBsZXR0ZXJzIGFuZCB1bmRlcnNjb3Jlcy4KCmBgYHtyfQp0aXRhbmljX3RyYWluIDwtIHRpdGFuaWNfdHJhaW4gJT4lIGNsZWFuX25hbWVzKCkKCmhlYWQodGl0YW5pY190cmFpbikKYGBgCgpJdCBpcyBhbHdheXMgYSBnb29kIGlkZWEgdG8gY2hlY2sgZm9yIGR1cGxpY2F0ZSByZWNvcmRzL2V4YW1wbGVzL3Jvd3MgaW4geW91ciBkYXRhc2V0LgoKYGBge3J9CmdldF9kdXBlcyh0aXRhbmljX3RyYWluKQpgYGAKCkRyb3AgdGhlIHVuaXF1ZSBpZGVudGlmaWVyczogKnBhc3Nlbmdlcl9pZCosICpuYW1lKiwgYW5kICp0aWNrZXQqLiAgQWxzbyBkcm9wICpjYWJpbiogYmVjYXVzZSBpdCBoYXMgYSBoaWdoIG1pc3NpbmcgcmF0ZS4KCmBgYHtyfQp0aXRhbmljX3RyYWluMiA8LSB0aXRhbmljX3RyYWluICU+JSBzZWxlY3QoLXBhc3Nlbmdlcl9pZCwgLW5hbWUsIC10aWNrZXQsIC1jYWJpbikgJT4lCiAgbXV0YXRlKAogICAgc3Vydml2ZWQgPSBhc19mYWN0b3Ioc3Vydml2ZWQpICwKICAgIHBjbGFzcyA9IGFzX2ZhY3RvcihwY2xhc3MpLAogICAgc2V4ID0gYXNfZmFjdG9yKHNleCksCiAgICBlbWJhcmtlZCA9IGFzX2ZhY3RvcihlbWJhcmtlZCkKICApCgpoZWFkKHRpdGFuaWNfdHJhaW4yKQpgYGAKCmBgYHtyfQp0aXRhbmljX3Rlc3QgPC0gdGl0YW5pY190ZXN0ICU+JSBjbGVhbl9uYW1lcygpCgpoZWFkKHRpdGFuaWNfdGVzdCkKYGBgCgpJdCBpcyBhbHdheXMgYSBnb29kIGlkZWEgdG8gY2hlY2sgZm9yIGR1cGxpY2F0ZSByZWNvcmRzL2V4YW1wbGVzL3Jvd3MgaW4geW91ciBkYXRhc2V0LgoKYGBge3J9CmdldF9kdXBlcyh0aXRhbmljX3Rlc3QpCmBgYAoKYGBge3J9CnRpdGFuaWNfdGVzdDIgPC0gdGl0YW5pY190ZXN0ICU+JSBzZWxlY3QoLXBhc3Nlbmdlcl9pZCwgLW5hbWUsIC10aWNrZXQsIC1jYWJpbikgJT4lCiAgbXV0YXRlKAogICAgcGNsYXNzID0gYXNfZmFjdG9yKHBjbGFzcyksCiAgICBzZXggPSBhc19mYWN0b3Ioc2V4KSwKICAgIGVtYmFya2VkID0gYXNfZmFjdG9yKGVtYmFya2VkKQogICkKCmhlYWQodGl0YW5pY190ZXN0MikKCmBgYAoKClN0YXJ0IGJ5IGludmVzdGlnYXRpbmcgdGhlIG1pc3NpbmcgdmFsdWVzIGFuZCBjb21wbGV0ZW5lc3Mgb2YgdGhlIGZlYXR1cmVzIGluIHRoZSBkYXRhLiAgTm90ZSB0aGF0IHRoZSAqYWdlKiB2YXJpYWJsZSBjb250YWlucyBzb21lIG1pc3NpbmcgdmFsdWVzLgoKYGBge3J9CnZpc19taXNzKHRpdGFuaWNfdHJhaW4yKQpnZ19taXNzX3Zhcih0aXRhbmljX3RyYWluMikKZ2dfbWlzc192YXIodGl0YW5pY190cmFpbjIsIHNob3dfcGN0ID0gVFJVRSkKYGBgCgpgYGB7ciBldmFsID0gRkFMU0V9CmNyZWF0ZV9yZXBvcnQodGl0YW5pY190cmFpbjIsIHkgPSAic3Vydml2ZWQiLCBvdXRwdXRfZmlsZSA9ICJyZXBvcnQuaHRtbCIsIG91dHB1dF9kaXIgPSBnZXR3ZCgpKQpgYGAKCk5vdyB0cnkgdGhlIE1MIGFsZ29yaXRobXMuCgoKIyMgTW9kZWwgMDoKClN1bW1hcml6ZSB0aGUgeS12YXJpYWJsZS4gIE51bGwgTW9kZWwuCgpgYGB7cn0KdGl0YW5pY190cmFpbjIgJT4lIGdyb3VwX2J5KHN1cnZpdmVkKSAlPiUKICBzdW1tYXJpemUobiA9IG4oKSkgJT4lCiAgbXV0YXRlKGZyZXEgPSBuIC8gc3VtKG4pKQpgYGAKCk1ha2UgdGhlIGZpcnN0IHNwbGl0IHdpdGggODAlIG9mIHRoZSBkYXRhIGJlaW5nIGluIHRoZSB0cmFpbmluZyBkYXRhIHNldC4KCmBgYHtyfQp0aXRhbmljX3RyYWluMl9zcGxpdCA8LSBpbml0aWFsX3NwbGl0KHRpdGFuaWNfdHJhaW4yLCBwcm9wID0gMC44KQp0aXRhbmljX3RyYWluMl9zcGxpdApgYGAKClRyYWluaW5nIGRhdGEuCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfc3BsaXQgJT4lCiAgdHJhaW5pbmcoKSAKYGBgCgpDcmVhdGUgdGhlIHJlY2lwZSBmb3IgYXBwbHlpbmcgdGhlIHByZXByb2Nlc3NpbmcuICBOb3RlIHRoZSB1c2Ugb2Ygc3RlcF9uenYoKSwgd2hpY2ggcmVtb3ZlcyBhbnkgY29sdW1ucyB0aGF0IGhhdmUgdmVyeSBsb3cgdmFyaWFiaWxpdHksIGFuZCB0aGUgdXNlIG9mIHRoZSBzdGVwX21lYW5pbXB1dGUoKSBmdW5jdGlvbiwgd2hpY2ggZmlsbHMgaW4gdGhlIGNlbGxzIHRoYXQgYXJlIG1pc3Npbmcgd2l0aCB0aGUgbWVhbiBvZiB0aGUgY29sdW1uLgoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX3JlY2lwZSA8LSB0cmFpbmluZyh0aXRhbmljX3RyYWluMl9zcGxpdCkgJT4lCiAgcmVjaXBlKHN1cnZpdmVkIH4gLikgJT4lCiAgc3RlcF9ybShwY2xhc3MsIHNleCwgZW1iYXJrZWQpICU+JSAKICBzdGVwX256dihhbGxfcHJlZGljdG9ycygpKSAlPiUKICBzdGVwX21lYW5pbXB1dGUoYWdlKSAlPiUKICBwcmVwKCkKCnN1bW1hcnkodGl0YW5pY190cmFpbjJfcmVjaXBlKQoKdGlkeSh0aXRhbmljX3RyYWluMl9yZWNpcGUpCmBgYAoKQXBwbHkgdGhlIHJlY2VpcGUsIHNvIHRoZSAqYWdlKiB2YXJpYWJsZSBzaG91bGQgYmUgY29tcGxldGUgYWZ0ZXIgdGhlIGltcHV0YXRpb24uCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfdGVzdGluZyA8LSB0aXRhbmljX3RyYWluMl9yZWNpcGUgJT4lCiAgYmFrZSh0ZXN0aW5nKHRpdGFuaWNfdHJhaW4yX3NwbGl0KSkgCgp0aXRhbmljX3RyYWluMl90ZXN0aW5nCmBgYAoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX3RyYWluaW5nIDwtIGp1aWNlKHRpdGFuaWNfdHJhaW4yX3JlY2lwZSkKCnRpdGFuaWNfdHJhaW4yX3RyYWluaW5nCmBgYAoKIyMjIE1vZGVsIDA6IG51bGwKCgpgYGB7cn0KCnRpdGFuaWNfdHJhaW4yX251bGwgPC0gbnVsbF9tb2RlbCgpICU+JQogIHNldF9tb2RlKCJjbGFzc2lmaWNhdGlvbiIpICU+JQogIGZpdChzdXJ2aXZlZCB+IC4sIGRhdGEgPSB0aXRhbmljX3RyYWluMl90cmFpbmluZykKCmBgYAoKYGBge3J9CnByZWRpY3QodGl0YW5pY190cmFpbjJfbnVsbCwgdGl0YW5pY190cmFpbjJfdHJhaW5pbmcpCmBgYAoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX251bGwgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgCmBgYAoKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9udWxsICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIG1ldHJpY3ModHJ1dGggPSBzdXJ2aXZlZCwgZXN0aW1hdGUgPSAucHJlZF9jbGFzcykKYGBgCgoKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9udWxsICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIGNvbmZfbWF0KHRydXRoID0gc3Vydml2ZWQsIGVzdGltYXRlID0gLnByZWRfY2xhc3MpCmBgYAoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX251bGwgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nLCB0eXBlID0gInByb2IiKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgcm9jX2N1cnZlKHN1cnZpdmVkLCAucHJlZF8xKSAlPiUKICBhdXRvcGxvdCgpCmBgYAoKIyMjIE1vZGVsIDI6IEM1LjAKClNldHVwIHRoZSBtb2RlbC4KCmBgYHtyfQp0aXRhbmljX3RyYWluMl9DNTAgPC0gYm9vc3RfdHJlZSh0cmVlcyA9IDIwKSAlPiUgCiAgc2V0X2VuZ2luZSgiQzUuMCIpICU+JQogIHNldF9tb2RlKCJjbGFzc2lmaWNhdGlvbiIpICU+JQogIGZpdChzdXJ2aXZlZCB+IC4sIGRhdGEgPSB0aXRhbmljX3RyYWluMl90cmFpbmluZykKCmBgYAoKCgpgYGB7cn0KcHJlZGljdCh0aXRhbmljX3RyYWluMl9DNTAsIHRpdGFuaWNfdHJhaW4yX3RyYWluaW5nKQpgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9DNTAgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgCmBgYAoKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9DNTAgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgbWV0cmljcyh0cnV0aCA9IHN1cnZpdmVkLCBlc3RpbWF0ZSA9IC5wcmVkX2NsYXNzKQpgYGAKCgoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX0M1MCAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBjb25mX21hdCh0cnV0aCA9IHN1cnZpdmVkLCBlc3RpbWF0ZSA9IC5wcmVkX2NsYXNzKQpgYGAKCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfQzUwICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZywgdHlwZSA9ICJwcm9iIikgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIHJvY19jdXJ2ZShzdXJ2aXZlZCwgLnByZWRfMCkgJT4lCiAgYXV0b3Bsb3QoKQpgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9DNTAgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nLCB0eXBlID0gInByb2IiKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgcm9jX2F1YyhzdXJ2aXZlZCwgLnByZWRfMCkgCmBgYAoKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9DNTAgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nLCB0eXBlID0gInByb2IiKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgZ2dwbG90KCkgKwogIGdlb21fZGVuc2l0eShhZXMoeCA9IC5wcmVkXzEsIGZpbGwgPSBzdXJ2aXZlZCksIAogICAgICAgICAgICAgICBhbHBoYSA9IDAuNSkKYGBgCgoKCiMjIyBNb2RlbCAyOiBYR0Jvb3N0CgpTZXR1cCB0aGUgbW9kZWwuCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfeGdiIDwtIGJvb3N0X3RyZWUodHJlZXMgPSAyMCkgJT4lIAogIHNldF9lbmdpbmUoInhnYm9vc3QiKSAlPiUKICBzZXRfbW9kZSgiY2xhc3NpZmljYXRpb24iKSAlPiUKICBmaXQoc3Vydml2ZWQgfiAuLCBkYXRhID0gdGl0YW5pY190cmFpbjJfdHJhaW5pbmcpCgpgYGAKCgoKYGBge3J9CnByZWRpY3QodGl0YW5pY190cmFpbjJfeGdiLCB0aXRhbmljX3RyYWluMl90cmFpbmluZykKYGBgCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfeGdiICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpIApgYGAKCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfeGdiICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIG1ldHJpY3ModHJ1dGggPSBzdXJ2aXZlZCwgZXN0aW1hdGUgPSAucHJlZF9jbGFzcykKYGBgCgoKCmBgYHtyfQp0aXRhbmljX3RyYWluMl94Z2IgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgY29uZl9tYXQodHJ1dGggPSBzdXJ2aXZlZCwgZXN0aW1hdGUgPSAucHJlZF9jbGFzcykKYGBgCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfeGdiICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZywgdHlwZSA9ICJwcm9iIikgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIHJvY19jdXJ2ZShzdXJ2aXZlZCwgLnByZWRfMCkgJT4lCiAgYXV0b3Bsb3QoKSAKYGBgCgpgYGB7cn0KdGl0YW5pY190cmFpbjJfeGdiICU+JQogIHByZWRpY3QodGl0YW5pY190cmFpbjJfdGVzdGluZywgdHlwZSA9ICJwcm9iIikgJT4lCiAgYmluZF9jb2xzKHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIHJvY19hdWMoc3Vydml2ZWQsIC5wcmVkXzApIApgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl94Z2IgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nLCB0eXBlID0gInByb2IiKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgZ2dwbG90KCkgKwogIGdlb21fZGVuc2l0eShhZXMoeCA9IC5wcmVkXzEsIGZpbGwgPSBzdXJ2aXZlZCksIAogICAgICAgICAgICAgICBhbHBoYSA9IDAuNSkKYGBgCgojIyMgTW9kZWwgMzogUmFuZG9tIEZvcmVzdAoKU2V0dXAgdGhlIG1vZGVsLgoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX3JhbmdlciA8LSByYW5kX2ZvcmVzdCh0cmVlcyA9IDEwMCkgJT4lIAogIHNldF9lbmdpbmUoInJhbmdlciIpICU+JQogIHNldF9tb2RlKCJjbGFzc2lmaWNhdGlvbiIpICU+JQogIGZpdChzdXJ2aXZlZCB+IC4sIGRhdGEgPSB0aXRhbmljX3RyYWluMl90cmFpbmluZykKCmBgYAoKCgpgYGB7cn0KcHJlZGljdCh0aXRhbmljX3RyYWluMl9yYW5nZXIsIHRpdGFuaWNfdHJhaW4yX3RyYWluaW5nKQpgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9yYW5nZXIgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgCmBgYAoKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9yYW5nZXIgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgbWV0cmljcyh0cnV0aCA9IHN1cnZpdmVkLCBlc3RpbWF0ZSA9IC5wcmVkX2NsYXNzKQpgYGAKCgoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX3JhbmdlciAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBjb25mX21hdCh0cnV0aCA9IHN1cnZpdmVkLCBlc3RpbWF0ZSA9IC5wcmVkX2NsYXNzKQpgYGAKCmBgYHtyfQp0aXRhbmljX3RyYWluMl9yYW5nZXIgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nLCB0eXBlID0gInByb2IiKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgcm9jX2N1cnZlKHN1cnZpdmVkLCAucHJlZF8xKSAlPiUKICBnZ3Bsb3QoYWVzKHggPSAxIC0gc3BlY2lmaWNpdHksIHkgPSBzZW5zaXRpdml0eSkpICsKICBnZW9tX3BhdGgoKSArCiAgZ2VvbV9hYmxpbmUobHR5ID0gMykgKwogIGNvb3JkX2VxdWFsKCkgCgp0aXRhbmljX3RyYWluMl9yYW5nZXIgJT4lCiAgcHJlZGljdCh0aXRhbmljX3RyYWluMl90ZXN0aW5nLCB0eXBlID0gInByb2IiKSAlPiUKICBiaW5kX2NvbHModGl0YW5pY190cmFpbjJfdGVzdGluZykgJT4lCiAgcm9jX2N1cnZlKHN1cnZpdmVkLCAucHJlZF8wKSAlPiUKICBhdXRvcGxvdCgpCgoKCmBgYAoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX3JhbmdlciAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcsIHR5cGUgPSAicHJvYiIpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICByb2NfYXVjKHN1cnZpdmVkLCAucHJlZF8wKSAKYGBgCgoKYGBge3J9CnRpdGFuaWNfdHJhaW4yX3JhbmdlciAlPiUKICBwcmVkaWN0KHRpdGFuaWNfdHJhaW4yX3Rlc3RpbmcsIHR5cGUgPSAicHJvYiIpICU+JQogIGJpbmRfY29scyh0aXRhbmljX3RyYWluMl90ZXN0aW5nKSAlPiUKICBnZ3Bsb3QoKSArCiAgZ2VvbV9kZW5zaXR5KGFlcyh4ID0gLnByZWRfMSwgZmlsbCA9IHN1cnZpdmVkKSwgCiAgICAgICAgICAgICAgIGFscGhhID0gMC41KQpgYGA=