text2vec - GloVe

Read A hands-on intuitive approach to Deep Learning Methods for Text Data — Word2Vec, GloVe and FastText

If you are interested in learning more about applications of Deep Learning to NLP applications. Intel AI Developer Program NLP

library(text2vec)

text8_file = "text8"

if (!file.exists(text8_file)) {
  download.file("http://mattmahoney.net/dc/text8.zip", "text8.zip")
  unzip ("text8.zip")
}

wiki = readLines(text8_file, n = 1, warn = FALSE)

# Create iterator over tokens
tokens = space_tokenizer(wiki)
# Create vocabulary. Terms will be unigrams (simple words).
it = itoken(tokens, progressbar = TRUE)
vocab = create_vocabulary(it)

vocab = prune_vocabulary(vocab, term_count_min = 5L)

# Use our filtered vocabulary
vectorizer = vocab_vectorizer(vocab)
# use window of 5 for context words
tcm = create_tcm(it, vectorizer, skip_grams_window = 5L)

Fit the GloVe neural network encoder. Dimension reductions.

glove = GlobalVectors$new(rank = 50, x_max = 20)
wv_main = glove$fit_transform(tcm, n_iter = 10, convergence_tol = 0.01, n_threads = 8)

dim(wv_main)

wv_context = glove$components
dim(wv_context)

word_vectors = wv_main + t(wv_context)

We can find the closest word vectors for our paris - france + germany example:

berlin = word_vectors["paris", , drop = FALSE] - 
  word_vectors["france", , drop = FALSE] + 
  word_vectors["germany", , drop = FALSE]
cos_sim = sim2(x = word_vectors, y = berlin, method = "cosine", norm = "l2")
head(sort(cos_sim[,1], decreasing = TRUE), 5)