Courses‎ > ‎Python worksheets‎ > ‎

Python demo: using a pre-trained space

# what corpora and pre-trained models does gensim have?
import gensim.downloader as gensim_api

# the whole info: not easy to read
print(gensim_api.info())

# okay, let's break this down: we have corpora, and models
obj = gensim_api.info()
print("\n\nGensim has:", obj.keys())

# what models are there?
print("\n\nGensim models:\n", obj["models"].keys())

# let's find out more about one of them
print("\n\nHere is the info on the word2vec google news 300 model\n", obj["models"]["word2vec-google-news-300"])

# okay, let's download that one
# warning: this takes a while, it's 1.6G
space = gensim_api.load("word2vec-google-news-300")
# if you call the same command again, it doesn't re-download the space,
# it just accesses what you have on disk. It still takes a while.

# a few tests to see what we have
print(space.most_similar("science", topn=3))
print(space.most_similar("football", topn=3))
print(space.most_similar("idea", topn=3))

# let's try an analogy
# the idea of doing analogies this way is from Mikolov et al 2015
king_analog = space["king"] - space["man"] + space["woman"]               
# what are the 10 nearest neighbors?
space.similar_by_vector(king_analog)

# Now we look at gender bias in word embeddings.
# The idea of doing "debiasing" in this way is from Andrew Ng,
# and is similar to an approach in
# Bolukbasi et al (2016): "Man is to Computer Programmer as Woman is to Homemaker?
# Debiasing Word Embeddings" (though Ng's approach is simpler)

# getting a bias vector: vec(woman) - vec(man)
bias = space["woman"] - space["man"]
# what are the words closest to this bias vector?
space.similar_by_vector(bias)

# let's check some particular occupations.
# these are the top-7 "he"-associated and "she"-associated occupations
# from Bolukbasi et al (2016)
occupations = ["homemaker", "nurse", "receptionist", "librarian", "socialite",
               "hairdresser", "nanny", "maestro", "skipper", "protege",
               "philosopher", "captain", "architect", "financier"]

# this function computes the similarity of the vector
# to each of the members of the list of words.
# it returns a list of pairs of (word, similarity)
def vector_similarities_to_words(vec, words):
    # 'distances' gives you 1 - cosine_similarity
    distances = space.distances(vec,  other_words = words)
    similarities = [1 - d for d in distances]
    return list(zip(words, similarities))

# we apply this new function to our occupations
# to see how close each of them is to our "bias direction"
for occupation, sim in vector_similarities_to_words(bias, occupations):
    print(occupation, sim)

###
# Now we use vector rejections to remove bias from vectors.
# For this, we need a bit of math.
import math

# length of a vector: square root of sum of squared entries
def veclen(vec):
    return math.sqrt(sum(dim*dim for dim in vec))

# vector projection:
# proj_p(x) = [ (x * p)/ veclen(p) ] * p
# This returns a vector.
def vecproject(vec, direction):
    factor = sum(v * d for v, d in zip(vec, direction)) / veclen(direction)
    return [ d * factor for d in direction ]

# vector rejection:
# rej_p(x) = x - proj_p(x)
# This returns a vector.
def vecreject(vec, direction):
    vecprojection = vecproject(vec, direction)

    return [ v - p for v, p in zip(vec, vecprojection)]

# debiased vector:
# vector rejection removing the bias (vec(woman) - vec(man)) from the vector
# This returns a vector.
def debias(vec):
    return vecreject(vec, bias)

# try it out:
# for each of our occupations above,
# let's look at the similarities to "man" and "woman"
# for both the original and the debiased vector
for occupation in occupations:
    print("---")
    for word, sim in vector_similarities_to_words(space[occupation], ["man", "woman"]):
        print("original", occupation, word, sim)

    for word, sim in vector_similarities_to_words(debias(space[occupation]), ["man", "woman"]):
        print("debiased", occupation, word, sim)
   
# looks like the debiasing overshoots a little.

Comments