Browse Source

temp work on word embeddigns

master
jrtechs 4 years ago
parent
commit
ebdfc2290c
1 changed files with 284 additions and 0 deletions
  1. +284
    -0
      notebooks/word-embeddings/Untitled.ipynb

+ 284
- 0
notebooks/word-embeddings/Untitled.ipynb View File

@ -0,0 +1,284 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import gensim\n"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"model = gensim.models.KeyedVectors.load_word2vec_format('./GoogleNews-vectors-negative300.bin', binary=True) "
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[('hi', 0.654898464679718),\n",
" ('goodbye', 0.639905571937561),\n",
" ('howdy', 0.6310957074165344),\n",
" ('goodnight', 0.5920578241348267),\n",
" ('greeting', 0.5855878591537476),\n",
" ('Hello', 0.5842196941375732),\n",
" (\"g'day\", 0.5754077434539795),\n",
" ('See_ya', 0.5688871145248413),\n",
" ('ya_doin', 0.5643119812011719),\n",
" ('greet', 0.5636603832244873)]"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.most_similar(\"hello\")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[('coders', 0.6104331612586975),\n",
" ('coder', 0.6063331365585327),\n",
" ('Coding', 0.5804804563522339),\n",
" ('formatting', 0.5671651363372803),\n",
" ('soluble_receptors', 0.5576372146606445),\n",
" ('ICD9', 0.5571348667144775),\n",
" ('refactoring', 0.5495434999465942),\n",
" ('database_schemas', 0.5372464656829834),\n",
" ('recode', 0.534299373626709),\n",
" ('XHTML_CSS', 0.5328801870346069)]"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.most_similar(\"coding\")"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[('cats', 0.8099379539489746),\n",
" ('dog', 0.7609456777572632),\n",
" ('kitten', 0.7464985251426697),\n",
" ('feline', 0.7326233983039856),\n",
" ('beagle', 0.7150583267211914),\n",
" ('puppy', 0.7075453996658325),\n",
" ('pup', 0.6934291124343872),\n",
" ('pet', 0.6891531348228455),\n",
" ('felines', 0.6755931377410889),\n",
" ('chihuahua', 0.6709762215614319)]"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.most_similar(\"cat\")"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"hi globe \n"
]
}
],
"source": [
"def transformSentence(sentence):\n",
" outputSentence = \"\"\n",
" \n",
" for word in sentence.split(\" \"):\n",
" try:\n",
" outputSentence += model.most_similar(word)[0][0] + \" \"\n",
" except Exception:\n",
" outputSentence += word + \" \"\n",
" return outputSentence\n",
"\n",
"print(transformSentence(\"hello world\"))"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"looks Mom No hand \n"
]
}
],
"source": [
"print(transformSentence(\"look mom no hands\"))"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"This gen_eral concept of Clustering was to groups Data wtih similiar trait \n"
]
}
],
"source": [
"print(transformSentence(\"The general idea of clustering is to group data with similar traits\"))"
]
},
{
"cell_type": "code",
"execution_count": 52,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"This manager concept of clusters was to groups datasets wtih similiar traits. \n"
]
}
],
"source": [
"def removeFromString(string, chars):\n",
" for c in chars:\n",
" string = string.replace(c, \"\")\n",
" return string\n",
"\n",
"\n",
"def transformSentenceWithHeuristic(sentence):\n",
" outputSentence = \"\"\n",
" \n",
" for word in sentence.split(\" \"):\n",
" try:\n",
" changed = False\n",
" for w, _ in model.most_similar(word):\n",
" clean = removeFromString(w, [' ', '_']).lower()\n",
" if clean not in word.lower() and \"_\" not in w:\n",
" outputSentence += w + \" \"\n",
" changed = True\n",
" break\n",
" outputSentence = outputSentence if changed else outputSentence + word + \" \"\n",
" except Exception:\n",
" outputSentence += word + \" \"\n",
" return outputSentence\n",
"print(transformSentenceWithHeuristic(\"The general idea of clustering is to group data with similar traits.\"))"
]
},
{
"cell_type": "code",
"execution_count": 53,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Relax up and grabbing a drinks but that was day it I talking abut this hallucinogenic trips it was this 1981 film Fever Treatment. \n"
]
}
],
"source": [
"print(transformSentenceWithHeuristic(\"Sit down and grab a drink because it is time that we talk about the LSD trip that is the 1981 movie Shock Treatment.\"))"
]
},
{
"cell_type": "code",
"execution_count": 54,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.decomposition import IncrementalPCA # inital reduction\n",
"from sklearn.manifold import TSNE # final reduction\n",
"import numpy as np # array handling\n",
"\n",
"\n",
"def reduce_dimensions(model):\n",
" num_dimensions = 2 # final num dimensions (2D, 3D, etc)\n",
"\n",
" vectors = [] # positions in vector space\n",
" labels = [] # keep track of words to label our data again later\n",
" for word in model.wv.vocab:\n",
" vectors.append(model.wv[word])\n",
" labels.append(word)\n",
"\n",
" # convert both lists into numpy vectors for reduction\n",
" vectors = np.asarray(vectors)\n",
" labels = np.asarray(labels)\n",
"\n",
" # reduce using t-SNE\n",
" vectors = np.asarray(vectors)\n",
" tsne = TSNE(n_components=num_dimensions, random_state=0)\n",
" vectors = tsne.fit_transform(vectors)\n",
"\n",
" x_vals = [v[0] for v in vectors]\n",
" y_vals = [v[1] for v in vectors]\n",
" return x_vals, y_vals, labels\n",
"\n",
"\n",
"#x_vals, y_vals, labels = reduce_dimensions(model)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.1"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

Loading…
Cancel
Save