如何解决Word2Vec散点图生成时的'list'无shape属性错误

问题场景 #

在Google Colab中运行Word2Vec算法生成词相似度散点图时，触发以下错误：

AttributeError: 'list' object has no attribute 'shape'

相关代码：

import numpy as np
from IPython.display import display, HTML
import matplotlib.pyplot as plt
from sklearn.manifold import TSNE
import pandas as pd
import re
import nltk

def word_similarity_scatter_plot(index_to_word,weight,plot_title,fig,axes):
    
    labels = []
    tokens = []
    
    for key,value in index_to_word.items():
        tokens.append(weight[key])
        labels.append(value)
        
    #TSNE : Compressing the weights to 2 dimensions to plot the data
    tsne_model = TSNE(perplexity=40, n_components=2, init='pca', n_iter=2500, random_state=23)
    new_values = tsne_model.fit_transform(tokens)
    
    x = []
    y = []
    for value in new_values:
        x.append(value[0])
        y.append(value[1])
    
    
    #plt.figure(figsize=(5, 5)) 
    for i in range(len(x)):
        axes.scatter(x[i],y[i])
        axes.annotate(labels[i],
                     xy=(x[i], y[i]),
                     xytext=(5, 2),
                     textcoords='offset points',
                     ha='right',
                     va='bottom')
    #plt.title(plot_title)
    axes.set_title(plot_title, loc='center')

完整报错栈：

AttributeError                            Traceback (most recent call last)
<ipython-input-46-ce4c95c25962> in <cell line: 447>()
    450     loss_epoch.update( {dim: epoch_loss} )
    451 
--> 452     word_similarity_scatter_plot(
    453         index_to_word,
    454         weights_1[epochs -1],

2 frames
<ipython-input-46-ce4c95c25962> in word_similarity_scatter_plot(index_to_word, weight, plot_title, fig, axes)
    387     #TSNE : Compressing the weights to 2 dimensions to plot the data
    388     tsne_model = TSNE(perplexity=40, n_components=2, init='pca', n_iter=2500, random_state=23)
--> 389     new_values = tsne_model.fit_transform(tokens)
    390 
    391     x = []

/usr/local/lib/python3.10/dist-packages/sklearn/manifold/_t_sne.py in fit_transform(self, X, y)
   1116         """
   1117         self._validate_params()
-> 1118         self._check_params_vs_input(X)
   1119         embedding = self._fit(X)
   1120         self.embedding_ = embedding

/usr/local/lib/python3.10/dist-packages/sklearn/manifold/_t_sne.py in _check_params_vs_input(self, X)
    826 
    827     def _check_params_vs_input(self, X):
--> 828         if self.perplexity >= X.shape[0]:
    829             raise ValueError("perplexity must be less than n_samples")
    830 

AttributeError: 'list' object has no attribute 'shape'

错误原因 #

Scikit-learn的TSNE模块fit_transform()方法要求输入必须是NumPy数组或Pandas DataFrame，这类数据结构自带shape属性用于维度检查。但当前代码中tokens是Python原生列表，没有shape属性，导致TSNE在验证输入参数时抛出错误。

解决方案 #

在调用fit_transform()前，将tokens列表转换为NumPy数组即可解决问题。

修改后的代码 #

import numpy as np
from IPython.display import display, HTML
import matplotlib.pyplot as plt
from sklearn.manifold import TSNE
import pandas as pd
import re
import nltk

def word_similarity_scatter_plot(index_to_word,weight,plot_title,fig,axes):
    
    labels = []
    tokens = []
    
    for key,value in index_to_word.items():
        tokens.append(weight[key])
        labels.append(value)
        
    # 将tokens列表转换为NumPy数组
    tokens = np.array(tokens)
    
    #TSNE : Compressing the weights to 2 dimensions to plot the data
    tsne_model = TSNE(perplexity=40, n_components=2, init='pca', n_iter=2500, random_state=23)
    new_values = tsne_model.fit_transform(tokens)
    
    x = []
    y = []
    for value in new_values:
        x.append(value[0])
        y.append(value[1])
    
    
    #plt.figure(figsize=(5, 5)) 
    for i in range(len(x)):
        axes.scatter(x[i],y[i])
        axes.annotate(labels[i],
                     xy=(x[i], y[i]),
                     xytext=(5, 2),
                     textcoords='offset points',
                     ha='right',
                     va='bottom')
    #plt.title(plot_title)
    axes.set_title(plot_title, loc='center')

额外优化建议 #

可以简化后续提取x、y坐标的代码，直接利用NumPy数组的索引特性：

# 替换原x、y循环提取代码
x = new_values[:, 0]
y = new_values[:, 1]

内容的提问来源于stack exchange，提问作者Porkodi Ishwarya