from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
 
tokenizer=Tokenizer()
tokenizer.fit_on_texts(documents_df.documents_cleaned)
tokenized_documents=tokenizer.texts_to_sequences(documents_df.documents_cleaned)
tokenized_paded_documents=pad_sequences(tokenized_documents,maxlen=64,padding='post')
vocab_size=len(tokenizer.word_index)+1
 
# reading Glove word embeddings into a dictionary with "word" as key and values as word vectors
embeddings_index = dict()
 
with open("D:\Clustering\glove.6B.100d.txt", 'r', encoding="utf8") as file:
    for line in file:
        values = line.split()
        word = values[0]
        coefs = np.asarray(values[1:], dtype='float32')
        embeddings_index[word] = coefs
     
# creating embedding matrix, every row is a vector representation from the vocabulary indexed by the tokenizer index. 
embedding_matrix=np.zeros((vocab_size,100))
 
for word,i in tokenizer.word_index.items():
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
        embedding_matrix[i] = embedding_vector
         
# calculating average of word vectors of a document weighted by tf-idf
document_embeddings=np.zeros((len(tokenized_paded_documents),100))
words=tfidfvectoriser.get_feature_names()
 
# instead of creating document-word embeddings, directly creating document embeddings
for i in range(documents_df.shape[0]):
    for j in range(len(words)):
        document_embeddings[i]+=embedding_matrix[tokenizer.word_index[words[j]]]*tfidf_vectors[i][j]
         
 
pairwise_similarities=cosine_similarity(document_embeddings)
pairwise_differences=euclidean_distances(document_embeddings)

Error:
Traceback (most recent call last):
  File "D:/Clustering/text-cluster-master/Cos_Sim_Eucliden_distance.py", line 156, in <module>
    document_embeddings[i] += embedding_matrix[tokenizer.word_index[words[j]]] * tfidf_vectors[i][j]
  File "D:\Python3.8.0\Python\lib\site-packages\scipy\sparse\base.py", line 550, in __rmul__
    return (self.transpose() * tr).transpose()
  File "D:\Python3.8.0\Python\lib\site-packages\scipy\sparse\base.py", line 498, in __mul__
    raise ValueError('dimension mismatch')
ValueError: dimension mismatch