Python中通用的文本相似度计算方法详解

## 最长公共子序列计算最长公共子串-------------------
def LCS(str_a, str_b):
	if len(str_a) == 0 or len(str_b) == 0:
		return 0
	dp = [0 for _ in range(len(str_b) + 1)]
	for i in range(1, len(str_a) + 1):
		left_up = 0
		dp[0] = 0
		for j in range(1, len(str_b) + 1):
			left = dp[j-1]
			up = dp[j]
			if str_a[i-1] == str_b[j-1]:
				dp[j] = left_up + 1
			else:
				dp[j] = max([left, up])
			left_up = up
	return dp[len(str_b)]

#转换最长公共子序列为0-1之间的数值，结果越接近1，相似度越大
def LCS_Score(str_a, str_b):
	return np.round(LCS(str_a, str_b)*2/(len(str_a)+len(str_b)),2)

#LCS_Score(str_a, str_b)    


## 对dataframe的2列按照最长公共子序列计算相似度
## df: 数据来源变量
## col_name1、col_name2：用于计算相似度的2个列名
## simarity_score_name： 返回的相似度结果的列名
## 返回数据框，则simarity_score_name是用该计算方法对2列文本相似度的计算结果
def df_simarity_lcs(df , col_name1 , col_name2 , simarity_score_name):
    df[simarity_score_name] = list(map(lambda str_a, str_b:LCS_Score(str_a, str_b),df[col_name1],df[col_name2]))
    return df

## 采用集合的方法计算2个集合的相似度
def similarity(a, b):
    try:
        return len(a & b) / len(a | b)
    except ZeroDivisionError:
        return -1e-4

## 采用集合的方法计算一个数据框中2个列的文本相似度
## df: 数据来源变量
## col_name1、col_name2：用于计算相似度的2个列名
## simarity_score_name： 返回的相似度结果的列名
## 返回数据框，则simarity_score_name是用该计算方法对2列文本相似度的计算结果
#对dataframe的2列按照集合的方法计算相似度          
def df_simarity_jh(df , col_name1 , col_name2 , simarity_score_name):
    df[simarity_score_name] = list(map(lambda str_a, str_b:similarity(set(str_a), set(str_b)),df[col_name1],df[col_name2]))
    return df 

## vec1, vec2:待计算的向量
## 返回2个向量的相似度
def cosine_simi(vec1, vec2):
    from scipy import spatial
    return 1 - spatial.distance.cosine(vec1, vec2)

## 对dataframe的2列按照最长公共子序列计算相似度
## df: 数据来源变量
## col_name1、col_name2：用于计算相似度的2个列名
## simarity_score_name： 返回的相似度结果的列名
## 返回数据框，则simarity_score_name是用该计算方法对2列文本相似度的计算结果 
def df_simarity_cosine(df , col_name1 , col_name2 , simarity_score_name):
    df[simarity_score_name] = list(map(lambda str_a, str_b:cosine_simi(str_a, str_b),df[col_name1],df[col_name2]))
    return df

from sklearn.feature_extraction.text import TfidfVectorizer

def calculate_tfidf_cosine_similarity(text1, text2):
    vectorizer = TfidfVectorizer()
    corpus = [text1, text2]
    vectors = vectorizer.fit_transform(corpus)
    similarity = cosine_similarity(vectors)
    return similarity[0][1]

text1 = "I love Python programming"
text2 = "Python programming is great"

tfidf_cosine_similarity = calculate_tfidf_cosine_similarity(text1, text2)
print(tfidf_cosine_similarity)

import gensim.downloader as api
from gensim import matutils
import numpy as np

def calculate_word2vec_similarity(text1, text2):
    model = api.load("word2vec-google-news-300")
    tokens1 = text1.split()
    tokens2 = text2.split()
    vec1 = np.mean([model[token] for token in tokens1 if token in model], axis=0)
    vec2 = np.mean([model[token] for token in tokens2 if token in model], axis=0)
    return matutils.cosine(vec1, vec2)

text1 = "I love Python programming"
text2 = "Python programming is great"

word2vec_similarity = calculate_word2vec_similarity(text1, text2)
print(word2vec_similarity)

from gensim.models import Doc2Vec
from gensim.models.doc2vec import TaggedDocument

def calculate_doc2vec_similarity(text1, text2):
    corpus = [TaggedDocument(text1.split(), ["text1"]), TaggedDocument(text2.split(), ["text2"])]
    model = Doc2Vec(corpus, vector_size=100, window=5, min_count=1, workers=4)
    vec1 = model.docvecs["text1"]
    vec2 = model.docvecs["text2"]
    return matutils.cosine(vec1, vec2)

text1 = "I love Python programming"
text2 = "Python programming is great"

doc2vec_similarity = calculate_doc2vec_similarity(text1, text2)
print(doc2vec_similarity)

from sentence_transformers import SentenceTransformer

def calculate_bert_similarity(text1, text2):
    model = SentenceTransformer("bert-base-nli-mean-tokens")
    embeddings = model.encode([text1, text2])
    similarity = cosine_similarity(embeddings)
    return similarity[0][1]

text1 = "I love Python programming"
text2 = "Python programming is great"

bert_similarity = calculate_bert_similarity(text1, text2)
print(bert_similarity)