| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677 |
- # -*- coding: utf-8 -*-
- # @Author: privacy
- # @Date: 2024-06-27 09:33:01
- # @Last Modified by: privacy
- # @Last Modified time: 2024-06-27 14:44:43
- import torch
- import numpy as np
- import pandas as pd
- from sklearn.metrics.pairwise import cosine_similarity
- from transformers import AutoTokenizer, AutoModel
- class Matcher:
- def __init__(self):
- # Load model directly
- # # # 加载预训练的text2vec模型和分词器
- base_dir = "/home/stf/.cache/huggingface/hub/models--GanymedeNil--text2vec-base-chinese/snapshots/f13ec0b6396814e1352f3d30fe80bb7079625777"
- self.tokenizer = AutoTokenizer.from_pretrained(base_dir)
- self.model = AutoModel.from_pretrained(base_dir)
- # 计算给定文本（title）与一组关键词（keywords）之间的相似度，并确定其中最相似的关键词
- # title：输入文本标题
- # keywords：关键词
- # query_embedding：表示标题文本的embedding vector
- # option_embedding：表示待比较关键词列表的embedding vector
- def TopK1(self, title: str, keywords: list, query_embedding, option_embeddings: list) -> pd.Series:
- # 计算相似度
- similarities = [cosine_similarity([query_embedding], [embedding])[0][0] for embedding in option_embeddings]
-
- # 找到最相近的关键词
- most_similar_keyword = keywords[similarities.index(max(similarities))]
-
- print(f"和 {title} 最相近的关键词是：{most_similar_keyword}")
- # 返pandas Series对象，包含与title最相近的关键词和其对应的最大相似度值
- return pd.Series([most_similar_keyword, max(similarities)])
- # 调用self.tokenizer和self.model获取text embedding
- def get_embedding(self, text: str):
- encoded_input = self.tokenizer(text, return_tensors='pt')
- with torch.no_grad():
- output = self.model(**encoded_input)
- text_embedding = np.mean(output.last_hidden_state.mean(dim=1).numpy(), axis=0)
- return text_embedding
- # 获取text_list中每个text embedding
- def get_embeddings(self, text_list: list) -> list:
- text_embeddings = []
- for text in text_list:
- encoded_input = self.tokenizer(text, return_tensors='pt')
- with torch.no_grad():
- output = self.model(**encoded_input)
- text_embeddings.append(np.mean(output.last_hidden_state.mean(dim=1).numpy(), axis=0))
- return text_embeddings
- if __name__ == '__main__':
- matcher = Matcher()
- 招标因素 = ['投标人名称', '投标文件封面、投标函签字盖章', '投标文件格式', '报价唯一', '营业执照', '安全生产许可证', '资质条件', '财务要求', '业绩要求', '人员要求', '信誉要求', '不得存在的情形', '其他要求', '投标报价', '投标内容', '工期', '工程质量', '投标有效期', '投标保证金', '权利义务', '己标价工程量清单', '技术标准和要求', '其他', '以往同类项目业绩、经验', '信用评价', '财务状况', '投标报价合理性', '施工组织设计', '无机磨石品牌及质量', '无机磨石地坪的施工工艺及质量控制', '投标关键技术、设备、部件及材料的来源及供应可靠性', '施工安全和文明施工', '组织机构及施工管理人员', '价格得分']
- df = pd.read_json("投标文件-修改版9-5-1-1.json")
- del df['bbox']
- keyword_embeddings = matcher.get_embeddings(招标因素)
- result = df['text'].apply(lambda x: matcher.TopK1(x, 招标因素, matcher.get_embedding(x), keyword_embeddings))
- result.columns = ['因素', '相似度']
- df['因素'] = result['因素']
- df['相似度'] = result['相似度']
- max_sim_idx = df.groupby('因素')['相似度'].idxmax()
- max_sim_rows = df.loc[max_sim_idx]
- max_sim_rows.to_json('相似度.json', orient='records', lines=True, force_ascii=False)
|
