pdf_title_image/matcher.py

# -*- coding: utf-8 -*-
# @Author: privacy
# @Date:   2024-06-27 09:33:01
# @Last Modified by:   privacy
# @Last Modified time: 2024-08-23 12:10:09
import os
os.environ['TRANSFORMERS_OFFLINE'] = '1'
os.environ['HF_DATASETS_OFFLINE'] = '1'

import torch
import numpy as np
import pandas as pd
from sklearn.metrics.pairwise import cosine_similarity
from transformers import AutoTokenizer, AutoModel


class Matcher:
    def __init__(self):
        # Load model directly
        # # # 加载预训练的text2vec模型和分词器
        # self.tokenizer = AutoTokenizer.from_pretrained("GanymedeNil/text2vec-base-chinese")
        # self.model = AutoModel.from_pretrained("GanymedeNil/text2vec-base-chinese")
        self.tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path='code/model/models--GanymedeNil--text2vec-base-chinese/snapshots/f13ec0b6396814e1352f3d30fe80bb7079625777')
        self.model = AutoModel.from_pretrained(pretrained_model_name_or_path='code/model/models--GanymedeNil--text2vec-base-chinese/snapshots/f13ec0b6396814e1352f3d30fe80bb7079625777')

    
    def TopK1(self, title: str, keywords: list, query_embedding, option_embeddings: list) -> pd.Series:
        # 计算相似度
        similarities = [cosine_similarity([query_embedding], [embedding])[0][0] for embedding in option_embeddings]
    
        # 找到最相近的关键词
        most_similar_keyword = keywords[similarities.index(max(similarities))]
    
        # print(f"和 {title} 最相近的关键词是：{most_similar_keyword}")
    
        return pd.Series([most_similar_keyword, max(similarities)])

    def get_embedding(self, text: str):
        encoded_input = self.tokenizer(text, return_tensors='pt',truncation=True,padding=True,max_length=512)
        with torch.no_grad():
            try:
                output = self.model(**encoded_input)
            except:
                print(encoded_input['input_ids'].size())
                raise ValueError(text)
        text_embedding = np.mean(output.last_hidden_state.mean(dim=1).numpy(), axis=0)
        return text_embedding
    
    def get_embeddings(self, text_list: list) -> list:
        text_embeddings = []
        for text in text_list:
            encoded_input = self.tokenizer(text, return_tensors='pt')
            with torch.no_grad():
                output = self.model(**encoded_input)
            text_embeddings.append(np.mean(output.last_hidden_state.mean(dim=1).numpy(), axis=0))
        return text_embeddings


if __name__ == '__main__':
    matcher = Matcher()

    招标因素 = ['投标人名称', '投标文件封面、投标函签字盖章', '投标文件格式', '报价唯一', '营业执照', '安全生产许可证', '资质条件', '财务要求', '业绩要求', '人员要求', '信誉要求', '不得存在的情形', '其他要求', '投标报价', '投标内容', '工期', '工程质量', '投标有效期', '投标保证金', '权利义务', '己标价工程量清单', '技术标准和要求', '其他', '以往同类项目业绩、经验', '信用评价', '财务状况', '投标报价合理性', '施工组织设计', '无机磨石品牌及质量', '无机磨石地坪的施工工艺及质量控制', '投标关键技术、设备、部件及材料的来源及供应可靠性', '施工安全和文明施工', '组织机构及施工管理人员', '价格得分']

    df = pd.read_json("投标文件-修改版9-5-1-1.json")
    del df['bbox']

    keyword_embeddings = matcher.get_embeddings(招标因素)

    result = df['text'].apply(lambda x: matcher.TopK1(x, 招标因素, matcher.get_embedding(x), keyword_embeddings))

    result.columns = ['因素', '相似度']

    df['因素'] = result['因素']
    df['相似度'] = result['相似度']

    max_sim_idx = df.groupby('因素')['相似度'].idxmax()

    max_sim_rows = df.loc[max_sim_idx]

    max_sim_rows.to_json('相似度.json', orient='records', lines=True, force_ascii=False)