document_py更新

document_.py

@@ -77,8 +77,8 @@ class DocumentPreReview():
     def get_Bidding_table(self):
         ''' get table data
         '''
-        # file_path = "data/预审查数据/all_tables.json"
-        file_path = "data/预审查数据/2023年档案管理系统功能优化项目采购程序文件-table.json"
+        file_path = "data/预审查数据/三峡左岸及电源电站中央空调系统管网及末端改造（发布稿）-table.json"
+        # file_path = "data/预审查数据/2023年档案管理系统功能优化项目采购程序文件-table.json"
         all_tables = self.bm.json_read(file_path)
         return all_tables
     
@@ -95,7 +95,7 @@ class DocumentPreReview():
         tender_context = self.bm.json_read(file_path)
         return tender_context
     
-    def _scrutinize_judge(self, tag:str):
+    def _scrutinize_judge(self, tag:str, threshold_value:int=3):
         ''' Clause number content judgment 
             商务 技术 报价 评审 评分 标准
         '''
@@ -103,43 +103,95 @@ class DocumentPreReview():
         hit_num = 0
         for scru in scrutinize_tuple:
             if scru in tag: hit_num+= 1
-        if hit_num>=3: return True
+        if hit_num>=threshold_value: return True
         else: return False
 
-    def check_table(self):
+    def check_table(self, all_tables):
         ''' check the form to assess quailty'''
-        all_tables = self.Bidding_tables
+        # all_tables = self.Bidding_tables
 
         tables_list = []
+
+        previous_page_number = 0
+        criteria_sign = False
+
         for partial_form in all_tables:
+            record_num = 1
             table_name = partial_form['table_name']
             page_number = partial_form['page_numbers']
+            title_len = partial_form['title_len']
+            col_len = partial_form['col_len']
             tables = partial_form["table"]
 
-            for table in tables:
-                if '须知' in table and '前附表' in table:
-                    pass
-
-            form_ = {'table_name':table_name, 'page_numbers':page_number, 'table':[]}
-
-
-
-            for table_index, table in enumerate(tables):
-                if '前附表' not in table_name:
-                    form_['table'] = tables
-                    tables_list.append(form_)
-                    break
-
-                regulation_number_index = table.index("条款号")
-                evaluation_factor_index = table.index("评审因素")
-                evaluation_criteria_index = table.index("评审标准")
+            form_ = {'table_name':table_name, 'page_numbers':page_number, 'table':[],
+                     'col_len':col_len, 'title_len':title_len}
+
+            # if '须知' in table_name and '前附表' in table_name:
+            #     regulation_number_index,clause_name_index,clause_content_index = '','',''
+            #     for table_index, table in enumerate(tables):
+            #         if '条款号' in table and '编列内容' in table:
+            #             regulation_number_index = table.index("条款号")
+            #             clause_name_index = table.index("条款名称")
+            #             clause_content_index = table.index("编列内容")
+            #             form_['table'].append(table)
+            #             continue
+            #         elif not table[clause_name_index]:
+            #             form_['table'][table_index-1][clause_content_index] += table[clause_content_index]
+            #         else: form_['table'].append(table)
+            #     tables_list.append(form_)
+
+            if '办法' in table_name and '前附表' in table_name:
+                previous_page_number = page_number[0]
+                regulation_number_index,evaluation_factor_index,evaluation_criteria_index = 0,0,0
+                regulation_number_index_,score_factor_index,score_criteria_index = 0,0,0
+                for table_index, table in enumerate(tables):
+                    if '评审因素' in table and '评审标准' in table:
+                        regulation_number_index = table.index("条款号")
+                        evaluation_factor_index = table.index("评审因素")
+                        evaluation_criteria_index = table.index("评审标准")
+                        form_['table'].append(table)
+                        continue
+                    elif not table[evaluation_factor_index] and table[evaluation_criteria_index]:
+                        form_['table'][table_index-1][evaluation_criteria_index] += table[evaluation_factor_index]
+                    else: 
+                        if table not in form_['table']: form_['table'].append(table)
 
-                
-                if '评分因素' in table and '评分标准' in table:
-                    regulation_number_index = table.index("条款号")
-                    evaluation_factor_index = table.index("评分因素")
-                    evaluation_criteria_index = table.index("评分标准")
-                    weights_index = table.index("权重")
+                    if '评分因素' in table and '评分标准' in table:
+                        regulation_number_index_ = table.index("条款号")
+                        score_factor_index = table.index("评分因素")
+                        score_criteria_index = table.index("评分标准")
+                        weights_index = table.index("权重")
+                        form_['table'].append(table)
+                        criteria_sign = True
+                        continue
+                    elif criteria_sign and self._scrutinize_judge(table[regulation_number_index_+1],2) and not table[score_factor_index]: 
+                        form_['table'][table_index-record_num][score_factor_index-1] += table[score_factor_index-1]
+                        form_['table'][table_index-record_num][score_criteria_index] += table[score_criteria_index]
+                        form_['table'][table_index-record_num][weights_index] += table[weights_index]
+                        record_num += 1
+                    else: 
+                        if table not in form_['table']: form_['table'].append(table)
+                tables_list.append(form_)
+            elif previous_page_number and page_number[-1]<previous_page_number+3:
+                for table_index, table in enumerate(tables):
+                    if '评分因素' in table and '评分标准' in table:
+                        regulation_number_index_ = table.index("条款号")
+                        score_factor_index = table.index("评分因素")
+                        score_criteria_index = table.index("评分标准")
+                        weights_index = table.index("权重")
+                        form_['table'].append(table)
+                        criteria_sign = True
+                        continue
+                    elif criteria_sign and self._scrutinize_judge(table[regulation_number_index_+1],2) and not table[score_factor_index]: 
+                        form_['table'][table_index-record_num][score_factor_index-1] += table[score_factor_index-1]
+                        form_['table'][table_index-record_num][score_criteria_index] += table[score_criteria_index]
+                        form_['table'][table_index-record_num][weights_index] += table[weights_index]
+                        record_num += 1
+                    else: form_['table'].append(table)
+                tables_list.append(form_)
+            else:
+                tables_list.append(partial_form)
+        return tables_list
 
 
 
@@ -147,7 +199,8 @@ class DocumentPreReview():
     def get_table(self):
         ''' parse the Bidding_tables.json file to get the table data from it.
         '''
-        all_tables = self.Bidding_tables
+        all_tables = self.check_table(self.Bidding_tables)
+        # all_tables = self.Bidding_tables
 
         # 招标文件内容中预审查
         tag_sign = ''
@@ -161,6 +214,7 @@ class DocumentPreReview():
         scrutinize_page = 0
         scrutinize_index = -1
         scrutinize_Initial_title_len = 0  # 详审位置标记
+        scrutinize_sign = False
 
         record_page = 0
         bidder_know = {}   # 投标人须知前附表
@@ -183,7 +237,7 @@ class DocumentPreReview():
             
             form_sign = re.findall('评\w+法前附表',table_name)
             if form_sign:
-                table_page_num = page_number[0]
+                table_page_num = page_number[-1]
                 inital_data = tables[0]
                 # confirm data location
                 regulation_number_index = inital_data.index("条款号")
@@ -206,72 +260,91 @@ class DocumentPreReview():
 
             ''' scrutinize '''
             if (scrutinize_page == page_number[0] and scrutinize_Initial_title_len) or scrutinize_page == page_number[0]:
-                regulation_number_index,evaluation_factor_index,evaluation_criteria_index,weights_index = 0,0,0,0
+                regulation_number_index_,evaluation_factor_index,evaluation_criteria_index,weights_index = 0,0,0,0
+                scrutinize_sign = True
                 if not scrutinize_Initial_title_len: scrutinize_Initial_title_len = title_len
                 for table in tables:
                     if '评分因素' in table and '评分标准' in table:
-                        regulation_number_index = table.index("条款号")
+                        regulation_number_index_ = table.index("条款号")
                         evaluation_factor_index = table.index("评分因素")
                         evaluation_criteria_index = table.index("评分标准")
                         weights_index = table.index("权重")
-                        tag_sign = ''
+                        tag_sign_ = ''
                         scrutinize_index = tables.index(table)
                         break
+                    elif '评分因素' in table and '评分标准' not in table:
+                        scrutinize_index = tables.index(table)
+                        table_split = table[-1].replace(' ','').split()
+                        if '评分标准' in table_split and '权重' in table_split:
+                            table = table[:-1]
+                            table.extend(table_split)
+                        regulation_number_index_ = table.index("条款号")
+                        evaluation_factor_index = table.index("评分因素")
+                        evaluation_criteria_index = table.index("评分标准")
+                        weights_index = table.index("权重")
+                        tag_sign_ = ''
+                        break
                 if scrutinize_index != -1:
                     for table in tables[scrutinize_index+1:]:
-                        if table[regulation_number_index+1]: tag = table[regulation_number_index+1]
-                        elif self._scrutinize_judge(table[regulation_number_index+2]): tag = table[regulation_number_index+2]
-                        else: tag = table[regulation_number_index]
+                        if table[regulation_number_index_+1]: tag = table[regulation_number_index_+1]
+                        elif self._scrutinize_judge(table[regulation_number_index_+2]): tag = table[regulation_number_index_+2]
+                        else: tag = table[regulation_number_index_]
                         if tag: 
                             tag = tag.strip().replace("\n","")
                             tag = ''.join(re.findall(r"[\u4e00-\u9fa5]+", tag))
                         if tag and self._scrutinize_judge(tag):
-                            tag_sign = tag
-                            if tag_sign not in scrutinize_dict: scrutinize_dict[tag_sign] = []
-                        evaluation_factor,evaluation_criteria,weights = table[evaluation_factor_index],table[evaluation_criteria_index],table[weights_index]
+                            tag_sign_ = tag
+                            if tag_sign_ not in scrutinize_dict: scrutinize_dict[tag_sign_] = []
+                        try:
+                            evaluation_factor,evaluation_criteria,weights = table[evaluation_factor_index],table[evaluation_criteria_index],table[weights_index]
+                        except:
+                            print()
                         if not weights: value = {"评分因素":evaluation_factor.strip().replace("\n",""),"评分标准":evaluation_criteria.strip().replace("\n","")}
                         else: value = {"评分因素":evaluation_factor.strip().replace("\n",""),
                                         "评分标准":evaluation_criteria.strip().replace("\n",""),
                                         "权重":weights.strip().replace("\n","")}
-                        scrutinize_dict[tag_sign].append(value)
-
-                        if '报价' in tag_sign and '标准' in tag_sign:
-                            scrutinize_dict = {key: value for key, value in scrutinize_dict.items() if value}
-                            scrutinize_Initial_title_len = 0
-                            break
-            elif scrutinize_page+1 == page_number[0] and '报价' not in tag_sign and tag_sign:
+                        scrutinize_dict[tag_sign_].append(value)
+                        if table[regulation_number_index_]:
+                            if table[regulation_number_index_][0] == '3':
+                                scrutinize_dict = {key: value for key, value in scrutinize_dict.items() if value}
+                                scrutinize_Initial_title_len = 0
+                                break
+            elif scrutinize_page+1 == page_number[0] and scrutinize_sign:
                 difference_value = scrutinize_Initial_title_len - title_len
-                if scrutinize_Initial_title_len:
+                if difference_value:
+                    table_length = len(table)
                     evaluation_factor_index -= difference_value
                     evaluation_criteria_index -= difference_value
                     weights_index -= difference_value
+                    if weights_index >= table_length:
+                        evaluation_factor_index = table_length-3
+                        evaluation_criteria_index = table_length-2
+                        weights_index = table_length-1
                 for table in tables:
                     if not table[2]:
-                        scrutinize_dict[tag_sign][-1]['评分标准'] += table[3]
+                        scrutinize_dict[tag_sign_][-1]['评分标准'] += table[3]
                         continue
-                    if table[regulation_number_index+1]: tag = table[regulation_number_index+1]
-                    elif self._scrutinize_judge(table[regulation_number_index+2]): tag = table[regulation_number_index+2]
-                    else: tag = table[regulation_number_index]
+                    if table[regulation_number_index_+1]: tag = table[regulation_number_index_+1]
+                    elif self._scrutinize_judge(table[regulation_number_index_+2]): tag = table[regulation_number_index_+2]
+                    else: tag = table[regulation_number_index_]
                     if tag: 
                         tag = tag.strip().replace("\n","")
                         tag = re.findall("[\u4e00-\u9fff]+", tag)[0]
                     if tag and self._scrutinize_judge(tag):
-                        tag_sign = tag
-                        if tag_sign not in scrutinize_dict: scrutinize_dict[tag_sign] = []
+                        tag_sign_ = tag
+                        if tag_sign_ not in scrutinize_dict: scrutinize_dict[tag_sign_] = []
                     evaluation_factor,evaluation_criteria,weights = table[evaluation_factor_index],table[evaluation_criteria_index],table[weights_index]
                     if not weights: value = {"评分因素":evaluation_factor.strip().replace("\n",""), "评分标准":evaluation_criteria.strip().replace("\n","")}
                     else: value = {"评分因素":evaluation_factor.strip().replace("\n",""),
                                     "评分标准":evaluation_criteria.strip().replace("\n",""),
                                     "权重":weights.strip().replace("\n","")}
-                    try:
-                        scrutinize_dict[tag_sign].append(value)
-                    except:
-                        print()
-                    if '报价' in tag_sign and '标准' in tag_sign:
-                        scrutinize_dict = {key: value for key, value in scrutinize_dict.items() if value}
-                        scrutinize_Initial_title_len = 0
-                        break
-            elif scrutinize_page+2 == page_number[0] and '报价' not in tag_sign and tag_sign:
+                    scrutinize_dict[tag_sign_].append(value)
+                    if table[regulation_number_index_]:
+                            if table[regulation_number_index_][0] == '3':
+                                scrutinize_dict = {key: value for key, value in scrutinize_dict.items() if value}
+                                scrutinize_Initial_title_len = 0
+                                break
+            elif scrutinize_page+2 == page_number[0] and scrutinize_sign:
                 difference_value = scrutinize_Initial_title_len - title_len
                 if scrutinize_Initial_title_len:
                     evaluation_factor_index -= difference_value
@@ -279,27 +352,28 @@ class DocumentPreReview():
                     weights_index -= difference_value
                 for table in tables:
                     if not table[2]:
-                        scrutinize_dict[tag_sign][-1]['评分标准'] += table[3]
+                        scrutinize_dict[tag_sign_][-1]['评分标准'] += table[3]
                         continue
-                    if table[regulation_number_index+1]: tag = table[regulation_number_index+1]
-                    elif self._scrutinize_judge(table[regulation_number_index+2]): tag = table[regulation_number_index+2]
-                    else: tag = table[regulation_number_index]
+                    if table[regulation_number_index_+1]: tag = table[regulation_number_index_+1]
+                    elif self._scrutinize_judge(table[regulation_number_index_+2]): tag = table[regulation_number_index_+2]
+                    else: tag = table[regulation_number_index_]
                     if tag: 
                         tag = tag.strip().replace("\n","")
                         tag = re.findall("[\u4e00-\u9fff]+", tag)[0]
                     if tag and self._scrutinize_judge(tag):
-                        tag_sign = tag
-                        if tag_sign not in scrutinize_dict: scrutinize_dict[tag_sign] = []
+                        tag_sign_ = tag
+                        if tag_sign_ not in scrutinize_dict: scrutinize_dict[tag_sign_] = []
                     evaluation_factor,evaluation_criteria,weights = table[evaluation_factor_index],table[evaluation_criteria_index],table[weights_index]
                     if not weights: value = {"评分因素":evaluation_factor.strip().replace("\n",""), "评分标准":evaluation_criteria.strip().replace("\n","")}
                     else: value = {"评分因素":evaluation_factor.strip().replace("\n",""),
                                 "评分标准":evaluation_criteria.strip().replace("\n",""),
                                 "权重":weights.strip().replace("\n","")}
-                    scrutinize_dict[tag_sign].append(value)
-                    if '报价' in tag_sign and '标准' in tag_sign:
-                        scrutinize_dict = {key: value for key, value in scrutinize_dict.items() if value}
-                        scrutinize_Initial_title_len = 0
-                        break
+                    scrutinize_dict[tag_sign_].append(value)
+                    if table[regulation_number_index_]:
+                            if table[regulation_number_index_][0] == '3':
+                                scrutinize_dict = {key: value for key, value in scrutinize_dict.items() if value}
+                                scrutinize_Initial_title_len = 0
+                                break
 
         # pprint(tag_dict)
         pprint(scrutinize_dict)
@@ -553,8 +627,9 @@ def get_pre_review():
 
 if __name__ == '__main__':
     dpr = DocumentPreReview()
-
+    # dpr.check_table(dpr.Bidding_tables)
     dpr.get_table()
+
     # dpr.content_parsing()
 
     # formal_review_criteria = [