document_.py更新

document_.py

@@ -54,46 +54,20 @@ log_path = "code/logs/logs.log"
 logger = create_logger(log_path=log_path)
 
 class DocumentPreReview():
-    def __init__(self) -> None:
+    def __init__(self, file_path) -> None:
         self.bm = BaseMethods()
-        # self.agent_ = PdfExtractAttr_(file_path=self.file_path)
-        # self.agent = PdfExtractAttr(file_path=self.file_path)
-
-        self.Bidding_tables = self.get_Bidding_table()
-        self.contexts = self.get_Bidding_contexts()
-        self.announcement = self.get_announcement()
-        self.Bidding_context = self.get_Bidding_json()
-        self.tender_context = self.get_tender_context()
-
-        self.chinese_num_map = chinese_num_map
+        self.Bidding_tables = self.get_Bidding_table(file_path)
     
-
-    def get_Bidding_contexts(self, file_path:str = 'data/预审查数据/contexts.json'):
-        ''' get contexts by page
-        '''
-        contexts = self.bm.json_read(file_path)
-        return contexts
     
-    def get_Bidding_table(self):
+    def get_Bidding_table(self, file_path:str):
         ''' get table data
         '''
-        file_path = "data/预审查数据/三峡左岸及电源电站中央空调系统管网及末端改造（发布稿）-table.json"
+        # file_path = "data/预审查数据/三峡左岸及电源电站中央空调系统管网及末端改造（发布稿）-table.json"
         # file_path = "data/预审查数据/2023年档案管理系统功能优化项目采购程序文件-table.json"
         all_tables = self.bm.json_read(file_path)
         return all_tables
     
-    def get_Bidding_json(self):
-        ''' read json to get context
-        '''
-        file_path = "data/预审查数据/三峡左岸及电源电站中央空调系统管网及末端改造（发布稿）-table.json"
-        Bidding_context = self.bm.json_read(file_path)
-        return Bidding_context
     
-    def get_tender_context(self):
-        ''' read the tender context '''
-        file_path = "data/预审查数据/南方电网数字研究院有限公司_bidding_content.json"
-        tender_context = self.bm.json_read(file_path)
-        return tender_context
     
     def _scrutinize_judge(self, tag:str, threshold_value:int=3):
         ''' Clause number content judgment 
@@ -126,20 +100,6 @@ class DocumentPreReview():
             form_ = {'table_name':table_name, 'page_numbers':page_number, 'table':[],
                      'col_len':col_len, 'title_len':title_len}
 
-            # if '须知' in table_name and '前附表' in table_name:
-            #     regulation_number_index,clause_name_index,clause_content_index = '','',''
-            #     for table_index, table in enumerate(tables):
-            #         if '条款号' in table and '编列内容' in table:
-            #             regulation_number_index = table.index("条款号")
-            #             clause_name_index = table.index("条款名称")
-            #             clause_content_index = table.index("编列内容")
-            #             form_['table'].append(table)
-            #             continue
-            #         elif not table[clause_name_index]:
-            #             form_['table'][table_index-1][clause_content_index] += table[clause_content_index]
-            #         else: form_['table'].append(table)
-            #     tables_list.append(form_)
-
             if '办法' in table_name and '前附表' in table_name:
                 previous_page_number = page_number[0]
                 regulation_number_index,evaluation_factor_index,evaluation_criteria_index = 0,0,0
@@ -200,16 +160,12 @@ class DocumentPreReview():
         ''' parse the Bidding_tables.json file to get the table data from it.
         '''
         all_tables = self.check_table(self.Bidding_tables)
-        # all_tables = self.Bidding_tables
 
         # 招标文件内容中预审查
         tag_sign = ''
         tag_list = ("形式评审标准", "资格评审标准", "响应性评审标准")
         tag_dict = dict([(tag,[]) for tag in tag_list])
         
-        # 招标文件内容中清标表格数据
-        # scrutinize_tuple = ("商务部分评分标准","技术部分评审标准","技术部分评分标准","投标报价评审标准","报价部分评审标准","报价评分标准","报价部分评分标准")
-        
         scrutinize_dict = {}
         scrutinize_page = 0
         scrutinize_index = -1
@@ -375,239 +331,8 @@ class DocumentPreReview():
                                 scrutinize_Initial_title_len = 0
                                 break
 
-        # pprint(tag_dict)
         pprint(scrutinize_dict)
-        # pprint(bidder_know)
-        return tag_dict,bidder_know,scrutinize_dict
-    
-    def get_announcement(self)->str:
-        ''' bidder announcement
-        '''
-        announcements = ''
-        announcement_contexts = self.contexts[2:8]
-        for index, announcement in enumerate(announcement_contexts):
-            finder = re.findall("^第一章",announcement['text'])
-            if finder:
-                for text in announcement_contexts[index:]:
-                    if re.findall("^第二章", text["text"]): break
-                    announcements += text["text"]
-                break
-        return announcements
-    
-    def contexts_extract(self, evaluation_criteria:str):
-        ''' 招标文件正文抓取
-        '''
-        comp1 = re.compile("(第.*?章)")
-        comp2 = re.compile("“(.*?)”")
-        title = comp1.findall(evaluation_criteria)[0]+comp2.findall(evaluation_criteria)[0]
-        comp3 = re.compile("第(.*?)章")
-        title_list = []
-
-        format_index,sta_page = -1,-1
-        sign = True
-        title_next = ''
-        for context in self.Bidding_context: # 取招标文件内容
-            text = context['text'].strip().replace(" ","")
-
-            if text == '目录': 
-                sta_page = context['page_number']
-            if sta_page != -1 and context['page_number'] < 4:
-                finder = comp3.findall(context['text'])
-                if finder and sign:
-                    if title_list:
-                        chinese_num = self.chinese_num_map.get(comp3.findall(title_list[-1])[0],None)
-                        if chinese_num > self.chinese_num_map.get(finder[0],0):
-                            sign = False
-                        else:
-                            title_list.append(context['text'].split(' ')[0])
-                    else:
-                        title_list.append(context['text'].split(' ')[0])
-                
-            if text == title and format_index == -1:
-                format_index = self.Bidding_context.index(context)
-                break
-
-        title_index = title_list.index(title)
-        if title_index != len(title_list)-1:
-            title_next = title_list[title_index+1]
-
-        file_format = {title:{}}
-        for context in self.Bidding_context[format_index+1:]:
-            text = context['text'].strip().replace(" ","").replace("\n","——>")
-            if title_next and title_next == text:
-                break
-            if context['page_number'] not in file_format[title]:
-                file_format[title][context['page_number']] = []
-            file_format[title][context['page_number']].append(context['text'])
-
-        return file_format
-
-
-    def formal_criteria(self, review_criteria_list:list):
-        ''' Analysis of formal review criteria
-        形式评审标准
-        [{'评审因素': '投标人名称', '评审标准': '与营业执照书一致'},
-            {'评审因素': '投标文件封面、投标函签字盖章',
-             '评审标准': '投标文件封面、投标函须有法定代表人（或其委托代理人）签字（或签章）并加盖单位章，由委托代理人签字的须具有有效的授权委托书'},
-            {'评审因素': '投标文件格式', '评审标准': '符合第八章“投标文件格式”的要求'},
-            {'评审因素': '联合体投标人（如有）', '评审标准': '不适用'},
-            {'评审因素': '报价唯一', '评审标准': '只能有一个有效报价'}]
-        '''
-        formal_result = {}
-        for review_criteria in review_criteria_list:
-            evaluation_factor = review_criteria['评审因素']
-            evaluation_criteria = review_criteria['评审标准']
-            if '投标人名称' in evaluation_factor or '供应商名称' in evaluation_factor:
-                ['营业执照','资质证书']
-                '''
-                要求投标文件中 投标公司 与 其提供的营业执照或资质证书中的名称相同
-                '''
-                pass
-            elif '报价函签字盖章' in evaluation_factor or '投标文件封面、投标函签字盖章' in evaluation_factor:
-                '''
-                要求投标文件中 投标公司的 法人或委托人签字或是 存在单位盖章
-                '''
-                pass
-            elif '投标文件格式' in evaluation_factor:
-                file_format = self.contexts_extract(evaluation_criteria)
-                pprint(file_format)
-                '''
-                招标文件 file_format 与投标文件内容对比，投标文件中只要存在file_format内容即可
-                '''
-                chinese_map_list = list(self.chinese_num_map)
-                catelogue_list = []
-                tender_start = 0
-                catelogue_value = ''
-                add_index = 0
-                hit_nums = 0
-                numbers = 0
-                for format_values in file_format.values():
-                    for format in format_values.values():
-                        numbers += 1
-                        catelogue_update_sign = False
-                        first_value = format[0].replace(" ","").replace("\n","")
-                        if '目录' == first_value:
-                            for i in format[1:]:
-                                for j in chinese_map_list:
-                                    if j in i and i not in catelogue_list:
-                                        catelogue_list.append(i)
-                        if catelogue_list and not tender_start:
-                            catelogue = catelogue_list[0]
-                            comp1 = re.compile(f'^{catelogue}')
-                            for tender_context in self.tender_context:
-                                context = tender_context['text']
-                                finder = comp1.findall(context)
-                                if finder:
-                                    tender_start = self.tender_context.index(tender_context)
-                                    break 
-
-                        if first_value in catelogue_list: 
-                            catelogue_update_sign = True
-                            catelogue_value = first_value
-                            catelogue_index = catelogue_list.index(catelogue_value)
-                            if catelogue_list[-1] != catelogue_value:
-                                catelogue_value_next = catelogue_list[catelogue_index+1]
-                            else:
-                                catelogue_value_next = catelogue_value
-                        
-                        if catelogue_value:
-                            hit_num = 0
-                            if catelogue_update_sign:
-                                tender_start += add_index
-                                add_index = 0
-                            for tender_index, tender_contents in enumerate(self.tender_context[tender_start:]):
-                                tender_context = tender_contents['text'].split("\n")
-                                if tender_context[0] == catelogue_value_next: 
-                                    add_index = tender_index
-                                    break
-                                for value in format:
-                                    if value in tender_context:
-                                        hit_num += 1
-                                
-                            hit_nums += hit_num
-                hit_rate = round(hit_nums/numbers,4)
-                if hit_rate>0.70:
-                    formal_result[evaluation_factor] = (True, evaluation_criteria)
-                else:
-                    formal_result[evaluation_factor] = (False, evaluation_criteria)
-            
-            elif '联合体投标人' in evaluation_factor:
-                if '不适用' in evaluation_criteria: continue
-                
-            elif '报价唯一' in evaluation_factor:
-                '''
-                需要在投标文件中比对三个位置的报价总和值抽取
-                '''
-                pass
-    
-
-    def qualification_criteria(self, review_criteria_list:list, bidder_know:dict):
-        ''' Qualification assessment criteria
-        资格评审标准
-        '''
-        for review_criteria in review_criteria_list:
-            evaluation_factor = review_criteria['评审因素']
-            evaluation_criteria = review_criteria['评审标准']
-
-            if '营业执照' in evaluation_factor:
-                '''
-                在投标文件中 对营业执照识别营业期限；长期识别认为可以；只有开始时间没有结束时间给提示。
-                '''
-                pass
-            elif '资质' in evaluation_factor:
-                comp1 = re.compile('(第.*?章)')
-                comp2 = re.compile('“(.*?)”')
-                comp3 = re.compile('第([\d+\.]+)项规定')
-                
-                finder1 = comp1.findall(evaluation_criteria)[0]
-                finder2 = comp2.findall(evaluation_criteria)[0]
-                finder3 = comp3.findall(evaluation_criteria)[0]
-
-                chapter_name = finder1+finder2
-                stipulation = finder3
-
-                if '投标人须知' in chapter_name:
-                    bidder_data = bidder_know.get(stipulation,None)
-                    if not bidder_data: continue  ## 需要修改
-                    clause_name = bidder_data[0]['条款名称'].replace("\n","")
-                    list_content = bidder_data[0]['编列内容']
-
-                    if '招标公告' in list_content:
-                        cert_index = self.announcement.index('资质')   ## 默认 资质条件 不变
-                        cert_required = re.findall("：(.*?)\\n",self.announcement[cert_index:cert_index+500])[0]
-                        print(cert_required)
-                        # 具备法人资格
-                        '''
-                        big model
-
-                        需要设计prompt，可将内容及情况在线上glm4中使用，测出合适prompt
-                        '''
-
-
-        def responsive_criteria(self, review_criteria_list:list, bidder_know:dict):
-            ''' Responsive review criteria
-            响应性评审
-            '''
-            for review_criteria in review_criteria_list:
-                evaluation_factor = review_criteria['评审因素']
-                evaluation_criteria = review_criteria['评审标准']
-
-                if evaluation_factor == '权利义务' or '合同' in evaluation_criteria:
-                    '''不对合同进行处理'''
-                    continue
-                        
-                
-    
-
-    def content_parsing(self):
-        ''' data analysis aggregate function
-        '''
-        tag_dict,bidder_know,scrutinize_dict = dpr.get_table()
-        # {}
-        # self.formal_criteria(tag_dict['形式评审标准'])
-
-
-        self.qualification_criteria(tag_dict['资格评审标准'], bidder_know)
+        return scrutinize_dict
 
 
 
@@ -626,17 +351,11 @@ def get_pre_review():
 
 
 if __name__ == '__main__':
-    dpr = DocumentPreReview()
-    # dpr.check_table(dpr.Bidding_tables)
-    dpr.get_table()
-
-    # dpr.content_parsing()
+    path_list = []
+    for path_ in path_list:
+        dpr = DocumentPreReview(path_)
+        scrutinize_dict = dpr.get_table()  # TODO scrutinize_dict是需要的结果
 
-    # formal_review_criteria = [
-    #         {'评审因素': '投标文件格式', '评审标准': '符合第八章“投标文件格式”的要求'}
-    #         # {'评审因素': '投标文件格式', '评审标准': '符合第四章“合同条款及格式”规定'}
-    #         ]
-    # dpr.formal_criteria(formal_review_criteria)
 
 
 

get_info.py

@@ -2,7 +2,7 @@
 # @Author: privacy
 # @Date:   2024-06-11 13:43:14
 # @Last Modified by:   privacy
-# @Last Modified time: 2024-08-02 11:19:17
+# @Last Modified time: 2024-08-08 17:07:49
 
 # import os
 
@@ -82,14 +82,12 @@ from io import BytesIO
 from pprint import pprint
 
 # 第三方包导入
+import cv2
 import numpy as np
 import pandas as pd
-import cv2
 from pdfminer.high_level import extract_pages
 from pdfminer.layout import LTRect, LTTextBoxHorizontal, LTLine, LTFigure, LTCurve, LTImage, LTChar
 from pdfminer.pdfcolor import LITERAL_DEVICE_CMYK
-from pdfminer.pdfcolor import LITERAL_DEVICE_GRAY
-from pdfminer.pdfcolor import LITERAL_DEVICE_RGB
 from pdfminer.pdftypes import (
     LITERALS_DCT_DECODE,
     LITERALS_JBIG2_DECODE,
@@ -98,7 +96,6 @@ from pdfminer.pdftypes import (
 )
 from pdfminer.pdfparser import PDFParser, PDFSyntaxError
 from pdfminer.pdfdocument import PDFDocument, PDFNoOutlines
-from pdfminer.image import BMPWriter
 import pdfplumber
 import camelot
 
@@ -109,7 +106,8 @@ HEADERS = set({'序号', '项目编码', '项目名称', '项目特征', '单位
 
 
 def is_title(line: str) -> bool:
-    title_word = re.findall('^[（\(][一二三四五六七八九十]+[\)）]|^\d\.|^1\d\.|^2\d\.|^[第][一二三四五六七八九十\d]+[章节条]|[一二三四五六七八九十]+[、要是]', line.strip())
+    # title_word = re.findall('^[（\(][一二三四五六七八九十]+[\)）]|^\d\.|^1\d\.|^2\d\.|^[第][一二三四五六七八九十\d]+[章节条]|[一二三四五六七八九十]+[、要是]', line.strip())
+    title_word = re.findall('^第[一二三四五六七八九十]+|^[一二三四五六七八九十\d]+、|^[\（\(][一二三四五六七八九十]+[\）\)]', line.strip())
     if title_word:
         return True
     title_word = re.findall('^附录|^参考文献|^附表', line.strip())
@@ -131,20 +129,6 @@ def export_image(image: LTImage, path: str) -> str:
         name = _save_jpeg2000(image, path)
         return name
 
-    # elif image.bits == 1:
-    #     name = _save_bmp(image, width, height, (width + 7) // 8, image.bits, path)
-
-    # elif image.bits == 8 and LITERAL_DEVICE_RGB in image.colorspace:
-    #     name = _save_bmp(image, width, height, width * 3, image.bits * 3, path)
-
-    # elif image.bits == 8 and LITERAL_DEVICE_GRAY in image.colorspace:
-    #     name = _save_bmp(image, width, height, width, image.bits, path)
-
-    # elif len(filters) == 1 and filters[0][0] in LITERALS_FLATE_DECODE:
-    #     name = _save_bytes(image)
-
-    # else:
-    #     name = _save_raw(image)
     data = image.stream.get_data()
     raw_data = image.stream.get_rawdata()
 
@@ -283,37 +267,6 @@ def _save_bmp(image: LTImage, width: int, height: int, bytes_per_line: int, bits
         fp.write(data)
     return path
 
-def main_parse(pdf_path: str, title_path: str, image_dir: str) -> None:
-    texts = []
-    images = []
-    # 读取PDF文件并提取页面
-    for page_number, page_layout in enumerate(extract_pages(pdf_path)):
-        title_index = 0
-        image_index = 0
-        for element in page_layout:
-            if isinstance(element, LTLine):
-                pass
-            elif isinstance(element, LTRect):
-                pass
-            elif isinstance(element, LTTextBoxHorizontal) and len(element._objs) == 1:
-                text = element.get_text().strip()
-                # # 假设标题通常是一行且字体较大
-                if text and (is_title(text) or element.height > 15):
-                    texts.append({'index': title_index, 'page_number': page_number, 'bbox': element.bbox, 'text': text})
-                    title_index += 1
-            elif isinstance(element, LTFigure):
-                for e_obj in element._objs:
-                    if isinstance(e_obj, LTImage):
-                        # 提取图片数据
-                        image_file = os.path.join(image_dir, f'image_page_{page_number}_{image_index}')
-                        image_file = export_image(e_obj, image_file)
-                        images.append(image_file)
-                        pprint(f'Image saved: {image_file}')
-                        image_index += 1
-
-    with open(title_path, 'w', encoding='utf-8') as fp:
-        json.dump(texts, fp, indent=4, ensure_ascii=False)
-
 
 def table_parse(pdf_path: str, title_path: str, start_title: str = '六、已标价工程量清单', end_title: str = '七、施工组织设计', table_path: str = 'table.json', start_page_number: int = None, end_page_number: int = None) -> list:
     """pdf表格解析功能
@@ -374,17 +327,185 @@ class PdfExtractAttr(object):
         self.details = []
         self.tables = []
         self.content = []
+        self.chapters = []
+        self.references = []
+        self.detail_df = None
+        self.outlines = None
+
+    def can_merge_lines(self, line1, line2) -> bool:
+        """判断两行文本是否可以合并为一段
+        """
+        # line1 已结束
+        if line1.x1 < self.right:
+            return False
+        # line2 有缩进
+        if line2.x0 > self.left:
+            return False
+        return True
+
+    def main_parse(self, title_path: str = None, section_path: str = None, image_dir: str = None) -> None:
+        """解析PDF
+        参数:
+        - title_path: str, 标题保存路径
+        - sections_path: str, 正文保存目录
+        - image_dir: str, 图片保存目录
+        """
+        self.outlines['text'] = ''
+        # 标题
+        texts = []
+        # 图片
+        images = []
+
+        # 读取PDF文件并提取页面
+        for page_number, page_layout in enumerate(extract_pages(self.file_path)):
+
+            max_start_row = self.outlines.query(f''' page_number <= {page_number+1} ''').query(''' page_number == page_number.max() ''').query(''' level == level.max() ''')
+
+            if not max_start_row.empty:
+                idx = max_start_row.index.values[0]
+            else:
+                idx = len(self.outlines.index)
+                self.outlines.loc[idx] = {'level': 6, 'title': '', 'page_number': 0, 'text': ''}
+
+            # 左侧坐标
+            x0s = []
+            # 右侧坐标
+            x1s = []
+
+            title_index = 0
+            image_index = 0
+
+            for element in page_layout:
+                if isinstance(element, LTTextBoxHorizontal):
+                    x0s.append(element.x0)
+                    x1s.append(element.x1)
+
+            if x0s and x1s:
+                # 左侧边缘
+                self.left = min(x0s) + 15
+                # 右侧边缘
+                self.right = max(x1s) - 15
+
+            current = None
+
+            for element in page_layout:
+
+                if isinstance(element, LTLine):
+                    pass
+
+                elif isinstance(element, LTRect):
+                    pass
+
+                elif isinstance(element, LTTextBoxHorizontal):
+                    # 文本
+                    text = element.get_text().strip()
+
+                    # 假设标题通常是一行且字体较大
+                    if len(element._objs) == 1 and text and (is_title(text) or element.height > 15):
+                        texts.append({'index': title_index, 'page_number': page_number, 'bbox': element.bbox, 'text': text})
+                        title_index += 1
+                        self.outlines.at[idx, 'text'] += '\n'
+                        self.outlines.at[idx, 'text'] += text
+
+                    # 正文部分
+                    elif not current or self.can_merge_lines(current, element):# 可以合并
+                        current = element
+                        for line in element:
+                            self.outlines.at[idx, 'text'] += line.get_text().strip()
+
+                    else:# 不可以合并
+                        for line in element:
+                            self.outlines.at[idx, 'text'] += '\n'
+                            self.outlines.at[idx, 'text'] += line.get_text().strip()
+
+                elif image_dir and isinstance(element, LTFigure):
+                    for e_obj in element._objs:
+                        if isinstance(e_obj, LTImage):
+                            # 提取图片数据
+                            image_file = os.path.join(image_dir, f'image_page_{page_number}_{image_index}')
+                            image_file = export_image(e_obj, image_file)
+                            images.append(image_file)
+                            pprint(f'Image saved: {image_file}')
+                            image_index += 1
+
+
+        if title_path:
+            with open(title_path, 'w', encoding='utf-8') as fp:
+                json.dump(texts, fp, indent=4, ensure_ascii=False)
+
+        if section_path:
+            self.outlines.to_json(section_path, orient='records', lines=True, force_ascii=False)
+
+    def extract_toc(self) -> list:
+        """PDF大纲解析，依据内容解析
+        """
+        results = []
+
+        for page_number, page in enumerate(extract_pages(self.file_path)):
+
+            is_outline = False
+
+            if page_number < 1:
+                continue
+
+            if page_number > 20:
+                break
+
+            lines = []
+            for element in page:
+                if isinstance(element, LTTextBoxHorizontal):
+                    for line in element:
+                        lines.append(line.get_text().strip())
+
+            for line in lines:
+                # 检查是否符合目录格式
+                if line and '......' in line and (line[0].isdigit() or '\u4e00' <= line[0] <= '\u9fff') and line[-1].isdigit():
+                    is_outline = True
+                    # 计算缩进级别
+                    indent_level = 1
+                    # 获取内容
+                    title = re.findall('^[\d\.、]{0,}[\u4e00-\u9fff、（）\s]+', line).pop()
+                    # 计算页码
+                    page_n = int(re.findall('\d+$', line).pop())
+                    # 添加到目录结构中
+                    # directory_structure.append({
+                    results.append({
+                        "level": indent_level,
+                        "title": title,
+                        "page_number": page_n
+                    })
+
+            if not is_outline:
+                break
+
+        return results
+
+    def extract_content(self, content_path: str = None) -> list:
+        with pdfplumber.open(self.file_path) as pdf:
+            for page in pdf.pages:
+                self.content.append({
+                    'page_number': page.page_number - 1,
+                    'text': page.extract_text()
+                })
+
+        if content_path:
+            with open(content_path, 'w', encoding='utf-8') as fp:
+                json.dump(self.content, fp, indent=4, ensure_ascii=False)
+
+        return self.content
 
-    def parse_outline(self):
-        """PDF大纲解析
+    def parse_outline(self, outline_path: str = None) -> list:
+        """PDF大纲解析，依据元数据解析，解析失败则调用内容解析
         """
         results = []
+
         with open(self.file_path, "rb") as fp:
             try:
                 parser = PDFParser(fp)
                 document = PDFDocument(parser)
                 ref_pagenum_resolver = RefPageNumberResolver(document)
                 outlines = document.get_outlines()
+
                 for (level, title, dest, a, se) in outlines:
                     if dest:
                         page_num = ref_pagenum_resolver.resolve(dest)
@@ -395,6 +516,7 @@ class PdfExtractAttr(object):
                     else:
                         page_num = None
                     results.append({'level': level, 'title': title, 'page_number': page_num})
+
             except PDFNoOutlines:
                 print("No outlines found.")
             except PDFSyntaxError:
@@ -402,19 +524,16 @@ class PdfExtractAttr(object):
             finally:
                 parser.close()
 
-        with open('outlines.json', 'w', encoding='utf-8') as op:
-            json.dump(results, op, indent=4, ensure_ascii=False)
+        if not results:
+            results = self.extract_toc()
 
-        print(results)
+        if outline_path:
+            with open(outline_path, 'w', encoding='utf-8') as op:
+                json.dump(results, op, indent=4, ensure_ascii=False)
 
-    def extract_content(self) -> list:
-        with pdfplumber.open(self.file_path) as pdf:
-            for page in pdf.pages:
-                self.content.append({
-                    'page_number': page.page_number - 1,
-                    'text': page.extract_text()
-                })
-        return self.content
+        self.outlines = pd.DataFrame(results)
+
+        return results
 
     def parse_text(self) -> None:
         """文本解析
@@ -485,47 +604,37 @@ class PdfExtractAttr(object):
         else:
             self.tables.append({"page_numbers": [page_number], "title_len": len(first), "col_len": len(table[-1]), "table": table, "confidence": 0, "table_name": table_name if table_name else ""})
 
-    def parse_table(self) -> None:
+    def parse_table_pro(self, table_path: str = 'all_tables.json') -> None:
         """表格解析
         """
-        with pdfplumber.open(self.file_path) as pdf:
-            for page_number, page_layout in enumerate(pdf.pages):
-                # 查询是否存在表格
-                tables = page_layout.find_tables()
-                # 检测到该页面存在一个表格，对其进行合并判断
-                if len(tables) == 1:
-                    table = tables[0]
-                    x0, y0, x1, y1 = table.bbox
-                    table_title_df = self.detail_df.query(f''' page_number == {page_number} and is_table_name == True and alignment == "center" ''')
-                    if table_title_df.empty:
-                        self.concat_table(table.extract(), page_number=page_number)
-                    else:
-                        table_title_name = table_title_df.iloc[0]['text']
-                        self.concat_table(table.extract(), page_number=page_number, table_name=table_title_name)
-                    table = tables[0]
-                    #self.concat_table(table.extract(), table_title_name)
-                # 检测到存在多个表格，对第一个表格进行合并判断之后的表格一定不相干
-                elif len(tables) > 1:
-                    first_table = tables[0]
-                    self.concat_table(first_table.extract(), page_number=page_number)
-                    for table_index in range(1, len(tables)):
-                        self.concat_table(tables[table_index].extract(), page_number=page_number, new=True)
+        if self.detail_df == None:
+            self.parse_text()
 
-    def parse_table_pro(self) -> None:
         with pdfplumber.open(self.file_path) as pdf:
             for page_number, page_layout in enumerate(pdf.pages):
                 # 查询是否存在表格
                 tables = page_layout.find_tables()
 
+                if not tables:
+                    continue
+
+
                 tables_pro = camelot.read_pdf(
                     self.file_path,
                     # flavor='stream',
                     pages=str(page_number+1),
                     # edge_tol=200,
-                    # row_tol=50,
                 )
+
+                if not tables_pro:
+                    continue
+
+                print(len(tables), len(tables_pro))
+
                 # 检测到该页面存在一个表格，对其进行合并判断
-                if len(tables) == 1:
+                if (len(tables) != 0) and (len(tables_pro) == 1):
+                    print(f"解析PDF{page_number}页的表格")
+                    # print(f"解析PDF{page_number}页的表格")
                     table = tables[0]
                     table_pro = tables_pro[0].df.to_dict(orient='split')['data']
                     x0, y0, x1, y1 = table.bbox
@@ -538,36 +647,41 @@ class PdfExtractAttr(object):
                     table = tables[0]
                 # 检测到存在多个表格，对第一个表格进行合并判断之后的表格一定不相干
                 elif len(tables_pro) > 1:
+                    print(f"解析PDF{page_number}页的表格")
                     first_table = tables_pro[0]
                     self.concat_table(first_table.df.to_dict(orient='split')['data'], page_number=page_number)
                     for table_index in range(1, len(tables_pro)):
                         self.concat_table(tables_pro[table_index].df.to_dict(orient='split')['data'], page_number=page_number, new=True)
 
-    def output(self, table_path: str = 'all_tables.json'):
-        """结果输出
-        """
-        with open(table_path, 'w', encoding='utf-8') as fp:
-            json.dump(self.tables, fp, indent=4, ensure_ascii=False)
+        if table_path:
+            with open(table_path, 'w', encoding='utf-8') as fp:
+                json.dump(self.tables, fp, indent=4, ensure_ascii=False)
 
         return self.tables
 
 
 if __name__ == '__main__':
-    pdf_path = 'data/预审查数据/2022-2025年度三峡电站9台机组检修密封加工制作重新招标招标文件印刷版.pdf'
-    image_dir = 'data/预审查数据/extracted_images'
-    title_path = 'data/预审查数据/2022-2025年度三峡电站9台机组检修密封加工制作重新招标招标文件印刷版.json'
-
+    pdf_path = './投标文件-修改版9-5-1-1.pdf'
+    # pdf_path = './南方电网数字研究院有限公司.pdf'
+    # pdf_path = './2022年度工程类-公招采购资料/2022-2025年度三峡电站9台机组检修密封加工制作重新招标/2022-2025年度三峡电站9台机组检修密封加工制作重新招标招标文件印刷版.pdf'
+    # title_path = './投标文件-修改版9-5-1-1.json'
+    # title_path = './投标文件-修改版9-5-1-1-title.json'
+    # title_path = './南方电网数字研究院有限公司.json'
+    # section_path = './投标文件-修改版9-5-1-1-section.json'
+    # section_path = './南方电网数字研究院有限公司-section.json'
+    # image_dir = './extracted_images'
     # os.makedirs(image_dir, exist_ok=True)
-    # main_parse(pdf_path=pdf_path, title_path=title_path, image_dir=image_dir)
-
-    # table_path = 'data/预审查数据/all_tables_2022-2025年度三峡电站9台机组检修密封加工制作重新招标招标文件印刷版.json'
-    # content_path = 'data/预审查数据/contexts_2022-2025年度三峡电站9台机组检修密封加工制作重新招标招标文件印刷版.json'
-    agent = PdfExtractAttr_(file_path=pdf_path)
-
-    # agent.extract_content()
-    # contents = agent.output_()  
-    
-    agent.parse_text()
-    # agent.parse_table()
-    agent.parse_table_pro()
-    all_tables = agent.output()
+
+    # tables = table_parse(pdf_path=pdf_path, title_path=title_path, start_title='六、已标价工程量清单', end_title = '七、施工组织设计')
+    # tables = table_parse(pdf_path=pdf_path, title_path=title_path, start_page_number=0, end_page_number=725)
+
+    # pdf_path = './2022年度工程类-公招采购资料/三峡右岸电站35kV及10kV厂用电系统保护装置换型/三峡右岸电站35kV和10kV厂用电系统保护装置换型招标文件审批稿 (3).pdf'
+    # table_path = './2022年度工程类-公招采购资料/三峡右岸电站35kV及10kV厂用电系统保护装置换型/三峡右岸电站35kV和10kV厂用电系统保护装置换型招标文件审批稿 (3)-table.json'
+
+    pdf_path = './2022年度工程类-公招采购资料/三峡左岸及地下电站地坪整治/三峡左岸及地下电站地坪整治招标文件（发售版）.pdf'
+    table_path = './2022年度工程类-公招采购资料/三峡左岸及地下电站地坪整治/三峡左岸及地下电站地坪整治招标文件（发售版）-table.json'
+
+    agent = PdfExtractAttr(file_path=pdf_path)
+    # agent.parse_outline()
+    # agent.main_parse(title_path=title_path, section_path=section_path)
+    agent.parse_table_pro(table_path=table_path)