|
|
@@ -2,7 +2,7 @@
|
|
|
# @Author: privacy
|
|
|
# @Date: 2024-06-11 13:43:14
|
|
|
# @Last Modified by: privacy
|
|
|
-# @Last Modified time: 2024-08-02 11:19:17
|
|
|
+# @Last Modified time: 2024-08-08 17:07:49
|
|
|
|
|
|
# import os
|
|
|
|
|
|
@@ -82,14 +82,12 @@ from io import BytesIO
|
|
|
from pprint import pprint
|
|
|
|
|
|
# 第三方包导入
|
|
|
+import cv2
|
|
|
import numpy as np
|
|
|
import pandas as pd
|
|
|
-import cv2
|
|
|
from pdfminer.high_level import extract_pages
|
|
|
from pdfminer.layout import LTRect, LTTextBoxHorizontal, LTLine, LTFigure, LTCurve, LTImage, LTChar
|
|
|
from pdfminer.pdfcolor import LITERAL_DEVICE_CMYK
|
|
|
-from pdfminer.pdfcolor import LITERAL_DEVICE_GRAY
|
|
|
-from pdfminer.pdfcolor import LITERAL_DEVICE_RGB
|
|
|
from pdfminer.pdftypes import (
|
|
|
LITERALS_DCT_DECODE,
|
|
|
LITERALS_JBIG2_DECODE,
|
|
|
@@ -98,7 +96,6 @@ from pdfminer.pdftypes import (
|
|
|
)
|
|
|
from pdfminer.pdfparser import PDFParser, PDFSyntaxError
|
|
|
from pdfminer.pdfdocument import PDFDocument, PDFNoOutlines
|
|
|
-from pdfminer.image import BMPWriter
|
|
|
import pdfplumber
|
|
|
import camelot
|
|
|
|
|
|
@@ -109,7 +106,8 @@ HEADERS = set({'序号', '项目编码', '项目名称', '项目特征', '单位
|
|
|
|
|
|
|
|
|
def is_title(line: str) -> bool:
|
|
|
- title_word = re.findall('^[（\(][一二三四五六七八九十]+[\)）]|^\d\.|^1\d\.|^2\d\.|^[第][一二三四五六七八九十\d]+[章节条]|[一二三四五六七八九十]+[、要是]', line.strip())
|
|
|
+ # title_word = re.findall('^[（\(][一二三四五六七八九十]+[\)）]|^\d\.|^1\d\.|^2\d\.|^[第][一二三四五六七八九十\d]+[章节条]|[一二三四五六七八九十]+[、要是]', line.strip())
|
|
|
+ title_word = re.findall('^第[一二三四五六七八九十]+|^[一二三四五六七八九十\d]+、|^[\（\(][一二三四五六七八九十]+[\）\)]', line.strip())
|
|
|
if title_word:
|
|
|
return True
|
|
|
title_word = re.findall('^附录|^参考文献|^附表', line.strip())
|
|
|
@@ -131,20 +129,6 @@ def export_image(image: LTImage, path: str) -> str:
|
|
|
name = _save_jpeg2000(image, path)
|
|
|
return name
|
|
|
|
|
|
- # elif image.bits == 1:
|
|
|
- # name = _save_bmp(image, width, height, (width + 7) // 8, image.bits, path)
|
|
|
-
|
|
|
- # elif image.bits == 8 and LITERAL_DEVICE_RGB in image.colorspace:
|
|
|
- # name = _save_bmp(image, width, height, width * 3, image.bits * 3, path)
|
|
|
-
|
|
|
- # elif image.bits == 8 and LITERAL_DEVICE_GRAY in image.colorspace:
|
|
|
- # name = _save_bmp(image, width, height, width, image.bits, path)
|
|
|
-
|
|
|
- # elif len(filters) == 1 and filters[0][0] in LITERALS_FLATE_DECODE:
|
|
|
- # name = _save_bytes(image)
|
|
|
-
|
|
|
- # else:
|
|
|
- # name = _save_raw(image)
|
|
|
data = image.stream.get_data()
|
|
|
raw_data = image.stream.get_rawdata()
|
|
|
|
|
|
@@ -283,37 +267,6 @@ def _save_bmp(image: LTImage, width: int, height: int, bytes_per_line: int, bits
|
|
|
fp.write(data)
|
|
|
return path
|
|
|
|
|
|
-def main_parse(pdf_path: str, title_path: str, image_dir: str) -> None:
|
|
|
- texts = []
|
|
|
- images = []
|
|
|
- # 读取PDF文件并提取页面
|
|
|
- for page_number, page_layout in enumerate(extract_pages(pdf_path)):
|
|
|
- title_index = 0
|
|
|
- image_index = 0
|
|
|
- for element in page_layout:
|
|
|
- if isinstance(element, LTLine):
|
|
|
- pass
|
|
|
- elif isinstance(element, LTRect):
|
|
|
- pass
|
|
|
- elif isinstance(element, LTTextBoxHorizontal) and len(element._objs) == 1:
|
|
|
- text = element.get_text().strip()
|
|
|
- # # 假设标题通常是一行且字体较大
|
|
|
- if text and (is_title(text) or element.height > 15):
|
|
|
- texts.append({'index': title_index, 'page_number': page_number, 'bbox': element.bbox, 'text': text})
|
|
|
- title_index += 1
|
|
|
- elif isinstance(element, LTFigure):
|
|
|
- for e_obj in element._objs:
|
|
|
- if isinstance(e_obj, LTImage):
|
|
|
- # 提取图片数据
|
|
|
- image_file = os.path.join(image_dir, f'image_page_{page_number}_{image_index}')
|
|
|
- image_file = export_image(e_obj, image_file)
|
|
|
- images.append(image_file)
|
|
|
- pprint(f'Image saved: {image_file}')
|
|
|
- image_index += 1
|
|
|
-
|
|
|
- with open(title_path, 'w', encoding='utf-8') as fp:
|
|
|
- json.dump(texts, fp, indent=4, ensure_ascii=False)
|
|
|
-
|
|
|
|
|
|
def table_parse(pdf_path: str, title_path: str, start_title: str = '六、已标价工程量清单', end_title: str = '七、施工组织设计', table_path: str = 'table.json', start_page_number: int = None, end_page_number: int = None) -> list:
|
|
|
"""pdf表格解析功能
|
|
|
@@ -374,17 +327,185 @@ class PdfExtractAttr(object):
|
|
|
self.details = []
|
|
|
self.tables = []
|
|
|
self.content = []
|
|
|
+ self.chapters = []
|
|
|
+ self.references = []
|
|
|
+ self.detail_df = None
|
|
|
+ self.outlines = None
|
|
|
+
|
|
|
+ def can_merge_lines(self, line1, line2) -> bool:
|
|
|
+ """判断两行文本是否可以合并为一段
|
|
|
+ """
|
|
|
+ # line1 已结束
|
|
|
+ if line1.x1 < self.right:
|
|
|
+ return False
|
|
|
+ # line2 有缩进
|
|
|
+ if line2.x0 > self.left:
|
|
|
+ return False
|
|
|
+ return True
|
|
|
+
|
|
|
+ def main_parse(self, title_path: str = None, section_path: str = None, image_dir: str = None) -> None:
|
|
|
+ """解析PDF
|
|
|
+ 参数:
|
|
|
+ - title_path: str, 标题保存路径
|
|
|
+ - sections_path: str, 正文保存目录
|
|
|
+ - image_dir: str, 图片保存目录
|
|
|
+ """
|
|
|
+ self.outlines['text'] = ''
|
|
|
+ # 标题
|
|
|
+ texts = []
|
|
|
+ # 图片
|
|
|
+ images = []
|
|
|
+
|
|
|
+ # 读取PDF文件并提取页面
|
|
|
+ for page_number, page_layout in enumerate(extract_pages(self.file_path)):
|
|
|
+
|
|
|
+ max_start_row = self.outlines.query(f''' page_number <= {page_number+1} ''').query(''' page_number == page_number.max() ''').query(''' level == level.max() ''')
|
|
|
+
|
|
|
+ if not max_start_row.empty:
|
|
|
+ idx = max_start_row.index.values[0]
|
|
|
+ else:
|
|
|
+ idx = len(self.outlines.index)
|
|
|
+ self.outlines.loc[idx] = {'level': 6, 'title': '', 'page_number': 0, 'text': ''}
|
|
|
+
|
|
|
+ # 左侧坐标
|
|
|
+ x0s = []
|
|
|
+ # 右侧坐标
|
|
|
+ x1s = []
|
|
|
+
|
|
|
+ title_index = 0
|
|
|
+ image_index = 0
|
|
|
+
|
|
|
+ for element in page_layout:
|
|
|
+ if isinstance(element, LTTextBoxHorizontal):
|
|
|
+ x0s.append(element.x0)
|
|
|
+ x1s.append(element.x1)
|
|
|
+
|
|
|
+ if x0s and x1s:
|
|
|
+ # 左侧边缘
|
|
|
+ self.left = min(x0s) + 15
|
|
|
+ # 右侧边缘
|
|
|
+ self.right = max(x1s) - 15
|
|
|
+
|
|
|
+ current = None
|
|
|
+
|
|
|
+ for element in page_layout:
|
|
|
+
|
|
|
+ if isinstance(element, LTLine):
|
|
|
+ pass
|
|
|
+
|
|
|
+ elif isinstance(element, LTRect):
|
|
|
+ pass
|
|
|
+
|
|
|
+ elif isinstance(element, LTTextBoxHorizontal):
|
|
|
+ # 文本
|
|
|
+ text = element.get_text().strip()
|
|
|
+
|
|
|
+ # 假设标题通常是一行且字体较大
|
|
|
+ if len(element._objs) == 1 and text and (is_title(text) or element.height > 15):
|
|
|
+ texts.append({'index': title_index, 'page_number': page_number, 'bbox': element.bbox, 'text': text})
|
|
|
+ title_index += 1
|
|
|
+ self.outlines.at[idx, 'text'] += '\n'
|
|
|
+ self.outlines.at[idx, 'text'] += text
|
|
|
+
|
|
|
+ # 正文部分
|
|
|
+ elif not current or self.can_merge_lines(current, element):# 可以合并
|
|
|
+ current = element
|
|
|
+ for line in element:
|
|
|
+ self.outlines.at[idx, 'text'] += line.get_text().strip()
|
|
|
+
|
|
|
+ else:# 不可以合并
|
|
|
+ for line in element:
|
|
|
+ self.outlines.at[idx, 'text'] += '\n'
|
|
|
+ self.outlines.at[idx, 'text'] += line.get_text().strip()
|
|
|
+
|
|
|
+ elif image_dir and isinstance(element, LTFigure):
|
|
|
+ for e_obj in element._objs:
|
|
|
+ if isinstance(e_obj, LTImage):
|
|
|
+ # 提取图片数据
|
|
|
+ image_file = os.path.join(image_dir, f'image_page_{page_number}_{image_index}')
|
|
|
+ image_file = export_image(e_obj, image_file)
|
|
|
+ images.append(image_file)
|
|
|
+ pprint(f'Image saved: {image_file}')
|
|
|
+ image_index += 1
|
|
|
+
|
|
|
+
|
|
|
+ if title_path:
|
|
|
+ with open(title_path, 'w', encoding='utf-8') as fp:
|
|
|
+ json.dump(texts, fp, indent=4, ensure_ascii=False)
|
|
|
+
|
|
|
+ if section_path:
|
|
|
+ self.outlines.to_json(section_path, orient='records', lines=True, force_ascii=False)
|
|
|
+
|
|
|
+ def extract_toc(self) -> list:
|
|
|
+ """PDF大纲解析，依据内容解析
|
|
|
+ """
|
|
|
+ results = []
|
|
|
+
|
|
|
+ for page_number, page in enumerate(extract_pages(self.file_path)):
|
|
|
+
|
|
|
+ is_outline = False
|
|
|
+
|
|
|
+ if page_number < 1:
|
|
|
+ continue
|
|
|
+
|
|
|
+ if page_number > 20:
|
|
|
+ break
|
|
|
+
|
|
|
+ lines = []
|
|
|
+ for element in page:
|
|
|
+ if isinstance(element, LTTextBoxHorizontal):
|
|
|
+ for line in element:
|
|
|
+ lines.append(line.get_text().strip())
|
|
|
+
|
|
|
+ for line in lines:
|
|
|
+ # 检查是否符合目录格式
|
|
|
+ if line and '......' in line and (line[0].isdigit() or '\u4e00' <= line[0] <= '\u9fff') and line[-1].isdigit():
|
|
|
+ is_outline = True
|
|
|
+ # 计算缩进级别
|
|
|
+ indent_level = 1
|
|
|
+ # 获取内容
|
|
|
+ title = re.findall('^[\d\.、]{0,}[\u4e00-\u9fff、（）\s]+', line).pop()
|
|
|
+ # 计算页码
|
|
|
+ page_n = int(re.findall('\d+$', line).pop())
|
|
|
+ # 添加到目录结构中
|
|
|
+ # directory_structure.append({
|
|
|
+ results.append({
|
|
|
+ "level": indent_level,
|
|
|
+ "title": title,
|
|
|
+ "page_number": page_n
|
|
|
+ })
|
|
|
+
|
|
|
+ if not is_outline:
|
|
|
+ break
|
|
|
+
|
|
|
+ return results
|
|
|
+
|
|
|
+ def extract_content(self, content_path: str = None) -> list:
|
|
|
+ with pdfplumber.open(self.file_path) as pdf:
|
|
|
+ for page in pdf.pages:
|
|
|
+ self.content.append({
|
|
|
+ 'page_number': page.page_number - 1,
|
|
|
+ 'text': page.extract_text()
|
|
|
+ })
|
|
|
+
|
|
|
+ if content_path:
|
|
|
+ with open(content_path, 'w', encoding='utf-8') as fp:
|
|
|
+ json.dump(self.content, fp, indent=4, ensure_ascii=False)
|
|
|
+
|
|
|
+ return self.content
|
|
|
|
|
|
- def parse_outline(self):
|
|
|
- """PDF大纲解析
|
|
|
+ def parse_outline(self, outline_path: str = None) -> list:
|
|
|
+ """PDF大纲解析，依据元数据解析，解析失败则调用内容解析
|
|
|
"""
|
|
|
results = []
|
|
|
+
|
|
|
with open(self.file_path, "rb") as fp:
|
|
|
try:
|
|
|
parser = PDFParser(fp)
|
|
|
document = PDFDocument(parser)
|
|
|
ref_pagenum_resolver = RefPageNumberResolver(document)
|
|
|
outlines = document.get_outlines()
|
|
|
+
|
|
|
for (level, title, dest, a, se) in outlines:
|
|
|
if dest:
|
|
|
page_num = ref_pagenum_resolver.resolve(dest)
|
|
|
@@ -395,6 +516,7 @@ class PdfExtractAttr(object):
|
|
|
else:
|
|
|
page_num = None
|
|
|
results.append({'level': level, 'title': title, 'page_number': page_num})
|
|
|
+
|
|
|
except PDFNoOutlines:
|
|
|
print("No outlines found.")
|
|
|
except PDFSyntaxError:
|
|
|
@@ -402,19 +524,16 @@ class PdfExtractAttr(object):
|
|
|
finally:
|
|
|
parser.close()
|
|
|
|
|
|
- with open('outlines.json', 'w', encoding='utf-8') as op:
|
|
|
- json.dump(results, op, indent=4, ensure_ascii=False)
|
|
|
+ if not results:
|
|
|
+ results = self.extract_toc()
|
|
|
|
|
|
- print(results)
|
|
|
+ if outline_path:
|
|
|
+ with open(outline_path, 'w', encoding='utf-8') as op:
|
|
|
+ json.dump(results, op, indent=4, ensure_ascii=False)
|
|
|
|
|
|
- def extract_content(self) -> list:
|
|
|
- with pdfplumber.open(self.file_path) as pdf:
|
|
|
- for page in pdf.pages:
|
|
|
- self.content.append({
|
|
|
- 'page_number': page.page_number - 1,
|
|
|
- 'text': page.extract_text()
|
|
|
- })
|
|
|
- return self.content
|
|
|
+ self.outlines = pd.DataFrame(results)
|
|
|
+
|
|
|
+ return results
|
|
|
|
|
|
def parse_text(self) -> None:
|
|
|
"""文本解析
|
|
|
@@ -485,47 +604,37 @@ class PdfExtractAttr(object):
|
|
|
else:
|
|
|
self.tables.append({"page_numbers": [page_number], "title_len": len(first), "col_len": len(table[-1]), "table": table, "confidence": 0, "table_name": table_name if table_name else ""})
|
|
|
|
|
|
- def parse_table(self) -> None:
|
|
|
+ def parse_table_pro(self, table_path: str = 'all_tables.json') -> None:
|
|
|
"""表格解析
|
|
|
"""
|
|
|
- with pdfplumber.open(self.file_path) as pdf:
|
|
|
- for page_number, page_layout in enumerate(pdf.pages):
|
|
|
- # 查询是否存在表格
|
|
|
- tables = page_layout.find_tables()
|
|
|
- # 检测到该页面存在一个表格，对其进行合并判断
|
|
|
- if len(tables) == 1:
|
|
|
- table = tables[0]
|
|
|
- x0, y0, x1, y1 = table.bbox
|
|
|
- table_title_df = self.detail_df.query(f''' page_number == {page_number} and is_table_name == True and alignment == "center" ''')
|
|
|
- if table_title_df.empty:
|
|
|
- self.concat_table(table.extract(), page_number=page_number)
|
|
|
- else:
|
|
|
- table_title_name = table_title_df.iloc[0]['text']
|
|
|
- self.concat_table(table.extract(), page_number=page_number, table_name=table_title_name)
|
|
|
- table = tables[0]
|
|
|
- #self.concat_table(table.extract(), table_title_name)
|
|
|
- # 检测到存在多个表格，对第一个表格进行合并判断之后的表格一定不相干
|
|
|
- elif len(tables) > 1:
|
|
|
- first_table = tables[0]
|
|
|
- self.concat_table(first_table.extract(), page_number=page_number)
|
|
|
- for table_index in range(1, len(tables)):
|
|
|
- self.concat_table(tables[table_index].extract(), page_number=page_number, new=True)
|
|
|
+ if self.detail_df == None:
|
|
|
+ self.parse_text()
|
|
|
|
|
|
- def parse_table_pro(self) -> None:
|
|
|
with pdfplumber.open(self.file_path) as pdf:
|
|
|
for page_number, page_layout in enumerate(pdf.pages):
|
|
|
# 查询是否存在表格
|
|
|
tables = page_layout.find_tables()
|
|
|
|
|
|
+ if not tables:
|
|
|
+ continue
|
|
|
+
|
|
|
+
|
|
|
tables_pro = camelot.read_pdf(
|
|
|
self.file_path,
|
|
|
# flavor='stream',
|
|
|
pages=str(page_number+1),
|
|
|
# edge_tol=200,
|
|
|
- # row_tol=50,
|
|
|
)
|
|
|
+
|
|
|
+ if not tables_pro:
|
|
|
+ continue
|
|
|
+
|
|
|
+ print(len(tables), len(tables_pro))
|
|
|
+
|
|
|
# 检测到该页面存在一个表格，对其进行合并判断
|
|
|
- if len(tables) == 1:
|
|
|
+ if (len(tables) != 0) and (len(tables_pro) == 1):
|
|
|
+ print(f"解析PDF{page_number}页的表格")
|
|
|
+ # print(f"解析PDF{page_number}页的表格")
|
|
|
table = tables[0]
|
|
|
table_pro = tables_pro[0].df.to_dict(orient='split')['data']
|
|
|
x0, y0, x1, y1 = table.bbox
|
|
|
@@ -538,36 +647,41 @@ class PdfExtractAttr(object):
|
|
|
table = tables[0]
|
|
|
# 检测到存在多个表格，对第一个表格进行合并判断之后的表格一定不相干
|
|
|
elif len(tables_pro) > 1:
|
|
|
+ print(f"解析PDF{page_number}页的表格")
|
|
|
first_table = tables_pro[0]
|
|
|
self.concat_table(first_table.df.to_dict(orient='split')['data'], page_number=page_number)
|
|
|
for table_index in range(1, len(tables_pro)):
|
|
|
self.concat_table(tables_pro[table_index].df.to_dict(orient='split')['data'], page_number=page_number, new=True)
|
|
|
|
|
|
- def output(self, table_path: str = 'all_tables.json'):
|
|
|
- """结果输出
|
|
|
- """
|
|
|
- with open(table_path, 'w', encoding='utf-8') as fp:
|
|
|
- json.dump(self.tables, fp, indent=4, ensure_ascii=False)
|
|
|
+ if table_path:
|
|
|
+ with open(table_path, 'w', encoding='utf-8') as fp:
|
|
|
+ json.dump(self.tables, fp, indent=4, ensure_ascii=False)
|
|
|
|
|
|
return self.tables
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
- pdf_path = 'data/预审查数据/2022-2025年度三峡电站9台机组检修密封加工制作重新招标招标文件印刷版.pdf'
|
|
|
- image_dir = 'data/预审查数据/extracted_images'
|
|
|
- title_path = 'data/预审查数据/2022-2025年度三峡电站9台机组检修密封加工制作重新招标招标文件印刷版.json'
|
|
|
-
|
|
|
+ pdf_path = './投标文件-修改版9-5-1-1.pdf'
|
|
|
+ # pdf_path = './南方电网数字研究院有限公司.pdf'
|
|
|
+ # pdf_path = './2022年度工程类-公招采购资料/2022-2025年度三峡电站9台机组检修密封加工制作重新招标/2022-2025年度三峡电站9台机组检修密封加工制作重新招标招标文件印刷版.pdf'
|
|
|
+ # title_path = './投标文件-修改版9-5-1-1.json'
|
|
|
+ # title_path = './投标文件-修改版9-5-1-1-title.json'
|
|
|
+ # title_path = './南方电网数字研究院有限公司.json'
|
|
|
+ # section_path = './投标文件-修改版9-5-1-1-section.json'
|
|
|
+ # section_path = './南方电网数字研究院有限公司-section.json'
|
|
|
+ # image_dir = './extracted_images'
|
|
|
# os.makedirs(image_dir, exist_ok=True)
|
|
|
- # main_parse(pdf_path=pdf_path, title_path=title_path, image_dir=image_dir)
|
|
|
-
|
|
|
- # table_path = 'data/预审查数据/all_tables_2022-2025年度三峡电站9台机组检修密封加工制作重新招标招标文件印刷版.json'
|
|
|
- # content_path = 'data/预审查数据/contexts_2022-2025年度三峡电站9台机组检修密封加工制作重新招标招标文件印刷版.json'
|
|
|
- agent = PdfExtractAttr_(file_path=pdf_path)
|
|
|
-
|
|
|
- # agent.extract_content()
|
|
|
- # contents = agent.output_()
|
|
|
-
|
|
|
- agent.parse_text()
|
|
|
- # agent.parse_table()
|
|
|
- agent.parse_table_pro()
|
|
|
- all_tables = agent.output()
|
|
|
+
|
|
|
+ # tables = table_parse(pdf_path=pdf_path, title_path=title_path, start_title='六、已标价工程量清单', end_title = '七、施工组织设计')
|
|
|
+ # tables = table_parse(pdf_path=pdf_path, title_path=title_path, start_page_number=0, end_page_number=725)
|
|
|
+
|
|
|
+ # pdf_path = './2022年度工程类-公招采购资料/三峡右岸电站35kV及10kV厂用电系统保护装置换型/三峡右岸电站35kV和10kV厂用电系统保护装置换型招标文件审批稿 (3).pdf'
|
|
|
+ # table_path = './2022年度工程类-公招采购资料/三峡右岸电站35kV及10kV厂用电系统保护装置换型/三峡右岸电站35kV和10kV厂用电系统保护装置换型招标文件审批稿 (3)-table.json'
|
|
|
+
|
|
|
+ pdf_path = './2022年度工程类-公招采购资料/三峡左岸及地下电站地坪整治/三峡左岸及地下电站地坪整治招标文件（发售版）.pdf'
|
|
|
+ table_path = './2022年度工程类-公招采购资料/三峡左岸及地下电站地坪整治/三峡左岸及地下电站地坪整治招标文件（发售版）-table.json'
|
|
|
+
|
|
|
+ agent = PdfExtractAttr(file_path=pdf_path)
|
|
|
+ # agent.parse_outline()
|
|
|
+ # agent.main_parse(title_path=title_path, section_path=section_path)
|
|
|
+ agent.parse_table_pro(table_path=table_path)
|
lfygithub01