#!/usr/bin/env python3
"""
PDF to Markdown Converter
Uses PyMuPDF to extract PDF text content and convert to Markdown format.
Supports heading levels, bold, italic, and list detection.
"""
import argparse
import hashlib
import json
import os
import re
import sys
from pathlib import Path
from collections import Counter
_SCRIPTS_DIR = Path(__file__).resolve().parents[1]
if str(_SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(_SCRIPTS_DIR))
from console_encoding import configure_utf8_stdio # noqa: E402
from _batch import run_path_batch # noqa: E402
from _conversion_profile import write_conversion_profile_best_effort # noqa: E402
configure_utf8_stdio()
try:
import fitz # PyMuPDF
except ImportError:
print("[ERROR] PyMuPDF not installed. Run: pip install PyMuPDF", file=sys.stderr)
sys.exit(1)
FONT_BODY_SIZE = 12
FONT_H1_SIZE = 24
FONT_H2_SIZE = 18
FONT_H3_SIZE = 14
HEADER_FOOTER_SAMPLE_LIMIT = 40
HEADER_FOOTER_EDGE_SAMPLE_SIZE = 20
CONTROL_CHARS_RE = re.compile(r'[\x00-\x08\x0b\x0c\x0e-\x1f]')
def analyze_font_sizes(doc: fitz.Document) -> dict[str, float]:
"""Analyze font size distribution to infer heading levels.
Args:
doc: Open PDF document.
Returns:
A size mapping containing body and inferred heading sizes.
"""
size_counter = Counter()
for page in doc:
blocks = page.get_text("dict")["blocks"]
for block in blocks:
if block["type"] == 0:
for line in block["lines"]:
for span in line["spans"]:
size = round(span["size"], 1)
text = span["text"].strip()
if text:
size_counter[size] += len(text)
if not size_counter:
return {
"body": FONT_BODY_SIZE,
"h1": FONT_H1_SIZE,
"h2": FONT_H2_SIZE,
"h3": FONT_H3_SIZE,
}
sorted_sizes = sorted(size_counter.items(), key=lambda x: x[1], reverse=True)
body_size = sorted_sizes[0][0]
all_sizes = sorted(size_counter.keys(), reverse=True)
larger_sizes = [s for s in all_sizes if s > body_size + 1]
size_map = {"body": body_size}
if len(larger_sizes) >= 1:
size_map["h1"] = larger_sizes[0]
if len(larger_sizes) >= 2:
size_map["h2"] = larger_sizes[1]
if len(larger_sizes) >= 3:
size_map["h3"] = larger_sizes[2]
return size_map
def get_heading_level(size: float, size_map: dict, text: str = "",
flags: int = 0, strict: bool = True) -> int:
"""
Determine heading level using multiple heuristics.
Args:
size: Font size
size_map: Font size mapping
text: Text content (used for additional heuristics)
flags: Font flags (bit 4 = bold)
strict: Strict mode, requires more conditions to be met
Returns:
Heading level (0 = body text, 1-3 = H1-H3)
"""
# Initial determination based on font size
level = 0
if "h1" in size_map and size >= size_map["h1"] - 0.5:
level = 1
elif "h2" in size_map and size >= size_map["h2"] - 0.5:
level = 2
elif "h3" in size_map and size >= size_map["h3"] - 0.5:
level = 3
if level == 0:
return 0
# Non-strict mode returns directly (backward compatible)
if not strict or not text:
return level
# Strict mode: additional validation conditions
text = text.strip()
# Exclusion: text too long is unlikely to be a heading
if len(text) > 80:
return 0
# Exclusion: complete sentences ending with punctuation
sentence_endings = '.。!！?？'
if text and text[-1] in sentence_endings:
# But keep numbered headings like "1. Overview" or "Chapter 1."
if not re.match(r'^[\d第]+[.、章节]', text):
return 0
# Bonus: bold text is more likely to be a heading
is_bold = flags & 16
if not is_bold and level >= 2:
# Non-bold subheadings require a larger font size difference
body_size = size_map.get("body", 12)
if size < body_size + 2:
return 0
return level
def is_monospace_font(font_name: str) -> bool:
"""
Determine if the font is monospace (typically used for code).
"""
if not font_name:
return False
font_lower = font_name.lower()
mono_fonts = [
'courier', 'consolas', 'monaco', 'menlo', 'monospace',
'source code', 'fira code', 'jetbrains', 'inconsolata',
'dejavu sans mono', 'liberation mono', 'ubuntu mono',
'roboto mono', 'robotomono', 'sf mono', 'cascadia', 'hack'
]
return any(f in font_lower for f in mono_fonts)
def format_span_text(text: str, flags: int) -> str:
"""Format text based on font flags (bold, italic)."""
text = CONTROL_CHARS_RE.sub('', text)
text = text.strip()
if not text:
return ""
is_bold = flags & 16
is_italic = flags & 2
if is_bold and is_italic:
return f"***{text}***"
elif is_bold:
return f"**{text}**"
elif is_italic:
return f"*{text}*"
return text
def detect_list_item(text: str) -> tuple:
"""Detect if the text is a list item. Returns (is_list, list_type, content)."""
text = text.strip()
ul_patterns = [
(r'^[•●○◦▪▸►]\s*', '-'),
(r'^[-–—]\s+', '-'),
(r'^\*\s+', '-'),
]
for pattern, marker in ul_patterns:
match = re.match(pattern, text)
if match:
return (True, 'ul', marker + ' ' + text[match.end():])
ol_pattern = r'^(\d+)[.、)]\s*'
match = re.match(ol_pattern, text)
if match:
num = match.group(1)
return (True, 'ol', f"{num}. " + text[match.end():])
return (False, None, text)
def remove_page_footer(text: str) -> str:
"""
Remove page number patterns from footers, e.g. 'November 2025 8' or '2025年11月 8'.
"""
# English month + year + page number
months_en = r'(?:January|February|March|April|May|June|July|August|September|October|November|December)'
pattern_en = rf'\s*{months_en}\s+\d{{4}}\s+\d{{1,3}}\s*$'
text = re.sub(pattern_en, '', text, flags=re.IGNORECASE)
# Chinese format: 2025年11月 8
pattern_cn = r'\s*\d{4}年\d{1,2}月\s+\d{1,3}\s*$'
text = re.sub(pattern_cn, '', text)
return text.rstrip()
def detect_headers_footers(doc: fitz.Document, threshold_ratio: float = 0.6) -> set[str]:
"""
Detect headers and footers statistically.
Principle: Headers and footers typically appear at fixed positions (top or bottom)
on each page with the same content. We collect top and bottom text from all pages,
and if certain text appears more frequently than the threshold, it is treated as noise.
"""
if len(doc) < 3:
return set()
headers = []
footers = []
# Sample first 20 and last 20 pages (avoid processing too slowly)
pages_to_scan = list(range(len(doc)))
if len(doc) > HEADER_FOOTER_SAMPLE_LIMIT:
pages_to_scan = (
pages_to_scan[:HEADER_FOOTER_EDGE_SAMPLE_SIZE]
+ pages_to_scan[-HEADER_FOOTER_EDGE_SAMPLE_SIZE:]
)
for i in pages_to_scan:
page = doc[i]
rect = page.rect
h = rect.height
# Define top and bottom regions (15% each)
top_rect = fitz.Rect(0, 0, rect.width, h * 0.15)
bottom_rect = fitz.Rect(0, h * 0.85, rect.width, h)
# Extract text blocks
blocks = page.get_text("blocks")
for b in blocks:
b_rect = fitz.Rect(b[:4])
text = b[4].strip()
if not text:
continue
# Simple spatial determination
if b_rect.intersects(top_rect):
headers.append(text)
elif b_rect.intersects(bottom_rect):
footers.append(text)
# Count frequencies
noise_texts = set()
total_scanned = len(pages_to_scan)
for collection in [headers, footers]:
counter = Counter(collection)
for text, count in counter.items():
# if text appears in > 60% of scanned pages, mark as noise
if count / total_scanned > threshold_ratio:
noise_texts.add(text)
return noise_texts
def merge_adjacent_headings(elements: list) -> list:
"""
Merge adjacent same-level short headings.
Example: '# Agent Tools &' + '# Interoperability' -> '# Agent Tools & Interoperability'
"""
if not elements:
return elements
merged = []
i = 0
while i < len(elements):
el = elements[i]
# Only process heading elements
if el.get("type") != 0 or not el.get("is_heading"):
merged.append(el)
i += 1
continue
content = el["content"]
# Extract heading level
match = re.match(r'^(#{1,6})\s+(.+)$', content)
if not match:
merged.append(el)
i += 1
continue
level = match.group(1)
title_text = match.group(2)
# If heading is short and the next one is also the same level, try to merge
j = i + 1
while j < len(elements) and len(title_text) < 60:
next_el = elements[j]
if next_el.get("type") != 0 or not next_el.get("is_heading"):
break
next_match = re.match(r'^(#{1,6})\s+(.+)$', next_el["content"])
if not next_match or next_match.group(1) != level:
break
next_text = next_match.group(2)
# Only merge short heading fragments
if len(next_text) > 40:
break
# Merge
title_text += " " + next_text
j += 1
# Create merged element
merged_el = el.copy()
merged_el["content"] = f"{level} {title_text}"
merged.append(merged_el)
i = j
return merged
# Image filtering thresholds
MIN_IMAGE_PIXELS = 100 # Minimum pixel dimension (width AND height)
MIN_IMAGE_AREA = 30000 # Minimum pixel area (e.g. 200x150)
MIN_IMAGE_BYTES = 2048 # Minimum image data size (2KB)
MIN_PAGE_RATIO = 0.05 # Minimum render size relative to page (5%)
MIN_VISIBLE_IMAGE_WIDTH = 40
MIN_VISIBLE_IMAGE_HEIGHT = 40
MIN_VISIBLE_IMAGE_AREA_RATIO = 0.01
MAX_ASPECT_RATIO = 12 # Maximum aspect ratio (filters decorative bars)
MAX_LOW_INFO_BPP = 0.08 # Bytes-per-pixel threshold for low-info images
MAX_LOW_INFO_AREA = 500000 # Area threshold: only apply bpp filter below this
MIN_VECTOR_FIGURE_WIDTH = 100
MIN_VECTOR_FIGURE_HEIGHT = 80
MIN_VECTOR_FIGURE_AREA = 30000
MAX_VECTOR_FIGURE_ASPECT_RATIO = 8
VECTOR_FIGURE_PADDING = 4
VECTOR_FIGURE_DPI = 180
VECTOR_CAPTION_SEARCH_HEIGHT = 380
VECTOR_CAPTION_HORIZONTAL_GAP = 90
MAX_VECTOR_BACKGROUND_AREA_RATIO = 1.9
# Caption delimiters: ``Figure 1:`` / ``Figure 1.`` (classic) and ``Figure 1 |``
# (the DeepMind / Distill / Nature house style used by many ML papers, incl.
# full-width ``｜``). Without the pipe variants, captioned vector figures route
# to the generic per-drawing fallback, which discards fine-grained line plots
# (each plot is hundreds of tiny primitives, none large enough on its own).
FIGURE_CAPTION_RE = re.compile(r'^(?:Figure|Fig\.?)\s*\d+\s*[:.|｜]', re.IGNORECASE)
TABLE_CAPTION_RE = re.compile(
r'^\u8868\s*\d+(?:\.\d+)*\s+(?!(?:\u7684|\u5217\u793a|\u6240\u793a|\u4e3a)).+'
)
TABLE_REFERENCE_PROSE_RE = re.compile(
r'^\u8868\s*\d+(?:\.\d+)*\s*(?:\u7684|\u5217\u793a|\u6240\u793a|\u4e3a)'
)
SECTION_HEADING_RE = re.compile(r'^\d+(?:\.\d+){1,3}\s+\S')
NUMBER_RE = re.compile(r'[-+]?\d+(?:\.\d+)?')
MODEL_COLUMN_RE = re.compile(r'^[（(]\d+[）)]$')
TABLE_NOTE_PREFIX = '\u6ce8'
TABLE_CONTINUATION_Y_RATIO = 0.88
TABLE_SCAN_BOTTOM_RATIO = 0.92
def should_keep_image(
block: dict[str, object],
page_rect: fitz.Rect,
seen_hashes: set[str] | None = None,
) -> bool:
"""Filter out small, decorative, or duplicate images.
Args:
block: Image block extracted from PyMuPDF.
page_rect: Current page rectangle.
seen_hashes: Optional set used to deduplicate image payloads.
Returns:
Whether the image should be kept in the Markdown output.
"""
w, h = block.get("width", 0), block.get("height", 0)
bbox = block.get("bbox", (0, 0, 0, 0))
render_w = bbox[2] - bbox[0]
render_h = bbox[3] - bbox[1]
page_area = page_rect.width * page_rect.height
render_area_ratio = (render_w * render_h) / page_area if page_area > 0 else 0
visibly_placed = (
render_w >= MIN_VISIBLE_IMAGE_WIDTH
and render_h >= MIN_VISIBLE_IMAGE_HEIGHT
and render_area_ratio >= MIN_VISIBLE_IMAGE_AREA_RATIO
)
# Pixel dimension filter
if not visibly_placed and (w < MIN_IMAGE_PIXELS or h < MIN_IMAGE_PIXELS):
return False
# Pixel area filter
area = w * h
if not visibly_placed and area < MIN_IMAGE_AREA:
return False
image_data = block.get("image", b"")
if not visibly_placed and len(image_data) < MIN_IMAGE_BYTES:
return False
# Deduplicate tiny repeats, but preserve visibly placed logos / charts on
# distinct pages. Academic PDFs often reuse a small logo on title pages.
if seen_hashes is not None:
img_hash = hashlib.md5(image_data).hexdigest()
if img_hash in seen_hashes and not visibly_placed:
return False
seen_hashes.add(img_hash)
# Check render size relative to page
page_w = page_rect.width
page_h = page_rect.height
if page_w > 0 and page_h > 0:
if render_w / page_w < MIN_PAGE_RATIO and render_h / page_h < MIN_PAGE_RATIO:
return False
# Filter extreme aspect ratios (decorative bars/separators)
aspect = max(w, h) / max(min(w, h), 1)
if aspect > MAX_ASPECT_RATIO:
return False
# Filter low-info images: solid color blocks / gradients have very high
# compression ratios (low bytes-per-pixel). Only apply to smaller images
# to avoid filtering large photos with dark/uniform backgrounds.
bpp = len(image_data) / area
if bpp < MAX_LOW_INFO_BPP and area < MAX_LOW_INFO_AREA and not visibly_placed:
return False
return True
def _clip_rect_to_page(rect: fitz.Rect, page_rect: fitz.Rect) -> fitz.Rect:
"""Clamp a rectangle to the current PDF page."""
return fitz.Rect(
max(page_rect.x0, rect.x0),
max(page_rect.y0, rect.y0),
min(page_rect.x1, rect.x1),
min(page_rect.y1, rect.y1),
)
def _is_rect_contained(inner: fitz.Rect, outer: fitz.Rect, threshold: float = 0.9) -> bool:
"""Return whether ``inner`` is mostly covered by ``outer``."""
inner_area = inner.get_area()
if inner_area <= 0:
return False
return ((inner & outer).get_area() / inner_area) >= threshold
def _overlaps_table(rect: fitz.Rect, tab_rects: list[fitz.Rect]) -> bool:
"""Skip vector regions that are already handled as extracted tables."""
rect_area = rect.get_area()
if rect_area <= 0:
return False
for tab_rect in tab_rects:
intersection = rect & tab_rect
if intersection.get_area() > 0.5 * min(rect_area, tab_rect.get_area()):
return True
return False
def _is_white(color: tuple[float, ...] | None) -> bool:
"""Return whether a PDF drawing color is visually white."""
if color is None:
return False
return all(channel >= 0.98 for channel in color[:3])
def _is_background_drawing(drawing: dict[str, object]) -> bool:
"""Identify white background rectangles that should not drive crops."""
return _is_white(drawing.get("fill")) and drawing.get("color") is None
def find_figure_caption_rects(page: fitz.Page) -> list[fitz.Rect]:
"""Return text-line rectangles that look like figure captions."""
caption_rects = []
for block in page.get_text("dict")["blocks"]:
if block.get("type") != 0:
continue
for line in block["lines"]:
text = "".join(span["text"] for span in line["spans"]).strip()
if FIGURE_CAPTION_RE.match(text):
caption_rects.append(fitz.Rect(line["bbox"]))
return caption_rects
def _expand_rect(rect: fitz.Rect, padding: float, page_rect: fitz.Rect) -> fitz.Rect:
"""Pad a rectangle and clamp it to the current PDF page."""
return _clip_rect_to_page(
fitz.Rect(
rect.x0 - padding,
rect.y0 - padding,
rect.x1 + padding,
rect.y1 + padding,
),
page_rect,
)
def _union_rects(rects: list[fitz.Rect]) -> fitz.Rect:
"""Return the bounding union for one or more rectangles."""
result = fitz.Rect(rects[0])
for rect in rects[1:]:
result |= rect
return result
def _find_captioned_vector_figures(
page: fitz.Page,
drawing_rects: list[fitz.Rect],
background_rects: list[fitz.Rect],
caption_rects: list[fitz.Rect],
) -> list[fitz.Rect]:
"""Build tight figure crops from non-background drawings above captions."""
figure_rects = []
page_rect = page.rect
for caption_rect in caption_rects:
related = []
for rect in drawing_rects:
horizontal_gap = max(caption_rect.x0 - rect.x1, rect.x0 - caption_rect.x1, 0)
if horizontal_gap > VECTOR_CAPTION_HORIZONTAL_GAP:
continue
if rect.y0 > caption_rect.y0:
continue
if caption_rect.y0 - rect.y1 > VECTOR_CAPTION_SEARCH_HEIGHT:
continue
related.append(rect)
if not related:
continue
content_rect = _union_rects(related)
rect = content_rect
for background_rect in background_rects:
background_rect = _clip_rect_to_page(background_rect, page_rect)
if not _is_rect_contained(content_rect, background_rect, threshold=0.95):
continue
if background_rect.get_area() > content_rect.get_area() * MAX_VECTOR_BACKGROUND_AREA_RATIO:
continue
rect = background_rect
break
rect = _expand_rect(rect, 10, page_rect)
rect.y1 = min(rect.y1, caption_rect.y0 - 2)
if rect.width >= MIN_VECTOR_FIGURE_WIDTH and rect.height >= MIN_VECTOR_FIGURE_HEIGHT:
figure_rects.append(rect)
return figure_rects
def detect_vector_figure_rects(page: fitz.Page, tab_rects: list[fitz.Rect]) -> list[fitz.Rect]:
"""Detect large vector drawing regions that should be rasterized as figures.
Some academic PDFs store charts and diagrams as vector drawing commands,
not image XObjects. ``page.get_text("dict")`` exposes only raster image
blocks, so those figures need a separate drawing-region fallback.
"""
candidates = []
page_rect = page.rect
caption_rects = find_figure_caption_rects(page)
drawing_rects = []
background_rects = []
for drawing in page.get_drawings():
rect = drawing.get("rect")
if not rect:
continue
rect = fitz.Rect(rect)
if rect.is_empty:
continue
if _is_background_drawing(drawing):
background_rects.append(rect)
continue
drawing_rects.append(rect)
if caption_rects:
return _find_captioned_vector_figures(page, drawing_rects, background_rects, caption_rects)
for rect in drawing_rects:
rect = _expand_rect(rect, VECTOR_FIGURE_PADDING, page_rect)
rect = _clip_rect_to_page(rect, page_rect)
width = rect.width
height = rect.height
if width < MIN_VECTOR_FIGURE_WIDTH or height < MIN_VECTOR_FIGURE_HEIGHT:
continue
area = rect.get_area()
if area < MIN_VECTOR_FIGURE_AREA:
continue
aspect = max(width, height) / max(min(width, height), 1)
if aspect > MAX_VECTOR_FIGURE_ASPECT_RATIO:
continue
if _overlaps_table(rect, tab_rects):
continue
candidates.append(rect)
candidates.sort(key=lambda r: r.get_area(), reverse=True)
kept = []
for rect in candidates:
if any(_is_rect_contained(rect, existing) for existing in kept):
continue
kept.append(rect)
return sorted(kept, key=lambda r: (r.y0, r.x0))
def clean_text(text: str) -> str:
"""Clean extracted text."""
lines = text.split('\n')
cleaned_lines = []
prev_empty = False
for line in lines:
line = line.rstrip()
is_empty = len(line.strip()) == 0
if is_empty:
if not prev_empty:
cleaned_lines.append('')
prev_empty = True
else:
cleaned_lines.append(line)
prev_empty = False
return '\n'.join(cleaned_lines)
def _extract_text_lines(page: fitz.Page) -> list[tuple[fitz.Rect, str]]:
"""Return text lines with their page rectangles in reading order."""
lines = []
for block in page.get_text("dict")["blocks"]:
if block.get("type") != 0:
continue
for line in block["lines"]:
text = "".join(span["text"] for span in line["spans"]).strip()
if text:
lines.append((fitz.Rect(line["bbox"]), text))
return sorted(lines, key=lambda item: (item[0].y0, item[0].x0))
def _is_table_caption(text: str) -> bool:
"""Return whether a line is a real table caption, not prose citing a table."""
return bool(TABLE_CAPTION_RE.match(text.strip()))
def _caption_table_start_y(
caption_rect: fitz.Rect,
lines: list[tuple[fitz.Rect, str]],
) -> float:
"""Start below a caption and its adjacent English translation line."""
start_y = caption_rect.y1 + 1
for rect, text in lines:
if rect.y0 < caption_rect.y1 - 1 or rect.y0 > caption_rect.y1 + 35:
continue
if text.startswith("Table"):
start_y = max(start_y, rect.y1 + 1)
return start_y
def _looks_like_numeric_table_line(text: str) -> bool:
"""Detect long data rows so they are not mistaken for prose boundaries."""
numbers = NUMBER_RE.findall(text)
if len(numbers) >= 3:
return True
tokens = [token for token in re.split(r'\s+', text.strip()) if token]
return len(tokens) >= 4 and len(numbers) >= 2
def _is_table_region_boundary(
rect: fitz.Rect,
text: str,
page: fitz.Page,
start_y: float,
) -> bool:
"""Return whether a line likely starts prose after a text-detected table."""
text = text.strip()
if rect.y0 < start_y + 45:
return False
if text.startswith(TABLE_NOTE_PREFIX):
return True
if SECTION_HEADING_RE.match(text):
return True
if TABLE_REFERENCE_PROSE_RE.match(text):
return True
if _looks_like_numeric_table_line(text):
return False
width_ratio = rect.width / page.rect.width if page.rect.width > 0 else 0
return len(text) >= 26 and width_ratio > 0.52 and rect.x0 < page.rect.width * 0.25
def _table_region_bottom(
page: fitz.Page,
lines: list[tuple[fitz.Rect, str]],
start_y: float,
) -> float:
"""Find a conservative bottom edge for a caption-guided text table scan."""
for rect, text in lines:
if rect.y0 <= start_y:
continue
if _is_table_region_boundary(rect, text, page, start_y):
return max(start_y + 20, rect.y0 - 2)
return page.rect.height * TABLE_SCAN_BOTTOM_RATIO
def _normalize_table_cell(value: object) -> str:
"""Normalize one extracted table cell for Markdown output."""
if value is None:
return ""
text = CONTROL_CHARS_RE.sub('', str(value))
text = re.sub(r'\s*\n\s*', '
', text.strip())
text = re.sub(r'[ \t]+', ' ', text)
return text.replace('|', r'\|')
def _clean_table_rows(rows: list[list[object]]) -> list[list[str]]:
"""Remove empty rows / columns from PyMuPDF table extraction output."""
normalized = [[_normalize_table_cell(cell) for cell in row] for row in rows]
normalized = [row for row in normalized if any(cell for cell in row)]
if not normalized:
return []
max_cols = max(len(row) for row in normalized)
padded = [row + [""] * (max_cols - len(row)) for row in normalized]
keep_cols = [
idx
for idx in range(max_cols)
if any(row[idx] for row in padded)
]
if len(keep_cols) < 2:
return []
return _postprocess_table_rows([[row[idx] for idx in keep_cols] for row in padded])
def _nonempty_cell_indexes(row: list[str]) -> list[int]:
"""Return indexes of non-empty cells in a row."""
return [idx for idx, cell in enumerate(row) if cell]
def _merge_label_underscore_rows(rows: list[list[str]]) -> list[list[str]]:
"""Join rows where PDF extraction split a leading underscore from a label."""
merged = []
for row in rows:
nonempty = _nonempty_cell_indexes(row)
if nonempty == [0] and row[0] == "_" and merged and merged[-1][0]:
merged[-1][0] = f"_{merged[-1][0]}"
continue
merged.append(row)
return merged
def _merge_single_cell_continuations(rows: list[list[str]]) -> list[list[str]]:
"""Merge wrapped single-cell table rows into the previous row."""
merged: list[list[str]] = []
for row in rows:
nonempty = _nonempty_cell_indexes(row)
if (
len(nonempty) == 1
and nonempty[0] > 0
and merged
and not MODEL_COLUMN_RE.match(row[nonempty[0]])
):
index = nonempty[0]
separator = "
" if merged[-1][index] else ""
merged[-1][index] = f"{merged[-1][index]}{separator}{row[index]}"
continue
merged.append(row)
return merged
def _looks_like_model_row(row: list[str]) -> bool:
"""Return whether a row contains model-number table headings."""
values = [cell for cell in row[1:] if cell]
return len(values) >= 2 and all(MODEL_COLUMN_RE.match(cell) for cell in values)
def _looks_like_outcome_row(row: list[str]) -> bool:
"""Return whether a row contains regression outcome labels."""
values = [cell for cell in row[1:] if cell]
if len(values) < 2:
return False
short_values = [cell for cell in values if len(cell) <= 12 and not NUMBER_RE.search(cell)]
return len(short_values) == len(values)
def _regression_group_labels(row: list[str], data_cols: int) -> list[str]:
"""Infer repeated group labels for common regression-table headings."""
compact = "".join(row)
if "总样本" in compact and "国有" in compact and "非国有" in compact and data_cols == 7:
return ["总样本"] * 3 + ["国有企业"] * 2 + ["非国有企业"] * 2
return [""] * data_cols
def _flatten_regression_header(rows: list[list[str]]) -> list[list[str]]:
"""Flatten multi-line regression headings into one Markdown header row."""
if len(rows) < 3:
return rows
if (
len(rows) >= 2
and rows[0][0]
and not any(rows[0][1:])
and _looks_like_model_row(rows[1])
):
outcome = rows[0][0]
header = ["变量"]
header.extend(
f"{model} {outcome}".strip()
for model in rows[1][1:]
)
return [header] + rows[2:]
header_offset = 0
groups = [""] * (len(rows[0]) - 1)
if not _looks_like_model_row(rows[0]) and _looks_like_model_row(rows[1]):
header_offset = 1
groups = _regression_group_labels(rows[0], len(rows[1]) - 1)
if not _looks_like_model_row(rows[header_offset]):
return rows
if len(rows) <= header_offset + 1 or not _looks_like_outcome_row(rows[header_offset + 1]):
return rows
model_row = rows[header_offset]
outcome_row = rows[header_offset + 1]
header = ["变量"]
for idx, model in enumerate(model_row[1:]):
pieces = []
if idx < len(groups) and groups[idx]:
pieces.append(groups[idx])
if model:
pieces.append(model)
if idx + 1 < len(outcome_row) and outcome_row[idx + 1]:
pieces.append(outcome_row[idx + 1])
header.append(" ".join(pieces).strip())
return [header] + rows[header_offset + 2:]
def _fix_paired_sample_t_table(rows: list[list[str]]) -> list[list[str]]:
"""Collapse multi-row paired-sample T-test headings into readable columns."""
if not rows or not any("成对差分" in cell for cell in rows[0]):
return rows
body = [row for row in rows if row and row[0].startswith("对")]
if len(body) < 1:
return rows
header = [
"配对",
"变量",
"均值",
"标准差",
"均值的标准误",
"差分95%置信区间下限",
"差分95%置信区间上限",
"t",
"Df",
"Sig.(双侧)",
]
fixed_rows = [header]
for row in body:
fixed_rows.append(row[:len(header)] + [""] * max(0, len(header) - len(row)))
return fixed_rows
def _fix_variable_definition_table(rows: list[list[str]]) -> list[list[str]]:
"""Repeat variable-category labels for common variable definition tables."""
if not rows or rows[0] != ["变量类型", "变量名称", "符号", "变量说明"]:
return rows
fixed = [rows[0]]
for row in rows[1:]:
if not any(row):
continue
name = row[1] if len(row) > 1 else ""
symbol = row[2] if len(row) > 2 else ""
description = row[3] if len(row) > 3 else ""
if not name or not symbol:
continue
if symbol in {"R＆D", "Fixed", "Hc"}:
category = "被解释变量"
elif symbol == "Vat":
category = "解释变量"
else:
category = "控制变量"
fixed.append([category, name, symbol, description])
return fixed
def _fix_correlation_triangle(rows: list[list[str]]) -> list[list[str]]:
"""Restore the missing last self-correlation column in triangular tables."""
if len(rows) < 4 or not rows[0] or rows[0][0] != "变量":
return rows
body_names = [row[0] for row in rows[1:] if row and row[0]]
header_names = rows[0][1:]
if len(body_names) != len(header_names) + 1:
return rows
missing_name = body_names[-1]
fixed = [rows[0] + [missing_name]]
for row in rows[1:-1]:
fixed.append(row + [""])
fixed.append(rows[-1] + ["1"])
return fixed
def _postprocess_table_rows(rows: list[list[str]]) -> list[list[str]]:
"""Apply Markdown-oriented cleanup to extracted table rows."""
rows = _merge_label_underscore_rows(rows)
rows = _merge_single_cell_continuations(rows)
rows = _fix_variable_definition_table(rows)
rows = _fix_paired_sample_t_table(rows)
rows = _flatten_regression_header(rows)
rows = _fix_correlation_triangle(rows)
return rows
def _rows_to_markdown(rows: list[list[str]]) -> str:
"""Convert cleaned table rows to GitHub-flavored Markdown."""
if len(rows) < 2:
return ""
col_count = max(len(row) for row in rows)
padded = [row + [""] * (col_count - len(row)) for row in rows]
header = padded[0]
body = padded[1:]
lines = [
"|" + "|".join(header) + "|",
"|" + "|".join(["---"] * col_count) + "|",
]
lines.extend("|" + "|".join(row) + "|" for row in body)
return "\n".join(lines)
def _table_to_markdown(tab: object) -> str:
"""Convert a PyMuPDF table object to cleaned Markdown."""
try:
rows = tab.extract() or []
except Exception:
return ""
return _rows_to_markdown(_clean_table_rows(rows))
def _is_valid_table_markdown(markdown: str) -> bool:
"""Return whether generated Markdown contains a minimally useful table."""
return markdown.count("\n") >= 2 and markdown.startswith("|")
def _markdown_col_count(markdown: str) -> int:
"""Return the column count implied by the first Markdown table row."""
first_line = markdown.splitlines()[0] if markdown else ""
return max(0, first_line.count("|") - 1)
def _append_table_markdown_candidate(
candidates: list[dict[str, object]],
bbox: fitz.Rect,
markdown: str,
method: str,
replace_narrow: bool = False,
) -> None:
"""Append or replace a table candidate after overlap deduplication."""
if not _is_valid_table_markdown(markdown):
return
for index, candidate in enumerate(candidates):
existing = candidate["bbox"]
if not isinstance(existing, fitz.Rect):
continue
overlap = (bbox & existing).get_area()
if overlap <= 0.8 * min(bbox.get_area(), existing.get_area()):
continue
existing_markdown = str(candidate.get("content", ""))
can_replace = (
replace_narrow
and bbox.width > existing.width * 1.4
and _markdown_col_count(markdown) >= _markdown_col_count(existing_markdown)
)
if can_replace:
candidates[index] = {
"bbox": bbox,
"content": markdown,
"method": method,
}
return
candidates.append({
"bbox": bbox,
"content": markdown,
"method": method,
})
def _add_table_candidate(
candidates: list[dict[str, object]],
tab: object,
method: str,
) -> None:
"""Append a table candidate if it has useful Markdown and is not duplicate."""
markdown = _table_to_markdown(tab)
if not _is_valid_table_markdown(markdown):
return
bbox = fitz.Rect(tab.bbox)
_append_table_markdown_candidate(candidates, bbox, markdown, method)
def _merge_word_runs(words: list[tuple]) -> list[dict[str, object]]:
"""Group PyMuPDF words into row-level text runs."""
rows: list[list[tuple]] = []
for word in sorted(words, key=lambda item: (item[1], item[0])):
if not rows or abs(rows[-1][0][1] - word[1]) > 4:
rows.append([word])
else:
rows[-1].append(word)
runs = []
for row in rows:
row_runs = []
for word in sorted(row, key=lambda item: item[0]):
x0, y0, x1, y1, text = word[:5]
if row_runs and x0 - row_runs[-1]["x1"] <= 8:
row_runs[-1]["x1"] = x1
row_runs[-1]["y0"] = min(row_runs[-1]["y0"], y0)
row_runs[-1]["y1"] = max(row_runs[-1]["y1"], y1)
row_runs[-1]["text"] = f"{row_runs[-1]['text']} {text}"
else:
row_runs.append({
"x0": x0,
"y0": y0,
"x1": x1,
"y1": y1,
"text": text,
})
runs.extend(row_runs)
return runs
def _cluster_word_columns(runs: list[dict[str, object]]) -> list[float]:
"""Infer stable table columns from word-run centers."""
centers = sorted((float(run["x0"]) + float(run["x1"])) / 2 for run in runs)
columns: list[float] = []
for center in centers:
if not columns or abs(center - columns[-1]) > 14:
columns.append(center)
else:
columns[-1] = (columns[-1] + center) / 2
return columns
def _word_runs_to_rows(
runs: list[dict[str, object]],
columns: list[float],
) -> list[list[str]]:
"""Place word runs into inferred columns and return table rows."""
row_groups: list[list[dict[str, object]]] = []
for run in sorted(runs, key=lambda item: (float(item["y0"]), float(item["x0"]))):
if not row_groups or abs(float(row_groups[-1][0]["y0"]) - float(run["y0"])) > 4:
row_groups.append([run])
else:
row_groups[-1].append(run)
rows = []
for group in row_groups:
row = [""] * len(columns)
for run in group:
center = (float(run["x0"]) + float(run["x1"])) / 2
col_index = min(range(len(columns)), key=lambda idx: abs(columns[idx] - center))
text = _normalize_table_cell(run["text"])
row[col_index] = f"{row[col_index]} {text}".strip() if row[col_index] else text
rows.append(row)
return rows
def _words_to_markdown_table(
page: fitz.Page,
clip: fitz.Rect,
) -> tuple[fitz.Rect, str] | None:
"""Build a simple table from word coordinates inside a clipped region."""
words = page.get_text("words", clip=clip)
if len(words) < 6:
return None
runs = _merge_word_runs(words)
if len(runs) < 6:
return None
columns = _cluster_word_columns(runs)
if len(columns) < 3:
return None
rows = _word_runs_to_rows(runs, columns)
cleaned_rows = _clean_table_rows(rows)
markdown = _rows_to_markdown(cleaned_rows)
if not _is_valid_table_markdown(markdown):
return None
x0 = min(float(run["x0"]) for run in runs)
y0 = min(float(run["y0"]) for run in runs)
x1 = max(float(run["x1"]) for run in runs)
y1 = max(float(run["y1"]) for run in runs)
return fitz.Rect(x0, y0, x1, y1), markdown
def _find_tables_in_clip(
page: fitz.Page,
clip: fitz.Rect,
) -> list[object]:
"""Find text-strategy tables inside a clipped page region."""
if clip.height < 20 or clip.width < 80:
return []
try:
return list(page.find_tables(strategy="text", clip=clip))
except Exception:
return []
def find_page_tables(
page: fitz.Page,
include_top_continuation: bool = False,
) -> tuple[list[dict[str, object]], bool]:
"""Find line-detected tables plus caption-guided text tables on one page."""
candidates: list[dict[str, object]] = []
try:
for tab in page.find_tables():
_add_table_candidate(candidates, tab, "lines")
except Exception:
pass
lines = _extract_text_lines(page)
for rect, text in lines:
if not _is_table_caption(text):
continue
start_y = _caption_table_start_y(rect, lines)
bottom_y = _table_region_bottom(page, lines, start_y)
clip = fitz.Rect(0, start_y, page.rect.width, bottom_y)
for tab in _find_tables_in_clip(page, clip):
_add_table_candidate(candidates, tab, "caption-text")
word_table = _words_to_markdown_table(page, clip)
if word_table:
bbox, markdown = word_table
_append_table_markdown_candidate(
candidates,
bbox,
markdown,
"caption-words",
replace_narrow=True,
)
if include_top_continuation:
start_y = page.rect.height * 0.08
bottom_y = _table_region_bottom(page, lines, start_y)
clip = fitz.Rect(0, start_y, page.rect.width, bottom_y)
for tab in _find_tables_in_clip(page, clip):
_add_table_candidate(candidates, tab, "continuation-text")
word_table = _words_to_markdown_table(page, clip)
if word_table:
bbox, markdown = word_table
_append_table_markdown_candidate(
candidates,
bbox,
markdown,
"continuation-words",
replace_narrow=True,
)
candidates.sort(key=lambda candidate: candidate["bbox"].y0)
table_continues = any(
isinstance(candidate["bbox"], fitz.Rect)
and candidate["bbox"].y1 >= page.rect.height * TABLE_CONTINUATION_Y_RATIO
for candidate in candidates
)
return candidates, table_continues
def _is_markdown_table_line(line: str) -> bool:
"""Return whether a Markdown line belongs to a pipe table."""
return line.startswith("|")
def _is_markdown_separator_line(line: str) -> bool:
"""Return whether a Markdown table line is the separator row."""
cells = [cell.strip() for cell in line.strip("|").split("|")]
return bool(cells) and all(cell and set(cell) <= {"-", ":"} for cell in cells)
def _compatible_table_headers(first: list[str], second: list[str]) -> bool:
"""Return whether two Markdown table blocks have the same flattened header."""
if len(first) < 2 or len(second) < 2:
return False
if not _is_markdown_separator_line(first[1]) or not _is_markdown_separator_line(second[1]):
return False
return first[0] == second[0] and _markdown_col_count(first[0]) > 2
def _is_table_continuation_noise(line: str) -> bool:
"""Allow only page/header noise between split table parts."""
text = line.strip()
if not text:
return True
if text.startswith(""):
return True
if re.fullmatch(r'\d+', text):
return True
if "重庆大学硕士学位论文" in text:
return True
continuation_labels = [
"营改增",
"深化增值税改革",
"国有企业",
"非国有企业",
]
return any(label in text for label in continuation_labels)
def _read_markdown_table_block(lines: list[str], start: int) -> tuple[list[str], int]:
"""Read a contiguous Markdown table block from ``start``."""
end = start
while end < len(lines) and _is_markdown_table_line(lines[end]):
end += 1
return lines[start:end], end
def merge_markdown_continuation_tables(markdown: str) -> str:
"""Merge split cross-page Markdown tables with repeated headers."""
lines = markdown.splitlines()
result = []
index = 0
while index < len(lines):
if not _is_markdown_table_line(lines[index]):
result.append(lines[index])
index += 1
continue
table, table_end = _read_markdown_table_block(lines, index)
search = table_end
while True:
between_start = search
while search < len(lines) and not _is_markdown_table_line(lines[search]):
if not _is_table_continuation_noise(lines[search]):
break
search += 1
if search >= len(lines) or not _is_markdown_table_line(lines[search]):
break
if any(
not _is_table_continuation_noise(line)
for line in lines[between_start:search]
):
break
next_table, next_end = _read_markdown_table_block(lines, search)
if not _compatible_table_headers(table, next_table):
break
table.extend(next_table[2:])
search = next_end
result.extend(table)
index = search if search != table_end else table_end
return "\n".join(result)
def merge_adjacent_formatting(text: str) -> str:
"""Merge adjacent same-style formatted spans split across PDF tokens.
PyMuPDF often emits a phrase as several spans, so per-span wrapping in
``format_span_text`` produces ``**X****Y**`` (bold) or ``***X******Y***``
(bold-italic) where one phrase is intended. Collapse the abutting markers
so the run reads as a single phrase: ``**X Y**`` / ``***X Y***``.
Italic-italic adjacency (``*X**Y*``) is indistinguishable from a plain
bold span's open/close pair and is left alone — merging it would corrupt
every legitimate ``**bold**`` phrase on the page. The previous regexes
``\\*\\s*\\*`` and ``\\*\\*\\*\\s*\\*\\*\\*`` did exactly that, deleting
all bold formatting from the converted Markdown.
"""
# Bold-italic adjacency first so the inner ``******`` isn't half-eaten
# by the bold pass.
text = re.sub(r'\*{6}', ' ', text)
text = re.sub(r'\*{4}', ' ', text)
return text
def is_sentence_end(text: str) -> bool:
"""Check if the text ends with sentence-ending punctuation."""
text = text.rstrip()
if not text:
return True
end_puncts = '.。!！?？:：;；'
return text[-1] in end_puncts
def should_merge_lines(current: dict, next_line: dict) -> bool:
"""Determine if two lines should be merged into the same paragraph."""
if current.get("is_heading") or next_line.get("is_heading"):
return False
if current.get("is_list") or next_line.get("is_list"):
return False
if is_sentence_end(current.get("content", "")):
return False
return True
def extract_pdf_to_markdown(
pdf_path: str,
output_path: str = None,
images: str = "filtered",
render_vector_figures: bool = False,
vector_figure_dpi: int = VECTOR_FIGURE_DPI,
) -> str:
"""Extract text, images, and tables from a PDF and convert to Markdown.
Args:
pdf_path: Path to the PDF file.
output_path: Optional output path for the Markdown file.
images: Image extraction mode.
"filtered" = apply size/quality filters (default),
"all" = extract all images without filtering,
"none" = skip all images.
render_vector_figures: Rasterize large vector drawing regions as PNGs.
vector_figure_dpi: DPI used for rendered vector figure PNGs.
"""
try:
doc = fitz.open(pdf_path)
except Exception as e:
print(f"[ERROR] Failed to open PDF file: {e}")
return ""
if len(doc) >= 200:
print(f"[HINT] {len(doc)} pages — for very large PDFs, consider splitting "
f"the source by chapter beforehand (e.g. with pdftk / qpdf / PyPDF2) "
f"and converting each part individually.")
filename = Path(pdf_path).stem
title = re.sub(r'^\d+-', '', filename).strip()
print(f"[INFO] Analyzing document structure...")
size_map = analyze_font_sizes(doc)
print(f" Font size mapping: body={size_map.get('body', 'N/A')}, " +
f"H1={size_map.get('h1', 'N/A')}, H2={size_map.get('h2', 'N/A')}, H3={size_map.get('h3', 'N/A')}")
print(f"[INFO] Detecting repeated headers/footers...")
noise_texts = detect_headers_footers(doc)
if noise_texts:
print(f" Found {len(noise_texts)} repeated noise texts (will be removed):")
for t in list(noise_texts)[:3]:
print(f" - {t[:30]}...")
markdown_content = f"# {title}\n\n"
seen_image_hashes = set() # Track seen image hashes for deduplication
img_dir = None
rel_img_dir = None
if output_path:
output_path = Path(output_path)
rel_img_dir = f"{output_path.stem}_files"
img_dir = output_path.parent / rel_img_dir
img_count = 0
image_manifest: list[dict[str, object]] = []
previous_table_continues = False
for page_num, page in enumerate(doc, 1):
if page_num > 1:
# Add page break marker to help LLM understand context segmentation
markdown_content += f"\n\n\n\n"
table_candidates, previous_table_continues = find_page_tables(
page,
include_top_continuation=previous_table_continues,
)
tab_rects = [
candidate["bbox"]
for candidate in table_candidates
if isinstance(candidate["bbox"], fitz.Rect)
]
page_elements = []
for table in table_candidates:
bbox = table["bbox"]
if not isinstance(bbox, fitz.Rect):
continue
page_elements.append({
"y0": bbox.y0,
"type": 2,
"content": table["content"]
})
print(f" [OK] Found table: P{page_num} ({table['method']})")
if render_vector_figures:
for figure_rect in detect_vector_figure_rects(page, tab_rects):
page_elements.append({
"y0": figure_rect.y0,
"type": 3,
"content": figure_rect,
})
print(f" [OK] Found vector figure region: P{page_num} {tuple(round(v, 1) for v in figure_rect)}")
blocks = page.get_text("dict")["blocks"]
for block in blocks:
block_rect = fitz.Rect(block["bbox"])
# Check if this is table content
is_in_table = False
for tab_rect in tab_rects:
intersect = block_rect & tab_rect
if intersect.get_area() > 0.6 * block_rect.get_area():
is_in_table = True
break
if is_in_table:
continue
if block["type"] == 0:
# Check if this is noise text to be filtered (whole block match)
block_text_full = "".join([span["text"] for line in block["lines"] for span in line["spans"]]).strip()
if block_text_full in noise_texts:
continue
for line in block["lines"]:
line_text = ""
line_size = 0
line_flags = 0
span_count = 0
is_code_line = False
formatted_spans = []
for span in line["spans"]:
span_text = CONTROL_CHARS_RE.sub('', span["text"])
if not span_text.strip():
if span_text:
formatted_spans.append(span_text)
continue
span_size = span["size"]
span_flags = span["flags"]
line_size = max(line_size, span_size)
line_flags |= span_flags
span_count += 1
heading_level = get_heading_level(span_size, size_map, span_text, span_flags)
# Detect code font
font_name = span.get("font", "")
if is_monospace_font(font_name):
is_code_line = True
formatted_spans.append(span_text) # No formatting for code
elif heading_level > 0:
formatted_spans.append(span_text.strip())
else:
formatted_spans.append(format_span_text(span_text, span_flags))
line_text = ''.join(formatted_spans).strip()
if not line_text:
continue
# Secondary check: line-level noise match (sometimes blocks are split)
if line_text in noise_texts:
continue
line_text = merge_adjacent_formatting(line_text)
heading_level = get_heading_level(line_size, size_map, line_text, line_flags)
is_list, list_type, list_content = detect_list_item(line_text)
if heading_level > 0:
prefix = '#' * heading_level + ' '
clean_line = re.sub(r'\*+([^*]+)\*+', r'\1', line_text)
final_text = prefix + clean_line
elif is_list:
final_text = list_content
else:
final_text = line_text
page_elements.append({
"y0": line["bbox"][1],
"type": 0,
"content": final_text,
"is_heading": heading_level > 0,
"is_list": is_list,
"is_code": is_code_line
})
elif block["type"] == 1:
if images == "none":
pass
elif images == "all" or should_keep_image(block, page.rect, seen_image_hashes):
page_elements.append({
"y0": block["bbox"][1],
"type": 1,
"content": block
})
else:
w, h = block.get("width", 0), block.get("height", 0)
print(f" [SKIP] Filtered small/decorative image: {w}x{h}px, {len(block.get('image', b''))} bytes")
page_elements.sort(key=lambda x: x["y0"])
# Merge adjacent same-level short headings
page_elements = merge_adjacent_headings(page_elements)
merged_elements = []
i = 0
while i < len(page_elements):
el = page_elements[i]
if el["type"] == 0 and not el.get("is_heading") and not el.get("is_list"):
merged_content = el["content"]
j = i + 1
while j < len(page_elements):
next_el = page_elements[j]
if next_el["type"] != 0:
break
if not should_merge_lines({"content": merged_content, "is_heading": False, "is_list": False}, next_el):
break
merged_content += " " + next_el["content"]
j += 1
merged_elements.append({
"type": 0,
"content": remove_page_footer(merged_content),
"is_heading": False,
"is_list": False
})
i = j
else:
merged_elements.append(el)
i += 1
prev_was_list = False
prev_was_code = False
code_block_lines = []
def flush_code_block():
"""Flush accumulated code block."""
nonlocal code_block_lines, markdown_content
if code_block_lines:
markdown_content += "```\n"
markdown_content += "\n".join(code_block_lines) + "\n"
markdown_content += "```\n\n"
code_block_lines = []
for el in merged_elements:
if el["type"] == 0:
is_list = el.get("is_list", False)
is_heading = el.get("is_heading", False)
is_code = el.get("is_code", False)
if is_code:
# Accumulate code lines
if prev_was_list:
markdown_content += "\n"
prev_was_list = False
code_block_lines.append(el["content"])
prev_was_code = True
else:
# Non-code line, flush accumulated code block first
if prev_was_code:
flush_code_block()
prev_was_code = False
if is_heading:
if prev_was_list:
markdown_content += "\n"
markdown_content += el["content"] + "\n\n"
prev_was_list = False
elif is_list:
markdown_content += el["content"] + "\n"
prev_was_list = True
else:
if prev_was_list:
markdown_content += "\n"
markdown_content += el["content"] + "\n\n"
prev_was_list = False
elif el["type"] == 2:
if prev_was_code:
flush_code_block()
prev_was_code = False
if prev_was_list:
markdown_content += "\n"
markdown_content += el["content"] + "\n\n"
prev_was_list = False
elif el["type"] == 1:
if prev_was_code:
flush_code_block()
prev_was_code = False
if img_dir:
block = el["content"]
ext = block["ext"]
image_data = block["image"]
safe_filename = filename.replace(" ", "_")
image_name = f"{safe_filename}_p{page_num}_{img_count}.{ext}"
image_path = img_dir / image_name
try:
img_dir.mkdir(parents=True, exist_ok=True)
with open(image_path, "wb") as f:
f.write(image_data)
if prev_was_list:
markdown_content += "\n"
markdown_content += f"\n\n"
width = int(block.get("width", 0) or 0)
height = int(block.get("height", 0) or 0)
ratio = width / height if width > 0 and height > 0 else None
image_manifest.append({
"index": len(image_manifest) + 1,
"filename": image_name,
"original_filename": image_name,
"asset_kind": "bitmap",
"svg_renderable": True,
"pptx_native_supported": True,
"source_kind": "pdf_image",
"source_ext": f".{ext}",
"page_index": page_num,
"occurrence_index": img_count + 1,
"pixel_width": width or None,
"pixel_height": height or None,
"pixel_ratio": round(ratio, 6) if ratio else None,
"display_ratio": round(ratio, 6) if ratio else None,
"source_sha256": hashlib.sha256(image_data).hexdigest(),
"bbox": list(block.get("bbox", [])),
})
img_count += 1
prev_was_list = False
print(f" [OK] Extracted image: {image_name}")
except Exception as e:
print(f" [WARN] Failed to save image: {e}")
elif el["type"] == 3:
if prev_was_code:
flush_code_block()
prev_was_code = False
if img_dir:
figure_rect = el["content"]
safe_filename = filename.replace(" ", "_")
image_name = f"{safe_filename}_p{page_num}_figure_{img_count}.png"
image_path = img_dir / image_name
try:
img_dir.mkdir(parents=True, exist_ok=True)
scale = vector_figure_dpi / 72
pix = page.get_pixmap(
matrix=fitz.Matrix(scale, scale),
clip=figure_rect,
alpha=False,
)
pix.save(str(image_path))
if prev_was_list:
markdown_content += "\n"
markdown_content += f"\n\n"
ratio = pix.width / pix.height if pix.width > 0 and pix.height > 0 else None
image_manifest.append({
"index": len(image_manifest) + 1,
"filename": image_name,
"original_filename": image_name,
"asset_kind": "bitmap",
"svg_renderable": True,
"pptx_native_supported": True,
"source_kind": "pdf_vector_figure",
"source_ext": ".png",
"page_index": page_num,
"occurrence_index": img_count + 1,
"pixel_width": pix.width,
"pixel_height": pix.height,
"pixel_ratio": round(ratio, 6) if ratio else None,
"display_ratio": round(ratio, 6) if ratio else None,
"bbox": [
figure_rect.x0,
figure_rect.y0,
figure_rect.x1,
figure_rect.y1,
],
})
img_count += 1
prev_was_list = False
print(f" [OK] Rendered vector figure: {image_name}")
except Exception as e:
print(f" [WARN] Failed to render vector figure: {e}")
# Flush code block at end of page
if prev_was_code:
flush_code_block()
doc.close()
markdown_content = merge_markdown_continuation_tables(markdown_content)
markdown_content = CONTROL_CHARS_RE.sub('', markdown_content)
markdown_content = re.sub(r'\n{3,}', '\n\n', markdown_content)
markdown_content = markdown_content.strip() + "\n"
if output_path:
os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True)
with open(output_path, 'w', encoding='utf-8') as f:
f.write(markdown_content)
if img_dir and image_manifest:
(img_dir / "image_manifest.json").write_text(
json.dumps(image_manifest, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
profile_path = write_conversion_profile_best_effort(
input_path=pdf_path,
markdown_path=output_path,
converter="pdf_to_md.py",
conversion_type="pdf",
asset_dir=img_dir,
)
print(f"[OK] Saved Markdown to: {output_path}")
if profile_path:
print(f" Wrote conversion profile -> {profile_path}")
return markdown_content
def main() -> int:
"""Run the CLI entry point."""
parser = argparse.ArgumentParser(
description='PDF to Markdown converter (with structure detection and LLM optimization)',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog='''
Examples:
python pdf_to_md.py book.pdf # Convert a single file
python pdf_to_md.py book.pdf appendix.pdf # Convert multiple files
python pdf_to_md.py ./pdfs -o ./markdown # Convert PDFs in a directory
python pdf_to_md.py book.pdf -o output.md # Specify output file
python pdf_to_md.py book.pdf --render-vector-figures
Structure detection features:
- Auto-detect heading levels (based on font size)
- Detect bold and italic text
- Detect ordered and unordered lists
- Extract tables and convert to Markdown format (with deduplication)
- [New] Smart detection and removal of repeated page headers/footers
- [New] Add page break markers to help LLM understanding
'''
)
parser.add_argument('inputs', nargs='+', help='PDF file(s) or directories')
parser.add_argument(
'-o',
'--output',
help='Output Markdown file for one input, or output directory for multiple inputs/directories',
)
parser.add_argument(
'--images',
choices=['all', 'filtered', 'none'],
default='filtered',
help='Image extraction mode: filtered=apply size/quality filters (default), all=no filtering, none=skip images',
)
parser.add_argument(
'--render-vector-figures',
action='store_true',
help='Render large vector drawing regions as PNG figure assets',
)
parser.add_argument(
'--vector-figure-dpi',
type=int,
default=VECTOR_FIGURE_DPI,
help=f'DPI for --render-vector-figures output (default: {VECTOR_FIGURE_DPI})',
)
args = parser.parse_args()
return run_path_batch(
args.inputs,
{'.pdf'},
args.output,
lambda source, output: bool(
extract_pdf_to_markdown(
str(source),
str(output),
images=args.images,
render_vector_figures=args.render_vector_figures,
vector_figure_dpi=args.vector_figure_dpi,
)
),
)
if __name__ == '__main__':
raise SystemExit(main())