mirror of
https://github.com/github/awesome-copilot.git
synced 2026-07-18 11:50:00 +00:00
chore: publish from main
This commit is contained in:
github-actions[bot]
@@ -0,0 +1,321 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Convert PDF documents to Markdown using Microsoft's MarkItDown, with
|
||||
embedded images extracted to real files via PyMuPDF (MarkItDown's PDF
|
||||
converter only extracts text/tables -- it does not detect or emit anything
|
||||
for embedded images at all).
|
||||
|
||||
Usage:
|
||||
python convert_pdf_to_md.py <input> [-o OUTPUT] [--recursive]
|
||||
|
||||
<input> may be either:
|
||||
- a path to a single .pdf file, or
|
||||
- a path to a directory (batch mode: every .pdf file directly inside it
|
||||
is converted; pass --recursive to also descend into subdirectories).
|
||||
|
||||
Output:
|
||||
For each source .pdf (named "<name>.pdf"), a folder is created containing
|
||||
the Markdown and its images, in this layout:
|
||||
|
||||
<name>/
|
||||
img/
|
||||
page001_img001.<ext>
|
||||
page001_img002.<ext>
|
||||
page002_img001.<ext>
|
||||
...
|
||||
<name>.md
|
||||
|
||||
IMPORTANT: MarkItDown's PDF text extraction does not preserve reliable
|
||||
per-page markers in the returned Markdown (pages are simply joined
|
||||
together, or in some cases returned as a single unmarked block of text).
|
||||
That means there is no safe way to know exactly where, inline, an image
|
||||
should go. Rather than guess and risk misplacing an image next to the
|
||||
wrong paragraph, this script appends a clearly labeled "## Extracted
|
||||
Images" section at the end of the Markdown, with a "### Page N"
|
||||
subheading per page that contains images. This is a deliberate, honest
|
||||
tradeoff -- read the images section separately from the main body text.
|
||||
|
||||
- Single file mode: the "<name>/" folder is created next to the source
|
||||
file, or at -o/--output (treated as the exact destination folder) if
|
||||
given.
|
||||
- Batch/directory mode: a "<name>/" folder is created next to each source
|
||||
file, or under -o/--output (treated as a parent directory, created if
|
||||
missing) if given, preserving relative subfolder structure when
|
||||
--recursive is used.
|
||||
- If a document has no embedded images, no "img/" folder or "Extracted
|
||||
Images" section is created.
|
||||
|
||||
Exit codes:
|
||||
0 - all requested conversions succeeded
|
||||
1 - one or more conversions failed (partial success in batch mode)
|
||||
2 - a required dependency ("markitdown" or "pymupdf") is not installed
|
||||
3 - invalid input (path not found, or single-file input is not .pdf)
|
||||
"""
|
||||
import argparse
|
||||
import sys
|
||||
import hashlib
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
EXIT_OK = 0
|
||||
EXIT_CONVERSION_FAILED = 1
|
||||
EXIT_MISSING_DEPENDENCY = 2
|
||||
EXIT_INVALID_INPUT = 3
|
||||
|
||||
|
||||
def _import_markitdown():
|
||||
"""Import MarkItDown, failing with a clear, actionable message if absent."""
|
||||
try:
|
||||
from markitdown import MarkItDown
|
||||
return MarkItDown
|
||||
except ImportError:
|
||||
print(
|
||||
"ERROR: The 'markitdown' package is not installed.\n"
|
||||
"See references/setup.md for this skill, or run:\n"
|
||||
' pip install "markitdown[pdf]"',
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(EXIT_MISSING_DEPENDENCY)
|
||||
|
||||
|
||||
def _import_fitz():
|
||||
"""Import PyMuPDF (module name 'fitz'), failing with a clear message if absent."""
|
||||
try:
|
||||
import fitz
|
||||
import hashlib
|
||||
return fitz
|
||||
except ImportError:
|
||||
print(
|
||||
"ERROR: The 'pymupdf' package is not installed (needed for image "
|
||||
"extraction).\nSee references/setup.md for this skill, or run:\n"
|
||||
" pip install pymupdf",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(EXIT_MISSING_DEPENDENCY)
|
||||
|
||||
|
||||
def extract_images(fitz, pdf_path: Path, img_dir: Path):
|
||||
"""Extract embedded images from pdf_path, grouped by 1-based page number.
|
||||
Returns {page_num: [filename, ...]} in per-page image order. Files are
|
||||
named 'page{P:03d}_img{N:03d}.<ext>'. Corrupt/unreadable images are
|
||||
skipped with a warning rather than aborting the whole conversion.
|
||||
|
||||
Two sources are combined and deduplicated:
|
||||
1. Image XObjects via page.get_images(full=True) -- covers most embedded
|
||||
images in modern PDFs.
|
||||
2. Inline image blocks via page.get_text("dict") -- covers images stored
|
||||
directly in the page content stream, which get_images() misses entirely.
|
||||
Deduplication is by image bytes hash so the same raster is never written twice
|
||||
on the same page regardless of which source reported it."""
|
||||
written_by_page = {}
|
||||
try:
|
||||
doc = fitz.open(str(pdf_path))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"WARNING: could not open {pdf_path} for image extraction: {exc}", file=sys.stderr)
|
||||
return written_by_page
|
||||
|
||||
try:
|
||||
for page_index in range(len(doc)):
|
||||
page = doc[page_index]
|
||||
page_label = page_index + 1
|
||||
seen_hashes: set = set()
|
||||
raw_images: list[tuple[bytes, str]] = [] # (image_bytes, ext)
|
||||
|
||||
# --- Source 1: XObject images ---
|
||||
try:
|
||||
xobjects = page.get_images(full=True)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(
|
||||
f"WARNING: failed to enumerate XObject images on page {page_label} "
|
||||
f"of {pdf_path}: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
xobjects = []
|
||||
|
||||
for img in xobjects:
|
||||
xref = img[0]
|
||||
try:
|
||||
base_image = doc.extract_image(xref)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(
|
||||
f"WARNING: failed to extract XObject image xref={xref} on page "
|
||||
f"{page_label} of {pdf_path}: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
continue
|
||||
img_bytes = base_image.get("image") or b""
|
||||
if not img_bytes:
|
||||
continue
|
||||
ext = (base_image.get("ext") or "png").lower()
|
||||
raw_images.append((img_bytes, ext))
|
||||
|
||||
# --- Source 2: Inline images via get_text("dict") ---
|
||||
try:
|
||||
blocks = page.get_text("dict", flags=fitz.TEXT_PRESERVE_IMAGES).get("blocks", [])
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(
|
||||
f"WARNING: failed to extract text/image dict on page {page_label} "
|
||||
f"of {pdf_path}: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
blocks = []
|
||||
|
||||
for block in blocks:
|
||||
# Image blocks have type == 1
|
||||
if block.get("type") != 1:
|
||||
continue
|
||||
img_bytes = block.get("image") or b""
|
||||
if not img_bytes:
|
||||
continue
|
||||
# Derive extension from the block's "ext" key (fitz sets this)
|
||||
ext = (block.get("ext") or "png").lower()
|
||||
raw_images.append((img_bytes, ext))
|
||||
|
||||
# --- Write deduplicated images ---
|
||||
page_files = []
|
||||
img_idx = 1
|
||||
for img_bytes, ext in raw_images:
|
||||
h = hashlib.sha256(img_bytes).digest()
|
||||
if h in seen_hashes:
|
||||
continue
|
||||
seen_hashes.add(h)
|
||||
out_name = f"page{page_label:03d}_img{img_idx:03d}.{ext}"
|
||||
img_dir.mkdir(parents=True, exist_ok=True)
|
||||
(img_dir / out_name).write_bytes(img_bytes)
|
||||
page_files.append(out_name)
|
||||
img_idx += 1
|
||||
|
||||
if page_files:
|
||||
written_by_page[page_label] = page_files
|
||||
finally:
|
||||
doc.close()
|
||||
|
||||
return written_by_page
|
||||
|
||||
|
||||
def build_image_appendix(written_by_page) -> str:
|
||||
"""Build the '## Extracted Images' appendix text. Returns "" if empty."""
|
||||
if not written_by_page:
|
||||
return ""
|
||||
lines = ["", "## Extracted Images", ""]
|
||||
for page_num in sorted(written_by_page):
|
||||
lines.append(f"### Page {page_num}")
|
||||
lines.append("")
|
||||
for name in written_by_page[page_num]:
|
||||
lines.append(f"")
|
||||
lines.append("")
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def convert_one(md, fitz, source: Path, dest_dir: Path) -> bool:
|
||||
"""Convert a single .pdf file to a '<name>/' folder containing the
|
||||
Markdown file and an 'img/' folder of extracted images. Returns True on
|
||||
success."""
|
||||
try:
|
||||
result = md.convert(str(source))
|
||||
except Exception as exc: # noqa: BLE001 - surface any conversion error
|
||||
print(f"FAILED {source} -> {exc}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
try:
|
||||
if dest_dir.exists():
|
||||
shutil.rmtree(dest_dir)
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
written_by_page = extract_images(fitz, source, dest_dir / "img")
|
||||
appendix = build_image_appendix(written_by_page)
|
||||
text = result.text_content.rstrip("\n")
|
||||
full_text = f"{text}\n{appendix}" if appendix else f"{text}\n"
|
||||
md_path = dest_dir / f"{source.stem}.md"
|
||||
md_path.write_text(full_text, encoding="utf-8")
|
||||
except OSError as exc:
|
||||
print(f"FAILED {source} -> could not write output in {dest_dir}: {exc}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
img_count = sum(len(v) for v in written_by_page.values())
|
||||
img_note = f", {img_count} image(s)" if img_count else ""
|
||||
print(f"OK {source} -> {md_path}{img_note}")
|
||||
return True
|
||||
|
||||
|
||||
def find_pdf_files(root: Path, recursive: bool):
|
||||
"""Return (pdf_files, skipped_count) for files directly/recursively under root."""
|
||||
pattern_iter = root.rglob("*") if recursive else root.iterdir()
|
||||
pdf_files = []
|
||||
skipped = 0
|
||||
for entry in pattern_iter:
|
||||
if entry.is_dir():
|
||||
continue
|
||||
if entry.suffix.lower() == ".pdf":
|
||||
pdf_files.append(entry)
|
||||
else:
|
||||
skipped += 1
|
||||
return sorted(pdf_files), skipped
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("input", help="Path to a .pdf file or a directory of .pdf files")
|
||||
parser.add_argument(
|
||||
"-o", "--output",
|
||||
help=(
|
||||
"Destination folder for the '<name>/' output (single-file mode), "
|
||||
"or parent directory under which each '<name>/' output folder is "
|
||||
"created (batch mode)"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--recursive", action="store_true",
|
||||
help="When input is a directory, also search subdirectories",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
#MarkItDown = _import_markitdown()
|
||||
#fitz = _import_fitz()
|
||||
#md = MarkItDown()
|
||||
|
||||
source = Path(args.input)
|
||||
if not source.exists():
|
||||
print(f"ERROR: Input path not found: {source}", file=sys.stderr)
|
||||
return EXIT_INVALID_INPUT
|
||||
|
||||
if source.is_file() and source.suffix.lower() != ".pdf":
|
||||
print(
|
||||
f"ERROR: Unsupported file type '{source.suffix}'. "
|
||||
"This skill only converts .pdf files.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return EXIT_INVALID_INPUT
|
||||
|
||||
MarkItDown = _import_markitdown()
|
||||
fitz = _import_fitz()
|
||||
md = MarkItDown()
|
||||
|
||||
if source.is_file():
|
||||
dest_dir = Path(args.output) if args.output else source.parent / source.stem
|
||||
return EXIT_OK if convert_one(md, fitz, source, dest_dir) else EXIT_CONVERSION_FAILED
|
||||
|
||||
# Directory / batch mode
|
||||
pdf_files, skipped = find_pdf_files(source, args.recursive)
|
||||
if skipped:
|
||||
print(f"NOTE: skipped {skipped} non-.pdf file(s) in {source}")
|
||||
if not pdf_files:
|
||||
print(f"ERROR: No .pdf files found under {source}", file=sys.stderr)
|
||||
return EXIT_INVALID_INPUT
|
||||
|
||||
out_dir = Path(args.output) if args.output else None
|
||||
success_count = 0
|
||||
for pdf_path in pdf_files:
|
||||
if out_dir is not None:
|
||||
rel = pdf_path.relative_to(source)
|
||||
dest_dir = out_dir / rel.parent / pdf_path.stem
|
||||
else:
|
||||
dest_dir = pdf_path.parent / pdf_path.stem
|
||||
if convert_one(md, fitz, pdf_path, dest_dir):
|
||||
success_count += 1
|
||||
|
||||
total = len(pdf_files)
|
||||
print(f"\nConverted {success_count}/{total} file(s).")
|
||||
return EXIT_OK if success_count == total else EXIT_CONVERSION_FAILED
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,2 @@
|
||||
markitdown[pdf]>=0.1.0
|
||||
pymupdf>=1.24.0
|
||||
Reference in New Issue
Block a user