add-api-extract-pdf-content

PdfExtract.py

@@ -0,0 +1,107 @@
+import os
+import uuid
+import tempfile
+import requests
+from minio import Minio
+from pdf2image import convert_from_path
+import pytesseract
+import concurrent.futures
+import PyPDF2
+
+# ================= CONFIG =================
+MINIO_ENDPOINT = "192.168.10.76:9000"
+MINIO_ACCESS = "minioadmin"
+MINIO_SECRET = "Admin@!@2025"
+BUCKET_NAME = "pdf-storage"
+
+POPPLER_PATH = r"C:\Program Files\poppler\Library\bin"  # Ubuntu = None
+MAX_FILE_SIZE_MB = 20
+
+# ================= INIT MINIO =================
+minio_client = Minio(
+    MINIO_ENDPOINT,
+    access_key=MINIO_ACCESS,
+    secret_key=MINIO_SECRET,
+    secure=False
+)
+
+if not minio_client.bucket_exists(BUCKET_NAME):
+    minio_client.make_bucket(BUCKET_NAME)
+
+
+# ================= HELPERS =================
+
+def pdf_has_text(file_path):
+    reader = PyPDF2.PdfReader(file_path)
+    for page in reader.pages:
+        if page.extract_text():
+            return True
+    return False
+
+
+def ocr_page(page):
+    return pytesseract.image_to_string(page, config="--psm 6")
+
+
+def process_pdf(url):
+    response = requests.get(url, timeout=30)
+
+    if response.status_code != 200:
+        raise Exception("Download failed")
+
+    if "application/pdf" not in response.headers.get("Content-Type", ""):
+        raise Exception("Not a valid PDF")
+
+    if len(response.content) > MAX_FILE_SIZE_MB * 1024 * 1024:
+        raise Exception("File too large")
+
+    return _process_bytes(response.content)
+
+
+def process_uploaded_pdf(file_bytes):
+    return _process_bytes(file_bytes)
+
+
+def _process_bytes(file_bytes):
+    file_id = str(uuid.uuid4())
+
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp:
+        tmp.write(file_bytes)
+        local_pdf = tmp.name
+
+    # Upload PDF
+    minio_client.fput_object(
+        BUCKET_NAME,
+        f"{file_id}.pdf",
+        local_pdf,
+        content_type="application/pdf"
+    )
+
+    # Detect digital PDF
+    if pdf_has_text(local_pdf):
+        reader = PyPDF2.PdfReader(local_pdf)
+        full_text = ""
+        for page in reader.pages:
+            full_text += page.extract_text() or ""
+        source_type = "digital_pdf"
+    else:
+        pages = convert_from_path(
+            local_pdf,
+            dpi=300,
+            poppler_path=POPPLER_PATH
+        )
+
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            results = executor.map(ocr_page, pages)
+
+        full_text = "\n".join(results)
+        source_type = "scanned_pdf"
+
+    os.remove(local_pdf)
+
+    return {
+        "file_id": file_id,
+        "type": source_type,
+        "text_length": len(full_text),
+        "text": full_text
+    }