diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0c0f5d8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ +# Virtual environment +myenv/ +venv/ +.env/ + +# Python cache +__pycache__/ +*.pyc +*.pyo +*.pyd + +# VSCode +.vscode/ + +# PyCharm +.idea/ + +# Environment variables +.env + +# MacOS +.DS_Store \ No newline at end of file diff --git a/MinioExtract.py b/MinioExtract.py new file mode 100644 index 0000000..d3b50fa --- /dev/null +++ b/MinioExtract.py @@ -0,0 +1,9 @@ +from pdf2image import convert_from_path +import pytesseract + +pages = convert_from_path("images/sample.pdf", dpi=300) + +for i, page in enumerate(pages): + text = pytesseract.image_to_string(page, config="--psm 6") + print(f"--- Page {i+1} ---") + print(text) \ No newline at end of file diff --git a/PdfExtract.py b/PdfExtract.py new file mode 100644 index 0000000..bb8fd3a --- /dev/null +++ b/PdfExtract.py @@ -0,0 +1,107 @@ +import os +import uuid +import tempfile +import requests +from minio import Minio +from pdf2image import convert_from_path +import pytesseract +import concurrent.futures +import PyPDF2 + +# ================= CONFIG ================= +MINIO_ENDPOINT = "192.168.10.76:9000" +MINIO_ACCESS = "minioadmin" +MINIO_SECRET = "Admin@!@2025" +BUCKET_NAME = "pdf-storage" + +POPPLER_PATH = r"C:\Program Files\poppler\Library\bin" # Ubuntu = None +MAX_FILE_SIZE_MB = 20 + +# ================= INIT MINIO ================= +minio_client = Minio( + MINIO_ENDPOINT, + access_key=MINIO_ACCESS, + secret_key=MINIO_SECRET, + secure=False +) + +if not minio_client.bucket_exists(BUCKET_NAME): + minio_client.make_bucket(BUCKET_NAME) + + +# ================= HELPERS ================= + +def pdf_has_text(file_path): + reader = PyPDF2.PdfReader(file_path) + for page in reader.pages: + if page.extract_text(): + return True + return False + + +def ocr_page(page): + return pytesseract.image_to_string(page, config="--psm 6") + + +def process_pdf(url): + response = requests.get(url, timeout=30) + + if response.status_code != 200: + raise Exception("Download failed") + + if "application/pdf" not in response.headers.get("Content-Type", ""): + raise Exception("Not a valid PDF") + + if len(response.content) > MAX_FILE_SIZE_MB * 1024 * 1024: + raise Exception("File too large") + + return _process_bytes(response.content) + + +def process_uploaded_pdf(file_bytes): + return _process_bytes(file_bytes) + + +def _process_bytes(file_bytes): + file_id = str(uuid.uuid4()) + + with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp: + tmp.write(file_bytes) + local_pdf = tmp.name + + # Upload PDF + minio_client.fput_object( + BUCKET_NAME, + f"{file_id}.pdf", + local_pdf, + content_type="application/pdf" + ) + + # Detect digital PDF + if pdf_has_text(local_pdf): + reader = PyPDF2.PdfReader(local_pdf) + full_text = "" + for page in reader.pages: + full_text += page.extract_text() or "" + source_type = "digital_pdf" + else: + pages = convert_from_path( + local_pdf, + dpi=300, + poppler_path=POPPLER_PATH + ) + + with concurrent.futures.ThreadPoolExecutor() as executor: + results = executor.map(ocr_page, pages) + + full_text = "\n".join(results) + source_type = "scanned_pdf" + + os.remove(local_pdf) + + return { + "file_id": file_id, + "type": source_type, + "text_length": len(full_text), + "text": full_text + } \ No newline at end of file diff --git a/app.py b/app.py index da357e5..c5cbcaf 100644 --- a/app.py +++ b/app.py @@ -2,6 +2,8 @@ from flask import Flask, request, jsonify import requests from bs4 import BeautifulSoup from DuckDuckGo import search_duckduckgo +from PdfExtract import process_pdf, process_uploaded_pdf + app = Flask(__name__) @@ -14,6 +16,41 @@ def search(): results = search_duckduckgo(query) return jsonify(results) +# ================= API 1: Extract from URL ================= +@app.route("/extract", methods=["POST"]) +def extract(): + try: + data = request.get_json() + if not data or "url" not in data: + return jsonify({"error": "Missing URL"}), 400 + + result = process_pdf(data["url"]) + return jsonify(result) + + except Exception as e: + return jsonify({"error": str(e)}), 400 + + +# ================= API 2: Upload file ================= +@app.route("/upload", methods=["POST"]) +def upload(): + try: + if "file" not in request.files: + return jsonify({"error": "No file field"}), 400 + + file = request.files["file"] + + if file.filename == "": + return jsonify({"error": "No selected file"}), 400 + + file_bytes = file.read() + + result = process_uploaded_pdf(file_bytes) + return jsonify(result) + + except Exception as e: + return jsonify({"error": str(e)}), 400 + if __name__ == "__main__": - app.run(debug=True) \ No newline at end of file + app.run(host="0.0.0.0", port=8000) \ No newline at end of file diff --git a/images/sample.pdf b/images/sample.pdf new file mode 100644 index 0000000..c01805e Binary files /dev/null and b/images/sample.pdf differ diff --git a/images/toc.pdf b/images/toc.pdf new file mode 100644 index 0000000..39ffae9 Binary files /dev/null and b/images/toc.pdf differ diff --git a/images/toc.png b/images/toc.png new file mode 100644 index 0000000..e4785ce Binary files /dev/null and b/images/toc.png differ