Stop Feeding Your RAG
Garbage PDFs

# Create and activate a virtual environment
python3 -m venv venv
source venv/bin/activate

# Install Docling
pip install docling

# For GPU acceleration (recommended if available)
pip install docling[gpu]

from docling.document_converter import DocumentConverter

# Initialize the converter
converter = DocumentConverter()

# Parse a clinical PDF — ACOG bulletin, SMFM consult, scanned referral, etc.
result = converter.convert("acog_gdm_guidelines.pdf")

# Export to Markdown (preserves structure, tables, headings)
markdown_output = result.document.export_to_markdown()
print(markdown_output)

# Or export to JSON for programmatic access
json_output = result.document.export_to_dict()

from docling.document_converter import DocumentConverter

converter = DocumentConverter()
result = converter.convert("smfm_consult_periviability.pdf")
doc = result.document

# Iterate over all content elements
for element, level in doc.iterate_items():
    print(type(element).__name__, ":", str(element)[:80])

# Access tables directly
for table in doc.tables:
    print("\n--- TABLE ---")
    print(table.export_to_markdown())

pip install langchain-docling langchain-core langchain langchain-openai chromadb

from langchain_docling import DoclingLoader
from langchain_docling.loader import ExportType

# Your clinical document library
# Can be local file paths or URLs
CLINICAL_DOCS = [
    "acog_gdm_practice_bulletin.pdf",
    "smfm_periviability_consult.pdf",
    "acog_preeclampsia_task_force.pdf",
    "smfm_fgr_consult_series.pdf",
]

# DOC_CHUNKS: splits document into structured chunks
# Each chunk preserves section context (heading, page, table vs. paragraph)
loader = DoclingLoader(
    file_path=CLINICAL_DOCS,
    export_type=ExportType.DOC_CHUNKS
)

docs = loader.load()
print(f"Loaded {len(docs)} chunks from {len(CLINICAL_DOCS)} documents")
print("\nSample chunk metadata:")
print(docs[0].metadata)

from langchain_openai import OpenAIEmbeddings
from langchain_community.vectorstores import Chroma

# Using OpenAI embeddings — swap for a local model if needed
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")

# Build the vector store from Docling chunks
vectorstore = Chroma.from_documents(
    documents=docs,
    embedding=embeddings,
    collection_name="clinical_guidelines",
    persist_directory="./chroma_db"
)

print("Vector store built. Documents indexed.")

from langchain_openai import ChatOpenAI
from langchain_core.prompts import PromptTemplate
from langchain_core.runnables import RunnablePassthrough
from langchain_core.output_parsers import StrOutputParser

# Load the persisted vector store
vectorstore = Chroma(
    collection_name="clinical_guidelines",
    embedding_function=OpenAIEmbeddings(model="text-embedding-3-small"),
    persist_directory="./chroma_db"
)

retriever = vectorstore.as_retriever(search_kwargs={"k": 5})

# Clinical-context prompt
PROMPT_TEMPLATE = """You are a Maternal-Fetal Medicine clinical decision support tool.
Answer the question using only the retrieved guideline excerpts below.
Cite the source document when possible.
If the answer is not in the context, say so — do not speculate.

Context:
{context}

Question: {question}

Answer:"""

prompt = PromptTemplate.from_template(PROMPT_TEMPLATE)
llm = ChatOpenAI(model="gpt-4o", temperature=0)

def format_docs(docs):
    return "\n\n".join(d.page_content for d in docs)

chain = (
    {"context": retriever | format_docs, "question": RunnablePassthrough()}
    | prompt
    | llm
    | StrOutputParser()
)

# Ask a clinical question
question = "What are the diagnostic criteria for gestational diabetes using the two-step approach?"
answer = chain.invoke(question)
print(answer)

pip install llama-index-core llama-index-readers-docling llama-index-node-parser-docling

from llama_index.core import VectorStoreIndex
from llama_index.readers.docling import DoclingReader
from llama_index.node_parser.docling import DoclingNodeParser

# DoclingReader: loads and converts the PDFs
reader = DoclingReader()

# Load your documents
documents = reader.load_data(
    file_path=[
        "acog_gdm_practice_bulletin.pdf",
        "smfm_fgr_consult_series.pdf",
    ]
)

# DoclingNodeParser: produces structure-aware nodes for indexing
node_parser = DoclingNodeParser()
nodes = node_parser.get_nodes_from_documents(documents)

# Build the vector index
index = VectorStoreIndex(nodes)

# Query
query_engine = index.as_query_engine()
response = query_engine.query(
    "What fetal weight percentile defines fetal growth restriction?"
)
print(response)

from docling.document_converter import DocumentConverter
from docling.datamodel.pipeline_options import PdfPipelineOptions

# Enable full OCR pipeline for scanned documents
pipeline_options = PdfPipelineOptions()
pipeline_options.do_ocr = True
pipeline_options.do_table_structure = True

converter = DocumentConverter(
    format_options={
        "pdf": {"pipeline_options": pipeline_options}
    }
)

result = converter.convert("scanned_outside_records.pdf")
markdown = result.document.export_to_markdown()

# Fully local stack: Docling + Chroma + Ollama
# pip install docling chromadb langchain-community

from langchain_docling import DoclingLoader
from langchain_docling.loader import ExportType
from langchain_community.embeddings import OllamaEmbeddings
from langchain_community.vectorstores import Chroma
from langchain_community.llms import Ollama

# Parse with Docling — local, no API
loader = DoclingLoader(
    file_path=["patient_records/"],
    export_type=ExportType.DOC_CHUNKS
)
docs = loader.load()

# Embed with local Ollama model
embeddings = OllamaEmbeddings(model="nomic-embed-text")

# Store in local Chroma DB
vectorstore = Chroma.from_documents(
    documents=docs,
    embedding=embeddings,
    persist_directory="./local_chroma_db"
)

# Query with local LLM — zero data egress
llm = Ollama(model="llama3.2")
retriever = vectorstore.as_retriever()