Undatasio 集成

import getpass
import os

if "UNDATASIO_TOKEN" not in os.environ:
    os.environ["UNDATASIO_TOKEN"] = getpass.getpass(
        "Enter your UnDatasIO API token: "
    )

# 安装软件包，兼容 API 分区
pip install langchain-undatasio

from langchain_undatasio import UnDatasIOLoader

loader = UnDatasIOLoader(
    token=os.environ["UNDATASIO_TOKEN"],
    file_path="demo.pdf"
)

docs = loader.load()
docs[0]

Document(
    metadata={'source': 'demo.pdf', 'task_id': 't1', 'file_id': 'f1'},
    page_content='Growing a Tail: Increasing Output Diversity in Large Language Models\n\nAuthors: Michal Shur-Ofry1, Bar Horowitz-Amsalem1†, Adir Rahamim2, Yonatan Belinkov2*\n\nAffiliations:\n\n1Law Faculty, Hebrew University of Jerusalem; Jerusalem, Israel.\n\n2Faculty of Computer Science, Technion – I'
)

print(docs[0].page_content[:300])

Growing a Tail: Increasing Output Diversity in Large Language Models

Authors: Michal Shur-Ofry1, Bar Horowitz-Amsalem1†, Adir Rahamim2, Yonatan Belinkov2*

Affiliations:

1Law Faculty, Hebrew University of Jerusalem; Jerusalem, Israel.

2Faculty of Computer Science, Technion – I

pages = []
for doc in loader.lazy_load():
    pages.append(doc)

pages[0]

Document(
    metadata={'source': 'demo.pdf', 'task_id': 't1', 'file_id': 'f1'},
    page_content='Growing a Tail: Increasing Output Diversity in Large Language Models\n\nAuthors: Michal Shur-Ofry1, Bar Horowitz-Amsalem1†, Adir Rahamim2, Yonatan Belinkov2*\n\nAffiliations:\n\n1Law Faculty, Hebrew University of Jerusalem; Jerusalem, Israel.\n\n2Faculty of Computer Science, Technion – I'
)