BSHTMLLoader 集成

os.environ["LANGSMITH_API_KEY"] = getpass.getpass("Enter your LangSmith API key: ")
os.environ["LANGSMITH_TRACING"] = "true"

pip install -qU langchain-community bs4

from langchain_community.document_loaders import BSHTMLLoader

loader = BSHTMLLoader(
    file_path="./example_data/fake-content.html",
)

docs = loader.load()
docs[0]

Document(metadata={'source': './example_data/fake-content.html', 'title': 'Test Title'}, page_content='\nTest Title\n\n\nMy First Heading\nMy first paragraph.\n\n\n')

print(docs[0].metadata)

{'source': './example_data/fake-content.html', 'title': 'Test Title'}

page = []
for doc in loader.lazy_load():
    page.append(doc)
    if len(page) >= 10:
        # 执行一些分页操作，例如
        # index.upsert(page)

        page = []
page[0]

Document(metadata={'source': './example_data/fake-content.html', 'title': 'Test Title'}, page_content='\nTest Title\n\n\nMy First Heading\nMy first paragraph.\n\n\n')

loader = BSHTMLLoader(
    file_path="./example_data/fake-content.html", get_text_separator=", "
)

docs = loader.load()
print(docs[0])

page_content='
, Test Title,
,
,
, My First Heading,
, My first paragraph.,
,
,
' metadata={'source': './example_data/fake-content.html', 'title': 'Test Title'}