UnstructuredLoader 集成

docker run -p 8000:8000 -d --rm --name unstructured-api downloads.unstructured.io/unstructured-io/unstructured-api:latest --port 8000 --host 0.0.0.0

export UNSTRUCTURED_API_KEY="your-api-key"

import { UnstructuredLoader } from "@langchain/community/document_loaders/fs/unstructured"

const loader = new UnstructuredLoader("../../../../../../examples/src/document_loaders/example_data/notion.mdx")

const docs = await loader.load()
docs[0]

Document {
  pageContent: '# Testing the notion markdownloader',
  metadata: {
    filename: 'notion.mdx',
    languages: [ 'eng' ],
    filetype: 'text/plain',
    category: 'NarrativeText'
  },
  id: undefined
}

console.log(docs[0].metadata)

{
  filename: 'notion.mdx',
  languages: [ 'eng' ],
  filetype: 'text/plain',
  category: 'NarrativeText'
}

import { UnstructuredDirectoryLoader } from "@langchain/community/document_loaders/fs/unstructured";

const directoryLoader = new UnstructuredDirectoryLoader(
  "../../../../../../examples/src/document_loaders/example_data/",
  {}
);
const directoryDocs = await directoryLoader.load();
console.log("directoryDocs.length: ", directoryDocs.length);
console.log(directoryDocs[0])

Unknown file type: Star_Wars_The_Clone_Wars_S06E07_Crisis_at_the_Heart.srt
Unknown file type: test.mp3

directoryDocs.length:  247
Document {
  pageContent: 'Bitcoin: A Peer-to-Peer Electronic Cash System',
  metadata: {
    filetype: 'application/pdf',
    languages: [ 'eng' ],
    page_number: 1,
    filename: 'bitcoin.pdf',
    category: 'Title'
  },
  id: undefined
}