Pinecone 混合搜索集成

pip install -qU  pinecone pinecone-text pinecone-notebooks

# 连接到 Pinecone 并获取 API 密钥。
from pinecone_notebooks.colab import Authenticate

Authenticate()

import os

api_key = os.environ["PINECONE_API_KEY"]

from langchain_community.retrievers import (
    PineconeHybridSearchRetriever,
)

import getpass

if "OPENAI_API_KEY" not in os.environ:
    os.environ["OPENAI_API_KEY"] = getpass.getpass("OpenAI API Key:")

import os

from pinecone import Pinecone, ServerlessSpec

index_name = "langchain-pinecone-hybrid-search"

# 初始化 Pinecone 客户端
pc = Pinecone(api_key=api_key)

# 创建索引
if index_name not in pc.list_indexes().names():
    pc.create_index(
        name=index_name,
        dimension=1536,  # 稠密模型的维度
        metric="dotproduct",  # 仅点积支持稀疏值
        spec=ServerlessSpec(cloud="aws", region="us-east-1"),
    )

WhoAmIResponse(username='load', user_label='label', projectname='load-test')

index = pc.Index(index_name)

from langchain_openai import OpenAIEmbeddings

embeddings = OpenAIEmbeddings()

from pinecone_text.sparse import BM25Encoder

# 或者如果您希望使用 SPLADE，则 from pinecone_text.sparse import SpladeEncoder

# 使用默认的 tf-idf 值
bm25_encoder = BM25Encoder().default()

corpus = ["foo", "bar", "world", "hello"]

# 在您的语料库上拟合 tf-idf 值
bm25_encoder.fit(corpus)

# 将值存储到 json 文件
bm25_encoder.dump("bm25_values.json")

# 加载到您的 BM25Encoder 对象
bm25_encoder = BM25Encoder().load("bm25_values.json")

retriever = PineconeHybridSearchRetriever(
    embeddings=embeddings, sparse_encoder=bm25_encoder, index=index
)

retriever.add_texts(["foo", "bar", "world", "hello"])

100%|██████████| 1/1 [00:02<00:00,  2.27s/it]

result = retriever.invoke("foo")

result[0]

Document(page_content='foo', metadata={})