Azure 认知服务工具包集成

pip install -qU langchain-community
pip install -qU azure-ai-formrecognizer
pip install -qU azure-cognitiveservices-speech
pip install -qU azure-ai-textanalytics

# Windows / Linux only
pip install -qU azure-ai-vision

import os

os.environ["OPENAI_API_KEY"] = "sk-..."
os.environ["AZURE_COGS_KEY"] = ""
os.environ["AZURE_COGS_ENDPOINT"] = ""
os.environ["AZURE_COGS_REGION"] = ""

## 创建工具包
from langchain_community.agent_toolkits import AzureCognitiveServicesToolkit

toolkit = AzureCognitiveServicesToolkit()
toolkit.get_tools()

['Azure Cognitive Services Image Analysis',
 'Azure Cognitive Services Form Recognizer',
 'Azure Cognitive Services Speech2Text',
 'Azure Cognitive Services Text2Speech']

## 在代理中使用
from langchain.agents import create_agent
from langchain_openai import OpenAI

llm = OpenAI(temperature=0)

agent = create_agent(
    model=llm,
    tools=toolkit.get_tools(),
)

agent.invoke(
    "What can I make with these ingredients? "
    "https://images.openai.com/blob/9ad5a2ab-041f-475f-ad6a-b51899c50182/ingredients.png"
)

{
  "action": "Azure Cognitive Services Image Analysis",
  "action_input": "https://images.openai.com/blob/9ad5a2ab-041f-475f-ad6a-b51899c50182/ingredients.png"
}

{
  "action": "Final Answer",
  "action_input": "You can make pancakes, omelettes, or quiches with these ingredients!"
}

audio_file = agent.invoke("Tell me a joke and read it out loud.")

audio_file

{
  "action": "Azure Cognitive Services Text2Speech",
  "action_input": "Why did the chicken cross the playground? To get to the other slide!"
}

{
  "action": "Final Answer",
  "action_input": "/tmp/tmpa3uu_j6b.wav"
}

from IPython import display

display.Audio(audio_file)

agent.invoke(
    """The patient is a 54-year-old gentleman with a history of progressive angina over the past several months.
The patient had a cardiac catheterization in July of this year revealing total occlusion of the RCA and 50% left main disease ,
with a strong family history of coronary artery disease with a brother dying at the age of 52 from a myocardial infarction and
another brother who is status post coronary artery bypass grafting. The patient had a stress echocardiogram done on July , 2001 ,
which showed no wall motion abnormalities , but this was a difficult study due to body habitus. The patient went for six minutes with
minimal ST depressions in the anterior lateral leads , thought due to fatigue and wrist pain , his anginal equivalent. Due to the patient's
increased symptoms and family history and history left main disease with total occasional of his RCA was referred for revascularization with open heart surgery.

List all the diagnoses.
"""
)

{
  "action": "azure_cognitive_services_text_analyics_health",
  "action_input": "The patient is a 54-year-old gentleman with a history of progressive angina over the past several months. The patient had a cardiac catheterization in July of this year revealing total occlusion of the RCA and 50% left main disease, with a strong family history of coronary artery disease with a brother dying at the age of 52 from a myocardial infarction and another brother who is status post coronary artery bypass grafting. The patient had a stress echocardiogram done on July, 2001, which showed no wall motion abnormalities, but this was a difficult study due to body habitus. The patient went for six minutes with minimal ST depressions in the anterior lateral leads, thought due to fatigue and wrist pain, his anginal equivalent. Due to the patient's increased symptoms and family history and history left main disease with total occasional of his RCA was referred for revascularization with open heart surgery."
}

{
  "action": "Final Answer",
  "action_input": "The text contains the following diagnoses: progressive angina, coronary artery disease, myocardial infarction, and coronary artery bypass grafting."
}