Trying to connect to Azure Managed Instance for Llama 3.2 with AzureChatOpenAI in langchain_openai and with AzureMLChatOnlineEndpoint

[mindful-time]

Checked other resources

• I added a very descriptive title to this question.
• I searched the LangChain documentation with the integrated search.
• I used the GitHub search to find a similar question and didn't find it.

Commit to Help

• I commit to help with one of those options 👆

Example Code

azure_llm = AzureChatOpenAI(
    azure_endpoint="https://xxx.xxx.inference.ml.azure.com/score",
    deployment_name="llama-3-2-90b-vision-instruct",
    model_version="1",
    model_name="Llama-3.2-90B-Vision-Instruct",
    api_version=os.getenv("AZURE_API_VERSION"),
    api_key=os.environ["AZURE_LLAMA_3_2_API_KEY"],
    temperature=0
)

azure_llm.invoke("hello")


errors
PIStatusError                            Traceback (most recent call last)
Cell In[22], line 11
      1 azure_llm = AzureChatOpenAI(
      2     azure_endpoint="https://xxx.xxx.inference.ml.azure.com/score",
      3     deployment_name="llama-3-2-90b-vision-instruct",
   (...)
      8     temperature=0
      9 )
---> 11 azure_llm.invoke("hello")

File c:\POC\sandbox\notebooks-for-testing\.venv\Lib\site-packages\langchain_core\language_models\chat_models.py:284, in BaseChatModel.invoke(self, input, config, stop, **kwargs)
    273 def invoke(
    274     self,
    275     input: LanguageModelInput,
   (...)
    279     **kwargs: Any,
    280 ) -> BaseMessage:
    281     config = ensure_config(config)
    282     return cast(
    283         ChatGeneration,
--> 284         self.generate_prompt(
    285             [self._convert_input(input)],
    286             stop=stop,
    287             callbacks=config.get("callbacks"),
    288             tags=config.get("tags"),
    289             metadata=config.get("metadata"),
    290             run_name=config.get("run_name"),
    291             run_id=config.pop("run_id", None),
    292             **kwargs,
    293         ).generations[0][0],
    294     ).message

File c:\POC\sandbox\notebooks-for-testing\.venv\Lib\site-packages\langchain_core\language_models\chat_models.py:784, in BaseChatModel.generate_prompt(self, prompts, stop, callbacks, **kwargs)
    776 def generate_prompt(
    777     self,
    778     prompts: list[PromptValue],
   (...)
    781     **kwargs: Any,
    782 ) -> LLMResult:
    783     prompt_messages = [p.to_messages() for p in prompts]
--> 784     return self.generate(prompt_messages, stop=stop, callbacks=callbacks, **kwargs)

File c:\POC\sandbox\notebooks-for-testing\.venv\Lib\site-packages\langchain_core\language_models\chat_models.py:641, in BaseChatModel.generate(self, messages, stop, callbacks, tags, metadata, run_name, run_id, **kwargs)
    639         if run_managers:
    640             run_managers[i].on_llm_error(e, response=LLMResult(generations=[]))
--> 641         raise e
    642 flattened_outputs = [
    643     LLMResult(generations=[res.generations], llm_output=res.llm_output)  # type: ignore[list-item]
    644     for res in results
    645 ]
    646 llm_output = self._combine_llm_outputs([res.llm_output for res in results])

File c:\POC\sandbox\notebooks-for-testing\.venv\Lib\site-packages\langchain_core\language_models\chat_models.py:631, in BaseChatModel.generate(self, messages, stop, callbacks, tags, metadata, run_name, run_id, **kwargs)
    628 for i, m in enumerate(messages):
    629     try:
    630         results.append(
--> 631             self._generate_with_cache(
    632                 m,
    633                 stop=stop,
    634                 run_manager=run_managers[i] if run_managers else None,
    635                 **kwargs,
    636             )
    637         )
    638     except BaseException as e:
    639         if run_managers:

File c:\POC\sandbox\notebooks-for-testing\.venv\Lib\site-packages\langchain_core\language_models\chat_models.py:853, in BaseChatModel._generate_with_cache(self, messages, stop, run_manager, **kwargs)
    851 else:
    852     if inspect.signature(self._generate).parameters.get("run_manager"):
--> 853         result = self._generate(
    854             messages, stop=stop, run_manager=run_manager, **kwargs
    855         )
    856     else:
    857         result = self._generate(messages, stop=stop, **kwargs)

File c:\POC\sandbox\notebooks-for-testing\.venv\Lib\site-packages\langchain_openai\chat_models\base.py:670, in BaseChatOpenAI._generate(self, messages, stop, run_manager, **kwargs)
    668     generation_info = {"headers": dict(raw_response.headers)}
    669 else:
--> 670     response = self.client.create(**payload)
    671 return self._create_chat_result(response, generation_info)

File c:\POC\sandbox\notebooks-for-testing\.venv\Lib\site-packages\openai\_utils\_utils.py:274, in required_args.<locals>.inner.<locals>.wrapper(*args, **kwargs)
    272             msg = f"Missing required argument: {quote(missing[0])}"
    273     raise TypeError(msg)
--> 274 return func(*args, **kwargs)

File c:\POC\sandbox\notebooks-for-testing\.venv\Lib\site-packages\openai\resources\chat\completions.py:704, in Completions.create(self, messages, model, frequency_penalty, function_call, functions, logit_bias, logprobs, max_completion_tokens, max_tokens, n, parallel_tool_calls, presence_penalty, response_format, seed, service_tier, stop, stream, stream_options, temperature, tool_choice, tools, top_logprobs, top_p, user, extra_headers, extra_query, extra_body, timeout)
    668 @required_args(["messages", "model"], ["messages", "model", "stream"])
    669 def create(
    670     self,
   (...)
    701     timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    702 ) -> ChatCompletion | Stream[ChatCompletionChunk]:
    703     validate_response_format(response_format)
--> 704     return self._post(
    705         "/chat/completions",
    706         body=maybe_transform(
    707             {
    708                 "messages": messages,
    709                 "model": model,
    710                 "frequency_penalty": frequency_penalty,
    711                 "function_call": function_call,
    712                 "functions": functions,
    713                 "logit_bias": logit_bias,
    714                 "logprobs": logprobs,
    715                 "max_completion_tokens": max_completion_tokens,
    716                 "max_tokens": max_tokens,
    717                 "n": n,
    718                 "parallel_tool_calls": parallel_tool_calls,
    719                 "presence_penalty": presence_penalty,
    720                 "response_format": response_format,
    721                 "seed": seed,
    722                 "service_tier": service_tier,
    723                 "stop": stop,
    724                 "stream": stream,
    725                 "stream_options": stream_options,
    726                 "temperature": temperature,
    727                 "tool_choice": tool_choice,
    728                 "tools": tools,
    729                 "top_logprobs": top_logprobs,
    730                 "top_p": top_p,
    731                 "user": user,
    732             },
    733             completion_create_params.CompletionCreateParams,
    734         ),
    735         options=make_request_options(
    736             extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
    737         ),
    738         cast_to=ChatCompletion,
    739         stream=stream or False,
    740         stream_cls=Stream[ChatCompletionChunk],
    741     )

File c:\POC\sandbox\notebooks-for-testing\.venv\Lib\site-packages\openai\_base_client.py:1268, in SyncAPIClient.post(self, path, cast_to, body, options, files, stream, stream_cls)
   1254 def post(
   1255     self,
   1256     path: str,
   (...)
   1263     stream_cls: type[_StreamT] | None = None,
   1264 ) -> ResponseT | _StreamT:
   1265     opts = FinalRequestOptions.construct(
   1266         method="post", url=path, json_data=body, files=to_httpx_files(files), **options
   1267     )
-> 1268     return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))

File c:\POC\sandbox\notebooks-for-testing\.venv\Lib\site-packages\openai\_base_client.py:945, in SyncAPIClient.request(self, cast_to, options, remaining_retries, stream, stream_cls)
    942 else:
    943     retries_taken = 0
--> 945 return self._request(
    946     cast_to=cast_to,
    947     options=options,
    948     stream=stream,
    949     stream_cls=stream_cls,
    950     retries_taken=retries_taken,
    951 )

File c:\POC\sandbox\notebooks-for-testing\.venv\Lib\site-packages\openai\_base_client.py:1049, in SyncAPIClient._request(self, cast_to, options, retries_taken, stream, stream_cls)
   1046         err.response.read()
   1048     log.debug("Re-raising status error")
-> 1049     raise self._make_status_error_from_response(err.response) from None
   1051 return self._process_response(
   1052     cast_to=cast_to,
   1053     options=options,
   (...)
   1057     retries_taken=retries_taken,
   1058 )

APIStatusError: Error code: 424 - {'detail': 'Not Found'}



azure_llm.invoke("hello")

I also tried this

from langchain_community.chat_models.azureml_endpoint import (
    AzureMLEndpointApiType,
    CustomOpenAIChatContentFormatter,
    AzureMLChatOnlineEndpoint
)
from langchain_core.messages import HumanMessage

chat = AzureMLChatOnlineEndpoint(
    endpoint_url="https://xxx.xxxx.inference.ml.azure.com/score",
    endpoint_api_type=AzureMLEndpointApiType.dedicated,
    content_formatter=CustomOpenAIChatContentFormatter(),
    endpoint_api_key=os.getenv("AZURE_LLAMA_3_2_API_KEY"),
    model_kwargs={"temperature": 0}
)

response = chat.invoke(
    [HumanMessage(content="Will the Collatz conjecture ever be solved?")]
)
print(response)

with the following error

Error                                  Traceback (most recent call last)
File c:\POC\sandbox\notebooks-for-testing\.venv\Lib\site-packages\langchain_community\chat_models\azureml_endpoint.py:140, in CustomOpenAIChatContentFormatter.format_response_payload(self, output, api_type)
    [139](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:139) try:
--> [140](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:140)     choice = json.loads(output)["output"]
    [141](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:141) except (KeyError, IndexError, TypeError) as e:

KeyError: 'output'

The above exception was the direct cause of the following exception:

ValueError                                Traceback (most recent call last)
Cell In[63], [line 16](vscode-notebook-cell:?execution_count=63&line=16)
      [6](vscode-notebook-cell:?execution_count=63&line=6) from langchain_core.messages import HumanMessage
      [8](vscode-notebook-cell:?execution_count=63&line=8) chat = AzureMLChatOnlineEndpoint(
      [9](vscode-notebook-cell:?execution_count=63&line=9)     endpoint_url="https://xxx.xxx.inference.ml.azure.com/score",
     [10](vscode-notebook-cell:?execution_count=63&line=10)     endpoint_api_type=AzureMLEndpointApiType.dedicated,
   (...)
     [13](vscode-notebook-cell:?execution_count=63&line=13)     model_kwargs={"temperature": 0}
     [14](vscode-notebook-cell:?execution_count=63&line=14) )
---> [16](vscode-notebook-cell:?execution_count=63&line=16) response = chat.invoke(
     [17](vscode-notebook-cell:?execution_count=63&line=17)     [HumanMessage(content="Will the Collatz conjecture ever be solved?")]
     [18](vscode-notebook-cell:?execution_count=63&line=18) )
     [19](vscode-notebook-cell:?execution_count=63&line=19) print(response)

File c:\POC\sandbox\notebooks-for-testing\.venv\Lib\site-packages\langchain_core\language_models\chat_models.py:284, in BaseChatModel.invoke(self, input, config, stop, **kwargs)
    [273](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:273) def invoke(
    [274](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:274)     self,
    [275](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:275)     input: LanguageModelInput,
   (...)
    [279](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:279)     **kwargs: Any,
    [280](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:280) ) -> BaseMessage:
    [281](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:281)     config = ensure_config(config)
    [282](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:282)     return cast(
    [283](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:283)         ChatGeneration,
--> [284](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:284)         self.generate_prompt(
    [285](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:285)             [self._convert_input(input)],
    [286](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:286)             stop=stop,
    [287](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:287)             callbacks=config.get("callbacks"),
    [288](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:288)             tags=config.get("tags"),
    [289](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:289)             metadata=config.get("metadata"),
    [290](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:290)             run_name=config.get("run_name"),
    [291](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:291)             run_id=config.pop("run_id", None),
    [292](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:292)             **kwargs,
    [293](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:293)         ).generations[0][0],
    [294](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:294)     ).message

File c:\POC\sandbox\notebooks-for-testing\.venv\Lib\site-packages\langchain_core\language_models\chat_models.py:784, in BaseChatModel.generate_prompt(self, prompts, stop, callbacks, **kwargs)
    [776](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:776) def generate_prompt(
    [777](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:777)     self,
    [778](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:778)     prompts: list[PromptValue],
   (...)
    [781](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:781)     **kwargs: Any,
    [782](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:782) ) -> LLMResult:
    [783](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:783)     prompt_messages = [p.to_messages() for p in prompts]
--> [784](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:784)     return self.generate(prompt_messages, stop=stop, callbacks=callbacks, **kwargs)

File c:\POC\sandbox\notebooks-for-testing\.venv\Lib\site-packages\langchain_core\language_models\chat_models.py:641, in BaseChatModel.generate(self, messages, stop, callbacks, tags, metadata, run_name, run_id, **kwargs)
    [639](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:639)         if run_managers:
    [640](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:640)             run_managers[i].on_llm_error(e, response=LLMResult(generations=[]))
--> [641](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:641)         raise e
    [642](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:642) flattened_outputs = [
    [643](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:643)     LLMResult(generations=[res.generations], llm_output=res.llm_output)  # type: ignore[list-item]
    [644](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:644)     for res in results
    [645](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:645) ]
    [646](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:646) llm_output = self._combine_llm_outputs([res.llm_output for res in results])

File c:\POC\sandbox\notebooks-for-testing\.venv\Lib\site-packages\langchain_core\language_models\chat_models.py:631, in BaseChatModel.generate(self, messages, stop, callbacks, tags, metadata, run_name, run_id, **kwargs)
    [628](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:628) for i, m in enumerate(messages):
    [629](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:629)     try:
    [630](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:630)         results.append(
--> [631](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:631)             self._generate_with_cache(
    [632](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:632)                 m,
    [633](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:633)                 stop=stop,
    [634](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:634)                 run_manager=run_managers[i] if run_managers else None,
    [635](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:635)                 **kwargs,
    [636](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:636)             )
    [637](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:637)         )
    [638](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:638)     except BaseException as e:
    [639](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:639)         if run_managers:

File c:\POC\sandbox\notebooks-for-testing\.venv\Lib\site-packages\langchain_core\language_models\chat_models.py:853, in BaseChatModel._generate_with_cache(self, messages, stop, run_manager, **kwargs)
    [851](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:851) else:
    [852](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:852)     if inspect.signature(self._generate).parameters.get("run_manager"):
--> [853](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:853)         result = self._generate(
    [854](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:854)             messages, stop=stop, run_manager=run_manager, **kwargs
    [855](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:855)         )
    [856](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:856)     else:
    [857](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_core/language_models/chat_models.py:857)         result = self._generate(messages, stop=stop, **kwargs)

File c:\POC\sandbox\notebooks-for-testing\.venv\Lib\site-packages\langchain_community\chat_models\azureml_endpoint.py:280, in AzureMLChatOnlineEndpoint._generate(self, messages, stop, run_manager, **kwargs)
    [274](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:274) request_payload = self.content_formatter.format_messages_request_payload(
    [275](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:275)     messages, _model_kwargs, self.endpoint_api_type
    [276](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:276) )
    [277](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:277) response_payload = self.http_client.call(
    [278](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:278)     body=request_payload, run_manager=run_manager
    [279](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:279) )
--> [280](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:280) generations = self.content_formatter.format_response_payload(
    [281](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:281)     response_payload, self.endpoint_api_type
    [282](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:282) )
    [283](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:283) return ChatResult(generations=[generations])

File c:\POC\sandbox\notebooks-for-testing\.venv\Lib\site-packages\langchain_community\chat_models\azureml_endpoint.py:142, in CustomOpenAIChatContentFormatter.format_response_payload(self, output, api_type)
    [140](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:140)         choice = json.loads(output)["output"]
    [141](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:141)     except (KeyError, IndexError, TypeError) as e:
--> [142](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:142)         raise ValueError(self.format_error_msg.format(api_type=api_type)) from e
    [143](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:143)     return ChatGeneration(
    [144](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:144)         message=AIMessage(
    [145](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:145)             content=choice.strip(),
    [146](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:146)         ),
    [147](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:147)         generation_info=None,
    [148](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:148)     )
    [149](file:///C:/POC/sandbox/notebooks-for-testing/.venv/Lib/site-packages/langchain_community/chat_models/azureml_endpoint.py:149) if api_type == AzureMLEndpointApiType.serverless:

ValueError: Error while formatting response payload for chat model of type  `AzureMLEndpointApiType.dedicated`. Are you using the right formatter for the deployed  model and endpoint type?



### Description

Error code: 424 - {'detail': 'Not Found'}

What is the best way to use langchain for managed instance in Azure

### System Info

langchain==0.3.0
langchain-community==0.3.0
langchain-core==0.3.5
langchain-huggingface==0.1.0
langchain-openai==0.2.0
langchain-text-splitters==0.3.0