Merge branch 'master' into fix-build-2

.github/workflows/check-dead-links.yml

@@ -15,7 +15,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Install dependencies
         run: |

.github/workflows/clean-acr.yml

@@ -31,7 +31,7 @@ jobs:
          with:
            creds: ${{ secrets.clean_acr }}
        - name: checkout repo content
-         uses: actions/checkout@v3 # checkout the repo
+         uses: actions/checkout@v4 # checkout the repo
        - name: setup python
          uses: actions/setup-python@v4
          with:

.github/workflows/codeql.yml

@@ -42,7 +42,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/on-pull-request-target-review.yml

@@ -10,7 +10,7 @@ jobs:
     name: Azure OpenAI PR Comment
     steps:
       - id: review
-        uses: microsoft/[email protected].4
+        uses: microsoft/[email protected].5
         with:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           AZURE_OPENAI_API: ${{ secrets.AZURE_OPENAI_API }}

.github/workflows/scorecards.yml

@@ -32,7 +32,7 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+        uses: actions/checkout@v4 # v3.1.0
         with:
           persist-credentials: false
 

.gitignore

@@ -86,3 +86,4 @@ metastore_db/
 **/build/*
 **/dist/*
 **/*.egg-info/*
+

CONTRIBUTING.md

@@ -45,7 +45,7 @@ this process:
 
 #### Implement tests
 
--   Set up build environment using the [developer guide](https://microsoft.github.io/SynapseML/docs/reference/developer-readme/)
+-   Set up build environment using the [developer guide](https://microsoft.github.io/SynapseML/docs/Reference/Developer%20Setup/)
 -   Test your code locally.
 -   Add tests using ScalaTests — unit tests are required.
 -   A sample notebook is required as an end-to-end test.

README.md

@@ -12,9 +12,10 @@ SynapseML requires Scala 2.12, Spark 3.2+, and Python 3.8+.
 | :------ | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
 | Build   | [![Build Status](https://msdata.visualstudio.com/A365/_apis/build/status/microsoft.SynapseML?branchName=master)](https://msdata.visualstudio.com/A365/_build/latest?definitionId=17563&branchName=master) [![codecov](https://codecov.io/gh/Microsoft/SynapseML/branch/master/graph/badge.svg)](https://codecov.io/gh/Microsoft/SynapseML) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)                     |
 | Version | [![Version](https://img.shields.io/badge/version-0.11.2-blue)](https://github.com/Microsoft/SynapseML/releases) [![Release Notes](https://img.shields.io/badge/release-notes-blue)](https://github.com/Microsoft/SynapseML/releases) [![Snapshot Version](https://mmlspark.blob.core.windows.net/icons/badges/master_version3.svg)](#sbt)                                                                                                                                       |
-| Docs    | [![Scala Docs](https://img.shields.io/static/v1?label=api%20docs&message=scala&color=blue&logo=scala)](https://mmlspark.blob.core.windows.net/docs/0.11.2/scala/index.html#package) [![PySpark Docs](https://img.shields.io/static/v1?label=api%20docs&message=python&color=blue&logo=python)](https://mmlspark.blob.core.windows.net/docs/0.11.2/pyspark/index.html) [![Academic Paper](https://img.shields.io/badge/academic-paper-7fdcf7)](https://arxiv.org/abs/1810.08744) |
+| Docs    | [![Website](https://img.shields.io/badge/SynapseML-Website-blue)](https://aka.ms/spark) [![Scala Docs](https://img.shields.io/static/v1?label=api%20docs&message=scala&color=blue&logo=scala)](https://mmlspark.blob.core.windows.net/docs/0.11.2/scala/index.html#package) [![PySpark Docs](https://img.shields.io/static/v1?label=api%20docs&message=python&color=blue&logo=python)](https://mmlspark.blob.core.windows.net/docs/0.11.2/pyspark/index.html) [![Academic Paper](https://img.shields.io/badge/academic-paper-7fdcf7)](https://arxiv.org/abs/1810.08744) |
 | Support | [![Gitter](https://badges.gitter.im/Microsoft/MMLSpark.svg)](https://gitter.im/Microsoft/MMLSpark?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Mail](https://img.shields.io/badge/mail-synapseml--support-brightgreen)](mailto:[email protected])                                                                                                                                                                                                  |
 | Binder  | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/microsoft/SynapseML/v0.11.2?labpath=notebooks%2Ffeatures)                                                                                                                                                                                                                                                                                                                                           |
+| Usage | [![Downloads](https://static.pepy.tech/badge/synapseml)](https://pepy.tech/project/synapseml) |
 <!-- markdownlint-disable MD033 -->
 <details open>
 <summary>
@@ -238,25 +239,25 @@ docker run -it -p 8888:8888 -e ACCEPT_EULA=yes mcr.microsoft.com/mmlspark/releas
 ```
 
 Navigate to <http://localhost:8888/> in your web browser to run the sample
-notebooks.  See the [documentation](website/docs/reference/docker.md) for more on Docker use.
+notebooks.  See the [documentation](https://microsoft.github.io/SynapseML/docs/Reference/Docker%20Setup/) for more on Docker use.
 
 > To read the EULA for using the docker image, run `docker run -it -p 8888:8888 mcr.microsoft.com/mmlspark/release eula`
 
 ### R
 
 To try out SynapseML using the R autogenerated wrappers [see our
-instructions](website/docs/reference/R-setup.md).  Note: This feature is still under development
+instructions](https://microsoft.github.io/SynapseML/docs/Reference/R%20Setup/).  Note: This feature is still under development
 and some necessary custom wrappers may be missing.
 
 ### C# (.NET)
 
-To try out SynapseML with .NET, please follow the [.NET Installation Guide](website/docs/reference/dotnet-setup.md).
+To try out SynapseML with .NET, please follow the [.NET Installation Guide](https://microsoft.github.io/SynapseML/docs/Reference/Dotnet%20Setup/).
 Please note that some classes including the `AzureSearchWriter`, `DiagnosticInfo`, `UDPyFParam`, `ParamSpaceParam`, `BallTreeParam`, `ConditionalBallTreeParam`, `LightGBMBoosterParam` are still under development and not exposed in .NET yet.
 
 ### Building from source
 
 SynapseML has recently transitioned to a new build infrastructure.
-For detailed developer docs please see the [Developer Readme](website/docs/reference/developer-readme.md)
+For detailed developer docs please see the [Developer Readme](https://microsoft.github.io/SynapseML/docs/Reference/Developer%20Setup/)
 
 If you are an existing synapsemldeveloper, you will need to reconfigure your
 development setup. We now support platform independent development and

cognitive/src/main/python/synapse/ml/cognitive/langchain/LangchainTransform.py

@@ -44,6 +44,7 @@
 )
 from pyspark.sql.functions import udf
 from typing import cast, Optional, TypeVar, Type
+from synapse.ml.core.platform import running_on_synapse_internal
 
 OPENAI_API_VERSION = "2022-12-01"
 RL = TypeVar("RL", bound="MLReadable")
@@ -125,6 +126,14 @@ def __init__(
         self.subscriptionKey = Param(self, "subscriptionKey", "openai api key")
         self.url = Param(self, "url", "openai api base")
         self.apiVersion = Param(self, "apiVersion", "openai api version")
+        self.running_on_synapse_internal = running_on_synapse_internal()
+        if running_on_synapse_internal():
+            from synapse.ml.fabric.service_discovery import get_fabric_env_config
+
+            self._setDefault(
+                url=get_fabric_env_config().fabric_env_config.ml_workload_endpoint
+                + "cognitive/openai"
+            )
         kwargs = self._input_kwargs
         if subscriptionKey:
             kwargs["subscriptionKey"] = subscriptionKey
@@ -196,10 +205,15 @@ def _transform(self, dataset):
         def udfFunction(x):
             import openai
 
-            openai.api_type = "azure"
-            openai.api_key = self.getSubscriptionKey()
-            openai.api_base = self.getUrl()
-            openai.api_version = self.getApiVersion()
+            if self.running_on_synapse_internal and not self.isSet(self.url):
+                from synapse.ml.fabric.prerun.openai_prerun import OpenAIPrerun
+
+                OpenAIPrerun(api_base=self.getUrl()).init_personalized_session(None)
+            else:
+                openai.api_type = "azure"
+                openai.api_key = self.getSubscriptionKey()
+                openai.api_base = self.getUrl()
+                openai.api_version = self.getApiVersion()
             return self.getChain().run(x)
 
         outCol = self.getOutputCol()