refactor: automate adding tons of codeblock helpers in website samples (

#1253) * delete python imports and helpers in markdowns & modify doctest to auto add those * fix ) in markdown & format * fix mergePyCodeDir error * fix announcement color * test letting job fail on errors * fix pipeline * fix pipeline * fix SynapseE2e job * fixing pipeline * fix python version * fix sphinx version caused bug in publish artifacts
microsoft · Nov 12, 2021 · b2751eb · b2751eb
1 parent c0b516b
commit b2751eb
Show file tree

Hide file tree

Showing 33 changed files with 271 additions and 2,299 deletions.
diff --git a/environment.yaml b/environment.yaml
@@ -13,7 +13,7 @@ dependencies:
   - r-devtools
   - pip:
     - wheel
-    - sphinx
+    - sphinx==4.2.0
     - sphinx_rtd_theme
     - coverage
     - pytest

diff --git a/pipeline.yaml b/pipeline.yaml
@@ -56,6 +56,7 @@ jobs:
         azureSubscription: 'MMLSpark Build'
         keyVaultName: mmlspark-keys
     - bash: |
+        set -e
         source activate synapseml
         sbt packagePython
         sbt publishBlob publishDocs publishR publishPython
@@ -71,7 +72,9 @@ jobs:
         PGP-PRIVATE: $(pgp-private)
         PGP-PUBLIC: $(pgp-public)
         PGP-PW: $(pgp-pw)
-    - bash: sbt publishBadges
+    - bash: |
+        set -e
+        sbt publishBadges
       condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
       displayName: Publish Badges
       env:
@@ -98,6 +101,7 @@ jobs:
         azureSubscription: 'MMLSpark Build'
         keyVaultName: mmlspark-keys
     - bash: |
+        set -e
         source activate synapseml
         sbt packagePython
         sbt publishBlob
@@ -139,8 +143,9 @@ jobs:
         azureSubscription: 'MMLSpark Build'
         keyVaultName: mmlspark-keys
     - bash: |
+        set -e
         source activate synapseml
-        jupyter nbconvert --to script ./notebooks/*.ipynb*
+        jupyter nbconvert --to script ./notebooks/features/*/*.ipynb*
         sbt packagePython
         sbt publishBlob
       displayName: Publish Blob Artifacts
@@ -246,6 +251,7 @@ jobs:
         echo '##vso[task.setvariable variable=tag]'$(git tag -l --points-at HEAD)
       displayName: 'Get Git Tag'
     - bash: |
+        set -e
         wget https://github.com/git-chglog/git-chglog/releases/download/0.8.0/git-chglog_linux_amd64
         chmod +x git-chglog_linux_amd64
         ./git-chglog_linux_amd64 -o CHANGELOG.md $TAG
@@ -274,6 +280,7 @@ jobs:
         azureSubscription: 'MMLSpark Build'
         keyVaultName: mmlspark-keys
     - bash: |
+        set -e
         source activate synapseml
         sbt publishPypi
       condition: startsWith(variables['tag'], 'v')
@@ -327,6 +334,7 @@ jobs:
         keyVaultName: mmlspark-keys
       condition: succeededOrFailed()
     - bash: |
+        set -e
         curl -s https://codecov.io/bash > .codecov
         chmod +x .codecov
         echo "Starting Codecov Upload"
@@ -377,6 +385,7 @@ jobs:
         keyVaultName: mmlspark-keys
       condition: succeededOrFailed()
     - bash: |
+        set -e
         curl -s https://codecov.io/bash > .codecov
         chmod +x .codecov
         echo "Starting Codecov Upload"
@@ -424,6 +433,7 @@ jobs:
         keyVaultName: mmlspark-keys
       condition: succeededOrFailed()
     - bash: |
+        set -e
         curl -s https://codecov.io/bash > .codecov
         chmod +x .codecov
         echo "Starting Codecov Upload"
@@ -460,12 +470,14 @@ jobs:
           source activate synapseml
           sbt convertNotebooks
     - bash: |
+        set -e
         yarn install
         cd website
         yarn
         yarn build
       displayName: 'yarn install and build'
     - bash: |
+        set -e
         git config --global user.name "${GH_NAME}"
         git config --global user.email "${GH_EMAIL}"
         git checkout -b main
@@ -594,6 +606,7 @@ jobs:
         keyVaultName: mmlspark-keys
       condition: succeededOrFailed()
     - bash: |
+        set -e
         curl -s https://codecov.io/bash > .codecov
         chmod +x .codecov
         echo "Starting Codecov Upload"

diff --git a/project/CodegenPlugin.scala b/project/CodegenPlugin.scala
@@ -224,7 +224,7 @@ object CodegenPlugin extends AutoPlugin {
       artifactPath.in(packageBin).in(Compile).value.getParentFile
     },
     mergePyCodeDir := {
-      join(baseDirectory.value.getParent, "target", "scala-2.12", "sbt-1.0", "generated")
+      join(baseDirectory.value.getParent, "target", "scala-2.12", "generated")
     },
     codegenDir := {
       join(targetDir.value, "generated")

diff --git a/website/docs/documentation/estimators/_LightGBM.md b/website/docs/documentation/estimators/_LightGBM.md
@@ -2,26 +2,8 @@ import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 import DocTable from "@theme/DocumentationTable";
 
-<!-- 
-```python
-import pyspark
-import os
-import json
-from IPython.display import display
-
-spark = (pyspark.sql.SparkSession.builder.appName("MyApp")
-        .config("spark.jars.packages", "com.microsoft.azure:synapseml:0.9.2")
-        .config("spark.jars.repositories", "https://mmlspark.azureedge.net/maven")
-        .getOrCreate())
-
-def getSecret(secretName):
-        get_secret_cmd = 'az keyvault secret show --vault-name mmlspark-build-keys --name {}'.format(secretName)
-        value = json.loads(os.popen(get_secret_cmd).read())["value"]
-        return value
-
-import synapse.ml
-```
--->
+
+
 
 ## LightGBMClassifier
 
@@ -87,27 +69,8 @@ values={[
 ]}>
 <TabItem value="py">
 
-<!-- 
-```python
-import pyspark
-import os
-import json
-from IPython.display import display
-from pyspark.sql.functions import *
-
-spark = (pyspark.sql.SparkSession.builder.appName("MyApp")
-        .config("spark.jars.packages", "com.microsoft.azure:synapseml:0.9.2")
-        .config("spark.jars.repositories", "https://mmlspark.azureedge.net/maven")
-        .getOrCreate())
-
-def getSecret(secretName):
-        get_secret_cmd = 'az keyvault secret show --vault-name mmlspark-build-keys --name {}'.format(secretName)
-        value = json.loads(os.popen(get_secret_cmd).read())["value"]
-        return value
-
-import synapse.ml
-```
--->
+
+
 
 <!--pytest-codeblocks:cont-->
 
@@ -159,27 +122,8 @@ values={[
 ]}>
 <TabItem value="py">
 
-<!-- 
-```python
-import pyspark
-import os
-import json
-from IPython.display import display
-from pyspark.sql.functions import *
-
-spark = (pyspark.sql.SparkSession.builder.appName("MyApp")
-        .config("spark.jars.packages", "com.microsoft.azure:synapseml:0.9.2")
-        .config("spark.jars.repositories", "https://mmlspark.azureedge.net/maven")
-        .getOrCreate())
-
-def getSecret(secretName):
-        get_secret_cmd = 'az keyvault secret show --vault-name mmlspark-build-keys --name {}'.format(secretName)
-        value = json.loads(os.popen(get_secret_cmd).read())["value"]
-        return value
-
-import synapse.ml
-```
--->
+
+
 
 <!--pytest-codeblocks:cont-->
 

diff --git a/website/docs/documentation/estimators/_VW.md b/website/docs/documentation/estimators/_VW.md
@@ -2,26 +2,8 @@ import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 import DocTable from "@theme/DocumentationTable";
 
-<!-- 
-```python
-import pyspark
-import os
-import json
-from IPython.display import display
-
-spark = (pyspark.sql.SparkSession.builder.appName("MyApp")
-        .config("spark.jars.packages", "com.microsoft.azure:synapseml:0.9.2")
-        .config("spark.jars.repositories", "https://mmlspark.azureedge.net/maven")
-        .getOrCreate())
-
-def getSecret(secretName):
-        get_secret_cmd = 'az keyvault secret show --vault-name mmlspark-build-keys --name {}'.format(secretName)
-        value = json.loads(os.popen(get_secret_cmd).read())["value"]
-        return value
-
-import synapse.ml
-```
--->
+
+
 
 ## VowpalWabbitRegressor
 
@@ -84,27 +66,8 @@ values={[
 ]}>
 <TabItem value="py">
 
-<!-- 
-```python
-import pyspark
-import os
-import json
-from IPython.display import display
-from pyspark.sql.functions import *
-
-spark = (pyspark.sql.SparkSession.builder.appName("MyApp")
-        .config("spark.jars.packages", "com.microsoft.azure:synapseml:0.9.2")
-        .config("spark.jars.repositories", "https://mmlspark.azureedge.net/maven")
-        .getOrCreate())
-
-def getSecret(secretName):
-        get_secret_cmd = 'az keyvault secret show --vault-name mmlspark-build-keys --name {}'.format(secretName)
-        value = json.loads(os.popen(get_secret_cmd).read())["value"]
-        return value
-
-import synapse.ml
-```
--->
+
+
 
 <!--pytest-codeblocks:cont-->
 

diff --git a/website/docs/documentation/estimators/core/_AutoML.md b/website/docs/documentation/estimators/core/_AutoML.md
@@ -2,26 +2,6 @@ import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 import DocTable from "@theme/DocumentationTable";
 
-<!-- 
-```python
-import pyspark
-import os
-import json
-from IPython.display import display
-
-spark = (pyspark.sql.SparkSession.builder.appName("MyApp")
-        .config("spark.jars.packages", "com.microsoft.azure:synapseml:0.9.2")
-        .config("spark.jars.repositories", "https://mmlspark.azureedge.net/maven")
-        .getOrCreate())
-
-def getSecret(secretName):
-        get_secret_cmd = 'az keyvault secret show --vault-name mmlspark-build-keys --name {}'.format(secretName)
-        value = json.loads(os.popen(get_secret_cmd).read())["value"]
-        return value
-
-import synapse.ml
-``` 
--->
 
 ## AutoML
 
@@ -143,27 +123,6 @@ values={[
 ]}>
 <TabItem value="py">
 
-<!-- 
-```python
-import pyspark
-import os
-import json
-from IPython.display import display
-from pyspark.sql.functions import *
-
-spark = (pyspark.sql.SparkSession.builder.appName("MyApp")
-        .config("spark.jars.packages", "com.microsoft.azure:synapseml:0.9.2")
-        .config("spark.jars.repositories", "https://mmlspark.azureedge.net/maven")
-        .getOrCreate())
-
-def getSecret(secretName):
-        get_secret_cmd = 'az keyvault secret show --vault-name mmlspark-build-keys --name {}'.format(secretName)
-        value = json.loads(os.popen(get_secret_cmd).read())["value"]
-        return value
-
-import synapse.ml
-```
--->
 
 <!--pytest-codeblocks:cont-->