From c6660d0b78a598a6767c1860d221216dcec8438c Mon Sep 17 00:00:00 2001 From: Mark Hamilton Date: Mon, 15 Nov 2021 12:04:52 -0500 Subject: [PATCH] docs: clean up some spacing issues in website (#1256) --- .../Interpretability - Image Explainers.ipynb | 114 +++--- website/docusaurus.config.js | 2 +- website/src/pages/index.js | 27 +- .../documentation/estimators/_LightGBM.md | 68 +--- .../documentation/estimators/_VW.md | 45 +-- .../documentation/estimators/core/_AutoML.md | 41 -- .../estimators/core/_Featurize.md | 112 +----- .../estimators/core/_IsolationForest.md | 22 +- .../documentation/estimators/core/_NN.md | 45 +-- .../estimators/core/_Recommendation.md | 45 +-- .../documentation/estimators/core/_Stages.md | 68 +--- .../documentation/estimators/core/_Train.md | 45 +-- .../documentation/transformers/_OpenCV.md | 44 +-- .../documentation/transformers/_VW.md | 88 +---- .../cognitive/_AnomalyDetection.md | 69 +--- .../transformers/cognitive/_AzureSearch.md | 24 -- .../cognitive/_BingImageSearch.md | 24 -- .../transformers/cognitive/_ComputerVision.md | 184 +-------- .../transformers/cognitive/_Face.md | 115 +----- .../transformers/cognitive/_FormRecognizer.md | 178 +-------- .../transformers/cognitive/_SpeechToText.md | 44 +-- .../transformers/cognitive/_TextAnalytics.md | 134 +------ .../transformers/cognitive/_Translator.md | 159 +------- .../transformers/core/_Explainers.md | 176 +-------- .../transformers/core/_Featurize.md | 88 +---- .../documentation/transformers/core/_IO.md | 154 +------- .../documentation/transformers/core/_Image.md | 66 +--- .../transformers/core/_Stages.md | 374 ++---------------- .../core/_SuperpixelTransformer.md | 22 +- .../documentation/transformers/core/_Train.md | 44 +-- .../transformers/deep_learning/_ONNXModel.md | 17 +- ...a Multilingual Search Engine from Forms.md | 7 +- .../Interpretability - Image Explainers.md | 57 --- 33 files changed, 282 insertions(+), 2420 deletions(-) diff --git a/notebooks/features/responsible_ai/Interpretability - Image Explainers.ipynb b/notebooks/features/responsible_ai/Interpretability - Image Explainers.ipynb index aaabe25faf..50829ab5e9 100644 --- a/notebooks/features/responsible_ai/Interpretability - Image Explainers.ipynb +++ b/notebooks/features/responsible_ai/Interpretability - Image Explainers.ipynb @@ -29,33 +29,33 @@ "import urllib.request\n", "import matplotlib.pyplot as plt\n", "import PIL, io\n", - "from PIL import Image\r\n", - "\r\n", - "vec_slice = udf(lambda vec, indices: (vec.toArray())[indices].tolist(), ArrayType(FloatType()))\r\n", - "arg_top_k = udf(lambda vec, k: (-vec.toArray()).argsort()[:k].tolist(), ArrayType(IntegerType()))\r\n", - "\r\n", - "def downloadBytes(url: str):\r\n", - " with urllib.request.urlopen(url) as url:\r\n", - " barr = url.read()\r\n", - " return barr\r\n", - "\r\n", - "def rotate_color_channel(bgr_image_array, height, width, nChannels):\r\n", - " B, G, R, *_ = np.asarray(bgr_image_array).reshape(height, width, nChannels).T\r\n", - " rgb_image_array = np.array((R, G, B)).T\r\n", - " return rgb_image_array\r\n", - " \r\n", - "def plot_superpixels(image_rgb_array, sp_clusters, weights, green_threshold=99):\r\n", - " superpixels = sp_clusters\r\n", - " green_value = np.percentile(weights, green_threshold)\r\n", - " img = Image.fromarray(image_rgb_array, mode='RGB').convert(\"RGBA\")\r\n", - " image_array = np.asarray(img).copy()\r\n", - " for (sp, v) in zip(superpixels, weights):\r\n", - " if v > green_value:\r\n", - " for (x, y) in sp:\r\n", - " image_array[y, x, 1] = 255\r\n", - " image_array[y, x, 3] = 200\r\n", - " plt.clf()\r\n", - " plt.imshow(image_array)\r\n", + "from PIL import Image\n", + "\n", + "vec_slice = udf(lambda vec, indices: (vec.toArray())[indices].tolist(), ArrayType(FloatType()))\n", + "arg_top_k = udf(lambda vec, k: (-vec.toArray()).argsort()[:k].tolist(), ArrayType(IntegerType()))\n", + "\n", + "def downloadBytes(url: str):\n", + " with urllib.request.urlopen(url) as url:\n", + " barr = url.read()\n", + " return barr\n", + "\n", + "def rotate_color_channel(bgr_image_array, height, width, nChannels):\n", + " B, G, R, *_ = np.asarray(bgr_image_array).reshape(height, width, nChannels).T\n", + " rgb_image_array = np.array((R, G, B)).T\n", + " return rgb_image_array\n", + " \n", + "def plot_superpixels(image_rgb_array, sp_clusters, weights, green_threshold=99):\n", + " superpixels = sp_clusters\n", + " green_value = np.percentile(weights, green_threshold)\n", + " img = Image.fromarray(image_rgb_array, mode='RGB').convert(\"RGBA\")\n", + " image_array = np.asarray(img).copy()\n", + " for (sp, v) in zip(superpixels, weights):\n", + " if v > green_value:\n", + " for (x, y) in sp:\n", + " image_array[y, x, 1] = 255\n", + " image_array[y, x, 3] = 200\n", + " plt.clf()\n", + " plt.imshow(image_array)\n", " display()" ], "outputs": [], @@ -74,36 +74,36 @@ "cell_type": "code", "execution_count": null, "source": [ - "from synapse.ml.io import *\r\n", - "\r\n", - "image_df = spark.read.image().load(\"wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg\")\r\n", - "display(image_df)\r\n", - "\r\n", - "# Rotate the image array from BGR into RGB channels for visualization later.\r\n", - "row = image_df.select(\"image.height\", \"image.width\", \"image.nChannels\", \"image.data\").head()\r\n", - "locals().update(row.asDict())\r\n", - "rgb_image_array = rotate_color_channel(data, height, width, nChannels)\r\n", - "\r\n", - "# Download the ONNX model\r\n", - "modelPayload = downloadBytes(\"https://mmlspark.blob.core.windows.net/publicwasb/ONNXModels/resnet50-v2-7.onnx\")\r\n", - "\r\n", - "featurizer = (\r\n", - " ImageTransformer(inputCol=\"image\", outputCol=\"features\")\r\n", - " .resize(224, True)\r\n", - " .centerCrop(224, 224)\r\n", - " .normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], color_scale_factor = 1/255)\r\n", - " .setTensorElementType(FloatType())\r\n", - ")\r\n", - "\r\n", - "onnx = (\r\n", - " ONNXModel()\r\n", - " .setModelPayload(modelPayload)\r\n", - " .setFeedDict({\"data\": \"features\"})\r\n", - " .setFetchDict({\"rawPrediction\": \"resnetv24_dense0_fwd\"})\r\n", - " .setSoftMaxDict({\"rawPrediction\": \"probability\"})\r\n", - " .setMiniBatchSize(1)\r\n", - ")\r\n", - "\r\n", + "from synapse.ml.io import *\n", + "\n", + "image_df = spark.read.image().load(\"wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg\")\n", + "display(image_df)\n", + "\n", + "# Rotate the image array from BGR into RGB channels for visualization later.\n", + "row = image_df.select(\"image.height\", \"image.width\", \"image.nChannels\", \"image.data\").head()\n", + "locals().update(row.asDict())\n", + "rgb_image_array = rotate_color_channel(data, height, width, nChannels)\n", + "\n", + "# Download the ONNX model\n", + "modelPayload = downloadBytes(\"https://mmlspark.blob.core.windows.net/publicwasb/ONNXModels/resnet50-v2-7.onnx\")\n", + "\n", + "featurizer = (\n", + " ImageTransformer(inputCol=\"image\", outputCol=\"features\")\n", + " .resize(224, True)\n", + " .centerCrop(224, 224)\n", + " .normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], color_scale_factor = 1/255)\n", + " .setTensorElementType(FloatType())\n", + ")\n", + "\n", + "onnx = (\n", + " ONNXModel()\n", + " .setModelPayload(modelPayload)\n", + " .setFeedDict({\"data\": \"features\"})\n", + " .setFetchDict({\"rawPrediction\": \"resnetv24_dense0_fwd\"})\n", + " .setSoftMaxDict({\"rawPrediction\": \"probability\"})\n", + " .setMiniBatchSize(1)\n", + ")\n", + "\n", "model = Pipeline(stages=[featurizer, onnx]).fit(image_df)" ], "outputs": [], diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index fe672fbfac..4499523ca3 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -3,7 +3,7 @@ const {all_examples} = require('./src/plugins/examples'); let version = "0.9.1"; module.exports = { - title: 'Synapse ML', + title: 'SynapseML', tagline: 'Simple and Distributed Machine Learning', url: 'https://microsoft.github.io', baseUrl: '/SynapseML/', diff --git a/website/src/pages/index.js b/website/src/pages/index.js index 624ee5c3ee..52906b6b40 100644 --- a/website/src/pages/index.js +++ b/website/src/pages/index.js @@ -61,13 +61,26 @@ interpretation_df = (TabularSHAP() config: `from synapse.ml.lightgbm import * quantile_df = (LightGBMRegressor() - .setApplication('quantile') - .setAlpha(0.3) - .setLearningRate(0.3) - .setNumIterations(100) - .setNumLeaves(31) - .fit(train_df) - .transform(test_df))`, + .setApplication('quantile') + .setAlpha(0.3) + .setLearningRate(0.3) + .setNumIterations(100) + .setNumLeaves(31) + .fit(train_df) + .transform(test_df))`, + }, + { + label: "OpenCV", + further: "docs/features/opencv/OpenCV%20-%20Pipeline%20Image%20Transformations", + config: `from synapse.ml.opencv import * + +image_df = (ImageTransformer() + .setInputCol("images") + .setOutputCol("transformed_images") + .resize(224, True) + .centerCrop(224, 224) + .normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], color_scale_factor = 1/255) + .transform(input_df))`, }, ]; diff --git a/website/versioned_docs/version-0.9.1/documentation/estimators/_LightGBM.md b/website/versioned_docs/version-0.9.1/documentation/estimators/_LightGBM.md index a9927b744f..26fea2654c 100644 --- a/website/versioned_docs/version-0.9.1/documentation/estimators/_LightGBM.md +++ b/website/versioned_docs/version-0.9.1/documentation/estimators/_LightGBM.md @@ -2,26 +2,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - + + ## LightGBMClassifier @@ -87,27 +69,8 @@ values={[ ]}> - + + @@ -159,27 +122,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/estimators/_VW.md b/website/versioned_docs/version-0.9.1/documentation/estimators/_VW.md index 2e1348fef7..ca108aed1a 100644 --- a/website/versioned_docs/version-0.9.1/documentation/estimators/_VW.md +++ b/website/versioned_docs/version-0.9.1/documentation/estimators/_VW.md @@ -2,26 +2,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - + + ## VowpalWabbitRegressor @@ -84,27 +66,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/estimators/core/_AutoML.md b/website/versioned_docs/version-0.9.1/documentation/estimators/core/_AutoML.md index b4d092ca52..0f6a009c6a 100644 --- a/website/versioned_docs/version-0.9.1/documentation/estimators/core/_AutoML.md +++ b/website/versioned_docs/version-0.9.1/documentation/estimators/core/_AutoML.md @@ -2,26 +2,6 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - ## AutoML @@ -143,27 +123,6 @@ values={[ ]}> - diff --git a/website/versioned_docs/version-0.9.1/documentation/estimators/core/_Featurize.md b/website/versioned_docs/version-0.9.1/documentation/estimators/core/_Featurize.md index 92157a5d34..3f5dbfbaea 100644 --- a/website/versioned_docs/version-0.9.1/documentation/estimators/core/_Featurize.md +++ b/website/versioned_docs/version-0.9.1/documentation/estimators/core/_Featurize.md @@ -2,26 +2,6 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - ## Featurize @@ -112,27 +92,8 @@ values={[ ]}> - + + @@ -189,27 +150,8 @@ values={[ ]}> - + + @@ -277,27 +219,8 @@ values={[ ]}> - + + @@ -352,27 +275,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/estimators/core/_IsolationForest.md b/website/versioned_docs/version-0.9.1/documentation/estimators/core/_IsolationForest.md index ae84a4936f..7153fe995d 100644 --- a/website/versioned_docs/version-0.9.1/documentation/estimators/core/_IsolationForest.md +++ b/website/versioned_docs/version-0.9.1/documentation/estimators/core/_IsolationForest.md @@ -2,26 +2,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - + + ## Isolation Forest diff --git a/website/versioned_docs/version-0.9.1/documentation/estimators/core/_NN.md b/website/versioned_docs/version-0.9.1/documentation/estimators/core/_NN.md index 02092cdcf5..4b917bd8ff 100644 --- a/website/versioned_docs/version-0.9.1/documentation/estimators/core/_NN.md +++ b/website/versioned_docs/version-0.9.1/documentation/estimators/core/_NN.md @@ -2,26 +2,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - + + ## NN @@ -76,27 +58,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/estimators/core/_Recommendation.md b/website/versioned_docs/version-0.9.1/documentation/estimators/core/_Recommendation.md index 992ad410ca..3c4c4b5924 100644 --- a/website/versioned_docs/version-0.9.1/documentation/estimators/core/_Recommendation.md +++ b/website/versioned_docs/version-0.9.1/documentation/estimators/core/_Recommendation.md @@ -2,26 +2,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - + + ## Recommendation @@ -244,27 +226,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/estimators/core/_Stages.md b/website/versioned_docs/version-0.9.1/documentation/estimators/core/_Stages.md index c1c910ad58..0d2b1a2c1a 100644 --- a/website/versioned_docs/version-0.9.1/documentation/estimators/core/_Stages.md +++ b/website/versioned_docs/version-0.9.1/documentation/estimators/core/_Stages.md @@ -2,26 +2,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - + + ## Stages @@ -100,27 +82,8 @@ values={[ ]}> - + + @@ -186,27 +149,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/estimators/core/_Train.md b/website/versioned_docs/version-0.9.1/documentation/estimators/core/_Train.md index 0cda3a70b0..7bae6555ab 100644 --- a/website/versioned_docs/version-0.9.1/documentation/estimators/core/_Train.md +++ b/website/versioned_docs/version-0.9.1/documentation/estimators/core/_Train.md @@ -2,26 +2,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - + + ## Train @@ -112,27 +94,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/transformers/_OpenCV.md b/website/versioned_docs/version-0.9.1/documentation/transformers/_OpenCV.md index 3864bfe6d7..49744e75f0 100644 --- a/website/versioned_docs/version-0.9.1/documentation/transformers/_OpenCV.md +++ b/website/versioned_docs/version-0.9.1/documentation/transformers/_OpenCV.md @@ -2,26 +2,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - + + ## ImageTransformer @@ -88,26 +70,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/transformers/_VW.md b/website/versioned_docs/version-0.9.1/documentation/transformers/_VW.md index f647433eb9..1f46e7e9a1 100644 --- a/website/versioned_docs/version-0.9.1/documentation/transformers/_VW.md +++ b/website/versioned_docs/version-0.9.1/documentation/transformers/_VW.md @@ -2,26 +2,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - + + ## VectorZipper @@ -112,26 +94,8 @@ values={[ - + + @@ -178,26 +142,8 @@ values={[ - + + @@ -246,26 +192,8 @@ values={[ - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_AnomalyDetection.md b/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_AnomalyDetection.md index 185783181d..7aaa87ebcf 100644 --- a/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_AnomalyDetection.md +++ b/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_AnomalyDetection.md @@ -2,27 +2,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - + + ## Anomaly Detection @@ -136,27 +117,8 @@ values={[ ]}> - + + @@ -254,27 +216,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_AzureSearch.md b/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_AzureSearch.md index bedacee18b..05dae98558 100644 --- a/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_AzureSearch.md +++ b/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_AzureSearch.md @@ -2,30 +2,6 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - ## Azure Search diff --git a/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_BingImageSearch.md b/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_BingImageSearch.md index 7855fbd15f..d3eb0b2000 100644 --- a/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_BingImageSearch.md +++ b/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_BingImageSearch.md @@ -2,30 +2,6 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - ## Bing Image Search diff --git a/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_ComputerVision.md b/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_ComputerVision.md index 671d7267da..ab3c4b83a3 100644 --- a/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_ComputerVision.md +++ b/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_ComputerVision.md @@ -2,27 +2,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - + + ## Computer Vision @@ -99,27 +80,8 @@ values={[ ]}> - + + @@ -191,27 +153,8 @@ values={[ ]}> - + + @@ -280,27 +223,8 @@ values={[ ]}> - + + @@ -367,27 +291,8 @@ values={[ ]}> - + + @@ -450,27 +355,8 @@ values={[ ]}> - + + @@ -537,27 +423,8 @@ values={[ ]}> - + + @@ -618,27 +485,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_Face.md b/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_Face.md index 625cf6f5c9..d2f0ed4616 100644 --- a/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_Face.md +++ b/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_Face.md @@ -2,27 +2,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - + + ## Face @@ -104,27 +85,8 @@ values={[ ]}> - + + @@ -216,27 +178,8 @@ values={[ ]}> - + + @@ -326,27 +269,8 @@ values={[ ]}> - + + @@ -401,27 +325,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_FormRecognizer.md b/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_FormRecognizer.md index de5dd48ed9..5ec86fa76d 100644 --- a/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_FormRecognizer.md +++ b/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_FormRecognizer.md @@ -2,27 +2,6 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - ## Form Recognizer @@ -104,27 +83,8 @@ values={[ ]}> - + + @@ -189,27 +149,8 @@ values={[ ]}> - + + @@ -272,27 +213,6 @@ values={[ ]}> - @@ -358,27 +278,6 @@ values={[ ]}> - @@ -444,27 +343,8 @@ values={[ ]}> - + + @@ -537,27 +417,8 @@ values={[ ]}> - + + @@ -624,27 +485,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_SpeechToText.md b/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_SpeechToText.md index 068c9ea148..d5dba97d92 100644 --- a/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_SpeechToText.md +++ b/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_SpeechToText.md @@ -2,27 +2,6 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - ## Speech To Text @@ -105,27 +84,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_TextAnalytics.md b/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_TextAnalytics.md index 6d11f26d86..882e1abf2d 100644 --- a/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_TextAnalytics.md +++ b/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_TextAnalytics.md @@ -2,27 +2,6 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - ## Text Analytics @@ -99,27 +78,6 @@ values={[ ]}> - @@ -186,27 +144,8 @@ values={[ ]}> - + + @@ -275,27 +214,8 @@ values={[ ]}> - + + @@ -359,27 +279,8 @@ values={[ ]}> - + + @@ -446,27 +347,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_Translator.md b/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_Translator.md index 557041141d..1e14de7703 100644 --- a/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_Translator.md +++ b/website/versioned_docs/version-0.9.1/documentation/transformers/cognitive/_Translator.md @@ -2,27 +2,6 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - ## Translator @@ -106,27 +85,8 @@ values={[ ]}> - + + @@ -200,27 +160,8 @@ values={[ ]}> - + + @@ -286,27 +227,8 @@ values={[ ]}> - + + @@ -372,27 +294,8 @@ values={[ ]}> - + + @@ -464,27 +367,8 @@ values={[ ]}> - + + @@ -552,27 +436,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Explainers.md b/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Explainers.md index bfc381e17f..dbe6bcf076 100644 --- a/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Explainers.md +++ b/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Explainers.md @@ -2,26 +2,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - + + ## Explainers @@ -96,26 +78,8 @@ values={[ ]}> - + + @@ -181,26 +145,8 @@ values={[ ]}> - + + @@ -273,26 +219,8 @@ values={[ ]}> - + + @@ -363,26 +291,8 @@ values={[ ]}> - + + @@ -443,26 +353,8 @@ values={[ ]}> - + + @@ -521,26 +413,8 @@ values={[ ]}> - + + @@ -621,26 +495,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Featurize.md b/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Featurize.md index dd8f7b0fcb..82beef09ba 100644 --- a/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Featurize.md +++ b/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Featurize.md @@ -2,26 +2,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - + + ## Featurize @@ -92,26 +74,8 @@ values={[ ]}> - + + @@ -175,26 +139,8 @@ values={[ ]}> - + + @@ -271,26 +217,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/transformers/core/_IO.md b/website/versioned_docs/version-0.9.1/documentation/transformers/core/_IO.md index 91cd9e8b02..19569f157a 100644 --- a/website/versioned_docs/version-0.9.1/documentation/transformers/core/_IO.md +++ b/website/versioned_docs/version-0.9.1/documentation/transformers/core/_IO.md @@ -2,26 +2,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - + + ## IO @@ -87,26 +69,8 @@ values={[ ]}> - + + @@ -158,26 +122,8 @@ values={[ ]}> - + + @@ -221,26 +167,8 @@ values={[ ]}> - + + @@ -286,26 +214,8 @@ values={[ ]}> - + + @@ -347,26 +257,8 @@ values={[ ]}> - + + @@ -409,26 +301,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Image.md b/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Image.md index a96de7be85..3d6c452145 100644 --- a/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Image.md +++ b/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Image.md @@ -2,26 +2,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - + + ## Image @@ -90,26 +72,8 @@ values={[ ]}> - + + @@ -179,26 +143,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Stages.md b/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Stages.md index 82b57e7561..b8cc3a0ac1 100644 --- a/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Stages.md +++ b/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Stages.md @@ -2,26 +2,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - + + ## Stages @@ -105,26 +87,8 @@ values={[ ]}> - + + @@ -178,26 +142,8 @@ values={[ ]}> - + + @@ -259,26 +205,8 @@ values={[ ]}> - + + @@ -332,26 +260,8 @@ values={[ ]}> - + + @@ -416,26 +326,8 @@ values={[ ]}> - + + @@ -482,26 +374,8 @@ values={[ ]}> - + + @@ -543,26 +417,8 @@ values={[ ]}> - + + @@ -612,26 +468,8 @@ values={[ ]}> - + + @@ -681,26 +519,8 @@ values={[ ]}> - + + @@ -754,26 +574,8 @@ values={[ ]}> - + + @@ -845,26 +647,8 @@ values={[ ]}> - + + @@ -918,26 +702,8 @@ values={[ ]}> - + + @@ -1007,26 +773,8 @@ values={[ ]}> - + + @@ -1080,26 +828,8 @@ values={[ ]}> - + + @@ -1173,26 +903,8 @@ values={[ ]}> - + + @@ -1258,26 +970,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/transformers/core/_SuperpixelTransformer.md b/website/versioned_docs/version-0.9.1/documentation/transformers/core/_SuperpixelTransformer.md index ae2583711b..1c7c06890d 100644 --- a/website/versioned_docs/version-0.9.1/documentation/transformers/core/_SuperpixelTransformer.md +++ b/website/versioned_docs/version-0.9.1/documentation/transformers/core/_SuperpixelTransformer.md @@ -2,26 +2,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - + + ## LIME diff --git a/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Train.md b/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Train.md index 5077cfaa31..7247fc87f2 100644 --- a/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Train.md +++ b/website/versioned_docs/version-0.9.1/documentation/transformers/core/_Train.md @@ -2,26 +2,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - + + ## Train @@ -92,26 +74,8 @@ values={[ ]}> - + + diff --git a/website/versioned_docs/version-0.9.1/documentation/transformers/deep_learning/_ONNXModel.md b/website/versioned_docs/version-0.9.1/documentation/transformers/deep_learning/_ONNXModel.md index 9e696c891e..84fd7f1df6 100644 --- a/website/versioned_docs/version-0.9.1/documentation/transformers/deep_learning/_ONNXModel.md +++ b/website/versioned_docs/version-0.9.1/documentation/transformers/deep_learning/_ONNXModel.md @@ -2,16 +2,6 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import DocTable from "@theme/DocumentationTable"; - ## ONNXModel @@ -24,14 +14,13 @@ values={[ ```py -import synapse.ml from synapse.ml.onnx import ONNXModel model_path = "PUT_YOUR_MODEL_PATH" onnx_ml = (ONNXModel() .setModelLocation(model_path) .setFeedDict({"float_input": "features"}) - .setFetchDict({"prediction": "output_label", "rawProbability": "output_probability"}) + .setFetchDict({"prediction": "output_label", "rawProbability": "output_probability"})) ``` @@ -41,10 +30,10 @@ onnx_ml = (ONNXModel() import com.microsoft.azure.synapse.ml.onnx._ val model_path = "PUT_YOUR_MODEL_PATH" -val onnx_ml = new ONNXModel() +val onnx_ml = (new ONNXModel() .setModelLocation(model_path) .setFeedDict(Map("float_input" -> "features")) - .setFetchDict(Map("prediction" -> "output_label", "rawProbability" -> "output_probability")) + .setFetchDict(Map("prediction" -> "output_label", "rawProbability" -> "output_probability"))) ``` diff --git a/website/versioned_docs/version-0.9.1/features/cognitive_services/CognitiveServices - Create a Multilingual Search Engine from Forms.md b/website/versioned_docs/version-0.9.1/features/cognitive_services/CognitiveServices - Create a Multilingual Search Engine from Forms.md index 0397538a50..f8c47e8e84 100644 --- a/website/versioned_docs/version-0.9.1/features/cognitive_services/CognitiveServices - Create a Multilingual Search Engine from Forms.md +++ b/website/versioned_docs/version-0.9.1/features/cognitive_services/CognitiveServices - Create a Multilingual Search Engine from Forms.md @@ -28,9 +28,8 @@ def blob_to_url(blob): df2 = (spark.read.format("binaryFile") - .load("wasbs://ignite2021@mmlsparkdemo.blob.core.windows.net/forms/*") + .load("wasbs://ignite2021@mmlsparkdemo.blob.core.windows.net/form_subset/*") .select("path") - .coalesce(24) .limit(10) .select(udf(blob_to_url, StringType())("path").alias("url")) .cache() @@ -78,7 +77,7 @@ from synapse.ml.cognitive import FormOntologyLearner organized_df = (FormOntologyLearner() .setInputCol("invoices") .setOutputCol("extracted") - .fit(analyzed_df.limit(10)) + .fit(analyzed_df) .transform(analyzed_df) .select("url", "extracted.*") .cache()) @@ -107,7 +106,7 @@ display(itemized_df) ```python -display(itemized_df.where(col("ProductCode") == 6)) +display(itemized_df.where(col("ProductCode") == 48)) ``` diff --git a/website/versioned_docs/version-0.9.1/features/responsible_ai/Interpretability - Image Explainers.md b/website/versioned_docs/version-0.9.1/features/responsible_ai/Interpretability - Image Explainers.md index 4eea455b27..ca1048c138 100644 --- a/website/versioned_docs/version-0.9.1/features/responsible_ai/Interpretability - Image Explainers.md +++ b/website/versioned_docs/version-0.9.1/features/responsible_ai/Interpretability - Image Explainers.md @@ -27,58 +27,31 @@ import matplotlib.pyplot as plt import PIL, io from PIL import Image - - vec_slice = udf(lambda vec, indices: (vec.toArray())[indices].tolist(), ArrayType(FloatType())) - arg_top_k = udf(lambda vec, k: (-vec.toArray()).argsort()[:k].tolist(), ArrayType(IntegerType())) - - def downloadBytes(url: str): - with urllib.request.urlopen(url) as url: - barr = url.read() - return barr - - def rotate_color_channel(bgr_image_array, height, width, nChannels): - B, G, R, *_ = np.asarray(bgr_image_array).reshape(height, width, nChannels).T - rgb_image_array = np.array((R, G, B)).T - return rgb_image_array - - def plot_superpixels(image_rgb_array, sp_clusters, weights, green_threshold=99): - superpixels = sp_clusters - green_value = np.percentile(weights, green_threshold) - img = Image.fromarray(image_rgb_array, mode='RGB').convert("RGBA") - image_array = np.asarray(img).copy() - for (sp, v) in zip(superpixels, weights): - if v > green_value: - for (x, y) in sp: - image_array[y, x, 1] = 255 - image_array[y, x, 3] = 200 - plt.clf() - plt.imshow(image_array) - display() ``` @@ -90,64 +63,34 @@ The result shows 39.6% probability of "violin" (889), and 38.4% probability of " ```python from synapse.ml.io import * - - image_df = spark.read.image().load("wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg") - display(image_df) - - # Rotate the image array from BGR into RGB channels for visualization later. - row = image_df.select("image.height", "image.width", "image.nChannels", "image.data").head() - locals().update(row.asDict()) - rgb_image_array = rotate_color_channel(data, height, width, nChannels) - - # Download the ONNX model - modelPayload = downloadBytes("https://mmlspark.blob.core.windows.net/publicwasb/ONNXModels/resnet50-v2-7.onnx") - - featurizer = ( - ImageTransformer(inputCol="image", outputCol="features") - .resize(224, True) - .centerCrop(224, 224) - .normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], color_scale_factor = 1/255) - .setTensorElementType(FloatType()) - ) - - onnx = ( - ONNXModel() - .setModelPayload(modelPayload) - .setFeedDict({"data": "features"}) - .setFetchDict({"rawPrediction": "resnetv24_dense0_fwd"}) - .setSoftMaxDict({"rawPrediction": "probability"}) - .setMiniBatchSize(1) - ) - - model = Pipeline(stages=[featurizer, onnx]).fit(image_df) ```