Skip to content

Commit

Permalink
feat: add certified event feature categories (#2109)
Browse files Browse the repository at this point in the history
* add certified event feature categories

* fix scala style

* rename cognitiveservices to aiservices
  • Loading branch information
eisber authored Oct 26, 2023
1 parent 2f31f74 commit 5c0b803
Show file tree
Hide file tree
Showing 110 changed files with 353 additions and 311 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import com.microsoft.azure.synapse.ml.cognitive.anomaly.AnomalyDetectorProtocol.
import com.microsoft.azure.synapse.ml.core.contracts.HasOutputCol
import com.microsoft.azure.synapse.ml.core.schema.DatasetExtensions
import com.microsoft.azure.synapse.ml.io.http.ErrorUtils
import com.microsoft.azure.synapse.ml.logging.SynapseMLLogging
import com.microsoft.azure.synapse.ml.logging.{FeatureNames, SynapseMLLogging}
import com.microsoft.azure.synapse.ml.param.ServiceParam
import org.apache.http.entity.{AbstractHttpEntity, StringEntity}
import org.apache.spark.injections.UDFUtils
Expand Down Expand Up @@ -148,7 +148,7 @@ abstract class AnomalyDetectorBase(override val uid: String) extends CognitiveSe
object DetectLastAnomaly extends ComplexParamsReadable[DetectLastAnomaly] with Serializable

class DetectLastAnomaly(override val uid: String) extends AnomalyDetectorBase(uid) with SynapseMLLogging {
logClass()
logClass(FeatureNames.AiServices.Anomaly)

def this() = this(Identifiable.randomUID("DetectLastAnomaly"))

Expand All @@ -165,7 +165,7 @@ class DetectLastAnomaly(override val uid: String) extends AnomalyDetectorBase(ui
object DetectAnomalies extends ComplexParamsReadable[DetectAnomalies] with Serializable

class DetectAnomalies(override val uid: String) extends AnomalyDetectorBase(uid) with SynapseMLLogging {
logClass()
logClass(FeatureNames.AiServices.Anomaly)

def this() = this(Identifiable.randomUID("DetectAnomalies"))

Expand All @@ -183,7 +183,7 @@ object SimpleDetectAnomalies extends ComplexParamsReadable[SimpleDetectAnomalies

class SimpleDetectAnomalies(override val uid: String) extends AnomalyDetectorBase(uid)
with HasOutputCol with SynapseMLLogging {
logClass()
logClass(FeatureNames.AiServices.Anomaly)

def this() = this(Identifiable.randomUID("SimpleDetectAnomalies"))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import com.microsoft.azure.synapse.ml.core.schema.DatasetExtensions
import com.microsoft.azure.synapse.ml.io.http.HandlingUtils.{convertAndClose, sendWithRetries}
import com.microsoft.azure.synapse.ml.io.http.RESTHelpers.{Client, retry}
import com.microsoft.azure.synapse.ml.io.http._
import com.microsoft.azure.synapse.ml.logging.SynapseMLLogging
import com.microsoft.azure.synapse.ml.logging.{FeatureNames, SynapseMLLogging}
import com.microsoft.azure.synapse.ml.stages._
import com.microsoft.azure.synapse.ml.param.CognitiveServiceStructParam
import org.apache.commons.io.IOUtils
Expand Down Expand Up @@ -416,7 +416,7 @@ object SimpleFitMultivariateAnomaly extends ComplexParamsReadable[SimpleFitMulti

class SimpleFitMultivariateAnomaly(override val uid: String) extends Estimator[SimpleDetectMultivariateAnomaly]
with MADBase {
logClass()
logClass(FeatureNames.AiServices.Anomaly)

def this() = this(Identifiable.randomUID("SimpleFitMultivariateAnomaly"))

Expand Down Expand Up @@ -569,7 +569,7 @@ object SimpleDetectMultivariateAnomaly extends ComplexParamsReadable[SimpleDetec

class SimpleDetectMultivariateAnomaly(override val uid: String) extends Model[SimpleDetectMultivariateAnomaly]
with MADBase with HasHandler with DetectMAParams {
logClass()
logClass(FeatureNames.AiServices.Anomaly)

def this() = this(Identifiable.randomUID("SimpleDetectMultivariateAnomaly"))

Expand Down Expand Up @@ -654,7 +654,7 @@ class DetectLastMultivariateAnomaly(override val uid: String) extends CognitiveS
with HasSetLocation with HasCognitiveServiceInput with HasBatchSize
with ComplexParamsWritable with Wrappable
with HasErrorCol with SynapseMLLogging with DetectMAParams {
logClass()
logClass(FeatureNames.AiServices.Anomaly)

def this() = this(Identifiable.randomUID("DetectLastMultivariateAnomaly"))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ package com.microsoft.azure.synapse.ml.cognitive.bing

import com.microsoft.azure.synapse.ml.cognitive._
import com.microsoft.azure.synapse.ml.core.utils.AsyncUtils
import com.microsoft.azure.synapse.ml.logging.SynapseMLLogging
import com.microsoft.azure.synapse.ml.logging.{FeatureNames, SynapseMLLogging}
import com.microsoft.azure.synapse.ml.param.ServiceParam
import com.microsoft.azure.synapse.ml.stages.Lambda
import org.apache.commons.io.IOUtils
Expand Down Expand Up @@ -67,7 +67,7 @@ object BingImageSearch extends ComplexParamsReadable[BingImageSearch] with Seria
class BingImageSearch(override val uid: String)
extends CognitiveServicesBase(uid)
with HasCognitiveServiceInput with HasInternalJsonOutputParser with SynapseMLLogging with HasSetLinkedService {
logClass()
logClass(FeatureNames.AiServices.BingImage)

override protected lazy val pyInternalWrapper = true

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ package com.microsoft.azure.synapse.ml.cognitive.face

import com.microsoft.azure.synapse.ml.cognitive._
import com.microsoft.azure.synapse.ml.cognitive.vision.HasImageUrl
import com.microsoft.azure.synapse.ml.logging.SynapseMLLogging
import com.microsoft.azure.synapse.ml.logging.{FeatureNames, SynapseMLLogging}
import com.microsoft.azure.synapse.ml.param.ServiceParam
import org.apache.http.entity.{AbstractHttpEntity, StringEntity}
import org.apache.spark.ml.ComplexParamsReadable
Expand All @@ -21,7 +21,7 @@ class DetectFace(override val uid: String)
extends CognitiveServicesBase(uid) with HasImageUrl with HasServiceParams
with HasCognitiveServiceInput with HasInternalJsonOutputParser with HasSetLocation with SynapseMLLogging
with HasSetLinkedService {
logClass()
logClass(FeatureNames.AiServices.Face)

def this() = this(Identifiable.randomUID("DetectFace"))

Expand Down Expand Up @@ -100,7 +100,7 @@ class FindSimilarFace(override val uid: String)
with HasMaxNumOfCandidatesReturned with HasFaceIds
with HasCognitiveServiceInput with HasInternalJsonOutputParser with HasSetLocation with SynapseMLLogging
with HasSetLinkedService {
logClass()
logClass(FeatureNames.AiServices.Face)

def this() = this(Identifiable.randomUID("FindSimilarFace"))

Expand Down Expand Up @@ -189,7 +189,7 @@ class GroupFaces(override val uid: String)
extends CognitiveServicesBase(uid) with HasServiceParams
with HasFaceIds with HasSetLocation
with HasCognitiveServiceInput with HasInternalJsonOutputParser with SynapseMLLogging with HasSetLinkedService {
logClass()
logClass(FeatureNames.AiServices.Face)

def this() = this(Identifiable.randomUID("GroupFaces"))

Expand All @@ -212,7 +212,7 @@ class IdentifyFaces(override val uid: String)
with HasMaxNumOfCandidatesReturned with HasFaceIds
with HasCognitiveServiceInput with HasInternalJsonOutputParser with HasSetLocation with SynapseMLLogging
with HasSetLinkedService {
logClass()
logClass(FeatureNames.AiServices.Face)

def this() = this(Identifiable.randomUID("IdentifyFaces"))

Expand Down Expand Up @@ -281,7 +281,7 @@ class VerifyFaces(override val uid: String)
extends CognitiveServicesBase(uid) with HasServiceParams
with HasCognitiveServiceInput with HasInternalJsonOutputParser with HasSetLocation with SynapseMLLogging
with HasSetLinkedService {
logClass()
logClass(FeatureNames.AiServices.Face)

def this() = this(Identifiable.randomUID("VerifyFaces"))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ package com.microsoft.azure.synapse.ml.cognitive.form

import com.microsoft.azure.synapse.ml.codegen.Wrappable
import com.microsoft.azure.synapse.ml.core.contracts.{HasInputCol, HasOutputCol}
import com.microsoft.azure.synapse.ml.logging.SynapseMLLogging
import com.microsoft.azure.synapse.ml.logging.{FeatureNames, SynapseMLLogging}
import com.microsoft.azure.synapse.ml.param.DataTypeParam
import org.apache.spark.injections.UDFUtils
import org.apache.spark.ml.param.ParamMap
Expand Down Expand Up @@ -42,7 +42,7 @@ object FormOntologyLearner extends DefaultParamsReadable[FormOntologyLearner] {

class FormOntologyLearner(override val uid: String) extends Estimator[FormOntologyTransformer]
with SynapseMLLogging with DefaultParamsWritable with HasInputCol with HasOutputCol with Wrappable {
logClass()
logClass(FeatureNames.AiServices.Form)

def this() = this(Identifiable.randomUID("FormOntologyLearner"))

Expand Down Expand Up @@ -87,7 +87,7 @@ object FormOntologyTransformer extends ComplexParamsReadable[FormOntologyTransfo

class FormOntologyTransformer(override val uid: String) extends Model[FormOntologyTransformer]
with SynapseMLLogging with ComplexParamsWritable with HasInputCol with HasOutputCol with Wrappable {
logClass()
logClass(FeatureNames.AiServices.Form)

val ontology: DataTypeParam = new DataTypeParam(
parent = this,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ package com.microsoft.azure.synapse.ml.cognitive.form

import com.microsoft.azure.synapse.ml.cognitive._
import com.microsoft.azure.synapse.ml.cognitive.vision.{BasicAsyncReply, HasImageInput, ReadLine}
import com.microsoft.azure.synapse.ml.logging.SynapseMLLogging
import com.microsoft.azure.synapse.ml.logging.{FeatureNames, SynapseMLLogging}
import com.microsoft.azure.synapse.ml.param.ServiceParam
import com.microsoft.azure.synapse.ml.stages.UDFTransformer
import org.apache.http.client.methods.{HttpGet, HttpRequestBase}
Expand Down Expand Up @@ -187,7 +187,7 @@ object AnalyzeLayout extends ComplexParamsReadable[AnalyzeLayout]

class AnalyzeLayout(override val uid: String) extends FormRecognizerBase(uid)
with SynapseMLLogging with HasPages {
logClass()
logClass(FeatureNames.AiServices.Form)

def this() = this(Identifiable.randomUID("AnalyzeLayout"))

Expand Down Expand Up @@ -220,7 +220,7 @@ object AnalyzeReceipts extends ComplexParamsReadable[AnalyzeReceipts]

class AnalyzeReceipts(override val uid: String) extends FormRecognizerBase(uid)
with SynapseMLLogging with HasPages with HasTextDetails with HasLocale {
logClass()
logClass(FeatureNames.AiServices.Form)

def this() = this(Identifiable.randomUID("AnalyzeReceipts"))

Expand All @@ -234,7 +234,7 @@ object AnalyzeBusinessCards extends ComplexParamsReadable[AnalyzeBusinessCards]

class AnalyzeBusinessCards(override val uid: String) extends FormRecognizerBase(uid)
with SynapseMLLogging with HasPages with HasTextDetails with HasLocale {
logClass()
logClass(FeatureNames.AiServices.Form)

def this() = this(Identifiable.randomUID("AnalyzeBusinessCards"))

Expand All @@ -248,7 +248,7 @@ object AnalyzeInvoices extends ComplexParamsReadable[AnalyzeInvoices]

class AnalyzeInvoices(override val uid: String) extends FormRecognizerBase(uid)
with SynapseMLLogging with HasPages with HasTextDetails with HasLocale {
logClass()
logClass(FeatureNames.AiServices.Form)

def this() = this(Identifiable.randomUID("AnalyzeInvoices"))

Expand All @@ -262,7 +262,7 @@ object AnalyzeIDDocuments extends ComplexParamsReadable[AnalyzeIDDocuments]

class AnalyzeIDDocuments(override val uid: String) extends FormRecognizerBase(uid)
with SynapseMLLogging with HasPages with HasTextDetails {
logClass()
logClass(FeatureNames.AiServices.Form)

def this() = this(Identifiable.randomUID("AnalyzeIDDocuments"))

Expand All @@ -277,7 +277,7 @@ object ListCustomModels extends ComplexParamsReadable[ListCustomModels]
class ListCustomModels(override val uid: String) extends CognitiveServicesBase(uid)
with HasCognitiveServiceInput with HasInternalJsonOutputParser
with HasSetLocation with HasSetLinkedService with SynapseMLLogging {
logClass()
logClass(FeatureNames.AiServices.Form)

def this() = this(Identifiable.randomUID("ListCustomModels"))

Expand All @@ -302,7 +302,7 @@ object GetCustomModel extends ComplexParamsReadable[GetCustomModel]
class GetCustomModel(override val uid: String) extends CognitiveServicesBase(uid)
with HasCognitiveServiceInput with HasInternalJsonOutputParser
with HasSetLocation with HasSetLinkedService with SynapseMLLogging with HasModelID {
logClass()
logClass(FeatureNames.AiServices.Form)

def this() = this(Identifiable.randomUID("GetCustomModel"))

Expand Down Expand Up @@ -330,7 +330,7 @@ object AnalyzeCustomModel extends ComplexParamsReadable[AnalyzeCustomModel]

class AnalyzeCustomModel(override val uid: String) extends FormRecognizerBase(uid)
with SynapseMLLogging with HasTextDetails with HasModelID {
logClass()
logClass(FeatureNames.AiServices.Form)

def this() = this(Identifiable.randomUID("AnalyzeCustomModel"))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ package com.microsoft.azure.synapse.ml.cognitive.form

import com.microsoft.azure.synapse.ml.cognitive._
import com.microsoft.azure.synapse.ml.cognitive.vision.{BasicAsyncReply, HasImageInput}
import com.microsoft.azure.synapse.ml.logging.SynapseMLLogging
import com.microsoft.azure.synapse.ml.logging.{FeatureNames, SynapseMLLogging}
import com.microsoft.azure.synapse.ml.param.ServiceParam
import org.apache.http.entity.{AbstractHttpEntity, ByteArrayEntity, ContentType, StringEntity}
import org.apache.spark.ml.ComplexParamsReadable
Expand Down Expand Up @@ -38,7 +38,7 @@ class AnalyzeDocument(override val uid: String) extends CognitiveServicesBaseNoH
with HasCognitiveServiceInput with HasInternalJsonOutputParser with BasicAsyncReply
with HasPrebuiltModelID with HasPages with HasLocale with HasAPIVersion
with HasImageInput with HasSetLocation with SynapseMLLogging with HasSetLinkedService {
logClass()
logClass(FeatureNames.AiServices.Anomaly)

setDefault(apiVersion -> Left("2022-08-31"))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ package com.microsoft.azure.synapse.ml.cognitive.language

import com.microsoft.azure.synapse.ml.cognitive._
import com.microsoft.azure.synapse.ml.cognitive.text.{TADocument, TextAnalyticsAutoBatch}
import com.microsoft.azure.synapse.ml.logging.SynapseMLLogging
import com.microsoft.azure.synapse.ml.logging.{FeatureNames, SynapseMLLogging}
import com.microsoft.azure.synapse.ml.param.ServiceParam
import com.microsoft.azure.synapse.ml.stages.{FixedMiniBatchTransformer, FlattenBatch, HasBatchSize, UDFTransformer}
import org.apache.http.entity.{AbstractHttpEntity, StringEntity}
Expand Down Expand Up @@ -127,7 +127,7 @@ class AnalyzeText(override val uid: String) extends CognitiveServicesBase(uid)
with HasCognitiveServiceInput with HasInternalJsonOutputParser with HasSetLocation
with HasAPIVersion with HasCountryHint with TextAnalyticsAutoBatch with HasBatchSize
with AnalyzeTextTaskParameters with SynapseMLLogging {
logClass()
logClass(FeatureNames.AiServices.Language)

def this() = this(Identifiable.randomUID("AnalyzeText"))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,9 @@

package com.microsoft.azure.synapse.ml.cognitive.openai

import com.microsoft.azure.synapse.ml.cognitive.{
CognitiveServicesBase, HasCognitiveServiceInput, HasInternalJsonOutputParser
}
import com.microsoft.azure.synapse.ml.logging.SynapseMLLogging
import com.microsoft.azure.synapse.ml.cognitive.{CognitiveServicesBase, HasCognitiveServiceInput,
HasInternalJsonOutputParser}
import com.microsoft.azure.synapse.ml.logging.{FeatureNames, SynapseMLLogging}
import com.microsoft.azure.synapse.ml.param.AnyJsonFormat.anyFormat
import org.apache.http.entity.{AbstractHttpEntity, ContentType, StringEntity}
import org.apache.spark.ml.ComplexParamsReadable
Expand All @@ -24,7 +23,7 @@ object OpenAIChatCompletion extends ComplexParamsReadable[OpenAIChatCompletion]
class OpenAIChatCompletion(override val uid: String) extends CognitiveServicesBase(uid)
with HasOpenAITextParams with HasCognitiveServiceInput
with HasInternalJsonOutputParser with SynapseMLLogging {
logClass()
logClass(FeatureNames.AiServices.OpenAI)

val messagesCol: Param[String] = new Param[String](
this, "messagesCol", "The column messages to generate chat completions for," +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,9 @@

package com.microsoft.azure.synapse.ml.cognitive.openai

import com.microsoft.azure.synapse.ml.cognitive.{
CognitiveServicesBase, HasCognitiveServiceInput, HasInternalJsonOutputParser
}
import com.microsoft.azure.synapse.ml.logging.SynapseMLLogging
import com.microsoft.azure.synapse.ml.cognitive.{CognitiveServicesBase,
HasCognitiveServiceInput, HasInternalJsonOutputParser}
import com.microsoft.azure.synapse.ml.logging.{FeatureNames, SynapseMLLogging}
import com.microsoft.azure.synapse.ml.param.AnyJsonFormat.anyFormat
import org.apache.http.entity.{AbstractHttpEntity, ContentType, StringEntity}
import org.apache.spark.ml.ComplexParamsReadable
Expand All @@ -23,7 +22,7 @@ object OpenAICompletion extends ComplexParamsReadable[OpenAICompletion]
class OpenAICompletion(override val uid: String) extends CognitiveServicesBase(uid)
with HasOpenAITextParams with HasPromptInputs with HasCognitiveServiceInput
with HasInternalJsonOutputParser with SynapseMLLogging {
logClass()
logClass(FeatureNames.AiServices.OpenAI)

def this() = this(Identifiable.randomUID("OpenAICompletion"))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ package com.microsoft.azure.synapse.ml.cognitive.openai
import com.microsoft.azure.synapse.ml.cognitive.{CognitiveServicesBase, HasCognitiveServiceInput, HasServiceParams}
import com.microsoft.azure.synapse.ml.core.contracts.HasInputCol
import com.microsoft.azure.synapse.ml.io.http.JSONOutputParser
import com.microsoft.azure.synapse.ml.logging.SynapseMLLogging
import com.microsoft.azure.synapse.ml.logging.{FeatureNames, SynapseMLLogging}
import com.microsoft.azure.synapse.ml.param.ServiceParam
import org.apache.http.entity.{AbstractHttpEntity, ContentType, StringEntity}
import org.apache.spark.ml.ComplexParamsReadable
Expand All @@ -24,7 +24,7 @@ object OpenAIEmbedding extends ComplexParamsReadable[OpenAIEmbedding]

class OpenAIEmbedding (override val uid: String) extends CognitiveServicesBase(uid)
with HasOpenAISharedParams with HasCognitiveServiceInput with SynapseMLLogging {
logClass()
logClass(FeatureNames.AiServices.OpenAI)

def this() = this(Identifiable.randomUID("OpenAIEmbedding"))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import com.microsoft.azure.synapse.ml.cognitive._
import com.microsoft.azure.synapse.ml.core.contracts.HasOutputCol
import com.microsoft.azure.synapse.ml.core.spark.Functions
import com.microsoft.azure.synapse.ml.io.http.{ConcurrencyParams, HasErrorCol, HasURL}
import com.microsoft.azure.synapse.ml.logging.SynapseMLLogging
import com.microsoft.azure.synapse.ml.logging.{FeatureNames, SynapseMLLogging}
import com.microsoft.azure.synapse.ml.param.StringStringMapParam
import org.apache.spark.ml.param.{BooleanParam, Param, ParamMap, ParamValidators}
import org.apache.spark.ml.util.Identifiable
Expand All @@ -26,7 +26,7 @@ class OpenAIPrompt(override val uid: String) extends Transformer
with HasSubscriptionKey with HasAADToken with HasCustomAuthHeader
with ComplexParamsWritable with SynapseMLLogging {

logClass()
logClass(FeatureNames.AiServices.OpenAI)

def this() = this(Identifiable.randomUID("OpenAIPrompt"))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@
package com.microsoft.azure.synapse.ml.cognitive.search

import com.microsoft.azure.synapse.ml.cognitive.search.AzureSearchProtocol._
import com.microsoft.azure.synapse.ml.cognitive.{CognitiveServicesBase,
HasCognitiveServiceInput, HasInternalJsonOutputParser, HasServiceParams}
import com.microsoft.azure.synapse.ml.cognitive.{CognitiveServicesBase, HasCognitiveServiceInput,
HasInternalJsonOutputParser, HasServiceParams}
import com.microsoft.azure.synapse.ml.io.http.{ErrorUtils, SimpleHTTPTransformer}
import com.microsoft.azure.synapse.ml.io.powerbi.StreamMaterializer
import com.microsoft.azure.synapse.ml.logging.SynapseMLLogging
import com.microsoft.azure.synapse.ml.logging.{FeatureNames, SynapseMLLogging}
import com.microsoft.azure.synapse.ml.stages.{FixedMiniBatchTransformer, HasBatchSize, Lambda}
import org.apache.http.Consts
import org.apache.http.entity.{AbstractHttpEntity, ContentType, StringEntity}
Expand Down Expand Up @@ -90,7 +90,7 @@ class AddDocuments(override val uid: String) extends CognitiveServicesBase(uid)
with HasCognitiveServiceInput with HasInternalJsonOutputParser
with HasActionCol with HasServiceName with HasIndexName with HasBatchSize
with SynapseMLLogging {
logClass()
logClass(FeatureNames.AiServices.Search)

def this() = this(Identifiable.randomUID("AddDocuments"))

Expand Down
Loading

0 comments on commit 5c0b803

Please sign in to comment.