diff --git a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/contentunderstanding/ContentUnderstanding.scala b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/contentunderstanding/ContentUnderstanding.scala
new file mode 100644
index 0000000000..5c70aff802
--- /dev/null
+++ b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/contentunderstanding/ContentUnderstanding.scala
@@ -0,0 +1,361 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in project root for information.
+
+package com.microsoft.azure.synapse.ml.services.contentunderstanding
+
+import com.microsoft.azure.synapse.ml.core.schema.DatasetExtensions
+import com.microsoft.azure.synapse.ml.io.http.{HTTPRequestData, HTTPResponseData, HTTPSchema, SimpleHTTPTransformer}
+import com.microsoft.azure.synapse.ml.logging.FeatureNames
+import com.microsoft.azure.synapse.ml.param.ServiceParam
+import com.microsoft.azure.synapse.ml.services.{
+ CognitiveServicesBaseNoHandler, HasAPIVersion, HasCognitiveServiceInput, HasInternalJsonOutputParser}
+import com.microsoft.azure.synapse.ml.stages.{DropColumns, Lambda}
+import org.apache.http.client.methods.{HttpGet, HttpPost, HttpPut, HttpRequestBase}
+import org.apache.http.client.utils.URIBuilder
+import org.apache.http.entity.{AbstractHttpEntity, ContentType, StringEntity}
+import org.apache.http.impl.client.CloseableHttpClient
+import org.apache.spark.TaskContext
+import org.apache.spark.ml.param.ParamMap
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.ml.{ComplexParamsReadable, NamespaceInjections, PipelineModel}
+import org.apache.spark.sql.functions.{coalesce, col, lit, struct, when}
+import org.apache.spark.sql.types.{BinaryType, DataType, MapType, StringType, StructType}
+import org.apache.spark.sql.{DataFrame, Row}
+import spray.json._
+
+import java.nio.charset.StandardCharsets
+import java.security.MessageDigest
+import java.util.Base64
+
+object ContentUnderstanding extends ComplexParamsReadable[ContentUnderstanding]
+
+/** Lazy, one-document-per-row Content Understanding analysis with resumable operation handles.
+ *
+ * Service failures are retained in the output struct and errorCol. A Running or NotStarted
+ * response is not a completed result, even when the service has included an empty result object.
+ */
+class ContentUnderstanding(override val uid: String) extends CognitiveServicesBaseNoHandler(uid)
+ with HasCognitiveServiceInput with HasInternalJsonOutputParser with HasAPIVersion
+ with ContentUnderstandingParams with ContentUnderstandingPersistence with ContentUnderstandingPython {
+
+ import ContentUnderstandingProtocol._
+
+ logClass(FeatureNames.AiServices.Form)
+ setDefault(apiVersion -> Left(DefaultApiVersion))
+
+ def this() = this(Identifiable.randomUID("ContentUnderstanding"))
+
+ override def urlPath: String = AnalyzersPath.stripPrefix("/")
+
+ override def setEndpoint(value: String): this.type = setUrl(endpointUrl(value))
+
+ override def copy(extra: ParamMap): ContentUnderstanding = defaultCopy(extra)
+
+ // OpenAI global URL/version/key settings and Fabric MWC tokens are not credentials for this resource.
+ override private[ml] def transferGlobalParamsToParamMap(): Unit = ()
+
+ override protected def getFabricFallbackAuthHeader(row: Row): Option[String] = None
+
+ override protected def responseDataType: DataType =
+ StructType(ContentUnderstandingResponse.schema.fields.map(_.copy(nullable = true)))
+
+ private def authParams: Seq[ServiceParam[_]] =
+ Seq(subscriptionKey, AADToken, CustomAuthHeader, customHeaders, telemHeaders)
+
+ private def activeParams(poll: Boolean): Seq[ServiceParam[_]] = {
+ val inputParams = if (poll) {
+ Seq(operationLocation)
+ } else {
+ Seq(analyzerId, apiVersion, documentUrl, documentBytes, range, mimeType, documentName,
+ modelDeployments, stringEncoding, processingLocation)
+ }
+ authParams ++ inputParams
+ }
+
+ override protected def getVectorParamMap: Map[String, String] = {
+ val names = activeParams(getOperationMode == "poll").map(_.name).toSet
+ super.getVectorParamMap.filter { case (name, _) => names(name) }
+ }
+
+ private def configured(param: ServiceParam[_]): Boolean = get(param).orElse(getDefault(param)).isDefined
+
+ private def validateConfiguration(): Unit = {
+ require(isSet(url), "Content Understanding requires an explicit endpoint. Call setEndpoint(resourceRoot).")
+ validateEndpoint(getUrl)
+ require(!isSet(customUrlRoot), "customUrlRoot is not supported; use setEndpoint.")
+ require(getConcurrency > 0, "concurrency must be positive.")
+ val maxTimeout = Int.MaxValue.toDouble / MillisecondsPerSecond
+ require(java.lang.Double.isFinite(getTimeout) && getTimeout > 0 && getTimeout <= maxTimeout,
+ "timeout must be positive, finite, and representable in milliseconds.")
+ get(concurrentTimeout).foreach(value =>
+ require(java.lang.Double.isFinite(value) && value > 0, "concurrentTimeout must be positive and finite."))
+ require(Option(getOutputCol).exists(_.nonEmpty) && Option(getErrorCol).exists(_.nonEmpty) &&
+ getOutputCol != getErrorCol,
+ "outputCol and errorCol must be nonempty and distinct.")
+ }
+
+ private def settings: Settings = {
+ validateConfiguration()
+ Settings(getUrl, math.ceil(getTimeout * MillisecondsPerSecond).toInt,
+ getMaxPollAttempts, getPollingDelay, getMaxResponseBytes)
+ }
+
+ private def expectedType(param: ServiceParam[_]): DataType = {
+ if (param == documentBytes) {
+ BinaryType
+ } else if (Set(modelDeployments.name, customHeaders.name, telemHeaders.name)(param.name)) {
+ MapType(StringType, StringType)
+ } else {
+ StringType
+ }
+ }
+
+ private def matchesType(actual: DataType, expected: DataType): Boolean = (actual, expected) match {
+ case (MapType(StringType, StringType, _), MapType(StringType, StringType, _)) => true
+ case _ => actual == expected
+ }
+
+ private def validateValue(param: ServiceParam[_], value: Any): Unit = {
+ require(Option(value).isDefined, s"${param.name} cannot be null.")
+ val validType = (value, expectedType(param)) match {
+ case (_: Array[Byte], BinaryType) => true
+ case (values: Map[_, _], _: MapType) =>
+ values.forall { case (key, item) => key.isInstanceOf[String] && item.isInstanceOf[String] }
+ case (_: String, StringType) => true
+ case _ => false
+ }
+ require(validType, s"${param.name} has an invalid value type.")
+ if (!authParams.contains(param)) {
+ validateInputValue(param, value)
+ }
+ }
+
+ private def validateInputValue(param: ServiceParam[_], value: Any): Unit = {
+ validateNonemptyValue(param, value)
+ validateOptionValue(param, value)
+ }
+
+ private def validateNonemptyValue(param: ServiceParam[_], value: Any): Unit = {
+ value match {
+ case text: String => require(text.trim.nonEmpty, s"${param.name} cannot be empty.")
+ case bytes: Array[Byte] => require(bytes.nonEmpty, "documentBytes cannot be empty.")
+ case values: Map[_, _] =>
+ require(values.forall { case (key, item) => key.toString.trim.nonEmpty && item.toString.trim.nonEmpty },
+ s"${param.name} keys and values cannot be empty.")
+ case _ =>
+ }
+ }
+
+ private def validateDocumentUrl(value: String): Unit = {
+ val uri = parseUri(value, "documentUrl")
+ require(Set("http", "https")(Option(uri.getScheme).getOrElse("").toLowerCase(java.util.Locale.ROOT)) &&
+ Option(uri.getHost).isDefined && Option(uri.getRawUserInfo).isEmpty && Option(uri.getRawFragment).isEmpty,
+ "documentUrl must be an absolute HTTP(S) URL without user information or a fragment.")
+ }
+
+ private def validateOptionValue(param: ServiceParam[_], value: Any): Unit = {
+ param.name match {
+ case "analyzerId" => validateAnalyzerId(value.asInstanceOf[String])
+ case "apiVersion" =>
+ require(validApiVersion(value.asInstanceOf[String]), "apiVersion must be YYYY-MM-DD or YYYY-MM-DD-preview.")
+ case "documentUrl" => validateDocumentUrl(value.asInstanceOf[String])
+ case "operationLocation" => validateOperationLocation(getUrl, value.asInstanceOf[String])
+ case "stringEncoding" =>
+ require(Set("codePoint", "utf16", "utf8")(value.asInstanceOf[String]), "Unsupported stringEncoding.")
+ case "processingLocation" =>
+ require(Set("geography", "dataZone", "global")(value.asInstanceOf[String]), "Unsupported processingLocation.")
+ case _ =>
+ }
+ }
+
+ private def validateParamSchema(param: ServiceParam[_], schema: StructType): Unit = {
+ get(param).orElse(getDefault(param)).foreach {
+ case Left(value) => validateValue(param, value)
+ case Right(name) =>
+ require(Option(name).exists(_.nonEmpty), s"${param.name} column name cannot be empty.")
+ val field = schema.fields.find(_.name == name)
+ require(field.isDefined, s"The column configured for ${param.name} is missing.")
+ require(matchesType(field.get.dataType, expectedType(param)),
+ s"The column configured for ${param.name} must have type ${expectedType(param).simpleString}.")
+ }
+ }
+
+ private[contentunderstanding] def validateInputSchema(schema: StructType): Unit = {
+ validateConfiguration()
+ val poll = getOperationMode == "poll"
+ if (poll) {
+ require(configured(operationLocation), "poll mode requires operationLocation or operationLocationCol.")
+ } else {
+ require(configured(documentUrl) != configured(documentBytes),
+ "Configure exactly one of documentUrl and documentBytes, using a scalar value or a column.")
+ }
+ activeParams(poll).foreach(validateParamSchema(_, schema))
+ }
+
+ private def validateRow(row: Row, poll: Boolean): Unit = {
+ validateConfiguration()
+ activeParams(poll).foreach { param =>
+ getValueAnyOpt(row, param).foreach(validateValue(param, _))
+ }
+ if (poll) {
+ require(getValueOpt(row, operationLocation).isDefined, "operationLocation cannot be null.")
+ } else {
+ require(configured(documentUrl) != configured(documentBytes),
+ "Configure exactly one of documentUrl and documentBytes.")
+ require(getValueOpt(row, documentUrl).isDefined || getValueOpt(row, documentBytes).isDefined,
+ "The selected document input cannot be null.")
+ require(getValueOpt(row, analyzerId).isDefined && getValueOpt(row, apiVersion).isDefined,
+ "analyzerId and apiVersion cannot be null.")
+ }
+ }
+
+ private def requestBody(row: Row): String = {
+ val source = getValueOpt(row, documentUrl).map(value => "url" -> JsString(value))
+ .orElse(getValueOpt(row, documentBytes).map(value => "data" -> JsString(Base64.getEncoder.encodeToString(value))))
+ .getOrElse(throw new IllegalArgumentException("Document input cannot be null."))
+ val optionalInput = Seq(documentName -> "name", mimeType -> "mimeType", range -> "range")
+ .flatMap { case (param, name) => getValueOpt(row, param).map(value => name -> JsString(value)) }
+ val input = JsObject((optionalInput :+ source).toMap)
+ val models = getValueOpt(row, modelDeployments).map(values =>
+ "modelDeployments" -> JsObject(values.map { case (name, deployment) => name -> JsString(deployment) }))
+ canonicalJson(JsObject(Map[String, JsValue]("inputs" -> JsArray(input)) ++ models))
+ }
+
+ private def buildUrl(path: String, query: Seq[(String, String)]): String = {
+ val builder = new URIBuilder(getUrl).setPath(path)
+ query.sortBy(_._1).foreach { case (name, value) => builder.setParameter(name, value) }
+ builder.build().toString
+ }
+
+ private def analyzeUrl(row: Row): String = {
+ val query = Seq("api-version" -> getValue(row, apiVersion)) ++
+ Seq(stringEncoding, processingLocation).flatMap(param =>
+ getValueOpt(row, param).map(value => param.payloadName -> value))
+ buildUrl(s"$AnalyzersPath/${getValue(row, analyzerId)}:analyze", query)
+ }
+
+ override protected def prepareUrl: Row => String = analyzeUrl _
+
+ override protected def prepareEntity: Row => Option[AbstractHttpEntity] = row =>
+ Some(new StringEntity(requestBody(row), ContentType.APPLICATION_JSON))
+
+ private def prepareRequest(row: Row, poll: Boolean): HttpRequestBase = {
+ validateRow(row, poll)
+ val request = if (poll) {
+ new HttpGet(validateOperationLocation(getUrl, getValue(row, operationLocation)))
+ } else {
+ val post = new HttpPost(analyzeUrl(row))
+ post.setEntity(new StringEntity(requestBody(row), ContentType.APPLICATION_JSON))
+ post
+ }
+ addHeaders(request, row)
+ request
+ }
+
+ override protected def inputFunc(schema: StructType): Row => Option[HttpRequestBase] =
+ row => Some(prepareRequest(row, getOperationMode == "poll"))
+
+ override protected def handlingFunc(client: CloseableHttpClient, request: HTTPRequestData): HTTPResponseData = {
+ // The inherited client has automatic retries. Use a no-retry client, still scheduled by the shared HTTP pipeline.
+ val response = execute(settings, request, getOperationMode)
+ // Keep service failures in the output rather than letting SimpleHTTPTransformer discard the response.
+ HTTPSchema.stringToResponse(response.toJson.compactPrint, org.apache.http.HttpStatus.SC_OK, "OK")
+ }
+
+ private def quoted(name: String): String = "`" + name.replace("`", "``") + "`"
+
+ private def addServiceErrors(frame: DataFrame): DataFrame = {
+ val output = col(quoted(getOutputCol))
+ val serviceError = struct(output.getField("error").alias("response"),
+ struct(struct(lit("HTTP").alias("protocol"), lit(1).alias("major"), lit(1).alias("minor"))
+ .alias("protocolVersion"), coalesce(output.getField("httpStatus"), lit(0)).alias("statusCode"),
+ output.getField("status").alias("reasonPhrase")).alias("status"))
+ frame.withColumn(getErrorCol,
+ when(output.getField("error").isNotNull, serviceError).otherwise(col(quoted(getErrorCol))))
+ }
+
+ override protected def getInternalTransformer(schema: StructType): PipelineModel = {
+ validateInputSchema(schema)
+ require(!schema.fieldNames.contains(getOutputCol) && !schema.fieldNames.contains(getErrorCol),
+ "outputCol and errorCol must not overwrite input columns.")
+ val reserved = schema.fieldNames.toSet ++ Set(getOutputCol, getErrorCol)
+ val inputColumn = DatasetExtensions.findUnusedColumnName("contentUnderstandingInput")(reserved)
+ val resultColumn = DatasetExtensions.findUnusedColumnName("contentUnderstandingResult")(reserved + inputColumn)
+ val errorColumn = DatasetExtensions.findUnusedColumnName("contentUnderstandingError")(
+ reserved ++ Set(inputColumn, resultColumn))
+ val columns = getVectorParamMap.values.toSeq.distinct.map(name => col(quoted(name)).alias(name))
+ val inputs = if (columns.nonEmpty) columns else Seq(lit(false).alias("placeholder"))
+ NamespaceInjections.pipelineModel(Array(
+ Lambda(_.withColumn(inputColumn, struct(inputs: _*))),
+ new SimpleHTTPTransformer()
+ .setInputCol(inputColumn)
+ .setOutputCol(resultColumn)
+ .setErrorCol(errorColumn)
+ .setInputParser(getInternalInputParser(schema))
+ .setOutputParser(getInternalOutputParser(schema))
+ .setHandler(handlingFunc _)
+ .setConcurrency(getConcurrency)
+ .setConcurrentTimeout(get(concurrentTimeout))
+ .setTimeout(getTimeout),
+ Lambda(frame => addServiceErrors(frame.withColumnRenamed(resultColumn, getOutputCol)
+ .withColumnRenamed(errorColumn, getErrorCol))),
+ new DropColumns().setCol(inputColumn)))
+ }
+
+ private[contentunderstanding] def requestFingerprint(row: Row): String = {
+ validateRow(row, poll = false)
+ val bytes = (analyzeUrl(row) + "\n" + requestBody(row)).getBytes(StandardCharsets.UTF_8)
+ MessageDigest.getInstance("SHA-256").digest(bytes).map(byte => f"${byte & 0xff}%02x").mkString
+ }
+
+ private[contentunderstanding] def submitOne(row: Row): ContentUnderstandingResponse =
+ execute(settings, new HTTPRequestData(prepareRequest(row, poll = false)), "submit")
+
+ private[contentunderstanding] def pollOne(row: Row, location: String): ContentUnderstandingResponse = {
+ val config = settings
+ val request = new HttpGet(validateOperationLocation(getUrl, location))
+ authParams.foreach(param => getValueAnyOpt(row, param).foreach(validateValue(param, _)))
+ addHeaders(request, row)
+ execute(config, new HTTPRequestData(request), "poll", failOnClientError = true)
+ }
+
+ private def validateManagement(): Unit = {
+ require(Option(TaskContext.get()).isEmpty, "Analyzer management is driver-only.")
+ validateConfiguration()
+ (authParams ++ Seq(analyzerId, apiVersion)).foreach { param =>
+ require(get(param).orElse(getDefault(param)).forall(_.isLeft),
+ s"Analyzer management requires scalar ${param.name}, not a column.")
+ getValueAnyOpt(Row.empty, param).foreach(validateValue(param, _))
+ }
+ }
+
+ private def analyzerRequest: HTTPRequestData = {
+ val request = new HttpGet(buildUrl(s"$AnalyzersPath/$getAnalyzerId", Seq("api-version" -> getApiVersion)))
+ addHeaders(request, Row.empty)
+ new HTTPRequestData(request)
+ }
+
+ /** Provision a custom analyzer explicitly. Resource defaults are never changed.
+ * Throws ContentUnderstandingException with the response on service failure or poll exhaustion.
+ */
+ def createAnalyzer(definitionJson: String, allowReplace: Boolean): String = {
+ validateManagement()
+ require(Option(definitionJson).exists(_.trim.nonEmpty), "definitionJson must be a nonempty analyzer JSON object.")
+ val definition = try {
+ definitionJson.parseJson
+ } catch {
+ case _: JsonParser.ParsingException => throw new IllegalArgumentException("definitionJson must be valid JSON.")
+ }
+ require(definition.isInstanceOf[JsObject], "definitionJson must be an analyzer JSON object.")
+ val request = new HttpPut(buildUrl(s"$AnalyzersPath/$getAnalyzerId",
+ Seq("api-version" -> getApiVersion, "allowReplace" -> allowReplace.toString)))
+ request.setEntity(new StringEntity(definitionJson, ContentType.APPLICATION_JSON))
+ addHeaders(request, Row.empty)
+ ContentUnderstandingProtocol.createAnalyzer(settings, new HTTPRequestData(request), analyzerRequest, getAnalyzerId)
+ }
+
+ def getAnalyzer(): String = {
+ validateManagement()
+ ContentUnderstandingProtocol.getAnalyzer(settings, analyzerRequest)
+ }
+}
diff --git a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/contentunderstanding/ContentUnderstandingParams.scala b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/contentunderstanding/ContentUnderstandingParams.scala
new file mode 100644
index 0000000000..be89cd0d2e
--- /dev/null
+++ b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/contentunderstanding/ContentUnderstandingParams.scala
@@ -0,0 +1,207 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in project root for information.
+
+package com.microsoft.azure.synapse.ml.services.contentunderstanding
+
+import com.microsoft.azure.synapse.ml.codegen.Wrappable
+import com.microsoft.azure.synapse.ml.param.ServiceParam
+import com.microsoft.azure.synapse.ml.services.HasServiceParams
+import org.apache.spark.ml.param.{IntParam, Param, ParamValidators}
+import spray.json.DefaultJsonProtocol._
+
+trait ContentUnderstandingParams extends HasServiceParams {
+
+ val analyzerId = new ServiceParam[String](this, "analyzerId",
+ "Prebuilt or explicitly provisioned custom analyzer identifier.")
+
+ def setAnalyzerId(value: String): this.type = setScalarParam(analyzerId, value)
+ def setAnalyzerIdCol(value: String): this.type = setVectorParam(analyzerId, value)
+ def getAnalyzerId: String = getScalarParam(analyzerId)
+ def getAnalyzerIdCol: String = getVectorParam(analyzerId)
+
+ val documentUrl = new ServiceParam[String](this, "documentUrl",
+ "URL the service can read. Configure exactly one of documentUrl and documentBytes.")
+
+ def setDocumentUrl(value: String): this.type = setScalarParam(documentUrl, value)
+ def setDocumentUrlCol(value: String): this.type = setVectorParam(documentUrl, value)
+ def getDocumentUrl: String = getScalarParam(documentUrl)
+ def getDocumentUrlCol: String = getVectorParam(documentUrl)
+
+ val documentBytes: ServiceParam[Array[Byte]] = new ServiceParam[Array[Byte]](this, "documentBytes",
+ "Document bytes, sent as inputs[].data using base64. Use BinaryType for a column.") {
+ override def pyValue(value: Either[Array[Byte], String]): String = value match {
+ case Left(bytes) =>
+ val unsignedByteMask = 0xff
+ bytes.map(_ & unsignedByteMask).mkString("bytearray([", ", ", "])")
+ case Right(column) => super.pyValue(Right(column))
+ }
+ }
+
+ def setDocumentBytes(value: Array[Byte]): this.type = setScalarParam(documentBytes, value)
+ def setDocumentBytesCol(value: String): this.type = setVectorParam(documentBytes, value)
+ def getDocumentBytes: Array[Byte] = getScalarParam(documentBytes)
+ def getDocumentBytesCol: String = getVectorParam(documentBytes)
+
+ val range = new ServiceParam[String](this, "range",
+ "Optional service range: 1-based document pages, or integer milliseconds for audio/video.")
+
+ def setRange(value: String): this.type = setScalarParam(range, value)
+ def setRangeCol(value: String): this.type = setVectorParam(range, value)
+ def getRange: String = getScalarParam(range)
+ def getRangeCol: String = getVectorParam(range)
+
+ val mimeType = new ServiceParam[String](this, "mimeType", "Optional MIME type of the document.")
+
+ def setMimeType(value: String): this.type = setScalarParam(mimeType, value)
+ def setMimeTypeCol(value: String): this.type = setVectorParam(mimeType, value)
+ def getMimeType: String = getScalarParam(mimeType)
+ def getMimeTypeCol: String = getVectorParam(mimeType)
+
+ val documentName = new ServiceParam[String](this, "documentName", "Optional input document name.")
+
+ def setDocumentName(value: String): this.type = setScalarParam(documentName, value)
+ def setDocumentNameCol(value: String): this.type = setVectorParam(documentName, value)
+ def getDocumentName: String = getScalarParam(documentName)
+ def getDocumentNameCol: String = getVectorParam(documentName)
+
+ val modelDeployments = new ServiceParam[Map[String, String]](this, "modelDeployments",
+ "Per-request model-name or prebuilt-alias to deployment-name mapping; does not modify resource defaults.")
+
+ def setModelDeployments(value: Map[String, String]): this.type = setScalarParam(modelDeployments, value)
+ def setModelDeploymentsCol(value: String): this.type = setVectorParam(modelDeployments, value)
+ def getModelDeployments: Map[String, String] = getScalarParam(modelDeployments)
+ def getModelDeploymentsCol: String = getVectorParam(modelDeployments)
+
+ val stringEncoding = new ServiceParam[String](this, "stringEncoding",
+ "String offset encoding, for example codePoint, utf16, or utf8.", isURLParam = true)
+
+ def setStringEncoding(value: String): this.type = setScalarParam(stringEncoding, value)
+ def setStringEncodingCol(value: String): this.type = setVectorParam(stringEncoding, value)
+ def getStringEncoding: String = getScalarParam(stringEncoding)
+ def getStringEncodingCol: String = getVectorParam(stringEncoding)
+
+ val processingLocation = new ServiceParam[String](this, "processingLocation",
+ "Service processing location, for example geography, dataZone, or global.", isURLParam = true)
+
+ def setProcessingLocation(value: String): this.type = setScalarParam(processingLocation, value)
+ def setProcessingLocationCol(value: String): this.type = setVectorParam(processingLocation, value)
+ def getProcessingLocation: String = getScalarParam(processingLocation)
+ def getProcessingLocationCol: String = getVectorParam(processingLocation)
+
+ val operationLocation = new ServiceParam[String](this, "operationLocation",
+ "Previously accepted operation URL. Used only in poll mode and restricted to the configured endpoint.")
+
+ def setOperationLocation(value: String): this.type = setScalarParam(operationLocation, value)
+ def setOperationLocationCol(value: String): this.type = setVectorParam(operationLocation, value)
+ def getOperationLocation: String = getScalarParam(operationLocation)
+ def getOperationLocationCol: String = getVectorParam(operationLocation)
+
+ val operationMode = new Param[String](this, "operationMode",
+ "analyze submits and polls; submit only submits; poll resumes an operation without document input.",
+ ParamValidators.inArray(Array("analyze", "submit", "poll")))
+
+ def setOperationMode(value: String): this.type = set(operationMode, value)
+ def getOperationMode: String = $(operationMode)
+
+ val maxPollAttempts = new IntParam(this, "maxPollAttempts",
+ "Maximum GET attempts, including transient failures. Exhaustion preserves the last running operation.",
+ ParamValidators.gt(0))
+
+ def setMaxPollAttempts(value: Int): this.type = set(maxPollAttempts, value)
+ def getMaxPollAttempts: Int = $(maxPollAttempts)
+
+ val pollingDelay = new IntParam(this, "pollingDelay",
+ "Milliseconds between polls when Retry-After is absent. Zero is useful for local testing.",
+ ParamValidators.gtEq(0))
+
+ def setPollingDelay(value: Int): this.type = set(pollingDelay, value)
+ def getPollingDelay: Int = $(pollingDelay)
+
+ val maxResponseBytes = new IntParam(this, "maxResponseBytes",
+ "Maximum bytes per HTTP response before JSON parsing. Split large documents into explicit ranges.",
+ ParamValidators.gt(0))
+
+ def setMaxResponseBytes(value: Int): this.type = set(maxResponseBytes, value)
+ def getMaxResponseBytes: Int = $(maxResponseBytes)
+
+ setDefault(
+ analyzerId -> Left("prebuilt-read"),
+ operationMode -> "analyze",
+ maxPollAttempts -> 120,
+ pollingDelay -> 1000,
+ maxResponseBytes -> 32 * 1024 * 1024)
+}
+
+private[contentunderstanding] trait ContentUnderstandingPython extends Wrappable {
+ this: ContentUnderstanding =>
+
+ override protected def pyParamSetter(param: Param[_]): String = {
+ if (param.name == "documentBytes") {
+ """
+ |def setDocumentBytes(self, value):
+ | '''Set bytes, bytearray, or a sequence of unsigned byte values.'''
+ | self._java_obj = self._java_obj.setDocumentBytes(bytearray(value))
+ | return self
+ |
+ |def setDocumentBytesCol(self, value):
+ | self._java_obj = self._java_obj.setDocumentBytesCol(value)
+ | return self
+ |""".stripMargin
+ } else {
+ super.pyParamSetter(param)
+ }
+ }
+
+ override protected def pySetParamsFunc: String = {
+ """
+ |def setParams(self, **kwargs):
+ | '''Set parameters using the same JVM setters as the constructor.'''
+ | for name, value in kwargs.items():
+ | if value is not None:
+ | getattr(self, "set" + name[0].upper() + name[1:])(value)
+ | return self
+ |""".stripMargin
+ }
+
+ override def pyAdditionalMethods: String = super.pyAdditionalMethods + {
+ """
+ |def _transfer_params_from_java(self):
+ | from pyspark.ml.common import _java2py
+ | sc = SparkContext._active_spark_context
+ | # JVM-backed service params must not become None scalars or pickle-decoded document bytes.
+ | for param in self.params:
+ | if param.doc.startswith("ServiceParam:"):
+ | self._paramMap.pop(param, None)
+ | elif self._java_obj.hasParam(param.name):
+ | java_param = self._java_obj.getParam(param.name)
+ | if self._java_obj.isSet(java_param):
+ | self._set(**{param.name: _java2py(sc, self._java_obj.getOrDefault(java_param))})
+ |
+ |def _transfer_params_to_java(self):
+ | # Use the generated setters for ParamMap/copy values, including bytes and model maps.
+ | for param in list(self._paramMap):
+ | if param.doc.startswith("ServiceParam:"):
+ | value = self._paramMap.pop(param)
+ | if value is not None:
+ | getattr(self, "set" + param.name[0].upper() + param.name[1:])(value)
+ | super()._transfer_params_to_java()
+ |
+ |def clear(self, param):
+ | param = self._resolveParam(param)
+ | self._java_obj.clear(self._java_obj.getParam(param.name))
+ | return super().clear(param)
+ |
+ |def createAnalyzer(self, definition: "dict | str", allowReplace: bool = False) -> str:
+ | '''Explicit driver-only provisioning. Never changes resource defaults.'''
+ | import json
+ | self._transfer_params_to_java()
+ | payload = json.dumps(definition) if isinstance(definition, dict) else definition
+ | return self._java_obj.createAnalyzer(payload, allowReplace)
+ |
+ |def getAnalyzer(self) -> str:
+ | '''Get the current scalar analyzer definition from the service.'''
+ | self._transfer_params_to_java()
+ | return self._java_obj.getAnalyzer()
+ |""".stripMargin
+ }
+}
diff --git a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/contentunderstanding/ContentUnderstandingPersistence.scala b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/contentunderstanding/ContentUnderstandingPersistence.scala
new file mode 100644
index 0000000000..8bd646357b
--- /dev/null
+++ b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/contentunderstanding/ContentUnderstandingPersistence.scala
@@ -0,0 +1,77 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in project root for information.
+
+package com.microsoft.azure.synapse.ml.services.contentunderstanding
+
+import com.microsoft.azure.synapse.ml.codegen.Wrappable
+import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
+
+trait ContentUnderstandingPersistence extends Wrappable { this: ContentUnderstanding =>
+
+ def writeToTable(dataset: Dataset[_],
+ idCol: String,
+ tableName: String,
+ format: String = "delta",
+ batchSize: Int = 1): DataFrame =
+ ContentUnderstandingWriter.writeToTable(dataset, this, idCol, tableName, format, batchSize)
+
+ def writeToPath(dataset: Dataset[_],
+ idCol: String,
+ path: String,
+ format: String = "delta",
+ batchSize: Int = 1): DataFrame =
+ ContentUnderstandingWriter.writeToPath(dataset, this, idCol, path, format, batchSize)
+
+ def readTable(spark: SparkSession, tableName: String): DataFrame =
+ ContentUnderstandingWriter.readTable(spark, tableName)
+
+ def readPath(spark: SparkSession, path: String, format: String = "delta"): DataFrame =
+ ContentUnderstandingWriter.readPath(spark, path, format)
+
+ override def pyAdditionalMethods: String = super.pyAdditionalMethods + {
+ """
+ |def writeToTable(self, dataset, idCol, tableName, format="delta", batchSize=1):
+ | '''
+ | Eagerly analyze documents and commit resumable state to a table.
+ |
+ | Use unique, stable string IDs, including the range for split documents.
+ | The table is an append-only journal. The returned DataFrame contains
+ | the latest state per ID. Only one writer may own a destination.
+ | In submit mode, save handles only; rerun in analyze mode to collect results.
+ | '''
+ | from pyspark.sql import DataFrame
+ | self._transfer_params_to_java()
+ | result = self._java_obj.writeToTable(
+ | dataset._jdf, idCol, tableName, format, batchSize
+ | )
+ | return DataFrame(result, dataset.sparkSession)
+ |
+ |def writeToPath(self, dataset, idCol, path, format="delta", batchSize=1):
+ | '''
+ | Eagerly analyze documents and commit resumable state to a lakehouse path.
+ |
+ | Accepted operation handles are committed before polling. Each result
+ | is committed separately. Rerun with the same IDs and options to resume.
+ | In submit mode, save handles only; rerun in analyze mode to collect results.
+ | '''
+ | from pyspark.sql import DataFrame
+ | self._transfer_params_to_java()
+ | result = self._java_obj.writeToPath(
+ | dataset._jdf, idCol, path, format, batchSize
+ | )
+ | return DataFrame(result, dataset.sparkSession)
+ |
+ |def readTable(self, spark, tableName):
+ | '''Read the latest persisted state per document/range ID.'''
+ | from pyspark.sql import DataFrame
+ | result = self._java_obj.readTable(spark._jsparkSession, tableName)
+ | return DataFrame(result, spark)
+ |
+ |def readPath(self, spark, path, format="delta"):
+ | '''Read the latest persisted state per ID from a lakehouse path.'''
+ | from pyspark.sql import DataFrame
+ | result = self._java_obj.readPath(spark._jsparkSession, path, format)
+ | return DataFrame(result, spark)
+ |""".stripMargin
+ }
+}
diff --git a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/contentunderstanding/ContentUnderstandingProtocol.scala b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/contentunderstanding/ContentUnderstandingProtocol.scala
new file mode 100644
index 0000000000..5c76c5e83f
--- /dev/null
+++ b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/contentunderstanding/ContentUnderstandingProtocol.scala
@@ -0,0 +1,528 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in project root for information.
+
+package com.microsoft.azure.synapse.ml.services.contentunderstanding
+
+import com.microsoft.azure.synapse.ml.core.schema.SparkBindings
+import com.microsoft.azure.synapse.ml.io.http.{HTTPRequestData, RequestLineData}
+import org.apache.http.HttpStatus
+import org.apache.http.client.config.RequestConfig
+import org.apache.http.client.methods.HttpRequestBase
+import org.apache.http.impl.client.{CloseableHttpClient, HttpClients}
+import spray.json.DefaultJsonProtocol._
+import spray.json._
+
+import java.io.{ByteArrayOutputStream, IOException, InputStream, InterruptedIOException}
+import java.net.{URI, URISyntaxException}
+import java.nio.charset.StandardCharsets
+import java.time.ZonedDateTime
+import java.time.format.DateTimeFormatter
+import scala.annotation.tailrec
+import scala.util.Try
+
+case class ContentUnderstandingResponse(operationLocation: Option[String],
+ id: Option[String],
+ status: String,
+ httpStatus: Int,
+ rawResponse: String,
+ error: Option[String])
+
+object ContentUnderstandingResponse extends SparkBindings[ContentUnderstandingResponse] {
+ implicit val JsonFormat: RootJsonFormat[ContentUnderstandingResponse] =
+ jsonFormat6(ContentUnderstandingResponse.apply)
+}
+
+/** The response retains the service error and operation handle, including provisioning timeouts. */
+class ContentUnderstandingException(val response: ContentUnderstandingResponse)
+ extends IllegalStateException(
+ s"Content Understanding returned ${response.status} (HTTP ${response.httpStatus}): " +
+ response.error.getOrElse("Polling budget exhausted; the operation may still be running."))
+
+private[contentunderstanding] object ContentUnderstandingProtocol {
+ val AnalyzersPath = "/contentunderstanding/analyzers"
+ val ResultsPath = "/contentunderstanding/analyzerResults/"
+ val DefaultApiVersion = "2025-11-01"
+ val MillisecondsPerSecond = 1000
+ val MaxRetryDelayMs = 60000L
+ val TooManyRequests = 429
+ private val BufferSize = 8192
+ private val MaxPort = 65535
+ private val HttpsPort = 443
+ private val HttpPort = 80
+ private val MaxAnalyzerIdLength = 64
+ private val ManagementGetAttempts = 3
+ private val StatusNames = Seq("NotStarted", "Running", "Succeeded", "Failed", "Canceled")
+ .map(name => name.toLowerCase(java.util.Locale.ROOT) -> name).toMap
+ private val TransientStatuses = Set(HttpStatus.SC_REQUEST_TIMEOUT, TooManyRequests,
+ HttpStatus.SC_INTERNAL_SERVER_ERROR, HttpStatus.SC_BAD_GATEWAY,
+ HttpStatus.SC_SERVICE_UNAVAILABLE, HttpStatus.SC_GATEWAY_TIMEOUT)
+ private val AdmissionRejections = Set(HttpStatus.SC_UNAUTHORIZED, HttpStatus.SC_FORBIDDEN, TooManyRequests)
+ private val UnavailableResults = Set(HttpStatus.SC_NOT_FOUND, HttpStatus.SC_GONE)
+
+ case class Settings(endpoint: String,
+ timeoutMs: Int,
+ maxPollAttempts: Int,
+ pollingDelay: Int,
+ maxResponseBytes: Int) {
+ def requestConfig: RequestConfig = RequestConfig.custom()
+ .setConnectTimeout(timeoutMs)
+ .setConnectionRequestTimeout(timeoutMs)
+ .setSocketTimeout(timeoutMs)
+ .setRedirectsEnabled(false)
+ .build()
+ }
+
+ private case class WireResponse(httpStatus: Int,
+ body: String,
+ location: Option[String],
+ retryAfter: Option[String],
+ diagnostic: Option[String] = None,
+ transportFailure: Boolean = false) {
+ def successful: Boolean = httpStatus >= HttpStatus.SC_OK && httpStatus < HttpStatus.SC_MULTIPLE_CHOICES
+ def retryable: Boolean = transportFailure || (diagnostic.isEmpty && TransientStatuses(httpStatus))
+ lazy val json: Option[JsObject] = asObject(body)
+ }
+
+ private class ResponseTooLarge extends RuntimeException
+
+ def diagnostic(code: String, message: String): String =
+ JsObject("code" -> JsString(code), "message" -> JsString(message)).compactPrint
+
+ def parseUri(value: String, description: String): URI = {
+ require(Option(value).exists(_.nonEmpty), s"$description must be nonempty.")
+ try {
+ new URI(value)
+ } catch {
+ case _: URISyntaxException => throw new IllegalArgumentException(s"$description must be a valid URI.")
+ }
+ }
+
+ def validApiVersion(value: String): Boolean =
+ Option(value).exists(_.matches("[0-9]{4}-[0-9]{2}-[0-9]{2}(-preview)?"))
+
+ private def safeSegment(value: String): Boolean =
+ value.matches("[a-zA-Z0-9._-]+") && value != "." && value != ".."
+
+ def validateAnalyzerId(value: String): Unit = {
+ require(Option(value).exists(v => v.length <= MaxAnalyzerIdLength && safeSegment(v)),
+ "analyzerId must contain 1-64 letters, digits, periods, underscores, or hyphens, and cannot be a dot segment.")
+ }
+
+ private def literalLoopback(host: String): Boolean = {
+ val ipv6 = Set("[::1]", "::1", "[0:0:0:0:0:0:0:1]", "0:0:0:0:0:0:0:1")
+ val parts = host.split("\\.", -1)
+ ipv6(host) || (parts.length == 4 && parts.head == "127" &&
+ parts.forall(p => p.matches("[0-9]{1,3}") && p.toInt <= 255))
+ }
+
+ private def validateAuthority(uri: URI): Unit = {
+ require(uri.isAbsolute && !uri.isOpaque && Option(uri.getHost).exists(_.nonEmpty),
+ "The endpoint must have an absolute HTTP(S) authority.")
+ require(Option(uri.getRawUserInfo).isEmpty && Option(uri.getRawFragment).isEmpty,
+ "Endpoint and operation URLs cannot contain user information or fragments.")
+ require(uri.getPort >= -1 && uri.getPort <= MaxPort && uri.getPort != 0, "Invalid endpoint port.")
+ val scheme = uri.getScheme.toLowerCase(java.util.Locale.ROOT)
+ require(scheme == "https" || (scheme == "http" && literalLoopback(uri.getHost)),
+ "Use HTTPS. HTTP is allowed only for literal loopback test endpoints.")
+ }
+
+ def endpointUrl(value: String): String = {
+ val uri = parseUri(value, "endpoint")
+ validateAuthority(uri)
+ require(Option(uri.getRawQuery).isEmpty, "endpoint cannot contain a query.")
+ val path = Option(uri.getRawPath).getOrElse("").reverse.dropWhile(_ == '/').reverse
+ require(path.isEmpty || path == AnalyzersPath,
+ "endpoint must be the resource root or its contentunderstanding/analyzers URL.")
+ new URI(uri.getScheme, uri.getRawAuthority, AnalyzersPath, None.orNull, None.orNull).toString
+ }
+
+ def validateEndpoint(value: String): URI = {
+ val uri = parseUri(value, "url")
+ validateAuthority(uri)
+ require(uri.getRawPath == AnalyzersPath && Option(uri.getRawQuery).isEmpty,
+ "url must end with /contentunderstanding/analyzers and have no query. Use setEndpoint(resourceRoot).")
+ uri
+ }
+
+ private def effectivePort(uri: URI): Int =
+ if (uri.getPort >= 0) uri.getPort else if (uri.getScheme.equalsIgnoreCase("https")) HttpsPort else HttpPort
+
+ private def sameOrigin(endpoint: URI, uri: URI): Boolean =
+ endpoint.getScheme.equalsIgnoreCase(uri.getScheme) &&
+ endpoint.getHost.equalsIgnoreCase(uri.getHost) && effectivePort(endpoint) == effectivePort(uri)
+
+ def validateOperationLocation(endpoint: String,
+ location: String,
+ analyzer: Option[String] = None): URI = {
+ val root = validateEndpoint(endpoint)
+ val uri = parseUri(location, "operationLocation")
+ validateAuthority(uri)
+ require(sameOrigin(root, uri), "operationLocation must use the configured endpoint's scheme, host, and port.")
+ val prefix = analyzer.map(id => s"$AnalyzersPath/$id/operations/").getOrElse(ResultsPath)
+ val path = uri.getRawPath
+ require(path.startsWith(prefix) && safeSegment(path.substring(prefix.length)),
+ "operationLocation must identify one operation in the expected Content Understanding result path.")
+ require(Option(uri.getRawQuery).exists(q =>
+ q.startsWith("api-version=") && validApiVersion(q.stripPrefix("api-version="))),
+ "operationLocation must have exactly one api-version query parameter.")
+ uri
+ }
+
+ def canonicalJson(value: JsValue): String = value match {
+ case JsObject(fields) =>
+ fields.toSeq.sortBy(_._1).map { case (name, field) =>
+ JsString(name).compactPrint + ":" + canonicalJson(field)
+ }.mkString("{", ",", "}")
+ case JsArray(values) => values.map(canonicalJson).mkString("[", ",", "]")
+ case other => other.compactPrint
+ }
+
+ private def checkInterrupted(): Unit = {
+ if (Thread.currentThread().isInterrupted) {
+ throw new InterruptedException("Content Understanding request interrupted.")
+ }
+ }
+
+ private def pause(delay: Long): Unit = {
+ checkInterrupted()
+ if (delay > 0) {
+ try {
+ Thread.sleep(delay)
+ } catch {
+ case error: InterruptedException =>
+ Thread.currentThread().interrupt()
+ throw error
+ }
+ }
+ }
+
+ private[contentunderstanding] def retryDelay(value: Option[String], fallback: Int): Long = {
+ val seconds = value.flatMap(v => Try(v.trim.toLong).toOption).filter(_ >= 0)
+ .map(s => math.min(s, MaxRetryDelayMs / MillisecondsPerSecond) * MillisecondsPerSecond)
+ val date = value.flatMap(v => Try(
+ ZonedDateTime.parse(v, DateTimeFormatter.RFC_1123_DATE_TIME).toInstant.toEpochMilli).toOption)
+ .map(t => math.min(MaxRetryDelayMs, math.max(0L, t - System.currentTimeMillis())))
+ seconds.orElse(date).getOrElse(fallback.toLong)
+ }
+
+ private def readBounded(input: InputStream, limit: Int, request: HttpRequestBase): String = {
+ val output = new ByteArrayOutputStream(math.min(limit, BufferSize))
+ val buffer = new Array[Byte](BufferSize)
+ @tailrec
+ def read(): Unit = {
+ checkInterrupted()
+ val remaining = limit.toLong - output.size() + 1
+ val count = input.read(buffer, 0, math.min(buffer.length.toLong, remaining).toInt)
+ if (count >= 0) {
+ if (output.size().toLong + count > limit) {
+ request.abort()
+ throw new ResponseTooLarge
+ }
+ output.write(buffer, 0, count)
+ read()
+ }
+ }
+ // The enclosing HTTP response owns the stream; closing it here after abort can mask the original failure.
+ try {
+ read()
+ new String(output.toByteArray, StandardCharsets.UTF_8)
+ } catch {
+ case error: InterruptedException =>
+ request.abort()
+ throw error
+ case error: IOException =>
+ request.abort()
+ throw error
+ }
+ }
+
+ private def exchange(client: CloseableHttpClient, settings: Settings, data: HTTPRequestData): WireResponse = {
+ checkInterrupted()
+ val request = data.toHTTPCore
+ request.setConfig(settings.requestConfig)
+ var code = 0
+ var location = Option.empty[String]
+ var retryAfter = Option.empty[String]
+ try {
+ val response = client.execute(request)
+ try {
+ code = response.getStatusLine.getStatusCode
+ location = Option(response.getFirstHeader("Operation-Location")).map(_.getValue)
+ retryAfter = Option(response.getFirstHeader("Retry-After")).map(_.getValue)
+ val body = Option(response.getEntity).map { entity =>
+ if (entity.getContentLength > settings.maxResponseBytes) {
+ request.abort()
+ throw new ResponseTooLarge
+ }
+ readBounded(entity.getContent, settings.maxResponseBytes, request)
+ }.getOrElse("")
+ WireResponse(code, body, location, retryAfter)
+ } finally {
+ response.close()
+ }
+ } catch {
+ case _: ResponseTooLarge =>
+ WireResponse(code, "", location, retryAfter, Some(diagnostic("ResponseTooLarge",
+ "Response exceeded maxResponseBytes. Use explicit input ranges or increase the configured bound.")))
+ case error: InterruptedIOException if Thread.currentThread().isInterrupted => throw error
+ case error: IOException =>
+ request.abort()
+ val message = if (data.requestLine.method == "GET") {
+ "HTTP polling transport failed. Retain the operation handle and resume polling."
+ } else {
+ "HTTP transport failed. A submitted request may have been accepted; it was not resubmitted."
+ }
+ WireResponse(code, "", location, retryAfter,
+ Some(diagnostic("TransportError", s"${error.getClass.getSimpleName}: $message")),
+ transportFailure = true)
+ } finally {
+ request.releaseConnection()
+ }
+ }
+
+ private def withClient[T](settings: Settings)(body: CloseableHttpClient => T): T = {
+ val client = HttpClients.custom().disableAutomaticRetries().disableRedirectHandling()
+ .disableCookieManagement().setDefaultRequestConfig(settings.requestConfig).build()
+ try {
+ body(client)
+ } finally {
+ client.close()
+ }
+ }
+
+ private def asObject(raw: String): Option[JsObject] = {
+ try {
+ raw.parseJson match {
+ case value: JsObject => Some(value)
+ case _ => None
+ }
+ } catch {
+ case _: JsonParser.ParsingException => None
+ }
+ }
+
+ private def stringField(value: JsObject, field: String): Option[String] =
+ value.fields.get(field).collect { case JsString(text) => text }
+
+ private def checkClientResponse(raw: WireResponse,
+ response: ContentUnderstandingResponse,
+ invalidStatus: Boolean,
+ failOnClientError: Boolean): Unit = {
+ if (failOnClientError && !raw.retryable && (!raw.successful || raw.diagnostic.isDefined || invalidStatus)) {
+ throw new ContentUnderstandingException(response)
+ }
+ }
+
+ private def decode(raw: WireResponse,
+ location: Option[String],
+ failOnClientError: Boolean = false): ContentUnderstandingResponse = {
+ val json = raw.json
+ val id = json.flatMap(stringField(_, "id"))
+ val serviceError = json.flatMap(_.fields.get("error")).filterNot(_ == JsNull).map(_.compactPrint)
+ val serviceStatus = json.flatMap(stringField(_, "status"))
+ .flatMap(s => StatusNames.get(s.toLowerCase(java.util.Locale.ROOT)))
+ val status = if (raw.transportFailure) {
+ "Unknown"
+ } else if (!raw.successful || raw.diagnostic.isDefined) {
+ "Failed"
+ } else {
+ serviceStatus.getOrElse("Failed")
+ }
+ val missingStatusError = if (raw.successful && serviceStatus.isEmpty) {
+ Some(diagnostic("InvalidResponse", "Expected a JSON operation object with a recognized status."))
+ } else {
+ None
+ }
+ val failedError = if (Set("Failed", "Canceled")(status)) {
+ Some(diagnostic("OperationFailed", "The service operation did not succeed."))
+ } else {
+ None
+ }
+ val httpError = if (!raw.successful) {
+ Some(diagnostic("HttpError", s"HTTP ${raw.httpStatus}; redirects are not followed."))
+ } else {
+ None
+ }
+ val response = ContentUnderstandingResponse(location, id, status, raw.httpStatus, raw.body,
+ raw.diagnostic.orElse(serviceError).orElse(httpError).orElse(missingStatusError).orElse(failedError))
+ checkClientResponse(raw, response, missingStatusError.isDefined, failOnClientError)
+ response
+ }
+
+ private def ongoing(response: ContentUnderstandingResponse): Boolean =
+ Set("Running", "NotStarted")(response.status)
+
+ private def uncertainSubmission(raw: WireResponse, response: ContentUnderstandingResponse): Boolean = {
+ val unreadable = raw.diagnostic.isDefined || raw.json.flatMap(stringField(_, "status"))
+ .flatMap(status => StatusNames.get(status.toLowerCase(java.util.Locale.ROOT))).isEmpty
+ raw.transportFailure || raw.httpStatus == HttpStatus.SC_REQUEST_TIMEOUT ||
+ raw.httpStatus >= HttpStatus.SC_INTERNAL_SERVER_ERROR ||
+ (raw.successful && unreadable) || (!raw.successful && response.operationLocation.isDefined)
+ }
+
+ private def decodeSubmission(raw: WireResponse, location: Option[String]): ContentUnderstandingResponse = {
+ val response = decode(raw, location)
+ if (response.operationLocation.isEmpty && AdmissionRejections(raw.httpStatus)) {
+ response.copy(status = "Rejected")
+ } else if (uncertainSubmission(raw, response)) {
+ response.copy(status = "Unknown")
+ } else {
+ response
+ }
+ }
+
+ private def safeSubmission(settings: Settings, raw: WireResponse): ContentUnderstandingResponse = {
+ val location = raw.location.map { value =>
+ try {
+ Right(validateOperationLocation(settings.endpoint, value).toString)
+ } catch {
+ case _: IllegalArgumentException => Left(diagnostic("InvalidOperationLocation",
+ "The service returned an unsafe or malformed Operation-Location; it was not requested."))
+ }
+ }
+ val response = decodeSubmission(raw, location.flatMap(_.right.toOption))
+ val locationError = location.flatMap(_.left.toOption)
+ val missingLocation = raw.successful && (ongoing(response) || raw.httpStatus == HttpStatus.SC_ACCEPTED) &&
+ response.operationLocation.isEmpty
+ if (locationError.isDefined || missingLocation) {
+ response.copy(status = if (raw.successful) "Unknown" else response.status,
+ error = locationError.orElse(Some(diagnostic("MissingOperationLocation",
+ "An accepted operation did not include Operation-Location and cannot be resumed."))))
+ } else {
+ response
+ }
+ }
+
+ private def getRequest(source: HTTPRequestData, location: String): HTTPRequestData =
+ source.copy(requestLine = RequestLineData("GET", location, None), entity = None,
+ headers = source.headers.filterNot(h =>
+ Set("content-length", "transfer-encoding", "host")(h.name.toLowerCase(java.util.Locale.ROOT))))
+
+ private def decodePoll(raw: WireResponse,
+ location: String,
+ analyzer: Option[String],
+ failOnClientError: Boolean): ContentUnderstandingResponse = {
+ val unavailable = analyzer.isEmpty && !raw.transportFailure && UnavailableResults(raw.httpStatus)
+ val response = decode(raw, Some(location), failOnClientError && !unavailable)
+ if (unavailable) response.copy(status = "ResultUnavailable") else response
+ }
+
+ private def pollWithClient(client: CloseableHttpClient,
+ settings: Settings,
+ source: HTTPRequestData,
+ location: String,
+ initial: Option[ContentUnderstandingResponse],
+ firstDelay: Long,
+ analyzer: Option[String] = None,
+ failOnClientError: Boolean = false): ContentUnderstandingResponse = {
+ validateOperationLocation(settings.endpoint, location, analyzer)
+ val request = getRequest(source, location)
+ @tailrec
+ def loop(remaining: Int, previous: Option[ContentUnderstandingResponse], delay: Long)
+ : ContentUnderstandingResponse = {
+ pause(delay)
+ val raw = exchange(client, settings, request)
+ val parsed = decodePoll(raw, location, analyzer, failOnClientError)
+ val current = if (raw.retryable) {
+ previous.map(_.copy(httpStatus = parsed.httpStatus, error = parsed.error))
+ .getOrElse(parsed.copy(status = "Unknown"))
+ } else {
+ parsed
+ }
+ if (remaining == 1 || (!raw.retryable && !ongoing(current))) {
+ if (failOnClientError && raw.retryable) {
+ throw new ContentUnderstandingException(current)
+ }
+ current
+ } else {
+ loop(remaining - 1, Some(current), retryDelay(raw.retryAfter, settings.pollingDelay))
+ }
+ }
+ loop(settings.maxPollAttempts, initial, firstDelay)
+ }
+
+ def execute(settings: Settings,
+ request: HTTPRequestData,
+ mode: String,
+ failOnClientError: Boolean = false): ContentUnderstandingResponse = {
+ withClient(settings) { client =>
+ if (mode == "poll") {
+ pollWithClient(client, settings, request, request.requestLine.uri, None, 0L,
+ failOnClientError = failOnClientError)
+ } else {
+ val raw = exchange(client, settings, request)
+ val submitted = safeSubmission(settings, raw)
+ val resumable = ongoing(submitted) || (submitted.status == "Unknown" && submitted.operationLocation.isDefined)
+ if (mode == "analyze" && resumable) {
+ pollWithClient(client, settings, request, submitted.operationLocation.get, Some(submitted),
+ retryDelay(raw.retryAfter, settings.pollingDelay), failOnClientError = failOnClientError)
+ } else {
+ submitted
+ }
+ }
+ }
+ }
+
+ private def analyzerBody(raw: WireResponse): String = {
+ val json = raw.json
+ val hasError = json.exists(_.fields.get("error").exists(_ != JsNull))
+ if (!raw.successful || raw.diagnostic.isDefined || json.isEmpty || hasError) {
+ throw new ContentUnderstandingException(decode(raw, None))
+ }
+ raw.body
+ }
+
+ private def getAnalyzerWithClient(client: CloseableHttpClient,
+ settings: Settings,
+ request: HTTPRequestData): String = {
+ @tailrec
+ def get(remaining: Int): WireResponse = {
+ val response = exchange(client, settings, request)
+ if (response.retryable && remaining > 1) {
+ pause(retryDelay(response.retryAfter, settings.pollingDelay))
+ get(remaining - 1)
+ } else {
+ response
+ }
+ }
+ analyzerBody(get(ManagementGetAttempts))
+ }
+
+ def getAnalyzer(settings: Settings, request: HTTPRequestData): String =
+ withClient(settings)(getAnalyzerWithClient(_, settings, request))
+
+ def createAnalyzer(settings: Settings,
+ request: HTTPRequestData,
+ getRequestData: HTTPRequestData,
+ analyzerId: String): String = {
+ withClient(settings) { client =>
+ val raw = exchange(client, settings, request)
+ analyzerBody(raw)
+ raw.location match {
+ case Some(location) =>
+ try {
+ validateOperationLocation(settings.endpoint, location, Some(analyzerId))
+ } catch {
+ case _: IllegalArgumentException =>
+ throw new ContentUnderstandingException(decode(raw, None).copy(status = "Failed",
+ error = Some(diagnostic("InvalidOperationLocation",
+ "The analyzer creation operation URL was unsafe; it was not requested."))))
+ }
+ val completed = pollWithClient(client, settings, request, location, None,
+ retryDelay(raw.retryAfter, settings.pollingDelay), Some(analyzerId))
+ if (completed.status != "Succeeded" || completed.error.isDefined) {
+ throw new ContentUnderstandingException(completed)
+ }
+ getAnalyzerWithClient(client, settings, getRequestData)
+ case None =>
+ val status = raw.json.flatMap(stringField(_, "status"))
+ if (!status.exists(_.equalsIgnoreCase("ready"))) {
+ throw new ContentUnderstandingException(decode(raw, None).copy(status = "Failed",
+ error = Some(diagnostic("MissingOperationLocation",
+ "Analyzer creation was not ready and supplied no management operation URL."))))
+ }
+ raw.body
+ }
+ }
+ }
+}
diff --git a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/contentunderstanding/ContentUnderstandingWriter.scala b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/contentunderstanding/ContentUnderstandingWriter.scala
new file mode 100644
index 0000000000..1e4b9d737a
--- /dev/null
+++ b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/contentunderstanding/ContentUnderstandingWriter.scala
@@ -0,0 +1,227 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in project root for information.
+
+package com.microsoft.azure.synapse.ml.services.contentunderstanding
+
+import com.microsoft.azure.synapse.ml.param.ServiceParam
+import org.apache.hadoop.fs.Path
+import org.apache.spark.sql.expressions.Window
+import org.apache.spark.sql.functions.{col, lit, row_number}
+import org.apache.spark.sql.types.{LongType, StringType, StructField, StructType}
+import org.apache.spark.sql.{Column, DataFrame, Dataset, Row, SparkSession}
+
+import scala.annotation.tailrec
+import scala.collection.JavaConverters._
+
+/**
+ * Driver-side, single-writer orchestration with an append-only operation journal.
+ * Every accepted operation is committed before polling; each completed unit is
+ * committed independently. Spark task retries cannot repeat these driver-side POSTs.
+ * A crash between a service POST and its journal commit can still repeat that POST.
+ */
+object ContentUnderstandingWriter {
+
+ val Schema: StructType = StructType(Seq(
+ StructField("documentId", StringType),
+ StructField("requestHash", StringType),
+ StructField("sequence", LongType)
+ ) ++ ContentUnderstandingResponse.schema.fields)
+
+ private val MetadataColumns = Seq("documentId", "requestHash", "sequence", "operationLocation", "status")
+ private val TerminalStatuses = Set("Succeeded", "Failed", "Canceled", "Cancelled", "ResultUnavailable")
+
+ private def quoted(name: String): Column = col("`" + name.replace("`", "``") + "`")
+
+ /** Return one latest state per document/range ID, including pending and failed operations. */
+ def latest(journal: DataFrame): DataFrame = {
+ validateJournal(journal.schema)
+ val window = Window.partitionBy("documentId").orderBy(col("sequence").desc)
+ journal.withColumn("_cu_rank", row_number().over(window))
+ .filter(col("_cu_rank") === lit(1))
+ .drop("_cu_rank")
+ }
+
+ def readTable(spark: SparkSession, tableName: String): DataFrame = latest(spark.table(tableName))
+
+ def readPath(spark: SparkSession, path: String, format: String = "delta"): DataFrame =
+ latest(spark.read.format(format).load(path))
+
+ def writeToTable(dataset: Dataset[_],
+ analyzer: ContentUnderstanding,
+ idCol: String,
+ tableName: String,
+ format: String = "delta",
+ batchSize: Int = 1): DataFrame = {
+ require(tableName != null && tableName.trim.nonEmpty, "tableName must not be empty")
+ val spark = dataset.sparkSession
+ val store = new Journal {
+ override def exists: Boolean = spark.catalog.tableExists(tableName)
+ override def read(): DataFrame = spark.table(tableName)
+ override def append(data: DataFrame): Unit =
+ data.write.format(format).mode("append").saveAsTable(tableName)
+ }
+ write(dataset, analyzer, idCol, format, batchSize, store)
+ }
+
+ def writeToPath(dataset: Dataset[_],
+ analyzer: ContentUnderstanding,
+ idCol: String,
+ path: String,
+ format: String = "delta",
+ batchSize: Int = 1): DataFrame = {
+ require(path != null && path.trim.nonEmpty, "path must not be empty")
+ val spark = dataset.sparkSession
+ val output = new Path(path)
+ val fs = output.getFileSystem(spark.sessionState.newHadoopConf())
+ val store = new Journal {
+ override def exists: Boolean = fs.exists(output)
+ override def read(): DataFrame = spark.read.format(format).load(path)
+ override def append(data: DataFrame): Unit = data.write.format(format).mode("append").save(path)
+ }
+ write(dataset, analyzer, idCol, format, batchSize, store)
+ }
+
+ private trait Journal {
+ def exists: Boolean
+ def read(): DataFrame
+ def append(data: DataFrame): Unit
+ }
+
+ private def validateJournal(schema: StructType): Unit = {
+ require(
+ schema.fields.map(f => f.name -> f.dataType).toSeq == Schema.fields.map(f => f.name -> f.dataType).toSeq,
+ "The destination must be a Content Understanding operation journal with the expected schema"
+ )
+ }
+
+ private def inputColumns(analyzer: ContentUnderstanding): Seq[String] =
+ analyzer.params.toSeq.flatMap {
+ case p: ServiceParam[_] if analyzer.isDefined(p) => analyzer.getOrDefault(p).right.toOption
+ case _ => None
+ }
+
+ private def validateInput(dataset: Dataset[_], analyzer: ContentUnderstanding, idCol: String): Unit = {
+ require(!dataset.isStreaming, "Use the writer inside foreachBatch for streaming input")
+ require(idCol != null && dataset.schema.fieldNames.contains(idCol), "idCol must name an input column")
+ require(dataset.schema(idCol).dataType == StringType, "idCol must have StringType")
+ require(analyzer.getOperationMode != "poll", "The durable writer requires document input, not poll mode")
+ analyzer.validateInputSchema(dataset.schema)
+ val ids = dataset.select(quoted(idCol).alias("documentId"))
+ require(
+ ids.filter(col("documentId").isNull || col("documentId").rlike("(?U)^\\s*$")).limit(1).count() == 0,
+ "Document IDs must not be null or blank"
+ )
+ require(
+ ids.groupBy("documentId").count().filter(col("count") > lit(1)).limit(1).count() == 0,
+ "Document IDs must be unique; include the selected page/time range in each ID"
+ )
+ }
+
+ private def append(spark: SparkSession,
+ journal: Journal,
+ documentId: String,
+ requestHash: String,
+ sequence: Long,
+ response: ContentUnderstandingResponse): Unit = {
+ val responseRow = ContentUnderstandingResponse.makeToRowConverter(response)
+ val record = Row.fromSeq(Seq(documentId, requestHash, sequence) ++ responseRow.toSeq)
+ journal.append(spark.createDataFrame(Seq(record).asJava, Schema))
+ }
+
+ private def recordSubmission(spark: SparkSession,
+ journal: Journal,
+ documentId: String,
+ requestHash: String,
+ sequence: Long,
+ response: ContentUnderstandingResponse): Unit = {
+ if (response.status == "Rejected") {
+ throw new ContentUnderstandingException(response)
+ }
+ append(spark, journal, documentId, requestHash, sequence, response)
+ if (!TerminalStatuses(response.status) && response.operationLocation.isEmpty) {
+ throw new ContentUnderstandingException(response)
+ }
+ }
+
+ private def process(spark: SparkSession,
+ journal: Journal,
+ analyzer: ContentUnderstanding,
+ row: Row,
+ documentId: String,
+ previous: Option[Row]): Unit = {
+ val requestHash = analyzer.requestFingerprint(row)
+ previous.foreach { state =>
+ require(state.getAs[String]("requestHash") == requestHash,
+ "A document ID was reused with different content or analysis options; use a new ID or a new journal")
+ }
+ val previousStatus = previous.map(_.getAs[String]("status"))
+ if (!previousStatus.exists(TerminalStatuses)) {
+ val sequence = previous.map(_.getAs[Long]("sequence")).getOrElse(-1L)
+ require(sequence < Long.MaxValue - 1, "The operation journal sequence is exhausted")
+ val submitted = previous match {
+ case Some(state) =>
+ val location = state.getAs[String]("operationLocation")
+ val missingHandle = if (state.getAs[String]("status") == "Unknown") {
+ "An earlier submission has an unknown outcome and no operation handle. " +
+ "Inspect the journal and service outcome before intentionally submitting it with a new ID."
+ } else {
+ "A pending journal entry is missing its operationLocation"
+ }
+ require(location != null && location.nonEmpty, missingHandle)
+ None
+ case None =>
+ val response = analyzer.submitOne(row)
+ recordSubmission(spark, journal, documentId, requestHash, sequence + 1, response)
+ Some(response)
+ }
+ val location = submitted.flatMap(_.operationLocation)
+ .orElse(previous.map(_.getAs[String]("operationLocation")))
+ if (analyzer.getOperationMode == "analyze" && !submitted.exists(r => TerminalStatuses(r.status))) {
+ require(location.isDefined, "An accepted analysis is missing its operationLocation")
+ val response = analyzer.pollOne(row, location.get)
+ val nextSequence = if (previous.isDefined) sequence + 1 else sequence + 2
+ append(spark, journal, documentId, requestHash, nextSequence, response)
+ }
+ }
+ }
+
+ private def write(dataset: Dataset[_],
+ analyzer: ContentUnderstanding,
+ idCol: String,
+ format: String,
+ batchSize: Int,
+ journal: Journal): DataFrame = {
+ require(batchSize > 0, "batchSize must be positive")
+ require(Set("delta", "parquet").contains(format),
+ "format must be delta or parquet; use Delta for transactional lakehouse tables")
+ validateInput(dataset, analyzer, idCol)
+ val spark = dataset.sparkSession
+ if (journal.exists) {
+ validateJournal(journal.read().schema)
+ }
+ // Check the destination and provider before issuing a billable request.
+ journal.append(spark.createDataFrame(Seq.empty[Row].asJava, Schema))
+ val selected = (Seq(idCol) ++ inputColumns(analyzer)).distinct.map(quoted)
+ val input = dataset.select(selected: _*)
+ @tailrec
+ def writeBatch(after: Option[String]): Unit = {
+ val remaining = after.map(value => input.filter(quoted(idCol) > lit(value))).getOrElse(input)
+ val batch = remaining.orderBy(quoted(idCol)).limit(batchSize).collect()
+ if (batch.nonEmpty) {
+ val ids = batch.map(_.getAs[String](idCol))
+ val relevant = journal.read().filter(col("documentId").isin(ids.toSeq: _*))
+ val states = latest(relevant)
+ .select(MetadataColumns.map(col): _*)
+ .collect()
+ .map(state => state.getAs[String]("documentId") -> state).toMap
+ batch.foreach { row =>
+ val id = row.getAs[String](idCol)
+ process(spark, journal, analyzer, row, id, states.get(id))
+ }
+ writeBatch(Some(ids.last))
+ }
+ }
+ writeBatch(None)
+ latest(journal.read())
+ }
+}
diff --git a/cognitive/src/test/python/synapsemltest/services/content_understanding_fixtures.py b/cognitive/src/test/python/synapsemltest/services/content_understanding_fixtures.py
new file mode 100644
index 0000000000..fff3723250
--- /dev/null
+++ b/cognitive/src/test/python/synapsemltest/services/content_understanding_fixtures.py
@@ -0,0 +1,121 @@
+# Copyright (C) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See LICENSE in project root for information.
+
+"""Small synthetic PDF and DOCX inputs, generated without document libraries."""
+
+from io import BytesIO
+from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo
+
+
+DOCX_MIME_TYPE = (
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+)
+DOCX_FIRST_PAGE = "SYNAPSEML DOCX FIRST PAGE"
+DOCX_SECOND_PAGE = "SYNAPSEML DOCX SECOND PAGE"
+
+
+def synthetic_pdf(page_count=4):
+ if page_count < 1:
+ raise ValueError("page_count must be positive")
+ objects = [
+ b"<< /Type /Catalog /Pages 2 0 R >>",
+ b"",
+ b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+ ]
+ kids = []
+ for number in range(1, page_count + 1):
+ page_id = len(objects) + 1
+ content_id = page_id + 1
+ kids.append(f"{page_id} 0 R")
+ objects.append(
+ (
+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
+ f"/Resources << /Font << /F1 3 0 R >> >> /Contents {content_id} 0 R >>"
+ ).encode("ascii")
+ )
+ text = (
+ f"BT /F1 16 Tf 50 730 Td (SYNTHETIC INVOICE CU-{number:03d}) Tj "
+ "0 -35 Td (Example Company - synthetic test data only) Tj "
+ f"0 -35 Td (Page {number} of {page_count}) Tj "
+ "0 -35 Td (Invoice date: 2026-09-04) Tj "
+ f"0 -35 Td (Widgets: {number} x 10.00 USD) Tj "
+ f"0 -35 Td (Total due: {number * 10}.00 USD) Tj ET"
+ ).encode("ascii")
+ objects.append(
+ f"<< /Length {len(text)} >>\nstream\n".encode("ascii")
+ + text
+ + b"\nendstream"
+ )
+ objects[1] = (
+ f"<< /Type /Pages /Kids [{' '.join(kids)}] /Count {page_count} >>"
+ ).encode("ascii")
+ data = bytearray(b"%PDF-1.4\n")
+ offsets = [0]
+ for index, obj in enumerate(objects, 1):
+ offsets.append(len(data))
+ data.extend(f"{index} 0 obj\n".encode("ascii") + obj + b"\nendobj\n")
+ xref = len(data)
+ data.extend(f"xref\n0 {len(offsets)}\n0000000000 65535 f \n".encode("ascii"))
+ for offset in offsets[1:]:
+ data.extend(f"{offset:010d} 00000 n \n".encode("ascii"))
+ data.extend(
+ (
+ f"trailer\n<< /Size {len(offsets)} /Root 1 0 R >>\n"
+ f"startxref\n{xref}\n%%EOF\n"
+ ).encode("ascii")
+ )
+ return bytes(data)
+
+
+def synthetic_docx():
+ content_types = """
+
+
+
+
+"""
+ relationships = """
+
+
+"""
+ document = f"""
+
+
+ {DOCX_FIRST_PAGE}
+ Synthetic receipt CU-DOCX-001
+
+
+
+
+ Item
+ Amount
+
+
+ Synthetic widgets
+ 42.00 USD
+
+
+
+ {DOCX_SECOND_PAGE}
+ Synthetic delivery reference CU-DOCX-002
+
+
+
+
+
+"""
+ output = BytesIO()
+ with ZipFile(output, "w") as archive:
+ for name, text in (
+ ("[Content_Types].xml", content_types),
+ ("_rels/.rels", relationships),
+ ("word/document.xml", document),
+ ):
+ # Stable timestamps keep the document request fingerprint identical on resume.
+ entry = ZipInfo(name, date_time=(2020, 1, 1, 0, 0, 0))
+ entry.compress_type = ZIP_DEFLATED
+ archive.writestr(entry, text.encode("utf-8"))
+ return output.getvalue()
diff --git a/cognitive/src/test/python/synapsemltest/services/test_ContentUnderstanding.py b/cognitive/src/test/python/synapsemltest/services/test_ContentUnderstanding.py
new file mode 100644
index 0000000000..f7ba9c8699
--- /dev/null
+++ b/cognitive/src/test/python/synapsemltest/services/test_ContentUnderstanding.py
@@ -0,0 +1,210 @@
+# Copyright (C) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See LICENSE in project root for information.
+
+import base64
+import json
+import os
+import tempfile
+import threading
+import unittest
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from urllib.parse import urlsplit
+
+from synapse.ml.core.init_spark import init_spark
+from synapse.ml.services.contentunderstanding import ContentUnderstanding
+
+spark = init_spark()
+
+
+class TestContentUnderstanding(unittest.TestCase):
+ def setUp(self):
+ self.requests = []
+ requests = self.requests
+
+ class Handler(BaseHTTPRequestHandler):
+ def log_message(self, format, *args):
+ pass
+
+ def send_json(self, status, body, location=None):
+ payload = json.dumps(body).encode("utf-8")
+ self.send_response(status)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(payload)))
+ if location:
+ self.send_header("Operation-Location", location)
+ self.end_headers()
+ self.wfile.write(payload)
+
+ def read_body(self):
+ return json.loads(self.rfile.read(int(self.headers["Content-Length"])))
+
+ def do_POST(self):
+ requests.append(("POST", self.path, self.read_body()))
+ operation = "123e4567-e89b-12d3-a456-426614174000"
+ location = (
+ f"http://127.0.0.1:{self.server.server_port}"
+ f"/contentunderstanding/analyzerResults/{operation}"
+ "?api-version=2025-11-01"
+ )
+ self.send_json(
+ 202,
+ {"id": operation, "status": "Running", "result": {"contents": []}},
+ location,
+ )
+
+ def do_GET(self):
+ requests.append(("GET", self.path, None))
+ if "/analyzerResults/" in self.path:
+ body = {
+ "id": "123e4567-e89b-12d3-a456-426614174000",
+ "status": "Succeeded",
+ "usage": {"tokens": {"future-model-input": 3}},
+ "result": {
+ "contents": [
+ {
+ "markdown": "synthetic result",
+ "fields": {"Optional": {"type": "string"}},
+ }
+ ]
+ },
+ }
+ else:
+ body = {
+ "analyzerId": urlsplit(self.path).path.split("/")[-1],
+ "status": "ready",
+ }
+ self.send_json(200, body)
+
+ def do_PUT(self):
+ requests.append(("PUT", self.path, self.read_body()))
+ self.send_json(
+ 201,
+ {
+ "analyzerId": urlsplit(self.path).path.split("/")[-1],
+ "status": "ready",
+ },
+ )
+
+ self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
+ self.thread.start()
+
+ def tearDown(self):
+ self.server.shutdown()
+ self.server.server_close()
+ self.thread.join(timeout=5)
+
+ def analyzer(self):
+ return (
+ ContentUnderstanding()
+ .setEndpoint(f"http://127.0.0.1:{self.server.server_port}")
+ .setSubscriptionKey("synthetic-test-key")
+ .setDocumentBytesCol("body")
+ .setMimeType("text/plain")
+ .setDocumentNameCol("documentId")
+ .setOutputCol("analysis")
+ .setMaxPollAttempts(2)
+ .setPollingDelay(0)
+ )
+
+ def documents(self):
+ return spark.createDataFrame(
+ [("one", bytearray("Synthetic caf\u00e9 document".encode("utf-8")))],
+ "documentId string, body binary",
+ )
+
+ def test_generated_wrapper_copy_and_persistence_preserve_request_options(self):
+ analyzer = (
+ self.analyzer()
+ .setModelDeployments({"completion": "my-deployment"})
+ .setStringEncoding("utf16")
+ .setRange("1")
+ )
+ with tempfile.TemporaryDirectory() as directory:
+ path = os.path.join(directory, "analyzer")
+ analyzer.copy({}).write().save(path)
+ loaded = ContentUnderstanding.load(path)
+ row = loaded.transform(self.documents()).collect()[0]
+ self.assertEqual(row.analysis.status, "Succeeded")
+ raw = json.loads(row.analysis.rawResponse)
+ self.assertIn("future-model-input", raw["usage"]["tokens"])
+ submitted = next(request for request in self.requests if request[0] == "POST")
+ self.assertIn("stringEncoding=utf16", submitted[1])
+ self.assertEqual(
+ submitted[2]["modelDeployments"], {"completion": "my-deployment"}
+ )
+ document = submitted[2]["inputs"][0]
+ self.assertEqual(document["range"], "1")
+ self.assertEqual(
+ base64.b64decode(document["data"]).decode("utf-8"),
+ "Synthetic caf\u00e9 document",
+ )
+ self.assertNotIn("dataBase64", document)
+
+ def test_generated_durable_path_helpers_resume_without_resubmission(self):
+ analyzer = self.analyzer()
+ with tempfile.TemporaryDirectory() as directory:
+ path = os.path.join(directory, "journal")
+ first = analyzer.writeToPath(
+ self.documents(), idCol="documentId", path=path, format="parquet"
+ )
+ self.assertEqual(first.select("status").first()[0], "Succeeded")
+ analyzer.writeToPath(
+ self.documents(), idCol="documentId", path=path, format="parquet"
+ )
+ self.assertEqual(analyzer.readPath(spark, path, "parquet").count(), 1)
+ self.assertEqual(
+ sum(request[0] == "POST" for request in self.requests),
+ 1,
+ )
+
+ def test_custom_analyzer_dictionary_is_forwarded_only_by_explicit_driver_call(self):
+ analyzer = self.analyzer().setAnalyzerId("custom-v1")
+ definition = {
+ "baseAnalyzerId": "prebuilt-document",
+ "config": {"returnDetails": True},
+ "fieldSchema": {"fields": {"Supplier": {"type": "string"}}},
+ }
+ created = json.loads(analyzer.createAnalyzer(definition, allowReplace=False))
+ current = json.loads(analyzer.getAnalyzer())
+ self.assertEqual(created["status"], "ready")
+ self.assertEqual(current["analyzerId"], "custom-v1")
+ self.assertEqual(self.requests[0][0], "PUT")
+ self.assertIn("allowReplace=false", self.requests[0][1])
+ self.assertEqual(self.requests[0][2], definition)
+
+ def test_scalar_bytes_survive_constructor_save_load_copy_and_clear(self):
+ payload = bytes([0, 127, 128, 255])
+ analyzer = (
+ ContentUnderstanding(documentBytes=payload)
+ .setEndpoint(f"http://127.0.0.1:{self.server.server_port}")
+ .setOutputCol("analysis")
+ .setOperationMode("submit")
+ )
+ with tempfile.TemporaryDirectory() as directory:
+ path = os.path.join(directory, "analyzer")
+ analyzer.write().save(path)
+ loaded = ContentUnderstanding.load(path)
+ self.assertEqual(bytes(loaded.getDocumentBytes()), payload)
+ replacement = bytes([255, 128, 1, 0])
+ copied = loaded.copy({loaded.documentBytes: replacement})
+ copied.transform(self.documents()).collect()
+ submitted = next(
+ request for request in self.requests if request[0] == "POST"
+ )
+ self.assertEqual(
+ base64.b64decode(submitted[2]["inputs"][0]["data"]), replacement
+ )
+ loaded.clear(loaded.documentBytes)
+ loaded.setDocumentBytesCol("body").transform(self.documents()).collect()
+ submitted = [request for request in self.requests if request[0] == "POST"][
+ -1
+ ]
+ self.assertEqual(
+ base64.b64decode(submitted[2]["inputs"][0]["data"]),
+ "Synthetic caf\u00e9 document".encode("utf-8"),
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/cognitive/src/test/python/synapsemltest/services/test_ContentUnderstandingE2E.py b/cognitive/src/test/python/synapsemltest/services/test_ContentUnderstandingE2E.py
new file mode 100644
index 0000000000..9f4e077859
--- /dev/null
+++ b/cognitive/src/test/python/synapsemltest/services/test_ContentUnderstandingE2E.py
@@ -0,0 +1,330 @@
+# Copyright (C) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See LICENSE in project root for information.
+
+"""Opt-in live Content Understanding tests using synthetic documents only.
+
+Set CONTENT_UNDERSTANDING_ENDPOINT and either CONTENT_UNDERSTANDING_API_KEY
+or CONTENT_UNDERSTANDING_AAD_TOKEN through the test process environment.
+CONTENT_UNDERSTANDING_TEST_PREVIEW=1 enables the preview case.
+On Fabric, set CONTENT_UNDERSTANDING_TEST_FORMAT=delta and
+CONTENT_UNDERSTANDING_TEST_OUTPUT_ROOT to a scratch lakehouse Files path.
+The suite creates and removes one uniquely named table and one journal path.
+It never provisions analyzers or changes model deployments or resource defaults.
+"""
+
+import json
+import os
+import tempfile
+import unittest
+import uuid
+from contextlib import contextmanager
+
+from py4j.protocol import Py4JJavaError
+
+from synapsemltest.services.content_understanding_fixtures import (
+ DOCX_FIRST_PAGE,
+ DOCX_MIME_TYPE,
+ DOCX_SECOND_PAGE,
+ synthetic_docx,
+ synthetic_pdf,
+)
+
+
+GA_VERSION = "2025-11-01"
+PREVIEW_VERSION = "2026-06-01-preview"
+
+
+class TestContentUnderstandingE2E(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.endpoint = os.environ.get("CONTENT_UNDERSTANDING_ENDPOINT")
+ cls.key = os.environ.get("CONTENT_UNDERSTANDING_API_KEY")
+ cls.token = os.environ.get("CONTENT_UNDERSTANDING_AAD_TOKEN")
+ if not cls.endpoint or not (cls.key or cls.token):
+ raise unittest.SkipTest(
+ "Content Understanding live tests require an explicit endpoint "
+ "and API key or AAD token in the test process environment."
+ )
+ from synapse.ml.core.init_spark import init_spark
+ from synapse.ml.services.contentunderstanding import ContentUnderstanding
+
+ cls.spark = init_spark()
+ cls.stage_type = ContentUnderstanding
+ cls.format = os.environ.get("CONTENT_UNDERSTANDING_TEST_FORMAT", "parquet")
+ if cls.format not in ("delta", "parquet"):
+ raise ValueError(
+ "CONTENT_UNDERSTANDING_TEST_FORMAT must be delta or parquet"
+ )
+
+ def analyzer(self):
+ stage = (
+ self.stage_type()
+ .setEndpoint(self.endpoint)
+ .setAnalyzerId("prebuilt-read")
+ .setDocumentBytesCol("body")
+ .setDocumentNameCol("documentId")
+ .setMimeTypeCol("mimeType")
+ .setRangeCol("pageRange")
+ .setApiVersionCol("apiVersion")
+ .setOutputCol("analysis")
+ .setErrorCol("requestError")
+ .setMaxPollAttempts(120)
+ .setPollingDelay(1000)
+ )
+ if self.key:
+ return stage.setSubscriptionKey(self.key)
+ return stage.setAADToken(self.token)
+
+ def documents(self, rows):
+ return self.spark.createDataFrame(
+ rows,
+ "documentId string, body binary, mimeType string, "
+ "pageRange string, apiVersion string",
+ ).coalesce(1)
+
+ def pdf_document(self, pages="2"):
+ return self.documents(
+ [
+ (
+ "synthetic.pdf",
+ bytearray(synthetic_pdf()),
+ "application/pdf",
+ pages,
+ GA_VERSION,
+ )
+ ]
+ )
+
+ def docx_document(self, version=GA_VERSION):
+ return self.documents(
+ [
+ (
+ "synthetic.docx",
+ bytearray(synthetic_docx()),
+ DOCX_MIME_TYPE,
+ None,
+ version,
+ )
+ ]
+ )
+
+ def successful_body(self, response):
+ self.assertEqual(response.status, "Succeeded")
+ self.assertIsNone(response.error)
+ body = json.loads(response.rawResponse)
+ self.assertEqual(body["status"], "Succeeded")
+ self.assertTrue(body["result"]["contents"])
+ self.assertTrue(body["usage"])
+ return body
+
+ def assert_docx_content(self, body):
+ contents = body["result"]["contents"]
+ markdown = "\n".join(content["markdown"] for content in contents)
+ for text in (
+ DOCX_FIRST_PAGE,
+ DOCX_SECOND_PAGE,
+ "Synthetic widgets",
+ "42.00 USD",
+ ):
+ self.assertIn(text, markdown)
+ self.assertEqual(contents[0]["mimeType"], DOCX_MIME_TYPE)
+
+ def record(self, case, body, **facts):
+ contents = body["result"]["contents"]
+ print(
+ "CONTENT_UNDERSTANDING_E2E="
+ + json.dumps(
+ {
+ "case": case,
+ "status": body["status"],
+ "apiVersion": body["result"]["apiVersion"],
+ "usage": body["usage"],
+ "pages": [
+ page["pageNumber"]
+ for content in contents
+ for page in content.get("pages", [])
+ ],
+ **facts,
+ },
+ sort_keys=True,
+ )
+ )
+
+ @contextmanager
+ def output_path(self):
+ root = os.environ.get("CONTENT_UNDERSTANDING_TEST_OUTPUT_ROOT")
+ if not root:
+ with tempfile.TemporaryDirectory(prefix="cu-e2e-") as directory:
+ yield os.path.join(directory, "journal")
+ else:
+ path = root.rstrip("/") + "/journal-" + uuid.uuid4().hex
+ jpath = self.spark._jvm.org.apache.hadoop.fs.Path(path)
+ fs = jpath.getFileSystem(
+ self.spark._jsparkSession.sessionState().newHadoopConf()
+ )
+ if fs.exists(jpath):
+ raise RuntimeError(
+ "The unique Content Understanding test path already exists"
+ )
+ try:
+ yield path
+ finally:
+ if fs.exists(jpath) and not fs.delete(jpath, True):
+ raise RuntimeError(
+ "Could not remove the Content Understanding test journal"
+ )
+
+ def test_pdf_range_through_generated_transformer(self):
+ row = self.analyzer().transform(self.pdf_document()).first()
+ self.assertIsNone(row.requestError)
+ body = self.successful_body(row.analysis)
+ pages = [
+ page["pageNumber"]
+ for content in body["result"]["contents"]
+ for page in content["pages"]
+ ]
+ self.assertEqual(pages, [2])
+ self.assertEqual(body["usage"]["documentPagesBasic"], 1)
+ markdown = "\n".join(
+ content["markdown"] for content in body["result"]["contents"]
+ )
+ self.assertIn("CU-002", markdown)
+ self.assertNotIn("CU-001", markdown)
+ self.record("pdf-ga-range", body)
+
+ def test_docx_text_and_table_through_generated_transformer(self):
+ row = self.analyzer().transform(self.docx_document()).first()
+ self.assertIsNone(row.requestError)
+ body = self.successful_body(row.analysis)
+ self.assert_docx_content(body)
+ self.record("docx-ga-whole-document", body)
+
+ @unittest.skipUnless(
+ os.environ.get("CONTENT_UNDERSTANDING_TEST_PREVIEW") == "1",
+ "Set CONTENT_UNDERSTANDING_TEST_PREVIEW=1 to opt into the preview API",
+ )
+ def test_preview_docx_layout_preserves_metadata(self):
+ stage = self.analyzer().setAnalyzerId("prebuilt-layout")
+ row = stage.transform(self.docx_document(PREVIEW_VERSION)).first()
+ body = self.successful_body(row.analysis)
+ self.assert_docx_content(body)
+ self.assertEqual(body["result"]["apiVersion"], PREVIEW_VERSION)
+ self.assertTrue(
+ any("metadata" in content for content in body["result"]["contents"])
+ )
+ self.record("docx-preview-layout", body, metadataPreserved=True)
+
+ def test_completed_docx_survives_a_later_pdf_response_limit_and_resumes(self):
+ table = "cu_e2e_" + uuid.uuid4().hex
+ documents = self.documents(
+ [
+ (
+ "a.docx",
+ bytearray(synthetic_docx()),
+ DOCX_MIME_TYPE,
+ None,
+ GA_VERSION,
+ ),
+ (
+ "b.pdf",
+ bytearray(synthetic_pdf()),
+ "application/pdf",
+ "3-4",
+ GA_VERSION,
+ ),
+ ]
+ )
+ stage = (
+ self.analyzer()
+ .setMaxResponseBytes(2048)
+ .setOutputCol("body")
+ .setErrorCol("mimeType")
+ )
+ try:
+ with self.assertRaises(Py4JJavaError) as raised:
+ stage.writeToTable(documents, "documentId", table, self.format)
+ error = raised.exception.java_exception
+ self.assertEqual(
+ str(error.getClass().getName()),
+ "com.microsoft.azure.synapse.ml.services.contentunderstanding.ContentUnderstandingException",
+ )
+ self.assertEqual(
+ json.loads(str(error.response().error().get()))["code"],
+ "ResponseTooLarge",
+ )
+ partial = {
+ row.documentId: row
+ for row in stage.readTable(self.spark, table).collect()
+ }
+ self.assertEqual(partial["a.docx"].status, "Succeeded")
+ self.assertEqual(partial["b.pdf"].status, "Running")
+ self.assertTrue(partial["b.pdf"].operationLocation)
+ self.assertEqual(partial["b.pdf"].sequence, 0)
+ self.assert_docx_content(self.successful_body(partial["a.docx"]))
+
+ stage.setMaxResponseBytes(32 * 1024 * 1024)
+ resumed = {
+ row.documentId: row
+ for row in stage.writeToTable(
+ documents, "documentId", table, self.format
+ ).collect()
+ }
+ for document_id, response in resumed.items():
+ self.assertEqual(response.status, "Succeeded")
+ self.assertEqual(
+ response.operationLocation, partial[document_id].operationLocation
+ )
+ self.assertEqual(response.sequence, 1)
+ self.assertEqual(
+ resumed["a.docx"].rawResponse, partial["a.docx"].rawResponse
+ )
+ body = self.successful_body(resumed["b.pdf"])
+ pages = [
+ page["pageNumber"]
+ for content in body["result"]["contents"]
+ for page in content["pages"]
+ ]
+ self.assertEqual(pages, [3, 4])
+ self.assertEqual(self.spark.table(table).count(), 4)
+ self.record(
+ "mixed-table-recovery",
+ body,
+ preservedDocx=True,
+ sameHandles=True,
+ journalRows=4,
+ )
+ finally:
+ self.spark.sql(f"DROP TABLE IF EXISTS `{table}`")
+
+ def test_submit_only_path_resumes_the_original_pdf_operation(self):
+ stage = (
+ self.analyzer()
+ .setOperationMode("submit")
+ .setOutputCol("body")
+ .setErrorCol("documentId")
+ )
+ documents = self.pdf_document()
+ with self.output_path() as path:
+ submitted = stage.writeToPath(
+ documents, "documentId", path, self.format
+ ).first()
+ self.assertEqual(submitted.sequence, 0)
+ self.assertTrue(submitted.operationLocation)
+ self.assertEqual(submitted.status, "Running")
+ stage.setOperationMode("analyze")
+ resumed = stage.writeToPath(
+ documents, "documentId", path, self.format
+ ).first()
+ self.assertEqual(resumed.operationLocation, submitted.operationLocation)
+ self.assertEqual(resumed.sequence, 1)
+ body = self.successful_body(resumed)
+ stage.writeToPath(documents, "documentId", path, self.format)
+ self.assertEqual(stage.readPath(self.spark, path, self.format).count(), 1)
+ self.assertEqual(self.spark.read.format(self.format).load(path).count(), 2)
+ self.record(
+ "submit-only-path-recovery", body, sameHandle=True, journalRows=2
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/contentunderstanding/ContentUnderstandingFileSystemSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/contentunderstanding/ContentUnderstandingFileSystemSuite.scala
new file mode 100644
index 0000000000..fca5bc1d17
--- /dev/null
+++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/contentunderstanding/ContentUnderstandingFileSystemSuite.scala
@@ -0,0 +1,46 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in project root for information.
+
+package com.microsoft.azure.synapse.ml.services.form.contentunderstanding
+
+import com.microsoft.azure.synapse.ml.core.test.base.TestBase
+import com.microsoft.azure.synapse.ml.services.contentunderstanding.ContentUnderstanding
+import org.apache.commons.io.FileUtils
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.RawLocalFileSystem
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.types.{StringType, StructType}
+
+import java.net.URI
+import java.nio.file.Files
+import scala.collection.JavaConverters._
+
+class ContentUnderstandingSessionFileSystem extends RawLocalFileSystem {
+ override def getScheme: String = "cu-session"
+ override def getUri: URI = URI.create("cu-session:///")
+
+ override def initialize(name: URI, conf: Configuration): Unit = {
+ require(conf.get("cu.session.marker") == "configured", "The writer must use session Hadoop configuration")
+ super.initialize(name, conf)
+ }
+}
+
+class ContentUnderstandingFileSystemSuite extends TestBase {
+ test("path writer uses the configured session filesystem without changing SparkContext configuration") {
+ val session = spark.newSession()
+ session.conf.set("fs.cu-session.impl", classOf[ContentUnderstandingSessionFileSystem].getName)
+ session.conf.set("fs.cu-session.impl.disable.cache", "true")
+ session.conf.set("cu.session.marker", "configured")
+ assert(Option(spark.sparkContext.hadoopConfiguration.get("fs.cu-session.impl")).isEmpty)
+ val directory = Files.createTempDirectory("cu-session-")
+ try {
+ val path = new URI("cu-session", None.orNull, directory.resolve("journal").toUri.getPath, None.orNull).toString
+ val input = session.createDataFrame(Seq.empty[Row].asJava, new StructType().add("documentId", StringType))
+ val stage = new ContentUnderstanding().setEndpoint("https://example.invalid").setDocumentBytes(Array[Byte](1))
+ assert(stage.writeToPath(input, "documentId", path, "parquet").count() == 0)
+ assert(stage.readPath(session, path, "parquet").count() == 0)
+ } finally {
+ FileUtils.deleteDirectory(directory.toFile)
+ }
+ }
+}
diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/contentunderstanding/ContentUnderstandingFuzzingSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/contentunderstanding/ContentUnderstandingFuzzingSuite.scala
new file mode 100644
index 0000000000..2ce1896572
--- /dev/null
+++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/contentunderstanding/ContentUnderstandingFuzzingSuite.scala
@@ -0,0 +1,51 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in project root for information.
+
+package com.microsoft.azure.synapse.ml.services.form.contentunderstanding
+
+import com.microsoft.azure.synapse.ml.core.test.fuzzing.{TestObject, TransformerFuzzing}
+import com.microsoft.azure.synapse.ml.services.contentunderstanding.ContentUnderstanding
+import org.apache.http.HttpStatus
+import org.apache.spark.ml.util.MLReadable
+import org.apache.spark.sql.Row
+
+class ContentUnderstandingFuzzingSuite extends TransformerFuzzing[ContentUnderstanding] {
+ import spark.implicits._
+
+ private var endpoint = "https://example.invalid"
+
+ override def testObjects(): Seq[TestObject[ContentUnderstanding]] = {
+ val stage = new ContentUnderstanding().setEndpoint(endpoint).setDocumentUrlCol("documentUrl")
+ .setDocumentName("synthetic.pdf").setMimeType("application/pdf").setRange("1")
+ .setOutputCol("analysis").setErrorCol("error").setMaxPollAttempts(1).setPollingDelay(0)
+ val input = Seq("https://example.invalid/synthetic.pdf").toDF("documentUrl")
+ Seq(new TestObject(stage, input))
+ }
+
+ override def reader: MLReadable[_] = ContentUnderstanding
+
+ private def withService(testCode: => Unit): Unit = {
+ val reply = ContentUnderstandingStubReply(HttpStatus.SC_OK, ContentUnderstandingFixtures.Succeeded)
+ ContentUnderstandingStub.withReplies(Seq(reply)) { service =>
+ val original = endpoint
+ endpoint = service.endpoint
+ try {
+ testCode
+ assert(service.requests.nonEmpty)
+ } finally {
+ endpoint = original
+ }
+ }
+ }
+
+ override def testExperiments(): Unit = withService {
+ experimentTestObjects().foreach { testObject =>
+ val result = runExperiment(testObject.stage, testObject.fitDF, testObject.transDF).head()
+ assert(result.getAs[Row]("analysis").getAs[String]("status") == "Succeeded")
+ }
+ }
+
+ override def testSerialization(): Unit = withService {
+ super.testSerialization()
+ }
+}
diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/contentunderstanding/ContentUnderstandingRecoverySuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/contentunderstanding/ContentUnderstandingRecoverySuite.scala
new file mode 100644
index 0000000000..6cbc0085d4
--- /dev/null
+++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/contentunderstanding/ContentUnderstandingRecoverySuite.scala
@@ -0,0 +1,172 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in project root for information.
+
+package com.microsoft.azure.synapse.ml.services.form.contentunderstanding
+
+import com.microsoft.azure.synapse.ml.core.test.base.TestBase
+import com.microsoft.azure.synapse.ml.services.contentunderstanding.{
+ ContentUnderstanding, ContentUnderstandingException}
+import org.apache.commons.io.FileUtils
+import org.apache.http.HttpStatus
+import org.apache.spark.sql.types.{StringType, StructType}
+import org.apache.spark.sql.{DataFrame, Row}
+import spray.json._
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import scala.collection.JavaConverters._
+
+class ContentUnderstandingRecoverySuite extends TestBase {
+ import ContentUnderstandingFixtures.{
+ Accepted => accepted, LocationPath => locationPath, Succeeded => succeeded, TooManyRequests}
+ import ContentUnderstandingStub.withReplies
+
+ private def withJournal(test: String => Unit): Unit = {
+ val directory = Files.createTempDirectory("cu-recovery-")
+ try {
+ test(directory.resolve("journal").toString)
+ } finally {
+ FileUtils.deleteDirectory(directory.toFile)
+ }
+ }
+
+ private def stage(server: ContentUnderstandingStub): ContentUnderstanding =
+ new ContentUnderstanding().setEndpoint(server.endpoint).setOutputCol("result").setErrorCol("error")
+ .setPollingDelay(0).setMaxPollAttempts(3)
+
+ private def input: DataFrame =
+ spark.createDataFrame(Seq(Row("doc")).asJava, new StructType().add("id", StringType)).coalesce(1)
+
+ test("large chunked and compressed responses preserve the size-limit error and saved handle") {
+ val bytes = new Array[Byte](65536)
+ new scala.util.Random(1234).nextBytes(bytes)
+ val body = JsObject(succeeded.parseJson.asJsObject.fields +
+ ("padding" -> JsString(java.util.Base64.getEncoder.encodeToString(bytes)))).compactPrint
+ Seq(false, true).foreach { gzip =>
+ withReplies(Seq(accepted, ContentUnderstandingStubReply(HttpStatus.SC_OK, body,
+ chunked = true, gzip = gzip))) { server =>
+ withJournal { path =>
+ val transformer = stage(server).setDocumentBytes(Array[Byte](1))
+ .setMaxResponseBytes(1024).setMaxPollAttempts(2)
+ val failure = intercept[ContentUnderstandingException] {
+ transformer.writeToPath(input, "id", path, "parquet")
+ }
+ assert(failure.response.error.exists(_.contains("ResponseTooLarge")))
+ assert(server.requests.map(_.method) == Seq("POST", "GET"))
+ assert(transformer.readPath(spark, path, "parquet").head().getAs[String]("status") == "Running")
+ val resumed = transformer.setMaxResponseBytes(body.getBytes(StandardCharsets.UTF_8).length)
+ .writeToPath(input, "id", path, "parquet").head()
+ assert(resumed.getAs[String]("status") == "Succeeded")
+ assert(resumed.getAs[String]("rawResponse") == body)
+ assert(server.requests.map(_.method) == Seq("POST", "GET", "GET"))
+ }
+ }
+ }
+ }
+
+ test("writer polling leaves oversized results resumable after the response cap is raised") {
+ withReplies(Seq(accepted, ContentUnderstandingStubReply(HttpStatus.SC_OK, succeeded))) { server =>
+ withJournal { path =>
+ val transformer = stage(server).setDocumentBytes(Array[Byte](1)).setMaxResponseBytes(128)
+ .setMaxPollAttempts(1)
+ val failure = intercept[ContentUnderstandingException] {
+ transformer.writeToPath(input, "id", path, "parquet")
+ }
+ assert(failure.response.error.exists(_.contains("ResponseTooLarge")))
+ assert(failure.response.operationLocation.contains(server.endpoint + locationPath))
+ val pending = transformer.readPath(spark, path, "parquet").collect().head
+ assert(pending.getAs[String]("status") == "Running")
+ assert(pending.getAs[String]("operationLocation") == server.endpoint + locationPath)
+ val resumed = transformer.setMaxResponseBytes(4096).writeToPath(input, "id", path, "parquet").collect().head
+ assert(resumed.getAs[String]("status") == "Succeeded")
+ assert(server.requests.count(_.method == "POST") == 1)
+ assert(server.requests.count(_.method == "GET") == 2)
+ assert(server.requests.filter(_.method == "GET").map(_.path).distinct.size == 1)
+ }
+ }
+ }
+
+ test("an accepted handle survives an oversized or malformed submission response") {
+ val complete = """{"id":"op-1","status":"Succeeded","result":{"contents":[]}}"""
+ Seq("x" * 129, "not-json").foreach { body =>
+ withReplies(Seq(accepted.copy(body = body),
+ ContentUnderstandingStubReply(HttpStatus.SC_OK, complete))) { server =>
+ withJournal { path =>
+ val transformer = stage(server).setDocumentBytes(Array[Byte](1)).setMaxResponseBytes(128)
+ val result = transformer.writeToPath(input, "id", path, "parquet").head()
+ assert(result.getAs[String]("status") == "Succeeded")
+ assert(result.getAs[String]("operationLocation") == server.endpoint + locationPath)
+ assert(server.requests.map(_.method) == Seq("POST", "GET"))
+ val history = spark.read.parquet(path).orderBy("sequence").select("status").collect()
+ assert(history.map(_.getString(0)).toSeq == Seq("Unknown", "Succeeded"))
+ }
+ }
+ }
+ }
+
+ test("an unknown submission is journaled and is not resubmitted without an operation handle") {
+ val replies = Seq(
+ ContentUnderstandingStubReply(0, "", disconnect = true) -> "TransportError",
+ ContentUnderstandingStubReply(HttpStatus.SC_OK, "not-json") -> "InvalidResponse",
+ accepted.copy(headers = Map.empty) -> "MissingOperationLocation",
+ ContentUnderstandingStubReply(HttpStatus.SC_INTERNAL_SERVER_ERROR, "{}",
+ Map("Operation-Location" -> "https://example.invalid/unsafe")) -> "InvalidOperationLocation")
+ replies.foreach { case (reply, errorCode) =>
+ withReplies(Seq(reply)) { server =>
+ withJournal { path =>
+ val transformer = stage(server).setDocumentBytes(Array[Byte](1))
+ val first = intercept[ContentUnderstandingException] {
+ transformer.writeToPath(input, "id", path, "parquet")
+ }
+ assert(first.response.status == "Unknown")
+ val recorded = transformer.readPath(spark, path, "parquet").head()
+ assert(recorded.getAs[String]("status") == "Unknown")
+ assert(recorded.getAs[String]("error").contains(errorCode))
+ val retry = intercept[IllegalArgumentException] {
+ transformer.writeToPath(input, "id", path, "parquet")
+ }
+ assert(retry.getMessage.contains("unknown"))
+ assert(server.requests.map(_.method) == Seq("POST"))
+ }
+ }
+ }
+ }
+
+ test("exhausted transient polling errors leave the committed handle available for retry") {
+ val throttled = ContentUnderstandingStubReply(TooManyRequests,
+ """{"error":{"code":"TooManyRequests"}}""", Map("Retry-After" -> "0"))
+ withReplies(Seq(accepted, throttled)) { server =>
+ withJournal { path =>
+ val transformer = stage(server).setDocumentBytes(Array[Byte](1)).setMaxPollAttempts(1)
+ val failure = intercept[ContentUnderstandingException] {
+ transformer.writeToPath(input, "id", path, "parquet")
+ }
+ assert(failure.response.httpStatus == TooManyRequests)
+ val pending = transformer.readPath(spark, path, "parquet").head()
+ assert(pending.getAs[String]("status") == "Running")
+ assert(pending.getAs[String]("operationLocation") == server.endpoint + locationPath)
+ assert(server.requests.map(_.method) == Seq("POST", "GET"))
+ }
+ }
+ }
+
+ test("polling configuration and credential errors do not terminalize an accepted operation") {
+ Seq(HttpStatus.SC_BAD_REQUEST, HttpStatus.SC_UNAUTHORIZED, HttpStatus.SC_FORBIDDEN).foreach { code =>
+ val rejected = ContentUnderstandingStubReply(code, """{"error":{"code":"PollingRejected"}}""")
+ withReplies(Seq(accepted, rejected, ContentUnderstandingStubReply(HttpStatus.SC_OK, succeeded))) { server =>
+ withJournal { path =>
+ val transformer = stage(server).setDocumentBytes(Array[Byte](1)).setAADToken("expired-test-token")
+ val failure = intercept[ContentUnderstandingException] {
+ transformer.writeToPath(input, "id", path, "parquet")
+ }
+ assert(failure.response.httpStatus == code)
+ assert(transformer.readPath(spark, path, "parquet").head().getAs[String]("status") == "Running")
+ val resumed = transformer.setAADToken("refreshed-test-token").writeToPath(input, "id", path, "parquet")
+ assert(resumed.head().getAs[String]("status") == "Succeeded")
+ assert(server.requests.count(_.method == "POST") == 1)
+ assert(server.requests.last.headers("authorization") == "Bearer refreshed-test-token")
+ }
+ }
+ }
+ }
+}
diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/contentunderstanding/ContentUnderstandingStub.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/contentunderstanding/ContentUnderstandingStub.scala
new file mode 100644
index 0000000000..1e19d548bc
--- /dev/null
+++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/contentunderstanding/ContentUnderstandingStub.scala
@@ -0,0 +1,134 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in project root for information.
+
+package com.microsoft.azure.synapse.ml.services.form.contentunderstanding
+
+import com.sun.net.httpserver.{HttpExchange, HttpHandler, HttpServer}
+import org.apache.commons.io.IOUtils
+import org.apache.http.HttpStatus
+
+import java.io.ByteArrayOutputStream
+import java.net.InetSocketAddress
+import java.nio.charset.StandardCharsets
+import java.util.concurrent.{CopyOnWriteArrayList, Executors}
+import java.util.concurrent.atomic.AtomicInteger
+import java.util.zip.GZIPOutputStream
+import scala.collection.JavaConverters._
+
+private[contentunderstanding] object ContentUnderstandingFixtures {
+ val AnalyzersPath = "/contentunderstanding/analyzers"
+ val ResultsPath = "/contentunderstanding/analyzerResults/"
+ val DefaultApiVersion = "2025-11-01"
+ val TooManyRequests = 429
+
+ val LocationPath = ResultsPath + "op-1?api-version=" + DefaultApiVersion
+ val Running = """{"id":"op-1","status":"Running","result":{"contents":[]}}"""
+ val Succeeded =
+ """{
+ | "id":"op-1",
+ | "status":"Succeeded",
+ | "result":{"contents":[{"metadata":{"preview":true},"fields":{"A":{"type":"string","confidence":0.8}}}]},
+ | "usage":{"documentPagesBasic":2,"gpt-5.2-input":51},
+ | "warnings":[{"code":"ExampleWarning"}],
+ | "futureProperty":{"nested":[1,true,null]}
+ |}""".stripMargin
+ val Failed =
+ """{"id":"op-1","status":"Failed","result":{"contents":[]},"error":{"code":"ResourceError","innererror":""" +
+ """{"code":"DeploymentNotFound","message":"Missing model deployment."}}}"""
+ val Accepted = ContentUnderstandingStubReply(HttpStatus.SC_ACCEPTED, Running,
+ Map("Operation-Location" -> ("$ROOT" + LocationPath), "Retry-After" -> "0"))
+}
+
+private[contentunderstanding] case class ContentUnderstandingStubReply(status: Int,
+ body: String,
+ headers: Map[String, String] = Map.empty,
+ chunked: Boolean = false,
+ disconnect: Boolean = false,
+ gzip: Boolean = false)
+
+private[contentunderstanding] case class ContentUnderstandingStubRequest(method: String,
+ path: String,
+ query: String,
+ headers: Map[String, String],
+ body: String)
+
+private[contentunderstanding] object ContentUnderstandingStub {
+ def withReplies(replies: Seq[ContentUnderstandingStubReply])(test: ContentUnderstandingStub => Unit): Unit = {
+ require(replies.nonEmpty)
+ val next = new AtomicInteger()
+ val server = new ContentUnderstandingStub(_ => replies(math.min(next.getAndIncrement(), replies.size - 1)))
+ try {
+ test(server)
+ } finally {
+ server.close()
+ }
+ }
+}
+
+private[contentunderstanding] class ContentUnderstandingStub(
+ respond: ContentUnderstandingStubRequest => ContentUnderstandingStubReply) extends AutoCloseable {
+
+ private val server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0)
+ private val executor = Executors.newCachedThreadPool()
+ private val recorded = new CopyOnWriteArrayList[ContentUnderstandingStubRequest]()
+ val endpoint: String = s"http://127.0.0.1:${server.getAddress.getPort}"
+
+ def requests: Seq[ContentUnderstandingStubRequest] = recorded.asScala.toVector
+
+ private def responseBytes(response: ContentUnderstandingStubReply): Array[Byte] = {
+ val bytes = response.body.getBytes(StandardCharsets.UTF_8)
+ if (response.gzip) {
+ val output = new ByteArrayOutputStream()
+ val gzip = new GZIPOutputStream(output)
+ try {
+ gzip.write(bytes)
+ } finally {
+ gzip.close()
+ }
+ output.toByteArray
+ } else {
+ bytes
+ }
+ }
+
+ server.setExecutor(executor)
+ server.createContext("/", new HttpHandler {
+ override def handle(exchange: HttpExchange): Unit = {
+ try {
+ val input = exchange.getRequestBody
+ val body = try {
+ IOUtils.toString(input, StandardCharsets.UTF_8)
+ } finally {
+ input.close()
+ }
+ val request = ContentUnderstandingStubRequest(exchange.getRequestMethod,
+ exchange.getRequestURI.getRawPath, Option(exchange.getRequestURI.getRawQuery).getOrElse(""),
+ exchange.getRequestHeaders.asScala.map { case (name, values) =>
+ name.toLowerCase(java.util.Locale.ROOT) -> values.asScala.mkString(",")
+ }.toMap, body)
+ recorded.add(request)
+ val response = respond(request)
+ if (!response.disconnect) {
+ response.headers.foreach { case (name, value) =>
+ exchange.getResponseHeaders.set(name, value.replace("$ROOT", endpoint))
+ }
+ exchange.getResponseHeaders.set("Content-Type", "application/json; charset=utf-8")
+ if (response.gzip) {
+ exchange.getResponseHeaders.set("Content-Encoding", "gzip")
+ }
+ val bytes = responseBytes(response)
+ exchange.sendResponseHeaders(response.status, if (response.chunked) 0L else bytes.length.toLong)
+ exchange.getResponseBody.write(bytes)
+ }
+ } finally {
+ exchange.close()
+ }
+ }
+ })
+ server.start()
+
+ override def close(): Unit = {
+ server.stop(0)
+ executor.shutdownNow()
+ }
+}
diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/contentunderstanding/ContentUnderstandingSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/contentunderstanding/ContentUnderstandingSuite.scala
new file mode 100644
index 0000000000..d2edf10dff
--- /dev/null
+++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/contentunderstanding/ContentUnderstandingSuite.scala
@@ -0,0 +1,593 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in project root for information.
+
+package com.microsoft.azure.synapse.ml.services.form.contentunderstanding
+
+import com.microsoft.azure.synapse.ml.core.test.base.TestBase
+import com.microsoft.azure.synapse.ml.io.http.ErrorUtils
+import com.microsoft.azure.synapse.ml.services.contentunderstanding.{
+ ContentUnderstanding, ContentUnderstandingException, ContentUnderstandingResponse}
+import org.apache.commons.io.FileUtils
+import org.apache.http.HttpStatus
+import org.apache.spark.ml.param.ParamMap
+import org.apache.spark.sql.types._
+import org.apache.spark.sql.{DataFrame, Row}
+import spray.json.DefaultJsonProtocol._
+import spray.json._
+
+import java.io.File
+import java.util.UUID
+import scala.collection.JavaConverters._
+
+class ContentUnderstandingSuite extends TestBase {
+ import ContentUnderstandingFixtures.{
+ Accepted => accepted, Failed => failed, LocationPath => locationPath, Running => running, Succeeded => succeeded, _}
+ import ContentUnderstandingStub.withReplies
+
+ private def stage(server: ContentUnderstandingStub): ContentUnderstanding =
+ new ContentUnderstanding().setEndpoint(server.endpoint).setOutputCol("result").setErrorCol("error")
+ .setPollingDelay(0).setMaxPollAttempts(3)
+
+ private def dataFrame(rows: Seq[Row], schema: StructType): DataFrame =
+ spark.createDataFrame(rows.asJava, schema).coalesce(1)
+
+ private def input: DataFrame =
+ dataFrame(Seq(Row("doc")), new StructType().add("id", StringType))
+
+ private def response(row: Row): ContentUnderstandingResponse =
+ ContentUnderstandingResponse.makeFromRowConverter(row.getAs[Row]("result"))
+
+ private def resultOf(transformer: ContentUnderstanding, frame: DataFrame): ContentUnderstandingResponse = {
+ val row = transformer.transform(frame).collect().head
+ ContentUnderstandingResponse.makeFromRowConverter(row.getAs[Row](transformer.getOutputCol))
+ }
+
+ private def submit(transformer: ContentUnderstanding): ContentUnderstandingResponse =
+ resultOf(transformer.copy(ParamMap.empty).setOperationMode("submit"), input)
+
+ private def poll(transformer: ContentUnderstanding, location: String): ContentUnderstandingResponse =
+ resultOf(transformer.copy(ParamMap.empty).setOperationMode("poll").setOperationLocation(location), input)
+
+ private def exceptionContains(error: Throwable, text: String): Boolean =
+ Option(error).exists(value => Option(value.getMessage).exists(_.contains(text)) ||
+ exceptionContains(value.getCause, text))
+
+ private class PythonSourceStage extends ContentUnderstanding("python-source") {
+ def pythonSource: String = pythonClass()
+ }
+
+ private class HeaderGuard extends ContentUnderstanding("header-guard") {
+ override protected def getCustomAuthHeader(row: Row): Option[String] =
+ throw new IllegalStateException("Headers must not be prepared for an unsafe operation URL.")
+ }
+
+ test("public transform is lazy and only Succeeded completes a response with empty running contents") {
+ withReplies(Seq(accepted, ContentUnderstandingStubReply(HttpStatus.SC_OK, running),
+ ContentUnderstandingStubReply(HttpStatus.SC_OK, succeeded))) { server =>
+ val transformer = stage(server).setDocumentBytes(Array[Byte](1, 2, 3))
+ val transformed = transformer.transform(input)
+ assert(server.requests.isEmpty)
+ val row = transformed.collect().head
+ val result = response(row)
+ assert(server.requests.map(_.method) == Seq("POST", "GET", "GET"))
+ assert(result.status == "Succeeded")
+ assert(result.rawResponse == succeeded)
+ assert(result.operationLocation.contains(server.endpoint + locationPath))
+ assert(row.getAs[Row]("error") == None.orNull)
+ assert(result.error.isEmpty)
+ assert(result.rawResponse.parseJson.asJsObject.fields("usage").asJsObject.fields("gpt-5.2-input") == JsNumber(51))
+ assert(transformer.transformSchema(input.schema) == transformed.schema)
+ }
+ }
+
+ test("column inputs preserve binary data, options, preview query parameters, and explicit key authentication") {
+ withReplies(Seq(accepted)) { server =>
+ val bytes = Array[Byte](0, 1, -1, -128)
+ val schema = new StructType().add("source.bytes", BinaryType).add("analyzer", StringType)
+ .add("name", StringType).add("mime", StringType).add("pages", StringType)
+ .add("models", MapType(StringType, StringType)).add("key", StringType)
+ .add("version", StringType).add("encoding", StringType).add("processing", StringType)
+ val models = Map("prebuilt-analyzer-embedding" -> "text-embedding-3-large",
+ "prebuilt-analyzer-completion" -> "gpt-5.2")
+ val frame = dataFrame(Seq(Row(bytes, "prebuilt-invoice", "invoice.pdf", "application/pdf", "319-320",
+ models, "test-key", "2026-06-01-preview", "utf16", "geography")), schema)
+ val transformer = stage(server).setOperationMode("submit")
+ .setDocumentBytesCol("source.bytes").setAnalyzerIdCol("analyzer").setDocumentNameCol("name")
+ .setMimeTypeCol("mime").setRangeCol("pages").setModelDeploymentsCol("models")
+ .setSubscriptionKeyCol("key").setApiVersionCol("version").setStringEncodingCol("encoding")
+ .setProcessingLocationCol("processing")
+ assert(response(transformer.transform(frame).collect().head).status == "Running")
+ val sent = server.requests.head
+ assert(sent.path.endsWith("/prebuilt-invoice:analyze"))
+ assert(sent.query == "api-version=2026-06-01-preview&processingLocation=geography&stringEncoding=utf16")
+ assert(sent.headers("ocp-apim-subscription-key") == "test-key")
+ assert(!sent.headers.contains("authorization"))
+ val json = sent.body.parseJson.asJsObject
+ val item = json.fields("inputs").asInstanceOf[JsArray].elements.head.asJsObject
+ assert(item.fields("data") == JsString(java.util.Base64.getEncoder.encodeToString(bytes)))
+ assert(!item.fields.contains("dataBase64"))
+ assert(!item.fields.contains("url"))
+ assert(item.fields("name") == JsString("invoice.pdf"))
+ assert(item.fields("mimeType") == JsString("application/pdf"))
+ assert(item.fields("range") == JsString("319-320"))
+ assert(json.fields("modelDeployments").convertTo[Map[String, String]] == models)
+ }
+ }
+
+ test("URL requests use singleton inputs and AAD bearer authentication") {
+ withReplies(Seq(accepted)) { server =>
+ val transformer = stage(server).setDocumentUrl("https://example.invalid/document.pdf?signature=test")
+ .setAADToken("test-token").setDocumentName("document.pdf")
+ val submitted = submit(transformer)
+ assert(submitted.status == "Running")
+ assert(server.requests.size == 1)
+ val sent = server.requests.head
+ assert(sent.headers("authorization") == "Bearer test-token")
+ assert(!sent.headers.contains("ocp-apim-subscription-key"))
+ val items = sent.body.parseJson.asJsObject.fields("inputs").asInstanceOf[JsArray].elements
+ assert(items.size == 1)
+ assert(items.head.asJsObject.fields("url") == JsString(transformer.getDocumentUrl))
+ assert(!items.head.asJsObject.fields.contains("data"))
+ }
+ }
+
+ test("public output names remain literal and cannot collide with internal HTTP columns") {
+ withReplies(Seq(accepted)) { server =>
+ val transformer = stage(server).setOperationMode("submit").setDocumentBytes(Array[Byte](1))
+ .setOutputCol("contentUnderstandingInput").setErrorCol("service.error")
+ val output = transformer.transform(input)
+ val row = output.collect().head
+ val result = ContentUnderstandingResponse.makeFromRowConverter(row.getAs[Row]("contentUnderstandingInput"))
+ assert(result.status == "Running")
+ assert(output.columns.toSet == Set("id", "contentUnderstandingInput", "service.error"))
+ assert(output.schema == transformer.transformSchema(input.schema))
+ }
+ }
+
+ test("transform rejects input columns matching public output names before service calls") {
+ withReplies(Seq(accepted)) { server =>
+ val base = stage(server).setDocumentBytes(Array[Byte](1))
+ Seq(base.copy(ParamMap.empty).setOutputCol("id"), base.copy(ParamMap.empty).setErrorCol("id"))
+ .foreach { transformer =>
+ intercept[IllegalArgumentException](transformer.transformSchema(input.schema))
+ intercept[IllegalArgumentException](transformer.transform(input))
+ }
+ assert(server.requests.isEmpty)
+ }
+ }
+
+ test("HTTP 200 Failed operations retain the nested service error and fill errorCol") {
+ withReplies(Seq(accepted, ContentUnderstandingStubReply(HttpStatus.SC_OK, failed))) { server =>
+ val row = stage(server).setDocumentBytes(Array[Byte](1)).transform(input).collect().head
+ val result = response(row)
+ assert(result.status == "Failed")
+ assert(result.httpStatus == HttpStatus.SC_OK)
+ assert(result.rawResponse == failed)
+ assert(result.error.contains(failed.parseJson.asJsObject.fields("error").compactPrint))
+ val error = row.getAs[Row]("error")
+ assert(error.getAs[String]("response") == result.error.get)
+ assert(error.getAs[Row]("status").getAs[Int]("statusCode") == HttpStatus.SC_OK)
+ }
+ }
+
+ test("initial HTTP failures remain useful output and are never automatically resubmitted") {
+ val body = """{"error":{"code":"InvalidPagesOutOfRange","message":"Out of range."}}"""
+ val statuses = Seq(HttpStatus.SC_BAD_REQUEST -> "Failed",
+ HttpStatus.SC_INTERNAL_SERVER_ERROR -> "Unknown", TooManyRequests -> "Rejected")
+ statuses.foreach { case (code, status) =>
+ withReplies(Seq(ContentUnderstandingStubReply(code, body, Map("Retry-After" -> "0")))) { server =>
+ val row = stage(server).setDocumentBytes(Array[Byte](1)).transform(input).collect().head
+ val result = response(row)
+ assert(result.status == status)
+ assert(result.httpStatus == code)
+ assert(result.rawResponse == body)
+ assert(result.error.exists(_.contains("InvalidPagesOutOfRange")))
+ assert(Option(row.getAs[Row]("error")).isDefined)
+ assert(server.requests.size == 1)
+ }
+ }
+ }
+
+ test("malformed JSON and missing operation status leave submission outcomes unknown") {
+ Seq("not-json", """{"result":{"contents":[]}}""", """{"status":"unexpected"}""").foreach { body =>
+ withReplies(Seq(ContentUnderstandingStubReply(HttpStatus.SC_OK, body))) { server =>
+ val result = submit(stage(server).setDocumentBytes(Array[Byte](1)))
+ assert(result.status == "Unknown")
+ assert(result.rawResponse == body)
+ assert(result.error.exists(_.contains("InvalidResponse")))
+ }
+ }
+ }
+
+ test("missing or unsafe service operation locations do not trigger polling") {
+ val unsafe = Seq(
+ Map.empty[String, String],
+ Map("Operation-Location" -> ("https://example.invalid" + locationPath)),
+ Map("Operation-Location" -> ("$ROOT" + locationPath + "&redirect=other")),
+ Map("Operation-Location" -> ("$ROOT" + ResultsPath + "../analyzers?api-version=" + DefaultApiVersion)))
+ unsafe.foreach { headers =>
+ withReplies(Seq(ContentUnderstandingStubReply(HttpStatus.SC_ACCEPTED, running, headers))) { server =>
+ val result = response(stage(server).setDocumentBytes(Array[Byte](1)).transform(input).collect().head)
+ assert(result.status == "Unknown")
+ assert(result.operationLocation.isEmpty)
+ assert(result.error.exists(_.contains("OperationLocation")))
+ assert(result.rawResponse == running)
+ assert(server.requests.size == 1)
+ }
+ }
+ }
+
+ test("poll URLs are validated before credential resolution") {
+ val transformer = new HeaderGuard().setEndpoint("https://example.invalid").setSubscriptionKey("test-key")
+ .setOperationMode("poll")
+ val bad = Seq(
+ "http://example.invalid" + locationPath,
+ "https://other.invalid" + locationPath,
+ "https://example.invalid:444" + locationPath,
+ "https://user:password@example.invalid" + locationPath,
+ "https://example.invalid" + locationPath + "#fragment",
+ locationPath,
+ "https://example.invalid" + ResultsPath + "%2e%2e?api-version=" + DefaultApiVersion,
+ "https://example.invalid" + ResultsPath + "op%2F1?api-version=" + DefaultApiVersion,
+ "https://example.invalid" + ResultsPath + "op-1/child?api-version=" + DefaultApiVersion,
+ "https://example.invalid" + ResultsPath + "op-1?other=value",
+ "https://example.invalid" + locationPath + "&api-version=" + DefaultApiVersion)
+ bad.foreach { location =>
+ intercept[IllegalArgumentException] {
+ transformer.setOperationLocation(location).transformSchema(StructType(Nil))
+ }
+ }
+ val valid = new ContentUnderstanding().setEndpoint("https://example.invalid").setOperationMode("poll")
+ .setOperationLocation("https://example.invalid:443" + locationPath)
+ assert(valid.transformSchema(StructType(Nil)).fieldNames.contains(valid.getOutputCol))
+ }
+
+ test("HTTP redirects from submission and polling are never followed") {
+ val redirect = ContentUnderstandingStubReply(HttpStatus.SC_MOVED_TEMPORARILY, """{"redirect":true}""",
+ Map("Location" -> ("$ROOT" + locationPath)))
+ Seq(Seq(redirect), Seq(accepted, redirect)).foreach { replies =>
+ withReplies(replies) { server =>
+ val result = response(stage(server).setDocumentBytes(Array[Byte](1)).transform(input).collect().head)
+ assert(result.status == "Failed")
+ assert(result.httpStatus == HttpStatus.SC_MOVED_TEMPORARILY)
+ assert(result.error.isDefined)
+ assert(server.requests.size == replies.size)
+ }
+ }
+ }
+
+ test("poll budgets retain the last Running response and allow resumption without a new POST") {
+ withReplies(Seq(accepted, ContentUnderstandingStubReply(HttpStatus.SC_OK, running),
+ ContentUnderstandingStubReply(HttpStatus.SC_OK, running),
+ ContentUnderstandingStubReply(HttpStatus.SC_OK, succeeded))) { server =>
+ val transformer = stage(server).setDocumentBytes(Array[Byte](1)).setMaxPollAttempts(2)
+ val pending = response(transformer.transform(input).collect().head)
+ assert(pending.status == "Running")
+ assert(pending.rawResponse == running)
+ assert(pending.error.isEmpty)
+ assert(pending.operationLocation.contains(server.endpoint + locationPath))
+ val resumed = poll(transformer, pending.operationLocation.get)
+ assert(resumed.status == "Succeeded")
+ assert(server.requests.count(_.method == "POST") == 1)
+ assert(server.requests.count(_.method == "GET") == 3)
+ }
+ }
+
+ test("GET transient retries including 429 consume the poll budget") {
+ val throttled = ContentUnderstandingStubReply(TooManyRequests,
+ """{"error":{"code":"TooManyRequests"}}""", Map("Retry-After" -> "0"))
+ withReplies(Seq(accepted, throttled)) { server =>
+ val result = response(stage(server).setDocumentBytes(Array[Byte](1)).setMaxPollAttempts(2)
+ .transform(input).collect().head)
+ assert(server.requests.size == 3)
+ assert(result.status == "Running")
+ assert(result.rawResponse == running)
+ assert(result.httpStatus == TooManyRequests)
+ assert(result.error.exists(_.contains("TooManyRequests")))
+ assert(result.operationLocation.isDefined)
+ }
+ withReplies(Seq(throttled, ContentUnderstandingStubReply(HttpStatus.SC_SERVICE_UNAVAILABLE,
+ """{"error":{"code":"Unavailable"}}"""), ContentUnderstandingStubReply(HttpStatus.SC_OK, succeeded))) { server =>
+ val result = poll(stage(server), server.endpoint + locationPath)
+ assert(result.status == "Succeeded")
+ assert(server.requests.size == 3)
+ assert(server.requests.forall(_.method == "GET"))
+ }
+ }
+
+ test("Retry-After seconds, past HTTP dates, and malformed values allow bounded public polling") {
+ Seq("0", "Thu, 01 Jan 1970 00:00:00 GMT", "not a date", "-1").foreach { retryAfter =>
+ val submission = accepted.copy(headers = accepted.headers + ("Retry-After" -> retryAfter))
+ withReplies(Seq(submission, ContentUnderstandingStubReply(HttpStatus.SC_OK, succeeded))) { server =>
+ assert(resultOf(stage(server).setDocumentBytes(Array[Byte](1)), input).status == "Succeeded")
+ assert(server.requests.map(_.method) == Seq("POST", "GET"))
+ }
+ }
+ }
+
+ test("response bounds apply to declared and chunked bodies before JSON parsing") {
+ val limit = 64
+ Seq(false, true).foreach { chunked =>
+ withReplies(Seq(ContentUnderstandingStubReply(HttpStatus.SC_OK, "x" * (limit + 1),
+ chunked = chunked))) { server =>
+ val result = submit(stage(server).setDocumentBytes(Array[Byte](1)).setMaxResponseBytes(limit))
+ assert(result.status == "Unknown")
+ assert(result.httpStatus == HttpStatus.SC_OK)
+ assert(result.error.exists(_.contains("ResponseTooLarge")))
+ assert(result.rawResponse.isEmpty)
+ assert(server.requests.size == 1)
+ }
+ }
+ }
+
+ test("response stream cleanup cannot replace the size-limit failure") {
+ val input = new java.io.ByteArrayInputStream(Array[Byte](1, 2, 3)) {
+ override def close(): Unit = throw new java.io.IOException("Synthetic stream close failure")
+ }
+ val protocolClass = Class.forName(
+ "com.microsoft.azure.synapse.ml.services.contentunderstanding.ContentUnderstandingProtocol$")
+ val protocol = protocolClass.getField("MODULE$").get(None.orNull)
+ val read = protocolClass.getDeclaredMethod("readBounded", classOf[java.io.InputStream],
+ java.lang.Integer.TYPE, classOf[org.apache.http.client.methods.HttpRequestBase])
+ read.setAccessible(true)
+ val request = new org.apache.http.client.methods.HttpGet("http://127.0.0.1/")
+ try {
+ val failure = intercept[java.lang.reflect.InvocationTargetException] {
+ read.invoke(protocol, input, Int.box(2), request)
+ }
+ assert(failure.getCause.getClass.getSimpleName == "ResponseTooLarge")
+ assert(request.isAborted)
+ } finally {
+ intercept[java.io.IOException](input.close())
+ }
+ }
+
+ test("ambiguous POST transport failures are marked Unknown and are not retried") {
+ withReplies(Seq(ContentUnderstandingStubReply(0, "", disconnect = true))) { server =>
+ val result = submit(stage(server).setDocumentBytes(Array[Byte](1)))
+ assert(result.status == "Unknown")
+ assert(result.httpStatus == 0)
+ assert(result.error.exists(_.contains("TransportError")))
+ assert(server.requests.size == 1)
+ }
+ }
+
+ test("exhausted polling transport retries retain the last operation and expose the I/O error") {
+ withReplies(Seq(ContentUnderstandingStubReply(HttpStatus.SC_OK, running),
+ ContentUnderstandingStubReply(0, "", disconnect = true))) { server =>
+ val result = poll(stage(server).setMaxPollAttempts(2), server.endpoint + locationPath)
+ assert(result.status == "Running")
+ assert(result.rawResponse == running)
+ assert(result.operationLocation.contains(server.endpoint + locationPath))
+ assert(result.error.exists(_.contains("TransportError")))
+ assert(server.requests.size == 2)
+ }
+ }
+
+ test("interruption stops requests without clearing the thread interruption") {
+ withReplies(Seq(accepted)) { server =>
+ val transformer = stage(server).setAnalyzerId("custom")
+ Thread.currentThread().interrupt()
+ try {
+ intercept[InterruptedException] {
+ transformer.createAnalyzer("{}", allowReplace = false)
+ }
+ assert(Thread.currentThread().isInterrupted)
+ } finally {
+ Thread.interrupted()
+ }
+ assert(server.requests.isEmpty)
+ }
+ }
+
+ test("poll mode accepts operationLocation columns without document input") {
+ withReplies(Seq(ContentUnderstandingStubReply(HttpStatus.SC_OK, succeeded))) { server =>
+ val frame = dataFrame(Seq(Row(server.endpoint + locationPath, "test-token")),
+ new StructType().add("handle", StringType).add("token", StringType))
+ val transformer = stage(server).setOperationMode("poll").setOperationLocationCol("handle")
+ .setAADTokenCol("token")
+ val result = response(transformer.transform(frame).collect().head)
+ assert(result.status == "Succeeded")
+ assert(server.requests.map(_.method) == Seq("GET"))
+ assert(server.requests.head.headers("authorization") == "Bearer test-token")
+ assert(server.requests.head.body.isEmpty)
+ }
+ }
+
+ test("public submit and poll modes reuse configured document and API-version columns without resubmission") {
+ val version = "2026-06-01-preview"
+ val location = ResultsPath + "op-1?api-version=" + version
+ val submission = accepted.copy(headers = Map("Operation-Location" -> ("$ROOT" + location), "Retry-After" -> "0"))
+ withReplies(Seq(submission, ContentUnderstandingStubReply(HttpStatus.SC_OK, succeeded))) { server =>
+ val schema = new StructType().add("id", StringType).add("bytes", BinaryType).add("pages", StringType)
+ .add("version", StringType)
+ val frame = dataFrame(Seq(Row("document-1", Array[Byte](1), "1-2", version)), schema)
+ val transformer = stage(server).setOperationMode("submit").setDocumentBytesCol("bytes").setRangeCol("pages")
+ .setApiVersionCol("version")
+ val submitted = resultOf(transformer, frame)
+ assert(submitted.status == "Running")
+ val resumed = transformer.setOperationMode("poll").setOperationLocation(submitted.operationLocation.get)
+ assert(resultOf(resumed, input).status == "Succeeded")
+ assert(server.requests.size == 2)
+ assert(server.requests.last.query == "api-version=" + version)
+ assert(server.requests.head.body.parseJson.asJsObject.fields("inputs")
+ .asInstanceOf[JsArray].elements.head.asJsObject.fields("range") == JsString("1-2"))
+ }
+ }
+
+ test("canonical analyze URL and body do not depend on authentication, execution controls, or map ordering") {
+ withReplies(Seq(accepted)) { server =>
+ val first = stage(server).setDocumentBytes(Array[Byte](1)).setSubscriptionKey("first-key")
+ .setModelDeployments(Map("z" -> "last", "a" -> "first"))
+ val second = first.copy(ParamMap.empty).setSubscriptionKey("other-key").setAADToken("other-token")
+ .setCustomHeaders(Map("x-test-header" -> "value")).setConcurrency(3)
+ .setTimeout(3).setPollingDelay(0).setMaxPollAttempts(1).setMaxResponseBytes(1024)
+ .setOperationMode("submit").setOutputCol("other-output").setErrorCol("other-error")
+ .setModelDeployments(scala.collection.immutable.ListMap("a" -> "first", "z" -> "last"))
+ assert(submit(first).status == "Running")
+ assert(submit(second).status == "Running")
+ val original = server.requests.head
+ val equivalent = server.requests(1)
+ assert(original.path == equivalent.path)
+ assert(original.query == equivalent.query)
+ assert(original.body == equivalent.body)
+ assert(original.headers("ocp-apim-subscription-key") != equivalent.headers("ocp-apim-subscription-key"))
+ submit(second.setRange("1-2"))
+ assert(server.requests.last.body != original.body)
+ }
+ }
+
+ test("schema validation rejects missing and incorrectly typed inputs and invalid scalar configuration") {
+ val schema = new StructType().add("text", StringType).add("number", IntegerType)
+ .add("bytes", BinaryType).add("models", MapType(StringType, IntegerType))
+ val base = new ContentUnderstanding().setEndpoint("https://example.invalid").setOutputCol("result")
+ .setErrorCol("error")
+ val invalid = Seq(
+ base.copy(ParamMap.empty),
+ base.copy(ParamMap.empty).setDocumentBytes(Array[Byte](1)).setDocumentUrl("https://example.invalid/doc"),
+ base.copy(ParamMap.empty).setDocumentBytesCol("text"),
+ base.copy(ParamMap.empty).setDocumentUrlCol("number"),
+ base.copy(ParamMap.empty).setDocumentBytesCol("missing"),
+ base.copy(ParamMap.empty).setDocumentBytesCol("bytes").setModelDeploymentsCol("models"),
+ base.copy(ParamMap.empty).setDocumentBytes(Array.emptyByteArray),
+ base.copy(ParamMap.empty).setDocumentUrl(""),
+ base.copy(ParamMap.empty).setDocumentUrl("file:///document.pdf"),
+ base.copy(ParamMap.empty).setDocumentBytes(Array[Byte](1)).setAnalyzerId("../other"),
+ base.copy(ParamMap.empty).setDocumentBytes(Array[Byte](1)).setApiVersion("not-a-version"),
+ base.copy(ParamMap.empty).setDocumentBytes(Array[Byte](1)).setStringEncoding("bad"),
+ base.copy(ParamMap.empty).setDocumentBytes(Array[Byte](1)).setProcessingLocation("bad"),
+ base.copy(ParamMap.empty).setDocumentBytes(Array[Byte](1)).setConcurrency(0),
+ base.copy(ParamMap.empty).setDocumentBytes(Array[Byte](1)).setTimeout(Double.PositiveInfinity),
+ base.copy(ParamMap.empty).setOperationMode("poll"))
+ invalid.foreach(transformer => intercept[IllegalArgumentException](transformer.transformSchema(schema)))
+ intercept[IllegalArgumentException](base.setPollingDelay(-1))
+ intercept[IllegalArgumentException](base.setMaxPollAttempts(0))
+ intercept[IllegalArgumentException](base.setMaxResponseBytes(0))
+ intercept[IllegalArgumentException](base.setOperationMode("invalid"))
+ }
+
+ test("null configured row inputs fail explicitly rather than being silently skipped") {
+ val schema = new StructType().add("document", BinaryType)
+ val frame = dataFrame(Seq(Row(None.orNull)), schema)
+ val transformer = new ContentUnderstanding().setEndpoint("https://example.invalid").setDocumentBytesCol("document")
+ transformer.transformSchema(schema)
+ val error = intercept[Exception](transformer.transform(frame).collect())
+ assert(exceptionContains(error, "selected document input cannot be null"))
+ val scalar = transformer.copy(ParamMap.empty).setDocumentBytes(None.orNull)
+ intercept[IllegalArgumentException](scalar.transformSchema(schema))
+ }
+
+ test("endpoint joining requires explicit configuration and rejects insecure or ambiguous authorities") {
+ val unset = new ContentUnderstanding().setDocumentBytes(Array[Byte](1))
+ .setDefaultInternalEndpoint("https://fabric.invalid")
+ intercept[IllegalArgumentException](unset.transformSchema(StructType(Nil)))
+ val base = new ContentUnderstanding().setEndpoint("https://example.invalid")
+ assert(base.getUrl == "https://example.invalid" + AnalyzersPath)
+ assert(base.setEndpoint("https://example.invalid/").getUrl == "https://example.invalid" + AnalyzersPath)
+ assert(base.setEndpoint("https://example.invalid" + AnalyzersPath + "/").getUrl ==
+ "https://example.invalid" + AnalyzersPath)
+ Seq("http://example.invalid", "http://localhost", "https://user:password@example.invalid",
+ "https://example.invalid?api-version=other", "https://example.invalid/prefix",
+ "https://example.invalid#fragment").foreach(value =>
+ intercept[IllegalArgumentException](base.setEndpoint(value)))
+ assert(base.setEndpoint("http://127.0.0.1:1234").getUrl.startsWith("http://127.0.0.1:1234/"))
+ }
+
+ test("copy and persistence retain service parameters, binary scalars, and response schemas") {
+ val original = new ContentUnderstanding("saved-cu").setEndpoint("https://example.invalid")
+ .setDocumentBytes(Array[Byte](0, -1, -128, 127)).setAnalyzerId("custom.invoice")
+ .setApiVersion("2026-06-01-preview").setModelDeployments(Map("prebuilt-analyzer-completion" -> "gpt-5.2"))
+ .setOperationMode("submit").setPollingDelay(0).setOutputCol("result").setErrorCol("error")
+ val copied = original.copy(ParamMap(original.outputCol -> "copied"))
+ assert(copied.uid == original.uid)
+ assert(copied.getOutputCol == "copied")
+ assert(copied.getDocumentBytes.sameElements(original.getDocumentBytes))
+ val destination = new File("cu-stage-test-" + UUID.randomUUID().toString)
+ try {
+ original.write.save(destination.getAbsolutePath)
+ val restored = ContentUnderstanding.load(destination.getAbsolutePath)
+ assert(restored.uid == original.uid)
+ assert(restored.getUrl == original.getUrl)
+ assert(restored.isSet(restored.url))
+ assert(restored.getDocumentBytes.sameElements(original.getDocumentBytes))
+ assert(restored.getModelDeployments == original.getModelDeployments)
+ assert(restored.getApiVersion == original.getApiVersion)
+ assert(restored.getAnalyzerId == original.getAnalyzerId)
+ assert(restored.getOperationMode == original.getOperationMode)
+ val schema = restored.transformSchema(new StructType().add("id", StringType))
+ assert(schema("result").dataType ==
+ StructType(ContentUnderstandingResponse.schema.fields.map(_.copy(nullable = true))))
+ assert(schema("error").dataType == ErrorUtils.ErrorSchema)
+ } finally {
+ FileUtils.deleteDirectory(destination)
+ }
+ }
+
+ test("generated Python source supports unsigned binary scalars and composes provisioning and writer methods") {
+ val transformer = new PythonSourceStage
+ val source = transformer.pythonSource
+ assert(source.contains("self._java_obj.setDocumentBytes(bytearray(value))"))
+ assert(source.contains("def setDocumentBytesCol(self, value)"))
+ assert(source.contains("def setParams(self, **kwargs)"))
+ assert(source.contains("def _transfer_params_from_java(self)"))
+ assert(source.contains("self._paramMap.pop(param, None)"))
+ assert(source.contains("json.dumps(definition) if isinstance(definition, dict)"))
+ assert(source.contains("def writeToTable("))
+ assert(source.contains("def writeToPath("))
+ assert(source.contains("def getAnalyzer("))
+ assert(transformer.documentBytes.pyValue(Left(Array[Byte](-1, -128))) == "bytearray([255, 128])")
+ assert(transformer.documentBytes.jsonDecode(transformer.documentBytes.jsonEncode(Left(Array[Byte](-1))))
+ .left.get.sameElements(Array[Byte](-1)))
+ }
+
+ test("explicit provisioning awaits only management operations and returns the raw analyzer definition") {
+ val analyzer = """{"analyzerId":"custom","status":"ready","config":{"returnDetails":true},"future":{"value":1}}"""
+ val operation = AnalyzersPath + "/custom/operations/create-1?api-version=" + DefaultApiVersion
+ val creating = ContentUnderstandingStubReply(HttpStatus.SC_CREATED,
+ """{"analyzerId":"custom","status":"creating"}""",
+ Map("Operation-Location" -> ("$ROOT" + operation), "Retry-After" -> "0"))
+ withReplies(Seq(creating, ContentUnderstandingStubReply(HttpStatus.SC_OK, running),
+ ContentUnderstandingStubReply(HttpStatus.SC_OK, """{"id":"create-1","status":"Succeeded","result":{}}"""),
+ ContentUnderstandingStubReply(HttpStatus.SC_OK, analyzer))) { server =>
+ val transformer = stage(server).setAnalyzerId("custom").setSubscriptionKey("test-key")
+ val definition = """{"baseAnalyzerId":"prebuilt-document","config":{"returnDetails":true}}"""
+ assert(transformer.createAnalyzer(definition, allowReplace = false) == analyzer)
+ assert(server.requests.map(_.method) == Seq("PUT", "GET", "GET", "GET"))
+ assert(server.requests.head.query == "allowReplace=false&api-version=" + DefaultApiVersion)
+ assert(server.requests.head.body == definition)
+ assert(server.requests.slice(1, 3).forall(_.path == operation.takeWhile(_ != '?')))
+ assert(server.requests.last.path == AnalyzersPath + "/custom")
+ assert(server.requests.forall(_.headers("ocp-apim-subscription-key") == "test-key"))
+ assert(!server.requests.exists(_.path.contains("/defaults")))
+ }
+ }
+
+ test("provisioning preserves DefaultsNotSet and never changes shared resource defaults") {
+ val body = """{"error":{"code":"InvalidRequest","innererror":""" +
+ """{"code":"DefaultsNotSet","message":"Set defaults."}}}"""
+ withReplies(Seq(ContentUnderstandingStubReply(HttpStatus.SC_BAD_REQUEST, body))) { server =>
+ val error = intercept[ContentUnderstandingException] {
+ stage(server).setAnalyzerId("custom").createAnalyzer("""{"baseAnalyzerId":"prebuilt-document"}""",
+ allowReplace = false)
+ }
+ assert(error.response.rawResponse == body)
+ assert(error.response.error.exists(_.contains("DefaultsNotSet")))
+ assert(error.getMessage.contains("DefaultsNotSet"))
+ assert(server.requests.map(_.method) == Seq("PUT"))
+ }
+ }
+
+ test("provisioning rejects analysis-result locations and column-based management configuration") {
+ withReplies(Seq(ContentUnderstandingStubReply(HttpStatus.SC_CREATED,
+ """{"analyzerId":"custom","status":"creating"}""",
+ Map("Operation-Location" -> ("$ROOT" + locationPath))))) { server =>
+ val transformer = stage(server).setAnalyzerId("custom")
+ val error = intercept[ContentUnderstandingException] {
+ transformer.createAnalyzer("{}", allowReplace = true)
+ }
+ assert(error.response.error.exists(_.contains("InvalidOperationLocation")))
+ assert(server.requests.size == 1)
+ intercept[IllegalArgumentException](transformer.setAnalyzerIdCol("analyzer").getAnalyzer())
+ }
+ }
+}
diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/contentunderstanding/ContentUnderstandingWriterSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/contentunderstanding/ContentUnderstandingWriterSuite.scala
new file mode 100644
index 0000000000..d7e82e6056
--- /dev/null
+++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/contentunderstanding/ContentUnderstandingWriterSuite.scala
@@ -0,0 +1,396 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in project root for information.
+
+package com.microsoft.azure.synapse.ml.services.form.contentunderstanding
+
+import com.microsoft.azure.synapse.ml.core.test.base.TestBase
+import com.microsoft.azure.synapse.ml.services.contentunderstanding.{
+ ContentUnderstanding, ContentUnderstandingException, ContentUnderstandingWriter}
+import com.sun.net.httpserver.{HttpExchange, HttpHandler, HttpServer}
+import org.apache.commons.io.{FileUtils, IOUtils}
+import org.apache.spark.sql.DataFrame
+import org.apache.spark.sql.functions.col
+import spray.json._
+
+import java.net.InetSocketAddress
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.util.UUID
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.atomic.{AtomicInteger, AtomicReference}
+
+class ContentUnderstandingWriterSuite extends TestBase {
+ import spark.implicits._
+
+ private class Service extends AutoCloseable {
+ private val server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0)
+ private val operations = new ConcurrentHashMap[String, String]()
+ private val submissions = new ConcurrentHashMap[String, AtomicInteger]()
+ val malformed = new AtomicReference[String]("")
+ val pending = new AtomicReference[String]("")
+ val failed = new AtomicReference[String]("")
+
+ def endpoint: String = s"http://127.0.0.1:${server.getAddress.getPort}"
+ def submitted(name: String): Int = Option(submissions.get(name)).map(_.get()).getOrElse(0)
+ def totalSubmissions: Int = {
+ import scala.collection.JavaConverters._
+ submissions.values().asScala.map(_.get()).sum
+ }
+
+ private def send(exchange: HttpExchange, code: Int, body: String): Unit = {
+ val bytes = body.getBytes(StandardCharsets.UTF_8)
+ exchange.getResponseHeaders.add("Content-Type", "application/json")
+ exchange.sendResponseHeaders(code, bytes.length)
+ exchange.getResponseBody.write(bytes)
+ }
+
+ server.createContext("/contentunderstanding", new HttpHandler {
+ override def handle(exchange: HttpExchange): Unit = {
+ try {
+ if (exchange.getRequestMethod == "POST") {
+ val json = IOUtils.toString(exchange.getRequestBody, StandardCharsets.UTF_8).parseJson.asJsObject
+ val input = json.fields("inputs").asInstanceOf[JsArray].elements.head.asJsObject
+ val name = input.fields("name").asInstanceOf[JsString].value
+ submissions.computeIfAbsent(name, _ => new AtomicInteger()).incrementAndGet()
+ val id = UUID.nameUUIDFromBytes(name.getBytes(StandardCharsets.UTF_8)).toString
+ operations.put(id, name)
+ val location = s"$endpoint/contentunderstanding/analyzerResults/$id?api-version=2025-11-01"
+ exchange.getResponseHeaders.add("Operation-Location", location)
+ send(exchange, 202, s"""{"id":"$id","status":"Running","result":{"contents":[]}}""")
+ } else {
+ val id = exchange.getRequestURI.getPath.split("/").last
+ val name = operations.get(id)
+ if (name == malformed.get()) {
+ send(exchange, 200, "{")
+ } else if (name == pending.get()) {
+ send(exchange, 200, s"""{"id":"$id","status":"Running","result":{"contents":[]}}""")
+ } else if (name == failed.get()) {
+ send(exchange, 200,
+ s"""{"id":"$id","status":"Failed","error":{"code":"InvalidRequest"},"result":{"contents":[]}}""")
+ } else {
+ send(exchange, 200,
+ s"""{"id":"$id","status":"Succeeded","usage":{"documentPagesBasic":1},
+ |"result":{"contents":[{"kind":"document","markdown":"synthetic $name",
+ |"fields":{"Optional":{"type":"string","confidence":0.9}}}]},
+ |"futureProperty":{"preserve":true}}""".stripMargin)
+ }
+ }
+ } finally {
+ exchange.close()
+ }
+ }
+ })
+ server.start()
+
+ override def close(): Unit = server.stop(0)
+ }
+
+ private def withService(testCode: Service => Unit): Unit = {
+ val service = new Service
+ try {
+ testCode(service)
+ } finally {
+ service.close()
+ }
+ }
+
+ private def withOutput(testCode: String => Unit): Unit = {
+ val directory = Files.createTempDirectory("cu-writer-")
+ try {
+ testCode(directory.resolve("journal").toString)
+ } finally {
+ FileUtils.deleteDirectory(directory.toFile)
+ }
+ }
+
+ private def analyzer(service: Service): ContentUnderstanding =
+ new ContentUnderstanding()
+ .setEndpoint(service.endpoint)
+ .setSubscriptionKey("synthetic-test-key")
+ .setDocumentUrlCol("source")
+ .setDocumentNameCol("docId")
+ .setMaxPollAttempts(1)
+ .setPollingDelay(0)
+
+ private def documents: DataFrame =
+ Seq(("a", "https://example.test/a.pdf"), ("b", "https://example.test/b.pdf")).toDF("docId", "source")
+
+ test("committed results and operation handles survive a later polling failure and resume without POSTs") {
+ withService { service =>
+ withOutput { path =>
+ val stage = analyzer(service)
+ service.malformed.set("b")
+ intercept[Exception] {
+ ContentUnderstandingWriter.writeToPath(documents, stage, "docId", path, "parquet")
+ }
+ val partial = ContentUnderstandingWriter.readPath(spark, path, "parquet")
+ val states = partial.select("documentId", "status").collect().map(r => r.getString(0) -> r.getString(1)).toMap
+ assert(states == Map("a" -> "Succeeded", "b" -> "Running"))
+ assert(partial.filter(col("documentId") === "b").select("operationLocation").head().getString(0).nonEmpty)
+ assert(service.submitted("a") == 1)
+ assert(service.submitted("b") == 1)
+
+ service.malformed.set("")
+ val resumed = ContentUnderstandingWriter.writeToPath(documents, stage, "docId", path, "parquet")
+ assert(resumed.filter(col("status") === "Succeeded").count() == 2)
+ assert(service.totalSubmissions == 2)
+ val raw = resumed.select("rawResponse").head().getString(0).parseJson.asJsObject
+ assert(raw.fields.contains("usage"))
+ assert(raw.fields.contains("futureProperty"))
+ val journal = spark.read.parquet(path)
+ assert(journal.count() == 4)
+ }
+ }
+ }
+
+ test("poll budget exhaustion remains resumable and does not submit the document again") {
+ withService { service =>
+ withOutput { path =>
+ val stage = analyzer(service)
+ val input = documents.filter(col("docId") === "a")
+ service.pending.set("a")
+ val pending = ContentUnderstandingWriter.writeToPath(input, stage, "docId", path, "parquet")
+ assert(pending.select("status").head().getString(0) == "Running")
+ service.pending.set("")
+ val done = ContentUnderstandingWriter.writeToPath(input, stage, "docId", path, "parquet")
+ assert(done.select("status").head().getString(0) == "Succeeded")
+ assert(service.submitted("a") == 1)
+ }
+ }
+ }
+
+ test("retains service failures as terminal records without repeating invalid requests") {
+ withService { service =>
+ withOutput { path =>
+ service.failed.set("a")
+ val stage = analyzer(service)
+ val first = ContentUnderstandingWriter.writeToPath(documents, stage, "docId", path, "parquet", 2)
+ assert(first.filter(col("status") === "Failed").count() == 1)
+ assert(first.filter(col("documentId") === "a").select("error").head().getString(0).contains("InvalidRequest"))
+ ContentUnderstandingWriter.writeToPath(documents, stage, "docId", path, "parquet", 2)
+ assert(service.totalSubmissions == 2)
+ }
+ }
+ }
+
+ test("rejects an ID reused with changed content or analysis configuration") {
+ withService { service =>
+ withOutput { path =>
+ val stage = analyzer(service)
+ val input = documents.filter(col("docId") === "a")
+ ContentUnderstandingWriter.writeToPath(input, stage, "docId", path, "parquet")
+ intercept[IllegalArgumentException] {
+ ContentUnderstandingWriter.writeToPath(
+ Seq(("a", "https://example.test/changed.pdf")).toDF("docId", "source"),
+ stage, "docId", path, "parquet")
+ }
+ intercept[IllegalArgumentException] {
+ ContentUnderstandingWriter.writeToPath(
+ input, analyzer(service).setRange("1-2"), "docId", path, "parquet")
+ }
+ assert(service.totalSubmissions == 1)
+ }
+ }
+ }
+
+ test("validates IDs and the destination before making any service request") {
+ withService { service =>
+ withOutput { path =>
+ val stage = analyzer(service)
+ val invalidInputs = Seq(
+ Seq(("a", "https://example.test/a"), ("a", "https://example.test/b")).toDF("docId", "source"),
+ Seq((Option.empty[String].orNull, "https://example.test/a")).toDF("docId", "source"),
+ Seq((" ", "https://example.test/a")).toDF("docId", "source"),
+ Seq(("\t\n", "https://example.test/a")).toDF("docId", "source")
+ )
+ invalidInputs.foreach { input =>
+ intercept[IllegalArgumentException] {
+ ContentUnderstandingWriter.writeToPath(input, stage, "docId", path, "parquet")
+ }
+ }
+ Seq("unrelated").toDF("value").write.parquet(path)
+ intercept[IllegalArgumentException] {
+ ContentUnderstandingWriter.writeToPath(documents, stage, "docId", path, "parquet")
+ }
+ assert(service.totalSubmissions == 0)
+ }
+ }
+ }
+
+ test("writes and resumes a catalog table through the same public journal API") {
+ withService { service =>
+ val tableName = "cu_writer_" + UUID.randomUUID().toString.replace("-", "")
+ try {
+ val stage = analyzer(service)
+ val result = ContentUnderstandingWriter.writeToTable(documents, stage, "docId", tableName, "parquet", 2)
+ assert(result.count() == 2)
+ assert(ContentUnderstandingWriter.readTable(spark, tableName).count() == 2)
+ ContentUnderstandingWriter.writeToTable(documents, stage, "docId", tableName, "parquet", 1)
+ assert(service.totalSubmissions == 2)
+ } finally {
+ spark.sql(s"DROP TABLE IF EXISTS `$tableName`")
+ }
+ }
+ }
+
+ test("durable writers accept input columns matching transform output names") {
+ withService { service =>
+ withOutput { path =>
+ val tableName = "cu_writer_" + UUID.randomUUID().toString.replace("-", "")
+ try {
+ val stage = analyzer(service).setOutputCol("source").setErrorCol("docId")
+ val input = documents.filter(col("docId") === "a")
+ val pathResult = stage.writeToPath(input, "docId", path, "parquet").head()
+ val tableResult = stage.writeToTable(input, "docId", tableName, "parquet").head()
+ Seq(pathResult, tableResult).foreach { row =>
+ assert(row.getAs[String]("documentId") == "a")
+ assert(row.getAs[String]("status") == "Succeeded")
+ assert(row.getAs[String]("rawResponse").contains("synthetic a"))
+ }
+ assert(service.submitted("a") == 2)
+ } finally {
+ spark.sql(s"DROP TABLE IF EXISTS `$tableName`")
+ }
+ }
+ }
+ }
+
+ test("empty input creates a readable empty journal without calling the service") {
+ withService { service =>
+ withOutput { path =>
+ val result = ContentUnderstandingWriter.writeToPath(
+ documents.limit(0), analyzer(service), "docId", path, "parquet")
+ assert(result.count() == 0)
+ assert(service.totalSubmissions == 0)
+ }
+ }
+ }
+
+ test("treats punctuation in input column names literally and allows credential rotation on resume") {
+ withService { service =>
+ withOutput { path =>
+ val input = Seq(("a", "https://example.test/a.pdf")).toDF("doc.`id", "file.uri")
+ val stage = analyzer(service).setDocumentUrlCol("file.uri").setDocumentNameCol("doc.`id")
+ val first = stage.writeToPath(input, "doc.`id", path, "parquet")
+ assert(first.select("documentId").head().getString(0) == "a")
+ stage.setSubscriptionKey("synthetic-rotated-key")
+ stage.writeToPath(input, "doc.`id", path, "parquet")
+ assert(stage.readPath(spark, path, "parquet").count() == 1)
+ assert(service.totalSubmissions == 1)
+ }
+ }
+ }
+
+ test("rejects invalid batch sizes and poll-only configuration before submission") {
+ withService { service =>
+ withOutput { path =>
+ val stage = analyzer(service)
+ intercept[IllegalArgumentException] {
+ ContentUnderstandingWriter.writeToPath(documents, stage, "docId", path, "parquet", 0)
+ }
+ intercept[IllegalArgumentException] {
+ ContentUnderstandingWriter.writeToPath(
+ documents, stage.setOperationMode("poll"), "docId", path, "parquet")
+ }
+ assert(service.totalSubmissions == 0)
+ }
+ }
+ }
+
+ private val accepted = ContentUnderstandingStubReply(202, """{"id":"op","status":"Running"}""",
+ Map("Operation-Location" -> "$ROOT/contentunderstanding/analyzerResults/op?api-version=2025-11-01"))
+ private val completed = ContentUnderstandingStubReply(200,
+ """{"id":"op","status":"Succeeded","result":{"contents":[]}}""")
+
+ private def stubAnalyzer(server: ContentUnderstandingStub): ContentUnderstanding =
+ new ContentUnderstanding().setEndpoint(server.endpoint).setDocumentBytes(Array[Byte](1))
+ .setDocumentNameCol("docId").setPollingDelay(0).setMaxPollAttempts(1)
+
+ test("submit-only writer respects the configured operation mode and later resumes saved handles") {
+ ContentUnderstandingStub.withReplies(Seq(accepted, completed)) { server =>
+ withOutput { path =>
+ val stage = stubAnalyzer(server).setOperationMode("submit")
+ val input = documents.filter(col("docId") === "a")
+ val submitted = stage.writeToPath(input, "docId", path, "parquet").head()
+ assert(submitted.getAs[String]("status") == "Running")
+ assert(submitted.getAs[Long]("sequence") == 0L)
+ assert(server.requests.map(_.method) == Seq("POST"))
+
+ stage.writeToPath(input, "docId", path, "parquet")
+ assert(server.requests.map(_.method) == Seq("POST"))
+ assert(spark.read.parquet(path).count() == 1)
+
+ val resumed = stage.setOperationMode("analyze").writeToPath(input, "docId", path, "parquet").head()
+ assert(resumed.getAs[String]("status") == "Succeeded")
+ assert(resumed.getAs[String]("operationLocation") == submitted.getAs[String]("operationLocation"))
+ assert(resumed.getAs[Long]("sequence") == 1L)
+ assert(server.requests.map(_.method) == Seq("POST", "GET"))
+ }
+ }
+ }
+
+ test("writer recovery retries definite admission rejections without poisoning completed IDs") {
+ Seq(401, 403, 429).foreach { code =>
+ val rejected = ContentUnderstandingStubReply(code, """{"error":{"code":"Rejected"}}""")
+ ContentUnderstandingStub.withReplies(Seq(accepted, completed, rejected, accepted, completed)) { server =>
+ withOutput { path =>
+ val stage = stubAnalyzer(server)
+ val failure = intercept[ContentUnderstandingException] {
+ stage.writeToPath(documents, "docId", path, "parquet")
+ }
+ assert(failure.response.status == "Rejected")
+ assert(failure.response.httpStatus == code)
+ val partial = stage.readPath(spark, path, "parquet").collect()
+ assert(partial.length == 1)
+ assert(partial.head.getAs[String]("documentId") == "a")
+ assert(partial.head.getAs[String]("status") == "Succeeded")
+ val resumed = stage.setAADToken("refreshed-test-token")
+ .writeToPath(documents, "docId", path, "parquet")
+ assert(resumed.filter(col("status") === "Succeeded").count() == 2)
+ assert(server.requests.count(_.method == "POST") == 3)
+ }
+ }
+ }
+ }
+
+ test("writer recovery records indeterminate server errors without resubmitting them") {
+ Seq(408, 500, 503).foreach { code =>
+ ContentUnderstandingStub.withReplies(Seq(
+ ContentUnderstandingStubReply(code, """{"error":{"code":"Indeterminate"}}"""))) { server =>
+ withOutput { path =>
+ val stage = stubAnalyzer(server)
+ val failure = intercept[ContentUnderstandingException] {
+ stage.writeToPath(documents, "docId", path, "parquet")
+ }
+ assert(failure.response.status == "Unknown")
+ assert(stage.readPath(spark, path, "parquet").head().getAs[String]("status") == "Unknown")
+ intercept[IllegalArgumentException] {
+ stage.writeToPath(documents, "docId", path, "parquet")
+ }
+ assert(server.requests.count(_.method == "POST") == 1)
+ }
+ }
+ }
+ }
+
+ test("writer recovery records unavailable results and continues with later documents") {
+ Seq(404, 410).foreach { code =>
+ val unavailable = ContentUnderstandingStubReply(code, """{"error":{"code":"ResultNotFound"}}""")
+ ContentUnderstandingStub.withReplies(Seq(accepted, accepted, unavailable, accepted, completed)) { server =>
+ withOutput { path =>
+ val stage = stubAnalyzer(server)
+ val first = stage.writeToPath(documents.filter(col("docId") === "a"), "docId", path, "parquet")
+ assert(first.head().getAs[String]("status") == "Running")
+ val resumed = stage.writeToPath(documents, "docId", path, "parquet")
+ val missing = resumed.filter(col("documentId") === "a").head()
+ assert(missing.getAs[String]("status") == "ResultUnavailable")
+ assert(missing.getAs[Int]("httpStatus") == code)
+ assert(missing.getAs[String]("error").contains("ResultNotFound"))
+ assert(resumed.filter(col("documentId") === "b").head().getAs[String]("status") == "Succeeded")
+ stage.writeToPath(documents, "docId", path, "parquet")
+ assert(server.requests.map(_.method) == Seq("POST", "GET", "GET", "POST", "GET"))
+ }
+ }
+ }
+ }
+}
diff --git a/docs/Explore Algorithms/AI Services/Content Understanding.md b/docs/Explore Algorithms/AI Services/Content Understanding.md
new file mode 100644
index 0000000000..6742904803
--- /dev/null
+++ b/docs/Explore Algorithms/AI Services/Content Understanding.md
@@ -0,0 +1,473 @@
+---
+title: Content Understanding
+hide_title: true
+sidebar_label: Content Understanding
+description: Analyze content with Azure Content Understanding and save resumable results to a Fabric lakehouse or Spark table.
+---
+
+# Azure Content Understanding
+
+`ContentUnderstanding` calls Azure Content Understanding from a Spark DataFrame.
+It supports prebuilt and custom analyzers, document URLs or bytes, page and time
+ranges, and per-request model deployment mappings. The default API version is
+the latest generally available version, `2025-11-01`. Preview versions require
+an explicit opt-in.
+
+Use `transform` for ordinary Spark pipelines. Use `writeToTable` or `writeToPath`
+when completed documents must survive a later failure. The durable methods save
+each accepted operation handle before polling, then commit each result separately.
+They do not wait for the entire input DataFrame to finish before writing results.
+
+## API at a glance
+
+Import `ContentUnderstanding` from `synapse.ml.services.contentunderstanding`.
+These are instance methods on a configured analyzer:
+
+| Python method | Behavior and return value |
+| --- | --- |
+| `transform(dataset)` | Lazy Spark transformation. Returns the input columns plus `outputCol` and `errorCol`. |
+| `writeToTable(dataset, idCol, tableName, format="delta", batchSize=1)` | Eagerly processes input and persists each operation. Returns a DataFrame with the latest state of every ID in the journal. |
+| `writeToPath(dataset, idCol, path, format="delta", batchSize=1)` | Same persistence and return schema, using a lakehouse or filesystem path. |
+| `readTable(spark, tableName)` | Reads the latest persisted state per ID without contacting the service. |
+| `readPath(spark, path, format="delta")` | Reads the latest persisted state from a path without contacting the service. |
+| `createAnalyzer(definition, allowReplace=False)` | Explicit driver call. Accepts a dictionary or JSON string and returns the analyzer definition as a JSON string. |
+| `getAnalyzer()` | Explicit driver call that returns the configured analyzer's definition as a JSON string. |
+
+`transform` adds a response struct containing `operationLocation`, `id`,
+`status`, `httpStatus`, `rawResponse`, and `error`. The durable methods return
+those fields as top-level columns, with `documentId`, `requestHash`, and
+`sequence`. They do not copy the input document bytes into the journal.
+The writers do not add `outputCol` or `errorCol` to the input, so existing
+columns with those names are allowed. `transform` rejects those collisions
+to avoid overwriting input columns.
+
+### REST calls under the hood
+
+The Scala implementation uses the same SynapseML HTTP and Spark pipeline
+infrastructure as other cognitive-service stages. It sends REST requests
+directly, without a Content Understanding Python SDK dependency:
+
+| Method | REST operation |
+| --- | --- |
+| Analyze or submit | `POST {endpoint}/contentunderstanding/analyzers/{analyzerId}:analyze?api-version={version}` |
+| Poll an accepted operation | `GET {Operation-Location}` |
+| `createAnalyzer` | `PUT {endpoint}/contentunderstanding/analyzers/{analyzerId}?api-version={version}&allowReplace={bool}`, then poll its management operation |
+| `getAnalyzer` | `GET {endpoint}/contentunderstanding/analyzers/{analyzerId}?api-version={version}` |
+
+The analyze body contains one `inputs` entry with either `url` or base64 `data`,
+optional `name`, `mimeType`, and `range`, and an optional top-level
+`modelDeployments` map. `stringEncoding` and `processingLocation` are query
+parameters. Analyzer configuration is sent only by `createAnalyzer`.
+
+## Configure an analyzer
+
+Install the SynapseML Python package and matching JVM artifacts that include this
+class. Fabric's preinstalled SynapseML version may not include a newly released
+class. Follow the [installation guide](../../Get%20Started/Install%20SynapseML.md)
+for your runtime rather than mixing Python and JVM versions.
+
+In a Fabric notebook, retrieve the key from Key Vault at runtime:
+
+```python
+import notebookutils
+from synapse.ml.services.contentunderstanding import ContentUnderstanding
+
+key = notebookutils.credentials.getSecret(
+ "https://.vault.azure.net/", ""
+)
+
+analyzer = (
+ ContentUnderstanding()
+ .setEndpoint("https://.cognitiveservices.azure.com")
+ .setSubscriptionKey(key)
+ .setAnalyzerId("prebuilt-read")
+ .setDocumentUrlCol("documentUrl")
+ .setOutputCol("analysis")
+ .setErrorCol("requestError")
+)
+```
+
+Outside Fabric, obtain the key from your secret manager. Microsoft Entra ID
+authentication is also supported through `setAADToken` or `setAADTokenCol`.
+Acquire a token for `https://cognitiveservices.azure.com/.default` using an
+identity authorized on your Content Understanding resource. Refresh it for
+long-running work. No implicit Fabric service endpoint or identity is used.
+
+Do not put credentials in notebook arguments, Spark configuration, source code,
+or displayed output. SparkML persistence includes configured scalar parameters:
+save an unauthenticated stage and inject credentials after loading it.
+
+```python
+documents = spark.createDataFrame(
+ [("invoice-v1", "https:////invoice.pdf")],
+ ["documentId", "documentUrl"],
+)
+
+results = analyzer.transform(documents)
+results.selectExpr(
+ "documentId",
+ "analysis.status",
+ "get_json_object(analysis.rawResponse, '$.result.contents[0].markdown') AS markdown",
+ "requestError",
+).show(truncate=False)
+```
+
+A URL must be accessible to the Content Understanding service, not just the
+notebook. For private lakehouse files, use a Spark binary column instead:
+
+```python
+files = spark.read.format("binaryFile").load("Files/documents/")
+binaryAnalyzer = (
+ ContentUnderstanding()
+ .setEndpoint("https://.cognitiveservices.azure.com")
+ .setSubscriptionKey(key)
+ .setDocumentBytesCol("content")
+ .setDocumentNameCol("path")
+)
+```
+
+Configure exactly one source, `documentUrl` or `documentBytes`. Do not set both
+on the same stage. Binary input is base64-encoded in the request's `inputs[].data`
+property. Prefer URLs for large files when access requirements permit it, since
+base64 encoding increases memory and request size.
+
+PDF and DOCX files can use the same bytes API. For DOCX, supply a `.docx` name
+and, when setting `mimeType`, use
+`application/vnd.openxmlformats-officedocument.wordprocessingml.document`.
+Use `setMimeTypeCol` for mixed-format input.
+
+Do not assume every document response contains a `pages` array. In live
+`prebuilt-read` tests, PDFs returned page-level output and DOCX returned text and
+tables as Markdown with `documentPagesMinimal` usage. A DOCX request with
+`range="2"` still returned the whole document. Use whole-document IDs for DOCX;
+do not use its page range as a checkpoint boundary. The PDF range examples below
+are not a DOCX pagination guarantee.
+
+### Request options
+
+Options backed by service parameters accept either a scalar setter or a column
+setter, such as `setRange("1-5")` and `setRangeCol("pageRange")`.
+
+| Option | Purpose |
+| --- | --- |
+| `analyzerId` | Prebuilt analyzer or an existing custom analyzer. Defaults to `prebuilt-read`. |
+| `documentUrl`, `documentBytes` | Mutually exclusive input source. |
+| `documentName`, `mimeType` | Optional name and content type. |
+| `range` | Original 1-based pages for documents, or integer milliseconds for audio/video. |
+| `modelDeployments` | Map the analyzer's model names or aliases to deployments in your resource. |
+| `stringEncoding` | `codePoint`, `utf16`, or `utf8` for response offsets. |
+| `processingLocation` | Service-supported processing-location policy. |
+| `apiVersion` | Defaults to `2025-11-01`; use `setApiVersion` to override. |
+| `operationMode` | `analyze`, `submit`, or `poll`. Defaults to `analyze`. |
+| `operationLocation` | Accepted operation URL used by poll-only mode. |
+
+Polling and memory controls use ordinary scalar setters:
+
+| Setter | Default | Purpose |
+| --- | --- | --- |
+| `setMaxPollAttempts` | 120 | Bound GET attempts, including retries. |
+| `setPollingDelay` | 1000 | Milliseconds between polls when `Retry-After` is absent. |
+| `setMaxResponseBytes` | 33554432 | Bound each response before parsing JSON. |
+| `setTimeout` | 60 | HTTP connection and read timeout in seconds. |
+| `setConcurrency` | 1 | Concurrent operations per Spark partition for `transform`. |
+
+The client honors `Retry-After`, capped at one minute per delay. A response-size
+failure includes `ResponseTooLarge`; increase the bound or choose smaller ranges
+rather than treating an unread response as an empty result.
+
+For example, a prebuilt invoice analyzer can use request-level model aliases:
+
+```python
+invoiceAnalyzer = analyzer.copy({}).setAnalyzerId("prebuilt-invoice")
+invoiceAnalyzer.setModelDeployments(
+ {
+ "prebuilt-analyzer-completion": "",
+ "prebuilt-analyzer-embedding": "",
+ }
+)
+```
+
+The selected model must be supported by the analyzer. A deployment name alone
+does not identify the alias the analyzer expects. See
+[models and deployments](https://learn.microsoft.com/azure/ai-services/content-understanding/concepts/models-deployments).
+Request-level mappings do not modify the resource's shared defaults.
+
+To opt into the `2026-06-01-preview` API and its layout behavior:
+
+```python
+previewAnalyzer = (
+ analyzer.copy({})
+ .setApiVersion("2026-06-01-preview")
+ .setAnalyzerId("prebuilt-layout")
+ .setStringEncoding("utf16")
+)
+```
+
+### Custom extraction and analyzer configuration
+
+Layout details, segmentation, and extraction fields belong in an analyzer
+definition, not an `options` object in an analyze request. `createAnalyzer`
+accepts a Python dictionary or JSON string. It is an explicit driver operation
+and never runs automatically during `transform`.
+
+```python
+custom = analyzer.copy({}).setAnalyzerId("purchase-order-v1")
+definition = {
+ "baseAnalyzerId": "prebuilt-document",
+ "description": "Extract the supplier from a purchase order.",
+ "config": {"returnDetails": True},
+ "fieldSchema": {
+ "name": "PurchaseOrder",
+ "fields": {
+ "Supplier": {
+ "type": "string",
+ "description": "The supplier's legal business name.",
+ }
+ },
+ },
+}
+
+created = custom.createAnalyzer(definition, allowReplace=False)
+current = custom.getAnalyzer()
+```
+
+The service creates analyzers asynchronously. `createAnalyzer` waits within the
+configured polling budget and returns the final analyzer definition. A service
+failure or exhausted creation budget raises an exception with the operation
+response. Use `getAnalyzer` to inspect the analyzer afterward.
+`allowReplace=False` protects an existing analyzer. Replacing an analyzer is an
+explicit administrative choice.
+
+Your resource must have its required default model deployments configured
+before creating a custom analyzer. If the service reports `DefaultsNotSet`,
+have the resource administrator configure those defaults. SynapseML does not
+silently change them. The complete definition is forwarded to the
+[create-analyzer API](https://learn.microsoft.com/rest/api/contentunderstanding/content-analyzers/create-or-replace?view=rest-contentunderstanding-2025-11-01),
+so new service configuration fields do not require a new Python wrapper.
+
+## Save completed work to a lakehouse or table
+
+Use a stable, unique string ID for each immutable document version and selected
+range. `writeToTable` and `writeToPath` are eager actions, unlike `transform`.
+The examples below use Delta, which is the default and the recommended format
+for lakehouse use.
+
+```python
+latest = analyzer.writeToTable(
+ documents,
+ idCol="documentId",
+ tableName="content_understanding_operations",
+)
+
+# An alternative destination using a lakehouse path:
+latest = analyzer.writeToPath(
+ documents,
+ idCol="documentId",
+ path="Files/content-understanding/operations",
+ format="delta",
+)
+```
+
+Choose one destination for a given workload. These two calls are alternatives:
+calling both starts separate analyses in separate journals.
+
+If an action fails, read the already-committed work without contacting the
+service:
+
+```python
+partial = analyzer.readTable(spark, "content_understanding_operations")
+completed = partial.where("status = 'Succeeded'")
+failures = partial.where("status IN ('Failed', 'Canceled', 'ResultUnavailable')")
+```
+
+Rerun `writeToTable` with the same input, IDs, options, and destination to resume.
+It skips terminal records and polls saved handles for unfinished operations.
+It does not POST those documents again. Changing the bytes, URL, analyzer, or
+analysis options for an existing ID raises an error instead of silently mixing
+different analyses. Rotating the resource's authentication credential does not
+change the request fingerprint.
+
+Content changes behind an unchanged URL cannot be detected. Use immutable URLs
+or versioned IDs. A new SAS URL changes the request fingerprint, so retain the
+original manifest when resuming an operation that was already submitted.
+Likewise, version custom analyzers and model deployments: the fingerprint does
+not fetch remote analyzer definitions or model revisions.
+
+To persist accepted handles without waiting for analysis to complete, use
+submit-only mode. Resume against the same destination in analyze mode:
+
+```python
+analyzer.setOperationMode("submit").writeToTable(
+ documents, "documentId", "content_understanding_operations"
+)
+latest = analyzer.setOperationMode("analyze").writeToTable(
+ documents, "documentId", "content_understanding_operations"
+)
+```
+
+Repeated submit-only writes skip IDs that already have saved handles.
+Poll-only `transform` can also consume operation handles through
+`setOperationLocationCol`. It uses the API version in each saved URL, so the
+original document and API-version columns are not required. The durable writers
+require the original document manifest and use `analyze` or `submit` mode.
+
+For a path destination, use
+`analyzer.readPath(spark, "Files/content-understanding/operations")`.
+The read helpers and write return values contain the latest state per ID.
+Reading the physical table or path directly returns the append-only operation
+history, usually multiple rows per document:
+
+| Column | Meaning |
+| --- | --- |
+| `documentId` | Your stable document/range ID. |
+| `requestHash` | SHA-256 fingerprint of the analysis request, excluding resource authentication. |
+| `sequence` | Increasing journal sequence within that ID. |
+| `operationLocation`, `id` | Service operation handle and service-generated ID. |
+| `status`, `httpStatus` | Service operation status or SDK recovery status, and last HTTP status. |
+| `rawResponse` | Complete operation JSON, including results, usage, warnings, and unknown fields. |
+| `error` | Service error or client diagnostic JSON when present. |
+
+Only `Succeeded` means successful completion. A `Running` response can already
+contain an empty `result`, and an HTTP 200 response can report `Failed`. Inspect
+the status and error columns before downstream processing. Terminal failures
+are retained rather than retried indefinitely. Correct the cause and use a new
+versioned ID for an intentional retry.
+
+Submission HTTP 401, 403, or 429 without an operation handle is a definite
+rejection. The writer stops with `Rejected` before recording that ID. Fix the
+credential or throttling condition and rerun against the same journal. Completed
+IDs remain untouched, and the rejected ID can be submitted again.
+
+A submission transport failure, HTTP 408, HTTP 5xx, or unreadable response can
+leave acceptance or completion uncertain. An accepted response with a missing
+or invalid handle is also `Unknown`. The writer records `Unknown` and stops if
+no valid handle was received. It will not automatically submit that ID again.
+Resolve the service outcome before choosing a new ID for an intentional retry.
+To process unrelated documents while investigating, exclude the unresolved IDs
+from the input manifest and keep the same journal.
+
+Polling HTTP 404 or 410 produces a terminal `ResultUnavailable` record with the
+original handle and error. The result may be missing or past the service's
+retention period. The writer continues with later IDs without resubmitting the
+unavailable operation. Only the affected document or range needs a new versioned
+ID if you intentionally analyze it again.
+
+### Large documents and explicit ranges
+
+For range-by-range durability, create one input row and ID per range:
+
+```python
+ranges = spark.createDataFrame(
+ [
+ ("report-v1/pages/1-2", "https:///report.pdf", "1-2"),
+ ("report-v1/pages/3-4", "https:///report.pdf", "3-4"),
+ ],
+ ["documentId", "documentUrl", "pageRange"],
+)
+
+rangeAnalyzer = analyzer.copy({}).setRangeCol("pageRange")
+latest = rangeAnalyzer.writeToTable(
+ ranges, idCol="documentId", tableName="report_analysis_operations"
+)
+```
+
+Ranges refer to the original input. They do not restart numbering at page 1.
+Splitting a document changes the context available for cross-page extraction
+and table reasoning, so SynapseML never splits it automatically.
+
+This saves complete documents or selected ranges. The service does not promise
+usable page-by-page output while an operation is still running. Follow the
+[service limits](https://learn.microsoft.com/azure/ai-services/content-understanding/service-limits);
+selecting a range does not remove input-byte or response-size limits.
+
+### Durability and resource limits
+
+Only one writer may own a destination at a time. Use a dedicated output table
+or path, not an existing business table. Delta and Parquet are supported; Delta
+provides the stronger transactional storage behavior.
+
+The writer processes requests sequentially on the driver. `batchSize` controls
+how many input rows it collects at a time, not how many results share a commit.
+It defaults to 1 to limit retained document bytes. It projects only the ID and
+configured parameter columns, but a single binary document or response must
+still fit in memory. `setConcurrency` applies to `transform`, not to the durable
+writer.
+
+The input must be a static, deterministic manifest that does not change during
+the call. Bounded keyset batches can rescan and sort that manifest. For very
+large manifests, call the writer from controlled input batches or a structured
+streaming `foreachBatch` callback using the same journal and globally stable
+IDs. Do not pass a streaming DataFrame directly.
+
+Polling has a finite attempt budget. Exhausting it leaves a pending record that
+can be resumed with a fresh credential or a larger budget. Transport errors and
+malformed responses fail the action instead of being reported as success.
+Already-committed handles and results remain available.
+Other polling HTTP errors fail the action without marking the accepted service
+operation as failed. For an expired credential, set a fresh key or token and
+rerun against the same journal. Read helpers expose the last committed state;
+the exception contains the unsuccessful polling response.
+
+There is an unavoidable crash window between the service accepting a POST and
+the journal committing its handle. A crash in that window can cause another
+submission on retry. Neither the writer nor Spark task retries provide
+exactly-once external service calls. POST requests are never automatically
+retried within an invocation.
+
+Service results are retained for
+[up to 24 hours](https://learn.microsoft.com/azure/foundry/responsible-ai/content-understanding/data-privacy).
+Resume pending operations promptly. A saved handle is not permanent result
+storage; the journal's committed `rawResponse` is.
+
+## Scala
+
+Scala uses the same implementation and persistence behavior:
+
+```scala
+def transform(dataset: Dataset[_]): DataFrame
+def writeToTable(dataset: Dataset[_], idCol: String, tableName: String,
+ format: String = "delta", batchSize: Int = 1): DataFrame
+def writeToPath(dataset: Dataset[_], idCol: String, path: String,
+ format: String = "delta", batchSize: Int = 1): DataFrame
+def readTable(spark: SparkSession, tableName: String): DataFrame
+def readPath(spark: SparkSession, path: String, format: String = "delta"): DataFrame
+def createAnalyzer(definitionJson: String, allowReplace: Boolean): String
+def getAnalyzer(): String
+```
+
+```scala
+import com.microsoft.azure.synapse.ml.services.contentunderstanding.ContentUnderstanding
+
+val analyzer = new ContentUnderstanding()
+ .setEndpoint(endpoint)
+ .setSubscriptionKey(key)
+ .setDocumentUrlCol("documentUrl")
+ .setAnalyzerId("prebuilt-read")
+
+val latest = analyzer.writeToTable(
+ documents, idCol = "documentId", tableName = "content_understanding_operations")
+val resumed = analyzer.readTable(spark, "content_understanding_operations")
+```
+
+No Content Understanding Python SDK is required. The public transformer,
+request handling, and journal logic live in the JVM implementation.
+
+## Content Understanding tests
+
+Offline Scala suites are under
+`cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/contentunderstanding`.
+They remain in the existing document-service CI group, with separate files for
+the public API and protocol, journal writes, recovery, session filesystem
+configuration, and framework fuzzing. Run only this feature with:
+
+```bash
+sbt "cognitive/testOnly *ContentUnderstanding*Suite"
+```
+
+`test_ContentUnderstanding.py` exercises the generated Python wrapper against a
+loopback REST fixture. `test_ContentUnderstandingE2E.py` is a separate, opt-in
+Azure suite that generates synthetic PDF and DOCX files. It covers PDF ranges,
+DOCX text and tables, optional preview metadata, partial table results, and
+submit-only path resumption. Its module docstring lists the environment
+variables and the scratch resources it creates and removes. Without explicitly
+configured live-service credentials, the live suite is skipped.
diff --git a/website/sidebars.js b/website/sidebars.js
index 66df2e5dab..569c9382a9 100644
--- a/website/sidebars.js
+++ b/website/sidebars.js
@@ -33,6 +33,7 @@ module.exports = {
items: [
"Explore Algorithms/AI Services/Overview",
"Explore Algorithms/AI Services/Geospatial Services",
+ "Explore Algorithms/AI Services/Content Understanding",
"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key",
"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes",
"Explore Algorithms/AI Services/Quickstart - Analyze Text",