Merge branch 'main' into daymon-fix-tests

Author: daymxn

firebase-ai/CHANGELOG.md

@@ -13,3 +13,5 @@
 * [fixed] Fixed an issue with `LiveContentResponse` audio data not being present when the model was
   interrupted or the turn completed. (#6870)
 * [fixed] Fixed an issue with `LiveSession` not converting exceptions to `FirebaseVertexAIException`. (#6870)
+* [feature] Add support for specifying response modalities in `GenerationConfig`. (#6921)
+* [feature] Added a helper field for getting all the `InlineDataPart` from a `GenerateContentResponse`. (#6922)

firebase-ai/api.txt

@@ -330,11 +330,13 @@ package com.google.firebase.ai.type {
     ctor public GenerateContentResponse(java.util.List<com.google.firebase.ai.type.Candidate> candidates, com.google.firebase.ai.type.PromptFeedback? promptFeedback, com.google.firebase.ai.type.UsageMetadata? usageMetadata);
     method public java.util.List<com.google.firebase.ai.type.Candidate> getCandidates();
     method public java.util.List<com.google.firebase.ai.type.FunctionCallPart> getFunctionCalls();
+    method public java.util.List<com.google.firebase.ai.type.InlineDataPart> getInlineDataParts();
     method public com.google.firebase.ai.type.PromptFeedback? getPromptFeedback();
     method public String? getText();
     method public com.google.firebase.ai.type.UsageMetadata? getUsageMetadata();
     property public final java.util.List<com.google.firebase.ai.type.Candidate> candidates;
     property public final java.util.List<com.google.firebase.ai.type.FunctionCallPart> functionCalls;
+    property public final java.util.List<com.google.firebase.ai.type.InlineDataPart> inlineDataParts;
     property public final com.google.firebase.ai.type.PromptFeedback? promptFeedback;
     property public final String? text;
     property public final com.google.firebase.ai.type.UsageMetadata? usageMetadata;
@@ -352,6 +354,7 @@ package com.google.firebase.ai.type {
     field public Integer? maxOutputTokens;
     field public Float? presencePenalty;
     field public String? responseMimeType;
+    field public java.util.List<com.google.firebase.ai.type.ResponseModality>? responseModalities;
     field public com.google.firebase.ai.type.Schema? responseSchema;
     field public java.util.List<java.lang.String>? stopSequences;
     field public Float? temperature;

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/GenerateContentResponse.kt

@@ -44,6 +44,18 @@ public class GenerateContentResponse(
     candidates.first().content.parts.filterIsInstance<FunctionCallPart>()
   }
 
+  /**
+   * Convenience field representing all the [InlineDataPart]s in the first candidate, if they exist.
+   *
+   * This also includes any [ImagePart], but they will be represented as [InlineDataPart] instead.
+   */
+  public val inlineDataParts: List<InlineDataPart> by lazy {
+    candidates.first().content.parts.let { parts ->
+      parts.filterIsInstance<ImagePart>().map { it.toInlineDataPart() } +
+        parts.filterIsInstance<InlineDataPart>()
+    }
+  }
+
   @Serializable
   internal data class Internal(
     val candidates: List<Candidate.Internal>? = null,

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/GenerationConfig.kt

@@ -75,6 +75,8 @@ import kotlinx.serialization.Serializable
  * Refer to the
  * [Control generated output](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/control-generated-output)
  * guide for more details.
+ *
+ * @property responseModalities The format of data in which the model should respond with.
  */
 public class GenerationConfig
 private constructor(
@@ -88,6 +90,7 @@ private constructor(
   internal val stopSequences: List<String>?,
   internal val responseMimeType: String?,
   internal val responseSchema: Schema?,
+  internal val responseModalities: List<ResponseModality>?,
 ) {
 
   /**
@@ -115,6 +118,9 @@ private constructor(
    * @property responseMimeType See [GenerationConfig.responseMimeType].
    *
    * @property responseSchema See [GenerationConfig.responseSchema].
+   *
+   * @property responseModalities See [GenerationConfig.responseModalities].
+   *
    * @see [generationConfig]
    */
   public class Builder {
@@ -128,6 +134,7 @@ private constructor(
     @JvmField public var stopSequences: List<String>? = null
     @JvmField public var responseMimeType: String? = null
     @JvmField public var responseSchema: Schema? = null
+    @JvmField public var responseModalities: List<ResponseModality>? = null
 
     /** Create a new [GenerationConfig] with the attached arguments. */
     public fun build(): GenerationConfig =
@@ -142,6 +149,7 @@ private constructor(
         frequencyPenalty = frequencyPenalty,
         responseMimeType = responseMimeType,
         responseSchema = responseSchema,
+        responseModalities = responseModalities
       )
   }
 
@@ -156,7 +164,8 @@ private constructor(
       frequencyPenalty = frequencyPenalty,
       presencePenalty = presencePenalty,
       responseMimeType = responseMimeType,
-      responseSchema = responseSchema?.toInternal()
+      responseSchema = responseSchema?.toInternal(),
+      responseModalities = responseModalities?.map { it.toInternal() }
     )
 
   @Serializable
@@ -171,6 +180,7 @@ private constructor(
     @SerialName("presence_penalty") val presencePenalty: Float? = null,
     @SerialName("frequency_penalty") val frequencyPenalty: Float? = null,
     @SerialName("response_schema") val responseSchema: Schema.Internal? = null,
+    @SerialName("response_modalities") val responseModalities: List<String>? = null
   )
 
   public companion object {

plugins/src/main/java/com/google/firebase/gradle/plugins/PostReleasePlugin.kt

@@ -65,9 +65,9 @@ class PostReleasePlugin : Plugin<Project> {
   fun registerVersionBumpTask(project: Project) =
     project.tasks.register<VersionBumpTask>("versionBump") {
       // TODO(b/285892320): Remove condition when bug fixed
-      bumpVersion.set(project.firebaseLibrary.artifactId.map {
-        it !== "protolite-well-known-types"
-      })
+      bumpVersion.set(
+        project.firebaseLibrary.artifactId.map { it !== "protolite-well-known-types" }
+      )
     }
 
   /**

plugins/src/main/java/com/google/firebase/gradle/plugins/PublishingPlugin.kt

@@ -418,6 +418,7 @@ abstract class PublishingPlugin : Plugin<Project> {
           "com.google.firebase:firebase-crashlytics",
           "com.google.firebase:firebase-perf",
           "com.google.firebase:firebase-vertexai",
+          "com.google.firebase:firebase-ai",
           "com.google.firebase:firebase-messaging",
           "com.google.firebase:firebase-auth",
           "com.google.firebase:firebase-database",
@@ -816,6 +817,7 @@ abstract class PublishingPlugin : Plugin<Project> {
         "com.google.firebase:firebase-storage",
         "com.google.firebase:firebase-storage-ktx",
         "com.google.firebase:firebase-vertexai",
+        "com.google.firebase:firebase-ai",
       )
 
     /** Artifacts that we use in the tutorial bundle, but _not_ in the bom. */

plugins/src/main/java/com/google/firebase/gradle/plugins/VersionBumpTask.kt

@@ -40,7 +40,8 @@ import org.gradle.kotlin.dsl.provideDelegate
  *   version.
  * @property newVersion A [ModuleVersion] of what to set the version to. Defaults to one patch
  *   higher than [releasedVersion].
- * @property bumpVersion If set, then the default provided by [newVersion] will be bumped by one patch. Defaults to `true`.
+ * @property bumpVersion If set, then the default provided by [newVersion] will be bumped by one
+ *   patch. Defaults to `true`.
  * @see PostReleasePlugin
  */
 abstract class VersionBumpTask : DefaultTask() {
@@ -63,7 +64,8 @@ abstract class VersionBumpTask : DefaultTask() {
   @TaskAction
   fun build() {
     val latestVersion = releasedVersion.get()
-    val version = newVersion.orNull ?: if(bumpVersion.get()) latestVersion.bump() else latestVersion
+    val version =
+      newVersion.orNull ?: if (bumpVersion.get()) latestVersion.bump() else latestVersion
 
     versionFile.get().asFile.rewriteLines {
       when {