feat: [google-ai-generativelanguage] Add video metadata to files API (#12704)

BEGIN_COMMIT_OVERRIDE
feat(v1beta): Add video metadata to files API
feat(v1beta): Update timeouts for generate content
docs(v1beta): Minor updates
feat(v1): Add usage metadata to GenerateContentResponse
feat(v1): Add generate_content_request to CountTokensRequest
feat(v1): Update timeouts
docs(v1): Minor updates
END_COMMIT_OVERRIDE


- [ ] Regenerate this pull request now.



PiperOrigin-RevId: 634425922

Source-Link:
googleapis/googleapis@298a53b

Source-Link:
googleapis/googleapis-gen@e549e3f
Copy-Tag:
eyJwIjoicGFja2FnZXMvZ29vZ2xlLWFpLWdlbmVyYXRpdmVsYW5ndWFnZS8uT3dsQm90LnlhbWwiLCJoIjoiZTU0OWUzZmY0ZmJmZGMzNTc5ZjJlZWY1OWVhMDhjNzA3MDZkNDZmMiJ9

BEGIN_NESTED_COMMIT
feat: [google-ai-generativelanguage] Add usage metadata to
GenerateContentResponse
feat: Add generate_content_request to CountTokensRequest
feat: Update timeouts
docs: Minor updates

PiperOrigin-RevId: 634424772

Source-Link:
googleapis/googleapis@b546bed

Source-Link:
googleapis/googleapis-gen@c09c0f6
Copy-Tag:
eyJwIjoicGFja2FnZXMvZ29vZ2xlLWFpLWdlbmVyYXRpdmVsYW5ndWFnZS8uT3dsQm90LnlhbWwiLCJoIjoiYzA5YzBmNjI5MjIwMDVmNjczZTczZDAzNGI2MTI2NTc2MzE4ZGY3NCJ9
END_NESTED_COMMIT

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
Co-authored-by: Anthonios Partheniou <partheniou@google.com>

Author: gcf-owl-bot[bot]

packages/google-ai-generativelanguage/google/ai/generativelanguage/__init__.py

@@ -89,7 +89,7 @@
     Message,
     MessagePrompt,
 )
-from google.ai.generativelanguage_v1beta.types.file import File
+from google.ai.generativelanguage_v1beta.types.file import File, VideoMetadata
 from google.ai.generativelanguage_v1beta.types.file_service import (
     CreateFileRequest,
     CreateFileResponse,
@@ -251,6 +251,7 @@
     "Message",
     "MessagePrompt",
     "File",
+    "VideoMetadata",
     "CreateFileRequest",
     "CreateFileResponse",
     "DeleteFileRequest",

packages/google-ai-generativelanguage/google/ai/generativelanguage_v1/services/generative_service/async_client.py

@@ -283,6 +283,12 @@ async def generate_content(
         r"""Generates a response from the model given an input
         ``GenerateContentRequest``.
 
+        Input capabilities differ between models, including tuned
+        models. See the `model
+        guide <https://ai.google.dev/models/gemini>`__ and `tuning
+        guide <https://ai.google.dev/docs/model_tuning_guidance>`__ for
+        details.
+
         .. code-block:: python
 
             # This snippet has been automatically generated and should be regarded as a
@@ -847,8 +853,9 @@ async def sample_count_tokens():
                 on the ``request`` instance; if ``request`` is provided, this
                 should not be set.
             contents (:class:`MutableSequence[google.ai.generativelanguage_v1.types.Content]`):
-                Required. The input given to the
-                model as a prompt.
+                Optional. The input given to the model as a prompt. This
+                field is ignored when ``generate_content_request`` is
+                set.
 
                 This corresponds to the ``contents`` field
                 on the ``request`` instance; if ``request`` is provided, this

packages/google-ai-generativelanguage/google/ai/generativelanguage_v1/services/generative_service/client.py

@@ -696,6 +696,12 @@ def generate_content(
         r"""Generates a response from the model given an input
         ``GenerateContentRequest``.
 
+        Input capabilities differ between models, including tuned
+        models. See the `model
+        guide <https://ai.google.dev/models/gemini>`__ and `tuning
+        guide <https://ai.google.dev/docs/model_tuning_guidance>`__ for
+        details.
+
         .. code-block:: python
 
             # This snippet has been automatically generated and should be regarded as a
@@ -1248,8 +1254,9 @@ def sample_count_tokens():
                 on the ``request`` instance; if ``request`` is provided, this
                 should not be set.
             contents (MutableSequence[google.ai.generativelanguage_v1.types.Content]):
-                Required. The input given to the
-                model as a prompt.
+                Optional. The input given to the model as a prompt. This
+                field is ignored when ``generate_content_request`` is
+                set.
 
                 This corresponds to the ``contents`` field
                 on the ``request`` instance; if ``request`` is provided, this

packages/google-ai-generativelanguage/google/ai/generativelanguage_v1/services/generative_service/transports/base.py

@@ -136,9 +136,9 @@ def _prep_wrapped_messages(self, client_info):
                     predicate=retries.if_exception_type(
                         core_exceptions.ServiceUnavailable,
                     ),
-                    deadline=60.0,
+                    deadline=600.0,
                 ),
-                default_timeout=60.0,
+                default_timeout=600.0,
                 client_info=client_info,
             ),
             self.stream_generate_content: gapic_v1.method.wrap_method(
@@ -150,9 +150,9 @@ def _prep_wrapped_messages(self, client_info):
                     predicate=retries.if_exception_type(
                         core_exceptions.ServiceUnavailable,
                     ),
-                    deadline=60.0,
+                    deadline=600.0,
                 ),
-                default_timeout=60.0,
+                default_timeout=600.0,
                 client_info=client_info,
             ),
             self.embed_content: gapic_v1.method.wrap_method(

packages/google-ai-generativelanguage/google/ai/generativelanguage_v1/services/generative_service/transports/grpc.py

@@ -247,6 +247,12 @@ def generate_content(
         Generates a response from the model given an input
         ``GenerateContentRequest``.
 
+        Input capabilities differ between models, including tuned
+        models. See the `model
+        guide <https://ai.google.dev/models/gemini>`__ and `tuning
+        guide <https://ai.google.dev/docs/model_tuning_guidance>`__ for
+        details.
+
         Returns:
             Callable[[~.GenerateContentRequest],
                     ~.GenerateContentResponse]:

packages/google-ai-generativelanguage/google/ai/generativelanguage_v1/services/generative_service/transports/grpc_asyncio.py

@@ -251,6 +251,12 @@ def generate_content(
         Generates a response from the model given an input
         ``GenerateContentRequest``.
 
+        Input capabilities differ between models, including tuned
+        models. See the `model
+        guide <https://ai.google.dev/models/gemini>`__ and `tuning
+        guide <https://ai.google.dev/docs/model_tuning_guidance>`__ for
+        details.
+
         Returns:
             Callable[[~.GenerateContentRequest],
                     Awaitable[~.GenerateContentResponse]]:
@@ -401,9 +407,9 @@ def _prep_wrapped_messages(self, client_info):
                     predicate=retries.if_exception_type(
                         core_exceptions.ServiceUnavailable,
                     ),
-                    deadline=60.0,
+                    deadline=600.0,
                 ),
-                default_timeout=60.0,
+                default_timeout=600.0,
                 client_info=client_info,
             ),
             self.stream_generate_content: gapic_v1.method_async.wrap_method(
@@ -415,9 +421,9 @@ def _prep_wrapped_messages(self, client_info):
                     predicate=retries.if_exception_type(
                         core_exceptions.ServiceUnavailable,
                     ),
-                    deadline=60.0,
+                    deadline=600.0,
                 ),
-                default_timeout=60.0,
+                default_timeout=600.0,
                 client_info=client_info,
             ),
             self.embed_content: gapic_v1.method_async.wrap_method(

packages/google-ai-generativelanguage/google/ai/generativelanguage_v1/types/content.py

@@ -109,10 +109,13 @@ class Blob(proto.Message):
 
     Attributes:
         mime_type (str):
-            The IANA standard MIME type of the source
-            data. Accepted types include: "image/png",
-            "image/jpeg", "image/heic", "image/heif",
-            "image/webp".
+            The IANA standard MIME type of the source data. Examples:
+
+            -  image/png
+            -  image/jpeg If an unsupported MIME type is provided, an
+               error will be returned. For a complete list of supported
+               types, see `Supported file
+               formats <https://ai.google.dev/gemini-api/docs/prompting_with_media#supported_file_formats>`__.
         data (bytes):
             Raw bytes for media formats.
     """

packages/google-ai-generativelanguage/google/ai/generativelanguage_v1/types/generative_service.py

@@ -273,6 +273,9 @@ class GenerateContentResponse(proto.Message):
         prompt_feedback (google.ai.generativelanguage_v1.types.GenerateContentResponse.PromptFeedback):
             Returns the prompt's feedback related to the
             content filters.
+        usage_metadata (google.ai.generativelanguage_v1.types.GenerateContentResponse.UsageMetadata):
+            Output only. Metadata on the generation
+            requests' token usage.
     """
 
     class PromptFeedback(proto.Message):
@@ -319,6 +322,33 @@ class BlockReason(proto.Enum):
             message=safety.SafetyRating,
         )
 
+    class UsageMetadata(proto.Message):
+        r"""Metadata on the generation request's token usage.
+
+        Attributes:
+            prompt_token_count (int):
+                Number of tokens in the prompt.
+            candidates_token_count (int):
+                Total number of tokens across the generated
+                candidates.
+            total_token_count (int):
+                Total token count for the generation request
+                (prompt + candidates).
+        """
+
+        prompt_token_count: int = proto.Field(
+            proto.INT32,
+            number=1,
+        )
+        candidates_token_count: int = proto.Field(
+            proto.INT32,
+            number=2,
+        )
+        total_token_count: int = proto.Field(
+            proto.INT32,
+            number=3,
+        )
+
     candidates: MutableSequence["Candidate"] = proto.RepeatedField(
         proto.MESSAGE,
         number=1,
@@ -329,6 +359,11 @@ class BlockReason(proto.Enum):
         number=2,
         message=PromptFeedback,
     )
+    usage_metadata: UsageMetadata = proto.Field(
+        proto.MESSAGE,
+        number=3,
+        message=UsageMetadata,
+    )
 
 
 class Candidate(proto.Message):
@@ -458,8 +493,9 @@ class EmbedContentRequest(proto.Message):
         output_dimensionality (int):
             Optional. Optional reduced dimension for the output
             embedding. If set, excessive values in the output embedding
-            are truncated from the end. Supported by
-            ``models/text-embedding-latest``.
+            are truncated from the end. Supported by newer models since
+            2024, and the earlier model (``models/embedding-001``)
+            cannot specify this value.
 
             This field is a member of `oneof`_ ``_output_dimensionality``.
     """
@@ -584,8 +620,12 @@ class CountTokensRequest(proto.Message):
 
             Format: ``models/{model}``
         contents (MutableSequence[google.ai.generativelanguage_v1.types.Content]):
-            Required. The input given to the model as a
-            prompt.
+            Optional. The input given to the model as a prompt. This
+            field is ignored when ``generate_content_request`` is set.
+        generate_content_request (google.ai.generativelanguage_v1.types.GenerateContentRequest):
+            Optional. The overall input given to the
+            model. CountTokens will count prompt, function
+            calling, etc.
     """
 
     model: str = proto.Field(
@@ -597,6 +637,11 @@ class CountTokensRequest(proto.Message):
         number=2,
         message=gag_content.Content,
     )
+    generate_content_request: "GenerateContentRequest" = proto.Field(
+        proto.MESSAGE,
+        number=3,
+        message="GenerateContentRequest",
+    )
 
 
 class CountTokensResponse(proto.Message):

packages/google-ai-generativelanguage/google/ai/generativelanguage_v1/types/safety.py

@@ -144,7 +144,7 @@ class SafetySetting(proto.Message):
     r"""Safety setting, affecting the safety-blocking behavior.
 
     Passing a safety setting for a category changes the allowed
-    proability that content is blocked.
+    probability that content is blocked.
 
     Attributes:
         category (google.ai.generativelanguage_v1.types.HarmCategory):

packages/google-ai-generativelanguage/google/ai/generativelanguage_v1beta/__init__.py

@@ -60,7 +60,7 @@
     Message,
     MessagePrompt,
 )
-from .types.file import File
+from .types.file import File, VideoMetadata
 from .types.file_service import (
     CreateFileRequest,
     CreateFileResponse,
@@ -325,4 +325,5 @@
     "UpdateDocumentRequest",
     "UpdatePermissionRequest",
     "UpdateTunedModelRequest",
+    "VideoMetadata",
 )