Fix protobuf 6.x compatibility: regenerate v6 with gencode floor 6.30.0 (#840)

The v6 stubs were generated with grpcio-tools 1.76.0, which embeds a gencode floor of 6.31.1. The ValidateProtobufRuntimeVersion check is patch-strict, so any user on protobuf 6.0-6.31.0 (all allowed by requirements.txt protobuf<7.0.0) failed at `import ydb` with a VersionError.

Regenerate v6 with grpcio-tools 1.72.1 (bundled protoc 30.0), lowering the floor to 6.30.0 -- the lowest existing 6.x release -- so the entire protobuf 6 major is covered. This also lowers the grpc-side floor (GRPC_GENERATED_VERSION) from 1.76.0 to 1.72.1, the minimum achievable for protoc 30.0.

Clean up the generation script: protoc is taken solely from the protoc bundled inside grpcio-tools, so drop the unused standalone protoc download, PROTOC_VER, PY_PROTOBUF_VER and grpclib from the Dockerfile; make GRPCIO_VER explicit per target in the Makefile, document the grpcio-tools -> protoc -> gencode mapping, and add the missing protobuf-6 to the aggregate `protobuf` target.

Author: vgvoleg

CHANGELOG.md

@@ -1,3 +1,5 @@
+* Fix incompatibility with protobuf 6.30–6.31.0: regenerate v6 stubs with the lowest 6.x gencode floor (6.30.0) instead of 6.31.1
+
 ## 3.29.4 ##
 * Fix leaked topic reader stream when close interrupts stream creation during reconnect
 * Include reconnector and read session ids in topic reader logs

Makefile

@@ -1,17 +1,29 @@
-protobuf: protobuf-3 protobuf-4 protobuf-5
+# protoc version (hence the generated gencode and, for protobuf >= 6, its embedded
+# runtime-version floor) is determined SOLELY by the grpcio-tools version pinned via
+# GRPCIO_VER below (see generate-protobuf.Dockerfile). Keep this table in sync:
+#
+#   v3: grpcio-tools 1.39.0 -> protoc 3.17 -> protobuf 3.x old-style  (runs on protobuf >= 3.13)
+#   v4: grpcio-tools 1.50.0 -> protoc 21.x -> protobuf 4.21 builder   (no runtime floor)
+#   v5: grpcio-tools 1.50.0 -> protoc 21.x -> protobuf 4.21 builder   (no runtime floor; serves 5.x)
+#   v6: grpcio-tools 1.72.1 -> protoc 30.0 -> protobuf 6.30.0 gencode (floor 6.30.0, the lowest 6.x)
+#
+# v5 deliberately uses the same no-floor 4.21 gencode as v4: it maximizes the range of
+# installed protobuf 5.x runtimes it tolerates. Raising it to a "true" 5.x gencode would
+# only add a runtime-version floor and strand users below it.
+protobuf: protobuf-3 protobuf-4 protobuf-5 protobuf-6
 
 protobuf-3:
-	docker build -f generate-protobuf.Dockerfile . -t ydb-python-sdk-proto-generator-env-3 --build-arg GRPCIO_VER=1.39.0 --build-arg PY_PROTOBUF_VER=3.20.3
+	docker build -f generate-protobuf.Dockerfile . -t ydb-python-sdk-proto-generator-env-3 --build-arg GRPCIO_VER=1.39.0
 	docker run --rm -it -v $${PWD}:$${PWD} -w $${PWD} ydb-python-sdk-proto-generator-env-3 python generate_protoc.py --target-version=v3
 
 protobuf-4:
-	docker build -f generate-protobuf.Dockerfile . -t ydb-python-sdk-proto-generator-env-4
+	docker build -f generate-protobuf.Dockerfile . -t ydb-python-sdk-proto-generator-env-4 --build-arg GRPCIO_VER=1.50.0
 	docker run --rm -it -v $${PWD}:$${PWD} -w $${PWD} ydb-python-sdk-proto-generator-env-4 python generate_protoc.py --target-version=v4
 
 protobuf-5:
-	docker build -f generate-protobuf.Dockerfile . -t ydb-python-sdk-proto-generator-env-5 --build-arg PY_PROTOBUF_VER=5.26.1 --build-arg PROTOC_VER=26.1
+	docker build -f generate-protobuf.Dockerfile . -t ydb-python-sdk-proto-generator-env-5 --build-arg GRPCIO_VER=1.50.0
 	docker run --rm -it -v $${PWD}:$${PWD} -w $${PWD} ydb-python-sdk-proto-generator-env-5 python generate_protoc.py --target-version=v5
 
 protobuf-6:
-	docker build -f generate-protobuf.Dockerfile . -t ydb-python-sdk-proto-generator-env-6 --build-arg GRPCIO_VER=1.76.0 --build-arg PY_PROTOBUF_VER=6.30.2 --build-arg PROTOC_VER=30.2
-	docker run --rm -it -v $${PWD}:$${PWD} -w $${PWD} ydb-python-sdk-proto-generator-env-6 python generate_protoc.py --target-version=v6
+	docker build -f generate-protobuf.Dockerfile . -t ydb-python-sdk-proto-generator-env-6 --build-arg GRPCIO_VER=1.72.1
+	docker run --rm -it -v $${PWD}:$${PWD} -w $${PWD} ydb-python-sdk-proto-generator-env-6 python generate_protoc.py --target-version=v6

generate-protobuf.Dockerfile

@@ -1,17 +1,15 @@
+# The *_pb2.py / *_pb2_grpc.py stubs are produced by the protoc that ships INSIDE
+# grpcio-tools (generate_protoc.py -> grpc_tools.command.build_package_protos uses the
+# bundled protoc, NOT a standalone one). Therefore GRPCIO_VER is the ONLY knob that
+# controls the generated gencode version and, for protobuf >= 6, its embedded
+# runtime-version floor. grpcio-tools pulls a compatible protobuf runtime transitively.
+#
+#   grpcio-tools 1.39.0 -> protoc 3.17 -> protobuf 3.x old-style gencode         (v3)
+#   grpcio-tools 1.50.0 -> protoc 21.x -> protobuf 4.21 builder gencode          (v4, v5)
+#   grpcio-tools 1.72.1 -> protoc 30.0 -> protobuf 6.30.0 gencode (floor 6.30.0) (v6)
 FROM python:3.9.15
-ENV GRPCLIB_VER=0.4.3
 ARG GRPCIO_VER=1.50.0
-ARG PY_PROTOBUF_VER=4.21.9
-RUN \
-    python -m pip install --upgrade pip && \
-    python -m pip install grpcio==${GRPCIO_VER} && \
-    python -m pip install grpclib==${GRPCLIB_VER} && \
-    python -m pip install grpcio-tools==${GRPCIO_VER} && \
-    python -m pip install protobuf==${PY_PROTOBUF_VER}
-
-ARG PROTOC_VER=21.8
-RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VER}/protoc-${PROTOC_VER}-linux-x86_64.zip && \
-    unzip protoc-*.zip && \
-    rm -f protoc-*.zip && \
-    mv bin/protoc /usr/local/bin/protoc && \
-    mv include well-known-protos
+RUN python -m pip install --upgrade pip && \
+    python -m pip install "grpcio-tools==${GRPCIO_VER}"
+# Surface the real protoc version that will generate the stubs (catches silent drift).
+RUN python -m grpc_tools.protoc --version

ydb/_grpc/v6/draft/protos/ydb_dynamic_config_pb2.py

@@ -4,7 +4,7 @@
 import warnings
 
 
-GRPC_GENERATED_VERSION = '1.76.0'
+GRPC_GENERATED_VERSION = '1.72.1'
 GRPC_VERSION = grpc.__version__
 _version_not_supported = False
 
@@ -17,7 +17,7 @@
 if _version_not_supported:
     raise RuntimeError(
         f'The grpc package installed is at version {GRPC_VERSION},'
-        + ' but the generated code in draft/protos/ydb_dynamic_config_pb2_grpc.py depends on'
+        + f' but the generated code in draft/protos/ydb_dynamic_config_pb2_grpc.py depends on'
         + f' grpcio>={GRPC_GENERATED_VERSION}.'
         + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}'
         + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.'

ydb/_grpc/v6/draft/protos/ydb_dynamic_config_pb2_grpc.py

@@ -1,5 +1,3 @@
-import datetime
-
 from protos.annotations import sensitive_pb2 as _sensitive_pb2
 from protos.annotations import validation_pb2 as _validation_pb2
 from protos import ydb_operation_pb2 as _ydb_operation_pb2
@@ -101,20 +99,20 @@ class Limits(_message.Message):
     result_ttl: _duration_pb2.Duration
     execution_timeout: _duration_pb2.Duration
     execution_deadline: _timestamp_pb2.Timestamp
-    def __init__(self, vcpu_rate_limit: _Optional[int] = ..., flow_rate_limit: _Optional[int] = ..., vcpu_time_limit: _Optional[int] = ..., max_result_size: _Optional[int] = ..., max_result_rows: _Optional[int] = ..., memory_limit: _Optional[int] = ..., result_ttl: _Optional[_Union[datetime.timedelta, _duration_pb2.Duration, _Mapping]] = ..., execution_timeout: _Optional[_Union[datetime.timedelta, _duration_pb2.Duration, _Mapping]] = ..., execution_deadline: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ...
+    def __init__(self, vcpu_rate_limit: _Optional[int] = ..., flow_rate_limit: _Optional[int] = ..., vcpu_time_limit: _Optional[int] = ..., max_result_size: _Optional[int] = ..., max_result_rows: _Optional[int] = ..., memory_limit: _Optional[int] = ..., result_ttl: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., execution_timeout: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., execution_deadline: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ...
 
 class StreamingDisposition(_message.Message):
     __slots__ = ("oldest", "fresh", "from_time", "time_ago", "from_last_checkpoint")
     class FromTime(_message.Message):
         __slots__ = ("timestamp",)
         TIMESTAMP_FIELD_NUMBER: _ClassVar[int]
         timestamp: _timestamp_pb2.Timestamp
-        def __init__(self, timestamp: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ...
+        def __init__(self, timestamp: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ...
     class TimeAgo(_message.Message):
         __slots__ = ("duration",)
         DURATION_FIELD_NUMBER: _ClassVar[int]
         duration: _duration_pb2.Duration
-        def __init__(self, duration: _Optional[_Union[datetime.timedelta, _duration_pb2.Duration, _Mapping]] = ...) -> None: ...
+        def __init__(self, duration: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ...) -> None: ...
     class FromLastCheckpoint(_message.Message):
         __slots__ = ("force",)
         FORCE_FIELD_NUMBER: _ClassVar[int]
@@ -200,7 +198,7 @@ class CommonMeta(_message.Message):
     created_at: _timestamp_pb2.Timestamp
     modified_at: _timestamp_pb2.Timestamp
     revision: int
-    def __init__(self, id: _Optional[str] = ..., created_by: _Optional[str] = ..., modified_by: _Optional[str] = ..., created_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., modified_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., revision: _Optional[int] = ...) -> None: ...
+    def __init__(self, id: _Optional[str] = ..., created_by: _Optional[str] = ..., modified_by: _Optional[str] = ..., created_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., modified_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., revision: _Optional[int] = ...) -> None: ...
 
 class QueryMeta(_message.Message):
     __slots__ = ("common", "submitted_at", "started_at", "finished_at", "execute_mode", "status", "last_job_query_revision", "last_job_id", "expire_at", "result_expire_at", "started_by", "aborted_by", "paused_by", "has_saved_checkpoints")
@@ -262,7 +260,7 @@ class QueryMeta(_message.Message):
     aborted_by: str
     paused_by: str
     has_saved_checkpoints: bool
-    def __init__(self, common: _Optional[_Union[CommonMeta, _Mapping]] = ..., submitted_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., started_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., finished_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., execute_mode: _Optional[_Union[ExecuteMode, str]] = ..., status: _Optional[_Union[QueryMeta.ComputeStatus, str]] = ..., last_job_query_revision: _Optional[int] = ..., last_job_id: _Optional[str] = ..., expire_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., result_expire_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., started_by: _Optional[str] = ..., aborted_by: _Optional[str] = ..., paused_by: _Optional[str] = ..., has_saved_checkpoints: bool = ...) -> None: ...
+    def __init__(self, common: _Optional[_Union[CommonMeta, _Mapping]] = ..., submitted_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., started_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., finished_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., execute_mode: _Optional[_Union[ExecuteMode, str]] = ..., status: _Optional[_Union[QueryMeta.ComputeStatus, str]] = ..., last_job_query_revision: _Optional[int] = ..., last_job_id: _Optional[str] = ..., expire_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., result_expire_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., started_by: _Optional[str] = ..., aborted_by: _Optional[str] = ..., paused_by: _Optional[str] = ..., has_saved_checkpoints: bool = ...) -> None: ...
 
 class BriefQuery(_message.Message):
     __slots__ = ("type", "name", "meta", "visibility", "automatic")
@@ -535,7 +533,7 @@ class BriefJob(_message.Message):
     visibility: Acl.Visibility
     automatic: bool
     expire_at: _timestamp_pb2.Timestamp
-    def __init__(self, meta: _Optional[_Union[CommonMeta, _Mapping]] = ..., query_meta: _Optional[_Union[QueryMeta, _Mapping]] = ..., query_name: _Optional[str] = ..., visibility: _Optional[_Union[Acl.Visibility, str]] = ..., automatic: bool = ..., expire_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ...
+    def __init__(self, meta: _Optional[_Union[CommonMeta, _Mapping]] = ..., query_meta: _Optional[_Union[QueryMeta, _Mapping]] = ..., query_name: _Optional[str] = ..., visibility: _Optional[_Union[Acl.Visibility, str]] = ..., automatic: bool = ..., expire_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ...
 
 class Job(_message.Message):
     __slots__ = ("meta", "text", "query_meta", "plan", "issue", "statistics", "result_set_meta", "ast", "query_name", "acl", "automatic", "expire_at", "syntax", "parameters")
@@ -574,7 +572,7 @@ class Job(_message.Message):
     expire_at: _timestamp_pb2.Timestamp
     syntax: QueryContent.QuerySyntax
     parameters: _containers.MessageMap[str, _ydb_value_pb2.TypedValue]
-    def __init__(self, meta: _Optional[_Union[CommonMeta, _Mapping]] = ..., text: _Optional[str] = ..., query_meta: _Optional[_Union[QueryMeta, _Mapping]] = ..., plan: _Optional[_Union[QueryPlan, _Mapping]] = ..., issue: _Optional[_Iterable[_Union[_ydb_issue_message_pb2.IssueMessage, _Mapping]]] = ..., statistics: _Optional[_Union[QueryStatistics, _Mapping]] = ..., result_set_meta: _Optional[_Iterable[_Union[ResultSetMeta, _Mapping]]] = ..., ast: _Optional[_Union[QueryAst, _Mapping]] = ..., query_name: _Optional[str] = ..., acl: _Optional[_Union[Acl, _Mapping]] = ..., automatic: bool = ..., expire_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., syntax: _Optional[_Union[QueryContent.QuerySyntax, str]] = ..., parameters: _Optional[_Mapping[str, _ydb_value_pb2.TypedValue]] = ...) -> None: ...
+    def __init__(self, meta: _Optional[_Union[CommonMeta, _Mapping]] = ..., text: _Optional[str] = ..., query_meta: _Optional[_Union[QueryMeta, _Mapping]] = ..., plan: _Optional[_Union[QueryPlan, _Mapping]] = ..., issue: _Optional[_Iterable[_Union[_ydb_issue_message_pb2.IssueMessage, _Mapping]]] = ..., statistics: _Optional[_Union[QueryStatistics, _Mapping]] = ..., result_set_meta: _Optional[_Iterable[_Union[ResultSetMeta, _Mapping]]] = ..., ast: _Optional[_Union[QueryAst, _Mapping]] = ..., query_name: _Optional[str] = ..., acl: _Optional[_Union[Acl, _Mapping]] = ..., automatic: bool = ..., expire_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., syntax: _Optional[_Union[QueryContent.QuerySyntax, str]] = ..., parameters: _Optional[_Mapping[str, _ydb_value_pb2.TypedValue]] = ...) -> None: ...
 
 class ListJobsRequest(_message.Message):
     __slots__ = ("operation_params", "page_token", "limit", "query_id", "filter")

ydb/_grpc/v6/draft/protos/ydb_federated_query_pb2.py

@@ -4,7 +4,7 @@
 import warnings
 
 
-GRPC_GENERATED_VERSION = '1.76.0'
+GRPC_GENERATED_VERSION = '1.72.1'
 GRPC_VERSION = grpc.__version__
 _version_not_supported = False
 
@@ -17,7 +17,7 @@
 if _version_not_supported:
     raise RuntimeError(
         f'The grpc package installed is at version {GRPC_VERSION},'
-        + ' but the generated code in draft/protos/ydb_federated_query_pb2_grpc.py depends on'
+        + f' but the generated code in draft/protos/ydb_federated_query_pb2_grpc.py depends on'
         + f' grpcio>={GRPC_GENERATED_VERSION}.'
         + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}'
         + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.'

ydb/_grpc/v6/draft/protos/ydb_federated_query_pb2.pyi

@@ -4,7 +4,7 @@
 import warnings
 
 
-GRPC_GENERATED_VERSION = '1.76.0'
+GRPC_GENERATED_VERSION = '1.72.1'
 GRPC_VERSION = grpc.__version__
 _version_not_supported = False
 
@@ -17,7 +17,7 @@
 if _version_not_supported:
     raise RuntimeError(
         f'The grpc package installed is at version {GRPC_VERSION},'
-        + ' but the generated code in draft/protos/ydb_keyvalue_pb2_grpc.py depends on'
+        + f' but the generated code in draft/protos/ydb_keyvalue_pb2_grpc.py depends on'
         + f' grpcio>={GRPC_GENERATED_VERSION}.'
         + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}'
         + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.'