Skip to content

Commit ff3c590

Browse files
Port Pekko ShardStopped handler + handoff safety net (#7500) (#8055)
Shards can fail to HandOff indefinitely during scale-up when the RebalanceWorker times out before receiving ShardStopped. The coordinator never deallocates the shard, causing an endless GetShardHome/ShardHome loop. - Add ShardStopped handler to ShardCoordinator.Active() (Pekko port): cleans up unAckedHostShards and performs late deallocation when no rebalance is in progress for the shard - ShardRegion sends backup ShardStopped to coordinator on handoff completion, ensuring the coordinator learns about it even when the RebalanceWorker has already timed out
1 parent 8a9e6f0 commit ff3c590

2 files changed

Lines changed: 31 additions & 0 deletions

File tree

src/contrib/cluster/Akka.Cluster.Sharding/ShardCoordinator.cs

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1790,6 +1790,31 @@ internal bool Active(object message)
17901790
}
17911791
return true;
17921792

1793+
case ShardStopped m:
1794+
// Ported from Pekko: clean up unAckedHostShards when shard reports stopped
1795+
if (_unAckedHostShards.TryGetValue(m.Shard, out var stopCancel))
1796+
{
1797+
stopCancel.Cancel();
1798+
_unAckedHostShards = _unAckedHostShards.Remove(m.Shard);
1799+
}
1800+
1801+
// Safety net: if no rebalance is in progress for this shard (RebalanceWorker
1802+
// already timed out), deallocate the shard so it can be reallocated elsewhere.
1803+
// This prevents the shard from being endlessly recreated via GetShardHome/ShardHome.
1804+
if (!_rebalanceInProgress.ContainsKey(m.Shard) && State.Shards.ContainsKey(m.Shard))
1805+
{
1806+
Log.Info("{0}: Shard [{1}] stopped - performing late deallocation (rebalance worker timed out).",
1807+
TypeName, m.Shard);
1808+
Update(new ShardHomeDeallocated(m.Shard), evt =>
1809+
{
1810+
State = State.Updated(evt);
1811+
Log.Debug("{0}: Shard [{1}] deallocated (late)", TypeName, m.Shard);
1812+
AllocateShardHomesForRememberEntities();
1813+
_context.Self.Tell(new GetShardHome(m.Shard), _ignoreRef);
1814+
});
1815+
}
1816+
return true;
1817+
17931818
case ResendShardHost m:
17941819
{
17951820
if (State.Shards.TryGetValue(m.Shard, out var region) && region.Equals(m.Region))

src/contrib/cluster/Akka.Cluster.Sharding/ShardRegion.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1410,6 +1410,12 @@ private void HandleTerminated(Terminated terminated)
14101410
{
14111411
_handingOff = _handingOff.Remove(terminated.ActorRef);
14121412
_log.Debug("{0}: Shard [{1}] handoff complete", _typeName, shard);
1413+
1414+
// Send backup ShardStopped to coordinator in case the RebalanceWorker
1415+
// has already timed out and missed the ShardStopped from HandOffStopper.
1416+
// The coordinator's Active handler will only deallocate if no rebalance
1417+
// is currently in progress for this shard.
1418+
_coordinator?.Tell(new ShardCoordinator.ShardStopped(shard));
14131419
}
14141420
else
14151421
{

0 commit comments

Comments
 (0)