Skip to content

Commit 530a1d2

Browse files
gaius-qigaius-qi
authored
feat(scheduler): handle HitLocalCache in peer register & upgrade v2.2.32 (#4881)
* feat(scheduler): handle HitLocalCache in peer register & upgrade v2.2.32 - Add early-return path for `HitLocalCache` flag in peer register - Transition task/peer FSM states and send `HitLocalCacheResponse` directly without scheduling or triggering seed peer - Bump `d7y.io/api/v2` dependency from v2.2.30 to v2.2.32 Signed-off-by: Gaius <gaius.qi@gmail.com> * docs(scheduler): shorten hit local cache comment in service_v2 Simplifies the comment for the local cache hit case by removing details about HitLocalCacheResponse and peer reporting behavior. Signed-off-by: Gaius <gaius.qi@gmail.com> * fix(scheduler): replace HitLocalCache with MetadataOnly handling - Replace `HitLocalCache`/`HitLocalCacheResponse` with `MetadataOnly`/`MetadataOnlyResponse` in peer registration handling - Update log message to reflect new response type - Bump `google.golang.org/grpc` from v1.82.0 to v1.82.1 Signed-off-by: Gaius <gaius.qi@gmail.com> * build: upgrade node from 20 to 24 in manager console build - Bump Node.js base image from 20 to 24 (Alpine) in Dockerfile and build script - Add `yarn.lock` copy step to improve layer caching and ensure reproducible installs - Update manager/console submodule to latest commit Signed-off-by: Gaius <gaius.qi@gmail.com> --------- Signed-off-by: Gaius <gaius.qi@gmail.com>
1 parent 7aaad68 commit 530a1d2

6 files changed

Lines changed: 58 additions & 9 deletions

File tree

‎build/images/manager/Dockerfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
ARG BASE_IMAGE=alpine:3.23.4@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11
22

3-
FROM node:20-alpine@sha256:658d0f63e501824d6c23e06d4bb95c71e7d704537c9d9272f488ac03a370d448 AS console-builder
3+
FROM node:24-alpine@sha256:a0b9bf06e4e6193cf7a0f58816cc935ff8c2a908f81e6f1a95432d679c54fbfd AS console-builder
44

55
WORKDIR /build
66

77
COPY ./manager/console/package.json /build
8+
COPY ./manager/console/yarn.lock /build
89

910
RUN yarn install --frozen-lockfile --network-timeout 1000000
1011

‎go.mod‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module d7y.io/dragonfly/v2
33
go 1.25.5
44

55
require (
6-
d7y.io/api/v2 v2.2.30
6+
d7y.io/api/v2 v2.2.32
77
github.com/Showmax/go-fqdn v1.0.0
88
github.com/VividCortex/mysqlerr v1.0.0
99
github.com/appleboy/gin-jwt/v2 v2.10.3
@@ -71,7 +71,7 @@ require (
7171
golang.org/x/sync v0.22.0
7272
golang.org/x/sys v0.47.0
7373
google.golang.org/api v0.288.0
74-
google.golang.org/grpc v1.82.0
74+
google.golang.org/grpc v1.82.1
7575
google.golang.org/protobuf v1.36.11
7676
gopkg.in/natefinch/lumberjack.v2 v2.0.0
7777
gopkg.in/yaml.v3 v3.0.1

‎go.sum‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
5353
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
5454
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
5555
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
56-
d7y.io/api/v2 v2.2.30 h1:hUG7MmCbS/evGJvlBKClCehiolFufUv8DQcScwTaIFc=
57-
d7y.io/api/v2 v2.2.30/go.mod h1:q05190yfo1T5VR2/cK7RVQqIESPVA2VKRX4rS/7x9rc=
56+
d7y.io/api/v2 v2.2.32 h1:tGK/llLBoWdHVN8bwB1k7tXGw3FNUDv82XdhzAcXJWg=
57+
d7y.io/api/v2 v2.2.32/go.mod h1:8qiLrU0pd3yVZdjraORYLdVPD1was3AXp4dRyCo7N1Q=
5858
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
5959
filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw=
6060
filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
@@ -1293,8 +1293,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM
12931293
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
12941294
google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
12951295
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
1296-
google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU=
1297-
google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
1296+
google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE=
1297+
google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
12981298
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
12991299
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
13001300
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

‎hack/build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ build-manager-console() {
9292
CONSOLE_ASSETS=$CONSOLE_DIR/dist/*
9393
MANAGER_ASSETS_DIR=$MANAGER_DIR/dist
9494
docker run --workdir=/build \
95-
--rm -v ${CONSOLE_DIR}:/build node:20-alpine \
95+
--rm -v ${CONSOLE_DIR}:/build node:24-alpine \
9696
sh -c "yarn install --network-timeout 1000000 && yarn build"
9797
cp -r $CONSOLE_ASSETS $MANAGER_ASSETS_DIR
9898
}

‎manager/console‎

‎scheduler/service/service_v2.go‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1311,6 +1311,54 @@ func (v *V2) handleRegisterPeerRequest(ctx context.Context, stream schedulerv2.S
13111311
metrics.RegisterPeerCount.WithLabelValues(priority.String(), peer.Task.Type.String(),
13121312
peer.Host.Type.Name()).Inc()
13131313

1314+
// If the download hits the local cache of the peer completely, the peer only reports
1315+
// the metadata to the scheduler and no need to be scheduled or trigger the seed peer
1316+
// to download back-to-source.
1317+
if req.GetDownload().GetMetadataOnly() {
1318+
// Handle task with peer register request.
1319+
if !peer.Task.FSM.Is(standard.TaskStateRunning) {
1320+
if err := peer.Task.FSM.Event(ctx, standard.TaskEventDownload); err != nil {
1321+
// Collect RegisterPeerFailureCount metrics.
1322+
metrics.RegisterPeerFailureCount.WithLabelValues(priority.String(), peer.Task.Type.String(),
1323+
peer.Host.Type.Name()).Inc()
1324+
return status.Error(codes.Internal, err.Error())
1325+
}
1326+
} else {
1327+
peer.Task.UpdatedAt.Store(time.Now())
1328+
}
1329+
1330+
stream, loaded := peer.LoadAnnouncePeerStream()
1331+
if !loaded {
1332+
// Collect RegisterPeerFailureCount metrics.
1333+
metrics.RegisterPeerFailureCount.WithLabelValues(priority.String(), peer.Task.Type.String(),
1334+
peer.Host.Type.Name()).Inc()
1335+
return status.Error(codes.NotFound, "AnnouncePeerStream not found")
1336+
}
1337+
1338+
if err := peer.FSM.Event(ctx, standard.PeerEventRegisterNormal); err != nil {
1339+
// Collect RegisterPeerFailureCount metrics.
1340+
metrics.RegisterPeerFailureCount.WithLabelValues(priority.String(), peer.Task.Type.String(),
1341+
peer.Host.Type.Name()).Inc()
1342+
return status.Error(codes.Internal, err.Error())
1343+
}
1344+
1345+
peer.Log.Info("peer hits local cache, send MetadataOnlyResponse")
1346+
if err := stream.Send(&schedulerv2.AnnouncePeerResponse{
1347+
Response: &schedulerv2.AnnouncePeerResponse_MetadataOnlyResponse{
1348+
MetadataOnlyResponse: &schedulerv2.MetadataOnlyResponse{},
1349+
},
1350+
}); err != nil {
1351+
peer.Log.Error(err)
1352+
1353+
// Collect RegisterPeerFailureCount metrics.
1354+
metrics.RegisterPeerFailureCount.WithLabelValues(priority.String(), peer.Task.Type.String(),
1355+
peer.Host.Type.Name()).Inc()
1356+
return status.Error(codes.Internal, err.Error())
1357+
}
1358+
1359+
return nil
1360+
}
1361+
13141362
// Provides an exponential delay with jitter to prevent thundering herd problems. When a host has many
13151363
// concurrent registration requests, later requests are delayed progressively to avoid overwhelming the
13161364
// source with simultaneous back-to-source tasks from a single host.

0 commit comments

Comments
 (0)