Skip to content

Commit e6befb5

Browse files
bert-ebert-e
committed
Merge branch 'improvement/CLDSRV-972/call-isquotaenabled' into q/9.4
2 parents 752c888 + eeb84c5 commit e6befb5

2 files changed

Lines changed: 156 additions & 110 deletions

File tree

‎lib/utilities/monitoringHandler.js‎

Lines changed: 89 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -65,118 +65,112 @@ const httpResponseSizeBytes = new client.Summary({
6565
help: 'Cloudserver HTTP response sizes in bytes',
6666
});
6767

68-
let quotaEvaluationDuration;
69-
let utilizationMetricsRetrievalDuration;
70-
let utilizationServiceAvailable;
71-
let bucketsWithQuota;
72-
let accountsWithQuota;
73-
let requestWithQuotaMetricsUnavailable;
74-
75-
if (config.isQuotaEnabled) {
76-
quotaEvaluationDuration = new client.Histogram({
77-
name: 's3_cloudserver_quota_evaluation_duration_seconds',
78-
help: 'Duration of the quota evaluation operation',
79-
labelNames: ['action', 'code', 'type'],
80-
buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1],
81-
});
68+
const quotaEvaluationDuration = new client.Histogram({
69+
name: 's3_cloudserver_quota_evaluation_duration_seconds',
70+
help: 'Duration of the quota evaluation operation',
71+
labelNames: ['action', 'code', 'type'],
72+
buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1],
73+
});
8274

83-
utilizationMetricsRetrievalDuration = new client.Histogram({
84-
name: 's3_cloudserver_quota_metrics_retrieval_duration_seconds',
85-
help: 'Duration of the utilization metrics retrieval operation',
86-
labelNames: ['code', 'class'],
87-
buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5],
88-
});
75+
const utilizationMetricsRetrievalDuration = new client.Histogram({
76+
name: 's3_cloudserver_quota_metrics_retrieval_duration_seconds',
77+
help: 'Duration of the utilization metrics retrieval operation',
78+
labelNames: ['code', 'class'],
79+
buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5],
80+
});
8981

90-
utilizationServiceAvailable = new client.Gauge({
91-
name: 's3_cloudserver_quota_utilization_service_available',
92-
help: 'Availability of the utilization service',
93-
});
82+
const utilizationServiceAvailable = new client.Gauge({
83+
name: 's3_cloudserver_quota_utilization_service_available',
84+
help: 'Availability of the utilization service',
85+
});
9486

95-
bucketsWithQuota = new client.Gauge({
96-
name: 's3_cloudserver_quota_buckets_count',
97-
help: 'Total number of buckets quota',
98-
});
87+
const bucketsWithQuota = new client.Gauge({
88+
name: 's3_cloudserver_quota_buckets_count',
89+
help: 'Total number of buckets quota',
90+
});
9991

100-
accountsWithQuota = new client.Gauge({
101-
name: 's3_cloudserver_quota_accounts_count',
102-
help: 'Total number of account quota',
103-
});
92+
const accountsWithQuota = new client.Gauge({
93+
name: 's3_cloudserver_quota_accounts_count',
94+
help: 'Total number of account quota',
95+
});
10496

105-
requestWithQuotaMetricsUnavailable = new client.Counter({
106-
name: 's3_cloudserver_quota_unavailable_count',
107-
help: 'Total number of requests with quota metrics unavailable',
108-
});
109-
}
97+
const requestWithQuotaMetricsUnavailable = new client.Counter({
98+
name: 's3_cloudserver_quota_unavailable_count',
99+
help: 'Total number of requests with quota metrics unavailable',
100+
});
110101

111102
// Lifecycle duration metric, to track the completion of restore.
112103
// This metric is used to track the time it takes to complete the lifecycle operation (restore).
113104
// NOTE : this metric is the same as the one defined in Backbeat, and must keep the same name,
114105
// labels and buckets.
115106
const lifecycleDuration = new client.Histogram({
116107
name: 's3_lifecycle_duration_seconds',
117-
help: 'Duration of the lifecycle operation, calculated from the theoretical date to the end ' +
118-
'of the operation',
108+
help: 'Duration of the lifecycle operation, calculated from the theoretical date to the end ' + 'of the operation',
119109
labelNames: ['type', 'location'],
120110
buckets: [0.2, 1, 5, 30, 120, 600, 3600, 4 * 3600, 8 * 3600, 16 * 3600, 24 * 3600],
121111
});
122112

123-
function promMetrics(method, bucketName, code, action,
124-
newByteLength, oldByteLength, isVersionedObj,
125-
numOfObjectsRemoved, ingestSize) {
113+
function promMetrics(
114+
method,
115+
bucketName,
116+
code,
117+
action,
118+
newByteLength,
119+
oldByteLength,
120+
isVersionedObj,
121+
numOfObjectsRemoved,
122+
ingestSize,
123+
) {
126124
let bytes;
127125

128126
switch (action) {
129-
case 'putObject':
130-
case 'copyObject':
131-
case 'putObjectPart':
132-
if (code === '200') {
133-
bytes = newByteLength - (isVersionedObj ? 0 : oldByteLength);
134-
httpRequestSizeBytes
135-
.labels(method, action, code)
136-
.observe(newByteLength);
137-
dataDiskAvailable.dec(bytes);
138-
dataDiskFree.dec(bytes);
139-
if (ingestSize) {
140-
numberOfIngestedObjects.inc();
141-
dataIngested.inc(ingestSize);
127+
case 'putObject':
128+
case 'copyObject':
129+
case 'putObjectPart':
130+
if (code === '200') {
131+
bytes = newByteLength - (isVersionedObj ? 0 : oldByteLength);
132+
httpRequestSizeBytes.labels(method, action, code).observe(newByteLength);
133+
dataDiskAvailable.dec(bytes);
134+
dataDiskFree.dec(bytes);
135+
if (ingestSize) {
136+
numberOfIngestedObjects.inc();
137+
dataIngested.inc(ingestSize);
138+
}
139+
numberOfObjects.inc();
142140
}
143-
numberOfObjects.inc();
144-
}
145-
break;
146-
case 'createBucket':
147-
if (code === '200') {
148-
numberOfBuckets.inc();
149-
}
150-
break;
151-
case 'getObject':
152-
if (code === '200') {
153-
httpResponseSizeBytes
154-
.labels(method, action, code)
155-
.observe(newByteLength);
156-
}
157-
break;
158-
case 'deleteBucket':
159-
case 'deleteBucketWebsite':
160-
if (code === '200' || code === '204') {
161-
numberOfBuckets.dec();
162-
}
163-
break;
164-
case 'deleteObject':
165-
case 'abortMultipartUpload':
166-
case 'multiObjectDelete':
167-
if (code === '200') {
168-
dataDiskAvailable.inc(newByteLength);
169-
dataDiskFree.inc(newByteLength);
170-
const objs = numOfObjectsRemoved || 1;
171-
numberOfObjects.dec(objs);
172-
if (ingestSize) {
173-
numberOfIngestedObjects.dec(objs);
174-
dataIngested.dec(ingestSize);
141+
break;
142+
case 'createBucket':
143+
if (code === '200') {
144+
numberOfBuckets.inc();
175145
}
176-
}
177-
break;
178-
default:
179-
break;
146+
break;
147+
case 'getObject':
148+
if (code === '200') {
149+
httpResponseSizeBytes.labels(method, action, code).observe(newByteLength);
150+
}
151+
break;
152+
case 'deleteBucket':
153+
case 'deleteBucketWebsite':
154+
if (code === '200' || code === '204') {
155+
numberOfBuckets.dec();
156+
}
157+
break;
158+
case 'deleteObject':
159+
case 'abortMultipartUpload':
160+
case 'multiObjectDelete':
161+
if (code === '200') {
162+
dataDiskAvailable.inc(newByteLength);
163+
dataDiskFree.inc(newByteLength);
164+
const objs = numOfObjectsRemoved || 1;
165+
numberOfObjects.dec(objs);
166+
if (ingestSize) {
167+
numberOfIngestedObjects.dec(objs);
168+
dataIngested.dec(ingestSize);
169+
}
170+
}
171+
break;
172+
default:
173+
break;
180174
}
181175
}
182176

@@ -186,10 +180,10 @@ function crrCacheToProm(crrResults) {
186180
if (crrResults.getObjectCount) {
187181
numberOfBuckets.set(crrResults.getObjectCount.buckets || 0);
188182
numberOfObjects.set(crrResults.getObjectCount.objects || 0);
183+
bucketsWithQuota.set(crrResults.getObjectCount.bucketWithQuotaCount || 0);
189184
}
190-
if (config.isQuotaEnabled) {
191-
bucketsWithQuota.set(crrResults?.getObjectCount?.bucketWithQuotaCount || 0);
192-
accountsWithQuota.set(crrResults?.getVaultReport?.accountWithQuotaCount || 0);
185+
if (crrResults.getVaultReport) {
186+
accountsWithQuota.set(crrResults.getVaultReport.accountWithQuotaCount || 0);
193187
}
194188
if (crrResults.getDataDiskUsage) {
195189
dataDiskAvailable.set(crrResults.getDataDiskUsage.available || 0);
@@ -216,8 +210,7 @@ function writeResponse(res, error, results, cb) {
216210
}
217211

218212
const registry = config.isCluster ? new client.AggregatorRegistry() : client.register;
219-
const getMetrics = config.isCluster ?
220-
registry.clusterMetrics.bind(registry) : registry.metrics.bind(registry);
213+
const getMetrics = config.isCluster ? registry.clusterMetrics.bind(registry) : registry.metrics.bind(registry);
221214

222215
async function routeHandler(req, res, cb) {
223216
if (req.method !== 'GET') {

‎tests/unit/utils/monitoring.js‎

Lines changed: 67 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,17 @@ const promclient = require('prom-client');
33
const sinon = require('sinon');
44

55
const monitoring = require('../../../lib/utilities/monitoringHandler');
6+
const { config } = require('../../../lib/Config');
67

78
describe('Monitoring: endpoint', () => {
89
const sandbox = sinon.createSandbox();
910
const res = {
10-
writeHead(/* result, headers */) { return this; },
11-
write(/* body */) { return this; },
11+
writeHead(/* result, headers */) {
12+
return this;
13+
},
14+
write(/* body */) {
15+
return this;
16+
},
1217
end(/* body */) {},
1318
};
1419
monitoring.collectDefaultMetrics();
@@ -23,9 +28,20 @@ describe('Monitoring: endpoint', () => {
2328
});
2429

2530
async function fetchMetrics(req, res) {
26-
await new Promise(resolve => monitoring.monitoringHandler(null, req, {
27-
...res, end: (...body) => { res.end(...body); resolve(); }
28-
}, null));
31+
await new Promise(resolve =>
32+
monitoring.monitoringHandler(
33+
null,
34+
req,
35+
{
36+
...res,
37+
end: (...body) => {
38+
res.end(...body);
39+
resolve();
40+
},
41+
},
42+
null,
43+
),
44+
);
2945
}
3046

3147
it('should return an error is method is not GET', async () => {
@@ -80,29 +96,36 @@ describe('Monitoring: endpoint', () => {
8096
});
8197

8298
function parseMetric(metrics, name, labels) {
83-
const labelsString = Object.entries(labels).map(e => `${e[0]}="${e[1]}"`).join(',');
99+
const labelsString = Object.entries(labels)
100+
.map(e => `${e[0]}="${e[1]}"`)
101+
.join(',');
84102
const metric = metrics.match(new RegExp(`^${name}{${labelsString}} (.*)$`, 'm'));
85103
return metric ? metric[1] : null;
86104
}
87105

88106
function parseHttpRequestSize(metrics, action = 'putObject') {
89-
const value = parseMetric(metrics, 's3_cloudserver_http_request_size_bytes_sum',
90-
{ method: 'PUT', action, code: '200' });
107+
const value = parseMetric(metrics, 's3_cloudserver_http_request_size_bytes_sum', {
108+
method: 'PUT',
109+
action,
110+
code: '200',
111+
});
91112
return value ? parseInt(value, 10) : 0;
92113
}
93114

94115
function parseHttpResponseSize(metrics, action = 'getObject') {
95-
const value = parseMetric(metrics, 's3_cloudserver_http_response_size_bytes_sum',
96-
{ method: 'GET', action, code: '200' });
116+
const value = parseMetric(metrics, 's3_cloudserver_http_response_size_bytes_sum', {
117+
method: 'GET',
118+
action,
119+
code: '200',
120+
});
97121
return value ? parseInt(value, 10) : 0;
98122
}
99123

100124
it('should measure http requests size on putObject', async () => {
101125
await fetchMetrics({ method: 'GET', url: '/metrics' }, res);
102126
const requestSize = parseHttpRequestSize(res.end.args[0][0]);
103127

104-
monitoring.promMetrics('PUT', 'stuff', '200',
105-
'putObject', 2357, 3572, false, null, 5723);
128+
monitoring.promMetrics('PUT', 'stuff', '200', 'putObject', 2357, 3572, false, null, 5723);
106129

107130
await fetchMetrics({ method: 'GET', url: '/metrics' }, res);
108131
assert(parseHttpRequestSize(res.end.args[1][0]) === requestSize + 2357);
@@ -112,10 +135,40 @@ describe('Monitoring: endpoint', () => {
112135
await fetchMetrics({ method: 'GET', url: '/metrics' }, res);
113136
const responseSize = parseHttpResponseSize(res.end.args[0][0]);
114137

115-
monitoring.promMetrics('GET', 'stuff', '200',
116-
'getObject', 7532);
138+
monitoring.promMetrics('GET', 'stuff', '200', 'getObject', 7532);
117139

118140
await fetchMetrics({ method: 'GET', url: '/metrics' }, res);
119141
assert(parseHttpResponseSize(res.end.args[1][0]) === responseSize + 7532);
120142
});
143+
144+
const quotaMetricNames = [
145+
's3_cloudserver_quota_evaluation_duration_seconds',
146+
's3_cloudserver_quota_metrics_retrieval_duration_seconds',
147+
's3_cloudserver_quota_utilization_service_available',
148+
's3_cloudserver_quota_buckets_count',
149+
's3_cloudserver_quota_accounts_count',
150+
's3_cloudserver_quota_unavailable_count',
151+
];
152+
153+
it('should register quota metrics even when quota is disabled', async () => {
154+
assert.strictEqual(config.isQuotaEnabled(), false);
155+
156+
await fetchMetrics({ method: 'GET', url: '/metrics' }, res);
157+
const metrics = res.end.args[0][0];
158+
159+
quotaMetricNames.forEach(name => assert(metrics.includes(name), `${name} is not registered`));
160+
});
161+
162+
it('should report the quota counts from crrCacheToProm', async () => {
163+
monitoring.crrCacheToProm({
164+
getObjectCount: { buckets: 1, objects: 2, bucketWithQuotaCount: 3 },
165+
getVaultReport: { accountWithQuotaCount: 4 },
166+
});
167+
168+
await fetchMetrics({ method: 'GET', url: '/metrics' }, res);
169+
const metrics = res.end.args[0][0];
170+
171+
assert(metrics.includes('\ns3_cloudserver_quota_buckets_count 3'));
172+
assert(metrics.includes('\ns3_cloudserver_quota_accounts_count 4'));
173+
});
121174
});

0 commit comments

Comments
 (0)