Skip to content

Commit 9a35f87

Browse files
sanjaytripathikaruturi
sanjaytripathi
authored and
karuturi
committed
CLOUDSTACK-8835: Added alerts incase of template download failure
Reviewed-By: Devdeep
1 parent 2d90f18 commit 9a35f87

5 files changed

Lines changed: 53 additions & 19 deletions

File tree

‎api/src/org/apache/cloudstack/alert/AlertService.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ private AlertType(short type, String name, boolean isDefault) {
6565
public static final AlertType ALERT_TYPE_LOCAL_STORAGE = new AlertType((short)25, "ALERT.STORAGE.LOCAL", true);
6666
public static final AlertType ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED = new AlertType((short)26, "ALERT.RESOURCE.EXCEED", true);
6767
public static final AlertType ALERT_TYPE_SYNC = new AlertType((short)27, "ALERT.TYPE.SYNC", true);
68+
public static final AlertType ALERT_TYPE_UPLOAD_FAILED = new AlertType((short)28, "ALERT.UPLOAD.FAILED", true);
6869

6970
public short getType() {
7071
return type;

‎engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,7 @@ public void handleTemplateSync(DataStore store) {
334334
String msg = "Template " + tmplt.getName() + ":" + tmplt.getId() + " is corrupted on secondary storage " + tmpltStore.getId();
335335
tmpltStore.setErrorString(msg);
336336
s_logger.info(msg);
337+
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPLOAD_FAILED, zoneId, null, msg, msg);
337338
if (tmplt.getState() == VirtualMachineTemplate.State.NotUploaded || tmplt.getState() == VirtualMachineTemplate.State.UploadInProgress) {
338339
s_logger.info("Template Sync found " + uniqueName + " on image store " + storeId + " uploaded using SSVM as corrupted, marking it as failed");
339340
tmpltStore.setState(State.Failed);

‎engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,12 @@
5050
import com.cloud.agent.api.storage.Proxy;
5151
import com.cloud.agent.api.to.DataObjectType;
5252
import com.cloud.agent.api.to.DataTO;
53+
import com.cloud.alert.AlertManager;
5354
import com.cloud.storage.VMTemplateStorageResourceAssoc;
5455
import com.cloud.storage.VMTemplateVO;
5556
import com.cloud.storage.VolumeVO;
5657
import com.cloud.storage.dao.VMTemplateDao;
58+
import com.cloud.storage.dao.VMTemplateZoneDao;
5759
import com.cloud.storage.dao.VolumeDao;
5860
import com.cloud.storage.download.DownloadMonitor;
5961

@@ -73,6 +75,10 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver {
7375
EndPointSelector _epSelector;
7476
@Inject
7577
ConfigurationDao configDao;
78+
@Inject
79+
VMTemplateZoneDao _vmTemplateZoneDao;
80+
@Inject
81+
AlertManager _alertMgr;
7682
protected String _proxy = null;
7783

7884
protected Proxy getHttpProxy() {
@@ -177,6 +183,9 @@ protected Void createTemplateAsyncCallback(AsyncCallbackDispatcher<? extends Bas
177183
result.setSuccess(false);
178184
result.setResult(answer.getErrorString());
179185
caller.complete(result);
186+
String msg = "Failed to register template: " + obj.getUuid() + " with error: " + answer.getErrorString();
187+
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPLOAD_FAILED, _vmTemplateZoneDao.listByTemplateId(obj.getId()).get(0).getZoneId(), null, msg, msg);
188+
s_logger.error(msg);
180189
} else if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) {
181190
if (answer.getCheckSum() != null) {
182191
VMTemplateVO templateDaoBuilder = _templateDao.createForUpdate();
@@ -229,6 +238,9 @@ protected Void createTemplateAsyncCallback(AsyncCallbackDispatcher<? extends Bas
229238
result.setSuccess(false);
230239
result.setResult(answer.getErrorString());
231240
caller.complete(result);
241+
String msg = "Failed to upload volume: " + obj.getUuid() + " with error: " + answer.getErrorString();
242+
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPLOAD_FAILED, volStoreVO.getZoneId(), null, msg, msg);
243+
s_logger.error(msg);
232244
} else if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) {
233245
CreateCmdResult result = new CreateCmdResult(null, null);
234246
caller.complete(result);

‎server/src/com/cloud/alert/AlertManagerImpl.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -759,7 +759,8 @@ public void sendAlert(AlertType alertType, long dataCenterId, Long podId, Long c
759759
(alertType != AlertManager.AlertType.ALERT_TYPE_SSVM) &&
760760
(alertType != AlertManager.AlertType.ALERT_TYPE_STORAGE_MISC) &&
761761
(alertType != AlertManager.AlertType.ALERT_TYPE_MANAGMENT_NODE) &&
762-
(alertType != AlertManager.AlertType.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED)) {
762+
(alertType != AlertManager.AlertType.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED) &&
763+
(alertType != AlertManager.AlertType.ALERT_TYPE_UPLOAD_FAILED)) {
763764
alert = _alertDao.getLastAlert(alertType.getType(), dataCenterId, podId, clusterId);
764765
}
765766

‎server/src/com/cloud/storage/ImageStoreUploadMonitorImpl.java‎

Lines changed: 37 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -54,12 +54,14 @@
5454
import com.cloud.agent.api.Answer;
5555
import com.cloud.agent.api.Command;
5656
import com.cloud.agent.api.StartupCommand;
57+
import com.cloud.alert.AlertManager;
5758
import com.cloud.exception.ConnectionException;
5859
import com.cloud.host.Host;
5960
import com.cloud.host.Status;
6061
import com.cloud.host.dao.HostDao;
6162
import com.cloud.storage.Volume.Event;
6263
import com.cloud.storage.dao.VMTemplateDao;
64+
import com.cloud.storage.dao.VMTemplateZoneDao;
6365
import com.cloud.storage.dao.VolumeDao;
6466
import com.cloud.template.VirtualMachineTemplate;
6567
import com.cloud.utils.component.ManagerBase;
@@ -96,6 +98,10 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto
9698
private DataStoreManager storeMgr;
9799
@Inject
98100
ResourceLimitService _resourceLimitMgr;
101+
@Inject
102+
private AlertManager _alertMgr;
103+
@Inject
104+
private VMTemplateZoneDao _vmTemplateZoneDao;
99105

100106
private long _nodeId;
101107
private ScheduledExecutorService _executor = null;
@@ -275,6 +281,8 @@ private void handleVolumeStatusResponse(final UploadStatusAnswer answer, final V
275281
public void doInTransactionWithoutResult(TransactionStatus status) {
276282
VolumeVO tmpVolume = _volumeDao.findById(volume.getId());
277283
VolumeDataStoreVO tmpVolumeDataStore = _volumeDataStoreDao.findById(volumeDataStore.getId());
284+
boolean sendAlert = false;
285+
String msg = null;
278286
try {
279287
switch (answer.getStatus()) {
280288
case COMPLETED:
@@ -305,9 +313,9 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
305313
tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
306314
tmpVolumeDataStore.setState(State.Failed);
307315
stateMachine.transitTo(tmpVolume, Event.OperationFailed, null, _volumeDao);
308-
if (s_logger.isDebugEnabled()) {
309-
s_logger.debug("Volume " + tmpVolume.getUuid() + " failed to upload due to operation timed out");
310-
}
316+
msg = "Volume " + tmpVolume.getUuid() + " failed to upload due to operation timed out";
317+
s_logger.error(msg);
318+
sendAlert = true;
311319
} else {
312320
tmpVolumeDataStore.setDownloadPercent(answer.getDownloadPercent());
313321
}
@@ -317,26 +325,30 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
317325
tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
318326
tmpVolumeDataStore.setState(State.Failed);
319327
stateMachine.transitTo(tmpVolume, Event.OperationFailed, null, _volumeDao);
320-
if (s_logger.isDebugEnabled()) {
321-
s_logger.debug("Volume " + tmpVolume.getUuid() + " failed to upload. Error details: " + answer.getDetails());
322-
}
328+
msg = "Volume " + tmpVolume.getUuid() + " failed to upload. Error details: " + answer.getDetails();
329+
s_logger.error(msg);
330+
sendAlert = true;
323331
break;
324332
case UNKNOWN:
325333
if (tmpVolume.getState() == Volume.State.NotUploaded) { // check for timeout
326334
if (System.currentTimeMillis() - tmpVolumeDataStore.getCreated().getTime() > _uploadOperationTimeout) {
327335
tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.ABANDONED);
328336
tmpVolumeDataStore.setState(State.Failed);
329337
stateMachine.transitTo(tmpVolume, Event.OperationTimeout, null, _volumeDao);
330-
if (s_logger.isDebugEnabled()) {
331-
s_logger.debug("Volume " + tmpVolume.getUuid() + " failed to upload due to operation timed out");
332-
}
338+
msg = "Volume " + tmpVolume.getUuid() + " failed to upload due to operation timed out";
339+
s_logger.error(msg);
340+
sendAlert = true;
333341
}
334342
}
335343
break;
336344
}
337345
_volumeDataStoreDao.update(tmpVolumeDataStore.getId(), tmpVolumeDataStore);
338346
} catch (NoTransitionException e) {
339347
s_logger.error("Unexpected error " + e.getMessage());
348+
} finally {
349+
if (sendAlert) {
350+
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPLOAD_FAILED, tmpVolume.getDataCenterId(), null, msg, msg);
351+
}
340352
}
341353
}
342354
});
@@ -349,6 +361,8 @@ private void handleTemplateStatusResponse(final UploadStatusAnswer answer, final
349361
public void doInTransactionWithoutResult(TransactionStatus status) {
350362
VMTemplateVO tmpTemplate = _templateDao.findById(template.getId());
351363
TemplateDataStoreVO tmpTemplateDataStore = _templateDataStoreDao.findById(templateDataStore.getId());
364+
boolean sendAlert = false;
365+
String msg = null;
352366
try {
353367
switch (answer.getStatus()) {
354368
case COMPLETED:
@@ -380,9 +394,9 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
380394
tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
381395
tmpTemplateDataStore.setState(State.Failed);
382396
stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao);
383-
if (s_logger.isDebugEnabled()) {
384-
s_logger.debug("Template " + tmpTemplate.getUuid() + " failed to upload due to operation timed out");
385-
}
397+
msg = "Template " + tmpTemplate.getUuid() + " failed to upload due to operation timed out";
398+
s_logger.error(msg);
399+
sendAlert = true;
386400
} else {
387401
tmpTemplateDataStore.setDownloadPercent(answer.getDownloadPercent());
388402
}
@@ -392,26 +406,31 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
392406
tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
393407
tmpTemplateDataStore.setState(State.Failed);
394408
stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao);
395-
if (s_logger.isDebugEnabled()) {
396-
s_logger.debug("Template " + tmpTemplate.getUuid() + " failed to upload. Error details: " + answer.getDetails());
397-
}
409+
msg = "Template " + tmpTemplate.getUuid() + " failed to upload. Error details: " + answer.getDetails();
410+
s_logger.error(msg);
411+
sendAlert = true;
398412
break;
399413
case UNKNOWN:
400414
if (tmpTemplate.getState() == VirtualMachineTemplate.State.NotUploaded) { // check for timeout
401415
if (System.currentTimeMillis() - tmpTemplateDataStore.getCreated().getTime() > _uploadOperationTimeout) {
402416
tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.ABANDONED);
403417
tmpTemplateDataStore.setState(State.Failed);
404418
stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationTimeout, null, _templateDao);
405-
if (s_logger.isDebugEnabled()) {
406-
s_logger.debug("Template " + tmpTemplate.getUuid() + " failed to upload due to operation timed out");
407-
}
419+
msg = "Template " + tmpTemplate.getUuid() + " failed to upload due to operation timed out";
420+
s_logger.error(msg);
421+
sendAlert = true;
408422
}
409423
}
410424
break;
411425
}
412426
_templateDataStoreDao.update(tmpTemplateDataStore.getId(), tmpTemplateDataStore);
413427
} catch (NoTransitionException e) {
414428
s_logger.error("Unexpected error " + e.getMessage());
429+
} finally {
430+
if (sendAlert) {
431+
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPLOAD_FAILED,
432+
_vmTemplateZoneDao.listByTemplateId(tmpTemplate.getId()).get(0).getZoneId(), null, msg, msg);
433+
}
415434
}
416435
}
417436
});

0 commit comments

Comments
 (0)