Skip to content

Commit 976b3b7

Browse files
ustcweizhouustcweizhou
committed
CLOUDSTACK-6928: fix issue disk I/O throttling not applied
1 parent 7a0b37a commit 976b3b7

5 files changed

Lines changed: 98 additions & 20 deletions

File tree

server/src/com/cloud/storage/StorageManager.java renamed to engine/components-api/src/com/cloud/storage/StorageManager.java

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,16 +27,20 @@
2727
import com.cloud.agent.api.Answer;
2828
import com.cloud.agent.api.Command;
2929
import com.cloud.agent.api.StoragePoolInfo;
30+
import com.cloud.agent.api.to.DataTO;
31+
import com.cloud.agent.api.to.DiskTO;
3032
import com.cloud.agent.manager.Commands;
3133
import com.cloud.capacity.CapacityVO;
3234
import com.cloud.exception.ConnectionException;
3335
import com.cloud.exception.StorageConflictException;
3436
import com.cloud.exception.StorageUnavailableException;
3537
import com.cloud.host.Host;
3638
import com.cloud.hypervisor.Hypervisor.HypervisorType;
37-
import com.cloud.service.ServiceOfferingVO;
39+
import com.cloud.offering.DiskOffering;
40+
import com.cloud.offering.ServiceOffering;
3841
import com.cloud.storage.Storage.ImageFormat;
3942
import com.cloud.utils.Pair;
43+
import com.cloud.vm.DiskProfile;
4044
import com.cloud.vm.VMInstanceVO;
4145

4246
public interface StorageManager extends StorageService {
@@ -116,13 +120,18 @@ public interface StorageManager extends StorageService {
116120

117121
BigDecimal getStorageOverProvisioningFactor(Long dcId);
118122

119-
Long getDiskBytesReadRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering);
123+
Long getDiskBytesReadRate(ServiceOffering offering, DiskOffering diskOffering);
120124

121-
Long getDiskBytesWriteRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering);
125+
Long getDiskBytesWriteRate(ServiceOffering offering, DiskOffering diskOffering);
122126

123-
Long getDiskIopsReadRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering);
127+
Long getDiskIopsReadRate(ServiceOffering offering, DiskOffering diskOffering);
124128

125-
Long getDiskIopsWriteRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering);
129+
Long getDiskIopsWriteRate(ServiceOffering offering, DiskOffering diskOffering);
126130

127131
void cleanupDownloadUrls();
132+
133+
void setDiskProfileThrottling(DiskProfile dskCh, ServiceOffering offering, DiskOffering diskOffering);
134+
135+
DiskTO getDiskWithThrottling(DataTO volTO, Volume.Type volumeType, long deviceId, String path, long offeringId, long diskOfferingId);
136+
128137
}

‎engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@
9494
import com.cloud.storage.Snapshot;
9595
import com.cloud.storage.Storage;
9696
import com.cloud.storage.Storage.ImageFormat;
97+
import com.cloud.storage.StorageManager;
9798
import com.cloud.storage.StoragePool;
9899
import com.cloud.storage.VMTemplateStorageResourceAssoc;
99100
import com.cloud.storage.Volume;
@@ -175,6 +176,8 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
175176
protected AsyncJobManager _jobMgr;
176177
@Inject
177178
ClusterManager clusterManager;
179+
@Inject
180+
StorageManager storageMgr;
178181

179182
private final StateMachine2<Volume.State, Volume.Event, Volume> _volStateMachine;
180183
protected List<StoragePoolAllocator> _storagePoolAllocators;
@@ -207,10 +210,11 @@ public VolumeInfo moveVolume(VolumeInfo volume, long destPoolDcId, Long destPool
207210

208211
// Find a destination storage pool with the specified criteria
209212
DiskOffering diskOffering = _entityMgr.findById(DiskOffering.class, volume.getDiskOfferingId());
210-
;
211213
DiskProfile dskCh = new DiskProfile(volume.getId(), volume.getVolumeType(), volume.getName(), diskOffering.getId(), diskOffering.getDiskSize(),
212214
diskOffering.getTagsArray(), diskOffering.getUseLocalStorage(), diskOffering.isRecreatable(), null);
213215
dskCh.setHyperType(dataDiskHyperType);
216+
storageMgr.setDiskProfileThrottling(dskCh, null, diskOffering);
217+
214218
DataCenter destPoolDataCenter = _entityMgr.findById(DataCenter.class, destPoolDcId);
215219
Pod destPoolPod = _entityMgr.findById(Pod.class, destPoolPodId);
216220

@@ -458,6 +462,8 @@ public VolumeInfo copyVolumeFromSecToPrimary(VolumeInfo volume, VirtualMachine v
458462
final HashSet<StoragePool> avoidPools = new HashSet<StoragePool>(avoids);
459463
DiskProfile dskCh = createDiskCharacteristics(volume, template, dc, diskOffering);
460464
dskCh.setHyperType(vm.getHypervisorType());
465+
storageMgr.setDiskProfileThrottling(dskCh, null, diskOffering);
466+
461467
// Find a suitable storage to create volume on
462468
StoragePool destPool = findStoragePool(dskCh, dc, pod, clusterId, null, vm, avoidPools);
463469
DataStore destStore = dataStoreMgr.getDataStore(destPool.getId(), DataStoreRole.Primary);
@@ -490,8 +496,10 @@ public VolumeInfo createVolume(VolumeInfo volume, VirtualMachine vm, VirtualMach
490496
DiskProfile dskCh = null;
491497
if (volume.getVolumeType() == Type.ROOT && Storage.ImageFormat.ISO != template.getFormat()) {
492498
dskCh = createDiskCharacteristics(volume, template, dc, offering);
499+
storageMgr.setDiskProfileThrottling(dskCh, offering, diskOffering);
493500
} else {
494501
dskCh = createDiskCharacteristics(volume, template, dc, diskOffering);
502+
storageMgr.setDiskProfileThrottling(dskCh, null, diskOffering);
495503
}
496504

497505
if (diskOffering != null && diskOffering.isCustomized()) {
@@ -1054,9 +1062,10 @@ public void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest
10541062
}
10551063

10561064
for (VolumeVO vol : vols) {
1057-
DataTO volTO = volFactory.getVolume(vol.getId()).getTO();
1058-
DiskTO disk = new DiskTO(volTO, vol.getDeviceId(), vol.getPath(), vol.getVolumeType());
10591065
VolumeInfo volumeInfo = volFactory.getVolume(vol.getId());
1066+
DataTO volTO = volumeInfo.getTO();
1067+
DiskTO disk = storageMgr.getDiskWithThrottling(volTO, vol.getVolumeType(), vol.getDeviceId(), vol.getPath(),
1068+
vm.getServiceOfferingId(), vol.getDiskOfferingId());
10601069
DataStore dataStore = dataStoreMgr.getDataStore(vol.getPoolId(), DataStoreRole.Primary);
10611070

10621071
disk.setDetails(getDetails(volumeInfo, dataStore));
@@ -1337,9 +1346,10 @@ public void prepare(VirtualMachineProfile vm, DeployDestination dest) throws Sto
13371346
pool = (StoragePool)dataStoreMgr.getDataStore(result.second().getId(), DataStoreRole.Primary);
13381347
vol = result.first();
13391348
}
1340-
DataTO volumeTO = volFactory.getVolume(vol.getId()).getTO();
1341-
DiskTO disk = new DiskTO(volumeTO, vol.getDeviceId(), vol.getPath(), vol.getVolumeType());
13421349
VolumeInfo volumeInfo = volFactory.getVolume(vol.getId());
1350+
DataTO volTO = volumeInfo.getTO();
1351+
DiskTO disk = storageMgr.getDiskWithThrottling(volTO, vol.getVolumeType(), vol.getDeviceId(), vol.getPath(),
1352+
vm.getServiceOfferingId(), vol.getDiskOfferingId());
13431353
DataStore dataStore = dataStoreMgr.getDataStore(vol.getPoolId(), DataStoreRole.Primary);
13441354

13451355
disk.setDetails(getDetails(volumeInfo, dataStore));

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -944,8 +944,8 @@ protected synchronized String attachOrDetachDevice(final Connect conn, final boo
944944
return null;
945945
}
946946

947-
protected synchronized String attachOrDetachDisk(final Connect conn, final boolean attach, final String vmName, final KVMPhysicalDisk attachingDisk, final int devId, final String serial) throws LibvirtException,
948-
InternalErrorException {
947+
protected synchronized String attachOrDetachDisk(final Connect conn, final boolean attach, final String vmName, final KVMPhysicalDisk attachingDisk, final int devId, final String serial,
948+
final Long bytesReadRate, final Long bytesWriteRate, final Long iopsReadRate, final Long iopsWriteRate) throws LibvirtException, InternalErrorException {
949949
List<DiskDef> disks = null;
950950
Domain dm = null;
951951
DiskDef diskdef = null;
@@ -1006,6 +1006,19 @@ protected synchronized String attachOrDetachDisk(final Connect conn, final boole
10061006
} else if (attachingDisk.getFormat() == PhysicalDiskFormat.RAW) {
10071007
diskdef.defBlockBasedDisk(attachingDisk.getPath(), devId, DiskDef.DiskBus.VIRTIO);
10081008
}
1009+
1010+
if ((bytesReadRate != null) && (bytesReadRate > 0)) {
1011+
diskdef.setBytesReadRate(bytesReadRate);
1012+
}
1013+
if ((bytesWriteRate != null) && (bytesWriteRate > 0)) {
1014+
diskdef.setBytesWriteRate(bytesWriteRate);
1015+
}
1016+
if ((iopsReadRate != null) && (iopsReadRate > 0)) {
1017+
diskdef.setIopsReadRate(iopsReadRate);
1018+
}
1019+
if ((iopsWriteRate != null) && (iopsWriteRate > 0)) {
1020+
diskdef.setIopsWriteRate(iopsWriteRate);
1021+
}
10091022
}
10101023

10111024
final String xml = diskdef.toString();
@@ -1031,7 +1044,7 @@ public Answer attachVolume(final AttachCommand cmd) {
10311044

10321045
final KVMPhysicalDisk phyDisk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), vol.getPath());
10331046

1034-
attachOrDetachDisk(conn, true, vmName, phyDisk, disk.getDiskSeq().intValue(), serial);
1047+
attachOrDetachDisk(conn, true, vmName, phyDisk, disk.getDiskSeq().intValue(), serial, vol.getBytesReadRate(), vol.getBytesWriteRate(), vol.getIopsReadRate(), vol.getIopsWriteRate());
10351048

10361049
return new AttachAnswer(disk);
10371050
} catch (final LibvirtException e) {
@@ -1056,7 +1069,7 @@ public Answer dettachVolume(final DettachCommand cmd) {
10561069

10571070
final KVMPhysicalDisk phyDisk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), vol.getPath());
10581071

1059-
attachOrDetachDisk(conn, false, vmName, phyDisk, disk.getDiskSeq().intValue(), serial);
1072+
attachOrDetachDisk(conn, false, vmName, phyDisk, disk.getDiskSeq().intValue(), serial, vol.getBytesReadRate(), vol.getBytesWriteRate(), vol.getIopsReadRate(), vol.getIopsWriteRate());
10601073

10611074
storagePoolMgr.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), vol.getPath());
10621075

‎server/src/com/cloud/storage/StorageManagerImpl.java‎

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -95,11 +95,14 @@
9595
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
9696
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
9797
import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
98+
import org.apache.cloudstack.storage.to.VolumeObjectTO;
9899

99100
import com.cloud.agent.AgentManager;
100101
import com.cloud.agent.api.Answer;
101102
import com.cloud.agent.api.Command;
102103
import com.cloud.agent.api.StoragePoolInfo;
104+
import com.cloud.agent.api.to.DataTO;
105+
import com.cloud.agent.api.to.DiskTO;
103106
import com.cloud.agent.manager.Commands;
104107
import com.cloud.api.ApiDBUtils;
105108
import com.cloud.api.query.dao.TemplateJoinDao;
@@ -138,13 +141,14 @@
138141
import com.cloud.host.dao.HostDao;
139142
import com.cloud.hypervisor.Hypervisor.HypervisorType;
140143
import com.cloud.hypervisor.HypervisorGuruManager;
144+
import com.cloud.offering.DiskOffering;
145+
import com.cloud.offering.ServiceOffering;
141146
import com.cloud.org.Grouping;
142147
import com.cloud.org.Grouping.AllocationState;
143148
import com.cloud.resource.ResourceState;
144149
import com.cloud.server.ConfigurationServer;
145150
import com.cloud.server.ManagementServer;
146151
import com.cloud.server.StatsCollector;
147-
import com.cloud.service.ServiceOfferingVO;
148152
import com.cloud.storage.Storage.ImageFormat;
149153
import com.cloud.storage.Storage.StoragePoolType;
150154
import com.cloud.storage.Volume.Type;
@@ -173,6 +177,7 @@
173177
import com.cloud.utils.component.ManagerBase;
174178
import com.cloud.utils.concurrency.NamedThreadFactory;
175179
import com.cloud.utils.db.DB;
180+
import com.cloud.utils.db.EntityManager;
176181
import com.cloud.utils.db.GenericSearchBuilder;
177182
import com.cloud.utils.db.GlobalLock;
178183
import com.cloud.utils.db.JoinBuilder;
@@ -185,6 +190,7 @@
185190
import com.cloud.utils.db.TransactionLegacy;
186191
import com.cloud.utils.db.TransactionStatus;
187192
import com.cloud.utils.exception.CloudRuntimeException;
193+
import com.cloud.vm.DiskProfile;
188194
import com.cloud.vm.VMInstanceVO;
189195
import com.cloud.vm.VirtualMachine.State;
190196
import com.cloud.vm.dao.VMInstanceDao;
@@ -280,6 +286,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
280286
private DiskOfferingDao _diskOfferingDao;
281287
@Inject
282288
ResourceLimitService _resourceLimitMgr;
289+
@Inject
290+
EntityManager _entityMgr;
283291

284292
protected List<StoragePoolDiscoverer> _discoverers;
285293

@@ -2237,7 +2245,7 @@ public void cleanupDownloadUrls(){
22372245

22382246
// get bytesReadRate from service_offering, disk_offering and vm.disk.throttling.bytes_read_rate
22392247
@Override
2240-
public Long getDiskBytesReadRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering) {
2248+
public Long getDiskBytesReadRate(final ServiceOffering offering, final DiskOffering diskOffering) {
22412249
if ((offering != null) && (offering.getBytesReadRate() != null) && (offering.getBytesReadRate() > 0)) {
22422250
return offering.getBytesReadRate();
22432251
} else if ((diskOffering != null) && (diskOffering.getBytesReadRate() != null) && (diskOffering.getBytesReadRate() > 0)) {
@@ -2253,7 +2261,7 @@ public Long getDiskBytesReadRate(ServiceOfferingVO offering, DiskOfferingVO disk
22532261

22542262
// get bytesWriteRate from service_offering, disk_offering and vm.disk.throttling.bytes_write_rate
22552263
@Override
2256-
public Long getDiskBytesWriteRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering) {
2264+
public Long getDiskBytesWriteRate(final ServiceOffering offering, final DiskOffering diskOffering) {
22572265
if ((offering != null) && (offering.getBytesWriteRate() != null) && (offering.getBytesWriteRate() > 0)) {
22582266
return offering.getBytesWriteRate();
22592267
} else if ((diskOffering != null) && (diskOffering.getBytesWriteRate() != null) && (diskOffering.getBytesWriteRate() > 0)) {
@@ -2269,7 +2277,7 @@ public Long getDiskBytesWriteRate(ServiceOfferingVO offering, DiskOfferingVO dis
22692277

22702278
// get iopsReadRate from service_offering, disk_offering and vm.disk.throttling.iops_read_rate
22712279
@Override
2272-
public Long getDiskIopsReadRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering) {
2280+
public Long getDiskIopsReadRate(final ServiceOffering offering, final DiskOffering diskOffering) {
22732281
if ((offering != null) && (offering.getIopsReadRate() != null) && (offering.getIopsReadRate() > 0)) {
22742282
return offering.getIopsReadRate();
22752283
} else if ((diskOffering != null) && (diskOffering.getIopsReadRate() != null) && (diskOffering.getIopsReadRate() > 0)) {
@@ -2285,7 +2293,7 @@ public Long getDiskIopsReadRate(ServiceOfferingVO offering, DiskOfferingVO diskO
22852293

22862294
// get iopsWriteRate from service_offering, disk_offering and vm.disk.throttling.iops_write_rate
22872295
@Override
2288-
public Long getDiskIopsWriteRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering) {
2296+
public Long getDiskIopsWriteRate(final ServiceOffering offering, final DiskOffering diskOffering) {
22892297
if ((offering != null) && (offering.getIopsWriteRate() != null) && (offering.getIopsWriteRate() > 0)) {
22902298
return offering.getIopsWriteRate();
22912299
} else if ((diskOffering != null) && (diskOffering.getIopsWriteRate() != null) && (diskOffering.getIopsWriteRate() > 0)) {
@@ -2308,4 +2316,39 @@ public String getConfigComponentName() {
23082316
public ConfigKey<?>[] getConfigKeys() {
23092317
return new ConfigKey<?>[] {StorageCleanupInterval, StorageCleanupDelay, StorageCleanupEnabled};
23102318
}
2319+
2320+
@Override
2321+
public void setDiskProfileThrottling(DiskProfile dskCh, final ServiceOffering offering, final DiskOffering diskOffering) {
2322+
dskCh.setBytesReadRate(getDiskBytesReadRate(offering, diskOffering));
2323+
dskCh.setBytesWriteRate(getDiskBytesWriteRate(offering, diskOffering));
2324+
dskCh.setIopsReadRate(getDiskIopsReadRate(offering, diskOffering));
2325+
dskCh.setIopsWriteRate(getDiskIopsWriteRate(offering, diskOffering));
2326+
}
2327+
2328+
@Override
2329+
public DiskTO getDiskWithThrottling(final DataTO volTO, final Volume.Type volumeType, final long deviceId, final String path, final long offeringId, final long diskOfferingId) {
2330+
DiskTO disk = null;
2331+
if (volTO != null && volTO instanceof VolumeObjectTO) {
2332+
VolumeObjectTO volumeTO = (VolumeObjectTO) volTO;
2333+
ServiceOffering offering = _entityMgr.findById(ServiceOffering.class, offeringId);
2334+
DiskOffering diskOffering = _entityMgr.findById(DiskOffering.class, diskOfferingId);
2335+
if (volumeType == Volume.Type.ROOT) {
2336+
setVolumeObjectTOThrottling(volumeTO, offering, diskOffering);
2337+
} else {
2338+
setVolumeObjectTOThrottling(volumeTO, null, diskOffering);
2339+
}
2340+
disk = new DiskTO(volumeTO, deviceId, path, volumeType);
2341+
} else {
2342+
disk = new DiskTO(volTO, deviceId, path, volumeType);
2343+
}
2344+
return disk;
2345+
}
2346+
2347+
private void setVolumeObjectTOThrottling(VolumeObjectTO volumeTO, final ServiceOffering offering, final DiskOffering diskOffering) {
2348+
volumeTO.setBytesReadRate(getDiskBytesReadRate(offering, diskOffering));
2349+
volumeTO.setBytesWriteRate(getDiskBytesWriteRate(offering, diskOffering));
2350+
volumeTO.setIopsReadRate(getDiskIopsReadRate(offering, diskOffering));
2351+
volumeTO.setIopsWriteRate(getDiskIopsWriteRate(offering, diskOffering));
2352+
}
2353+
23112354
}

‎server/src/com/cloud/storage/VolumeApiServiceImpl.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
251251
@Inject
252252
UserVmManager _userVmMgr;
253253
protected Gson _gson;
254+
@Inject
255+
StorageManager storageMgr;
254256

255257
private List<StoragePoolAllocator> _storagePoolAllocators;
256258

@@ -2479,7 +2481,8 @@ private VolumeVO sendAttachVolumeCommand(UserVmVO vm, VolumeVO volumeToAttach, L
24792481

24802482
deviceId = getDeviceId(vm.getId(), deviceId);
24812483

2482-
DiskTO disk = new DiskTO(volTO, deviceId, volumeToAttach.getPath(), volumeToAttach.getVolumeType());
2484+
DiskTO disk = storageMgr.getDiskWithThrottling(volTO, volumeToAttach.getVolumeType(), deviceId, volumeToAttach.getPath(),
2485+
vm.getServiceOfferingId(), volumeToAttach.getDiskOfferingId());
24832486

24842487
AttachCommand cmd = new AttachCommand(disk, vm.getInstanceName());
24852488

0 commit comments

Comments
 (0)