switch to resourceproviders usages api for quotas

the quota api seems to be stricly rate limited, maybe not a good choice for this api

Signed-off-by: Markus Blaschke <mblaschke82@gmail.com>

Author: mblaschke

config/config_quota.go

@@ -4,6 +4,11 @@ type (
 	CollectorQuota struct {
 		*CollectorBase `yaml:",inline"`
 
-		ResourceProviders []string `json:"resourceProviders"`
+		ResourceProviders []CollectorQuotaResourceProvider `json:"resourceProviders"`
+	}
+
+	CollectorQuotaResourceProvider struct {
+		Provider   string `json:"provider"`
+		ApiVersion string `json:"apiVersion"`
 	}
 )

example.yaml

@@ -35,11 +35,11 @@ collectors:
     scrapeTime: 5m
 
     resourceProviders:
-      - Microsoft.App
-      - Microsoft.Compute
-      - Microsoft.Network
-      - Microsoft.Storage
-      - Microsoft.MachineLearningServices
+      - {provider: Microsoft.App, apiVersion: "2025-06-01"}
+      - {provider: Microsoft.Compute, apiVersion: "2025-04-01"}
+      - {provider: Microsoft.Network, apiVersion: "2025-05-01"}
+      - {provider: Microsoft.Storage, apiVersion: "2025-06-01"}
+      - {provider: Microsoft.MachineLearningServices, apiVersion: "2025-06-01"}
 
   # Azure Advisor recommendations
   advisor:

metrics_azurerm_quota.go

@@ -1,7 +1,9 @@
 package main
 
 import (
-	"context"
+	"errors"
+	"fmt"
+	"io"
 	"log/slog"
 	"net/http"
 	"net/url"
@@ -10,11 +12,13 @@ import (
 	armruntime "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm/runtime"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
-	armquota "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/quota/armquota/v2"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/webdevops/go-common/prometheus/collector"
 	"github.com/webdevops/go-common/utils/to"
+
+	"github.com/webdevops/azure-resourcemanager-exporter/config"
+	"github.com/webdevops/azure-resourcemanager-exporter/models/quota"
 )
 
 type MetricsCollectorAzureRmQuota struct {
@@ -105,10 +109,15 @@ func (m *MetricsCollectorAzureRmQuota) Collect(callback chan<- func()) {
 
 		if registered, err := AzureClient.IsResourceProviderRegistered(m.Context(), *subscription.SubscriptionID, "Microsoft.Capacity"); registered {
 			for _, provider := range Config.Collectors.Quota.ResourceProviders {
-				if registered, err := AzureClient.IsResourceProviderRegistered(m.Context(), *subscription.SubscriptionID, provider); registered {
-					m.collectQuotaUsage(subscription, provider, logger, callback)
+				if registered, err := AzureClient.IsResourceProviderRegistered(m.Context(), *subscription.SubscriptionID, provider.Provider); registered {
+
+					for _, location := range Config.Azure.Locations {
+						quotaLogger := logger.With(slog.String("provider", provider.Provider), slog.String("location", location))
+						m.collectQuotaUsage(subscription, provider, location, quotaLogger, callback)
+					}
+
 				} else if err != nil {
-					logger.Error("quota for resourceProvider requested, but not registered", slog.String("resourceProvider", provider), slog.Any("error", err))
+					logger.Error("quota for resourceProvider requested, but not registered", slog.String("resourceProvider", provider.Provider), slog.Any("error", err))
 				}
 			}
 		} else if err != nil {
@@ -138,15 +147,14 @@ func (m *MetricsCollectorAzureRmQuota) collectAuthorizationUsage(subscription *a
 	quotaLimitMetric := m.Collector.GetMetricList("quotaLimit")
 	quotaUsageMetric := m.Collector.GetMetricList("quotaUsage")
 
-	ctx := context.Background()
-
 	urlPath := "/subscriptions/{subscriptionId}/providers/Microsoft.Authorization/roleassignmentsusagemetrics"
 	urlPath = strings.ReplaceAll(urlPath, "{subscriptionId}", url.PathEscape(*subscription.SubscriptionID))
 
-	req, err := runtime.NewRequest(ctx, http.MethodGet, runtime.JoinPaths(ep, urlPath))
+	req, err := runtime.NewRequest(m.Context(), http.MethodGet, runtime.JoinPaths(ep, urlPath))
 	if err != nil {
 		panic(err)
 	}
+	defer req.Close()
 	reqQP := req.Raw().URL.Query()
 	reqQP.Set("api-version", "2019-08-01-preview")
 	req.Raw().URL.RawQuery = reqQP.Encode()
@@ -159,11 +167,7 @@ func (m *MetricsCollectorAzureRmQuota) collectAuthorizationUsage(subscription *a
 	defer resp.Body.Close()
 
 	if runtime.HasStatusCode(resp, http.StatusOK) {
-		result := struct {
-			RoleAssignmentsLimit          float64 `json:"roleAssignmentsLimit"`
-			RoleAssignmentsCurrentCount   float64 `json:"roleAssignmentsCurrentCount"`
-			RoleAssignmentsRemainingCount float64 `json:"roleAssignmentsRemainingCount"`
-		}{}
+		result := quota.RoleAssignmentUsage{}
 
 		if err := runtime.UnmarshalAsJSON(resp, &result); err == nil {
 			currentValue := result.RoleAssignmentsCurrentCount
@@ -187,121 +191,90 @@ func (m *MetricsCollectorAzureRmQuota) collectAuthorizationUsage(subscription *a
 	}
 }
 
-type (
-	Quota struct {
-		Labels  prometheus.Labels
-		Limit   *float64
-		Current *float64
-	}
-
-	QuotaList map[string]*Quota
-)
-
-func (q *QuotaList) Get(name string) *Quota {
-	name = strings.ToLower(name)
-	if _, ok := (*q)[name]; !ok {
-		(*q)[name] = &Quota{}
-	}
-
-	return (*q)[name]
-}
-
 // collectQuotaUsage collect generic quota usages
-func (m *MetricsCollectorAzureRmQuota) collectQuotaUsage(subscription *armsubscriptions.Subscription, provider string, logger *slog.Logger, callback chan<- func()) {
-	quotaClient, err := armquota.NewClient(AzureClient.GetCred(), AzureClient.NewArmClientOptions())
-	if err != nil {
-		panic(err)
-	}
-
-	usageClient, err := armquota.NewUsagesClient(AzureClient.GetCred(), AzureClient.NewArmClientOptions())
-	if err != nil {
-		panic(err)
-	}
-
+func (m *MetricsCollectorAzureRmQuota) collectQuotaUsage(subscription *armsubscriptions.Subscription, provider config.CollectorQuotaResourceProvider, location string, logger *slog.Logger, callback chan<- func()) {
 	quotaMetric := m.Collector.GetMetricList("quota")
 	quotaCurrentMetric := m.Collector.GetMetricList("quotaCurrent")
 	quotaLimitMetric := m.Collector.GetMetricList("quotaLimit")
 	quotaUsageMetric := m.Collector.GetMetricList("quotaUsage")
 
-	for _, location := range Config.Azure.Locations {
-		scope := "/subscriptions/{subscriptionId}/providers/{provider}/locations/{location}"
-		scope = strings.ReplaceAll(scope, "{subscriptionId}", url.PathEscape(*subscription.SubscriptionID))
-		scope = strings.ReplaceAll(scope, "{provider}", url.PathEscape(provider))
-		scope = strings.ReplaceAll(scope, "{location}", url.PathEscape(location))
-
-		quotaList := QuotaList{}
-
-		// -----------------------
-		// Quotas
-		quotaPager := quotaClient.NewListPager(scope, nil)
-		for quotaPager.More() {
-			result, err := quotaPager.NextPage(m.Context())
-			if err != nil {
-				panic(err)
-			}
+	options := AzureClient.NewArmClientOptions()
+	ep := cloud.AzurePublic.Services[cloud.ResourceManager].Endpoint
+	if c, ok := options.Cloud.Services[cloud.ResourceManager]; ok {
+		ep = c.Endpoint
+	}
 
-			if result.Value == nil {
-				continue
-			}
+	pl, err := armruntime.NewPipeline("azurerm-quota", gitTag, AzureClient.GetCred(), runtime.PipelineOptions{}, options)
+	if err != nil {
+		logger.Error("failed to create arm client", slog.Any("error", err))
+		panic(err)
+	}
 
-			for _, row := range result.Value {
-				switch v := row.Properties.Limit.(type) {
-				case *armquota.LimitObject:
-					if v.Value != nil {
-						quotaName := to.String(row.Name)
-
-						labels := prometheus.Labels{
-							"subscriptionID": to.StringLower(subscription.SubscriptionID),
-							"location":       strings.ToLower(location),
-							"provider":       provider,
-							"quota":          to.String(row.Properties.Name.Value),
-							"quotaName":      to.String(row.Properties.Name.LocalizedValue),
-						}
-
-						quotaList.Get(quotaName).Limit = to.Ptr(float64(to.Number(v.Value)))
-						quotaList.Get(quotaName).Labels = labels
-					}
-				}
-			}
+	urlPath := "/subscriptions/{subscriptionId}/providers/{provider}/locations/{location}/usages"
+	urlPath = strings.ReplaceAll(urlPath, "{subscriptionId}", url.PathEscape(*subscription.SubscriptionID))
+	urlPath = strings.ReplaceAll(urlPath, "{provider}", url.PathEscape(provider.Provider))
+	urlPath = strings.ReplaceAll(urlPath, "{location}", url.PathEscape(location))
+
+	requestUrl := runtime.JoinPaths(ep, urlPath)
+	fmt.Println(requestUrl)
+	for {
+		req, err := runtime.NewRequest(m.Context(), http.MethodGet, requestUrl)
+		if err != nil {
+			logger.Error("failed to create request", slog.Any("error", err))
+			panic(err)
+		}
+		defer req.Close()
+		reqQP := req.Raw().URL.Query()
+		reqQP.Set("api-version", provider.ApiVersion)
+		req.Raw().URL.RawQuery = reqQP.Encode()
+		req.Raw().Header["Accept"] = []string{"application/json"}
+
+		resp, err := pl.Do(req)
+		if err != nil {
+			logger.Error("failed to send request", slog.Any("error", err))
+			panic(err)
 		}
+		defer resp.Body.Close()
 
-		// -----------------------
-		// Usages
-		usagePager := usageClient.NewListPager(scope, nil)
-		for usagePager.More() {
-			result, err := usagePager.NextPage(m.Context())
+		if !runtime.HasStatusCode(resp, http.StatusOK) {
+			buf := new(strings.Builder)
+			_, err := io.Copy(buf, resp.Body)
 			if err != nil {
 				panic(err)
 			}
+			logger.Error("request failed", slog.String("statusCode", resp.Status), slog.Any("error", errors.New(buf.String())))
+			panic("request failed")
+		}
 
-			if result.Value == nil {
-				continue
-			}
-
-			for _, row := range result.Value {
-				quotaName := to.String(row.Name)
-				value := float64(to.Number(row.Properties.Usages.Value))
+		result := quota.ListUsageResult{}
+		if err := runtime.UnmarshalAsJSON(resp, &result); err == nil {
+			for _, quotaUsage := range result.Value {
 
 				labels := prometheus.Labels{
 					"subscriptionID": to.StringLower(subscription.SubscriptionID),
 					"location":       strings.ToLower(location),
-					"provider":       provider,
-					"quota":          to.String(row.Properties.Name.Value),
-					"quotaName":      to.String(row.Properties.Name.LocalizedValue),
+					"provider":       provider.Provider,
+					"quota":          to.String(quotaUsage.Name.Value),
+					"quotaName":      to.String(quotaUsage.Name.LocalizedValue),
 				}
 
-				quotaList.Get(quotaName).Current = to.Ptr(value)
-				quotaList.Get(quotaName).Labels = labels
+				quotaMetric.Add(labels, 1)
+				quotaCurrentMetric.AddIfNotNil(labels, quotaUsage.CurrentValue)
+				quotaLimitMetric.AddIfNotNil(labels, quotaUsage.Limit)
+				if quotaUsage.CurrentValue != nil && quotaUsage.Limit != nil && *quotaUsage.Limit != 0 {
+					quotaUsageMetric.Add(labels, *quotaUsage.CurrentValue / *quotaUsage.Limit)
+				}
 			}
+		} else {
+			logger.Error("failed to parse request", slog.Any("error", err))
+			panic("parsing failed")
 		}
 
-		for _, quota := range quotaList {
-			quotaMetric.Add(quota.Labels, 1)
-			quotaCurrentMetric.AddIfNotNil(quota.Labels, quota.Current)
-			quotaLimitMetric.AddIfNotNil(quota.Labels, quota.Limit)
-			if quota.Current != nil && quota.Limit != nil && *quota.Limit != 0 {
-				quotaUsageMetric.Add(quota.Labels, *quota.Current / *quota.Limit)
-			}
+		if result.NextLink != nil && *result.NextLink != "" {
+			requestUrl = *result.NextLink
+			continue
 		}
+
+		break
 	}
 }

models/quota/roleassignmentusages.go

@@ -0,0 +1,9 @@
+package quota
+
+type (
+	RoleAssignmentUsage struct {
+		RoleAssignmentsLimit          float64 `json:"roleAssignmentsLimit"`
+		RoleAssignmentsCurrentCount   float64 `json:"roleAssignmentsCurrentCount"`
+		RoleAssignmentsRemainingCount float64 `json:"roleAssignmentsRemainingCount"`
+	}
+)

models/quota/usages.go

@@ -0,0 +1,35 @@
+package quota
+
+type (
+	ListUsageResult struct {
+		// REQUIRED; The list of compute resource usages.
+		Value []*Usage
+
+		// The URI to fetch the next page of compute resource usage information. Call ListNext() with this to fetch the next page
+		// of compute resource usage information.
+		NextLink *string
+	}
+
+	Usage struct {
+		// REQUIRED; The current usage of the resource.
+		CurrentValue *float64
+
+		// REQUIRED; The maximum permitted usage of the resource.
+		Limit *float64
+
+		// REQUIRED; The name of the type of usage.
+		Name *UsageName
+
+		// REQUIRED; An enum describing the unit of usage measurement.
+		Unit *string
+	}
+
+	// UsageName - The Usage Names.
+	UsageName struct {
+		// The localized name of the resource.
+		LocalizedValue *string
+
+		// The name of the resource.
+		Value *string
+	}
+)