Skip to content

Commit dcf435d

Browse files
mblaschkemblaschke
committed
add metric azurejanitor_deployment and refactoring metric collection
Signed-off-by: Markus Blaschke <mblaschke82@gmail.com>
1 parent 48b03a8 commit dcf435d

6 files changed

Lines changed: 138 additions & 132 deletions

File tree

‎janitor/deployments.go‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import (
1212
"go.uber.org/zap"
1313
)
1414

15-
func (j *Janitor) runDeployments(ctx context.Context, logger *zap.SugaredLogger, subscription *armsubscriptions.Subscription, ttlMetricsChan chan<- *prometheusCommon.MetricList) {
15+
func (j *Janitor) runDeployments(ctx context.Context, logger *zap.SugaredLogger, subscription *armsubscriptions.Subscription, callback chan<- func()) {
1616
var deploymentCounter, deploymentFinalCounter int64
1717
contextLogger := logger.With(zap.String("task", "deployment"))
1818

@@ -21,7 +21,7 @@ func (j *Janitor) runDeployments(ctx context.Context, logger *zap.SugaredLogger,
2121
logger.Panic(err)
2222
}
2323

24-
resourceTtl := prometheusCommon.NewMetricsList()
24+
deploymentMetric := prometheusCommon.NewMetricsList()
2525

2626
deploymentClient, err := armresources.NewDeploymentsClient(*subscription.SubscriptionID, j.Azure.Client.GetCred(), j.Azure.Client.NewArmClientOptions())
2727
if err != nil {
@@ -84,6 +84,11 @@ func (j *Janitor) runDeployments(ctx context.Context, logger *zap.SugaredLogger,
8484
}
8585
}
8686

87+
deploymentMetric.Add(prometheus.Labels{
88+
"subscriptionID": to.String(subscription.SubscriptionID),
89+
"resourceGroup": "",
90+
}, float64(deploymentFinalCounter))
91+
8792
contextLogger.Infof("found %v deployments on Subscription scope, %v still existing, %v deleted", deploymentCounter, deploymentFinalCounter, deploymentCounter-deploymentFinalCounter)
8893

8994
// -------------------------------------
@@ -151,9 +156,16 @@ func (j *Janitor) runDeployments(ctx context.Context, logger *zap.SugaredLogger,
151156
}
152157
}
153158

159+
deploymentMetric.Add(prometheus.Labels{
160+
"subscriptionID": to.String(subscription.SubscriptionID),
161+
"resourceGroup": to.String(resourceGroup.Name),
162+
}, float64(deploymentFinalCounter))
163+
154164
resourceLogger.Infof("found %v deployments on ResourceGroup scope, %v still existing, %v deleted", deploymentCounter, deploymentFinalCounter, deploymentCounter-deploymentFinalCounter)
155165
}
156166
}
157167

158-
ttlMetricsChan <- resourceTtl
168+
callback <- func() {
169+
deploymentMetric.GaugeSet(j.Prometheus.MetricDeployment)
170+
}
159171
}

‎janitor/janitor.go‎

Lines changed: 20 additions & 123 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import (
44
"context"
55
"fmt"
66
"strings"
7-
"sync"
87
"time"
98

109
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
@@ -13,7 +12,6 @@ import (
1312
"github.com/prometheus/client_golang/prometheus"
1413
"github.com/rickb777/date/period"
1514
"github.com/webdevops/go-common/azuresdk/armclient"
16-
prometheusCommon "github.com/webdevops/go-common/prometheus"
1715
"github.com/webdevops/go-common/utils/to"
1816
"go.uber.org/zap"
1917

@@ -37,6 +35,7 @@ type (
3735

3836
Prometheus struct {
3937
MetricDuration *prometheus.GaugeVec
38+
MetricDeployment *prometheus.GaugeVec
4039
MetricTtlResources *prometheus.GaugeVec
4140
MetricTtlRoleAssignments *prometheus.GaugeVec
4241
MetricDeletedResource *prometheus.CounterVec
@@ -83,80 +82,6 @@ func (j *Janitor) Init() {
8382
j.initAuzreApiVersions()
8483
}
8584

86-
func (j *Janitor) initPrometheus() {
87-
var err error
88-
j.Azure.ResourceTagManager, err = j.Azure.Client.TagManager.ParseTagConfig(j.Conf.Azure.ResourceTags)
89-
if err != nil {
90-
j.Logger.Fatal(`unable to parse resourceTag configuration "%s": %v"`, j.Conf.Azure.ResourceTags, err.Error())
91-
}
92-
93-
j.Prometheus.MetricDuration = prometheus.NewGaugeVec(
94-
prometheus.GaugeOpts{
95-
Name: "azurejanitor_duration",
96-
Help: "AzureJanitor cleanup duration",
97-
},
98-
[]string{},
99-
)
100-
prometheus.MustRegister(j.Prometheus.MetricDuration)
101-
102-
j.Prometheus.MetricTtlResources = prometheus.NewGaugeVec(
103-
prometheus.GaugeOpts{
104-
Name: "azurejanitor_resource_ttl",
105-
Help: "AzureJanitor resources with expiry time",
106-
},
107-
j.Azure.ResourceTagManager.AddToPrometheusLabels(
108-
[]string{
109-
"resourceID",
110-
"subscriptionID",
111-
"resourceGroup",
112-
"resourceType",
113-
},
114-
),
115-
)
116-
prometheus.MustRegister(j.Prometheus.MetricTtlResources)
117-
118-
j.Prometheus.MetricTtlRoleAssignments = prometheus.NewGaugeVec(
119-
prometheus.GaugeOpts{
120-
Name: "azurejanitor_roleassignment_ttl",
121-
Help: "AzureJanitor roleassignments with expiry time",
122-
},
123-
[]string{
124-
"roleAssignmentId",
125-
"scope",
126-
"principalId",
127-
"principalType",
128-
"roleDefinitionId",
129-
"subscriptionID",
130-
"resourceGroup",
131-
},
132-
)
133-
prometheus.MustRegister(j.Prometheus.MetricTtlRoleAssignments)
134-
135-
j.Prometheus.MetricDeletedResource = prometheus.NewCounterVec(
136-
prometheus.CounterOpts{
137-
Name: "azurejanitor_resource_deleted_count",
138-
Help: "AzureJanitor deleted resources",
139-
},
140-
[]string{
141-
"subscriptionID",
142-
"resourceType",
143-
},
144-
)
145-
prometheus.MustRegister(j.Prometheus.MetricDeletedResource)
146-
147-
j.Prometheus.MetricErrors = prometheus.NewCounterVec(
148-
prometheus.CounterOpts{
149-
Name: "azurejanitor_error_count",
150-
Help: "AzureJanitor error counter",
151-
},
152-
[]string{
153-
"subscriptionID",
154-
"resourceType",
155-
},
156-
)
157-
prometheus.MustRegister(j.Prometheus.MetricErrors)
158-
}
159-
16085
func (j *Janitor) Run() {
16186
ctx := context.Background()
16287

@@ -166,10 +91,8 @@ func (j *Janitor) Run() {
16691

16792
startTime := time.Now()
16893
runLogger.Infof("start janitor run")
169-
var wgMetrics sync.WaitGroup
17094

171-
callbackTtlResourcesMetrics := make(chan *prometheusCommon.MetricList)
172-
callbackTtlRoleAssignmentsMetrics := make(chan *prometheusCommon.MetricList)
95+
callbackFuncs := make(chan func())
17396

17497
// subscription processing
17598
go func() {
@@ -180,70 +103,44 @@ func (j *Janitor) Run() {
180103
)
181104

182105
if j.Conf.Janitor.Deployments.Enable {
183-
j.runDeployments(ctx, contextLogger, subscription, callbackTtlResourcesMetrics)
106+
j.runDeployments(ctx, contextLogger, subscription, callbackFuncs)
184107
}
185108

186109
if j.Conf.Janitor.Resources.Enable {
187-
j.runResources(ctx, contextLogger, subscription, j.Conf.Janitor.Resources.Filter, callbackTtlResourcesMetrics)
110+
j.runResources(ctx, contextLogger, subscription, j.Conf.Janitor.Resources.Filter, callbackFuncs)
188111
}
189112

190113
if j.Conf.Janitor.RoleAssignments.Enable {
191-
j.runRoleAssignments(ctx, contextLogger, subscription, j.Conf.Janitor.RoleAssignments.Filter, callbackTtlRoleAssignmentsMetrics)
114+
j.runRoleAssignments(ctx, contextLogger, subscription, j.Conf.Janitor.RoleAssignments.Filter, callbackFuncs)
192115
}
193116

194117
if j.Conf.Janitor.ResourceGroups.Enable {
195-
j.runResourceGroups(ctx, contextLogger, subscription, j.Conf.Janitor.ResourceGroups.Filter, callbackTtlResourcesMetrics)
118+
j.runResourceGroups(ctx, contextLogger, subscription, j.Conf.Janitor.ResourceGroups.Filter, callbackFuncs)
196119
}
197120
})
198121
if err != nil {
199122
runLogger.Panic(err)
200123
}
201124

202-
close(callbackTtlResourcesMetrics)
203-
close(callbackTtlRoleAssignmentsMetrics)
204-
}()
205-
206-
// channel collecting gofunc
207-
wgMetrics.Add(1)
208-
go func() {
209-
defer wgMetrics.Done()
210-
211-
// store metriclists from channel
212-
ttlMetricListList := []prometheusCommon.MetricList{}
213-
for ttlMetrics := range callbackTtlRoleAssignmentsMetrics {
214-
if ttlMetrics != nil {
215-
ttlMetricListList = append(ttlMetricListList, *ttlMetrics)
216-
}
217-
}
218-
219-
// after channel is closed: reset metric and set them to the new state
220-
j.Prometheus.MetricTtlRoleAssignments.Reset()
221-
for _, ttlMetrics := range ttlMetricListList {
222-
ttlMetrics.GaugeSet(j.Prometheus.MetricTtlRoleAssignments)
223-
}
125+
close(callbackFuncs)
224126
}()
225127

226-
wgMetrics.Add(1)
227-
go func() {
228-
defer wgMetrics.Done()
229-
230-
// store metriclists from channel
231-
ttlMetricListList := []prometheusCommon.MetricList{}
232-
for ttlMetrics := range callbackTtlResourcesMetrics {
233-
if ttlMetrics != nil {
234-
ttlMetricListList = append(ttlMetricListList, *ttlMetrics)
235-
}
128+
// store metriclists from channel
129+
callbackFuncList := []func(){}
130+
for callbackFunc := range callbackFuncs {
131+
if callbackFunc != nil {
132+
callbackFuncList = append(callbackFuncList, callbackFunc)
236133
}
134+
}
237135

238-
// after channel is closed: reset metric and set them to the new state
239-
j.Prometheus.MetricTtlResources.Reset()
240-
for _, ttlMetrics := range ttlMetricListList {
241-
ttlMetrics.GaugeSet(j.Prometheus.MetricTtlResources)
242-
}
243-
}()
136+
// after channel is closed: reset metric and set them to the new state
137+
j.Prometheus.MetricDeployment.Reset()
138+
j.Prometheus.MetricTtlResources.Reset()
139+
j.Prometheus.MetricTtlRoleAssignments.Reset()
244140

245-
// wait for metrics processing
246-
wgMetrics.Wait()
141+
for _, callbackFunc := range callbackFuncList {
142+
callbackFunc()
143+
}
247144

248145
duration := time.Since(startTime)
249146
j.Prometheus.MetricDuration.With(prometheus.Labels{}).Set(duration.Seconds())

‎janitor/metrics.go‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
package janitor
2+
3+
import (
4+
"github.com/prometheus/client_golang/prometheus"
5+
)
6+
7+
func (j *Janitor) initPrometheus() {
8+
var err error
9+
j.Azure.ResourceTagManager, err = j.Azure.Client.TagManager.ParseTagConfig(j.Conf.Azure.ResourceTags)
10+
if err != nil {
11+
j.Logger.Fatal(`unable to parse resourceTag configuration "%s": %v"`, j.Conf.Azure.ResourceTags, err.Error())
12+
}
13+
14+
j.Prometheus.MetricDuration = prometheus.NewGaugeVec(
15+
prometheus.GaugeOpts{
16+
Name: "azurejanitor_duration",
17+
Help: "AzureJanitor cleanup duration",
18+
},
19+
[]string{},
20+
)
21+
prometheus.MustRegister(j.Prometheus.MetricDuration)
22+
23+
j.Prometheus.MetricDeployment = prometheus.NewGaugeVec(
24+
prometheus.GaugeOpts{
25+
Name: "azurejanitor_deployment",
26+
Help: "AzureJanitor count of deployments on scope",
27+
},
28+
[]string{
29+
"subscriptionID",
30+
"resourceGroup",
31+
},
32+
)
33+
prometheus.MustRegister(j.Prometheus.MetricDeployment)
34+
35+
j.Prometheus.MetricTtlResources = prometheus.NewGaugeVec(
36+
prometheus.GaugeOpts{
37+
Name: "azurejanitor_resource_ttl",
38+
Help: "AzureJanitor resources with expiry time",
39+
},
40+
j.Azure.ResourceTagManager.AddToPrometheusLabels(
41+
[]string{
42+
"resourceID",
43+
"subscriptionID",
44+
"resourceGroup",
45+
"resourceType",
46+
},
47+
),
48+
)
49+
prometheus.MustRegister(j.Prometheus.MetricTtlResources)
50+
51+
j.Prometheus.MetricTtlRoleAssignments = prometheus.NewGaugeVec(
52+
prometheus.GaugeOpts{
53+
Name: "azurejanitor_roleassignment_ttl",
54+
Help: "AzureJanitor roleassignments with expiry time",
55+
},
56+
[]string{
57+
"roleAssignmentId",
58+
"scope",
59+
"principalId",
60+
"principalType",
61+
"roleDefinitionId",
62+
"subscriptionID",
63+
"resourceGroup",
64+
},
65+
)
66+
prometheus.MustRegister(j.Prometheus.MetricTtlRoleAssignments)
67+
68+
j.Prometheus.MetricDeletedResource = prometheus.NewCounterVec(
69+
prometheus.CounterOpts{
70+
Name: "azurejanitor_resource_deleted_count",
71+
Help: "AzureJanitor deleted resources",
72+
},
73+
[]string{
74+
"subscriptionID",
75+
"resourceType",
76+
},
77+
)
78+
prometheus.MustRegister(j.Prometheus.MetricDeletedResource)
79+
80+
j.Prometheus.MetricErrors = prometheus.NewCounterVec(
81+
prometheus.CounterOpts{
82+
Name: "azurejanitor_error_count",
83+
Help: "AzureJanitor error counter",
84+
},
85+
[]string{
86+
"subscriptionID",
87+
"resourceType",
88+
},
89+
)
90+
prometheus.MustRegister(j.Prometheus.MetricErrors)
91+
}

‎janitor/resourcegroups.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111
"go.uber.org/zap"
1212
)
1313

14-
func (j *Janitor) runResourceGroups(ctx context.Context, logger *zap.SugaredLogger, subscription *armsubscriptions.Subscription, filter string, ttlMetricsChan chan<- *prometheusCommon.MetricList) {
14+
func (j *Janitor) runResourceGroups(ctx context.Context, logger *zap.SugaredLogger, subscription *armsubscriptions.Subscription, filter string, callback chan<- func()) {
1515
contextLogger := logger.With(zap.String("task", "resourceGroup"))
1616
resourceType := "Microsoft.Resources/resourceGroups"
1717

@@ -90,5 +90,7 @@ func (j *Janitor) runResourceGroups(ctx context.Context, logger *zap.SugaredLogg
9090
}
9191
}
9292

93-
ttlMetricsChan <- resourceTtl
93+
callback <- func() {
94+
resourceTtl.GaugeSet(j.Prometheus.MetricTtlResources)
95+
}
9496
}

‎janitor/resources.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import (
1212
"go.uber.org/zap"
1313
)
1414

15-
func (j *Janitor) runResources(ctx context.Context, logger *zap.SugaredLogger, subscription *armsubscriptions.Subscription, filter string, ttlMetricsChan chan<- *prometheusCommon.MetricList) {
15+
func (j *Janitor) runResources(ctx context.Context, logger *zap.SugaredLogger, subscription *armsubscriptions.Subscription, filter string, callback chan<- func()) {
1616
contextLogger := logger.With(zap.String("task", "resource"))
1717

1818
client, err := armresources.NewClient(*subscription.SubscriptionID, j.Azure.Client.GetCred(), j.Azure.Client.NewArmClientOptions())
@@ -111,5 +111,7 @@ func (j *Janitor) runResources(ctx context.Context, logger *zap.SugaredLogger, s
111111
}
112112
}
113113

114-
ttlMetricsChan <- resourceTtl
114+
callback <- func() {
115+
resourceTtl.GaugeSet(j.Prometheus.MetricTtlResources)
116+
}
115117
}

‎janitor/roleassignments.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import (
1313
"go.uber.org/zap"
1414
)
1515

16-
func (j *Janitor) runRoleAssignments(ctx context.Context, logger *zap.SugaredLogger, subscription *armsubscriptions.Subscription, filter string, ttlMetricsChan chan<- *prometheusCommon.MetricList) {
16+
func (j *Janitor) runRoleAssignments(ctx context.Context, logger *zap.SugaredLogger, subscription *armsubscriptions.Subscription, filter string, callback chan<- func()) {
1717
contextLogger := logger.With(zap.String("task", "roleAssignment"))
1818

1919
resourceTtl := prometheusCommon.NewMetricsList()
@@ -116,5 +116,7 @@ func (j *Janitor) runRoleAssignments(ctx context.Context, logger *zap.SugaredLog
116116
}
117117
}
118118

119-
ttlMetricsChan <- resourceTtl
119+
callback <- func() {
120+
resourceTtl.GaugeSet(j.Prometheus.MetricTtlRoleAssignments)
121+
}
120122
}

0 commit comments

Comments
 (0)