Skip to content

Commit 5c9bc03

Browse files
authored
feat: metric plugin system based on hashicorp go-plugin (#2514)
Signed-off-by: zachaller <zachaller@users.noreply.github.com>
1 parent 93add01 commit 5c9bc03

47 files changed

Lines changed: 2536 additions & 525 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ junit.xml
1313
rerunreport.txt
1414
site/
1515
vendor/
16+
plugin-bin/
1617
# static
1718
server/static/*
1819
!server/static/.gitkeep

Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ CMD ["dashboard"]
8484
####################################################################################################
8585
# Final image
8686
####################################################################################################
87-
FROM scratch
87+
FROM gcr.io/distroless/static-debian11
8888

8989
COPY --from=argo-rollouts-build /go/src/github.com/argoproj/argo-rollouts/dist/rollouts-controller /bin/
9090
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/

Dockerfile.dev

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y \
88
apt-get clean && \
99
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
1010

11-
FROM scratch
11+
FROM gcr.io/distroless/static-debian11
1212

1313
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
1414
COPY rollouts-controller-linux-amd64 /bin/rollouts-controller

Makefile

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ lint: go-mod-vendor
209209

210210
.PHONY: test
211211
test: test-kustomize
212-
go test -covermode=count -coverprofile=coverage.out ${TEST_TARGET}
212+
@make test-unit
213213

214214
.PHONY: test-kustomize
215215
test-kustomize:
@@ -225,7 +225,7 @@ test-e2e: install-devtools-local
225225

226226
.PHONY: test-unit
227227
test-unit: install-devtools-local
228-
${DIST_DIR}/gotestsum --junitfile=junit.xml --format=testname --packages="./..." -- -covermode=count -coverprofile=coverage.out ./...
228+
${DIST_DIR}/gotestsum --junitfile=junit.xml --format=testname -- -covermode=count -coverprofile=coverage.out `go list ./... | grep -v ./test/cmd/sample-metrics-plugin`
229229

230230

231231
.PHONY: coverage
@@ -279,3 +279,10 @@ trivy:
279279
.PHONY: checksums
280280
checksums:
281281
shasum -a 256 ./dist/kubectl-argo-rollouts-* | awk -F './dist/' '{print $$1 $$2}' > ./dist/argo-rollouts-checksums.txt
282+
283+
# Build sample plugin with debug info
284+
# https://www.jetbrains.com/help/go/attach-to-running-go-processes-with-debugger.html
285+
.PHONY: build-sample-metric-plugin-debug
286+
build-sample-metric-plugin-debug:
287+
go build -gcflags="all=-N -l" -o metric-plugin test/cmd/sample-metrics-plugin/main.go
288+

analysis/analysis.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -327,7 +327,7 @@ func (c *Controller) runMeasurements(run *v1alpha1.AnalysisRun, tasks []metricTa
327327

328328
provider, err := c.newProvider(*logger, t.metric)
329329
if err != nil {
330-
log.Errorf("Error in getting provider :%v", err)
330+
log.Errorf("Error in getting metric provider :%v", err)
331331
return err
332332
}
333333
if metricResult == nil {

analysis/controller.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import (
55
"sync"
66
"time"
77

8+
"github.com/argoproj/argo-rollouts/metric"
9+
810
unstructuredutil "github.com/argoproj/argo-rollouts/utils/unstructured"
911

1012
log "github.com/sirupsen/logrus"
@@ -44,7 +46,7 @@ type Controller struct {
4446

4547
metricsServer *metrics.MetricsServer
4648

47-
newProvider func(logCtx log.Entry, metric v1alpha1.Metric) (metricproviders.Provider, error)
49+
newProvider func(logCtx log.Entry, metric v1alpha1.Metric) (metric.Provider, error)
4850

4951
// used for unit testing
5052
enqueueAnalysis func(obj interface{})

analysis/controller_test.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ import (
77
"testing"
88
"time"
99

10+
"github.com/argoproj/argo-rollouts/metric"
11+
1012
timeutil "github.com/argoproj/argo-rollouts/utils/time"
1113

1214
"github.com/argoproj/argo-rollouts/utils/queue"
@@ -25,7 +27,6 @@ import (
2527
"k8s.io/client-go/util/workqueue"
2628

2729
"github.com/argoproj/argo-rollouts/controller/metrics"
28-
"github.com/argoproj/argo-rollouts/metricproviders"
2930
"github.com/argoproj/argo-rollouts/metricproviders/mocks"
3031
"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
3132
"github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/fake"
@@ -129,7 +130,7 @@ func (f *fixture) newController(resync resyncFunc) (*Controller, informers.Share
129130
c.enqueueAnalysis(obj)
130131
}
131132
f.provider = &mocks.Provider{}
132-
c.newProvider = func(logCtx log.Entry, metric v1alpha1.Metric) (metricproviders.Provider, error) {
133+
c.newProvider = func(logCtx log.Entry, metric v1alpha1.Metric) (metric.Provider, error) {
133134
return f.provider, nil
134135
}
135136

cmd/rollouts-controller/main.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"strings"
77
"time"
88

9+
"github.com/argoproj/pkg/kubeclientmetrics"
910
smiclientset "github.com/servicemeshinterface/smi-sdk-go/pkg/gen/client/split/clientset/versioned"
1011
log "github.com/sirupsen/logrus"
1112
"github.com/spf13/cobra"
@@ -20,8 +21,6 @@ import (
2021
_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
2122
"k8s.io/client-go/tools/clientcmd"
2223

23-
"github.com/argoproj/pkg/kubeclientmetrics"
24-
2524
"github.com/argoproj/argo-rollouts/controller"
2625
"github.com/argoproj/argo-rollouts/controller/metrics"
2726
jobprovider "github.com/argoproj/argo-rollouts/metricproviders/job"

controller/controller.go

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,15 @@ import (
99
"sync"
1010
"time"
1111

12-
"k8s.io/apimachinery/pkg/util/wait"
12+
"github.com/argoproj/argo-rollouts/utils/plugin"
1313

1414
istioutil "github.com/argoproj/argo-rollouts/utils/istio"
15+
16+
rolloutsConfig "github.com/argoproj/argo-rollouts/utils/config"
17+
goPlugin "github.com/hashicorp/go-plugin"
18+
19+
"k8s.io/apimachinery/pkg/util/wait"
20+
1521
"k8s.io/client-go/dynamic/dynamicinformer"
1622
kubeinformers "k8s.io/client-go/informers"
1723

@@ -352,6 +358,16 @@ func NewManager(
352358
istioPrimaryDynamicClient: istioPrimaryDynamicClient,
353359
}
354360

361+
_, err := rolloutsConfig.InitializeConfig(kubeclientset, defaults.DefaultRolloutsConfigMapName)
362+
if err != nil {
363+
log.Fatalf("Failed to init config: %v", err)
364+
}
365+
366+
err = plugin.DownloadPlugins(plugin.FileDownloaderImpl{})
367+
if err != nil {
368+
log.Fatalf("Failed to download plugins: %v", err)
369+
}
370+
355371
return cm
356372
}
357373

@@ -423,6 +439,7 @@ func (c *Manager) Run(ctx context.Context, rolloutThreadiness, serviceThreadines
423439
})
424440
}
425441
log.Info("Shutting down workers")
442+
goPlugin.CleanupClients()
426443

427444
c.serviceWorkqueue.ShutDownWithDrain()
428445
c.ingressWorkqueue.ShutDownWithDrain()

docs/analysis/plugins.md

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# Metric Plugins
2+
3+
!!! important Available since v1.5
4+
5+
Argo Rollouts supports getting analysis metrics via 3rd party plugin system. This allows users to extend the capabilities of Rollouts
6+
to support metric providers that are not natively supported. Rollout's uses a plugin library called
7+
[go-plugin](https://github.com/hashicorp/go-plugin) to do this. You can find a sample plugin
8+
here: [sample-rollouts-metric-plugin](https://github.com/argoproj-labs/sample-rollouts-metric-plugin)
9+
10+
## Using a Metric Plugin
11+
12+
There are two methods of installing and using an argo rollouts plugin. The first method is to mount up the plugin executable
13+
into the rollouts controller container. The second method is to use a HTTP(S) server to host the plugin executable.
14+
15+
### Mounting the plugin executable into the rollouts controller container
16+
17+
To use this method, you will need to build or download the plugin executable and then mount it into the rollouts controller container.
18+
The plugin executable must be mounted into the rollouts controller container at the path specified by the `--metric-plugin-location` flag.
19+
20+
There are a few ways to mount the plugin executable into the rollouts controller container. Some of these will depend on your
21+
particular infrastructure. Here are a few methods:
22+
23+
* Using an init container to download the plugin executable
24+
* Using a Kubernetes volume mount with a shared volume such as NFS, EBS, etc.
25+
* Building the plugin into the rollouts controller container
26+
27+
Then you can use the configmap to point to the plugin executable. Example:
28+
29+
```yaml
30+
apiVersion: v1
31+
kind: ConfigMap
32+
metadata:
33+
name: argo-rollouts-config
34+
data:
35+
plugins: |-
36+
metrics:
37+
- name: "prometheus" # name the plugin uses to find this configuration, it must match the name required by the plugin
38+
pluginLocation: "file://./my-custom-plugin" # supports http(s):// urls and file://
39+
```
40+
41+
### Using a HTTP(S) server to host the plugin executable
42+
43+
Argo Rollouts supports downloading the plugin executable from a HTTP(S) server. To use this method, you will need to
44+
configure the controller via the `argo-rollouts-config` configmap and set `pluginLocation` to a http(s) url. Example:
45+
46+
```yaml
47+
apiVersion: v1
48+
kind: ConfigMap
49+
metadata:
50+
name: argo-rollouts-config
51+
data:
52+
plugins: |-
53+
metrics:
54+
- name: "prometheus" # name the plugin uses to find this configuration, it must match the name required by the plugin
55+
pluginLocation: "https://github.com/argoproj-labs/sample-rollouts-metric-plugin/releases/download/v0.0.3/metric-plugin-linux-amd64" # supports http(s):// urls and file://
56+
pluginSha256: "08f588b1c799a37bbe8d0fc74cc1b1492dd70b2c" #optional sha256 checksum of the plugin executable
57+
```
58+
59+
## Some words of caution
60+
61+
Depending on which method you use to install and the plugin, there are some things to be aware of.
62+
The rollouts controller will not start if it can not download or find the plugin executable. This means that if you are using
63+
a method of installation that requires a download of the plugin and the server hosting the plugin for some reason is not available and the rollouts
64+
controllers pod got deleted while the server was down or is coming up for the first time, it will not be able to start until
65+
the server hosting the plugin is available again.
66+
67+
Argo Rollouts will download the plugin at startup only once but if the pod is deleted it will need to download the plugin again on next startup. Running
68+
Argo Rollouts in HA mode can help a little with this situation because each pod will download the plugin at startup. So if a single pod gets
69+
deleted during a server outage, the other pods will still be able to take over because there will already be a plugin executable available to it. However,
70+
it is up to you to define your risk for and decide how you want to install the plugin executable.
71+
72+
## List of Available Plugins (alphabetical order)
73+
74+
#### Add Your Plugin Here
75+
* If you have created a plugin, please submit a PR to add it to this list.
76+
#### [sample-rollouts-metric-plugin](https://github.com/argoproj-labs/sample-rollouts-metric-plugin)
77+
* This is just a sample plugin that can be used as a starting point for creating your own plugin.
78+
It is not meant to be used in production. It is based on the built-in prometheus provider.

0 commit comments

Comments
 (0)