Skip to content

Commit 2b1e125

Browse files
committed
feat: wip on plugin system using hasicorp go-plugin
Signed-off-by: zachaller <zachaller@users.noreply.github.com>
1 parent 72ab46d commit 2b1e125

26 files changed

Lines changed: 1929 additions & 550 deletions

Makefile

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,3 +279,10 @@ trivy:
279279
.PHONY: checksums
280280
checksums:
281281
shasum -a 256 ./dist/kubectl-argo-rollouts-* | awk -F './dist/' '{print $$1 $$2}' > ./dist/argo-rollouts-checksums.txt
282+
283+
# Build sample plugin with debug info
284+
# https://www.jetbrains.com/help/go/attach-to-running-go-processes-with-debugger.html
285+
.PHONY: build-sample-plugin-debug
286+
build-sample-plugin-debug:
287+
go build -gcflags="all=-N -l" -o metrics-plugin cmd/sample-metrics-plugin/main.go
288+

cmd/rollouts-controller/main.go

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ import (
66
"strings"
77
"time"
88

9+
"github.com/argoproj/argo-rollouts/utils/plugin"
10+
11+
"github.com/argoproj/pkg/kubeclientmetrics"
912
smiclientset "github.com/servicemeshinterface/smi-sdk-go/pkg/gen/client/split/clientset/versioned"
1013
log "github.com/sirupsen/logrus"
1114
"github.com/spf13/cobra"
@@ -20,8 +23,6 @@ import (
2023
_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
2124
"k8s.io/client-go/tools/clientcmd"
2225

23-
"github.com/argoproj/pkg/kubeclientmetrics"
24-
2526
"github.com/argoproj/argo-rollouts/controller"
2627
"github.com/argoproj/argo-rollouts/controller/metrics"
2728
jobprovider "github.com/argoproj/argo-rollouts/metricproviders/job"
@@ -70,6 +71,8 @@ func newCommand() *cobra.Command {
7071
awsVerifyTargetGroup bool
7172
namespaced bool
7273
printVersion bool
74+
metricPluginLocation string
75+
metricPluginSha256 string
7376
)
7477
electOpts := controller.NewLeaderElectionOptions()
7578
var command = cobra.Command{
@@ -199,6 +202,12 @@ func newCommand() *cobra.Command {
199202
controllerNamespaceInformerFactory,
200203
jobInformerFactory)
201204

205+
defaults.SetMetricPluginLocation(metricPluginLocation)
206+
err = plugin.InitMetricsPlugin(metricPluginLocation, plugin.FileDownloaderImpl{}, metricPluginSha256)
207+
if err != nil {
208+
log.Fatalf("Failed to init metric plugin: %v", err)
209+
}
210+
202211
if err = cm.Run(ctx, rolloutThreads, serviceThreads, ingressThreads, experimentThreads, analysisThreads, electOpts); err != nil {
203212
log.Fatalf("Error running controller: %s", err.Error())
204213
}
@@ -240,6 +249,8 @@ func newCommand() *cobra.Command {
240249
command.Flags().DurationVar(&electOpts.LeaderElectionLeaseDuration, "leader-election-lease-duration", controller.DefaultLeaderElectionLeaseDuration, "The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.")
241250
command.Flags().DurationVar(&electOpts.LeaderElectionRenewDeadline, "leader-election-renew-deadline", controller.DefaultLeaderElectionRenewDeadline, "The interval between attempts by the acting master to renew a leadership slot before it stops leading. This must be less than or equal to the lease duration. This is only applicable if leader election is enabled.")
242251
command.Flags().DurationVar(&electOpts.LeaderElectionRetryPeriod, "leader-election-retry-period", controller.DefaultLeaderElectionRetryPeriod, "The duration the clients should wait between attempting acquisition and renewal of a leadership. This is only applicable if leader election is enabled.")
252+
command.Flags().StringVar(&metricPluginLocation, "metric-plugin-location", defaults.DefaultMetricsPluginLocation, "The file path to the location of the metric plugin binary")
253+
command.Flags().StringVar(&metricPluginSha256, "metric-plugin-sha256", "", "The expected sha256 of the metric plugin binary")
243254
return &command
244255
}
245256

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
package plugin
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"errors"
7+
"fmt"
8+
"net/url"
9+
"os"
10+
"time"
11+
12+
"github.com/argoproj/argo-rollouts/metricproviders/plugin"
13+
"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
14+
"github.com/argoproj/argo-rollouts/utils/evaluate"
15+
metricutil "github.com/argoproj/argo-rollouts/utils/metric"
16+
timeutil "github.com/argoproj/argo-rollouts/utils/time"
17+
"github.com/prometheus/client_golang/api"
18+
v1 "github.com/prometheus/client_golang/api/prometheus/v1"
19+
"github.com/prometheus/common/model"
20+
log "github.com/sirupsen/logrus"
21+
)
22+
23+
const EnvVarArgoRolloutsPrometheusAddress string = "ARGO_ROLLOUTS_PROMETHEUS_ADDRESS"
24+
25+
// Here is a real implementation of MetricsPlugin
26+
type RpcPlugin struct {
27+
LogCtx log.Entry
28+
api v1.API
29+
}
30+
31+
type Config struct {
32+
// Address is the HTTP address and port of the prometheus server
33+
Address string `json:"address,omitempty" protobuf:"bytes,1,opt,name=address"`
34+
// Query is a raw prometheus query to perform
35+
Query string `json:"query,omitempty" protobuf:"bytes,2,opt,name=query"`
36+
}
37+
38+
func (g *RpcPlugin) NewMetricsPlugin(metric v1alpha1.Metric) error {
39+
config := Config{}
40+
err := json.Unmarshal(metric.Provider.Plugin.Config, &config)
41+
if err != nil {
42+
return err
43+
}
44+
45+
api, err := newPrometheusAPI(config.Address)
46+
g.api = api
47+
48+
return err
49+
}
50+
51+
func (g *RpcPlugin) Run(anaysisRun *v1alpha1.AnalysisRun, metric v1alpha1.Metric) v1alpha1.Measurement {
52+
startTime := timeutil.MetaNow()
53+
newMeasurement := v1alpha1.Measurement{
54+
StartedAt: &startTime,
55+
}
56+
57+
config := Config{}
58+
json.Unmarshal(metric.Provider.Plugin.Config, &config)
59+
60+
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
61+
defer cancel()
62+
63+
response, warnings, err := g.api.Query(ctx, config.Query, time.Now())
64+
if err != nil {
65+
return metricutil.MarkMeasurementError(newMeasurement, err)
66+
}
67+
68+
newValue, newStatus, err := g.processResponse(metric, response)
69+
if err != nil {
70+
return metricutil.MarkMeasurementError(newMeasurement, err)
71+
72+
}
73+
newMeasurement.Value = newValue
74+
if len(warnings) > 0 {
75+
warningMetadata := ""
76+
for _, warning := range warnings {
77+
warningMetadata = fmt.Sprintf(`%s"%s", `, warningMetadata, warning)
78+
}
79+
warningMetadata = warningMetadata[:len(warningMetadata)-2]
80+
if warningMetadata != "" {
81+
newMeasurement.Metadata = map[string]string{"warnings": warningMetadata}
82+
g.LogCtx.Warnf("Prometheus returned the following warnings: %s", warningMetadata)
83+
}
84+
}
85+
86+
newMeasurement.Phase = newStatus
87+
finishedTime := timeutil.MetaNow()
88+
newMeasurement.FinishedAt = &finishedTime
89+
return newMeasurement
90+
}
91+
92+
func (g *RpcPlugin) Resume(analysisRun *v1alpha1.AnalysisRun, metric v1alpha1.Metric, measurement v1alpha1.Measurement) v1alpha1.Measurement {
93+
return measurement
94+
}
95+
96+
func (g *RpcPlugin) Terminate(analysisRun *v1alpha1.AnalysisRun, metric v1alpha1.Metric, measurement v1alpha1.Measurement) v1alpha1.Measurement {
97+
return measurement
98+
}
99+
100+
func (g *RpcPlugin) GarbageCollect(*v1alpha1.AnalysisRun, v1alpha1.Metric, int) error {
101+
return nil
102+
}
103+
104+
func (g *RpcPlugin) Type() string {
105+
return plugin.ProviderType
106+
}
107+
108+
func (g *RpcPlugin) GetMetadata(metric v1alpha1.Metric) map[string]string {
109+
metricsMetadata := make(map[string]string)
110+
111+
config := Config{}
112+
json.Unmarshal(metric.Provider.Plugin.Config, &config)
113+
if config.Query != "" {
114+
metricsMetadata["ResolvedPrometheusQuery"] = config.Query
115+
}
116+
return metricsMetadata
117+
}
118+
119+
func (g *RpcPlugin) processResponse(metric v1alpha1.Metric, response model.Value) (string, v1alpha1.AnalysisPhase, error) {
120+
switch value := response.(type) {
121+
case *model.Scalar:
122+
valueStr := value.Value.String()
123+
result := float64(value.Value)
124+
newStatus, err := evaluate.EvaluateResult(result, metric, g.LogCtx)
125+
return valueStr, newStatus, err
126+
case model.Vector:
127+
results := make([]float64, 0, len(value))
128+
valueStr := "["
129+
for _, s := range value {
130+
if s != nil {
131+
valueStr = valueStr + s.Value.String() + ","
132+
results = append(results, float64(s.Value))
133+
}
134+
}
135+
// if we appended to the string, we should remove the last comma on the string
136+
if len(valueStr) > 1 {
137+
valueStr = valueStr[:len(valueStr)-1]
138+
}
139+
valueStr = valueStr + "]"
140+
newStatus, err := evaluate.EvaluateResult(results, metric, g.LogCtx)
141+
return valueStr, newStatus, err
142+
default:
143+
return "", v1alpha1.AnalysisPhaseError, fmt.Errorf("Prometheus metric type not supported")
144+
}
145+
}
146+
147+
func newPrometheusAPI(address string) (v1.API, error) {
148+
envValuesByKey := make(map[string]string)
149+
if value, ok := os.LookupEnv(fmt.Sprintf("%s", EnvVarArgoRolloutsPrometheusAddress)); ok {
150+
envValuesByKey[EnvVarArgoRolloutsPrometheusAddress] = value
151+
log.Debugf("ARGO_ROLLOUTS_PROMETHEUS_ADDRESS: %v", envValuesByKey[EnvVarArgoRolloutsPrometheusAddress])
152+
}
153+
if len(address) != 0 {
154+
if !isUrl(address) {
155+
return nil, errors.New("prometheus address is not is url format")
156+
}
157+
} else if envValuesByKey[EnvVarArgoRolloutsPrometheusAddress] != "" {
158+
if isUrl(envValuesByKey[EnvVarArgoRolloutsPrometheusAddress]) {
159+
address = envValuesByKey[EnvVarArgoRolloutsPrometheusAddress]
160+
} else {
161+
return nil, errors.New("prometheus address is not is url format")
162+
}
163+
} else {
164+
return nil, errors.New("prometheus address is not configured")
165+
}
166+
client, err := api.NewClient(api.Config{
167+
Address: address,
168+
})
169+
if err != nil {
170+
log.Errorf("Error in getting prometheus client: %v", err)
171+
return nil, err
172+
}
173+
return v1.NewAPI(client), nil
174+
}
175+
176+
func isUrl(str string) bool {
177+
u, err := url.Parse(str)
178+
if err != nil {
179+
log.Errorf("Error in parsing url: %v", err)
180+
}
181+
log.Debugf("Parsed url: %v", u)
182+
return err == nil && u.Scheme != "" && u.Host != ""
183+
}
Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
package plugin
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"testing"
7+
"time"
8+
9+
rolloutsPlugin "github.com/argoproj/argo-rollouts/metricproviders/plugin"
10+
"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
11+
log "github.com/sirupsen/logrus"
12+
13+
goPlugin "github.com/hashicorp/go-plugin"
14+
)
15+
16+
var testHandshake = goPlugin.HandshakeConfig{
17+
ProtocolVersion: 1,
18+
MagicCookieKey: "ARGO_ROLLOUTS_RPC_PLUGIN",
19+
MagicCookieValue: "metrics",
20+
}
21+
22+
// This is just an example of how to test a plugin.
23+
func TestRunSuccessfully(t *testing.T) {
24+
//Skip test because this is just an example of how to test a plugin.
25+
t.Skip("Skipping test because it requires a running prometheus server")
26+
27+
ctx, cancel := context.WithCancel(context.Background())
28+
defer cancel()
29+
30+
logCtx := *log.WithFields(log.Fields{"plugin-test": "prometheus"})
31+
32+
rpcPluginImp := &RpcPlugin{
33+
LogCtx: logCtx,
34+
}
35+
36+
// pluginMap is the map of plugins we can dispense.
37+
var pluginMap = map[string]goPlugin.Plugin{
38+
"RpcMetricsPlugin": &rolloutsPlugin.RpcMetricsPlugin{Impl: rpcPluginImp},
39+
}
40+
41+
ch := make(chan *goPlugin.ReattachConfig, 1)
42+
closeCh := make(chan struct{})
43+
go goPlugin.Serve(&goPlugin.ServeConfig{
44+
HandshakeConfig: testHandshake,
45+
Plugins: pluginMap,
46+
Test: &goPlugin.ServeTestConfig{
47+
Context: ctx,
48+
ReattachConfigCh: ch,
49+
CloseCh: closeCh,
50+
},
51+
})
52+
53+
// We should get a config
54+
var config *goPlugin.ReattachConfig
55+
select {
56+
case config = <-ch:
57+
case <-time.After(2000 * time.Millisecond):
58+
t.Fatal("should've received reattach")
59+
}
60+
if config == nil {
61+
t.Fatal("config should not be nil")
62+
}
63+
64+
// Connect!
65+
c := goPlugin.NewClient(&goPlugin.ClientConfig{
66+
Cmd: nil,
67+
HandshakeConfig: testHandshake,
68+
Plugins: pluginMap,
69+
Reattach: config,
70+
})
71+
client, err := c.Client()
72+
if err != nil {
73+
t.Fatalf("err: %s", err)
74+
}
75+
76+
// Pinging should work
77+
if err := client.Ping(); err != nil {
78+
t.Fatalf("should not err: %s", err)
79+
}
80+
81+
// Kill which should do nothing
82+
c.Kill()
83+
if err := client.Ping(); err != nil {
84+
t.Fatalf("should not err: %s", err)
85+
}
86+
87+
// Request the plugin
88+
raw, err := client.Dispense("RpcMetricsPlugin")
89+
if err != nil {
90+
t.Fail()
91+
}
92+
93+
plugin := raw.(rolloutsPlugin.MetricsPlugin)
94+
95+
err = plugin.NewMetricsPlugin(v1alpha1.Metric{
96+
Provider: v1alpha1.MetricProvider{
97+
Plugin: &v1alpha1.PluginMetric{Config: json.RawMessage(`{"address":"http://prometheus.local", "query":"machine_cpu_cores"}`)},
98+
},
99+
})
100+
if err != nil {
101+
t.Fail()
102+
}
103+
104+
// Canceling should cause an exit
105+
cancel()
106+
<-closeCh
107+
}

cmd/sample-metrics-plugin/main.go

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package main
2+
3+
import (
4+
"github.com/argoproj/argo-rollouts/cmd/sample-metrics-plugin/internal/plugin"
5+
rolloutsPlugin "github.com/argoproj/argo-rollouts/metricproviders/plugin"
6+
goPlugin "github.com/hashicorp/go-plugin"
7+
log "github.com/sirupsen/logrus"
8+
)
9+
10+
// handshakeConfigs are used to just do a basic handshake between
11+
// a plugin and host. If the handshake fails, a user friendly error is shown.
12+
// This prevents users from executing bad plugins or executing a plugin
13+
// directory. It is a UX feature, not a security feature.
14+
var handshakeConfig = goPlugin.HandshakeConfig{
15+
ProtocolVersion: 1,
16+
MagicCookieKey: "ARGO_ROLLOUTS_RPC_PLUGIN",
17+
MagicCookieValue: "metrics",
18+
}
19+
20+
func main() {
21+
logCtx := *log.WithFields(log.Fields{"plugin": "prometheus"})
22+
23+
rpcPluginImp := &plugin.RpcPlugin{
24+
LogCtx: logCtx,
25+
}
26+
// pluginMap is the map of plugins we can dispense.
27+
var pluginMap = map[string]goPlugin.Plugin{
28+
"RpcMetricsPlugin": &rolloutsPlugin.RpcMetricsPlugin{Impl: rpcPluginImp},
29+
}
30+
31+
logCtx.Debug("message from plugin", "foo", "bar")
32+
33+
goPlugin.Serve(&goPlugin.ServeConfig{
34+
HandshakeConfig: handshakeConfig,
35+
Plugins: pluginMap,
36+
})
37+
}

0 commit comments

Comments
 (0)