diff --git a/cmd/operator/README.md b/cmd/operator/README.md index 57217700a..1c73fb966 100644 --- a/cmd/operator/README.md +++ b/cmd/operator/README.md @@ -98,6 +98,20 @@ NAME DESIRED READY NODE SELECTOR AGE gpudeviceplugin-sample 1 1 5s ``` +In order to limit the deployment to a specific device type, +use one of kustomizations under deployments/operator/device. + +For example, to limit the deployment to FPGA, use: + +```bash +$ kubectl apply -k deployments/operator/device/fpga +``` + +Operator also supports deployments with multiple selected device types. +In this case, create a new kustomization with the necessary resources +that passes the desired device types to the operator using `--device` +command line argument multiple times. + ## Known issues When the operator is run with leader election enabled, that is with the option @@ -106,3 +120,7 @@ number of pods. Otherwise a heart beat used by the leader election code may trig a timeout and crash. We are going to use different clients for the controller and leader election code to alleviate the issue. See more details in https://github.com/intel/intel-device-plugins-for-kubernetes/issues/476. + +In case the deployment is limited to specific device type(s), +the CRDs for other device types are still created, but no controllers +for them are registered. diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 7a7aee9aa..ea96b1923 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -16,7 +16,9 @@ package main import ( "flag" + "fmt" "os" + "strings" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -54,18 +56,58 @@ func init() { type devicePluginControllerAndWebhook map[string](func(ctrl.Manager, string, bool) error) +type flagList []string + +var supportedDevices = flagList{"dsa", "fpga", "gpu", "qat", "sgx"} +var devices flagList + +func (flag *flagList) String() string { + return strings.Join(*flag, ", ") +} + +func (flag *flagList) Set(value string) error { + if !contains(supportedDevices, value) { + setupLog.Error(nil, fmt.Sprintf("Unsupported device: %s", value)) + os.Exit(1) + } + + if contains(devices, value) { + setupLog.Error(nil, fmt.Sprintf("Duplicate device: %s", value)) + os.Exit(1) + } + + *flag = append(*flag, value) + return nil +} + +func contains(arr []string, val string) bool { + for _, s := range arr { + if s == val { + return true + } + } + return false +} + func main() { var metricsAddr string var devicePluginNamespace string var enableLeaderElection bool + var pm *patcher.PatcherManager + + ctrl.SetLogger(klogr.New()) + flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&devicePluginNamespace, "deviceplugin-namespace", metav1.NamespaceSystem, "The namespace where deviceplugin daemonsets are created") flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") + flag.Var(&devices, "devices", "Device(s) to set up.") flag.Parse() - ctrl.SetLogger(klogr.New()) + if len(devices) == 0 { + devices = supportedDevices + } setupControllerAndWebhook := devicePluginControllerAndWebhook{ "dsa": dsa.SetupReconciler, @@ -95,40 +137,43 @@ func main() { withWebhook := true - // TODO(mythi): add a StringVar flag to select which controllers to enable - for _, device := range []string{"dsa", "fpga", "gpu", "qat", "sgx"} { + for _, device := range devices { if err = setupControllerAndWebhook[device](mgr, ns, withWebhook); err != nil { setupLog.Error(err, "unable to initialize controller", "controller", device) os.Exit(1) } } - pm := patcher.NewPatcherManager(mgr.GetLogger().WithName("webhooks").WithName("Fpga")) + if contains(devices, "sgx") { + mgr.GetWebhookServer().Register("/pods-sgx", &webhook.Admission{ + Handler: &sgxwebhook.SgxMutator{Client: mgr.GetClient()}, + }) + } - mgr.GetWebhookServer().Register("/pods", &webhook.Admission{ - Handler: admission.HandlerFunc(pm.GetPodMutator()), - }) + if contains(devices, "fpga") { + pm = patcher.NewPatcherManager(mgr.GetLogger().WithName("webhooks").WithName("Fpga")) - mgr.GetWebhookServer().Register("/pods-sgx", &webhook.Admission{ - Handler: &sgxwebhook.SgxMutator{Client: mgr.GetClient()}, - }) + mgr.GetWebhookServer().Register("/pods", &webhook.Admission{ + Handler: admission.HandlerFunc(pm.GetPodMutator()), + }) - if err = (&fpgacontroller.AcceleratorFunctionReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - PatcherManager: pm, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "AcceleratorFunction") - os.Exit(1) - } + if err = (&fpgacontroller.AcceleratorFunctionReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + PatcherManager: pm, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "AcceleratorFunction") + os.Exit(1) + } - if err = (&fpgacontroller.FpgaRegionReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - PatcherManager: pm, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "FpgaRegion") - os.Exit(1) + if err = (&fpgacontroller.FpgaRegionReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + PatcherManager: pm, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "FpgaRegion") + os.Exit(1) + } } setupLog.Info("starting manager") diff --git a/deployments/operator/device/dsa/dsa.yaml b/deployments/operator/device/dsa/dsa.yaml new file mode 100644 index 000000000..ca93f359b --- /dev/null +++ b/deployments/operator/device/dsa/dsa.yaml @@ -0,0 +1,14 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inteldeviceplugins-controller-manager + namespace: inteldeviceplugins-system +spec: + template: + spec: + containers: + - args: + - --metrics-addr=127.0.0.1:8080 + - --enable-leader-election + - --device=dsa + name: manager diff --git a/deployments/operator/device/dsa/kustomization.yaml b/deployments/operator/device/dsa/kustomization.yaml new file mode 100644 index 000000000..3a605359c --- /dev/null +++ b/deployments/operator/device/dsa/kustomization.yaml @@ -0,0 +1,5 @@ +bases: + - ../../default + +patchesStrategicMerge: + - dsa.yaml diff --git a/deployments/operator/device/fpga/fpga.yaml b/deployments/operator/device/fpga/fpga.yaml new file mode 100644 index 000000000..9c99d4e2c --- /dev/null +++ b/deployments/operator/device/fpga/fpga.yaml @@ -0,0 +1,14 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inteldeviceplugins-controller-manager + namespace: inteldeviceplugins-system +spec: + template: + spec: + containers: + - args: + - --metrics-addr=127.0.0.1:8080 + - --enable-leader-election + - --device=fpga + name: manager diff --git a/deployments/operator/device/fpga/kustomization.yaml b/deployments/operator/device/fpga/kustomization.yaml new file mode 100644 index 000000000..cea26f2ac --- /dev/null +++ b/deployments/operator/device/fpga/kustomization.yaml @@ -0,0 +1,5 @@ +bases: + - ../../default + +patchesStrategicMerge: + - fpga.yaml diff --git a/deployments/operator/device/gpu/gpu.yaml b/deployments/operator/device/gpu/gpu.yaml new file mode 100644 index 000000000..0af00ec76 --- /dev/null +++ b/deployments/operator/device/gpu/gpu.yaml @@ -0,0 +1,14 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inteldeviceplugins-controller-manager + namespace: inteldeviceplugins-system +spec: + template: + spec: + containers: + - args: + - --metrics-addr=127.0.0.1:8080 + - --enable-leader-election + - --device=gpu + name: manager diff --git a/deployments/operator/device/gpu/kustomization.yaml b/deployments/operator/device/gpu/kustomization.yaml new file mode 100644 index 000000000..19793e950 --- /dev/null +++ b/deployments/operator/device/gpu/kustomization.yaml @@ -0,0 +1,5 @@ +bases: + - ../../default + +patchesStrategicMerge: + - gpu.yaml diff --git a/deployments/operator/device/qat/kustomization.yaml b/deployments/operator/device/qat/kustomization.yaml new file mode 100644 index 000000000..74e100f9a --- /dev/null +++ b/deployments/operator/device/qat/kustomization.yaml @@ -0,0 +1,5 @@ +bases: + - ../../default + +patchesStrategicMerge: + - qat.yaml diff --git a/deployments/operator/device/qat/qat.yaml b/deployments/operator/device/qat/qat.yaml new file mode 100644 index 000000000..bf0e23bb3 --- /dev/null +++ b/deployments/operator/device/qat/qat.yaml @@ -0,0 +1,14 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inteldeviceplugins-controller-manager + namespace: inteldeviceplugins-system +spec: + template: + spec: + containers: + - args: + - --metrics-addr=127.0.0.1:8080 + - --enable-leader-election + - --device=qat + name: manager diff --git a/deployments/operator/device/sgx/kustomization.yaml b/deployments/operator/device/sgx/kustomization.yaml new file mode 100644 index 000000000..c99b56ab4 --- /dev/null +++ b/deployments/operator/device/sgx/kustomization.yaml @@ -0,0 +1,5 @@ +bases: + - ../../default + +patchesStrategicMerge: + - sgx.yaml diff --git a/deployments/operator/device/sgx/sgx.yaml b/deployments/operator/device/sgx/sgx.yaml new file mode 100644 index 000000000..09bdc0e03 --- /dev/null +++ b/deployments/operator/device/sgx/sgx.yaml @@ -0,0 +1,14 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inteldeviceplugins-controller-manager + namespace: inteldeviceplugins-system +spec: + template: + spec: + containers: + - args: + - --metrics-addr=127.0.0.1:8080 + - --enable-leader-election + - --device=sgx + name: manager