From 5a7e59ae0eb0ce13102f4cafc0dff078d6d9e9e8 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 9 Jan 2024 14:40:07 +0100 Subject: [PATCH 01/24] initial ca bundle injection draft --- .../ADR033-foundation-webhooks-ca-bundle.adoc | 211 ++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc new file mode 100644 index 000000000..dce167da0 --- /dev/null +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -0,0 +1,211 @@ += ADR033: Foundation for admission or conversion webhooks - CA bundle injection +Doc Writer +v0.1 +:status: draft +:date: 2024-01-09 + +* Status: {status} +* Deciders: +** Andrew Kenworthy +** Malte Sander +** Sascha Lautenschlaeger +* Date: {date} + +Technical Story: https://github.com/stackabletech/issues/issues/361 + +== Context + +There are many use cases for the future development of the SDP that involve validating, mutating and conversion webhooks. Tasks that must be tackled in the near future include: + +- The proper versioning of our CRDs and the possibility of smoothly up- and downgrading between versions including breaking changes (see https://github.com/stackabletech/documentation/issues/273) +- Fixing the commons operator initial restarting problem (see https://github.com/stackabletech/commons-operator/issues/111 and spike https://github.com/stackabletech/commons-operator/tree/spike/sts-restarter-webhook) + +When creating an API request, e.g. creating a custom resource, the Kubernetes API server performs the following steps before storing into etcd. + +1. Authentication / Authorization +2. Mutating admission +3. Schema validation +4. Validation admission +5. Conversion to storage version + +The steps 2 to 4 may run in a loop if multiple mutating admissions are performed and have side effects. + +In order for the Kubernetes API server to contact webhook controllers, the webhook endpoints have to present a certificate trusted by the Kubernetes API server. This can be configured similar for conversion and validating / mutating webhooks as follows. + +=== Conversion webhooks CA configuration + +[source,yaml] +---- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + # name must match the spec fields below, and be in the form: . + name: crontabs.example.com +spec: + # group name to use for REST API: /apis// + group: example.com + # list of versions supported by this CustomResourceDefinition + versions: + - name: v1beta1 + # Each version can be enabled/disabled by Served flag. + served: true + # One and only one version must be marked as the storage version. + storage: true + # Each version can define its own schema when there is no top-level + # schema is defined. + schema: + openAPIV3Schema: + type: object + properties: + hostPort: + type: string + - name: v1 + served: true + storage: false + schema: + openAPIV3Schema: + type: object + properties: + host: + type: string + port: + type: string + conversion: + # a Webhook strategy instruct API server to call an external webhook for any conversion between custom resources. + strategy: Webhook + # webhook is required when strategy is `Webhook` and it configures the webhook endpoint to be called by API server. + webhook: + # conversionReviewVersions indicates what ConversionReview versions are understood/preferred by the webhook. + # The first version in the list understood by the API server is sent to the webhook. + # The webhook must respond with a ConversionReview object in the same version it received. + conversionReviewVersions: ["v1","v1beta1"] + clientConfig: + service: + namespace: default + name: example-conversion-webhook-server + path: /crdconvert + caBundle: "Ci0tLS0tQk......tLS0K" + # either Namespaced or Cluster + scope: Namespaced + names: + # plural name to be used in the URL: /apis/// + plural: crontabs + # singular name to be used as an alias on the CLI and for display + singular: crontab + # kind is normally the CamelCased singular type. Your resource manifests use this. + kind: CronTab + # shortNames allow shorter string to match your resource on the CLI + shortNames: + - ct +---- + +=== Validating / Mutating webhooks CA configuration + +[source,yaml] +---- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration # or MutatingWebhookConfiguration +webhooks: + - name: my-webhook.example.com + matchPolicy: Equivalent + rules: + - operations: ['CREATE','UPDATE'] + apiGroups: ['*'] + apiVersions: ['*'] + resources: ['*'] + failurePolicy: "Ignore" # Fail-open (optional) + sideEffects: None + clientConfig: + service: + namespace: default + name: example-validation-webhook-server # or example-mutating-webhook-server + path: /validate # or /mutate + caBundle: "Ci0tLS0tQk......tLS0K" +---- + +The `clientConfig.caBundle` can not be shipped by us as it will differ from cluster to cluster and must be injected at runtime. + +This ADR is about how to achieve the CA bundle injection using external tools or a self-made solution via the secret-operator. + +== Problem Statement + +The required CA bundles for the webhooks endpoints must be injected at runtime. There exist tools like Cert Manager that do exactly this, most of them using annotations like: + +[source,yaml] +---- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: my-webhook.example.com + annotations: + cert-manager.io/inject-ca-from-secret: default/my-webhook-example-com-ca +---- + +This can be used for CRDs and conversion webhooks as well. + +The SDP should be compatible with as many of these as possible. The normal way of configuration seems to be adding an annotation to the objects containing the CA, which we can probably somehow support in our Helm chart, or solve via documentation. + +In a second step the operator needs to obtain a certificate signed by the CA, for which we could use the k8search backend of the secret operator and point this at a configmap which again, can be filled by certmanager or the Service CA operator mentioned above, configured via an annotation. + +For clusters without any existing manager we should provide our own, lightweight caBundle injector that can work in tandem with the secret operator. + +== Decision Drivers + +* Generic solution to be compatible with as many external cert providers as possible +* Openshift compatible +* How to activate / deactivate if e.g no conversion webhooks should be applied? +* Do not get vendor-locked + +== Considered Options + +* https://cert-manager.io/[Cert Manager] +* https://github.com/openshift/service-ca-operator/blob/master/README.md[OpenShift Service CA operator] +* Lightweight self-made solution via secret-operator + +== Decision Outcome + +Chosen option: "[option 1]", because [justification. e.g., only option, which meets k.o. criterion decision driver | which resolves force | … | comes out best (see below)]. + +=== Positive Consequences + +* [e.g., improvement of quality attribute satisfaction, follow-up decisions required, …] +* … + +=== Negative Consequences + +* [e.g., compromising quality attribute, follow-up decisions required, …] +* … + +== Pros and Cons of the Options + +=== [option 1] + +[example | description | pointer to more information | …] + +* Good, because [argument a] +* Good, because [argument b] +* Bad, because [argument c] +* … + +=== [option 2] + +[example | description | pointer to more information | …] + +* Good, because [argument a] +* Good, because [argument b] +* Bad, because [argument c] +* … + +=== [option 3] + +[example | description | pointer to more information | …] + +* Good, because [argument a] +* Good, because [argument b] +* Bad, because [argument c] +* … + +== Links + +* [Link type] [Link to ADR] +* … \ No newline at end of file From 2e0487a840e004b8a500bbc9952fc15652a80c59 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 9 Jan 2024 15:03:46 +0100 Subject: [PATCH 02/24] Apply suggestions from code review Co-authored-by: Andrew Kenworthy --- .../pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index dce167da0..5271e3de1 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -30,7 +30,7 @@ When creating an API request, e.g. creating a custom resource, the Kubernetes AP The steps 2 to 4 may run in a loop if multiple mutating admissions are performed and have side effects. -In order for the Kubernetes API server to contact webhook controllers, the webhook endpoints have to present a certificate trusted by the Kubernetes API server. This can be configured similar for conversion and validating / mutating webhooks as follows. +In order for the Kubernetes API server to contact webhook controllers, the webhook endpoints have to present a certificate trusted by the Kubernetes API server. This can be configured similarly for conversion and validating / mutating webhooks as follows. === Conversion webhooks CA configuration @@ -129,7 +129,7 @@ This ADR is about how to achieve the CA bundle injection using external tools or == Problem Statement -The required CA bundles for the webhooks endpoints must be injected at runtime. There exist tools like Cert Manager that do exactly this, most of them using annotations like: +The required CA bundles for the webhooks endpoints must be injected at runtime. There exist tools like [Cert Manager]([url](https://cert-manager.io/docs/concepts/ca-injector/#injecting-the-kubernetes-api-server-ca)) that do exactly this, adding one of these annotations to an injectionable source: cert-manager.io/inject-ca-from, cert-manager.io/inject-ca-from-secret, or cert-manager.io/inject-apiserver-ca (depending on the injection source). For example: [source,yaml] ---- @@ -154,7 +154,7 @@ For clusters without any existing manager we should provide our own, lightweight * Generic solution to be compatible with as many external cert providers as possible * Openshift compatible * How to activate / deactivate if e.g no conversion webhooks should be applied? -* Do not get vendor-locked +* Avoid vendor lock-in == Considered Options From d9364a9d82b77d0b96450b0e07db0bff496d69fa Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 9 Jan 2024 17:28:04 +0100 Subject: [PATCH 03/24] Apply suggestions from code review Co-authored-by: Techassi --- .../adr/ADR033-foundation-webhooks-ca-bundle.adoc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index 5271e3de1..133fa2154 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -20,7 +20,7 @@ There are many use cases for the future development of the SDP that involve vali - The proper versioning of our CRDs and the possibility of smoothly up- and downgrading between versions including breaking changes (see https://github.com/stackabletech/documentation/issues/273) - Fixing the commons operator initial restarting problem (see https://github.com/stackabletech/commons-operator/issues/111 and spike https://github.com/stackabletech/commons-operator/tree/spike/sts-restarter-webhook) -When creating an API request, e.g. creating a custom resource, the Kubernetes API server performs the following steps before storing into etcd. +When the Kubernetes API server receives an incoming request, for example when a custom resource should be created, webhooks can intercept that request after authentication and authorization. The following steps are performed before the server finally persists the object into etdc. 1. Authentication / Authorization 2. Mutating admission @@ -28,7 +28,7 @@ When creating an API request, e.g. creating a custom resource, the Kubernetes AP 4. Validation admission 5. Conversion to storage version -The steps 2 to 4 may run in a loop if multiple mutating admissions are performed and have side effects. +The steps 2 to 4 may run in a loop if multiple mutating admissions are performed and can have side effects. In order for the Kubernetes API server to contact webhook controllers, the webhook endpoints have to present a certificate trusted by the Kubernetes API server. This can be configured similarly for conversion and validating / mutating webhooks as follows. @@ -129,7 +129,14 @@ This ADR is about how to achieve the CA bundle injection using external tools or == Problem Statement -The required CA bundles for the webhooks endpoints must be injected at runtime. There exist tools like [Cert Manager]([url](https://cert-manager.io/docs/concepts/ca-injector/#injecting-the-kubernetes-api-server-ca)) that do exactly this, adding one of these annotations to an injectionable source: cert-manager.io/inject-ca-from, cert-manager.io/inject-ca-from-secret, or cert-manager.io/inject-apiserver-ca (depending on the injection source). For example: +The required CA bundles for the webhooks endpoints must be injected at runtime. There exist tools like https://cert-manager.io/docs/concepts/ca-injector/#injecting-the-kubernetes-api-server-ca[Cert Manager] that do exactly this. +Adding one of these annotations to an injectionable source: + +. `cert-manager.io/inject-ca-from` +. `cert-manager.io/inject-ca-from-secret` +. `cert-manager.io/inject-apiserver-ca` + +For example: [source,yaml] ---- From a743c4a72bc2bb422f696a3d3cbf5939acb3acbf Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jan 2024 12:42:15 +0100 Subject: [PATCH 04/24] fixes and more explanation --- .../ADR033-foundation-webhooks-ca-bundle.adoc | 77 ++++++++++--------- 1 file changed, 42 insertions(+), 35 deletions(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index 133fa2154..c4bd68b88 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -19,8 +19,9 @@ There are many use cases for the future development of the SDP that involve vali - The proper versioning of our CRDs and the possibility of smoothly up- and downgrading between versions including breaking changes (see https://github.com/stackabletech/documentation/issues/273) - Fixing the commons operator initial restarting problem (see https://github.com/stackabletech/commons-operator/issues/111 and spike https://github.com/stackabletech/commons-operator/tree/spike/sts-restarter-webhook) +- Inject logging / vector sidecar containers (mutating) -When the Kubernetes API server receives an incoming request, for example when a custom resource should be created, webhooks can intercept that request after authentication and authorization. The following steps are performed before the server finally persists the object into etdc. +When the Kubernetes API server receives an incoming request, for example when a custom resource should be created, webhooks can intercept that request after authentication and authorization. The following steps are performed before the server finally persists the object into etc. 1. Authentication / Authorization 2. Mutating admission @@ -123,14 +124,14 @@ webhooks: caBundle: "Ci0tLS0tQk......tLS0K" ---- -The `clientConfig.caBundle` can not be shipped by us as it will differ from cluster to cluster and must be injected at runtime. +The `clientConfig.caBundle` cannot be shipped by us as it will differ from cluster to cluster and must be injected at runtime. This ADR is about how to achieve the CA bundle injection using external tools or a self-made solution via the secret-operator. == Problem Statement The required CA bundles for the webhooks endpoints must be injected at runtime. There exist tools like https://cert-manager.io/docs/concepts/ca-injector/#injecting-the-kubernetes-api-server-ca[Cert Manager] that do exactly this. -Adding one of these annotations to an injectionable source: +Adding one of these annotations to an injectable source: . `cert-manager.io/inject-ca-from` . `cert-manager.io/inject-ca-from-secret` @@ -152,7 +153,7 @@ This can be used for CRDs and conversion webhooks as well. The SDP should be compatible with as many of these as possible. The normal way of configuration seems to be adding an annotation to the objects containing the CA, which we can probably somehow support in our Helm chart, or solve via documentation. -In a second step the operator needs to obtain a certificate signed by the CA, for which we could use the k8search backend of the secret operator and point this at a configmap which again, can be filled by certmanager or the Service CA operator mentioned above, configured via an annotation. +In a second step the operator needs to obtain a certificate signed by the CA, for which we could use the k8search backend of the secret operator and point this at a configmap which again, can be filled by cert-manager or the Service CA operator mentioned above, configured via an annotation. For clusters without any existing manager we should provide our own, lightweight caBundle injector that can work in tandem with the secret operator. @@ -160,57 +161,63 @@ For clusters without any existing manager we should provide our own, lightweight * Generic solution to be compatible with as many external cert providers as possible * Openshift compatible -* How to activate / deactivate if e.g no conversion webhooks should be applied? +* How to activate / deactivate if e.g. no conversion webhooks should be applied? * Avoid vendor lock-in == Considered Options -* https://cert-manager.io/[Cert Manager] -* https://github.com/openshift/service-ca-operator/blob/master/README.md[OpenShift Service CA operator] -* Lightweight self-made solution via secret-operator +[[option1]] +=== https://cert-manager.io/[Cert-Manager] -== Decision Outcome +The cert-manager is a powerful and extensible X.509 certificate controller for Kubernetes and OpenShift workloads. It supports certificates from a variety of popular private and public Issuers (HashiCorp, Lets encrypt and many more). The cert-manager ensures that the certificates are valid and up-to-date, and will attempt to renew certificates at a configured time before expiry. -Chosen option: "[option 1]", because [justification. e.g., only option, which meets k.o. criterion decision driver | which resolves force | … | comes out best (see below)]. +[[option2]] +=== https://github.com/openshift/service-ca-operator/blob/master/README.md[OpenShift Service CA operator] -=== Positive Consequences +The OpenShift Service CA operator is an OpenShift ClusterOperator and contains several controllers: -* [e.g., improvement of quality attribute satisfaction, follow-up decisions required, …] -* … +* *Serving cert signer*: Issues a signed serving certificate/key pair to services annotated with `service.beta.openshift.io/serving-cert-secret-name` via a secret +* *ConfigMap CA bundle injector*: Watches for configmaps annotated with `service.beta.openshift.io/inject-cabundle=true` and adds or updates a data item (key `service-ca.crt`) containing the PEM-encoded CA signing bundle. Consumers of the configmap can then trust `service-ca.crt` in their TLS client configuration, allowing connections to services that utilize service-serving certificates. Pods referencing the `service-ca.crt` in a VolumeMount will not start before the CA bundle was injected. -=== Negative Consequences +[[option3]] +=== Lightweight self-made solution via secret-operator -* [e.g., compromising quality attribute, follow-up decisions required, …] -* … +This would be the Stackable internal solution to avoid any external party tools. It would work similar to the OpenShift Service CA operator but rather injecting the bundles via the CSI instead of ConfigMap mounts. -== Pros and Cons of the Options +== Pros and Cons of the Options -=== [option 1] +=== <> -[example | description | pointer to more information | …] +* Good, because covers both Kubernetes and Openshift +* Good, because widely used and de-facto standard for cert injection +* Good, because cert injection works via annotations (compatibility) +* Bad, because another tool we have to be experienced with, check for updates and breaking features etc. -* Good, because [argument a] -* Good, because [argument b] -* Bad, because [argument c] -* … +=== <> -=== [option 2] +* Good, because cert injection works via annotations (compatibility) +* Bad, because specific to OpenShift +* Bad, because another tool we have to be experienced with, check for updates and breaking features etc. -[example | description | pointer to more information | …] +=== <> -* Good, because [argument a] -* Good, because [argument b] -* Bad, because [argument c] -* … +* Good, because no external tools are required +* Good, because reusing secret-operator and cert management should be an internal part of the SDP +* Bad, because requires more time and coding -=== [option 3] +== Decision Outcome -[example | description | pointer to more information | …] +Chosen option: "[option 1]", because [justification. e.g., only option, which meets k.o. criterion decision driver | which resolves force | … | comes out best (see below)]. -* Good, because [argument a] -* Good, because [argument b] -* Bad, because [argument c] -* … +=== Positive Consequences + +* [e.g., improvement of quality attribute satisfaction, follow-up decisions required, …] +* … + +=== Negative Consequences + +* [e.g., compromising quality attribute, follow-up decisions required, …] +* … == Links From a61a1737eb141aa11a59e9569fcc416abe006974 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jan 2024 14:04:38 +0100 Subject: [PATCH 05/24] Update modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc Co-authored-by: Andrew Kenworthy --- .../pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index c4bd68b88..33204bf04 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -21,7 +21,7 @@ There are many use cases for the future development of the SDP that involve vali - Fixing the commons operator initial restarting problem (see https://github.com/stackabletech/commons-operator/issues/111 and spike https://github.com/stackabletech/commons-operator/tree/spike/sts-restarter-webhook) - Inject logging / vector sidecar containers (mutating) -When the Kubernetes API server receives an incoming request, for example when a custom resource should be created, webhooks can intercept that request after authentication and authorization. The following steps are performed before the server finally persists the object into etc. +When the Kubernetes API server receives an incoming request, for example when a custom resource should be created, webhooks can intercept that request after authentication and authorization. The following steps are performed before the server finally persists the object into etcd. 1. Authentication / Authorization 2. Mutating admission From f5159cec1fc1108365c1f6802eb4eb85cdbbc210 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jan 2024 15:38:35 +0100 Subject: [PATCH 06/24] remove section, add kubernetes doc links --- .../pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index 33204bf04..16488ff72 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -15,7 +15,7 @@ Technical Story: https://github.com/stackabletech/issues/issues/361 == Context -There are many use cases for the future development of the SDP that involve validating, mutating and conversion webhooks. Tasks that must be tackled in the near future include: +There are many use cases for the future development of the SDP that involve https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/[validating, mutating] and https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definition-versioning/[conversion] webhooks. Tasks that must be tackled in the near future include: - The proper versioning of our CRDs and the possibility of smoothly up- and downgrading between versions including breaking changes (see https://github.com/stackabletech/documentation/issues/273) - Fixing the commons operator initial restarting problem (see https://github.com/stackabletech/commons-operator/issues/111 and spike https://github.com/stackabletech/commons-operator/tree/spike/sts-restarter-webhook) @@ -153,8 +153,6 @@ This can be used for CRDs and conversion webhooks as well. The SDP should be compatible with as many of these as possible. The normal way of configuration seems to be adding an annotation to the objects containing the CA, which we can probably somehow support in our Helm chart, or solve via documentation. -In a second step the operator needs to obtain a certificate signed by the CA, for which we could use the k8search backend of the secret operator and point this at a configmap which again, can be filled by cert-manager or the Service CA operator mentioned above, configured via an annotation. - For clusters without any existing manager we should provide our own, lightweight caBundle injector that can work in tandem with the secret operator. == Decision Drivers From 256812e2ca3119b74b926f08ebd29c67fe78149a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jan 2024 17:00:48 +0100 Subject: [PATCH 07/24] clarify injection example --- .../adr/ADR033-foundation-webhooks-ca-bundle.adoc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index 16488ff72..56959aaa5 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -42,6 +42,8 @@ kind: CustomResourceDefinition metadata: # name must match the spec fields below, and be in the form: . name: crontabs.example.com + annotations: + cert-manager.io/inject-ca-from-secret: default/example-conversion-webhook-ca spec: # group name to use for REST API: /apis// group: example.com @@ -85,7 +87,7 @@ spec: namespace: default name: example-conversion-webhook-server path: /crdconvert - caBundle: "Ci0tLS0tQk......tLS0K" + #caBundle: will be injected from a Secret 'default/example-conversion-webhook-ca' with a 'ca.crt' data key # either Namespaced or Cluster scope: Namespaced names: @@ -106,6 +108,10 @@ spec: ---- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration # or MutatingWebhookConfiguration +metadata: + name: "example-validating-webhook-server" + annotations: + cert-manager.io/inject-ca-from-secret: default/example-conversion-webhook-ca webhooks: - name: my-webhook.example.com matchPolicy: Equivalent @@ -121,7 +127,7 @@ webhooks: namespace: default name: example-validation-webhook-server # or example-mutating-webhook-server path: /validate # or /mutate - caBundle: "Ci0tLS0tQk......tLS0K" + #caBundle: will be injected from a Secret 'default/example-conversion-webhook-ca' with a 'ca.crt' data key ---- The `clientConfig.caBundle` cannot be shipped by us as it will differ from cluster to cluster and must be injected at runtime. From 2c625c2a55320ded42e6b0bbd1797c0f8db3dbb5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jan 2024 17:02:52 +0100 Subject: [PATCH 08/24] change injection annotation link --- .../pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index 56959aaa5..452b70406 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -136,7 +136,7 @@ This ADR is about how to achieve the CA bundle injection using external tools or == Problem Statement -The required CA bundles for the webhooks endpoints must be injected at runtime. There exist tools like https://cert-manager.io/docs/concepts/ca-injector/#injecting-the-kubernetes-api-server-ca[Cert Manager] that do exactly this. +The required CA bundles for the webhooks endpoints must be injected at runtime. There exist tools like https://cert-manager.io/docs/concepts/ca-injector/#injecting-ca-data-from-a-certificate-resource[Cert Manager] that do exactly this. Adding one of these annotations to an injectable source: . `cert-manager.io/inject-ca-from` From c36501a2675f4a21dc82ccd8814db4479eb7b604 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jan 2024 17:40:04 +0100 Subject: [PATCH 09/24] add 4th option for cert injection as suggested by nat --- .../adr/ADR033-foundation-webhooks-ca-bundle.adoc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index 452b70406..a4603a1f1 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -143,7 +143,7 @@ Adding one of these annotations to an injectable source: . `cert-manager.io/inject-ca-from-secret` . `cert-manager.io/inject-apiserver-ca` -For example: +For example injecting a CA from a secret using Cert-Manager: [source,yaml] ---- @@ -188,19 +188,25 @@ The OpenShift Service CA operator is an OpenShift ClusterOperator and contains s This would be the Stackable internal solution to avoid any external party tools. It would work similar to the OpenShift Service CA operator but rather injecting the bundles via the CSI instead of ConfigMap mounts. +[[option4]] +=== Common library for cert management / injection in operator-rs + +Put the cert management / injection stuff into a library (op-rs?) that we run in-process for each operator that has a webhook. + == Pros and Cons of the Options === <> * Good, because covers both Kubernetes and Openshift * Good, because widely used and de-facto standard for cert injection -* Good, because cert injection works via annotations (compatibility) +* Good, because cert injection works via annotations (compatibility) for various Kubernetes Resources (CRDs, Validating/Mutating webhooks) * Bad, because another tool we have to be experienced with, check for updates and breaking features etc. === <> * Good, because cert injection works via annotations (compatibility) * Bad, because specific to OpenShift +* Bad, because only injecting to ConfigMaps * Bad, because another tool we have to be experienced with, check for updates and breaking features etc. === <> @@ -209,6 +215,10 @@ This would be the Stackable internal solution to avoid any external party tools. * Good, because reusing secret-operator and cert management should be an internal part of the SDP * Bad, because requires more time and coding +=== <> + +TBD. + == Decision Outcome Chosen option: "[option 1]", because [justification. e.g., only option, which meets k.o. criterion decision driver | which resolves force | … | comes out best (see below)]. From c2bb208f421517c28a7d42c6adfaf37f46b70a69 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 16 Jan 2024 10:37:50 +0100 Subject: [PATCH 10/24] Clarify to include one CA injector but keep compatibility to others. --- .../pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index a4603a1f1..96c3f104f 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -157,7 +157,7 @@ metadata: This can be used for CRDs and conversion webhooks as well. -The SDP should be compatible with as many of these as possible. The normal way of configuration seems to be adding an annotation to the objects containing the CA, which we can probably somehow support in our Helm chart, or solve via documentation. +The SDP should focus on one solution for CA injection, internally or externally, but strive for compatibility with as many others as possible. The normal way of configuration seems to be adding an annotation to the objects containing the CA, which we can probably somehow support in our Helm chart, or solve via documentation. For clusters without any existing manager we should provide our own, lightweight caBundle injector that can work in tandem with the secret operator. @@ -214,10 +214,12 @@ Put the cert management / injection stuff into a library (op-rs?) that we run in * Good, because no external tools are required * Good, because reusing secret-operator and cert management should be an internal part of the SDP * Bad, because requires more time and coding +* Bad, because secret-operator must version its own CRDs as well which could be a predicament === <> -TBD. +* Good, because no external tools are required +* Bad, because requires more time and coding == Decision Outcome From 37357b3ac9c3e8c8bd8ca67974911df7eb5f3073 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 21 Jan 2024 14:43:39 +0100 Subject: [PATCH 11/24] added feedback and decision from adr meeting --- .../ADR033-foundation-webhooks-ca-bundle.adoc | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index 96c3f104f..ebdef8e5f 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -191,7 +191,7 @@ This would be the Stackable internal solution to avoid any external party tools. [[option4]] === Common library for cert management / injection in operator-rs -Put the cert management / injection stuff into a library (op-rs?) that we run in-process for each operator that has a webhook. +Put the cert management / injection stuff into a library in operator-rs (similar to the code in secret-operator) that we run in-process for each operator that has a webhook. See https://github.com/stackabletech/commons-operator/tree/spike/sts-restarter-webhook[commons-operator spike]. == Pros and Cons of the Options @@ -219,23 +219,16 @@ Put the cert management / injection stuff into a library (op-rs?) that we run in === <> * Good, because no external tools are required +* Good, because all included within the SDP +* Good, because external tools can be used if required * Bad, because requires more time and coding == Decision Outcome -Chosen option: "[option 1]", because [justification. e.g., only option, which meets k.o. criterion decision driver | which resolves force | … | comes out best (see below)]. +Chosen option <>, because the CA injection will be handled as part of the SDP and no external dependencies are required. The common library will reside in the operator-rs and be used in every operator. OpenShift should not pose a problem since no extra components are necessary. The required infrastructure (e.g. operator-templating, ca injection / generation) can be developed in parallel to the operators e.g. version conversion logic. -=== Positive Consequences +=== Positive Consequences -* [e.g., improvement of quality attribute satisfaction, follow-up decisions required, …] -* … - -=== Negative Consequences - -* [e.g., compromising quality attribute, follow-up decisions required, …] -* … - -== Links - -* [Link type] [Link to ADR] -* … \ No newline at end of file +* No external dependencies +* CA bundle injection as basic part of the SDP +* Opt-out (e.g. removing the conversion webhook from CRD) possible From a050b0adcf3fe915570837b21fc7b1f2bc0d0ac6 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 22 Jan 2024 14:15:03 +0100 Subject: [PATCH 12/24] fix grammar --- .../pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index ebdef8e5f..d1d8c4439 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -225,7 +225,7 @@ Put the cert management / injection stuff into a library in operator-rs (similar == Decision Outcome -Chosen option <>, because the CA injection will be handled as part of the SDP and no external dependencies are required. The common library will reside in the operator-rs and be used in every operator. OpenShift should not pose a problem since no extra components are necessary. The required infrastructure (e.g. operator-templating, ca injection / generation) can be developed in parallel to the operators e.g. version conversion logic. +Chosen option <>, because the CA injection will be handled as part of the SDP and no external dependencies are required. The common library will reside in the operator-rs and used in every operator. OpenShift should not pose a problem since no extra components are necessary. The required infrastructure (e.g. operator-templating, ca injection / generation) can be developed in parallel to the operators e.g. version conversion logic. === Positive Consequences From 1d43f176de8f6e2abdefbf131bf425d8d339adae Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 22 Jan 2024 17:41:05 +0100 Subject: [PATCH 13/24] Apply suggestions from code review Co-authored-by: Andrew Kenworthy --- .../pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index d1d8c4439..bf99c3577 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -136,8 +136,7 @@ This ADR is about how to achieve the CA bundle injection using external tools or == Problem Statement -The required CA bundles for the webhooks endpoints must be injected at runtime. There exist tools like https://cert-manager.io/docs/concepts/ca-injector/#injecting-ca-data-from-a-certificate-resource[Cert Manager] that do exactly this. -Adding one of these annotations to an injectable source: +The required CA bundles for the webhooks endpoints must be injected at runtime. There exist tools like https://cert-manager.io/docs/concepts/ca-injector/#injecting-ca-data-from-a-certificate-resource[Cert Manager] that do exactly this, adding one of these annotations to an injectable source: . `cert-manager.io/inject-ca-from` . `cert-manager.io/inject-ca-from-secret` @@ -157,7 +156,7 @@ metadata: This can be used for CRDs and conversion webhooks as well. -The SDP should focus on one solution for CA injection, internally or externally, but strive for compatibility with as many others as possible. The normal way of configuration seems to be adding an annotation to the objects containing the CA, which we can probably somehow support in our Helm chart, or solve via documentation. +The SDP should focus on one solution for CA injection, internally or externally, but strive for compatibility with as many others as possible. The normal way of configuration seems to be adding an annotation to the objects containing the CA, which can be supported in our Helm chart or solved via documentation. For clusters without any existing manager we should provide our own, lightweight caBundle injector that can work in tandem with the secret operator. @@ -186,7 +185,7 @@ The OpenShift Service CA operator is an OpenShift ClusterOperator and contains s [[option3]] === Lightweight self-made solution via secret-operator -This would be the Stackable internal solution to avoid any external party tools. It would work similar to the OpenShift Service CA operator but rather injecting the bundles via the CSI instead of ConfigMap mounts. +This would be the Stackable internal solution to avoid any external party tools. It would work in a similar way to the OpenShift Service CA operator but would rather inject the bundles via the CSI instead of ConfigMap mounts. [[option4]] === Common library for cert management / injection in operator-rs @@ -225,7 +224,7 @@ Put the cert management / injection stuff into a library in operator-rs (similar == Decision Outcome -Chosen option <>, because the CA injection will be handled as part of the SDP and no external dependencies are required. The common library will reside in the operator-rs and used in every operator. OpenShift should not pose a problem since no extra components are necessary. The required infrastructure (e.g. operator-templating, ca injection / generation) can be developed in parallel to the operators e.g. version conversion logic. +Chosen option <>, because the CA injection will be handled as part of the SDP and no external dependencies are required. The common library will reside in the operator-rs and be used in every operator. OpenShift should not pose a problem since no extra components are necessary. The required infrastructure (e.g. operator-templating, ca injection / generation) can be developed in parallel to the operators e.g. version conversion logic. === Positive Consequences From 4d708503de5e0059e8b397394bc0be62e24e5271 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 22 Jan 2024 17:45:50 +0100 Subject: [PATCH 14/24] Update modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc Co-authored-by: Andrew Kenworthy --- .../pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index bf99c3577..b9ee0ad98 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -54,7 +54,7 @@ spec: served: true # One and only one version must be marked as the storage version. storage: true - # Each version can define its own schema when there is no top-level + # Each version can define its own schema when no top-level # schema is defined. schema: openAPIV3Schema: From 82135fbea74a909ec3d802dd7718b9f62e45efc9 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 24 Jan 2024 15:17:03 +0100 Subject: [PATCH 15/24] set status to accepted --- .../pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index d1d8c4439..3c4ff0637 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -1,7 +1,7 @@ = ADR033: Foundation for admission or conversion webhooks - CA bundle injection Doc Writer v0.1 -:status: draft +:status: accepted :date: 2024-01-09 * Status: {status} From 00a89cd2092283328567ee5fa015c8be5f6f0adc Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 31 Jan 2024 14:38:03 +0100 Subject: [PATCH 16/24] add introductory text for webhook ca injection examples --- .../pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index 29c99c296..203b694d5 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -35,6 +35,10 @@ In order for the Kubernetes API server to contact webhook controllers, the webho === Conversion webhooks CA configuration +The following snippet shows a CRD containing multiple versions (`v1beta1`, `v1`) with different schemas. The `v1beta1` is still the storage version. Due to schema changes, any custom resource deployed as version `v1` requires a conversion webhook. + +The `conversion.webhook.clientConfig.service` points to an endpoint where the actual conversion from an applied `v1` to the stored `v1beta1` takes place. The Kubernetes API server must trust the endpoint for which a certificate is provided / injected in `conversion.webhook.clientConfig.caBundle`. The CA bundle in this case is injected via a Cert-Manager annotation that is discussed later in this document. + [source,yaml] ---- apiVersion: apiextensions.k8s.io/v1 @@ -104,6 +108,8 @@ spec: === Validating / Mutating webhooks CA configuration +The following snippet shows a `ValidatingWebhookConfiguration` which is similar to a `MutatingWebhookConfiguration`. The `webhooks.clientConfig` is configured as for the conversion webhook above. The `webhooks.clientConfig.service` points to the endpoint where the webhook is served an must be trusted by the Kubernetes API server via `webhooks.clientConfig.caBundle`. The CA bundle in this case is injected via a Cert-Manager annotation that is discussed later in this document. + [source,yaml] ---- apiVersion: admissionregistration.k8s.io/v1 From ffdea691ca4c0c08e8fd898ce6759f364ae668a7 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 31 Jan 2024 14:41:46 +0100 Subject: [PATCH 17/24] clarify the cert-manager injection for conversion webhooks --- .../pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index 203b694d5..e44c33b78 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -33,6 +33,7 @@ The steps 2 to 4 may run in a loop if multiple mutating admissions are performed In order for the Kubernetes API server to contact webhook controllers, the webhook endpoints have to present a certificate trusted by the Kubernetes API server. This can be configured similarly for conversion and validating / mutating webhooks as follows. +[[conversion_webhooks_ca_example]] === Conversion webhooks CA configuration The following snippet shows a CRD containing multiple versions (`v1beta1`, `v1`) with different schemas. The `v1beta1` is still the storage version. Due to schema changes, any custom resource deployed as version `v1` requires a conversion webhook. @@ -160,7 +161,7 @@ metadata: cert-manager.io/inject-ca-from-secret: default/my-webhook-example-com-ca ---- -This can be used for CRDs and conversion webhooks as well. +This can be used for CRDs and conversion webhooks as well as shown in <>. The SDP should focus on one solution for CA injection, internally or externally, but strive for compatibility with as many others as possible. The normal way of configuration seems to be adding an annotation to the objects containing the CA, which can be supported in our Helm chart or solved via documentation. From d061e00517c4107f3409347542d28fafe060c501 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 31 Jan 2024 15:41:08 +0100 Subject: [PATCH 18/24] clarify decision drivers --- .../pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index e44c33b78..e47c82168 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -169,10 +169,9 @@ For clusters without any existing manager we should provide our own, lightweight == Decision Drivers -* Generic solution to be compatible with as many external cert providers as possible -* Openshift compatible -* How to activate / deactivate if e.g. no conversion webhooks should be applied? -* Avoid vendor lock-in +* Generic solution to be compatible with as many external cert providers as possible to avoid vendor lock-in. This means a possible abstraction to support switching out cert providers / "backends". +* Openshift compatibility. +* How to activate / deactivate if e.g. no conversion webhooks should be applied? This is about how we e.g. set inject annotations via templating / Helm. == Considered Options From da4e2f5097ebaeaf0687ddec9660152582985df8 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 31 Jan 2024 17:17:08 +0100 Subject: [PATCH 19/24] Apply suggestions from code review Co-authored-by: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> --- .../pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index e47c82168..53b35c568 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -205,6 +205,8 @@ Put the cert management / injection stuff into a library in operator-rs (similar * Good, because covers both Kubernetes and Openshift * Good, because widely used and de-facto standard for cert injection * Good, because cert injection works via annotations (compatibility) for various Kubernetes Resources (CRDs, Validating/Mutating webhooks) +* Good, because it's already popular and familiar to users +* Good, because it frees maintenance and development resources for us * Bad, because another tool we have to be experienced with, check for updates and breaking features etc. === <> @@ -224,7 +226,6 @@ Put the cert management / injection stuff into a library in operator-rs (similar === <> * Good, because no external tools are required -* Good, because all included within the SDP * Good, because external tools can be used if required * Bad, because requires more time and coding From 294bc67565b023590442d6413be6ccc5591873d7 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 31 Jan 2024 17:18:29 +0100 Subject: [PATCH 20/24] adapt to review --- .../pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index 53b35c568..6a05dd26b 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -203,9 +203,8 @@ Put the cert management / injection stuff into a library in operator-rs (similar === <> * Good, because covers both Kubernetes and Openshift -* Good, because widely used and de-facto standard for cert injection * Good, because cert injection works via annotations (compatibility) for various Kubernetes Resources (CRDs, Validating/Mutating webhooks) -* Good, because it's already popular and familiar to users +* Good, because it is already popular and familiar to users * Good, because it frees maintenance and development resources for us * Bad, because another tool we have to be experienced with, check for updates and breaking features etc. From 8159f7b7f39402963439eb4ebff0ff1612d7a9a3 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 8 Feb 2024 10:45:40 +0100 Subject: [PATCH 21/24] Added notes on OLM --- .../adr/ADR033-foundation-webhooks-ca-bundle.adoc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index 6a05dd26b..1006967a4 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -167,6 +167,20 @@ The SDP should focus on one solution for CA injection, internally or externally, For clusters without any existing manager we should provide our own, lightweight caBundle injector that can work in tandem with the secret operator. +=== Notes on OLM + +The Operator Lifecycle Manager (OLM) is a core component of OpenShift. It is used to install, manage, and upgrade the lifecycle of all Operators and their associated services running across a cluster. It is the recommended way to install and manage operators on OpenShift. OLM is also used to install the Stackable Operator Framework. + +Operators and webhooks managed by OLM are automatically injected with the CA bundle from the cluster. This is done by the OLM itself and does not require any additional configuration. + +OLM patches the CustomResourceDefinition (CRD) marked as `owned` with the CA bundle from the cluster and mounts certificates and keys in the webhook Pods. The keys are in the EC format (as of version 4.14 of OpenShift). + +OLM mounts the TLS key and certificate for the webhook at the following locations: + + * The TLS certificate file is mounted to the deployment at `/apiserver.local.config/certificates/apiserver.crt`. +* The TLS key file is mounted to the deployment at `/apiserver.local.config/certificates/apiserver.key`. + +For more details regarding OLM constraints for webhooks, see the OpenShift Container Platform https://docs.openshift.com/container-platform/4.14/operators/operator_sdk/osdk-generating-csvs.html#olm-webhook-considerations_osdk-generating-csvs[documentation]. == Decision Drivers * Generic solution to be compatible with as many external cert providers as possible to avoid vendor lock-in. This means a possible abstraction to support switching out cert providers / "backends". From 8939c3245034393ead42594f356476931f46d927 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 23 Feb 2024 14:24:59 +0100 Subject: [PATCH 22/24] Put one sentence per line --- .../ADR033-foundation-webhooks-ca-bundle.adoc | 58 ++++++++++++++----- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index 1006967a4..b8103286a 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -21,7 +21,8 @@ There are many use cases for the future development of the SDP that involve http - Fixing the commons operator initial restarting problem (see https://github.com/stackabletech/commons-operator/issues/111 and spike https://github.com/stackabletech/commons-operator/tree/spike/sts-restarter-webhook) - Inject logging / vector sidecar containers (mutating) -When the Kubernetes API server receives an incoming request, for example when a custom resource should be created, webhooks can intercept that request after authentication and authorization. The following steps are performed before the server finally persists the object into etcd. +When the Kubernetes API server receives an incoming request, for example when a custom resource should be created, webhooks can intercept that request after authentication and authorization. +The following steps are performed before the server finally persists the object into etcd. 1. Authentication / Authorization 2. Mutating admission @@ -31,14 +32,19 @@ When the Kubernetes API server receives an incoming request, for example when a The steps 2 to 4 may run in a loop if multiple mutating admissions are performed and can have side effects. -In order for the Kubernetes API server to contact webhook controllers, the webhook endpoints have to present a certificate trusted by the Kubernetes API server. This can be configured similarly for conversion and validating / mutating webhooks as follows. +In order for the Kubernetes API server to contact webhook controllers, the webhook endpoints have to present a certificate trusted by the Kubernetes API server. +This can be configured similarly for conversion and validating / mutating webhooks as follows. [[conversion_webhooks_ca_example]] === Conversion webhooks CA configuration -The following snippet shows a CRD containing multiple versions (`v1beta1`, `v1`) with different schemas. The `v1beta1` is still the storage version. Due to schema changes, any custom resource deployed as version `v1` requires a conversion webhook. +The following snippet shows a CRD containing multiple versions (`v1beta1`, `v1`) with different schemas. +The `v1beta1` is still the storage version. +Due to schema changes, any custom resource deployed as version `v1` requires a conversion webhook. -The `conversion.webhook.clientConfig.service` points to an endpoint where the actual conversion from an applied `v1` to the stored `v1beta1` takes place. The Kubernetes API server must trust the endpoint for which a certificate is provided / injected in `conversion.webhook.clientConfig.caBundle`. The CA bundle in this case is injected via a Cert-Manager annotation that is discussed later in this document. +The `conversion.webhook.clientConfig.service` points to an endpoint where the actual conversion from an applied `v1` to the stored `v1beta1` takes place. +The Kubernetes API server must trust the endpoint for which a certificate is provided / injected in `conversion.webhook.clientConfig.caBundle`. +The CA bundle in this case is injected via a Cert-Manager annotation that is discussed later in this document. [source,yaml] ---- @@ -109,7 +115,10 @@ spec: === Validating / Mutating webhooks CA configuration -The following snippet shows a `ValidatingWebhookConfiguration` which is similar to a `MutatingWebhookConfiguration`. The `webhooks.clientConfig` is configured as for the conversion webhook above. The `webhooks.clientConfig.service` points to the endpoint where the webhook is served an must be trusted by the Kubernetes API server via `webhooks.clientConfig.caBundle`. The CA bundle in this case is injected via a Cert-Manager annotation that is discussed later in this document. +The following snippet shows a `ValidatingWebhookConfiguration` which is similar to a `MutatingWebhookConfiguration`. +The `webhooks.clientConfig` is configured as for the conversion webhook above. +The `webhooks.clientConfig.service` points to the endpoint where the webhook is served an must be trusted by the Kubernetes API server via `webhooks.clientConfig.caBundle`. +The CA bundle in this case is injected via a Cert-Manager annotation that is discussed later in this document. [source,yaml] ---- @@ -143,7 +152,8 @@ This ADR is about how to achieve the CA bundle injection using external tools or == Problem Statement -The required CA bundles for the webhooks endpoints must be injected at runtime. There exist tools like https://cert-manager.io/docs/concepts/ca-injector/#injecting-ca-data-from-a-certificate-resource[Cert Manager] that do exactly this, adding one of these annotations to an injectable source: +The required CA bundles for the webhooks endpoints must be injected at runtime. +There exist tools like https://cert-manager.io/docs/concepts/ca-injector/#injecting-ca-data-from-a-certificate-resource[Cert Manager] that do exactly this, adding one of these annotations to an injectable source: . `cert-manager.io/inject-ca-from` . `cert-manager.io/inject-ca-from-secret` @@ -163,17 +173,22 @@ metadata: This can be used for CRDs and conversion webhooks as well as shown in <>. -The SDP should focus on one solution for CA injection, internally or externally, but strive for compatibility with as many others as possible. The normal way of configuration seems to be adding an annotation to the objects containing the CA, which can be supported in our Helm chart or solved via documentation. +The SDP should focus on one solution for CA injection, internally or externally, but strive for compatibility with as many others as possible. +The normal way of configuration seems to be adding an annotation to the objects containing the CA, which can be supported in our Helm chart or solved via documentation. For clusters without any existing manager we should provide our own, lightweight caBundle injector that can work in tandem with the secret operator. === Notes on OLM -The Operator Lifecycle Manager (OLM) is a core component of OpenShift. It is used to install, manage, and upgrade the lifecycle of all Operators and their associated services running across a cluster. It is the recommended way to install and manage operators on OpenShift. OLM is also used to install the Stackable Operator Framework. +The Operator Lifecycle Manager (OLM) is a core component of OpenShift. +It is used to install, manage, and upgrade the lifecycle of all Operators and their associated services running across a cluster. +It is the recommended way to install and manage operators on OpenShift. OLM is also used to install the Stackable Operator Framework. -Operators and webhooks managed by OLM are automatically injected with the CA bundle from the cluster. This is done by the OLM itself and does not require any additional configuration. +Operators and webhooks managed by OLM are automatically injected with the CA bundle from the cluster. +This is done by the OLM itself and does not require any additional configuration. -OLM patches the CustomResourceDefinition (CRD) marked as `owned` with the CA bundle from the cluster and mounts certificates and keys in the webhook Pods. The keys are in the EC format (as of version 4.14 of OpenShift). +OLM patches the CustomResourceDefinition (CRD) marked as `owned` with the CA bundle from the cluster and mounts certificates and keys in the webhook Pods. +The keys are in the EC format (as of version 4.14 of OpenShift). OLM mounts the TLS key and certificate for the webhook at the following locations: @@ -181,18 +196,23 @@ OLM mounts the TLS key and certificate for the webhook at the following location * The TLS key file is mounted to the deployment at `/apiserver.local.config/certificates/apiserver.key`. For more details regarding OLM constraints for webhooks, see the OpenShift Container Platform https://docs.openshift.com/container-platform/4.14/operators/operator_sdk/osdk-generating-csvs.html#olm-webhook-considerations_osdk-generating-csvs[documentation]. + == Decision Drivers -* Generic solution to be compatible with as many external cert providers as possible to avoid vendor lock-in. This means a possible abstraction to support switching out cert providers / "backends". +* Generic solution to be compatible with as many external cert providers as possible to avoid vendor lock-in. + This means a possible abstraction to support switching out cert providers / "backends". * Openshift compatibility. -* How to activate / deactivate if e.g. no conversion webhooks should be applied? This is about how we e.g. set inject annotations via templating / Helm. +* How to activate / deactivate if e.g. no conversion webhooks should be applied? + This is about how we e.g. set inject annotations via templating / Helm. == Considered Options [[option1]] === https://cert-manager.io/[Cert-Manager] -The cert-manager is a powerful and extensible X.509 certificate controller for Kubernetes and OpenShift workloads. It supports certificates from a variety of popular private and public Issuers (HashiCorp, Lets encrypt and many more). The cert-manager ensures that the certificates are valid and up-to-date, and will attempt to renew certificates at a configured time before expiry. +The cert-manager is a powerful and extensible X.509 certificate controller for Kubernetes and OpenShift workloads. +It supports certificates from a variety of popular private and public Issuers (HashiCorp, Lets encrypt and many more). +The cert-manager ensures that the certificates are valid and up-to-date, and will attempt to renew certificates at a configured time before expiry. [[option2]] === https://github.com/openshift/service-ca-operator/blob/master/README.md[OpenShift Service CA operator] @@ -200,12 +220,15 @@ The cert-manager is a powerful and extensible X.509 certificate controller for K The OpenShift Service CA operator is an OpenShift ClusterOperator and contains several controllers: * *Serving cert signer*: Issues a signed serving certificate/key pair to services annotated with `service.beta.openshift.io/serving-cert-secret-name` via a secret -* *ConfigMap CA bundle injector*: Watches for configmaps annotated with `service.beta.openshift.io/inject-cabundle=true` and adds or updates a data item (key `service-ca.crt`) containing the PEM-encoded CA signing bundle. Consumers of the configmap can then trust `service-ca.crt` in their TLS client configuration, allowing connections to services that utilize service-serving certificates. Pods referencing the `service-ca.crt` in a VolumeMount will not start before the CA bundle was injected. +* *ConfigMap CA bundle injector*: Watches for configmaps annotated with `service.beta.openshift.io/inject-cabundle=true` and adds or updates a data item (key `service-ca.crt`) containing the PEM-encoded CA signing bundle. + Consumers of the configmap can then trust `service-ca.crt` in their TLS client configuration, allowing connections to services that utilize service-serving certificates. + Pods referencing the `service-ca.crt` in a VolumeMount will not start before the CA bundle was injected. [[option3]] === Lightweight self-made solution via secret-operator -This would be the Stackable internal solution to avoid any external party tools. It would work in a similar way to the OpenShift Service CA operator but would rather inject the bundles via the CSI instead of ConfigMap mounts. +This would be the Stackable internal solution to avoid any external party tools. +It would work in a similar way to the OpenShift Service CA operator but would rather inject the bundles via the CSI instead of ConfigMap mounts. [[option4]] === Common library for cert management / injection in operator-rs @@ -244,7 +267,10 @@ Put the cert management / injection stuff into a library in operator-rs (similar == Decision Outcome -Chosen option <>, because the CA injection will be handled as part of the SDP and no external dependencies are required. The common library will reside in the operator-rs and be used in every operator. OpenShift should not pose a problem since no extra components are necessary. The required infrastructure (e.g. operator-templating, ca injection / generation) can be developed in parallel to the operators e.g. version conversion logic. +Chosen option <>, because the CA injection will be handled as part of the SDP and no external dependencies are required. +The common library will reside in the operator-rs and be used in every operator. +OpenShift should not pose a problem since no extra components are necessary. +The required infrastructure (e.g. operator-templating, ca injection / generation) can be developed in parallel to the operators e.g. version conversion logic. === Positive Consequences From 327722469e98a457ebe6581b5ff5ffca2b93befa Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 23 Feb 2024 14:25:20 +0100 Subject: [PATCH 23/24] Add more details for option 4 --- .../pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index b8103286a..7a70f3326 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -233,7 +233,12 @@ It would work in a similar way to the OpenShift Service CA operator but would ra [[option4]] === Common library for cert management / injection in operator-rs -Put the cert management / injection stuff into a library in operator-rs (similar to the code in secret-operator) that we run in-process for each operator that has a webhook. See https://github.com/stackabletech/commons-operator/tree/spike/sts-restarter-webhook[commons-operator spike]. +Put the cert management / injection stuff into a library in operator-rs (similar to the code in secret-operator) that we run in-process for each operator that has a webhook. +The operator / webhook code will use a common TLS utility crate which handles creation of CAs and other certificates on-the-fly. +These certificates will be rotated automatically when they expire. +The webhook server will pick up the renewed certificate without the need for a restart. +Webhooks created by the common `stackable-webhook` crate can also use a certificate which is not autogenerated by us and provided via a Kubernetes Secret from external sources. +For reference, see the https://github.com/stackabletech/commons-operator/tree/spike/sts-restarter-webhook[commons-operator spike] and the https://github.com/stackabletech/operator-rs/pull/736[TLS utility crate PR]. == Pros and Cons of the Options From f10f0092f9612e0978a75c9ead3c6d9cbc76404a Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 23 Feb 2024 14:37:58 +0100 Subject: [PATCH 24/24] Add webhook server PR link --- .../pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc index 7a70f3326..ee50f5a16 100644 --- a/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc +++ b/modules/contributor/pages/adr/ADR033-foundation-webhooks-ca-bundle.adoc @@ -238,7 +238,11 @@ The operator / webhook code will use a common TLS utility crate which handles cr These certificates will be rotated automatically when they expire. The webhook server will pick up the renewed certificate without the need for a restart. Webhooks created by the common `stackable-webhook` crate can also use a certificate which is not autogenerated by us and provided via a Kubernetes Secret from external sources. -For reference, see the https://github.com/stackabletech/commons-operator/tree/spike/sts-restarter-webhook[commons-operator spike] and the https://github.com/stackabletech/operator-rs/pull/736[TLS utility crate PR]. +For references, see: + +* the https://github.com/stackabletech/commons-operator/tree/spike/sts-restarter-webhook[commons-operator spike], +* the https://github.com/stackabletech/operator-rs/pull/736[TLS utility crate PR], and +* the https://github.com/stackabletech/operator-rs/pull/730[webhook server PR]. == Pros and Cons of the Options