title: “Renew Certificate manually”
date: 2021-03-04T17:35:22
slug: renew-certificate-manually

oc adm ca create-signer-cert --key=/root/metrics-signer.key --cert=/root/ca.crt --serial=/root/ca.serial.txt --name=metrics-signer@$(date +%s)
oc adm ca create-server-cert --signer-cert=/root/ca.crt --signer-key=/root/metrics-signer.key --hostnames=metrics-server,metrics-server.openshift-metrics-server.svc,metrics-server.openshift-metrics-server.svc.cluster.local --cert=/root/tls.crt --key=/root/tls.key --signer-serial=/etc/origin/master/ca.serial.txt
oc create secret generic metrics-server-certs --from-file=tls.crt,tls.key,ca.crt -o json --dry-run | oc replace -f -

Content of api.yaml (caBundle is Output of 'cat ca.crt | base64 -w0' ):

apiVersion: apiregistration.k8s.io/v1beta1
kind: APIService
metadata:
 name: v1beta1.metrics.k8s.io
 labels:
 kubernetes.io/cluster-service: "true"
 metrics-server-infra: support
spec:
 service:
 name: metrics-server
 namespace: openshift-metrics-server
 group: metrics.k8s.io
 version: v1beta1
 insecureSkipTLSVerify: false
 groupPriorityMinimum: 100
 versionPriority: 100
 caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0sdFpYUnkKYVdOekxYTnBaMjVsY2tBeE5qRTBPRFV6TWpFNE1CNFhEVEl4TURNd05ERXdNakF4T0ZvWERUSTJNRE13TXpFdwpNakF4T1Zvd0pERWlNQ0FHQTFVRUF3d1piV1YwY21samN5MXphV2R1WlhKQU1UWXhORGcxTXpJeE9EQ0NBU0l3CkRRWUpLb1pJaHZjTkFRRUJCUUFEZ2dFUEFEQ0NBUW9DZ2dFQkFLbkdVSGtmZ3F3VTQvb1VzMSs4QWhqMS8rVWoKWE9Wd3kwK1oyazkwakpKVmIwMmo3TXhrcjEvUWRXREkva0NNcExKM1d5Sm9WbWJUOWNEWGxoVkV4VGRkOVd3RwpQbnIvMVMwVXB4aHJmWmdHNFgydVJnYzFEb0hxOWdzbCt3TTcrRTRPaDlaZ0NxZkhOU0ozQWF0UlFZUThiaGtVCnovMG9HT1FUUFV1aDJ3YVU1ZjZqcjJSN1VsS21Ua3RuVVBBV080SWpBRS9PSDBSZmNOS1V6V01XK1d5bVZXOGUKNExiUGc1TmpIT1YzTGVGQ0hkMTR1R3ArV3dYVm9GY3ZoaUViMHM4ZkRFTTBGVnJtYUxYOWJUTlNkYXV2cGNubgpqVXBJK1MzZ2VHRE5xdEk1U2lGQjkralg4eE5ma1gyUzZyOHZ6b0kwaHpUNTdrb2NndDhZb0tWcVFEc0NBd0VBCkFhTWpNQ0V3RGdZRFZSMFBBUUgvQkFRREFnS2tNQThHQTFVZEV3RUIvd1FGTUFNQkFmOHdEUVlKS29aSWh2Y04KQVFFTEJRQURnZ0VCQUcwYmhvancwNVFwam1oakMwSStBTU9pNVZMRWFlb1RIZks0eU91SVJRUkllQUtYWjUzbQpDOFo2clA1N3NUTmgvOFVoWk9rRzdndlNlTExwaGtaRlVtLzNWM3J1cFFCY2ZaNGZmZ3VGaEQ1bmludjRkR25SCkZ3TzREVm9mN0RROWhPUVlIMVh0bGpzUTBCTnpjNS9jOXBlTWc4eWtmekpCWm50dlZTcTN3TFA2Q29scWkzdGoKNWV5N012RkF6bjBxNHZVQm1MQ25BQmU5SWtFUFdibWt6Sytza2hpb3lCMUdoNGFEeml3ei9ZbkFvZjArVGVFdAp1V1l1K2xBR3U1MmJHSWN1emgyOURFaVg4YWEzQlBkU0p2N1RUczVNMnNsSVd0S3ByL0VHM1hiS1Z3SW8yUVdtCngzSUJUaE1NQUJ5Tll5UWQ3WngvYVEwQmg1a3FMdEV3K2lRPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==

oc replace -f api.yaml

title: “Check apiservices”
date: 2021-03-03T13:15:50
slug: check-apiservices

oc get apiservices

title: “Call Api by CLI / OC Command”
date: 2021-03-03T13:14:04
slug: call-api-by-cli-oc-command

oc api-versions
oc get --raw /apis/metrics.k8s.io/v1beta1
oc get --raw /apis/oauth.openshift.io/v1/oauthaccesstokens/e\_qNXwWgq4fWAyibMHELBJMRm5lsGgGS9DLLolPmvng

title: “Delete Custom Prometheus Alerting Rule”
date: 2021-02-24T08:33:03
slug: delete-custom-prometheus-alerting-rule

[root@master02vp ~]# oc get prometheusrule
NAME AGE
deployment-monitoring-rules 96d
prometheus-k8s-rules 1y
prometheus-noris-rules 1y
-------------------------------------------------------------
[root@master02vp ~]# oc delete prometheusrule deployment-monitoring-rules
prometheusrule.monitoring.coreos.com "deployment-monitoring-rules" deleted
[root@master02vp ~]# oc get prometheusrule
NAME AGE
prometheus-k8s-rules 1y
prometheus-noris-rules 1y
-----------------------------------------------------------------------
[root@master02vp ~]# oc delete pod cluster-monitoring-operator-8578656f6f-nrd2w alertmanager-main-2 alertmanager-main-1 alertmanager-main-0 prometheus-k8s-0 prometheus-k8s-1 prometheus-operator-6644b8cd54-n7mjg
pod "cluster-monitoring-operator-8578656f6f-nrd2w" deleted
pod "alertmanager-main-2" deleted
pod "alertmanager-main-1" deleted
pod "alertmanager-main-0" deleted
pod "prometheus-k8s-0" deleted
pod "prometheus-k8s-1" deleted
pod "prometheus-operator-6644b8cd54-n7mjg" deleted

title: “Explanation of replica, count and worker scaling for Red Hat OpenShift Container Storage 4.x”
date: 2021-01-20T17:23:57
slug: explanation-of-replica-count-and-worker-scaling-for-red-hat-openshift-container-storage-4-x

Umgebung

Red Hat OpenShift Container Storage 4.x

Problembeschreibung

Explanation of replica, count and worker scaling for Red Hat OpenShift Container Storage 4.x

When creating the StorageCluster kind object with name ocs-storagecluster to deploy OpenShift Container Storage (OCS), administrators can set spec.storageDeviceSets[0].count and spec.storageDeviceSets[0].replica. What should the values of these fields be set to?

For example:

Raw

cat <<'EOF' | oc apply -f -
apiVersion: ocs.openshift.io/v1
kind: StorageCluster
metadata:
 name: ocs-storagecluster
 namespace: openshift-storage
spec:
 manageNodes: false
 resources:
 mds:
 limits:
 cpu: 3
 memory: 8Gi
 requests:
 cpu: 1
 memory: 8Gi
 noobaa-core:
 limits:
 cpu: 2
 memory: 8Gi
 requests:
 cpu: 1
 memory: 8Gi
 noobaa-db:
 limits:
 cpu: 2
 memory: 8Gi
 requests:
 cpu: 1
 memory: 8Gi
 monDataDirHostPath: /var/lib/rook
 storageDeviceSets:
 - count: 2
 dataPVCTemplate:
 spec:
 accessModes:
 - ReadWriteOnce
 resources:
 requests:
 storage: 2328Gi
 storageClassName: localblock
 volumeMode: Block
 name: ocs-deviceset
 placement: {}
 portable: false
 replica: 3
 resources: {}
EOF

Lösung

Rules for node/disk scaling

• The only supported value for replica: is 3.

• OCS worker nodes must be scaled out in multiples of 3. Meaning that 3,6,9,… OCS worker nodes are o.k., but for example 4 or 5 are not.

• Each OCS worker node must have the same number [number of OCS PVs per worker] of equally sized PVs that can be used for OCS

• count: must be set to [number of OCS worker nodes] * [number of OCS PVs per worker] / 3 (the last 3 is explained by [replica size = 3]

Grundursache

The reason for this lies in the following

• The pools’ replicated size is hard coded to 3 with OCS. If the rack count is not divisible by 3, then the count: parameter must be a multiple of 3. The total number of OSDs needs to be divisible by 3.

• One could in theory change the rack count by changing the replica: parameter, but this negatively influences scale-outs, forcing administrators to add node multiples of rack count (= replica: parameter) at every single scale-out.

• One can only add node capacity in multiples of rack count. Rack count is set by the replica: parameter. The ideal rack count is 3, as this is the smallest supported unit one can have with Ceph due to the pools’ replicated size of 3.

When looking at ocs-deviceset-x-y, replica: controls max x. And count: controls max y.

Each storageDeviceSet will be tied to a specific rack according to replica::

Raw

$ oc get jobs -o name | while read job; do echo === $job === ; oc get $job -o yaml | egrep 'rack[0-9]+' ; done
=== job.batch/rook-ceph-osd-prepare-ocs-deviceset-0-0-nd5jp ===
 - rack0
=== job.batch/rook-ceph-osd-prepare-ocs-deviceset-0-1-ns9jx ===
 - rack0
=== job.batch/rook-ceph-osd-prepare-ocs-deviceset-1-0-qbf59 ===
 - rack1
=== job.batch/rook-ceph-osd-prepare-ocs-deviceset-1-1-j7bs8 ===
 - rack1
=== job.batch/rook-ceph-osd-prepare-ocs-deviceset-2-0-jwshs ===
 - rack2
=== job.batch/rook-ceph-osd-prepare-ocs-deviceset-2-1-49ldb ===
 - rack2

Each node will be evenly distributed into a specific rack:

Raw

$ oc get nodes -l topology.rook.io/rack=rack2
NAME STATUS ROLES AGE VERSION
ip-10-0-202-249.eu-west-1.compute.internal Ready worker 8d v1.17.1+b83bc57
$ oc get nodes -l topology.rook.io/rack=rack0
NAME STATUS ROLES AGE VERSION
ip-10-0-198-152.eu-west-1.compute.internal Ready worker 8d v1.17.1+b83bc57
$ oc get nodes -l topology.rook.io/rack=rack1
NAME STATUS ROLES AGE VERSION
ip-10-0-197-77.eu-west-1.compute.internal Ready worker 8d v1.17.1+b83bc57
$ oc get nodes -l topology.rook.io/rack=rack2
NAME STATUS ROLES AGE VERSION
ip-10-0-202-249.eu-west-1.compute.internal Ready worker 8d v1.17.1+b83bc57

When labeling nodes, nodes will be added to the racks. For example, when adding:

Raw

oc label node/ip-10-0-210-157.eu-west-1.compute.internal cluster.ocs.openshift.io/openshift-storage=''

The node will be added to rack0:

Raw

$ oc get nodes -l topology.rook.io/rack=rack0
NAME STATUS ROLES AGE VERSION
ip-10-0-198-152.eu-west-1.compute.internal Ready worker 8d v1.17.1+b83bc57
ip-10-0-210-157.eu-west-1.compute.internal Ready worker 8d v1.17.1+b83bc57
$ oc get nodes -l topology.rook.io/rack=rack1
NAME STATUS ROLES AGE VERSION
ip-10-0-197-77.eu-west-1.compute.internal Ready worker 8d v1.17.1+b83bc57
$ oc get nodes -l topology.rook.io/rack=rack2
NAME STATUS ROLES AGE VERSION
ip-10-0-202-249.eu-west-1.compute.internal Ready worker 8d v1.17.1+b83bc57

In the above example, 2 more nodes must added for a supported configuration in this case, until each rack shows the same number of nodes.

When increasing count: by 1, one new OSD will be created in each rack, adding a total of 3 OSDs.

If the OCS worker node count cannot be divided by 3, then OCS cannot create all OSDs for the scale-out. The prepare OSD job’s pod will remain in Pending with a message similar to the following:

Raw

$ oc get pods | grep rook-ceph-osd-prepare-ocs-deviceset-2-2-vp5m4-2gmkv
rook-ceph-osd-prepare-ocs-deviceset-2-2-vp5m4-2gmkv 0/1 Pending 0 32m
$ oc describe pod rook-ceph-osd-prepare-ocs-deviceset-2-2-vp5m4-2gmkv | tail -1
 Warning FailedScheduling 22s (x29 over 32m) default-scheduler 0/10 nodes are available: 1 node(s) didn't find available persistent volumes to bind, 9 node(s) didn't match node selector.

title: “RouterCertsDegraded: secret/v4-0-config-system-router-certs.spec.data -n openshift-authentication: certificate could not validate route hostname example.com: x509: certificate signed by unknown authority in OCP4”
date: 2021-01-13T10:34:42
slug: outercertsdegraded-secret-v4-0-config-system-router-certs-spec-data-n-openshift-authentication-certificate-could-not-validate-route-hostname-example-com-x509-certificate-signed-by-unknown-authori

Umgebung

• Red Hat OpenShift Container Platform

• 4.2

Problembeschreibung

• After replacing the default ingress certificate with a custom one, the authentication operator becomes degraded.

Lösung

Just follow the steps from Replacing the default ingress certificate
but in the step Create a secret that contains the wildcard certificate and key the certificate provided should contain the wildcard and the root CA (with the entire chain if there are intermediates).

Create a bundle file with the custom certificate and the chain CA in the following order:

Raw

wildcard certificate
intermediate CA (if available)
root CA

Create the secret using this bundle file and proceed with the next steps from the documentation:

Raw

$ oc create secret tls <certificate> --cert=</path/to/bundle-cert.crt> --key=</path/to/cert.key> -n openshift-ingress

Grundursache

The root CA from the new custom certificate provided is not recognized by the system.

Diagnostische Schritte

The cluster operator is in a degraded state:

Raw

$ oc get co authentication
NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE
authentication 4.2.0 True False True 11d

From a oauth-openshift pod there are logs like:

Raw

...
RouterCertsDegraded: secret/v4-0-config-system-router-certs.spec.data[apps.example.com] -n openshift-authentication: certificate could not validate route hostname oauth-openshift.apps.example.com: x509: certificate signed by unknown authority
...
Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-authentication-operator", Name:"authentication-operator", UID:"<UID>",
APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'OperatorStatusChanged' Status for clusteroperator/authentication changed: Degraded changed from False to True ("RouterCertsDegraded: secret/v4-0-config-system-router-certs.spec.data[apps.example.com] -n openshift-authentication: certificate could not validate route hostname oauth-openshift.apps.example.com: x509: certificate signed by unknown authority")
...

title: “Check Operator Status”
date: 2021-01-13T10:24:05
slug: check-operator-status

oc get co

title: “Openshift – Container Stroage OCS”
date: 2021-01-08T12:26:04
slug: openshift-container-stroage-ocs

Warning alert:
A minimal cluster deployment will be performed.Tech Preview
The selected nodes do not match the OCS storage cluster requirement of an aggregated 30 CPUs and 72 GiB of RAM. If the selection cannot be modified, a minimal cluster will be deployed.

title: “Create Buildconfig and trigger from github”
date: 2020-12-11T10:37:15
slug: create-buildconfig-and-trigger-from-github

kind: BuildConfig
apiVersion: build.openshift.io/v1
metadata:
 name: sec-git
 namespace: sec
 selfLink: /apis/build.openshift.io/v1/namespaces/sec/buildconfigs/sec-git
 uid: 176a09a4-324a-4208-847a-d5fd4c409a18
 resourceVersion: '6483992'
 creationTimestamp: '2020-12-11T08:23:22Z'
 labels:
 app: sec-git
 app.kubernetes.io/component: sec-git
 app.kubernetes.io/instance: sec-git
 app.kubernetes.io/name: php
 app.kubernetes.io/part-of: sec
 app.openshift.io/runtime: php
 app.openshift.io/runtime-version: 7.3-ubi7
 annotations:
 app.openshift.io/vcs-ref: master
 app.openshift.io/vcs-uri: 'https://github.com/xforze/sec.git'
 openshift.io/generated-by: OpenShiftWebConsole
 managedFields:
 - manager: openshift-apiserver
 operation: Update
 apiVersion: build.openshift.io/v1
 time: '2020-12-11T10:10:26Z'
 fieldsType: FieldsV1
 fieldsV1:
 'f:status':
 'f:lastVersion': {}
 - manager: Mozilla
 operation: Update
 apiVersion: build.openshift.io/v1
 time: '2020-12-11T10:34:43Z'
 fieldsType: FieldsV1
 fieldsV1:
 'f:metadata':
 'f:annotations':
 .: {}
 'f:app.openshift.io/vcs-ref': {}
 'f:app.openshift.io/vcs-uri': {}
 'f:openshift.io/generated-by': {}
 'f:labels':
 .: {}
 'f:app': {}
 'f:app.kubernetes.io/component': {}
 'f:app.kubernetes.io/instance': {}
 'f:app.kubernetes.io/name': {}
 'f:app.kubernetes.io/part-of': {}
 'f:app.openshift.io/runtime': {}
 'f:app.openshift.io/runtime-version': {}
 'f:spec':
 'f:output':
 'f:to':
 .: {}
 'f:kind': {}
 'f:name': {}
 'f:runPolicy': {}
 'f:source':
 'f:contextDir': {}
 'f:git':
 .: {}
 'f:uri': {}
 'f:type': {}
 'f:strategy':
 'f:sourceStrategy':
 .: {}
 'f:from':
 .: {}
 'f:kind': {}
 'f:name': {}
 'f:namespace': {}
 'f:type': {}
 'f:triggers': {}
spec:
 triggers:
 - type: GitHub
 github:
 secretReference:
 name: github
 - type: GitHub
 github:
 secretReference:
 name: sec-git-github-webhook-secret
 - type: ImageChange
 imageChange:
 lastTriggeredImageID: >-
 image-registry.openshift-image-registry.svc:5000/openshift/php@sha256:4173a6d7361c1d5d1154b0d24580c4abd2954c9116b9f91e79f297689e9fd9f9
 - type: ConfigChange
 runPolicy: Serial
 source:
 type: Git
 git:
 uri: 'https://github.com/xforze/sec.git'
 contextDir: /
 strategy:
 type: Source
 sourceStrategy:
 from:
 kind: ImageStreamTag
 namespace: openshift
 name: 'php:7.3-ubi7'
 output:
 to:
 kind: ImageStreamTag
 name: 'sec-git:latest'
 resources: {}
 postCommit: {}
 nodeSelector: null
 successfulBuildsHistoryLimit: 5
 failedBuildsHistoryLimit: 5

Create the github Secret

oc create secret generic github --from-literal=WebHookSecretKey=adfawertwetwer

Get the Webhook Url:

oc describe bc sec

Replace with the Secret Value:
https://api.ocp4-thasanger.paas.pop.noris.de:6443/apis/build.openshift.io/v1/namespaces/sec/buildconfigs/sec-git/webhooks//github

Activate Image Trigger in Deploymentconfig:

oc set triggers dc/sec --auto

title: “WARNING: No container image registry has been configured with the server. Automatic builds and deployments may not function.”
date: 2020-12-11T09:04:19
slug: warning-no-container-image-registry-has-been-configured-with-the-server-automatic-builds-and-deployments-may-not-function

If this error message appears, check if the image registry is in managed state:

oc edit configs.imageregistry.operator.openshift.io
apiVersion: imageregistry.operator.openshift.io/v1
kind: Config
metadata:
 creationTimestamp:
 finalizers:
 - imageregistry.operator.openshift.io/finalizer
 generation: 3
 name: cluster
 resourceVersion:
 selfLink:
spec:
 readOnly: false
 disableRedirect: false
 requests:
 read:
 maxInQueue: 0
 maxRunning: 0
 maxWaitInQueue: 0s
 write:
 maxInQueue: 0
 maxRunning: 0
 maxWaitInQueue: 0s
 defaultRoute: true
 managementState: Managed

To let the registry use local storage

oc patch configs.imageregistry.operator.openshift.io cluster --type merge --patch '{"spec":{"storage":{"emptyDir":{}}}}'