Search OpenShift CI

periodic-ci-openshift-release-master-ci-4.10-upgrade-from-stable-4.9-e2e-azure-upgrade (all) - 10 runs, 90% failed, 111% of failures match = 100% impact
#1694201672059850752	junit	3 hours ago
ns/openshift-cluster-node-tuning-operator pod/tuned-cc45v node/ci-op-czq2p38m-253f3-g5kms-worker-eastus23-nblz4 container/tuned - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"tuned-cc45v","generateName":"tuned-","namespace":"openshift-cluster-node-tuning-operator","uid":"85da2dc1-9249-4d55-af9e-7f2d24ee846c","resourceVersion":"64764","creationTimestamp":"2023-08-23T05:35:18Z","labels":{"controller-revision-hash":"6d84d77867","openshift-app":"tuned","pod-template-generation":"2"},"annotations":{"openshift.io/scc":"privileged"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"tuned","uid":"665d4783-bc8f-4cf1-8f46-5b75f83086c9","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-23T05:35:18Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:openshift-app":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"665d4783-bc8f-4cf1-8f46-5b75f83086c9\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"tuned\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"OCP_NODE_NAME\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"RELEASE_VERSION\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RESYNC_PERIOD\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"WATCH_NAMESPACE\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/modprobe.d\"}":{".":{},"f:mountPath":{},"f:name":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/sysconfig\"}":{".":{},"f:mountPath":{},"f:name":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/sysctl.conf\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/sysctl.d\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/systemd\"}":{".":{},"f:mountPath":{},"f:name":{},"f:subPath":{}},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/lib/modules\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/run/systemd/system\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/sys\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/lib/tuned/profiles-data\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/dbus\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostIPC":{},"f:hostNetwork":{},"f:hostPID":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"etc\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"lib-modules\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"run-systemd-system\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"sys\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"var-lib-tuned-profiles-data\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:items":{},"f:name":{},"f:optional":{}},"f:name":{}},"k:{\"name\":\"var-run-dbus\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-23T05:58:09Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-23T05:58:48Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"etc","hostPath":{"path":"/etc","type":"Directory"}},{"name":"sys","hostPath":{"path":"/sys","type":"Directory"}},{"name":"var-run-dbus","hostPath":{"path":"/var/run/dbus","type":"Directory"}},{"name":"run-systemd-system","hostPath":{"path":"/run/systemd/system","type":"Directory"}},{"name":"lib-modules","hostPath":{"path":"/lib/modules","type":"Directory"}},{"name":"host","hostPath":{"path":"/","type":"Directory"}},{"name":"var-lib-tuned-profiles-data","configMap":{"name":"tuned-profiles","items":[{"key":"tuned-profiles-data","path":"tuned-profiles.yaml"}],"defaultMode":420,"optional":true}},{"name":"kube-api-access-vrlss","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"tuned","image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:a46e73a9d6e06a067d466c1a255bec4292c66b4fb7ef2b96fed282df394a17ae","command":["/var/lib/tuned/bin/run","start"],"env":[{"name":"WATCH_NAMESPACE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}},{"name":"OCP_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"RESYNC_PERIOD","value":"600"},{"name":"RELEASE_VERSION","value":"4.10.0-0.ci-2023-08-22-222817"}],"resources":{"requests":{"cpu":"10m","memory":"50Mi"}},"volumeMounts":[{"name":"var-lib-tuned-profiles-data","mountPath":"/var/lib/tuned/profiles-data"},{"name":"etc","mountPath":"/etc/modprobe.d","subPath":"modprobe.d"},{"name":"etc","mountPath":"/etc/sysconfig","subPath":"sysconfig"},{"name":"etc","readOnly":true,"mountPath":"/etc/sysctl.d","subPath":"sysctl.d"},{"name":"etc","readOnly":true,"mountPath":"/etc/sysctl.conf","subPath":"sysctl.conf"},{"name":"etc","mountPath":"/etc/systemd","subPath":"systemd"},{"name":"sys","mountPath":"/sys"},{"name":"var-run-dbus","readOnly":true,"mountPath":"/var/run/dbus"},{"name":"run-systemd-system","readOnly":true,"mountPath":"/run/systemd/system"},{"name":"lib-modules","readOnly":true,"mountPath":"/lib/modules"},{"name":"host","mountPath":"/host"},{"name":"kube-api-access-vrlss","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"lifecycle":{"preStop":{"exec":{"command":["/var/lib/tuned/bin/run","stop"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"tuned","serviceAccount":"tuned","nodeName":"ci-op-czq2p38m-253f3-g5kms-worker-eastus23-nblz4","hostNetwork":true,"hostPID":true,"hostIPC":true,"securityContext":{},"imagePullSecrets":[{"name":"tuned-dockercfg-mphqs"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-czq2p38m-253f3-g5kms-worker-eastus23-nblz4"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-23T05:35:18Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T05:58:09Z","reason":"ContainersNotReady","message":"containers with unready status: [tuned]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T05:58:48Z","reason":"ContainersNotReady","message":"containers with unready status: [tuned]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-23T05:35:18Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-23T05:35:18Z","containerStatuses":[{"name":"tuned","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage1018805216/4\": happened during read: read tcp 10.0.128.5:54960-\u003e18.67.79.214:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:a46e73a9d6e06a067d466c1a255bec4292c66b4fb7ef2b96fed282df394a17ae","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-sdn pod/sdn-hgfks node/ci-op-czq2p38m-253f3-g5kms-worker-eastus23-nblz4 container/sdn - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"sdn-hgfks","generateName":"sdn-","namespace":"openshift-sdn","uid":"ef9a2f4e-dc3e-4767-b112-46d6311048e9","resourceVersion":"64765","creationTimestamp":"2023-08-23T05:41:30Z","labels":{"app":"sdn","component":"network","controller-revision-hash":"d8f6457cd","openshift.io/component":"network","pod-template-generation":"2","type":"infra"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"sdn","uid":"b8b06efa-f53c-4332-9e81-c5f1266bfd17","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-23T05:41:30Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:component":{},"f:controller-revision-hash":{},"f:openshift.io/component":{},"f:pod-template-generation":{},"f:type":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"b8b06efa-f53c-4332-9e81-c5f1266bfd17\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"drop-icmp\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:name":{}}}},"k:{\"name\":\"kube-rbac-proxy\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":9101,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/pki/tls/metrics-certs\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}},"k:{\"name\":\"sdn\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE_IP\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"K8S_NODE_NAME\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"KUBERNETES_SERVICE_HOST\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_PORT\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"OPENSHIFT_DNS_DOMAIN\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":10256,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:readinessProbe":{".":{},"f:exec":{".":{},"f:command":{}},"f:failureThreshold":{},"f:initialDelaySeconds":{},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/config\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/env\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/sysconfig\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host-cni-bin\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/var/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/lib/modules\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/lib/cni/networks/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/dbus/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/kubernetes/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/openvswitch/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:hostPID":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"config\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"env-overrides\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{},"f:optional":{}},"f:name":{}},"k:{\"name\":\"etc-sysconfig\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-bin\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-conf\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-modules\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-slash\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-lib-cni-networks-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-dbus\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-kubernetes\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-ovs\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"sdn-metrics-certs\"}":{".":{},"f:name":{},"f:secret":{".":{},"f:defaultMode":{},"f:optional":{},"f:secretName":{}}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-23T05:58:09Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-23T05:58:48Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"config","configMap":{"name":"sdn-config","defaultMode":420}},{"name":"env-overrides","configMap":{"name":"env-overrides","defaultMode":420,"optional":true}},{"name":"etc-sysconfig","hostPath":{"path":"/etc/sysconfig","type":""}},{"name":"host-modules","hostPath":{"path":"/lib/modules","type":""}},{"name":"host-var-run","hostPath":{"path":"/var/run","type":""}},{"name":"host-run-netns","hostPath":{"path":"/run/netns","type":""}},{"name":"host-var-run-netns","hostPath":{"path":"/var/run/netns","type":""}},{"name":"host-var-run-dbus","hostPath":{"path":"/var/run/dbus","type":""}},{"name":"host-var-run-ovs","hostPath":{"path":"/var/run/openvswitch","type":""}},{"name":"host-var-run-kubernetes","hostPath":{"path":"/var/run/kubernetes","type":""}},{"name":"host-var-run-openshift-sdn","hostPath":{"path":"/var/run/openshift-sdn","type":""}},{"name":"host-slash","hostPath":{"path":"/","type":""}},{"name":"host-cni-bin","hostPath":{"path":"/var/lib/cni/bin","type":""}},{"name":"host-cni-conf","hostPath":{"path":"/var/run/multus/cni/net.d","type":""}},{"name":"host-var-lib-cni-networks-openshift-sdn","hostPath":{"path":"/var/lib/cni/networks/openshift-sdn","type":""}},{"name":"sdn-metrics-certs","secret":{"secretName":"sdn-metrics-certs","defaultMode":420,"optional":true}},{"name":"kube-api-access-x2wb8","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"sdn","image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\n\n# if another process is listening on the cni-server socket, wait until it exits\ntrap 'kill $(jobs -p); rm -f /etc/cni/net.d/80-openshift-network.conf ; exit 0' TERM\nretries=0\nwhile true; do\n if echo 'test' \| socat - UNIX-CONNECT:/var/run/openshift-sdn/cniserver/socket \u0026\u003e/dev/null; then\n echo \"warning: Another process is currently listening on the CNI socket, waiting 15s ...\" 2\u003e\u00261\n sleep 15 \u0026 wait\n (( retries += 1 ))\n else\n break\n fi\n if [[ \"${retries}\" -gt 40 ]]; then\n echo \"error: Another process is currently listening on the CNI socket, exiting\" 2\u003e\u00261\n exit 1\n fi\ndone\n\n# local environment overrides\nif [[ -f /etc/sysconfig/openshift-sdn ]]; then\n set -o allexport\n source /etc/sysconfig/openshift-sdn\n set +o allexport\nfi\n#BUG: cdc accidentally mounted /etc/sysconfig/openshift-sdn as DirectoryOrCreate; clean it up so we can ultimately mount /etc/sysconfig/openshift-sdn as FileOrCreate\n# Once this is released, then we can mount it properly\nif [[ -d /etc/sysconfig/openshift-sdn ]]; then\n rmdir /etc/sysconfig/openshift-sdn \|\| true\nfi\n\n# configmap-based overrides\nif [[ -f /env/${K8S_NODE_NAME} ]]; then\n set -o allexport\n source /env/${K8S_NODE_NAME}\n set +o allexport\nfi\n\n# Take over network functions on the node\nrm -f /etc/cni/net.d/80-openshift-network.conf\ncp -f /opt/cni/bin/openshift-sdn /host-cni-bin/\n\nmtu_override_flag=\nif [[ -f /config/mtu.yaml ]]; then\n mtu_override_flag=\"--mtu-override /config/mtu.yaml\"\nfi\n\n# Launch the network process\nexec /usr/bin/openshift-sdn-node \\\n --node-name ${K8S_NODE_NAME} --node-ip ${K8S_NODE_IP} \\\n --platform-type Azure \\\n --proxy-config /config/kube-proxy-config.yaml \\\n ${mtu_override_flag} \\\n --v ${OPENSHIFT_SDN_LOG_LEVEL:-2}\n"],"ports":[{"name":"healthz","hostPort":10256,"containerPort":10256,"protocol":"TCP"}],"env":[{"name":"KUBERNETES_SERVICE_PORT","value":"6443"},{"name":"KUBERNETES_SERVICE_HOST","value":"api-int.ci-op-czq2p38m-253f3.ci.azure.devcluster.openshift.com"},{"name":"OPENSHIFT_DNS_DOMAIN","value":"cluster.local"},{"name":"K8S_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"K8S_NODE_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.hostIP"}}}],"resources":{"requests":{"cpu":"100m","memory":"200Mi"}},"volumeMounts":[{"name":"config","readOnly":true,"mountPath":"/config"},{"name":"env-overrides","mountPath":"/env"},{"name":"host-var-run","mountPath":"/var/run"},{"name":"host-var-run-dbus","readOnly":true,"mountPath":"/var/run/dbus/"},{"name":"host-var-run-ovs","readOnly":true,"mountPath":"/var/run/openvswitch/"},{"name":"host-var-run-kubernetes","readOnly":true,"mountPath":"/var/run/kubernetes/"},{"name":"host-run-netns","readOnly":true,"mountPath":"/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-netns","readOnly":true,"mountPath":"/host/var/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-openshift-sdn","mountPath":"/var/run/openshift-sdn"},{"name":"host-slash","readOnly":true,"mountPath":"/host","mountPropagation":"HostToContainer"},{"name":"host-cni-bin","mountPath":"/host-cni-bin"},{"name":"host-cni-conf","mountPath":"/etc/cni/net.d"},{"name":"host-var-lib-cni-networks-openshift-sdn","mountPath":"/var/lib/cni/networks/openshift-sdn"},{"name":"host-modules","readOnly":true,"mountPath":"/lib/modules"},{"name":"etc-sysconfig","readOnly":true,"mountPath":"/etc/sysconfig"},{"name":"kube-api-access-x2wb8","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"readinessProbe":{"exec":{"command":["test","-f","/etc/cni/net.d/80-openshift-network.conf"]},"initialDelaySeconds":5,"timeoutSeconds":1,"periodSeconds":5,"successThreshold":1,"failureThreshold":3},"lifecycle":{"preStop":{"exec":{"command":["rm","-f","/etc/cni/net.d/80-openshift-network.conf","/host-cni-bin/openshift-sdn"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}},{"name":"kube-rbac-proxy","image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\nTLS_PK=/etc/pki/tls/metrics-certs/tls.key\nTLS_CERT=/etc/pki/tls/metrics-certs/tls.crt\n\n# As the secret mount is optional we must wait for the files to be present.\n# The service is created in monitor.yaml and this is created in sdn.yaml.\n# If it isn't created there is probably an issue so we want to crashloop.\nTS=$(date +%s)\nWARN_TS=$(( ${TS} + $(( 20 * 60)) ))\nHAS_LOGGED_INFO=0\n\nlog_missing_certs(){\n CUR_TS=$(date +%s)\n if [[ \"${CUR_TS}\" -gt \"WARN_TS\" ]]; then\n echo $(date -Iseconds) WARN: sdn-metrics-certs not mounted after 20 minutes.\n elif [[ \"${HAS_LOGGED_INFO}\" -eq 0 ]] ; then\n echo $(date -Iseconds) INFO: sdn-metrics-certs not mounted. Waiting 20 minutes.\n HAS_LOGGED_INFO=1\n fi\n}\n\nwhile [[ ! -f \"${TLS_PK}\" \|\| ! -f \"${TLS_CERT}\" ]] ; do\n log_missing_certs\n sleep 5\ndone\n\necho $(date -Iseconds) INFO: sdn-metrics-certs mounted, starting kube-rbac-proxy\nexec /usr/bin/kube-rbac-proxy \\\n --logtostderr \\\n --secure-listen-address=:9101 \\\n --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \\\n --upstream=http://127.0.0.1:29101/ \\\n --tls-private-key-file=${TLS_PK} \\\n --tls-cert-file=${TLS_CERT}\n"],"ports":[{"name":"https","hostPort":9101,"containerPort":9101,"protocol":"TCP"}],"resources":{"requests":{"cpu":"10m","memory":"20Mi"}},"volumeMounts":[{"name":"sdn-metrics-certs","readOnly":true,"mountPath":"/etc/pki/tls/metrics-certs"},{"name":"kube-api-access-x2wb8","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent"},{"name":"drop-icmp","image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","set -xe\n\ntouch /var/run/add_iptables.sh\nchmod 0755 /var/run/add_iptables.sh\ncat \u003c\u003c'EOF' \u003e /var/run/add_iptables.sh\n#!/bin/sh\nif [ -z \"$3\" ]\nthen\n echo \"Called with host address missing, ignore\"\n exit 0\nfi\necho \"Adding ICMP drop rule for '$3' \"\nif iptables -C AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nthen\n echo \"iptables already set for $3\"\nelse\n iptables -A AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nfi\nEOF\n\necho \"I$(date \"+%m%d %H:%M:%S.%N\") - drop-icmp - start drop-icmp ${K8S_NODE}\"\niptables -X AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -N AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -F AZURE_CHECK_ICMP_SOURCE\niptables -D INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -I INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE\niptables -N AZURE_ICMP_ACTION \|\| true\niptables -F AZURE_ICMP_ACTION\niptables -A AZURE_ICMP_ACTION -j LOG\niptables -A AZURE_ICMP_ACTION -j DROP\n/host/usr/bin/oc observe pods -n openshift-sdn -l app=sdn -a '{ .status.hostIP }' -- /var/run/add_iptables.sh\n"],"env":[{"name":"K8S_NODE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}}],"resources":{"requests":{"cpu":"5m","memory":"20Mi"}},"volumeMounts":[{"name":"host-slash","mountPath":"/host"},{"name":"kube-api-access-x2wb8","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"lifecycle":{"preStop":{"exec":{"command":["/bin/bash","-c","echo drop-icmp done"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"sdn","serviceAccount":"sdn","nodeName":"ci-op-czq2p38m-253f3-g5kms-worker-eastus23-nblz4","hostNetwork":true,"hostPID":true,"securityContext":{},"imagePullSecrets":[{"name":"sdn-dockercfg-6jttg"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-czq2p38m-253f3-g5kms-worker-eastus23-nblz4"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-23T05:41:30Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T05:58:09Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T05:58:48Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-23T05:41:30Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-23T05:41:30Z","containerStatuses":[{"name":"drop-icmp","state":{"waiting":{"reason":"ImagePullBackOff","message":"Back-off pulling image \"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c\""}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false},{"name":"kube-rbac-proxy","state":{"running":{"startedAt":"2023-08-23T06:04:07Z"}},"lastState":{},"ready":true,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","imageID":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","containerID":"cri-o://c8327cdd1d5577a4c08594fc2ca06d4ecb33f3dda13b896dd78cf16d4b3e5dda","started":true},{"name":"sdn","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage637240159/4\": happened during read: read tcp 10.0.128.5:49018-\u003e18.67.79.178:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-sdn pod/sdn-hgfks node/ci-op-czq2p38m-253f3-g5kms-worker-eastus23-nblz4 container/drop-icmp - reason/ContainerWait cause/ImagePullBackOff unable to calculate container transition time in pod: {"metadata":{"name":"sdn-hgfks","generateName":"sdn-","namespace":"openshift-sdn","uid":"ef9a2f4e-dc3e-4767-b112-46d6311048e9","resourceVersion":"64765","creationTimestamp":"2023-08-23T05:41:30Z","labels":{"app":"sdn","component":"network","controller-revision-hash":"d8f6457cd","openshift.io/component":"network","pod-template-generation":"2","type":"infra"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"sdn","uid":"b8b06efa-f53c-4332-9e81-c5f1266bfd17","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-23T05:41:30Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:component":{},"f:controller-revision-hash":{},"f:openshift.io/component":{},"f:pod-template-generation":{},"f:type":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"b8b06efa-f53c-4332-9e81-c5f1266bfd17\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"drop-icmp\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:name":{}}}},"k:{\"name\":\"kube-rbac-proxy\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":9101,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/pki/tls/metrics-certs\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}},"k:{\"name\":\"sdn\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE_IP\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"K8S_NODE_NAME\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"KUBERNETES_SERVICE_HOST\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_PORT\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"OPENSHIFT_DNS_DOMAIN\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":10256,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:readinessProbe":{".":{},"f:exec":{".":{},"f:command":{}},"f:failureThreshold":{},"f:initialDelaySeconds":{},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/config\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/env\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/sysconfig\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host-cni-bin\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/var/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/lib/modules\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/lib/cni/networks/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/dbus/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/kubernetes/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/openvswitch/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:hostPID":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"config\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"env-overrides\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{},"f:optional":{}},"f:name":{}},"k:{\"name\":\"etc-sysconfig\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-bin\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-conf\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-modules\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-slash\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-lib-cni-networks-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-dbus\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-kubernetes\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-ovs\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"sdn-metrics-certs\"}":{".":{},"f:name":{},"f:secret":{".":{},"f:defaultMode":{},"f:optional":{},"f:secretName":{}}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-23T05:58:09Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-23T05:58:48Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"config","configMap":{"name":"sdn-config","defaultMode":420}},{"name":"env-overrides","configMap":{"name":"env-overrides","defaultMode":420,"optional":true}},{"name":"etc-sysconfig","hostPath":{"path":"/etc/sysconfig","type":""}},{"name":"host-modules","hostPath":{"path":"/lib/modules","type":""}},{"name":"host-var-run","hostPath":{"path":"/var/run","type":""}},{"name":"host-run-netns","hostPath":{"path":"/run/netns","type":""}},{"name":"host-var-run-netns","hostPath":{"path":"/var/run/netns","type":""}},{"name":"host-var-run-dbus","hostPath":{"path":"/var/run/dbus","type":""}},{"name":"host-var-run-ovs","hostPath":{"path":"/var/run/openvswitch","type":""}},{"name":"host-var-run-kubernetes","hostPath":{"path":"/var/run/kubernetes","type":""}},{"name":"host-var-run-openshift-sdn","hostPath":{"path":"/var/run/openshift-sdn","type":""}},{"name":"host-slash","hostPath":{"path":"/","type":""}},{"name":"host-cni-bin","hostPath":{"path":"/var/lib/cni/bin","type":""}},{"name":"host-cni-conf","hostPath":{"path":"/var/run/multus/cni/net.d","type":""}},{"name":"host-var-lib-cni-networks-openshift-sdn","hostPath":{"path":"/var/lib/cni/networks/openshift-sdn","type":""}},{"name":"sdn-metrics-certs","secret":{"secretName":"sdn-metrics-certs","defaultMode":420,"optional":true}},{"name":"kube-api-access-x2wb8","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"sdn","image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\n\n# if another process is listening on the cni-server socket, wait until it exits\ntrap 'kill $(jobs -p); rm -f /etc/cni/net.d/80-openshift-network.conf ; exit 0' TERM\nretries=0\nwhile true; do\n if echo 'test' \| socat - UNIX-CONNECT:/var/run/openshift-sdn/cniserver/socket \u0026\u003e/dev/null; then\n echo \"warning: Another process is currently listening on the CNI socket, waiting 15s ...\" 2\u003e\u00261\n sleep 15 \u0026 wait\n (( retries += 1 ))\n else\n break\n fi\n if [[ \"${retries}\" -gt 40 ]]; then\n echo \"error: Another process is currently listening on the CNI socket, exiting\" 2\u003e\u00261\n exit 1\n fi\ndone\n\n# local environment overrides\nif [[ -f /etc/sysconfig/openshift-sdn ]]; then\n set -o allexport\n source /etc/sysconfig/openshift-sdn\n set +o allexport\nfi\n#BUG: cdc accidentally mounted /etc/sysconfig/openshift-sdn as DirectoryOrCreate; clean it up so we can ultimately mount /etc/sysconfig/openshift-sdn as FileOrCreate\n# Once this is released, then we can mount it properly\nif [[ -d /etc/sysconfig/openshift-sdn ]]; then\n rmdir /etc/sysconfig/openshift-sdn \|\| true\nfi\n\n# configmap-based overrides\nif [[ -f /env/${K8S_NODE_NAME} ]]; then\n set -o allexport\n source /env/${K8S_NODE_NAME}\n set +o allexport\nfi\n\n# Take over network functions on the node\nrm -f /etc/cni/net.d/80-openshift-network.conf\ncp -f /opt/cni/bin/openshift-sdn /host-cni-bin/\n\nmtu_override_flag=\nif [[ -f /config/mtu.yaml ]]; then\n mtu_override_flag=\"--mtu-override /config/mtu.yaml\"\nfi\n\n# Launch the network process\nexec /usr/bin/openshift-sdn-node \\\n --node-name ${K8S_NODE_NAME} --node-ip ${K8S_NODE_IP} \\\n --platform-type Azure \\\n --proxy-config /config/kube-proxy-config.yaml \\\n ${mtu_override_flag} \\\n --v ${OPENSHIFT_SDN_LOG_LEVEL:-2}\n"],"ports":[{"name":"healthz","hostPort":10256,"containerPort":10256,"protocol":"TCP"}],"env":[{"name":"KUBERNETES_SERVICE_PORT","value":"6443"},{"name":"KUBERNETES_SERVICE_HOST","value":"api-int.ci-op-czq2p38m-253f3.ci.azure.devcluster.openshift.com"},{"name":"OPENSHIFT_DNS_DOMAIN","value":"cluster.local"},{"name":"K8S_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"K8S_NODE_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.hostIP"}}}],"resources":{"requests":{"cpu":"100m","memory":"200Mi"}},"volumeMounts":[{"name":"config","readOnly":true,"mountPath":"/config"},{"name":"env-overrides","mountPath":"/env"},{"name":"host-var-run","mountPath":"/var/run"},{"name":"host-var-run-dbus","readOnly":true,"mountPath":"/var/run/dbus/"},{"name":"host-var-run-ovs","readOnly":true,"mountPath":"/var/run/openvswitch/"},{"name":"host-var-run-kubernetes","readOnly":true,"mountPath":"/var/run/kubernetes/"},{"name":"host-run-netns","readOnly":true,"mountPath":"/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-netns","readOnly":true,"mountPath":"/host/var/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-openshift-sdn","mountPath":"/var/run/openshift-sdn"},{"name":"host-slash","readOnly":true,"mountPath":"/host","mountPropagation":"HostToContainer"},{"name":"host-cni-bin","mountPath":"/host-cni-bin"},{"name":"host-cni-conf","mountPath":"/etc/cni/net.d"},{"name":"host-var-lib-cni-networks-openshift-sdn","mountPath":"/var/lib/cni/networks/openshift-sdn"},{"name":"host-modules","readOnly":true,"mountPath":"/lib/modules"},{"name":"etc-sysconfig","readOnly":true,"mountPath":"/etc/sysconfig"},{"name":"kube-api-access-x2wb8","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"readinessProbe":{"exec":{"command":["test","-f","/etc/cni/net.d/80-openshift-network.conf"]},"initialDelaySeconds":5,"timeoutSeconds":1,"periodSeconds":5,"successThreshold":1,"failureThreshold":3},"lifecycle":{"preStop":{"exec":{"command":["rm","-f","/etc/cni/net.d/80-openshift-network.conf","/host-cni-bin/openshift-sdn"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}},{"name":"kube-rbac-proxy","image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\nTLS_PK=/etc/pki/tls/metrics-certs/tls.key\nTLS_CERT=/etc/pki/tls/metrics-certs/tls.crt\n\n# As the secret mount is optional we must wait for the files to be present.\n# The service is created in monitor.yaml and this is created in sdn.yaml.\n# If it isn't created there is probably an issue so we want to crashloop.\nTS=$(date +%s)\nWARN_TS=$(( ${TS} + $(( 20 * 60)) ))\nHAS_LOGGED_INFO=0\n\nlog_missing_certs(){\n CUR_TS=$(date +%s)\n if [[ \"${CUR_TS}\" -gt \"WARN_TS\" ]]; then\n echo $(date -Iseconds) WARN: sdn-metrics-certs not mounted after 20 minutes.\n elif [[ \"${HAS_LOGGED_INFO}\" -eq 0 ]] ; then\n echo $(date -Iseconds) INFO: sdn-metrics-certs not mounted. Waiting 20 minutes.\n HAS_LOGGED_INFO=1\n fi\n}\n\nwhile [[ ! -f \"${TLS_PK}\" \|\| ! -f \"${TLS_CERT}\" ]] ; do\n log_missing_certs\n sleep 5\ndone\n\necho $(date -Iseconds) INFO: sdn-metrics-certs mounted, starting kube-rbac-proxy\nexec /usr/bin/kube-rbac-proxy \\\n --logtostderr \\\n --secure-listen-address=:9101 \\\n --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \\\n --upstream=http://127.0.0.1:29101/ \\\n --tls-private-key-file=${TLS_PK} \\\n --tls-cert-file=${TLS_CERT}\n"],"ports":[{"name":"https","hostPort":9101,"containerPort":9101,"protocol":"TCP"}],"resources":{"requests":{"cpu":"10m","memory":"20Mi"}},"volumeMounts":[{"name":"sdn-metrics-certs","readOnly":true,"mountPath":"/etc/pki/tls/metrics-certs"},{"name":"kube-api-access-x2wb8","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent"},{"name":"drop-icmp","image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","set -xe\n\ntouch /var/run/add_iptables.sh\nchmod 0755 /var/run/add_iptables.sh\ncat \u003c\u003c'EOF' \u003e /var/run/add_iptables.sh\n#!/bin/sh\nif [ -z \"$3\" ]\nthen\n echo \"Called with host address missing, ignore\"\n exit 0\nfi\necho \"Adding ICMP drop rule for '$3' \"\nif iptables -C AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nthen\n echo \"iptables already set for $3\"\nelse\n iptables -A AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nfi\nEOF\n\necho \"I$(date \"+%m%d %H:%M:%S.%N\") - drop-icmp - start drop-icmp ${K8S_NODE}\"\niptables -X AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -N AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -F AZURE_CHECK_ICMP_SOURCE\niptables -D INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -I INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE\niptables -N AZURE_ICMP_ACTION \|\| true\niptables -F AZURE_ICMP_ACTION\niptables -A AZURE_ICMP_ACTION -j LOG\niptables -A AZURE_ICMP_ACTION -j DROP\n/host/usr/bin/oc observe pods -n openshift-sdn -l app=sdn -a '{ .status.hostIP }' -- /var/run/add_iptables.sh\n"],"env":[{"name":"K8S_NODE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}}],"resources":{"requests":{"cpu":"5m","memory":"20Mi"}},"volumeMounts":[{"name":"host-slash","mountPath":"/host"},{"name":"kube-api-access-x2wb8","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"lifecycle":{"preStop":{"exec":{"command":["/bin/bash","-c","echo drop-icmp done"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"sdn","serviceAccount":"sdn","nodeName":"ci-op-czq2p38m-253f3-g5kms-worker-eastus23-nblz4","hostNetwork":true,"hostPID":true,"securityContext":{},"imagePullSecrets":[{"name":"sdn-dockercfg-6jttg"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-czq2p38m-253f3-g5kms-worker-eastus23-nblz4"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-23T05:41:30Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T05:58:09Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T05:58:48Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-23T05:41:30Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-23T05:41:30Z","containerStatuses":[{"name":"drop-icmp","state":{"waiting":{"reason":"ImagePullBackOff","message":"Back-off pulling image \"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c\""}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false},{"name":"kube-rbac-proxy","state":{"running":{"startedAt":"2023-08-23T06:04:07Z"}},"lastState":{},"ready":true,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","imageID":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","containerID":"cri-o://c8327cdd1d5577a4c08594fc2ca06d4ecb33f3dda13b896dd78cf16d4b3e5dda","started":true},{"name":"sdn","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage637240159/4\": happened during read: read tcp 10.0.128.5:49018-\u003e18.67.79.178:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-dns pod/node-resolver-g2w5c node/ci-op-czq2p38m-253f3-g5kms-worker-eastus23-nblz4 container/dns-node-resolver - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"node-resolver-g2w5c","generateName":"node-resolver-","namespace":"openshift-dns","uid":"601705ac-12ad-44b2-9c9e-b4c575ed3899","resourceVersion":"64766","creationTimestamp":"2023-08-23T05:46:52Z","labels":{"controller-revision-hash":"5d6c9f964f","dns.operator.openshift.io/daemonset-node-resolver":"","pod-template-generation":"2"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"node-resolver","uid":"886b76e1-0f6c-45e0-aaeb-13095a2bb489","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-23T05:46:52Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:dns.operator.openshift.io/daemonset-node-resolver":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"886b76e1-0f6c-45e0-aaeb-13095a2bb489\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"dns-node-resolver\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"CLUSTER_DOMAIN\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"NAMESERVER\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"SERVICES\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/hosts\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"hosts-file\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-23T05:58:09Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-23T05:58:48Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"hosts-file","hostPath":{"path":"/etc/hosts","type":"File"}},{"name":"kube-api-access-7zx2w","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"dns-node-resolver","image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:80cc077c52369f7a4e73e7676d03fd7659f3b23e4033c26c3edff4d18096f27c","command":["/bin/bash","-c","#!/bin/bash\nset -uo pipefail\n\ntrap 'jobs -p \| xargs kill \|\| true; wait; exit 0' TERM\n\nOPENSHIFT_MARKER=\"openshift-generated-node-resolver\"\nHOSTS_FILE=\"/etc/hosts\"\nTEMP_FILE=\"/etc/hosts.tmp\"\n\nIFS=', ' read -r -a services \u003c\u003c\u003c \"${SERVICES}\"\n\n# Make a temporary file with the old hosts file's attributes.\ncp -f --attributes-only \"${HOSTS_FILE}\" \"${TEMP_FILE}\"\n\nwhile true; do\n declare -A svc_ips\n for svc in \"${services[@]}\"; do\n # Fetch service IP from cluster dns if present. We make several tries\n # to do it: IPv4, IPv6, IPv4 over TCP and IPv6 over TCP. The two last ones\n # are for deployments with Kuryr on older OpenStack (OSP13) - those do not\n # support UDP loadbalancers and require reaching DNS through TCP.\n cmds=('dig -t A @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"'\n 'dig -t AAAA @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"'\n 'dig -t A +tcp +retry=0 @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"'\n 'dig -t AAAA +tcp +retry=0 @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"')\n for i in ${!cmds[]}\n do\n ips=($(eval \"${cmds[i]}\"))\n if [[ \"$?\" -eq 0 \u0026\u0026 \"${#ips[@]}\" -ne 0 ]]; then\n svc_ips[\"${svc}\"]=\"${ips[@]}\"\n break\n fi\n done\n done\n\n # Update /etc/hosts only if we get valid service IPs\n # We will not update /etc/hosts when there is coredns service outage or api unavailability\n # Stale entries could exist in /etc/hosts if the service is deleted\n if [[ -n \"${svc_ips[]-}\" ]]; then\n # Build a new hosts file from /etc/hosts with our custom entries filtered out\n grep -v \"# ${OPENSHIFT_MARKER}\" \"${HOSTS_FILE}\" \u003e \"${TEMP_FILE}\"\n\n # Append resolver entries for services\n for svc in \"${!svc_ips[@]}\"; do\n for ip in ${svc_ips[${svc}]}; do\n echo \"${ip} ${svc} ${svc}.${CLUSTER_DOMAIN} # ${OPENSHIFT_MARKER}\" \u003e\u003e \"${TEMP_FILE}\"\n done\n done\n\n # TODO: Update /etc/hosts atomically to avoid any inconsistent behavior\n # Replace /etc/hosts with our modified version if needed\n cmp \"${TEMP_FILE}\" \"${HOSTS_FILE}\" \|\| cp -f \"${TEMP_FILE}\" \"${HOSTS_FILE}\"\n # TEMP_FILE is not removed to avoid file create/delete and attributes copy churn\n fi\n sleep 60 \u0026 wait\n unset svc_ips\ndone\n"],"env":[{"name":"SERVICES","value":"image-registry.openshift-image-registry.svc"},{"name":"NAMESERVER","value":"172.30.0.10"},{"name":"CLUSTER_DOMAIN","value":"cluster.local"}],"resources":{"requests":{"cpu":"5m","memory":"21Mi"}},"volumeMounts":[{"name":"hosts-file","mountPath":"/etc/hosts"},{"name":"kube-api-access-7zx2w","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"node-resolver","serviceAccount":"node-resolver","nodeName":"ci-op-czq2p38m-253f3-g5kms-worker-eastus23-nblz4","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"node-resolver-dockercfg-9zrdq"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-czq2p38m-253f3-g5kms-worker-eastus23-nblz4"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-23T05:46:52Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T05:58:09Z","reason":"ContainersNotReady","message":"containers with unready status: [dns-node-resolver]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T05:58:48Z","reason":"ContainersNotReady","message":"containers with unready status: [dns-node-resolver]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-23T05:46:52Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-23T05:46:52Z","containerStatuses":[{"name":"dns-node-resolver","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage2955257692/1\": happened during read: read tcp 10.0.128.5:55004-\u003e18.67.79.214:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:80cc077c52369f7a4e73e7676d03fd7659f3b23e4033c26c3edff4d18096f27c","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-cluster-csi-drivers pod/azure-disk-csi-driver-node-nhdr9 node/ci-op-czq2p38m-253f3-g5kms-worker-eastus23-nblz4 container/azure-inject-credentials - reason/ContainerWait cause/ErrImagePull duration/1799.14s rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage1607325213/1": happened during read: read tcp 10.0.128.5:54928->18.67.79.214:443: read: connection timed out
#1693747503431684096	junit	33 hours ago
ns/openshift-machine-config-operator pod/machine-config-daemon-phnms node/ci-op-g60kx8lh-253f3-kprg9-worker-eastus2-dwrlr container/oauth-proxy - reason/ContainerWait cause/ErrImagePull duration/953.43s rpc error: code = Unknown desc = reading blob sha256:7174a0ad1981455258e04e490057d880cac4c4f2134fcf62682f458d0a13a39a: Get "https://quay.io/v2/openshift-release-dev/ocp-v4.0-art-dev/blobs/sha256:7174a0ad1981455258e04e490057d880cac4c4f2134fcf62682f458d0a13a39a": read tcp 10.0.128.5:59928->54.80.206.196:443: read: connection timed out ns/openshift-sdn pod/sdn-jfd4p node/ci-op-g60kx8lh-253f3-kprg9-worker-eastus3-kbncf container/sdn - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"sdn-jfd4p","generateName":"sdn-","namespace":"openshift-sdn","uid":"2e36cff3-85e4-474c-a1e7-a367964856fb","resourceVersion":"59967","creationTimestamp":"2023-08-21T23:31:46Z","labels":{"app":"sdn","component":"network","controller-revision-hash":"686c9456cf","openshift.io/component":"network","pod-template-generation":"2","type":"infra"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"sdn","uid":"21d5dda7-13e0-4fe0-b339-7b26a214d479","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T23:31:46Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:component":{},"f:controller-revision-hash":{},"f:openshift.io/component":{},"f:pod-template-generation":{},"f:type":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"21d5dda7-13e0-4fe0-b339-7b26a214d479\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"drop-icmp\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:name":{}}}},"k:{\"name\":\"kube-rbac-proxy\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":9101,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/pki/tls/metrics-certs\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}},"k:{\"name\":\"sdn\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE_IP\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"K8S_NODE_NAME\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"KUBERNETES_SERVICE_HOST\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_PORT\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"OPENSHIFT_DNS_DOMAIN\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":10256,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:readinessProbe":{".":{},"f:exec":{".":{},"f:command":{}},"f:failureThreshold":{},"f:initialDelaySeconds":{},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/config\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/env\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/sysconfig\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host-cni-bin\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/var/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/lib/modules\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/lib/cni/networks/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/dbus/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/kubernetes/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/openvswitch/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:hostPID":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"config\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"env-overrides\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{},"f:optional":{}},"f:name":{}},"k:{\"name\":\"etc-sysconfig\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-bin\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-conf\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-modules\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-slash\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-lib-cni-networks-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-dbus\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-kubernetes\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-ovs\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"sdn-metrics-certs\"}":{".":{},"f:name":{},"f:secret":{".":{},"f:defaultMode":{},"f:optional":{},"f:secretName":{}}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T23:46:22Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-21T23:46:56Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.4\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"config","configMap":{"name":"sdn-config","defaultMode":420}},{"name":"env-overrides","configMap":{"name":"env-overrides","defaultMode":420,"optional":true}},{"name":"etc-sysconfig","hostPath":{"path":"/etc/sysconfig","type":""}},{"name":"host-modules","hostPath":{"path":"/lib/modules","type":""}},{"name":"host-var-run","hostPath":{"path":"/var/run","type":""}},{"name":"host-run-netns","hostPath":{"path":"/run/netns","type":""}},{"name":"host-var-run-netns","hostPath":{"path":"/var/run/netns","type":""}},{"name":"host-var-run-dbus","hostPath":{"path":"/var/run/dbus","type":""}},{"name":"host-var-run-ovs","hostPath":{"path":"/var/run/openvswitch","type":""}},{"name":"host-var-run-kubernetes","hostPath":{"path":"/var/run/kubernetes","type":""}},{"name":"host-var-run-openshift-sdn","hostPath":{"path":"/var/run/openshift-sdn","type":""}},{"name":"host-slash","hostPath":{"path":"/","type":""}},{"name":"host-cni-bin","hostPath":{"path":"/var/lib/cni/bin","type":""}},{"name":"host-cni-conf","hostPath":{"path":"/var/run/multus/cni/net.d","type":""}},{"name":"host-var-lib-cni-networks-openshift-sdn","hostPath":{"path":"/var/lib/cni/networks/openshift-sdn","type":""}},{"name":"sdn-metrics-certs","secret":{"secretName":"sdn-metrics-certs","defaultMode":420,"optional":true}},{"name":"kube-api-access-g2bw9","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"sdn","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\n\n# if another process is listening on the cni-server socket, wait until it exits\ntrap 'kill $(jobs -p); rm -f /etc/cni/net.d/80-openshift-network.conf ; exit 0' TERM\nretries=0\nwhile true; do\n if echo 'test' \| socat - UNIX-CONNECT:/var/run/openshift-sdn/cniserver/socket \u0026\u003e/dev/null; then\n echo \"warning: Another process is currently listening on the CNI socket, waiting 15s ...\" 2\u003e\u00261\n sleep 15 \u0026 wait\n (( retries += 1 ))\n else\n break\n fi\n if [[ \"${retries}\" -gt 40 ]]; then\n echo \"error: Another process is currently listening on the CNI socket, exiting\" 2\u003e\u00261\n exit 1\n fi\ndone\n\n# local environment overrides\nif [[ -f /etc/sysconfig/openshift-sdn ]]; then\n set -o allexport\n source /etc/sysconfig/openshift-sdn\n set +o allexport\nfi\n#BUG: cdc accidentally mounted /etc/sysconfig/openshift-sdn as DirectoryOrCreate; clean it up so we can ultimately mount /etc/sysconfig/openshift-sdn as FileOrCreate\n# Once this is released, then we can mount it properly\nif [[ -d /etc/sysconfig/openshift-sdn ]]; then\n rmdir /etc/sysconfig/openshift-sdn \|\| true\nfi\n\n# configmap-based overrides\nif [[ -f /env/${K8S_NODE_NAME} ]]; then\n set -o allexport\n source /env/${K8S_NODE_NAME}\n set +o allexport\nfi\n\n# Take over network functions on the node\nrm -f /etc/cni/net.d/80-openshift-network.conf\ncp -f /opt/cni/bin/openshift-sdn /host-cni-bin/\n\nmtu_override_flag=\nif [[ -f /config/mtu.yaml ]]; then\n mtu_override_flag=\"--mtu-override /config/mtu.yaml\"\nfi\n\n# Launch the network process\nexec /usr/bin/openshift-sdn-node \\\n --node-name ${K8S_NODE_NAME} --node-ip ${K8S_NODE_IP} \\\n --platform-type Azure \\\n --proxy-config /config/kube-proxy-config.yaml \\\n ${mtu_override_flag} \\\n --v ${OPENSHIFT_SDN_LOG_LEVEL:-2}\n"],"ports":[{"name":"healthz","hostPort":10256,"containerPort":10256,"protocol":"TCP"}],"env":[{"name":"KUBERNETES_SERVICE_PORT","value":"6443"},{"name":"KUBERNETES_SERVICE_HOST","value":"api-int.ci-op-g60kx8lh-253f3.ci.azure.devcluster.openshift.com"},{"name":"OPENSHIFT_DNS_DOMAIN","value":"cluster.local"},{"name":"K8S_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"K8S_NODE_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.hostIP"}}}],"resources":{"requests":{"cpu":"100m","memory":"200Mi"}},"volumeMounts":[{"name":"config","readOnly":true,"mountPath":"/config"},{"name":"env-overrides","mountPath":"/env"},{"name":"host-var-run","mountPath":"/var/run"},{"name":"host-var-run-dbus","readOnly":true,"mountPath":"/var/run/dbus/"},{"name":"host-var-run-ovs","readOnly":true,"mountPath":"/var/run/openvswitch/"},{"name":"host-var-run-kubernetes","readOnly":true,"mountPath":"/var/run/kubernetes/"},{"name":"host-run-netns","readOnly":true,"mountPath":"/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-netns","readOnly":true,"mountPath":"/host/var/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-openshift-sdn","mountPath":"/var/run/openshift-sdn"},{"name":"host-slash","readOnly":true,"mountPath":"/host","mountPropagation":"HostToContainer"},{"name":"host-cni-bin","mountPath":"/host-cni-bin"},{"name":"host-cni-conf","mountPath":"/etc/cni/net.d"},{"name":"host-var-lib-cni-networks-openshift-sdn","mountPath":"/var/lib/cni/networks/openshift-sdn"},{"name":"host-modules","readOnly":true,"mountPath":"/lib/modules"},{"name":"etc-sysconfig","readOnly":true,"mountPath":"/etc/sysconfig"},{"name":"kube-api-access-g2bw9","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"readinessProbe":{"exec":{"command":["test","-f","/etc/cni/net.d/80-openshift-network.conf"]},"initialDelaySeconds":5,"timeoutSeconds":1,"periodSeconds":5,"successThreshold":1,"failureThreshold":3},"lifecycle":{"preStop":{"exec":{"command":["rm","-f","/etc/cni/net.d/80-openshift-network.conf","/host-cni-bin/openshift-sdn"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}},{"name":"kube-rbac-proxy","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\nTLS_PK=/etc/pki/tls/metrics-certs/tls.key\nTLS_CERT=/etc/pki/tls/metrics-certs/tls.crt\n\n# As the secret mount is optional we must wait for the files to be present.\n# The service is created in monitor.yaml and this is created in sdn.yaml.\n# If it isn't created there is probably an issue so we want to crashloop.\nTS=$(date +%s)\nWARN_TS=$(( ${TS} + $(( 20 * 60)) ))\nHAS_LOGGED_INFO=0\n\nlog_missing_certs(){\n CUR_TS=$(date +%s)\n if [[ \"${CUR_TS}\" -gt \"WARN_TS\" ]]; then\n echo $(date -Iseconds) WARN: sdn-metrics-certs not mounted after 20 minutes.\n elif [[ \"${HAS_LOGGED_INFO}\" -eq 0 ]] ; then\n echo $(date -Iseconds) INFO: sdn-metrics-certs not mounted. Waiting 20 minutes.\n HAS_LOGGED_INFO=1\n fi\n}\n\nwhile [[ ! -f \"${TLS_PK}\" \|\| ! -f \"${TLS_CERT}\" ]] ; do\n log_missing_certs\n sleep 5\ndone\n\necho $(date -Iseconds) INFO: sdn-metrics-certs mounted, starting kube-rbac-proxy\nexec /usr/bin/kube-rbac-proxy \\\n --logtostderr \\\n --secure-listen-address=:9101 \\\n --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \\\n --upstream=http://127.0.0.1:29101/ \\\n --tls-private-key-file=${TLS_PK} \\\n --tls-cert-file=${TLS_CERT}\n"],"ports":[{"name":"https","hostPort":9101,"containerPort":9101,"protocol":"TCP"}],"resources":{"requests":{"cpu":"10m","memory":"20Mi"}},"volumeMounts":[{"name":"sdn-metrics-certs","readOnly":true,"mountPath":"/etc/pki/tls/metrics-certs"},{"name":"kube-api-access-g2bw9","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent"},{"name":"drop-icmp","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","set -xe\n\ntouch /var/run/add_iptables.sh\nchmod 0755 /var/run/add_iptables.sh\ncat \u003c\u003c'EOF' \u003e /var/run/add_iptables.sh\n#!/bin/sh\nif [ -z \"$3\" ]\nthen\n echo \"Called with host address missing, ignore\"\n exit 0\nfi\necho \"Adding ICMP drop rule for '$3' \"\nif iptables -C AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nthen\n echo \"iptables already set for $3\"\nelse\n iptables -A AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nfi\nEOF\n\necho \"I$(date \"+%m%d %H:%M:%S.%N\") - drop-icmp - start drop-icmp ${K8S_NODE}\"\niptables -X AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -N AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -F AZURE_CHECK_ICMP_SOURCE\niptables -D INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -I INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE\niptables -N AZURE_ICMP_ACTION \|\| true\niptables -F AZURE_ICMP_ACTION\niptables -A AZURE_ICMP_ACTION -j LOG\niptables -A AZURE_ICMP_ACTION -j DROP\n/host/usr/bin/oc observe pods -n openshift-sdn -l app=sdn -a '{ .status.hostIP }' -- /var/run/add_iptables.sh\n"],"env":[{"name":"K8S_NODE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}}],"resources":{"requests":{"cpu":"5m","memory":"20Mi"}},"volumeMounts":[{"name":"host-slash","mountPath":"/host"},{"name":"kube-api-access-g2bw9","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"lifecycle":{"preStop":{"exec":{"command":["/bin/bash","-c","echo drop-icmp done"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"sdn","serviceAccount":"sdn","nodeName":"ci-op-g60kx8lh-253f3-kprg9-worker-eastus3-kbncf","hostNetwork":true,"hostPID":true,"securityContext":{},"imagePullSecrets":[{"name":"sdn-dockercfg-j8cfm"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-g60kx8lh-253f3-kprg9-worker-eastus3-kbncf"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T23:31:46Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T23:46:22Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T23:46:56Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T23:31:46Z"}],"hostIP":"10.0.128.4","podIP":"10.0.128.4","podIPs":[{"ip":"10.0.128.4"}],"startTime":"2023-08-21T23:31:46Z","containerStatuses":[{"name":"drop-icmp","state":{"waiting":{"reason":"ImagePullBackOff","message":"Back-off pulling image \"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c\""}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false},{"name":"kube-rbac-proxy","state":{"running":{"startedAt":"2023-08-21T23:52:16Z"}},"lastState":{},"ready":true,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","imageID":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","containerID":"cri-o://0a75e7e331d2c5baacf586c81541eb7d0f79a0da1fa4f5a178b8b257ef5a9954","started":true},{"name":"sdn","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage2351272120/1\": happened during read: read tcp 10.0.128.4:52612-\u003e18.67.79.208:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-sdn pod/sdn-jfd4p node/ci-op-g60kx8lh-253f3-kprg9-worker-eastus3-kbncf container/drop-icmp - reason/ContainerWait cause/ImagePullBackOff unable to calculate container transition time in pod: {"metadata":{"name":"sdn-jfd4p","generateName":"sdn-","namespace":"openshift-sdn","uid":"2e36cff3-85e4-474c-a1e7-a367964856fb","resourceVersion":"59967","creationTimestamp":"2023-08-21T23:31:46Z","labels":{"app":"sdn","component":"network","controller-revision-hash":"686c9456cf","openshift.io/component":"network","pod-template-generation":"2","type":"infra"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"sdn","uid":"21d5dda7-13e0-4fe0-b339-7b26a214d479","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T23:31:46Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:component":{},"f:controller-revision-hash":{},"f:openshift.io/component":{},"f:pod-template-generation":{},"f:type":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"21d5dda7-13e0-4fe0-b339-7b26a214d479\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"drop-icmp\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:name":{}}}},"k:{\"name\":\"kube-rbac-proxy\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":9101,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/pki/tls/metrics-certs\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}},"k:{\"name\":\"sdn\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE_IP\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"K8S_NODE_NAME\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"KUBERNETES_SERVICE_HOST\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_PORT\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"OPENSHIFT_DNS_DOMAIN\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":10256,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:readinessProbe":{".":{},"f:exec":{".":{},"f:command":{}},"f:failureThreshold":{},"f:initialDelaySeconds":{},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/config\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/env\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/sysconfig\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host-cni-bin\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/var/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/lib/modules\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/lib/cni/networks/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/dbus/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/kubernetes/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/openvswitch/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:hostPID":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"config\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"env-overrides\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{},"f:optional":{}},"f:name":{}},"k:{\"name\":\"etc-sysconfig\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-bin\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-conf\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-modules\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-slash\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-lib-cni-networks-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-dbus\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-kubernetes\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-ovs\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"sdn-metrics-certs\"}":{".":{},"f:name":{},"f:secret":{".":{},"f:defaultMode":{},"f:optional":{},"f:secretName":{}}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T23:46:22Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-21T23:46:56Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.4\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"config","configMap":{"name":"sdn-config","defaultMode":420}},{"name":"env-overrides","configMap":{"name":"env-overrides","defaultMode":420,"optional":true}},{"name":"etc-sysconfig","hostPath":{"path":"/etc/sysconfig","type":""}},{"name":"host-modules","hostPath":{"path":"/lib/modules","type":""}},{"name":"host-var-run","hostPath":{"path":"/var/run","type":""}},{"name":"host-run-netns","hostPath":{"path":"/run/netns","type":""}},{"name":"host-var-run-netns","hostPath":{"path":"/var/run/netns","type":""}},{"name":"host-var-run-dbus","hostPath":{"path":"/var/run/dbus","type":""}},{"name":"host-var-run-ovs","hostPath":{"path":"/var/run/openvswitch","type":""}},{"name":"host-var-run-kubernetes","hostPath":{"path":"/var/run/kubernetes","type":""}},{"name":"host-var-run-openshift-sdn","hostPath":{"path":"/var/run/openshift-sdn","type":""}},{"name":"host-slash","hostPath":{"path":"/","type":""}},{"name":"host-cni-bin","hostPath":{"path":"/var/lib/cni/bin","type":""}},{"name":"host-cni-conf","hostPath":{"path":"/var/run/multus/cni/net.d","type":""}},{"name":"host-var-lib-cni-networks-openshift-sdn","hostPath":{"path":"/var/lib/cni/networks/openshift-sdn","type":""}},{"name":"sdn-metrics-certs","secret":{"secretName":"sdn-metrics-certs","defaultMode":420,"optional":true}},{"name":"kube-api-access-g2bw9","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"sdn","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\n\n# if another process is listening on the cni-server socket, wait until it exits\ntrap 'kill $(jobs -p); rm -f /etc/cni/net.d/80-openshift-network.conf ; exit 0' TERM\nretries=0\nwhile true; do\n if echo 'test' \| socat - UNIX-CONNECT:/var/run/openshift-sdn/cniserver/socket \u0026\u003e/dev/null; then\n echo \"warning: Another process is currently listening on the CNI socket, waiting 15s ...\" 2\u003e\u00261\n sleep 15 \u0026 wait\n (( retries += 1 ))\n else\n break\n fi\n if [[ \"${retries}\" -gt 40 ]]; then\n echo \"error: Another process is currently listening on the CNI socket, exiting\" 2\u003e\u00261\n exit 1\n fi\ndone\n\n# local environment overrides\nif [[ -f /etc/sysconfig/openshift-sdn ]]; then\n set -o allexport\n source /etc/sysconfig/openshift-sdn\n set +o allexport\nfi\n#BUG: cdc accidentally mounted /etc/sysconfig/openshift-sdn as DirectoryOrCreate; clean it up so we can ultimately mount /etc/sysconfig/openshift-sdn as FileOrCreate\n# Once this is released, then we can mount it properly\nif [[ -d /etc/sysconfig/openshift-sdn ]]; then\n rmdir /etc/sysconfig/openshift-sdn \|\| true\nfi\n\n# configmap-based overrides\nif [[ -f /env/${K8S_NODE_NAME} ]]; then\n set -o allexport\n source /env/${K8S_NODE_NAME}\n set +o allexport\nfi\n\n# Take over network functions on the node\nrm -f /etc/cni/net.d/80-openshift-network.conf\ncp -f /opt/cni/bin/openshift-sdn /host-cni-bin/\n\nmtu_override_flag=\nif [[ -f /config/mtu.yaml ]]; then\n mtu_override_flag=\"--mtu-override /config/mtu.yaml\"\nfi\n\n# Launch the network process\nexec /usr/bin/openshift-sdn-node \\\n --node-name ${K8S_NODE_NAME} --node-ip ${K8S_NODE_IP} \\\n --platform-type Azure \\\n --proxy-config /config/kube-proxy-config.yaml \\\n ${mtu_override_flag} \\\n --v ${OPENSHIFT_SDN_LOG_LEVEL:-2}\n"],"ports":[{"name":"healthz","hostPort":10256,"containerPort":10256,"protocol":"TCP"}],"env":[{"name":"KUBERNETES_SERVICE_PORT","value":"6443"},{"name":"KUBERNETES_SERVICE_HOST","value":"api-int.ci-op-g60kx8lh-253f3.ci.azure.devcluster.openshift.com"},{"name":"OPENSHIFT_DNS_DOMAIN","value":"cluster.local"},{"name":"K8S_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"K8S_NODE_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.hostIP"}}}],"resources":{"requests":{"cpu":"100m","memory":"200Mi"}},"volumeMounts":[{"name":"config","readOnly":true,"mountPath":"/config"},{"name":"env-overrides","mountPath":"/env"},{"name":"host-var-run","mountPath":"/var/run"},{"name":"host-var-run-dbus","readOnly":true,"mountPath":"/var/run/dbus/"},{"name":"host-var-run-ovs","readOnly":true,"mountPath":"/var/run/openvswitch/"},{"name":"host-var-run-kubernetes","readOnly":true,"mountPath":"/var/run/kubernetes/"},{"name":"host-run-netns","readOnly":true,"mountPath":"/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-netns","readOnly":true,"mountPath":"/host/var/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-openshift-sdn","mountPath":"/var/run/openshift-sdn"},{"name":"host-slash","readOnly":true,"mountPath":"/host","mountPropagation":"HostToContainer"},{"name":"host-cni-bin","mountPath":"/host-cni-bin"},{"name":"host-cni-conf","mountPath":"/etc/cni/net.d"},{"name":"host-var-lib-cni-networks-openshift-sdn","mountPath":"/var/lib/cni/networks/openshift-sdn"},{"name":"host-modules","readOnly":true,"mountPath":"/lib/modules"},{"name":"etc-sysconfig","readOnly":true,"mountPath":"/etc/sysconfig"},{"name":"kube-api-access-g2bw9","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"readinessProbe":{"exec":{"command":["test","-f","/etc/cni/net.d/80-openshift-network.conf"]},"initialDelaySeconds":5,"timeoutSeconds":1,"periodSeconds":5,"successThreshold":1,"failureThreshold":3},"lifecycle":{"preStop":{"exec":{"command":["rm","-f","/etc/cni/net.d/80-openshift-network.conf","/host-cni-bin/openshift-sdn"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}},{"name":"kube-rbac-proxy","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\nTLS_PK=/etc/pki/tls/metrics-certs/tls.key\nTLS_CERT=/etc/pki/tls/metrics-certs/tls.crt\n\n# As the secret mount is optional we must wait for the files to be present.\n# The service is created in monitor.yaml and this is created in sdn.yaml.\n# If it isn't created there is probably an issue so we want to crashloop.\nTS=$(date +%s)\nWARN_TS=$(( ${TS} + $(( 20 * 60)) ))\nHAS_LOGGED_INFO=0\n\nlog_missing_certs(){\n CUR_TS=$(date +%s)\n if [[ \"${CUR_TS}\" -gt \"WARN_TS\" ]]; then\n echo $(date -Iseconds) WARN: sdn-metrics-certs not mounted after 20 minutes.\n elif [[ \"${HAS_LOGGED_INFO}\" -eq 0 ]] ; then\n echo $(date -Iseconds) INFO: sdn-metrics-certs not mounted. Waiting 20 minutes.\n HAS_LOGGED_INFO=1\n fi\n}\n\nwhile [[ ! -f \"${TLS_PK}\" \|\| ! -f \"${TLS_CERT}\" ]] ; do\n log_missing_certs\n sleep 5\ndone\n\necho $(date -Iseconds) INFO: sdn-metrics-certs mounted, starting kube-rbac-proxy\nexec /usr/bin/kube-rbac-proxy \\\n --logtostderr \\\n --secure-listen-address=:9101 \\\n --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \\\n --upstream=http://127.0.0.1:29101/ \\\n --tls-private-key-file=${TLS_PK} \\\n --tls-cert-file=${TLS_CERT}\n"],"ports":[{"name":"https","hostPort":9101,"containerPort":9101,"protocol":"TCP"}],"resources":{"requests":{"cpu":"10m","memory":"20Mi"}},"volumeMounts":[{"name":"sdn-metrics-certs","readOnly":true,"mountPath":"/etc/pki/tls/metrics-certs"},{"name":"kube-api-access-g2bw9","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent"},{"name":"drop-icmp","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","set -xe\n\ntouch /var/run/add_iptables.sh\nchmod 0755 /var/run/add_iptables.sh\ncat \u003c\u003c'EOF' \u003e /var/run/add_iptables.sh\n#!/bin/sh\nif [ -z \"$3\" ]\nthen\n echo \"Called with host address missing, ignore\"\n exit 0\nfi\necho \"Adding ICMP drop rule for '$3' \"\nif iptables -C AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nthen\n echo \"iptables already set for $3\"\nelse\n iptables -A AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nfi\nEOF\n\necho \"I$(date \"+%m%d %H:%M:%S.%N\") - drop-icmp - start drop-icmp ${K8S_NODE}\"\niptables -X AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -N AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -F AZURE_CHECK_ICMP_SOURCE\niptables -D INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -I INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE\niptables -N AZURE_ICMP_ACTION \|\| true\niptables -F AZURE_ICMP_ACTION\niptables -A AZURE_ICMP_ACTION -j LOG\niptables -A AZURE_ICMP_ACTION -j DROP\n/host/usr/bin/oc observe pods -n openshift-sdn -l app=sdn -a '{ .status.hostIP }' -- /var/run/add_iptables.sh\n"],"env":[{"name":"K8S_NODE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}}],"resources":{"requests":{"cpu":"5m","memory":"20Mi"}},"volumeMounts":[{"name":"host-slash","mountPath":"/host"},{"name":"kube-api-access-g2bw9","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"lifecycle":{"preStop":{"exec":{"command":["/bin/bash","-c","echo drop-icmp done"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"sdn","serviceAccount":"sdn","nodeName":"ci-op-g60kx8lh-253f3-kprg9-worker-eastus3-kbncf","hostNetwork":true,"hostPID":true,"securityContext":{},"imagePullSecrets":[{"name":"sdn-dockercfg-j8cfm"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-g60kx8lh-253f3-kprg9-worker-eastus3-kbncf"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T23:31:46Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T23:46:22Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T23:46:56Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T23:31:46Z"}],"hostIP":"10.0.128.4","podIP":"10.0.128.4","podIPs":[{"ip":"10.0.128.4"}],"startTime":"2023-08-21T23:31:46Z","containerStatuses":[{"name":"drop-icmp","state":{"waiting":{"reason":"ImagePullBackOff","message":"Back-off pulling image \"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c\""}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false},{"name":"kube-rbac-proxy","state":{"running":{"startedAt":"2023-08-21T23:52:16Z"}},"lastState":{},"ready":true,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","imageID":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","containerID":"cri-o://0a75e7e331d2c5baacf586c81541eb7d0f79a0da1fa4f5a178b8b257ef5a9954","started":true},{"name":"sdn","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage2351272120/1\": happened during read: read tcp 10.0.128.4:52612-\u003e18.67.79.208:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-monitoring pod/node-exporter-6zlxp node/ci-op-g60kx8lh-253f3-kprg9-worker-eastus1-brknc container/init-textfile - reason/ContainerWait cause/ErrImagePull duration/2217.46s rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage3317448148/1": happened during read: read tcp 10.0.128.6:35020->18.67.79.73:443: read: connection timed out ns/openshift-image-registry pod/node-ca-k78t2 node/ci-op-g60kx8lh-253f3-kprg9-worker-eastus1-brknc container/node-ca - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"node-ca-k78t2","generateName":"node-ca-","namespace":"openshift-image-registry","uid":"d02de284-ff3b-4c6b-a1db-acd6bfc02b21","resourceVersion":"67691","creationTimestamp":"2023-08-21T23:27:19Z","labels":{"controller-revision-hash":"8c9ffcc77","name":"node-ca","pod-template-generation":"2"},"annotations":{"openshift.io/scc":"privileged"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"node-ca","uid":"af0aa3b9-200d-453d-856c-acb7e2803959","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T23:27:19Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:name":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"af0aa3b9-200d-453d-856c-acb7e2803959\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"node-ca\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{},"f:runAsGroup":{},"f:runAsUser":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/docker/certs.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/tmp/serviceca\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"host\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"serviceca\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T23:57:14Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-21T23:57:46Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.6\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"host","hostPath":{"path":"/etc/docker/certs.d","type":""}},{"name":"serviceca","configMap":{"name":"image-registry-certificates","defaultMode":420}},{"name":"kube-api-access-dj2pn","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"node-ca","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:26b626f9b0de3609f607fd85f4fb6865880dc100273846fcc8947cbcc8a2840c","command":["/bin/sh","-c","trap 'jobs -p \| xargs -r kill; echo shutting down node-ca; exit 0' TERM\nwhile [ true ];\ndo\n for f in $(ls /tmp/serviceca); do\n echo $f\n ca_file_path=\"/tmp/serviceca/${f}\"\n f=$(echo $f \| sed -r 's/(.)\\.\\./\\1:/')\n reg_dir_path=\"/etc/docker/certs.d/${f}\"\n if [ -e \"${reg_dir_path}\" ]; then\n cp -u $ca_file_path $reg_dir_path/ca.crt\n else\n mkdir $reg_dir_path\n cp $ca_file_path $reg_dir_path/ca.crt\n fi\n done\n for d in $(ls /etc/docker/certs.d); do\n echo $d\n dp=$(echo $d \| sed -r 's/(.):/\\1\\.\\./')\n reg_conf_path=\"/tmp/serviceca/${dp}\"\n if [ ! -e \"${reg_conf_path}\" ]; then\n rm -rf /etc/docker/certs.d/$d\n fi\n done\n sleep 60 \u0026 wait ${!}\ndone\n"],"resources":{"requests":{"cpu":"10m","memory":"10Mi"}},"volumeMounts":[{"name":"serviceca","mountPath":"/tmp/serviceca"},{"name":"host","mountPath":"/etc/docker/certs.d"},{"name":"kube-api-access-dj2pn","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true,"runAsUser":1001,"runAsGroup":0}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"node-ca","serviceAccount":"node-ca","nodeName":"ci-op-g60kx8lh-253f3-kprg9-worker-eastus1-brknc","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"node-ca-dockercfg-8dh8c"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-g60kx8lh-253f3-kprg9-worker-eastus1-brknc"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-cluster-critical","priority":2000000000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T23:27:19Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T23:57:14Z","reason":"ContainersNotReady","message":"containers with unready status: [node-ca]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T23:57:46Z","reason":"ContainersNotReady","message":"containers with unready status: [node-ca]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T23:27:19Z"}],"hostIP":"10.0.128.6","podIP":"10.0.128.6","podIPs":[{"ip":"10.0.128.6"}],"startTime":"2023-08-21T23:27:19Z","containerStatuses":[{"name":"node-ca","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage3929465431/3\": happened during read: read tcp 10.0.128.6:41062-\u003e18.67.79.178:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:26b626f9b0de3609f607fd85f4fb6865880dc100273846fcc8947cbcc8a2840c","imageID":"","started":false}],"qosClass":"Burstable"}}
#1694158169615372288	junit	6 hours ago
ns/openshift-image-registry pod/node-ca-f2jxr node/ci-op-d8jhnf1i-253f3-vd5bm-worker-centralus2-84z7b container/node-ca - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"node-ca-f2jxr","generateName":"node-ca-","namespace":"openshift-image-registry","uid":"be8b1c94-1a14-4e50-8e3d-b0c7ac690ca9","resourceVersion":"62111","creationTimestamp":"2023-08-23T02:33:23Z","labels":{"controller-revision-hash":"66c54747dd","name":"node-ca","pod-template-generation":"2"},"annotations":{"openshift.io/scc":"privileged"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"node-ca","uid":"b2bc3809-9f32-4564-8466-e7a84258e9c5","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-23T02:33:23Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:name":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"b2bc3809-9f32-4564-8466-e7a84258e9c5\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"node-ca\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{},"f:runAsGroup":{},"f:runAsUser":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/docker/certs.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/tmp/serviceca\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"host\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"serviceca\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-23T03:01:40Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-23T03:02:03Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"host","hostPath":{"path":"/etc/docker/certs.d","type":""}},{"name":"serviceca","configMap":{"name":"image-registry-certificates","defaultMode":420}},{"name":"kube-api-access-hzhmn","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"node-ca","image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:26b626f9b0de3609f607fd85f4fb6865880dc100273846fcc8947cbcc8a2840c","command":["/bin/sh","-c","trap 'jobs -p \| xargs -r kill; echo shutting down node-ca; exit 0' TERM\nwhile [ true ];\ndo\n for f in $(ls /tmp/serviceca); do\n echo $f\n ca_file_path=\"/tmp/serviceca/${f}\"\n f=$(echo $f \| sed -r 's/(.)\\.\\./\\1:/')\n reg_dir_path=\"/etc/docker/certs.d/${f}\"\n if [ -e \"${reg_dir_path}\" ]; then\n cp -u $ca_file_path $reg_dir_path/ca.crt\n else\n mkdir $reg_dir_path\n cp $ca_file_path $reg_dir_path/ca.crt\n fi\n done\n for d in $(ls /etc/docker/certs.d); do\n echo $d\n dp=$(echo $d \| sed -r 's/(.):/\\1\\.\\./')\n reg_conf_path=\"/tmp/serviceca/${dp}\"\n if [ ! -e \"${reg_conf_path}\" ]; then\n rm -rf /etc/docker/certs.d/$d\n fi\n done\n sleep 60 \u0026 wait ${!}\ndone\n"],"resources":{"requests":{"cpu":"10m","memory":"10Mi"}},"volumeMounts":[{"name":"serviceca","mountPath":"/tmp/serviceca"},{"name":"host","mountPath":"/etc/docker/certs.d"},{"name":"kube-api-access-hzhmn","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true,"runAsUser":1001,"runAsGroup":0}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"node-ca","serviceAccount":"node-ca","nodeName":"ci-op-d8jhnf1i-253f3-vd5bm-worker-centralus2-84z7b","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"node-ca-dockercfg-qt8w2"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-d8jhnf1i-253f3-vd5bm-worker-centralus2-84z7b"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-cluster-critical","priority":2000000000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-23T02:33:23Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T03:01:40Z","reason":"ContainersNotReady","message":"containers with unready status: [node-ca]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T03:02:03Z","reason":"ContainersNotReady","message":"containers with unready status: [node-ca]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-23T02:33:23Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-23T02:33:23Z","containerStatuses":[{"name":"node-ca","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage1459933444/2\": happened during read: read tcp 10.0.128.5:37972-\u003e65.8.55.85:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:26b626f9b0de3609f607fd85f4fb6865880dc100273846fcc8947cbcc8a2840c","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-dns pod/node-resolver-mzgkm node/ci-op-d8jhnf1i-253f3-vd5bm-worker-centralus2-84z7b container/dns-node-resolver - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"node-resolver-mzgkm","generateName":"node-resolver-","namespace":"openshift-dns","uid":"214fb1a0-09c3-424e-a2b4-171ac034f181","resourceVersion":"62113","creationTimestamp":"2023-08-23T02:46:24Z","labels":{"controller-revision-hash":"5d6c9f964f","dns.operator.openshift.io/daemonset-node-resolver":"","pod-template-generation":"2"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"node-resolver","uid":"039b1024-d9c3-494c-862d-761cb23c2fad","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-23T02:46:24Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:dns.operator.openshift.io/daemonset-node-resolver":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"039b1024-d9c3-494c-862d-761cb23c2fad\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"dns-node-resolver\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"CLUSTER_DOMAIN\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"NAMESERVER\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"SERVICES\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/hosts\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"hosts-file\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-23T03:01:40Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-23T03:02:03Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"hosts-file","hostPath":{"path":"/etc/hosts","type":"File"}},{"name":"kube-api-access-8rbgb","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"dns-node-resolver","image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:80cc077c52369f7a4e73e7676d03fd7659f3b23e4033c26c3edff4d18096f27c","command":["/bin/bash","-c","#!/bin/bash\nset -uo pipefail\n\ntrap 'jobs -p \| xargs kill \|\| true; wait; exit 0' TERM\n\nOPENSHIFT_MARKER=\"openshift-generated-node-resolver\"\nHOSTS_FILE=\"/etc/hosts\"\nTEMP_FILE=\"/etc/hosts.tmp\"\n\nIFS=', ' read -r -a services \u003c\u003c\u003c \"${SERVICES}\"\n\n# Make a temporary file with the old hosts file's attributes.\ncp -f --attributes-only \"${HOSTS_FILE}\" \"${TEMP_FILE}\"\n\nwhile true; do\n declare -A svc_ips\n for svc in \"${services[@]}\"; do\n # Fetch service IP from cluster dns if present. We make several tries\n # to do it: IPv4, IPv6, IPv4 over TCP and IPv6 over TCP. The two last ones\n # are for deployments with Kuryr on older OpenStack (OSP13) - those do not\n # support UDP loadbalancers and require reaching DNS through TCP.\n cmds=('dig -t A @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"'\n 'dig -t AAAA @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"'\n 'dig -t A +tcp +retry=0 @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"'\n 'dig -t AAAA +tcp +retry=0 @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"')\n for i in ${!cmds[]}\n do\n ips=($(eval \"${cmds[i]}\"))\n if [[ \"$?\" -eq 0 \u0026\u0026 \"${#ips[@]}\" -ne 0 ]]; then\n svc_ips[\"${svc}\"]=\"${ips[@]}\"\n break\n fi\n done\n done\n\n # Update /etc/hosts only if we get valid service IPs\n # We will not update /etc/hosts when there is coredns service outage or api unavailability\n # Stale entries could exist in /etc/hosts if the service is deleted\n if [[ -n \"${svc_ips[]-}\" ]]; then\n # Build a new hosts file from /etc/hosts with our custom entries filtered out\n grep -v \"# ${OPENSHIFT_MARKER}\" \"${HOSTS_FILE}\" \u003e \"${TEMP_FILE}\"\n\n # Append resolver entries for services\n for svc in \"${!svc_ips[@]}\"; do\n for ip in ${svc_ips[${svc}]}; do\n echo \"${ip} ${svc} ${svc}.${CLUSTER_DOMAIN} # ${OPENSHIFT_MARKER}\" \u003e\u003e \"${TEMP_FILE}\"\n done\n done\n\n # TODO: Update /etc/hosts atomically to avoid any inconsistent behavior\n # Replace /etc/hosts with our modified version if needed\n cmp \"${TEMP_FILE}\" \"${HOSTS_FILE}\" \|\| cp -f \"${TEMP_FILE}\" \"${HOSTS_FILE}\"\n # TEMP_FILE is not removed to avoid file create/delete and attributes copy churn\n fi\n sleep 60 \u0026 wait\n unset svc_ips\ndone\n"],"env":[{"name":"SERVICES","value":"image-registry.openshift-image-registry.svc"},{"name":"NAMESERVER","value":"172.30.0.10"},{"name":"CLUSTER_DOMAIN","value":"cluster.local"}],"resources":{"requests":{"cpu":"5m","memory":"21Mi"}},"volumeMounts":[{"name":"hosts-file","mountPath":"/etc/hosts"},{"name":"kube-api-access-8rbgb","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"node-resolver","serviceAccount":"node-resolver","nodeName":"ci-op-d8jhnf1i-253f3-vd5bm-worker-centralus2-84z7b","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"node-resolver-dockercfg-2dvxh"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-d8jhnf1i-253f3-vd5bm-worker-centralus2-84z7b"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-23T02:46:24Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T03:01:40Z","reason":"ContainersNotReady","message":"containers with unready status: [dns-node-resolver]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T03:02:03Z","reason":"ContainersNotReady","message":"containers with unready status: [dns-node-resolver]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-23T02:46:24Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-23T02:46:24Z","containerStatuses":[{"name":"dns-node-resolver","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage2130470821/1\": happened during read: read tcp 10.0.128.5:39370-\u003e65.8.55.215:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:80cc077c52369f7a4e73e7676d03fd7659f3b23e4033c26c3edff4d18096f27c","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-cluster-csi-drivers pod/azure-disk-csi-driver-node-nhwpm node/ci-op-d8jhnf1i-253f3-vd5bm-worker-centralus2-84z7b container/azure-inject-credentials - reason/ContainerWait cause/ErrImagePull duration/1960.49s rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage3176474738/2": happened during read: read tcp 10.0.128.5:37978->65.8.55.85:443: read: connection timed out ns/openshift-cluster-node-tuning-operator pod/tuned-cjp49 node/ci-op-d8jhnf1i-253f3-vd5bm-worker-centralus2-84z7b container/tuned - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"tuned-cjp49","generateName":"tuned-","namespace":"openshift-cluster-node-tuning-operator","uid":"f692f423-0ff4-444a-a4f9-689da661b6de","resourceVersion":"62115","creationTimestamp":"2023-08-23T02:33:00Z","labels":{"controller-revision-hash":"6d84d77867","openshift-app":"tuned","pod-template-generation":"2"},"annotations":{"openshift.io/scc":"privileged"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"tuned","uid":"045cdcc4-01e2-4e61-87a5-ab4f9c9c6c49","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-23T02:33:00Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:openshift-app":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"045cdcc4-01e2-4e61-87a5-ab4f9c9c6c49\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"tuned\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"OCP_NODE_NAME\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"RELEASE_VERSION\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RESYNC_PERIOD\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"WATCH_NAMESPACE\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/modprobe.d\"}":{".":{},"f:mountPath":{},"f:name":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/sysconfig\"}":{".":{},"f:mountPath":{},"f:name":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/sysctl.conf\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/sysctl.d\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/systemd\"}":{".":{},"f:mountPath":{},"f:name":{},"f:subPath":{}},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/lib/modules\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/run/systemd/system\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/sys\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/lib/tuned/profiles-data\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/dbus\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostIPC":{},"f:hostNetwork":{},"f:hostPID":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"etc\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"lib-modules\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"run-systemd-system\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"sys\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"var-lib-tuned-profiles-data\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:items":{},"f:name":{},"f:optional":{}},"f:name":{}},"k:{\"name\":\"var-run-dbus\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-23T03:01:40Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-23T03:02:03Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"etc","hostPath":{"path":"/etc","type":"Directory"}},{"name":"sys","hostPath":{"path":"/sys","type":"Directory"}},{"name":"var-run-dbus","hostPath":{"path":"/var/run/dbus","type":"Directory"}},{"name":"run-systemd-system","hostPath":{"path":"/run/systemd/system","type":"Directory"}},{"name":"lib-modules","hostPath":{"path":"/lib/modules","type":"Directory"}},{"name":"host","hostPath":{"path":"/","type":"Directory"}},{"name":"var-lib-tuned-profiles-data","configMap":{"name":"tuned-profiles","items":[{"key":"tuned-profiles-data","path":"tuned-profiles.yaml"}],"defaultMode":420,"optional":true}},{"name":"kube-api-access-n2hqj","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"tuned","image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:a46e73a9d6e06a067d466c1a255bec4292c66b4fb7ef2b96fed282df394a17ae","command":["/var/lib/tuned/bin/run","start"],"env":[{"name":"WATCH_NAMESPACE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}},{"name":"OCP_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"RESYNC_PERIOD","value":"600"},{"name":"RELEASE_VERSION","value":"4.10.0-0.ci-2023-08-22-222817"}],"resources":{"requests":{"cpu":"10m","memory":"50Mi"}},"volumeMounts":[{"name":"var-lib-tuned-profiles-data","mountPath":"/var/lib/tuned/profiles-data"},{"name":"etc","mountPath":"/etc/modprobe.d","subPath":"modprobe.d"},{"name":"etc","mountPath":"/etc/sysconfig","subPath":"sysconfig"},{"name":"etc","readOnly":true,"mountPath":"/etc/sysctl.d","subPath":"sysctl.d"},{"name":"etc","readOnly":true,"mountPath":"/etc/sysctl.conf","subPath":"sysctl.conf"},{"name":"etc","mountPath":"/etc/systemd","subPath":"systemd"},{"name":"sys","mountPath":"/sys"},{"name":"var-run-dbus","readOnly":true,"mountPath":"/var/run/dbus"},{"name":"run-systemd-system","readOnly":true,"mountPath":"/run/systemd/system"},{"name":"lib-modules","readOnly":true,"mountPath":"/lib/modules"},{"name":"host","mountPath":"/host"},{"name":"kube-api-access-n2hqj","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"lifecycle":{"preStop":{"exec":{"command":["/var/lib/tuned/bin/run","stop"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"tuned","serviceAccount":"tuned","nodeName":"ci-op-d8jhnf1i-253f3-vd5bm-worker-centralus2-84z7b","hostNetwork":true,"hostPID":true,"hostIPC":true,"securityContext":{},"imagePullSecrets":[{"name":"tuned-dockercfg-2kxwv"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-d8jhnf1i-253f3-vd5bm-worker-centralus2-84z7b"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-23T02:33:00Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T03:01:40Z","reason":"ContainersNotReady","message":"containers with unready status: [tuned]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T03:02:03Z","reason":"ContainersNotReady","message":"containers with unready status: [tuned]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-23T02:33:00Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-23T02:33:00Z","containerStatuses":[{"name":"tuned","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage1461032928/2\": happened during read: read tcp 10.0.128.5:39470-\u003e65.8.55.215:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:a46e73a9d6e06a067d466c1a255bec4292c66b4fb7ef2b96fed282df394a17ae","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-multus pod/multus-additional-cni-plugins-6mnsn node/ci-op-d8jhnf1i-253f3-vd5bm-worker-centralus2-84z7b container/egress-router-binary-copy - reason/ContainerWait cause/ErrImagePull duration/1307.52s rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage506112386/1": happened during read: read tcp 10.0.128.5:40548->65.8.55.213:443: read: connection timed out
#1693835857468854272	junit	27 hours ago
ns/openshift-cluster-node-tuning-operator pod/tuned-r2vst node/ci-op-shx4vyfl-253f3-649rr-worker-eastus21-nnffh container/tuned - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"tuned-r2vst","generateName":"tuned-","namespace":"openshift-cluster-node-tuning-operator","uid":"13d899ca-6f8f-4b77-8a94-0bb9dd6ce73e","resourceVersion":"62362","creationTimestamp":"2023-08-22T05:13:05Z","labels":{"controller-revision-hash":"fcf6c6675","openshift-app":"tuned","pod-template-generation":"2"},"annotations":{"openshift.io/scc":"privileged"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"tuned","uid":"a2680da2-1b08-4e19-b097-c7fc1431135c","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-22T05:13:05Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:openshift-app":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"a2680da2-1b08-4e19-b097-c7fc1431135c\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"tuned\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"OCP_NODE_NAME\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"RELEASE_VERSION\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RESYNC_PERIOD\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"WATCH_NAMESPACE\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/modprobe.d\"}":{".":{},"f:mountPath":{},"f:name":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/sysconfig\"}":{".":{},"f:mountPath":{},"f:name":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/sysctl.conf\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/sysctl.d\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/systemd\"}":{".":{},"f:mountPath":{},"f:name":{},"f:subPath":{}},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/lib/modules\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/run/systemd/system\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/sys\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/lib/tuned/profiles-data\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/dbus\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostIPC":{},"f:hostNetwork":{},"f:hostPID":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"etc\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"lib-modules\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"run-systemd-system\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"sys\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"var-lib-tuned-profiles-data\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:items":{},"f:name":{},"f:optional":{}},"f:name":{}},"k:{\"name\":\"var-run-dbus\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-22T05:35:53Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-22T05:36:40Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"etc","hostPath":{"path":"/etc","type":"Directory"}},{"name":"sys","hostPath":{"path":"/sys","type":"Directory"}},{"name":"var-run-dbus","hostPath":{"path":"/var/run/dbus","type":"Directory"}},{"name":"run-systemd-system","hostPath":{"path":"/run/systemd/system","type":"Directory"}},{"name":"lib-modules","hostPath":{"path":"/lib/modules","type":"Directory"}},{"name":"host","hostPath":{"path":"/","type":"Directory"}},{"name":"var-lib-tuned-profiles-data","configMap":{"name":"tuned-profiles","items":[{"key":"tuned-profiles-data","path":"tuned-profiles.yaml"}],"defaultMode":420,"optional":true}},{"name":"kube-api-access-ct57w","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"tuned","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:a46e73a9d6e06a067d466c1a255bec4292c66b4fb7ef2b96fed282df394a17ae","command":["/var/lib/tuned/bin/run","start"],"env":[{"name":"WATCH_NAMESPACE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}},{"name":"OCP_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"RESYNC_PERIOD","value":"600"},{"name":"RELEASE_VERSION","value":"4.10.0-0.ci-2023-08-21-220954"}],"resources":{"requests":{"cpu":"10m","memory":"50Mi"}},"volumeMounts":[{"name":"var-lib-tuned-profiles-data","mountPath":"/var/lib/tuned/profiles-data"},{"name":"etc","mountPath":"/etc/modprobe.d","subPath":"modprobe.d"},{"name":"etc","mountPath":"/etc/sysconfig","subPath":"sysconfig"},{"name":"etc","readOnly":true,"mountPath":"/etc/sysctl.d","subPath":"sysctl.d"},{"name":"etc","readOnly":true,"mountPath":"/etc/sysctl.conf","subPath":"sysctl.conf"},{"name":"etc","mountPath":"/etc/systemd","subPath":"systemd"},{"name":"sys","mountPath":"/sys"},{"name":"var-run-dbus","readOnly":true,"mountPath":"/var/run/dbus"},{"name":"run-systemd-system","readOnly":true,"mountPath":"/run/systemd/system"},{"name":"lib-modules","readOnly":true,"mountPath":"/lib/modules"},{"name":"host","mountPath":"/host"},{"name":"kube-api-access-ct57w","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"lifecycle":{"preStop":{"exec":{"command":["/var/lib/tuned/bin/run","stop"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"tuned","serviceAccount":"tuned","nodeName":"ci-op-shx4vyfl-253f3-649rr-worker-eastus21-nnffh","hostNetwork":true,"hostPID":true,"hostIPC":true,"securityContext":{},"imagePullSecrets":[{"name":"tuned-dockercfg-c6tgv"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-shx4vyfl-253f3-649rr-worker-eastus21-nnffh"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T05:13:05Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-22T05:35:53Z","reason":"ContainersNotReady","message":"containers with unready status: [tuned]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-22T05:36:40Z","reason":"ContainersNotReady","message":"containers with unready status: [tuned]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T05:13:05Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-22T05:13:05Z","containerStatuses":[{"name":"tuned","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage3033867004/2\": happened during read: read tcp 10.0.128.5:44136-\u003e18.67.79.73:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:a46e73a9d6e06a067d466c1a255bec4292c66b4fb7ef2b96fed282df394a17ae","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-multus pod/multus-vjwhn node/ci-op-shx4vyfl-253f3-649rr-worker-eastus21-nnffh container/kube-multus - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"multus-vjwhn","generateName":"multus-","namespace":"openshift-multus","uid":"b6d0f19f-5291-4025-944f-2f27b0cdb27d","resourceVersion":"62363","creationTimestamp":"2023-08-22T05:19:30Z","labels":{"app":"multus","component":"network","controller-revision-hash":"5b8bf7f795","openshift.io/component":"network","pod-template-generation":"2","type":"infra"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"multus","uid":"36f3712f-dbcf-460e-b853-2e553e80fa43","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-22T05:19:30Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:component":{},"f:controller-revision-hash":{},"f:openshift.io/component":{},"f:pod-template-generation":{},"f:type":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"36f3712f-dbcf-460e-b853-2e553e80fa43\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"kube-multus\"}":{".":{},"f:args":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"DEFAULT_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_HOST\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_PORT\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RHEL7_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RHEL8_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/entrypoint\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/etc/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/etc/os-release\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/opt/cni/bin\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/var/run/multus/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"cni-binary-copy\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"cnibin\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"multus-cni-dir\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"os-release\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"system-cni-dir\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-22T05:35:53Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-22T05:36:40Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"system-cni-dir","hostPath":{"path":"/etc/kubernetes/cni/net.d","type":""}},{"name":"multus-cni-dir","hostPath":{"path":"/var/run/multus/cni/net.d","type":""}},{"name":"cnibin","hostPath":{"path":"/var/lib/cni/bin","type":""}},{"name":"os-release","hostPath":{"path":"/etc/os-release","type":"File"}},{"name":"cni-binary-copy","configMap":{"name":"cni-binary-copy-script","defaultMode":484}},{"name":"kube-api-access-xj6x9","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"kube-multus","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:528a2e0fb6ca1a3c6aaefcbf1fa7bcce000182565a8900bcc27cfd8db47a6ca5","command":["/bin/bash","-ec","--"],"args":["/entrypoint/cnibincopy.sh; exec /entrypoint.sh --multus-conf-file=auto --multus-autoconfig-dir=/host/var/run/multus/cni/net.d --multus-kubeconfig-file-host=/etc/kubernetes/cni/net.d/multus.d/multus.kubeconfig --readiness-indicator-file=/var/run/multus/cni/net.d/80-openshift-network.conf --cleanup-config-on-exit=true --namespace-isolation=true --multus-log-level=verbose --cni-version=0.3.1 --additional-bin-dir=/opt/multus/bin --skip-multus-binary-copy=true - \"--global-namespaces=default,openshift-multus,openshift-sriov-network-operator\"\n"],"env":[{"name":"RHEL7_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/rhel7/bin/"},{"name":"RHEL8_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/rhel8/bin/"},{"name":"DEFAULT_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/bin/"},{"name":"KUBERNETES_SERVICE_PORT","value":"6443"},{"name":"KUBERNETES_SERVICE_HOST","value":"api-int.ci-op-shx4vyfl-253f3.ci.azure.devcluster.openshift.com"}],"resources":{"requests":{"cpu":"10m","memory":"65Mi"}},"volumeMounts":[{"name":"cni-binary-copy","mountPath":"/entrypoint"},{"name":"os-release","mountPath":"/host/etc/os-release"},{"name":"system-cni-dir","mountPath":"/host/etc/cni/net.d"},{"name":"multus-cni-dir","mountPath":"/host/var/run/multus/cni/net.d"},{"name":"cnibin","mountPath":"/host/opt/cni/bin"},{"name":"kube-api-access-xj6x9","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"multus","serviceAccount":"multus","nodeName":"ci-op-shx4vyfl-253f3-649rr-worker-eastus21-nnffh","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"multus-dockercfg-2xnkz"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-shx4vyfl-253f3-649rr-worker-eastus21-nnffh"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T05:19:30Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-22T05:35:53Z","reason":"ContainersNotReady","message":"containers with unready status: [kube-multus]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-22T05:36:40Z","reason":"ContainersNotReady","message":"containers with unready status: [kube-multus]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T05:19:30Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-22T05:19:30Z","containerStatuses":[{"name":"kube-multus","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage2567926871/4\": happened during read: read tcp 10.0.128.5:42990-\u003e18.67.79.178:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:528a2e0fb6ca1a3c6aaefcbf1fa7bcce000182565a8900bcc27cfd8db47a6ca5","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-multus pod/multus-additional-cni-plugins-9r7rc node/ci-op-shx4vyfl-253f3-649rr-worker-eastus21-nnffh container/egress-router-binary-copy - reason/ContainerWait cause/ErrImagePull duration/1352.64s rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage3439164528/4": happened during read: read tcp 10.0.128.5:44158->18.67.79.73:443: read: connection timed out ns/openshift-image-registry pod/node-ca-knsmf node/ci-op-shx4vyfl-253f3-649rr-worker-eastus21-nnffh container/node-ca - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"node-ca-knsmf","generateName":"node-ca-","namespace":"openshift-image-registry","uid":"631ae146-34c5-4cff-94fc-c4521b7bcdde","resourceVersion":"62366","creationTimestamp":"2023-08-22T05:13:14Z","labels":{"controller-revision-hash":"8c9ffcc77","name":"node-ca","pod-template-generation":"2"},"annotations":{"openshift.io/scc":"privileged"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"node-ca","uid":"08ce4c1d-1b6e-4784-ae58-8ea1648af68c","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-22T05:13:14Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:name":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"08ce4c1d-1b6e-4784-ae58-8ea1648af68c\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"node-ca\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{},"f:runAsGroup":{},"f:runAsUser":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/docker/certs.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/tmp/serviceca\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"host\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"serviceca\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-22T05:35:53Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-22T05:36:40Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"host","hostPath":{"path":"/etc/docker/certs.d","type":""}},{"name":"serviceca","configMap":{"name":"image-registry-certificates","defaultMode":420}},{"name":"kube-api-access-6fggc","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"node-ca","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:26b626f9b0de3609f607fd85f4fb6865880dc100273846fcc8947cbcc8a2840c","command":["/bin/sh","-c","trap 'jobs -p \| xargs -r kill; echo shutting down node-ca; exit 0' TERM\nwhile [ true ];\ndo\n for f in $(ls /tmp/serviceca); do\n echo $f\n ca_file_path=\"/tmp/serviceca/${f}\"\n f=$(echo $f \| sed -r 's/(.)\\.\\./\\1:/')\n reg_dir_path=\"/etc/docker/certs.d/${f}\"\n if [ -e \"${reg_dir_path}\" ]; then\n cp -u $ca_file_path $reg_dir_path/ca.crt\n else\n mkdir $reg_dir_path\n cp $ca_file_path $reg_dir_path/ca.crt\n fi\n done\n for d in $(ls /etc/docker/certs.d); do\n echo $d\n dp=$(echo $d \| sed -r 's/(.):/\\1\\.\\./')\n reg_conf_path=\"/tmp/serviceca/${dp}\"\n if [ ! -e \"${reg_conf_path}\" ]; then\n rm -rf /etc/docker/certs.d/$d\n fi\n done\n sleep 60 \u0026 wait ${!}\ndone\n"],"resources":{"requests":{"cpu":"10m","memory":"10Mi"}},"volumeMounts":[{"name":"serviceca","mountPath":"/tmp/serviceca"},{"name":"host","mountPath":"/etc/docker/certs.d"},{"name":"kube-api-access-6fggc","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true,"runAsUser":1001,"runAsGroup":0}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"node-ca","serviceAccount":"node-ca","nodeName":"ci-op-shx4vyfl-253f3-649rr-worker-eastus21-nnffh","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"node-ca-dockercfg-c8hjd"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-shx4vyfl-253f3-649rr-worker-eastus21-nnffh"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-cluster-critical","priority":2000000000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T05:13:14Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-22T05:35:53Z","reason":"ContainersNotReady","message":"containers with unready status: [node-ca]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-22T05:36:40Z","reason":"ContainersNotReady","message":"containers with unready status: [node-ca]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T05:13:14Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-22T05:13:14Z","containerStatuses":[{"name":"node-ca","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage292876294/3\": happened during read: read tcp 10.0.128.5:44228-\u003e18.67.79.73:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:26b626f9b0de3609f607fd85f4fb6865880dc100273846fcc8947cbcc8a2840c","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-sdn pod/sdn-h7z49 node/ci-op-shx4vyfl-253f3-649rr-worker-eastus21-nnffh container/sdn - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"sdn-h7z49","generateName":"sdn-","namespace":"openshift-sdn","uid":"09e40b1a-473b-4c5c-b957-8155c797c8cd","resourceVersion":"62367","creationTimestamp":"2023-08-22T05:20:27Z","labels":{"app":"sdn","component":"network","controller-revision-hash":"dcfbb587f","openshift.io/component":"network","pod-template-generation":"2","type":"infra"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"sdn","uid":"964156fa-5f5f-4283-bb56-d39ab3cf5d8a","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-22T05:20:27Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:component":{},"f:controller-revision-hash":{},"f:openshift.io/component":{},"f:pod-template-generation":{},"f:type":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"964156fa-5f5f-4283-bb56-d39ab3cf5d8a\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"drop-icmp\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:name":{}}}},"k:{\"name\":\"kube-rbac-proxy\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":9101,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/pki/tls/metrics-certs\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}},"k:{\"name\":\"sdn\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE_IP\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"K8S_NODE_NAME\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"KUBERNETES_SERVICE_HOST\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_PORT\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"OPENSHIFT_DNS_DOMAIN\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":10256,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:readinessProbe":{".":{},"f:exec":{".":{},"f:command":{}},"f:failureThreshold":{},"f:initialDelaySeconds":{},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/config\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/env\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/sysconfig\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host-cni-bin\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/var/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/lib/modules\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/lib/cni/networks/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/dbus/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/kubernetes/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/openvswitch/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:hostPID":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"config\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"env-overrides\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{},"f:optional":{}},"f:name":{}},"k:{\"name\":\"etc-sysconfig\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-bin\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-conf\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-modules\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-slash\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-lib-cni-networks-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-dbus\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-kubernetes\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-ovs\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"sdn-metrics-certs\"}":{".":{},"f:name":{},"f:secret":{".":{},"f:defaultMode":{},"f:optional":{},"f:secretName":{}}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-22T05:35:53Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-22T05:36:40Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"config","configMap":{"name":"sdn-config","defaultMode":420}},{"name":"env-overrides","configMap":{"name":"env-overrides","defaultMode":420,"optional":true}},{"name":"etc-sysconfig","hostPath":{"path":"/etc/sysconfig","type":""}},{"name":"host-modules","hostPath":{"path":"/lib/modules","type":""}},{"name":"host-var-run","hostPath":{"path":"/var/run","type":""}},{"name":"host-run-netns","hostPath":{"path":"/run/netns","type":""}},{"name":"host-var-run-netns","hostPath":{"path":"/var/run/netns","type":""}},{"name":"host-var-run-dbus","hostPath":{"path":"/var/run/dbus","type":""}},{"name":"host-var-run-ovs","hostPath":{"path":"/var/run/openvswitch","type":""}},{"name":"host-var-run-kubernetes","hostPath":{"path":"/var/run/kubernetes","type":""}},{"name":"host-var-run-openshift-sdn","hostPath":{"path":"/var/run/openshift-sdn","type":""}},{"name":"host-slash","hostPath":{"path":"/","type":""}},{"name":"host-cni-bin","hostPath":{"path":"/var/lib/cni/bin","type":""}},{"name":"host-cni-conf","hostPath":{"path":"/var/run/multus/cni/net.d","type":""}},{"name":"host-var-lib-cni-networks-openshift-sdn","hostPath":{"path":"/var/lib/cni/networks/openshift-sdn","type":""}},{"name":"sdn-metrics-certs","secret":{"secretName":"sdn-metrics-certs","defaultMode":420,"optional":true}},{"name":"kube-api-access-5dfwl","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"sdn","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\n\n# if another process is listening on the cni-server socket, wait until it exits\ntrap 'kill $(jobs -p); rm -f /etc/cni/net.d/80-openshift-network.conf ; exit 0' TERM\nretries=0\nwhile true; do\n if echo 'test' \| socat - UNIX-CONNECT:/var/run/openshift-sdn/cniserver/socket \u0026\u003e/dev/null; then\n echo \"warning: Another process is currently listening on the CNI socket, waiting 15s ...\" 2\u003e\u00261\n sleep 15 \u0026 wait\n (( retries += 1 ))\n else\n break\n fi\n if [[ \"${retries}\" -gt 40 ]]; then\n echo \"error: Another process is currently listening on the CNI socket, exiting\" 2\u003e\u00261\n exit 1\n fi\ndone\n\n# local environment overrides\nif [[ -f /etc/sysconfig/openshift-sdn ]]; then\n set -o allexport\n source /etc/sysconfig/openshift-sdn\n set +o allexport\nfi\n#BUG: cdc accidentally mounted /etc/sysconfig/openshift-sdn as DirectoryOrCreate; clean it up so we can ultimately mount /etc/sysconfig/openshift-sdn as FileOrCreate\n# Once this is released, then we can mount it properly\nif [[ -d /etc/sysconfig/openshift-sdn ]]; then\n rmdir /etc/sysconfig/openshift-sdn \|\| true\nfi\n\n# configmap-based overrides\nif [[ -f /env/${K8S_NODE_NAME} ]]; then\n set -o allexport\n source /env/${K8S_NODE_NAME}\n set +o allexport\nfi\n\n# Take over network functions on the node\nrm -f /etc/cni/net.d/80-openshift-network.conf\ncp -f /opt/cni/bin/openshift-sdn /host-cni-bin/\n\nmtu_override_flag=\nif [[ -f /config/mtu.yaml ]]; then\n mtu_override_flag=\"--mtu-override /config/mtu.yaml\"\nfi\n\n# Launch the network process\nexec /usr/bin/openshift-sdn-node \\\n --node-name ${K8S_NODE_NAME} --node-ip ${K8S_NODE_IP} \\\n --platform-type Azure \\\n --proxy-config /config/kube-proxy-config.yaml \\\n ${mtu_override_flag} \\\n --v ${OPENSHIFT_SDN_LOG_LEVEL:-2}\n"],"ports":[{"name":"healthz","hostPort":10256,"containerPort":10256,"protocol":"TCP"}],"env":[{"name":"KUBERNETES_SERVICE_PORT","value":"6443"},{"name":"KUBERNETES_SERVICE_HOST","value":"api-int.ci-op-shx4vyfl-253f3.ci.azure.devcluster.openshift.com"},{"name":"OPENSHIFT_DNS_DOMAIN","value":"cluster.local"},{"name":"K8S_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"K8S_NODE_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.hostIP"}}}],"resources":{"requests":{"cpu":"100m","memory":"200Mi"}},"volumeMounts":[{"name":"config","readOnly":true,"mountPath":"/config"},{"name":"env-overrides","mountPath":"/env"},{"name":"host-var-run","mountPath":"/var/run"},{"name":"host-var-run-dbus","readOnly":true,"mountPath":"/var/run/dbus/"},{"name":"host-var-run-ovs","readOnly":true,"mountPath":"/var/run/openvswitch/"},{"name":"host-var-run-kubernetes","readOnly":true,"mountPath":"/var/run/kubernetes/"},{"name":"host-run-netns","readOnly":true,"mountPath":"/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-netns","readOnly":true,"mountPath":"/host/var/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-openshift-sdn","mountPath":"/var/run/openshift-sdn"},{"name":"host-slash","readOnly":true,"mountPath":"/host","mountPropagation":"HostToContainer"},{"name":"host-cni-bin","mountPath":"/host-cni-bin"},{"name":"host-cni-conf","mountPath":"/etc/cni/net.d"},{"name":"host-var-lib-cni-networks-openshift-sdn","mountPath":"/var/lib/cni/networks/openshift-sdn"},{"name":"host-modules","readOnly":true,"mountPath":"/lib/modules"},{"name":"etc-sysconfig","readOnly":true,"mountPath":"/etc/sysconfig"},{"name":"kube-api-access-5dfwl","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"readinessProbe":{"exec":{"command":["test","-f","/etc/cni/net.d/80-openshift-network.conf"]},"initialDelaySeconds":5,"timeoutSeconds":1,"periodSeconds":5,"successThreshold":1,"failureThreshold":3},"lifecycle":{"preStop":{"exec":{"command":["rm","-f","/etc/cni/net.d/80-openshift-network.conf","/host-cni-bin/openshift-sdn"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}},{"name":"kube-rbac-proxy","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\nTLS_PK=/etc/pki/tls/metrics-certs/tls.key\nTLS_CERT=/etc/pki/tls/metrics-certs/tls.crt\n\n# As the secret mount is optional we must wait for the files to be present.\n# The service is created in monitor.yaml and this is created in sdn.yaml.\n# If it isn't created there is probably an issue so we want to crashloop.\nTS=$(date +%s)\nWARN_TS=$(( ${TS} + $(( 20 * 60)) ))\nHAS_LOGGED_INFO=0\n\nlog_missing_certs(){\n CUR_TS=$(date +%s)\n if [[ \"${CUR_TS}\" -gt \"WARN_TS\" ]]; then\n echo $(date -Iseconds) WARN: sdn-metrics-certs not mounted after 20 minutes.\n elif [[ \"${HAS_LOGGED_INFO}\" -eq 0 ]] ; then\n echo $(date -Iseconds) INFO: sdn-metrics-certs not mounted. Waiting 20 minutes.\n HAS_LOGGED_INFO=1\n fi\n}\n\nwhile [[ ! -f \"${TLS_PK}\" \|\| ! -f \"${TLS_CERT}\" ]] ; do\n log_missing_certs\n sleep 5\ndone\n\necho $(date -Iseconds) INFO: sdn-metrics-certs mounted, starting kube-rbac-proxy\nexec /usr/bin/kube-rbac-proxy \\\n --logtostderr \\\n --secure-listen-address=:9101 \\\n --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \\\n --upstream=http://127.0.0.1:29101/ \\\n --tls-private-key-file=${TLS_PK} \\\n --tls-cert-file=${TLS_CERT}\n"],"ports":[{"name":"https","hostPort":9101,"containerPort":9101,"protocol":"TCP"}],"resources":{"requests":{"cpu":"10m","memory":"20Mi"}},"volumeMounts":[{"name":"sdn-metrics-certs","readOnly":true,"mountPath":"/etc/pki/tls/metrics-certs"},{"name":"kube-api-access-5dfwl","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent"},{"name":"drop-icmp","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","set -xe\n\ntouch /var/run/add_iptables.sh\nchmod 0755 /var/run/add_iptables.sh\ncat \u003c\u003c'EOF' \u003e /var/run/add_iptables.sh\n#!/bin/sh\nif [ -z \"$3\" ]\nthen\n echo \"Called with host address missing, ignore\"\n exit 0\nfi\necho \"Adding ICMP drop rule for '$3' \"\nif iptables -C AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nthen\n echo \"iptables already set for $3\"\nelse\n iptables -A AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nfi\nEOF\n\necho \"I$(date \"+%m%d %H:%M:%S.%N\") - drop-icmp - start drop-icmp ${K8S_NODE}\"\niptables -X AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -N AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -F AZURE_CHECK_ICMP_SOURCE\niptables -D INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -I INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE\niptables -N AZURE_ICMP_ACTION \|\| true\niptables -F AZURE_ICMP_ACTION\niptables -A AZURE_ICMP_ACTION -j LOG\niptables -A AZURE_ICMP_ACTION -j DROP\n/host/usr/bin/oc observe pods -n openshift-sdn -l app=sdn -a '{ .status.hostIP }' -- /var/run/add_iptables.sh\n"],"env":[{"name":"K8S_NODE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}}],"resources":{"requests":{"cpu":"5m","memory":"20Mi"}},"volumeMounts":[{"name":"host-slash","mountPath":"/host"},{"name":"kube-api-access-5dfwl","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"lifecycle":{"preStop":{"exec":{"command":["/bin/bash","-c","echo drop-icmp done"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"sdn","serviceAccount":"sdn","nodeName":"ci-op-shx4vyfl-253f3-649rr-worker-eastus21-nnffh","hostNetwork":true,"hostPID":true,"securityContext":{},"imagePullSecrets":[{"name":"sdn-dockercfg-xhsf5"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-shx4vyfl-253f3-649rr-worker-eastus21-nnffh"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T05:20:27Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-22T05:35:53Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-22T05:36:40Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T05:20:27Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-22T05:20:27Z","containerStatuses":[{"name":"drop-icmp","state":{"waiting":{"reason":"ImagePullBackOff","message":"Back-off pulling image \"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c\""}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false},{"name":"kube-rbac-proxy","state":{"running":{"startedAt":"2023-08-22T05:41:59Z"}},"lastState":{},"ready":true,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","imageID":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","containerID":"cri-o://471fd2d9fee674bb41fff76876631a5bb1b9de2f6651805e1ccd46a02d210a4b","started":true},{"name":"sdn","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage598752273/3\": happened during read: read tcp 10.0.128.5:43006-\u003e18.67.79.178:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false}],"qosClass":"Burstable"}}
#1693651607373221888	junit	39 hours ago
ns/openshift-dns pod/node-resolver-qk6px node/ci-op-h0w5q3d8-253f3-prxcj-worker-centralus2-v8gsn container/dns-node-resolver - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"node-resolver-qk6px","generateName":"node-resolver-","namespace":"openshift-dns","uid":"5c9e7d94-9ee6-469e-b5a0-a68dcadf9820","resourceVersion":"64274","creationTimestamp":"2023-08-21T17:23:11Z","labels":{"controller-revision-hash":"7db5bfd767","dns.operator.openshift.io/daemonset-node-resolver":"","pod-template-generation":"2"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"node-resolver","uid":"629ab1b7-8ba8-4bd6-9662-f2fd90ad7f97","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T17:23:11Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:dns.operator.openshift.io/daemonset-node-resolver":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"629ab1b7-8ba8-4bd6-9662-f2fd90ad7f97\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"dns-node-resolver\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"CLUSTER_DOMAIN\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"NAMESERVER\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"SERVICES\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/hosts\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"hosts-file\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T17:37:57Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-21T17:38:24Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.4\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"hosts-file","hostPath":{"path":"/etc/hosts","type":"File"}},{"name":"kube-api-access-wwjns","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"dns-node-resolver","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:80cc077c52369f7a4e73e7676d03fd7659f3b23e4033c26c3edff4d18096f27c","command":["/bin/bash","-c","#!/bin/bash\nset -uo pipefail\n\ntrap 'jobs -p \| xargs kill \|\| true; wait; exit 0' TERM\n\nOPENSHIFT_MARKER=\"openshift-generated-node-resolver\"\nHOSTS_FILE=\"/etc/hosts\"\nTEMP_FILE=\"/etc/hosts.tmp\"\n\nIFS=', ' read -r -a services \u003c\u003c\u003c \"${SERVICES}\"\n\n# Make a temporary file with the old hosts file's attributes.\ncp -f --attributes-only \"${HOSTS_FILE}\" \"${TEMP_FILE}\"\n\nwhile true; do\n declare -A svc_ips\n for svc in \"${services[@]}\"; do\n # Fetch service IP from cluster dns if present. We make several tries\n # to do it: IPv4, IPv6, IPv4 over TCP and IPv6 over TCP. The two last ones\n # are for deployments with Kuryr on older OpenStack (OSP13) - those do not\n # support UDP loadbalancers and require reaching DNS through TCP.\n cmds=('dig -t A @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"'\n 'dig -t AAAA @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"'\n 'dig -t A +tcp +retry=0 @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"'\n 'dig -t AAAA +tcp +retry=0 @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"')\n for i in ${!cmds[]}\n do\n ips=($(eval \"${cmds[i]}\"))\n if [[ \"$?\" -eq 0 \u0026\u0026 \"${#ips[@]}\" -ne 0 ]]; then\n svc_ips[\"${svc}\"]=\"${ips[@]}\"\n break\n fi\n done\n done\n\n # Update /etc/hosts only if we get valid service IPs\n # We will not update /etc/hosts when there is coredns service outage or api unavailability\n # Stale entries could exist in /etc/hosts if the service is deleted\n if [[ -n \"${svc_ips[]-}\" ]]; then\n # Build a new hosts file from /etc/hosts with our custom entries filtered out\n grep -v \"# ${OPENSHIFT_MARKER}\" \"${HOSTS_FILE}\" \u003e \"${TEMP_FILE}\"\n\n # Append resolver entries for services\n for svc in \"${!svc_ips[@]}\"; do\n for ip in ${svc_ips[${svc}]}; do\n echo \"${ip} ${svc} ${svc}.${CLUSTER_DOMAIN} # ${OPENSHIFT_MARKER}\" \u003e\u003e \"${TEMP_FILE}\"\n done\n done\n\n # TODO: Update /etc/hosts atomically to avoid any inconsistent behavior\n # Replace /etc/hosts with our modified version if needed\n cmp \"${TEMP_FILE}\" \"${HOSTS_FILE}\" \|\| cp -f \"${TEMP_FILE}\" \"${HOSTS_FILE}\"\n # TEMP_FILE is not removed to avoid file create/delete and attributes copy churn\n fi\n sleep 60 \u0026 wait\n unset svc_ips\ndone\n"],"env":[{"name":"SERVICES","value":"image-registry.openshift-image-registry.svc"},{"name":"NAMESERVER","value":"172.30.0.10"},{"name":"CLUSTER_DOMAIN","value":"cluster.local"}],"resources":{"requests":{"cpu":"5m","memory":"21Mi"}},"volumeMounts":[{"name":"hosts-file","mountPath":"/etc/hosts"},{"name":"kube-api-access-wwjns","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"node-resolver","serviceAccount":"node-resolver","nodeName":"ci-op-h0w5q3d8-253f3-prxcj-worker-centralus2-v8gsn","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"node-resolver-dockercfg-lkk5d"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-h0w5q3d8-253f3-prxcj-worker-centralus2-v8gsn"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T17:23:11Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T17:37:57Z","reason":"ContainersNotReady","message":"containers with unready status: [dns-node-resolver]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T17:38:23Z","reason":"ContainersNotReady","message":"containers with unready status: [dns-node-resolver]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T17:23:11Z"}],"hostIP":"10.0.128.4","podIP":"10.0.128.4","podIPs":[{"ip":"10.0.128.4"}],"startTime":"2023-08-21T17:23:11Z","containerStatuses":[{"name":"dns-node-resolver","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage2035820380/3\": happened during read: read tcp 10.0.128.4:48098-\u003e54.230.17.85:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:80cc077c52369f7a4e73e7676d03fd7659f3b23e4033c26c3edff4d18096f27c","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-image-registry pod/node-ca-v24b5 node/ci-op-h0w5q3d8-253f3-prxcj-worker-centralus2-v8gsn container/node-ca - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"node-ca-v24b5","generateName":"node-ca-","namespace":"openshift-image-registry","uid":"6ed8470c-e7ef-4eae-bb77-ee9527e1f119","resourceVersion":"64275","creationTimestamp":"2023-08-21T17:14:49Z","labels":{"controller-revision-hash":"7c956f79fd","name":"node-ca","pod-template-generation":"2"},"annotations":{"openshift.io/scc":"privileged"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"node-ca","uid":"e476c922-a272-48ee-af8a-c29e70563a84","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T17:14:47Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:name":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"e476c922-a272-48ee-af8a-c29e70563a84\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"node-ca\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{},"f:runAsGroup":{},"f:runAsUser":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/docker/certs.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/tmp/serviceca\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"host\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"serviceca\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T17:37:57Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-21T17:38:24Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.4\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"host","hostPath":{"path":"/etc/docker/certs.d","type":""}},{"name":"serviceca","configMap":{"name":"image-registry-certificates","defaultMode":420}},{"name":"kube-api-access-zrjw9","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"node-ca","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:26b626f9b0de3609f607fd85f4fb6865880dc100273846fcc8947cbcc8a2840c","command":["/bin/sh","-c","trap 'jobs -p \| xargs -r kill; echo shutting down node-ca; exit 0' TERM\nwhile [ true ];\ndo\n for f in $(ls /tmp/serviceca); do\n echo $f\n ca_file_path=\"/tmp/serviceca/${f}\"\n f=$(echo $f \| sed -r 's/(.)\\.\\./\\1:/')\n reg_dir_path=\"/etc/docker/certs.d/${f}\"\n if [ -e \"${reg_dir_path}\" ]; then\n cp -u $ca_file_path $reg_dir_path/ca.crt\n else\n mkdir $reg_dir_path\n cp $ca_file_path $reg_dir_path/ca.crt\n fi\n done\n for d in $(ls /etc/docker/certs.d); do\n echo $d\n dp=$(echo $d \| sed -r 's/(.):/\\1\\.\\./')\n reg_conf_path=\"/tmp/serviceca/${dp}\"\n if [ ! -e \"${reg_conf_path}\" ]; then\n rm -rf /etc/docker/certs.d/$d\n fi\n done\n sleep 60 \u0026 wait ${!}\ndone\n"],"resources":{"requests":{"cpu":"10m","memory":"10Mi"}},"volumeMounts":[{"name":"serviceca","mountPath":"/tmp/serviceca"},{"name":"host","mountPath":"/etc/docker/certs.d"},{"name":"kube-api-access-zrjw9","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true,"runAsUser":1001,"runAsGroup":0}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"node-ca","serviceAccount":"node-ca","nodeName":"ci-op-h0w5q3d8-253f3-prxcj-worker-centralus2-v8gsn","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"node-ca-dockercfg-jl6n8"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-h0w5q3d8-253f3-prxcj-worker-centralus2-v8gsn"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-cluster-critical","priority":2000000000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T17:14:49Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T17:37:57Z","reason":"ContainersNotReady","message":"containers with unready status: [node-ca]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T17:38:23Z","reason":"ContainersNotReady","message":"containers with unready status: [node-ca]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T17:14:49Z"}],"hostIP":"10.0.128.4","podIP":"10.0.128.4","podIPs":[{"ip":"10.0.128.4"}],"startTime":"2023-08-21T17:14:49Z","containerStatuses":[{"name":"node-ca","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage2876476718/1\": happened during read: read tcp 10.0.128.4:48072-\u003e54.230.17.85:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:26b626f9b0de3609f607fd85f4fb6865880dc100273846fcc8947cbcc8a2840c","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-multus pod/multus-ltdxf node/ci-op-h0w5q3d8-253f3-prxcj-worker-centralus2-v8gsn container/kube-multus - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"multus-ltdxf","generateName":"multus-","namespace":"openshift-multus","uid":"bd31feb9-1760-4a11-9328-56c65c3ca39d","resourceVersion":"64276","creationTimestamp":"2023-08-21T17:19:48Z","labels":{"app":"multus","component":"network","controller-revision-hash":"7df49bc4d8","openshift.io/component":"network","pod-template-generation":"2","type":"infra"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"multus","uid":"a1760432-1c1b-4ca3-95d4-cf4b5e35c50d","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T17:19:48Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:component":{},"f:controller-revision-hash":{},"f:openshift.io/component":{},"f:pod-template-generation":{},"f:type":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"a1760432-1c1b-4ca3-95d4-cf4b5e35c50d\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"kube-multus\"}":{".":{},"f:args":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"DEFAULT_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_HOST\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_PORT\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RHEL7_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RHEL8_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/entrypoint\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/etc/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/etc/os-release\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/opt/cni/bin\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/var/run/multus/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"cni-binary-copy\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"cnibin\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"multus-cni-dir\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"os-release\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"system-cni-dir\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T17:37:57Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-21T17:38:24Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.4\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"system-cni-dir","hostPath":{"path":"/etc/kubernetes/cni/net.d","type":""}},{"name":"multus-cni-dir","hostPath":{"path":"/var/run/multus/cni/net.d","type":""}},{"name":"cnibin","hostPath":{"path":"/var/lib/cni/bin","type":""}},{"name":"os-release","hostPath":{"path":"/etc/os-release","type":"File"}},{"name":"cni-binary-copy","configMap":{"name":"cni-binary-copy-script","defaultMode":484}},{"name":"kube-api-access-hk765","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"kube-multus","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:528a2e0fb6ca1a3c6aaefcbf1fa7bcce000182565a8900bcc27cfd8db47a6ca5","command":["/bin/bash","-ec","--"],"args":["/entrypoint/cnibincopy.sh; exec /entrypoint.sh --multus-conf-file=auto --multus-autoconfig-dir=/host/var/run/multus/cni/net.d --multus-kubeconfig-file-host=/etc/kubernetes/cni/net.d/multus.d/multus.kubeconfig --readiness-indicator-file=/var/run/multus/cni/net.d/80-openshift-network.conf --cleanup-config-on-exit=true --namespace-isolation=true --multus-log-level=verbose --cni-version=0.3.1 --additional-bin-dir=/opt/multus/bin --skip-multus-binary-copy=true - \"--global-namespaces=default,openshift-multus,openshift-sriov-network-operator\"\n"],"env":[{"name":"RHEL7_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/rhel7/bin/"},{"name":"RHEL8_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/rhel8/bin/"},{"name":"DEFAULT_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/bin/"},{"name":"KUBERNETES_SERVICE_PORT","value":"6443"},{"name":"KUBERNETES_SERVICE_HOST","value":"api-int.ci-op-h0w5q3d8-253f3.ci.azure.devcluster.openshift.com"}],"resources":{"requests":{"cpu":"10m","memory":"65Mi"}},"volumeMounts":[{"name":"cni-binary-copy","mountPath":"/entrypoint"},{"name":"os-release","mountPath":"/host/etc/os-release"},{"name":"system-cni-dir","mountPath":"/host/etc/cni/net.d"},{"name":"multus-cni-dir","mountPath":"/host/var/run/multus/cni/net.d"},{"name":"cnibin","mountPath":"/host/opt/cni/bin"},{"name":"kube-api-access-hk765","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"multus","serviceAccount":"multus","nodeName":"ci-op-h0w5q3d8-253f3-prxcj-worker-centralus2-v8gsn","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"multus-dockercfg-mm649"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-h0w5q3d8-253f3-prxcj-worker-centralus2-v8gsn"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T17:19:48Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T17:37:57Z","reason":"ContainersNotReady","message":"containers with unready status: [kube-multus]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T17:38:23Z","reason":"ContainersNotReady","message":"containers with unready status: [kube-multus]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T17:19:48Z"}],"hostIP":"10.0.128.4","podIP":"10.0.128.4","podIPs":[{"ip":"10.0.128.4"}],"startTime":"2023-08-21T17:19:48Z","containerStatuses":[{"name":"kube-multus","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage3979214308/2\": happened during read: read tcp 10.0.128.4:54316-\u003e54.230.17.223:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:528a2e0fb6ca1a3c6aaefcbf1fa7bcce000182565a8900bcc27cfd8db47a6ca5","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-cluster-csi-drivers pod/azure-disk-csi-driver-node-h9d6w node/ci-op-h0w5q3d8-253f3-prxcj-worker-centralus2-v8gsn container/azure-inject-credentials - reason/ContainerWait cause/ErrImagePull duration/1824.17s rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage3585477338/3": happened during read: read tcp 10.0.128.4:48086->54.230.17.85:443: read: connection timed out ns/openshift-sdn pod/sdn-skssz node/ci-op-h0w5q3d8-253f3-prxcj-worker-centralus2-v8gsn container/sdn - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"sdn-skssz","generateName":"sdn-","namespace":"openshift-sdn","uid":"6832760d-c6c5-456d-aa47-bf1fffa2d703","resourceVersion":"64280","creationTimestamp":"2023-08-21T17:19:53Z","labels":{"app":"sdn","component":"network","controller-revision-hash":"568fbcd95b","openshift.io/component":"network","pod-template-generation":"2","type":"infra"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"sdn","uid":"917e0044-8b54-4bea-ac35-1408fdfd212a","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T17:19:53Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:component":{},"f:controller-revision-hash":{},"f:openshift.io/component":{},"f:pod-template-generation":{},"f:type":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"917e0044-8b54-4bea-ac35-1408fdfd212a\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"drop-icmp\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:name":{}}}},"k:{\"name\":\"kube-rbac-proxy\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":9101,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/pki/tls/metrics-certs\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}},"k:{\"name\":\"sdn\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE_IP\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"K8S_NODE_NAME\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"KUBERNETES_SERVICE_HOST\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_PORT\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"OPENSHIFT_DNS_DOMAIN\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":10256,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:readinessProbe":{".":{},"f:exec":{".":{},"f:command":{}},"f:failureThreshold":{},"f:initialDelaySeconds":{},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/config\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/env\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/sysconfig\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host-cni-bin\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/var/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/lib/modules\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/lib/cni/networks/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/dbus/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/kubernetes/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/openvswitch/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:hostPID":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"config\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"env-overrides\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{},"f:optional":{}},"f:name":{}},"k:{\"name\":\"etc-sysconfig\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-bin\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-conf\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-modules\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-slash\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-lib-cni-networks-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-dbus\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-kubernetes\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-ovs\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"sdn-metrics-certs\"}":{".":{},"f:name":{},"f:secret":{".":{},"f:defaultMode":{},"f:optional":{},"f:secretName":{}}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T17:37:57Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-21T17:38:24Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.4\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"config","configMap":{"name":"sdn-config","defaultMode":420}},{"name":"env-overrides","configMap":{"name":"env-overrides","defaultMode":420,"optional":true}},{"name":"etc-sysconfig","hostPath":{"path":"/etc/sysconfig","type":""}},{"name":"host-modules","hostPath":{"path":"/lib/modules","type":""}},{"name":"host-var-run","hostPath":{"path":"/var/run","type":""}},{"name":"host-run-netns","hostPath":{"path":"/run/netns","type":""}},{"name":"host-var-run-netns","hostPath":{"path":"/var/run/netns","type":""}},{"name":"host-var-run-dbus","hostPath":{"path":"/var/run/dbus","type":""}},{"name":"host-var-run-ovs","hostPath":{"path":"/var/run/openvswitch","type":""}},{"name":"host-var-run-kubernetes","hostPath":{"path":"/var/run/kubernetes","type":""}},{"name":"host-var-run-openshift-sdn","hostPath":{"path":"/var/run/openshift-sdn","type":""}},{"name":"host-slash","hostPath":{"path":"/","type":""}},{"name":"host-cni-bin","hostPath":{"path":"/var/lib/cni/bin","type":""}},{"name":"host-cni-conf","hostPath":{"path":"/var/run/multus/cni/net.d","type":""}},{"name":"host-var-lib-cni-networks-openshift-sdn","hostPath":{"path":"/var/lib/cni/networks/openshift-sdn","type":""}},{"name":"sdn-metrics-certs","secret":{"secretName":"sdn-metrics-certs","defaultMode":420,"optional":true}},{"name":"kube-api-access-s76fx","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"sdn","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\n\n# if another process is listening on the cni-server socket, wait until it exits\ntrap 'kill $(jobs -p); rm -f /etc/cni/net.d/80-openshift-network.conf ; exit 0' TERM\nretries=0\nwhile true; do\n if echo 'test' \| socat - UNIX-CONNECT:/var/run/openshift-sdn/cniserver/socket \u0026\u003e/dev/null; then\n echo \"warning: Another process is currently listening on the CNI socket, waiting 15s ...\" 2\u003e\u00261\n sleep 15 \u0026 wait\n (( retries += 1 ))\n else\n break\n fi\n if [[ \"${retries}\" -gt 40 ]]; then\n echo \"error: Another process is currently listening on the CNI socket, exiting\" 2\u003e\u00261\n exit 1\n fi\ndone\n\n# local environment overrides\nif [[ -f /etc/sysconfig/openshift-sdn ]]; then\n set -o allexport\n source /etc/sysconfig/openshift-sdn\n set +o allexport\nfi\n#BUG: cdc accidentally mounted /etc/sysconfig/openshift-sdn as DirectoryOrCreate; clean it up so we can ultimately mount /etc/sysconfig/openshift-sdn as FileOrCreate\n# Once this is released, then we can mount it properly\nif [[ -d /etc/sysconfig/openshift-sdn ]]; then\n rmdir /etc/sysconfig/openshift-sdn \|\| true\nfi\n\n# configmap-based overrides\nif [[ -f /env/${K8S_NODE_NAME} ]]; then\n set -o allexport\n source /env/${K8S_NODE_NAME}\n set +o allexport\nfi\n\n# Take over network functions on the node\nrm -f /etc/cni/net.d/80-openshift-network.conf\ncp -f /opt/cni/bin/openshift-sdn /host-cni-bin/\n\nmtu_override_flag=\nif [[ -f /config/mtu.yaml ]]; then\n mtu_override_flag=\"--mtu-override /config/mtu.yaml\"\nfi\n\n# Launch the network process\nexec /usr/bin/openshift-sdn-node \\\n --node-name ${K8S_NODE_NAME} --node-ip ${K8S_NODE_IP} \\\n --platform-type Azure \\\n --proxy-config /config/kube-proxy-config.yaml \\\n ${mtu_override_flag} \\\n --v ${OPENSHIFT_SDN_LOG_LEVEL:-2}\n"],"ports":[{"name":"healthz","hostPort":10256,"containerPort":10256,"protocol":"TCP"}],"env":[{"name":"KUBERNETES_SERVICE_PORT","value":"6443"},{"name":"KUBERNETES_SERVICE_HOST","value":"api-int.ci-op-h0w5q3d8-253f3.ci.azure.devcluster.openshift.com"},{"name":"OPENSHIFT_DNS_DOMAIN","value":"cluster.local"},{"name":"K8S_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"K8S_NODE_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.hostIP"}}}],"resources":{"requests":{"cpu":"100m","memory":"200Mi"}},"volumeMounts":[{"name":"config","readOnly":true,"mountPath":"/config"},{"name":"env-overrides","mountPath":"/env"},{"name":"host-var-run","mountPath":"/var/run"},{"name":"host-var-run-dbus","readOnly":true,"mountPath":"/var/run/dbus/"},{"name":"host-var-run-ovs","readOnly":true,"mountPath":"/var/run/openvswitch/"},{"name":"host-var-run-kubernetes","readOnly":true,"mountPath":"/var/run/kubernetes/"},{"name":"host-run-netns","readOnly":true,"mountPath":"/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-netns","readOnly":true,"mountPath":"/host/var/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-openshift-sdn","mountPath":"/var/run/openshift-sdn"},{"name":"host-slash","readOnly":true,"mountPath":"/host","mountPropagation":"HostToContainer"},{"name":"host-cni-bin","mountPath":"/host-cni-bin"},{"name":"host-cni-conf","mountPath":"/etc/cni/net.d"},{"name":"host-var-lib-cni-networks-openshift-sdn","mountPath":"/var/lib/cni/networks/openshift-sdn"},{"name":"host-modules","readOnly":true,"mountPath":"/lib/modules"},{"name":"etc-sysconfig","readOnly":true,"mountPath":"/etc/sysconfig"},{"name":"kube-api-access-s76fx","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"readinessProbe":{"exec":{"command":["test","-f","/etc/cni/net.d/80-openshift-network.conf"]},"initialDelaySeconds":5,"timeoutSeconds":1,"periodSeconds":5,"successThreshold":1,"failureThreshold":3},"lifecycle":{"preStop":{"exec":{"command":["rm","-f","/etc/cni/net.d/80-openshift-network.conf","/host-cni-bin/openshift-sdn"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}},{"name":"kube-rbac-proxy","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\nTLS_PK=/etc/pki/tls/metrics-certs/tls.key\nTLS_CERT=/etc/pki/tls/metrics-certs/tls.crt\n\n# As the secret mount is optional we must wait for the files to be present.\n# The service is created in monitor.yaml and this is created in sdn.yaml.\n# If it isn't created there is probably an issue so we want to crashloop.\nTS=$(date +%s)\nWARN_TS=$(( ${TS} + $(( 20 * 60)) ))\nHAS_LOGGED_INFO=0\n\nlog_missing_certs(){\n CUR_TS=$(date +%s)\n if [[ \"${CUR_TS}\" -gt \"WARN_TS\" ]]; then\n echo $(date -Iseconds) WARN: sdn-metrics-certs not mounted after 20 minutes.\n elif [[ \"${HAS_LOGGED_INFO}\" -eq 0 ]] ; then\n echo $(date -Iseconds) INFO: sdn-metrics-certs not mounted. Waiting 20 minutes.\n HAS_LOGGED_INFO=1\n fi\n}\n\nwhile [[ ! -f \"${TLS_PK}\" \|\| ! -f \"${TLS_CERT}\" ]] ; do\n log_missing_certs\n sleep 5\ndone\n\necho $(date -Iseconds) INFO: sdn-metrics-certs mounted, starting kube-rbac-proxy\nexec /usr/bin/kube-rbac-proxy \\\n --logtostderr \\\n --secure-listen-address=:9101 \\\n --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \\\n --upstream=http://127.0.0.1:29101/ \\\n --tls-private-key-file=${TLS_PK} \\\n --tls-cert-file=${TLS_CERT}\n"],"ports":[{"name":"https","hostPort":9101,"containerPort":9101,"protocol":"TCP"}],"resources":{"requests":{"cpu":"10m","memory":"20Mi"}},"volumeMounts":[{"name":"sdn-metrics-certs","readOnly":true,"mountPath":"/etc/pki/tls/metrics-certs"},{"name":"kube-api-access-s76fx","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent"},{"name":"drop-icmp","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","set -xe\n\ntouch /var/run/add_iptables.sh\nchmod 0755 /var/run/add_iptables.sh\ncat \u003c\u003c'EOF' \u003e /var/run/add_iptables.sh\n#!/bin/sh\nif [ -z \"$3\" ]\nthen\n echo \"Called with host address missing, ignore\"\n exit 0\nfi\necho \"Adding ICMP drop rule for '$3' \"\nif iptables -C AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nthen\n echo \"iptables already set for $3\"\nelse\n iptables -A AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nfi\nEOF\n\necho \"I$(date \"+%m%d %H:%M:%S.%N\") - drop-icmp - start drop-icmp ${K8S_NODE}\"\niptables -X AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -N AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -F AZURE_CHECK_ICMP_SOURCE\niptables -D INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -I INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE\niptables -N AZURE_ICMP_ACTION \|\| true\niptables -F AZURE_ICMP_ACTION\niptables -A AZURE_ICMP_ACTION -j LOG\niptables -A AZURE_ICMP_ACTION -j DROP\n/host/usr/bin/oc observe pods -n openshift-sdn -l app=sdn -a '{ .status.hostIP }' -- /var/run/add_iptables.sh\n"],"env":[{"name":"K8S_NODE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}}],"resources":{"requests":{"cpu":"5m","memory":"20Mi"}},"volumeMounts":[{"name":"host-slash","mountPath":"/host"},{"name":"kube-api-access-s76fx","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"lifecycle":{"preStop":{"exec":{"command":["/bin/bash","-c","echo drop-icmp done"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"sdn","serviceAccount":"sdn","nodeName":"ci-op-h0w5q3d8-253f3-prxcj-worker-centralus2-v8gsn","hostNetwork":true,"hostPID":true,"securityContext":{},"imagePullSecrets":[{"name":"sdn-dockercfg-hrzfq"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-h0w5q3d8-253f3-prxcj-worker-centralus2-v8gsn"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T17:19:53Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T17:37:57Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T17:38:23Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T17:19:53Z"}],"hostIP":"10.0.128.4","podIP":"10.0.128.4","podIPs":[{"ip":"10.0.128.4"}],"startTime":"2023-08-21T17:19:53Z","containerStatuses":[{"name":"drop-icmp","state":{"waiting":{"reason":"ImagePullBackOff","message":"Back-off pulling image \"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c\""}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false},{"name":"kube-rbac-proxy","state":{"running":{"startedAt":"2023-08-21T17:43:41Z"}},"lastState":{},"ready":true,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","imageID":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","containerID":"cri-o://b9dcdf711e41ce2658223aa037ac7868cd5dda39b049fd0f17551e6f27b12e0f","started":true},{"name":"sdn","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage3882456229/3\": happened during read: read tcp 10.0.128.4:48054-\u003e54.230.17.85:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false}],"qosClass":"Burstable"}}
#1693698494000271360	junit	36 hours ago
ns/openshift-cluster-node-tuning-operator pod/tuned-qxg2l node/ci-op-ltz2z0yv-253f3-5g2rk-worker-westus-psk9h container/tuned - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"tuned-qxg2l","generateName":"tuned-","namespace":"openshift-cluster-node-tuning-operator","uid":"917e933f-6436-4ba8-8758-f4aed850d983","resourceVersion":"70072","creationTimestamp":"2023-08-21T20:17:32Z","labels":{"controller-revision-hash":"8bb6bb4cf","openshift-app":"tuned","pod-template-generation":"2"},"annotations":{"openshift.io/scc":"privileged"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"tuned","uid":"aa755580-e3c1-4878-93d3-f8b81b5603f0","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T20:17:32Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:openshift-app":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"aa755580-e3c1-4878-93d3-f8b81b5603f0\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"tuned\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"OCP_NODE_NAME\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"RELEASE_VERSION\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RESYNC_PERIOD\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"WATCH_NAMESPACE\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/modprobe.d\"}":{".":{},"f:mountPath":{},"f:name":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/sysconfig\"}":{".":{},"f:mountPath":{},"f:name":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/sysctl.conf\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/sysctl.d\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/systemd\"}":{".":{},"f:mountPath":{},"f:name":{},"f:subPath":{}},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/lib/modules\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/run/systemd/system\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/sys\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/lib/tuned/profiles-data\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/dbus\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostIPC":{},"f:hostNetwork":{},"f:hostPID":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"etc\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"lib-modules\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"run-systemd-system\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"sys\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"var-lib-tuned-profiles-data\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:items":{},"f:name":{},"f:optional":{}},"f:name":{}},"k:{\"name\":\"var-run-dbus\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T20:53:56Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-21T20:54:43Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"etc","hostPath":{"path":"/etc","type":"Directory"}},{"name":"sys","hostPath":{"path":"/sys","type":"Directory"}},{"name":"var-run-dbus","hostPath":{"path":"/var/run/dbus","type":"Directory"}},{"name":"run-systemd-system","hostPath":{"path":"/run/systemd/system","type":"Directory"}},{"name":"lib-modules","hostPath":{"path":"/lib/modules","type":"Directory"}},{"name":"host","hostPath":{"path":"/","type":"Directory"}},{"name":"var-lib-tuned-profiles-data","configMap":{"name":"tuned-profiles","items":[{"key":"tuned-profiles-data","path":"tuned-profiles.yaml"}],"defaultMode":420,"optional":true}},{"name":"kube-api-access-x6zhk","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"tuned","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:a46e73a9d6e06a067d466c1a255bec4292c66b4fb7ef2b96fed282df394a17ae","command":["/var/lib/tuned/bin/run","start"],"env":[{"name":"WATCH_NAMESPACE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}},{"name":"OCP_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"RESYNC_PERIOD","value":"600"},{"name":"RELEASE_VERSION","value":"4.10.0-0.ci-2023-08-21-091209"}],"resources":{"requests":{"cpu":"10m","memory":"50Mi"}},"volumeMounts":[{"name":"var-lib-tuned-profiles-data","mountPath":"/var/lib/tuned/profiles-data"},{"name":"etc","mountPath":"/etc/modprobe.d","subPath":"modprobe.d"},{"name":"etc","mountPath":"/etc/sysconfig","subPath":"sysconfig"},{"name":"etc","readOnly":true,"mountPath":"/etc/sysctl.d","subPath":"sysctl.d"},{"name":"etc","readOnly":true,"mountPath":"/etc/sysctl.conf","subPath":"sysctl.conf"},{"name":"etc","mountPath":"/etc/systemd","subPath":"systemd"},{"name":"sys","mountPath":"/sys"},{"name":"var-run-dbus","readOnly":true,"mountPath":"/var/run/dbus"},{"name":"run-systemd-system","readOnly":true,"mountPath":"/run/systemd/system"},{"name":"lib-modules","readOnly":true,"mountPath":"/lib/modules"},{"name":"host","mountPath":"/host"},{"name":"kube-api-access-x6zhk","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"lifecycle":{"preStop":{"exec":{"command":["/var/lib/tuned/bin/run","stop"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"tuned","serviceAccount":"tuned","nodeName":"ci-op-ltz2z0yv-253f3-5g2rk-worker-westus-psk9h","hostNetwork":true,"hostPID":true,"hostIPC":true,"securityContext":{},"imagePullSecrets":[{"name":"tuned-dockercfg-s482r"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-ltz2z0yv-253f3-5g2rk-worker-westus-psk9h"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T20:17:32Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T20:53:56Z","reason":"ContainersNotReady","message":"containers with unready status: [tuned]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T20:54:43Z","reason":"ContainersNotReady","message":"containers with unready status: [tuned]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T20:17:32Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-21T20:17:32Z","containerStatuses":[{"name":"tuned","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = reading signatures: Get \"https://registry.ci.openshift.org/extensions/v2/ocp/4.10-2023-08-21-091209/signatures/sha256:a46e73a9d6e06a067d466c1a255bec4292c66b4fb7ef2b96fed282df394a17ae\": read tcp 10.0.128.5:38142-\u003e34.234.122.5:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:a46e73a9d6e06a067d466c1a255bec4292c66b4fb7ef2b96fed282df394a17ae","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-multus pod/multus-additional-cni-plugins-8pcrk node/ci-op-ltz2z0yv-253f3-5g2rk-worker-westus-psk9h container/egress-router-binary-copy - reason/ContainerWait cause/ErrImagePull duration/1645.10s rpc error: code = Unknown desc = reading signatures: Get "https://registry.ci.openshift.org/extensions/v2/ocp/4.10-2023-08-21-091209/signatures/sha256:f4f7d1197b4a3098ee1b4c9ff1e1a4bf8d2de7cd5e6662b7b345dd6bd511b8d2": read tcp 10.0.128.5:44140->44.217.124.80:443: read: connection timed out ns/openshift-image-registry pod/node-ca-kcvsc node/ci-op-ltz2z0yv-253f3-5g2rk-worker-westus-psk9h container/node-ca - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"node-ca-kcvsc","generateName":"node-ca-","namespace":"openshift-image-registry","uid":"3769e507-2abb-4593-919b-91bd2dab655b","resourceVersion":"70074","creationTimestamp":"2023-08-21T20:19:15Z","labels":{"controller-revision-hash":"7c956f79fd","name":"node-ca","pod-template-generation":"2"},"annotations":{"openshift.io/scc":"privileged"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"node-ca","uid":"7615f3c8-5d61-4a40-b352-da15037ef8b6","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T20:19:15Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:name":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"7615f3c8-5d61-4a40-b352-da15037ef8b6\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"node-ca\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{},"f:runAsGroup":{},"f:runAsUser":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/docker/certs.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/tmp/serviceca\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"host\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"serviceca\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T20:53:56Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-21T20:54:43Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"host","hostPath":{"path":"/etc/docker/certs.d","type":""}},{"name":"serviceca","configMap":{"name":"image-registry-certificates","defaultMode":420}},{"name":"kube-api-access-rtznq","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"node-ca","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:26b626f9b0de3609f607fd85f4fb6865880dc100273846fcc8947cbcc8a2840c","command":["/bin/sh","-c","trap 'jobs -p \| xargs -r kill; echo shutting down node-ca; exit 0' TERM\nwhile [ true ];\ndo\n for f in $(ls /tmp/serviceca); do\n echo $f\n ca_file_path=\"/tmp/serviceca/${f}\"\n f=$(echo $f \| sed -r 's/(.)\\.\\./\\1:/')\n reg_dir_path=\"/etc/docker/certs.d/${f}\"\n if [ -e \"${reg_dir_path}\" ]; then\n cp -u $ca_file_path $reg_dir_path/ca.crt\n else\n mkdir $reg_dir_path\n cp $ca_file_path $reg_dir_path/ca.crt\n fi\n done\n for d in $(ls /etc/docker/certs.d); do\n echo $d\n dp=$(echo $d \| sed -r 's/(.):/\\1\\.\\./')\n reg_conf_path=\"/tmp/serviceca/${dp}\"\n if [ ! -e \"${reg_conf_path}\" ]; then\n rm -rf /etc/docker/certs.d/$d\n fi\n done\n sleep 60 \u0026 wait ${!}\ndone\n"],"resources":{"requests":{"cpu":"10m","memory":"10Mi"}},"volumeMounts":[{"name":"serviceca","mountPath":"/tmp/serviceca"},{"name":"host","mountPath":"/etc/docker/certs.d"},{"name":"kube-api-access-rtznq","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true,"runAsUser":1001,"runAsGroup":0}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"node-ca","serviceAccount":"node-ca","nodeName":"ci-op-ltz2z0yv-253f3-5g2rk-worker-westus-psk9h","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"node-ca-dockercfg-txn7t"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-ltz2z0yv-253f3-5g2rk-worker-westus-psk9h"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-cluster-critical","priority":2000000000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T20:19:15Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T20:53:56Z","reason":"ContainersNotReady","message":"containers with unready status: [node-ca]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T20:54:43Z","reason":"ContainersNotReady","message":"containers with unready status: [node-ca]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T20:19:15Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-21T20:19:15Z","containerStatuses":[{"name":"node-ca","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = reading signatures: Get \"https://registry.ci.openshift.org/extensions/v2/ocp/4.10-2023-08-21-091209/signatures/sha256:26b626f9b0de3609f607fd85f4fb6865880dc100273846fcc8947cbcc8a2840c\": read tcp 10.0.128.5:38118-\u003e34.234.122.5:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:26b626f9b0de3609f607fd85f4fb6865880dc100273846fcc8947cbcc8a2840c","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-multus pod/multus-h2kgz node/ci-op-ltz2z0yv-253f3-5g2rk-worker-westus-psk9h container/kube-multus - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"multus-h2kgz","generateName":"multus-","namespace":"openshift-multus","uid":"5353800f-908e-4d3f-960f-d2393ee6e0dc","resourceVersion":"70076","creationTimestamp":"2023-08-21T20:27:12Z","labels":{"app":"multus","component":"network","controller-revision-hash":"7d8fbd8b9c","openshift.io/component":"network","pod-template-generation":"2","type":"infra"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"multus","uid":"bc8da206-b9ac-4712-ad9a-f4349351a7ca","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T20:27:12Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:component":{},"f:controller-revision-hash":{},"f:openshift.io/component":{},"f:pod-template-generation":{},"f:type":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"bc8da206-b9ac-4712-ad9a-f4349351a7ca\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"kube-multus\"}":{".":{},"f:args":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"DEFAULT_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_HOST\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_PORT\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RHEL7_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RHEL8_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/entrypoint\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/etc/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/etc/os-release\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/opt/cni/bin\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/var/run/multus/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"cni-binary-copy\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"cnibin\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"multus-cni-dir\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"os-release\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"system-cni-dir\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T20:53:56Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-21T20:54:43Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"system-cni-dir","hostPath":{"path":"/etc/kubernetes/cni/net.d","type":""}},{"name":"multus-cni-dir","hostPath":{"path":"/var/run/multus/cni/net.d","type":""}},{"name":"cnibin","hostPath":{"path":"/var/lib/cni/bin","type":""}},{"name":"os-release","hostPath":{"path":"/etc/os-release","type":"File"}},{"name":"cni-binary-copy","configMap":{"name":"cni-binary-copy-script","defaultMode":484}},{"name":"kube-api-access-mkrw7","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"kube-multus","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:528a2e0fb6ca1a3c6aaefcbf1fa7bcce000182565a8900bcc27cfd8db47a6ca5","command":["/bin/bash","-ec","--"],"args":["/entrypoint/cnibincopy.sh; exec /entrypoint.sh --multus-conf-file=auto --multus-autoconfig-dir=/host/var/run/multus/cni/net.d --multus-kubeconfig-file-host=/etc/kubernetes/cni/net.d/multus.d/multus.kubeconfig --readiness-indicator-file=/var/run/multus/cni/net.d/80-openshift-network.conf --cleanup-config-on-exit=true --namespace-isolation=true --multus-log-level=verbose --cni-version=0.3.1 --additional-bin-dir=/opt/multus/bin --skip-multus-binary-copy=true - \"--global-namespaces=default,openshift-multus,openshift-sriov-network-operator\"\n"],"env":[{"name":"RHEL7_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/rhel7/bin/"},{"name":"RHEL8_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/rhel8/bin/"},{"name":"DEFAULT_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/bin/"},{"name":"KUBERNETES_SERVICE_PORT","value":"6443"},{"name":"KUBERNETES_SERVICE_HOST","value":"api-int.ci-op-ltz2z0yv-253f3.ci.azure.devcluster.openshift.com"}],"resources":{"requests":{"cpu":"10m","memory":"65Mi"}},"volumeMounts":[{"name":"cni-binary-copy","mountPath":"/entrypoint"},{"name":"os-release","mountPath":"/host/etc/os-release"},{"name":"system-cni-dir","mountPath":"/host/etc/cni/net.d"},{"name":"multus-cni-dir","mountPath":"/host/var/run/multus/cni/net.d"},{"name":"cnibin","mountPath":"/host/opt/cni/bin"},{"name":"kube-api-access-mkrw7","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"multus","serviceAccount":"multus","nodeName":"ci-op-ltz2z0yv-253f3-5g2rk-worker-westus-psk9h","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"multus-dockercfg-fqsbd"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-ltz2z0yv-253f3-5g2rk-worker-westus-psk9h"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T20:27:12Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T20:53:56Z","reason":"ContainersNotReady","message":"containers with unready status: [kube-multus]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T20:54:43Z","reason":"ContainersNotReady","message":"containers with unready status: [kube-multus]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T20:27:12Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-21T20:27:12Z","containerStatuses":[{"name":"kube-multus","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = reading signatures: Get \"https://registry.ci.openshift.org/extensions/v2/ocp/4.10-2023-08-21-091209/signatures/sha256:528a2e0fb6ca1a3c6aaefcbf1fa7bcce000182565a8900bcc27cfd8db47a6ca5\": read tcp 10.0.128.5:38120-\u003e34.234.122.5:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:528a2e0fb6ca1a3c6aaefcbf1fa7bcce000182565a8900bcc27cfd8db47a6ca5","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-dns pod/node-resolver-57lhd node/ci-op-ltz2z0yv-253f3-5g2rk-worker-westus-psk9h container/dns-node-resolver - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"node-resolver-57lhd","generateName":"node-resolver-","namespace":"openshift-dns","uid":"c88bfd1c-39b1-40a5-b6dd-536d1c5e30d4","resourceVersion":"70078","creationTimestamp":"2023-08-21T20:33:48Z","labels":{"controller-revision-hash":"7db5bfd767","dns.operator.openshift.io/daemonset-node-resolver":"","pod-template-generation":"2"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"node-resolver","uid":"b2b25345-4e51-4913-a5a1-f8987c5c6527","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T20:33:48Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:dns.operator.openshift.io/daemonset-node-resolver":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"b2b25345-4e51-4913-a5a1-f8987c5c6527\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"dns-node-resolver\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"CLUSTER_DOMAIN\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"NAMESERVER\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"SERVICES\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/hosts\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"hosts-file\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T20:53:56Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-21T20:54:43Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"hosts-file","hostPath":{"path":"/etc/hosts","type":"File"}},{"name":"kube-api-access-86lnb","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"dns-node-resolver","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:80cc077c52369f7a4e73e7676d03fd7659f3b23e4033c26c3edff4d18096f27c","command":["/bin/bash","-c","#!/bin/bash\nset -uo pipefail\n\ntrap 'jobs -p \| xargs kill \|\| true; wait; exit 0' TERM\n\nOPENSHIFT_MARKER=\"openshift-generated-node-resolver\"\nHOSTS_FILE=\"/etc/hosts\"\nTEMP_FILE=\"/etc/hosts.tmp\"\n\nIFS=', ' read -r -a services \u003c\u003c\u003c \"${SERVICES}\"\n\n# Make a temporary file with the old hosts file's attributes.\ncp -f --attributes-only \"${HOSTS_FILE}\" \"${TEMP_FILE}\"\n\nwhile true; do\n declare -A svc_ips\n for svc in \"${services[@]}\"; do\n # Fetch service IP from cluster dns if present. We make several tries\n # to do it: IPv4, IPv6, IPv4 over TCP and IPv6 over TCP. The two last ones\n # are for deployments with Kuryr on older OpenStack (OSP13) - those do not\n # support UDP loadbalancers and require reaching DNS through TCP.\n cmds=('dig -t A @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"'\n 'dig -t AAAA @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"'\n 'dig -t A +tcp +retry=0 @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"'\n 'dig -t AAAA +tcp +retry=0 @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"')\n for i in ${!cmds[]}\n do\n ips=($(eval \"${cmds[i]}\"))\n if [[ \"$?\" -eq 0 \u0026\u0026 \"${#ips[@]}\" -ne 0 ]]; then\n svc_ips[\"${svc}\"]=\"${ips[@]}\"\n break\n fi\n done\n done\n\n # Update /etc/hosts only if we get valid service IPs\n # We will not update /etc/hosts when there is coredns service outage or api unavailability\n # Stale entries could exist in /etc/hosts if the service is deleted\n if [[ -n \"${svc_ips[]-}\" ]]; then\n # Build a new hosts file from /etc/hosts with our custom entries filtered out\n grep -v \"# ${OPENSHIFT_MARKER}\" \"${HOSTS_FILE}\" \u003e \"${TEMP_FILE}\"\n\n # Append resolver entries for services\n for svc in \"${!svc_ips[@]}\"; do\n for ip in ${svc_ips[${svc}]}; do\n echo \"${ip} ${svc} ${svc}.${CLUSTER_DOMAIN} # ${OPENSHIFT_MARKER}\" \u003e\u003e \"${TEMP_FILE}\"\n done\n done\n\n # TODO: Update /etc/hosts atomically to avoid any inconsistent behavior\n # Replace /etc/hosts with our modified version if needed\n cmp \"${TEMP_FILE}\" \"${HOSTS_FILE}\" \|\| cp -f \"${TEMP_FILE}\" \"${HOSTS_FILE}\"\n # TEMP_FILE is not removed to avoid file create/delete and attributes copy churn\n fi\n sleep 60 \u0026 wait\n unset svc_ips\ndone\n"],"env":[{"name":"SERVICES","value":"image-registry.openshift-image-registry.svc"},{"name":"NAMESERVER","value":"172.30.0.10"},{"name":"CLUSTER_DOMAIN","value":"cluster.local"}],"resources":{"requests":{"cpu":"5m","memory":"21Mi"}},"volumeMounts":[{"name":"hosts-file","mountPath":"/etc/hosts"},{"name":"kube-api-access-86lnb","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"node-resolver","serviceAccount":"node-resolver","nodeName":"ci-op-ltz2z0yv-253f3-5g2rk-worker-westus-psk9h","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"node-resolver-dockercfg-9hc62"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-ltz2z0yv-253f3-5g2rk-worker-westus-psk9h"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T20:33:48Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T20:53:56Z","reason":"ContainersNotReady","message":"containers with unready status: [dns-node-resolver]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T20:54:43Z","reason":"ContainersNotReady","message":"containers with unready status: [dns-node-resolver]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T20:33:48Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-21T20:33:48Z","containerStatuses":[{"name":"dns-node-resolver","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = reading signatures: Get \"https://registry.ci.openshift.org/extensions/v2/ocp/4.10-2023-08-21-091209/signatures/sha256:80cc077c52369f7a4e73e7676d03fd7659f3b23e4033c26c3edff4d18096f27c\": read tcp 10.0.128.5:38036-\u003e34.234.122.5:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:80cc077c52369f7a4e73e7676d03fd7659f3b23e4033c26c3edff4d18096f27c","imageID":"","started":false}],"qosClass":"Burstable"}}
#1693553695679057920	junit	45 hours ago
ns/openshift-multus pod/multus-7rpwh node/ci-op-l48s32yd-253f3-7fqzj-worker-centralus2-q2kqx container/kube-multus - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"multus-7rpwh","generateName":"multus-","namespace":"openshift-multus","uid":"81cf6087-e7b7-4169-b98f-b8de008cafd0","resourceVersion":"77578","creationTimestamp":"2023-08-21T10:56:47Z","labels":{"app":"multus","component":"network","controller-revision-hash":"f4f447b49","openshift.io/component":"network","pod-template-generation":"2","type":"infra"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"multus","uid":"044fb15e-93e2-44c6-b26f-4f37113e60f5","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T10:56:47Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:component":{},"f:controller-revision-hash":{},"f:openshift.io/component":{},"f:pod-template-generation":{},"f:type":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"044fb15e-93e2-44c6-b26f-4f37113e60f5\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"kube-multus\"}":{".":{},"f:args":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"DEFAULT_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_HOST\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_PORT\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RHEL7_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RHEL8_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/entrypoint\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/etc/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/etc/os-release\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/opt/cni/bin\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/var/run/multus/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"cni-binary-copy\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"cnibin\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"multus-cni-dir\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"os-release\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"system-cni-dir\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T11:27:39Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-21T11:28:15Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"system-cni-dir","hostPath":{"path":"/etc/kubernetes/cni/net.d","type":""}},{"name":"multus-cni-dir","hostPath":{"path":"/var/run/multus/cni/net.d","type":""}},{"name":"cnibin","hostPath":{"path":"/var/lib/cni/bin","type":""}},{"name":"os-release","hostPath":{"path":"/etc/os-release","type":"File"}},{"name":"cni-binary-copy","configMap":{"name":"cni-binary-copy-script","defaultMode":484}},{"name":"kube-api-access-l4ts9","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"kube-multus","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:528a2e0fb6ca1a3c6aaefcbf1fa7bcce000182565a8900bcc27cfd8db47a6ca5","command":["/bin/bash","-ec","--"],"args":["/entrypoint/cnibincopy.sh; exec /entrypoint.sh --multus-conf-file=auto --multus-autoconfig-dir=/host/var/run/multus/cni/net.d --multus-kubeconfig-file-host=/etc/kubernetes/cni/net.d/multus.d/multus.kubeconfig --readiness-indicator-file=/var/run/multus/cni/net.d/80-openshift-network.conf --cleanup-config-on-exit=true --namespace-isolation=true --multus-log-level=verbose --cni-version=0.3.1 --additional-bin-dir=/opt/multus/bin --skip-multus-binary-copy=true - \"--global-namespaces=default,openshift-multus,openshift-sriov-network-operator\"\n"],"env":[{"name":"RHEL7_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/rhel7/bin/"},{"name":"RHEL8_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/rhel8/bin/"},{"name":"DEFAULT_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/bin/"},{"name":"KUBERNETES_SERVICE_PORT","value":"6443"},{"name":"KUBERNETES_SERVICE_HOST","value":"api-int.ci-op-l48s32yd-253f3.ci.azure.devcluster.openshift.com"}],"resources":{"requests":{"cpu":"10m","memory":"65Mi"}},"volumeMounts":[{"name":"cni-binary-copy","mountPath":"/entrypoint"},{"name":"os-release","mountPath":"/host/etc/os-release"},{"name":"system-cni-dir","mountPath":"/host/etc/cni/net.d"},{"name":"multus-cni-dir","mountPath":"/host/var/run/multus/cni/net.d"},{"name":"cnibin","mountPath":"/host/opt/cni/bin"},{"name":"kube-api-access-l4ts9","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"multus","serviceAccount":"multus","nodeName":"ci-op-l48s32yd-253f3-7fqzj-worker-centralus2-q2kqx","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"multus-dockercfg-9vs78"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-l48s32yd-253f3-7fqzj-worker-centralus2-q2kqx"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T10:56:47Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T11:27:39Z","reason":"ContainersNotReady","message":"containers with unready status: [kube-multus]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T11:28:15Z","reason":"ContainersNotReady","message":"containers with unready status: [kube-multus]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T10:56:47Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-21T10:56:47Z","containerStatuses":[{"name":"kube-multus","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage2182359866/4\": happened during read: read tcp 10.0.128.5:36644-\u003e65.8.55.213:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:528a2e0fb6ca1a3c6aaefcbf1fa7bcce000182565a8900bcc27cfd8db47a6ca5","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-dns pod/node-resolver-zxfsc node/ci-op-l48s32yd-253f3-7fqzj-worker-centralus2-q2kqx container/dns-node-resolver - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"node-resolver-zxfsc","generateName":"node-resolver-","namespace":"openshift-dns","uid":"a34387e2-7477-4cf3-94bc-4832d93c9151","resourceVersion":"77581","creationTimestamp":"2023-08-21T11:00:43Z","labels":{"controller-revision-hash":"7db5bfd767","dns.operator.openshift.io/daemonset-node-resolver":"","pod-template-generation":"2"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"node-resolver","uid":"e0dd7b72-74d3-48c7-8175-60883cdf5fa9","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T11:00:43Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:dns.operator.openshift.io/daemonset-node-resolver":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"e0dd7b72-74d3-48c7-8175-60883cdf5fa9\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"dns-node-resolver\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"CLUSTER_DOMAIN\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"NAMESERVER\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"SERVICES\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/hosts\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"hosts-file\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T11:27:39Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-21T11:28:15Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"hosts-file","hostPath":{"path":"/etc/hosts","type":"File"}},{"name":"kube-api-access-fc49k","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"dns-node-resolver","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:80cc077c52369f7a4e73e7676d03fd7659f3b23e4033c26c3edff4d18096f27c","command":["/bin/bash","-c","#!/bin/bash\nset -uo pipefail\n\ntrap 'jobs -p \| xargs kill \|\| true; wait; exit 0' TERM\n\nOPENSHIFT_MARKER=\"openshift-generated-node-resolver\"\nHOSTS_FILE=\"/etc/hosts\"\nTEMP_FILE=\"/etc/hosts.tmp\"\n\nIFS=', ' read -r -a services \u003c\u003c\u003c \"${SERVICES}\"\n\n# Make a temporary file with the old hosts file's attributes.\ncp -f --attributes-only \"${HOSTS_FILE}\" \"${TEMP_FILE}\"\n\nwhile true; do\n declare -A svc_ips\n for svc in \"${services[@]}\"; do\n # Fetch service IP from cluster dns if present. We make several tries\n # to do it: IPv4, IPv6, IPv4 over TCP and IPv6 over TCP. The two last ones\n # are for deployments with Kuryr on older OpenStack (OSP13) - those do not\n # support UDP loadbalancers and require reaching DNS through TCP.\n cmds=('dig -t A @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"'\n 'dig -t AAAA @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"'\n 'dig -t A +tcp +retry=0 @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"'\n 'dig -t AAAA +tcp +retry=0 @\"${NAMESERVER}\" +short \"${svc}.${CLUSTER_DOMAIN}\"\|grep -v \"^;\"')\n for i in ${!cmds[]}\n do\n ips=($(eval \"${cmds[i]}\"))\n if [[ \"$?\" -eq 0 \u0026\u0026 \"${#ips[@]}\" -ne 0 ]]; then\n svc_ips[\"${svc}\"]=\"${ips[@]}\"\n break\n fi\n done\n done\n\n # Update /etc/hosts only if we get valid service IPs\n # We will not update /etc/hosts when there is coredns service outage or api unavailability\n # Stale entries could exist in /etc/hosts if the service is deleted\n if [[ -n \"${svc_ips[]-}\" ]]; then\n # Build a new hosts file from /etc/hosts with our custom entries filtered out\n grep -v \"# ${OPENSHIFT_MARKER}\" \"${HOSTS_FILE}\" \u003e \"${TEMP_FILE}\"\n\n # Append resolver entries for services\n for svc in \"${!svc_ips[@]}\"; do\n for ip in ${svc_ips[${svc}]}; do\n echo \"${ip} ${svc} ${svc}.${CLUSTER_DOMAIN} # ${OPENSHIFT_MARKER}\" \u003e\u003e \"${TEMP_FILE}\"\n done\n done\n\n # TODO: Update /etc/hosts atomically to avoid any inconsistent behavior\n # Replace /etc/hosts with our modified version if needed\n cmp \"${TEMP_FILE}\" \"${HOSTS_FILE}\" \|\| cp -f \"${TEMP_FILE}\" \"${HOSTS_FILE}\"\n # TEMP_FILE is not removed to avoid file create/delete and attributes copy churn\n fi\n sleep 60 \u0026 wait\n unset svc_ips\ndone\n"],"env":[{"name":"SERVICES","value":"image-registry.openshift-image-registry.svc"},{"name":"NAMESERVER","value":"172.30.0.10"},{"name":"CLUSTER_DOMAIN","value":"cluster.local"}],"resources":{"requests":{"cpu":"5m","memory":"21Mi"}},"volumeMounts":[{"name":"hosts-file","mountPath":"/etc/hosts"},{"name":"kube-api-access-fc49k","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"node-resolver","serviceAccount":"node-resolver","nodeName":"ci-op-l48s32yd-253f3-7fqzj-worker-centralus2-q2kqx","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"node-resolver-dockercfg-kh9ln"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-l48s32yd-253f3-7fqzj-worker-centralus2-q2kqx"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T11:00:43Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T11:27:39Z","reason":"ContainersNotReady","message":"containers with unready status: [dns-node-resolver]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T11:28:15Z","reason":"ContainersNotReady","message":"containers with unready status: [dns-node-resolver]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T11:00:43Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-21T11:00:43Z","containerStatuses":[{"name":"dns-node-resolver","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage196750838/4\": happened during read: read tcp 10.0.128.5:36662-\u003e65.8.55.213:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:80cc077c52369f7a4e73e7676d03fd7659f3b23e4033c26c3edff4d18096f27c","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-cluster-csi-drivers pod/azure-disk-csi-driver-node-rfr7l node/ci-op-l48s32yd-253f3-7fqzj-worker-centralus2-q2kqx container/azure-inject-credentials - reason/ContainerWait cause/ErrImagePull duration/2447.53s rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage2781211055/3": happened during read: read tcp 10.0.128.5:49396->54.230.17.219:443: read: connection timed out ns/openshift-multus pod/multus-additional-cni-plugins-7t9nm node/ci-op-l48s32yd-253f3-7fqzj-worker-centralus2-q2kqx container/egress-router-binary-copy - reason/ContainerWait cause/ErrImagePull duration/2222.54s rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage2775515296/2": happened during read: read tcp 10.0.128.5:49422->54.230.17.219:443: read: connection timed out ns/openshift-image-registry pod/node-ca-wcn6r node/ci-op-l48s32yd-253f3-7fqzj-worker-centralus2-q2kqx container/node-ca - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"node-ca-wcn6r","generateName":"node-ca-","namespace":"openshift-image-registry","uid":"cdd397e6-ae01-4e44-b0e6-c063660df375","resourceVersion":"77584","creationTimestamp":"2023-08-21T10:51:23Z","labels":{"controller-revision-hash":"7c956f79fd","name":"node-ca","pod-template-generation":"2"},"annotations":{"openshift.io/scc":"privileged"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"node-ca","uid":"fa80085b-a378-425b-b7f9-1be9a61edde0","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T10:51:23Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:name":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"fa80085b-a378-425b-b7f9-1be9a61edde0\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"node-ca\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{},"f:runAsGroup":{},"f:runAsUser":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/docker/certs.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/tmp/serviceca\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"host\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"serviceca\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T11:27:40Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-21T11:28:15Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"host","hostPath":{"path":"/etc/docker/certs.d","type":""}},{"name":"serviceca","configMap":{"name":"image-registry-certificates","defaultMode":420}},{"name":"kube-api-access-2k25d","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"node-ca","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:26b626f9b0de3609f607fd85f4fb6865880dc100273846fcc8947cbcc8a2840c","command":["/bin/sh","-c","trap 'jobs -p \| xargs -r kill; echo shutting down node-ca; exit 0' TERM\nwhile [ true ];\ndo\n for f in $(ls /tmp/serviceca); do\n echo $f\n ca_file_path=\"/tmp/serviceca/${f}\"\n f=$(echo $f \| sed -r 's/(.)\\.\\./\\1:/')\n reg_dir_path=\"/etc/docker/certs.d/${f}\"\n if [ -e \"${reg_dir_path}\" ]; then\n cp -u $ca_file_path $reg_dir_path/ca.crt\n else\n mkdir $reg_dir_path\n cp $ca_file_path $reg_dir_path/ca.crt\n fi\n done\n for d in $(ls /etc/docker/certs.d); do\n echo $d\n dp=$(echo $d \| sed -r 's/(.):/\\1\\.\\./')\n reg_conf_path=\"/tmp/serviceca/${dp}\"\n if [ ! -e \"${reg_conf_path}\" ]; then\n rm -rf /etc/docker/certs.d/$d\n fi\n done\n sleep 60 \u0026 wait ${!}\ndone\n"],"resources":{"requests":{"cpu":"10m","memory":"10Mi"}},"volumeMounts":[{"name":"serviceca","mountPath":"/tmp/serviceca"},{"name":"host","mountPath":"/etc/docker/certs.d"},{"name":"kube-api-access-2k25d","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true,"runAsUser":1001,"runAsGroup":0}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"node-ca","serviceAccount":"node-ca","nodeName":"ci-op-l48s32yd-253f3-7fqzj-worker-centralus2-q2kqx","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"node-ca-dockercfg-tt5zk"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-l48s32yd-253f3-7fqzj-worker-centralus2-q2kqx"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-cluster-critical","priority":2000000000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T10:51:23Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T11:27:40Z","reason":"ContainersNotReady","message":"containers with unready status: [node-ca]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T11:28:15Z","reason":"ContainersNotReady","message":"containers with unready status: [node-ca]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T10:51:23Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-21T10:51:23Z","containerStatuses":[{"name":"node-ca","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage4093882563/3\": happened during read: read tcp 10.0.128.5:36678-\u003e65.8.55.213:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:26b626f9b0de3609f607fd85f4fb6865880dc100273846fcc8947cbcc8a2840c","imageID":"","started":false}],"qosClass":"Burstable"}}
#1694114620681228288	junit	9 hours ago
ns/openshift-multus pod/multus-additional-cni-plugins-ql7rz node/ci-op-v23dx441-253f3-vhnpj-worker-centralus1-7626l container/egress-router-binary-copy - reason/ContainerWait cause/ErrImagePull duration/1206.04s rpc error: code = Unknown desc = reading signatures: Get "https://registry.ci.openshift.org/extensions/v2/ocp/4.10-2023-08-22-222817/signatures/sha256:f4f7d1197b4a3098ee1b4c9ff1e1a4bf8d2de7cd5e6662b7b345dd6bd511b8d2": read tcp 10.0.128.5:51134->34.234.122.5:443: read: connection timed out ns/openshift-cluster-node-tuning-operator pod/tuned-8d6rk node/ci-op-v23dx441-253f3-vhnpj-worker-centralus1-7626l container/tuned - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"tuned-8d6rk","generateName":"tuned-","namespace":"openshift-cluster-node-tuning-operator","uid":"1fb378e7-3120-401e-97a4-9e96d4886e03","resourceVersion":"63420","creationTimestamp":"2023-08-22T23:46:49Z","labels":{"controller-revision-hash":"6d84d77867","openshift-app":"tuned","pod-template-generation":"2"},"annotations":{"openshift.io/scc":"privileged"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"tuned","uid":"a8b29cad-3e57-4b6b-9752-510008656896","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-22T23:46:49Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:openshift-app":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"a8b29cad-3e57-4b6b-9752-510008656896\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"tuned\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"OCP_NODE_NAME\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"RELEASE_VERSION\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RESYNC_PERIOD\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"WATCH_NAMESPACE\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/modprobe.d\"}":{".":{},"f:mountPath":{},"f:name":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/sysconfig\"}":{".":{},"f:mountPath":{},"f:name":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/sysctl.conf\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/sysctl.d\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{},"f:subPath":{}},"k:{\"mountPath\":\"/etc/systemd\"}":{".":{},"f:mountPath":{},"f:name":{},"f:subPath":{}},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/lib/modules\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/run/systemd/system\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/sys\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/lib/tuned/profiles-data\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/dbus\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostIPC":{},"f:hostNetwork":{},"f:hostPID":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"etc\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"lib-modules\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"run-systemd-system\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"sys\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"var-lib-tuned-profiles-data\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:items":{},"f:name":{},"f:optional":{}},"f:name":{}},"k:{\"name\":\"var-run-dbus\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-23T00:08:17Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-23T00:08:54Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"etc","hostPath":{"path":"/etc","type":"Directory"}},{"name":"sys","hostPath":{"path":"/sys","type":"Directory"}},{"name":"var-run-dbus","hostPath":{"path":"/var/run/dbus","type":"Directory"}},{"name":"run-systemd-system","hostPath":{"path":"/run/systemd/system","type":"Directory"}},{"name":"lib-modules","hostPath":{"path":"/lib/modules","type":"Directory"}},{"name":"host","hostPath":{"path":"/","type":"Directory"}},{"name":"var-lib-tuned-profiles-data","configMap":{"name":"tuned-profiles","items":[{"key":"tuned-profiles-data","path":"tuned-profiles.yaml"}],"defaultMode":420,"optional":true}},{"name":"kube-api-access-2bqvd","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"tuned","image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:a46e73a9d6e06a067d466c1a255bec4292c66b4fb7ef2b96fed282df394a17ae","command":["/var/lib/tuned/bin/run","start"],"env":[{"name":"WATCH_NAMESPACE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}},{"name":"OCP_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"RESYNC_PERIOD","value":"600"},{"name":"RELEASE_VERSION","value":"4.10.0-0.ci-2023-08-22-222817"}],"resources":{"requests":{"cpu":"10m","memory":"50Mi"}},"volumeMounts":[{"name":"var-lib-tuned-profiles-data","mountPath":"/var/lib/tuned/profiles-data"},{"name":"etc","mountPath":"/etc/modprobe.d","subPath":"modprobe.d"},{"name":"etc","mountPath":"/etc/sysconfig","subPath":"sysconfig"},{"name":"etc","readOnly":true,"mountPath":"/etc/sysctl.d","subPath":"sysctl.d"},{"name":"etc","readOnly":true,"mountPath":"/etc/sysctl.conf","subPath":"sysctl.conf"},{"name":"etc","mountPath":"/etc/systemd","subPath":"systemd"},{"name":"sys","mountPath":"/sys"},{"name":"var-run-dbus","readOnly":true,"mountPath":"/var/run/dbus"},{"name":"run-systemd-system","readOnly":true,"mountPath":"/run/systemd/system"},{"name":"lib-modules","readOnly":true,"mountPath":"/lib/modules"},{"name":"host","mountPath":"/host"},{"name":"kube-api-access-2bqvd","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"lifecycle":{"preStop":{"exec":{"command":["/var/lib/tuned/bin/run","stop"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"tuned","serviceAccount":"tuned","nodeName":"ci-op-v23dx441-253f3-vhnpj-worker-centralus1-7626l","hostNetwork":true,"hostPID":true,"hostIPC":true,"securityContext":{},"imagePullSecrets":[{"name":"tuned-dockercfg-5xkmz"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-v23dx441-253f3-vhnpj-worker-centralus1-7626l"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T23:46:49Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T00:08:17Z","reason":"ContainersNotReady","message":"containers with unready status: [tuned]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T00:08:53Z","reason":"ContainersNotReady","message":"containers with unready status: [tuned]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T23:46:49Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-22T23:46:49Z","containerStatuses":[{"name":"tuned","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = reading signatures: Get \"https://registry.ci.openshift.org/extensions/v2/ocp/4.10-2023-08-22-222817/signatures/sha256:a46e73a9d6e06a067d466c1a255bec4292c66b4fb7ef2b96fed282df394a17ae\": read tcp 10.0.128.5:51166-\u003e34.234.122.5:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:a46e73a9d6e06a067d466c1a255bec4292c66b4fb7ef2b96fed282df394a17ae","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-monitoring pod/node-exporter-8mpg5 node/ci-op-v23dx441-253f3-vhnpj-worker-centralus1-7626l container/init-textfile - reason/ContainerWait cause/ErrImagePull duration/1631.04s rpc error: code = Unknown desc = reading signatures: Get "https://registry.ci.openshift.org/extensions/v2/ocp/4.10-2023-08-22-222817/signatures/sha256:9e69356edf4d1e36bb53156906c69762e987a454aa4ec5c83f7ac55483bede9a": read tcp 10.0.128.5:51172->34.234.122.5:443: read: connection timed out ns/openshift-multus pod/multus-cxl27 node/ci-op-v23dx441-253f3-vhnpj-worker-centralus1-7626l container/kube-multus - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"multus-cxl27","generateName":"multus-","namespace":"openshift-multus","uid":"f6174840-1f79-4f4d-9ac6-65b2808a7818","resourceVersion":"63434","creationTimestamp":"2023-08-22T23:49:59Z","labels":{"app":"multus","component":"network","controller-revision-hash":"647694dd6","openshift.io/component":"network","pod-template-generation":"2","type":"infra"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"multus","uid":"11d0bba4-cc76-48ea-b32e-92bf1b33ca94","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-22T23:49:59Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:component":{},"f:controller-revision-hash":{},"f:openshift.io/component":{},"f:pod-template-generation":{},"f:type":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"11d0bba4-cc76-48ea-b32e-92bf1b33ca94\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"kube-multus\"}":{".":{},"f:args":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"DEFAULT_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_HOST\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_PORT\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RHEL7_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RHEL8_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/entrypoint\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/etc/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/etc/os-release\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/opt/cni/bin\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/var/run/multus/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"cni-binary-copy\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"cnibin\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"multus-cni-dir\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"os-release\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"system-cni-dir\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-23T00:08:17Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-23T00:08:54Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"system-cni-dir","hostPath":{"path":"/etc/kubernetes/cni/net.d","type":""}},{"name":"multus-cni-dir","hostPath":{"path":"/var/run/multus/cni/net.d","type":""}},{"name":"cnibin","hostPath":{"path":"/var/lib/cni/bin","type":""}},{"name":"os-release","hostPath":{"path":"/etc/os-release","type":"File"}},{"name":"cni-binary-copy","configMap":{"name":"cni-binary-copy-script","defaultMode":484}},{"name":"kube-api-access-4v5ss","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"kube-multus","image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:528a2e0fb6ca1a3c6aaefcbf1fa7bcce000182565a8900bcc27cfd8db47a6ca5","command":["/bin/bash","-ec","--"],"args":["/entrypoint/cnibincopy.sh; exec /entrypoint.sh --multus-conf-file=auto --multus-autoconfig-dir=/host/var/run/multus/cni/net.d --multus-kubeconfig-file-host=/etc/kubernetes/cni/net.d/multus.d/multus.kubeconfig --readiness-indicator-file=/var/run/multus/cni/net.d/80-openshift-network.conf --cleanup-config-on-exit=true --namespace-isolation=true --multus-log-level=verbose --cni-version=0.3.1 --additional-bin-dir=/opt/multus/bin --skip-multus-binary-copy=true - \"--global-namespaces=default,openshift-multus,openshift-sriov-network-operator\"\n"],"env":[{"name":"RHEL7_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/rhel7/bin/"},{"name":"RHEL8_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/rhel8/bin/"},{"name":"DEFAULT_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/bin/"},{"name":"KUBERNETES_SERVICE_PORT","value":"6443"},{"name":"KUBERNETES_SERVICE_HOST","value":"api-int.ci-op-v23dx441-253f3.ci.azure.devcluster.openshift.com"}],"resources":{"requests":{"cpu":"10m","memory":"65Mi"}},"volumeMounts":[{"name":"cni-binary-copy","mountPath":"/entrypoint"},{"name":"os-release","mountPath":"/host/etc/os-release"},{"name":"system-cni-dir","mountPath":"/host/etc/cni/net.d"},{"name":"multus-cni-dir","mountPath":"/host/var/run/multus/cni/net.d"},{"name":"cnibin","mountPath":"/host/opt/cni/bin"},{"name":"kube-api-access-4v5ss","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"multus","serviceAccount":"multus","nodeName":"ci-op-v23dx441-253f3-vhnpj-worker-centralus1-7626l","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"multus-dockercfg-2xv7q"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-v23dx441-253f3-vhnpj-worker-centralus1-7626l"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T23:49:59Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T00:08:17Z","reason":"ContainersNotReady","message":"containers with unready status: [kube-multus]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T00:08:53Z","reason":"ContainersNotReady","message":"containers with unready status: [kube-multus]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T23:49:59Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-22T23:49:59Z","containerStatuses":[{"name":"kube-multus","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = reading blob sha256:105c65c5c5cb89a834430f16db02441dc03869352f2a9ea6a86a51d3946f5a88: Get \"https://d1wgmlmph9kt2c.cloudfront.net/docker/registry/v2/blobs/sha256/10/105c65c5c5cb89a834430f16db02441dc03869352f2a9ea6a86a51d3946f5a88/data?Expires=1692750549\u0026Signature=0pwtbFmHGGEkaRjKmr60LMSyuuJgCXzap4UpxIPBiL3G-UH~SrUt43hGAQJ-lgbyi99CFWJBn0Dyg0xCE2eVNdAcuMAa~Q6fOrqaBp2dtExEiDv-iuVLvJn8A1AQOGdJP4yqe2wg02A4DZasBRSQNkFACkHH6BCFjK5QccvHoCHyuDaBvW7YRbwediqOaUBJ7hucnXWzaVwtqwmiPRRntP3ezbXOOccF93eAuNPeZbPasQkdmwkDklqxQlv49Bag9V6-JW0yq14dJWdG1V0eJqL1EQF3ZUk6lne-Hjms2MuFJWV0vcrgKE1YF2WcuI5azuGeMaG66BE7-jc5J4XodQ__\u0026Key-Pair-Id=K2SWMQXKFN57UD\": read tcp 10.0.128.5:33112-\u003e65.8.55.198:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:528a2e0fb6ca1a3c6aaefcbf1fa7bcce000182565a8900bcc27cfd8db47a6ca5","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-image-registry pod/node-ca-9rrdv node/ci-op-v23dx441-253f3-vhnpj-worker-centralus1-7626l container/node-ca - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"node-ca-9rrdv","generateName":"node-ca-","namespace":"openshift-image-registry","uid":"3bc3e5d6-d772-41c9-8c39-d218d9f1ea50","resourceVersion":"63436","creationTimestamp":"2023-08-22T23:46:32Z","labels":{"controller-revision-hash":"66c54747dd","name":"node-ca","pod-template-generation":"2"},"annotations":{"openshift.io/scc":"privileged"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"node-ca","uid":"8029314a-a769-4956-bf1e-2f37a595c256","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-22T23:46:32Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:name":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"8029314a-a769-4956-bf1e-2f37a595c256\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"node-ca\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{},"f:runAsGroup":{},"f:runAsUser":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/docker/certs.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/tmp/serviceca\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"host\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"serviceca\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-23T00:08:17Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-23T00:08:54Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"host","hostPath":{"path":"/etc/docker/certs.d","type":""}},{"name":"serviceca","configMap":{"name":"image-registry-certificates","defaultMode":420}},{"name":"kube-api-access-zntqd","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"node-ca","image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:26b626f9b0de3609f607fd85f4fb6865880dc100273846fcc8947cbcc8a2840c","command":["/bin/sh","-c","trap 'jobs -p \| xargs -r kill; echo shutting down node-ca; exit 0' TERM\nwhile [ true ];\ndo\n for f in $(ls /tmp/serviceca); do\n echo $f\n ca_file_path=\"/tmp/serviceca/${f}\"\n f=$(echo $f \| sed -r 's/(.)\\.\\./\\1:/')\n reg_dir_path=\"/etc/docker/certs.d/${f}\"\n if [ -e \"${reg_dir_path}\" ]; then\n cp -u $ca_file_path $reg_dir_path/ca.crt\n else\n mkdir $reg_dir_path\n cp $ca_file_path $reg_dir_path/ca.crt\n fi\n done\n for d in $(ls /etc/docker/certs.d); do\n echo $d\n dp=$(echo $d \| sed -r 's/(.):/\\1\\.\\./')\n reg_conf_path=\"/tmp/serviceca/${dp}\"\n if [ ! -e \"${reg_conf_path}\" ]; then\n rm -rf /etc/docker/certs.d/$d\n fi\n done\n sleep 60 \u0026 wait ${!}\ndone\n"],"resources":{"requests":{"cpu":"10m","memory":"10Mi"}},"volumeMounts":[{"name":"serviceca","mountPath":"/tmp/serviceca"},{"name":"host","mountPath":"/etc/docker/certs.d"},{"name":"kube-api-access-zntqd","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true,"runAsUser":1001,"runAsGroup":0}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"node-ca","serviceAccount":"node-ca","nodeName":"ci-op-v23dx441-253f3-vhnpj-worker-centralus1-7626l","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"node-ca-dockercfg-t4js2"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-v23dx441-253f3-vhnpj-worker-centralus1-7626l"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-cluster-critical","priority":2000000000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T23:46:33Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T00:08:17Z","reason":"ContainersNotReady","message":"containers with unready status: [node-ca]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-23T00:08:53Z","reason":"ContainersNotReady","message":"containers with unready status: [node-ca]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T23:46:32Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-22T23:46:33Z","containerStatuses":[{"name":"node-ca","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage3211750930/3\": happened during read: read tcp 10.0.128.5:39272-\u003e65.8.55.213:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-22-222817@sha256:26b626f9b0de3609f607fd85f4fb6865880dc100273846fcc8947cbcc8a2840c","imageID":"","started":false}],"qosClass":"Burstable"}}
#1693791357040070656	junit	30 hours ago
ns/openshift-multus pod/multus-9x9l5 node/ci-op-7xm3fr31-253f3-kqls5-worker-centralus1-mvzcn container/kube-multus - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"multus-9x9l5","generateName":"multus-","namespace":"openshift-multus","uid":"9307774a-456c-4760-b3e9-8b3e3960d85f","resourceVersion":"61466","creationTimestamp":"2023-08-22T02:21:38Z","labels":{"app":"multus","component":"network","controller-revision-hash":"5c5c46f488","openshift.io/component":"network","pod-template-generation":"2","type":"infra"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"multus","uid":"5abf8892-b4c1-4198-87eb-a7ee6694cad6","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-22T02:21:38Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:component":{},"f:controller-revision-hash":{},"f:openshift.io/component":{},"f:pod-template-generation":{},"f:type":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"5abf8892-b4c1-4198-87eb-a7ee6694cad6\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"kube-multus\"}":{".":{},"f:args":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"DEFAULT_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_HOST\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_PORT\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RHEL7_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RHEL8_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/entrypoint\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/etc/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/etc/os-release\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/opt/cni/bin\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/var/run/multus/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"cni-binary-copy\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"cnibin\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"multus-cni-dir\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"os-release\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"system-cni-dir\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-22T02:41:27Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-22T02:42:01Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.4\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"system-cni-dir","hostPath":{"path":"/etc/kubernetes/cni/net.d","type":""}},{"name":"multus-cni-dir","hostPath":{"path":"/var/run/multus/cni/net.d","type":""}},{"name":"cnibin","hostPath":{"path":"/var/lib/cni/bin","type":""}},{"name":"os-release","hostPath":{"path":"/etc/os-release","type":"File"}},{"name":"cni-binary-copy","configMap":{"name":"cni-binary-copy-script","defaultMode":484}},{"name":"kube-api-access-vldzz","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"kube-multus","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:528a2e0fb6ca1a3c6aaefcbf1fa7bcce000182565a8900bcc27cfd8db47a6ca5","command":["/bin/bash","-ec","--"],"args":["/entrypoint/cnibincopy.sh; exec /entrypoint.sh --multus-conf-file=auto --multus-autoconfig-dir=/host/var/run/multus/cni/net.d --multus-kubeconfig-file-host=/etc/kubernetes/cni/net.d/multus.d/multus.kubeconfig --readiness-indicator-file=/var/run/multus/cni/net.d/80-openshift-network.conf --cleanup-config-on-exit=true --namespace-isolation=true --multus-log-level=verbose --cni-version=0.3.1 --additional-bin-dir=/opt/multus/bin --skip-multus-binary-copy=true - \"--global-namespaces=default,openshift-multus,openshift-sriov-network-operator\"\n"],"env":[{"name":"RHEL7_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/rhel7/bin/"},{"name":"RHEL8_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/rhel8/bin/"},{"name":"DEFAULT_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/bin/"},{"name":"KUBERNETES_SERVICE_PORT","value":"6443"},{"name":"KUBERNETES_SERVICE_HOST","value":"api-int.ci-op-7xm3fr31-253f3.ci.azure.devcluster.openshift.com"}],"resources":{"requests":{"cpu":"10m","memory":"65Mi"}},"volumeMounts":[{"name":"cni-binary-copy","mountPath":"/entrypoint"},{"name":"os-release","mountPath":"/host/etc/os-release"},{"name":"system-cni-dir","mountPath":"/host/etc/cni/net.d"},{"name":"multus-cni-dir","mountPath":"/host/var/run/multus/cni/net.d"},{"name":"cnibin","mountPath":"/host/opt/cni/bin"},{"name":"kube-api-access-vldzz","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"multus","serviceAccount":"multus","nodeName":"ci-op-7xm3fr31-253f3-kqls5-worker-centralus1-mvzcn","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"multus-dockercfg-cjgm2"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-7xm3fr31-253f3-kqls5-worker-centralus1-mvzcn"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T02:21:38Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-22T02:41:27Z","reason":"ContainersNotReady","message":"containers with unready status: [kube-multus]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-22T02:42:01Z","reason":"ContainersNotReady","message":"containers with unready status: [kube-multus]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T02:21:38Z"}],"hostIP":"10.0.128.4","podIP":"10.0.128.4","podIPs":[{"ip":"10.0.128.4"}],"startTime":"2023-08-22T02:21:38Z","containerStatuses":[{"name":"kube-multus","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage484306194/1\": happened during read: read tcp 10.0.128.4:43420-\u003e54.230.17.75:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:528a2e0fb6ca1a3c6aaefcbf1fa7bcce000182565a8900bcc27cfd8db47a6ca5","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-sdn pod/sdn-6gmfx node/ci-op-7xm3fr31-253f3-kqls5-worker-centralus1-mvzcn container/sdn - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"sdn-6gmfx","generateName":"sdn-","namespace":"openshift-sdn","uid":"ce3ce501-d35b-4530-97a6-24d69c0d50be","resourceVersion":"61467","creationTimestamp":"2023-08-22T02:21:08Z","labels":{"app":"sdn","component":"network","controller-revision-hash":"7f597fdd4b","openshift.io/component":"network","pod-template-generation":"2","type":"infra"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"sdn","uid":"077118ac-ccb8-4b8f-8da4-8cf95f45d847","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-22T02:21:08Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:component":{},"f:controller-revision-hash":{},"f:openshift.io/component":{},"f:pod-template-generation":{},"f:type":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"077118ac-ccb8-4b8f-8da4-8cf95f45d847\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"drop-icmp\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:name":{}}}},"k:{\"name\":\"kube-rbac-proxy\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":9101,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/pki/tls/metrics-certs\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}},"k:{\"name\":\"sdn\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE_IP\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"K8S_NODE_NAME\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"KUBERNETES_SERVICE_HOST\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_PORT\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"OPENSHIFT_DNS_DOMAIN\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":10256,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:readinessProbe":{".":{},"f:exec":{".":{},"f:command":{}},"f:failureThreshold":{},"f:initialDelaySeconds":{},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/config\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/env\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/sysconfig\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host-cni-bin\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/var/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/lib/modules\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/lib/cni/networks/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/dbus/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/kubernetes/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/openvswitch/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:hostPID":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"config\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"env-overrides\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{},"f:optional":{}},"f:name":{}},"k:{\"name\":\"etc-sysconfig\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-bin\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-conf\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-modules\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-slash\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-lib-cni-networks-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-dbus\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-kubernetes\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-ovs\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"sdn-metrics-certs\"}":{".":{},"f:name":{},"f:secret":{".":{},"f:defaultMode":{},"f:optional":{},"f:secretName":{}}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-22T02:41:27Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-22T02:42:01Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.4\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"config","configMap":{"name":"sdn-config","defaultMode":420}},{"name":"env-overrides","configMap":{"name":"env-overrides","defaultMode":420,"optional":true}},{"name":"etc-sysconfig","hostPath":{"path":"/etc/sysconfig","type":""}},{"name":"host-modules","hostPath":{"path":"/lib/modules","type":""}},{"name":"host-var-run","hostPath":{"path":"/var/run","type":""}},{"name":"host-run-netns","hostPath":{"path":"/run/netns","type":""}},{"name":"host-var-run-netns","hostPath":{"path":"/var/run/netns","type":""}},{"name":"host-var-run-dbus","hostPath":{"path":"/var/run/dbus","type":""}},{"name":"host-var-run-ovs","hostPath":{"path":"/var/run/openvswitch","type":""}},{"name":"host-var-run-kubernetes","hostPath":{"path":"/var/run/kubernetes","type":""}},{"name":"host-var-run-openshift-sdn","hostPath":{"path":"/var/run/openshift-sdn","type":""}},{"name":"host-slash","hostPath":{"path":"/","type":""}},{"name":"host-cni-bin","hostPath":{"path":"/var/lib/cni/bin","type":""}},{"name":"host-cni-conf","hostPath":{"path":"/var/run/multus/cni/net.d","type":""}},{"name":"host-var-lib-cni-networks-openshift-sdn","hostPath":{"path":"/var/lib/cni/networks/openshift-sdn","type":""}},{"name":"sdn-metrics-certs","secret":{"secretName":"sdn-metrics-certs","defaultMode":420,"optional":true}},{"name":"kube-api-access-r6ffx","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"sdn","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\n\n# if another process is listening on the cni-server socket, wait until it exits\ntrap 'kill $(jobs -p); rm -f /etc/cni/net.d/80-openshift-network.conf ; exit 0' TERM\nretries=0\nwhile true; do\n if echo 'test' \| socat - UNIX-CONNECT:/var/run/openshift-sdn/cniserver/socket \u0026\u003e/dev/null; then\n echo \"warning: Another process is currently listening on the CNI socket, waiting 15s ...\" 2\u003e\u00261\n sleep 15 \u0026 wait\n (( retries += 1 ))\n else\n break\n fi\n if [[ \"${retries}\" -gt 40 ]]; then\n echo \"error: Another process is currently listening on the CNI socket, exiting\" 2\u003e\u00261\n exit 1\n fi\ndone\n\n# local environment overrides\nif [[ -f /etc/sysconfig/openshift-sdn ]]; then\n set -o allexport\n source /etc/sysconfig/openshift-sdn\n set +o allexport\nfi\n#BUG: cdc accidentally mounted /etc/sysconfig/openshift-sdn as DirectoryOrCreate; clean it up so we can ultimately mount /etc/sysconfig/openshift-sdn as FileOrCreate\n# Once this is released, then we can mount it properly\nif [[ -d /etc/sysconfig/openshift-sdn ]]; then\n rmdir /etc/sysconfig/openshift-sdn \|\| true\nfi\n\n# configmap-based overrides\nif [[ -f /env/${K8S_NODE_NAME} ]]; then\n set -o allexport\n source /env/${K8S_NODE_NAME}\n set +o allexport\nfi\n\n# Take over network functions on the node\nrm -f /etc/cni/net.d/80-openshift-network.conf\ncp -f /opt/cni/bin/openshift-sdn /host-cni-bin/\n\nmtu_override_flag=\nif [[ -f /config/mtu.yaml ]]; then\n mtu_override_flag=\"--mtu-override /config/mtu.yaml\"\nfi\n\n# Launch the network process\nexec /usr/bin/openshift-sdn-node \\\n --node-name ${K8S_NODE_NAME} --node-ip ${K8S_NODE_IP} \\\n --platform-type Azure \\\n --proxy-config /config/kube-proxy-config.yaml \\\n ${mtu_override_flag} \\\n --v ${OPENSHIFT_SDN_LOG_LEVEL:-2}\n"],"ports":[{"name":"healthz","hostPort":10256,"containerPort":10256,"protocol":"TCP"}],"env":[{"name":"KUBERNETES_SERVICE_PORT","value":"6443"},{"name":"KUBERNETES_SERVICE_HOST","value":"api-int.ci-op-7xm3fr31-253f3.ci.azure.devcluster.openshift.com"},{"name":"OPENSHIFT_DNS_DOMAIN","value":"cluster.local"},{"name":"K8S_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"K8S_NODE_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.hostIP"}}}],"resources":{"requests":{"cpu":"100m","memory":"200Mi"}},"volumeMounts":[{"name":"config","readOnly":true,"mountPath":"/config"},{"name":"env-overrides","mountPath":"/env"},{"name":"host-var-run","mountPath":"/var/run"},{"name":"host-var-run-dbus","readOnly":true,"mountPath":"/var/run/dbus/"},{"name":"host-var-run-ovs","readOnly":true,"mountPath":"/var/run/openvswitch/"},{"name":"host-var-run-kubernetes","readOnly":true,"mountPath":"/var/run/kubernetes/"},{"name":"host-run-netns","readOnly":true,"mountPath":"/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-netns","readOnly":true,"mountPath":"/host/var/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-openshift-sdn","mountPath":"/var/run/openshift-sdn"},{"name":"host-slash","readOnly":true,"mountPath":"/host","mountPropagation":"HostToContainer"},{"name":"host-cni-bin","mountPath":"/host-cni-bin"},{"name":"host-cni-conf","mountPath":"/etc/cni/net.d"},{"name":"host-var-lib-cni-networks-openshift-sdn","mountPath":"/var/lib/cni/networks/openshift-sdn"},{"name":"host-modules","readOnly":true,"mountPath":"/lib/modules"},{"name":"etc-sysconfig","readOnly":true,"mountPath":"/etc/sysconfig"},{"name":"kube-api-access-r6ffx","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"readinessProbe":{"exec":{"command":["test","-f","/etc/cni/net.d/80-openshift-network.conf"]},"initialDelaySeconds":5,"timeoutSeconds":1,"periodSeconds":5,"successThreshold":1,"failureThreshold":3},"lifecycle":{"preStop":{"exec":{"command":["rm","-f","/etc/cni/net.d/80-openshift-network.conf","/host-cni-bin/openshift-sdn"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}},{"name":"kube-rbac-proxy","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\nTLS_PK=/etc/pki/tls/metrics-certs/tls.key\nTLS_CERT=/etc/pki/tls/metrics-certs/tls.crt\n\n# As the secret mount is optional we must wait for the files to be present.\n# The service is created in monitor.yaml and this is created in sdn.yaml.\n# If it isn't created there is probably an issue so we want to crashloop.\nTS=$(date +%s)\nWARN_TS=$(( ${TS} + $(( 20 * 60)) ))\nHAS_LOGGED_INFO=0\n\nlog_missing_certs(){\n CUR_TS=$(date +%s)\n if [[ \"${CUR_TS}\" -gt \"WARN_TS\" ]]; then\n echo $(date -Iseconds) WARN: sdn-metrics-certs not mounted after 20 minutes.\n elif [[ \"${HAS_LOGGED_INFO}\" -eq 0 ]] ; then\n echo $(date -Iseconds) INFO: sdn-metrics-certs not mounted. Waiting 20 minutes.\n HAS_LOGGED_INFO=1\n fi\n}\n\nwhile [[ ! -f \"${TLS_PK}\" \|\| ! -f \"${TLS_CERT}\" ]] ; do\n log_missing_certs\n sleep 5\ndone\n\necho $(date -Iseconds) INFO: sdn-metrics-certs mounted, starting kube-rbac-proxy\nexec /usr/bin/kube-rbac-proxy \\\n --logtostderr \\\n --secure-listen-address=:9101 \\\n --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \\\n --upstream=http://127.0.0.1:29101/ \\\n --tls-private-key-file=${TLS_PK} \\\n --tls-cert-file=${TLS_CERT}\n"],"ports":[{"name":"https","hostPort":9101,"containerPort":9101,"protocol":"TCP"}],"resources":{"requests":{"cpu":"10m","memory":"20Mi"}},"volumeMounts":[{"name":"sdn-metrics-certs","readOnly":true,"mountPath":"/etc/pki/tls/metrics-certs"},{"name":"kube-api-access-r6ffx","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent"},{"name":"drop-icmp","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","set -xe\n\ntouch /var/run/add_iptables.sh\nchmod 0755 /var/run/add_iptables.sh\ncat \u003c\u003c'EOF' \u003e /var/run/add_iptables.sh\n#!/bin/sh\nif [ -z \"$3\" ]\nthen\n echo \"Called with host address missing, ignore\"\n exit 0\nfi\necho \"Adding ICMP drop rule for '$3' \"\nif iptables -C AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nthen\n echo \"iptables already set for $3\"\nelse\n iptables -A AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nfi\nEOF\n\necho \"I$(date \"+%m%d %H:%M:%S.%N\") - drop-icmp - start drop-icmp ${K8S_NODE}\"\niptables -X AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -N AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -F AZURE_CHECK_ICMP_SOURCE\niptables -D INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -I INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE\niptables -N AZURE_ICMP_ACTION \|\| true\niptables -F AZURE_ICMP_ACTION\niptables -A AZURE_ICMP_ACTION -j LOG\niptables -A AZURE_ICMP_ACTION -j DROP\n/host/usr/bin/oc observe pods -n openshift-sdn -l app=sdn -a '{ .status.hostIP }' -- /var/run/add_iptables.sh\n"],"env":[{"name":"K8S_NODE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}}],"resources":{"requests":{"cpu":"5m","memory":"20Mi"}},"volumeMounts":[{"name":"host-slash","mountPath":"/host"},{"name":"kube-api-access-r6ffx","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"lifecycle":{"preStop":{"exec":{"command":["/bin/bash","-c","echo drop-icmp done"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"sdn","serviceAccount":"sdn","nodeName":"ci-op-7xm3fr31-253f3-kqls5-worker-centralus1-mvzcn","hostNetwork":true,"hostPID":true,"securityContext":{},"imagePullSecrets":[{"name":"sdn-dockercfg-rk7q4"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-7xm3fr31-253f3-kqls5-worker-centralus1-mvzcn"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T02:21:08Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-22T02:41:27Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-22T02:42:01Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T02:21:08Z"}],"hostIP":"10.0.128.4","podIP":"10.0.128.4","podIPs":[{"ip":"10.0.128.4"}],"startTime":"2023-08-22T02:21:08Z","containerStatuses":[{"name":"drop-icmp","state":{"waiting":{"reason":"ImagePullBackOff","message":"Back-off pulling image \"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c\""}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false},{"name":"kube-rbac-proxy","state":{"running":{"startedAt":"2023-08-22T02:47:20Z"}},"lastState":{},"ready":true,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","imageID":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","containerID":"cri-o://db247e1aa145e43357977452f738b14f2870790f5f6890e0fb63b2860912464a","started":true},{"name":"sdn","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage508193225/2\": happened during read: read tcp 10.0.128.4:44794-\u003e54.230.17.75:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-sdn pod/sdn-6gmfx node/ci-op-7xm3fr31-253f3-kqls5-worker-centralus1-mvzcn container/drop-icmp - reason/ContainerWait cause/ImagePullBackOff unable to calculate container transition time in pod: {"metadata":{"name":"sdn-6gmfx","generateName":"sdn-","namespace":"openshift-sdn","uid":"ce3ce501-d35b-4530-97a6-24d69c0d50be","resourceVersion":"61467","creationTimestamp":"2023-08-22T02:21:08Z","labels":{"app":"sdn","component":"network","controller-revision-hash":"7f597fdd4b","openshift.io/component":"network","pod-template-generation":"2","type":"infra"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"sdn","uid":"077118ac-ccb8-4b8f-8da4-8cf95f45d847","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-22T02:21:08Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:component":{},"f:controller-revision-hash":{},"f:openshift.io/component":{},"f:pod-template-generation":{},"f:type":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"077118ac-ccb8-4b8f-8da4-8cf95f45d847\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"drop-icmp\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:name":{}}}},"k:{\"name\":\"kube-rbac-proxy\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":9101,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/pki/tls/metrics-certs\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}},"k:{\"name\":\"sdn\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE_IP\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"K8S_NODE_NAME\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"KUBERNETES_SERVICE_HOST\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_PORT\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"OPENSHIFT_DNS_DOMAIN\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":10256,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:readinessProbe":{".":{},"f:exec":{".":{},"f:command":{}},"f:failureThreshold":{},"f:initialDelaySeconds":{},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/config\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/env\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/sysconfig\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host-cni-bin\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/var/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/lib/modules\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/lib/cni/networks/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/dbus/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/kubernetes/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/openvswitch/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:hostPID":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"config\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"env-overrides\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{},"f:optional":{}},"f:name":{}},"k:{\"name\":\"etc-sysconfig\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-bin\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-conf\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-modules\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-slash\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-lib-cni-networks-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-dbus\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-kubernetes\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-ovs\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"sdn-metrics-certs\"}":{".":{},"f:name":{},"f:secret":{".":{},"f:defaultMode":{},"f:optional":{},"f:secretName":{}}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-22T02:41:27Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-22T02:42:01Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.4\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"config","configMap":{"name":"sdn-config","defaultMode":420}},{"name":"env-overrides","configMap":{"name":"env-overrides","defaultMode":420,"optional":true}},{"name":"etc-sysconfig","hostPath":{"path":"/etc/sysconfig","type":""}},{"name":"host-modules","hostPath":{"path":"/lib/modules","type":""}},{"name":"host-var-run","hostPath":{"path":"/var/run","type":""}},{"name":"host-run-netns","hostPath":{"path":"/run/netns","type":""}},{"name":"host-var-run-netns","hostPath":{"path":"/var/run/netns","type":""}},{"name":"host-var-run-dbus","hostPath":{"path":"/var/run/dbus","type":""}},{"name":"host-var-run-ovs","hostPath":{"path":"/var/run/openvswitch","type":""}},{"name":"host-var-run-kubernetes","hostPath":{"path":"/var/run/kubernetes","type":""}},{"name":"host-var-run-openshift-sdn","hostPath":{"path":"/var/run/openshift-sdn","type":""}},{"name":"host-slash","hostPath":{"path":"/","type":""}},{"name":"host-cni-bin","hostPath":{"path":"/var/lib/cni/bin","type":""}},{"name":"host-cni-conf","hostPath":{"path":"/var/run/multus/cni/net.d","type":""}},{"name":"host-var-lib-cni-networks-openshift-sdn","hostPath":{"path":"/var/lib/cni/networks/openshift-sdn","type":""}},{"name":"sdn-metrics-certs","secret":{"secretName":"sdn-metrics-certs","defaultMode":420,"optional":true}},{"name":"kube-api-access-r6ffx","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"sdn","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\n\n# if another process is listening on the cni-server socket, wait until it exits\ntrap 'kill $(jobs -p); rm -f /etc/cni/net.d/80-openshift-network.conf ; exit 0' TERM\nretries=0\nwhile true; do\n if echo 'test' \| socat - UNIX-CONNECT:/var/run/openshift-sdn/cniserver/socket \u0026\u003e/dev/null; then\n echo \"warning: Another process is currently listening on the CNI socket, waiting 15s ...\" 2\u003e\u00261\n sleep 15 \u0026 wait\n (( retries += 1 ))\n else\n break\n fi\n if [[ \"${retries}\" -gt 40 ]]; then\n echo \"error: Another process is currently listening on the CNI socket, exiting\" 2\u003e\u00261\n exit 1\n fi\ndone\n\n# local environment overrides\nif [[ -f /etc/sysconfig/openshift-sdn ]]; then\n set -o allexport\n source /etc/sysconfig/openshift-sdn\n set +o allexport\nfi\n#BUG: cdc accidentally mounted /etc/sysconfig/openshift-sdn as DirectoryOrCreate; clean it up so we can ultimately mount /etc/sysconfig/openshift-sdn as FileOrCreate\n# Once this is released, then we can mount it properly\nif [[ -d /etc/sysconfig/openshift-sdn ]]; then\n rmdir /etc/sysconfig/openshift-sdn \|\| true\nfi\n\n# configmap-based overrides\nif [[ -f /env/${K8S_NODE_NAME} ]]; then\n set -o allexport\n source /env/${K8S_NODE_NAME}\n set +o allexport\nfi\n\n# Take over network functions on the node\nrm -f /etc/cni/net.d/80-openshift-network.conf\ncp -f /opt/cni/bin/openshift-sdn /host-cni-bin/\n\nmtu_override_flag=\nif [[ -f /config/mtu.yaml ]]; then\n mtu_override_flag=\"--mtu-override /config/mtu.yaml\"\nfi\n\n# Launch the network process\nexec /usr/bin/openshift-sdn-node \\\n --node-name ${K8S_NODE_NAME} --node-ip ${K8S_NODE_IP} \\\n --platform-type Azure \\\n --proxy-config /config/kube-proxy-config.yaml \\\n ${mtu_override_flag} \\\n --v ${OPENSHIFT_SDN_LOG_LEVEL:-2}\n"],"ports":[{"name":"healthz","hostPort":10256,"containerPort":10256,"protocol":"TCP"}],"env":[{"name":"KUBERNETES_SERVICE_PORT","value":"6443"},{"name":"KUBERNETES_SERVICE_HOST","value":"api-int.ci-op-7xm3fr31-253f3.ci.azure.devcluster.openshift.com"},{"name":"OPENSHIFT_DNS_DOMAIN","value":"cluster.local"},{"name":"K8S_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"K8S_NODE_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.hostIP"}}}],"resources":{"requests":{"cpu":"100m","memory":"200Mi"}},"volumeMounts":[{"name":"config","readOnly":true,"mountPath":"/config"},{"name":"env-overrides","mountPath":"/env"},{"name":"host-var-run","mountPath":"/var/run"},{"name":"host-var-run-dbus","readOnly":true,"mountPath":"/var/run/dbus/"},{"name":"host-var-run-ovs","readOnly":true,"mountPath":"/var/run/openvswitch/"},{"name":"host-var-run-kubernetes","readOnly":true,"mountPath":"/var/run/kubernetes/"},{"name":"host-run-netns","readOnly":true,"mountPath":"/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-netns","readOnly":true,"mountPath":"/host/var/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-openshift-sdn","mountPath":"/var/run/openshift-sdn"},{"name":"host-slash","readOnly":true,"mountPath":"/host","mountPropagation":"HostToContainer"},{"name":"host-cni-bin","mountPath":"/host-cni-bin"},{"name":"host-cni-conf","mountPath":"/etc/cni/net.d"},{"name":"host-var-lib-cni-networks-openshift-sdn","mountPath":"/var/lib/cni/networks/openshift-sdn"},{"name":"host-modules","readOnly":true,"mountPath":"/lib/modules"},{"name":"etc-sysconfig","readOnly":true,"mountPath":"/etc/sysconfig"},{"name":"kube-api-access-r6ffx","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"readinessProbe":{"exec":{"command":["test","-f","/etc/cni/net.d/80-openshift-network.conf"]},"initialDelaySeconds":5,"timeoutSeconds":1,"periodSeconds":5,"successThreshold":1,"failureThreshold":3},"lifecycle":{"preStop":{"exec":{"command":["rm","-f","/etc/cni/net.d/80-openshift-network.conf","/host-cni-bin/openshift-sdn"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}},{"name":"kube-rbac-proxy","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\nTLS_PK=/etc/pki/tls/metrics-certs/tls.key\nTLS_CERT=/etc/pki/tls/metrics-certs/tls.crt\n\n# As the secret mount is optional we must wait for the files to be present.\n# The service is created in monitor.yaml and this is created in sdn.yaml.\n# If it isn't created there is probably an issue so we want to crashloop.\nTS=$(date +%s)\nWARN_TS=$(( ${TS} + $(( 20 * 60)) ))\nHAS_LOGGED_INFO=0\n\nlog_missing_certs(){\n CUR_TS=$(date +%s)\n if [[ \"${CUR_TS}\" -gt \"WARN_TS\" ]]; then\n echo $(date -Iseconds) WARN: sdn-metrics-certs not mounted after 20 minutes.\n elif [[ \"${HAS_LOGGED_INFO}\" -eq 0 ]] ; then\n echo $(date -Iseconds) INFO: sdn-metrics-certs not mounted. Waiting 20 minutes.\n HAS_LOGGED_INFO=1\n fi\n}\n\nwhile [[ ! -f \"${TLS_PK}\" \|\| ! -f \"${TLS_CERT}\" ]] ; do\n log_missing_certs\n sleep 5\ndone\n\necho $(date -Iseconds) INFO: sdn-metrics-certs mounted, starting kube-rbac-proxy\nexec /usr/bin/kube-rbac-proxy \\\n --logtostderr \\\n --secure-listen-address=:9101 \\\n --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \\\n --upstream=http://127.0.0.1:29101/ \\\n --tls-private-key-file=${TLS_PK} \\\n --tls-cert-file=${TLS_CERT}\n"],"ports":[{"name":"https","hostPort":9101,"containerPort":9101,"protocol":"TCP"}],"resources":{"requests":{"cpu":"10m","memory":"20Mi"}},"volumeMounts":[{"name":"sdn-metrics-certs","readOnly":true,"mountPath":"/etc/pki/tls/metrics-certs"},{"name":"kube-api-access-r6ffx","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent"},{"name":"drop-icmp","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","set -xe\n\ntouch /var/run/add_iptables.sh\nchmod 0755 /var/run/add_iptables.sh\ncat \u003c\u003c'EOF' \u003e /var/run/add_iptables.sh\n#!/bin/sh\nif [ -z \"$3\" ]\nthen\n echo \"Called with host address missing, ignore\"\n exit 0\nfi\necho \"Adding ICMP drop rule for '$3' \"\nif iptables -C AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nthen\n echo \"iptables already set for $3\"\nelse\n iptables -A AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nfi\nEOF\n\necho \"I$(date \"+%m%d %H:%M:%S.%N\") - drop-icmp - start drop-icmp ${K8S_NODE}\"\niptables -X AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -N AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -F AZURE_CHECK_ICMP_SOURCE\niptables -D INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -I INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE\niptables -N AZURE_ICMP_ACTION \|\| true\niptables -F AZURE_ICMP_ACTION\niptables -A AZURE_ICMP_ACTION -j LOG\niptables -A AZURE_ICMP_ACTION -j DROP\n/host/usr/bin/oc observe pods -n openshift-sdn -l app=sdn -a '{ .status.hostIP }' -- /var/run/add_iptables.sh\n"],"env":[{"name":"K8S_NODE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}}],"resources":{"requests":{"cpu":"5m","memory":"20Mi"}},"volumeMounts":[{"name":"host-slash","mountPath":"/host"},{"name":"kube-api-access-r6ffx","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"lifecycle":{"preStop":{"exec":{"command":["/bin/bash","-c","echo drop-icmp done"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"sdn","serviceAccount":"sdn","nodeName":"ci-op-7xm3fr31-253f3-kqls5-worker-centralus1-mvzcn","hostNetwork":true,"hostPID":true,"securityContext":{},"imagePullSecrets":[{"name":"sdn-dockercfg-rk7q4"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-7xm3fr31-253f3-kqls5-worker-centralus1-mvzcn"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T02:21:08Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-22T02:41:27Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-22T02:42:01Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T02:21:08Z"}],"hostIP":"10.0.128.4","podIP":"10.0.128.4","podIPs":[{"ip":"10.0.128.4"}],"startTime":"2023-08-22T02:21:08Z","containerStatuses":[{"name":"drop-icmp","state":{"waiting":{"reason":"ImagePullBackOff","message":"Back-off pulling image \"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c\""}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false},{"name":"kube-rbac-proxy","state":{"running":{"startedAt":"2023-08-22T02:47:20Z"}},"lastState":{},"ready":true,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","imageID":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","containerID":"cri-o://db247e1aa145e43357977452f738b14f2870790f5f6890e0fb63b2860912464a","started":true},{"name":"sdn","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage508193225/2\": happened during read: read tcp 10.0.128.4:44794-\u003e54.230.17.75:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-cluster-csi-drivers pod/azure-disk-csi-driver-node-nzcjr node/ci-op-7xm3fr31-253f3-kqls5-worker-centralus1-mvzcn container/azure-inject-credentials - reason/ContainerWait cause/ErrImagePull duration/1854.73s rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage41290218/1": happened during read: read tcp 10.0.128.4:43404->54.230.17.75:443: read: connection timed out ns/openshift-multus pod/multus-mb8f8 node/ci-op-7xm3fr31-253f3-kqls5-worker-centralus2-7t5cb container/kube-multus - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"multus-mb8f8","generateName":"multus-","namespace":"openshift-multus","uid":"5647f06b-153c-460d-b28c-6bff9139051d","resourceVersion":"69993","creationTimestamp":"2023-08-22T02:21:49Z","labels":{"app":"multus","component":"network","controller-revision-hash":"5c5c46f488","openshift.io/component":"network","pod-template-generation":"2","type":"infra"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"multus","uid":"5abf8892-b4c1-4198-87eb-a7ee6694cad6","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-22T02:21:49Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:component":{},"f:controller-revision-hash":{},"f:openshift.io/component":{},"f:pod-template-generation":{},"f:type":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"5abf8892-b4c1-4198-87eb-a7ee6694cad6\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"kube-multus\"}":{".":{},"f:args":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"DEFAULT_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_HOST\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_PORT\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RHEL7_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RHEL8_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/entrypoint\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/etc/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/etc/os-release\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/opt/cni/bin\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/var/run/multus/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"cni-binary-copy\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"cnibin\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"multus-cni-dir\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"os-release\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"system-cni-dir\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-22T02:51:57Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.5\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"system-cni-dir","hostPath":{"path":"/etc/kubernetes/cni/net.d","type":""}},{"name":"multus-cni-dir","hostPath":{"path":"/var/run/multus/cni/net.d","type":""}},{"name":"cnibin","hostPath":{"path":"/var/lib/cni/bin","type":""}},{"name":"os-release","hostPath":{"path":"/etc/os-release","type":"File"}},{"name":"cni-binary-copy","configMap":{"name":"cni-binary-copy-script","defaultMode":484}},{"name":"kube-api-access-kwpdn","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"kube-multus","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:528a2e0fb6ca1a3c6aaefcbf1fa7bcce000182565a8900bcc27cfd8db47a6ca5","command":["/bin/bash","-ec","--"],"args":["/entrypoint/cnibincopy.sh; exec /entrypoint.sh --multus-conf-file=auto --multus-autoconfig-dir=/host/var/run/multus/cni/net.d --multus-kubeconfig-file-host=/etc/kubernetes/cni/net.d/multus.d/multus.kubeconfig --readiness-indicator-file=/var/run/multus/cni/net.d/80-openshift-network.conf --cleanup-config-on-exit=true --namespace-isolation=true --multus-log-level=verbose --cni-version=0.3.1 --additional-bin-dir=/opt/multus/bin --skip-multus-binary-copy=true - \"--global-namespaces=default,openshift-multus,openshift-sriov-network-operator\"\n"],"env":[{"name":"RHEL7_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/rhel7/bin/"},{"name":"RHEL8_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/rhel8/bin/"},{"name":"DEFAULT_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/bin/"},{"name":"KUBERNETES_SERVICE_PORT","value":"6443"},{"name":"KUBERNETES_SERVICE_HOST","value":"api-int.ci-op-7xm3fr31-253f3.ci.azure.devcluster.openshift.com"}],"resources":{"requests":{"cpu":"10m","memory":"65Mi"}},"volumeMounts":[{"name":"cni-binary-copy","mountPath":"/entrypoint"},{"name":"os-release","mountPath":"/host/etc/os-release"},{"name":"system-cni-dir","mountPath":"/host/etc/cni/net.d"},{"name":"multus-cni-dir","mountPath":"/host/var/run/multus/cni/net.d"},{"name":"cnibin","mountPath":"/host/opt/cni/bin"},{"name":"kube-api-access-kwpdn","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"multus","serviceAccount":"multus","nodeName":"ci-op-7xm3fr31-253f3-kqls5-worker-centralus2-7t5cb","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"multus-dockercfg-cjgm2"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-7xm3fr31-253f3-kqls5-worker-centralus2-7t5cb"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T02:21:49Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-22T02:51:57Z","reason":"ContainersNotReady","message":"containers with unready status: [kube-multus]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-22T02:51:57Z","reason":"ContainersNotReady","message":"containers with unready status: [kube-multus]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-22T02:21:49Z"}],"hostIP":"10.0.128.5","podIP":"10.0.128.5","podIPs":[{"ip":"10.0.128.5"}],"startTime":"2023-08-22T02:21:49Z","containerStatuses":[{"name":"kube-multus","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = initializing source docker://registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:528a2e0fb6ca1a3c6aaefcbf1fa7bcce000182565a8900bcc27cfd8db47a6ca5: Get \"https://registry.ci.openshift.org/v2/ocp/4.10-2023-08-21-220954/manifests/sha256:528a2e0fb6ca1a3c6aaefcbf1fa7bcce000182565a8900bcc27cfd8db47a6ca5\": read tcp 10.0.128.5:45256-\u003e34.234.122.5:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-220954@sha256:528a2e0fb6ca1a3c6aaefcbf1fa7bcce000182565a8900bcc27cfd8db47a6ca5","imageID":"","started":false}],"qosClass":"Burstable"}}
#1693601850890129408	junit	42 hours ago
ns/openshift-sdn pod/sdn-mxgvh node/ci-op-tbc012m5-253f3-jndsh-worker-centralus1-m2wwl container/sdn - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"sdn-mxgvh","generateName":"sdn-","namespace":"openshift-sdn","uid":"e5ae884d-0865-452b-afc1-b3b60e8123e1","resourceVersion":"68969","creationTimestamp":"2023-08-21T14:05:08Z","labels":{"app":"sdn","component":"network","controller-revision-hash":"568d97c8fb","openshift.io/component":"network","pod-template-generation":"2","type":"infra"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"sdn","uid":"af45f05d-f41d-4ded-9a8a-e9ec388318f8","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T14:05:08Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:component":{},"f:controller-revision-hash":{},"f:openshift.io/component":{},"f:pod-template-generation":{},"f:type":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"af45f05d-f41d-4ded-9a8a-e9ec388318f8\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"drop-icmp\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:name":{}}}},"k:{\"name\":\"kube-rbac-proxy\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":9101,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/pki/tls/metrics-certs\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}},"k:{\"name\":\"sdn\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE_IP\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"K8S_NODE_NAME\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"KUBERNETES_SERVICE_HOST\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_PORT\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"OPENSHIFT_DNS_DOMAIN\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":10256,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:readinessProbe":{".":{},"f:exec":{".":{},"f:command":{}},"f:failureThreshold":{},"f:initialDelaySeconds":{},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/config\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/env\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/sysconfig\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host-cni-bin\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/var/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/lib/modules\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/lib/cni/networks/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/dbus/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/kubernetes/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/openvswitch/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:hostPID":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"config\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"env-overrides\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{},"f:optional":{}},"f:name":{}},"k:{\"name\":\"etc-sysconfig\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-bin\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-conf\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-modules\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-slash\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-lib-cni-networks-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-dbus\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-kubernetes\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-ovs\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"sdn-metrics-certs\"}":{".":{},"f:name":{},"f:secret":{".":{},"f:defaultMode":{},"f:optional":{},"f:secretName":{}}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T14:24:34Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-21T14:24:42Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.6\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"config","configMap":{"name":"sdn-config","defaultMode":420}},{"name":"env-overrides","configMap":{"name":"env-overrides","defaultMode":420,"optional":true}},{"name":"etc-sysconfig","hostPath":{"path":"/etc/sysconfig","type":""}},{"name":"host-modules","hostPath":{"path":"/lib/modules","type":""}},{"name":"host-var-run","hostPath":{"path":"/var/run","type":""}},{"name":"host-run-netns","hostPath":{"path":"/run/netns","type":""}},{"name":"host-var-run-netns","hostPath":{"path":"/var/run/netns","type":""}},{"name":"host-var-run-dbus","hostPath":{"path":"/var/run/dbus","type":""}},{"name":"host-var-run-ovs","hostPath":{"path":"/var/run/openvswitch","type":""}},{"name":"host-var-run-kubernetes","hostPath":{"path":"/var/run/kubernetes","type":""}},{"name":"host-var-run-openshift-sdn","hostPath":{"path":"/var/run/openshift-sdn","type":""}},{"name":"host-slash","hostPath":{"path":"/","type":""}},{"name":"host-cni-bin","hostPath":{"path":"/var/lib/cni/bin","type":""}},{"name":"host-cni-conf","hostPath":{"path":"/var/run/multus/cni/net.d","type":""}},{"name":"host-var-lib-cni-networks-openshift-sdn","hostPath":{"path":"/var/lib/cni/networks/openshift-sdn","type":""}},{"name":"sdn-metrics-certs","secret":{"secretName":"sdn-metrics-certs","defaultMode":420,"optional":true}},{"name":"kube-api-access-zsdjd","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"sdn","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\n\n# if another process is listening on the cni-server socket, wait until it exits\ntrap 'kill $(jobs -p); rm -f /etc/cni/net.d/80-openshift-network.conf ; exit 0' TERM\nretries=0\nwhile true; do\n if echo 'test' \| socat - UNIX-CONNECT:/var/run/openshift-sdn/cniserver/socket \u0026\u003e/dev/null; then\n echo \"warning: Another process is currently listening on the CNI socket, waiting 15s ...\" 2\u003e\u00261\n sleep 15 \u0026 wait\n (( retries += 1 ))\n else\n break\n fi\n if [[ \"${retries}\" -gt 40 ]]; then\n echo \"error: Another process is currently listening on the CNI socket, exiting\" 2\u003e\u00261\n exit 1\n fi\ndone\n\n# local environment overrides\nif [[ -f /etc/sysconfig/openshift-sdn ]]; then\n set -o allexport\n source /etc/sysconfig/openshift-sdn\n set +o allexport\nfi\n#BUG: cdc accidentally mounted /etc/sysconfig/openshift-sdn as DirectoryOrCreate; clean it up so we can ultimately mount /etc/sysconfig/openshift-sdn as FileOrCreate\n# Once this is released, then we can mount it properly\nif [[ -d /etc/sysconfig/openshift-sdn ]]; then\n rmdir /etc/sysconfig/openshift-sdn \|\| true\nfi\n\n# configmap-based overrides\nif [[ -f /env/${K8S_NODE_NAME} ]]; then\n set -o allexport\n source /env/${K8S_NODE_NAME}\n set +o allexport\nfi\n\n# Take over network functions on the node\nrm -f /etc/cni/net.d/80-openshift-network.conf\ncp -f /opt/cni/bin/openshift-sdn /host-cni-bin/\n\nmtu_override_flag=\nif [[ -f /config/mtu.yaml ]]; then\n mtu_override_flag=\"--mtu-override /config/mtu.yaml\"\nfi\n\n# Launch the network process\nexec /usr/bin/openshift-sdn-node \\\n --node-name ${K8S_NODE_NAME} --node-ip ${K8S_NODE_IP} \\\n --platform-type Azure \\\n --proxy-config /config/kube-proxy-config.yaml \\\n ${mtu_override_flag} \\\n --v ${OPENSHIFT_SDN_LOG_LEVEL:-2}\n"],"ports":[{"name":"healthz","hostPort":10256,"containerPort":10256,"protocol":"TCP"}],"env":[{"name":"KUBERNETES_SERVICE_PORT","value":"6443"},{"name":"KUBERNETES_SERVICE_HOST","value":"api-int.ci-op-tbc012m5-253f3.ci.azure.devcluster.openshift.com"},{"name":"OPENSHIFT_DNS_DOMAIN","value":"cluster.local"},{"name":"K8S_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"K8S_NODE_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.hostIP"}}}],"resources":{"requests":{"cpu":"100m","memory":"200Mi"}},"volumeMounts":[{"name":"config","readOnly":true,"mountPath":"/config"},{"name":"env-overrides","mountPath":"/env"},{"name":"host-var-run","mountPath":"/var/run"},{"name":"host-var-run-dbus","readOnly":true,"mountPath":"/var/run/dbus/"},{"name":"host-var-run-ovs","readOnly":true,"mountPath":"/var/run/openvswitch/"},{"name":"host-var-run-kubernetes","readOnly":true,"mountPath":"/var/run/kubernetes/"},{"name":"host-run-netns","readOnly":true,"mountPath":"/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-netns","readOnly":true,"mountPath":"/host/var/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-openshift-sdn","mountPath":"/var/run/openshift-sdn"},{"name":"host-slash","readOnly":true,"mountPath":"/host","mountPropagation":"HostToContainer"},{"name":"host-cni-bin","mountPath":"/host-cni-bin"},{"name":"host-cni-conf","mountPath":"/etc/cni/net.d"},{"name":"host-var-lib-cni-networks-openshift-sdn","mountPath":"/var/lib/cni/networks/openshift-sdn"},{"name":"host-modules","readOnly":true,"mountPath":"/lib/modules"},{"name":"etc-sysconfig","readOnly":true,"mountPath":"/etc/sysconfig"},{"name":"kube-api-access-zsdjd","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"readinessProbe":{"exec":{"command":["test","-f","/etc/cni/net.d/80-openshift-network.conf"]},"initialDelaySeconds":5,"timeoutSeconds":1,"periodSeconds":5,"successThreshold":1,"failureThreshold":3},"lifecycle":{"preStop":{"exec":{"command":["rm","-f","/etc/cni/net.d/80-openshift-network.conf","/host-cni-bin/openshift-sdn"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}},{"name":"kube-rbac-proxy","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\nTLS_PK=/etc/pki/tls/metrics-certs/tls.key\nTLS_CERT=/etc/pki/tls/metrics-certs/tls.crt\n\n# As the secret mount is optional we must wait for the files to be present.\n# The service is created in monitor.yaml and this is created in sdn.yaml.\n# If it isn't created there is probably an issue so we want to crashloop.\nTS=$(date +%s)\nWARN_TS=$(( ${TS} + $(( 20 * 60)) ))\nHAS_LOGGED_INFO=0\n\nlog_missing_certs(){\n CUR_TS=$(date +%s)\n if [[ \"${CUR_TS}\" -gt \"WARN_TS\" ]]; then\n echo $(date -Iseconds) WARN: sdn-metrics-certs not mounted after 20 minutes.\n elif [[ \"${HAS_LOGGED_INFO}\" -eq 0 ]] ; then\n echo $(date -Iseconds) INFO: sdn-metrics-certs not mounted. Waiting 20 minutes.\n HAS_LOGGED_INFO=1\n fi\n}\n\nwhile [[ ! -f \"${TLS_PK}\" \|\| ! -f \"${TLS_CERT}\" ]] ; do\n log_missing_certs\n sleep 5\ndone\n\necho $(date -Iseconds) INFO: sdn-metrics-certs mounted, starting kube-rbac-proxy\nexec /usr/bin/kube-rbac-proxy \\\n --logtostderr \\\n --secure-listen-address=:9101 \\\n --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \\\n --upstream=http://127.0.0.1:29101/ \\\n --tls-private-key-file=${TLS_PK} \\\n --tls-cert-file=${TLS_CERT}\n"],"ports":[{"name":"https","hostPort":9101,"containerPort":9101,"protocol":"TCP"}],"resources":{"requests":{"cpu":"10m","memory":"20Mi"}},"volumeMounts":[{"name":"sdn-metrics-certs","readOnly":true,"mountPath":"/etc/pki/tls/metrics-certs"},{"name":"kube-api-access-zsdjd","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent"},{"name":"drop-icmp","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","set -xe\n\ntouch /var/run/add_iptables.sh\nchmod 0755 /var/run/add_iptables.sh\ncat \u003c\u003c'EOF' \u003e /var/run/add_iptables.sh\n#!/bin/sh\nif [ -z \"$3\" ]\nthen\n echo \"Called with host address missing, ignore\"\n exit 0\nfi\necho \"Adding ICMP drop rule for '$3' \"\nif iptables -C AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nthen\n echo \"iptables already set for $3\"\nelse\n iptables -A AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nfi\nEOF\n\necho \"I$(date \"+%m%d %H:%M:%S.%N\") - drop-icmp - start drop-icmp ${K8S_NODE}\"\niptables -X AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -N AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -F AZURE_CHECK_ICMP_SOURCE\niptables -D INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -I INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE\niptables -N AZURE_ICMP_ACTION \|\| true\niptables -F AZURE_ICMP_ACTION\niptables -A AZURE_ICMP_ACTION -j LOG\niptables -A AZURE_ICMP_ACTION -j DROP\n/host/usr/bin/oc observe pods -n openshift-sdn -l app=sdn -a '{ .status.hostIP }' -- /var/run/add_iptables.sh\n"],"env":[{"name":"K8S_NODE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}}],"resources":{"requests":{"cpu":"5m","memory":"20Mi"}},"volumeMounts":[{"name":"host-slash","mountPath":"/host"},{"name":"kube-api-access-zsdjd","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"lifecycle":{"preStop":{"exec":{"command":["/bin/bash","-c","echo drop-icmp done"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"sdn","serviceAccount":"sdn","nodeName":"ci-op-tbc012m5-253f3-jndsh-worker-centralus1-m2wwl","hostNetwork":true,"hostPID":true,"securityContext":{},"imagePullSecrets":[{"name":"sdn-dockercfg-jkr7n"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-tbc012m5-253f3-jndsh-worker-centralus1-m2wwl"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T14:05:08Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T14:24:34Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T14:24:42Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T14:05:08Z"}],"hostIP":"10.0.128.6","podIP":"10.0.128.6","podIPs":[{"ip":"10.0.128.6"}],"startTime":"2023-08-21T14:05:08Z","containerStatuses":[{"name":"drop-icmp","state":{"waiting":{"reason":"ImagePullBackOff","message":"Back-off pulling image \"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c\""}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false},{"name":"kube-rbac-proxy","state":{"running":{"startedAt":"2023-08-21T14:30:02Z"}},"lastState":{},"ready":true,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","imageID":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","containerID":"cri-o://2b7fd22bb934461cb328b309990e9d916f87b35fe5bb10becb2a6a22980df341","started":true},{"name":"sdn","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage1871968393/4\": happened during read: read tcp 10.0.128.6:44230-\u003e54.230.17.85:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-sdn pod/sdn-mxgvh node/ci-op-tbc012m5-253f3-jndsh-worker-centralus1-m2wwl container/drop-icmp - reason/ContainerWait cause/ImagePullBackOff unable to calculate container transition time in pod: {"metadata":{"name":"sdn-mxgvh","generateName":"sdn-","namespace":"openshift-sdn","uid":"e5ae884d-0865-452b-afc1-b3b60e8123e1","resourceVersion":"68969","creationTimestamp":"2023-08-21T14:05:08Z","labels":{"app":"sdn","component":"network","controller-revision-hash":"568d97c8fb","openshift.io/component":"network","pod-template-generation":"2","type":"infra"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"sdn","uid":"af45f05d-f41d-4ded-9a8a-e9ec388318f8","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T14:05:08Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:component":{},"f:controller-revision-hash":{},"f:openshift.io/component":{},"f:pod-template-generation":{},"f:type":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"af45f05d-f41d-4ded-9a8a-e9ec388318f8\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"drop-icmp\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:name":{}}}},"k:{\"name\":\"kube-rbac-proxy\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":9101,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/pki/tls/metrics-certs\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}},"k:{\"name\":\"sdn\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"K8S_NODE_IP\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"K8S_NODE_NAME\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}},"k:{\"name\":\"KUBERNETES_SERVICE_HOST\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_PORT\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"OPENSHIFT_DNS_DOMAIN\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:lifecycle":{".":{},"f:preStop":{".":{},"f:exec":{".":{},"f:command":{}}}},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":10256,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:hostPort":{},"f:name":{},"f:protocol":{}}},"f:readinessProbe":{".":{},"f:exec":{".":{},"f:command":{}},"f:failureThreshold":{},"f:initialDelaySeconds":{},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/config\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/env\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/etc/sysconfig\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/host-cni-bin\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/var/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/lib/modules\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/run/netns\"}":{".":{},"f:mountPath":{},"f:mountPropagation":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/lib/cni/networks/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/dbus/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/kubernetes/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/run/openshift-sdn\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/run/openvswitch/\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:hostPID":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"config\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"env-overrides\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{},"f:optional":{}},"f:name":{}},"k:{\"name\":\"etc-sysconfig\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-bin\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-cni-conf\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-modules\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-slash\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-lib-cni-networks-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-dbus\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-kubernetes\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-netns\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-openshift-sdn\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"host-var-run-ovs\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"sdn-metrics-certs\"}":{".":{},"f:name":{},"f:secret":{".":{},"f:defaultMode":{},"f:optional":{},"f:secretName":{}}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T14:24:34Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-21T14:24:42Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.6\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"config","configMap":{"name":"sdn-config","defaultMode":420}},{"name":"env-overrides","configMap":{"name":"env-overrides","defaultMode":420,"optional":true}},{"name":"etc-sysconfig","hostPath":{"path":"/etc/sysconfig","type":""}},{"name":"host-modules","hostPath":{"path":"/lib/modules","type":""}},{"name":"host-var-run","hostPath":{"path":"/var/run","type":""}},{"name":"host-run-netns","hostPath":{"path":"/run/netns","type":""}},{"name":"host-var-run-netns","hostPath":{"path":"/var/run/netns","type":""}},{"name":"host-var-run-dbus","hostPath":{"path":"/var/run/dbus","type":""}},{"name":"host-var-run-ovs","hostPath":{"path":"/var/run/openvswitch","type":""}},{"name":"host-var-run-kubernetes","hostPath":{"path":"/var/run/kubernetes","type":""}},{"name":"host-var-run-openshift-sdn","hostPath":{"path":"/var/run/openshift-sdn","type":""}},{"name":"host-slash","hostPath":{"path":"/","type":""}},{"name":"host-cni-bin","hostPath":{"path":"/var/lib/cni/bin","type":""}},{"name":"host-cni-conf","hostPath":{"path":"/var/run/multus/cni/net.d","type":""}},{"name":"host-var-lib-cni-networks-openshift-sdn","hostPath":{"path":"/var/lib/cni/networks/openshift-sdn","type":""}},{"name":"sdn-metrics-certs","secret":{"secretName":"sdn-metrics-certs","defaultMode":420,"optional":true}},{"name":"kube-api-access-zsdjd","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"sdn","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\n\n# if another process is listening on the cni-server socket, wait until it exits\ntrap 'kill $(jobs -p); rm -f /etc/cni/net.d/80-openshift-network.conf ; exit 0' TERM\nretries=0\nwhile true; do\n if echo 'test' \| socat - UNIX-CONNECT:/var/run/openshift-sdn/cniserver/socket \u0026\u003e/dev/null; then\n echo \"warning: Another process is currently listening on the CNI socket, waiting 15s ...\" 2\u003e\u00261\n sleep 15 \u0026 wait\n (( retries += 1 ))\n else\n break\n fi\n if [[ \"${retries}\" -gt 40 ]]; then\n echo \"error: Another process is currently listening on the CNI socket, exiting\" 2\u003e\u00261\n exit 1\n fi\ndone\n\n# local environment overrides\nif [[ -f /etc/sysconfig/openshift-sdn ]]; then\n set -o allexport\n source /etc/sysconfig/openshift-sdn\n set +o allexport\nfi\n#BUG: cdc accidentally mounted /etc/sysconfig/openshift-sdn as DirectoryOrCreate; clean it up so we can ultimately mount /etc/sysconfig/openshift-sdn as FileOrCreate\n# Once this is released, then we can mount it properly\nif [[ -d /etc/sysconfig/openshift-sdn ]]; then\n rmdir /etc/sysconfig/openshift-sdn \|\| true\nfi\n\n# configmap-based overrides\nif [[ -f /env/${K8S_NODE_NAME} ]]; then\n set -o allexport\n source /env/${K8S_NODE_NAME}\n set +o allexport\nfi\n\n# Take over network functions on the node\nrm -f /etc/cni/net.d/80-openshift-network.conf\ncp -f /opt/cni/bin/openshift-sdn /host-cni-bin/\n\nmtu_override_flag=\nif [[ -f /config/mtu.yaml ]]; then\n mtu_override_flag=\"--mtu-override /config/mtu.yaml\"\nfi\n\n# Launch the network process\nexec /usr/bin/openshift-sdn-node \\\n --node-name ${K8S_NODE_NAME} --node-ip ${K8S_NODE_IP} \\\n --platform-type Azure \\\n --proxy-config /config/kube-proxy-config.yaml \\\n ${mtu_override_flag} \\\n --v ${OPENSHIFT_SDN_LOG_LEVEL:-2}\n"],"ports":[{"name":"healthz","hostPort":10256,"containerPort":10256,"protocol":"TCP"}],"env":[{"name":"KUBERNETES_SERVICE_PORT","value":"6443"},{"name":"KUBERNETES_SERVICE_HOST","value":"api-int.ci-op-tbc012m5-253f3.ci.azure.devcluster.openshift.com"},{"name":"OPENSHIFT_DNS_DOMAIN","value":"cluster.local"},{"name":"K8S_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"K8S_NODE_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.hostIP"}}}],"resources":{"requests":{"cpu":"100m","memory":"200Mi"}},"volumeMounts":[{"name":"config","readOnly":true,"mountPath":"/config"},{"name":"env-overrides","mountPath":"/env"},{"name":"host-var-run","mountPath":"/var/run"},{"name":"host-var-run-dbus","readOnly":true,"mountPath":"/var/run/dbus/"},{"name":"host-var-run-ovs","readOnly":true,"mountPath":"/var/run/openvswitch/"},{"name":"host-var-run-kubernetes","readOnly":true,"mountPath":"/var/run/kubernetes/"},{"name":"host-run-netns","readOnly":true,"mountPath":"/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-netns","readOnly":true,"mountPath":"/host/var/run/netns","mountPropagation":"HostToContainer"},{"name":"host-var-run-openshift-sdn","mountPath":"/var/run/openshift-sdn"},{"name":"host-slash","readOnly":true,"mountPath":"/host","mountPropagation":"HostToContainer"},{"name":"host-cni-bin","mountPath":"/host-cni-bin"},{"name":"host-cni-conf","mountPath":"/etc/cni/net.d"},{"name":"host-var-lib-cni-networks-openshift-sdn","mountPath":"/var/lib/cni/networks/openshift-sdn"},{"name":"host-modules","readOnly":true,"mountPath":"/lib/modules"},{"name":"etc-sysconfig","readOnly":true,"mountPath":"/etc/sysconfig"},{"name":"kube-api-access-zsdjd","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"readinessProbe":{"exec":{"command":["test","-f","/etc/cni/net.d/80-openshift-network.conf"]},"initialDelaySeconds":5,"timeoutSeconds":1,"periodSeconds":5,"successThreshold":1,"failureThreshold":3},"lifecycle":{"preStop":{"exec":{"command":["rm","-f","/etc/cni/net.d/80-openshift-network.conf","/host-cni-bin/openshift-sdn"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}},{"name":"kube-rbac-proxy","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","command":["/bin/bash","-c","#!/bin/bash\nset -euo pipefail\nTLS_PK=/etc/pki/tls/metrics-certs/tls.key\nTLS_CERT=/etc/pki/tls/metrics-certs/tls.crt\n\n# As the secret mount is optional we must wait for the files to be present.\n# The service is created in monitor.yaml and this is created in sdn.yaml.\n# If it isn't created there is probably an issue so we want to crashloop.\nTS=$(date +%s)\nWARN_TS=$(( ${TS} + $(( 20 * 60)) ))\nHAS_LOGGED_INFO=0\n\nlog_missing_certs(){\n CUR_TS=$(date +%s)\n if [[ \"${CUR_TS}\" -gt \"WARN_TS\" ]]; then\n echo $(date -Iseconds) WARN: sdn-metrics-certs not mounted after 20 minutes.\n elif [[ \"${HAS_LOGGED_INFO}\" -eq 0 ]] ; then\n echo $(date -Iseconds) INFO: sdn-metrics-certs not mounted. Waiting 20 minutes.\n HAS_LOGGED_INFO=1\n fi\n}\n\nwhile [[ ! -f \"${TLS_PK}\" \|\| ! -f \"${TLS_CERT}\" ]] ; do\n log_missing_certs\n sleep 5\ndone\n\necho $(date -Iseconds) INFO: sdn-metrics-certs mounted, starting kube-rbac-proxy\nexec /usr/bin/kube-rbac-proxy \\\n --logtostderr \\\n --secure-listen-address=:9101 \\\n --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \\\n --upstream=http://127.0.0.1:29101/ \\\n --tls-private-key-file=${TLS_PK} \\\n --tls-cert-file=${TLS_CERT}\n"],"ports":[{"name":"https","hostPort":9101,"containerPort":9101,"protocol":"TCP"}],"resources":{"requests":{"cpu":"10m","memory":"20Mi"}},"volumeMounts":[{"name":"sdn-metrics-certs","readOnly":true,"mountPath":"/etc/pki/tls/metrics-certs"},{"name":"kube-api-access-zsdjd","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent"},{"name":"drop-icmp","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","command":["/bin/bash","-c","set -xe\n\ntouch /var/run/add_iptables.sh\nchmod 0755 /var/run/add_iptables.sh\ncat \u003c\u003c'EOF' \u003e /var/run/add_iptables.sh\n#!/bin/sh\nif [ -z \"$3\" ]\nthen\n echo \"Called with host address missing, ignore\"\n exit 0\nfi\necho \"Adding ICMP drop rule for '$3' \"\nif iptables -C AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nthen\n echo \"iptables already set for $3\"\nelse\n iptables -A AZURE_CHECK_ICMP_SOURCE -p icmp -s $3 -j AZURE_ICMP_ACTION\nfi\nEOF\n\necho \"I$(date \"+%m%d %H:%M:%S.%N\") - drop-icmp - start drop-icmp ${K8S_NODE}\"\niptables -X AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -N AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -F AZURE_CHECK_ICMP_SOURCE\niptables -D INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE \|\| true\niptables -I INPUT -p icmp --icmp-type fragmentation-needed -j AZURE_CHECK_ICMP_SOURCE\niptables -N AZURE_ICMP_ACTION \|\| true\niptables -F AZURE_ICMP_ACTION\niptables -A AZURE_ICMP_ACTION -j LOG\niptables -A AZURE_ICMP_ACTION -j DROP\n/host/usr/bin/oc observe pods -n openshift-sdn -l app=sdn -a '{ .status.hostIP }' -- /var/run/add_iptables.sh\n"],"env":[{"name":"K8S_NODE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}}],"resources":{"requests":{"cpu":"5m","memory":"20Mi"}},"volumeMounts":[{"name":"host-slash","mountPath":"/host"},{"name":"kube-api-access-zsdjd","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"lifecycle":{"preStop":{"exec":{"command":["/bin/bash","-c","echo drop-icmp done"]}}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"sdn","serviceAccount":"sdn","nodeName":"ci-op-tbc012m5-253f3-jndsh-worker-centralus1-m2wwl","hostNetwork":true,"hostPID":true,"securityContext":{},"imagePullSecrets":[{"name":"sdn-dockercfg-jkr7n"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-tbc012m5-253f3-jndsh-worker-centralus1-m2wwl"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T14:05:08Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T14:24:34Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T14:24:42Z","reason":"ContainersNotReady","message":"containers with unready status: [sdn drop-icmp]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T14:05:08Z"}],"hostIP":"10.0.128.6","podIP":"10.0.128.6","podIPs":[{"ip":"10.0.128.6"}],"startTime":"2023-08-21T14:05:08Z","containerStatuses":[{"name":"drop-icmp","state":{"waiting":{"reason":"ImagePullBackOff","message":"Back-off pulling image \"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c\""}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false},{"name":"kube-rbac-proxy","state":{"running":{"startedAt":"2023-08-21T14:30:02Z"}},"lastState":{},"ready":true,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","imageID":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901","containerID":"cri-o://2b7fd22bb934461cb328b309990e9d916f87b35fe5bb10becb2a6a22980df341","started":true},{"name":"sdn","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage1871968393/4\": happened during read: read tcp 10.0.128.6:44230-\u003e54.230.17.85:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:59620f760205f8e3868205a1961cced470602ccb05ed2f28e73f1f3ee2c2370c","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-multus pod/multus-q59lg node/ci-op-tbc012m5-253f3-jndsh-worker-centralus1-m2wwl container/kube-multus - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"multus-q59lg","generateName":"multus-","namespace":"openshift-multus","uid":"52305d06-e7ca-43bf-96c5-cd6c07e11008","resourceVersion":"68970","creationTimestamp":"2023-08-21T14:05:21Z","labels":{"app":"multus","component":"network","controller-revision-hash":"578b696cc6","openshift.io/component":"network","pod-template-generation":"2","type":"infra"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"multus","uid":"0238acc8-b154-4ece-9f37-d132d5aa37ae","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T14:05:21Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:component":{},"f:controller-revision-hash":{},"f:openshift.io/component":{},"f:pod-template-generation":{},"f:type":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"0238acc8-b154-4ece-9f37-d132d5aa37ae\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"kube-multus\"}":{".":{},"f:args":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"DEFAULT_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_HOST\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"KUBERNETES_SERVICE_PORT\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RHEL7_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"RHEL8_SOURCE_DIRECTORY\"}":{".":{},"f:name":{},"f:value":{}}},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/entrypoint\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/etc/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/etc/os-release\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/opt/cni/bin\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/host/var/run/multus/cni/net.d\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"cni-binary-copy\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"cnibin\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"multus-cni-dir\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"os-release\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"system-cni-dir\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T14:24:34Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-21T14:24:42Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.6\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"system-cni-dir","hostPath":{"path":"/etc/kubernetes/cni/net.d","type":""}},{"name":"multus-cni-dir","hostPath":{"path":"/var/run/multus/cni/net.d","type":""}},{"name":"cnibin","hostPath":{"path":"/var/lib/cni/bin","type":""}},{"name":"os-release","hostPath":{"path":"/etc/os-release","type":"File"}},{"name":"cni-binary-copy","configMap":{"name":"cni-binary-copy-script","defaultMode":484}},{"name":"kube-api-access-jgcg6","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"kube-multus","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:528a2e0fb6ca1a3c6aaefcbf1fa7bcce000182565a8900bcc27cfd8db47a6ca5","command":["/bin/bash","-ec","--"],"args":["/entrypoint/cnibincopy.sh; exec /entrypoint.sh --multus-conf-file=auto --multus-autoconfig-dir=/host/var/run/multus/cni/net.d --multus-kubeconfig-file-host=/etc/kubernetes/cni/net.d/multus.d/multus.kubeconfig --readiness-indicator-file=/var/run/multus/cni/net.d/80-openshift-network.conf --cleanup-config-on-exit=true --namespace-isolation=true --multus-log-level=verbose --cni-version=0.3.1 --additional-bin-dir=/opt/multus/bin --skip-multus-binary-copy=true - \"--global-namespaces=default,openshift-multus,openshift-sriov-network-operator\"\n"],"env":[{"name":"RHEL7_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/rhel7/bin/"},{"name":"RHEL8_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/rhel8/bin/"},{"name":"DEFAULT_SOURCE_DIRECTORY","value":"/usr/src/multus-cni/bin/"},{"name":"KUBERNETES_SERVICE_PORT","value":"6443"},{"name":"KUBERNETES_SERVICE_HOST","value":"api-int.ci-op-tbc012m5-253f3.ci.azure.devcluster.openshift.com"}],"resources":{"requests":{"cpu":"10m","memory":"65Mi"}},"volumeMounts":[{"name":"cni-binary-copy","mountPath":"/entrypoint"},{"name":"os-release","mountPath":"/host/etc/os-release"},{"name":"system-cni-dir","mountPath":"/host/etc/cni/net.d"},{"name":"multus-cni-dir","mountPath":"/host/var/run/multus/cni/net.d"},{"name":"cnibin","mountPath":"/host/opt/cni/bin"},{"name":"kube-api-access-jgcg6","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"multus","serviceAccount":"multus","nodeName":"ci-op-tbc012m5-253f3-jndsh-worker-centralus1-m2wwl","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"multus-dockercfg-x7pdt"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-tbc012m5-253f3-jndsh-worker-centralus1-m2wwl"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T14:05:21Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T14:24:34Z","reason":"ContainersNotReady","message":"containers with unready status: [kube-multus]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T14:24:42Z","reason":"ContainersNotReady","message":"containers with unready status: [kube-multus]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T14:05:21Z"}],"hostIP":"10.0.128.6","podIP":"10.0.128.6","podIPs":[{"ip":"10.0.128.6"}],"startTime":"2023-08-21T14:05:21Z","containerStatuses":[{"name":"kube-multus","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage2666210933/2\": happened during read: read tcp 10.0.128.6:47300-\u003e65.8.55.85:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:528a2e0fb6ca1a3c6aaefcbf1fa7bcce000182565a8900bcc27cfd8db47a6ca5","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-image-registry pod/node-ca-9g8cr node/ci-op-tbc012m5-253f3-jndsh-worker-centralus1-m2wwl container/node-ca - reason/ContainerWait cause/ErrImagePull unable to calculate container transition time in pod: {"metadata":{"name":"node-ca-9g8cr","generateName":"node-ca-","namespace":"openshift-image-registry","uid":"f59bf181-60a6-463a-b7f3-23eb3d88cb34","resourceVersion":"68971","creationTimestamp":"2023-08-21T13:59:24Z","labels":{"controller-revision-hash":"7c956f79fd","name":"node-ca","pod-template-generation":"2"},"annotations":{"openshift.io/scc":"privileged"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"node-ca","uid":"d73d1148-9de6-4382-a7a0-0cd8851c5087","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T13:59:24Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:target.workload.openshift.io/management":{}},"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:name":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"d73d1148-9de6-4382-a7a0-0cd8851c5087\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"node-ca\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:privileged":{},"f:runAsGroup":{},"f:runAsUser":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/docker/certs.d\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/tmp/serviceca\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"host\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"serviceca\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}}}}}},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-08-21T14:24:34Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"Ready\"}":{"f:lastTransitionTime":{},"f:status":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-08-21T14:24:42Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:message":{},"f:reason":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.0.128.6\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"host","hostPath":{"path":"/etc/docker/certs.d","type":""}},{"name":"serviceca","configMap":{"name":"image-registry-certificates","defaultMode":420}},{"name":"kube-api-access-25hbj","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}}],"containers":[{"name":"node-ca","image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:26b626f9b0de3609f607fd85f4fb6865880dc100273846fcc8947cbcc8a2840c","command":["/bin/sh","-c","trap 'jobs -p \| xargs -r kill; echo shutting down node-ca; exit 0' TERM\nwhile [ true ];\ndo\n for f in $(ls /tmp/serviceca); do\n echo $f\n ca_file_path=\"/tmp/serviceca/${f}\"\n f=$(echo $f \| sed -r 's/(.)\\.\\./\\1:/')\n reg_dir_path=\"/etc/docker/certs.d/${f}\"\n if [ -e \"${reg_dir_path}\" ]; then\n cp -u $ca_file_path $reg_dir_path/ca.crt\n else\n mkdir $reg_dir_path\n cp $ca_file_path $reg_dir_path/ca.crt\n fi\n done\n for d in $(ls /etc/docker/certs.d); do\n echo $d\n dp=$(echo $d \| sed -r 's/(.):/\\1\\.\\./')\n reg_conf_path=\"/tmp/serviceca/${dp}\"\n if [ ! -e \"${reg_conf_path}\" ]; then\n rm -rf /etc/docker/certs.d/$d\n fi\n done\n sleep 60 \u0026 wait ${!}\ndone\n"],"resources":{"requests":{"cpu":"10m","memory":"10Mi"}},"volumeMounts":[{"name":"serviceca","mountPath":"/tmp/serviceca"},{"name":"host","mountPath":"/etc/docker/certs.d"},{"name":"kube-api-access-25hbj","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true,"runAsUser":1001,"runAsGroup":0}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"node-ca","serviceAccount":"node-ca","nodeName":"ci-op-tbc012m5-253f3-jndsh-worker-centralus1-m2wwl","hostNetwork":true,"securityContext":{},"imagePullSecrets":[{"name":"node-ca-dockercfg-gshrh"}],"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["ci-op-tbc012m5-253f3-jndsh-worker-centralus1-m2wwl"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"}],"priorityClassName":"system-cluster-critical","priority":2000000000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T13:59:24Z"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T14:24:34Z","reason":"ContainersNotReady","message":"containers with unready status: [node-ca]"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-08-21T14:24:42Z","reason":"ContainersNotReady","message":"containers with unready status: [node-ca]"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-08-21T13:59:24Z"}],"hostIP":"10.0.128.6","podIP":"10.0.128.6","podIPs":[{"ip":"10.0.128.6"}],"startTime":"2023-08-21T13:59:24Z","containerStatuses":[{"name":"node-ca","state":{"waiting":{"reason":"ErrImagePull","message":"rpc error: code = Unknown desc = writing blob: storing blob to file \"/var/tmp/storage1830439418/4\": happened during read: read tcp 10.0.128.6:44228-\u003e54.230.17.85:443: read: connection timed out"}},"lastState":{"terminated":{"exitCode":137,"reason":"ContainerStatusUnknown","message":"The container could not be located when the pod was deleted. The container used to be Running","startedAt":null,"finishedAt":null}},"ready":false,"restartCount":1,"image":"registry.ci.openshift.org/ocp/4.10-2023-08-21-091209@sha256:26b626f9b0de3609f607fd85f4fb6865880dc100273846fcc8947cbcc8a2840c","imageID":"","started":false}],"qosClass":"Burstable"}} ns/openshift-cluster-csi-drivers pod/azure-disk-csi-driver-node-mr8kj node/ci-op-tbc012m5-253f3-jndsh-worker-centralus1-m2wwl container/azure-inject-credentials - reason/ContainerWait cause/ErrImagePull duration/1729.24s rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage1566525556/4": happened during read: read tcp 10.0.128.6:45338->65.8.55.215:443: read: connection timed out
periodic-ci-openshift-release-master-ci-4.10-e2e-azure-ovn-upgrade (all) - 2 runs, 100% failed, 50% of failures match = 50% impact
#1693667812716843008	junit	39 hours ago
ns/openshift-marketplace pod/redhat-marketplace-9zqfl node/ci-op-2i8mhyhc-14a78-wh7n7-master-2 container/registry-server - reason/ContainerWait cause/ErrImagePull duration/306.86s rpc error: code = Unknown desc = Get "https://registry.redhat.io/v2/redhat/redhat-marketplace-index/manifests/v4.10": read tcp 10.0.0.6:52254->184.29.195.16:443: read: connection timed out ns/openshift-marketplace pod/redhat-marketplace-9zqfl node/ci-op-2i8mhyhc-14a78-wh7n7-master-2 container/registry-server: reason/ContainerWait cause/ErrImagePull duration/306.86s rpc error: code = Unknown desc = Get "https://registry.redhat.io/v2/redhat/redhat-marketplace-index/manifests/v4.10": read tcp 10.0.0.6:52254->184.29.195.16:443: read: connection timed out
periodic-ci-openshift-release-master-ci-4.11-upgrade-from-stable-4.10-e2e-azure-upgrade (all) - 5 runs, 60% failed, 100% of failures match = 60% impact
#1694111960372613120	junit	8 hours ago
ns/openshift-multus pod/multus-8qdjc node/ci-op-p06lsvly-fde6e-6p2lt-worker-eastus21-w9xjm uid/9277dc42-7365-459e-8956-a3aab1afe3b1 container/kube-multus - reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage254835156/3": happened during read: read tcp 10.0.128.6:43546->18.67.79.208:443: read: connection timed out ns/openshift-multus pod/multus-8qdjc uid/9277dc42-7365-459e-8956-a3aab1afe3b1 container/kube-multus - constructed/true reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage254835156/3": happened during read: read tcp 10.0.128.6:43546->18.67.79.208:443: read: connection timed out ns/openshift-multus pod/multus-additional-cni-plugins-qzm42 node/ci-op-p06lsvly-fde6e-6p2lt-worker-eastus21-w9xjm uid/27259eb5-7648-44b5-9e2c-4081ce9c7359 container/egress-router-binary-copy - reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage4116004196/4": happened during read: read tcp 10.0.128.6:43168->18.67.79.73:443: read: connection timed out ns/openshift-multus pod/multus-additional-cni-plugins-qzm42 uid/27259eb5-7648-44b5-9e2c-4081ce9c7359 container/egress-router-binary-copy - constructed/true reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage4116004196/4": happened during read: read tcp 10.0.128.6:43168->18.67.79.73:443: read: connection timed out ns/openshift-cluster-node-tuning-operator pod/tuned-qfnm2 node/ci-op-p06lsvly-fde6e-6p2lt-worker-eastus21-w9xjm uid/050b2380-62d5-4b01-91a1-ef1da2761a9e container/tuned - reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage133266528/1": happened during read: read tcp 10.0.128.6:37260->18.67.79.178:443: read: connection timed out
#1693838389494681600	junit	27 hours ago
ns/openshift-cluster-node-tuning-operator pod/tuned-7p5pv node/ci-op-ljdt3t7l-fde6e-lb5kp-worker-centralus1-79sl5 uid/a5b5a301-b7a8-4660-b5c1-fe9639fe5bc9 container/tuned - reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage2531722425/1": happened during read: read tcp 10.0.128.6:53480->65.8.55.213:443: read: connection timed out ns/openshift-cluster-node-tuning-operator pod/tuned-7p5pv uid/a5b5a301-b7a8-4660-b5c1-fe9639fe5bc9 container/tuned - constructed/true reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage2531722425/1": happened during read: read tcp 10.0.128.6:53480->65.8.55.213:443: read: connection timed out ns/openshift-multus pod/multus-k4psf node/ci-op-ljdt3t7l-fde6e-lb5kp-worker-centralus1-79sl5 uid/a69a2877-f1e7-45cb-adcc-18f11ee3e20a container/kube-multus - reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage342295801/1": happened during read: read tcp 10.0.128.6:53466->65.8.55.213:443: read: connection timed out ns/openshift-multus pod/multus-k4psf uid/a69a2877-f1e7-45cb-adcc-18f11ee3e20a container/kube-multus - constructed/true reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage342295801/1": happened during read: read tcp 10.0.128.6:53466->65.8.55.213:443: read: connection timed out ns/openshift-image-registry pod/node-ca-2r6bm node/ci-op-ljdt3t7l-fde6e-lb5kp-worker-centralus1-79sl5 uid/cc075183-e31b-47af-b380-d0d7f6270e89 container/node-ca - reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage948036236/3": happened during read: read tcp 10.0.128.6:48678->54.230.17.219:443: read: connection timed out
#1693536115207704576	junit	47 hours ago
ns/openshift-cluster-node-tuning-operator pod/tuned-7nwnz node/ci-op-4s46wi83-fde6e-rg6wf-worker-centralus3-vtqw5 uid/8161198d-8303-4814-bc59-57720afcba9c container/tuned - reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage1927804125/1": happened during read: read tcp 10.0.128.6:34824->18.161.108.68:443: read: connection timed out ns/openshift-cluster-node-tuning-operator pod/tuned-7nwnz uid/8161198d-8303-4814-bc59-57720afcba9c container/tuned - constructed/true reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage1927804125/1": happened during read: read tcp 10.0.128.6:34824->18.161.108.68:443: read: connection timed out ns/openshift-sdn pod/sdn-2zbdt node/ci-op-4s46wi83-fde6e-rg6wf-worker-centralus3-vtqw5 uid/bcdc0896-a279-4c76-98a6-32836c71a63d container/sdn - reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage175976709/1": happened during read: read tcp 10.0.128.6:35002->18.67.21.201:443: read: connection timed out ns/openshift-sdn pod/sdn-2zbdt uid/bcdc0896-a279-4c76-98a6-32836c71a63d container/sdn - constructed/true reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage175976709/1": happened during read: read tcp 10.0.128.6:35002->18.67.21.201:443: read: connection timed out ns/openshift-multus pod/multus-additional-cni-plugins-q8tsm node/ci-op-4s46wi83-fde6e-rg6wf-worker-centralus3-vtqw5 uid/63ad4e96-350f-4760-9f84-42865280e864 container/egress-router-binary-copy - reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage930931520/3": happened during read: read tcp 10.0.128.6:59452->52.222.137.97:443: read: connection timed out
periodic-ci-openshift-release-master-ci-4.11-upgrade-from-stable-4.10-e2e-azure-ovn-upgrade (all) - 2 runs, 100% failed, 50% of failures match = 50% impact
#1693555269809737728	junit	45 hours ago
ns/openshift-cluster-node-tuning-operator pod/tuned-9pj8d node/ci-op-45b62gx4-7215c-46qcf-worker-centralus2-6wmgd uid/c58daa97-40f6-4d5a-9a11-740c18c3f323 container/tuned - reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage537446517/1": happened during read: read tcp 10.0.128.5:37006->65.8.55.215:443: read: connection timed out ns/openshift-cluster-node-tuning-operator pod/tuned-9pj8d uid/c58daa97-40f6-4d5a-9a11-740c18c3f323 container/tuned - constructed/true reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage537446517/1": happened during read: read tcp 10.0.128.5:37006->65.8.55.215:443: read: connection timed out ns/openshift-ovn-kubernetes pod/ovnkube-node-m224w node/ci-op-45b62gx4-7215c-46qcf-worker-centralus2-6wmgd uid/a91ee5ce-02fb-488d-9639-ac9181ab2666 container/ovn-controller - reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage4194457462/1": happened during read: read tcp 10.0.128.5:56028->65.8.55.85:443: read: connection timed out ns/openshift-ovn-kubernetes pod/ovnkube-node-m224w uid/a91ee5ce-02fb-488d-9639-ac9181ab2666 container/ovn-controller - constructed/true reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage4194457462/1": happened during read: read tcp 10.0.128.5:56028->65.8.55.85:443: read: connection timed out ns/openshift-multus pod/multus-additional-cni-plugins-jlxrd node/ci-op-45b62gx4-7215c-46qcf-worker-centralus2-6wmgd uid/3d3288c0-956c-451c-938b-ef369c5f40e2 container/egress-router-binary-copy - reason/ContainerWait cause/ErrImagePull: rpc error: code = Unknown desc = writing blob: storing blob to file "/var/tmp/storage882637123/6": happened during read: read tcp 10.0.128.5:37020->65.8.55.215:443: read: connection timed out

Found in 37.50% of runs (75.00% of failures) across 40 total runs and 11 jobs (50.00% failed) in 72ms - clear search | chart view - source code located on github