# [sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes

# [sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes
upgrade to registry.build01.ci.openshift.org/ci-op-yj6pt103/release@sha256:0429fe0d28cca2104d5e9315a299cb665bd3db492586fab581333b1feb92e447 took too long: 122.34 minutes

# [sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes
upgrade to registry.build01.ci.openshift.org/ci-op-1bf3tt9x/release@sha256:481c56d9925ee5e6a80a665ef6a1630acca4ed9636600636abc2e42507956c8a took too long: 116.68 minutes

# [sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes

# step graph.Run multi-stage test e2e-gcp-ovn-upgrade - e2e-gcp-ovn-upgrade-openshift-e2e-test container test
"2023-06-08T01:38:08.1227Z","deleted_at":null,"status":"New","last_change_time":"2023-04-16T00:01:45Z","summary":"Backport TRT-596 to release-4.12","affects_versions":["4.12"],"fix_versions":[],"components":["Test Framework"],"labels":[],"url":"https://issues.redhat.com/browse/OCPBUGS-3644"},{"id":15192076,"key":"OCPBUGS-11467","created_at":"2023-06-06T14:30:12.617612Z","updated_at":"2023-06-08T01:38:08.168027Z","deleted_at":null,"status":"MODIFIED","last_change_time":"2023-04-11T06:37:23Z","summary":"loki-promtail readiness probe errors causes `[sig-arch] events should not repeat pathologically` test to fail","affects_versions":["4.9.z"],"fix_versions":[],"components":["Test Framework"],"labels":[],"url":"https://issues.redhat.com/browse/OCPBUGS-11467"},{"id":15250799,"key":"OCPBUGS-13366","created_at":"2023-06-06T14:30:12.162522Z","updated_at":"2023-06-08T01:38:08.330809Z","deleted_at":null,"status":"New","last_change_time":"2023-05-30T14:04:46Z","summary":"DNS operator prone to spamming TopologyAwareHintsDisable events on GCP/Azure since May 5","affects_versions":["4.14.0"],"fix_versions":[],"components":["Networking / DNS"],"labels":["sippy-watchlist","sippy-watchlist","trt-standup","trt-standup"],"url":"https://issues.redhat.com/browse/OCPBUGS-13366"}]},{"Name":"[sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes","Risk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Unable to find matching test results for variants: [amd64 gcp ha ovn upgrade upgrade-minor]"]},"OpenBugs":null}],"OverallRisk":{"Level":{"Name":"High","Level":100},"Reasons":["Maximum failed test risk: High"]},"OpenBugs":[]}

2023-06-08T02:28:13Z: Call to sippy finished after: 2.329805961s
response Body: {"ProwJobName":"periodic-ci-openshift-release-master-ci-4.13-upgrade-from-stable-4.12-e2e-gcp-ovn-upgrade","ProwJobRunID":1666582784040243200,"Release":"4.13","CompareRelease":"4.13","Tests":[{"Name":"[sig-arch] events should not repeat pathologically","Risk":{"Level":{"Name":"High","Level":100},"Reasons":["This test has passed 100.00% of 54 runs on jobs ['periodic-ci-openshift-release-master-ci-4.13-upgrade-from-stable-4.12-e2e-gcp-ovn-upgrade'] in the last 14 days."]},"OpenBugs":[{"id":15167728,"key":"OCPBUGS-10841","created_at":"2023-06-06T14:30:11.806307Z","updated_at":"2023-06-08T01:38:08.791068Z","deleted_at":null,"status":"POST","last_change_time":"2023-05-30T08:34:50Z","summary":"CI fails on \"events should not repeat pathologically\" because of missing node annotations","affects_versions":["4.12","4.13","4.14"],"fix_versions":[],"components":["Networking / ovn-kubernetes"],"labels":[],"url":"https://issues.redhat.com/browse/OCPBUGS-10841"},{"id":15191271,"key":"OCPBUGS-11449","created_at":"2023-06-06T14:30:12.041099Z","updated_at":"2023-06-08T01:38:08.502097Z","deleted_at":null,"status":"POST","last_change_time":"2023-05-03T16:08:37Z","summary":"Excessive TopologyAwareHintsDisabled events due to service/dns-default with topology aware hints activated.","affects_versions":["4.13"],"fix_versions":[],"components":["Networking / DNS"],"labels":[],"url":"https://issues.redhat.com/browse/OCPBUGS-11449"},{"id":15292129,"key":"OCPBUGS-14400","created_at":"2023-06-06T14:30:12.322769Z","updated_at":"2023-06-08T01:38:08.359258Z","deleted_at":null,"status":"ASSIGNED","last_change_time":"2023-06-02T14:18:37Z","summary":"CI fails on \"events should not repeat pathologically\", reason \"FailedAttachVolume\"","affects_versions":["4.13","4.14"],"fix_versions":[],"components":["Storage / Kubernetes"],"labels":[],"url":"https://issues.redhat.com/browse/OCPBUGS-14400"},{"id":15028565,"key":"OCPBUGS-5943","created_at":"2023-06-06T14:30:12.056394Z","updated_at":"2023-06-08T01:38:08.473689Z","deleted_at":null,"status":"POST","last_change_time":"2023-05-31T06:46:15Z","summary":"Excessive TopologyAwareHintsDisabled events due to service/dns-default with topology aware hints activated.","affects_versions":["4.13"],"fix_versions":["4.13.0"],"components":["Networking / DNS"],"labels":[],"url":"https://issues.redhat.com/browse/OCPBUGS-5943"},{"id":15278627,"key":"OCPBUGS-14033","created_at":"2023-06-06T14:30:11.764386Z","updated_at":"2023-06-08T01:38:08.153681Z","deleted_at":null,"status":"ON_QA","last_change_time":"2023-06-05T11:30:18Z","summary":"Pathological test failing on reason/RecreatingFailedPod in openshift-monitoring","affects_versions":["4.14"],"fix_versions":[],"components":["Monitoring"],"labels":["trt-standup","trt-standup"],"url":"https://issues.redhat.com/browse/OCPBUGS-14033"},{"id":14935296,"key":"OCPBUGS-3644","created_at":"2023-04-10T01:00:10.55633Z","updated_at":"2023-06-08T01:38:08.1227Z","deleted_at":null,"status":"New","last_change_time":"2023-04-16T00:01:45Z","summary":"Backport TRT-596 to release-4.12","affects_versions":["4.12"],"fix_versions":[],"components":["Test Framework"],"labels":[],"url":"https://issues.redhat.com/browse/OCPBUGS-3644"},{"id":15192076,"key":"OCPBUGS-11467","created_at":"2023-06-06T14:30:12.617612Z","updated_at":"2023-06-08T01:38:08.168027Z","deleted_at":null,"status":"MODIFIED","last_change_time":"2023-04-11T06:37:23Z","summary":"loki-promtail readiness probe errors causes `[sig-arch] events should not repeat pathologically` test to fail","affects_versions":["4.9.z"],"fix_versions":[],"components":["Test Framework"],"labels":[],"url":"https://issues.redhat.com/browse/OCPBUGS-11467"},{"id":15250799,"key":"OCPBUGS-13366","created_at":"2023-06-06T14:30:12.162522Z","updated_at":"2023-06-08T01:38:08.330809Z","deleted_at":null,"status":"New","last_change_time":"2023-05-30T14:04:46Z","summary":"DNS operator prone to spamming TopologyAwareHintsDisable events on GCP/Azure since May 5","affects_versions":["4.14.0"],"fix_versions":[],"components":["Networking / DNS"],"labels":["sippy-watchlist","sippy-watchlist","trt-standup","trt-standup"],"url":"https://issues.redhat.com/browse/OCPBUGS-13366"}]},{"Name":"[sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes","Risk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Unable to find matching test results for variants: [amd64 gcp ha ovn upgrade upgrade-minor]"]},"OpenBugs":null}],"OverallRisk":{"Level":{"Name":"High","Level":100},"Reasons":["Maximum failed test risk: High"]},"OpenBugs":[]}

---
"2023-06-08T01:38:08.1227Z","deleted_at":null,"status":"New","last_change_time":"2023-04-16T00:01:45Z","summary":"Backport TRT-596 to release-4.12","affects_versions":["4.12"],"fix_versions":[],"components":["Test Framework"],"labels":[],"url":"https://issues.redhat.com/browse/OCPBUGS-3644"},{"id":15192076,"key":"OCPBUGS-11467","created_at":"2023-06-06T14:30:12.617612Z","updated_at":"2023-06-08T01:38:08.168027Z","deleted_at":null,"status":"MODIFIED","last_change_time":"2023-04-11T06:37:23Z","summary":"loki-promtail readiness probe errors causes `[sig-arch] events should not repeat pathologically` test to fail","affects_versions":["4.9.z"],"fix_versions":[],"components":["Test Framework"],"labels":[],"url":"https://issues.redhat.com/browse/OCPBUGS-11467"},{"id":15250799,"key":"OCPBUGS-13366","created_at":"2023-06-06T14:30:12.162522Z","updated_at":"2023-06-08T01:38:08.330809Z","deleted_at":null,"status":"New","last_change_time":"2023-05-30T14:04:46Z","summary":"DNS operator prone to spamming TopologyAwareHintsDisable events on GCP/Azure since May 5","affects_versions":["4.14.0"],"fix_versions":[],"components":["Networking / DNS"],"labels":["sippy-watchlist","sippy-watchlist","trt-standup","trt-standup"],"url":"https://issues.redhat.com/browse/OCPBUGS-13366"}]},{"Name":"[sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes","Risk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Unable to find matching test results for variants: [amd64 gcp ha ovn upgrade upgrade-minor]"]},"OpenBugs":null}],"OverallRisk":{"Level":{"Name":"High","Level":100},"Reasons":["Maximum failed test risk: High"]},"OpenBugs":[]}

# [sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes
upgrade to registry.build01.ci.openshift.org/ci-op-hlw9qvbv/release@sha256:57c84993858cc21787046993e6c55f30aebaf6ff01d2cbfa9f37ca326e8f7b1c took too long: 118.34 minutes

# [sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes
upgrade to registry.build01.ci.openshift.org/ci-op-cc1qgwbg/release@sha256:16c77b8c87bd99552a50be2e26f02d11ec9ca8df1dbe0f0bac4bc7c3bc0f7bec took too long: 107.74 minutes

# [sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes

2023-06-07T19:03:18Z: Call to sippy finished after: 4.986374344s
response Body: {"ProwJobName":"periodic-ci-openshift-release-master-ci-4.13-upgrade-from-stable-4.12-e2e-gcp-ovn-rt-upgrade","ProwJobRunID":1666468268656824320,"Release":"4.13","CompareRelease":"4.13","Tests":[{"Name":"[sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes","Risk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Unable to find matching test results for variants: [amd64 gcp ha ovn realtime upgrade upgrade-minor]"]},"OpenBugs":null},{"Name":"[sig-trt] disruption/ci-cluster-network-liveness connection/new should be available throughout the test","Risk":{"Level":{"Name":"Medium","Level":50},"Reasons":["This test has passed 97.70% of 87 runs on jobs ['periodic-ci-openshift-release-master-ci-4.13-upgrade-from-stable-4.12-e2e-gcp-ovn-rt-upgrade'] in the last 14 days."]},"OpenBugs":[]},{"Name":"[sig-trt] disruption/ci-cluster-network-liveness connection/reused should be available throughout the test","Risk":{"Level":{"Name":"Medium","Level":50},"Reasons":["This test has passed 96.55% of 87 runs on jobs ['periodic-ci-openshift-release-master-ci-4.13-upgrade-from-stable-4.12-e2e-gcp-ovn-rt-upgrade'] in the last 14 days."]},"OpenBugs":[]}],"OverallRisk":{"Level":{"Name":"Medium","Level":50},"Reasons":["Maximum failed test risk: Medium"]},"OpenBugs":[]}

2023-06-07T19:03:18Z: Call to sippy finished after: 4.986374344s
response Body: {"ProwJobName":"periodic-ci-openshift-release-master-ci-4.13-upgrade-from-stable-4.12-e2e-gcp-ovn-rt-upgrade","ProwJobRunID":1666468268656824320,"Release":"4.13","CompareRelease":"4.13","Tests":[{"Name":"[sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes","Risk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Unable to find matching test results for variants: [amd64 gcp ha ovn realtime upgrade upgrade-minor]"]},"OpenBugs":null},{"Name":"[sig-trt] disruption/ci-cluster-network-liveness connection/new should be available throughout the test","Risk":{"Level":{"Name":"Medium","Level":50},"Reasons":["This test has passed 97.70% of 87 runs on jobs ['periodic-ci-openshift-release-master-ci-4.13-upgrade-from-stable-4.12-e2e-gcp-ovn-rt-upgrade'] in the last 14 days."]},"OpenBugs":[]},{"Name":"[sig-trt] disruption/ci-cluster-network-liveness connection/reused should be available throughout the test","Risk":{"Level":{"Name":"Medium","Level":50},"Reasons":["This test has passed 96.55% of 87 runs on jobs ['periodic-ci-openshift-release-master-ci-4.13-upgrade-from-stable-4.12-e2e-gcp-ovn-rt-upgrade'] in the last 14 days."]},"OpenBugs":[]}],"OverallRisk":{"Level":{"Name":"Medium","Level":50},"Reasons":["Maximum failed test risk: Medium"]},"OpenBugs":[]}

2023-06-07T19:03:18Z: Call to sippy finished after: 4.986374344s
response Body: {"ProwJobName":"periodic-ci-openshift-release-master-ci-4.13-upgrade-from-stable-4.12-e2e-gcp-ovn-rt-upgrade","ProwJobRunID":1666468268656824320,"Release":"4.13","CompareRelease":"4.13","Tests":[{"Name":"[sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes","Risk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Unable to find matching test results for variants: [amd64 gcp ha ovn realtime upgrade upgrade-minor]"]},"OpenBugs":null},{"Name":"[sig-trt] disruption/ci-cluster-network-liveness connection/new should be available throughout the test","Risk":{"Level":{"Name":"Medium","Level":50},"Reasons":["This test has passed 97.70% of 87 runs on jobs ['periodic-ci-openshift-release-master-ci-4.13-upgrade-from-stable-4.12-e2e-gcp-ovn-rt-upgrade'] in the last 14 days."]},"OpenBugs":[]},{"Name":"[sig-trt] disruption/ci-cluster-network-liveness connection/reused should be available throughout the test","Risk":{"Level":{"Name":"Medium","Level":50},"Reasons":["This test has passed 96.55% of 87 runs on jobs ['periodic-ci-openshift-release-master-ci-4.13-upgrade-from-stable-4.12-e2e-gcp-ovn-rt-upgrade'] in the last 14 days."]},"OpenBugs":[]}],"OverallRisk":{"Level":{"Name":"Medium","Level":50},"Reasons":["Maximum failed test risk: Medium"]},"OpenBugs":[]}

# [sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes

# [sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes

# [sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes

# step graph.Run multi-stage test e2e-vsphere-ovn-upgrade-storage-data - e2e-vsphere-ovn-upgrade-storage-data-openshift-e2e-test container test
dy after being upgraded and go unready only once","Risk":{"Level":{"Name":"Low","Level":1},"Reasons":["This test has passed 50.00% of 12 runs on release 4.13 [amd64 ha ovn upgrade upgrade-minor vsphere-ipi] in the last week."]},"OpenBugs":[{"id":14922179,"key":"OCPBUGS-3299","created_at":"2023-05-06T01:13:53.193711Z","updated_at":"2023-06-06T17:54:59.485869Z","deleted_at":null,"status":"ASSIGNED","last_change_time":"2023-04-25T16:52:30Z","summary":"4.8-ci payload failure due to NodeControllerDegraded / \"[sig-node] nodes should not go unready after being upgraded and go unready only once\"","affects_versions":["4.8.z"],"fix_versions":[],"components":["Node / Kubelet"],"labels":["triaged","triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-3299"},{"id":15136559,"key":"OCPBUGS-9941","created_at":"2023-05-06T01:13:52.860973Z","updated_at":"2023-06-06T17:54:59.334274Z","deleted_at":null,"status":"New","last_change_time":"2023-06-05T20:12:20Z","summary":"\"[sig-node] nodes should not go unready after being upgraded and go unready only once\" failing on upgrade tests","affects_versions":["4.13.0"],"fix_versions":[],"components":["Test Framework / OpenStack"],"labels":["Triaged","Triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-9941"}]},{"Name":"[sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes","Risk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Unable to find matching test results for variants: [amd64 ha ovn upgrade upgrade-minor vsphere-ipi]"]},"OpenBugs":null}],"OverallRisk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Maximum failed test risk: Unknown"]},"OpenBugs":[]}

2023-06-06T18:47:55Z: Call to sippy finished after: 883.171843ms
response Body: {"ProwJobName":"periodic-ci-openshift-release-master-ci-4.13-upgrade-from-stable-4.12-e2e-vsphere-ovn-upgrade-storage-data","ProwJobRunID":1666101174161379328,"Release":"4.13","CompareRelease":"4.13","Tests":[{"Name":"[sig-node] nodes should not go unready after being upgraded and go unready only once","Risk":{"Level":{"Name":"Low","Level":1},"Reasons":["This test has passed 50.00% of 12 runs on release 4.13 [amd64 ha ovn upgrade upgrade-minor vsphere-ipi] in the last week."]},"OpenBugs":[{"id":14922179,"key":"OCPBUGS-3299","created_at":"2023-05-06T01:13:53.193711Z","updated_at":"2023-06-06T17:54:59.485869Z","deleted_at":null,"status":"ASSIGNED","last_change_time":"2023-04-25T16:52:30Z","summary":"4.8-ci payload failure due to NodeControllerDegraded / \"[sig-node] nodes should not go unready after being upgraded and go unready only once\"","affects_versions":["4.8.z"],"fix_versions":[],"components":["Node / Kubelet"],"labels":["triaged","triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-3299"},{"id":15136559,"key":"OCPBUGS-9941","created_at":"2023-05-06T01:13:52.860973Z","updated_at":"2023-06-06T17:54:59.334274Z","deleted_at":null,"status":"New","last_change_time":"2023-06-05T20:12:20Z","summary":"\"[sig-node] nodes should not go unready after being upgraded and go unready only once\" failing on upgrade tests","affects_versions":["4.13.0"],"fix_versions":[],"components":["Test Framework / OpenStack"],"labels":["Triaged","Triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-9941"}]},{"Name":"[sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes","Risk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Unable to find matching test results for variants: [amd64 ha ovn upgrade upgrade-minor vsphere-ipi]"]},"OpenBugs":null}],"OverallRisk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Maximum failed test risk: Unknown"]},"OpenBugs":[]}

---
dy after being upgraded and go unready only once","Risk":{"Level":{"Name":"Low","Level":1},"Reasons":["This test has passed 50.00% of 12 runs on release 4.13 [amd64 ha ovn upgrade upgrade-minor vsphere-ipi] in the last week."]},"OpenBugs":[{"id":14922179,"key":"OCPBUGS-3299","created_at":"2023-05-06T01:13:53.193711Z","updated_at":"2023-06-06T17:54:59.485869Z","deleted_at":null,"status":"ASSIGNED","last_change_time":"2023-04-25T16:52:30Z","summary":"4.8-ci payload failure due to NodeControllerDegraded / \"[sig-node] nodes should not go unready after being upgraded and go unready only once\"","affects_versions":["4.8.z"],"fix_versions":[],"components":["Node / Kubelet"],"labels":["triaged","triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-3299"},{"id":15136559,"key":"OCPBUGS-9941","created_at":"2023-05-06T01:13:52.860973Z","updated_at":"2023-06-06T17:54:59.334274Z","deleted_at":null,"status":"New","last_change_time":"2023-06-05T20:12:20Z","summary":"\"[sig-node] nodes should not go unready after being upgraded and go unready only once\" failing on upgrade tests","affects_versions":["4.13.0"],"fix_versions":[],"components":["Test Framework / OpenStack"],"labels":["Triaged","Triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-9941"}]},{"Name":"[sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes","Risk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Unable to find matching test results for variants: [amd64 ha ovn upgrade upgrade-minor vsphere-ipi]"]},"OpenBugs":null}],"OverallRisk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Maximum failed test risk: Unknown"]},"OpenBugs":[]}

# step graph.Run multi-stage test e2e-vsphere-ovn-upgrade-storage-data - e2e-vsphere-ovn-upgrade-storage-data-openshift-e2e-test container test
dy after being upgraded and go unready only once","Risk":{"Level":{"Name":"Low","Level":1},"Reasons":["This test has passed 50.00% of 12 runs on release 4.13 [amd64 ha ovn upgrade upgrade-minor vsphere-ipi] in the last week."]},"OpenBugs":[{"id":14922179,"key":"OCPBUGS-3299","created_at":"2023-05-06T01:13:53.193711Z","updated_at":"2023-06-05T17:56:45.539726Z","deleted_at":null,"status":"ASSIGNED","last_change_time":"2023-04-25T16:52:30Z","summary":"4.8-ci payload failure due to NodeControllerDegraded / \"[sig-node] nodes should not go unready after being upgraded and go unready only once\"","affects_versions":["4.8.z"],"fix_versions":[],"components":["Node / Kubelet"],"labels":["triaged","triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-3299"},{"id":15136559,"key":"OCPBUGS-9941","created_at":"2023-05-06T01:13:52.860973Z","updated_at":"2023-06-05T17:56:46.128779Z","deleted_at":null,"status":"New","last_change_time":"2023-06-01T10:42:11Z","summary":"\"[sig-node] nodes should not go unready after being upgraded and go unready only once\" failing on upgrade tests","affects_versions":["4.13.0"],"fix_versions":[],"components":["Test Framework / OpenStack"],"labels":["Triaged","Triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-9941"}]},{"Name":"[sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes","Risk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Unable to find matching test results for variants: [amd64 ha ovn upgrade upgrade-minor vsphere-ipi]"]},"OpenBugs":null}],"OverallRisk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Maximum failed test risk: Unknown"]},"OpenBugs":[]}

---
dy after being upgraded and go unready only once","Risk":{"Level":{"Name":"Low","Level":1},"Reasons":["This test has passed 50.00% of 12 runs on release 4.13 [amd64 ha ovn upgrade upgrade-minor vsphere-ipi] in the last week."]},"OpenBugs":[{"id":14922179,"key":"OCPBUGS-3299","created_at":"2023-05-06T01:13:53.193711Z","updated_at":"2023-06-05T17:56:45.539726Z","deleted_at":null,"status":"ASSIGNED","last_change_time":"2023-04-25T16:52:30Z","summary":"4.8-ci payload failure due to NodeControllerDegraded / \"[sig-node] nodes should not go unready after being upgraded and go unready only once\"","affects_versions":["4.8.z"],"fix_versions":[],"components":["Node / Kubelet"],"labels":["triaged","triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-3299"},{"id":15136559,"key":"OCPBUGS-9941","created_at":"2023-05-06T01:13:52.860973Z","updated_at":"2023-06-05T17:56:46.128779Z","deleted_at":null,"status":"New","last_change_time":"2023-06-01T10:42:11Z","summary":"\"[sig-node] nodes should not go unready after being upgraded and go unready only once\" failing on upgrade tests","affects_versions":["4.13.0"],"fix_versions":[],"components":["Test Framework / OpenStack"],"labels":["Triaged","Triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-9941"}]},{"Name":"[sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes","Risk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Unable to find matching test results for variants: [amd64 ha ovn upgrade upgrade-minor vsphere-ipi]"]},"OpenBugs":null}],"OverallRisk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Maximum failed test risk: Unknown"]},"OpenBugs":[]}

2023-06-05T18:23:45Z: Call to sippy finished after: 581.507956ms
response Body: {"ProwJobName":"periodic-ci-openshift-release-master-ci-4.13-upgrade-from-stable-4.12-e2e-vsphere-ovn-upgrade-storage-data","ProwJobRunID":1665738584633643008,"Release":"4.13","CompareRelease":"4.13","Tests":[{"Name":"[sig-node] nodes should not go unready after being upgraded and go unready only once","Risk":{"Level":{"Name":"Low","Level":1},"Reasons":["This test has passed 50.00% of 12 runs on release 4.13 [amd64 ha ovn upgrade upgrade-minor vsphere-ipi] in the last week."]},"OpenBugs":[{"id":14922179,"key":"OCPBUGS-3299","created_at":"2023-05-06T01:13:53.193711Z","updated_at":"2023-06-05T17:56:45.539726Z","deleted_at":null,"status":"ASSIGNED","last_change_time":"2023-04-25T16:52:30Z","summary":"4.8-ci payload failure due to NodeControllerDegraded / \"[sig-node] nodes should not go unready after being upgraded and go unready only once\"","affects_versions":["4.8.z"],"fix_versions":[],"components":["Node / Kubelet"],"labels":["triaged","triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-3299"},{"id":15136559,"key":"OCPBUGS-9941","created_at":"2023-05-06T01:13:52.860973Z","updated_at":"2023-06-05T17:56:46.128779Z","deleted_at":null,"status":"New","last_change_time":"2023-06-01T10:42:11Z","summary":"\"[sig-node] nodes should not go unready after being upgraded and go unready only once\" failing on upgrade tests","affects_versions":["4.13.0"],"fix_versions":[],"components":["Test Framework / OpenStack"],"labels":["Triaged","Triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-9941"}]},{"Name":"[sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes","Risk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Unable to find matching test results for variants: [amd64 ha ovn upgrade upgrade-minor vsphere-ipi]"]},"OpenBugs":null}],"OverallRisk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Maximum failed test risk: Unknown"]},"OpenBugs":[]}

---
dy after being upgraded and go unready only once","Risk":{"Level":{"Name":"Low","Level":1},"Reasons":["This test has passed 50.00% of 12 runs on release 4.13 [amd64 ha ovn upgrade upgrade-minor vsphere-ipi] in the last week."]},"OpenBugs":[{"id":14922179,"key":"OCPBUGS-3299","created_at":"2023-05-06T01:13:53.193711Z","updated_at":"2023-06-05T17:56:45.539726Z","deleted_at":null,"status":"ASSIGNED","last_change_time":"2023-04-25T16:52:30Z","summary":"4.8-ci payload failure due to NodeControllerDegraded / \"[sig-node] nodes should not go unready after being upgraded and go unready only once\"","affects_versions":["4.8.z"],"fix_versions":[],"components":["Node / Kubelet"],"labels":["triaged","triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-3299"},{"id":15136559,"key":"OCPBUGS-9941","created_at":"2023-05-06T01:13:52.860973Z","updated_at":"2023-06-05T17:56:46.128779Z","deleted_at":null,"status":"New","last_change_time":"2023-06-01T10:42:11Z","summary":"\"[sig-node] nodes should not go unready after being upgraded and go unready only once\" failing on upgrade tests","affects_versions":["4.13.0"],"fix_versions":[],"components":["Test Framework / OpenStack"],"labels":["Triaged","Triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-9941"}]},{"Name":"[sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes","Risk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Unable to find matching test results for variants: [amd64 ha ovn upgrade upgrade-minor vsphere-ipi]"]},"OpenBugs":null}],"OverallRisk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Maximum failed test risk: Unknown"]},"OpenBugs":[]}

# [sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes

# step graph.Run multi-stage test e2e-vsphere-ovn-upgrade-storage-data - e2e-vsphere-ovn-upgrade-storage-data-openshift-e2e-test container test
ady after being upgraded and go unready only once","Risk":{"Level":{"Name":"Low","Level":1},"Reasons":["This test has passed 45.45% of 11 runs on release 4.13 [amd64 ha ovn upgrade upgrade-minor vsphere-ipi] in the last week."]},"OpenBugs":[{"id":14922179,"key":"OCPBUGS-3299","created_at":"2023-05-06T01:13:53.193711Z","updated_at":"2023-06-02T16:48:05.36698Z","deleted_at":null,"status":"ASSIGNED","last_change_time":"2023-04-25T16:52:30Z","summary":"4.8-ci payload failure due to NodeControllerDegraded / \"[sig-node] nodes should not go unready after being upgraded and go unready only once\"","affects_versions":["4.8.z"],"fix_versions":[],"components":["Node / Kubelet"],"labels":["triaged","triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-3299"},{"id":15136559,"key":"OCPBUGS-9941","created_at":"2023-05-06T01:13:52.860973Z","updated_at":"2023-06-02T16:48:07.241122Z","deleted_at":null,"status":"New","last_change_time":"2023-06-01T10:42:11Z","summary":"\"[sig-node] nodes should not go unready after being upgraded and go unready only once\" failing on upgrade tests","affects_versions":["4.13.0"],"fix_versions":[],"components":["Test Framework / OpenStack"],"labels":["Triaged","Triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-9941"}]},{"Name":"[sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes","Risk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Unable to find matching test results for variants: [amd64 ha ovn upgrade upgrade-minor vsphere-ipi]"]},"OpenBugs":null}],"OverallRisk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Maximum failed test risk: Unknown"]},"OpenBugs":[]}

---
ady after being upgraded and go unready only once","Risk":{"Level":{"Name":"Low","Level":1},"Reasons":["This test has passed 45.45% of 11 runs on release 4.13 [amd64 ha ovn upgrade upgrade-minor vsphere-ipi] in the last week."]},"OpenBugs":[{"id":14922179,"key":"OCPBUGS-3299","created_at":"2023-05-06T01:13:53.193711Z","updated_at":"2023-06-02T16:48:05.36698Z","deleted_at":null,"status":"ASSIGNED","last_change_time":"2023-04-25T16:52:30Z","summary":"4.8-ci payload failure due to NodeControllerDegraded / \"[sig-node] nodes should not go unready after being upgraded and go unready only once\"","affects_versions":["4.8.z"],"fix_versions":[],"components":["Node / Kubelet"],"labels":["triaged","triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-3299"},{"id":15136559,"key":"OCPBUGS-9941","created_at":"2023-05-06T01:13:52.860973Z","updated_at":"2023-06-02T16:48:07.241122Z","deleted_at":null,"status":"New","last_change_time":"2023-06-01T10:42:11Z","summary":"\"[sig-node] nodes should not go unready after being upgraded and go unready only once\" failing on upgrade tests","affects_versions":["4.13.0"],"fix_versions":[],"components":["Test Framework / OpenStack"],"labels":["Triaged","Triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-9941"}]},{"Name":"[sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes","Risk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Unable to find matching test results for variants: [amd64 ha ovn upgrade upgrade-minor vsphere-ipi]"]},"OpenBugs":null}],"OverallRisk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Maximum failed test risk: Unknown"]},"OpenBugs":[]}

2023-06-02T17:47:42Z: Call to sippy finished after: 607.501238ms
response Body: {"ProwJobName":"periodic-ci-openshift-release-master-ci-4.13-upgrade-from-stable-4.12-e2e-vsphere-ovn-upgrade-storage-data","ProwJobRunID":1664651390988849152,"Release":"4.13","CompareRelease":"4.13","Tests":[{"Name":"[sig-node] nodes should not go unready after being upgraded and go unready only once","Risk":{"Level":{"Name":"Low","Level":1},"Reasons":["This test has passed 45.45% of 11 runs on release 4.13 [amd64 ha ovn upgrade upgrade-minor vsphere-ipi] in the last week."]},"OpenBugs":[{"id":14922179,"key":"OCPBUGS-3299","created_at":"2023-05-06T01:13:53.193711Z","updated_at":"2023-06-02T16:48:05.36698Z","deleted_at":null,"status":"ASSIGNED","last_change_time":"2023-04-25T16:52:30Z","summary":"4.8-ci payload failure due to NodeControllerDegraded / \"[sig-node] nodes should not go unready after being upgraded and go unready only once\"","affects_versions":["4.8.z"],"fix_versions":[],"components":["Node / Kubelet"],"labels":["triaged","triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-3299"},{"id":15136559,"key":"OCPBUGS-9941","created_at":"2023-05-06T01:13:52.860973Z","updated_at":"2023-06-02T16:48:07.241122Z","deleted_at":null,"status":"New","last_change_time":"2023-06-01T10:42:11Z","summary":"\"[sig-node] nodes should not go unready after being upgraded and go unready only once\" failing on upgrade tests","affects_versions":["4.13.0"],"fix_versions":[],"components":["Test Framework / OpenStack"],"labels":["Triaged","Triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-9941"}]},{"Name":"[sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes","Risk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Unable to find matching test results for variants: [amd64 ha ovn upgrade upgrade-minor vsphere-ipi]"]},"OpenBugs":null}],"OverallRisk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Maximum failed test risk: Unknown"]},"OpenBugs":[]}

---
ady after being upgraded and go unready only once","Risk":{"Level":{"Name":"Low","Level":1},"Reasons":["This test has passed 45.45% of 11 runs on release 4.13 [amd64 ha ovn upgrade upgrade-minor vsphere-ipi] in the last week."]},"OpenBugs":[{"id":14922179,"key":"OCPBUGS-3299","created_at":"2023-05-06T01:13:53.193711Z","updated_at":"2023-06-02T16:48:05.36698Z","deleted_at":null,"status":"ASSIGNED","last_change_time":"2023-04-25T16:52:30Z","summary":"4.8-ci payload failure due to NodeControllerDegraded / \"[sig-node] nodes should not go unready after being upgraded and go unready only once\"","affects_versions":["4.8.z"],"fix_versions":[],"components":["Node / Kubelet"],"labels":["triaged","triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-3299"},{"id":15136559,"key":"OCPBUGS-9941","created_at":"2023-05-06T01:13:52.860973Z","updated_at":"2023-06-02T16:48:07.241122Z","deleted_at":null,"status":"New","last_change_time":"2023-06-01T10:42:11Z","summary":"\"[sig-node] nodes should not go unready after being upgraded and go unready only once\" failing on upgrade tests","affects_versions":["4.13.0"],"fix_versions":[],"components":["Test Framework / OpenStack"],"labels":["Triaged","Triaged"],"url":"https://issues.redhat.com/browse/OCPBUGS-9941"}]},{"Name":"[sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes","Risk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Unable to find matching test results for variants: [amd64 ha ovn upgrade upgrade-minor vsphere-ipi]"]},"OpenBugs":null}],"OverallRisk":{"Level":{"Name":"Unknown","Level":25},"Reasons":["Maximum failed test risk: Unknown"]},"OpenBugs":[]}

# [sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes
upgrade to registry.build01.ci.openshift.org/ci-op-vy8s2j9t/release@sha256:c2b5db8eeaa59150c5a154ebe8eb365c6f9e12d14824f0f27556ce054f25bf8b took too long: 99.17 minutes

# [sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes
upgrade to registry.build01.ci.openshift.org/ci-op-nhzzhg2h/release@sha256:2e5051d962a2067d50b6d9c06bcfa13945b0f5b343a4706ef373a718caf204d3 took too long: 114.17 minutes

# [sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes
upgrade to registry.build01.ci.openshift.org/ci-op-vy2zg8tj/release@sha256:2e5051d962a2067d50b6d9c06bcfa13945b0f5b343a4706ef373a718caf204d3 took too long: 92.51 minutes

# [sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes
upgrade to registry.build01.ci.openshift.org/ci-op-vrynrz6d/release@sha256:2e5051d962a2067d50b6d9c06bcfa13945b0f5b343a4706ef373a718caf204d3 took too long: 90.34 minutes

# [sig-cluster-lifecycle] cluster upgrade should complete in 90.00 minutes