Repository navigation
Kraken timeout: gossip-router pods not recognized after restart #943
Description
Describe the bug
There seems to be an issue with Kraken recognizing the pods which are coming back for gossip-router and fails with a timeout message. Need to debug this, at least some form of regression from the past.
[chaos:kraken-pod-scenarios] 2024-08-15 14:26:54,129 [INFO] Executing scenarios for iteration 0
[chaos:kraken-pod-scenarios] 2024-08-15 14:26:54,144 [INFO] scenario scenarios/pod_scenario.yaml
[chaos:kraken-pod-scenarios] 2024-08-15 14:26:55,532 [INFO] {
[chaos:kraken-pod-scenarios] "output_id": "success",
[chaos:kraken-pod-scenarios] "output_data": {
[chaos:kraken-pod-scenarios] "pods": {
[chaos:kraken-pod-scenarios] "1723732014361533777": {
[chaos:kraken-pod-scenarios] "namespace": "runner-keycloak",
[chaos:kraken-pod-scenarios] "name": "infinispan-router-766c85c469-qd97c",
[chaos:kraken-pod-scenarios] "creation_timestamp": "2024-08-15 13:17:01+00:00"
[chaos:kraken-pod-scenarios] }
[chaos:kraken-pod-scenarios] }
[chaos:kraken-pod-scenarios] }
[chaos:kraken-pod-scenarios] }
[chaos:kraken-pod-scenarios]
[chaos:kraken-pod-scenarios] 2024-08-15 14:32:56,642 [INFO] {
[chaos:kraken-pod-scenarios] "output_id": "error",
[chaos:kraken-pod-scenarios] "output_data": {
[chaos:kraken-pod-scenarios] "error": "timeout while waiting for pods to come up"
[chaos:kraken-pod-scenarios] }
[chaos:kraken-pod-scenarios] }
[chaos:kraken-pod-scenarios]
[chaos:kraken-pod-scenarios] 2024-08-15 14:32:56,643 [ERROR] scenario: scenarios/pod_scenario.yaml failed with exception: <class 'Exception'> file: /root/kraken/kraken/plugins/__init__.py line: 247
[chaos:kraken-pod-scenarios] 2024-08-15 14:32:56,644 [INFO]
[chaos:kraken-pod-scenarios] 2024-08-15 14:33:08,219 [INFO] Telemetry data:
[chaos:kraken-pod-scenarios] {
[chaos:kraken-pod-scenarios] "scenarios": [
[chaos:kraken-pod-scenarios] {
[chaos:kraken-pod-scenarios] "startTimeStamp": 1723732014,
[chaos:kraken-pod-scenarios] "endTimeStamp": 1723732376.6440089,
[chaos:kraken-pod-scenarios] "scenario": "scenarios/pod_scenario.yaml",
[chaos:kraken-pod-scenarios] "exitStatus": 1,
[chaos:kraken-pod-scenarios] "parametersBase64": "",
[chaos:kraken-pod-scenarios] "parameters": [
[chaos:kraken-pod-scenarios] {
[chaos:kraken-pod-scenarios] "config": {
[chaos:kraken-pod-scenarios] "kill": 1,
[chaos:kraken-pod-scenarios] "label_selector": "app=infinispan-router-pod,clusterName=infinispan",
[chaos:kraken-pod-scenarios] "namespace_pattern": "runner-keycloak",
[chaos:kraken-pod-scenarios] "timeout": 180
[chaos:kraken-pod-scenarios] },
[chaos:kraken-pod-scenarios] "id": "kill-pods"
[chaos:kraken-pod-scenarios] },
[chaos:kraken-pod-scenarios] {
[chaos:kraken-pod-scenarios] "config": {
[chaos:kraken-pod-scenarios] "count": 3,
[chaos:kraken-pod-scenarios] "label_selector": "app=infinispan-router-pod,clusterName=infinispan",
[chaos:kraken-pod-scenarios] "namespace_pattern": "runner-keycloak",
[chaos:kraken-pod-scenarios] "timeout": 360
[chaos:kraken-pod-scenarios] },
[chaos:kraken-pod-scenarios] "id": "wait-for-pods"
[chaos:kraken-pod-scenarios] }
[chaos:kraken-pod-scenarios] ]
[chaos:kraken-pod-scenarios] }
[chaos:kraken-pod-scenarios] ],
[chaos:kraken-pod-scenarios] "node_infos": [
[chaos:kraken-pod-scenarios] {
[chaos:kraken-pod-scenarios] "taints": [],
[chaos:kraken-pod-scenarios] "instance_type": "unknown",
[chaos:kraken-pod-scenarios] "node_type": "worker",
[chaos:kraken-pod-scenarios] "name": "ip-10-0-12-12.ec2.internal",
[chaos:kraken-pod-scenarios] "architecture": "amd64",
[chaos:kraken-pod-scenarios] "kernel_version": "5.14.0-284.73.1.el9_2.x86_64",
[chaos:kraken-pod-scenarios] "kubelet_version": "v1.28.11+add48d0",
[chaos:kraken-pod-scenarios] "os_version": "Red Hat Enterprise Linux CoreOS 415.92.202407031156-0 (Plow)"
[chaos:kraken-pod-scenarios] },
[chaos:kraken-pod-scenarios] {
[chaos:kraken-pod-scenarios] "taints": [],
[chaos:kraken-pod-scenarios] "instance_type": "unknown",
[chaos:kraken-pod-scenarios] "node_type": "worker",
[chaos:kraken-pod-scenarios] "name": "ip-10-0-12-13.ec2.internal",
[chaos:kraken-pod-scenarios] "architecture": "amd64",
[chaos:kraken-pod-scenarios] "kernel_version": "5.14.0-284.73.1.el9_2.x86_64",
[chaos:kraken-pod-scenarios] "kubelet_version": "v1.28.11+add48d0",
[chaos:kraken-pod-scenarios] "os_version": "Red Hat Enterprise Linux CoreOS 415.92.202407031156-0 (Plow)"
[chaos:kraken-pod-scenarios] },
[chaos:kraken-pod-scenarios] {
[chaos:kraken-pod-scenarios] "taints": [],
[chaos:kraken-pod-scenarios] "instance_type": "unknown",
[chaos:kraken-pod-scenarios] "node_type": "worker",
[chaos:kraken-pod-scenarios] "name": "ip-10-0-12-4.ec2.internal",
[chaos:kraken-pod-scenarios] "architecture": "amd64",
[chaos:kraken-pod-scenarios] "kernel_version": "5.14.0-284.73.1.el9_2.x86_64",
[chaos:kraken-pod-scenarios] "kubelet_version": "v1.28.11+add48d0",
[chaos:kraken-pod-scenarios] "os_version": "Red Hat Enterprise Linux CoreOS 415.92.202407031156-0 (Plow)"
[chaos:kraken-pod-scenarios] },
[chaos:kraken-pod-scenarios] {
[chaos:kraken-pod-scenarios] "taints": [],
[chaos:kraken-pod-scenarios] "instance_type": "unknown",
[chaos:kraken-pod-scenarios] "node_type": "worker",
[chaos:kraken-pod-scenarios] "name": "ip-10-0-12-5.ec2.internal",
[chaos:kraken-pod-scenarios] "architecture": "amd64",
[chaos:kraken-pod-scenarios] "kernel_version": "5.14.0-284.73.1.el9_2.x86_64",
[chaos:kraken-pod-scenarios] "kubelet_version": "v1.28.11+add48d0",
[chaos:kraken-pod-scenarios] "os_version": "Red Hat Enterprise Linux CoreOS 415.92.202407031156-0 (Plow)"
[chaos:kraken-pod-scenarios] },
[chaos:kraken-pod-scenarios] {
[chaos:kraken-pod-scenarios] "taints": [],
[chaos:kraken-pod-scenarios] "instance_type": "unknown",
[chaos:kraken-pod-scenarios] "node_type": "worker",
[chaos:kraken-pod-scenarios] "name": "ip-10-0-12-7.ec2.internal",
[chaos:kraken-pod-scenarios] "architecture": "amd64",
[chaos:kraken-pod-scenarios] "kernel_version": "5.14.0-284.73.1.el9_2.x86_64",
[chaos:kraken-pod-scenarios] "kubelet_version": "v1.28.11+add48d0",
[chaos:kraken-pod-scenarios] "os_version": "Red Hat Enterprise Linux CoreOS 415.92.202407031156-0 (Plow)"
[chaos:kraken-pod-scenarios] }
[chaos:kraken-pod-scenarios] ],
[chaos:kraken-pod-scenarios] "kubernetes_objects_count": {
[chaos:kraken-pod-scenarios] "ConfigMap": 313,
[chaos:kraken-pod-scenarios] "Pod": 156,
[chaos:kraken-pod-scenarios] "Secret": 932,
[chaos:kraken-pod-scenarios] "Deployment": 38,
[chaos:kraken-pod-scenarios] "Route": 14,
[chaos:kraken-pod-scenarios] "Build": 1
[chaos:kraken-pod-scenarios] },
[chaos:kraken-pod-scenarios] "network_plugins": [
[chaos:kraken-pod-scenarios] "OVNKubernetes"
[chaos:kraken-pod-scenarios] ],
[chaos:kraken-pod-scenarios] "node_count": 5,
[chaos:kraken-pod-scenarios] "cloud_infrastructure": "AWS",
[chaos:kraken-pod-scenarios] "run_uuid": "44039d70-0f8b-4c30-a697-ab93a3c357db"
[chaos:kraken-pod-scenarios] }
[chaos:kraken-pod-scenarios] 2024-08-15 14:33:08,220 [INFO] telemetry collection disabled, skipping.
[chaos:kraken-pod-scenarios] 2024-08-15 14:33:08,220 [ERROR] Post scenarios are still failing at the end of all iterations
[chaos:kraken-pod-scenarios] _ _
[chaos:kraken-pod-scenarios] | | ___ __ __ _| | _____ _ __
[chaos:kraken-pod-scenarios] | |/ / '__/ _` | |/ / _ \ '_ \
[chaos:kraken-pod-scenarios] | <| | | (_| | < __/ | | |
[chaos:kraken-pod-scenarios] |_|\_\_| \__,_|_|\_\___|_| |_|
[chaos:kraken-pod-scenarios]
[chaos:kraken-pod-scenarios]
task: Failed to run task "chaos:kill-gossip-router": exit status 1
Version
keycloak-benchmark latest main
Expected behavior
No response
Actual behavior
No response
How to Reproduce?
Run the below command against a ROSA based Keycloak cluster from the provision/rosa-cross-dc` directory
task chaos:kill-gossip-router
Anything else?
No response
Activity