Job stuck in waiting for resource
Summary
Jobs get intermittently stuck in waiting for resource
Steps to reproduce
Unknown
Example Project
What is the current bug behavior?
The job got stuck in waiting for resource
What is the expected correct behavior?
The job would have been enqueued correctly.
Relevant logs and/or screenshots
Logs from Loki with this query:
{release="gitlab"} |= `` | json | line_format `{{.app}} | {{.message}} | {{.deduplication_options_if_deduplicated}} {{.duplicate_of}}` | class = `Ci::ResourceGroups::AssignResourceFromResourceGroupWorker`
In the logs, we can see that the jobs 7c4a8766e4ee058ae9b0e95d and dd37d45903003feb68e6e84a were deduplicated.
86cfc1a93b30741a83dc5fb8 (the job which was duplicated by dd37d45903003feb68e6e84a) was rescheduled just fine and the resource group relating to this job was not stuck.
However, we can see that the job c8e98e53d37e2a4721d7d92e (the job which was duplicated by 7c4a8766e4ee058ae9b0e95d) was not rescheduled.
The resource group relating to this job was stuck and the CI Job did not exit waiting_for_resource.
Canceling the CI job and retrying it unlocked the resource group again.
Full Logs
[
{
"line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-84f64c33182713312566cc8c: done: 0.034945 sec | ",
"timestamp": "1691011441709727487",
"fields": {
"Line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-84f64c33182713312566cc8c: done: 0.034945 sec | ",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-zllx2_gitlab_sidekiq-5ef7a7497f7769a88fff588641bb0fa96074a5ce8681baca4cc91b924eca0c84.log",
"app": "sidekiq",
"chart": "sidekiq-7.2.2",
"class": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker",
"completed_at": "2023-08-02T21:24:01.708Z",
"correlation_id": "01H6W25R6DVN24MQM65E1BQKH0",
"cpu_s": "0.012574",
"created_at": "2023-08-02T21:24:01.669Z",
"db_cached_count": "0",
"db_ci_cached_count": "0",
"db_ci_count": "4",
"db_ci_duration_s": "0.015",
"db_ci_replica_cached_count": "0",
"db_ci_replica_count": "0",
"db_ci_replica_duration_s": "0.0",
"db_ci_replica_wal_cached_count": "0",
"db_ci_replica_wal_count": "0",
"db_ci_wal_cached_count": "0",
"db_ci_wal_count": "0",
"db_count": "5",
"db_duration_s": "0.017203",
"db_main_cached_count": "0",
"db_main_count": "1",
"db_main_duration_s": "0.002",
"db_main_replica_cached_count": "0",
"db_main_replica_count": "0",
"db_main_replica_duration_s": "0.0",
"db_main_replica_wal_cached_count": "0",
"db_main_replica_wal_count": "0",
"db_main_wal_cached_count": "0",
"db_main_wal_count": "0",
"db_primary_cached_count": "0",
"db_primary_count": "5",
"db_primary_duration_s": "0.017",
"db_primary_wal_cached_count": "0",
"db_primary_wal_count": "0",
"db_replica_cached_count": "0",
"db_replica_count": "0",
"db_replica_duration_s": "0.0",
"db_replica_wal_cached_count": "0",
"db_replica_wal_count": "0",
"db_write_count": "0",
"duration_s": "0.034945",
"enqueued_at": "2023-08-02T21:24:01.671Z",
"environment": "aws",
"heritage": "Helm",
"idempotency_key": "resque:gitlab:duplicate:default:dece5d0ccd847f28f07fd9918d9a315955d505a95d71ec696d50b9394b3bd498",
"jid": "84f64c33182713312566cc8c",
"job_size_bytes": "5",
"job_status": "done",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "5ef7a7497f7769a88fff588641bb0fa96074a5ce8681baca4cc91b924eca0c84",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "74d4cc83-b6f2-48e7-9e9e-211e37899354",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-zllx2",
"load_balancing_strategy": "primary",
"logtag": "F",
"mem_bytes": "356984",
"mem_mallocs": "1742",
"mem_objects": "4680",
"mem_total_bytes": "544184",
"message": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-84f64c33182713312566cc8c: done: 0.034945 sec",
"meta_caller_id": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker",
"meta_client_id": "user/35",
"meta_feature_category": "continuous_delivery",
"meta_job_id": "411678",
"meta_pipeline_id": "41693",
"meta_project": "[FILTERED]",
"meta_remote_ip": "[FILTERED]",
"meta_root_caller_id": "PUT /api/:version/jobs/:id",
"meta_root_namespace": "[FILTERED]",
"meta_user": "[FILTERED]",
"meta_user_id": "35",
"namespace": "gitlab",
"pid": "25",
"pod_template_hash": "666cd666cc",
"queue": "default",
"queue_namespace": "pipeline_processing",
"queue_pod_name": "all-in-1",
"redis_calls": "4",
"redis_duration_s": "0.002441",
"redis_feature_flag_calls": "1",
"redis_feature_flag_duration_s": "0.00065",
"redis_feature_flag_read_bytes": "199",
"redis_feature_flag_write_bytes": "110",
"redis_queues_calls": "3",
"redis_queues_duration_s": "0.001791",
"redis_queues_read_bytes": "80",
"redis_queues_write_bytes": "309",
"redis_read_bytes": "279",
"redis_write_bytes": "419",
"release": "gitlab",
"retry": "3",
"scheduling_latency_s": "0.002399",
"severity": "INFO",
"size_limiter": "validated",
"stream": "stdout",
"target_duration_s": "300",
"time": "2023-08-02T21:24:01.708Z",
"urgency": "low",
"version": "0",
"worker_data_consistency": "always",
"worker_id": "sidekiq_0"
}
},
{
"line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-86cfc1a93b30741a83dc5fb8: done: 0.487941 sec | ",
"timestamp": "1691011441680037879",
"fields": {
"Line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-86cfc1a93b30741a83dc5fb8: done: 0.487941 sec | ",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-x8664_gitlab_sidekiq-a509deaa5f98b2a323bf399c69bb507d7e1d8e3abb99cb6b2df6f8b3c8ea1ab7.log",
"app": "sidekiq",
"chart": "sidekiq-7.2.2",
"class": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker",
"completed_at": "2023-08-02T21:24:01.679Z",
"correlation_id": "01H6W25R6DVN24MQM65E1BQKH0",
"cpu_s": "0.081208",
"created_at": "2023-08-02T21:24:00.995Z",
"db_cached_count": "1",
"db_ci_cached_count": "0",
"db_ci_count": "14",
"db_ci_duration_s": "0.028",
"db_ci_replica_cached_count": "0",
"db_ci_replica_count": "0",
"db_ci_replica_duration_s": "0.0",
"db_ci_replica_wal_cached_count": "0",
"db_ci_replica_wal_count": "0",
"db_ci_wal_cached_count": "0",
"db_ci_wal_count": "0",
"db_count": "33",
"db_duration_s": "0.065748",
"db_main_cached_count": "1",
"db_main_count": "19",
"db_main_duration_s": "0.033",
"db_main_replica_cached_count": "0",
"db_main_replica_count": "0",
"db_main_replica_duration_s": "0.0",
"db_main_replica_wal_cached_count": "0",
"db_main_replica_wal_count": "0",
"db_main_wal_cached_count": "0",
"db_main_wal_count": "0",
"db_primary_cached_count": "1",
"db_primary_count": "33",
"db_primary_duration_s": "0.061",
"db_primary_wal_cached_count": "0",
"db_primary_wal_count": "0",
"db_replica_cached_count": "0",
"db_replica_count": "0",
"db_replica_duration_s": "0.0",
"db_replica_wal_cached_count": "0",
"db_replica_wal_count": "0",
"db_write_count": "3",
"duration_s": "0.487941",
"enqueued_at": "2023-08-02T21:24:00.997Z",
"environment": "aws",
"gitaly_calls": "1",
"gitaly_duration_s": "0.00676",
"heritage": "Helm",
"idempotency_key": "resque:gitlab:duplicate:default:dece5d0ccd847f28f07fd9918d9a315955d505a95d71ec696d50b9394b3bd498",
"jid": "86cfc1a93b30741a83dc5fb8",
"job_size_bytes": "5",
"job_status": "done",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "a509deaa5f98b2a323bf399c69bb507d7e1d8e3abb99cb6b2df6f8b3c8ea1ab7",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "546ce723-9d8e-4720-a106-197269207b9f",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-x8664",
"load_balancing_strategy": "primary",
"logtag": "F",
"mem_bytes": "1860864",
"mem_mallocs": "9535",
"mem_objects": "28077",
"mem_total_bytes": "2983944",
"message": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-86cfc1a93b30741a83dc5fb8: done: 0.487941 sec",
"meta_caller_id": "PUT /api/:version/jobs/:id",
"meta_client_id": "user/35",
"meta_feature_category": "continuous_delivery",
"meta_job_id": "411678",
"meta_pipeline_id": "41693",
"meta_project": "[FILTERED]",
"meta_remote_ip": "[FILTERED]",
"meta_root_caller_id": "PUT /api/:version/jobs/:id",
"meta_root_namespace": "[FILTERED]",
"meta_user": "[FILTERED]",
"meta_user_id": "35",
"namespace": "gitlab",
"pid": "24",
"pod_template_hash": "666cd666cc",
"queue": "default",
"queue_namespace": "pipeline_processing",
"queue_pod_name": "all-in-1",
"redis_calls": "14",
"redis_duration_s": "0.020912",
"redis_feature_flag_calls": "1",
"redis_feature_flag_duration_s": "0.000682",
"redis_feature_flag_read_bytes": "199",
"redis_feature_flag_write_bytes": "59",
"redis_queues_calls": "12",
"redis_queues_duration_s": "0.013938",
"redis_queues_read_bytes": "107",
"redis_queues_write_bytes": "3764",
"redis_read_bytes": "308",
"redis_shared_state_calls": "1",
"redis_shared_state_duration_s": "0.006292",
"redis_shared_state_read_bytes": "2",
"redis_shared_state_write_bytes": "91",
"redis_write_bytes": "3914",
"release": "gitlab",
"retry": "3",
"scheduling_latency_s": "0.1942",
"severity": "INFO",
"size_limiter": "validated",
"stream": "stdout",
"target_duration_s": "300",
"time": "2023-08-02T21:24:01.679Z",
"urgency": "low",
"version": "0",
"worker_data_consistency": "always",
"worker_id": "sidekiq_0"
}
},
{
"line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-84f64c33182713312566cc8c: start | ",
"timestamp": "1691011441674590864",
"fields": {
"Line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-84f64c33182713312566cc8c: start | ",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-zllx2_gitlab_sidekiq-5ef7a7497f7769a88fff588641bb0fa96074a5ce8681baca4cc91b924eca0c84.log",
"app": "sidekiq",
"chart": "sidekiq-7.2.2",
"class": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker",
"correlation_id": "01H6W25R6DVN24MQM65E1BQKH0",
"created_at": "2023-08-02T21:24:01.669Z",
"enqueued_at": "2023-08-02T21:24:01.671Z",
"environment": "aws",
"heritage": "Helm",
"idempotency_key": "resque:gitlab:duplicate:default:dece5d0ccd847f28f07fd9918d9a315955d505a95d71ec696d50b9394b3bd498",
"jid": "84f64c33182713312566cc8c",
"job_size_bytes": "5",
"job_status": "start",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "5ef7a7497f7769a88fff588641bb0fa96074a5ce8681baca4cc91b924eca0c84",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "74d4cc83-b6f2-48e7-9e9e-211e37899354",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-zllx2",
"logtag": "F",
"message": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-84f64c33182713312566cc8c: start",
"meta_caller_id": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker",
"meta_client_id": "user/35",
"meta_feature_category": "continuous_delivery",
"meta_job_id": "411678",
"meta_pipeline_id": "41693",
"meta_project": "[FILTERED]",
"meta_remote_ip": "[FILTERED]",
"meta_root_caller_id": "PUT /api/:version/jobs/:id",
"meta_root_namespace": "[FILTERED]",
"meta_user": "[FILTERED]",
"meta_user_id": "35",
"namespace": "gitlab",
"pid": "25",
"pod_template_hash": "666cd666cc",
"queue": "default",
"queue_namespace": "pipeline_processing",
"queue_pod_name": "all-in-1",
"release": "gitlab",
"retry": "3",
"scheduling_latency_s": "0.002399",
"severity": "INFO",
"size_limiter": "validated",
"stream": "stdout",
"time": "2023-08-02T21:24:01.674Z",
"version": "0",
"worker_data_consistency": "always"
}
},
{
"line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-86cfc1a93b30741a83dc5fb8: rescheduled | ",
"timestamp": "1691011441669246933",
"fields": {
"Line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-86cfc1a93b30741a83dc5fb8: rescheduled | ",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-x8664_gitlab_sidekiq-a509deaa5f98b2a323bf399c69bb507d7e1d8e3abb99cb6b2df6f8b3c8ea1ab7.log",
"app": "sidekiq",
"chart": "sidekiq-7.2.2",
"class": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker",
"correlation_id": "01H6W25R6DVN24MQM65E1BQKH0",
"created_at": "2023-08-02T21:24:00.995Z",
"enqueued_at": "2023-08-02T21:24:00.997Z",
"environment": "aws",
"heritage": "Helm",
"idempotency_key": "resque:gitlab:duplicate:default:dece5d0ccd847f28f07fd9918d9a315955d505a95d71ec696d50b9394b3bd498",
"jid": "86cfc1a93b30741a83dc5fb8",
"job_size_bytes": "5",
"job_status": "rescheduled",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "a509deaa5f98b2a323bf399c69bb507d7e1d8e3abb99cb6b2df6f8b3c8ea1ab7",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "546ce723-9d8e-4720-a106-197269207b9f",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-x8664",
"load_balancing_strategy": "primary",
"logtag": "F",
"message": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-86cfc1a93b30741a83dc5fb8: rescheduled",
"meta_caller_id": "PUT /api/:version/jobs/:id",
"meta_client_id": "user/35",
"meta_feature_category": "continuous_delivery",
"meta_job_id": "411678",
"meta_pipeline_id": "41693",
"meta_project": "[FILTERED]",
"meta_remote_ip": "[FILTERED]",
"meta_root_caller_id": "PUT /api/:version/jobs/:id",
"meta_root_namespace": "[FILTERED]",
"meta_user": "[FILTERED]",
"meta_user_id": "35",
"namespace": "gitlab",
"pid": "24",
"pod_template_hash": "666cd666cc",
"queue": "default",
"queue_namespace": "pipeline_processing",
"queue_pod_name": "all-in-1",
"release": "gitlab",
"retry": "3",
"severity": "INFO",
"size_limiter": "validated",
"stream": "stdout",
"time": "2023-08-02T21:24:01.668Z",
"version": "0",
"worker_data_consistency": "always"
}
},
{
"line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-86cfc1a93b30741a83dc5fb8: start | ",
"timestamp": "1691011441387787738",
"fields": {
"Line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-86cfc1a93b30741a83dc5fb8: start | ",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-x8664_gitlab_sidekiq-a509deaa5f98b2a323bf399c69bb507d7e1d8e3abb99cb6b2df6f8b3c8ea1ab7.log",
"app": "sidekiq",
"chart": "sidekiq-7.2.2",
"class": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker",
"correlation_id": "01H6W25R6DVN24MQM65E1BQKH0",
"created_at": "2023-08-02T21:24:00.995Z",
"enqueued_at": "2023-08-02T21:24:00.997Z",
"environment": "aws",
"heritage": "Helm",
"idempotency_key": "resque:gitlab:duplicate:default:dece5d0ccd847f28f07fd9918d9a315955d505a95d71ec696d50b9394b3bd498",
"jid": "86cfc1a93b30741a83dc5fb8",
"job_size_bytes": "5",
"job_status": "start",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "a509deaa5f98b2a323bf399c69bb507d7e1d8e3abb99cb6b2df6f8b3c8ea1ab7",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "546ce723-9d8e-4720-a106-197269207b9f",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-x8664",
"logtag": "F",
"message": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-86cfc1a93b30741a83dc5fb8: start",
"meta_caller_id": "PUT /api/:version/jobs/:id",
"meta_client_id": "user/35",
"meta_feature_category": "continuous_delivery",
"meta_job_id": "411678",
"meta_pipeline_id": "41693",
"meta_project": "[FILTERED]",
"meta_remote_ip": "[FILTERED]",
"meta_root_caller_id": "PUT /api/:version/jobs/:id",
"meta_root_namespace": "[FILTERED]",
"meta_user": "[FILTERED]",
"meta_user_id": "35",
"namespace": "gitlab",
"pid": "24",
"pod_template_hash": "666cd666cc",
"queue": "default",
"queue_namespace": "pipeline_processing",
"queue_pod_name": "all-in-1",
"release": "gitlab",
"retry": "3",
"scheduling_latency_s": "0.1942",
"severity": "INFO",
"size_limiter": "validated",
"stream": "stdout",
"time": "2023-08-02T21:24:01.191Z",
"version": "0",
"worker_data_consistency": "always"
}
},
{
"line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-dd37d45903003feb68e6e84a: deduplicated: until executed | reschedule_once 86cfc1a93b30741a83dc5fb8",
"timestamp": "1691011441081767232",
"fields": {
"Line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-dd37d45903003feb68e6e84a: deduplicated: until executed | reschedule_once 86cfc1a93b30741a83dc5fb8",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-zllx2_gitlab_sidekiq-5ef7a7497f7769a88fff588641bb0fa96074a5ce8681baca4cc91b924eca0c84.log",
"app": "sidekiq",
"chart": "sidekiq-7.2.2",
"class": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker",
"correlation_id": "01H6W25R6DVN24MQM65E1BQKH0",
"created_at": "2023-08-02T21:24:01.060Z",
"deduplication_options_if_deduplicated": "reschedule_once",
"deduplication_type": "until executed",
"duplicate_of": "86cfc1a93b30741a83dc5fb8",
"environment": "aws",
"heritage": "Helm",
"idempotency_key": "resque:gitlab:duplicate:default:dece5d0ccd847f28f07fd9918d9a315955d505a95d71ec696d50b9394b3bd498",
"jid": "dd37d45903003feb68e6e84a",
"job_size_bytes": "5",
"job_status": "deduplicated",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "5ef7a7497f7769a88fff588641bb0fa96074a5ce8681baca4cc91b924eca0c84",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "74d4cc83-b6f2-48e7-9e9e-211e37899354",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-zllx2",
"logtag": "F",
"message": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-dd37d45903003feb68e6e84a: deduplicated: until executed",
"meta_caller_id": "PipelineProcessWorker",
"meta_client_id": "user/35",
"meta_feature_category": "continuous_delivery",
"meta_job_id": "411678",
"meta_pipeline_id": "41693",
"meta_project": "[FILTERED]",
"meta_remote_ip": "[FILTERED]",
"meta_root_caller_id": "PUT /api/:version/jobs/:id",
"meta_root_namespace": "[FILTERED]",
"meta_user": "[FILTERED]",
"meta_user_id": "35",
"namespace": "gitlab",
"pid": "25",
"pod_template_hash": "666cd666cc",
"queue": "default",
"queue_namespace": "pipeline_processing",
"queue_pod_name": "all-in-1",
"release": "gitlab",
"retry": "3",
"severity": "INFO",
"stream": "stdout",
"time": "2023-08-02T21:24:01.081Z",
"version": "0",
"worker_data_consistency": "always"
}
},
{
"line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-c8e98e53d37e2a4721d7d92e: done: 0.091843 sec | ",
"timestamp": "1691011436574996922",
"fields": {
"Line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-c8e98e53d37e2a4721d7d92e: done: 0.091843 sec | ",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-fq9zs_gitlab_sidekiq-7a6d2f7636e6566a460286abbaade42c663df4b9574170517283b4a550694278.log",
"app": "sidekiq",
"chart": "sidekiq-7.2.2",
"class": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker",
"completed_at": "2023-08-02T21:23:56.574Z",
"correlation_id": "01H6W25KQRMGE65S29371ZSYYV",
"cpu_s": "0.013707",
"created_at": "2023-08-02T21:23:56.464Z",
"db_cached_count": "0",
"db_ci_cached_count": "0",
"db_ci_count": "4",
"db_ci_duration_s": "0.022",
"db_ci_replica_cached_count": "0",
"db_ci_replica_count": "0",
"db_ci_replica_duration_s": "0.0",
"db_ci_replica_wal_cached_count": "0",
"db_ci_replica_wal_count": "0",
"db_ci_wal_cached_count": "0",
"db_ci_wal_count": "0",
"db_count": "5",
"db_duration_s": "0.02758",
"db_main_cached_count": "0",
"db_main_count": "1",
"db_main_duration_s": "0.006",
"db_main_replica_cached_count": "0",
"db_main_replica_count": "0",
"db_main_replica_duration_s": "0.0",
"db_main_replica_wal_cached_count": "0",
"db_main_replica_wal_count": "0",
"db_main_wal_cached_count": "0",
"db_main_wal_count": "0",
"db_primary_cached_count": "0",
"db_primary_count": "5",
"db_primary_duration_s": "0.028",
"db_primary_wal_cached_count": "0",
"db_primary_wal_count": "0",
"db_replica_cached_count": "0",
"db_replica_count": "0",
"db_replica_duration_s": "0.0",
"db_replica_wal_cached_count": "0",
"db_replica_wal_count": "0",
"db_write_count": "0",
"duration_s": "0.09184299999999999",
"enqueued_at": "2023-08-02T21:23:56.467Z",
"environment": "aws",
"heritage": "Helm",
"idempotency_key": "resque:gitlab:duplicate:default:442c81c35901275caca3e6fed7e3b10df8e1a23c234c2bf06c734340c783b1af",
"jid": "c8e98e53d37e2a4721d7d92e",
"job_size_bytes": "5",
"job_status": "done",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "7a6d2f7636e6566a460286abbaade42c663df4b9574170517283b4a550694278",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "c18fefdd-ec8f-4f2f-ae1d-fe4abeed6371",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-fq9zs",
"load_balancing_strategy": "primary",
"logtag": "F",
"mem_bytes": "306168",
"mem_mallocs": "1662",
"mem_objects": "4472",
"mem_total_bytes": "485048",
"message": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-c8e98e53d37e2a4721d7d92e: done: 0.091843 sec",
"meta_caller_id": "PUT /api/:version/jobs/:id",
"meta_client_id": "user/35",
"meta_feature_category": "continuous_delivery",
"meta_job_id": "411677",
"meta_pipeline_id": "41693",
"meta_project": "[FILTERED]",
"meta_remote_ip": "[FILTERED]",
"meta_root_caller_id": "PUT /api/:version/jobs/:id",
"meta_root_namespace": "[FILTERED]",
"meta_user": "[FILTERED]",
"meta_user_id": "35",
"namespace": "gitlab",
"pid": "25",
"pod_template_hash": "666cd666cc",
"queue": "default",
"queue_namespace": "pipeline_processing",
"queue_pod_name": "all-in-1",
"redis_calls": "3",
"redis_duration_s": "0.007792",
"redis_queues_calls": "3",
"redis_queues_duration_s": "0.007792",
"redis_queues_read_bytes": "80",
"redis_queues_write_bytes": "309",
"redis_read_bytes": "80",
"redis_write_bytes": "309",
"release": "gitlab",
"retry": "3",
"scheduling_latency_s": "0.015541",
"severity": "INFO",
"size_limiter": "validated",
"stream": "stdout",
"target_duration_s": "300",
"time": "2023-08-02T21:23:56.574Z",
"urgency": "low",
"version": "0",
"worker_data_consistency": "always",
"worker_id": "sidekiq_0"
}
},
{
"line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-7c4a8766e4ee058ae9b0e95d: deduplicated: until executed | reschedule_once c8e98e53d37e2a4721d7d92e",
"timestamp": "1691011436567311922",
"fields": {
"Line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-7c4a8766e4ee058ae9b0e95d: deduplicated: until executed | reschedule_once c8e98e53d37e2a4721d7d92e",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-zllx2_gitlab_sidekiq-5ef7a7497f7769a88fff588641bb0fa96074a5ce8681baca4cc91b924eca0c84.log",
"app": "sidekiq",
"chart": "sidekiq-7.2.2",
"class": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker",
"correlation_id": "01H6W25KK91GA0BBNQBMVFB7DC",
"created_at": "2023-08-02T21:23:56.564Z",
"deduplication_options_if_deduplicated": "reschedule_once",
"deduplication_type": "until executed",
"duplicate_of": "c8e98e53d37e2a4721d7d92e",
"environment": "aws",
"heritage": "Helm",
"idempotency_key": "resque:gitlab:duplicate:default:442c81c35901275caca3e6fed7e3b10df8e1a23c234c2bf06c734340c783b1af",
"jid": "7c4a8766e4ee058ae9b0e95d",
"job_size_bytes": "5",
"job_status": "deduplicated",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "5ef7a7497f7769a88fff588641bb0fa96074a5ce8681baca4cc91b924eca0c84",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "74d4cc83-b6f2-48e7-9e9e-211e37899354",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-zllx2",
"logtag": "F",
"message": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-7c4a8766e4ee058ae9b0e95d: deduplicated: until executed",
"meta_caller_id": "PipelineProcessWorker",
"meta_client_id": "user/35",
"meta_feature_category": "continuous_delivery",
"meta_job_id": "411675",
"meta_pipeline_id": "41693",
"meta_project": "[FILTERED]",
"meta_remote_ip": "[FILTERED]",
"meta_root_caller_id": "PUT /api/:version/jobs/:id",
"meta_root_namespace": "[FILTERED]",
"meta_user": "[FILTERED]",
"meta_user_id": "35",
"namespace": "gitlab",
"pid": "25",
"pod_template_hash": "666cd666cc",
"queue": "default",
"queue_namespace": "pipeline_processing",
"queue_pod_name": "all-in-1",
"release": "gitlab",
"retry": "3",
"severity": "INFO",
"stream": "stdout",
"time": "2023-08-02T21:23:56.566Z",
"version": "0",
"worker_data_consistency": "always"
}
},
{
"line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-a687429d39922a59e6ea4e7c: done: 0.124154 sec | ",
"timestamp": "1691011436514541959",
"fields": {
"Line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-a687429d39922a59e6ea4e7c: done: 0.124154 sec | ",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-x8664_gitlab_sidekiq-a509deaa5f98b2a323bf399c69bb507d7e1d8e3abb99cb6b2df6f8b3c8ea1ab7.log",
"app": "sidekiq",
"chart": "sidekiq-7.2.2",
"class": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker",
"completed_at": "2023-08-02T21:23:56.513Z",
"correlation_id": "01H6W25KK91GA0BBNQBMVFB7DC",
"cpu_s": "0.07442799999999999",
"created_at": "2023-08-02T21:23:56.382Z",
"db_cached_count": "1",
"db_ci_cached_count": "0",
"db_ci_count": "14",
"db_ci_duration_s": "0.012",
"db_ci_replica_cached_count": "0",
"db_ci_replica_count": "0",
"db_ci_replica_duration_s": "0.0",
"db_ci_replica_wal_cached_count": "0",
"db_ci_replica_wal_count": "0",
"db_ci_wal_cached_count": "0",
"db_ci_wal_count": "0",
"db_count": "33",
"db_duration_s": "0.038187",
"db_main_cached_count": "1",
"db_main_count": "19",
"db_main_duration_s": "0.026",
"db_main_replica_cached_count": "0",
"db_main_replica_count": "0",
"db_main_replica_duration_s": "0.0",
"db_main_replica_wal_cached_count": "0",
"db_main_replica_wal_count": "0",
"db_main_wal_cached_count": "0",
"db_main_wal_count": "0",
"db_primary_cached_count": "1",
"db_primary_count": "33",
"db_primary_duration_s": "0.038",
"db_primary_wal_cached_count": "0",
"db_primary_wal_count": "0",
"db_replica_cached_count": "0",
"db_replica_count": "0",
"db_replica_duration_s": "0.0",
"db_replica_wal_cached_count": "0",
"db_replica_wal_count": "0",
"db_write_count": "3",
"duration_s": "0.124154",
"enqueued_at": "2023-08-02T21:23:56.387Z",
"environment": "aws",
"gitaly_calls": "1",
"gitaly_duration_s": "0.008251",
"heritage": "Helm",
"idempotency_key": "resque:gitlab:duplicate:default:74aae84e74e26af9a6fd285135b7b5a0412f456143ea729189c18a2acc021a06",
"jid": "a687429d39922a59e6ea4e7c",
"job_size_bytes": "5",
"job_status": "done",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "a509deaa5f98b2a323bf399c69bb507d7e1d8e3abb99cb6b2df6f8b3c8ea1ab7",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "546ce723-9d8e-4720-a106-197269207b9f",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-x8664",
"load_balancing_strategy": "primary",
"logtag": "F",
"mem_bytes": "1923504",
"mem_mallocs": "9456",
"mem_objects": "27768",
"mem_total_bytes": "3034224",
"message": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-a687429d39922a59e6ea4e7c: done: 0.124154 sec",
"meta_caller_id": "PipelineProcessWorker",
"meta_client_id": "user/35",
"meta_feature_category": "continuous_delivery",
"meta_job_id": "411675",
"meta_pipeline_id": "41693",
"meta_project": "[FILTERED]",
"meta_remote_ip": "[FILTERED]",
"meta_root_caller_id": "PUT /api/:version/jobs/:id",
"meta_root_namespace": "[FILTERED]",
"meta_user": "[FILTERED]",
"meta_user_id": "35",
"namespace": "gitlab",
"pid": "24",
"pod_template_hash": "666cd666cc",
"queue": "default",
"queue_namespace": "pipeline_processing",
"queue_pod_name": "all-in-1",
"redis_calls": "13",
"redis_duration_s": "0.009565000000000001",
"redis_feature_flag_calls": "3",
"redis_feature_flag_duration_s": "0.002993",
"redis_feature_flag_read_bytes": "595",
"redis_feature_flag_write_bytes": "180",
"redis_queues_calls": "9",
"redis_queues_duration_s": "0.005889",
"redis_queues_read_bytes": "177",
"redis_queues_write_bytes": "2163",
"redis_read_bytes": "774",
"redis_shared_state_calls": "1",
"redis_shared_state_duration_s": "0.000683",
"redis_shared_state_read_bytes": "2",
"redis_shared_state_write_bytes": "91",
"redis_write_bytes": "2434",
"release": "gitlab",
"retry": "3",
"scheduling_latency_s": "0.002109",
"severity": "INFO",
"size_limiter": "validated",
"stream": "stdout",
"target_duration_s": "300",
"time": "2023-08-02T21:23:56.513Z",
"urgency": "low",
"version": "0",
"worker_data_consistency": "always",
"worker_id": "sidekiq_0"
}
},
{
"line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-c8e98e53d37e2a4721d7d92e: start | ",
"timestamp": "1691011436483657412",
"fields": {
"Line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-c8e98e53d37e2a4721d7d92e: start | ",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-fq9zs_gitlab_sidekiq-7a6d2f7636e6566a460286abbaade42c663df4b9574170517283b4a550694278.log",
"app": "sidekiq",
"chart": "sidekiq-7.2.2",
"class": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker",
"correlation_id": "01H6W25KQRMGE65S29371ZSYYV",
"created_at": "2023-08-02T21:23:56.464Z",
"enqueued_at": "2023-08-02T21:23:56.467Z",
"environment": "aws",
"heritage": "Helm",
"idempotency_key": "resque:gitlab:duplicate:default:442c81c35901275caca3e6fed7e3b10df8e1a23c234c2bf06c734340c783b1af",
"jid": "c8e98e53d37e2a4721d7d92e",
"job_size_bytes": "5",
"job_status": "start",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "7a6d2f7636e6566a460286abbaade42c663df4b9574170517283b4a550694278",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "c18fefdd-ec8f-4f2f-ae1d-fe4abeed6371",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-fq9zs",
"logtag": "F",
"message": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-c8e98e53d37e2a4721d7d92e: start",
"meta_caller_id": "PUT /api/:version/jobs/:id",
"meta_client_id": "user/35",
"meta_feature_category": "continuous_delivery",
"meta_job_id": "411677",
"meta_pipeline_id": "41693",
"meta_project": "[FILTERED]",
"meta_remote_ip": "[FILTERED]",
"meta_root_caller_id": "PUT /api/:version/jobs/:id",
"meta_root_namespace": "[FILTERED]",
"meta_user": "[FILTERED]",
"meta_user_id": "35",
"namespace": "gitlab",
"pid": "25",
"pod_template_hash": "666cd666cc",
"queue": "default",
"queue_namespace": "pipeline_processing",
"queue_pod_name": "all-in-1",
"release": "gitlab",
"retry": "3",
"scheduling_latency_s": "0.015541",
"severity": "INFO",
"size_limiter": "validated",
"stream": "stdout",
"time": "2023-08-02T21:23:56.482Z",
"version": "0",
"worker_data_consistency": "always"
}
},
{
"line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-a687429d39922a59e6ea4e7c: start | ",
"timestamp": "1691011436389962367",
"fields": {
"Line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-a687429d39922a59e6ea4e7c: start | ",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-x8664_gitlab_sidekiq-a509deaa5f98b2a323bf399c69bb507d7e1d8e3abb99cb6b2df6f8b3c8ea1ab7.log",
"app": "sidekiq",
"chart": "sidekiq-7.2.2",
"class": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker",
"correlation_id": "01H6W25KK91GA0BBNQBMVFB7DC",
"created_at": "2023-08-02T21:23:56.382Z",
"enqueued_at": "2023-08-02T21:23:56.387Z",
"environment": "aws",
"heritage": "Helm",
"idempotency_key": "resque:gitlab:duplicate:default:74aae84e74e26af9a6fd285135b7b5a0412f456143ea729189c18a2acc021a06",
"jid": "a687429d39922a59e6ea4e7c",
"job_size_bytes": "5",
"job_status": "start",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "a509deaa5f98b2a323bf399c69bb507d7e1d8e3abb99cb6b2df6f8b3c8ea1ab7",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "546ce723-9d8e-4720-a106-197269207b9f",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-x8664",
"logtag": "F",
"message": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-a687429d39922a59e6ea4e7c: start",
"meta_caller_id": "PipelineProcessWorker",
"meta_client_id": "user/35",
"meta_feature_category": "continuous_delivery",
"meta_job_id": "411675",
"meta_pipeline_id": "41693",
"meta_project": "[FILTERED]",
"meta_remote_ip": "[FILTERED]",
"meta_root_caller_id": "PUT /api/:version/jobs/:id",
"meta_root_namespace": "[FILTERED]",
"meta_user": "[FILTERED]",
"meta_user_id": "35",
"namespace": "gitlab",
"pid": "24",
"pod_template_hash": "666cd666cc",
"queue": "default",
"queue_namespace": "pipeline_processing",
"queue_pod_name": "all-in-1",
"release": "gitlab",
"retry": "3",
"scheduling_latency_s": "0.002109",
"severity": "INFO",
"size_limiter": "validated",
"stream": "stdout",
"time": "2023-08-02T21:23:56.389Z",
"version": "0",
"worker_data_consistency": "always"
}
},
{
"line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-b55ff4917f15c4cd5a7746e9: done: 0.052493 sec | ",
"timestamp": "1691011436378553860",
"fields": {
"Line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-b55ff4917f15c4cd5a7746e9: done: 0.052493 sec | ",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-x8664_gitlab_sidekiq-a509deaa5f98b2a323bf399c69bb507d7e1d8e3abb99cb6b2df6f8b3c8ea1ab7.log",
"app": "sidekiq",
"chart": "sidekiq-7.2.2",
"class": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker",
"completed_at": "2023-08-02T21:23:56.377Z",
"correlation_id": "01H6W25KK91GA0BBNQBMVFB7DC",
"cpu_s": "0.014862",
"created_at": "2023-08-02T21:23:56.318Z",
"db_cached_count": "0",
"db_ci_cached_count": "0",
"db_ci_count": "4",
"db_ci_duration_s": "0.007",
"db_ci_replica_cached_count": "0",
"db_ci_replica_count": "0",
"db_ci_replica_duration_s": "0.0",
"db_ci_replica_wal_cached_count": "0",
"db_ci_replica_wal_count": "0",
"db_ci_wal_cached_count": "0",
"db_ci_wal_count": "0",
"db_count": "5",
"db_duration_s": "0.009242",
"db_main_cached_count": "0",
"db_main_count": "1",
"db_main_duration_s": "0.003",
"db_main_replica_cached_count": "0",
"db_main_replica_count": "0",
"db_main_replica_duration_s": "0.0",
"db_main_replica_wal_cached_count": "0",
"db_main_replica_wal_count": "0",
"db_main_wal_cached_count": "0",
"db_main_wal_count": "0",
"db_primary_cached_count": "0",
"db_primary_count": "5",
"db_primary_duration_s": "0.008999999999999999",
"db_primary_wal_cached_count": "0",
"db_primary_wal_count": "0",
"db_replica_cached_count": "0",
"db_replica_count": "0",
"db_replica_duration_s": "0.0",
"db_replica_wal_cached_count": "0",
"db_replica_wal_count": "0",
"db_write_count": "0",
"duration_s": "0.052493",
"enqueued_at": "2023-08-02T21:23:56.320Z",
"environment": "aws",
"heritage": "Helm",
"idempotency_key": "resque:gitlab:duplicate:default:74aae84e74e26af9a6fd285135b7b5a0412f456143ea729189c18a2acc021a06",
"jid": "b55ff4917f15c4cd5a7746e9",
"job_size_bytes": "5",
"job_status": "done",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "a509deaa5f98b2a323bf399c69bb507d7e1d8e3abb99cb6b2df6f8b3c8ea1ab7",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "546ce723-9d8e-4720-a106-197269207b9f",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-x8664",
"load_balancing_strategy": "primary",
"logtag": "F",
"mem_bytes": "356984",
"mem_mallocs": "1742",
"mem_objects": "4679",
"mem_total_bytes": "544144",
"message": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-b55ff4917f15c4cd5a7746e9: done: 0.052493 sec",
"meta_caller_id": "PUT /api/:version/jobs/:id",
"meta_client_id": "user/35",
"meta_feature_category": "continuous_delivery",
"meta_job_id": "411675",
"meta_pipeline_id": "41693",
"meta_project": "[FILTERED]",
"meta_remote_ip": "[FILTERED]",
"meta_root_caller_id": "PUT /api/:version/jobs/:id",
"meta_root_namespace": "[FILTERED]",
"meta_user": "[FILTERED]",
"meta_user_id": "35",
"namespace": "gitlab",
"pid": "24",
"pod_template_hash": "666cd666cc",
"queue": "default",
"queue_namespace": "pipeline_processing",
"queue_pod_name": "all-in-1",
"redis_calls": "4",
"redis_duration_s": "0.024309",
"redis_feature_flag_calls": "1",
"redis_feature_flag_duration_s": "0.000674",
"redis_feature_flag_read_bytes": "199",
"redis_feature_flag_write_bytes": "110",
"redis_queues_calls": "3",
"redis_queues_duration_s": "0.023635",
"redis_queues_read_bytes": "80",
"redis_queues_write_bytes": "309",
"redis_read_bytes": "279",
"redis_write_bytes": "419",
"release": "gitlab",
"retry": "3",
"scheduling_latency_s": "0.004884",
"severity": "INFO",
"size_limiter": "validated",
"stream": "stdout",
"target_duration_s": "300",
"time": "2023-08-02T21:23:56.377Z",
"urgency": "low",
"version": "0",
"worker_data_consistency": "always",
"worker_id": "sidekiq_0"
}
},
{
"line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-b55ff4917f15c4cd5a7746e9: start | ",
"timestamp": "1691011436325960918",
"fields": {
"Line": "sidekiq | Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-b55ff4917f15c4cd5a7746e9: start | ",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-x8664_gitlab_sidekiq-a509deaa5f98b2a323bf399c69bb507d7e1d8e3abb99cb6b2df6f8b3c8ea1ab7.log",
"app": "sidekiq",
"chart": "sidekiq-7.2.2",
"class": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker",
"correlation_id": "01H6W25KK91GA0BBNQBMVFB7DC",
"created_at": "2023-08-02T21:23:56.318Z",
"enqueued_at": "2023-08-02T21:23:56.320Z",
"environment": "aws",
"heritage": "Helm",
"idempotency_key": "resque:gitlab:duplicate:default:74aae84e74e26af9a6fd285135b7b5a0412f456143ea729189c18a2acc021a06",
"jid": "b55ff4917f15c4cd5a7746e9",
"job_size_bytes": "5",
"job_status": "start",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "a509deaa5f98b2a323bf399c69bb507d7e1d8e3abb99cb6b2df6f8b3c8ea1ab7",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "546ce723-9d8e-4720-a106-197269207b9f",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-x8664",
"logtag": "F",
"message": "Ci::ResourceGroups::AssignResourceFromResourceGroupWorker JID-b55ff4917f15c4cd5a7746e9: start",
"meta_caller_id": "PUT /api/:version/jobs/:id",
"meta_client_id": "user/35",
"meta_feature_category": "continuous_delivery",
"meta_job_id": "411675",
"meta_pipeline_id": "41693",
"meta_project": "[FILTERED]",
"meta_remote_ip": "[FILTERED]",
"meta_root_caller_id": "PUT /api/:version/jobs/:id",
"meta_root_namespace": "[FILTERED]",
"meta_user": "[FILTERED]",
"meta_user_id": "35",
"namespace": "gitlab",
"pid": "24",
"pod_template_hash": "666cd666cc",
"queue": "default",
"queue_namespace": "pipeline_processing",
"queue_pod_name": "all-in-1",
"release": "gitlab",
"retry": "3",
"scheduling_latency_s": "0.004884",
"severity": "INFO",
"size_limiter": "validated",
"stream": "stdout",
"time": "2023-08-02T21:23:56.325Z",
"version": "0",
"worker_data_consistency": "always"
}
}
]Output of checks
Results of GitLab environment info
Expand for output related to GitLab environment info
System information System: Proxy: no Current User: git Using RVM: no Ruby Version: 3.0.6p216 Gem Version: 3.4.17 Bundler Version:2.4.17 Rake Version: 13.0.6 Redis Version: 6.0.16 Sidekiq Version:6.5.7 Go Version: unknown GitLab information Version: 16.2.2-ee Revision: 2e273e251ef Directory: /srv/gitlab DB Adapter: PostgreSQL DB Version: 14.7 URL: https://[FILTERED] HTTP Clone URL: https://[FILTERED]/some-group/some-project.git SSH Clone URL: git@[FILTERED]:some-group/some-project.git Elasticsearch: yes Geo: no Using LDAP: no Using Omniauth: yes Omniauth Providers: saml GitLab Shell Version: 14.23.0 Repository storages: - default: tcp://gitlab-gitaly-0.gitlab-gitaly.gitlab.svc:8075 GitLab Shell path: /home/git/gitlab-shell
Results of GitLab application Check
Expand for output related to the GitLab application check
(For installations with omnibus-gitlab package run and paste the output of:
sudo gitlab-rake gitlab:check SANITIZE=true)(For installations from source run and paste the output of:
sudo -u git -H bundle exec rake gitlab:check RAILS_ENV=production SANITIZE=true)(we will only investigate if the tests are passing)