From 574f5421ed15ffc5b0617ef9276f6b6d81f504eb Mon Sep 17 00:00:00 2001 From: "john.mcdonnell" Date: Thu, 12 Jan 2023 11:56:32 +0000 Subject: [PATCH 1/5] Adding ENV to allow customizing where docker runner images live --- qa/qa/resource/runner.rb | 148 +++++++++++++++++++++++++++++++++++++++ qa/qa/runtime/env.rb | 5 ++ 2 files changed, 153 insertions(+) create mode 100644 qa/qa/resource/runner.rb diff --git a/qa/qa/resource/runner.rb b/qa/qa/resource/runner.rb new file mode 100644 index 00000000000000..ed57183fe0b7cb --- /dev/null +++ b/qa/qa/resource/runner.rb @@ -0,0 +1,148 @@ +# frozen_string_literal: true + +module QA + module Resource + class Runner < Base + attributes :id, + :active, + :paused, + :runner_type, + :online, + :status, + :ip_address, + :token, + :tags, + :config, + :run_untagged, + :name, # This attribute == runner[:description] + :image, + :executor, + :executor_image + + attribute :project do + Project.fabricate_via_api! do |resource| + resource.name = 'project-with-ci-cd' + resource.description = 'Project with CI/CD Pipelines' + end + end + + def initialize + @tags = nil + @config = nil + @run_untagged = nil + @name = "qa-runner-#{SecureRandom.hex(4)}" + @image = "#{Runtime::Env.runner_image_registry}/gitlab-org/gitlab-runner:alpine" + @executor = :shell + @executor_image = "#{Runtime::Env.runner_image_registry}/gitlab-org/gitlab-build-images:gitlab-qa-alpine-ruby-2.7" + end + + # Initially we only support fabricate + # via API + def fabricate! + fabricate_via_api! + end + + # Start container and register runner + # Fetch via API and populate attributes + # + def fabricate_via_api! + start_container_and_register + populate_runner_attributes + end + + def remove_via_api! + super + ensure + @docker_container.remove! + end + + def reload! + populate_runner_attributes + end + + def api_delete_path + "/runners/#{id}" + end + + def api_get_path + "/runners" + end + + def api_post_path + "/runners" + end + + def api_post_body; end + + def not_found_by_tags? + url = "#{api_get_path}?tag_list=#{tags.compact.join(',')}" + auto_paginated_response(request_url(url)).empty? + end + + def runners_list + runners_list = nil + url = tags ? "#{api_get_path}?tag_list=#{tags.compact.join(',')}" : api_get_path + Runtime::Logger.info('Looking for list of runners via API...') + Support::Retrier.retry_until(max_duration: 60, sleep_interval: 1) do + runners_list = auto_paginated_response(request_url(url)) + runners_list.present? + end + + runners_list + end + + def wait_until_online + Runtime::Logger.info('Waiting for runner to come online...') + Support::Retrier.retry_until(max_duration: 60, sleep_interval: 1) do + this_runner[:status] == 'online' + end + end + + def restart + Runtime::Logger.info("Restarting runner container #{name}...") + @docker_container.restart + wait_until_online + end + + private + + def start_container_and_register + @docker_container = Service::DockerRun::GitlabRunner.new(name).tap do |runner| + Support::Retrier.retry_on_exception(sleep_interval: 5) do + runner.pull + end + + runner.token = @token ||= project.runners_token + runner.address = Runtime::Scenario.gitlab_address + runner.tags = tags if tags + runner.image = image + runner.config = config if config + runner.executor = executor + runner.executor_image = executor_image if executor == :docker + runner.run_untagged = run_untagged if run_untagged + runner.register! + end + end + + def this_runner + runner = nil + Support::Retrier.retry_until(max_duration: 60, sleep_interval: 1) do + runner = runners_list.find { |runner| runner[:description] == name } + !runner.nil? + end + runner + end + + def populate_runner_attributes + runner = this_runner + @id = runner[:id] + @active = runner[:active] + @paused = runner[:paused] + @runner_type = runner[:typed] + @online = runner[:online] + @status = runner[:status] + @ip_address = runner[:ip_address] + end + end + end +end diff --git a/qa/qa/runtime/env.rb b/qa/qa/runtime/env.rb index 3153c644b883cd..6be76de7e4bec9 100644 --- a/qa/qa/runtime/env.rb +++ b/qa/qa/runtime/env.rb @@ -479,6 +479,11 @@ def skip_smoke_reliable? enabled?(ENV['QA_SKIP_SMOKE_RELIABLE'], default: false) end + # The QA_RUNNER_IMAGE_REGISTRY defines where we can find docker images required for runner tests + def runner_image_registry + ENV['QA_RUNNER_IMAGE_REGISTRY'] || 'registry.gitlab.com' + end + # ENV variables for authenticating against a private container registry # These need to be set if using the # Service::DockerRun::Mixins::ThirdPartyDocker module -- GitLab From 064c12f1ab1e49a406c57db6249b7944aa7c1920 Mon Sep 17 00:00:00 2001 From: "john.mcdonnell" Date: Thu, 22 Dec 2022 12:48:12 +0000 Subject: [PATCH 2/5] Force remove runner container to allow for retries when it may fail --- qa/qa/service/docker_run/gitlab_runner.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/qa/qa/service/docker_run/gitlab_runner.rb b/qa/qa/service/docker_run/gitlab_runner.rb index a8fcf8f9332759..f71653c36b51ed 100644 --- a/qa/qa/service/docker_run/gitlab_runner.rb +++ b/qa/qa/service/docker_run/gitlab_runner.rb @@ -39,6 +39,7 @@ def register! raise("Missing runner token value!") unless token cmd = <<~CMD.tr("\n", ' ') + docker container rm -f #{@name} && docker run -d --rm --network #{runner_network} --name #{@name} #{'-v /var/run/docker.sock:/var/run/docker.sock' if @executor == :docker} --privileged -- GitLab From a08acec3a0248f95f9de250c5a79e740cb718215 Mon Sep 17 00:00:00 2001 From: "john.mcdonnell" Date: Thu, 22 Dec 2022 12:53:12 +0000 Subject: [PATCH 3/5] Using ENV in more place --- qa/qa/service/docker_run/gitlab_runner.rb | 6 +++--- .../6_release/deploy_key/clone_using_deploy_key_spec.rb | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/qa/qa/service/docker_run/gitlab_runner.rb b/qa/qa/service/docker_run/gitlab_runner.rb index f71653c36b51ed..b1684dd569bf50 100644 --- a/qa/qa/service/docker_run/gitlab_runner.rb +++ b/qa/qa/service/docker_run/gitlab_runner.rb @@ -16,12 +16,12 @@ class GitlabRunner < Base MSG def initialize(name) - @image = 'gitlab/gitlab-runner:alpine-v15.8.3' + @image = "#{Runtime::Env.runner_image_registry}/gitlab-org/gitlab-runner:alpine-v15.8.3" @name = name || "qa-runner-#{SecureRandom.hex(4)}" @run_untagged = true @executor = :shell - @executor_image = 'registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine-ruby-2.7' - + @executor_image = "#{Runtime::Env.runner_image_registry}/" \ + "gitlab-org/gitlab-build-images:gitlab-qa-alpine-ruby-2.7" super() end diff --git a/qa/qa/specs/features/browser_ui/6_release/deploy_key/clone_using_deploy_key_spec.rb b/qa/qa/specs/features/browser_ui/6_release/deploy_key/clone_using_deploy_key_spec.rb index 0f6bee951a727e..a435edcca771e1 100644 --- a/qa/qa/specs/features/browser_ui/6_release/deploy_key/clone_using_deploy_key_spec.rb +++ b/qa/qa/specs/features/browser_ui/6_release/deploy_key/clone_using_deploy_key_spec.rb @@ -19,7 +19,7 @@ module QA resource.project = project resource.name = runner_name resource.tags = [runner_name] - resource.image = 'gitlab/gitlab-runner:alpine-v15.8.3' + resource.image = "#{Runtime::Env.runner_image_registry}/gitlab-org/gitlab-runner:alpine-v15.8.3" end end -- GitLab From c895f21e74dcfbfe5324c102c04a8dbca510a413 Mon Sep 17 00:00:00 2001 From: "john.mcdonnell" Date: Thu, 22 Dec 2022 13:40:49 +0000 Subject: [PATCH 4/5] Remove airgap check as use of docker internal verifies that --- qa/qa/service/docker_run/gitlab_runner.rb | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/qa/qa/service/docker_run/gitlab_runner.rb b/qa/qa/service/docker_run/gitlab_runner.rb index b1684dd569bf50..c5ced4c0d119f5 100644 --- a/qa/qa/service/docker_run/gitlab_runner.rb +++ b/qa/qa/service/docker_run/gitlab_runner.rb @@ -49,9 +49,6 @@ def register! shell(cmd, mask_secrets: [@token]) wait_until_running_and_configured - - # Prove airgappedness - shell("docker exec #{@name} sh -c '#{prove_airgap}'") if runner_network == 'airgapped' end def tags=(tags) @@ -102,17 +99,6 @@ def register_command CMD end - # Ping Cloudflare DNS, should fail - # Ping Registry, should fail to resolve - def prove_airgap - gitlab_ip = Resolv.getaddress 'registry.gitlab.com' - <<~CMD - echo "Checking airgapped connectivity..." - nc -zv -w 10 #{gitlab_ip} 80 && (echo "Airgapped network faulty. Connectivity netcat check failed." && exit 1) || (echo "Connectivity netcat check passed." && exit 0) - wget --retry-connrefused --waitretry=1 --read-timeout=15 --timeout=10 -t 2 http://registry.gitlab.com > /dev/null 2>&1 && (echo "Airgapped network faulty. Connectivity wget check failed." && exit 1) || (echo "Airgapped network confirmed. Connectivity wget check passed." && exit 0) - CMD - end - def add_gitlab_tls_cert gitlab_tls_certificate = Tempfile.new('gitlab-cert') gitlab_tls_certificate.write(Runtime::Env.gitlab_tls_certificate) -- GitLab From 87867f543d1fae2d97f67d40a0878f9ec5c93dd4 Mon Sep 17 00:00:00 2001 From: "john.mcdonnell" Date: Thu, 12 Jan 2023 12:00:36 +0000 Subject: [PATCH 5/5] Tidy up --- qa/qa/resource/runner.rb | 148 ---------------------------------- qa/qa/resource/runner_base.rb | 5 +- 2 files changed, 3 insertions(+), 150 deletions(-) delete mode 100644 qa/qa/resource/runner.rb diff --git a/qa/qa/resource/runner.rb b/qa/qa/resource/runner.rb deleted file mode 100644 index ed57183fe0b7cb..00000000000000 --- a/qa/qa/resource/runner.rb +++ /dev/null @@ -1,148 +0,0 @@ -# frozen_string_literal: true - -module QA - module Resource - class Runner < Base - attributes :id, - :active, - :paused, - :runner_type, - :online, - :status, - :ip_address, - :token, - :tags, - :config, - :run_untagged, - :name, # This attribute == runner[:description] - :image, - :executor, - :executor_image - - attribute :project do - Project.fabricate_via_api! do |resource| - resource.name = 'project-with-ci-cd' - resource.description = 'Project with CI/CD Pipelines' - end - end - - def initialize - @tags = nil - @config = nil - @run_untagged = nil - @name = "qa-runner-#{SecureRandom.hex(4)}" - @image = "#{Runtime::Env.runner_image_registry}/gitlab-org/gitlab-runner:alpine" - @executor = :shell - @executor_image = "#{Runtime::Env.runner_image_registry}/gitlab-org/gitlab-build-images:gitlab-qa-alpine-ruby-2.7" - end - - # Initially we only support fabricate - # via API - def fabricate! - fabricate_via_api! - end - - # Start container and register runner - # Fetch via API and populate attributes - # - def fabricate_via_api! - start_container_and_register - populate_runner_attributes - end - - def remove_via_api! - super - ensure - @docker_container.remove! - end - - def reload! - populate_runner_attributes - end - - def api_delete_path - "/runners/#{id}" - end - - def api_get_path - "/runners" - end - - def api_post_path - "/runners" - end - - def api_post_body; end - - def not_found_by_tags? - url = "#{api_get_path}?tag_list=#{tags.compact.join(',')}" - auto_paginated_response(request_url(url)).empty? - end - - def runners_list - runners_list = nil - url = tags ? "#{api_get_path}?tag_list=#{tags.compact.join(',')}" : api_get_path - Runtime::Logger.info('Looking for list of runners via API...') - Support::Retrier.retry_until(max_duration: 60, sleep_interval: 1) do - runners_list = auto_paginated_response(request_url(url)) - runners_list.present? - end - - runners_list - end - - def wait_until_online - Runtime::Logger.info('Waiting for runner to come online...') - Support::Retrier.retry_until(max_duration: 60, sleep_interval: 1) do - this_runner[:status] == 'online' - end - end - - def restart - Runtime::Logger.info("Restarting runner container #{name}...") - @docker_container.restart - wait_until_online - end - - private - - def start_container_and_register - @docker_container = Service::DockerRun::GitlabRunner.new(name).tap do |runner| - Support::Retrier.retry_on_exception(sleep_interval: 5) do - runner.pull - end - - runner.token = @token ||= project.runners_token - runner.address = Runtime::Scenario.gitlab_address - runner.tags = tags if tags - runner.image = image - runner.config = config if config - runner.executor = executor - runner.executor_image = executor_image if executor == :docker - runner.run_untagged = run_untagged if run_untagged - runner.register! - end - end - - def this_runner - runner = nil - Support::Retrier.retry_until(max_duration: 60, sleep_interval: 1) do - runner = runners_list.find { |runner| runner[:description] == name } - !runner.nil? - end - runner - end - - def populate_runner_attributes - runner = this_runner - @id = runner[:id] - @active = runner[:active] - @paused = runner[:paused] - @runner_type = runner[:typed] - @online = runner[:online] - @status = runner[:status] - @ip_address = runner[:ip_address] - end - end - end -end diff --git a/qa/qa/resource/runner_base.rb b/qa/qa/resource/runner_base.rb index 399d1153dc2126..22c966143c5b6e 100644 --- a/qa/qa/resource/runner_base.rb +++ b/qa/qa/resource/runner_base.rb @@ -35,9 +35,10 @@ def initialize @config = nil @run_untagged = nil @name = "qa-runner-#{SecureRandom.hex(4)}" - @image = 'registry.gitlab.com/gitlab-org/gitlab-runner:alpine-v15.8.3' + @image = "#{Runtime::Env.runner_image_registry}/gitlab-org/gitlab-runner:alpine-v15.8.3" @executor = :shell - @executor_image = 'registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine-ruby-2.7' + @executor_image = "#{Runtime::Env.runner_image_registry} + /gitlab-org/gitlab-build-images:gitlab-qa-alpine-ruby-2.7" end # Initially we only support fabricate via API -- GitLab