From 90e9670207e3c1905e2a8277884d72f6b4de13ca Mon Sep 17 00:00:00 2001 From: "john.mcdonnell" Date: Fri, 16 Dec 2022 12:08:07 +0000 Subject: [PATCH 1/3] Adding custom registry component Tempoarily disable other jobs Adding offline registry to airgapped instance Remvove QA_RSPEC_TAGS excluding tags Just pull the required images rather so they are available locally Update network names to match gitlab-org/gitlab Update ci to use testing branch in gitlab Fix typo on network create Fix rubocop issues Always run downstream pipeline for test Fix typo where starting runner network with no name Add mechanism to allow docker network connect/disconnect Set the GITLAB_QA_CONTAINER_REGISTRY_HOST for downstream pipeline Add GITLAB_QA_CONTAINER_REGISTRY_HOST to pass to downstream jobs Use test branch image for pipeline Update variable to use QA_IAMGE rather than release Update pipeline to reflect new ENV name Use registry.gitlab as QA_CONTAINER_REGISTRY_HOST Use 'test' network name Update networking --- .gitlab-ci.yml | 5 +- lib/gitlab/qa/component/container_registry.rb | 55 ++++++++++ lib/gitlab/qa/component/gitaly_cluster.rb | 3 +- lib/gitlab/qa/docker/engine.rb | 40 ++++++- .../qa/scenario/test/instance/airgapped.rb | 100 ++++++------------ 5 files changed, 131 insertions(+), 72 deletions(-) create mode 100644 lib/gitlab/qa/component/container_registry.rb diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 89b78683..f5f7155c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -130,10 +130,11 @@ package-and-test: ALLURE_JOB_NAME: gitlab-qa UPDATE_QA_CACHE: $UPDATE_QA_CACHE GITLAB_QA_CACHE_KEY: $GITLAB_QA_CACHE_KEY + QA_CONTAINER_REGISTRY_HOST: 'localhost:5001' inherit: variables: - RUBY_VERSION - when: manual + when: on_success trigger: strategy: depend forward: @@ -141,5 +142,5 @@ package-and-test: pipeline_variables: true include: - project: gitlab-org/gitlab - ref: master + ref: jmd/offline-environment-pipeline file: .gitlab/ci/package-and-test/main.gitlab-ci.yml diff --git a/lib/gitlab/qa/component/container_registry.rb b/lib/gitlab/qa/component/container_registry.rb new file mode 100644 index 00000000..a308e5c5 --- /dev/null +++ b/lib/gitlab/qa/component/container_registry.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module QA + module Component + class ContainerRegistry < Base + DOCKER_IMAGE = 'registry' + DOCKER_IMAGE_TAG = '2' + + def name + @name ||= "containerregistry" + end + + def start(network: 'bridge', **kwargs) + docker.run(image: image, tag: tag) do |command| + command << '-d ' + command << "--name #{name}" + command << "--net #{network}" + command << "--publish 5001:5000" + command << "--dns=#{kwargs[:dns]}" unless kwargs[:dns].nil? + end + + seed_container_registry + end + + def disconnect_from(network_name) + docker.network_disconnect(network_name, name) + end + + def connect_to(network_name) + docker.network_connect(network_name, name) + end + + private + + def seed_container_registry + docker.pull(image: "registry.gitlab.com/gitlab-org/gitlab-runner", tag: "alpine") + docker.tag( + source_image: "registry.gitlab.com/gitlab-org/gitlab-runner", source_tag: "alpine", + target_image: "localhost:5001/gitlab-org/gitlab-runner", target_tag: "alpine" + ) + + docker.pull(image: "registry.gitlab.com/gitlab-org/gitlab-build-images", tag: "gitlab-qa-alpine-ruby-2.7") + docker.tag( + source_image: "registry.gitlab.com/gitlab-org/gitlab-build-images", source_tag: "gitlab-qa-alpine-ruby-2.7", + target_image: "localhost:5001/gitlab-org/gitlab-build-images", target_tag: "gitlab-qa-alpine-ruby-2.7" + ) + + docker.push(image: "localhost:5001/gitlab-org/gitlab-runner", tag: "alpine") + docker.push(image: "localhost:5001/gitlab-org/gitlab-build-images", tag: "gitlab-qa-alpine-ruby-2.7") + end + end + end + end +end diff --git a/lib/gitlab/qa/component/gitaly_cluster.rb b/lib/gitlab/qa/component/gitaly_cluster.rb index cd1fba3e..ae96bb82 100644 --- a/lib/gitlab/qa/component/gitaly_cluster.rb +++ b/lib/gitlab/qa/component/gitaly_cluster.rb @@ -6,7 +6,7 @@ module Gitlab class GitalyCluster class GitalyClusterConfig attr_accessor :gitlab_name, :network, :airgapped_network, - :praefect_node_name, :praefect_port, :praefect_ip, + :praefect_node_name, :praefect_port, :primary_node_name, :primary_node_port, :secondary_node_name, :secondary_node_port, :tertiary_node_name, :tertiary_node_port, @@ -90,7 +90,6 @@ module Gitlab end @praefect_node = praefect(release) - config.praefect_ip = praefect_node.ip_address Runtime::Logger.info("Gitaly Cluster Ready") end diff --git a/lib/gitlab/qa/docker/engine.rb b/lib/gitlab/qa/docker/engine.rb index c133a38d..8f7411cd 100644 --- a/lib/gitlab/qa/docker/engine.rb +++ b/lib/gitlab/qa/docker/engine.rb @@ -18,7 +18,10 @@ module Gitlab end def login(username:, password:, registry:) - Docker::Command.execute(%(login --username "#{username}" --password "#{password}" #{registry}), mask_secrets: password) + Docker::Command.execute( + %(login --username "#{username}" --password "#{password}" #{registry}), + mask_secrets: password + ) end def pull(image:, tag: nil, quiet: true) @@ -30,6 +33,21 @@ module Gitlab end end + def tag(source_image: nil, source_tag: nil, target_image: nil, target_tag: nil) + Docker::Command.new("tag").tap do |command| + command << "#{full_image_name(source_image, source_tag)} #{full_image_name(target_image, target_tag)}" + command.execute! + end + end + + def push(image:, tag: nil, quiet: true) + Docker::Command.new("push").tap do |command| + command << "-q" if quiet + command << full_image_name(image, tag) + command.execute! + end + end + def run(image:, tag: nil, args: []) Docker::Command.new('run', stream_output: stream_output).tap do |command| yield command if block_given? @@ -63,7 +81,8 @@ module Gitlab Class.new do # @param file The name of the file # @param contents The content of the file to write - # @param expand_vars Set false if you need to write an environment variable '$' to a file. The variable should be escaped \\\$ + # @param expand_vars Set false if you need to write an environment variable '$' to a file. + # The variable should be escaped \\\$ def self.write(file, contents, expand_vars = true) if expand_vars %(echo "#{contents}" > #{file};) @@ -82,7 +101,10 @@ module Gitlab def exec(name, command, mask_secrets: nil) cmd = ['exec'] cmd << '--privileged' if privileged_command?(command) - Docker::Command.execute(%(#{cmd.join(' ')} #{name} bash -c "#{command.gsub('"', '\\"')}"), mask_secrets: mask_secrets) + Docker::Command.execute( + %(#{cmd.join(' ')} #{name} bash -c "#{command.gsub('"', '\\"')}"), + mask_secrets: mask_secrets + ) end def read_file(image, tag, path, &block) @@ -130,6 +152,18 @@ module Gitlab Docker::Command.execute("network create #{name}") end + def network_remove(name) + Docker::Command.execute("network rm #{name}") + end + + def network_disconnect(network_name, container_name) + Docker::Command.execute("network disconnect #{network_name} #{container_name}") + end + + def network_connect(network_name, container_name) + Docker::Command.execute("network connect #{network_name} #{container_name}") + end + def port(name, port) Docker::Command.execute("port #{name} #{port}/tcp") end diff --git a/lib/gitlab/qa/scenario/test/instance/airgapped.rb b/lib/gitlab/qa/scenario/test/instance/airgapped.rb index 0d2053ad..8a68b330 100644 --- a/lib/gitlab/qa/scenario/test/instance/airgapped.rb +++ b/lib/gitlab/qa/scenario/test/instance/airgapped.rb @@ -7,43 +7,49 @@ module Gitlab module Instance class Airgapped < Scenario::Template require 'resolv' - attr_reader :config, :gitlab_air_gap_commands, :iptables_restricted_network, :airgapped_network_name + attr_reader :gitlab_name, :runner_network, :offline_network, :online_network, :cluster_config def initialize - # Uses https://docs.docker.com/engine/reference/commandline/network_create/#network-internal-mode - @airgapped_network_name = 'airgapped' - # Uses iptables to deny all network traffic, with a number of exceptions for required ports and IPs - @iptables_restricted_network = 'test' - @config = Component::GitalyCluster::GitalyClusterConfig.new( - gitlab_name: "gitlab-airgapped-#{SecureRandom.hex(4)}", + @gitlab_name = 'gitlab-airgapped' + @offline_network = 'test' + @online_network = 'online-network' + @cluster_config = Component::GitalyCluster::GitalyClusterConfig.new( + gitlab_name: gitlab_name, airgapped_network: true, - network: airgapped_network_name + network: offline_network ) end def perform(release, *rspec_args) + setup_networking + + # For runner tests we need to verify that we can pull from a container registry that is airgapped. + Component::ContainerRegistry.perform do |c| + c.start(network: online_network, dns: '0.0.0.0') + # TODO we need to identify a way to run 'docker pull' targeting an offline container registry + # c.disconnect_from(online_network) + # c.connect_to(offline_network) + end + Component::Gitlab.perform do |gitlab| - Component::GitalyCluster.perform do |cluster| - cluster.config = @config + cluster = Component::GitalyCluster.perform do |cluster| cluster.release = release - # we need to get an IP for praefect before proceeding so it cannot be run in parallel with gitlab - cluster.instance(true).join + cluster.config = cluster_config + cluster.instance end - gitlab.name = config.gitlab_name + gitlab.name = gitlab_name gitlab.release = release - gitlab.network = iptables_restricted_network # we use iptables to restrict access on the gitlab instance - gitlab.runner_network = config.network - gitlab.exec_commands = airgap_gitlab_commands + gitlab.network = offline_network gitlab.skip_availability_check = true gitlab.omnibus_configuration << gitlab_omnibus_configuration rspec_args << "--" unless rspec_args.include?('--') rspec_args << %w[--tag ~orchestrated] gitlab.instance do + cluster.join Component::Specs.perform do |specs| specs.suite = 'Test::Instance::Airgapped' specs.release = gitlab.release specs.network = gitlab.network - specs.runner_network = gitlab.runner_network specs.args = [gitlab.address, *rspec_args] end end @@ -52,59 +58,23 @@ module Gitlab private - def airgap_gitlab_commands - gitlab_ip = Resolv.getaddress('gitlab.com') - gitlab_registry_ip = Resolv.getaddress(QA::Release::COM_REGISTRY) - dev_gitlab_registry_ip = Resolv.getaddress(QA::Release::DEV_REGISTRY.split(':')[0]) - praefect_ip = config.praefect_ip - @commands = <<~AIRGAP_AND_VERIFY_COMMAND.split(/\n+/) - # Should not fail before airgapping due to eg. DNS failure - # Ping and wget check - apt-get update && apt-get install -y iptables ncat - if ncat -zv -w 10 #{gitlab_ip} 80; then echo 'Airgapped connectivity check passed.'; else echo 'Airgapped connectivity check failed - should be able to access gitlab_ip'; exit 1; fi; - - echo "Checking regular connectivity..." \ - && wget --retry-connrefused --waitretry=1 --read-timeout=15 --timeout=10 -t 2 http://registry.gitlab.com > /dev/null 2>&1 \ - && (echo "Regular connectivity wget check passed." && exit 0) || (echo "Regular connectivity wget check failed." && exit 1) - - iptables -P INPUT DROP && iptables -P OUTPUT DROP - iptables -A INPUT -i lo -j ACCEPT && iptables -A OUTPUT -o lo -j ACCEPT # LOOPBACK - iptables -I INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT - iptables -A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - - # Jenkins on port 8080 and 50000 - iptables -A OUTPUT -p tcp -m tcp --dport 8080 -m state --state NEW,ESTABLISHED -j ACCEPT \ - && iptables -A OUTPUT -p tcp -m tcp --dport 50000 -m state --state NEW,ESTABLISHED -j ACCEPT - iptables -A OUTPUT -p tcp -m tcp --sport 22 -m state --state NEW,ESTABLISHED -j ACCEPT - iptables -A INPUT -p tcp -m tcp --dport 22 -m state --state NEW,ESTABLISHED -j ACCEPT - iptables -A OUTPUT -p tcp -m tcp --sport 80 -m state --state NEW,ESTABLISHED -j ACCEPT - iptables -A INPUT -p tcp -m tcp --dport 80 -m state --state NEW,ESTABLISHED -j ACCEPT - - # some exceptions to allow runners access network https://gitlab.com/gitlab-org/gitlab-qa/-/issues/700 - iptables -A OUTPUT -p tcp -d #{gitlab_registry_ip} -j ACCEPT - iptables -A OUTPUT -p tcp -d #{dev_gitlab_registry_ip} -j ACCEPT - # allow access to praefect node - iptables -A OUTPUT -p tcp -d #{praefect_ip} -j ACCEPT + def setup_networking + docker = Docker::Engine.new - # Should now fail to ping gitlab_ip, port 22/80 should be open - if ncat -zv -w 10 #{gitlab_ip} 80; then echo 'Airgapped connectivity check failed - should not be able to access gitlab_ip'; exit 1; else echo 'Airgapped connectivity check passed.'; fi; - if ncat -zv -w 10 127.0.0.1 22; then echo 'Airgapped connectivity port 22 check passed.'; else echo 'Airgapped connectivity port 22 check failed.'; exit 1; fi; - if ncat -zv -w 10 127.0.0.1 80; then echo 'Airgapped connectivity port 80 check passed.'; else echo 'Airgapped connectivity port 80 check failed.'; exit 1 ; fi; - if ncat -zv -w 10 #{gitlab_registry_ip} 80; then echo 'Airgapped connectivity port gitlab_registry_ip check passed.'; else echo 'Airgapped connectivity port 80 check failed.'; exit 1; fi; + # to ensure we don't reuse a network that may not be 'internal' delete existing network of same name + docker.network_remove(offline_network) if docker.network_exists?(offline_network) + docker.network_create("--driver=bridge --internal #{offline_network}") - echo "Checking airgapped connectivity..." \ - && wget --retry-connrefused --waitretry=1 --read-timeout=15 --timeout=10 -t 2 http://registry.gitlab.com > /dev/null 2>&1 \ - && (echo "Airgapped network faulty. Connectivity wget check failed." && exit 1) || (echo "Airgapped network confirmed. Connectivity wget check passed." && exit 0) - AIRGAP_AND_VERIFY_COMMAND + docker.network_create(online_network) unless docker.network_exists?(online_network) end def gitlab_omnibus_configuration <<~OMNIBUS - external_url 'http://#{config.gitlab_name}.#{iptables_restricted_network}'; + external_url 'http://#{gitlab_name}.#{offline_network}'; git_data_dirs({ 'default' => { - 'gitaly_address' => 'tcp://#{config.praefect_addr}:#{config.praefect_port}', + 'gitaly_address' => 'tcp://#{cluster_config.praefect_addr}:#{cluster_config.praefect_port}', 'gitaly_token' => 'PRAEFECT_EXTERNAL_TOKEN' } }); @@ -115,7 +85,7 @@ module Gitlab 'job_name' => 'praefect', 'static_configs' => [ 'targets' => [ - '#{config.praefect_addr}:9652' + '#{cluster_config.praefect_addr}:9652' ] ] }, @@ -123,9 +93,9 @@ module Gitlab 'job_name' => 'praefect-gitaly', 'static_configs' => [ 'targets' => [ - '#{config.primary_node_addr}:9236', - '#{config.secondary_node_addr}:9236', - '#{config.tertiary_node_addr}:9236' + '#{cluster_config.primary_node_addr}:9236', + '#{cluster_config.secondary_node_addr}:9236', + '#{cluster_config.tertiary_node_addr}:9236' ] ] } -- GitLab From 92015edbc043221a05aa8e1922aea795fd556e43 Mon Sep 17 00:00:00 2001 From: John McDonnell Date: Wed, 26 Apr 2023 01:39:00 +0100 Subject: [PATCH 2/3] Use network name airgapped --- lib/gitlab/qa/scenario/test/instance/airgapped.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/qa/scenario/test/instance/airgapped.rb b/lib/gitlab/qa/scenario/test/instance/airgapped.rb index 8a68b330..64c20f2f 100644 --- a/lib/gitlab/qa/scenario/test/instance/airgapped.rb +++ b/lib/gitlab/qa/scenario/test/instance/airgapped.rb @@ -11,7 +11,7 @@ module Gitlab def initialize @gitlab_name = 'gitlab-airgapped' - @offline_network = 'test' + @offline_network = 'airgapped' @online_network = 'online-network' @cluster_config = Component::GitalyCluster::GitalyClusterConfig.new( gitlab_name: gitlab_name, -- GitLab From a60731408615a5059b29ed5955b39e296bc9458e Mon Sep 17 00:00:00 2001 From: John McDonnell Date: Wed, 26 Apr 2023 09:38:35 +0100 Subject: [PATCH 3/3] Move seeding of container registry to tests --- .gitlab-ci.yml | 1 - lib/gitlab/qa/component/container_registry.rb | 21 ------------------- .../qa/scenario/test/instance/airgapped.rb | 5 ++--- 3 files changed, 2 insertions(+), 25 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f5f7155c..15ace74a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -130,7 +130,6 @@ package-and-test: ALLURE_JOB_NAME: gitlab-qa UPDATE_QA_CACHE: $UPDATE_QA_CACHE GITLAB_QA_CACHE_KEY: $GITLAB_QA_CACHE_KEY - QA_CONTAINER_REGISTRY_HOST: 'localhost:5001' inherit: variables: - RUBY_VERSION diff --git a/lib/gitlab/qa/component/container_registry.rb b/lib/gitlab/qa/component/container_registry.rb index a308e5c5..66054aef 100644 --- a/lib/gitlab/qa/component/container_registry.rb +++ b/lib/gitlab/qa/component/container_registry.rb @@ -19,8 +19,6 @@ module Gitlab command << "--publish 5001:5000" command << "--dns=#{kwargs[:dns]}" unless kwargs[:dns].nil? end - - seed_container_registry end def disconnect_from(network_name) @@ -30,25 +28,6 @@ module Gitlab def connect_to(network_name) docker.network_connect(network_name, name) end - - private - - def seed_container_registry - docker.pull(image: "registry.gitlab.com/gitlab-org/gitlab-runner", tag: "alpine") - docker.tag( - source_image: "registry.gitlab.com/gitlab-org/gitlab-runner", source_tag: "alpine", - target_image: "localhost:5001/gitlab-org/gitlab-runner", target_tag: "alpine" - ) - - docker.pull(image: "registry.gitlab.com/gitlab-org/gitlab-build-images", tag: "gitlab-qa-alpine-ruby-2.7") - docker.tag( - source_image: "registry.gitlab.com/gitlab-org/gitlab-build-images", source_tag: "gitlab-qa-alpine-ruby-2.7", - target_image: "localhost:5001/gitlab-org/gitlab-build-images", target_tag: "gitlab-qa-alpine-ruby-2.7" - ) - - docker.push(image: "localhost:5001/gitlab-org/gitlab-runner", tag: "alpine") - docker.push(image: "localhost:5001/gitlab-org/gitlab-build-images", tag: "gitlab-qa-alpine-ruby-2.7") - end end end end diff --git a/lib/gitlab/qa/scenario/test/instance/airgapped.rb b/lib/gitlab/qa/scenario/test/instance/airgapped.rb index 64c20f2f..021dc138 100644 --- a/lib/gitlab/qa/scenario/test/instance/airgapped.rb +++ b/lib/gitlab/qa/scenario/test/instance/airgapped.rb @@ -7,12 +7,11 @@ module Gitlab module Instance class Airgapped < Scenario::Template require 'resolv' - attr_reader :gitlab_name, :runner_network, :offline_network, :online_network, :cluster_config + attr_reader :gitlab_name, :runner_network, :offline_network, :cluster_config def initialize @gitlab_name = 'gitlab-airgapped' @offline_network = 'airgapped' - @online_network = 'online-network' @cluster_config = Component::GitalyCluster::GitalyClusterConfig.new( gitlab_name: gitlab_name, airgapped_network: true, @@ -25,7 +24,7 @@ module Gitlab # For runner tests we need to verify that we can pull from a container registry that is airgapped. Component::ContainerRegistry.perform do |c| - c.start(network: online_network, dns: '0.0.0.0') + c.start(network: offline_network, dns: '0.0.0.0') # TODO we need to identify a way to run 'docker pull' targeting an offline container registry # c.disconnect_from(online_network) # c.connect_to(offline_network) -- GitLab