From 2b5fa1073cb7ec7f79add7bb8b417cf70b61aa46 Mon Sep 17 00:00:00 2001 From: John McDonnell Date: Fri, 7 Apr 2023 10:23:38 +0100 Subject: [PATCH 1/7] Use temporary test branch on gitlab/gitlab-org --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 89b78683..15ace74a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -133,7 +133,7 @@ package-and-test: inherit: variables: - RUBY_VERSION - when: manual + when: on_success trigger: strategy: depend forward: @@ -141,5 +141,5 @@ package-and-test: pipeline_variables: true include: - project: gitlab-org/gitlab - ref: master + ref: jmd/offline-environment-pipeline file: .gitlab/ci/package-and-test/main.gitlab-ci.yml -- GitLab From 09f140338e05affb0ed47ff5f6e9024aa75b193c Mon Sep 17 00:00:00 2001 From: John McDonnell Date: Fri, 7 Apr 2023 10:32:18 +0100 Subject: [PATCH 2/7] Update airgapped to use docker internal network --- lib/gitlab/qa/component/gitaly_cluster.rb | 3 +- .../qa/scenario/test/instance/airgapped.rb | 62 ++----------------- 2 files changed, 7 insertions(+), 58 deletions(-) diff --git a/lib/gitlab/qa/component/gitaly_cluster.rb b/lib/gitlab/qa/component/gitaly_cluster.rb index cd1fba3e..ae96bb82 100644 --- a/lib/gitlab/qa/component/gitaly_cluster.rb +++ b/lib/gitlab/qa/component/gitaly_cluster.rb @@ -6,7 +6,7 @@ module Gitlab class GitalyCluster class GitalyClusterConfig attr_accessor :gitlab_name, :network, :airgapped_network, - :praefect_node_name, :praefect_port, :praefect_ip, + :praefect_node_name, :praefect_port, :primary_node_name, :primary_node_port, :secondary_node_name, :secondary_node_port, :tertiary_node_name, :tertiary_node_port, @@ -90,7 +90,6 @@ module Gitlab end @praefect_node = praefect(release) - config.praefect_ip = praefect_node.ip_address Runtime::Logger.info("Gitaly Cluster Ready") end diff --git a/lib/gitlab/qa/scenario/test/instance/airgapped.rb b/lib/gitlab/qa/scenario/test/instance/airgapped.rb index 0d2053ad..76e06cc2 100644 --- a/lib/gitlab/qa/scenario/test/instance/airgapped.rb +++ b/lib/gitlab/qa/scenario/test/instance/airgapped.rb @@ -7,13 +7,11 @@ module Gitlab module Instance class Airgapped < Scenario::Template require 'resolv' - attr_reader :config, :gitlab_air_gap_commands, :iptables_restricted_network, :airgapped_network_name + attr_reader :config, :gitlab_air_gap_commands, :airgapped_network_name, :runner_network def initialize - # Uses https://docs.docker.com/engine/reference/commandline/network_create/#network-internal-mode - @airgapped_network_name = 'airgapped' - # Uses iptables to deny all network traffic, with a number of exceptions for required ports and IPs - @iptables_restricted_network = 'test' + @airgapped_network_name = 'test' + @runner_network = 'airgapped' @config = Component::GitalyCluster::GitalyClusterConfig.new( gitlab_name: "gitlab-airgapped-#{SecureRandom.hex(4)}", airgapped_network: true, @@ -26,14 +24,12 @@ module Gitlab Component::GitalyCluster.perform do |cluster| cluster.config = @config cluster.release = release - # we need to get an IP for praefect before proceeding so it cannot be run in parallel with gitlab cluster.instance(true).join end gitlab.name = config.gitlab_name gitlab.release = release - gitlab.network = iptables_restricted_network # we use iptables to restrict access on the gitlab instance - gitlab.runner_network = config.network - gitlab.exec_commands = airgap_gitlab_commands + gitlab.network = config.network + gitlab.runner_network = runner_network gitlab.skip_availability_check = true gitlab.omnibus_configuration << gitlab_omnibus_configuration rspec_args << "--" unless rspec_args.include?('--') @@ -52,55 +48,9 @@ module Gitlab private - def airgap_gitlab_commands - gitlab_ip = Resolv.getaddress('gitlab.com') - gitlab_registry_ip = Resolv.getaddress(QA::Release::COM_REGISTRY) - dev_gitlab_registry_ip = Resolv.getaddress(QA::Release::DEV_REGISTRY.split(':')[0]) - praefect_ip = config.praefect_ip - @commands = <<~AIRGAP_AND_VERIFY_COMMAND.split(/\n+/) - # Should not fail before airgapping due to eg. DNS failure - # Ping and wget check - apt-get update && apt-get install -y iptables ncat - if ncat -zv -w 10 #{gitlab_ip} 80; then echo 'Airgapped connectivity check passed.'; else echo 'Airgapped connectivity check failed - should be able to access gitlab_ip'; exit 1; fi; - - echo "Checking regular connectivity..." \ - && wget --retry-connrefused --waitretry=1 --read-timeout=15 --timeout=10 -t 2 http://registry.gitlab.com > /dev/null 2>&1 \ - && (echo "Regular connectivity wget check passed." && exit 0) || (echo "Regular connectivity wget check failed." && exit 1) - - iptables -P INPUT DROP && iptables -P OUTPUT DROP - iptables -A INPUT -i lo -j ACCEPT && iptables -A OUTPUT -o lo -j ACCEPT # LOOPBACK - iptables -I INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT - iptables -A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - - # Jenkins on port 8080 and 50000 - iptables -A OUTPUT -p tcp -m tcp --dport 8080 -m state --state NEW,ESTABLISHED -j ACCEPT \ - && iptables -A OUTPUT -p tcp -m tcp --dport 50000 -m state --state NEW,ESTABLISHED -j ACCEPT - iptables -A OUTPUT -p tcp -m tcp --sport 22 -m state --state NEW,ESTABLISHED -j ACCEPT - iptables -A INPUT -p tcp -m tcp --dport 22 -m state --state NEW,ESTABLISHED -j ACCEPT - iptables -A OUTPUT -p tcp -m tcp --sport 80 -m state --state NEW,ESTABLISHED -j ACCEPT - iptables -A INPUT -p tcp -m tcp --dport 80 -m state --state NEW,ESTABLISHED -j ACCEPT - - # some exceptions to allow runners access network https://gitlab.com/gitlab-org/gitlab-qa/-/issues/700 - iptables -A OUTPUT -p tcp -d #{gitlab_registry_ip} -j ACCEPT - iptables -A OUTPUT -p tcp -d #{dev_gitlab_registry_ip} -j ACCEPT - # allow access to praefect node - iptables -A OUTPUT -p tcp -d #{praefect_ip} -j ACCEPT - - # Should now fail to ping gitlab_ip, port 22/80 should be open - if ncat -zv -w 10 #{gitlab_ip} 80; then echo 'Airgapped connectivity check failed - should not be able to access gitlab_ip'; exit 1; else echo 'Airgapped connectivity check passed.'; fi; - if ncat -zv -w 10 127.0.0.1 22; then echo 'Airgapped connectivity port 22 check passed.'; else echo 'Airgapped connectivity port 22 check failed.'; exit 1; fi; - if ncat -zv -w 10 127.0.0.1 80; then echo 'Airgapped connectivity port 80 check passed.'; else echo 'Airgapped connectivity port 80 check failed.'; exit 1 ; fi; - if ncat -zv -w 10 #{gitlab_registry_ip} 80; then echo 'Airgapped connectivity port gitlab_registry_ip check passed.'; else echo 'Airgapped connectivity port 80 check failed.'; exit 1; fi; - - echo "Checking airgapped connectivity..." \ - && wget --retry-connrefused --waitretry=1 --read-timeout=15 --timeout=10 -t 2 http://registry.gitlab.com > /dev/null 2>&1 \ - && (echo "Airgapped network faulty. Connectivity wget check failed." && exit 1) || (echo "Airgapped network confirmed. Connectivity wget check passed." && exit 0) - AIRGAP_AND_VERIFY_COMMAND - end - def gitlab_omnibus_configuration <<~OMNIBUS - external_url 'http://#{config.gitlab_name}.#{iptables_restricted_network}'; + external_url 'http://#{config.gitlab_name}.#{airgapped_network_name}'; git_data_dirs({ 'default' => { -- GitLab From 1e110640df132a74e6b072e11d4c53737e23f8a2 Mon Sep 17 00:00:00 2001 From: John McDonnell Date: Fri, 7 Apr 2023 11:03:38 +0100 Subject: [PATCH 3/7] Start GitalyCluster in parallel to GitLab --- lib/gitlab/qa/scenario/test/instance/airgapped.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/qa/scenario/test/instance/airgapped.rb b/lib/gitlab/qa/scenario/test/instance/airgapped.rb index 76e06cc2..639f82ea 100644 --- a/lib/gitlab/qa/scenario/test/instance/airgapped.rb +++ b/lib/gitlab/qa/scenario/test/instance/airgapped.rb @@ -21,10 +21,10 @@ module Gitlab def perform(release, *rspec_args) Component::Gitlab.perform do |gitlab| - Component::GitalyCluster.perform do |cluster| + cluster = Component::GitalyCluster.perform do |cluster| cluster.config = @config cluster.release = release - cluster.instance(true).join + cluster.instance end gitlab.name = config.gitlab_name gitlab.release = release @@ -35,6 +35,7 @@ module Gitlab rspec_args << "--" unless rspec_args.include?('--') rspec_args << %w[--tag ~orchestrated] gitlab.instance do + cluster.join Component::Specs.perform do |specs| specs.suite = 'Test::Instance::Airgapped' specs.release = gitlab.release -- GitLab From 21aa18a85036b99590b61016a7572e268697233f Mon Sep 17 00:00:00 2001 From: John McDonnell Date: Tue, 11 Apr 2023 23:53:23 +0100 Subject: [PATCH 4/7] Use testing gitlab-ee-qa release image --- .gitlab-ci.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 15ace74a..0d0c8071 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -124,12 +124,42 @@ package-and-test: - package-and-test-env variables: RELEASE: EE + QA_IMAGE: registry.gitlab.com/gitlab-org/gitlab/gitlab-ee-qa:jmd-offline-environment-pipeline RUN_WITH_BUNDLE: "true" SKIP_OMNIBUS_TRIGGER: "true" SKIP_REPORT_IN_ISSUES: "true" ALLURE_JOB_NAME: gitlab-qa UPDATE_QA_CACHE: $UPDATE_QA_CACHE GITLAB_QA_CACHE_KEY: $GITLAB_QA_CACHE_KEY + QA_TESTS: qa/specs/features/ee/browser_ui/2_plan/epic/epics_management_spec.rb qa/specs/features/browser_ui/1_manage/integrations/jenkins/jenkins_build_status_spec.rb qa/specs/features/ee/browser_ui/2_plan/epic/epics_management_spec.rb + inherit: + variables: + - RUBY_VERSION + when: on_success + trigger: + strategy: depend + forward: + yaml_variables: true + pipeline_variables: true + include: + - project: gitlab-org/gitlab + ref: jmd/offline-environment-pipeline + file: .gitlab/ci/package-and-test/main.gitlab-ci.yml + + +package-and-test-with-bundle-false: + stage: qa + needs: + - package-and-test-env + variables: + RELEASE: EE + QA_IMAGE: registry.gitlab.com/gitlab-org/gitlab/gitlab-ee-qa:jmd-offline-environment-pipeline + RUN_WITH_BUNDLE: "false" + SKIP_OMNIBUS_TRIGGER: "true" + SKIP_REPORT_IN_ISSUES: "true" + ALLURE_JOB_NAME: gitlab-qa + UPDATE_QA_CACHE: $UPDATE_QA_CACHE + GITLAB_QA_CACHE_KEY: $GITLAB_QA_CACHE_KEY inherit: variables: - RUBY_VERSION -- GitLab From ab89204b6a574377a61a357676af378ccdb9c537 Mon Sep 17 00:00:00 2001 From: John McDonnell Date: Wed, 12 Apr 2023 13:00:03 +0100 Subject: [PATCH 5/7] Add temporary logic to reduce number of gitaly nodes --- lib/gitlab/qa/component/gitaly_cluster.rb | 19 +++++++++++-------- .../qa/scenario/test/instance/airgapped.rb | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/gitlab/qa/component/gitaly_cluster.rb b/lib/gitlab/qa/component/gitaly_cluster.rb index ae96bb82..4eb991e9 100644 --- a/lib/gitlab/qa/component/gitaly_cluster.rb +++ b/lib/gitlab/qa/component/gitaly_cluster.rb @@ -57,24 +57,24 @@ module Gitlab end # @param [Boolean] parallel_gitaly controls whether we start gitaly nodes in parallel to improve startup time - def instance(parallel_gitaly = false) - run_gitaly_cluster(QA::Release.new(release), parallel_gitaly) + def instance(parallel_gitaly = false, gitaly_nodes = 3) + run_gitaly_cluster(QA::Release.new(release), parallel_gitaly, gitaly_nodes) end # @param [Boolean] parallel_gitaly controls whether we start gitaly nodes in parallel to improve startup time - def run_gitaly_cluster(release, parallel_gitaly = false) + def run_gitaly_cluster(release, parallel_gitaly = false, gitaly_nodes = 3) # This also ensure that the docker network is created here, avoiding any potential race conditions later # if the gitaly-cluster and GitLab containers attempt to create a network in parallel @database_node = postgres Thread.new do Thread.current.abort_on_exception = true - start_gitaly_cluster(release, parallel_gitaly) + start_gitaly_cluster(release, parallel_gitaly, gitaly_nodes) end end # @param [Boolean] parallel_gitaly controls whether we start gitaly nodes in parallel to improve startup time - def start_gitaly_cluster(release, parallel_gitaly = false) # rubocop:disable Metrics/AbcSize + def start_gitaly_cluster(release, parallel_gitaly = false, nodes = 3) # rubocop:disable Metrics/AbcSize Runtime::Logger.info("Starting Gitaly Cluster") if parallel_gitaly @@ -84,9 +84,12 @@ module Gitlab threads << Thread.new { @gitaly_tertiary_node = gitaly(config.tertiary_node_name, config.tertiary_node_port, release) } threads.each(&:join) else - @gitaly_primary_node = gitaly(config.primary_node_name, config.primary_node_port, release) - @gitaly_secondary_node = gitaly(config.secondary_node_name, config.secondary_node_port, release) - @gitaly_tertiary_node = gitaly(config.tertiary_node_name, config.tertiary_node_port, release) + @gitaly_primary_node = gitaly(config.primary_node_name, config.primary_node_port, release) if nodes >= 1 + + name2 = config.secondary_node_name + @gitaly_secondary_node = gitaly(name2, config.secondary_node_port, release) if nodes >= 2 + + @gitaly_tertiary_node = gitaly(config.tertiary_node_name, config.tertiary_node_port, release) if nodes >= 3 end @praefect_node = praefect(release) diff --git a/lib/gitlab/qa/scenario/test/instance/airgapped.rb b/lib/gitlab/qa/scenario/test/instance/airgapped.rb index 639f82ea..e4948e92 100644 --- a/lib/gitlab/qa/scenario/test/instance/airgapped.rb +++ b/lib/gitlab/qa/scenario/test/instance/airgapped.rb @@ -24,7 +24,7 @@ module Gitlab cluster = Component::GitalyCluster.perform do |cluster| cluster.config = @config cluster.release = release - cluster.instance + cluster.instance(false, 1) end gitlab.name = config.gitlab_name gitlab.release = release -- GitLab From 538fee94d80a3c161fc4023f21dee7bba56b2568 Mon Sep 17 00:00:00 2001 From: John McDonnell Date: Thu, 20 Apr 2023 21:17:25 +0100 Subject: [PATCH 6/7] Set log level debug --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0d0c8071..fc17c98e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -160,6 +160,7 @@ package-and-test-with-bundle-false: ALLURE_JOB_NAME: gitlab-qa UPDATE_QA_CACHE: $UPDATE_QA_CACHE GITLAB_QA_CACHE_KEY: $GITLAB_QA_CACHE_KEY + QA_LOG_LEVEL: debug inherit: variables: - RUBY_VERSION -- GitLab From 94108dd927b948285e655524d984e5a9aa8c387b Mon Sep 17 00:00:00 2001 From: John McDonnell Date: Fri, 21 Apr 2023 12:36:59 +0100 Subject: [PATCH 7/7] Fixes --- .gitlab-ci.yml | 1 - lib/gitlab/qa/scenario/test/instance/airgapped.rb | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fc17c98e..0d0c8071 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -160,7 +160,6 @@ package-and-test-with-bundle-false: ALLURE_JOB_NAME: gitlab-qa UPDATE_QA_CACHE: $UPDATE_QA_CACHE GITLAB_QA_CACHE_KEY: $GITLAB_QA_CACHE_KEY - QA_LOG_LEVEL: debug inherit: variables: - RUBY_VERSION diff --git a/lib/gitlab/qa/scenario/test/instance/airgapped.rb b/lib/gitlab/qa/scenario/test/instance/airgapped.rb index e4948e92..e6ebaa9b 100644 --- a/lib/gitlab/qa/scenario/test/instance/airgapped.rb +++ b/lib/gitlab/qa/scenario/test/instance/airgapped.rb @@ -13,7 +13,7 @@ module Gitlab @airgapped_network_name = 'test' @runner_network = 'airgapped' @config = Component::GitalyCluster::GitalyClusterConfig.new( - gitlab_name: "gitlab-airgapped-#{SecureRandom.hex(4)}", + gitlab_name: "gitlab-airgapped", airgapped_network: true, network: airgapped_network_name ) @@ -24,7 +24,7 @@ module Gitlab cluster = Component::GitalyCluster.perform do |cluster| cluster.config = @config cluster.release = release - cluster.instance(false, 1) + cluster.instance(false, 3) end gitlab.name = config.gitlab_name gitlab.release = release -- GitLab