diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ca272452861f60d4a64bb3df6b33b60c858b58c2..bc1a35dbe490319a88772e4a36000db627f6af91 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1109,39 +1109,23 @@ no_ee_check: - //@gitlab-org/gitlab-ce # GitLab EE Review apps -review-app-image: - <<: *single-script-job - stage: test - allow_failure: true - variables: - <<: *single-script-job-variables - SCRIPT_NAME: trigger-build - script: - - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./$SCRIPT_NAME cng - when: manual - only: - refs: - - branches - except: - refs: - - master - - /(^docs[\/-].*|.*-docs$)/ - review: - <<: *single-script-job - image: registry.gitlab.com/charts/gitlab:latest - stage: post-test + <<: *dedicated-no-docs-pull-cache-job + image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base + stage: test allow_failure: true + before_script: [] variables: - <<: *single-script-job-variables - SCRIPT_NAME: review-apps.sh + GIT_DEPTH: "1" HOST_SUFFIX: "$CI_ENVIRONMENT_SLUG" DOMAIN: "-$CI_ENVIRONMENT_SLUG.$REVIEW_APPS_DOMAIN" GITLAB_HELM_CHART_REF: "master" script: - - source ./$SCRIPT_NAME - - export GITLAB_SHELL_VERSION=$(< GITLAB_SHELL_VERSION) - - export GITALY_VERSION=$(< GITALY_SERVER_VERSION) + - export GITLAB_SHELL_VERSION=$(&1) + command << '|' << %(grep "#{release_name}") + command << '|' << "awk '{print $1}'" + command << '|' << %(xargs kubectl -n "#{namespace}" delete) + command << '||' << 'true' + + run_command(command) + end + + private + + def run_command(command) + puts "Running command: `#{command.join(' ')}`" # rubocop:disable Rails/Output + + Gitlab::Popen.popen_with_detail(command) + end + end +end diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb new file mode 100755 index 0000000000000000000000000000000000000000..ea53f89c8448f1be7e3997d4a15ad9ee4e044ee5 --- /dev/null +++ b/scripts/review_apps/automated_cleanup.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +require 'gitlab' +require_relative File.expand_path('../../lib/quality/helm_client.rb', __dir__) +require_relative File.expand_path('../../lib/quality/kubernetes_client.rb', __dir__) + +class AutomatedCleanup + attr_reader :project_path, :gitlab_token, :cleaned_up_releases + + def initialize(project_path: ENV['CI_PROJECT_PATH'], gitlab_token: ENV['GITLAB_BOT_REVIEW_APPS_CLEANUP_TOKEN']) + @project_path = project_path + @gitlab_token = gitlab_token + @cleaned_up_releases = [] + end + + def gitlab + @gitlab ||= begin + Gitlab.configure do |config| + config.endpoint = 'https://gitlab.com/api/v4' + # gitlab-bot's token "GitLab review apps cleanup" + config.private_token = gitlab_token + end + + Gitlab + end + end + + def helm + @helm ||= Quality::HelmClient.new + end + + def kubernetes + @kubernetes ||= Quality::KubernetesClient.new + end + + def perform_gitlab_environment_cleanup!(days_for_stop:, days_for_delete:) + puts "Checking for review apps not updated in the last #{days_for_stop} days..." + + checked_environments = [] + delete_threshold = threshold_time(days: days_for_delete) + stop_threshold = threshold_time(days: days_for_stop) + gitlab.deployments(project_path, per_page: 50).auto_paginate do |deployment| + next unless deployment.environment.name.start_with?('review/') + next if checked_environments.include?(deployment.environment.slug) + + puts + + checked_environments << deployment.environment.slug + deployed_at = Time.parse(deployment.created_at) + + if deployed_at < delete_threshold + print_release_state(subject: 'Review app', release_name: deployment.environment.slug, release_date: deployment.created_at, action: 'deleting') + gitlab.delete_environment(project_path, deployment.environment.id) + cleaned_up_releases << deployment.environment.slug + elsif deployed_at < stop_threshold + print_release_state(subject: 'Review app', release_name: deployment.environment.slug, release_date: deployment.created_at, action: 'stopping') + gitlab.stop_environment(project_path, deployment.environment.id) + cleaned_up_releases << deployment.environment.slug + else + print_release_state(subject: 'Review app', release_name: deployment.environment.slug, release_date: deployment.created_at, action: 'leaving') + end + end + end + + def perform_helm_releases_cleanup!(days:) + puts "Checking for Helm releases not updated in the last #{days} days..." + + threshold_day = threshold_time(days: days) + helm.releases(args: ['--deployed', '--failed', '--date', '--reverse', '--max 25']).each do |release| + next if cleaned_up_releases.include?(release.name) + + if release.last_update < threshold_day + print_release_state(subject: 'Release', release_name: release.name, release_date: release.last_update, action: 'cleaning') + helm.delete(release_name: release.name) + kubernetes.cleanup(release_name: release.name) + else + print_release_state(subject: 'Release', release_name: release.name, release_date: release.last_update, action: 'leaving') + end + end + end + + def threshold_time(days:) + Time.now - days * 24 * 3600 + end + + def print_release_state(subject:, release_name:, release_date:, action:) + puts "\n#{subject} '#{release_name}' was last deployed on #{release_date}: #{action} it." + end +end + +def timed(task) + start = Time.now + yield(self) + puts "#{task} finished in #{Time.now - start} seconds.\n" +end + +automated_cleanup = AutomatedCleanup.new + +timed('Review apps cleanup') do + automated_cleanup.perform_gitlab_environment_cleanup!(days_for_stop: 5, days_for_delete: 6) +end + +puts + +timed('Helm releases cleanup') do + automated_cleanup.perform_helm_releases_cleanup!(days: 7) +end + +exit(0) diff --git a/scripts/review-apps.sh b/scripts/review_apps/review-apps.sh similarity index 89% rename from scripts/review-apps.sh rename to scripts/review_apps/review-apps.sh index 3ef070c87e23a7dc6998eaf750de9e935ac5b754..21f3ddb77a539f158a90bb8e261747bb01682dc0 100755 --- a/scripts/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -83,11 +83,15 @@ function deploy() { gitlab_migrations_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ce" gitlab_sidekiq_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ce" gitlab_unicorn_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ce" + gitlab_gitaly_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly" + gitlab_shell_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell" + gitlab_workhorse_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ce" if [[ "$CI_PROJECT_NAME" == "gitlab-ee" ]]; then gitlab_migrations_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ee" gitlab_sidekiq_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ee" gitlab_unicorn_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ee" + gitlab_workhorse_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ee" fi # canary uses stable db @@ -117,6 +121,7 @@ function deploy() { helm repo add gitlab https://charts.gitlab.io/ helm dep update . +HELM_CMD=$(cat << EOF helm upgrade --install \ --wait \ --timeout 600 \ @@ -142,10 +147,19 @@ function deploy() { --set gitlab.gitaly.image.tag="v$GITALY_VERSION" \ --set gitlab.gitlab-shell.image.repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell" \ --set gitlab.gitlab-shell.image.tag="v$GITLAB_SHELL_VERSION" \ + --set gitlab.unicorn.workhorse.image="$gitlab_workhorse_image_repository" \ + --set gitlab.unicorn.workhorse.tag="$CI_COMMIT_REF_NAME" \ --namespace="$KUBE_NAMESPACE" \ --version="$CI_PIPELINE_ID-$CI_JOB_ID" \ "$name" \ . +EOF +) + + echo "Deploying with:" + echo $HELM_CMD + + eval $HELM_CMD } function delete() { @@ -155,10 +169,13 @@ function delete() { if [[ "$track" != "stable" ]]; then name="$name-$track" fi + + echo "Deleting release '$name'..." helm delete --purge "$name" || true } function cleanup() { + echo "Cleaning up $CI_ENVIRONMENT_SLUG..." kubectl -n "$KUBE_NAMESPACE" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa 2>&1 \ | grep "$CI_ENVIRONMENT_SLUG" \ | awk '{print $1}' \ diff --git a/scripts/trigger-build b/scripts/trigger-build index 74fbb55613702e4a2c0c8733c25769e55d540e37..5ee3b77bc6fdbf23518e6aa42072431868277abd 100755 --- a/scripts/trigger-build +++ b/scripts/trigger-build @@ -158,7 +158,8 @@ module Trigger def status req = Net::HTTP::Get.new(@uri) - req['PRIVATE-TOKEN'] = ENV['GITLAB_QA_ACCESS_TOKEN'] + # gitlab-bot's token "GitLab multi-project pipeline polling" + req['PRIVATE-TOKEN'] = ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN'] res = Net::HTTP.start(@uri.hostname, @uri.port, use_ssl: true) do |http| http.request(req) diff --git a/spec/lib/quality/helm_client_spec.rb b/spec/lib/quality/helm_client_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..553cd8719de66faa5807f9fe5d1cca32aa27b784 --- /dev/null +++ b/spec/lib/quality/helm_client_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Quality::HelmClient do + let(:namespace) { 'review-apps-ee' } + let(:release_name) { 'my-release' } + let(:raw_helm_list_result) do + <<~OUTPUT + NAME REVISION UPDATED STATUS CHART NAMESPACE + review-improve-re-2dsd9d 1 Tue Jul 31 15:53:17 2018 FAILED gitlab-0.3.4 #{namespace} + review-11-1-stabl-3r2fso 1 Mon Jul 30 22:44:14 2018 FAILED gitlab-0.3.3 #{namespace} + review-49375-css-fk664j 1 Thu Jul 19 11:01:30 2018 FAILED gitlab-0.2.4 #{namespace} + OUTPUT + end + + subject { described_class.new(namespace: namespace) } + + describe '#releases' do + it 'calls helm list with default arguments' do + expect(Gitlab::Popen).to receive(:popen_with_detail) + .with([%(helm list --namespace "#{namespace}")]) + .and_return(Gitlab::Popen::Result.new([], '')) + + subject.releases + end + + it 'calls helm list with given arguments' do + expect(Gitlab::Popen).to receive(:popen_with_detail) + .with([%(helm list --namespace "#{namespace}" --deployed)]) + .and_return(Gitlab::Popen::Result.new([], '')) + + subject.releases(args: ['--deployed']) + end + + it 'returns a list of Release objects' do + expect(Gitlab::Popen).to receive(:popen_with_detail) + .with([%(helm list --namespace "#{namespace}" --deployed)]) + .and_return(Gitlab::Popen::Result.new([], raw_helm_list_result)) + + releases = subject.releases(args: ['--deployed']) + + expect(releases.size).to eq(3) + expect(releases[0].name).to eq('review-improve-re-2dsd9d') + expect(releases[0].revision).to eq(1) + expect(releases[0].last_update).to eq(Time.parse('Tue Jul 31 15:53:17 2018')) + expect(releases[0].status).to eq('FAILED') + expect(releases[0].chart).to eq('gitlab-0.3.4') + expect(releases[0].namespace).to eq(namespace) + end + end + + describe '#delete' do + it 'calls helm delete with default arguments' do + expect(Gitlab::Popen).to receive(:popen_with_detail) + .with(["helm delete --purge #{release_name}"]) + .and_return(Gitlab::Popen::Result.new([], '', '', 0)) + + expect(subject.delete(release_name: release_name).status).to eq(0) + end + end +end diff --git a/spec/lib/quality/kubernetes_client_spec.rb b/spec/lib/quality/kubernetes_client_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..3c0c0d0977a81faea95d94e47a7683a4d922967a --- /dev/null +++ b/spec/lib/quality/kubernetes_client_spec.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Quality::KubernetesClient do + subject { described_class.new(namespace: 'review-apps-ee') } + + describe '#cleanup' do + it 'calls kubectl with the correct arguments' do + # popen_with_detail will receive an array with a bunch of arguments; we're + # only concerned with it having the correct namespace and release name + expect(Gitlab::Popen).to receive(:popen_with_detail) do |args| + expect(args) + .to satisfy_one { |arg| arg.start_with?('-n "review-apps-ee" get') } + expect(args) + .to satisfy_one { |arg| arg == 'grep "my-release"' } + expect(args) + .to satisfy_one { |arg| arg.end_with?('-n "review-apps-ee" delete') } + end + + # We're not verifying the output here, just silencing it + expect { subject.cleanup(release_name: 'my-release') }.to output.to_stdout + end + end +end