From ede592f2dd61b0c51911803606118828d6bf4af9 Mon Sep 17 00:00:00 2001 From: SilentGhost Date: Mon, 28 Oct 2024 11:17:03 +0100 Subject: [PATCH 1/3] Ensure that new commits are created on top of fdroiddata:master Make sure new commits are created in a runner-specific branch that has parent of fdroidata.git:master Removed confused and outdated code --- .gitlab-ci.yml | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b9376e8..5fb29a8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,8 +20,6 @@ checkupdates_runner: image: debian:bookworm-slim - variables: - GIT_DEPTH: "1" parallel: 10 tags: - fdroid @@ -40,27 +38,16 @@ checkupdates_runner: - printf 'Ciphers -chacha20-poly1305@openssh.com,*-cbc\nMACs -*etm*,*-sha1*\n' > /etc/ssh/ssh_config.d/0-terrapin-workaround.conf - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh + - mkdir -pm 700 ~/.ssh - cp "${GITLAB_KNOWN_HOSTS}" ~/.ssh/known_hosts - chmod 644 ~/.ssh/known_hosts - eval $(ssh-agent -s) - echo "${CHECKUPDATES_SSH_DEPLOY_KEY}" | tr -d '\r' | base64 --decode | ssh-add - - - export job_branch="${CI_JOB_NAME//[^a-zA-Z0-9]/_}-$CI_JOB_ID" - - - TARGET_PROJECT_PATH=fdroid/checkupdates-bot-fdroiddata - - git clone --depth $GIT_DEPTH https://gitlab.com/${TARGET_PROJECT_PATH}.git fdroiddata + - git clone --depth 1 git@gitlab.com:fdroid/checkupdates-bot-fdroiddata.git fdroiddata - cd fdroiddata - - git pull --rebase --strategy-option=ours https://gitlab.com/fdroid/fdroiddata.git master || true - - git remote set-url --push origin git@gitlab.com:${TARGET_PROJECT_PATH}.git - # reset repo to origin state before adding new commits - - git rebase --abort || true - - rm -fr ".git/rebase-apply" - # https://gitlab.com/gitlab-org/gitlab-runner/-/issues/29187 - - rm -f ".git/logs/HEAD.lock" - - git checkout master || true - - git reset --hard || true + - git remote add -ft master upstream https://gitlab.com/fdroid/fdroiddata.git + - git checkout -b "${CI_JOB_NAME//[^a-zA-Z0-9]/_}-$CI_JOB_ID" upstream/master script: # Get app IDs in this batch - | @@ -78,7 +65,6 @@ checkupdates_runner: export CHECKUPDATES_APPIDS - - echo "$job_branch" - echo "$metadata_files_count" - echo "$batch_size" - echo "${metadata_files_batch[@]}" @@ -87,14 +73,11 @@ checkupdates_runner: - echo "${#CHECKUPDATES_APPIDS[@]}" - fdroid checkupdates --allow-dirty --auto --commit "${CHECKUPDATES_APPIDS[@]}" 2>&1 | tee /tmp/out || true - # when two jobs try to push at the same time they occasionally fail, so try it again - - while ! git push --force origin HEAD:$job_branch + - | + git push origin -o merge_request.create -o merge_request.title="$CI_JOB_NAME $CI_JOB_ID" -o merge_request.description="$CI_JOB_URL" -o merge_request.remove_source_branch - ; do - git pull --rebase --strategy-option=ours origin master; - done - echo "============== Summary =====================" - 'grep -v "INFO: Processing" /tmp/out || true' -- GitLab From 3acb3982308665047e015265507d861cb4cf141f Mon Sep 17 00:00:00 2001 From: SilentGhost Date: Wed, 30 Oct 2024 12:15:30 +0100 Subject: [PATCH 2/3] Single MR per app --- .gitlab-ci.yml | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5fb29a8..ae46f57 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,13 +43,10 @@ checkupdates_runner: - chmod 644 ~/.ssh/known_hosts - eval $(ssh-agent -s) - echo "${CHECKUPDATES_SSH_DEPLOY_KEY}" | tr -d '\r' | base64 --decode | ssh-add - - + script: - git clone --depth 1 git@gitlab.com:fdroid/checkupdates-bot-fdroiddata.git fdroiddata - cd fdroiddata - git remote add -ft master upstream https://gitlab.com/fdroid/fdroiddata.git - - git checkout -b "${CI_JOB_NAME//[^a-zA-Z0-9]/_}-$CI_JOB_ID" upstream/master - script: - # Get app IDs in this batch - | metadata_files=(metadata/*.yml) metadata_files_count=${#metadata_files[@]} @@ -72,12 +69,17 @@ checkupdates_runner: - echo "${CHECKUPDATES_APPIDS[@]}" - echo "${#CHECKUPDATES_APPIDS[@]}" - - fdroid checkupdates --allow-dirty --auto --commit "${CHECKUPDATES_APPIDS[@]}" 2>&1 | tee /tmp/out || true - | - git push origin - -o merge_request.create - -o merge_request.title="$CI_JOB_NAME $CI_JOB_ID" - -o merge_request.description="$CI_JOB_URL" - -o merge_request.remove_source_branch + for appid in "${CHECKUPDATES_APPIDS[@]}"; do + git checkout -b "$appid-$CI_JOB_ID" upstream/master + fdroid checkupdates --allow-dirty --auto --commit "$appid" 2>&1 | tee -a /tmp/out || true + if ! git diff --quiet upstream/master.. ; then + git push -u origin + -o merge_request.create + -o merge_request.title="checkupdates $appid $CI_JOB_ID" + -o merge_request.description="$CI_JOB_URL" + -o merge_request.remove_source_branch + fi + done - echo "============== Summary =====================" - 'grep -v "INFO: Processing" /tmp/out || true' -- GitLab From 60b2d2098d9d7935c0f788012bf748843fa6d099 Mon Sep 17 00:00:00 2001 From: SilentGhost Date: Wed, 30 Oct 2024 11:52:16 +0100 Subject: [PATCH 3/3] Filter out inactive apps early Skip processing of apps with UpdateCheckMode: None or Static. This saves time on parsing more than 1800 yaml files. Extract appid directly without a for loop. --- .gitlab-ci.yml | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ae46f57..06d3f7e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,24 +48,12 @@ checkupdates_runner: - cd fdroiddata - git remote add -ft master upstream https://gitlab.com/fdroid/fdroiddata.git - | - metadata_files=(metadata/*.yml) - metadata_files_count=${#metadata_files[@]} - batch_size=$((metadata_files_count / CI_NODE_TOTAL + 1)) - metadata_files_batch=("${metadata_files[@]:$(((CI_NODE_INDEX - 1) * batch_size)):$batch_size}") - - declare -a CHECKUPDATES_APPIDS - for file in "${metadata_files_batch[@]}"; do - filename="${file##*/}" - filename_without_extension="${filename%.*}" - CHECKUPDATES_APPIDS=("${CHECKUPDATES_APPIDS[@]}" "$filename_without_extension") - done - - export CHECKUPDATES_APPIDS - - - echo "$metadata_files_count" + appids=($(grep -PL '^UpdateCheckMode: (None|Static)' metadata/*yml | grep -oP '(?<=metadata/).*(?=.yml)')) + - appids_count=${#appids[@]} + - batch_size=$((appids_count / CI_NODE_TOTAL + 1)) + - CHECKUPDATES_APPIDS=("${appids[@]:$(((CI_NODE_INDEX - 1) * batch_size)):$batch_size}") + - echo "$appids_count" - echo "$batch_size" - - echo "${metadata_files_batch[@]}" - - echo "${#metadata_files_batch[@]}" - echo "${CHECKUPDATES_APPIDS[@]}" - echo "${#CHECKUPDATES_APPIDS[@]}" -- GitLab