diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b9376e80ed63ada4df41c24e7e938fc130023d0d..06d3f7e7fee984cc20e0888f2d406d8a4aa0b0f4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,8 +20,6 @@ checkupdates_runner: image: debian:bookworm-slim - variables: - GIT_DEPTH: "1" parallel: 10 tags: - fdroid @@ -40,61 +38,36 @@ checkupdates_runner: - printf 'Ciphers -chacha20-poly1305@openssh.com,*-cbc\nMACs -*etm*,*-sha1*\n' > /etc/ssh/ssh_config.d/0-terrapin-workaround.conf - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh + - mkdir -pm 700 ~/.ssh - cp "${GITLAB_KNOWN_HOSTS}" ~/.ssh/known_hosts - chmod 644 ~/.ssh/known_hosts - eval $(ssh-agent -s) - echo "${CHECKUPDATES_SSH_DEPLOY_KEY}" | tr -d '\r' | base64 --decode | ssh-add - - - - export job_branch="${CI_JOB_NAME//[^a-zA-Z0-9]/_}-$CI_JOB_ID" - - - TARGET_PROJECT_PATH=fdroid/checkupdates-bot-fdroiddata - - git clone --depth $GIT_DEPTH https://gitlab.com/${TARGET_PROJECT_PATH}.git fdroiddata - - cd fdroiddata - - git pull --rebase --strategy-option=ours https://gitlab.com/fdroid/fdroiddata.git master || true - - git remote set-url --push origin git@gitlab.com:${TARGET_PROJECT_PATH}.git - # reset repo to origin state before adding new commits - - git rebase --abort || true - - rm -fr ".git/rebase-apply" - # https://gitlab.com/gitlab-org/gitlab-runner/-/issues/29187 - - rm -f ".git/logs/HEAD.lock" - - git checkout master || true - - git reset --hard || true script: - # Get app IDs in this batch + - git clone --depth 1 git@gitlab.com:fdroid/checkupdates-bot-fdroiddata.git fdroiddata + - cd fdroiddata + - git remote add -ft master upstream https://gitlab.com/fdroid/fdroiddata.git - | - metadata_files=(metadata/*.yml) - metadata_files_count=${#metadata_files[@]} - batch_size=$((metadata_files_count / CI_NODE_TOTAL + 1)) - metadata_files_batch=("${metadata_files[@]:$(((CI_NODE_INDEX - 1) * batch_size)):$batch_size}") - - declare -a CHECKUPDATES_APPIDS - for file in "${metadata_files_batch[@]}"; do - filename="${file##*/}" - filename_without_extension="${filename%.*}" - CHECKUPDATES_APPIDS=("${CHECKUPDATES_APPIDS[@]}" "$filename_without_extension") - done - - export CHECKUPDATES_APPIDS - - - echo "$job_branch" - - echo "$metadata_files_count" + appids=($(grep -PL '^UpdateCheckMode: (None|Static)' metadata/*yml | grep -oP '(?<=metadata/).*(?=.yml)')) + - appids_count=${#appids[@]} + - batch_size=$((appids_count / CI_NODE_TOTAL + 1)) + - CHECKUPDATES_APPIDS=("${appids[@]:$(((CI_NODE_INDEX - 1) * batch_size)):$batch_size}") + - echo "$appids_count" - echo "$batch_size" - - echo "${metadata_files_batch[@]}" - - echo "${#metadata_files_batch[@]}" - echo "${CHECKUPDATES_APPIDS[@]}" - echo "${#CHECKUPDATES_APPIDS[@]}" - - fdroid checkupdates --allow-dirty --auto --commit "${CHECKUPDATES_APPIDS[@]}" 2>&1 | tee /tmp/out || true - # when two jobs try to push at the same time they occasionally fail, so try it again - - while ! git push --force origin HEAD:$job_branch - -o merge_request.create - -o merge_request.title="$CI_JOB_NAME $CI_JOB_ID" - -o merge_request.description="$CI_JOB_URL" - -o merge_request.remove_source_branch - ; do - git pull --rebase --strategy-option=ours origin master; + - | + for appid in "${CHECKUPDATES_APPIDS[@]}"; do + git checkout -b "$appid-$CI_JOB_ID" upstream/master + fdroid checkupdates --allow-dirty --auto --commit "$appid" 2>&1 | tee -a /tmp/out || true + if ! git diff --quiet upstream/master.. ; then + git push -u origin + -o merge_request.create + -o merge_request.title="checkupdates $appid $CI_JOB_ID" + -o merge_request.description="$CI_JOB_URL" + -o merge_request.remove_source_branch + fi done - echo "============== Summary =====================" - 'grep -v "INFO: Processing" /tmp/out || true'