diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8280332c2398b78dfe3b3045bc2d3d3e9b361ef..2a94e773560a355d07863092c194f901bca329a9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,30 +46,76 @@ lint commits: script: test/lint/commits allow_failure: true -# Build - +# Build Template .build_template: &build_definition stage: build script: - export RECALBOX_VERSION="${CI_COMMIT_TAG:-${CI_COMMIT_REF_SLUG:0:12} (${CI_COMMIT_SHA:0:8}) ${CI_PIPELINE_ID} ${ARCH} $(date '+%Y/%m/%d %H:%M:%S')}" - docker build -t "recalbox-${ARCH}" . - docker run --rm -v `pwd`:/work -v /recalbox-builds/dl:/share/dl -v "/recalbox-builds/ccaches/ccache-${ARCH}:/share/ccache" -e "ARCH=${ARCH}" -e "RECALBOX_VERSION=${RECALBOX_VERSION}" -e "RECALBOX_CCACHE_ENABLED=true" "recalbox-${ARCH}" 2>&1 | tee build.log | grep '>>>' || tac build.log | grep '>>>' -m 1 -B 9999 | tac - - export DIST_DIR="dist/${ARCH}" - - mkdir -p "${DIST_DIR}" - - cp output/images/recalbox/* "${DIST_DIR}" + - export BUILD_DIR="build-${CI_PIPELINE_ID}" + - export ARTIFACT_DIR="${BUILD_DIR}/${ARCH}" + - mkdir -p "${ARTIFACT_DIR}" + - cp output/images/recalbox/* "${ARTIFACT_DIR}" + - rm -rf output/ + - echo "${RECALBOX_VERSION}" >> "${ARTIFACT_DIR}/recalbox.version" + - cp CHANGELOG.md "${ARTIFACT_DIR}/recalbox.changelog" + - ./updatesv3/bucket.sh upload "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}" "recalbox-build" "${BUILD_DIR}" + after_script: + - rm -rf buildroot + artifacts: + name: ${ARCH}-${CI_PIPELINE_ID} + when: always + paths: + - build.log + expire_in: 2 mos + +# Build Template +fake_build: + stage: build + script: + - export ARCH='fake' + - export RECALBOX_VERSION="${CI_COMMIT_TAG:-${CI_COMMIT_REF_SLUG:0:12} (${CI_COMMIT_SHA:0:8}) ${CI_PIPELINE_ID} ${ARCH} $(date '+%Y/%m/%d %H:%M:%S')}" + - export BUILD_DIR="build-${CI_PIPELINE_ID}" + - export ARTIFACT_DIR="${BUILD_DIR}/${ARCH}" + - mkdir -p "${ARTIFACT_DIR}" + - echo "animage" > "${ARTIFACT_DIR}/image.tar.xz" + - echo "root" > "${ARTIFACT_DIR}/root.tar.xz" - rm -rf output/ - - echo "${RECALBOX_VERSION}" >> "${DIST_DIR}/recalbox.version" - - cp CHANGELOG.md "${DIST_DIR}/recalbox.changelog" + - echo "${RECALBOX_VERSION}" >> "${ARTIFACT_DIR}/recalbox.version" + - cp CHANGELOG.md "${ARTIFACT_DIR}/recalbox.changelog" + - ./updatesv3/bucket.sh upload "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}" "recalbox-build" "${BUILD_DIR}" after_script: - rm -rf buildroot artifacts: - name: dist-${ARCH}-${CI_BUILD_ID} + name: ${ARCH}-${CI_PIPELINE_ID} when: always paths: - - dist - build.log expire_in: 2 mos + +# Release Template +.release_template: &release_definition + stage: release + variables: + DEPLOY_TYPE: type of the deployment (prod, review) + SKIP_IMAGES: Release or without images if true + BUCKET: Bucket to deploy to + BUCKET_PATH: path in the bucket + script: + - export DOWNLOAD_DIR="build-${CI_PIPELINE_ID}" + - export RELEASE_DIR="${BUCKET_PATH}" + - ./updatesv3/bucket.sh download "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}" "recalbox-build" "${DOWNLOAD_DIR}" + - mv "${DOWNLOAD_DIR}" "${RELEASE_DIR}" + - cd updatesv2 + - ./prepare_assets.sh "${DEPLOY_TYPE}" "../release" "${RELEASE_DIR}" netlify "${SKIP_IMAGES}" + - echo "Releasing files" + - cd .. + - ./updatesv3/bucket.sh upload "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}" "${BUCKET}" "${RELEASE_DIR}" + - if [ "${DEPLOY_TYPE}" = "prod" ]; then docker run --rm -v $(pwd):/work -w /work node:10 bash -c "./upload_to_netlify.sh '${NETLIFY_AUTH_TOKEN}' 'netlify' '${NETLIFY_PROJECT_ID}'"; fi + +# Build build rpi1: <<: *build_definition only: @@ -170,21 +216,6 @@ build odroidxu4 for test: ### Release -.release_template: &release_definition - stage: release - variables: - DEPLOY_TYPE: type of the deployment (prod, review) - SKIP_IMAGES: Release or without images if true - BUCKET: Bucket to deploy to - BUCKET_PATH: path in the bucket - script: - - cd updatesv2 - - ./prepare_assets.sh "${DEPLOY_TYPE}" "../dist" release netlify "${SKIP_IMAGES}" - - echo "Releasing files" - - ./upload_to_bucket.sh "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}" "${BUCKET}" "release" "${BUCKET_PATH}" - - if [ "${DEPLOY_TYPE}" = "prod" ]; then docker run --rm -v $(pwd):/work -w /work node:10 bash -c "./upload_to_netlify.sh '${NETLIFY_AUTH_TOKEN}' 'netlify' '${NETLIFY_PROJECT_ID}'"; fi - - release prod: <<: *release_definition when: manual @@ -215,15 +246,14 @@ release review: ### Clean clean review: - image: registry.gitlab.com/recalbox/ops/rancher-cli:0.6.2 stage: clean when: manual - dependencies: [] only: - branches script: - - echo "cleaning ${ARCHIVE_IMAGE} from ${REVIEW_STACK_NAME}/${CI_ENVIRONMENT_SLUG}" - - rancher rm "${REVIEW_STACK_NAME}/${CI_ENVIRONMENT_SLUG}" + - export BUILD_DIR="build-${CI_PIPELINE_ID}" + - echo "Removing ${BUILD_DIR} from recalbox-build" + - ./updatesv3/bucket.sh delete "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}" "recalbox-build" "${BUILD_DIR}" environment: name: review/${CI_COMMIT_REF_SLUG} action: stop diff --git a/CHANGELOG.md b/CHANGELOG.md index b132915542c45d1c50d50f4eda9434026b5c6221..4d4a465f9cd3257ddfe5a7e5bf07c07da536425c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). ## [6.2] +- Fixed ci artifact upload for gigantic files - Add joystick rules for IPega PG-9037 - Fix Odroid XU4 fan issue - Fix Odroid XU4 sound issue diff --git a/updatesv2/prepare_assets.sh b/updatesv2/prepare_assets.sh index 3ed7fa8c6c765c1b935756154135c9fd7d176f60..61d2531cf49ebe82d98e9ca9e024be4e705fb2fb 100755 --- a/updatesv2/prepare_assets.sh +++ b/updatesv2/prepare_assets.sh @@ -24,7 +24,6 @@ if [ "${SKIP_IMAGES}" == "true" ]; then fi # Noobs compatibility - if [ "${RELEASE_TYPE}" == "prod" ]; then UPGRADE_DIR="${IMAGE_DIR}/" SOURCE_NOOBS_DIR="./noobs/" @@ -43,6 +42,7 @@ if [ "${RELEASE_TYPE}" == "prod" ]; then done done fi + # Template html cp "${RELEASE_TYPE}.template.html" "${RELEASE_DIR}/index.html" diff --git a/updatesv3/bucket.sh b/updatesv3/bucket.sh new file mode 100755 index 0000000000000000000000000000000000000000..119be920f4fd4b4c713aea5f588454674ebc92f8 --- /dev/null +++ b/updatesv3/bucket.sh @@ -0,0 +1,43 @@ +#!/bin/sh +set -eux + +if [ -z "${1}" -o -z "${2}" -o -z "${3}" -o -z "${4}" -o -z "${5}" ]; then + echo "This script need 5 parameters" + exit 1 +fi + +SCRIPT=`realpath $0` +BASEDIR=`dirname $SCRIPT` +OPERATION="${1}" # upload or download +AWS_ACCESS_KEY_ID="${2}" +AWS_SECRET_ACCESS_KEY="${3}" +BUCKET="${4}" # bucket name +FROM_FOLDER="${5}" # for download : bucket folder to download | for upload : folder to upload +TO_FOLDER="${6:-}" # for download : folder where to download | for upload : bucket folder where files will be uploaded + +cat >s3cfg <