From 92cc2e6232241e1f1da1e1b1c9695b69c92a8bca Mon Sep 17 00:00:00 2001 From: vjrj Date: Wed, 22 Jul 2026 09:56:59 +0200 Subject: [PATCH] ci(release): build fdroid references under /home/vagrant to match F-Droid (reproducibility) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F-Droid's reproducible-build verification failed: the native .so (Flutter libapp/libdartjni + tesseract4android libjpeg/leptonica/pngx/ libtesseract) differed byte-for-byte. Root cause: F-Droid builds under /home/vagrant/build/ (its buildserver path) while our reference built under /tmp/fdroiddata/build/ — the differing absolute build path leaks into the native libs (debug info / embedded paths). Build the reference under /home/vagrant so app + srclib paths match F-Droid's. Also: add a workflow_dispatch ref_tag input to rebuild+re-upload an existing release's reference APKs (idempotent asset replace) without cutting a new tag or re-deploying to Play — so reproducibility fixes can be iterated on the same v* release. --- .forgejo/workflows/release.yml | 119 +++++++++++++++++++++++++-------- 1 file changed, 92 insertions(+), 27 deletions(-) diff --git a/.forgejo/workflows/release.yml b/.forgejo/workflows/release.yml index 71d33aa..d0c9bcc 100644 --- a/.forgejo/workflows/release.yml +++ b/.forgejo/workflows/release.yml @@ -36,7 +36,15 @@ on: - 'v*' # Manual trigger to test the fdroid_reference job on a branch without cutting a # tag or deploying to Play (the play job below is guarded to tags only). + # Pass ref_tag (e.g. v0.1.10) to rebuild the reference APKs for an EXISTING + # release and re-upload them (idempotently), without cutting a new tag — used + # to iterate on build-reproducibility fixes without re-deploying to Play. workflow_dispatch: + inputs: + ref_tag: + description: 'Existing tag to rebuild+re-upload reference APKs for (e.g. v0.1.10). Empty = build-only, no upload.' + required: false + default: '' jobs: play: @@ -129,6 +137,7 @@ jobs: TANE_KEYSTORE_PASSWORD: ${{ secrets.TANE_KEYSTORE_PASSWORD }} TANE_KEY_ALIAS: ${{ secrets.TANE_KEY_ALIAS }} TANE_KEY_PASSWORD: ${{ secrets.TANE_KEY_PASSWORD }} + REF_TAG: ${{ github.event.inputs.ref_tag }} ABI: armeabi-v7a OFFSET: 1 run: | @@ -153,11 +162,15 @@ jobs: # 3) Fetch our in-repo recipe (the single source of truth) at this commit, # plus the pubspec versionCode base for this ABI's versionCode. + # Which app commit/tag to build: on a tag push, the pushed commit; on a + # manual dispatch with ref_tag set, that tag (to rebuild+re-upload an + # existing release's references without cutting a new tag / Play deploy). + BUILD_REF="${REF_TAG:-$GITHUB_SHA}" work=/tmp/tane-src mkdir -p "$work" && cd "$work" git init -q . git remote add origin "http://x-access-token:${TOKEN}@forgejo:3000/${GITHUB_REPOSITORY}.git" - git fetch -q --depth 1 origin "${GITHUB_SHA}" + git fetch -q --depth 1 origin "$BUILD_REF" git checkout -q FETCH_HEAD base=$(sed -n -E 's/^version:.*\+([0-9]+)$/\1/p' apps/app_seeds/pubspec.yaml) [ -n "$base" ] @@ -166,14 +179,16 @@ jobs: # 4) Assemble a minimal fdroiddata (config.yml + srclibs + our recipe) from the # in-repo files. No external host needed — the job container reaches # git.comunes.org (via forgejo:3000) but not gitlab.com. - fdd=/tmp/fdroiddata + # Match F-Droid's buildserver path (~/build/) so the native .so + # embed identical build paths — required for the byte-for-byte repro check. + fdd=/home/vagrant mkdir -p "$fdd/metadata" "$fdd/srclibs" cp "$work/fdroid-ci/config.yml" "$fdd/config.yml" cp "$work"/fdroid-ci/srclibs/*.yml "$fdd/srclibs/" cp "$work/docs/fdroid/org.comunes.tane.yml" "$fdd/metadata/org.comunes.tane.yml" cd "$fdd" # Build THIS commit; drop binary: (we PRODUCE the reference here, not verify it). - sed -i "s#^\( *commit: \).*#\1${GITHUB_SHA}#" metadata/org.comunes.tane.yml + sed -i "s#^\( *commit: \).*#\1${BUILD_REF}#" metadata/org.comunes.tane.yml sed -i '/^ *binary: /d' metadata/org.comunes.tane.yml # Pre-clone the app into fdroid's build dir: fdroidserver computes @@ -185,7 +200,7 @@ jobs: # by the insteadOf rule above). mkdir -p build git clone -q https://git.comunes.org/comunes/tane.git build/org.comunes.tane - git -C build/org.comunes.tane checkout -q "${GITHUB_SHA}" + git -C build/org.comunes.tane checkout -q "${BUILD_REF}" # 5) Build ONLY this ABI's split with F-Droid's own toolchain. Splitting # per-ABI into its own job/matrix-entry (rather than all 3 sequentially in @@ -216,20 +231,32 @@ jobs: "$apksigner" verify --print-certs "$out" | grep -i 'SHA-256' || true rm -f /tmp/ks.jks + # Upload target: the pushed tag, or (on a manual dispatch) the ref_tag + # input. A plain dispatch with no ref_tag just builds+signs, no upload. api="http://forgejo:3000/api/v1/repos/${GITHUB_REPOSITORY}" - tag="${GITHUB_REF_NAME}" + UPLOAD_TAG="" if [ "${GITHUB_REF_TYPE:-}" = tag ]; then + UPLOAD_TAG="${GITHUB_REF_NAME}" + elif [ -n "${REF_TAG:-}" ]; then + UPLOAD_TAG="${REF_TAG}" + fi + if [ -n "$UPLOAD_TAG" ]; then curl -sf -X POST "$api/releases" \ -H "Authorization: token ${TOKEN}" -H 'Content-Type: application/json' \ - -d "{\"tag_name\":\"${tag}\",\"name\":\"${tag}\"}" >/dev/null || true - rid=$(curl -sf -H "Authorization: token ${TOKEN}" "$api/releases/tags/${tag}" \ + -d "{\"tag_name\":\"${UPLOAD_TAG}\",\"name\":\"${UPLOAD_TAG}\"}" >/dev/null || true + rid=$(curl -sf -H "Authorization: token ${TOKEN}" "$api/releases/tags/${UPLOAD_TAG}" \ | python3 -c 'import sys,json;print(json.load(sys.stdin)["id"])') + # Idempotent re-upload: delete any existing asset of this name first + # (the assets POST rejects a duplicate name). + curl -sf -H "Authorization: token ${TOKEN}" "$api/releases/${rid}/assets" \ + | python3 -c "import sys,json;[print(a['id']) for a in json.load(sys.stdin) if a['name']=='app-${ABI}-release.apk']" 2>/dev/null \ + | while read -r aid; do [ -n "$aid" ] && curl -sf -X DELETE -H "Authorization: token ${TOKEN}" "$api/releases/${rid}/assets/${aid}" >/dev/null || true; done curl -sf -X POST "$api/releases/${rid}/assets?name=app-${ABI}-release.apk" \ -H "Authorization: token ${TOKEN}" \ -F "attachment=@${out};type=application/vnd.android.package-archive" - echo "uploaded app-${ABI}-release.apk (from ${f##*/})" + echo "uploaded app-${ABI}-release.apk to ${UPLOAD_TAG} (from ${f##*/})" else - echo "built+signed app-${ABI}-release.apk (dispatch: upload skipped)" + echo "built+signed app-${ABI}-release.apk (dispatch: no ref_tag, upload skipped)" fi fdroid_reference_arm64_v8a: @@ -245,6 +272,7 @@ jobs: TANE_KEYSTORE_PASSWORD: ${{ secrets.TANE_KEYSTORE_PASSWORD }} TANE_KEY_ALIAS: ${{ secrets.TANE_KEY_ALIAS }} TANE_KEY_PASSWORD: ${{ secrets.TANE_KEY_PASSWORD }} + REF_TAG: ${{ github.event.inputs.ref_tag }} ABI: arm64-v8a OFFSET: 2 run: | @@ -259,26 +287,32 @@ jobs: export PYTHONPATH="$fds:$fds/examples" git config --global --add safe.directory '*' git config --global url."http://x-access-token:${TOKEN}@forgejo:3000/".insteadOf "https://git.comunes.org/" + # Which app commit/tag to build: on a tag push, the pushed commit; on a + # manual dispatch with ref_tag set, that tag (to rebuild+re-upload an + # existing release's references without cutting a new tag / Play deploy). + BUILD_REF="${REF_TAG:-$GITHUB_SHA}" work=/tmp/tane-src mkdir -p "$work" && cd "$work" git init -q . git remote add origin "http://x-access-token:${TOKEN}@forgejo:3000/${GITHUB_REPOSITORY}.git" - git fetch -q --depth 1 origin "${GITHUB_SHA}" + git fetch -q --depth 1 origin "$BUILD_REF" git checkout -q FETCH_HEAD base=$(sed -n -E 's/^version:.*\+([0-9]+)$/\1/p' apps/app_seeds/pubspec.yaml) [ -n "$base" ] vc=$((base * 10 + OFFSET)) - fdd=/tmp/fdroiddata + # Match F-Droid's buildserver path (~/build/) so the native .so + # embed identical build paths — required for the byte-for-byte repro check. + fdd=/home/vagrant mkdir -p "$fdd/metadata" "$fdd/srclibs" cp "$work/fdroid-ci/config.yml" "$fdd/config.yml" cp "$work"/fdroid-ci/srclibs/*.yml "$fdd/srclibs/" cp "$work/docs/fdroid/org.comunes.tane.yml" "$fdd/metadata/org.comunes.tane.yml" cd "$fdd" - sed -i "s#^\( *commit: \).*#\1${GITHUB_SHA}#" metadata/org.comunes.tane.yml + sed -i "s#^\( *commit: \).*#\1${BUILD_REF}#" metadata/org.comunes.tane.yml sed -i '/^ *binary: /d' metadata/org.comunes.tane.yml mkdir -p build git clone -q https://git.comunes.org/comunes/tane.git build/org.comunes.tane - git -C build/org.comunes.tane checkout -q "${GITHUB_SHA}" + git -C build/org.comunes.tane checkout -q "${BUILD_REF}" fdroid build --verbose --no-tarball "org.comunes.tane:${vc}" f="unsigned/org.comunes.tane_${vc}.apk" if [ ! -f "$f" ]; then @@ -298,20 +332,32 @@ jobs: --out "$out" "$f" "$apksigner" verify --print-certs "$out" | grep -i 'SHA-256' || true rm -f /tmp/ks.jks + # Upload target: the pushed tag, or (on a manual dispatch) the ref_tag + # input. A plain dispatch with no ref_tag just builds+signs, no upload. api="http://forgejo:3000/api/v1/repos/${GITHUB_REPOSITORY}" - tag="${GITHUB_REF_NAME}" + UPLOAD_TAG="" if [ "${GITHUB_REF_TYPE:-}" = tag ]; then + UPLOAD_TAG="${GITHUB_REF_NAME}" + elif [ -n "${REF_TAG:-}" ]; then + UPLOAD_TAG="${REF_TAG}" + fi + if [ -n "$UPLOAD_TAG" ]; then curl -sf -X POST "$api/releases" \ -H "Authorization: token ${TOKEN}" -H 'Content-Type: application/json' \ - -d "{\"tag_name\":\"${tag}\",\"name\":\"${tag}\"}" >/dev/null || true - rid=$(curl -sf -H "Authorization: token ${TOKEN}" "$api/releases/tags/${tag}" \ + -d "{\"tag_name\":\"${UPLOAD_TAG}\",\"name\":\"${UPLOAD_TAG}\"}" >/dev/null || true + rid=$(curl -sf -H "Authorization: token ${TOKEN}" "$api/releases/tags/${UPLOAD_TAG}" \ | python3 -c 'import sys,json;print(json.load(sys.stdin)["id"])') + # Idempotent re-upload: delete any existing asset of this name first + # (the assets POST rejects a duplicate name). + curl -sf -H "Authorization: token ${TOKEN}" "$api/releases/${rid}/assets" \ + | python3 -c "import sys,json;[print(a['id']) for a in json.load(sys.stdin) if a['name']=='app-${ABI}-release.apk']" 2>/dev/null \ + | while read -r aid; do [ -n "$aid" ] && curl -sf -X DELETE -H "Authorization: token ${TOKEN}" "$api/releases/${rid}/assets/${aid}" >/dev/null || true; done curl -sf -X POST "$api/releases/${rid}/assets?name=app-${ABI}-release.apk" \ -H "Authorization: token ${TOKEN}" \ -F "attachment=@${out};type=application/vnd.android.package-archive" - echo "uploaded app-${ABI}-release.apk (from ${f##*/})" + echo "uploaded app-${ABI}-release.apk to ${UPLOAD_TAG} (from ${f##*/})" else - echo "built+signed app-${ABI}-release.apk (dispatch: upload skipped)" + echo "built+signed app-${ABI}-release.apk (dispatch: no ref_tag, upload skipped)" fi fdroid_reference_x86_64: @@ -327,6 +373,7 @@ jobs: TANE_KEYSTORE_PASSWORD: ${{ secrets.TANE_KEYSTORE_PASSWORD }} TANE_KEY_ALIAS: ${{ secrets.TANE_KEY_ALIAS }} TANE_KEY_PASSWORD: ${{ secrets.TANE_KEY_PASSWORD }} + REF_TAG: ${{ github.event.inputs.ref_tag }} ABI: x86_64 OFFSET: 3 run: | @@ -341,26 +388,32 @@ jobs: export PYTHONPATH="$fds:$fds/examples" git config --global --add safe.directory '*' git config --global url."http://x-access-token:${TOKEN}@forgejo:3000/".insteadOf "https://git.comunes.org/" + # Which app commit/tag to build: on a tag push, the pushed commit; on a + # manual dispatch with ref_tag set, that tag (to rebuild+re-upload an + # existing release's references without cutting a new tag / Play deploy). + BUILD_REF="${REF_TAG:-$GITHUB_SHA}" work=/tmp/tane-src mkdir -p "$work" && cd "$work" git init -q . git remote add origin "http://x-access-token:${TOKEN}@forgejo:3000/${GITHUB_REPOSITORY}.git" - git fetch -q --depth 1 origin "${GITHUB_SHA}" + git fetch -q --depth 1 origin "$BUILD_REF" git checkout -q FETCH_HEAD base=$(sed -n -E 's/^version:.*\+([0-9]+)$/\1/p' apps/app_seeds/pubspec.yaml) [ -n "$base" ] vc=$((base * 10 + OFFSET)) - fdd=/tmp/fdroiddata + # Match F-Droid's buildserver path (~/build/) so the native .so + # embed identical build paths — required for the byte-for-byte repro check. + fdd=/home/vagrant mkdir -p "$fdd/metadata" "$fdd/srclibs" cp "$work/fdroid-ci/config.yml" "$fdd/config.yml" cp "$work"/fdroid-ci/srclibs/*.yml "$fdd/srclibs/" cp "$work/docs/fdroid/org.comunes.tane.yml" "$fdd/metadata/org.comunes.tane.yml" cd "$fdd" - sed -i "s#^\( *commit: \).*#\1${GITHUB_SHA}#" metadata/org.comunes.tane.yml + sed -i "s#^\( *commit: \).*#\1${BUILD_REF}#" metadata/org.comunes.tane.yml sed -i '/^ *binary: /d' metadata/org.comunes.tane.yml mkdir -p build git clone -q https://git.comunes.org/comunes/tane.git build/org.comunes.tane - git -C build/org.comunes.tane checkout -q "${GITHUB_SHA}" + git -C build/org.comunes.tane checkout -q "${BUILD_REF}" fdroid build --verbose --no-tarball "org.comunes.tane:${vc}" f="unsigned/org.comunes.tane_${vc}.apk" if [ ! -f "$f" ]; then @@ -380,18 +433,30 @@ jobs: --out "$out" "$f" "$apksigner" verify --print-certs "$out" | grep -i 'SHA-256' || true rm -f /tmp/ks.jks + # Upload target: the pushed tag, or (on a manual dispatch) the ref_tag + # input. A plain dispatch with no ref_tag just builds+signs, no upload. api="http://forgejo:3000/api/v1/repos/${GITHUB_REPOSITORY}" - tag="${GITHUB_REF_NAME}" + UPLOAD_TAG="" if [ "${GITHUB_REF_TYPE:-}" = tag ]; then + UPLOAD_TAG="${GITHUB_REF_NAME}" + elif [ -n "${REF_TAG:-}" ]; then + UPLOAD_TAG="${REF_TAG}" + fi + if [ -n "$UPLOAD_TAG" ]; then curl -sf -X POST "$api/releases" \ -H "Authorization: token ${TOKEN}" -H 'Content-Type: application/json' \ - -d "{\"tag_name\":\"${tag}\",\"name\":\"${tag}\"}" >/dev/null || true - rid=$(curl -sf -H "Authorization: token ${TOKEN}" "$api/releases/tags/${tag}" \ + -d "{\"tag_name\":\"${UPLOAD_TAG}\",\"name\":\"${UPLOAD_TAG}\"}" >/dev/null || true + rid=$(curl -sf -H "Authorization: token ${TOKEN}" "$api/releases/tags/${UPLOAD_TAG}" \ | python3 -c 'import sys,json;print(json.load(sys.stdin)["id"])') + # Idempotent re-upload: delete any existing asset of this name first + # (the assets POST rejects a duplicate name). + curl -sf -H "Authorization: token ${TOKEN}" "$api/releases/${rid}/assets" \ + | python3 -c "import sys,json;[print(a['id']) for a in json.load(sys.stdin) if a['name']=='app-${ABI}-release.apk']" 2>/dev/null \ + | while read -r aid; do [ -n "$aid" ] && curl -sf -X DELETE -H "Authorization: token ${TOKEN}" "$api/releases/${rid}/assets/${aid}" >/dev/null || true; done curl -sf -X POST "$api/releases/${rid}/assets?name=app-${ABI}-release.apk" \ -H "Authorization: token ${TOKEN}" \ -F "attachment=@${out};type=application/vnd.android.package-archive" - echo "uploaded app-${ABI}-release.apk (from ${f##*/})" + echo "uploaded app-${ABI}-release.apk to ${UPLOAD_TAG} (from ${f##*/})" else - echo "built+signed app-${ABI}-release.apk (dispatch: upload skipped)" + echo "built+signed app-${ABI}-release.apk (dispatch: no ref_tag, upload skipped)" fi