From 9b46e590afa089943bd9c0ae4d38727b05a32b44 Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Fri, 11 Aug 2023 19:12:44 +0200 Subject: [PATCH] [TESTS] upgrade tests for storage (cherry picked from commit 884ca63738cc2e2c7cde31c649e9fa77cd590044) (cherry picked from commit 0a45d9c37b187249d347020ca79e71c523177d61) [TESTS] upgrade tests for storage (squash) relative paths (cherry picked from commit 3bb19285f365ae0b56dedc183db04504946a2a50) (cherry picked from commit c640c09e6110a8c684dfcc8f9fc05495f30c16f1) (cherry picked from commit 40ffe2d226f1ea1e78ae40eb0deeccc0838b5d48) [UPGRADE] S3 storage and fixtures (cherry picked from commit c466c9c6571b18b1426e07439e7aa6445f5d86dd) (cherry picked from commit e80abbe2cd7968c5c5601bf5e39de9b265e40ca4) (cherry picked from commit 50a47df1d17df60cda6504165d5b02ac42e63c86) [UPGRADE] add sanity checks for [storage*] (squash) speedup upgrade tests (cherry picked from commit f578279cfe85d4e23b83d3759ec782790fed869d) (cherry picked from commit 6dd677f151d3ba3b6f016d788585372958608cd6) (cherry picked from commit ee7e7eca27a2e123188cdf05fb493f51c8c8a3e4) (cherry picked from commit 879ffca697e6cd04d364ad46a6d4a2d957ba3eb3) (cherry picked from commit 60b33cc5af4f8ad4e2f27034a7cff98959f44128) (cherry picked from commit a75b450c7b45f6647ca432bce4cdaebdea70dc47) (cherry picked from commit 4e8712c52377533fada2cc747d20784275b03aa4) (cherry picked from commit f6447567a464d2320d26fcccf7fc7293dc6e2a44) (cherry picked from commit e0ab67536b89d161eaa26c7c69988498c986b227) (cherry picked from commit 2f9dc0b4cc2f5363e6b02da8bba5cd6e01c9ea8f) --- .forgejo/upgrades/default-app.ini | 30 ++ .forgejo/upgrades/fixtures.sh | 187 +++++++ .forgejo/upgrades/legagy-relative-app.ini | 32 ++ .forgejo/upgrades/merged-app.ini | 32 ++ .forgejo/upgrades/misplace-app.ini | 59 +++ .forgejo/upgrades/misplace-s3-app.ini | 89 ++++ .forgejo/upgrades/relative-app.ini | 44 ++ .forgejo/upgrades/specific-app.ini | 47 ++ .forgejo/upgrades/stable-s3-app.ini | 39 ++ .forgejo/upgrades/storage-relative-app.ini | 44 ++ .forgejo/upgrades/test-upgrade.sh | 590 +++++++++++++++++++++ .forgejo/workflows/upgrade.yml | 45 ++ 12 files changed, 1238 insertions(+) create mode 100644 .forgejo/upgrades/default-app.ini create mode 100644 .forgejo/upgrades/fixtures.sh create mode 100644 .forgejo/upgrades/legagy-relative-app.ini create mode 100644 .forgejo/upgrades/merged-app.ini create mode 100644 .forgejo/upgrades/misplace-app.ini create mode 100644 .forgejo/upgrades/misplace-s3-app.ini create mode 100644 .forgejo/upgrades/relative-app.ini create mode 100644 .forgejo/upgrades/specific-app.ini create mode 100644 .forgejo/upgrades/stable-s3-app.ini create mode 100644 .forgejo/upgrades/storage-relative-app.ini create mode 100755 .forgejo/upgrades/test-upgrade.sh create mode 100644 .forgejo/workflows/upgrade.yml diff --git a/.forgejo/upgrades/default-app.ini b/.forgejo/upgrades/default-app.ini new file mode 100644 index 0000000000..a51290a8b2 --- /dev/null +++ b/.forgejo/upgrades/default-app.ini @@ -0,0 +1,30 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/data +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 +PATH = ${WORK_PATH}/forgejo.db + +[log] +MODE = file +LEVEL = trace +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[actions] +ENABLED = true diff --git a/.forgejo/upgrades/fixtures.sh b/.forgejo/upgrades/fixtures.sh new file mode 100644 index 0000000000..1b8cb44cac --- /dev/null +++ b/.forgejo/upgrades/fixtures.sh @@ -0,0 +1,187 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT + +#ONEPIXEL="iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +# +# one pixel scaled to 290x290 because that's what versions lower or equal to v1.19.4-0 want +# by default and any other size will be transformed which make it difficult to compare. +# +ONEPIXEL="iVBORw0KGgoAAAANSUhEUgAAASIAAAEiCAYAAABdvt+2AAADrElEQVR4nOzUMRHAMADEsL9eeQd6AsOLhMCT/7udAYS+OgDAiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDuBQAA//+4jAPFe1H1tgAAAABJRU5ErkJggg==" + +function fixture_get_paths_s3() { + local path=$1 + + ( + echo -n $path/ + mc ls --quiet --recursive testS3/$path | sed -e 's/.* //' + ) > $DIR/path +} + +function fixture_get_paths_local() { + local path=$1 + local work_path=$DIR/forgejo-work-path + + ( cd $work_path ; find $path -type f) > $DIR/path +} + +function fixture_get_one_path() { + local storage=$1 + local path=$2 + + fixture_get_paths_$storage $path + + if test $(wc -l < $DIR/path) != 1 ; then + echo expected one path but got + cat $DIR/path + return 1 + fi + cat $DIR/path +} + +function fixture_repo_archive_create() { + retry curl -f -sS http://${HOST_PORT}/root/fixture/archive/main.zip -o /dev/null +} + +function fixture_repo_archive_assert_s3() { + mc ls --recursive testS3/forgejo/repo-archive | grep --quiet '.zip$' +} + +function fixture_repo_archive_assert_local() { + local path=$1 + local work_path=$DIR/forgejo-work-path + + find $work_path/$path | grep --quiet '.zip$' +} + +function fixture_lfs_create() { + ( + cd $DIR/fixture + git lfs track "*.txt" + echo CONTENT > file.txt + git add . + git commit -m 'lfs files' + git push + ) +} + +function fixture_lfs_assert_s3() { + local content=$(mc cat testS3/forgejo/lfs/d6/1e/5fa787e50330288923bd0c9866b44643925965144262288447cf52f9f9b7) + test "$content" = CONTENT +} + +function fixture_lfs_assert_local() { + local path=$1 + local work_path=$DIR/forgejo-work-path + + local content=$(mc cat $work_path/$path/d6/1e/5fa787e50330288923bd0c9866b44643925965144262288447cf52f9f9b7) + test "$content" = CONTENT +} + +function fixture_packages_create() { + echo PACKAGE_CONTENT > $DIR/fixture/package + $work_path/forgejo-api -X DELETE http://${HOST_PORT}/api/packages/${FORGEJO_USER}/generic/test_package/1.0.0/file.txt || true + $work_path/forgejo-api --upload-file $DIR/fixture/package http://${HOST_PORT}/api/packages/${FORGEJO_USER}/generic/test_package/1.0.0/file.txt +} + +function fixture_packages_assert_s3() { + local content=$(mc cat testS3/forgejo/packages/aa/cf/aacf02e660d813e95d2854e27926ba1ad5c87299dc5f7661d5f08f076c6bbc17) + test "$content" = PACKAGE_CONTENT +} + +function fixture_packages_assert_local() { + local path=$1 + + local content=$(cat $work_path/$path/aa/cf/aacf02e660d813e95d2854e27926ba1ad5c87299dc5f7661d5f08f076c6bbc17) + test "$content" = PACKAGE_CONTENT +} + +function fixture_avatars_create() { + echo -n $ONEPIXEL | base64 --decode > $DIR/avatar.png + $work_path/forgejo-client --form avatar=@$DIR/avatar.png http://${HOST_PORT}/user/settings/avatar +} + +function fixture_avatars_assert_s3() { + local filename=$(fixture_get_one_path s3 forgejo/avatars) + local content=$(mc cat testS3/$filename | base64 -w0) + test "$content" = "$ONEPIXEL" +} + +function fixture_avatars_assert_local() { + local path=$1 + + local filename=$(fixture_get_one_path local $path) + local content=$(cat $work_path/$filename | base64 -w0) + test "$content" = "$ONEPIXEL" +} + +function fixture_repo_avatars_create() { + echo -n $ONEPIXEL | base64 --decode > $DIR/repo-avatar.png + $work_path/forgejo-client --form avatar=@$DIR/repo-avatar.png http://${HOST_PORT}/${FORGEJO_USER}/${FORGEJO_REPO}/settings/avatar + # v1.21 only + #$work_path/forgejo-api -X POST --data-raw '{"body":"'$avatar'"}' http://${HOST_PORT}/api/v1/repos/${FORGEJO_USER}/${FORGEJO_REPO}/avatar +} + +function fixture_repo_avatars_assert_s3() { + local filename=$(fixture_get_one_path s3 forgejo/repo-avatars) + local content=$(mc cat testS3/$filename | base64 -w0) + test "$content" = "$ONEPIXEL" +} + +function fixture_repo_avatars_assert_local() { + local path=$1 + + local filename=$(fixture_get_one_path local $path) + local content=$(cat $work_path/$filename | base64 -w0) + test "$content" = "$ONEPIXEL" +} + +function fixture_attachments_create_1_18() { + echo -n $ONEPIXEL | base64 --decode > $DIR/attachment.png + $work_path/forgejo-client --trace-ascii - --form file=@$DIR/attachment.png http://${HOST_PORT}/${FORGEJO_USER}/${FORGEJO_REPO}/issues/attachments +} + +function fixture_attachments_create() { + if $work_path/forgejo-api http://${HOST_PORT}/api/v1/version | grep --quiet --fixed-strings 1.18. ; then + fixture_attachments_create_1_18 + return + fi + id=$($work_path/forgejo-api --data-raw '{"title":"TITLE"}' http://${HOST_PORT}/api/v1/repos/${FORGEJO_USER}/${FORGEJO_REPO}/issues | jq .id) + echo -n $ONEPIXEL | base64 --decode > $DIR/attachment.png + $work_path/forgejo-client -H @$DIR/forgejo-work-path/forgejo-header --form name=attachment.png --form attachment=@$DIR/attachment.png http://${HOST_PORT}/api/v1/repos/${FORGEJO_USER}/${FORGEJO_REPO}/issues/$id/assets +} + +function fixture_attachments_assert_s3() { + local filename=$(fixture_get_one_path s3 forgejo/attachments) + local content=$(mc cat testS3/$filename | base64 -w0) + test "$content" = "$ONEPIXEL" +} + +function fixture_attachments_assert_local() { + local path=$1 + + local filename=$(fixture_get_one_path local $path) + local content=$(cat $work_path/$filename | base64 -w0) + test "$content" = "$ONEPIXEL" +} + +function fixture_create() { + local work_path=$DIR/forgejo-work-path + + rm -fr $DIR/fixture + mkdir -p $DIR/fixture + ( + cd $DIR/fixture + git init + git checkout -b main + git remote add origin http://${FORGEJO_USER}:${FORGEJO_PASSWORD}@${HOST_PORT}/${FORGEJO_USER}/${FORGEJO_REPO} + git config user.email root@example.com + git config user.name username + echo SOMETHING > README + git add README + git commit -m 'initial commit' + git push --set-upstream --force origin main + ) + for fun in ${STORAGE_FUN} ; do + fixture_${fun}_create + done +} diff --git a/.forgejo/upgrades/legagy-relative-app.ini b/.forgejo/upgrades/legagy-relative-app.ini new file mode 100644 index 0000000000..130294a4aa --- /dev/null +++ b/.forgejo/upgrades/legagy-relative-app.ini @@ -0,0 +1,32 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/data +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true +LFS_CONTENT_PATH = relative-lfs + +[database] +DB_TYPE = sqlite3 +PATH = ${WORK_PATH}/forgejo.db + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[picture] +AVATAR_UPLOAD_PATH = relative-avatars +REPOSITORY_AVATAR_UPLOAD_PATH = relative-repo-avatars diff --git a/.forgejo/upgrades/merged-app.ini b/.forgejo/upgrades/merged-app.ini new file mode 100644 index 0000000000..0b7150c077 --- /dev/null +++ b/.forgejo/upgrades/merged-app.ini @@ -0,0 +1,32 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/data +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[actions] +ENABLED = true + +[storage] +PATH = ${WORK_PATH}/merged diff --git a/.forgejo/upgrades/misplace-app.ini b/.forgejo/upgrades/misplace-app.ini new file mode 100644 index 0000000000..0aeff45319 --- /dev/null +++ b/.forgejo/upgrades/misplace-app.ini @@ -0,0 +1,59 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/elsewhere +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[actions] +ENABLED = true + +[attachment] + +[storage.attachments] +PATH = ${WORK_PATH}/data/attachments + +[lfs] + +[storage.lfs] +PATH = ${WORK_PATH}/data/lfs + +[avatar] + +[storage.avatars] +PATH = ${WORK_PATH}/data/avatars + +[repo-avatar] + +[storage.repo-avatars] +PATH = ${WORK_PATH}/data/repo-avatars + +[repo-archive] + +[storage.repo-archive] +PATH = ${WORK_PATH}/data/repo-archive + +[packages] + +[storage.packages] +PATH = ${WORK_PATH}/data/packages diff --git a/.forgejo/upgrades/misplace-s3-app.ini b/.forgejo/upgrades/misplace-s3-app.ini new file mode 100644 index 0000000000..d9243dd250 --- /dev/null +++ b/.forgejo/upgrades/misplace-s3-app.ini @@ -0,0 +1,89 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/elsewhere +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[actions] +ENABLED = true + +[attachment] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false + +[lfs] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false + +[repo-avatar] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false + +[avatar] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false + +[repo-archive] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false + +[packages] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false diff --git a/.forgejo/upgrades/relative-app.ini b/.forgejo/upgrades/relative-app.ini new file mode 100644 index 0000000000..d53c291780 --- /dev/null +++ b/.forgejo/upgrades/relative-app.ini @@ -0,0 +1,44 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/data +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[attachment] +PATH = relative-attachments + +[lfs] +PATH = relative-lfs + +[avatar] +PATH = relative-avatars + +[repo-avatar] +PATH = relative-repo-avatars + +[repo-archive] +PATH = relative-repo-archive + +[packages] +PATH = relative-packages diff --git a/.forgejo/upgrades/specific-app.ini b/.forgejo/upgrades/specific-app.ini new file mode 100644 index 0000000000..d7a0badc13 --- /dev/null +++ b/.forgejo/upgrades/specific-app.ini @@ -0,0 +1,47 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/elsewhere +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[actions] +ENABLED = true + +[attachment] +PATH = ${WORK_PATH}/data/attachments + +[lfs] +PATH = ${WORK_PATH}/data/lfs + +[avatar] +PATH = ${WORK_PATH}/data/avatars + +[repo-avatar] +PATH = ${WORK_PATH}/data/repo-avatars + +[repo-archive] +PATH = ${WORK_PATH}/data/repo-archive + +[packages] +PATH = ${WORK_PATH}/data/packages diff --git a/.forgejo/upgrades/stable-s3-app.ini b/.forgejo/upgrades/stable-s3-app.ini new file mode 100644 index 0000000000..e8c48ae2e1 --- /dev/null +++ b/.forgejo/upgrades/stable-s3-app.ini @@ -0,0 +1,39 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/elsewhere +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[actions] +ENABLED = true + +[storage] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false diff --git a/.forgejo/upgrades/storage-relative-app.ini b/.forgejo/upgrades/storage-relative-app.ini new file mode 100644 index 0000000000..eba5232dd0 --- /dev/null +++ b/.forgejo/upgrades/storage-relative-app.ini @@ -0,0 +1,44 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/data +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[storage.attachments] +PATH = relative-attachments + +[storage.lfs] +PATH = relative-lfs + +[storage.avatars] +PATH = relative-avatars + +[storage.repo-avatars] +PATH = relative-repo-avatars + +[storage.repo-archive] +PATH = relative-repo-archive + +[storage.packages] +PATH = relative-packages diff --git a/.forgejo/upgrades/test-upgrade.sh b/.forgejo/upgrades/test-upgrade.sh new file mode 100755 index 0000000000..287a92a591 --- /dev/null +++ b/.forgejo/upgrades/test-upgrade.sh @@ -0,0 +1,590 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT + +# +# Debug loop from the source tree: +# +# ./.forgejo/upgrades/test-upgrade.sh dependencies +# ./.forgejo/upgrades/test-upgrade.sh build_all +# VERBOSE=true ./.forgejo/upgrades/test-upgrade.sh test_downgrade_1.20.2_fails +# +# Everything happens in /tmp/forgejo-upgrades +# + +PREFIX=============== +HOST_PORT=0.0.0.0:3000 +STORAGE_PATHS="attachments avatars lfs packages repo-archive repo-avatars" +STORAGE_FUN="attachments avatars lfs packages repo_archive repo_avatars" +DIR=/tmp/forgejo-upgrades +if ${VERBOSE:-false} ; then + set -ex + PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: ' +else + set -e +fi +SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +: ${FORGEJO_USER:=root} +: ${FORGEJO_REPO:=fixture} +: ${FORGEJO_PASSWORD:=admin1234} + +source $SELF_DIR/fixtures.sh + +function maybe_sudo() { + if test $(id -u) != 0 ; then + SUDO=sudo + fi +} + +function log_info() { + echo "$PREFIX $@" +} + +function dependencies() { + maybe_sudo + if ! which curl daemon jq git-lfs > /dev/null ; then + $SUDO apt-get install -y -qq curl daemon git-lfs jq + fi + + if ! test -f /usr/local/bin/mc || ! test -f /usr/local/bin/minio > /dev/null ; then + $SUDO curl --fail -sS https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc + $SUDO curl --fail -sS https://dl.min.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2023-08-23T10-07-06Z -o /usr/local/bin/minio + fi + if ! test -x /usr/local/bin/mc || ! test -x /usr/local/bin/minio > /dev/null ; then + $SUDO chmod +x /usr/local/bin/mc + $SUDO chmod +x /usr/local/bin/minio + fi + + if ! test -f /usr/local/bin/garage > /dev/null ; then + $SUDO curl --fail -sS https://garagehq.deuxfleurs.fr/_releases/v0.8.2/x86_64-unknown-linux-musl/garage -o /usr/local/bin/garage + fi + if ! test -x /usr/local/bin/garage > /dev/null ; then + $SUDO chmod +x /usr/local/bin/garage + fi +} + +function build() { + local version=$1 + local semver=$2 + + if ! test -f $DIR/forgejo-$version ; then + mkdir -p $DIR + make VERSION=v$version GITEA_VERSION=v$version FORGEJO_VERSION=$semver TAGS='bindata sqlite sqlite_unlock_notify' generate gitea + mv gitea $DIR/forgejo-$version + fi +} + +function build_all() { + test -f Makefile + build 1.21.0-0 6.0.0+0-gitea-1.21.0 +} + +function retry() { + rm -f $DIR/wait-for.out + success=false + for delay in 1 1 5 5 15 ; do + if "$@" >> $DIR/wait-for.out 2>&1 ; then + success=true + break + fi + cat $DIR/wait-for.out + echo waiting $delay + sleep $delay + done + if test $success = false ; then + cat $DIR/wait-for.out + return 1 + fi +} + +function download() { + local version=$1 + + if ! test -f $DIR/forgejo-$version ; then + mkdir -p $DIR + wget -O $DIR/forgejo-$version --quiet https://codeberg.org/forgejo/forgejo/releases/download/v$version/forgejo-$version-linux-amd64 + chmod +x $DIR/forgejo-$version + fi +} + +function cleanup_logs() { + local work_path=$DIR/forgejo-work-path + + rm -f $DIR/*.log + rm -f $work_path/log/*.log +} + +function clobber() { + rm -fr /tmp/forgejo-upgrades +} + +function start_forgejo() { + local version=$1 + + download $version + local work_path=$DIR/forgejo-work-path + daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/forgejo-pid --errlog=$DIR/forgejo-err.log --output=$DIR/forgejo-out.log -- $DIR/forgejo-$version --config $work_path/app.ini --work-path $work_path + if ! retry grep 'Starting server on' $work_path/log/forgejo.log ; then + cat $DIR/*.log + cat $work_path/log/*.log + return 1 + fi + create_user $version +} + +function start_minio() { + mkdir -p $DIR/minio + daemon --chdir=$DIR --unsafe \ + --env="PATH=$PATH" \ + --env=MINIO_ROOT_USER=123456 \ + --env=MINIO_ROOT_PASSWORD=12345678 \ + --env=MINIO_VOLUMES=$DIR/minio \ + --pidfile=$DIR/minio-pid --errlog=$DIR/minio-err.log --output=$DIR/minio-out.log -- /usr/local/bin/minio server + retry mc alias set testS3 http://127.0.0.1:9000 123456 12345678 +} + +function start_garage() { + mkdir -p $DIR/garage/{data,meta} + cat > $DIR/garage/garage.toml < $work_path/forgejo-token + ( echo -n 'Authorization: token ' ; cat $work_path/forgejo-token ) > $work_path/forgejo-header + ( echo "#!/bin/sh" ; echo 'curl -f -sS -H "Content-Type: application/json" -H @'$work_path/forgejo-header' "$@"' ) > $work_path/forgejo-api && chmod +x $work_path/forgejo-api + $work_path/forgejo-api http://${HOST_PORT}/api/v1/version + + # + # forgejo-client is to use with web endpoints + # + # + # login and obtain a CSRF, all stored in the cookie file + # + ( echo "#!/bin/sh" ; echo 'curl --cookie-jar '$DIR/cookies' --cookie '$DIR/cookies' -f -sS "$@"' ) > $work_path/forgejo-client-update-cookies && chmod +x $work_path/forgejo-client-update-cookies + $work_path/forgejo-client-update-cookies http://${HOST_PORT}/user/login -o /dev/null + $work_path/forgejo-client-update-cookies --verbose -X POST --data user_name=${FORGEJO_USER} --data password=${FORGEJO_PASSWORD} http://${HOST_PORT}/user/login >& $DIR/login.html + $work_path/forgejo-client-update-cookies http://${HOST_PORT}/user/login -o /dev/null + local csrf=$(sed -n -e '/csrf/s/.*csrf\t//p' $DIR/cookies) + # + # use the cookie file but do not modify it + # + ( echo "#!/bin/sh" ; echo 'curl --cookie '$DIR/cookies' -H "X-Csrf-Token: '$csrf'" -f -sS "$@"' ) > $work_path/forgejo-client && chmod +x $work_path/forgejo-client +} + +function stop_daemon() { + local daemon=$1 + + if test -f $DIR/$daemon-pid ; then + local pid=$(cat $DIR/$daemon-pid) + kill -TERM $pid + pidwait $pid || true + for delay in 1 1 2 2 5 5 ; do + if ! test -f $DIR/$daemon-pid ; then + break + fi + sleep $delay + done + ! test -f $DIR/$daemon-pid + fi +} + +function stop() { + stop_daemon forgejo + stop_daemon minio + stop_daemon garage + + cleanup_logs +} + +function reset_forgejo() { + local config=$1 + local work_path=$DIR/forgejo-work-path + rm -fr $work_path + mkdir -p $work_path + WORK_PATH=$work_path envsubst < $SELF_DIR/$config-app.ini > $work_path/app.ini +} + +function reset_minio() { + rm -fr $DIR/minio +} + +function reset_garage() { + rm -fr $DIR/garage +} + +function reset() { + local config=$1 + reset_forgejo $config + reset_minio + reset_garage +} + +function verify_storage() { + local work_path=$DIR/forgejo-work-path + + for path in ${STORAGE_PATHS} ; do + test -d $work_path/data/$path + done +} + +function cleanup_storage() { + local work_path=$DIR/forgejo-work-path + + for path in ${STORAGE_PATHS} ; do + rm -fr $work_path/data/$path + done +} + +function test_downgrade_1.20.2_fails() { + local work_path=$DIR/forgejo-work-path + + log_info "See also https://codeberg.org/forgejo/forgejo/pulls/1225" + + log_info "downgrading from 1.20.3-0 to 1.20.2-0 fails" + stop + reset default + start 1.20.3-0 + stop + download 1.20.2-0 + timeout 60 $DIR/forgejo-1.20.2-0 --config $work_path/app.ini --work-path $work_path || true + if ! grep --fixed-strings --quiet 'use the newer database' $work_path/log/forgejo.log ; then + cat $work_path/log/forgejo.log + return 1 + fi +} + +function test_bug_storage_merged() { + local work_path=$DIR/forgejo-work-path + + log_info "See also https://codeberg.org/forgejo/forgejo/pulls/1225" + + log_info "using < 1.20.3-0 and [storage].PATH merge all storage" + for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do + stop + reset merged + start $version + for path in ${STORAGE_PATHS} ; do + ! test -d $work_path/data/$path + done + for path in ${STORAGE_PATHS} ; do + ! test -d $work_path/merged/$path + done + test -d $work_path/merged + done + stop + + log_info "upgrading from 1.20.2-0 with [storage].PATH fails" + download 1.20.3-0 + timeout 60 $DIR/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true + if ! grep --fixed-strings --quiet '[storage].PATH is set and may create storage issues' $work_path/log/forgejo.log ; then + cat $work_path/log/forgejo.log + return 1 + fi +} + +function test_bug_storage_relative_path() { + local work_path=$DIR/forgejo-work-path + + log_info "using < 1.20.3-0 legacy [server].XXXX and [picture].XXXX are relative to WORK_PATH" + for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do + stop + reset legagy-relative + start $version + test -d $work_path/relative-lfs + test -d $work_path/relative-avatars + test -d $work_path/relative-repo-avatars + done + + log_info "using >= 1.20.3-0 legacy [server].XXXX and [picture].XXXX are relative to APP_DATA_PATH" + for version in 1.20.3-0 1.21.0-0 ; do + stop + reset legagy-relative + start $version + test -d $work_path/data/relative-lfs + test -d $work_path/data/relative-avatars + test -d $work_path/data/relative-repo-avatars + done + + log_info "using >= 1.20.3-0 relative [storage.XXXX].PATHS are relative to APP_DATA_PATH" + for version in 1.20.3-0 1.21.0-0 ; do + stop + reset storage-relative + start $version + for path in ${STORAGE_PATHS} ; do + test -d $work_path/data/relative-$path + done + done + + log_info "using 1.20.[12]-0 relative [storage.XXXX].PATHS are inconsistent" + for version in 1.20.2-0 ; do + stop + reset storage-relative + start $version + test -d $work_path/data/packages + test -d $work_path/relative-repo-archive + test -d $work_path/relative-attachments + test -d $work_path/relative-lfs + test -d $work_path/data/avatars + test -d $work_path/data/repo-avatars + done + + log_info "using < 1.20 relative [storage.XXXX].PATHS are inconsistent" + for version in 1.18.5-0 1.19.4-0 ; do + stop + reset storage-relative + start $version + test -d $work_path/relative-packages + test -d $work_path/relative-repo-archive + test -d $work_path/relative-attachments + test -d $work_path/data/lfs + test -d $work_path/data/avatars + test -d $work_path/data/repo-avatars + done + + log_info "using < 1.20.3-0 relative [XXXX].PATHS are relative to WORK_PATH" + for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do + stop + reset relative + start $version + for path in ${STORAGE_PATHS} ; do + test -d $work_path/relative-$path + done + done + + log_info "using >= 1.20.3-0 relative [XXXX].PATHS are relative to APP_DATA_PATH" + for version in 1.20.3-0 1.21.0-0 ; do + stop + reset relative + start $version + for path in ${STORAGE_PATHS} ; do + test -d $work_path/data/relative-$path + done + done + + stop +} + +function test_bug_storage_s3_misplace() { + local work_path=$DIR/forgejo-work-path + local s3_backend=${2:-minio} + + log_info "See also https://codeberg.org/forgejo/forgejo/issues/1338" + + for version in 1.20.2-0 1.20.3-0 ; do + log_info "Forgejo $version & $s3_backend" + stop + reset misplace-s3 + start $version $s3_backend + fixture_create + for fun in ${STORAGE_FUN} ; do + fixture_${fun}_assert_s3 + done + done + + for version in 1.18.5-0 1.19.4-0 ; do + log_info "Forgejo $version & $s3_backend" + stop + reset misplace-s3 + start $version $s3_backend + fixture_create + # + # some storage are in S3 + # + fixture_attachments_assert_s3 + fixture_lfs_assert_s3 + # + # others are in local + # + fixture_repo_archive_assert_local elsewhere/repo-archive + fixture_avatars_assert_local elsewhere/avatars + fixture_packages_assert_local elsewhere/packages + fixture_repo_avatars_assert_local elsewhere/repo-avatars + done +} + +function test_storage_stable_s3() { + local work_path=$DIR/forgejo-work-path + local s3_backend=${1:-minio} + + log_info "See also https://codeberg.org/forgejo/forgejo/issues/1338" + + for version in 1.18.5-0 1.19.4-0 1.20.2-0 1.20.3-0 ; do + log_info "Forgejo $version & $s3_backend" + stop + reset stable-s3 + start $version $s3_backend + fixture_create + for fun in ${STORAGE_FUN} ; do + fixture_${fun}_assert_s3 + done + done +} + +function test_bug_storage_misplace() { + local work_path=$DIR/forgejo-work-path + + log_info "See also https://codeberg.org/forgejo/forgejo/pulls/1225" + + log_info "using < 1.20 and conflicting sections misplace storage" + for version in 1.18.5-0 1.19.4-0 ; do + stop + reset misplace + start $version + # + # some storage are where they should be + # + test -d $work_path/data/packages + test -d $work_path/data/repo-archive + test -d $work_path/data/attachments + # + # others are under APP_DATA_PATH + # + test -d $work_path/elsewhere/lfs + test -d $work_path/elsewhere/avatars + test -d $work_path/elsewhere/repo-avatars + done + + log_info "using < 1.20.[12]-0 and conflicting sections ignores [storage.*]" + for version in 1.20.2-0 ; do + stop + reset misplace + start $version + for path in ${STORAGE_PATHS} ; do + test -d $work_path/elsewhere/$path + done + done + + stop + + log_info "upgrading from 1.20.2-0 with conflicting sections fails" + download 1.20.3-0 + timeout 60 $DIR/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true + for path in ${STORAGE_PATHS} ; do + if ! grep --fixed-strings --quiet "[storage.$path] may conflict" $work_path/log/forgejo.log ; then + cat $work_path/log/forgejo.log + return 1 + fi + done +} + +function test_successful_upgrades() { + for config in default specific ; do + log_info "using $config app.ini" + reset $config + + for version in 1.18.5-0 1.19.4-0 1.20.2-0 1.20.3-0 1.21.0-0 ; do + log_info "run $version" + cleanup_storage + start $version + verify_storage + stop + done + done +} + +function run() { + local fun=$1 + shift + + echo Start running $fun + mkdir -p $DIR + > $DIR/$fun.out + tail --follow $DIR/$fun.out | sed --unbuffered -n -e "/^$PREFIX/s/^$PREFIX //p" & + pid=$! + if ! VERBOSE=true ${BASH_SOURCE[0]} $fun "$@" >& $DIR/$fun.out ; then + kill $pid + cat $DIR/$fun.out + echo Failure running $fun + return 1 + fi + kill $pid + echo Success running $fun +} + +function test_upgrades() { + run stop + run dependencies + run build_all + run test_successful_upgrades + run test_bug_storage_misplace + run test_bug_storage_merged + run test_downgrade_1.20.2_fails + run test_bug_storage_s3_misplace + run test_storage_stable_s3 minio + run test_storage_stable_s3 garage +} + +"$@" diff --git a/.forgejo/workflows/upgrade.yml b/.forgejo/workflows/upgrade.yml new file mode 100644 index 0000000000..e07a349944 --- /dev/null +++ b/.forgejo/workflows/upgrade.yml @@ -0,0 +1,45 @@ +name: upgrade + +on: + pull_request_review: + push: + branches: + - 'forgejo*' + - 'v*/forgejo*' + +jobs: + upgrade: + runs-on: docker + container: + image: codeberg.org/forgejo/test_env:main + steps: + - run: apt-get install -y -qq zstd + + - name: cache S3 binaries + id: S3 + uses: https://code.forgejo.org/actions/cache@v3 + with: + path: | + /usr/local/bin/minio + /usr/local/bin/mc + /usr/local/bin/garage + key: S3 + + - name: skip if S3 cache hit + if: steps.S3.outputs.cache-hit != 'true' + run: echo no hit + + - uses: https://code.forgejo.org/actions/checkout@v3 + - uses: https://code.forgejo.org/actions/setup-go@v4 + with: + go-version: "1.21" + - run: | + git config --add safe.directory '*' + chown -R gitea:gitea . /go + - run: | + su gitea -c 'make deps-backend' + - run: | + script=$(pwd)/.forgejo/upgrades/test-upgrade.sh + $script run dependencies + $script clobber + su gitea -c "$script test_upgrades"