diff --git a/Dockerfile b/Dockerfile
index 303d7085bc..3ed86570ab 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -38,6 +38,19 @@ RUN make frontend
 RUN go build contrib/environment-to-ini/environment-to-ini.go && xx-verify environment-to-ini
 RUN make go-check generate-backend static-executable && xx-verify gitea
 
+# Copy local files
+COPY docker/root /tmp/local
+
+# Set permissions
+RUN chmod 755 /tmp/local/usr/bin/entrypoint \
+              /tmp/local/usr/local/bin/gitea \
+              /tmp/local/etc/s6/gitea/* \
+              /tmp/local/etc/s6/openssh/* \
+              /tmp/local/etc/s6/.s6-svscan/* \
+              /go/src/code.gitea.io/gitea/gitea \
+              /go/src/code.gitea.io/gitea/environment-to-ini
+RUN chmod 644 /go/src/code.gitea.io/gitea/contrib/autocompletion/bash_autocomplete
+
 FROM docker.io/library/alpine:3.18
 LABEL maintainer="contact@forgejo.org"
 
@@ -54,7 +67,8 @@ RUN apk --no-cache add \
     s6 \
     sqlite \
     su-exec \
-    gnupg
+    gnupg \
+    && rm -rf /var/cache/apk/*
 
 RUN addgroup \
     -S -g 1000 \
@@ -76,10 +90,7 @@ VOLUME ["/data"]
 ENTRYPOINT ["/usr/bin/entrypoint"]
 CMD ["/bin/s6-svscan", "/etc/s6"]
 
-COPY docker/root /
+COPY --from=build-env /tmp/local /
 COPY --from=build-env /go/src/code.gitea.io/gitea/gitea /app/gitea/gitea
 COPY --from=build-env /go/src/code.gitea.io/gitea/environment-to-ini /usr/local/bin/environment-to-ini
 COPY --from=build-env /go/src/code.gitea.io/gitea/contrib/autocompletion/bash_autocomplete /etc/profile.d/gitea_bash_autocomplete.sh
-RUN chmod 755 /usr/bin/entrypoint /app/gitea/gitea /usr/local/bin/gitea /usr/local/bin/environment-to-ini
-RUN chmod 755 /etc/s6/gitea/* /etc/s6/openssh/* /etc/s6/.s6-svscan/*
-RUN chmod 644 /etc/profile.d/gitea_bash_autocomplete.sh
diff --git a/Dockerfile.rootless b/Dockerfile.rootless
index 5fe529c228..e662669627 100644
--- a/Dockerfile.rootless
+++ b/Dockerfile.rootless
@@ -38,6 +38,17 @@ RUN make frontend
 RUN go build contrib/environment-to-ini/environment-to-ini.go && xx-verify environment-to-ini
 RUN make go-check generate-backend static-executable && xx-verify gitea
 
+# Copy local files
+COPY docker/rootless /tmp/local
+
+# Set permissions
+RUN chmod 755 /tmp/local/usr/local/bin/docker-entrypoint.sh \
+              /tmp/local/usr/local/bin/docker-setup.sh \
+              /tmp/local/usr/local/bin/gitea \
+              /go/src/code.gitea.io/gitea/gitea \
+              /go/src/code.gitea.io/gitea/environment-to-ini
+RUN chmod 644 /go/src/code.gitea.io/gitea/contrib/autocompletion/bash_autocomplete
+
 FROM docker.io/library/alpine:3.18
 LABEL maintainer="contact@forgejo.org"
 
@@ -50,7 +61,8 @@ RUN apk --no-cache add \
     gettext \
     git \
     curl \
-    gnupg
+    gnupg \
+    && rm -rf /var/cache/apk/*
 
 RUN addgroup \
     -S -g 1000 \
@@ -66,12 +78,10 @@ RUN addgroup \
 RUN mkdir -p /var/lib/gitea /etc/gitea
 RUN chown git:git /var/lib/gitea /etc/gitea
 
-COPY docker/rootless /
+COPY --from=build-env /tmp/local /
 COPY --from=build-env --chown=root:root /go/src/code.gitea.io/gitea/gitea /app/gitea/gitea
 COPY --from=build-env --chown=root:root /go/src/code.gitea.io/gitea/environment-to-ini /usr/local/bin/environment-to-ini
 COPY --from=build-env /go/src/code.gitea.io/gitea/contrib/autocompletion/bash_autocomplete /etc/profile.d/gitea_bash_autocomplete.sh
-RUN chmod 755 /usr/local/bin/docker-entrypoint.sh /usr/local/bin/docker-setup.sh /app/gitea/gitea /usr/local/bin/gitea /usr/local/bin/environment-to-ini
-RUN chmod 644 /etc/profile.d/gitea_bash_autocomplete.sh
 
 #git:git
 USER 1000:1000