From cd363fce3dbbd574b179d6802a8abcc1c76a6cfe Mon Sep 17 00:00:00 2001 From: zhangwenjian Date: Wed, 9 Sep 2026 07:45:40 +0800 Subject: [PATCH 1/3] =?UTF-8?q?perf=F0=9F=91=8C:=20stop=20shipping=20a=20C?= =?UTF-8?q?=20toolchain=20in=20the=20runtime=20image?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gcc and g++ were 273MB of a 381MB image, and nothing in the container ever invoked them: the binary is compiled and statically linked before the image is built and arrives as a COPY, and Go is not installed here either. The layer cost more than its size. apk resolves against an index that moves, so its digest differed on every build and no two images shared it - a host that keeps one image per deployed commit paid the full 273MB each time rather than storing it once. libc6-compat is kept although nothing measured needs it: a container built without it resolves a hostname and opens a database connection exactly as one built with it, but it costs half a megabyte and covers a ./main that was linked dynamically, which this Dockerfile cannot check. Verified by building this Dockerfile and running the result under the check the deploy script uses - captcha answering 200 and the log reporting the datastore connected. A control built from the current recipe passes the same check and carries a 273MB apk layer this one does not; a third build with a deliberately truncated binary fails the check, so it distinguishes a serving process from a dead one. --- Dockerfile | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index fb3e023d..22a060e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,10 +4,26 @@ FROM alpine RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories -RUN apk update --no-cache -RUN apk add --update gcc g++ libc6-compat -RUN apk add --no-cache ca-certificates -RUN apk add --no-cache tzdata +# Runtime packages only. +# +# gcc and g++ used to be installed here and were 273MB of a 381MB image. The +# binary this image runs is compiled and linked before the image is built and +# arrives as a COPY, so nothing in the container ever invokes a compiler - +# there is no toolchain to drive it with either, since Go itself is not +# installed. +# +# That layer was also why a host could not share storage between images. apk +# resolves against an index that moves, so the layer digest differed on every +# build and no two images shared it: a host keeping one image per deployed +# commit stored a private 273MB copy each time. 68 of them filled the disk +# and the next deployment could not pull. +# +# libc6-compat stays. Nothing measured needs it - the binary CI produces is +# statically linked, and a container built without libc6-compat resolves a +# hostname and opens a database connection exactly as one built with it - but +# it is half a megabyte and it covers a ./main that was linked dynamically, +# which this Dockerfile has no way to check. +RUN apk add --no-cache ca-certificates tzdata libc6-compat ENV TZ Asia/Shanghai COPY ./main /main @@ -15,4 +31,4 @@ COPY ./config/settings.demo.yml /config/settings.yml COPY ./go-admin-db.db /go-admin-db.db EXPOSE 8000 RUN chmod +x /main -CMD ["/main","server","-c", "/config/settings.yml"] \ No newline at end of file +CMD ["/main","server","-c", "/config/settings.yml"] From d12f40c9a066d40b33f6124bd6ea708dde51e6d0 Mon Sep 17 00:00:00 2001 From: zhangwenjian Date: Wed, 9 Sep 2026 07:45:51 +0800 Subject: [PATCH 2/3] =?UTF-8?q?ci=F0=9F=91=B7:=20remove=20this=20repositor?= =?UTF-8?q?y's=20old=20images=20after=20a=20healthy=20deploy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every deployment pulls an image tagged with its commit and nothing removed the previous one, so they only accumulated. 68 had built up when a deployment failed on a pull with no space left on the device. That is the harmless place to fail - the site kept serving the image it already had - but no later run would have recovered on its own. Three are kept so a release can be re-run by tag by hand. Only this repository's images are listed, because the host runs other services. The image the new container is on is excluded by id rather than by position, and rmi is called without -f so an image a container still holds is refused rather than taken from it. Verified against a real docker daemon: with five images newer than the running one, so position alone no longer protects it, the pipeline leaves three and does not select the live one. Removing the id exclusion from the same pipeline does select it, so that guard is load-bearing rather than decorative. --- .github/workflows/build.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 94cd4cea..33c9a8f3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -155,6 +155,31 @@ jobs: if [ "$ok" = "1" ]; then sudo docker rm -f "$PREV" >/dev/null 2>&1 || true + + # Old images of this repository are deleted here and nowhere + # else. Every deployment pulls one image tagged with its commit + # and nothing ever removed the previous one, so they only ever + # accumulated: 68 of them filled the disk and the next deployment + # could not pull. That one stopped at the pull, which + # is the harmless place to stop - the site kept serving the image + # it already had - but no later run would have recovered either. + # + # Three are kept so a release can be re-run by tag by hand. + # + # Only this repository's images are listed: the host runs other + # services whose images are not this script's business. The image + # the new container is on is excluded by id rather than by + # position, so it survives even if the listing order is not what + # it looks like. No -f, so an image some container still holds is + # refused rather than taken away from it, and a refusal does not + # fail a deployment that has already succeeded. + REPO="${IMG%:*}" + LIVE=$(sudo docker inspect -f '{{.Image}}' "$NAME" | sed 's/^sha256://' | cut -c1-12) + sudo docker images "$REPO" --format '{{.ID}} {{.Repository}}:{{.Tag}}' \ + | grep -v "^$LIVE" \ + | tail -n +3 \ + | awk '{print $2}' \ + | xargs -r -n1 sudo docker rmi >/dev/null 2>&1 || true else echo "健康检查失败，回滚到上一版本" sudo docker logs --tail 40 "$NAME" 2>&1 || true From 46c10f999acfda0dc687ca943527d3fadfe314b1 Mon Sep 17 00:00:00 2001 From: zhangwenjian Date: Wed, 9 Sep 2026 08:11:26 +0800 Subject: [PATCH 3/3] =?UTF-8?q?ci=F0=9F=91=B7:=20reclaim=20before=20the=20?= =?UTF-8?q?pull,=20not=20only=20after=20a=20healthy=20deploy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cleaning up after a successful deployment never runs on the host that needs it. The pull is the first thing in this script that needs space and it is where a full disk stops it, so the run ends before reaching any cleanup - and so does the next run, and the one after that. That is not hypothetical: a deployment failed on the pull with no space left on the device, and rerunning the workflow unchanged failed at the same place. The disk had to be cleared by hand before a deployment could go through. The pipeline is now a function called twice, before the pull and after the health check, so the window is bounded on both sides. Verified against a real docker daemon, in the function form rather than the inlined one: with five images newer than the running one, so position alone no longer protects it, it leaves three and does not select the live one; removing the id exclusion from the same function does select it. With NAME pointing at a container that does not exist it selects nothing - as it also does without the explicit guard, which is there because grep -v on an empty id reads like the opposite of what it does, not because it changes the outcome. --- .github/workflows/build.yml | 61 ++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 33c9a8f3..53a381d3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -117,7 +117,41 @@ jobs: test -f "$CFG" || { echo "宿主机配置缺失，中止部署"; exit 1; } + # Old images of this repository are removed here and nowhere else. + # Every deployment pulls one tagged with its commit and nothing ever + # removed the previous one, so they only accumulated: 68 of them + # filled the disk and the next deployment could not pull. + # + # Three are kept so a release can be re-run by tag by hand. + # + # Only this repository's images are listed, because the host runs + # other services whose images are not this script's business. The + # image the live container is on is excluded by id rather than by + # position, so it survives even if the listing order is not what + # it looks like. With no container to ask, the function returns + # rather than running the pipeline on an empty id - which would + # also delete nothing, but by way of grep -v matching every line, + # which reads like the opposite of what it does. No -f, so an image + # any container still holds - including the one kept for rollback - + # is refused rather than taken away from it. + prune_old_images() { + REPO="${IMG%:*}" + LIVE=$(sudo docker inspect -f '{{.Image}}' "$NAME" 2>/dev/null | sed 's/^sha256://' | cut -c1-12) + [ -n "$LIVE" ] || return 0 + sudo docker images "$REPO" --format '{{.ID}} {{.Repository}}:{{.Tag}}' \ + | grep -v "^$LIVE" \ + | tail -n +3 \ + | awk '{print $2}' \ + | xargs -r -n1 sudo docker rmi >/dev/null 2>&1 || true + } + sudo docker login --username=${{ secrets.DOCKER_USERNAME }} registry.ap-northeast-1.aliyuncs.com --password=${{ secrets.DOCKER_PASSWORD }} + # Before the pull, not only after a successful deploy. The pull + # is the first thing here that needs space and it is where a full + # disk stops this script, so a cleanup that only runs afterwards + # never runs on the host that needs it: rerunning the workflow + # fails at the same pull, and the disk has to be cleared by hand. + prune_old_images sudo docker pull "$IMG" || { echo "拉取镜像失败，中止部署"; exit 1; } # 迁移用新镜像跑。失败时线上仍是旧版本配旧 schema，是自洽的； @@ -156,30 +190,9 @@ jobs: if [ "$ok" = "1" ]; then sudo docker rm -f "$PREV" >/dev/null 2>&1 || true - # Old images of this repository are deleted here and nowhere - # else. Every deployment pulls one image tagged with its commit - # and nothing ever removed the previous one, so they only ever - # accumulated: 68 of them filled the disk and the next deployment - # could not pull. That one stopped at the pull, which - # is the harmless place to stop - the site kept serving the image - # it already had - but no later run would have recovered either. - # - # Three are kept so a release can be re-run by tag by hand. - # - # Only this repository's images are listed: the host runs other - # services whose images are not this script's business. The image - # the new container is on is excluded by id rather than by - # position, so it survives even if the listing order is not what - # it looks like. No -f, so an image some container still holds is - # refused rather than taken away from it, and a refusal does not - # fail a deployment that has already succeeded. - REPO="${IMG%:*}" - LIVE=$(sudo docker inspect -f '{{.Image}}' "$NAME" | sed 's/^sha256://' | cut -c1-12) - sudo docker images "$REPO" --format '{{.ID}} {{.Repository}}:{{.Tag}}' \ - | grep -v "^$LIVE" \ - | tail -n +3 \ - | awk '{print $2}' \ - | xargs -r -n1 sudo docker rmi >/dev/null 2>&1 || true + # Again, so the image this deployment replaced falls out of the + # window rather than waiting for the next deployment to notice. + prune_old_images else echo "健康检查失败，回滚到上一版本" sudo docker logs --tail 40 "$NAME" 2>&1 || true