Fix: Fix Error! App '90' state is 0x6 after update job. for hlds (s…

…tartersclan#54)

* Add github workflows `test-build-*` and `test-update-*` jobs for automated `hlds` and `srcds` builds.
* Add user variables `STEAM_USERNAME` and `STEAM_PASSWORD` for steam authentication.
* Add CI variable `STEAM_LOGIN` for toggling between authenticated and anonymous logins.
* Utilize `DOCKER_BUILDKIT=1` and `# syntax=docker/dockerfile:1` in Dockerfiles for usage of secret variables in docker builds.
* Fix order of `steamcmd` `+force_install_dir` parameter.
* Bump `hlds-appmanifest` to `v2.0.0` to fix `hlds` builds.
* Tweak clean-up steps in Dockerfiles for improved image size.
* Bump to use `docker:20.10.18-*` images in `.gitlab-ci.yml` `build-image` job for docker buildkit support.
* Update `env` key in `azure-pipelines.yml`.
* Update `parameters` and `environment` keys in `.circleci/config.yml`.
* Revise `NO_PUSH=true` to additionally skip `docker login` and `docker logout` commands.

Co-authored-by: leo <[email protected]>
Co-authored-by: joe <[email protected]>

.circleci/config.yml

@@ -36,6 +36,9 @@ parameters:
   NO_PUSH:
     type: string
     default: ''
+  STEAM_LOGIN:
+    type: string
+    default: ''
 
 workflows:
   version: 2
@@ -61,6 +64,7 @@ jobs:
       CACHE: << pipeline.parameters.CACHE >>
       NO_TEST: << pipeline.parameters.NO_TEST >>
       NO_PUSH: << pipeline.parameters.NO_PUSH >>
+      STEAM_LOGIN: << pipeline.parameters.STEAM_LOGIN >>
     steps:
     - checkout
     - run:

.github/workflows/ci-master-pr.yml

@@ -11,8 +11,56 @@ on:
     - master
 
 jobs:
-  test:
+  test-build-hlds:
     runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request' || github.repository_owner != 'startersclan'
     steps:
     - name: Checkout
       uses: actions/checkout@v3
+    - name: Test
+      run: |
+        . ./test/build-hlds-cstrike.env
+        . ./build.sh
+      env:
+        STEAM_USERNAME: ${{ secrets.STEAM_USERNAME }}
+        STEAM_PASSWORD: ${{ secrets.STEAM_PASSWORD }}
+
+  test-update-hlds:
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request' || github.repository_owner != 'startersclan'
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Test
+      run: |
+        . ./test/update-hlds-cstrike.env
+        . ./build.sh
+      env:
+        STEAM_USERNAME: ${{ secrets.STEAM_USERNAME }}
+        STEAM_PASSWORD: ${{ secrets.STEAM_PASSWORD }}
+
+  test-build-srcds:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Test
+      run: |
+        . ./test/build-srcds-hl2mp.env
+        . ./build.sh
+      env:
+        STEAM_USERNAME: ${{ secrets.STEAM_USERNAME }}
+        STEAM_PASSWORD: ${{ secrets.STEAM_PASSWORD }}
+
+  test-update-srcds:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Test
+      run: |
+        . ./test/update-srcds-hl2mp.env
+        . ./build.sh
+      env:
+        STEAM_USERNAME: ${{ secrets.STEAM_USERNAME }}
+        STEAM_PASSWORD: ${{ secrets.STEAM_PASSWORD }}

.gitlab-ci.yml

@@ -4,9 +4,9 @@ stages:
 
 build-image:
   stage: build
-  image: docker:18.09.2-git
+  image: docker:20.10.18-git
   services:
-    - docker:18.09.2-dind
+    - docker:20.10.18-dind
   rules:
     - if: $CI_PIPELINE_SOURCE == "trigger"
     - if: $CI_PIPELINE_SOURCE == "web"

azure-pipelines.yml

@@ -14,4 +14,6 @@ jobs:
       ./build.sh
     env:
       REGISTRY_PASSWORD: $(REGISTRY_PASSWORD)
+      STEAM_USERNAME: $(STEAM_USERNAME)
+      STEAM_PASSWORD: $(STEAM_PASSWORD)
     displayName: Build, test, push image

build.sh

@@ -11,6 +11,8 @@
 # REGISTRY_PASSWORD=      # string - docker hub password or api key - required
 # REGISTRY_GOLDSOURCE=    # string - docker hub username or organization for goldsource games - required
 # REGISTRY_SOURCE=        # string - docker hub username or organization for source games - required
+# STEAM_USERNAME=         # string - steam username - optional
+# STEAM_PASSWORD=         # string - steam password - optional
 
 ## Job variables ##
 # PIPELINE=               # string - 'build' for clean builds, 'update' for layered builds - required
@@ -26,6 +28,7 @@
 # CACHE=false             # bool - optional
 # NO_TEST=false           # bool - optional
 # NO_PUSH=false           # bool - optional
+# STEAM_LOGIN=false       # bool - optional
 
 ## update variables ##
 # GAME_VERSION=           # int - required
@@ -34,6 +37,7 @@
 # GAME_UPDATE_COUNT=      # int - required
 # NO_TEST=false           # bool - optional
 # NO_PUSH=false           # bool - optional
+# STEAM_LOGIN=false       # bool - optional
 
 #############################  End of CI variables  ##############################
 
@@ -42,6 +46,8 @@ REGISTRY_USER=${REGISTRY_USER:?err}
 REGISTRY_PASSWORD=${REGISTRY_PASSWORD:?err}
 REGISTRY_GOLDSOURCE=${REGISTRY_GOLDSOURCE:?err}
 REGISTRY_SOURCE=${REGISTRY_SOURCE:?err}
+export STEAM_USERNAME=${STEAM_USERNAME:-}
+export STEAM_PASSWORD=${STEAM_PASSWORD:-}
 
 # Process job variables
 PIPELINE=${PIPELINE:?err}
@@ -56,16 +62,19 @@ if [ "$PIPELINE" = 'build' ]; then
     CACHE=${CACHE:-}
     NO_TEST=${NO_TEST:-}
     NO_PUSH=${NO_PUSH:-}
+    STEAM_LOGIN=${STEAM_LOGIN:-}
 elif [ "$PIPELINE" = 'update' ]; then
     GAME_VERSION=${GAME_VERSION:?err}
     APPID=${APPID:?err}
     GAME=${GAME:?err}
     GAME_UPDATE_COUNT=${GAME_UPDATE_COUNT:?err}
     NO_TEST=${NO_TEST:-}
     NO_PUSH=${NO_PUSH:-}
+    STEAM_LOGIN=${STEAM_LOGIN:-}
 fi
 
 # Process default job variables
+export DOCKER_BUILDKIT=1
 if [ "$PIPELINE" = 'build' ]; then
     DOCKER_BUILD_CONTEXT='build/'
 elif [ "$PIPELINE" = 'update' ]; then
@@ -99,7 +108,9 @@ docker version
 set -e
 
 # Docker registry login
-echo "$REGISTRY_PASSWORD" | docker login -u "$REGISTRY_USER" --password-stdin
+if [ ! "$NO_PUSH" = 'true' ]; then
+    echo "$REGISTRY_PASSWORD" | docker login -u "$REGISTRY_USER" --password-stdin
+fi
 
 # Build / Update the game image
 if [ "$PIPELINE" = 'build' ]; then
@@ -111,10 +122,13 @@ if [ "$PIPELINE" = 'build' ]; then
     date
     time docker build \
         --cache-from "$GAME_IMAGE" \
+        --secret id=STEAM_USERNAME,env=STEAM_USERNAME \
+        --secret id=STEAM_PASSWORD,env=STEAM_PASSWORD \
         --build-arg APPID="$APPID" \
         --build-arg MOD="$MOD" \
         --build-arg FIX_APPMANIFEST="$FIX_APPMANIFEST" \
         --build-arg CLIENT_APPID="$CLIENT_APPID" \
+        --build-arg STEAM_LOGIN="$STEAM_LOGIN" \
         -t "$GAME_IMAGE" \
         --label "appid=$APPID" \
         --label "mod=$MOD" \
@@ -134,7 +148,10 @@ elif [ "$PIPELINE" = 'update' ]; then
     time docker pull "$GAME_IMAGE"
     date
     time docker build \
+        --secret id=STEAM_USERNAME,env=STEAM_USERNAME \
+        --secret id=STEAM_PASSWORD,env=STEAM_PASSWORD \
         --build-arg GAME_IMAGE="$GAME_IMAGE" \
+        --build-arg STEAM_LOGIN="$STEAM_LOGIN" \
         -t "$GAME_IMAGE" \
         --label "game_version=$GAME_VERSION" \
         --label "game_update_count=$GAME_UPDATE_COUNT" \
@@ -180,4 +197,6 @@ if [ ! "$NO_PUSH" = 'true' ]; then
 fi
 
 # Docker registry logout
-docker logout
+if [ ! "$NO_PUSH" = 'true' ]; then
+    docker logout
+fi

build/Dockerfile

@@ -1,28 +1,40 @@
+# syntax=docker/dockerfile:1
+
 # Builds a base image for the specified game.
 
 FROM startersclan/steamcmd:git-20190605.0.0
 
 ARG SERVER_DIR=/server
-ARG APPMANIFEST_AR_URL=https://github.com/startersclan/hlds-appmanifest/archive/v1.0.0.tar.gz
+ARG APPMANIFEST_AR_URL=https://github.com/startersclan/hlds-appmanifest/archive/refs/tags/v2.0.0.tar.gz
 ARG APPID
 ARG MOD
 ARG FIX_APPMANIFEST=false
 ARG CLIENT_APPID
 ARG INSTALL_COUNT=3
+ARG STEAM_LOGIN
 
 # Download game
-RUN echo "[BUILD] SERVER_DIR: $SERVER_DIR"; \
+RUN --mount=type=secret,id=STEAM_USERNAME \
+    --mount=type=secret,id=STEAM_PASSWORD \
+    STEAM_USERNAME=$( cat /run/secrets/STEAM_USERNAME ); \
+    STEAM_PASSWORD=$( cat /run/secrets/STEAM_PASSWORD ); \
+    echo "[BUILD] SERVER_DIR: $SERVER_DIR"; \
     echo "[BUILD] APPMANIFEST_AR_URL: $APPMANIFEST_AR_URL"; \
     echo "[BUILD] APPID: $APPID"; \
     echo "[BUILD] MOD: $MOD"; \
     echo "[BUILD] FIX_APPMANIFEST: $FIX_APPMANIFEST"; \
     echo "[BUILD] INSTALL_COUNT: $INSTALL_COUNT"; \
+    echo "[BUILD] STEAM_LOGIN: $STEAM_LOGIN"; \
+    if ( [ "$STEAM_LOGIN" = 'true' ] && [ -n "$STEAM_USERNAME" ] && [ -n "$STEAM_PASSWORD" ] || [ "$APPID" = 90 ] ); then \
+        steamcmdLoginArgs="$STEAM_USERNAME $STEAM_PASSWORD"; \
+    else \
+        steamcmdLoginArgs='anonymous'; \
+    fi; \
     if [ "$APPID" = 90 ] && [ -n "$MOD" ]; then \
-        steamcmdArgs="+login anonymous +force_install_dir $SERVER_DIR +app_set_config $APPID mod $MOD +app_update $APPID validate +quit"; \
+        steamcmdArgs="+force_install_dir $SERVER_DIR +login $steamcmdLoginArgs +app_set_config $APPID mod $MOD +app_update $APPID validate +quit"; \
     else \
-        steamcmdArgs="+login anonymous +force_install_dir $SERVER_DIR +app_update $APPID validate +quit"; \
+        steamcmdArgs="+force_install_dir $SERVER_DIR +login $steamcmdLoginArgs +app_update $APPID validate +quit"; \
     fi; \
-    echo "[BUILD] steamcmd arguments: $steamcmdArgs"; \
     echo "[BUILD] Downloading game"; \
     if [ "$APPID" = 90 ] && [ "$FIX_APPMANIFEST" = 'true' ]; then \
         echo "[BUILD] APPID is 90 and FIX_APPMANIFEST is true. Appmanifest fix and install count will apply."; \
@@ -33,8 +45,10 @@ RUN echo "[BUILD] SERVER_DIR: $SERVER_DIR"; \
             i=$(( i+1 )); \
             if [ "$i" -eq 1 ]; then \
                 echo "[BUILD] Downloading and applying appmanifest fix"; \
-                curl -sqL "$APPMANIFEST_AR_URL" | tar -zxvf - -C $SERVER_DIR/steamapps --strip-components=1; \
+                ls -al $SERVER_DIR/steamapps; \
+                curl -sqL "$APPMANIFEST_AR_URL" | tar -zxvf - -C $SERVER_DIR/steamapps --wildcards "$GAME/*" --strip-components=2; \
             fi; \
+            ls -al $SERVER_DIR/steamapps; \
         done; \
     else \
         echo "[BUILD] Either APPID is not 90 or FIX_APPMANIFEST is not true. Appmanifest fix and install count will be ignored."; \
@@ -45,12 +59,15 @@ RUN echo "[BUILD] SERVER_DIR: $SERVER_DIR"; \
         linux64 \
         package \
         public \
+        siteserverui \
         linux32/libtier0_s.so \
         linux32/libvstdlib_s.so \
         linux32/steamclient.so \
         linux32/steamconsole.so \
-    && rm -rf "~/.steam" \
-    && rm -rf "~/Steam"
+        update_hosts_cached.vdf \
+        /root/.steam \
+        /root/Steam
+
 
 # Apply game fixes
 RUN echo "[BUILD] Applying game fixes"; \

notify.circleci.sh

@@ -13,6 +13,7 @@ LATEST=${LATEST:-}
 CACHE=${CACHE:-}
 NO_TEST=${NO_TEST:-}
 NO_PUSH=${NO_PUSH:-}
+STEAM_LOGIN=${STEAM_LOGIN:-}
 
 # Send notification
 date
@@ -34,7 +35,8 @@ BODY=$( cat <<EOF
         "LATEST": "$LATEST",
         "CACHE": "$CACHE",
         "NO_TEST": "$NO_TEST",
-        "NO_PUSH": "$NO_PUSH"
+        "NO_PUSH": "$NO_PUSH",
+        "STEAM_LOGIN": "$STEAM_LOGIN"
     },
     "status": "$BUILD_STATUS"
 }

notify.sh

@@ -28,7 +28,8 @@ BODY=$( cat <<EOF
         "LATEST": "$LATEST",
         "CACHE": "$CACHE",
         "NO_TEST": "$NO_TEST",
-        "NO_PUSH": "$NO_PUSH"
+        "NO_PUSH": "$NO_PUSH",
+        "STEAM_LOGIN": "$STEAM_LOGIN"
     },
     "status": "$JOB_STATUS"
 }

test/build-hlds-cstrike.env

@@ -0,0 +1,17 @@
+REGISTRY_USER=foo
+REGISTRY_PASSWORD=foo
+REGISTRY_GOLDSOURCE=goldsourceservers
+REGISTRY_SOURCE=sourceservers
+
+PIPELINE=build
+
+GAME_VERSION=1127
+APPID=90
+CLIENT_APPID=10
+GAME=cstrike
+MOD=cstrike
+FIX_APPMANIFEST=true
+LATEST=true
+CACHE=
+NO_TEST=
+NO_PUSH=true

test/build-srcds-hl2mp.env

@@ -0,0 +1,17 @@
+REGISTRY_USER=foo
+REGISTRY_PASSWORD=foo
+REGISTRY_GOLDSOURCE=goldsourceservers
+REGISTRY_SOURCE=sourceservers
+
+PIPELINE=build
+
+GAME_VERSION=6630498
+APPID=232370
+CLIENT_APPID=320
+GAME=hl2mp
+MOD=
+FIX_APPMANIFEST=
+LATEST=true
+CACHE=
+NO_TEST=
+NO_PUSH=true

test/update-hlds-cstrike.env

@@ -0,0 +1,13 @@
+REGISTRY_USER=foo
+REGISTRY_PASSWORD=foo
+REGISTRY_GOLDSOURCE=goldsourceservers
+REGISTRY_SOURCE=sourceservers
+
+PIPELINE=update
+
+GAME_VERSION=1127
+APPID=90
+GAME=cstrike
+GAME_UPDATE_COUNT=1
+NO_TEST=
+NO_PUSH=true

test/update-srcds-hl2mp.env

@@ -0,0 +1,13 @@
+REGISTRY_USER=foo
+REGISTRY_PASSWORD=foo
+REGISTRY_GOLDSOURCE=goldsourceservers
+REGISTRY_SOURCE=sourceservers
+
+PIPELINE=update
+
+GAME_VERSION=6630498
+APPID=232370
+GAME=hl2mp
+GAME_UPDATE_COUNT=1
+NO_TEST=
+NO_PUSH=true