github-ci: use bundle.sh script for libhtp, suricata-update

Update the GitHub CI workflow to use the bundle.sh script to pull in
Suricata-Update and libhtp. This means one less place where defaults
are hardcoded and can get out of sync.

This also simplifies the variable names that can be embedded in a pull
request message to use the same variable names that bundle.sh
expects. Of note, this removes the _PR variant, instead a branch name
of "pr/N" can be used to specify a PR.

.github/PULL_REQUEST_TEMPLATE.md

@@ -11,12 +11,15 @@ Describe changes:
 -
 -
 
-#suricata-verify-pr:
+### Provide values to any of the below to override the defaults.
-#suricata-verify-repo:
+
-#suricata-verify-branch:
+To use a pull request use a branch name like `pr/N` where `N` is the pull request number.
-#suricata-update-pr:
+
-#suricata-update-repo:
+```
-#suricata-update-branch:
+SV_REPO=
-#libhtp-pr:
+SV_BRANCH=
-#libhtp-repo:
+SU_REPO=
-#libhtp-branch:
+SU_BRANCH=
+LIBHTP_REPO=
+LIBHTP_BRANCH=
+```

.github/workflows/builds.yml

@@ -7,17 +7,8 @@ on:
 permissions: read-all
 
 env:
-  DEFAULT_LIBHTP_REPO: https://github.com/OISF/libhtp
-  DEFAULT_LIBHTP_BRANCH: 0.5.x
-  DEFAULT_LIBHTP_PR:
-
-  DEFAULT_SU_REPO: https://github.com/OISF/suricata-update
-  DEFAULT_SU_BRANCH: master
-  DEFAULT_SU_PR:
-
   DEFAULT_SV_REPO: https://github.com/OISF/suricata-verify
   DEFAULT_SV_BRANCH: master
-  DEFAULT_SV_PR:
 
   DEFAULT_CFLAGS: "-Wall -Wextra -Werror -Wno-unused-parameter -Wno-unused-function"
 
@@ -56,82 +47,72 @@ jobs:
               echo "Parsing branch and PR info from:"
               echo "${body}"
 
-              libhtp_repo=$(echo "${body}" | awk '/^libhtp-repo/ { print $2 }')
+              LIBHTP_REPO=$(echo "${body}" | awk -F = '/^LIBHTP_REPO=/ { print $2 }')
-              libhtp_branch=$(echo "${body}" | awk '/^libhtp-branch/ { print $2 }')
+              LIBHTP_BRANCH=$(echo "${body}" | awk -F = '/^LIBHTP_BRANCH=/ { print $2 }')
-              libhtp_pr=$(echo "${body}" | awk '/^libhtp-pr/ { print $2 }')
 
-              su_repo=$(echo "${body}" | awk '/^suricata-update-repo/ { print $2 }')
+              SU_REPO=$(echo "${body}" | awk -F = '/^SU_REPO=/ { print $2 }')
-              su_branch=$(echo "${body}" | awk '/^suricata-update-branch/ { print $2 }')
+              SU_BRANCH=$(echo "${body}" | awk -F = '/^SU_BRANCH=/ { print $2 }')
-              su_pr=$(echo "${body}" | awk '/^suricata-update-pr/ { print $2 }')
 
-              sv_repo=$(echo "${body}" | awk '/^suricata-verify-repo/ { print $2 }')
+              SV_REPO=$(echo "${body}" | awk -F = '/^SV_REPO=/ { print $2 }')
-              sv_branch=$(echo "${body}" | awk '/^suricata-verify-branch/ { print $2 }')
+              SV_BRANCH=$(echo "${body}" | awk -F = '/^SV_BRANCH=/ { print $2 }')
-              sv_pr=$(echo "${body}" | awk '/^suricata-verify-pr/ { print $2 }')
           else
-              echo "PR_HREF is empty"
+              echo "No pull request body, will use defaults."
           fi
-          echo "libhtp_repo=${libhtp_repo:-${DEFAULT_LIBHTP_REPO}}" >> $GITHUB_ENV
-          echo "libhtp_branch=${libhtp_branch:-${DEFAULT_LIBHTP_BRANCH}}" >> $GITHUB_ENV
-          echo "libhtp_pr=${libhtp_pr:-${DEFAULT_LIBHTP_PR}}" >> $GITHUB_ENV
 
-          echo "su_repo=${su_repo:-${DEFAULT_SU_REPO}}" >> $GITHUB_ENV
+          echo LIBHTP_REPO=${LIBHTP_REPO} | tee -a ${GITHUB_ENV}
-          echo "su_branch=${su_branch:-${DEFAULT_SU_BRANCH}}" >> $GITHUB_ENV
+          echo LIBHTP_BRANCH=${LIBHTP_BRANCH} | tee -a ${GITHUB_ENV}
-          echo "su_pr=${su_pr:-${DEFAULT_SU_PR}}" >> $GITHUB_ENV
+
+          echo SU_REPO=${SU_REPO} | tee -a ${GITHUB_ENV}
+          echo SU_BRANCH=${SU_BRANCH} | tee -a ${GITHUB_ENV}
+
+          echo SV_REPO=${SV_REPO:-${DEFAULT_SV_REPO}} | tee -a ${GITHUB_ENV}
+          echo SV_BRANCH=${SV_BRANCH:-${DEFAULT_SV_BRANCH}} | tee -a ${GITHUB_ENV}
 
-          echo "sv_repo=${sv_repo:-${DEFAULT_SV_REPO}}" >> $GITHUB_ENV
-          echo "sv_branch=${sv_branch:-${DEFAULT_SV_BRANCH}}" >> $GITHUB_ENV
-          echo "sv_pr=${sv_pr:-${DEFAULT_SV_PR}}" >> $GITHUB_ENV
       - name: Annotate output
         run: |
-          echo "::notice:: LIBHTP_REPO=${libhtp_repo}"
+          echo "::notice:: LIBHTP_REPO=${LIBHTP_REPO}"
-          echo "::notice:: LIBHTP_BRANCH=${libhtp_branch}"
+          echo "::notice:: LIBHTP_BRANCH=${LIBHTP_BRANCH}"
-          echo "::notice:: LIBHTP_PR=${libhtp_pr}"
+          echo "::notice:: SU_REPO=${SU_REPO}"
-          echo "::notice:: SU_REPO=${su_repo}"
+          echo "::notice:: SU_BRANCH=${SU_BRANCH}"
-          echo "::notice:: SU_BRANCH=${su_branch}"
+          echo "::notice:: SV_REPO=${SV_REPO}"
-          echo "::notice:: SU_PR=${su_pr}"
+          echo "::notice:: SV_BRANCH=${SV_BRANCH}"
-          echo "::notice:: SV_REPO=${sv_repo}"
+
-          echo "::notice:: SV_BRANCH=${sv_branch}"
+      # Now checkout Suricata for the bundle script.
-          echo "::notice:: SV_PR=${sv_pr}"
+      - name: Checking out Suricata
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+
       - name: Fetching libhtp
         run: |
-          git clone --depth 1 ${libhtp_repo} -b ${libhtp_branch} libhtp
+          DESTDIR=./bundle ./scripts/bundle.sh libhtp
-          if [[ "${libhtp_pr}" != "" ]]; then
+          tar zcf libhtp.tar.gz -C bundle libhtp
-              cd libhtp
-              git fetch origin pull/${libhtp_pr}/head:prep
-              git checkout prep
-              cd ..
-          fi
-          tar zcf libhtp.tar.gz libhtp
       - name: Fetching suricata-update
         run: |
-          git clone --depth 1 ${su_repo} -b ${su_branch} suricata-update
+          DESTDIR=./bundle ./scripts/bundle.sh suricata-update
-          if [[ "${su_pr}" != "" ]]; then
+          tar zcf suricata-update.tar.gz -C bundle suricata-update
-              cd suricata-update
+
-              git fetch origin pull/${su_pr}/head:prep
-              git checkout prep
-              cd ..
-          fi
-          tar zcf suricata-update.tar.gz suricata-update
       - name: Fetching suricata-verify
         run: |
-          git clone ${sv_repo} -b ${sv_branch} suricata-verify
+          pr=$(echo "${SV_BRANCH}" | sed -n 's/^pr\/\([[:digit:]]\+\)$/\1/p')
-          if [[ "${sv_pr}" != "" ]]; then
+          if [ "${pr}" ]; then
-              cd suricata-verify
+              SV_BRANCH="refs/pull/${pr}/head"
-              git fetch origin pull/${sv_pr}/head:prep
+              echo "Using suricata-verify pull-request ${SV_BRANCH}"
-              git checkout prep
+          else
-              git config --global user.email you@example.com
+              echo "Using suricata-verify branch ${SV_BRANCH}"
-              git config --global user.name You
-              git rebase ${DEFAULT_SV_BRANCH}
-              cd ..
           fi
+          git clone --depth 1 ${SV_REPO} suricata-verify
+          cd suricata-verify
+          git fetch --depth 1 origin ${SV_BRANCH}
+          git -c advice.detachedHead=false checkout FETCH_HEAD
+          cd ..
           tar zcf suricata-verify.tar.gz suricata-verify
-      - name: Cleaning up
-        run: rm -rf libhtp suricata-update suricata-verify
       - name: Uploading prep archive
         uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
         with:
           name: prep
-          path: .
+          path: |
+            libhtp.tar.gz
+            suricata-update.tar.gz
+            suricata-verify.tar.gz
 
   prepare-cbindgen:
     name: Prepare cbindgen