bromite-buildtools/.github/workflows/build_bromite.yaml

name: Build x86
permissions:
  actions: none
  checks: none
  contents: none
  deployments: none
  issues: none
  packages: none
  pull-requests: none
  repository-projects: none
  security-events: none
  statuses: none
      
on:
  workflow_dispatch:
    inputs:
      sha:
        description: 'uazo/bromite SHA'
        required: true
        default: '76745fde6ed75542f005ea5528f486df41c56da4'
        
env:
  BROMITE_SHA: ${{ github.event.inputs.sha }}
  REMOVEDOCKERSUPPORT: true
  USELOCALIMAGE: true
  GOMAJOBS: 60

jobs:
  check_images:
    runs-on: self-hosted
    steps:
      - name: Checkout repo
        uses: actions/checkout@v2
        with:
          path: bromite-buildtools
          fetch-depth: 1

      - name: Enable proxy on container
        shell: bash
        run: |
          if ! [[ -z "${HTTP_PROXY}" ]]; then
            PROXY_ADDR=http://$(hostname -I | xargs):8118
            echo "PROXY_ADDR=$PROXY_ADDR" >> $GITHUB_ENV
            sudo iptables -D INPUT -p tcp -s localhost --dport 8118 -j ACCEPT
            sudo iptables -D INPUT -p tcp --dport 8118 -j DROP
          fi

      - name: Get current chromium version
        shell: bash
        run: |
          mkdir bromite
          cd bromite
          git init
          git remote add origin https://github.com/uazo/bromite
          git fetch origin $BROMITE_SHA
          git reset --hard FETCH_HEAD
          cd ..
          
          export VERSION=$( cat ./bromite/build/RELEASE )
          rm -rf bromite
          
          echo Current version is $VERSION
          echo "VERSION=$VERSION" >> $GITHUB_ENV
          
          cd bromite-buildtools
          
      - name: Checking build-deps for ${{ env.VERSION }}
        shell: bash
        run: |
          IS_PRESENT=$(docker inspect --type=image uazo/build-deps:$VERSION > /dev/null ; echo $?)
          if [ $IS_PRESENT -ne "0" ]; then
            IS_PRESENT=$(docker manifest inspect uazo/build-deps:$VERSION > /dev/null ; echo $?)
            if [ $IS_PRESENT -ne "0" ]; then
              DOCKER_BUILDKIT=1 docker build -t uazo/build-deps:$VERSION \
                --progress plain \
                --build-arg VERSION=$VERSION \
                --build-arg HTTP_PROXY="$PROXY_ADDR" \
                --no-cache \
                bromite-buildtools/images/build-deps/.
            fi
          fi
        
      - name: Checking chromium for ${{ env.VERSION }}
        shell: bash
        run: |
          IS_PRESENT=$(docker inspect --type=image uazo/chromium:$VERSION > /dev/null ; echo $?)
          if [ $IS_PRESENT -ne "0" ]; then
            IS_PRESENT=$(docker manifest inspect uazo/chromium:$VERSION > /dev/null ; echo $?)
            if [ $IS_PRESENT -ne "0" ]; then
              DOCKER_BUILDKIT=1 docker build -t uazo/chromium:$VERSION \
                --progress plain \
                --build-arg VERSION=$VERSION \
                --build-arg HTTP_PROXY="$PROXY_ADDR" \
                bromite-buildtools/images/chr-source/.
            fi
          fi

      - name: Checking bromite for ${{ env.BROMITE_SHA }}
        shell: bash
        run: |
          IS_PRESENT=$(docker inspect --type=image uazo/bromite:$BROMITE_SHA > /dev/null ; echo $?)
          if [ $IS_PRESENT -ne "0" ]; then
            IS_PRESENT=$(docker manifest inspect uazo/bromite:$BROMITE_SHA > /dev/null ; echo $?)
            if [ $IS_PRESENT -ne "0" ]; then
              DOCKER_BUILDKIT=1 docker build -t uazo/bromite:$BROMITE_SHA --progress plain \
                --build-arg BROMITE_SHA=$BROMITE_SHA \
                --build-arg VERSION=$VERSION \
                --build-arg HTTP_PROXY="$PROXY_ADDR" \
                bromite-buildtools/images/bromite-source/.
            fi
          fi
          
      - name: Checking bromite-build for ${{ env.BROMITE_SHA }}
        shell: bash
        run: |
          IS_PRESENT=$(docker inspect --type=image uazo/bromite-build:$BROMITE_SHA > /dev/null ; echo $?)
          if [ $IS_PRESENT -ne "0" ]; then
            IS_PRESENT=$(docker manifest inspect uazo/bromite-build:$BROMITE_SHA > /dev/null ; echo $?)
            if [ $IS_PRESENT -ne "0" ]; then
              DOCKER_BUILDKIT=1 docker build -t uazo/bromite-build:$BROMITE_SHA --progress plain \
                --build-arg BROMITE_SHA=$BROMITE_SHA \
                --build-arg HTTP_PROXY="$PROXY_ADDR" \
                --no-cache \
                bromite-buildtools/images/bromite-build/.
            fi
          fi
        
      - name: Mark image to build
        shell: bash
        run: |
          IS_PRESENT=$(docker inspect --type=image uazo/bromite-build:build > /dev/null ; echo $?)
          if [ $IS_PRESENT -eq "0" ]; then
            docker rmi uazo/bromite-build:build
          fi
          docker tag uazo/bromite-build:$BROMITE_SHA uazo/bromite-build:build
          
  build:
    runs-on: self-hosted
    needs: check_images
    if: success()
    timeout-minutes: 720

    services:      
      gomaserver:
        image: uazo/goma-server
        volumes: 
          - /tmp/proxy:/tmp/proxy
          - /redis:/var/lib/redis
        options: >-
          --health-cmd "hostname -I >/tmp/proxy/gomaserverip"
          --health-interval 30s
          --health-timeout 5s
          --health-retries 5
          
        env:
          REMOVEDOCKERSUPPORT: true
          USELOCALIMAGE: true
  
    container:
      image: uazo/bromite-build:build
      env:
        SERVER_HOST_GOMA: gomaserver
        REMOVEDOCKERSUPPORT: true # CUSTOM RUNNER: remove sharing of docker socket
        USELOCALIMAGE: true       # CUSTOM RUNNER: permit use of local images
        USEINTERNALNETWORK: true  # CUSTOM RUNNER: create the docker network as internal
        WORKSPACE: /home/lg/working_dir
        ARTIFACS_DIR: /home/lg/working_dir/artifacs
      volumes:
        - /storage/images/${{ github.event.inputs.sha }}:/home/lg/working_dir/artifacs
        - /tmp/proxy:/tmp/proxy
        
    steps:    
      - name: Prepare Build Container
        shell: bash
        run: |
          # set workspace paths
          PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
          cd $WORKSPACE
          
          # clean artifacts
          sudo chmod 777 $ARTIFACS_DIR
          rm -rf $ARTIFACS_DIR/*

          # reset proxy env
          HTTP_PROXY=
          HTTPS_PROXY=
          http_proxy=
          https_proxy=
          
          # update hosts
          #sudo echo "$(cat /tmp/proxy/gomaserverip | xargs) gomaserver" >/etc/hosts
          #sudo echo "$(cat /tmp/proxy/redisip | xargs) redis" >/etc/hosts
          
          # set goma options
          export SERVER_HOST_GOMA=$(cat /tmp/proxy/gomaserverip | xargs)
          export GOMA_SERVER_HOST=$SERVER_HOST_GOMA
          export GOMA_SERVER_PORT=5050
          export GOMA_USE_SSL=false
          export GOMA_HTTP_AUTHORIZATION_FILE=$WORKSPACE/.debug_auth_file
          export GOMA_HERMETIC=error
          export GOMA_USE_LOCAL=false
          export GOMA_FALLBACK=true
          export GOMA_ARBITRARY_TOOLCHAIN_SUPPORT=true

          # start goma client
          echo "::group::-------- start goma client"
          $WORKSPACE/goma/goma_ctl.py ensure_stop
          $WORKSPACE/goma/goma_ctl.py ensure_start
          echo "::endgroup::"
          
          cd chromium/src

          OUT_PRESENT=0
          test -d out/bromite && OUT_PRESENT=1
          if [[ OUT_PRESENT -eq 0 ]]; then

             echo "::group::-------- sync out folder"
             test -d $ARTIFACS_DIR/out/bromite && \
                mkdir -p out/bromite && \
                cp -arp $ARTIFACS_DIR/out/bromite/* out/bromite/
             echo "::endgroup::"

             echo "::group::-------- gn gen"
             gn gen --args="import(\"/home/lg/working_dir/bromite/build/GN_ARGS\") use_goma=true goma_dir=\"$WORKSPACE/goma\" $(cat ../../build_args.gni) " out/bromite
             echo "::endgroup::"

             echo "::group::-------- gn args"
             gn args out/bromite/ --list --short
             gn args out/bromite/ --list >$ARTIFACS_DIR/gn_list
             echo "::endgroup::"

             echo "::group::-------- apply .mtool"
             test -f out/bromite/.mtool && \
                cp out/bromite/.mtool .mtool && \
                $WORKSPACE/mtool/chromium/mtime.sh --restore
             echo "::endgroup::"

          fi

          if [[ -z "${GOMAJOBS}" ]]; then
              GOMAJOBS=40
          fi

          echo "::group::-------- pre-cache toolchain"
          sudo ../../casupload --cas-server=unix:/tmp/proxy/bots.sock --instance=default_instance \
                  third_party/android_ndk/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include \
                  third_party/android_ndk/toolchains/llvm/prebuilt/linux-x86_64/include \
                  third_party/llvm-build/Release+Asserts/lib \
                  third_party/llvm-build/Release+Asserts/bin \
                  buildtools/third_party/libc++ \
            chrome/android/profiles/afdo.prof
          echo "::endgroup::"

      - name: Build Bromite
        shell: bash
        run: |
          PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
          cd $WORKSPACE/chromium/src
          
          autoninja -j $GOMAJOBS -C out/bromite chrome_public_apk

      - name: Get goma logs
        shell: bash
        run: |
          # reset proxy env
          HTTP_PROXY=
          HTTPS_PROXY=
          http_proxy=
          https_proxy=
          
          wget http://127.0.0.1:8088/logz?INFO -O $ARTIFACS_DIR/goma-client.log
          
      - name: Generate breakpad symbols
        shell: bash
        run: |
          PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
          cd $WORKSPACE/chromium/src
          
          cp out/bromite/apks/* $WORKSPACE/artifacs/

          echo "::group::-------- generating breakpad symbols"
          autoninja -j $GOMAJOBS -C out/bromite minidump_stackwalk dump_syms
          components/crash/content/tools/generate_breakpad_symbols.py --build-dir=out/bromite \
             --symbols-dir=$ARTIFACS_DIR/symbols/ --binary=out/bromite/lib.unstripped/libchrome.so \
             --platform=android --clear --verbose
          cp out/bromite/lib.unstripped/libchrome.so $ARTIFACS_DIR/symbols/libchrome.lib.so
          cp out/bromite/minidump_stackwalk $ARTIFACS_DIR/symbols
          cp out/bromite/dump_syms $ARTIFACS_DIR/symbols
          echo "::endgroup::"

      - name: Build junit tests
        shell: bash
        run: |
          PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
          cd $WORKSPACE/chromium/src
          
          autoninja -j $GOMAJOBS -C out/bromite chrome_junit_tests
          autoninja -j $GOMAJOBS -C out/bromite components_junit_tests
          autoninja -j $GOMAJOBS -C out/bromite content_junit_tests
          autoninja -j $GOMAJOBS -C out/bromite base_junit_tests
          autoninja -j $GOMAJOBS -C out/bromite ui_junit_tests
          
      - name: Build c++ tests
        shell: bash
        run: |
          PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
          cd $WORKSPACE/chromium/src
          
          autoninja -j $GOMAJOBS -C out/bromite network_service
          autoninja -j $GOMAJOBS -C out/bromite unit_tests
          autoninja -j $GOMAJOBS -C out/bromite content_browsertests
          autoninja -j $GOMAJOBS -C out/bromite components_unittests
          
      - name: Build instrumentation tests
        shell: bash
        run: |
          PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
          cd $WORKSPACE/chromium/src
          autoninja -j $GOMAJOBS -C out/bromite content_shell_test_apk
          autoninja -j $GOMAJOBS -C out/bromite chrome_public_test_apk

      - name: Sync out folder
        shell: bash
        run: |
          PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
          cd $WORKSPACE/chromium/src

          echo "::group::-------- sync out folder"
          $WORKSPACE/mtool/chromium/mtime.sh --backup
          mv .mtool out/bromite/
          cp -arp out/bromite $ARTIFACS_DIR/out
          echo "::endgroup::"
          
      - name: Stop goma
        shell: bash
        run: |
          PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
          cd $WORKSPACE/chromium/src

          echo "::group::-------- stop goma"
          $WORKSPACE/goma/goma_ctl.py ensure_stop
          find /tmp/ -maxdepth 1 -name "gomacc*" -print0 | xargs -0 rm
          rm -rf /tmp/goma_lg/
          echo "::endgroup::"