Update and rename build_bromite.yaml to build_bromite_dev.yaml

.github/workflows/build_bromite_dev.yaml

@@ -0,0 +1,427 @@
+name: Build Bromite
+permissions:
+  actions: none
+  checks: none
+  contents: none
+  deployments: none
+  issues: none
+  packages: none
+  pull-requests: none
+  repository-projects: none
+  security-events: none
+  statuses: none
+      
+on:
+  workflow_dispatch:
+    inputs:
+      sha:
+        description: 'uazo/bromite SHA'
+        required: true
+        default: '76745fde6ed75542f005ea5528f486df41c56da4'
+      build:
+        description: 'arm64/x64'
+        required: true
+        default: 'x64'
+      type:
+        description: 'dev/ci'
+        required: true
+        default: 'dev'
+      debug:
+        description: 'true/false'
+        required: true
+        default: 'true'
+      usegoma:
+        description: 'Use goma?'
+        required: true
+        default: 'false'
+        
+env:
+  BROMITE_SHA: ${{ github.event.inputs.sha }}
+  USEGOMA: ${{ github.event.inputs.usegoma }}
+  REMOVEDOCKERSUPPORT: true
+  USELOCALIMAGE: true
+  GOMAJOBS: 60
+
+jobs:
+  check_images:
+    runs-on: ${{ github.event.inputs.type }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+        with:
+          path: bromite-buildtools
+          fetch-depth: 1
+
+      - name: Enable proxy on container
+        shell: bash
+        run: |
+          if ! [[ -z "${HTTP_PROXY}" ]]; then
+            PROXY_ADDR=http://$(hostname -I | cut -d' ' -f1 | xargs):8118
+            echo "PROXY_ADDR=$PROXY_ADDR" >> $GITHUB_ENV
+            sudo iptables -D INPUT -p tcp -s localhost --dport 8118 -j ACCEPT
+            sudo iptables -D INPUT -p tcp --dport 8118 -j DROP
+          fi
+
+      - name: Get current chromium version
+        shell: bash
+        run: |
+          mkdir bromite
+          cd bromite
+          git init
+          git remote add origin https://github.com/uazo/bromite
+          git fetch origin $BROMITE_SHA
+          git reset --hard FETCH_HEAD
+          cd ..
+          
+          export VERSION=$( cat ./bromite/build/RELEASE )
+          rm -rf bromite
+          
+          echo Current version is $VERSION
+          echo "VERSION=$VERSION" >> $GITHUB_ENV
+          
+          cd bromite-buildtools
+          
+      - name: Checking build-deps for ${{ env.VERSION }}
+        shell: bash
+        run: |
+          IS_PRESENT=$(docker inspect --type=image uazo/build-deps:$VERSION > /dev/null ; echo $?)
+          if [ $IS_PRESENT -ne "0" ]; then
+            IS_PRESENT=$(docker manifest inspect uazo/build-deps:$VERSION > /dev/null ; echo $?)
+            if [ $IS_PRESENT -ne "0" ]; then
+              DOCKER_BUILDKIT=1 docker build -t uazo/build-deps:$VERSION \
+                --progress plain \
+                --build-arg VERSION=$VERSION \
+                --build-arg HTTP_PROXY="$PROXY_ADDR" \
+                --no-cache \
+                bromite-buildtools/images/build-deps/.
+            fi
+          fi
+        
+      - name: Checking chromium for ${{ env.VERSION }}
+        shell: bash
+        run: |
+          IS_PRESENT=$(docker inspect --type=image uazo/chromium:$VERSION > /dev/null ; echo $?)
+          if [ $IS_PRESENT -ne "0" ]; then
+            IS_PRESENT=$(docker manifest inspect uazo/chromium:$VERSION > /dev/null ; echo $?)
+            if [ $IS_PRESENT -ne "0" ]; then
+              DOCKER_BUILDKIT=1 docker build -t uazo/chromium:$VERSION \
+                --progress plain \
+                --build-arg VERSION=$VERSION \
+                --build-arg HTTP_PROXY="$PROXY_ADDR" \
+                bromite-buildtools/images/chr-source/.
+            fi
+          fi
+
+      - name: Checking bromite for ${{ env.BROMITE_SHA }}
+        shell: bash
+        run: |
+          IS_PRESENT=$(docker inspect --type=image uazo/bromite:$BROMITE_SHA > /dev/null ; echo $?)
+          if [ $IS_PRESENT -ne "0" ]; then
+            IS_PRESENT=$(docker manifest inspect uazo/bromite:$BROMITE_SHA > /dev/null ; echo $?)
+            if [ $IS_PRESENT -ne "0" ]; then
+              DOCKER_BUILDKIT=1 docker build -t uazo/bromite:$BROMITE_SHA --progress plain \
+                --build-arg BROMITE_SHA=$BROMITE_SHA \
+                --build-arg VERSION=$VERSION \
+                --build-arg HTTP_PROXY="$PROXY_ADDR" \
+                bromite-buildtools/images/bromite-source/.
+            fi
+          fi
+          
+      - name: Checking bromite-build for ${{ env.BROMITE_SHA }}
+        shell: bash
+        run: |
+          IS_PRESENT=$(docker inspect --type=image uazo/bromite-build:$BROMITE_SHA > /dev/null ; echo $?)
+          if [ $IS_PRESENT -ne "0" ]; then
+            IS_PRESENT=$(docker manifest inspect uazo/bromite-build:$BROMITE_SHA > /dev/null ; echo $?)
+            if [ $IS_PRESENT -ne "0" ]; then
+              DOCKER_BUILDKIT=1 docker build -t uazo/bromite-build:$BROMITE_SHA --progress plain \
+                --build-arg BROMITE_SHA=$BROMITE_SHA \
+                --build-arg HTTP_PROXY="$PROXY_ADDR" \
+                --no-cache \
+                bromite-buildtools/images/bromite-build/.
+            fi
+          fi
+        
+      - name: Mark image to build
+        shell: bash
+        run: |
+          IS_PRESENT=$(docker inspect --type=image uazo/bromite-build:build > /dev/null ; echo $?)
+          if [ $IS_PRESENT -eq "0" ]; then
+            docker rmi uazo/bromite-build:build
+          fi
+          docker tag uazo/bromite-build:$BROMITE_SHA uazo/bromite-build:build
+          
+  build:
+    runs-on: ${{ github.event.inputs.type }}
+    needs: check_images
+    if: success()
+    timeout-minutes: 720
+
+    services:      
+      gomaserver:
+        image: uazo/goma-server
+        volumes: 
+          - /tmp/proxy:/tmp/proxy
+          - /redis:/var/lib/redis
+        options: >-
+          --health-cmd "hostname -I >/tmp/proxy/gomaserverip"
+          --health-interval 30s
+          --health-timeout 5s
+          --health-retries 5
+          
+        env:
+          REMOVEDOCKERSUPPORT: true
+          USELOCALIMAGE: true
+  
+    container:
+      image: uazo/bromite-build:build
+      env:
+        SERVER_HOST_GOMA: gomaserver
+        REMOVEDOCKERSUPPORT: true # CUSTOM RUNNER: remove sharing of docker socket
+        USELOCALIMAGE: true       # CUSTOM RUNNER: permit use of local images
+        USEINTERNALNETWORK: true  # CUSTOM RUNNER: create the docker network as internal
+        WORKSPACE: /home/lg/working_dir
+        KYTHE_CORPUS: chromium.googlesource.com/chromium/src
+        KYTHE_ROOT_DIRECTORY: /home/lg/working_dir/chromium/src
+        KYTHE_OUTPUT_DIRECTORY: /home/lg/working_dir/chromium/src/out/bromite/kythe
+        # compile in debug mode
+        TARGET_ISDEBUG: true 
+        TARGET_CPU: ${{ github.event.inputs.build }}
+      volumes:
+        - /storage/images/${{ github.event.inputs.build }}/${{ github.event.inputs.sha }}:/home/lg/working_dir/chromium/src/out/bromite
+        - /tmp/proxy:/tmp/proxy
+        
+    steps:    
+      - name: Prepare Build Container
+        shell: bash
+        run: |
+          # set workspace paths
+          PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
+          cd $WORKSPACE
+
+          # reset proxy env
+          HTTP_PROXY=
+          HTTPS_PROXY=
+          http_proxy=
+          https_proxy=
+          
+          # update hosts
+          #sudo echo "$(cat /tmp/proxy/gomaserverip | xargs) gomaserver" >/etc/hosts
+          #sudo echo "$(cat /tmp/proxy/redisip | xargs) redis" >/etc/hosts
+          
+          # set goma options
+          export SERVER_HOST_GOMA=$(cat /tmp/proxy/gomaserverip | xargs)
+          export GOMA_SERVER_HOST=$SERVER_HOST_GOMA
+          export GOMA_SERVER_PORT=5050
+          export GOMA_USE_SSL=false
+          export GOMA_HTTP_AUTHORIZATION_FILE=$WORKSPACE/.debug_auth_file
+          export GOMA_HERMETIC=error
+          export GOMA_USE_LOCAL=false
+          export GOMA_FALLBACK=true
+          export GOMA_ARBITRARY_TOOLCHAIN_SUPPORT=true
+          export GOMA_MAX_SUBPROCS_LOW=0
+          
+          # set out folder permissions
+          test -d chromium/src/out/bromite || sudo mkdir -p chromium/src/out/bromite && \
+            sudo chown lg chromium/src/out &&
+            sudo chown lg chromium/src/out/bromite &&
+          
+          # make kythe output directory
+          test -d $KYTHE_OUTPUT_DIRECTORY || mkdir -p $KYTHE_OUTPUT_DIRECTORY
+
+          sudo mkdir -p /run/user/1000/
+          sudo chown lg /run/user/1000/
+          sudo chmod g-rxw /run/user/1000/
+          sudo chmod o-rxw /run/user/1000/
+
+          # start goma client
+          [[ "$USEGOMA" = "true" ]] && \
+              echo "::group::-------- start goma client" && \
+              $WORKSPACE/goma/goma_ctl.py ensure_stop && \
+              $WORKSPACE/goma/goma_ctl.py ensure_start && \
+              echo "::endgroup::" \
+            || true
+          
+          cd chromium/src
+
+          OUT_PRESENT=0
+          test -f out/bromite/prepare_gn && OUT_PRESENT=1
+          if [[ OUT_PRESENT -eq 0 ]]; then
+
+             echo "::group::-------- gn gen"
+             [[ "$USEGOMA" = "true" ]] && \
+                gn gen --args="import(\"/home/lg/working_dir/bromite/build/GN_ARGS\") use_goma=true goma_dir=\"$WORKSPACE/goma\" $(cat ../../build_args.gni) " out/bromite \
+                || \
+                gn gen --args="import(\"/home/lg/working_dir/bromite/build/GN_ARGS\") $(cat ../../build_args.gni) " out/bromite              
+             echo "::endgroup::"
+
+             echo "::group::-------- gn args"
+             gn args out/bromite/ --list --short
+             gn args out/bromite/ --list >out/bromite/gn_list
+             echo "::endgroup::"
+
+             echo "::group::-------- apply .mtool"
+             test -f out/bromite/.mtool && \
+                cp out/bromite/.mtool .mtool && \
+                $WORKSPACE/mtool/chromium/mtime.sh --restore
+             echo "::endgroup::"
+             
+             echo "OK" >out/bromite/prepare_gn
+
+          fi
+
+          if [[ -z "${GOMAJOBS}" ]]; then
+              GOMAJOBS=40
+          fi
+
+          echo "::group::-------- pre-cache toolchain"
+          [[ "$USEGOMA" = "true" ]] && \
+            sudo ../../casupload --cas-server=unix:/tmp/proxy/bots.sock --instance=default_instance \
+                    third_party/android_ndk/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include \
+                    third_party/android_ndk/toolchains/llvm/prebuilt/linux-x86_64/include \
+                    third_party/llvm-build/Release+Asserts/lib \
+                    third_party/llvm-build/Release+Asserts/bin \
+                    buildtools/third_party/libc++ \
+              chrome/android/profiles/afdo.prof \
+            || true
+          echo "::endgroup::"
+
+      - name: Build Bromite
+        shell: bash
+        run: |
+          PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
+          cd $WORKSPACE/chromium/src
+          
+          [[ "$USEGOMA" = "true" ]] && \
+            autoninja -j $GOMAJOBS -C out/bromite chrome_public_apk \
+            || \
+            autoninja -C out/bromite chrome_public_apk
+
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          files: /home/lg/working_dir/chromium/src/out/bromite/apks/ChromePublic.apk
+          
+      - name: Get ninja logs
+        shell: bash
+        run: |
+          cd $WORKSPACE
+          $WORKSPACE/ninjatracing/ninjatracing $WORKSPACE/chromium/src/out/bromite/.ninja_log >$WORKSPACE/chromium/src/out/bromite/ninja_log_trace.json
+          $WORKSPACE/chromium/src/third_party/catapult/tracing/bin/trace2html $WORKSPACE/chromium/src/out/bromite/ninja_log_trace.json
+        
+      - name: Get goma logs
+        shell: bash
+        run: |
+          # reset proxy env
+          HTTP_PROXY=
+          HTTPS_PROXY=
+          http_proxy=
+          https_proxy=
+          
+           [[ "$USEGOMA" = "true" ]] && \
+             wget http://127.0.0.1:8088/logz?INFO -O out/bromite/goma-client.log \
+           || \
+             echo Skipped.
+          
+      - name: Generate breakpad symbols
+        shell: bash
+        run: |
+          PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
+          cd $WORKSPACE/chromium/src
+          
+          echo "::group::-------- generating breakpad symbols"
+          autoninja -j $GOMAJOBS -C out/bromite minidump_stackwalk dump_syms
+          components/crash/content/tools/generate_breakpad_symbols.py --build-dir=out/bromite \
+             --symbols-dir=out/bromite/symbols/ --binary=out/bromite/lib.unstripped/libchrome.so \
+             --platform=android --clear --verbose
+          cp out/bromite/lib.unstripped/libchrome.so out/bromite/symbols/libchrome.lib.so
+          cp out/bromite/minidump_stackwalk out/bromite/symbols
+          cp out/bromite/dump_syms out/bromite/symbols
+          echo "::endgroup::"
+
+      - name: Generate kythe kzip
+        shell: bash
+        run: |
+          PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
+          cd $WORKSPACE/chromium/src
+          
+          #python tools/clang/scripts/generate_compdb.py -p out/bromite/ -o out/bromite/compile_commands.json chrome_public_apk
+          #ninja -C out/bromite chrome_public_apk -t compdb cc cxx objc objcxx >out/bromite/compile_commands.json
+          $WORKSPACE/ninja/ninja -C $WORKSPACE/chromium/src/out/bromite -a chrome_public_apk \
+            -t compdb cc cxx objc objcxx >$WORKSPACE/chromium/src/out/bromite/compile_commands.json
+          gn desc out/bromite 'chrome_public_apk' --format=json >out/bromite/gn_targets.json
+          
+          #python tools/clang/scripts/run_tool.py --tool translation_unit -p out/bromite/ --all || true
+          tools/clang/scripts/run_tool.py --tool translation_unit -p out/bromite || true
+          python /home/lg/add_kythe_metadata.py out/bromite/ --corpus $KYTHE_CORPUS || true
+
+          /home/lg/package_index/latest/package_index \
+            --checkout_dir /home/lg/working_dir/chromium/src \
+            --path_to_compdb /home/lg/working_dir/chromium/src/out/bromite/compile_commands.json \
+            --path_to_gn_targets /home/lg/working_dir/chromium/src/out/bromite/gn_targets.json \
+            --path_to_java_kzips $KYTHE_OUTPUT_DIRECTORY \
+            --path_to_archive_output /home/lg/working_dir/chromium/src/out/bromite/chromium_linux.kzip \
+            --corpus $KYTHE_CORPUS \
+            --out_dir src/out/bromite || true
+
+      - name: Generate clangd index
+        shell: bash
+        run: |
+          PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
+          cd $WORKSPACE/chromium/src
+          /home/lg/working_dir/clangd_snapshot_20211205/bin/clangd-indexer --executor=all-TUs out/bromite/compile_commands.json >out/bromite/bromite.idx
+          
+      - name: Build junit tests
+        shell: bash
+        run: |
+          PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
+          cd $WORKSPACE/chromium/src
+          
+          git log | grep FILE:Fix-build-test-suite.patch && TEST=1
+          if [[ TEST -eq 1 ]]; then
+            autoninja -j $GOMAJOBS -C out/bromite chrome_junit_tests
+            autoninja -j $GOMAJOBS -C out/bromite components_junit_tests
+            autoninja -j $GOMAJOBS -C out/bromite content_junit_tests
+            autoninja -j $GOMAJOBS -C out/bromite base_junit_tests
+            autoninja -j $GOMAJOBS -C out/bromite ui_junit_tests
+          fi
+          
+      - name: Build c++ tests
+        shell: bash
+        run: |
+          PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
+          cd $WORKSPACE/chromium/src
+          
+          git log | grep FILE:Fix-build-test-suite.patch && TEST=1
+          if [[ TEST -eq 1 ]]; then
+            autoninja -j $GOMAJOBS -C out/bromite network_service
+            autoninja -j $GOMAJOBS -C out/bromite unit_tests
+            autoninja -j $GOMAJOBS -C out/bromite content_browsertests
+            autoninja -j $GOMAJOBS -C out/bromite components_unittests
+            autoninja -j $GOMAJOBS -C out/bromite components_browsertests
+          fi
+          
+      - name: Build instrumentation tests
+        shell: bash
+        run: |
+          PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
+          cd $WORKSPACE/chromium/src
+          
+          git log | grep FILE:Fix-build-test-suite.patch && TEST=1
+          if [[ TEST -eq 1 ]]; then
+            autoninja -j $GOMAJOBS -C out/bromite content_shell_test_apk
+            autoninja -j $GOMAJOBS -C out/bromite chrome_public_test_apk
+          fi
+          
+      - name: Stop goma
+        shell: bash
+        run: |
+          PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
+          cd $WORKSPACE/chromium/src
+
+          echo "::group::-------- stop goma"
+          [[ "$USEGOMA" = "true" ]] && $WORKSPACE/goma/goma_ctl.py ensure_stop || true
+          [[ "$USEGOMA" = "true" ]] && find /tmp/ -maxdepth 1 -name "gomacc*" -print0 | xargs -0 rm || true
+          [[ "$USEGOMA" = "true" ]] && rm -rf /tmp/goma_lg/ || true
+          echo "::endgroup::"