llvm 17 fixes (lifting-bits#688)

* llvm 17 fix * more instances * more fixes * add inline * bump ci * bump cxx common * add 17 case * bump xcode version * macos 13 * xcode 15 * bump docker to llvm 17 * change actual matrix value * opaque pointers are the default now * actually remove the option * debug size * install tree * fix install * fix install? * get tree after * build dir? * tree above * accumulate * bug? * try to remove intermediate packaging * fix opt * only inline * give up on function inlining * llvm 17 * remove debug tasks * Update scripts/build.sh Co-authored-by: William Tan <[email protected]> * Revert "Update scripts/build.sh" This reverts commit 6c727a5. * debug * try clear out old build * retest * try to save more space * up * Clear space and don't use cxx-common docker image (lifting-bits#689) * Clear space and don't use cxx-common docker image * Run on all PRs * Fix perms * Remove some unnecessary apt commands * Install LLVM version as root * Allow writing to external target * Build before running tests * Install with sudo * Simplify with container volume mounts --------- Co-authored-by: William Tan <[email protected]> Co-authored-by: Eric Kilmer <[email protected]>
ict-sys-group · Oct 12, 2023 · 0183248 · 0183248
1 parent 7182636
commit 0183248
Show file tree

Hide file tree

Showing 21 changed files with 123 additions and 150 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -3,32 +3,28 @@ name: VCPKG Continuous Integration
 on:
  # Run this workflow once every 6 hours against the master branch
  schedule:
- - cron: "0 */6 * * *"
+  - cron: "0 */6 * * *"
 
  push:
  branches:
- - 'master'
+ - "master"
 
  tags:
  - "*"
 
  pull_request:
- branches:
- - "*"
 
 jobs:
  build_linux:
  strategy:
  fail-fast: false
  matrix:
  image:
- - { name: 'ubuntu', tag: '22.04' }
- llvm: ['15', '16']
+ - { name: "ubuntu", tag: "22.04" }
+ llvm: ["17"]
  compiler:
- - { CC: 'clang', CXX: 'clang++' }
- - { CC: 'gcc', CXX: 'g++' }
- exclude:
- - { llvm: '15', compiler: { CC: 'gcc', CXX: 'g++' } }
+ - { CC: "clang", CXX: "clang++" }
+ - { CC: "gcc", CXX: "g++" }
 
  env:
  CC: ${{ matrix.compiler.CC }}
@@ -37,6 +33,8 @@ jobs:
  runs-on: ubuntu-22.04
  container:
  image: ghcr.io/lifting-bits/cxx-common/vcpkg-builder-${{ matrix.image.name }}:${{ matrix.image.tag }}
+ volumes:
+ - /:/gha-runner
  credentials:
  username: ${{ github.actor }}
  password: ${{ secrets.GITHUB_TOKEN }}
@@ -46,15 +44,25 @@ jobs:
  with:
  fetch-depth: 0
  - uses: ./.github/actions/prepare_git_user
- - name: Get Poetry
+
+ - name: Clear space
  shell: bash
  run: |
- python3 -m pip install poetry
+ df -h
+ rm -rf /gha-runner/usr/local/lib/android
+ rm -rf /gha-runner/usr/local/share/boost
+ df -h
+
  - name: Build with build script
  shell: bash
  run: |
  git config --global --add safe.directory "$GITHUB_WORKSPACE"
  ./scripts/build.sh --llvm-version ${{ matrix.llvm }}
+ mkdir remill-rel/
+ mv remill-build/*.deb ./remill-rel
+ mv remill-build/*.rpm ./remill-rel
+ mv remill-build/*.tar.gz ./remill-rel
+ rm -rf remill-build/_CPack_Packages
  - name: Build with build-presets script
  shell: bash
  run: |
@@ -65,13 +73,25 @@ jobs:
  shell: bash
  run: |
  pip3 install --user ./scripts/diff_tester_export_insns
+ - name: Tree size
+ shell: bash
+ run: |
+ du -hs
+ df -h
  - name: Run tests
  shell: bash
  working-directory: remill-build
  run: |
  cmake --build . --target install -- -j "$(nproc)"
  cmake --build . --target test_dependencies -- -j "$(nproc)"
  env CTEST_OUTPUT_ON_FAILURE=1 cmake --build . --target test -- -j "$(nproc)"
+ - name: Tree size after
+ shell: bash
+ if: failure()
+ run: |
+ df -h
+ du -h $(pwd)/../ | sort -h
+ df -h
  - name: Smoketests with installed executable
  shell: bash
  run: |
@@ -82,11 +102,11 @@ jobs:
  - name: Locate the packages
  id: package_names
  shell: bash
- working-directory: remill-build
+ working-directory: remill-rel
  run: |
- echo ::set-output name=DEB_PACKAGE_PATH::remill-build/$(ls *.deb)
- echo ::set-output name=RPM_PACKAGE_PATH::remill-build/$(ls *.rpm)
- echo ::set-output name=TGZ_PACKAGE_PATH::remill-build/$(ls *.tar.gz)
+ echo ::set-output name=DEB_PACKAGE_PATH::remill-rel/$(ls *.deb)
+ echo ::set-output name=RPM_PACKAGE_PATH::remill-rel/$(ls *.rpm)
+ echo ::set-output name=TGZ_PACKAGE_PATH::remill-rel/$(ls *.tar.gz)
 
  - name: Store the DEB package
  uses: actions/upload-artifact@v1
@@ -110,10 +130,10 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- os: ['macos-12']
- llvm: ['15', '16']
+ os: ["macos-13"]
+ llvm: ["17"]
 
- runs-on: macos-12
+ runs-on: macos-13
 
  steps:
  - uses: actions/checkout@v2
@@ -137,7 +157,7 @@ jobs:
  shell: bash
  run: |
  brew install ninja
- export CMAKE_TOOLCHAIN_FILE=$(pwd)/../lifting-bits-downloads/vcpkg_${{ matrix.os}}_llvm-${{ matrix.llvm }}_xcode-14.2_amd64/scripts/buildsystems/vcpkg.cmake
+ export CMAKE_TOOLCHAIN_FILE=$(pwd)/../lifting-bits-downloads/vcpkg_${{ matrix.os}}_llvm-${{ matrix.llvm }}_xcode-15.0_amd64/scripts/buildsystems/vcpkg.cmake
  export INSTALL_DIR=$(pwd)/remill-preset-install
  ./scripts/build-preset.sh release
  - name: Run tests
@@ -167,12 +187,9 @@ jobs:
  name: ${{ matrix.os }}_llvm${{ matrix.llvm }}_tgz_package
  path: ${{ steps.package_names.outputs.TGZ_PACKAGE_PATH }}
 
-
-
-
  release_packages:
  # Do not run the release procedure if any of the builds has failed
- needs: [ build_linux, build_mac ]
+ needs: [build_linux, build_mac]
  runs-on: ubuntu-22.04
  if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
 
@@ -238,20 +255,19 @@ jobs:
  asset_name: remill_macos-12_packages.zip
  asset_content_type: application/gzip
 
-
  Docker_Linux:
  runs-on: ubuntu-latest
  strategy:
  matrix:
- llvm: ["15", "16"]
+ llvm: ["17"]
  ubuntu: ["22.04"]
  steps:
- - uses: actions/checkout@v2
- - name: Build LLVM ${{ matrix.llvm }} on ${{ matrix.ubuntu }}
- run: |
- docker build . -t ghcr.io/lifting-bits/remill/remill-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest -f Dockerfile --build-arg UBUNTU_VERSION=${{ matrix.ubuntu }} --build-arg ARCH=amd64 --build-arg LLVM_VERSION=${{ matrix.llvm }}
- - name: Test Docker image
- run: |
- docker run --rm ghcr.io/lifting-bits/remill/remill-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --arch amd64 --ir_out /dev/stdout --bytes c704ba01000000
- docker run --rm ghcr.io/lifting-bits/remill/remill-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --arch aarch64 --ir_out /dev/stdout --address 0x400544 --bytes FD7BBFA90000009000601891FD030091B7FFFF97E0031F2AFD7BC1A8C0035FD6
- docker run --rm ghcr.io/lifting-bits/remill/remill-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --arch aarch32 -ir_out /dev/stderr --bytes 0cd04de208008de504108de500208de508309de504009de500109de5903122e0c20fa0e110109fe5001091e5002081e5040081e50cd08de21eff2fe14000000000000000
+  - uses: actions/checkout@v2
+  - name: Build LLVM ${{ matrix.llvm }} on ${{ matrix.ubuntu }}
+  run: |
+  docker build . -t ghcr.io/lifting-bits/remill/remill-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest -f Dockerfile --build-arg UBUNTU_VERSION=${{ matrix.ubuntu }} --build-arg ARCH=amd64 --build-arg LLVM_VERSION=${{ matrix.llvm }}
+  - name: Test Docker image
+  run: |
+  docker run --rm ghcr.io/lifting-bits/remill/remill-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --arch amd64 --ir_out /dev/stdout --bytes c704ba01000000
+  docker run --rm ghcr.io/lifting-bits/remill/remill-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --arch aarch64 --ir_out /dev/stdout --address 0x400544 --bytes FD7BBFA90000009000601891FD030091B7FFFF97E0031F2AFD7BC1A8C0035FD6
+  docker run --rm ghcr.io/lifting-bits/remill/remill-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --arch aarch32 -ir_out /dev/stderr --bytes 0cd04de208008de504108de500208de508309de504009de500109de5903122e0c20fa0e110109fe5001091e5002081e5040081e50cd08de21eff2fe14000000000000000
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -61,6 +61,8 @@ message(STATUS "LLVM Libraries: ${llvm_libs}")
 message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}")
 message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
 
+include_directories(SYSTEM ${LLVM_INCLUDE_DIRS})
+
 string(REPLACE "." ";" LLVM_VERSION_LIST ${LLVM_PACKAGE_VERSION})
 list(GET LLVM_VERSION_LIST 0 LLVM_MAJOR_VERSION)
 list(GET LLVM_VERSION_LIST 1 LLVM_MINOR_VERSION)

diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 # Choose your LLVM version
-ARG LLVM_VERSION=16
+ARG LLVM_VERSION=17
 ARG ARCH=amd64
 ARG UBUNTU_VERSION=22.04
 ARG DISTRO_BASE=ubuntu${UBUNTU_VERSION}

diff --git a/bin/differential_tester_x86/LiftAndCompare.cpp b/bin/differential_tester_x86/LiftAndCompare.cpp
@@ -16,6 +16,7 @@
 
 #include <glog/logging.h>
 #include <gtest/gtest.h>
+#include <llvm/ADT/StringExtras.h>
 #include <llvm/ExecutionEngine/ExecutionEngine.h>
 #include <llvm/ExecutionEngine/GenericValue.h>
 #include <llvm/ExecutionEngine/Interpreter.h>

diff --git a/cmake/utils.cmake b/cmake/utils.cmake
@@ -161,7 +161,7 @@ function(InstallExternalTarget target_name target_path install_type installed_fi
 
  install(FILES "${output_file_path}"
  TYPE ${install_type}
- PERMISSIONS OWNER_READ OWNER_EXECUTE
+ PERMISSIONS OWNER_READ OWNER_EXECUTE OWNER_WRITE
  GROUP_READ GROUP_EXECUTE
  WORLD_READ WORLD_EXECUTE
  )

diff --git a/include/remill/Arch/Arch.h b/include/remill/Arch/Arch.h
@@ -25,7 +25,7 @@
 #pragma clang diagnostic ignored "-Wswitch-enum"
 
 #include <llvm/ADT/SmallVector.h>
-#include <llvm/ADT/Triple.h>
+#include <llvm/TargetParser/Triple.h>
 #include <llvm/IR/DataLayout.h>
 #include <llvm/IR/IRBuilder.h>
 #include <remill/BC/InstructionLifter.h>

diff --git a/lib/Arch/AArch64/Arch.cpp b/lib/Arch/AArch64/Arch.cpp
@@ -16,12 +16,12 @@
 
 #include <gflags/gflags.h>
 #include <glog/logging.h>
-#include <llvm/ADT/Triple.h>
 #include <llvm/IR/Attributes.h>
 #include <llvm/IR/DataLayout.h>
 #include <llvm/IR/Function.h>
 #include <llvm/IR/IRBuilder.h>
 #include <llvm/IR/Module.h>
+#include <llvm/TargetParser/Triple.h>
 
 #include <algorithm>
 #include <cctype>

diff --git a/lib/Arch/Arch.cpp b/lib/Arch/Arch.cpp
@@ -20,6 +20,7 @@
 #include <glog/logging.h>
 #include <llvm/ADT/APInt.h>
 #include <llvm/ADT/SmallVector.h>
+#include <llvm/IR/AttributeMask.h>
 #include <llvm/IR/BasicBlock.h>
 #include <llvm/IR/Function.h>
 #include <llvm/IR/IRBuilder.h>

diff --git a/lib/Arch/Name.cpp b/lib/Arch/Name.cpp
@@ -16,7 +16,7 @@
 
 #include "remill/Arch/Name.h"
 
-#include <llvm/ADT/Triple.h>
+#include <llvm/TargetParser/Triple.h>
 
 namespace remill {
 

diff --git a/lib/Arch/Sleigh/AArch32Arch.cpp b/lib/Arch/Sleigh/AArch32Arch.cpp
@@ -17,12 +17,12 @@
 #include "AArch32Arch.h"
 
 #include <glog/logging.h>
-#include <llvm/ADT/Triple.h>
 #include <llvm/IR/Attributes.h>
 #include <llvm/IR/DataLayout.h>
 #include <llvm/IR/Function.h>
 #include <llvm/IR/IRBuilder.h>
 #include <llvm/IR/Module.h>
+#include <llvm/TargetParser/Triple.h>
 #include <remill/Arch/AArch32/ArchContext.h>
 
 #include <iomanip>

diff --git a/lib/Arch/Sleigh/ARMBase.cpp b/lib/Arch/Sleigh/ARMBase.cpp
@@ -1,10 +1,10 @@
 #include <glog/logging.h>
-#include <llvm/ADT/Triple.h>
 #include <llvm/IR/Attributes.h>
 #include <llvm/IR/DataLayout.h>
 #include <llvm/IR/Function.h>
 #include <llvm/IR/IRBuilder.h>
 #include <llvm/IR/Module.h>
+#include <llvm/TargetParser/Triple.h>
 #include <remill/Arch/AArch32/AArch32Base.h>
 #include <remill/Arch/Name.h>
 #include <remill/BC/ABI.h>

diff --git a/lib/Arch/Sleigh/X86Arch.cpp b/lib/Arch/Sleigh/X86Arch.cpp
@@ -15,12 +15,12 @@
  */
 
 #include <glog/logging.h>
-#include <llvm/ADT/Triple.h>
 #include <llvm/IR/Attributes.h>
 #include <llvm/IR/DataLayout.h>
 #include <llvm/IR/Function.h>
 #include <llvm/IR/IRBuilder.h>
 #include <llvm/IR/Module.h>
+#include <llvm/TargetParser/Triple.h>
 #include <remill/Arch/Context.h>
 #include <remill/Arch/Name.h>
 #include <remill/Arch/X86/X86Base.h>

diff --git a/lib/Arch/X86/Arch.cpp b/lib/Arch/X86/Arch.cpp
@@ -15,12 +15,12 @@
  */
 
 #include <glog/logging.h>
-#include <llvm/ADT/Triple.h>
 #include <llvm/IR/Attributes.h>
 #include <llvm/IR/DataLayout.h>
 #include <llvm/IR/Function.h>
 #include <llvm/IR/IRBuilder.h>
 #include <llvm/IR/Module.h>
+#include <llvm/TargetParser/Triple.h>
 #include <remill/Arch/ArchBase.h> // For `Arch` and `ArchImpl`.
 #include <remill/Arch/X86/X86Base.h>