From ad4ac9aa7082b97ed6592853bc698bf645fd36f8 Mon Sep 17 00:00:00 2001 From: mehrdadn Date: Fri, 7 Feb 2020 16:19:46 -0800 Subject: [PATCH] Add clang-iwyu (#7081) * Add iwyu Co-authored-by: GitHub Web Flow --- .bazelrc | 1 + .github/workflows/main.yml | 65 ++++++- BUILD.bazel | 31 ++++ ci/travis/iwyu.sh | 94 ++++++++++ thirdparty/protobuf/extra_actions_base.proto | 184 +++++++++++++++++++ 5 files changed, 373 insertions(+), 2 deletions(-) create mode 100755 ci/travis/iwyu.sh create mode 100644 thirdparty/protobuf/extra_actions_base.proto diff --git a/.bazelrc b/.bazelrc index 8ef0453f3..2aef06b72 100644 --- a/.bazelrc +++ b/.bazelrc @@ -26,6 +26,7 @@ build --host_copt="-Wno-microsoft-unqualified-friend" # This workaround is needed due to https://github.com/bazelbuild/bazel/issues/4341 build --per_file_copt="-\\.(asm|S)$,external/com_github_grpc_grpc/.*@-DGRPC_BAZEL_BUILD" build --http_timeout_scaling=5.0 +build:iwyu --experimental_action_listener=//:iwyu_cpp # This workaround is due to an incompatibility of # bazel_common/tools/maven/pom_file.bzl with Bazel 1.0 build --incompatible_depset_is_not_iterable=false diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f943534e2..d71505c04 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,10 +2,71 @@ name: CI env: LLVM_VERSION_WINDOWS: 9.0.0 + DEBIAN_FRONTEND: noninteractive on: [push, pull_request] jobs: + iwyu: + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + name: [ + ubuntu-clang-iwyu, + ] + include: + - name: ubuntu-clang-iwyu + os: ubuntu-latest + compiler: clang + steps: + - name: Checkout repository + uses: actions/checkout@v1 + with: + fetch-depth: 1 + - name: Setup Bazel + shell: bash + env: + BAZEL_CACHE_CREDENTIAL_B64: ${{ secrets.BAZEL_CACHE_CREDENTIAL_B64 }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: ./ci/travis/install-bazel.sh + - name: Setup Clang Include-What-You-Use + shell: bash + run: | + sudo apt-get install iwyu + - name: Perform build + continue-on-error: true + shell: bash + env: + CC: ${{ matrix.compiler }} + run: | + # TODO(mehrdadn): Replace this with the same build script as below, once we factor that out + set -euo pipefail + PATH="${HOME}/bin:${PATH}" + main() { + local startflags=() + startflags+=(--batch) + startflags+=(--nodeep_execroot) + local cmdflags=() + cmdflags+=(--attempt_to_print_relative_paths) + cmdflags+=(--color=yes) + cmdflags+=(--experimental_repository_cache_hardlinks) + cmdflags+=(--experimental_ui_deduplicate) + cmdflags+=(--incompatible_strict_action_env) + cmdflags+=(--keep_going) + cmdflags+=(--per_file_copt='-\.(asm|S)$@-fansi-escape-codes') + cmdflags+=(--per_file_copt='-\.(asm|S)$@-fcolor-diagnostics') + cmdflags+=(--show_progress_rate_limit=5) + cmdflags+=(--show_task_finish) + cmdflags+=(--show_timestamps) + cmdflags+=(--symlink_prefix=/) + cmdflags+=(--verbose_failures) + local packages=() + packages+=("//:ray_pkg") + bazel "${startflags[@]}" build "${cmdflags[@]}" "${packages[@]}" "$@" + } + main --compilation_mode=fastbuild --config=iwyu "$@" build: name: ${{ matrix.name }} runs-on: ${{ matrix.os }} @@ -98,8 +159,8 @@ jobs: cmdflags+=(--experimental_ui_deduplicate) cmdflags+=(--incompatible_strict_action_env) cmdflags+=(--keep_going) - cmdflags+=(--per_file_copt="-\\.(asm|S)$@-fansi-escape-codes") - cmdflags+=(--per_file_copt="-\\.(asm|S)$@-fcolor-diagnostics") + cmdflags+=(--per_file_copt='-\.(asm|S)$@-fansi-escape-codes') + cmdflags+=(--per_file_copt='-\.(asm|S)$@-fcolor-diagnostics') cmdflags+=(--show_progress_rate_limit=5) cmdflags+=(--show_task_finish) cmdflags+=(--show_timestamps) diff --git a/BUILD.bazel b/BUILD.bazel index 92c9f0b0f..94172b612 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -835,6 +835,37 @@ cc_library( ], ) +filegroup( + name = "iwyu_sh", + srcs = ["ci/travis/iwyu.sh"], +) + +filegroup( + name = "extra_actions_base_proto", + srcs = [ + # TODO: Replace our file with the built-in copy once this issue is resolved: + # https://github.com/bazelbuild/bazel/issues/8738 + "thirdparty/protobuf/extra_actions_base.proto", + #"@bazel_tools//src/main/protobuf:extra_actions_base.proto", + ], +) + +action_listener( + name = "iwyu_cpp", + extra_actions = [":iwyu_action"], + mnemonics = ["CppCompile"], +) + +extra_action( + name = "iwyu_action", + cmd = "$(location :iwyu_sh) $(location @com_google_protobuf//:protoc) $(location :extra_actions_base_proto) --extra_action_file=$(EXTRA_ACTION_FILE)", + tools = [ + ":extra_actions_base_proto", + ":iwyu_sh", + "@com_google_protobuf//:protoc", + ], +) + cc_library( name = "sha256", srcs = [ diff --git a/ci/travis/iwyu.sh b/ci/travis/iwyu.sh new file mode 100755 index 000000000..a3994fe80 --- /dev/null +++ b/ci/travis/iwyu.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [ "${OSTYPE-}" = msys ] && [ -z "${MINGW_DIR+x}" ]; then + if [ "${HOSTTYPE-}" = x86_64 ]; then + MINGW_DIR=/mingw64 + elif [ "${HOSTTYPE-}" = i686 ]; then + MINGW_DIR=/mingw32 + fi +fi +invoke_cc() { + local env_vars=() args=() env_parsed=0 + if [ "${OSTYPE}" = msys ]; then + env_vars+=(MSYS2_ARG_CONV_EXCL="*") + fi + local arg; for arg in "$@"; do + if [ 0 -ne "${env_parsed}" ]; then + args+=("${arg}") + elif [ "${arg}" == "--" ]; then + env_parsed=1 + else + if [ "${OSTYPE}" = msys ] && [ ! "${arg}" = "${arg#PATH=}" ]; then + local key="${arg%%=*}" + local value="${arg#*=}" + value="${value//'/'\\''}" + value="'${value//;/\' \'}'" + local paths; declare -a paths="(${value})" + value="" + local path; for path in "${paths[@]}"; do + if [ "${OSTYPE}" = msys ]; then + path="${path//\\//}" + fi + case "${path}" in + [a-zA-Z]:|[a-zA-Z]:/*) + local drive; drive="${path%%:*}" + path="/${drive,,}${path#*:}" + ;; + *) true;; + esac + if [ -n "${value}" ]; then + value="${value}:" + fi + value="${value}${path}" + done + arg="${key}=${value}" + fi + env_vars+=("${arg}") + fi + done + local cc; cc="${args[0]}" + case "${cc##*/}" in + clang*) + { PATH="${PATH}:/usr/bin" env -i "${env_vars[@]}" "${SHELL-/bin/bash}" -c 'iwyu -isystem "$("$1" -print-resource-dir "${@:2}")/include" "${@:2}"' exec "${args[@]}" 2>&1 || true; } | awk ' + { header = 0; } + /^(The full include-list for .*|.* should (add|remove) these lines:)$/ { keep = 1; header = 1; } + /^(The full include-list for ([^\/]*\/)*external\/.*|([^\/]*\/)*external\/.* should (add|remove) these lines:)$/ { keep = 0; header = 1; } + /^The full include-list for .*$/ { keep = 0; header = 1; } + /^---$/ { keep = 0; header = 1; } + keep { print; n += header ? 0 : 1; } + END { exit n; } + ' + ;; + esac +} + +main() { + # Parsing might fail due to various things like aspects (e.g. BazelCcProtoAspect), but we don't want to check generated files anyway + local data="" # initialize in case next line fails + data="$(exec "$1" --decode=blaze.CppCompileInfo "$2" < "${3#*=}" 2>&-)" || true + if [ -n "${data}" ]; then + # Convert output to JSON-like format so we can parse it + data="$(exec sed -e "/^[a-zA-Z]/d" -e "s/^\(\s*\)\([0-9]\+\):\s*\(.*\)/\1[\2, \3],/g" -e "s/^\(\s*\)\([0-9]\+\)\s*{/\1[\2, /g" -e "s/^\(\s*\)}/\1],/g" <<< "${data}")" + # Turn everything into a single line by replacing newlines with a special character that should never appear in the actual stream + data="$(exec tr "\n" "\a" <<< "${data}")" + # Remove some fields that we don't want, and put back newlines we removed + data="$(exec sed -e "s/\(0x[0-9a-fA-F]*\)]\(,\a\)/\"\1\"]\2/g" -e "s/,\(\a\s*\(]\|\$\)\)/\1/g" -e "s/\a/\n/g" <<< "${data}")" + # Parse the resulting JSON and select the actual fields we're interested in + data="$(PATH="${PATH}:${MINGW_DIR-/usr}/bin" && exec jq -r '( + [] + + [.[1:][] | select (.[0] == 6) | "\(.[1][1])=\(.[2][1])" | gsub("'\''"; "'\''\\'\'''\''") | "'\''\(.)'\''"] + + ["--"] + + [.[1:][] | select (.[0] == 1) | .[1] | gsub("'\''"; "'\''\\'\'''\''") | "'\''\(.)'\''"] + + [.[1:][] | select (.[0] == 2) | .[1] ] + ) | .[]' <<< "${data}")" + # On Windows, jq can insert carriage returns; remove them + data="$(exec tr -d "\r" <<< "${data}")" + # Wrap everything into a single line for passing as argument array + data="$(exec tr "\n" " " <<< "${data}")" + eval invoke_cc "${data}" + fi +} + +main "$@" diff --git a/thirdparty/protobuf/extra_actions_base.proto b/thirdparty/protobuf/extra_actions_base.proto new file mode 100644 index 000000000..3be65b9a3 --- /dev/null +++ b/thirdparty/protobuf/extra_actions_base.proto @@ -0,0 +1,184 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// proto definitions for the blaze extra_action feature. + +syntax = "proto2"; + +package blaze; + +option java_multiple_files = true; +option java_package = "com.google.devtools.build.lib.actions.extra"; +// option cc_api_version = 2; +// option java_api_version = 2; + +// A list of extra actions and metadata for the print_action command. +message ExtraActionSummary { + repeated DetailedExtraActionInfo action = 1; +} + +// An individual action printed by the print_action command. +message DetailedExtraActionInfo { + // If the given action was included in the output due to a request for a + // specific file, then this field contains the name of that file so that the + // caller can correctly associate the extra action with that file. + // + // The data in this message is currently not sufficient to run the action on a + // production machine, because not all necessary input files are identified, + // especially for C++. + // + // There is no easy way to fix this; we could require that all header files + // are declared and then add all of them here (which would be a huge superset + // of the files that are actually required), or we could run the include + // scanner and add those files here. + optional string triggering_file = 1; + // The actual action. + required ExtraActionInfo action = 2; +} + +// Provides information to an extra_action on the original action it is +// shadowing. +message ExtraActionInfo { + extensions 1000 to max; + + // The label of the ActionOwner of the shadowed action. + optional string owner = 1; + + // Only set if the owner is an Aspect. + // Corresponds to AspectValue.AspectKey.getAspectClass.getName() + // This field is deprecated as there might now be + // multiple aspects applied to the same target. + // This is the aspect name of the last aspect + // in 'aspects' (8) field. + optional string aspect_name = 6 [deprecated = true]; + + // Only set if the owner is an Aspect. + // Corresponds to AspectValue.AspectKey.getParameters() + // This field is deprecated as there might now be + // multiple aspects applied to the same target. + // These are the aspect parameters of the last aspect + // in 'aspects' (8) field. + map aspect_parameters = 7 [deprecated = true]; + message StringList { + option deprecated = true; + repeated string value = 1; + } + + message AspectDescriptor { + // Corresponds to AspectDescriptor.getName() + optional string aspect_name = 1; + // Corresponds to AspectDescriptor.getParameters() + map aspect_parameters = 2; + message StringList { + repeated string value = 1; + } + } + + // If the owner is an aspect, all aspects applied to the target + repeated AspectDescriptor aspects = 8; + + // An id uniquely describing the shadowed action at the ActionOwner level. + optional string id = 2; + + // The mnemonic of the shadowed action. Used to distinguish actions with the + // same ActionType. + optional string mnemonic = 5; +} + +message EnvironmentVariable { + // It is possible that this name is not a valid variable identifier. + required string name = 1; + // The value is unescaped and unquoted. + required string value = 2; +} + +// Provides access to data that is specific to spawn actions. +// Usually provided by actions using the "Spawn" & "Genrule" Mnemonics. +message SpawnInfo { + extend ExtraActionInfo { + optional SpawnInfo spawn_info = 1003; + } + + repeated string argument = 1; + // A list of environment variables and their values. No order is enforced. + repeated EnvironmentVariable variable = 2; + repeated string input_file = 4; + repeated string output_file = 5; +} + +// Provides access to data that is specific to C++ compile actions. +// Usually provided by actions using the "CppCompile" Mnemonic. +message CppCompileInfo { + extend ExtraActionInfo { + optional CppCompileInfo cpp_compile_info = 1001; + } + + optional string tool = 1; + repeated string compiler_option = 2; + optional string source_file = 3; + optional string output_file = 4; + // Due to header discovery, this won't include headers unless the build is + // actually performed. If set, this field will include the value of + // "source_file" in addition to the headers. + repeated string sources_and_headers = 5; + // A list of environment variables and their values. No order is enforced. + repeated EnvironmentVariable variable = 6; +} + +// Provides access to data that is specific to C++ link actions. +// Usually provided by actions using the "CppLink" Mnemonic. +message CppLinkInfo { + extend ExtraActionInfo { + optional CppLinkInfo cpp_link_info = 1002; + } + + repeated string input_file = 1; + optional string output_file = 2; + optional string interface_output_file = 3; + optional string link_target_type = 4; + optional string link_staticness = 5; + repeated string link_stamp = 6; + repeated string build_info_header_artifact = 7; + // The list of command line options used for running the linking tool. + repeated string link_opt = 8; +} + +// Provides access to data that is specific to java compile actions. +// Usually provided by actions using the "Javac" Mnemonic. +message JavaCompileInfo { + extend ExtraActionInfo { + optional JavaCompileInfo java_compile_info = 1000; + } + + optional string outputjar = 1; + repeated string classpath = 2; + repeated string sourcepath = 3; + repeated string source_file = 4; + repeated string javac_opt = 5; + repeated string processor = 6; + repeated string processorpath = 7; + repeated string bootclasspath = 8; + repeated string argument = 9; +} + +// Provides access to data that is specific to python rules. +// Usually provided by actions using the "Python" Mnemonic. +message PythonInfo { + extend ExtraActionInfo { + optional PythonInfo python_info = 1005; + } + + repeated string source_file = 1; + repeated string dep_file = 2; +}