Sync Java Agent (#63)

* Internal Change

PiperOrigin-RevId: 514417320

* Internal Code Change

PiperOrigin-RevId: 515270706

* Add missing proto build steps to Makefile builds

PiperOrigin-RevId: 518057341

* Internal Code Change

PiperOrigin-RevId: 520552498

* Internal Code Change

PiperOrigin-RevId: 520600713

* Internal Code Change

PiperOrigin-RevId: 520853442

* Prevent segfaults due to race conditions when Java Heap sampler is enabled.

PiperOrigin-RevId: 521492609

* Prevent other segfaults due to race conditions when Java Heap sampler is enabled.

PiperOrigin-RevId: 522395875

* Prevent memory leak due to race conditions when Java Heap sampler is enabled.

PiperOrigin-RevId: 523535577

* Internal testing logs

PiperOrigin-RevId: 533466682

* Internal testing logs

PiperOrigin-RevId: 534069370

* Split out ThreadProfile code to a separate target in order to add unit tests.

PiperOrigin-RevId: 548381574

* Internal Code Change

PiperOrigin-RevId: 555494681

* Remove uses of the deprecated DISALLOW_IMPLICIT_CONSTRUCTORS in favor of explicitly deleted constructors and assignment operators..

PiperOrigin-RevId: 556161979

* Internal Code Change

PiperOrigin-RevId: 557667699

* Change `T const&&` to `T`.

const r-value vector reference always results in a copy.

PiperOrigin-RevId: 572682185

* Fix SIGSEGV in `HeapMonitor` during shutdown

PiperOrigin-RevId: 576185633

* Update roots.pem to align with the current public set

PiperOrigin-RevId: 576940801

* Fix potential crash in shutdown due to null jvmti_ pointer.

PiperOrigin-RevId: 578310101

* Update profile.proto to match pprof's original.

PiperOrigin-RevId: 578657868

* Fix ClangTidy warnings and format code.

PiperOrigin-RevId: 580609161

* Make HeapMonitor::Enable()'s use_jvm_trace parameter default to false.

We plan to delete code path for use_jvm_trace == true in the future.

PiperOrigin-RevId: 588853678

* Allow users to configure the max number of garbage traces via a parameter to HeapMonitor::Enable().

PiperOrigin-RevId: 589244025

* Fix overflow in heap_sampler when a single object's size is greater than 2147483647.

Changed type for sample size from jint (int32_t) to jlong (int64_t).

PiperOrigin-RevId: 591017093

* Internal Code Change

PiperOrigin-RevId: 594103049

* Unconditionally define `DISALLOW_COPY_AND_ASSIGN`

PiperOrigin-RevId: 595165882

* Automated Code Change

PiperOrigin-RevId: 598691690

* Sync changes to profile.proto.

PiperOrigin-RevId: 599222268

* chore: update gRPC to v1.36.4

PiperOrigin-RevId: 607338479

---------

Co-authored-by: cloud-profiler-team <cloud-profiler-team@google.com>
Co-authored-by: aalexand <aalexand@google.com>

Author: 3 people

Dockerfile

@@ -20,11 +20,22 @@ FROM ubuntu:xenial
 #
 # Dependencies
 #
+# Install JDK 11 as sampling heap profiler depends on the new JVMTI APIs
+# Install Kitware apt repository. This repository contains newer version of
+# CMake. CMake > 3.13 is required to use "module" mode for gRPC dependencies.
+# See https://github.com/grpc/grpc/blob/master/BUILDING.md#install-after-build.
+# Kitware installs CMake v3.20.5
 
-# Install JDK 11 as sampling heap profiler depends on the new JVMTI APIs.
-RUN apt-get update && apt-get install -y software-properties-common
-RUN add-apt-repository -y ppa:openjdk-r/ppa
-RUN apt-get update
+RUN apt-get update && apt-get install -y software-properties-common apt-transport-https ca-certificates gnupg wget && \
+    add-apt-repository -y ppa:openjdk-r/ppa && \
+    test -f /usr/share/doc/kitware-archive-keyring/copyright || \
+    wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null \
+    | gpg --dearmor - \
+    | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null && \
+    echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ xenial main' \
+    | tee /etc/apt/sources.list.d/kitware.list >/dev/null && \
+    apt-get update && \
+    apt-get install kitware-archive-keyring
 
 # Installing openjdk-11-jdk-headless can be very slow due to repo download
 # speed.
@@ -41,6 +52,7 @@ RUN apt-get update && apt-get -y -q install \
   libtool \
   make \
   openjdk-11-jdk-headless \
+  pkg-config \
   unzip \
   zlib1g-dev
 
@@ -81,21 +93,47 @@ RUN mkdir /tmp/glog && cd /tmp/glog && \
     make -j && make install && \
     cd ~ && rm -rf /tmp/glog
 
-# gRPC & protobuf
+# gRPC & protobuf - build using CMake
 # Use the protobuf version from gRPC for everything to avoid conflicting
 # versions to be linked in. Disable OpenSSL embedding: when it's on, the build
 # process of gRPC puts the OpenSSL static object files into the gRPC archive
 # which causes link errors later when the agent is linked with the static
 # OpenSSL library itself.
 # Limit the number of threads used by make, as unlimited threads causes
 # memory exhausted error on the Kokoro VM.
-RUN git clone --depth=1 --recursive -b v1.28.1 https://github.com/grpc/grpc.git /tmp/grpc && \
-    cd /tmp/grpc && \
-    cd third_party/protobuf && \
-    ./autogen.sh && \
-    ./configure --with-pic CXXFLAGS="$(pkg-config --cflags protobuf)" LIBS="$(pkg-config --libs protobuf)" LDFLAGS="-Wl,--no-as-needed" && \
-    make -j4 && make install && ldconfig && \
-    cd ../.. && \
-    CPPFLAGS="-I /usr/local/ssl/include" LDFLAGS="-L /usr/local/ssl/lib/ -Wl,--no-as-needed" make -j4 CONFIG=opt EMBED_OPENSSL=false V=1 HAS_SYSTEM_OPENSSL_NPN=0 && \
-    CPPFLAGS="-I /usr/local/ssl/include" LDFLAGS="-L /usr/local/ssl/lib/ -Wl,--no-as-needed" make CONFIG=opt EMBED_OPENSSL=false V=1 HAS_SYSTEM_OPENSSL_NPN=0 install && \
-    rm -rf /tmp/grpc
+# -DCMAKE_CXX_STANDARD=11 is necessary since it is the minimum version supported
+# by gRPC dependencies. Xenial sets default version to c++98.
+#
+# See https://github.com/grpc/grpc/blob/v1.36.4/test/distrib/cpp/run_distrib_test_cmake_pkgconfig.sh
+RUN git clone --depth=1 --recursive -b v1.36.4 https://github.com/grpc/grpc.git /tmp/grpc && \
+    cd /tmp/grpc/ && \
+    # Install protobuf
+    mkdir -p third_party/protobuf/cmake/build && \
+    (cd third_party/protobuf/cmake/build && \
+    cmake -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_CXX_STANDARD=11 -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE -DCMAKE_BUILD_TYPE=Release .. && \
+    make -j4 install) && \
+    # Install finish - protobuf
+    # Install gRPC
+    mkdir -p cmake/build && \
+    (cd cmake/build && \
+    cmake \
+        -DOPENSSL_ROOT_DIR=/usr/local/ssl            \
+        -DOPENSSL_INCLUDE_DIR=/usr/local/ssl/include \
+        -DOPENSSL_CRYPTO_LIB=/usr/local/ssl/lib      \
+        -DCMAKE_BUILD_TYPE=Release                   \
+        -DCMAKE_CXX_STANDARD=11                      \
+        -DCMAKE_INSTALL_PREFIX=/usr/local/grpc       \
+        -DgRPC_INSTALL=ON                            \
+        -DgRPC_BUILD_TESTS=OFF                       \
+        -DgRPC_ABSL_PROVIDER=module                  \
+        -DgRPC_CARES_PROVIDER=module                 \
+        -DgRPC_RE2_PROVIDER=module                   \
+        -DgRPC_ZLIB_PROVIDER=module                  \
+        -DgRPC_PROTOBUF_PROVIDER=package             \
+        -DgRPC_SSL_PROVIDER=package                  \
+        ../.. && \
+    make -j4 install) && \
+    cd ~ && rm -rf /tmp/grpc
+
+ENV PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:/usr/local/ssl/lib/pkgconfig:/usr/local/grpc/lib/pkgconfig:/usr/local/lib64/pkgconfig"
+ENV PATH="${PATH}:/usr/local/grpc/bin"

Dockerfile.alpine

@@ -44,7 +44,6 @@ RUN apk --no-cache add \
 # openssl
 # This openssl (compiled with -fPIC) is used to statically link into the agent
 # shared library.
-ENV PKG_CONFIG_PATH=/usr/local/ssl/lib/pkgconfig
 ENV JAVA_PATH=/usr/lib/jvm/java-11-openjdk/
 RUN mkdir /tmp/openssl && cd /tmp/openssl && \
     curl -sL https://github.com/openssl/openssl/archive/OpenSSL_1_1_1t.tar.gz | \
@@ -80,21 +79,43 @@ RUN mkdir /tmp/glog && cd /tmp/glog && \
     make -j && make install && \
     cd ~ && rm -rf /tmp/glog
 
-# gRPC & protobuf
+# gRPC & protobuf - build using CMake
 # Use the protobuf version from gRPC for everything to avoid conflicting
 # versions to be linked in. Disable OpenSSL embedding: when it's on, the build
 # process of gRPC puts the OpenSSL static object files into the gRPC archive
 # which causes link errors later when the agent is linked with the static
 # OpenSSL library itself.
 # Limit the number of threads used by make, as unlimited threads causes
 # memory exhausted error on the Kokoro VM.
-RUN git clone --depth=1 --recursive -b v1.28.1 https://github.com/grpc/grpc.git /tmp/grpc && \
-    cd /tmp/grpc && \
-    cd third_party/protobuf && \
-    ./autogen.sh && \
-    ./configure --with-pic CXXFLAGS="$(pkg-config --cflags protobuf)" LIBS="$(pkg-config --libs protobuf)" LDFLAGS="-Wl,--no-as-needed" && \
-    make -j4 && make install && ldconfig / && \
-    cd ../.. && \
-    CPPFLAGS="-I /usr/local/ssl/include" LDFLAGS="-L /usr/local/ssl/lib/ -Wl,--no-as-needed" make -j4 CONFIG=opt EMBED_OPENSSL=false V=1 HAS_SYSTEM_OPENSSL_NPN=0 && \
-    CPPFLAGS="-I /usr/local/ssl/include" LDFLAGS="-L /usr/local/ssl/lib/ -Wl,--no-as-needed" make CONFIG=opt EMBED_OPENSSL=false V=1 HAS_SYSTEM_OPENSSL_NPN=0 install && \
-    rm -rf /tmp/grpc
+#
+# See https://github.com/grpc/grpc/blob/v1.36.4/test/distrib/cpp/run_distrib_test_cmake_pkgconfig.sh
+RUN git clone --depth=1 --recursive -b v1.36.4 https://github.com/grpc/grpc.git /tmp/grpc && \
+    cd /tmp/grpc/ && \
+    # Install protobuf
+    mkdir -p third_party/protobuf/cmake/build && \
+    (cd third_party/protobuf/cmake/build && \
+    cmake -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE -DCMAKE_BUILD_TYPE=Release .. && \
+    make -j4 install) && \
+    # Install gRPC
+    mkdir -p cmake/build && \
+    cd cmake/build && \
+    cmake \
+        -DOPENSSL_ROOT_DIR=/usr/local/ssl            \
+        -DOPENSSL_INCLUDE_DIR=/usr/local/ssl/include \
+        -DOPENSSL_CRYPTO_LIB=/usr/local/ssl/lib      \
+        -DCMAKE_BUILD_TYPE=Release                   \
+        -DCMAKE_INSTALL_PREFIX=/usr/local/grpc       \
+        -DgRPC_INSTALL=ON                            \
+        -DgRPC_BUILD_TESTS=OFF                       \
+        -DgRPC_ABSL_PROVIDER=module                  \
+        -DgRPC_CARES_PROVIDER=module                 \
+        -DgRPC_RE2_PROVIDER=module                   \
+        -DgRPC_ZLIB_PROVIDER=module                  \
+        -DgRPC_PROTOBUF_PROVIDER=package             \
+        -DgRPC_SSL_PROVIDER=package                  \
+        ../.. && \
+    make -j4 install && \
+    cd ~ && rm -rf /tmp/grpc
+
+ENV PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:/usr/local/ssl/lib/pkgconfig:/usr/local/grpc/lib/pkgconfig:/usr/local/lib64/pkgconfig"
+ENV PATH="${PATH}:/usr/local/grpc/bin"

Makefile

@@ -30,7 +30,8 @@ CFLAGS = \
 	-Wno-array-bounds \
 	-g0 \
 	-DSTANDALONE_BUILD \
-	-D_GNU_SOURCE
+	-D_GNU_SOURCE \
+	$(shell pkg-config --cflags protobuf grpc)
 
 ifeq ($(machine_type),$(filter $(machine_type),aarch64 arm64))
   # Building on an ARM64 machine.
@@ -50,7 +51,7 @@ endif
 SRC_ROOT_PATH=.
 
 PROTOC ?= /usr/local/bin/protoc
-PROTOC_GEN_GRPC ?= /usr/local/bin/grpc_cpp_plugin
+GRPC_CPP_PLUGIN_PATH ?= /usr/local/grpc/bin/grpc_cpp_plugin
 
 PROFILE_PROTO_PATH ?= third_party/perftools/profiles/proto
 JAVA_AGENT_PATH ?= src
@@ -82,6 +83,8 @@ PROFILER_API_SOURCES = \
 	$(GENFILES_PATH)/google/api/client.pb.cc \
 	$(GENFILES_PATH)/google/api/http.pb.cc \
 	$(GENFILES_PATH)/google/api/launch_stage.pb.cc \
+	$(GENFILES_PATH)/google/api/resource.pb.cc \
+	$(GENFILES_PATH)/google/api/field_behavior.pb.cc \
 	$(GENFILES_PATH)/google/devtools/cloudprofiler/v2/profiler.grpc.pb.cc \
 	$(GENFILES_PATH)/google/devtools/cloudprofiler/v2/profiler.pb.cc \
 	$(GENFILES_PATH)/google/protobuf/duration.pb.cc \
@@ -160,7 +163,7 @@ HEADERS = \
 
 VERSION_SCRIPT = $(JAVA_AGENT_PATH)/cloud_profiler_java_agent.lds
 OPT_FLAGS = -O3
-LDFLAGS = -static-libstdc++ -shared
+LDFLAGS = -L/usr/local/lib -static-libstdc++ -shared $(shell pkg-config --libs --static protobuf grpc++)
 LDS_FLAGS = -Wl,-z,defs -Wl,--version-script=$(VERSION_SCRIPT)
 
 LIB_ROOT_PATH ?= /usr/local
@@ -176,17 +179,6 @@ LIBS1= \
   $(LIB_ROOT_PATH)/lib/libglog.a \
 	$(LIB_ROOT_PATH)/lib/libgflags.a \
 
-LIBS2= \
-	$(LIB_ROOT_PATH)/lib/libprotobuf.a \
-	$(LIB_ROOT_PATH)/ssl/lib/libssl.a \
-	$(LIB_ROOT_PATH)/ssl/lib/libcrypto.a \
-	-lz \
-
-GRPC_LIBS= \
-	$(LIB_ROOT_PATH)/lib/libgrpc++.a \
-  $(LIB_ROOT_PATH)/lib/libgrpc.a \
-  $(LIB_ROOT_PATH)/lib/libgpr.a \
-
 # Detect if running on Alpine and modify various flags
 ifeq ("$(wildcard /etc/alpine-release)","/etc/alpine-release")
   # musl only supports global dynamic tls model, and is documented as
@@ -212,7 +204,7 @@ clean:
 
 $(TARGET_AGENT): $(SOURCES) $(HEADERS)
 	mkdir -p $(dir $@)
-	$(CC) $(INCLUDES) $(CFLAGS) $(OPT_FLAGS) $(LDFLAGS) $(SOURCES) $(LIBS1) $(GRPC_LIBS) $(LIBS2) -o $@ $(LDS_FLAGS)
+	$(CC) $(INCLUDES) $(CFLAGS) $(OPT_FLAGS) $(SOURCES) $(LIBS1) $(LDFLAGS) -o $@ $(LDS_FLAGS)
 
 $(TARGET_NOTICES): $(JAVA_AGENT_PATH)/NOTICES
 	mkdir -p $(dir $@)
@@ -225,7 +217,7 @@ $(GENFILES_PATH)/%profiler.pb.h $(GENFILES_PATH)/%profiler.pb.cc : third_party/g
 $(GENFILES_PATH)/%profiler.grpc.pb.h $(GENFILES_PATH)/%profiler.grpc.pb.cc : third_party/googleapis/%profiler.proto
 	mkdir -p $(dir $@)
 	$(PROTOC) -Ithird_party/googleapis -I$(PROTOBUF_INCLUDE_PATH) \
-			--plugin=protoc-gen-grpc=$(PROTOC_GEN_GRPC) --grpc_out=services_namespace=grpc:$(GENFILES_PATH) $<
+			--plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN_PATH) --grpc_out=services_namespace=grpc:$(GENFILES_PATH) $<
 
 $(GENFILES_PATH)/%annotations.pb.h $(GENFILES_PATH)/%annotations.pb.cc : third_party/googleapis/%annotations.proto
 	mkdir -p $(dir $@)
@@ -235,6 +227,14 @@ $(GENFILES_PATH)/%launch_stage.pb.h $(GENFILES_PATH)/%launch_stage.pb.cc : third
 	mkdir -p $(dir $@)
 	$(PROTOC) -Ithird_party/googleapis -I$(PROTOBUF_INCLUDE_PATH) --cpp_out=$(GENFILES_PATH) $<
 
+$(GENFILES_PATH)/%resource.pb.h $(GENFILES_PATH)/%resource.pb.cc : third_party/googleapis/%resource.proto
+	mkdir -p $(dir $@)
+	$(PROTOC) -Ithird_party/googleapis -I$(PROTOBUF_INCLUDE_PATH) --cpp_out=$(GENFILES_PATH) $<
+
+$(GENFILES_PATH)/%field_behavior.pb.h $(GENFILES_PATH)/%field_behavior.pb.cc : third_party/googleapis/%field_behavior.proto
+	mkdir -p $(dir $@)
+	$(PROTOC) -Ithird_party/googleapis -I$(PROTOBUF_INCLUDE_PATH) --cpp_out=$(GENFILES_PATH) $<
+
 $(GENFILES_PATH)/%client.pb.h $(GENFILES_PATH)/%client.pb.cc : third_party/googleapis/%client.proto
 	mkdir -p $(dir $@)
 	$(PROTOC) -Ithird_party/googleapis -I$(PROTOBUF_INCLUDE_PATH) --cpp_out=$(GENFILES_PATH) $<

build.sh

@@ -38,7 +38,7 @@ while getopts ":d:m:v:" opt; do
   m)
       TARGET=$OPTARG
       if [[ "${TARGET}" == "arm64" ]]; then
-        if [[ "${MACHINE_TYPE}" != "aarch64" ]] ||  [[ "${MACHINE_TYPE}" != "arm64" ]]; then
+        if [[ "${MACHINE_TYPE}" != "aarch64" ]] &&  [[ "${MACHINE_TYPE}" != "arm64" ]]; then
           echo "-m arm64 is supported only when running on an ARM64 system."
           exit;
         fi

src/cloud_env.cc

@@ -17,6 +17,7 @@
 #include <cstdlib>
 #include <sstream>
 #include <string>
+#include <vector>
 
 #include "src/clock.h"
 #include "src/http.h"

src/cloud_env.h

@@ -17,6 +17,8 @@
 #ifndef CLOUD_PROFILER_AGENT_JAVA_CLOUD_ENV_H_
 #define CLOUD_PROFILER_AGENT_JAVA_CLOUD_ENV_H_
 
+#include <string>
+
 #include "src/globals.h"
 
 namespace cloud {
@@ -29,6 +31,11 @@ class HTTPRequest;
 class CloudEnv {
  public:
   CloudEnv();
+
+  // This type is neither copyable nor movable.
+  CloudEnv(const CloudEnv&) = delete;
+  CloudEnv& operator=(const CloudEnv&) = delete;
+
   virtual ~CloudEnv() {}
 
   // Returns the current cloud project ID.
@@ -68,7 +75,6 @@ class CloudEnv {
   std::string zone_name_;
   std::string service_;
   std::string service_version_;
-  DISALLOW_COPY_AND_ASSIGN(CloudEnv);
 };
 
 // Returns the default instance of a cloud env object. The returned

src/entry.cc

@@ -15,6 +15,7 @@
 #include <limits.h>
 
 #include <string>
+#include <vector>
 
 #include "src/globals.h"
 #include "src/string.h"
@@ -122,11 +123,10 @@ void JNICALL OnVMInit(jvmtiEnv *jvmti, JNIEnv *jni_env, jthread thread) {
   }
 
   if (FLAGS_cprof_enable_heap_sampling) {
-    // TODO: Allow using the JVM's stack tracer with a flag once
-    // we can get the current context in a cloud build.
-    google::javaprofiler::HeapMonitor::Enable(
-        jvmti, jni_env, FLAGS_cprof_heap_sampling_interval,
-        false /* use_jvm_trace */);
+    if (!google::javaprofiler::HeapMonitor::Enable(
+            jvmti, jni_env, FLAGS_cprof_heap_sampling_interval)) {
+      LOG(WARNING) << "Failed to start HeapMonitor.";
+    }
   }
 
   worker->Start(jni_env);

src/globals.h

@@ -23,6 +23,7 @@
 #include <jvmti.h>
 #include <stdint.h>
 
+#include <string>
 #include <unordered_map>
 
 #include "third_party/javaprofiler/jvmti_error.h"
@@ -103,6 +104,10 @@ class JvmtiScopedPtr {
 
   JvmtiScopedPtr(jvmtiEnv *jvmti, T *ref) : jvmti_(jvmti), ref_(ref) {}
 
+  // This type is neither copyable nor movable.
+  JvmtiScopedPtr(const JvmtiScopedPtr &) = delete;
+  JvmtiScopedPtr &operator=(const JvmtiScopedPtr &) = delete;
+
   ~JvmtiScopedPtr() {
     if (NULL != ref_) {
       JVMTI_ERROR(jvmti_->Deallocate((unsigned char *)ref_));
@@ -121,8 +126,6 @@ class JvmtiScopedPtr {
  private:
   jvmtiEnv *jvmti_;
   T *ref_;
-
-  DISALLOW_IMPLICIT_CONSTRUCTORS(JvmtiScopedPtr);
 };
 
 // Things that should probably be user-configurable

src/http.cc

@@ -14,6 +14,8 @@
 
 #include "src/http.h"
 
+#include <string>
+
 #include "curl/curl.h"
 
 namespace cloud {

src/http.h

@@ -17,6 +17,8 @@
 #ifndef CLOUD_PROFILER_AGENT_JAVA_HTTP_H_
 #define CLOUD_PROFILER_AGENT_JAVA_HTTP_H_
 
+#include <string>
+
 #include "src/globals.h"
 
 typedef void CURL;
@@ -31,6 +33,11 @@ const int kHTTPStatusOK = 200;
 class HTTPRequest {
  public:
   HTTPRequest();
+
+  // This type is neither copyable nor movable.
+  HTTPRequest(const HTTPRequest&) = delete;
+  HTTPRequest& operator=(const HTTPRequest&) = delete;
+
   virtual ~HTTPRequest();
 
   virtual void AddAuthBearerHeader(const std::string& token);
@@ -51,7 +58,6 @@ class HTTPRequest {
 
   CURL* curl_;
   struct curl_slist* headers_;
-  DISALLOW_COPY_AND_ASSIGN(HTTPRequest);
 };
 
 }  // namespace profiler