Android进阶KOOM线上APM监控全面剖析

Object object = new Object();
ReferenceQueue<Object> referenceQueue = new ReferenceQueue<Object>();
WeakReference<Object> weak = new WeakReference<Object>(object,referenceQueue);
Log.e("Test","弱引用 "+weak.get());
object = null;
System.gc();
Thread.sleep(1000);
Log.e("Test","弱引用 "+weak.get());
Log.e("Test","弱引用队列 "+referenceQueue.poll());
System.gc();
Thread.sleep(2000);
Log.e("Test","弱引用 "+weak.get());
Log.e("Test","弱引用队列 "+referenceQueue.poll());

object Constant {
    private var any: Any? = null
    fun hold(any: Any?) {
        this.any = any
    }
}

ReferenceQueue<Object> referenceQueue = new ReferenceQueue<Object>();
WeakReference<Object> weak = new WeakReference<Object>(mObject,referenceQueue);
Log.e("Test","弱引用 "+weak.get());
Constant.INSTANCE.hold(mObject);
mObject = null;
System.gc();
Thread.sleep(2000);
Log.e("Test","弱引用 "+weak.get());
Log.e("Test","弱引用队列 "+referenceQueue.poll());

def VERSION_NAME = '2.2.0'
implementation "com.kuaishou.koom:koom-native-leak-static:${VERSION_NAME}"
implementation "com.kuaishou.koom:koom-java-leak-static:${VERSION_NAME}"
implementation "com.kuaishou.koom:koom-thread-leak-static:${VERSION_NAME}"
implementation "com.kuaishou.koom:xhook-static:${VERSION_NAME}"

val commonConfig = CommonConfig.Builder().build()
val oomMonitorConfig = OOMMonitorConfig.Builder().build()
OOMMonitor.init(commonConfig, oomMonitorConfig)
OOMMonitor.startLoop(clearQueue = true,postAtFront = true, delayMillis = 5000)

override fun startLoop(clearQueue: Boolean, postAtFront: Boolean, delayMillis: Long) {
  throwIfNotInitialized { return }
  /**要在主进程中开启*/
  if (!isMainProcess()) {
    return
  }
  MonitorLog.i(TAG, "startLoop()")
  if (mIsLoopStarted) {
    return
  }
  mIsLoopStarted = true
  super.startLoop(clearQueue, postAtFront, delayMillis)
  getLoopHandler().postDelayed({ async { processOldHprofFile() } }, delayMillis)
}

open fun startLoop(
    clearQueue: Boolean = true,
    postAtFront: Boolean = false,
    delayMillis: Long = 0L
) {
  if (clearQueue) getLoopHandler().removeCallbacks(mLoopRunnable)
  if (postAtFront) {
    getLoopHandler().postAtFrontOfQueue(mLoopRunnable)
  } else {
    getLoopHandler().postDelayed(mLoopRunnable, delayMillis)
  }
  mIsLoopStopped = false
}

private val mLoopRunnable = object : Runnable {
  override fun run() {
    /**进行内存泄漏、OOM检测*/
    if (call() == LoopState.Terminate) {
      return
    }
    if (mIsLoopStopped) {
      return
    }
    getLoopHandler().removeCallbacks(this)
    getLoopHandler().postDelayed(this, getLoopInterval())
  }
}

override fun call(): LoopState {
  if (!sdkVersionMatch()) {
    return LoopState.Terminate
  }
  if (mHasDumped) {
    return LoopState.Terminate
  }
  return trackOOM()
}

private fun trackOOM(): LoopState {
  SystemInfo.refresh()
  mTrackReasons.clear()
  for (oomTracker in mOOMTrackers) {
    if (oomTracker.track()) {
      mTrackReasons.add(oomTracker.reason())
    }
  }
  /**如果追踪到了OOM，那么就会异步分析*/
  if (mTrackReasons.isNotEmpty() && monitorConfig.enableHprofDumpAnalysis) {
    if (isExceedAnalysisPeriod() || isExceedAnalysisTimes()) {
      MonitorLog.e(TAG, "Triggered, but exceed analysis times or period!")
    } else {
      async {
        MonitorLog.i(TAG, "mTrackReasons:${mTrackReasons}")
        dumpAndAnalysis()
      }
    }
    return LoopState.Terminate
  }
  return LoopState.Continue
}

private val mOOMTrackers = mutableListOf(
  HeapOOMTracker(), ThreadOOMTracker(), FdOOMTracker(),
  PhysicalMemoryOOMTracker(), FastHugeMemoryOOMTracker()
)

for (oomTracker in mOOMTrackers) {
  if (oomTracker.track()) {
    mTrackReasons.add(oomTracker.reason())
  }
}

abstract class OOMTracker : Monitor<OOMMonitorConfig>() {
  /**
   * @return true 表示追踪到oom、 false 表示没有追踪到oom
   */
  abstract fun track(): Boolean
  /**
   * 重置track状态
   */
  abstract fun reset()
  /**
   * @return 追踪到的oom的标识
   */
  abstract fun reason(): String
}

override fun track(): Boolean {
  /**第一步：获取进程内存占用率*/
  val heapRatio = SystemInfo.javaHeap.rate
  /**利用内存占用率 与 配置文件中的阈值做比较*/
  if (heapRatio > monitorConfig.heapThreshold
      && heapRatio >= mLastHeapRatio - HEAP_RATIO_THRESHOLD_GAP) {
    mOverThresholdCount++
    MonitorLog.i(TAG,
        "[meet condition] "
            + "overThresholdCount: $mOverThresholdCount"
            + ", heapRatio: $heapRatio"
            + ", usedMem: ${SizeUnit.BYTE.toMB(SystemInfo.javaHeap.used)}mb"
            + ", max: ${SizeUnit.BYTE.toMB(SystemInfo.javaHeap.max)}mb")
  } else {
    reset()
  }
  mLastHeapRatio = heapRatio
  return mOverThresholdCount >= monitorConfig.maxOverThresholdCount
}

/**当前进程最大内存，-xmx*/
javaHeap.max = Runtime.getRuntime().maxMemory()
/**当前进程初始化申请的内存，-xms*/
javaHeap.total = Runtime.getRuntime().totalMemory()
/**当前进程剩余可用内存*/
javaHeap.free = Runtime.getRuntime().freeMemory()
javaHeap.used = javaHeap.total - javaHeap.free
javaHeap.rate = 1.0f * javaHeap.used / javaHeap.max

private var mLastHeapRatio = 0.0f
private var mOverThresholdCount = 0
private const val HEAP_RATIO_THRESHOLD_GAP = 0.05f
if (heapRatio > monitorConfig.heapThreshold
    && heapRatio >= mLastHeapRatio - HEAP_RATIO_THRESHOLD_GAP)

override fun track(): Boolean {
  val threadCount = getThreadCount()
  if (threadCount > monitorConfig.threadThreshold
      && threadCount >= mLastThreadCount - THREAD_COUNT_THRESHOLD_GAP) {
    mOverThresholdCount++
    MonitorLog.i(TAG,
        "[meet condition] "
            + "overThresholdCount:$mOverThresholdCount"
            + ", threadCount: $threadCount")
    dumpThreadIfNeed()
  } else {
    reset()
  }
  mLastThreadCount = threadCount
  return mOverThresholdCount >= monitorConfig.maxOverThresholdCount
}

File("/proc/self/status").forEachLineQuietly { line ->
  if (procStatus.vssInKb != 0 && procStatus.rssInKb != 0
      && procStatus.thread != 0) return@forEachLineQuietly
  when {
    line.startsWith("VmSize") -> {
      procStatus.vssInKb = VSS_REGEX.matchValue(line)
    }
    line.startsWith("VmRSS") -> {
      procStatus.rssInKb = RSS_REGEX.matchValue(line)
    }
    /**获取线程数*/
    line.startsWith("Threads") -> {
      procStatus.thread = THREADS_REGEX.matchValue(line)
    }
  }
}

override fun track(): Boolean {
  val javaHeap = SystemInfo.javaHeap
  // 高危阈值直接触发dump分析
  if (javaHeap.rate > monitorConfig.forceDumpJavaHeapMaxThreshold) {
    mDumpReason = REASON_HIGH_WATERMARK
    MonitorLog.i(TAG, "[meet condition] fast huge memory allocated detected, " +
        "high memory watermark, force dump analysis!")
    return true
  }
  // 高差值直接dump
  val lastJavaHeap = SystemInfo.lastJavaHeap
  if (lastJavaHeap.max != 0L && javaHeap.used - lastJavaHeap.used
      > SizeUnit.KB.toByte(monitorConfig.forceDumpJavaHeapDeltaThreshold)) {
    mDumpReason = REASON_HUGE_DELTA
    MonitorLog.i(TAG, "[meet condition] fast huge memory allocated detected, " +
        "over the delta threshold!")
    return true
  }
  return false
}

private fun dumpAndAnalysis() {
  MonitorLog.i(TAG, "dumpAndAnalysis");
  runCatching {
    if (!OOMFileManager.isSpaceEnough()) {
      MonitorLog.e(TAG, "available space not enough", true)
      return@runCatching
    }
    if (mHasDumped) {
      return
    }
    mHasDumped = true
    val date = Date()
    val jsonFile = OOMFileManager.createJsonAnalysisFile(date)
    val hprofFile = OOMFileManager.createHprofAnalysisFile(date).apply {
      createNewFile()
      setWritable(true)
      setReadable(true)
    }
    MonitorLog.i(TAG, "hprof analysis dir:$hprofAnalysisDir")
    /**核心代码 在这里完成内存镜像的dump*/
    ForkJvmHeapDumper.getInstance().run {
      dump(hprofFile.absolutePath)
    }
    MonitorLog.i(TAG, "end hprof dump", true)
    Thread.sleep(1000) // make sure file synced to disk.
    MonitorLog.i(TAG, "start hprof analysis")
    startAnalysisService(hprofFile, jsonFile, mTrackReasons.joinToString())
  }.onFailure {
    it.printStackTrace()
    MonitorLog.i(TAG, "onJvmThreshold Exception " + it.message, true)
  }
}

@Override
public synchronized boolean dump(String path) {
  MonitorLog.i(TAG, "dump " + path);
  if (!sdkVersionMatch()) {
    throw new UnsupportedOperationException("dump failed caused by sdk version not supported!");
  }
  /**第一步，调用init方法，加载so文件*/
  init();
  if (!mLoadSuccess) {
    MonitorLog.e(TAG, "dump failed caused by so not loaded!");
    return false;
  }
  boolean dumpRes = false;
  try {
    MonitorLog.i(TAG, "before suspend and fork.");
    /**第二步，fork出一个子进程*/
    int pid = suspendAndFork();
    /**第三步，在子进程中完成dump*/
    if (pid == 0) {
      // Child process
      Debug.dumpHprofData(path);
      exitProcess();
    } else if (pid > 0) {
      // Parent process
      dumpRes = resumeAndWait(pid);
      MonitorLog.i(TAG, "dump " + dumpRes + ", notify from pid " + pid);
    }
  } catch (IOException e) {
    MonitorLog.e(TAG, "dump failed caused by " + e);
    e.printStackTrace();
  }
  return dumpRes;
}

private void init () {
  if (mLoadSuccess) {
    return;
  }
  if (loadSoQuietly("koom-fast-dump")) {
    mLoadSuccess = true;
    nativeInit();
  }
}

/**
 * Suspend the whole ART, and then fork a process for dumping hprof.
 *
 * @return return value of fork
 */
private native int suspendAndFork();

async {
  MonitorLog.i(TAG, "mTrackReasons:${mTrackReasons}")
  dumpAndAnalysis()
}

public static void dumpHprofData(String fileName) throws IOException {
    VMDebug.dumpHprofData(fileName);
}

static void VMDebug_dumpHprofData(JNIEnv* env, jclass, jstring javaFilename, jint javaFd) {
  // Only one of these may be null.
  if (javaFilename == nullptr && javaFd < 0) {
        ScopedObjectAccess soa(env);
        ThrowNullPointerException("fileName == null && fd == null");
        return;
      }
  std::string filename;
  if (javaFilename != nullptr) {
        ScopedUtfChars chars(env, javaFilename);
        if (env->ExceptionCheck()) {
              return;
            }
        filename = chars.c_str();
      } else {
        filename = "[fd]";
      }
  int fd = javaFd;
  /**调用Hprof的DumpHeap函数*/
  hprof::DumpHeap(filename.c_str(), fd, false);
}

void DumpHeap(const char* filename, int fd, bool direct_to_ddms) {
      CHECK(filename != nullptr);
      Thread* self = Thread::Current();
      // Need to take a heap dump while GC isn't running. See the comment in Heap::VisitObjects().
      // Also we need the critical section to avoid visiting the same object twice. See b/34967844
      gc::ScopedGCCriticalSection gcs(self,
            1607                                  gc::kGcCauseHprof,
            1608                                  gc::kCollectorTypeHprof);
      ScopedSuspendAll ssa(__FUNCTION__, true /* long suspend */);
      Hprof hprof(filename, fd, direct_to_ddms);
      hprof.Dump();
    }

MonitorLog.i(TAG, "before suspend and fork.");
int pid = suspendAndFork();
if (pid == 0) {
  // Child process
  Log.e("TAG","父进程fork成功，子进程开始执行")
  Debug.dumpHprofData(path);
  exitProcess();
  Log.e("TAG","子进程执行完成，退出")
} else if (pid > 0) {
  Log.e("TAG","父进程fork成功，继续执行")
  // Parent process
  dumpRes = resumeAndWait(pid);
  MonitorLog.i(TAG, "dump " + dumpRes + ", notify from pid " + pid);
}

val thread = Thread{
   Log.e("TAG","do something")
}
thread.start()
MonitorLog.i(TAG, "before suspend and fork.");
int pid = suspendAndFork();
if (pid == 0) {
  // Child process
  Log.e("TAG","父进程fork成功，子进程开始执行")
  Debug.dumpHprofData(path);
  exitProcess();
  Log.e("TAG","子进程执行完成，退出")
} else if (pid > 0) {
  Log.e("TAG","父进程fork成功，继续执行")
  // Parent process
  dumpRes = resumeAndWait(pid);
  MonitorLog.i(TAG, "dump " + dumpRes + ", notify from pid " + pid);
}