#!/usr/bin/python3
# -*- coding: utf-8 -*-
#
# offwaketime   Summarize blocked time by kernel off-CPU stack + waker stack
#               For Linux, uses BCC, eBPF.
#
# USAGE: offwaketime [-h] [-p PID | -u | -k] [-U | -K] [-f] [duration]
#
# Copyright 2016 Netflix, Inc.
# Licensed under the Apache License, Version 2.0 (the "License")
#
# 20-Jan-2016   Brendan Gregg   Created this.
# 04-Apr-2023   Rocky Xing      Updated default stack storage size.
# 12-Nov-2025   Jack Zhao       - Generate flamegraph
#                               - Print top 5 longest blocking
#                               - Support raw tracepoint

from __future__ import print_function
from bcc import BPF
from time import sleep
import argparse
import signal
import errno
from sys import stderr
import subprocess
import os

# arg validation
def positive_int(val):
    dest = []
    # Filter up to 5 pids, arbitrary
    args_list = val.split(",", 5)
    pids_to_add = min(len(args_list), 5)
    for i in range(pids_to_add):
        dest.append(_positive_int(args_list[i]))

    return dest

def _positive_int(val):
    try:
        ival = int(val)
    except ValueError:
        raise argparse.ArgumentTypeError("must be an integer")

    if ival < 0:
        raise argparse.ArgumentTypeError("must be positive")
    return ival

def positive_nonzero_int(val):
    ival = _positive_int(val)
    if ival == 0:
        raise argparse.ArgumentTypeError("must be nonzero")
    return ival

def build_filter(filter_name, values):
    filter_string = "((%s == %d)" % (filter_name, values[0])

    for val in values[1:]:
        filter_string += " || (%s == %d )" % (filter_name , val)

    filter_string += ")"

    return filter_string

def stack_id_err(stack_id):
    # -EFAULT in get_stackid normally means the stack-trace is not available,
    # Such as getting kernel stack trace in userspace code
    return (stack_id < 0) and (stack_id != -errno.EFAULT)

def cleanup_folded_file(filename, force_cleanup=True):
    if os.path.exists(filename):
        try:
            os.remove(filename)
            safe_print(f"Cleaned up temporary file: {filename}")
            return True
        except OSError as e:
            safe_print(f"Warning: Failed to clean up {filename}: {e}")
            return False
    return False

def safe_print(*args, **kwargs):
    """Safe print function that handles closed stdout/stderr gracefully."""
    try:
        print(*args, **kwargs)
    except (OSError, IOError, BrokenPipeError):
        # stdout/stderr may be closed, try to reopen or ignore
        try:
            import sys
            # Try to write to stderr as fallback
            if 'file' not in kwargs or kwargs['file'] == sys.stdout:
                sys.stderr.write(' '.join(str(arg) for arg in args) + '\n')
        except:
            pass  # If all else fails, silently ignore

def aggregate_by_target(all_counts):
    """
    Aggregate blocking time by target name only.
    
    Returns:
        dict: Aggregated data keyed by target name
    """
    aggregated = {}
    for k, v in all_counts:
        # Create aggregation key based on target name only
        agg_key = k.target

        # Extract waker base name (remove CPU number suffix like "swapper/10" -> "swapper")
        waker_name_str = k.waker.decode('utf-8', 'replace')
        if '/' in waker_name_str and waker_name_str.split('/')[0] == 'swapper':
            waker_base_name = 'swapper'
        else:
            waker_base_name = waker_name_str

        if agg_key in aggregated:
            aggregated[agg_key]['total_time'] += v.value
            aggregated[agg_key]['count'] += 1
            # Keep the sample with largest individual time for better stack display
            if v.value > aggregated[agg_key]['max_single_time']:
                aggregated[agg_key]['max_single_time'] = v.value
                aggregated[agg_key]['sample_tgid'] = k.t_tgid
                aggregated[agg_key]['t_k_stack_id'] = k.t_k_stack_id
                aggregated[agg_key]['t_u_stack_id'] = k.t_u_stack_id
            # Aggregate waker info by base name
            if waker_base_name in aggregated[agg_key]['wakers']:
                aggregated[agg_key]['wakers'][waker_base_name]['time'] += v.value
                aggregated[agg_key]['wakers'][waker_base_name]['count'] += 1
                # Keep the waker stack with largest time
                if v.value > aggregated[agg_key]['wakers'][waker_base_name]['max_time']:
                    aggregated[agg_key]['wakers'][waker_base_name]['max_time'] = v.value
                    aggregated[agg_key]['wakers'][waker_base_name]['w_k_stack_id'] = k.w_k_stack_id
                    aggregated[agg_key]['wakers'][waker_base_name]['w_u_stack_id'] = k.w_u_stack_id
                    aggregated[agg_key]['wakers'][waker_base_name]['w_tgid'] = k.w_tgid
                    aggregated[agg_key]['wakers'][waker_base_name]['w_pid'] = k.w_pid
            else:
                aggregated[agg_key]['wakers'][waker_base_name] = {
                    'time': v.value,
                    'count': 1,
                    'max_time': v.value,
                    'w_k_stack_id': k.w_k_stack_id,
                    'w_u_stack_id': k.w_u_stack_id,
                    'w_tgid': k.w_tgid,
                    'w_pid': k.w_pid
                }
        else:
            aggregated[agg_key] = {
                'total_time': v.value,
                'count': 1,
                'max_single_time': v.value,
                'sample_tgid': k.t_tgid,
                't_k_stack_id': k.t_k_stack_id,
                't_u_stack_id': k.t_u_stack_id,
                'wakers': {
                    waker_base_name: {
                        'time': v.value,
                        'count': 1,
                        'max_time': v.value,
                        'w_k_stack_id': k.w_k_stack_id,
                        'w_u_stack_id': k.w_u_stack_id,
                        'w_tgid': k.w_tgid,
                        'w_pid': k.w_pid
                    }
                }
            }
    return aggregated

def print_top5(aggregated, stack_traces, b, args, need_delimiter):
    """
    Print top 5 blocking stacks from aggregated data.
    
    Args:
        aggregated: Aggregated data from aggregate_by_target()
        stack_traces: BPF stack_traces table
        b: BPF object for symbol resolution
        args: Command line arguments
        need_delimiter: Whether to insert delimiter between kernel/user stacks
    """
    # Sort by total time and get top 5
    top_5_aggregated = sorted(aggregated.items(), key=lambda x: x[1]['total_time'], reverse=True)[:5]

    for i, (target_name, agg_val) in enumerate(top_5_aggregated, 1):
        total_time = agg_val['total_time']
        count = agg_val['count']
        sample_tgid = agg_val['sample_tgid']
        t_k_stack_id = agg_val['t_k_stack_id']
        t_u_stack_id = agg_val['t_u_stack_id']
        wakers = agg_val['wakers']

        # Get top waker (by time)
        top_waker_name = max(wakers.keys(), key=lambda x: wakers[x]['time'])
        top_waker = wakers[top_waker_name]
        w_k_stack_id = top_waker['w_k_stack_id']
        w_u_stack_id = top_waker['w_u_stack_id']
        w_tgid = top_waker['w_tgid']
        w_pid = top_waker['w_pid']

        safe_print("\nTOP {}: {} us ({} entries, avg {} us)".format(
            i, total_time, count, total_time // count if count > 0 else 0))
        safe_print("-" * 60)

        # Get waker stack information
        waker_user_stack = [] if w_u_stack_id < 1 else \
            reversed(list(stack_traces.walk(w_u_stack_id))[1:])
        waker_kernel_stack = [] if w_k_stack_id < 1 else \
            reversed(list(stack_traces.walk(w_k_stack_id))[1:])

        # Print waker information first
        safe_print("    waker:           {} {}".format(top_waker_name, w_pid))

        # Print waker user stack
        if not args.kernel_stacks_only and not stack_id_err(w_u_stack_id):
            for addr in waker_user_stack:
                sym = b.sym(addr, w_tgid).decode('utf-8', 'replace')
                safe_print("    {}".format(sym))

        # Print waker kernel stack
        if not args.user_stacks_only and not stack_id_err(w_k_stack_id):
            if need_delimiter and w_u_stack_id > 0 and w_k_stack_id > 0:
                safe_print("    -")
            for addr in waker_kernel_stack:
                sym = b.ksym(addr).decode('utf-8', 'replace')
                safe_print("    {}".format(sym))

        # Print separator
        safe_print("    --               --")

        # Get target stack information
        target_user_stack = [] if t_u_stack_id < 1 else \
            stack_traces.walk(t_u_stack_id)
        target_kernel_stack = [] if t_k_stack_id < 1 else \
            stack_traces.walk(t_k_stack_id)

        # Print target kernel stack
        if not args.user_stacks_only and not stack_id_err(t_k_stack_id):
            for addr in target_kernel_stack:
                sym = b.ksym(addr).decode('utf-8', 'replace')
                safe_print("    {}".format(sym))

        # Print target user stack
        if not args.kernel_stacks_only and not stack_id_err(t_u_stack_id):
            if need_delimiter and t_u_stack_id > 0 and t_k_stack_id > 0:
                safe_print("    -")
            for addr in target_user_stack:
                sym = b.sym(addr, sample_tgid).decode('utf-8', 'replace')
                safe_print("    {}".format(sym))

        # Print target information
        safe_print("    target:          {}".format(target_name.decode('utf-8', 'replace')))
        safe_print("        {}\n".format(total_time))

def print_all_call_stacks(all_counts, stack_traces, b, args, need_delimiter, folded):
    """
    Print all call stacks.
    
    Args:
        all_counts: All count data
        stack_traces: BPF stack_traces table
        b: BPF object for symbol resolution
        args: Command line arguments
        need_delimiter: Whether to insert delimiter between kernel/user stacks
        folded: Whether to output in folded format
    
    Returns:
        tuple: (missing_stacks_count, has_enomem_flag)
    """
    missing_stacks_count = 0
    has_enomem_flag = False
    
    for k, v in sorted(all_counts, key=lambda counts: counts[1].value):
        # handle get_stackid errors
        if not args.user_stacks_only:
            missing_stacks_count += int(stack_id_err(k.w_k_stack_id))
            missing_stacks_count += int(stack_id_err(k.t_k_stack_id))
            has_enomem_flag = has_enomem_flag or (k.w_k_stack_id == -errno.ENOMEM) or \
                         (k.t_k_stack_id == -errno.ENOMEM)
        if not args.kernel_stacks_only:
            missing_stacks_count += int(stack_id_err(k.w_u_stack_id))
            missing_stacks_count += int(stack_id_err(k.t_u_stack_id))
            has_enomem_flag = has_enomem_flag or (k.w_u_stack_id == -errno.ENOMEM) or \
                         (k.t_u_stack_id == -errno.ENOMEM)
    
        waker_user_stack = [] if k.w_u_stack_id < 1 else \
            reversed(list(stack_traces.walk(k.w_u_stack_id))[1:])
        waker_kernel_stack = [] if k.w_k_stack_id < 1 else \
            reversed(list(stack_traces.walk(k.w_k_stack_id))[1:])
        target_user_stack = [] if k.t_u_stack_id < 1 else \
            stack_traces.walk(k.t_u_stack_id)
        target_kernel_stack = [] if k.t_k_stack_id < 1 else \
            stack_traces.walk(k.t_k_stack_id)
    
        if folded:
            # print folded stack output
            line = [k.target.decode('utf-8', 'replace')]
            if not args.kernel_stacks_only:
                if stack_id_err(k.t_u_stack_id):
                    line.append("[Missed User Stack] %d" % k.t_u_stack_id)
                else:
                    line.extend([b.sym(addr, k.t_tgid).decode('utf-8', 'replace')
                        for addr in reversed(list(target_user_stack)[1:])])
            if not args.user_stacks_only:
                line.extend(["-"] if (need_delimiter and k.t_k_stack_id > 0 and k.t_u_stack_id > 0) else [])
                if stack_id_err(k.t_k_stack_id):
                    line.append("[Missed Kernel Stack]")
                else:
                    line.extend([b.ksym(addr).decode('utf-8', 'replace')
                        for addr in reversed(list(target_kernel_stack)[1:])])
            line.append("--")
            if not args.user_stacks_only:
                if stack_id_err(k.w_k_stack_id):
                    line.append("[Missed Kernel Stack]")
                else:
                    line.extend([b.ksym(addr).decode('utf-8', 'replace')
                        for addr in reversed(list(waker_kernel_stack))])
            if not args.kernel_stacks_only:
                line.extend(["-"] if (need_delimiter and k.w_u_stack_id > 0 and k.w_k_stack_id > 0) else [])
                if stack_id_err(k.w_u_stack_id):
                    line.append("[Missed User Stack]")
                else:
                    line.extend([b.sym(addr, k.w_tgid).decode('utf-8', 'replace')
                        for addr in reversed(list(waker_user_stack))])
            line.append(k.waker.decode('utf-8', 'replace'))
            print("%s %d" % (";".join(line), v.value))
        else:
            # print wakeup name then stack in reverse order
            print("    %-16s %s %s" % ("waker:", k.waker.decode('utf-8', 'replace'), k.w_pid))
            if not args.kernel_stacks_only:
                if stack_id_err(k.w_u_stack_id):
                    print("    [Missed User Stack] %d" % k.w_u_stack_id)
                else:
                    for addr in waker_user_stack:
                        print("    %s" % b.sym(addr, k.w_tgid).decode('utf-8', 'replace'))
            if not args.user_stacks_only:
                if need_delimiter and k.w_u_stack_id > 0 and k.w_k_stack_id > 0:
                    print("    -")
                if stack_id_err(k.w_k_stack_id):
                    print("    [Missed Kernel Stack]")
                else:
                    for addr in waker_kernel_stack:
                        print("    %s" % b.ksym(addr).decode('utf-8', 'replace'))
    
            # print waker/wakee delimiter
            print("    %-16s %s" % ("--", "--"))
    
            if not args.user_stacks_only:
                if stack_id_err(k.t_k_stack_id):
                    print("    [Missed Kernel Stack]")
                else:
                    for addr in target_kernel_stack:
                        print("    %s" % b.ksym(addr).decode('utf-8', 'replace'))
            if not args.kernel_stacks_only:
                if need_delimiter and k.t_u_stack_id > 0 and k.t_k_stack_id > 0:
                    print("    -")
                if stack_id_err(k.t_u_stack_id):
                    print("    [Missed User Stack]")
                else:
                    for addr in target_user_stack:
                        print("    %s" % b.sym(addr, k.t_tgid).decode('utf-8', 'replace'))
            print("    %-16s %s %s" % ("target:", k.target.decode('utf-8', 'replace'), k.t_pid))
            print("        %d\n" % v.value)
    
    return (missing_stacks_count, has_enomem_flag)

# arguments
examples = """examples:
    ./offwaketime             # trace off-CPU + waker stack time until Ctrl-C
    ./offwaketime -d 5        # trace for 5 seconds only
    ./offwaketime -a -f -d 5  # 5 seconds, and output in folded format
    ./offwaketime -m 1000     # trace only events that last more than 1000 usec
    ./offwaketime -M 9000     # trace only events that last less than 9000 usec
    ./offwaketime -p 185      # only trace threads for PID 185
    ./offwaketime -t 188      # only trace thread 188
    ./offwaketime -u          # only trace user threads (no kernel)
    ./offwaketime -k          # only trace kernel threads (no user)
    ./offwaketime -U          # only show user space stacks (no kernel)
    ./offwaketime -K          # only show kernel space stacks (no user)
    ./offwaketime -g          # generate flame graph automatically (implies folded format) default output: /tmp/output_ext.dat
    ./offwaketime -g -o /path/to/output.svg  # generate flame graph with custom output path
    ./offwaketime             # output top 5 longest blocking stacks (default)
    ./offwaketime -a          # output all call stacks
    ./offwaketime -g -a       # generate flame graph + output all call stacks
"""
parser = argparse.ArgumentParser(
    description="Summarize blocked time by kernel stack trace + waker stack",
    formatter_class=argparse.RawDescriptionHelpFormatter,
    epilog=examples)
thread_group = parser.add_mutually_exclusive_group()
# Note: this script provides --pid and --tid flags but their arguments are
# referred to internally using kernel nomenclature: TGID and PID.
thread_group.add_argument("-p", "--pid", metavar="PIDS", dest="tgid",
     type=positive_int,
     help="trace these PIDS only. Can be a comma separated list of PIDS.")
thread_group.add_argument("-t", "--tid", metavar="TIDS", dest="pid",
    type=positive_int,
    help="trace these TIDS only. Can be a comma separated list of TIDS.")
thread_group.add_argument("-u", "--user-threads-only", action="store_true",
    help="user threads only (no kernel threads)")
thread_group.add_argument("-k", "--kernel-threads-only", action="store_true",
    help="kernel threads only (no user threads)")
stack_group = parser.add_mutually_exclusive_group()
stack_group.add_argument("-U", "--user-stacks-only", action="store_true",
    help="show stacks from user space only (no kernel space stacks)")
stack_group.add_argument("-K", "--kernel-stacks-only", action="store_true",
    help="show stacks from kernel space only (no user space stacks)")
parser.add_argument("-D", "--delimited", action="store_true",
    help="insert delimiter between kernel/user stacks")
parser.add_argument("-f", "--folded", action="store_true",
    help="output folded format")
parser.add_argument("-g", "--flamegraph", action="store_true",
    help="generate flame graph automatically (implies folded format)")
parser.add_argument("-a", "--all", action="store_true",
    help="output all call stacks (works with or without -g, default: top 5)")
parser.add_argument("-o", "--output", metavar="OUTPUT_FILE", dest="output_file",
    help="output file path for flame graph (default: /tmp/output_ext.dat)")
parser.add_argument("--stack-storage-size", default=16384,
    type=positive_nonzero_int,
    help="the number of unique stack traces that can be stored and "
         "displayed (default 16384)")
parser.add_argument("-d","--duration", default=99999999,
    type=positive_nonzero_int,
    help="duration of trace, in seconds")
parser.add_argument("-m", "--min-block-time", default=1,
    type=positive_nonzero_int,
    help="the amount of time in microseconds over which we " +
         "store traces (default 1)")
parser.add_argument("-M", "--max-block-time", default=3600000000,
    type=positive_nonzero_int,
    help="the amount of time in microseconds under which we " +
         "store traces (default U64_MAX)")
parser.add_argument("--state", type=_positive_int,
    help="filter on this thread state bitmask (eg, 2 == TASK_UNINTERRUPTIBLE" +
         ") see include/linux/sched.h")
parser.add_argument("--bpf-max-entries", default=10240,
    type=positive_nonzero_int,
    help="the maximum number of entries for BPF hash tables "
         "(default 10240)")
parser.add_argument("--ebpf", action="store_true",
    help=argparse.SUPPRESS)
args = parser.parse_args()
folded = args.folded
duration = int(args.duration)

global flamegraph_path
# Check if flamegraph.pl exists when --flamegraph is specified
if args.flamegraph:
    # Get the directory path of the current script
    script_dir = os.path.dirname(os.path.abspath(__file__))
    flamegraph_script = "flamegraph.pl"
    
    # First check if flamegraph.pl exists in the same directory as the script
    flamegraph_path = os.path.join(script_dir, flamegraph_script)
    
    if not os.path.exists(flamegraph_path):
        # If not in the same directory, try to find in PATH
        flamegraph_found = False
        for path in os.environ.get("PATH", "").split(":"):
            if os.path.exists(os.path.join(path, flamegraph_script)):
                flamegraph_path = os.path.join(path, flamegraph_script)
                flamegraph_found = True
                break
        
        if not flamegraph_found:
            print("ERROR: flamegraph.pl not found in script directory or PATH")
            print("Please ensure flamegraph.pl is available for flame graph generation")
            exit(1)
    
# signal handler
def signal_ignore(signal, frame):
    print()

# define BPF program
bpf_text = """
#include <uapi/linux/ptrace.h>
#include <linux/sched.h>

#define MINBLOCK_US    MINBLOCK_US_VALUEULL
#define MAXBLOCK_US    MAXBLOCK_US_VALUEULL

struct key_t {
    char waker[TASK_COMM_LEN];
    char target[TASK_COMM_LEN];
    s64 w_k_stack_id;
    s64 w_u_stack_id;
    s64 t_k_stack_id;
    s64 t_u_stack_id;
    u64 t_pid;
    u64 t_tgid;
    u32 w_pid;
    u32 w_tgid;
};
BPF_HASH(counts, struct key_t,u64,BPF_MAX_ENTRIES);

// Key of this hash is PID of waiting Process,
// value is timestamp and stack ids when it went into waiting
struct start_t {
    u64 ts;
    int k_stack_id;
    int u_stack_id;
};
BPF_HASH(start, u32, struct start_t, BPF_MAX_ENTRIES);

struct wokeby_t {
    char name[TASK_COMM_LEN];
    int k_stack_id;
    int u_stack_id;
    int w_pid;
    int w_tgid;
};
// Key of the hash is PID of the Process to be waken, value is information
// of the Process who wakes it
BPF_HASH(wokeby, u32, struct wokeby_t,BPF_MAX_ENTRIES);

BPF_STACK_TRACE(stack_traces, STACK_STORAGE_SIZE);

static int waker(ARG0, struct task_struct *p) {
    // PID and TGID of the target Process to be waken
    u32 pid = p->pid;
    u32 tgid = p->tgid;

    // Skip swapper/idle thread (pid == 0)
    if (pid == 0) {
        return 0;
    }

    if (!((THREAD_FILTER) && (STATE_FILTER))) {
        return 0;
    }

    // Construct information about current (the waker) Process
    struct wokeby_t woke = {};
    bpf_get_current_comm(&woke.name, sizeof(woke.name));
    woke.k_stack_id = KERNEL_STACK_GET;
    woke.u_stack_id = USER_STACK_GET;
    woke.w_pid = bpf_get_current_pid_tgid();
    woke.w_tgid = bpf_get_current_pid_tgid() >> 32;

    wokeby.update(&pid, &woke);
    return 0;
}

static int oncpu(ARG0, struct task_struct *p, struct task_struct *n) {
    // PID and TGID of the previous Process (Process going into waiting)
    u32 pid = p->pid;
    u32 tgid = p->tgid;
    struct start_t *stp;
    u64 ts = bpf_ktime_get_ns();

    // Record timestamp and stack ids for the previous Process (Process going into waiting)
    // Skip swapper/idle thread (pid == 0)
    if ((THREAD_FILTER) && (STATE_FILTER) && (pid != 0)) {
        struct start_t st = {};
        st.ts = ts;
#ifdef USE_RAW_TP
        // For raw_tracepoint sched_switch: context has NOT switched yet,
        // CPU is still running prev, so KERNEL_STACK_GET returns prev's stack (correct)
        st.k_stack_id = KERNEL_STACK_GET;
        st.u_stack_id = USER_STACK_GET;
#else
        // For kprobe finish_task_switch: context has ALREADY switched to next,
        // so KERNEL_STACK_GET would return next's stack (wrong for prev)
        // We don't save stack here, will get it when next wakes up
        st.k_stack_id = -1;
        st.u_stack_id = -1;
#endif
        start.update(&pid, &st);
    }

    // Calculate current Process's wait time by finding the timestamp of when
    // it went into waiting.
    // pid and tgid are now the PID and TGID of the current (waking) Process.
    //pid = bpf_get_current_pid_tgid();
    //tgid = bpf_get_current_pid_tgid() >> 32;
    pid = n->pid;
    tgid = n->tgid;

    // Skip swapper/idle thread (pid == 0)
    if (pid == 0) {
        return 0;
    }

    stp = start.lookup(&pid);
    if (stp == 0) {
        // Missed or filtered when the Process went into waiting
        return 0;
    }
    u64 delta = ts - stp->ts;
    start.delete(&pid);
    delta = delta / 1000;
    if ((delta < MINBLOCK_US) || (delta > MAXBLOCK_US)) {
        return 0;
    }

    // create map key
    struct key_t key = {};
    struct wokeby_t *woke;

    bpf_probe_read_kernel_str(&key.target, sizeof(key.target), n->comm);
    key.t_pid = pid;
    key.t_tgid = tgid;

#ifdef USE_RAW_TP
    // For raw_tracepoint sched_switch: context has NOT switched yet,
    // KERNEL_STACK_GET would return prev's stack (wrong for next/target)
    // Use the stack saved when this process went to sleep
    key.t_k_stack_id = stp->k_stack_id;
    key.t_u_stack_id = stp->u_stack_id;
#else
    // For kprobe finish_task_switch: context has ALREADY switched to next,
    // so KERNEL_STACK_GET returns next's stack (correct for target)
    key.t_k_stack_id = KERNEL_STACK_GET;
    key.t_u_stack_id = USER_STACK_GET;
#endif

    woke = wokeby.lookup(&pid);
    if (woke) {
        key.w_k_stack_id = woke->k_stack_id;
        key.w_u_stack_id = woke->u_stack_id;
        key.w_pid = woke->w_pid;
        key.w_tgid = woke->w_tgid;
        __builtin_memcpy(&key.waker, woke->name, TASK_COMM_LEN);
        wokeby.delete(&pid);
    }

    counts.increment(key, delta);
    return 0;
}
"""

bpf_text_kprobe = """ 
int oncpu_kprobe(struct pt_regs *ctx, struct task_struct *prev) {
    // In finish_task_switch, prev is the first parameter
    // The current task (next) is obtained via bpf_get_current_task()
    struct task_struct *next = (struct task_struct *)bpf_get_current_task();
    return oncpu(ctx, prev, next);
}

int waker_kprobe(struct pt_regs *ctx, struct task_struct *p) {
    // try_to_wake_up's first parameter is struct task_struct *p
    // BCC automatically extracts this from pt_regs
    return waker(ctx, p);
}
"""

bpf_text_raw_tp = """ 
RAW_TRACEPOINT_PROBE(sched_switch)
{
    // TP_PROTO(bool preempt, struct task_struct *prev, struct task_struct *next)
    // sched_switch args:bool preempt, struct task_struct *prev, struct task_struct *next
    // ctx->args:
    // args[0] = preempt
    // args[1] = prev 
    // args[2] = next 
    
    struct task_struct *prev = (struct task_struct *)ctx->args[1];
    struct task_struct *next = (struct task_struct *)ctx->args[2];
    
    return oncpu(ctx,prev,next);
}

RAW_TRACEPOINT_PROBE(sched_wakeup)
{
    // TP_PROTO(struct task_struct *p)
    struct task_struct *p = (struct task_struct *)ctx->args[0];
    return waker(ctx, p);
}
"""

is_supported_raw_tp = BPF.support_raw_tracepoint()
if is_supported_raw_tp:
    bpf_text = "#define USE_RAW_TP\n" + bpf_text
    bpf_text += bpf_text_raw_tp
else:
    bpf_text += bpf_text_kprobe

if is_supported_raw_tp:
    arg0 = 'struct bpf_raw_tracepoint_args *ctx'
else:
    arg0 = 'struct pt_regs *ctx'
bpf_text = bpf_text.replace('ARG0', arg0)

# set thread filter
if args.tgid is not None:
    thread_filter = build_filter("tgid", args.tgid)
elif args.pid is not None:
    thread_filter = build_filter("pid", args.pid)
elif args.user_threads_only:
    thread_filter = '!(p->flags & PF_KTHREAD)'
elif args.kernel_threads_only:
    thread_filter = 'p->flags & PF_KTHREAD'
else:
    thread_filter = '1'
if args.state == 0:
    state_filter = 'p->STATE_FIELD == 0'
elif args.state:
    # these states are sometimes bitmask checked
    state_filter = 'p->STATE_FIELD & %d' % args.state
else:
    state_filter = '1'
bpf_text = bpf_text.replace('THREAD_FILTER', thread_filter)
bpf_text = bpf_text.replace('STATE_FILTER', state_filter)
if BPF.kernel_struct_has_field(b'task_struct', b'__state') == 1:
    bpf_text = bpf_text.replace('STATE_FIELD', '__state')
else:
    bpf_text = bpf_text.replace('STATE_FIELD', 'state')

# set stack storage size and hash max entries
bpf_text = bpf_text.replace('STACK_STORAGE_SIZE', str(args.stack_storage_size))
bpf_text = bpf_text.replace('BPF_MAX_ENTRIES', str(args.bpf_max_entries))
bpf_text = bpf_text.replace('MINBLOCK_US_VALUE', str(args.min_block_time))
bpf_text = bpf_text.replace('MAXBLOCK_US_VALUE', str(args.max_block_time))

# handle stack args
kernel_stack_get = "stack_traces.get_stackid(ctx, 0)"
user_stack_get = "stack_traces.get_stackid(ctx, BPF_F_USER_STACK)"
stack_context = ""
if args.user_stacks_only:
    stack_context = "user"
    kernel_stack_get = "-1"
elif args.kernel_stacks_only:
    stack_context = "kernel"
    user_stack_get = "-1"
else:
    stack_context = "user + kernel"
bpf_text = bpf_text.replace('USER_STACK_GET', user_stack_get)
bpf_text = bpf_text.replace('KERNEL_STACK_GET', kernel_stack_get)
if args.ebpf:
    print(bpf_text)
    exit()

# initialize BPF
b = BPF(text=bpf_text)
if not is_supported_raw_tp:
    b.attach_kprobe(event_re=r'^finish_task_switch$|^finish_task_switch\.isra\.\d$',fn_name="oncpu_kprobe")
    b.attach_kprobe(event="try_to_wake_up", fn_name="waker_kprobe")
    matched = b.num_open_kprobes()
    if matched == 0:
        print("0 functions traced. Exiting.")
        exit(1)

# header
if not folded:
    print("Tracing blocked time (us) by %s off-CPU and waker stack" %
        stack_context, end="")
    if duration < 99999999:
        print(" for %d secs." % duration)
    else:
        print("... Hit Ctrl-C to end.")

try:
    sleep(duration)
except KeyboardInterrupt:
    # as cleanup can take many seconds, trap Ctrl-C:
    # print a newline for folded output on Ctrl-C
    signal.signal(signal.SIGINT, signal_ignore)


if not folded:
    print()

missing_stacks = 0
has_enomem = False
counts = b.get_table("counts")
stack_traces = b.get_table("stack_traces")
need_delimiter = args.delimited and not (args.kernel_stacks_only or
                                         args.user_stacks_only)
# Collect all data for processing
all_counts = list(counts.items())

# Output logic:
# 1. -g: output flamegraph (whether output top5 or all stacks is determined by -a)
# 2. no -g: no flamegraph output
# 3. -a: output all call stacks (works with or without -g)
# 4. no -a: default output top5 (works with or without -g)
#
# Handle all call stacks output when -a is used (without -g)
if not args.flamegraph and args.all:
    missing_count, enomem_flag = print_all_call_stacks(all_counts, stack_traces, b, args, need_delimiter, folded)
    missing_stacks += missing_count
    has_enomem = has_enomem or enomem_flag

# =============== NEW FEATURE  ===============
# ===== 1: Generate flamegraph  ======
# ===== 2: Print top 5 longest blocking ======
# Output flamegraph when -g is used (always outputs top5 after flamegraph)
if args.flamegraph:
    folded_filename = "offwaketime.stack.fold"
    safe_print("Generating folded format file: {}".format(folded_filename))
    
    with open(folded_filename, 'w') as fold_file:
        for k, v in sorted(all_counts, key=lambda x: x[1].value):
            # Handle get_stackid errors - skip invalid stacks
            skip_entry = False
            if not args.user_stacks_only:
                if stack_id_err(k.w_k_stack_id) or stack_id_err(k.t_k_stack_id):
                    skip_entry = True
            if not args.kernel_stacks_only:
                if stack_id_err(k.w_u_stack_id) or stack_id_err(k.t_u_stack_id):
                    skip_entry = True
            if skip_entry:
                continue

            # Get stack information
            waker_user_stack = [] if k.w_u_stack_id < 1 else \
                reversed(list(stack_traces.walk(k.w_u_stack_id))[1:])
            waker_kernel_stack = [] if k.w_k_stack_id < 1 else \
                reversed(list(stack_traces.walk(k.w_k_stack_id))[1:])
            target_user_stack = [] if k.t_u_stack_id < 1 else \
                stack_traces.walk(k.t_u_stack_id)
            target_kernel_stack = [] if k.t_k_stack_id < 1 else \
                stack_traces.walk(k.t_k_stack_id)

            # Build folded format line
            line = [k.target.decode('utf-8', 'replace')]
            
            # Target user stack
            if not args.kernel_stacks_only:
                if not stack_id_err(k.t_u_stack_id):
                    line.extend([b.sym(addr, k.t_tgid).decode('utf-8', 'replace')
                        for addr in reversed(list(target_user_stack)[1:])])
            
            # Target kernel stack
            if not args.user_stacks_only:
                if need_delimiter and k.t_k_stack_id > 0 and k.t_u_stack_id > 0:
                    line.append("-")
                if not stack_id_err(k.t_k_stack_id):
                    line.extend([b.ksym(addr).decode('utf-8', 'replace')
                        for addr in reversed(list(target_kernel_stack)[1:])])
            
            line.append("--")
            
            # Waker kernel stack
            if not args.user_stacks_only:
                if not stack_id_err(k.w_k_stack_id):
                    line.extend([b.ksym(addr).decode('utf-8', 'replace')
                        for addr in reversed(list(waker_kernel_stack))])
            
            # Waker user stack
            if not args.kernel_stacks_only:
                if need_delimiter and k.w_u_stack_id > 0 and k.w_k_stack_id > 0:
                    line.append("-")
                if not stack_id_err(k.w_u_stack_id):
                    line.extend([b.sym(addr, k.w_tgid).decode('utf-8', 'replace')
                        for addr in reversed(list(waker_user_stack))])
            
            line.append(k.waker.decode('utf-8', 'replace'))
            
            # Write to file
            fold_file.write("%s %d\n" % (";".join(line), v.value))

    safe_print("Generating flame graph...")
    try:
        result = subprocess.run([
            "perl", flamegraph_path,  # Use the global flamegraph path
            "--color=chain",
            "--title=Off-Wake Time Flame Graph",
            "--countname=us",
            folded_filename
        ],  stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, check=True)

        # Determine output file path
        output_file = args.output_file if args.output_file else "/tmp/output_ext.dat"
        with open(output_file, "w") as svg_file:
            svg_file.write(result.stdout)

        cleanup_folded_file(folded_filename)
        safe_print("Flame graph generated: {}".format(output_file))

    except subprocess.CalledProcessError as e:
        safe_print("ERROR: Failed to generate flame graph")
        safe_print("Error: {}".format(e.stderr))
    except FileNotFoundError:
        safe_print("ERROR: flamegraph.pl not found")
    except Exception as e:
        safe_print("ERROR: Unexpected error generating flame graph: {}".format(e))

    # After flamegraph generation, output top5 or all stacks based on -a flag
    if args.all:
        # Output all call stacks after flamegraph
        safe_print("\n" + "="*80)
        safe_print("All Call Stacks (after flamegraph)")
        safe_print("="*80)
        missing_count, enomem_flag = print_all_call_stacks(all_counts, stack_traces, b, args, need_delimiter, folded)
        missing_stacks += missing_count
        has_enomem = has_enomem or enomem_flag
    else:
        # Output top5 after flamegraph generation (default when -a is not used)
        safe_print("\n" + "="*80)
        safe_print("TOP 5 Blocking Stacks by Duration (aggregated by target name only)")
        safe_print("="*80)

        # Aggregate by target name only
        aggregated = aggregate_by_target(all_counts)
        
        # Print top 5
        print_top5(aggregated, stack_traces, b, args, need_delimiter)

# Output top5 when -g is not used and -a is not used (default behavior)
if not args.flamegraph and not args.all:
    safe_print("\n" + "="*80)
    safe_print("TOP 5 Blocking Stacks by Duration (aggregated by target name only)")
    safe_print("="*80)

    # Aggregate by target name only
    aggregated = aggregate_by_target(all_counts)
    
    # Print top 5
    print_top5(aggregated, stack_traces, b, args, need_delimiter)
    
    # Update missing_stacks count for top5 output
    for k, v in all_counts:
        if not args.user_stacks_only:
            missing_stacks += int(stack_id_err(k.w_k_stack_id))
            missing_stacks += int(stack_id_err(k.t_k_stack_id))
            has_enomem = has_enomem or (k.w_k_stack_id == -errno.ENOMEM) or \
                         (k.t_k_stack_id == -errno.ENOMEM)
        if not args.kernel_stacks_only:
            missing_stacks += int(stack_id_err(k.w_u_stack_id))
            missing_stacks += int(stack_id_err(k.t_u_stack_id))
            has_enomem = has_enomem or (k.w_u_stack_id == -errno.ENOMEM) or \
                         (k.t_u_stack_id == -errno.ENOMEM)

if missing_stacks > 0:
    enomem_str = " Consider increasing --stack-storage-size."
    safe_print("WARNING: %d stack traces lost and could not be displayed.%s" %
        (missing_stacks, (enomem_str if has_enomem else "")),
        file=stderr)
