golang beyla采集trace程序原理源码解析

// beyla/bpf/go_nethttp.c

SEC("uprobe/ServeHTTP")
int uprobe_ServeHTTP(struct pt_regs *ctx) {
    void *goroutine_addr = GOROUTINE_PTR(ctx);
    void *req = GO_PARAM4(ctx);
    http_func_invocation_t invocation = {
        .start_monotime_ns = bpf_ktime_get_ns(),
        .req_ptr = (u64)req,
        .tp = {0}
    };
    if (req) {
        // 处理trace信息，存入go_trace_map
        server_trace_parent(goroutine_addr, &invocation.tp, (void*)(req + req_header_ptr_pos));
    }
    // write event
    if (bpf_map_update_elem(&ongoing_http_server_requests, &goroutine_addr, &invocation, BPF_ANY)) {
        bpf_dbg_printk("can't update map element");
    }
    return 0;
}

// bpf/go_common.h

static __always_inline void server_trace_parent(void *goroutine_addr, tp_info_t *tp, void *req_header) {
    // May get overriden when decoding existing traceparent, but otherwise we set sample ON
    tp->flags = 1;
    // Get traceparent from the Request.Header
    void *traceparent_ptr = extract_traceparent_from_req_headers(req_header);
    if (traceparent_ptr != NULL) {    // 读到了traceparent
       ....
    } else {     // 未读到traceparent
        bpf_dbg_printk("No traceparent in headers, generating");
        urand_bytes(tp->trace_id, TRACE_ID_SIZE_BYTES);       // 生成随机的trace_id；
        *((u64 *)tp->parent_id) = 0;
    }

    urand_bytes(tp->span_id, SPAN_ID_SIZE_BYTES);
    bpf_map_update_elem(&go_trace_map, &goroutine_addr, tp, BPF_ANY);
}

struct {
    __uint(type, BPF_MAP_TYPE_LRU_HASH);
    __type(key, void *); // key: pointer to the goroutine
    __type(value, tp_info_t);  // value: traceparent info
    __uint(max_entries, MAX_CONCURRENT_SHARED_REQUESTS);
    __uint(pinning, LIBBPF_PIN_BY_NAME);
} go_trace_map SEC(".maps");

typedef struct tp_info {
    unsigned char trace_id[TRACE_ID_SIZE_BYTES];
    unsigned char span_id[SPAN_ID_SIZE_BYTES];
    unsigned char parent_id[SPAN_ID_SIZE_BYTES];
    u64 ts;
    u8  flags;
} tp_info_t;

// beyla/bpf/go_nethttp.c
SEC("uprobe/roundTrip")
int uprobe_roundTrip(struct pt_regs *ctx) {
    roundTripStartHelper(ctx);
    return 0;
}
static __always_inline void roundTripStartHelper(struct pt_regs *ctx) {
    void *goroutine_addr = GOROUTINE_PTR(ctx);
    void *req = GO_PARAM2(ctx);
    http_func_invocation_t invocation = {
        .start_monotime_ns = bpf_ktime_get_ns(),
        .req_ptr = (u64)req,
        .tp = {0}
    };
    // 根据request和goroutine_addr，查找trace信息
    __attribute__((__unused__)) u8 existing_tp = client_trace_parent(goroutine_addr, &invocation.tp, (void*)(req + req_header_ptr_pos));
    // 将trace信息写入ongoing_http_client_requests
    if (bpf_map_update_elem(&ongoing_http_client_requests, &goroutine_addr, &invocation, BPF_ANY)) {
        bpf_dbg_printk("can't update http client map element");
    }
}

// beyla/go_common.h
static __always_inline u8 client_trace_parent(void *goroutine_addr, tp_info_t *tp_i, void *req_header) {
    u8 found_trace_id = 0;
    u8 trace_id_exists = 0;    
    // May get overriden when decoding existing traceparent or finding a server span, but otherwise we set sample ON
    tp_i->flags = 1;
    // 首先尝试从request的header中提取traceparent
    if (req_header) {
        ...
    }
    // 然后再使用goroutine去go_trace_map中找
    if (!found_trace_id) {
        tp_info_t *tp = 0;
        u64 parent_id = find_parent_goroutine(goroutine_addr);
        if (parent_id) {// we found a parent request
            tp = (tp_info_t *)bpf_map_lookup_elem(&go_trace_map, &parent_id);
        }
        if (tp) {   // 找到了，copy traceId，当前span.parentId=上流span.spanId
            *((u64 *)tp_i->trace_id) = *((u64 *)tp->trace_id);
            *((u64 *)(tp_i->trace_id + 8)) = *((u64 *)(tp->trace_id + 8));
            *((u64 *)tp_i->parent_id) = *((u64 *)tp->span_id);
            tp_i->flags = tp->flags;
        } 
        ...
        // 生成当前span.spanId
        urand_bytes(tp_i->span_id, SPAN_ID_SIZE_BYTES);
    }
    return trace_id_exists;
}

// beyla/bpf/go_nethttp.c
SEC("uprobe/roundTrip_return")
int uprobe_roundTripReturn(struct pt_regs *ctx) {
    void *goroutine_addr = GOROUTINE_PTR(ctx);
    // 使用goroutine_addr找ongoing_http_client_requests
    http_func_invocation_t *invocation =
        bpf_map_lookup_elem(&ongoing_http_client_requests, &goroutine_addr);
    bpf_map_delete_elem(&ongoing_http_client_requests, &goroutine_addr);
    http_request_trace *trace = bpf_ringbuf_reserve(&events, sizeof(http_request_trace), 0);
    // 初始化http_request_trace
    task_pid(&trace->pid);
    trace->type = EVENT_HTTP_CLIENT;
    trace->start_monotime_ns = invocation->start_monotime_ns;
    trace->go_start_monotime_ns = invocation->start_monotime_ns;
    trace->end_monotime_ns = bpf_ktime_get_ns();
    void *req_ptr = (void *)invocation->req_ptr;
    void *resp_ptr = (void *)GO_PARAM1(ctx);
    // 从request中找method，赋值给trace->method
    if (!read_go_str("method", req_ptr, method_ptr_pos, &trace->method, sizeof(trace->method))) {
        ...
    }
    // 从request中找host，赋值给trace->host
    if (!read_go_str("host", req_ptr, host_ptr_pos, &trace->host, sizeof(trace->host))) {
        ...
    }
    // 从request中找url，赋值给trace->path
    void *url_ptr = 0;
    bpf_probe_read(&url_ptr, sizeof(url_ptr), (void *)(req_ptr + url_ptr_pos));
    if (!url_ptr || !read_go_str("path", url_ptr, path_ptr_pos, &trace->path, sizeof(trace->path))) {
        ...
    }
    // 赋值trace信息
    trace->tp = invocation->tp;
    // 从request中找content_length，赋值给trace->content_length
    bpf_probe_read(&trace->content_length, sizeof(trace->content_length), (void *)(req_ptr + content_length_ptr_pos));
    // 从resp中找status，赋值给trace->status
    bpf_probe_read(&trace->status, sizeof(trace->status), (void *)(resp_ptr + status_code_ptr_pos));
    // 提交trace到ringbuf
    bpf_ringbuf_submit(trace, get_flags());
    return 0;
}