From 42600f6e9b490b0c7638954a9a1487f5b73214b9 Mon Sep 17 00:00:00 2001 From: cle-b Date: Sun, 13 Apr 2025 18:36:18 +0200 Subject: [PATCH 1/3] go --- httpdbg/__init__.py | 2 +- httpdbg/__main__.py | 5 +- httpdbg/args.py | 8 +- httpdbg/hooks/all.py | 41 +++-- httpdbg/hooks/go.py | 85 ++++++++++ httpdbg/hooks/httpdbg-tracer.go | 271 ++++++++++++++++++++++++++++++++ httpdbg/mode_go.py | 68 ++++++++ tests/demo_go/go.mod | 3 + tests/demo_go/indir/mymod.go | 17 ++ tests/demo_go/main.go | 27 ++++ 10 files changed, 507 insertions(+), 20 deletions(-) create mode 100644 httpdbg/hooks/go.py create mode 100644 httpdbg/hooks/httpdbg-tracer.go create mode 100644 httpdbg/mode_go.py create mode 100644 tests/demo_go/go.mod create mode 100644 tests/demo_go/indir/mymod.go create mode 100644 tests/demo_go/main.go diff --git a/httpdbg/__init__.py b/httpdbg/__init__.py index bac8029..94a9a2a 100644 --- a/httpdbg/__init__.py +++ b/httpdbg/__init__.py @@ -3,6 +3,6 @@ from httpdbg.records import HTTPRecords -__version__ = "1.2.1" +__version__ = "2.0.0" __all__ = ["httprecord", "HTTPRecords"] diff --git a/httpdbg/__main__.py b/httpdbg/__main__.py index de4d3b8..1c5845d 100644 --- a/httpdbg/__main__.py +++ b/httpdbg/__main__.py @@ -8,6 +8,7 @@ from httpdbg.log import set_env_for_logging from httpdbg.server import httpdbg_srv from httpdbg.mode_console import run_console +from httpdbg.mode_go import run_go from httpdbg.mode_module import run_module from httpdbg.mode_script import run_script @@ -30,11 +31,13 @@ def pyhttpdbg(params, subparams, test_mode=False): with httpdbg_srv(params.host, params.port) as records: records.server = not params.only_client - with httprecord(records, params.initiator, server=records.server): + with httprecord(records, params.initiator, server=records.server, go=params.go): if params.module: run_module(subparams) elif params.script: run_script(subparams) + elif params.go: + run_go(subparams) else: run_console(records, test_mode) diff --git a/httpdbg/args.py b/httpdbg/args.py index e43bb47..e8cf197 100644 --- a/httpdbg/args.py +++ b/httpdbg/args.py @@ -9,7 +9,7 @@ def read_args(args: List[str]) -> Tuple[argparse.Namespace, List[str]]: httpdbg_args = args client_args = [] - for action in ["--console", "--module", "-m", "--script"]: + for action in ["--console", "--module", "-m", "--script", "--go"]: if action in args: httpdbg_args = args[: args.index(action) + 2] client_args = args[args.index(action) + 1 :] @@ -87,4 +87,10 @@ def read_args(args: List[str]) -> Tuple[argparse.Namespace, List[str]]: help="run a script (the next args are passed to the script as is)", ) + actions.add_argument( + "--go", + type=str, + help="run a go program using the 'go run' command (the next args are passed to the 'go' process as is) (experimental)", + ) + return parser.parse_args(httpdbg_args), client_args diff --git a/httpdbg/hooks/all.py b/httpdbg/hooks/all.py index 0ba1eac..05ccf9d 100644 --- a/httpdbg/hooks/all.py +++ b/httpdbg/hooks/all.py @@ -12,6 +12,7 @@ from httpdbg.hooks.generic import hook_generic from httpdbg.hooks.http import hook_http from httpdbg.hooks.httpx import hook_httpx +from httpdbg.hooks.go import watcher_go from httpdbg.hooks.pytest import hook_pytest from httpdbg.hooks.requests import hook_requests from httpdbg.hooks.socket import hook_socket @@ -30,24 +31,30 @@ def httprecord( client: bool = True, server: bool = False, ignore: Union[List[Tuple[str, int]], None] = None, + go: bool = False, ) -> Generator[HTTPRecords, None, None]: if records is None: records = HTTPRecords(client=client, server=server, ignore=ignore) - with watcher_external(records, initiators, server): - with hook_flask(records): - with hook_socket(records): - with hook_fastapi(records): - with hook_starlette(records): - with hook_uvicorn(records): - with hook_http(records): - with hook_httpx(records): - with hook_requests(records): - with hook_urllib3(records): - with hook_aiohttp(records): - with hook_pytest(records): - with hook_unittest(records): - with hook_generic( - records, initiators - ): - yield records + if not go: + with watcher_external(records, initiators, server): + with hook_flask(records): + with hook_socket(records): + with hook_fastapi(records): + with hook_starlette(records): + with hook_uvicorn(records): + with hook_http(records): + with hook_httpx(records): + with hook_requests(records): + with hook_urllib3(records): + with hook_aiohttp(records): + with hook_pytest(records): + with hook_unittest(records): + with hook_generic( + records, initiators + ): + yield records + else: + # if we trace the HTTP requests in go process, there is no reason to hook the Python call + with watcher_go(records): + yield records diff --git a/httpdbg/hooks/go.py b/httpdbg/hooks/go.py new file mode 100644 index 0000000..53ad6ca --- /dev/null +++ b/httpdbg/hooks/go.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +from contextlib import contextmanager +import json +import os +import tempfile +import time +import threading +from typing import Generator + +from httpdbg.env import HTTPDBG_MULTIPROCESS_DIR +from httpdbg.initiator import Group +from httpdbg.initiator import Initiator +from httpdbg.log import logger +from httpdbg.records import HTTPRecord +from httpdbg.records import HTTPRecords + + +@contextmanager +def watcher_go( + records: HTTPRecords, +) -> Generator[HTTPRecords, None, None]: + if HTTPDBG_MULTIPROCESS_DIR not in os.environ: + with tempfile.TemporaryDirectory(prefix="httpdbg_") as httpdbg_multiprocess_dir: + + logger().info(f"watcher_go {httpdbg_multiprocess_dir}") + os.environ[HTTPDBG_MULTIPROCESS_DIR] = httpdbg_multiprocess_dir + + try: + watcher = WatcherGoDirThread(records, httpdbg_multiprocess_dir, 1.0) + watcher.start() + + yield records + finally: + watcher.shutdown() + else: + yield records + + +class WatcherGoDirThread(threading.Thread): + def __init__(self, records: HTTPRecords, directory: str, delay: float) -> None: + self.records: HTTPRecords = records + self.directory: str = directory + self.delay: float = delay + self._running: bool = True + threading.Thread.__init__(self) + + def get_go_traces(self): + traces_filename = os.path.join(self.directory, "httpdbg-go-traces.json") + if os.path.exists(traces_filename): + with open(traces_filename, "r") as f: + traces = json.load(f) + for trace in traces: + if trace["trace_id"] not in self.records.requests.keys(): + label = trace["initiator"]["code"] + full_label = f'File "{trace["initiator"]["filename"]}", line {trace["initiator"]["lineno"]}, in {trace["initiator"]["func_name"]}' + full_label += f"\n {trace['initiator']['code']}" + stack = [] + for line in trace["initiator"]["stack"]: + stack.append(f"{line['location']}, in {line['func_name']}") + stack.append(f" {line['code'].replace('\t', ' ')}\n") + + initiator = Initiator(label, full_label, stack) + self.records.add_initiator(initiator) + group = Group(label, full_label, False) + self.records.add_group(group) + record = HTTPRecord( + initiator.id, + group.id, + ) + record.id = trace["trace_id"] + with open(trace["request_file"], "rb") as reqf: + record.send_data(reqf.read()) + with open(trace["response_file"], "rb") as resf: + record.receive_data(resf.read()) + self.records.requests[record.id] = record + + def run(self): + while self._running: + self.get_go_traces() + time.sleep(self.delay) + + def shutdown(self): + self._running = False + self.join(timeout=5) + self.get_go_traces() diff --git a/httpdbg/hooks/httpdbg-tracer.go b/httpdbg/hooks/httpdbg-tracer.go new file mode 100644 index 0000000..b40687a --- /dev/null +++ b/httpdbg/hooks/httpdbg-tracer.go @@ -0,0 +1,271 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log" + "math/rand" + "net/http" + "net/http/httputil" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "time" +) + +var traceLogMutex sync.Mutex +var traceList []TraceEntry + +type StackFrame struct { + FuncName string `json:"func_name"` + Location string `json:"location"` + Code string `json:"code"` +} + +type Initiator struct { + Filename string `json:"filename"` + Lineno int `json:"lineno"` + FuncName string `json:"func_name"` + Code string `json:"code"` + Stack []StackFrame `json:"stack"` +} + +type TraceEntry struct { + TraceID string `json:"trace_id"` + Timestamp string `json:"timestamp"` + EndTimestamp string `json:"end_timestamp"` + Initiator Initiator `json:"initiator"` + RequestFile string `json:"request_file"` + ResponseFile string `json:"response_file"` +} + +func init() { + defaultTransport := http.DefaultTransport.(*http.Transport) + + http.DefaultTransport = &tracingTransport{ + inner: defaultTransport, + } +} + +func min(a, b int) int { //TODO to replace by cmp.Min + if a < b { + return a + } + return b +} +func max(a, b int) int { //TODO to replace by cmp.Max + if a > b { + return a + } + return b +} + +func getHttpdbgDir() string { + outputDir := os.Getenv("HTTPDBG_MULTIPROCESS_DIR") + if outputDir == "" { + log.Printf("httpdbg -HTTPDBG_MULTIPROCESS_DIR not set") + } + return outputDir +} + +type tracingTransport struct { + inner *http.Transport +} + +func getInitiator() Initiator { + pcs := make([]uintptr, 32) + n := runtime.Callers(3, pcs) + frames := runtime.CallersFrames(pcs[:n]) + + var fullStack []StackFrame + var selectedFrame *runtime.Frame + + for { + frame, more := frames.Next() + + entry := StackFrame{ + FuncName: frame.Function, + Location: fmt.Sprintf("%s:%d", filepath.Base(frame.File), frame.Line), + Code: readSourceLine(frame.File, frame.Line), // default single line + } + + if selectedFrame == nil && + !strings.Contains(frame.Function, "net/http") && + !strings.Contains(frame.Function, "runtime.") { + + selectedFrame = &frame + + // Replace single line with full context as one string + contextLines := readCodeContextLines(frame.File, frame.Line, 4) + entry.Code = strings.Join(contextLines, "\n") + + fullStack = append(fullStack, entry) + break // stop after the initiator + } else { + fullStack = append(fullStack, entry) + } + + if !more { + break + } + } + + if selectedFrame == nil { + selectedFrame = &runtime.Frame{ + File: "unknown", + Line: 0, + Function: "unknown", + } + fullStack = append(fullStack, StackFrame{ + FuncName: "unknown", + Location: "unknown:0", + Code: "", + }) + } + + return Initiator{ + Filename: filepath.Base(selectedFrame.File), + Lineno: selectedFrame.Line, + FuncName: selectedFrame.Function, + Code: readSourceLine(selectedFrame.File, selectedFrame.Line), + Stack: fullStack, + } +} + +func readCodeContextLines(filename string, centerLine, padding int) []string { + data, err := os.ReadFile(filename) + if err != nil { + return []string{""} + } + lines := strings.Split(string(data), "\n") + start := max(centerLine-padding-1, 0) + end := min(centerLine+padding, len(lines)) + + var context []string + for i := start; i < end; i++ { + line := lines[i] + if i == centerLine-1 { + context = append(context, line+" <====") + } else { + context = append(context, line) + } + } + return context +} + +func readSourceLine(filename string, line int) string { + data, err := os.ReadFile(filename) + if err != nil { + return "source unavailable" + } + lines := strings.Split(string(data), "\n") + if line >= 1 && line <= len(lines) { + return strings.TrimSpace(lines[line-1]) + } + return "line out of range" +} + +func (t *tracingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + traceID := generateTraceID() + startTime := time.Now().UTC() + timestamp := startTime.Format(time.RFC3339Nano) + outputDir := getHttpdbgDir() + initiator := getInitiator() + + // Read request body (must buffer + re-inject) + var reqBody []byte + if req.Body != nil { + reqBody, _ = io.ReadAll(req.Body) + req.Body = io.NopCloser(bytes.NewReader(reqBody)) + } + + requestFile := filepath.Join(outputDir, fmt.Sprintf("%s-request.txt", traceID)) + dump, err := httputil.DumpRequestOut(req, true) + if err != nil { + log.Printf("httpdbg -Failed to dump request: %v", err) + } else { + err = os.WriteFile(requestFile, dump, 0644) + if err != nil { + log.Printf("httpdbg -Failed to write request file: %v", err) + } + } + + resp, err := t.inner.RoundTrip(req) + if err != nil { // TODO trace failure + return nil, err + } + + // Read response body (must buffer + re-inject) + var respBody []byte + if resp.Body != nil { + respBody, _ = io.ReadAll(resp.Body) + resp.Body = io.NopCloser(bytes.NewReader(respBody)) + } + + responseFile := filepath.Join(outputDir, fmt.Sprintf("%s-response.txt", traceID)) + dumpResp, err := httputil.DumpResponse(resp, true) + if err != nil { + log.Printf("httpdbg -Failed to dump response: %v", err) + } else { + err = os.WriteFile(responseFile, dumpResp, 0644) + if err != nil { + log.Printf("httpdbg -Failed to write response file: %v", err) + } + } + + endTime := time.Now().UTC() + endTimestamp := endTime.Format(time.RFC3339Nano) + + // Write trace index entry + entry := TraceEntry{ + TraceID: traceID, + Timestamp: timestamp, + EndTimestamp: endTimestamp, + Initiator: initiator, + RequestFile: requestFile, + ResponseFile: responseFile, + } + + appendToTraceIndex(entry) + + return resp, nil +} + +func generateTraceID() string { + return fmt.Sprintf("go-%d-%d", time.Now().UnixNano(), rand.Intn(10000)) +} + +func appendToTraceIndex(entry TraceEntry) { + traceLogMutex.Lock() + defer traceLogMutex.Unlock() + + traceList = append(traceList, entry) + + dir := getHttpdbgDir() + tmpFile := filepath.Join(dir, "httpdbg-go-traces.json.tmp") + finalFile := filepath.Join(dir, "httpdbg-go-traces.json") + + // Write to temp file + f, err := os.Create(tmpFile) + if err != nil { + log.Printf("httpdbg -Failed to create temp trace file: %v", err) + return + } + defer f.Close() + + encoder := json.NewEncoder(f) + encoder.SetIndent("", " ") + if err := encoder.Encode(traceList); err != nil { + log.Printf("httpdbg -Failed to encode trace list: %v", err) + return + } + + // Atomically rename to final file + if err := os.Rename(tmpFile, finalFile); err != nil { + log.Printf("httpdbg -Failed to rename temp trace file: %v", err) + } +} diff --git a/httpdbg/mode_go.py b/httpdbg/mode_go.py new file mode 100644 index 0000000..cc6c243 --- /dev/null +++ b/httpdbg/mode_go.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +import os +import subprocess +import sys +import threading +from typing import List + + +def run_process(args): + """ + Runs the go process and redirect the input/output in the console. + """ + process = subprocess.Popen( + args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + + def forward_output(src, dest): + for line in src: + dest.write(line) + dest.flush() + + threading.Thread(target=forward_output, args=(process.stdout, sys.stdout)).start() + threading.Thread(target=forward_output, args=(process.stderr, sys.stderr)).start() + + return process + + +def run_go(argv: List[str]) -> None: + httdbg_tracer_filename = "httpdbg-tracer.go" + try: + if argv[0] != "run": + print("only the 'go run' command is supported.") + else: + args: List[str] = argv.copy() + + args.insert(0, "go") + + # we inject the "tracer" in the Go program during the execution of the "go run" command + tracer = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "hooks", + httdbg_tracer_filename, + ) + os.symlink( + tracer, httdbg_tracer_filename + ) # the file must be in the same directory as the other files + + i = 2 + while i < len(args): + if args[i].endswith(".go") and os.path.join(os.path.dirname(os.path.abspath(__file__)), args[i]): + i += 1 + else: + break + args.insert(i, httdbg_tracer_filename) + + process = run_process(args) + + process.wait() + except SystemExit: + pass + finally: + if os.path.exists(httdbg_tracer_filename): + os.remove(httdbg_tracer_filename) diff --git a/tests/demo_go/go.mod b/tests/demo_go/go.mod new file mode 100644 index 0000000..721e4ea --- /dev/null +++ b/tests/demo_go/go.mod @@ -0,0 +1,3 @@ +module demo_go + +go 1.24.2 diff --git a/tests/demo_go/indir/mymod.go b/tests/demo_go/indir/mymod.go new file mode 100644 index 0000000..93a382e --- /dev/null +++ b/tests/demo_go/indir/mymod.go @@ -0,0 +1,17 @@ +package indir + +import ( + "fmt" + "log" + "net/http" +) + +func AnotherFunction(url string) { + resp, err := http.Get(url) + if err != nil { + log.Fatalf("Error making HTTP request in AnotherFunction: %v", err) + } + defer resp.Body.Close() + + fmt.Printf("mymod.go: Response status from %s: %s\n", url, resp.Status) +} diff --git a/tests/demo_go/main.go b/tests/demo_go/main.go new file mode 100644 index 0000000..f30bbb5 --- /dev/null +++ b/tests/demo_go/main.go @@ -0,0 +1,27 @@ +package main + +import ( + "fmt" + "log" + "net/http" + "os" + + "demo_go/indir" +) + +func main() { + if len(os.Args) < 2 { + log.Fatal("Usage: go run main.go ") + } + url := os.Args[1] + + resp, err := http.Get(url) + if err != nil { + log.Fatalf("Error making HTTP request: %v", err) + } + defer resp.Body.Close() + + fmt.Printf("main.go: Response status from %s: %s\n", url, resp.Status) + + indir.AnotherFunction(url) +} From e4afe2bab8eeed1797f392f0a47e8eb0e1cea771 Mon Sep 17 00:00:00 2001 From: cle-b Date: Sun, 13 Apr 2025 18:40:07 +0200 Subject: [PATCH 2/3] format --- httpdbg/mode_go.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/httpdbg/mode_go.py b/httpdbg/mode_go.py index cc6c243..cdc1139 100644 --- a/httpdbg/mode_go.py +++ b/httpdbg/mode_go.py @@ -50,12 +50,14 @@ def run_go(argv: List[str]) -> None: tracer, httdbg_tracer_filename ) # the file must be in the same directory as the other files - i = 2 + i = 2 while i < len(args): - if args[i].endswith(".go") and os.path.join(os.path.dirname(os.path.abspath(__file__)), args[i]): + if args[i].endswith(".go") and os.path.join( + os.path.dirname(os.path.abspath(__file__)), args[i] + ): i += 1 else: - break + break args.insert(i, httdbg_tracer_filename) process = run_process(args) From b2efe46aef37cce0b9189a4d00b77390c26b9955 Mon Sep 17 00:00:00 2001 From: cle-b Date: Sun, 13 Apr 2025 18:45:56 +0200 Subject: [PATCH 3/3] format --- httpdbg/hooks/go.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/httpdbg/hooks/go.py b/httpdbg/hooks/go.py index 53ad6ca..08791d4 100644 --- a/httpdbg/hooks/go.py +++ b/httpdbg/hooks/go.py @@ -57,7 +57,8 @@ def get_go_traces(self): stack = [] for line in trace["initiator"]["stack"]: stack.append(f"{line['location']}, in {line['func_name']}") - stack.append(f" {line['code'].replace('\t', ' ')}\n") + code = line["code"].replace("\t", " ") + stack.append(f" {code}\n") initiator = Initiator(label, full_label, stack) self.records.add_initiator(initiator)