From 7c48860ef67e3eb805aa496fb25158e77e683fc7 Mon Sep 17 00:00:00 2001 From: Gabriele Tornetta Date: Sun, 16 Oct 2022 14:43:36 +0100 Subject: [PATCH] refactor: libaustin --- .github/workflows/benchmarks.yml | 2 +- .github/workflows/release.yml | 10 +- .github/workflows/release_arch.yml | 2 +- .github/workflows/tests.yml | 13 ++- .gitignore | 1 + Dockerfile | 2 +- README.md | 72 +++++++++++- configure.ac | 3 + include/libaustin.h | 181 +++++++++++++++++++++++++++++ scripts/build_arch.sh | 10 +- src/Makefile.am | 20 +++- src/austin.c | 76 +++++++++++- src/austin.h | 2 +- src/code.h | 30 ++--- src/error.h | 13 --- src/frame.h | 15 +-- src/linux/py_proc.h | 3 + src/logging.c | 2 + src/logging.h | 22 ++++ src/py_proc.c | 129 +++++++++++++++++++- src/py_proc.h | 27 ++++- src/py_thread.c | 35 ++++++ src/py_thread.h | 9 ++ src/stack.h | 6 +- test/cunit/conftest.py | 13 ++- test/cunit/libaustin.py | 52 +++++++++ test/cunit/test_libaustin.py | 65 +++++++++++ test/libaustin/frame.c | 41 +++++++ test/libaustin/where.c | 43 +++++++ 29 files changed, 819 insertions(+), 80 deletions(-) create mode 100644 include/libaustin.h create mode 100644 test/cunit/libaustin.py create mode 100644 test/cunit/test_libaustin.py create mode 100644 test/libaustin/frame.c create mode 100644 test/libaustin/where.c diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index f337839a..3f6c438e 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -22,7 +22,7 @@ jobs: - name: Compile Austin run: | autoreconf --install - ./configure --enable-debug-symbols true + ./configure --enable-debug-symbols=yes make - name: Install runtime dependencies diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e06100f3..33bce916 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,7 +26,7 @@ jobs: pip install -r scripts/requirements-bw.txt sudo apt-get update - sudo apt-get -y install autoconf build-essential libunwind-dev binutils-dev libiberty-dev musl-tools zlib1g-dev + sudo apt-get -y install autoconf build-essential libunwind-dev binutils-dev libiberty-dev musl-tools zlib1g-dev libtool # Build austin autoreconf --install @@ -38,6 +38,10 @@ jobs: pushd src tar -Jcf austin-$VERSION-gnu-linux-amd64.tar.xz austin tar -Jcf austinp-$VERSION-gnu-linux-amd64.tar.xz austinp + + cp ../include/libaustin.h . + cp .libs/libaustin.{a,so.*.*} . + tar -Jcf libaustin-$VERSION-gnu-linux-amd64.tar.xz libaustin.* popd # Build gnu wheel @@ -47,7 +51,7 @@ jobs: --files austin:src/austin austinp:src/austinp # Build with musl - musl-gcc -O3 -Os -s -Wall -pthread src/*.c -o src/austin -D__MUSL__ + musl-gcc -O3 -Os -s -Wall -pthread src/*.c -Iinclude -o src/austin -D__MUSL__ pushd src tar -Jcf austin-$VERSION-musl-linux-amd64.tar.xz austin popd @@ -64,7 +68,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: src/austin*.tar.xz + file: src/*austin*.tar.xz tag: ${{ github.ref }} overwrite: true file_glob: true diff --git a/.github/workflows/release_arch.yml b/.github/workflows/release_arch.yml index 3c513b34..d28af88c 100644 --- a/.github/workflows/release_arch.yml +++ b/.github/workflows/release_arch.yml @@ -35,7 +35,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/austin*.tar.xz + file: artifacts/*austin*.tar.xz tag: ${{ github.ref }} overwrite: true file_glob: true diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 739b7dc2..9f5fc250 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,12 +21,12 @@ jobs: - name: Install build dependencies run: | sudo apt-get update - sudo apt-get -y install libunwind-dev binutils-dev libiberty-dev + sudo apt-get -y install libunwind-dev binutils-dev libiberty-dev libtool - name: Compile Austin run: | autoreconf --install - ./configure --enable-debug-symbols true + ./configure --enable-debug-symbols=yes make - uses: actions/upload-artifact@v3 @@ -35,6 +35,7 @@ jobs: path: | src/austin src/austinp + src/.libs/libaustin.so* build-linux-musl: runs-on: ubuntu-20.04 @@ -49,7 +50,7 @@ jobs: - name: Compile Austin run: | - musl-gcc -O3 -Os -s -Wall -pthread src/*.c -o src/austin.musl -D__MUSL__ + musl-gcc -O3 -Os -s -Wall -pthread src/*.c -Iinclude -o src/austin.musl -D__MUSL__ - uses: actions/upload-artifact@v3 with: @@ -159,7 +160,7 @@ jobs: - uses: actions/checkout@v2 - name: Compile Austin - run: gcc-11 -Wall -Werror -O3 -g src/*.c -o src/austin + run: gcc-11 -Wall -Werror -O3 -g src/*.c -Iinclude -o src/austin - uses: actions/upload-artifact@v3 with: @@ -278,7 +279,7 @@ jobs: - name: Compile Austin run: | gcc.exe --version - gcc.exe -O3 -g -o src/austin.exe src/*.c -lpsapi -lntdll -Wall -Werror + gcc.exe -O3 -g -o src/austin.exe src/*.c -Iinclude -lpsapi -lntdll -Wall -Werror src\austin.exe --help - uses: actions/upload-artifact@v3 @@ -395,7 +396,7 @@ jobs: - name: Compile Austin run: | autoreconf --install - ./configure --enable-debug-symbols true + ./configure --enable-debug-symbols=yes make - name: Install runtime dependencies diff --git a/.gitignore b/.gitignore index b283606c..05eeab95 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ src/austin src/austinp src/austin.exe +.libs/ # ----------------------------------------------------------------------------- # -- PYTHON SOURCES diff --git a/Dockerfile b/Dockerfile index 24836096..32b53ad5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:20.04 COPY . /austin RUN apt-get update && \ - apt-get install -y autoconf build-essential libunwind-dev binutils-dev libiberty-dev zlib1g-dev && \ + apt-get install -y autoconf build-essential libunwind-dev binutils-dev libiberty-dev zlib1g-dev libtool && \ cd /austin && \ autoreconf --install && \ ./configure && \ diff --git a/README.md b/README.md index e5dc0e61..d8f0f326 100644 --- a/README.md +++ b/README.md @@ -293,19 +293,19 @@ git clone --depth=1 https://github.com/P403n1x87/austin.git On Linux, one can then use the command ~~~ console -gcc -O3 -Os -Wall -pthread src/*.c -o src/austin +gcc -O3 -Os -Wall -pthread src/*.c -Iinclude -o src/austin ~~~ whereas on macOS it is enough to run ~~~ console -gcc -O3 -Os -Wall src/*.c -o src/austin +gcc -O3 -Os -Wall src/*.c -Iinclude -o src/austin ~~~ On Windows, the `-lpsapi -lntdll` switches are needed ~~~ console -gcc -O3 -Os -Wall -lpsapi -lntdll src/*.c -o src/austin +gcc -O3 -Os -Wall -lpsapi -lntdll src/*.c -Iinclude -o src/austin ~~~ Add `-DDEBUG` if you need a more verbose log. This is useful if you encounter a @@ -521,7 +521,7 @@ sudo apt install libunwind-dev binutils-dev and compile with ~~~ console -gcc -O3 -Os -Wall -pthread src/*.c -DAUSTINP -lunwind-ptrace -lunwind-generic -lbfd -o src/austinp +gcc -O3 -Os -Wall -pthread src/*.c -Iinclude -DAUSTINP -lunwind-ptrace -lunwind-generic -lbfd -o src/austinp ~~~ then use as per normal. The extra `-k/--kernel` option is available with @@ -557,6 +557,70 @@ show both native and Python frames. Highlighting helps tell frames apart. The > executable will be available as `austin.p` from the command line. +## libaustin + +The code base can also be used to generate a library to embed Austin in your +projects to unwind Python stacks or extract frame information from a running +Python process. + +A shared library on Linux can be obtained with + +~~~ console +gcc -O3 -Wall -pthread src/*.c -Iinclude -o src/libaustin.so -shared -fPIC -DLIBAUSTIN +~~~ + +The required headers are in the `include/` subfolder. More information on how +to use the library can be found in the `libaustin.h` header file. + +At a glance, this is a typical use. The library needs to be initialised at +runtime before it can be used. Hence, before you call any other of the exported +functions, you must do + +~~~ c +#include + +... + +{ + ... + + austin_up(); + + ... +} +~~~ + +After this call, you can attach any Python process with + +~~~ c +pid_t pid = ...; +... +austin_handle_t proc_handle = austin_attach(pid); +~~~ + +When you want to sample the call stack of an attached process, call + +~~~ c +austin_sample(proc_handle, cb); +~~~ + +where `cb` is a callback function with signature `void ()(pid_t pid, pid_t tid)` +that gets called once a thread stack is ready to be retrieved. To get the frames +from the sampled stack, call `austin_pop_frame()` until it returns `NULL`. + +To sample a single thread, use `austin_sample_thread(proc_handle, tid)` instead. +Then retrieve the sampled stack with `austin_pop_frame()` as above. + +Once you are done with the process, you should detach it with + +~~~ c +austin_detach(proc_handle); +~~~ + +Once you are done with libaustin entirely, make sure to release resources with +`austin_down()`. + + ## Logging Austin uses `syslog` on Linux and macOS, and `%TEMP%\austin.log` on Windows diff --git a/configure.ac b/configure.ac index 8cda77ff..50b661db 100644 --- a/configure.ac +++ b/configure.ac @@ -16,6 +16,9 @@ AM_INIT_AUTOMAKE AC_PROG_CC_C99 AC_PROG_CPP +# Use libtool for libaustin. +LT_INIT + # Use the C language and compiler for the following checks AC_LANG([C]) diff --git a/include/libaustin.h b/include/libaustin.h new file mode 100644 index 00000000..c6f1b4ec --- /dev/null +++ b/include/libaustin.h @@ -0,0 +1,181 @@ +// This file is part of "austin" which is released under GPL. +// +// See file LICENCE or go to http://www.gnu.org/licenses/ for full license +// details. +// +// Austin is a Python frame stack sampler for CPython. +// +// Copyright (c) 2018 Gabriele N. Tornetta . +// All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + + +/** + * Austin stack callback. + * + * This function is called back once the unwinding of thread stack has been + * completed, and frames are ready to be retrieved with austin_pop_frame(). + * The callback is called with the following arguments: + * + * @param pid_t the PID of the process the thread belongs to. + * @param pid_t the TID of the thread. +*/ +typedef void (*austin_callback_t)(pid_t, pid_t); + + +/** + * Austin process handle. + * + * This is a handle to an attached process that is generally required to perfom + * unwinding operations. +*/ +typedef struct _py_proc_t * austin_handle_t; + + +/** + * The Austin frame structure. + * + * This structure is used to return frame stack information to the user. +*/ +typedef struct { + uintptr_t key; // a key that uniquely identifies the frame. + char * filename; // the file name of the source containing the code. + char * scope; // the name of the scope, e.g. the function name + unsigned int line; // the line number. + unsigned int line_end; // the end line number. + unsigned int column; // the column number. + unsigned int column_end; // the end column number. +} austin_frame_t; + + +/** + * Initialise the Austin library. + * + * This function must be called before any other function in the library. +*/ +extern int +austin_up(); + + +/** + * Finalise the Austin library. + * + * This function should be called once the Austin library is no longer needed, + * to free up resources. +*/ +extern void +austin_down(); + + +/** + * Attach to a Python process. + * + * This function tries to attach to a running Python process, identified by its + * PID. If the process exists and is a valid Python process, a non-NULL handle + * is returned to the caller, to be used for further operations. + * + * Note that attaching to a Python process is a lightweight operation that does + * not interfere with the execution of the process in any way. + * + * @param pid_t The PID of the process to attach to. + * + * @return a handle to the process, or NULL if the process is not a valid + * Python process. +*/ +extern austin_handle_t +austin_attach(pid_t); + + +/** + * Detach from a Python process. + * + * This function detaches from a Python process, identified by its handle. +*/ +extern void +austin_detach(austin_handle_t); + + +/** + * Sample an attached Python process. + * + * This function samples the call stack of all the threads within the attached + * process. The passed callback function is called after every thread has been + * sampled. Frames are available to be retrieved with austin_pop_frame(). + * + * @param austin_handle_t the handle to the process to sample. + * @param austin_callback_t the callback function to call after sampling. + * + * @return 0 if the sampling was successful. +*/ +extern int +austin_sample(austin_handle_t, austin_callback_t); + + +/** + * Sample a single thread. + * + * This function samples the call stack of a single thread within the attached + * process. + * + * @param austin_handle_t the handle to the process to sample. + * @param pid_t the TID of the thread to sample. + * + * @return 0 if the sampling was successful. +*/ +extern int +austin_sample_thread(austin_handle_t, pid_t); + + +/** + * Pop a frame from the stack. + * + * This function pops a frame from the stack of the last sampled thread. This + * function should be called iteratively until it returns NULL, to retrieve all + * the frames in the stack. + * + * @return a valid reference to a frame structure, or NULL otherwise. +*/ +extern austin_frame_t * +austin_pop_frame(); + + +/** + * Read a single frame from the attached process. + * + * This function reads a single frame from an attached process, at the given + * remote memory location. This is useful if one is intercepting calls to, e.g. + * _PyEval_EvalFrameDefault and has access to the frame pointer (the second + * argument). This function can then be used to resolve the frame details. + * + * @param austin_handle_t the handle to the process. + * @param void * the remote memory location of the frame. + * + * @return a valid reference to a frame structure, or NULL otherwise. +*/ +extern austin_frame_t * +austin_read_frame(austin_handle_t, void *); + + +#ifdef __cplusplus +} +#endif diff --git a/scripts/build_arch.sh b/scripts/build_arch.sh index 070048a2..f3f5349c 100644 --- a/scripts/build_arch.sh +++ b/scripts/build_arch.sh @@ -12,7 +12,8 @@ apt-get -y install \ binutils-dev \ libiberty-dev \ musl-tools \ - zlib1g-dev + zlib1g-dev \ + libtool # Build Austin autoreconf --install @@ -28,12 +29,17 @@ pushd src cp austin /artifacts/austin cp austinp /artifacts/austinp - musl-gcc -O3 -Os -s -Wall -pthread *.c -o austin -D__MUSL__ + musl-gcc -O3 -Os -s -Wall -pthread *.c -I../include -o austin -D__MUSL__ tar -Jcf austin-$VERSION-musl-linux-$ARCH.tar.xz austin cp austin /artifacts/austin.musl + cp ../include/libaustin.h . + cp .libs/libaustin.{a,so.*.*} . + tar -Jcf libaustin-$VERSION-gnu-linux-$ARCH.tar.xz libaustin.* + mv austin-$VERSION-gnu-linux-$ARCH.tar.xz /artifacts mv austinp-$VERSION-gnu-linux-$ARCH.tar.xz /artifacts mv austin-$VERSION-musl-linux-$ARCH.tar.xz /artifacts + mv libaustin-$VERSION-gnu-linux-$ARCH.tar.xz /artifacts popd diff --git a/src/Makefile.am b/src/Makefile.am index 380bb12d..cf1da758 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -20,7 +20,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -AM_CFLAGS = -I$(srcdir) -Wall -Werror -Wno-unused-command-line-argument -pthread +AM_CFLAGS = -I$(srcdir) -I$(srcdir)/../include -Wall -Werror -Wno-unused-command-line-argument -pthread OPT_FLAGS = -O3 STRIP_FLAGS = -Os -s @@ -39,21 +39,24 @@ man_MANS = austin.1 bin_PROGRAMS = austin -# ---- Austin ---- +lib_LTLIBRARIES = libaustin.la -austin_CFLAGS = $(AM_CFLAGS) $(OPT_FLAGS) $(STRIP_FLAGS) $(COVERAGE_FLAGS) $(DEBUG_OPTS) -austin_SOURCES = \ +core_SOURCES = \ argparse.c \ austin.c \ cache.c \ error.c \ logging.c \ - stats.c \ platform.c \ py_proc_list.c \ py_proc.c \ py_thread.c +# ---- Austin ---- + +austin_CFLAGS = $(AM_CFLAGS) $(OPT_FLAGS) $(STRIP_FLAGS) $(COVERAGE_FLAGS) $(DEBUG_OPTS) +austin_SOURCES = $(core_SOURCES) \ + stats.c # ---- Austin P ---- @@ -64,3 +67,10 @@ austinp_SOURCES = $(austin_SOURCES) austinp_CFLAGS = $(austin_CFLAGS) @AUSTINP_CFLAGS@ austinp_LDADD = @AUSTINP_LDADD@ endif + + +# ---- libaustin ---- + +libaustin_la_CFLAGS = $(austin_CFLAGS) -DLIBAUSTIN +libaustin_la_SOURCES = $(core_SOURCES) +libaustin_la_LDFLAGS = -version-info 0:0:0 diff --git a/src/austin.c b/src/austin.c index a584243b..bd09d000 100644 --- a/src/austin.c +++ b/src/austin.c @@ -30,7 +30,6 @@ #include #include -#include "argparse.h" #include "austin.h" #include "error.h" #include "events.h" @@ -41,15 +40,82 @@ #include "msg.h" #include "platform.h" #include "python/abi.h" -#include "stats.h" #include "timing.h" #include "version.h" -#include "py_proc.h" #include "py_proc_list.h" #include "py_thread.h" +#if defined LIBAUSTIN + +#include "libaustin.h" + +int +austin_up() { + return py_thread_allocate(); +} + +void +austin_down() { + py_thread_free(); +} + +austin_handle_t +austin_attach(pid_t pid) { + py_proc_t * proc = py_proc_new(pid); + if (!isvalid(proc)) + return NULL; + + if (fail(py_proc__attach(proc, pid))) + goto failed; + + return (austin_handle_t) proc; + +failed: + py_proc__destroy(proc); + return NULL; +} + + +void +austin_detach(austin_handle_t proc) { + py_proc__destroy((py_proc_t *) proc); +} + +int +austin_sample(austin_handle_t proc, austin_callback_t callback) { + if (fail(py_proc__sample_cb((py_proc_t *) proc, callback))) + FAIL; + + SUCCESS; +} + +extern int +austin_sample_thread(austin_handle_t proc, pid_t tid) { + if (fail(py_proc__sample_thread((py_proc_t *) proc, tid))) + FAIL; + + SUCCESS; +} + +austin_frame_t * +austin_pop_frame() { + return (austin_frame_t *) py_thread_pop_frame(); +} + + +austin_frame_t * +austin_read_frame(austin_handle_t proc, void * frame_remote_address) { + return (austin_frame_t *) py_proc__read_frame((py_proc_t *) proc, frame_remote_address); +} + +#else /* LIBAUSTIN */ + +#include "argparse.h" +#include "stats.h" + + // ---- SIGNAL HANDLING ------------------------------------------------------- static int interrupt = FALSE; @@ -402,4 +468,6 @@ int main(int argc, char ** argv) { return retval; } /* main */ -#endif +#endif /* LIBAUSTIN */ + +#endif /* AUSTIN_C */ diff --git a/src/austin.h b/src/austin.h index d26722ce..ea415738 100644 --- a/src/austin.h +++ b/src/austin.h @@ -35,4 +35,4 @@ print(f'#define VERSION "{version()}"') #define VERSION "3.5.0" // [[[end]]] -#endif +#endif // AUSTIN_H diff --git a/src/code.h b/src/code.h index 76bbb8b8..eca8aa67 100644 --- a/src/code.h +++ b/src/code.h @@ -22,26 +22,20 @@ #pragma once - #include "py_string.h" +#define _code__get_filename(self, pref, py_v) \ + _string_from_raddr( \ + pref, *((void **)((void *)self + py_v->py_code.o_filename)), py_v) -#define _code__get_filename(self, pref, py_v) \ - _string_from_raddr( \ - pref, *((void **) ((void *) self + py_v->py_code.o_filename)), py_v \ - ) - -#define _code__get_name(self, pref, py_v) \ - _string_from_raddr( \ - pref, *((void **) ((void *) self + py_v->py_code.o_name)), py_v \ - ) +#define _code__get_name(self, pref, py_v) \ + _string_from_raddr( \ + pref, *((void **)((void *)self + py_v->py_code.o_name)), py_v) -#define _code__get_qualname(self, pref, py_v) \ - _string_from_raddr( \ - pref, *((void **) ((void *) self + py_v->py_code.o_qualname)), py_v \ - ) +#define _code__get_qualname(self, pref, py_v) \ + _string_from_raddr( \ + pref, *((void **)((void *)self + py_v->py_code.o_qualname)), py_v) -#define _code__get_lnotab(self, pref, len, py_v) \ - _bytes_from_raddr( \ - pref, *((void **) ((void *) self + py_v->py_code.o_lnotab)), len, py_v \ - ) +#define _code__get_lnotab(self, pref, len, py_v) \ + _bytes_from_raddr( \ + pref, *((void **)((void *)self + py_v->py_code.o_lnotab)), len, py_v) diff --git a/src/error.h b/src/error.h index 7fd69e92..251ea914 100644 --- a/src/error.h +++ b/src/error.h @@ -25,9 +25,6 @@ #include -#include "logging.h" - - // generic messages #define EOK 0 #define EMMAP 1 @@ -106,15 +103,6 @@ error_get_msg(error_t); const int is_fatal(error_t); - -/** - * Log the last error - */ -#define log_error() { \ - ( is_fatal(austin_errno) ? log_f(get_last_error()) : log_e(get_last_error()) ); \ -} - - /** * Set and log the given error. * @@ -122,7 +110,6 @@ is_fatal(error_t); */ #define set_error(x) { \ austin_errno = (x); \ - log_error(); \ } #endif // ERROR_H diff --git a/src/frame.h b/src/frame.h index c92ad212..e55ccb77 100644 --- a/src/frame.h +++ b/src/frame.h @@ -24,20 +24,10 @@ #include "cache.h" +#include "libaustin.h" #include "resources.h" -typedef struct { - key_dt key; - char * filename; - char * scope; - unsigned int line; - unsigned int line_end; - unsigned int column; - unsigned int column_end; -} frame_t; - - typedef struct { void * origin; void * code; @@ -45,6 +35,9 @@ typedef struct { } py_frame_t; +typedef austin_frame_t frame_t; + + // ---------------------------------------------------------------------------- static inline frame_t * frame_new( diff --git a/src/linux/py_proc.h b/src/linux/py_proc.h index 4f739c64..ab346cb9 100644 --- a/src/linux/py_proc.h +++ b/src/linux/py_proc.h @@ -686,6 +686,7 @@ _get_nspid(pid_t pid) { return nspid; } +#if !defined LIBAUSTIN // Support for CPU time on Linux. We need to retrieve the TID from the struct // pthread pointed to by the native thread ID stored by Python. We do not have @@ -733,4 +734,6 @@ _infer_tid_field_offset(py_thread_t * py_thread) { FAIL; } +#endif // !LIBAUSTIN + #endif diff --git a/src/logging.c b/src/logging.c index d465c771..c89318b1 100644 --- a/src/logging.c +++ b/src/logging.c @@ -95,6 +95,7 @@ logger_init(void) { #endif } +#if !defined LIBAUSTIN && !defined DEBUG void log_f(const char * fmt, ...) { @@ -144,6 +145,7 @@ log_m(const char * fmt, ...) { fflush(stderr); va_end(args); } +#endif // !defined LIBAUSTIN && !defined DEBUG #ifdef DEBUG void diff --git a/src/logging.h b/src/logging.h index 1c70aa37..c8270aad 100644 --- a/src/logging.h +++ b/src/logging.h @@ -29,6 +29,7 @@ #include "argparse.h" #include "austin.h" +#include "error.h" #define META_HEAD "# " @@ -70,6 +71,18 @@ void logger_init(void); +#if defined LIBAUSTIN && !defined DEBUG + +// Make logging a no-op when using non-debug libaustin + +#define log_f(f, args...) {} +#define log_e(f, args...) {} +#define log_w(f, args...) {} +#define log_i(f, args...) {} +#define log_m(f, args...) {} + +#else + /** * Log an entry at the various supported levels. */ @@ -88,6 +101,7 @@ log_i(const char *, ...); void log_m(const char *, ...); // metrics +#endif // LIBAUSTIN && !DEBUG /** * Log indirect error. @@ -126,4 +140,12 @@ logger_close(void); void log_meta_header(void); +/** + * Log the last error + */ +#define log_error() { \ + ( is_fatal(austin_errno) ? log_f(get_last_error()) : log_e(get_last_error()) ); \ +} + + #endif // LOGGING_H diff --git a/src/py_proc.c b/src/py_proc.c index 623815b6..1a3f4cb8 100644 --- a/src/py_proc.c +++ b/src/py_proc.c @@ -46,7 +46,6 @@ #include "hints.h" #include "logging.h" #include "mem.h" -#include "stack.h" #include "stats.h" #include "py_proc.h" @@ -413,6 +412,10 @@ _py_proc__check_interp_state(py_proc_t * self, void * raddr) { SUCCESS; } + #if defined LIBAUSTIN + // libaustin does not need the native thread ID information as the consumer of + // the library API would have it already. + #else // Try to determine the TID by reading the remote struct pthread structure. // We can then use this information to parse the appropriate procfs file and // determine the native thread's running state. @@ -429,7 +432,8 @@ _py_proc__check_interp_state(py_proc_t * self, void * raddr) { } log_d("tid field offset not ready"); FAIL; - #endif + #endif // LIBAUSTIN + #endif // PL_LINUX SUCCESS; } @@ -1379,10 +1383,12 @@ py_proc__log_version(py_proc_t * self, int parent) { } else { log_m(""); - if (patch == 0xFF) + if (patch == 0xFF) { log_m("🐍 \033[1mPython\033[0m version: \033[33;1m%d.%d.?\033[0m (from shared library)", major, minor); - else + } + else { log_m("🐍 \033[1mPython\033[0m version: \033[33;1m%d.%d.%d\033[0m", major, minor, patch); + } } } @@ -1422,3 +1428,118 @@ py_proc__destroy(py_proc_t * self) { free(self); } + + +#ifdef LIBAUSTIN +int +py_proc__sample_cb(py_proc_t * self, void (*cb)(pid_t, pid_t)) { + V_DESC(self->py_v); + + PyInterpreterState is; + if (fail(py_proc__get_type(self, self->is_raddr, is))) { + FAIL; + } + + void * tstate_head = V_FIELD(void *, is, py_is, o_tstate_head); + if (isvalid(tstate_head)) { + raddr_t raddr = { .pref = self->proc_ref, .addr = tstate_head }; + py_thread_t py_thread; + + if (fail(py_thread__fill_from_raddr(&py_thread, &raddr, self))) { + if (is_fatal(austin_errno)) { + FAIL; + } + SUCCESS; + } + + do { + py_thread__unwind_stack(&py_thread); + cb(self->pid, py_thread.tid); + } while (success(py_thread__next(&py_thread))); + } + + SUCCESS; +} + +int +py_proc__sample_thread(py_proc_t * self, pid_t tid) { + V_DESC(self->py_v); + + PyInterpreterState is; + if (fail(py_proc__get_type(self, self->is_raddr, is))) { + FAIL; + } + + void * tstate_head = V_FIELD(void *, is, py_is, o_tstate_head); + if (isvalid(tstate_head)) { + raddr_t raddr = { .pref = self->proc_ref, .addr = tstate_head }; + py_thread_t py_thread; + + if (fail(py_thread__fill_from_raddr(&py_thread, &raddr, self))) { + if (is_fatal(austin_errno)) { + FAIL; + } + SUCCESS; + } + + do { + if (py_thread.tid == tid) { + py_thread__unwind_stack(&py_thread); + SUCCESS; + } + } while (success(py_thread__next(&py_thread))); + } + + FAIL; +} + + +// ---------------------------------------------------------------------------- +frame_t * +py_proc__read_frame(py_proc_t * proc, void * remote_address) { + V_DESC(proc->py_v); + + lru_cache_t * cache = proc->frame_cache; + + void * code = NULL; + int lasti = 0; + + if (V_MIN(3, 11)) { + PyInterpreterFrame iframe; + + if (fail(copy_py(proc->proc_ref, remote_address, py_iframe, iframe))) { + log_ie("Cannot read remote PyInterpreterFrame"); + return NULL; + } + + code = V_FIELD(void *, iframe, py_iframe, o_code); + lasti = ((int)(V_FIELD(void *, iframe, py_iframe, o_prev_instr) - code)) - py_v->py_code.o_code; + } + else { + PyFrameObject frame_obj; + + if (fail(copy_py(proc->proc_ref, remote_address, py_frame, frame_obj))) { + log_ie("Cannot read remote PyFrameObject"); + return NULL; + } + + code = V_FIELD(void *, frame_obj, py_frame, o_code); + lasti = V_FIELD(int , frame_obj, py_frame, o_lasti); + } + + key_dt frame_key = py_frame_key(code, lasti); + frame_t * frame = lru_cache__maybe_hit(cache, frame_key); + if (!isvalid(frame)) { + frame = _frame_from_code_raddr(proc, code, lasti, py_v); + if (!isvalid(frame)) { + log_ie("Failed to get frame from code object"); + return NULL; + } + lru_cache__store(cache, frame_key, frame); + } + + return frame; +} + + +#endif /* LIBAUSTIN */ diff --git a/src/py_proc.h b/src/py_proc.h index 462ad174..2d943fa7 100644 --- a/src/py_proc.h +++ b/src/py_proc.h @@ -56,7 +56,7 @@ typedef struct { typedef struct _proc_extra_info proc_extra_info; // Forward declaration. -typedef struct { +typedef struct _py_proc_t { pid_t pid; proc_ref_t proc_ref; int child; @@ -233,4 +233,29 @@ py_proc__terminate(py_proc_t *); void py_proc__destroy(py_proc_t *); +#ifdef LIBAUSTIN +#include "frame.h" + +/** + * Sample the frame stack of each thread of the given Python process. + * + * @param py_proc_t * self. + * @param (void*) (*callback)(pid_t, pid_t) the callback function to cal + * when a thread stack is available. + + * @return 0 if the sampling succeeded; 1 otherwise. + */ +int +py_proc__sample_cb(py_proc_t *, void (*)(pid_t, pid_t)); + + +int +py_proc__sample_thread(py_proc_t *, pid_t); + + +frame_t * +py_proc__read_frame(py_proc_t *, void *); + +#endif + #endif // PY_PROC_H diff --git a/src/py_thread.c b/src/py_thread.c index a542ef16..c922dc08 100644 --- a/src/py_thread.c +++ b/src/py_thread.c @@ -803,6 +803,9 @@ py_thread__fill_from_raddr(py_thread_t * self, raddr_t * raddr, py_proc_t * proc } #endif } + #if defined LIBAUSTIN + // We do not need native thread ID information with libaustin + #else else if ( likely(proc->extra->pthread_tid_offset) && success(read_pthread_t(self->proc, (void *) self->tid @@ -822,6 +825,7 @@ py_thread__fill_from_raddr(py_thread_t * self, raddr_t * raddr, py_proc_t * proc } #endif } + #endif // LIBAUSTIN } #endif @@ -1116,3 +1120,34 @@ py_thread_free(void) { sfree(_kstacks); #endif } + + +#ifdef LIBAUSTIN +void +py_thread__unwind_stack(py_thread_t * self) { + + V_DESC(self->proc->py_v); + + if (isvalid(self->top_frame)) { + if (V_MIN(3, 11)) { + if (fail(_py_thread__unwind_cframe_stack(self))) { + } + } + else { + if (fail(_py_thread__unwind_frame_stack(self))) { + } + } + + if (fail(_py_thread__resolve_py_stack(self))) { + } + } +} /* py_thread__unwind_stack */ + + +frame_t * +py_thread_pop_frame() { + return stack_is_empty() ? NULL : stack_pop(); +} /* py_thread_pop_frame */ + + +#endif /* LIBAUSTIN */ diff --git a/src/py_thread.h b/src/py_thread.h index 1d2d5187..319c2529 100644 --- a/src/py_thread.h +++ b/src/py_thread.h @@ -27,6 +27,7 @@ #include #include "cache.h" +#include "frame.h" #include "mem.h" #include "py_proc.h" #include "stats.h" @@ -126,5 +127,13 @@ int py_thread__save_kernel_stack(py_thread_t *); #endif +#ifdef LIBAUSTIN +void +py_thread__unwind_stack(py_thread_t *); + +frame_t * +py_thread_pop_frame(); + +#endif // LIBAUSTIN #endif // PY_THREAD_H diff --git a/src/stack.h b/src/stack.h index 036dbf9d..57aa25c9 100644 --- a/src/stack.h +++ b/src/stack.h @@ -78,14 +78,14 @@ stack_deallocate(void) { if (!isvalid(_stack)) return; - free(_stack->base); - free(_stack->py_base); + sfree(_stack->base); + sfree(_stack->py_base); #ifdef NATIVE free(_stack->native_base); free(_stack->kernel_base); #endif - free(_stack); + sfree(_stack); } diff --git a/test/cunit/conftest.py b/test/cunit/conftest.py index 7e7d02d4..dec6ec61 100644 --- a/test/cunit/conftest.py +++ b/test/cunit/conftest.py @@ -11,6 +11,9 @@ import pytest +LIBS = SRC / ".libs" + + class SegmentationFault(Exception): pass @@ -55,12 +58,18 @@ def _(*_, **__): if result.returncode == -11: binary_name = Path(module).stem.replace("test_", "") - raise SegmentationFault(bt((SRC / binary_name).with_suffix(".so"))) + for prefix in (SRC, LIBS): + binary_path = (prefix / binary_name).with_suffix(".so") + if binary_path.exists(): + raise SegmentationFault(bt(binary_path)) + + print("No binary found for segfault analysis") raise CUnitTestFailure( f"\n{result.stdout.decode()}\n" + f"\n{result.stderr.decode()}\n" f"Process terminated with exit code {result.returncode} " - "(expected {exit_code})" + f"(expected {exit_code})" ) return _ diff --git a/test/cunit/libaustin.py b/test/cunit/libaustin.py new file mode 100644 index 00000000..aa6e1ee1 --- /dev/null +++ b/test/cunit/libaustin.py @@ -0,0 +1,52 @@ +import sys +import typing as t +from ctypes import CDLL +from ctypes import CFUNCTYPE +from ctypes import POINTER +from ctypes import Structure +from ctypes import c_char_p +from ctypes import c_int +from ctypes import c_ulong +from ctypes import c_void_p +from test.cunit import SRC +from types import ModuleType + + +la = CDLL(str(SRC / ".libs" / "libaustin.so")) + + +# ---- libaustin spec --------------------------------------------------------- + + +class Frame(Structure): + _fields_ = [ + ("key", c_ulong), + ("filename", c_char_p), + ("scope", c_char_p), + ("line", c_int), + ] + + +la.austin_callback = austin_callback = CFUNCTYPE(None, c_int, c_int) + +la.austin_up.restype = c_int + +la.austin_attach.argtypes = [c_int] +la.austin_attach.restype = c_void_p + +la.austin_detach.argtypes = [c_void_p] + +la.austin_sample.argtypes = [c_void_p, austin_callback] +la.austin_sample.restype = c_int + +la.austin_sample_thread.argtypes = [c_void_p, c_int] +la.austin_sample_thread.restype = c_int + +la.austin_pop_frame.restype = POINTER(Frame) + +la.austin_read_frame.argtypes = [c_void_p, c_void_p] +la.austin_read_frame.restype = POINTER(Frame) + +# ----------------------------------------------------------------------------- + +sys.modules[__name__] = t.cast(ModuleType, la) diff --git a/test/cunit/test_libaustin.py b/test/cunit/test_libaustin.py new file mode 100644 index 00000000..9269670c --- /dev/null +++ b/test/cunit/test_libaustin.py @@ -0,0 +1,65 @@ +import os +import sys +import test.cunit.libaustin as la +from test.utils import requires_sudo + +import pytest + + +@pytest.fixture +def handle(): + la.austin_up() + + pid = os.getpid() + handle = la.austin_attach(pid) + + assert handle + + yield handle + + la.austin_detach(handle) + + la.austin_down() + + +def test_libaustin_up_down(): + for _ in range(10): + la.austin_up() + la.austin_down() + + +@requires_sudo +def test_libaustin_sample(handle): + last_frame = None + seen_pid = None + + @la.austin_callback + def cb(pid, tid): + nonlocal last_frame, seen_pid + + seen_pid = pid + while True: + frame = la.austin_pop_frame() + if not frame: + return + + last_frame = frame.contents + + la.austin_sample(handle, cb) + + frame = sys._getframe() + + assert seen_pid == os.getpid() + + assert last_frame.filename.decode() == frame.f_code.co_filename + assert last_frame.scope.decode() == frame.f_code.co_name + + +@requires_sudo +def test_libaustin_read_frame(handle): + frame = sys._getframe() + + austin_frame = la.austin_read_frame(handle, id(frame)).contents + + assert austin_frame.filename.decode() == frame.f_code.co_filename + assert austin_frame.scope.decode() == frame.f_code.co_name diff --git a/test/libaustin/frame.c b/test/libaustin/frame.c new file mode 100644 index 00000000..44d94fea --- /dev/null +++ b/test/libaustin/frame.c @@ -0,0 +1,41 @@ +#include +#include +#include + +#include "libaustin.h" + +int main(int argc, char **argv) +{ + if (argc != 3) + { + fprintf(stderr, "Usage: frame \n"); + return -1; + } + + pid_t pid = atoi(argv[1]); + void *frame_addr = (void *)atoll(argv[2]); + + if (austin_up() != 0) + { + perror("austin_up() failed"); + return -1; + } + + austin_handle_t proc_handle = austin_attach(pid); + + if (proc_handle == NULL) + { + perror("Failed to attach to process"); + return -1; + } + + austin_frame_t *frame = austin_read_frame(proc_handle, frame_addr); + + printf("%s (%s:%d)\n", frame->scope, frame->filename, frame->line); + + austin_detach(proc_handle); + + austin_down(); + + return 0; +} diff --git a/test/libaustin/where.c b/test/libaustin/where.c new file mode 100644 index 00000000..13c56c7a --- /dev/null +++ b/test/libaustin/where.c @@ -0,0 +1,43 @@ +#include +#include +#include + +#include "libaustin.h" + +void unwind(pid_t pid, pid_t tid) { + printf("\n\npid: %d, tid: %d\n\n", pid, tid); + + austin_frame_t *frame = NULL; + while ((frame = austin_pop_frame()) != NULL) { + printf(" %s (%s:%d)\n", frame->scope, frame->filename, frame->line); + } +} + +int main(int argc, char **argv) { + if (argc != 2) { + fprintf(stderr, "Usage: test_libaustin \n"); + return -1; + } + + pid_t pid = atoi(argv[1]); + + if (austin_up() != 0) { + perror("austin_up() failed"); + return -1; + } + + austin_handle_t proc_handle = austin_attach(pid); + + if (proc_handle == NULL) { + perror("Failed to attach to process"); + return -1; + } + + austin_sample(proc_handle, unwind); + + austin_detach(proc_handle); + + austin_down(); + + return 0; +}