[{"data":1,"prerenderedAt":1303},["ShallowReactive",2],{"$B3SvcJdz-H":3,"pages-links":23,"posts-links":29,"posts-search":58,"blog-tracing-signals-in-linux":129,"decks-for-tracing-signals-in-linux":1298,"meta":1299},{"id":4,"title":5,"author":6,"description":14,"extension":15,"meta":16,"repository":17,"stem":21,"__hash__":22},"meta\u002Fmeta.yml","bmo.dev",{"name":7,"github":8,"gitlab":9,"email":10,"linkedin":11,"mastodon":12,"bio":13},"William Little","https:\u002F\u002Fgithub.com\u002Fbmodotdev\u002F","https:\u002F\u002Fgitlab.com\u002Fbmodotdev\u002F","L29hqTSwqROvoJ8hMTI2","https:\u002F\u002Fwww.linkedin.com\u002Fin\u002Fwilliam-e-little-jr\u002F","https:\u002F\u002Fdefcon.social\u002F@bmodotdev","I am a Full Stack Software & DevOps Engineer who loves building and experimenting with Software, Linux, & electronics.\n","A personal blog where I share my thoughts, projects, and anything I might find interesting or helpful.\n","yml",{},{"url":18,"git":19,"branch":20},"https:\u002F\u002Fgitlab.com\u002Fbmodotdev\u002Fbmo.dev\u002F","git@gitlab.com:bmodotdev\u002Fbmo.dev.git","main","meta","MPtcc2KHfBCBy4V4SDhZFIvjyZ_Ba4e-yDnx_Ok_KOs",[24],{"path":25,"extension":26,"title":27,"id":28},"\u002Fabout","md","About","pages\u002Fabout.md",[30,34,38,42,46,50,54],{"path":31,"extension":26,"title":32,"id":33},"\u002Fblog\u002Fconventional-commit-caveats","Conventional Commit Caveats","blog\u002Fblog\u002Fconventional-commit-caveats.md",{"path":35,"extension":26,"title":36,"id":37},"\u002Fblog\u002Fdodging-a-critical-cve-with-zta","Dodging a Critical CVE with ZTA","blog\u002Fblog\u002Fdodging-a-critical-cve-with-zta.md",{"path":39,"extension":26,"title":40,"id":41},"\u002Fblog\u002Flog4perl-mixed-screen-appender","Mixed Output Streams with Log4perl Screen Appender","blog\u002Fblog\u002Flog4perl-mixed-screen-appender.md",{"path":43,"extension":26,"title":44,"id":45},"\u002Fblog\u002Fmake-version","make version","blog\u002Fblog\u002Fmake-version.md",{"path":47,"extension":26,"title":48,"id":49},"\u002Fblog\u002Fpadding-oracle-attacks-rubiks-cow","Padding Oracle Attacks & Rubik’s Cow","blog\u002Fblog\u002Fpadding-oracle-attacks-rubiks-cow.md",{"path":51,"extension":26,"title":52,"id":53},"\u002Fblog\u002Ftracing-signals-in-linux","Tracing Signals in Linux","blog\u002Fblog\u002Ftracing-signals-in-linux.md",{"path":55,"extension":26,"title":56,"id":57},"\u002Fblog\u002Fwhy-are-go-modules-in-my-nodes-modules","Why are Go modules in my node_modules?","blog\u002Fblog\u002Fwhy-are-go-modules-in-my-nodes-modules.md",[59,69,81,90,100,108,117],{"path":55,"title":56,"description":60,"tags":61,"thumbnail":64,"created":68},"Let’s dissect this developer pattern that leads to arbitrary code execution across languages.",[62,63],"go","js",{"sm":65,"md":66,"lg":67},"\u002Fassets\u002Fdavid-mitchell-great-british-baking-show_sm.webp","\u002Fassets\u002Fdavid-mitchell-great-british-baking-show_md.webp","\u002Fassets\u002Fdavid-mitchell-great-british-baking-show_lg.webp","2026-04-07",{"path":35,"title":36,"description":70,"tags":71,"thumbnail":76,"created":80},"Let’s discuss tenets of Zero Trust Architecture, Go, gRPC, and why everyone dodged CVE-2026-33186, right?",[72,73,62,74,75],"awk","cve","grpc","zta",{"sm":77,"md":78,"lg":79},"\u002Fassets\u002Fdodging-the-flyer_sm.webp","\u002Fassets\u002Fdodging-the-flyer_md.webp","\u002Fassets\u002Fdodging-the-flyer_lg.webp","2026-04-02",{"path":47,"title":48,"description":82,"tags":83,"thumbnail":85,"created":89},"In this thought experiment, you’ll learn about cryptographic primitives, combinitorical group theory, and along the way, gain a deeper appreciation for 1999’s film, The Matrix. Oh yeah, and a Cow!",[84],"cryptography",{"sm":86,"md":87,"lg":88},"\u002Fassets\u002Fbournemouth_sm.webp","\u002Fassets\u002Fbournemouth_md.webp","\u002Fassets\u002Fbournemouth_lg.webp","2026-03-06",{"path":43,"title":44,"description":91,"tags":92,"thumbnail":95,"created":99},"Here is how I went about implementing a simple versioning system for my Go project using Conventional Commmits, awk, and make.",[72,93,62,94],"git","semver",{"sm":96,"md":97,"lg":98},"\u002Fassets\u002Flego-pieces_sm.webp","\u002Fassets\u002Flego-pieces_md.webp","\u002Fassets\u002Flego-pieces_lg.webp","2025-11-11",{"path":31,"title":32,"description":101,"tags":102,"thumbnail":103,"created":107},"The Conventional Commmits spec aims to bring structure to commit messages so that we write succint yet meaningful commits. That sounds great, but let’s examine some potential drawbacks.",[93,94],{"sm":104,"md":105,"lg":106},"\u002Fassets\u002Fgit-commit_sm.webp","\u002Fassets\u002Fgit-commit_md.webp","\u002Fassets\u002Fgit-commit_lg.webp","2025-11-07",{"path":39,"title":40,"description":109,"tags":110,"thumbnail":112,"created":116},"Log4perl provides a powerful logging framework, but how can we dynamically and meaninfully use stdout and stderr?",[111],"perl",{"sm":113,"md":114,"lg":115},"\u002Fassets\u002Fperl-latte-onion_100x100.webp","\u002Fassets\u002Fperl-latte-onion_250x250.webp","\u002Fassets\u002Fperl-latte-onion_600x600.webp","2022-02-14",{"path":51,"title":52,"description":118,"tags":119,"thumbnail":124,"created":128},"Signals, or software interrupts, are used to communicate with a process. They could be used to kill a process, restart, or signal a segmentation fault, but how can we know where that signal came from?",[120,121,122,123],"signals","systemtap","strace","linux",{"sm":125,"md":126,"lg":127},"\u002Fassets\u002Ftraffic-light_sm.webp","\u002Fassets\u002Ftraffic-light_md.webp","\u002Fassets\u002Ftraffic-light_lg.webp","2022-02-13",{"id":53,"title":52,"body":130,"created":128,"description":118,"extension":26,"meta":1291,"navigation":298,"path":51,"seo":1292,"stem":1293,"tags":1294,"thumbnail":1295,"updated":1296,"__hash__":1297},{"type":131,"value":132,"toc":1283},"minimark",[133,163,171,174,179,187,190,192,202,217,222,233,249,443,469,472,474,483,492,502,570,575,578,685,688,986,999,1179,1190,1235,1238,1263,1267,1276,1279],[134,135,136,144],"with-caption",{},[137,138,139],"p",{},[140,141],"img",{"alt":142,"src":127,"target":143,"title":142},"“A couple of red traffic lights against a blue sky” by Horia Varlan","_blank",[145,146,148],"template",{"v-slot:caption":147},"",[137,149,150,157,158],{},[151,152,156],"a",{"href":153,"rel":154,"target":143},"https:\u002F\u002Fwww.flickr.com\u002Fphotos\u002F10361931@N06\u002F4747872021",[155],"nofollow","“A couple of red traffic lights against a blue sky”"," by ",[151,159,162],{"href":160,"rel":161,"target":143},"https:\u002F\u002Fwww.flickr.com\u002Fphotos\u002F10361931@N06",[155],"Horia Varlan",[137,164,165,166,170],{},"Many programs in Linux use ",[151,167,120],{"href":168,"rel":169,"target":143},"https:\u002F\u002Fwww.man7.org\u002Flinux\u002Fman-pages\u002Fman7\u002Fsignal.7.html",[155]," to communicate with other processes.\nSignals are commonly used by init daemons, like init.d or systemd, to control services;\nfor example stopping or restarting a service gracefully.\nSignals are also used by multiprocess applications to control their children processes.",[137,172,173],{},"As a system administrator, how do you know what signals are sent to an application?\nA good application will log appropriately or provide some debug mode, but what if the application does not provide this?",[175,176,178],"h2",{"id":177},"the-mystery-of-the-restarting-ftp-server","The mystery of the restarting FTP server",[137,180,181,182,186],{},"In my former role as a ",[183,184,185],"em",{},"Senior Technical Analyst",", I was the last line of defense in solving difficult tickets we received from our End-users.\nOne particularly frustrating ticket was the mystery of the restarting FTP server.\nThe End-user reported that the FTP server was continuously restarting at seemingly random intervals.\nAs a result, their monitoring service was constantly complaining.",[137,188,189],{},"Checking the logs, we could see it was a normal graceful restart.\nTypically, this is seen when software is updated or perhaps when a new configuration is applied.\nThe restarts didn’t quite line up when cross-referencing other logs or checking file timestamps of the configurations.\nEven more frustrating and puzzling, it seemed to always happen just after our technicians gave up on investigating or monitoring.",[175,191,122],{"id":122},[137,193,194,195,201],{},"A goto tool in any system administrator’s toolbox is ",[151,196,199],{"href":197,"rel":198,"target":143},"https:\u002F\u002Fstrace.io\u002F",[155],[183,200,122],{},":",[203,204,206,209],"block-quote",{"link":205},"https:\u002F\u002Fwww.man7.org\u002Flinux\u002Fman-pages\u002Fman1\u002Fstrace.1.html",[137,207,208],{},"In the simplest case strace runs the specified command until it\nexits.  It intercepts and records the system calls which are\ncalled by a process and the signals which are received by a\nprocess.  The name of each system call, its arguments and its\nreturn value are printed on standard error or to the file\nspecified with the -o option.",[145,210,211],{"v-slot:cite":147},[137,212,213],{},[214,215,216],"code",{},"man 1 strace",[137,218,219,221],{},[183,220,122],{}," is such a powerful tool that the output or the number of options can be overwhelming to beginners; however,\nthe more you use it, the more you will begin to understand and recognize what is happening at a system level.\nI encourage you to just jump in!\nYou will find it archaic and perhaps cumbersome at first, but you will learn so much about how applications work\nand how they interact with the operating system.",[203,223,224,227],{"link":205},[137,225,226],{},"Students, hackers and the overly-curious will find that a\ngreat deal can be learned about a system and its system\ncalls by tracing even ordinary programs.",[145,228,229],{"v-slot:cite":147},[137,230,231],{},[214,232,216],{},[137,234,235,236,238,239,242,243,245,246,248],{},"The first step for an in depth look at what is happening to the FTP server is tracing the system calls.\nLet’s take a very basic example of how we can use ",[183,237,122],{}," to watch a process and why it might have exited.\nIn this example, the process we are inspecting is the ",[183,240,241],{},"sleep"," command, which we will throw into the background.\nWe will also throw our ",[183,244,122],{}," process into the background so we can interact with the original ",[183,247,241],{}," process.",[250,251,255],"pre",{"className":252,"code":253,"language":254,"meta":147,"style":147},"language-bash shiki shiki-themes houston","# Launch our example process\n🞂 sleep 300 &\n[3] 106834\n\n# Trace the process, to reduce noise, only trace signals\n🞂 sudo strace --attach 106834 -e trace=signal &\n[4] 107084\nstrace: Process 106834 attached\n\n# Send a signal to our example process\n🞂 kill -SIGTERM 106834\n--- SIGTERM {si_signo=SIGTERM, si_code=SI_USER, si_pid=74326, si_uid=1000} ---\n+++ killed by SIGTERM +++\n\n🞂 echo $$\n74326\n","bash",[214,256,257,266,284,293,300,306,331,340,354,359,365,379,403,420,425,437],{"__ignoreMap":147},[258,259,262],"span",{"class":260,"line":261},"line",1,[258,263,265],{"class":264},"sy89t","# Launch our example process\n",[258,267,269,273,277,280],{"class":260,"line":268},2,[258,270,272],{"class":271},"stHQG","🞂",[258,274,276],{"class":275},"se2J0"," sleep",[258,278,279],{"class":275}," 300",[258,281,283],{"class":282},"sLo_3"," &\n",[258,285,287,290],{"class":260,"line":286},3,[258,288,289],{"class":282},"[3] ",[258,291,292],{"class":271},"106834\n",[258,294,296],{"class":260,"line":295},4,[258,297,299],{"emptyLinePlaceholder":298},true,"\n",[258,301,303],{"class":260,"line":302},5,[258,304,305],{"class":264},"# Trace the process, to reduce noise, only trace signals\n",[258,307,309,311,314,317,320,323,326,329],{"class":260,"line":308},6,[258,310,272],{"class":271},[258,312,313],{"class":275}," sudo",[258,315,316],{"class":275}," strace",[258,318,319],{"class":275}," --attach",[258,321,322],{"class":275}," 106834",[258,324,325],{"class":275}," -e",[258,327,328],{"class":275}," trace=signal",[258,330,283],{"class":282},[258,332,334,337],{"class":260,"line":333},7,[258,335,336],{"class":282},"[4] ",[258,338,339],{"class":271},"107084\n",[258,341,343,346,349,351],{"class":260,"line":342},8,[258,344,345],{"class":271},"strace:",[258,347,348],{"class":275}," Process",[258,350,322],{"class":275},[258,352,353],{"class":275}," attached\n",[258,355,357],{"class":260,"line":356},9,[258,358,299],{"emptyLinePlaceholder":298},[258,360,362],{"class":260,"line":361},10,[258,363,364],{"class":264},"# Send a signal to our example process\n",[258,366,368,370,373,376],{"class":260,"line":367},11,[258,369,272],{"class":271},[258,371,372],{"class":275}," kill",[258,374,375],{"class":275}," -SIGTERM",[258,377,378],{"class":275}," 106834\n",[258,380,382,385,388,391,394,397,400],{"class":260,"line":381},12,[258,383,384],{"class":271},"---",[258,386,387],{"class":275}," SIGTERM",[258,389,390],{"class":275}," {si_signo=SIGTERM,",[258,392,393],{"class":275}," si_code=SI_USER,",[258,395,396],{"class":275}," si_pid=74326,",[258,398,399],{"class":275}," si_uid=1000}",[258,401,402],{"class":275}," ---\n",[258,404,406,409,412,415,417],{"class":260,"line":405},13,[258,407,408],{"class":271},"+++",[258,410,411],{"class":275}," killed",[258,413,414],{"class":275}," by",[258,416,387],{"class":275},[258,418,419],{"class":275}," +++\n",[258,421,423],{"class":260,"line":422},14,[258,424,299],{"emptyLinePlaceholder":298},[258,426,428,430,433],{"class":260,"line":427},15,[258,429,272],{"class":271},[258,431,432],{"class":275}," echo",[258,434,436],{"class":435},"s8HWQ"," $$\n",[258,438,440],{"class":260,"line":439},16,[258,441,442],{"class":271},"74326\n",[137,444,445,446,448,449,451,452,455,456,458,459,462,463,458,466,468],{},"We can see from the output of ",[183,447,122],{}," that our ",[183,450,241],{}," process received a SIGTERM, then promptly exited.\nWe can also see additional information, ",[183,453,454],{},"siginfo",", in the braces like the PID and UID that sent the signal.\nIn ",[183,457,254],{}," ",[214,460,461],{},"$$"," produces the PID of the current shell, which confirms the correlation in the ",[214,464,465],{},"si_pid",[183,467,454],{}," field.",[137,470,471],{},"So that makes identifying who sent this rather easy, but what if that PID is no longer running?\nIn the case of our FTP server, the process had always already exited before we could investigate.\nHow do we track this further?",[175,473,121],{"id":121},[137,475,476,482],{},[151,477,480],{"href":478,"rel":479,"target":143},"https:\u002F\u002Fsourceware.org\u002Fsystemtap\u002Fwiki",[155],[183,481,121],{}," allows you to write code and attach it directly to the kernel as a module.",[484,485,486,489],"blockquote",{},[137,487,488],{},"SystemTap provides free software (GPL) infrastructure to simplify the gathering of information about the running Linux system. This assists diagnosis of a performance or functional problem. SystemTap eliminates the need for the developer to go through the tedious and disruptive instrument, recompile, install, and reboot sequence that may be otherwise required to collect data.",[137,490,491],{},"SystemTap provides a simple command line interface and scripting language for writing instrumentation for a live running system. We are publishing samples, as well as enlarging the internal \"tapset\" script library to aid reuse and abstraction.",[137,493,494,495,497,498,501],{},"Before we get into ",[183,496,121],{},", let’s setup a test environment using ",[183,499,500],{},"vagrant"," and Ubuntu 20.04:",[250,503,505],{"className":252,"code":504,"language":254,"meta":147,"style":147},"🞂 mkdir systemtap && cd $_\n🞂 vagrant init generic\u002Fubuntu2004\n🞂 vagrant up\n🞂 vagrant ssh\n\n🞂 sudo su -\n",[214,506,507,523,536,545,554,558],{"__ignoreMap":147},[258,508,509,511,514,517,520],{"class":260,"line":261},[258,510,272],{"class":271},[258,512,513],{"class":275}," mkdir",[258,515,516],{"class":275}," systemtap",[258,518,519],{"class":282}," && cd ",[258,521,522],{"class":435},"$_\n",[258,524,525,527,530,533],{"class":260,"line":268},[258,526,272],{"class":271},[258,528,529],{"class":275}," vagrant",[258,531,532],{"class":275}," init",[258,534,535],{"class":275}," generic\u002Fubuntu2004\n",[258,537,538,540,542],{"class":260,"line":286},[258,539,272],{"class":271},[258,541,529],{"class":275},[258,543,544],{"class":275}," up\n",[258,546,547,549,551],{"class":260,"line":295},[258,548,272],{"class":271},[258,550,529],{"class":275},[258,552,553],{"class":275}," ssh\n",[258,555,556],{"class":260,"line":302},[258,557,299],{"emptyLinePlaceholder":298},[258,559,560,562,564,567],{"class":260,"line":308},[258,561,272],{"class":271},[258,563,313],{"class":275},[258,565,566],{"class":275}," su",[258,568,569],{"class":275}," -\n",[571,572,574],"h3",{"id":573},"installing-systemtap","Installing systemtap",[137,576,577],{},"We can now install systemtap, kernel debug symbols, and gcc in our test server:",[250,579,581],{"className":252,"code":580,"language":254,"meta":147,"style":147},"🞂 cat \u003C\u003CEOF > \u002Fetc\u002Fapt\u002Fsources.list.d\u002Fddebs.list\ndeb http:\u002F\u002Fddebs.ubuntu.com $(lsb_release -cs) main restricted universe multiverse\ndeb http:\u002F\u002Fddebs.ubuntu.com $(lsb_release -cs)-updates main restricted universe multiverse\ndeb http:\u002F\u002Fddebs.ubuntu.com $(lsb_release -cs)-proposed main restricted universe multiverse\nEOF\n\n🞂 apt install ubuntu-dbgsym-keyring\n🞂 apt-get update\n🞂 apt install linux-image-\"$(uname -r)\"-dbgsym systemtap gcc\n",[214,582,583,602,613,622,631,636,640,653,663],{"__ignoreMap":147},[258,584,585,587,590,593,596,599],{"class":260,"line":261},[258,586,272],{"class":271},[258,588,589],{"class":275}," cat",[258,591,592],{"class":282}," \u003C\u003C",[258,594,595],{"class":282},"EOF",[258,597,598],{"class":282}," > ",[258,600,601],{"class":275},"\u002Fetc\u002Fapt\u002Fsources.list.d\u002Fddebs.list\n",[258,603,604,607,610],{"class":260,"line":268},[258,605,606],{"class":275},"deb http:\u002F\u002Fddebs.ubuntu.com $(",[258,608,609],{"class":271},"lsb_release",[258,611,612],{"class":275}," -cs) main restricted universe multiverse\n",[258,614,615,617,619],{"class":260,"line":286},[258,616,606],{"class":275},[258,618,609],{"class":271},[258,620,621],{"class":275}," -cs)-updates main restricted universe multiverse\n",[258,623,624,626,628],{"class":260,"line":295},[258,625,606],{"class":275},[258,627,609],{"class":271},[258,629,630],{"class":275}," -cs)-proposed main restricted universe multiverse\n",[258,632,633],{"class":260,"line":302},[258,634,635],{"class":282},"EOF\n",[258,637,638],{"class":260,"line":308},[258,639,299],{"emptyLinePlaceholder":298},[258,641,642,644,647,650],{"class":260,"line":333},[258,643,272],{"class":271},[258,645,646],{"class":275}," apt",[258,648,649],{"class":275}," install",[258,651,652],{"class":275}," ubuntu-dbgsym-keyring\n",[258,654,655,657,660],{"class":260,"line":342},[258,656,272],{"class":271},[258,658,659],{"class":275}," apt-get",[258,661,662],{"class":275}," update\n",[258,664,665,667,669,671,674,677,680,682],{"class":260,"line":356},[258,666,272],{"class":271},[258,668,646],{"class":275},[258,670,649],{"class":275},[258,672,673],{"class":275}," linux-image-\"$(",[258,675,676],{"class":271},"uname",[258,678,679],{"class":275}," -r)\"-dbgsym",[258,681,516],{"class":275},[258,683,684],{"class":275}," gcc\n",[137,686,687],{},"Let’s do a simple test to ensure we have basic systemtap functionality with the command below:",[250,689,691],{"className":252,"code":690,"language":254,"meta":147,"style":147},"🞂 stap -ve 'probe begin { log(\"Hello World!\") }' &\n[1] 35145\nPass 1: parsed user script and 477 library scripts using 105496virt\u002F90832res\u002F7288shr\u002F83648data kb, in 120usr\u002F20sys\u002F135real ms.\nPass 2: analyzed script: 1 probe, 1 function, 0 embeds, 0 globals using 106948virt\u002F92656res\u002F7544shr\u002F85100data kb, in 0usr\u002F10sys\u002F6real ms.\nPass 3: translated to C into \"\u002Ftmp\u002FstapShbQhJ\u002Fstap_d5e149da6109a20d77c20a3618c3a15c_944_src.c\" using 106948virt\u002F92656res\u002F7544shr\u002F85100data kb, in 0usr\u002F0sys\u002F0real ms.\nPass 4: compiled C into \"stap_d5e149da6109a20d77c20a3618c3a15c_944.ko\" in 1260usr\u002F120sys\u002F1374real ms.\nPass 5: starting run.\nHello World!\n\n🞂 lsmod | grep stap\nstap_d5e149da6109a20d77c20a3618c3a1_35684   192512  2\n\n🞂 kill %1\nPass 5: run completed in 10usr\u002F10sys\u002F14417real ms.\n[1]+  Done                    stap -ve 'probe begin { log(\"Hello World!\") }'\n\n🞂 lsmod | grep stap\n🞂\n",[214,692,693,708,716,763,812,847,871,884,892,896,912,923,927,936,955,963,967,980],{"__ignoreMap":147},[258,694,695,697,700,703,706],{"class":260,"line":261},[258,696,272],{"class":271},[258,698,699],{"class":275}," stap",[258,701,702],{"class":275}," -ve",[258,704,705],{"class":275}," 'probe begin { log(\"Hello World!\") }'",[258,707,283],{"class":282},[258,709,710,713],{"class":260,"line":268},[258,711,712],{"class":282},"[1] ",[258,714,715],{"class":271},"35145\n",[258,717,718,721,724,727,730,733,736,739,742,745,748,751,754,757,760],{"class":260,"line":286},[258,719,720],{"class":271},"Pass",[258,722,723],{"class":275}," 1:",[258,725,726],{"class":275}," parsed",[258,728,729],{"class":275}," user",[258,731,732],{"class":275}," script",[258,734,735],{"class":275}," and",[258,737,738],{"class":275}," 477",[258,740,741],{"class":275}," library",[258,743,744],{"class":275}," scripts",[258,746,747],{"class":275}," using",[258,749,750],{"class":275}," 105496virt\u002F90832res\u002F7288shr\u002F83648data",[258,752,753],{"class":275}," kb,",[258,755,756],{"class":275}," in",[258,758,759],{"class":275}," 120usr\u002F20sys\u002F135real",[258,761,762],{"class":275}," ms.\n",[258,764,765,767,770,773,776,779,782,784,787,790,793,795,798,800,803,805,807,810],{"class":260,"line":295},[258,766,720],{"class":271},[258,768,769],{"class":275}," 2:",[258,771,772],{"class":275}," analyzed",[258,774,775],{"class":275}," script:",[258,777,778],{"class":275}," 1",[258,780,781],{"class":275}," probe,",[258,783,778],{"class":275},[258,785,786],{"class":275}," function,",[258,788,789],{"class":275}," 0",[258,791,792],{"class":275}," embeds,",[258,794,789],{"class":275},[258,796,797],{"class":275}," globals",[258,799,747],{"class":275},[258,801,802],{"class":275}," 106948virt\u002F92656res\u002F7544shr\u002F85100data",[258,804,753],{"class":275},[258,806,756],{"class":275},[258,808,809],{"class":275}," 0usr\u002F10sys\u002F6real",[258,811,762],{"class":275},[258,813,814,816,819,822,825,828,831,834,836,838,840,842,845],{"class":260,"line":302},[258,815,720],{"class":271},[258,817,818],{"class":275}," 3:",[258,820,821],{"class":275}," translated",[258,823,824],{"class":275}," to",[258,826,827],{"class":275}," C",[258,829,830],{"class":275}," into",[258,832,833],{"class":275}," \"\u002Ftmp\u002FstapShbQhJ\u002Fstap_d5e149da6109a20d77c20a3618c3a15c_944_src.c\"",[258,835,747],{"class":275},[258,837,802],{"class":275},[258,839,753],{"class":275},[258,841,756],{"class":275},[258,843,844],{"class":275}," 0usr\u002F0sys\u002F0real",[258,846,762],{"class":275},[258,848,849,851,854,857,859,861,864,866,869],{"class":260,"line":308},[258,850,720],{"class":271},[258,852,853],{"class":275}," 4:",[258,855,856],{"class":275}," compiled",[258,858,827],{"class":275},[258,860,830],{"class":275},[258,862,863],{"class":275}," \"stap_d5e149da6109a20d77c20a3618c3a15c_944.ko\"",[258,865,756],{"class":275},[258,867,868],{"class":275}," 1260usr\u002F120sys\u002F1374real",[258,870,762],{"class":275},[258,872,873,875,878,881],{"class":260,"line":333},[258,874,720],{"class":271},[258,876,877],{"class":275}," 5:",[258,879,880],{"class":275}," starting",[258,882,883],{"class":275}," run.\n",[258,885,886,889],{"class":260,"line":342},[258,887,888],{"class":271},"Hello",[258,890,891],{"class":275}," World!\n",[258,893,894],{"class":260,"line":356},[258,895,299],{"emptyLinePlaceholder":298},[258,897,898,900,903,906,909],{"class":260,"line":361},[258,899,272],{"class":271},[258,901,902],{"class":275}," lsmod",[258,904,905],{"class":282}," | ",[258,907,908],{"class":271},"grep",[258,910,911],{"class":275}," stap\n",[258,913,914,917,920],{"class":260,"line":367},[258,915,916],{"class":271},"stap_d5e149da6109a20d77c20a3618c3a1_35684",[258,918,919],{"class":275},"   192512",[258,921,922],{"class":275},"  2\n",[258,924,925],{"class":260,"line":381},[258,926,299],{"emptyLinePlaceholder":298},[258,928,929,931,933],{"class":260,"line":405},[258,930,272],{"class":271},[258,932,372],{"class":275},[258,934,935],{"class":275}," %1\n",[258,937,938,940,942,945,948,950,953],{"class":260,"line":422},[258,939,720],{"class":271},[258,941,877],{"class":275},[258,943,944],{"class":275}," run",[258,946,947],{"class":275}," completed",[258,949,756],{"class":275},[258,951,952],{"class":275}," 10usr\u002F10sys\u002F14417real",[258,954,762],{"class":275},[258,956,957,960],{"class":260,"line":427},[258,958,959],{"class":282},"[1]+  Done                    stap -ve ",[258,961,962],{"class":275},"'probe begin { log(\"Hello World!\") }'\n",[258,964,965],{"class":260,"line":439},[258,966,299],{"emptyLinePlaceholder":298},[258,968,970,972,974,976,978],{"class":260,"line":969},17,[258,971,272],{"class":271},[258,973,902],{"class":275},[258,975,905],{"class":282},[258,977,908],{"class":271},[258,979,911],{"class":275},[258,981,983],{"class":260,"line":982},18,[258,984,985],{"class":271},"🞂\n",[137,987,988,989,992,993,998],{},"Pairing the ",[214,990,991],{},"signal.send"," probe with ",[151,994,997],{"href":995,"rel":996,"target":143},"https:\u002F\u002Flinux.die.net\u002Fman\u002F5\u002Fstapfuncs",[155],"stapfuncs","\nto walk the parent process until we get to 0 or 1, we can track which process sent a signal, and where it came from.\nNow, even when a process has already exited, we can understand how it may have started, something we could not do earlier with strace alone.",[250,1000,1004],{"className":1001,"code":1002,"language":1003,"meta":147,"style":147},"language-sh shiki shiki-themes houston","#!\u002Fusr\u002Fbin\u002Fenv stap\n\nfunction walk_parent () {\n    current_proc_struct = task_current();\n    parent_proc_pid     = task_pid( task_parent( current_proc_struct ));\n\n    while ( parent_proc_pid != 0 ) {\n        printf( \"%s [%d] %d:%d \u003C \",\n            task_execname( current_proc_struct ),\n            task_pid( current_proc_struct ),\n            task_uid( current_proc_struct ),\n            task_gid( current_proc_struct ));\n\n        current_proc_struct     = task_parent( current_proc_struct );\n        parent_proc_pid         = task_pid( task_parent( current_proc_struct ));\n    }\n\n    if ( task_pid( current_proc_struct ) == 1 ) {\n        printf( \"%s [%d] %d:%d\\n\",\n            task_execname( current_proc_struct ),\n            task_pid( current_proc_struct ),\n            task_uid( current_proc_struct ),\n            task_gid( current_proc_struct ));\n    }\n}\n\nprobe signal.send {\n    if ( sig_name == \"SIGHUP\" || sig_name == \"SIGKILL\" || sig_name == \"SIGTERM\" ) {\n        printf( \"[%s] %s sent to %s [%d] by \",\n            ctime( gettimeofday_s() ), sig_name, pid_name, sig_pid );\n\n        walk_parent();\n    }\n}\n","sh",[214,1005,1006,1011,1015,1020,1025,1030,1034,1039,1044,1049,1054,1059,1064,1068,1073,1078,1083,1087,1092,1098,1103,1108,1113,1118,1123,1129,1134,1140,1146,1152,1158,1163,1169,1174],{"__ignoreMap":147},[258,1007,1008],{"class":260,"line":261},[258,1009,1010],{},"#!\u002Fusr\u002Fbin\u002Fenv stap\n",[258,1012,1013],{"class":260,"line":268},[258,1014,299],{"emptyLinePlaceholder":298},[258,1016,1017],{"class":260,"line":286},[258,1018,1019],{},"function walk_parent () {\n",[258,1021,1022],{"class":260,"line":295},[258,1023,1024],{},"    current_proc_struct = task_current();\n",[258,1026,1027],{"class":260,"line":302},[258,1028,1029],{},"    parent_proc_pid     = task_pid( task_parent( current_proc_struct ));\n",[258,1031,1032],{"class":260,"line":308},[258,1033,299],{"emptyLinePlaceholder":298},[258,1035,1036],{"class":260,"line":333},[258,1037,1038],{},"    while ( parent_proc_pid != 0 ) {\n",[258,1040,1041],{"class":260,"line":342},[258,1042,1043],{},"        printf( \"%s [%d] %d:%d \u003C \",\n",[258,1045,1046],{"class":260,"line":356},[258,1047,1048],{},"            task_execname( current_proc_struct ),\n",[258,1050,1051],{"class":260,"line":361},[258,1052,1053],{},"            task_pid( current_proc_struct ),\n",[258,1055,1056],{"class":260,"line":367},[258,1057,1058],{},"            task_uid( current_proc_struct ),\n",[258,1060,1061],{"class":260,"line":381},[258,1062,1063],{},"            task_gid( current_proc_struct ));\n",[258,1065,1066],{"class":260,"line":405},[258,1067,299],{"emptyLinePlaceholder":298},[258,1069,1070],{"class":260,"line":422},[258,1071,1072],{},"        current_proc_struct     = task_parent( current_proc_struct );\n",[258,1074,1075],{"class":260,"line":427},[258,1076,1077],{},"        parent_proc_pid         = task_pid( task_parent( current_proc_struct ));\n",[258,1079,1080],{"class":260,"line":439},[258,1081,1082],{},"    }\n",[258,1084,1085],{"class":260,"line":969},[258,1086,299],{"emptyLinePlaceholder":298},[258,1088,1089],{"class":260,"line":982},[258,1090,1091],{},"    if ( task_pid( current_proc_struct ) == 1 ) {\n",[258,1093,1095],{"class":260,"line":1094},19,[258,1096,1097],{},"        printf( \"%s [%d] %d:%d\\n\",\n",[258,1099,1101],{"class":260,"line":1100},20,[258,1102,1048],{},[258,1104,1106],{"class":260,"line":1105},21,[258,1107,1053],{},[258,1109,1111],{"class":260,"line":1110},22,[258,1112,1058],{},[258,1114,1116],{"class":260,"line":1115},23,[258,1117,1063],{},[258,1119,1121],{"class":260,"line":1120},24,[258,1122,1082],{},[258,1124,1126],{"class":260,"line":1125},25,[258,1127,1128],{},"}\n",[258,1130,1132],{"class":260,"line":1131},26,[258,1133,299],{"emptyLinePlaceholder":298},[258,1135,1137],{"class":260,"line":1136},27,[258,1138,1139],{},"probe signal.send {\n",[258,1141,1143],{"class":260,"line":1142},28,[258,1144,1145],{},"    if ( sig_name == \"SIGHUP\" || sig_name == \"SIGKILL\" || sig_name == \"SIGTERM\" ) {\n",[258,1147,1149],{"class":260,"line":1148},29,[258,1150,1151],{},"        printf( \"[%s] %s sent to %s [%d] by \",\n",[258,1153,1155],{"class":260,"line":1154},30,[258,1156,1157],{},"            ctime( gettimeofday_s() ), sig_name, pid_name, sig_pid );\n",[258,1159,1161],{"class":260,"line":1160},31,[258,1162,299],{"emptyLinePlaceholder":298},[258,1164,1166],{"class":260,"line":1165},32,[258,1167,1168],{},"        walk_parent();\n",[258,1170,1172],{"class":260,"line":1171},33,[258,1173,1082],{},[258,1175,1177],{"class":260,"line":1176},34,[258,1178,1128],{},[137,1180,1181,1182,1185,1186,1189],{},"To keep the process running, execute the script in a ",[183,1183,1184],{},"screen"," or ",[183,1187,1188],{},"tmux"," session and log the output.\nIn the case of our mysteriously restarting FTP server, we saw output like below:",[250,1191,1193],{"className":252,"code":1192,"language":254,"meta":147,"style":147},"🞂 stap signal_tracker.stp | tee -a signals.txt\n[Mon Feb 14 06:44:49 2022] SIGHUP sent to vsftpd [41795] by killall [46034] 0:0 \u003C bash [45925] 0:0 \u003C su [45924] 0:0 \u003C sudo [45922] 0:0 \u003C bash [45914] 1000:1000 \u003C sshd [45913] 1000:1000 \u003C sshd [45832] 0:0 \u003C sshd [1050] 0:0 \u003C systemd [1] 0:0\n[Mon Feb 14 06:44:49 2022] SIGHUP sent to bash [45914] by bash [45914] 1000:1000 \u003C sshd [45913] 1000:1000 \u003C sshd [45832] 0:0 \u003C sshd [1050] 0:0 \u003C systemd [1] 0:0\n",[214,1194,1195,1215,1226],{"__ignoreMap":147},[258,1196,1197,1199,1201,1204,1206,1209,1212],{"class":260,"line":261},[258,1198,272],{"class":271},[258,1200,699],{"class":275},[258,1202,1203],{"class":275}," signal_tracker.stp",[258,1205,905],{"class":282},[258,1207,1208],{"class":271},"tee",[258,1210,1211],{"class":275}," -a",[258,1213,1214],{"class":275}," signals.txt\n",[258,1216,1217,1220,1223],{"class":260,"line":268},[258,1218,1219],{"class":282},"[Mon Feb ",[258,1221,1222],{"class":275},"14",[258,1224,1225],{"class":282}," 06:44:49 2022] SIGHUP sent to vsftpd [41795] by killall [46034] 0:0 \u003C bash [45925] 0:0 \u003C su [45924] 0:0 \u003C sudo [45922] 0:0 \u003C bash [45914] 1000:1000 \u003C sshd [45913] 1000:1000 \u003C sshd [45832] 0:0 \u003C sshd [1050] 0:0 \u003C systemd [1] 0:0\n",[258,1227,1228,1230,1232],{"class":260,"line":286},[258,1229,1219],{"class":282},[258,1231,1222],{"class":275},[258,1233,1234],{"class":282}," 06:44:49 2022] SIGHUP sent to bash [45914] by bash [45914] 1000:1000 \u003C sshd [45913] 1000:1000 \u003C sshd [45832] 0:0 \u003C sshd [1050] 0:0 \u003C systemd [1] 0:0\n",[137,1236,1237],{},"The restart was actually coming from a SSH session, suspiciously close to when our technicians would logout after investigating or monitoring!\nLo and behold, someone had the bright idea to do the following:",[250,1239,1241],{"className":252,"code":1240,"language":254,"meta":147,"style":147},"🞂 cat ~\u002F.bash_logout\nkillall -SIGHUP vsftpd\n",[214,1242,1243,1252],{"__ignoreMap":147},[258,1244,1245,1247,1249],{"class":260,"line":261},[258,1246,272],{"class":271},[258,1248,589],{"class":275},[258,1250,1251],{"class":275}," ~\u002F.bash_logout\n",[258,1253,1254,1257,1260],{"class":260,"line":268},[258,1255,1256],{"class":271},"killall",[258,1258,1259],{"class":275}," -SIGHUP",[258,1261,1262],{"class":275}," vsftpd\n",[175,1264,1266],{"id":1265},"conclusion","Conclusion",[137,1268,1269,1270,1272,1273,1275],{},"Signals are used all the time to communicate between processes.\n",[183,1271,122],{}," allows us to see signals received by procsses;\nhowever, ",[183,1274,121],{}," allows us to trace and dynamically execute code we can use to print additional debugging information.",[137,1277,1278],{},"P.S. Please don’t restart services using logout scripts 🤮",[1280,1281,1282],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .sy89t, html code.shiki .sy89t{--shiki-default:#EEF0F98F;--shiki-default-font-style:italic}html pre.shiki code .stHQG, html code.shiki .stHQG{--shiki-default:#00DAEF}html pre.shiki code .se2J0, html code.shiki .se2J0{--shiki-default:#FFD493}html pre.shiki code .sLo_3, html code.shiki .sLo_3{--shiki-default:#EEF0F9}html pre.shiki code .s8HWQ, html code.shiki .s8HWQ{--shiki-default:#ACAFFF}",{"title":147,"searchDepth":268,"depth":268,"links":1284},[1285,1286,1287,1290],{"id":177,"depth":268,"text":178},{"id":122,"depth":268,"text":122},{"id":121,"depth":268,"text":121,"children":1288},[1289],{"id":573,"depth":286,"text":574},{"id":1265,"depth":268,"text":1266},{},{"title":52,"description":118},"blog\u002Ftracing-signals-in-linux",[120,121,122,123],{"sm":125,"md":126,"lg":127},null,"ka2kgY89ntACVpSARpGUSehOVR_He7CjrY3cdkRxu2I",[],{"id":4,"title":5,"author":1300,"description":14,"extension":15,"meta":1301,"repository":1302,"stem":21,"__hash__":22},{"name":7,"github":8,"gitlab":9,"email":10,"linkedin":11,"mastodon":12,"bio":13},{},{"url":18,"git":19,"branch":20},1793313045136]