diff --git a/my-gym b/my-gym index f5389f70..13f12b77 160000 --- a/my-gym +++ b/my-gym @@ -1 +1 @@ -Subproject commit f5389f70876c37db2c1629f22d9904546ffe10de +Subproject commit 13f12b77bb536238e31d14a595fd47a05a6ab9f9 diff --git a/mybin b/mybin index c888019f..4e84af15 160000 --- a/mybin +++ b/mybin @@ -1 +1 @@ -Subproject commit c888019fb4197708a2379874b98ab5f7fdab31a5 +Subproject commit 4e84af158ba0e1718e020d44d89685877f35e721 diff --git a/system_monitoring/README.md b/system_monitoring/README.md index 7a778c91..a49cd706 100644 --- a/system_monitoring/README.md +++ b/system_monitoring/README.md @@ -1,3 +1,32 @@ +# 2024-09-09 + +More OOM. + +Found PID via + + grep -5 -i "killed process" /var/log/syslog + +(or perhaps other means). Then + + atop -r + + to search for PID + forward back until you find it in history + to show command line view and hopefully you see something more informative than just "python" or whatever. I think the first entry is the full command. + + +No way to track PID to process (other than python). + +Try to enable this for next time: + + sudo apt install auditd + sudo auditctl -a exit,always -F arch=b64 -S execve + +Then (maybe) something like + + ausearch -k python_exec + + # 2024-09-06 sudo apt-get install atop @@ -19,6 +48,16 @@ # 2024 + +Reminders (for post mortem) + + sudo dmesg -T ... this might get you a PID before the OOM + + sar -r # Shows memory usage over time + sar -u # Shows CPU usage over time + sar -S # Shows swap usage over time + + * https://tomscii.sig7.se/2022/07/uMon-stupid-simple-monitoring * glances * systar (sar): @@ -26,10 +65,6 @@ sudo apt install sysstat # On Ubuntu/Debian sudo systemctl enable --now sysstat - sar -r # Shows memory usage over time - sar -u # Shows CPU usage over time - sar -S # Shows swap usage over time - sar -r 10 100 # Logs memory usage every 10 seconds for 100 intervals # post event debug