Macbook: Return a list of processes using a specific remote port number

I find this script useful for debugging which processes are talking to which remote port.

#!/bin/bash

# Network Connection Monitor with Color Coding
# Shows TCP/UDP connections with state and process info
# Refreshes every 5 seconds
# Usage: ./netmon.sh [--port PORT] [--ip IP_ADDRESS]

# Parse command line arguments
FILTER_PORT=""
FILTER_IP=""

while [[ $# -gt 0 ]]; do
    case $1 in
        --port|-p)
            FILTER_PORT="$2"
            shift 2
            ;;
        --ip|-i)
            FILTER_IP="$2"
            shift 2
            ;;
        --help|-h)
            echo "Usage: $0 [OPTIONS]"
            echo "Options:"
            echo "  --port, -p PORT    Filter by remote port"
            echo "  --ip, -i IP        Filter by remote IP address"
            echo "  --help, -h         Show this help message"
            echo ""
            echo "Examples:"
            echo "  $0 --port 443      Show only connections to port 443"
            echo "  $0 --ip 1.1.1.1    Show only connections to IP 1.1.1.1"
            echo "  $0 -p 80 -i 192.168.1.1  Show connections to 192.168.1.1:80"
            exit 0
            ;;
        *)
            echo "Unknown option: $1"
            echo "Use --help for usage information"
            exit 1
            ;;
    esac
done

# Color definitions
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
MAGENTA='\033[0;35m'
CYAN='\033[0;36m'
WHITE='\033[1;37m'
GRAY='\033[0;90m'
NC='\033[0m' # No Color
BOLD='\033[1m'

# Function to get process name from PID
get_process_name() {
    local pid=$1
    if [ "$pid" != "-" ] && [ "$pid" != "0" ] && [ -n "$pid" ]; then
        ps -p "$pid" -o comm= 2>/dev/null || echo "unknown"
    else
        echo "-"
    fi
}

# Function to color-code based on state
get_state_color() {
    local state=$1
    case "$state" in
        "ESTABLISHED")
            echo "${GREEN}"
            ;;
        "LISTEN")
            echo "${BLUE}"
            ;;
        "TIME_WAIT")
            echo "${YELLOW}"
            ;;
        "CLOSE_WAIT")
            echo "${MAGENTA}"
            ;;
        "SYN_SENT"|"SYN_RCVD")
            echo "${CYAN}"
            ;;
        "FIN_WAIT"*)
            echo "${GRAY}"
            ;;
        "CLOSING"|"LAST_ACK")
            echo "${RED}"
            ;;
        *)
            echo "${WHITE}"
            ;;
    esac
}

# Function to split address into IP and port
split_address() {
    local addr=$1
    local ip=""
    local port=""
    
    if [[ "$addr" == "*"* ]]; then
        ip="*"
        port="*"
    elif [[ "$addr" =~ ^(.+)\.([0-9]+)$ ]]; then
        ip="${BASH_REMATCH[1]}"
        port="${BASH_REMATCH[2]}"
    elif [[ "$addr" =~ ^(.+):([0-9]+)$ ]]; then
        # Handle IPv6 format
        ip="${BASH_REMATCH[1]}"
        port="${BASH_REMATCH[2]}"
    else
        ip="$addr"
        port="-"
    fi
    
    echo "$ip|$port"
}

# Function to check if connection matches filters
matches_filter() {
    local remote_ip=$1
    local remote_port=$2
    
    # Check port filter
    if [ -n "$FILTER_PORT" ] && [ "$remote_port" != "$FILTER_PORT" ]; then
        return 1
    fi
    
    # Check IP filter
    if [ -n "$FILTER_IP" ]; then
        # Handle partial IP matching
        if [[ "$remote_ip" != *"$FILTER_IP"* ]]; then
            return 1
        fi
    fi
    
    return 0
}

# Function to display connections
show_connections() {
    clear
    
    # Header
    echo -e "${BOLD}${WHITE}=== Network Connections Monitor ===${NC}"
    echo -e "${BOLD}${WHITE}$(date '+%Y-%m-%d %H:%M:%S')${NC}"
    
    # Show active filters
    if [ -n "$FILTER_PORT" ] || [ -n "$FILTER_IP" ]; then
        echo -e "${YELLOW}Active Filters:${NC}"
        [ -n "$FILTER_PORT" ] && echo -e "  Remote Port: ${BOLD}$FILTER_PORT${NC}"
        [ -n "$FILTER_IP" ] && echo -e "  Remote IP: ${BOLD}$FILTER_IP${NC}"
    fi
    echo ""
    
    # Legend
    echo -e "${BOLD}Color Legend:${NC}"
    echo -e "  ${GREEN}●${NC} ESTABLISHED    ${BLUE}●${NC} LISTEN         ${YELLOW}●${NC} TIME_WAIT"
    echo -e "  ${CYAN}●${NC} SYN_SENT/RCVD  ${MAGENTA}●${NC} CLOSE_WAIT     ${RED}●${NC} CLOSING/LAST_ACK"
    echo -e "  ${GRAY}●${NC} FIN_WAIT       ${WHITE}●${NC} OTHER/UDP"
    echo ""
    
    # Table header
    printf "${BOLD}%-6s %-22s %-22s %-7s %-12s %-8s %-30s${NC}\n" \
        "PROTO" "LOCAL ADDRESS" "REMOTE IP" "R.PORT" "STATE" "PID" "PROCESS"
    echo "$(printf '%.0s-' {1..120})"
    
    # Temporary file for storing connections
    TMPFILE=$(mktemp)
    
    # Get TCP connections with netstat
    # Note: On macOS, we need sudo to see process info for all connections
    if command -v sudo >/dev/null 2>&1; then
        # Try with sudo first (will show all processes)
        sudo netstat -anp tcp 2>/dev/null | grep -E '^tcp' > "$TMPFILE" 2>/dev/null || \
        netstat -an -p tcp 2>/dev/null | grep -E '^tcp' > "$TMPFILE"
    else
        netstat -an -p tcp 2>/dev/null | grep -E '^tcp' > "$TMPFILE"
    fi
    
    # Process TCP connections
    while IFS= read -r line; do
        # Parse netstat output (macOS format)
        proto=$(echo "$line" | awk '{print $1}')
        local_addr=$(echo "$line" | awk '{print $4}')
        remote_addr=$(echo "$line" | awk '{print $5}')
        state=$(echo "$line" | awk '{print $6}')
        
        # Split remote address into IP and port
        IFS='|' read -r remote_ip remote_port <<< "$(split_address "$remote_addr")"
        
        # Apply filters
        if ! matches_filter "$remote_ip" "$remote_port"; then
            continue
        fi
        
        # Try to get PID using lsof for the local address
        if [[ "$local_addr" =~ ([0-9.]+|[*])\.([0-9]+)$ ]] || [[ "$local_addr" =~ ([0-9a-f:]+)\.([0-9]+)$ ]]; then
            port="${BASH_REMATCH[2]}"
            # Use lsof to find the PID
            pid=$(sudo lsof -i TCP:$port -sTCP:$state 2>/dev/null | grep -v PID | head -1 | awk '{print $2}')
            if [ -z "$pid" ]; then
                pid="-"
                process="-"
            else
                process=$(get_process_name "$pid")
            fi
        else
            pid="-"
            process="-"
        fi
        
        # Get color based on state
        color=$(get_state_color "$state")
        
        # Format and print
        printf "${color}%-6s %-22s %-22s %-7s %-12s %-8s %-30s${NC}\n" \
            "$proto" \
            "${local_addr:0:22}" \
            "${remote_ip:0:22}" \
            "${remote_port:0:7}" \
            "$state" \
            "$pid" \
            "${process:0:30}"
    done < "$TMPFILE"
    
    # Get UDP connections
    echo ""
    if command -v sudo >/dev/null 2>&1; then
        sudo netstat -anp udp 2>/dev/null | grep -E '^udp' > "$TMPFILE" 2>/dev/null || \
        netstat -an -p udp 2>/dev/null | grep -E '^udp' > "$TMPFILE"
    else
        netstat -an -p udp 2>/dev/null | grep -E '^udp' > "$TMPFILE"
    fi
    
    # Process UDP connections
    while IFS= read -r line; do
        # Parse netstat output for UDP
        proto=$(echo "$line" | awk '{print $1}')
        local_addr=$(echo "$line" | awk '{print $4}')
        remote_addr=$(echo "$line" | awk '{print $5}')
        
        # Split remote address into IP and port
        IFS='|' read -r remote_ip remote_port <<< "$(split_address "$remote_addr")"
        
        # Apply filters
        if ! matches_filter "$remote_ip" "$remote_port"; then
            continue
        fi
        
        # UDP doesn't have state
        state="*"
        
        # Try to get PID using lsof for the local address
        if [[ "$local_addr" =~ ([0-9.]+|[*])\.([0-9]+)$ ]] || [[ "$local_addr" =~ ([0-9a-f:]+)\.([0-9]+)$ ]]; then
            port="${BASH_REMATCH[2]}"
            # Use lsof to find the PID
            pid=$(sudo lsof -i UDP:$port 2>/dev/null | grep -v PID | head -1 | awk '{print $2}')
            if [ -z "$pid" ]; then
                pid="-"
                process="-"
            else
                process=$(get_process_name "$pid")
            fi
        else
            pid="-"
            process="-"
        fi
        
        # White color for UDP
        printf "${WHITE}%-6s %-22s %-22s %-7s %-12s %-8s %-30s${NC}\n" \
            "$proto" \
            "${local_addr:0:22}" \
            "${remote_ip:0:22}" \
            "${remote_port:0:7}" \
            "$state" \
            "$pid" \
            "${process:0:30}"
    done < "$TMPFILE"
    
    # Clean up
    rm -f "$TMPFILE"
    
    # Footer
    echo ""
    echo "$(printf '%.0s-' {1..120})"
    echo -e "${BOLD}Press Ctrl+C to exit${NC} | Refreshing every 5 seconds..."
    
    # Show filter hint if no filters active
    if [ -z "$FILTER_PORT" ] && [ -z "$FILTER_IP" ]; then
        echo -e "${GRAY}Tip: Use --port PORT or --ip IP to filter connections${NC}"
    fi
}

# Trap Ctrl+C to exit cleanly
trap 'echo -e "\n${BOLD}Exiting...${NC}"; exit 0' INT

# Main loop
echo -e "${BOLD}${CYAN}Starting Network Connection Monitor...${NC}"
echo -e "${YELLOW}Note: Run with sudo for complete process information${NC}"

# Show active filters on startup
if [ -n "$FILTER_PORT" ] || [ -n "$FILTER_IP" ]; then
    echo -e "${GREEN}Filtering enabled:${NC}"
    [ -n "$FILTER_PORT" ] && echo -e "  Remote Port: ${BOLD}$FILTER_PORT${NC}"
    [ -n "$FILTER_IP" ] && echo -e "  Remote IP: ${BOLD}$FILTER_IP${NC}"
fi

sleep 2

while true; do
    show_connections
    sleep 5
done

Macbook: Script to monitor the top disk reads and writes

The script below tracks disk usage of a macbook for 20 seconds and the shows the processes with the highest disk utilisations

#!/bin/bash

# Disk I/O Monitor for macOS
# Shows which processes are using disk I/O the most with full paths

DURATION=20

echo "Disk I/O Monitor for macOS"
echo "========================================"
echo ""

# Check for sudo
if [[ $EUID -ne 0 ]]; then
    echo "ERROR: This script requires sudo privileges"
    echo "Please run: sudo $0"
    exit 1
fi

# Create temp file
TEMP_FILE="/tmp/disk_io_$$.txt"
export TEMP_FILE

# Collect data
echo "Collecting disk I/O data for $DURATION seconds..."
fs_usage -w -f filesys 2>/dev/null > "$TEMP_FILE" &
FS_PID=$!

# Progress bar
for i in $(seq 1 $DURATION); do
    printf "\rProgress: [%-20s] %d/%d seconds" "$(printf '#%.0s' $(seq 1 $((i*20/DURATION))))" $i $DURATION
    sleep 1
done
echo ""

# Stop collection
kill $FS_PID 2>/dev/null
wait $FS_PID 2>/dev/null

echo ""
echo "Processing data..."

# Parse with Python - pass temp file as argument
python3 - "$TEMP_FILE" << 'PYTHON_END'
import re
import os
import sys
from collections import defaultdict
import subprocess

# Get temp file from argument
temp_file = sys.argv[1] if len(sys.argv) > 1 else '/tmp/disk_io_temp.txt'

# Storage for process stats
stats = defaultdict(lambda: {'reads': 0, 'writes': 0, 'process_name': '', 'pid': ''})

# Parse fs_usage output
try:
    with open(temp_file, 'r') as f:
        for line in f:
            # Look for lines with process info (format: processname.pid at end of line)
            match = re.search(r'(\S+)\.(\d+)\s*$', line)
            if match:
                process_name = match.group(1)
                pid = match.group(2)
                key = f"{process_name}|{pid}"
                
                # Store process info
                stats[key]['process_name'] = process_name
                stats[key]['pid'] = pid
                
                # Categorize operation
                if any(op in line for op in ['RdData', 'read', 'READ', 'getattrlist', 'stat64', 'lstat64', 'open']):
                    stats[key]['reads'] += 1
                elif any(op in line for op in ['WrData', 'write', 'WRITE', 'close', 'fsync']):
                    stats[key]['writes'] += 1
except Exception as e:
    print(f"Error reading file: {e}")
    sys.exit(1)

# Calculate totals
total_ops = sum(s['reads'] + s['writes'] for s in stats.values())

# Get executable paths
def get_exe_path(process_name, pid):
    try:
        # Method 1: Try lsof with format output
        result = subprocess.run(['lsof', '-p', pid, '-Fn'], capture_output=True, text=True, stderr=subprocess.DEVNULL)
        paths = []
        for line in result.stdout.split('\n'):
            if line.startswith('n'):
                path = line[1:].strip()
                paths.append(path)
        
        # Look for the best path
        for path in paths:
            if '/Contents/MacOS/' in path and process_name in path:
                return path
            elif path.endswith('.app'):
                return path
            elif any(p in path for p in ['/bin/', '/sbin/', '/usr/']) and not any(path.endswith(ext) for ext in ['.dylib', '.so']):
                return path
        
        # Method 2: Try ps
        result = subprocess.run(['ps', '-p', pid, '-o', 'command='], capture_output=True, text=True, stderr=subprocess.DEVNULL)
        if result.stdout.strip():
            cmd = result.stdout.strip().split()[0]
            if os.path.exists(cmd):
                return cmd
        
        # Method 3: Return command name from ps
        result = subprocess.run(['ps', '-p', pid, '-o', 'comm='], capture_output=True, text=True, stderr=subprocess.DEVNULL)
        if result.stdout.strip():
            return result.stdout.strip()
            
    except Exception:
        pass
    
    # Last resort: return process name
    return process_name

# Sort by total operations
sorted_stats = sorted(stats.items(), key=lambda x: x[1]['reads'] + x[1]['writes'], reverse=True)

# Print header
print("\n%-30s %-8s %-45s %8s %8s %8s %7s %7s" % 
      ("Process Name", "PID", "Executable Path", "Reads", "Writes", "Total", "Read%", "Write%"))
print("=" * 140)

# Print top 20 processes
count = 0
for key, data in sorted_stats:
    if data['reads'] + data['writes'] == 0:
        continue
        
    total = data['reads'] + data['writes']
    read_pct = (data['reads'] * 100.0 / total_ops) if total_ops > 0 else 0
    write_pct = (data['writes'] * 100.0 / total_ops) if total_ops > 0 else 0
    
    # Get executable path
    exe_path = get_exe_path(data['process_name'], data['pid'])
    if len(exe_path) > 45:
        exe_path = "..." + exe_path[-42:]
    
    print("%-30s %-8s %-45s %8d %8d %8d %6.1f%% %6.1f%%" % 
          (data['process_name'][:30], 
           data['pid'], 
           exe_path,
           data['reads'], 
           data['writes'], 
           total,
           read_pct, 
           write_pct))
    
    count += 1
    if count >= 20:
        break

print("=" * 140)
print(f"Total I/O operations captured: {total_ops}")

PYTHON_END

# Cleanup
rm -f "$TEMP_FILE"

echo ""
echo "Monitoring complete."

Example output:

Disk I/O Monitor for macOS
========================================

Collecting disk I/O data for 20 seconds...
Progress: [####################] 20/20 seconds

Processing data...

Process Name                   PID      Executable Path                                  Reads   Writes    Total   Read%  Write%
============================================================================================================================================
Chrome                         4719678  Chrome                                             427      811     1238    3.1%    5.9%
UPMServiceController           4644625  UPMServiceController                               423      587     1010    3.1%    4.3%
UPMServiceController           4014337  UPMServiceController                               468      309      777    3.4%    2.2%
wsdlpd                         3060029  wsdlpd                                             154      370      524    1.1%    2.7%
tccd                           4743441  tccd                                               359       48      407    2.6%    0.3%
tccd                           4742031  tccd                                               358       48      406    2.6%    0.3%
com.crowdstrike.falcon.Agent   6174     com.crowdstrike.falcon.Agent                       301        5      306    2.2%    0.0%
UPMServiceContro               4644625  UPMServiceContro                                    12      285      297    0.1%    2.1%
mds_stores                     4736869  mds_stores                                         204       71      275    1.5%    0.5%
EndPointClassifier             6901     EndPointClassifier                                  40      231      271    0.3%    1.7%

MacOs: How to see which processes are using a specific port (eg 443)

Below is a useful script when you want to see which processes are using a specific port.

#!/bin/bash

# Port Monitor Script for macOS
# Usage: ./port_monitor.sh <port_number>
# Check if port number is provided

if [ $# -eq 0 ]; then
echo "Usage: $0 <port_number>"
echo "Example: $0 8080"
exit 1
fi

PORT=$1

# Validate port number

if ! [[ $PORT =~ ^[0-9]+$ ]] || [ $PORT -lt 1 ] || [ $PORT -gt 65535 ]; then
echo "Error: Please provide a valid port number (1-65535)"
exit 1
fi

# Function to display processes using the port

show_port_usage() {
local timestamp=$(date "+%Y-%m-%d %H:%M:%S")

# Clear screen for better readability
clear

echo "=================================="
echo "Port Monitor - Port $PORT"
echo "Last updated: $timestamp"
echo "Press Ctrl+C to exit"
echo "=================================="
echo

# Check for processes using the port with lsof - both TCP and UDP
if lsof -i :$PORT &>/dev/null || netstat -an | grep -E "[:.]$PORT[[:space:]]" &>/dev/null; then
    echo "Processes using port $PORT:"
    echo
    lsof -i :$PORT -P -n | head -1
    echo "--------------------------------------------------------------------------------"
    lsof -i :$PORT -P -n | tail -n +2
    echo
    
    # Also show netstat information for additional context
    echo "Network connections on port $PORT:"
    echo
    printf "%-6s %-30s %-30s %-12s\n" "PROTO" "LOCAL ADDRESS" "FOREIGN ADDRESS" "STATE"
    echo "--------------------------------------------------------------------------------------------"
    
    # Show all connections (LISTEN, ESTABLISHED, etc.)
    # Use netstat -n to show numeric addresses
    netstat -anp tcp | grep -E "\.$PORT[[:space:]]" | while read line; do
        # Extract the relevant fields from netstat output
        proto=$(echo "$line" | awk '{print $1}')
        local_addr=$(echo "$line" | awk '{print $4}')
        foreign_addr=$(echo "$line" | awk '{print $5}')
        state=$(echo "$line" | awk '{print $6}')
        
        # Only print if we have valid data
        if [ -n "$proto" ] && [ -n "$local_addr" ]; then
            printf "%-6s %-30s %-30s %-12s\n" "$proto" "$local_addr" "$foreign_addr" "$state"
        fi
    done
    
    # Also check UDP connections
    netstat -anp udp | grep -E "\.$PORT[[:space:]]" | while read line; do
        proto=$(echo "$line" | awk '{print $1}')
        local_addr=$(echo "$line" | awk '{print $4}')
        foreign_addr=$(echo "$line" | awk '{print $5}')
        printf "%-6s %-30s %-30s %-12s\n" "$proto" "$local_addr" "$foreign_addr" "-"
    done
    
    # Also check for any established connections using lsof
    echo
    echo "Active connections with processes:"
    echo "--------------------------------------------------------------------------------------------"
    lsof -i :$PORT -P -n 2>/dev/null | grep -v LISTEN | tail -n +2 | while read line; do
        if [ -n "$line" ]; then
            echo "$line"
        fi
    done
    
else
    echo "No processes found using port $PORT"
    echo
    
    # Check if the port might be in use but not showing up in lsof
    local netstat_result=$(netstat -anv | grep -E "\.$PORT ")
    if [ -n "$netstat_result" ]; then
        echo "However, netstat shows activity on port $PORT:"
        echo "$netstat_result"
    fi
fi

echo
echo "Refreshing in 20 seconds... (Press Ctrl+C to exit)"
}

# Trap Ctrl+C to exit gracefully

trap 'echo -e "\n\nExiting port monitor..."; exit 0' INT

# Main loop - refresh every 20 seconds

while true; do
show_port_usage
sleep 20
done

Windows Server: Polling critical DNS entries for any changes or errors

If you have tier 1 services that are dependant on a few DNS records, then you may want a simple batch job to monitor these dns records for changes or deletion.

The script below contains an example list of DNS entries (replace these records for the ones you want to monitor).

@echo off
setlocal enabledelayedexpansion

REM ============================================================================
REM DNS Monitor Script for Windows Server
REM Purpose: Monitor DNS entries for changes every 15 minutes
REM Author: Andrew Baker
REM Version: 1.0
REM Date: August 13, 2018
REM ============================================================================

REM Configuration Variables
set "LOG_FILE=dns_monitor.log"
set "PREVIOUS_FILE=dns_previous.tmp"
set "CURRENT_FILE=dns_current.tmp"
set "CHECK_INTERVAL=900"

REM DNS Entries to Monitor (Comma Separated List)
REM Add or modify domains as needed
set "DNS_LIST=google.com,microsoft.com,github.com,stackoverflow.com,amazon.com,facebook.com,twitter.com,linkedin.com,youtube.com,cloudflare.com"

REM Initialize log file with header if it doesn't exist
if not exist "%LOG_FILE%" (
    echo DNS Monitor Log - Started on %DATE% %TIME% > "%LOG_FILE%"
    echo ============================================================================ >> "%LOG_FILE%"
    echo. >> "%LOG_FILE%"
)

:MAIN_LOOP
echo [%DATE% %TIME%] Starting DNS monitoring cycle...
echo [%DATE% %TIME%] INFO: Starting DNS monitoring cycle >> "%LOG_FILE%"

REM Clear current results file
if exist "%CURRENT_FILE%" del "%CURRENT_FILE%"

REM Process each DNS entry
for %%d in (%DNS_LIST%) do (
    call :CHECK_DNS "%%d"
)

REM Compare with previous results if they exist
if exist "%PREVIOUS_FILE%" (
    call :COMPARE_RESULTS
) else (
    echo [%DATE% %TIME%] INFO: First run - establishing baseline >> "%LOG_FILE%"
)

REM Copy current results to previous for next comparison
copy "%CURRENT_FILE%" "%PREVIOUS_FILE%" >nul 2>&1

echo [%DATE% %TIME%] DNS monitoring cycle completed. Next check in 15 minutes...
echo [%DATE% %TIME%] INFO: DNS monitoring cycle completed >> "%LOG_FILE%"
echo. >> "%LOG_FILE%"

REM Wait 15 minutes (900 seconds) before next check
timeout /t %CHECK_INTERVAL% /nobreak >nul

goto MAIN_LOOP

REM ============================================================================
REM Function: CHECK_DNS
REM Purpose: Resolve DNS entry and log results
REM Parameter: %1 = Domain name to check
REM ============================================================================
:CHECK_DNS
set "DOMAIN=%~1"
echo Checking DNS for: %DOMAIN%

REM Perform nslookup and capture results
nslookup "%DOMAIN%" > temp_dns.txt 2>&1

REM Check if nslookup was successful
if %ERRORLEVEL% equ 0 (
    REM Extract IP addresses from nslookup output
    for /f "tokens=2" %%i in ('findstr /c:"Address:" temp_dns.txt ^| findstr /v "#53"') do (
        set "IP_ADDRESS=%%i"
        echo %DOMAIN%,!IP_ADDRESS! >> "%CURRENT_FILE%"
        echo [%DATE% %TIME%] INFO: %DOMAIN% resolves to !IP_ADDRESS! >> "%LOG_FILE%"
    )
    
    REM Handle case where no IP addresses were found in successful lookup
    findstr /c:"Address:" temp_dns.txt | findstr /v "#53" >nul
    if !ERRORLEVEL! neq 0 (
        echo %DOMAIN%,RESOLUTION_ERROR >> "%CURRENT_FILE%"
        echo [%DATE% %TIME%] ERROR: %DOMAIN% - No IP addresses found in DNS response >> "%LOG_FILE%"
        type temp_dns.txt >> "%LOG_FILE%"
        echo. >> "%LOG_FILE%"
    )
) else (
    REM DNS resolution failed
    echo %DOMAIN%,DNS_FAILURE >> "%CURRENT_FILE%"
    echo [%DATE% %TIME%] ERROR: %DOMAIN% - DNS resolution failed >> "%LOG_FILE%"
    type temp_dns.txt >> "%LOG_FILE%"
    echo. >> "%LOG_FILE%"
)

REM Clean up temporary file
if exist temp_dns.txt del temp_dns.txt

goto :EOF

REM ============================================================================
REM Function: COMPARE_RESULTS
REM Purpose: Compare current DNS results with previous results
REM ============================================================================
:COMPARE_RESULTS
echo Comparing DNS results for changes...

REM Read previous results into memory
if exist "%PREVIOUS_FILE%" (
    for /f "tokens=1,2 delims=," %%a in (%PREVIOUS_FILE%) do (
        set "PREV_%%a=%%b"
    )
)

REM Compare current results with previous
for /f "tokens=1,2 delims=," %%a in (%CURRENT_FILE%) do (
    set "CURRENT_DOMAIN=%%a"
    set "CURRENT_IP=%%b"
    
    REM Get previous IP for this domain
    set "PREVIOUS_IP=!PREV_%%a!"
    
    if "!PREVIOUS_IP!"=="" (
        REM New domain added
        echo [%DATE% %TIME%] INFO: New domain added to monitoring: !CURRENT_DOMAIN! = !CURRENT_IP! >> "%LOG_FILE%"
    ) else if "!PREVIOUS_IP!" neq "!CURRENT_IP!" (
        REM DNS change detected
        echo [%DATE% %TIME%] WARNING: DNS change detected for !CURRENT_DOMAIN! >> "%LOG_FILE%"
        echo [%DATE% %TIME%] WARNING: Previous IP: !PREVIOUS_IP! >> "%LOG_FILE%"
        echo [%DATE% %TIME%] WARNING: Current IP:  !CURRENT_IP! >> "%LOG_FILE%"
        echo [%DATE% %TIME%] WARNING: *** INVESTIGATE DNS CHANGE *** >> "%LOG_FILE%"
        echo. >> "%LOG_FILE%"
        
        REM Also display warning on console
        echo.
        echo *** WARNING: DNS CHANGE DETECTED ***
        echo Domain: !CURRENT_DOMAIN!
        echo Previous: !PREVIOUS_IP!
        echo Current:  !CURRENT_IP!
        echo Check log file for details: %LOG_FILE%
        echo.
    )
)

REM Check for domains that disappeared from current results
for /f "tokens=1,2 delims=," %%a in (%PREVIOUS_FILE%) do (
    set "CHECK_DOMAIN=%%a"
    set "FOUND=0"
    
    for /f "tokens=1 delims=," %%c in (%CURRENT_FILE%) do (
        if "%%c"=="!CHECK_DOMAIN!" set "FOUND=1"
    )
    
    if "!FOUND!"=="0" (
        echo [%DATE% %TIME%] WARNING: Domain !CHECK_DOMAIN! no longer resolving or removed from monitoring >> "%LOG_FILE%"
    )
)

goto :EOF

REM ============================================================================
REM End of Script
REM ============================================================================

Mac OSX: Altering the OS route table to re-direct the traffic of a website to a different interface (eg re-routing whatsapp traffic to en0)

This was a hard article to figure out the title for! Put simply, your mac book has a route table and if you want to move a specific IP address or dns from one interface to another, then follow the steps below:

First find the IP address of the website that you want to re-route the traffic for:

$ nslookup web.whatsapp.com
Server:		100.64.0.1
Address:	100.64.0.1#53

Non-authoritative answer:
web.whatsapp.com	canonical name = mmx-ds.cdn.whatsapp.net.
Name:	mmx-ds.cdn.whatsapp.net
Address: 102.132.99.60

We want to re-route traffic the traffic from: 102.132.99.60 to the default interface. So first lets find out which interface this traffic is currently being routed to?

$ route -n get web.whatsapp.com
   route to: 102.132.99.60
destination: 102.132.99.60
    gateway: 100.64.0.1
  interface: utun0
      flags: <UP,GATEWAY,HOST,DONE,WASCLONED,IFSCOPE,IFREF>
 recvpipe  sendpipe  ssthresh  rtt,msec    rttvar  hopcount      mtu     expire
       0         0         0        34        21         0      1400         0

So this is currently going to a tunnelled interface called utun0 on gateway 100.64.0.1.

Ok, so I want to move if off this tunnelled interface. So lets first display the kernel routing table. The -n option forces netstat to print the IP addresses. Without this option, netstat attempts to display the host names.

$ netstat - rn | head -n 5
Active Internet connections
Proto Recv-Q Send-Q  Local Address          Foreign Address        (state)
tcp4       0    126  100.64.0.1.64770       136.226.216.14.https   ESTABLISHED
tcp4       0      0  100.64.0.1.64768       whatsapp-cdn-shv.https ESTABLISHED
tcp4       0      0  100.64.0.1.64766       52.178.17.3.https      ESTABLISHED

Now we want to re-route whatsapp to the default interface. So lets get the IP address of the default interface.

$ netstat -nr | grep default
default            192.168.8.1        UGScg                 en0
default                                 fe80::%utun1                            UGcIg               utun1
default                                 fe80::%utun2                            UGcIg               utun2
default                                 fe80::%utun3                            UGcIg               utun3
default                                 fe80::%utun4                            UGcIg               utun4
default                                 fe80::%utun5                            UGcIg               utun5
default                                 fe80::%utun0                            UGcIg               utun0

We can see that our en0 interface is on IP address: 192.168.8.1. So lets re-route the traffic from Whatsapp’s ip address to this interace’s IP address:

$ sudo route add 102.132.99.60 192.168.0.1
route: writing to routing socket: File exists
add host 102.132.99.60: gateway 192.168.8.1: File exists

Now lets test if we are routing via the correct interface:

$ route -n get 102.132.99.60
   route to: 102.132.99.60
destination: 102.132.99.60
    gateway: 192.168.8.1
  interface: utun6
      flags: <UP,GATEWAY,HOST,DONE,STATIC>
 recvpipe  sendpipe  ssthresh  rtt,msec    rttvar  hopcount      mtu     expire
       0         0         0         0         0         0      1400         0

Finally delete the route and recheck the routing:

$ sudo route delete 102.132.99.60
delete host 102.132.99.60

$ route -n get 102.132.99.60
   route to: 102.132.99.60
destination: 102.132.99.60
    gateway: 100.64.0.1
  interface: utun6
      flags: <UP,GATEWAY,HOST,DONE,WASCLONED,IFSCOPE,IFREF>
 recvpipe  sendpipe  ssthresh  rtt,msec    rttvar  hopcount      mtu     expire
       0         0         0         0         0         0      1400         0

AWS: Use the AWS CLI to delete snapshots from your account

The Amazon EC2 console allows you to delete up to 50 Amazon Elastic Block Store (Amazon EBS) snapshots at once. To delete more than 50 snapshots, use the AWS Command Line Interface (AWS CLI) or the AWS SDK.

To see all the snapshots that you own in a specific region, run the following. Note, replace af-south-1 with your region:

aws ec2 describe-snapshots --owner-ids self  --query 'Snapshots[]' --region af-south-1

Note: To run the code below, first make sure your in the correct account (or life will become difficult for you). Next replace BOTH instances “af-south-1” with your particular region. Finally, you can use a specific account number in place of –owner-ids=self (eg –owner-ids=1234567890).

for SnapshotID in $(aws ec2 --region af-south-1 describe-snapshots --owner-ids=self --query 'Snapshots[*].SnapshotId' --output=text); do
aws ec2 --region af-south-1 delete-snapshot --snapshot-id ${SnapshotID}
done

Macbook OSX: Using Touch ID / fingerprints to enable SUDO and permanently enabling this after Mac OSX updates

Each day that I wake up I try and figure out if I can do less work than yesterday. With this in mind I was playing around to see if there is a way to save me typing my password each time I SUDO. It turns out this is quite a simple change…

Open Terminal and run the following to edit sudos behaviour:

sudo nano /etc/pam.d/sudo

Next add the following to the top of the file:

auth       sufficient     pam_tid.so

The only issue with this is that /etc/pam.d/sudo is overwritten on every macOS update (major, minor or patch – it is always overwritten and reset back to its default state).

MacOS: Sonoma

In their “What’s new for enterprise in macOS Sonoma” document Apple listed the following in the “Bug fixes and other improvements” section:

Touch ID can be allowed for sudo with a configuration that persists across software updates using /etc/pam.d/sudo_local. See /etc/pam.d/sudo_local.template for details.

So lets create a template file in /etc/pam.d/sudo_local.template:

sudo nano /etc/pam.d/sudo_local.template

Next uncomment the auth line, as per:

# sudo_local: local config file which survives system update and is included fo$
# uncomment following line to enable Touch ID for sudo
auth       sufficient     pam_tid.so

This should mean that Touch ID now survive system updates!

Quick tests:

sudo ls
# exit sudo
sudo -k
sudo ls

To enable Touch ID access on Iterm2. You need to do the following. Go to Prefs -> Advanced -> Allow sessions to survive logging out and back in and set value to no . Restart Iterm2 and touch ID authentication will work on Iterm2.

Macbook OSX: Change the default image type of your screenshots from PNG to JPEG, GIF or PDF

There are a few things that I tweak when I get a new Macbook, one of which is the screenshot format (mainly because it doesnt natively render in Whatsapp). So I thought I would share the code snippet that you can run in Terminal to alter the default image type of your screenshots:

For JPEG use:

$ defaults write com.apple.screencapture type JPG

For GIF use:

$ defaults write com.apple.screencapture type GIF

For PDF use:

$ defaults write com.apple.screencapture type PDF

For PNG use:

$ defaults write com.apple.screencapture type PNG

How to make an offline copy of a static website using wget and hosting on AWS S3 with CloudFront

I have an old website that I want to avoid the hosting costs and so just wanted to download the website and run it from an AWS S3 bucket using Cloud Front to publish the content. Below are the steps I took to do this:

First download the website to your laptop

$ wget \
     --recursive \
     --no-clobber \
     --page-requisites \
     --html-extension \
     --convert-links \
     --no-check-certificate \
     --restrict-file-names=unix \
     --domains archive.andrewbaker.ninja \
     --no-parent \
         http://archive.andrewbaker.ninja/
$ cd archive.andrewbaker.ninja
$ ls

Below is a summary of the parameters (inc common alternatives):

–recursive: Wget is capable of traversing parts of the Web (or a single HTTP or FTP server), following links and directory structure. We refer to this as to recursive retrieval, or recursion.

–no-clobber: If a file is downloaded more than once in the same directory, Wget’s behavior depends on a few options, including `-nc’. In certain cases, the local file will be clobbered, or overwritten, upon repeated download. In other cases it will be preserved. When running Wget without `-N’`-nc’, or `-r’, downloading the same file in the same directory will result in the original copy of file being preserved and the second copy being named `file.1′. If that file is downloaded yet again, the third copy will be named `file.2′, and so on. When `-nc’ is specified, this behavior is suppressed, and Wget will refuse to download newer copies of `file. Therefore, “no-clobber” is actually a misnomer in this mode–it’s not clobbering that’s prevented (as the numeric suffixes were already preventing clobbering), but rather the multiple version saving that’s prevented. When running Wget with `-r’, but without `-N’ or `-nc’, re-downloading a file will result in the new copy simply overwriting the old. Adding `-nc’ will prevent this behavior, instead causing the original version to be preserved and any newer copies on the server to be ignored. When running Wget with `-N’, with or without `-r’, the decision as to whether or not to download a newer copy of a file depends on the local and remote timestamp and size of the file (see section Time-Stamping). `-nc’ may not be specified at the same time as `-N’. Note that when `-nc’ is specified, files with the suffixes `.html’ or (yuck) `.htm’ will be loaded from the local disk and parsed as if they had been retrieved from the Web.

–page-requisites: This causes wget to download all the files that are necessary to properly display a given HTML page which includes images, css, js, etc. –adjust-extension Preserves proper file extensions for . html, . css, and other assets

–html-extension: This adds .html after the downloaded filename, to make sure it plays nicely on whatever system you’re going to view the archive on

–convert-links: After the download is complete, convert the links in the document to make them suitable for local viewing. This affects not only the visible hyperlinks, but any part of the document that links to external content, such as embedded images, links to style sheets, hyperlinks to non-HTML content, etc.

–no-check-certificate: Don’t check the server certificate against the available certificate authorities. Also don’t require the URL host name to match the common name presented by the certificate.

–restrict-file-names: By default, Wget escapes the characters that are not valid or safe as part of file names on your operating system, as well as control characters that are typically unprintable. This option is useful for changing these defaults, perhaps because you are downloading to a non-native partition”. So unless you are not downloading to non-native partition you do not need to restrict file names by OS. its automatic. Additionally: “The values ‘unix’ and ‘windows’ are mutually exclusive (one will override the other)”

–domains: Limit spanning to specified domains

–no-parent: If you don’t want wget to descend down to the parent directory, use -np or –no-parent option. This instructs wget not to ascend to the parent directory when it hits references like ../ in href links.

Upload Files to S3 Bucket

Next upload the files to your S3 bucket. First move into the relevant bucket, then perform the recursive upload.

$ cd archive.andrewbaker.ninja
$ ls .
$ aws s3 cp . s3://vbusers.com/ --recursive

Create a CloudFront Distribution from an S3 Bucket

Finally go to CloudFront and create a distribution from the S3 Bucket you just created. You can pretty much use the default settings. Note: you will need to wait a few minutes before you browse to the distributions domain name:

AWS: Automatically Stop and Start your EC2 Services

Below is a quick (am busy) outline on how to automatically stop and start your EC2 instances.

Step 1: Tag your resources

In order to decide which instances stop and start you first need to add an auto-start-stop: Yes tag to all the instances you want to be affected by the start / stop functions. Note: You can use “Resource Groups and Tag Editor” to bulk apply these tags to the resources you want to be affected by the lambda functions you are going to create. See below (click the orange button called “Manage tags of Selected Resources”).

Step 2: Create a new role for our lambda functions

First we need to create the IAM role to run the Lambda functions. Go to IAM and click the “Create Role” button. Then select “AWS Service” from the “Trusted entity options”, and select Lambda from the “Use Cases” options. Then click “Next”, followed by “Create Policy”. To specify the permission, simply Click the JSON button on the right of the screen and enter the below policy (swapping the region and account id for your region and account id):

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "VisualEditor0",
            "Effect": "Allow",
            "Action": [
                "ec2:DescribeInstances",
                "ec2:StartInstances",
                "ec2:DescribeTags",
                "logs:*",
                "ec2:DescribeInstanceTypes",
                "ec2:StopInstances",
                "ec2:DescribeInstanceStatus"
            ],
            "Resource": "arn:aws:ec2:<region>:<accountID>:instance/*",
            "Condition": {
                "StringEquals": {
                    "aws:ResourceTag/auto-start-stop": "Yes"
                }
            }
        }
    ]
}

Hit next and under “Review and create”, save the above policy as ec2-lambda-start-stop by clicking the “Create Policy” button. Next, search for this newly created policy and select it as per below and hit “Next”.

You will now see the “Name, review, and create” screen. Here you simply need to hit “Create Role” after you enter the role name as ec2-lambda-start-stop-role.

Note the policy is restricted to only have access to EC2 instances that contains auto-start-stop: Yes tags (least privileges).

If you want to review your role, this is how it should look. You can see I have filled in my region and account number in the policy:

Step 3: Create Lambda Functions To Start/Stop EC2 Instances

In this section we will create two lambda functions, one to start the instances and the other to stop the instances.

Step 3a: Add the Stop EC2 instance function

  • Goto Lambda console and click on create function
  • Create a lambda function with a function name of stop-ec2-instance-lambda, python3.11 runtime, and ec2-lambda-stop-start-role (see image below).

Next add the lamdba stop function and save it as stop-ec2-instance. Note, you will need to change the value of the region_name parameter accordingly.

import json
import boto3

ec2 = boto3.resource('ec2', region_name='af-south-1')
def lambda_handler(event, context):
   instances = ec2.instances.filter(Filters=[{'Name': 'instance-state-name', 'Values': ['running']},{'Name': 'tag:auto-start-stop','Values':['Yes']}])
   for instance in instances:
       id=instance.id
       ec2.instances.filter(InstanceIds=[id]).stop()
       print("Instance ID is stopped:- "+instance.id)
   return "success"

This is how your Lambda function should look:

Step 3b: Add the Start EC2 instance function

  • Goto Lambda console and click on create function
  • Create lambda functions with start-ec2-instance, python3.11 runtime, and ec2-lambda-stop-start-role.
  • Then add the below code and save the function as start-ec2-instance-lambda.

Note, you will need to change the value of the region_name parameter accordingly.

import json
import boto3

ec2 = boto3.resource('ec2', region_name='af-south-1')
def lambda_handler(event, context):
   instances = ec2.instances.filter(Filters=[{'Name': 'instance-state-name', 'Values': ['stopped']},{'Name': 'tag:auto-start-stop','Values':['Yes']}])
   for instance in instances:
       id=instance.id
       ec2.instances.filter(InstanceIds=[id]).stop()
       print("Instance ID is stopped:- "+instance.id)
   return "success"

4. Summary

If either of the above lambda functions are triggered, they will start or stop your EC2 instances based on the instance state and the value of auto-start-stop tag. To automate this you can simply setup up cron jobs, step functions, AWS Event Bridge, Jenkins etc.