prologic revised this gist . Go to revision
1 file changed, 140 insertions
caddy_log_parser.sh(file created)
| @@ -0,0 +1,140 @@ | |||
| 1 | + | #!/bin/bash | |
| 2 | + | ||
| 3 | + | ### | |
| 4 | + | ### Caddy webserver JSON log parser | |
| 5 | + | ### | |
| 6 | + | ||
| 7 | + | show_help() { | |
| 8 | + | cat <<END | |
| 9 | + | Usage: $0 [-c|--common] [-C|--combined] [-f|--files] [-h|--help] filename | |
| 10 | + | ||
| 11 | + | Options: | |
| 12 | + | -c, --common Apache Common Log Format (default) | |
| 13 | + | -C, --combined Apache Combined Log Format | |
| 14 | + | -f, --files Custom Log Format, focused on file transfers | |
| 15 | + | -h, --help Show this help message and exit | |
| 16 | + | END | |
| 17 | + | } | |
| 18 | + | ||
| 19 | + | # Function to output logs in common log format | |
| 20 | + | output_common_log_format() { | |
| 21 | + | jq -r ' | |
| 22 | + | . as $log | | |
| 23 | + | $log.request.client_ip as $h | | |
| 24 | + | "-" as $l | | |
| 25 | + | ($log.user_id // "-") as $u | | |
| 26 | + | ($log.ts | tostring | split(".") | .[1][:3]) as $ms | | |
| 27 | + | ($log.ts | floor | todateiso8601 | sub("T"; " ") | sub("Z"; "." + $ms + " +0000")) as $t | | |
| 28 | + | "\($log.request.method) \($log.request.uri) \($log.request.proto)" as $r | | |
| 29 | + | $log.status as $s | | |
| 30 | + | ($log.size // "-") as $b | | |
| 31 | + | "\($h) \($l) \($u)[\($t)] \"\($r)\" \($s) \($b)" | |
| 32 | + | ' "${1}" | |
| 33 | + | } | |
| 34 | + | ||
| 35 | + | # Function to output logs in combined log format | |
| 36 | + | output_combined_log_format() { | |
| 37 | + | jq -r ' | |
| 38 | + | . as $log | | |
| 39 | + | $log.request.client_ip as $h | | |
| 40 | + | "-" as $l | | |
| 41 | + | ($log.user_id // "-") as $u | | |
| 42 | + | ($log.ts | tostring | split(".") | .[1][:3]) as $ms | | |
| 43 | + | ($log.ts | floor | todateiso8601 | sub("T"; " ") | sub("Z"; "." + $ms + " +0000")) as $t | | |
| 44 | + | "\($log.request.method) \($log.request.uri) \($log.request.proto)" as $r | | |
| 45 | + | $log.status as $s | | |
| 46 | + | ($log.size // "-") as $b | | |
| 47 | + | ($log.request.headers.Referer[0] // "-") as $referrer | | |
| 48 | + | ($log.request.headers["User-Agent"][0] // "-") as $user_agent | | |
| 49 | + | "\($h) \($l) \($u)[\($t)] \"\($r)\" \($s) \($b) \"\($referrer)\" \"\($user_agent)\"" | |
| 50 | + | ' "${1}" | |
| 51 | + | } | |
| 52 | + | ||
| 53 | + | output_file_log_format() { | |
| 54 | + | jq -r ' | |
| 55 | + | . as $log | | |
| 56 | + | $log.request.client_ip as $h | | |
| 57 | + | "-" as $l | | |
| 58 | + | ($log.user_id // "-") as $u | | |
| 59 | + | ($log.ts | tostring | split(".") | .[1][:3]) as $ms | | |
| 60 | + | ($log.ts | floor | todateiso8601 | sub("T"; " ") | sub("Z"; "." + $ms + " +0000")) as $t | | |
| 61 | + | "\($log.request.method) \($log.request.uri) \($log.request.proto)" as $r | | |
| 62 | + | $log.status as $s | | |
| 63 | + | ($log.size // "-") as $b | | |
| 64 | + | ($log.request.headers.Referer[0] // "-") as $referrer | | |
| 65 | + | ($log.request.headers["User-Agent"][0] // "-") as $user_agent | | |
| 66 | + | "\($h) \($l) \($u)[\($t)] \"\($r)\" \($s) \($b) \"\($referrer)\" \"\($user_agent)\"" | |
| 67 | + | ' "${1}" | |
| 68 | + | } | |
| 69 | + | ||
| 70 | + | # Function to list served files | |
| 71 | + | list_served_files() { | |
| 72 | + | jq -r ' | |
| 73 | + | .request.client_ip as $client_ip | | |
| 74 | + | (.ts | floor | todateiso8601 | sub("T"; " ") | sub("Z"; " +0000")) as $datetime | | |
| 75 | + | .status as $status_code | | |
| 76 | + | (.size // "-") as $size | | |
| 77 | + | .request.uri as $filename | | |
| 78 | + | "\($client_ip) [\($datetime)] \($status_code) \($size) \"\($filename)\"" | |
| 79 | + | ' "${1}" | |
| 80 | + | } | |
| 81 | + | ||
| 82 | + | # Default to common log format | |
| 83 | + | format="common" | |
| 84 | + | ||
| 85 | + | # Parse command line options | |
| 86 | + | while [[ "$#" -gt 0 ]]; do | |
| 87 | + | case $1 in | |
| 88 | + | -c|--common) | |
| 89 | + | format="common" | |
| 90 | + | shift | |
| 91 | + | ;; | |
| 92 | + | -C|--combined) | |
| 93 | + | format="combined" | |
| 94 | + | shift | |
| 95 | + | ;; | |
| 96 | + | -f|--files) | |
| 97 | + | format="files" | |
| 98 | + | shift | |
| 99 | + | ;; | |
| 100 | + | -h|--help) | |
| 101 | + | show_help | |
| 102 | + | exit 0 | |
| 103 | + | ;; | |
| 104 | + | *) | |
| 105 | + | input="${1}" | |
| 106 | + | shift | |
| 107 | + | ;; | |
| 108 | + | esac | |
| 109 | + | done | |
| 110 | + | ||
| 111 | + | # Check if a filename is provided | |
| 112 | + | if [ -z "${input}" ]; then | |
| 113 | + | echo -e "Error: No input file provided. \n" | |
| 114 | + | show_help | |
| 115 | + | exit 1 | |
| 116 | + | fi | |
| 117 | + | ||
| 118 | + | ||
| 119 | + | # Output the logs in the chosen format | |
| 120 | + | if [ "$format" = "common" ]; then | |
| 121 | + | output_common_log_format "${input}" | |
| 122 | + | elif [ "$format" = "files" ]; then | |
| 123 | + | list_served_files "${input}" | |
| 124 | + | else | |
| 125 | + | output_combined_log_format "${input}" | |
| 126 | + | fi | |
| 127 | + | ||
| 128 | + | # Apache Commong Log format: | |
| 129 | + | # "%h %l %u %t \"%r\" %>s %b" | |
| 130 | + | ||
| 131 | + | # Apache Combined LogFormat: | |
| 132 | + | # "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" | |
| 133 | + | ||
| 134 | + | # %h is the remote host (client IP) | |
| 135 | + | # %l is the remote logname (not used, so we'll use -) | |
| 136 | + | # %u is the authenticated user | |
| 137 | + | # %t is the time the request was received | |
| 138 | + | # %r is the request line from the client ("method uri proto") | |
| 139 | + | # %>s is the status code | |
| 140 | + | # %b is the size of the object returned to the client | |
Newer
Older