feat(loganalyzer): complete tool suite for analysing erupe logs.

2026-03-22 15:43:49 +01:00 · 2025-11-18 00:02:02 +01:00
parent 79636af608
commit 7aafc71dcc
12 changed files with 2241 additions and 0 deletions
--- a/tools/loganalyzer/parser.go
+++ b/tools/loganalyzer/parser.go
@@ -0,0 +1,315 @@
+// Package main provides a comprehensive suite of tools for analyzing Erupe server logs.
+//
+// The log analyzer supports both JSON-formatted logs and tab-delimited timestamp logs,
+// providing commands for filtering, error analysis, connection tracking, statistics
+// generation, and real-time log following.
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+)
+
+// LogEntry represents a parsed log entry from either JSON or timestamp-based format.
+//
+// The parser supports two log formats:
+//   1. JSON format: {"level":"info","ts":1762989571.547817,"logger":"main","msg":"Starting"}
+//   2. Timestamp format: 2025-11-12T23:19:31.546Z	INFO	commands	Command Help: Enabled
+type LogEntry struct {
+	Raw        string                 // Original log line
+	Level      string                 // Log level: info, warn, error, fatal
+	Timestamp  time.Time              // Parsed timestamp
+	Logger     string                 // Logger name
+	Caller     string                 // Caller file:line
+	Message    string                 // Log message
+	Error      string                 // Error message (if present)
+	StackTrace string                 // Stack trace (if present)
+	ExtraData  map[string]interface{} // Additional fields
+	IsJSON     bool                   // True if parsed from JSON format
+}
+
+// ParseLogFile reads and parses an entire log file into memory.
+//
+// This function loads all log entries into memory and is suitable for smaller log files
+// or when random access to entries is needed. For large files or streaming operations,
+// use StreamLogFile instead.
+//
+// The function automatically handles both JSON and timestamp-based log formats,
+// skips empty lines and "nohup: ignoring input" messages, and uses a large buffer
+// (1MB) to handle long lines like stack traces.
+//
+// Parameters:
+//   - filename: Path to the log file to parse
+//
+// Returns:
+//   - A slice of LogEntry pointers containing all parsed entries
+//   - An error if the file cannot be opened or read
+//
+// Example:
+//   entries, err := ParseLogFile("erupe.log")
+//   if err != nil {
+//       log.Fatal(err)
+//   }
+//   fmt.Printf("Parsed %d entries\n", len(entries))
+func ParseLogFile(filename string) ([]*LogEntry, error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open log file: %w", err)
+	}
+	defer file.Close()
+
+	var entries []*LogEntry
+	scanner := bufio.NewScanner(file)
+
+	// Increase buffer size for long lines (like stack traces)
+	const maxCapacity = 1024 * 1024 // 1MB
+	buf := make([]byte, maxCapacity)
+	scanner.Buffer(buf, maxCapacity)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line == "" || line == "nohup: ignoring input" {
+			continue
+		}
+
+		entry := ParseLogLine(line)
+		if entry != nil {
+			entries = append(entries, entry)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("error reading log file: %w", err)
+	}
+
+	return entries, nil
+}
+
+// ParseLogLine parses a single log line into a LogEntry.
+//
+// This function attempts to parse the line in the following order:
+//   1. JSON format: Lines starting with '{' are parsed as JSON objects
+//   2. Timestamp format: Tab-delimited lines with RFC3339 timestamps
+//   3. Unknown format: Lines that don't match either format are marked as "unknown" level
+//
+// For JSON logs, all standard fields (level, ts, logger, caller, msg, error, stacktrace)
+// are extracted, and any additional fields are stored in ExtraData.
+//
+// Parameters:
+//   - line: A single line from the log file
+//
+// Returns:
+//   - A LogEntry pointer containing the parsed data, or nil if the line is invalid
+//
+// Example:
+//   entry := ParseLogLine(`{"level":"info","ts":1762989571.547817,"msg":"Starting"}`)
+//   fmt.Println(entry.Level, entry.Message)
+func ParseLogLine(line string) *LogEntry {
+	entry := &LogEntry{
+		Raw:       line,
+		ExtraData: make(map[string]interface{}),
+	}
+
+	// Try parsing as JSON first
+	if strings.HasPrefix(line, "{") {
+		var jsonData map[string]interface{}
+		if err := json.Unmarshal([]byte(line), &jsonData); err == nil {
+			entry.IsJSON = true
+
+			// Extract standard fields
+			if level, ok := jsonData["level"].(string); ok {
+				entry.Level = level
+			}
+
+			if ts, ok := jsonData["ts"].(float64); ok {
+				entry.Timestamp = time.Unix(int64(ts), int64((ts-float64(int64(ts)))*1e9))
+			}
+
+			if logger, ok := jsonData["logger"].(string); ok {
+				entry.Logger = logger
+			}
+
+			if caller, ok := jsonData["caller"].(string); ok {
+				entry.Caller = caller
+			}
+
+			if msg, ok := jsonData["msg"].(string); ok {
+				entry.Message = msg
+			}
+
+			if errMsg, ok := jsonData["error"].(string); ok {
+				entry.Error = errMsg
+			}
+
+			if stackTrace, ok := jsonData["stacktrace"].(string); ok {
+				entry.StackTrace = stackTrace
+			}
+
+			// Store any extra fields
+			for k, v := range jsonData {
+				if k != "level" && k != "ts" && k != "logger" && k != "caller" &&
+				   k != "msg" && k != "error" && k != "stacktrace" {
+					entry.ExtraData[k] = v
+				}
+			}
+
+			return entry
+		}
+	}
+
+	// Try parsing as timestamp-based log (2025-11-12T23:19:31.546Z INFO commands ...)
+	parts := strings.SplitN(line, "\t", 4)
+	if len(parts) >= 3 {
+		// Parse timestamp
+		if ts, err := time.Parse(time.RFC3339Nano, parts[0]); err == nil {
+			entry.Timestamp = ts
+			entry.Level = strings.ToLower(parts[1])
+			entry.Logger = parts[2]
+			if len(parts) == 4 {
+				entry.Message = parts[3]
+			}
+			return entry
+		}
+	}
+
+	// If we can't parse it, return a basic entry
+	entry.Level = "unknown"
+	entry.Message = line
+	return entry
+}
+
+// StreamLogFile reads a log file line by line and calls the callback for each entry.
+//
+// This function is memory-efficient and suitable for processing large log files as it
+// processes entries one at a time without loading the entire file into memory. The
+// callback function is called for each successfully parsed log entry.
+//
+// The function uses a 1MB buffer to handle long lines such as those containing stack traces.
+// Empty lines and "nohup: ignoring input" messages are automatically skipped.
+//
+// If the callback returns an error, processing stops immediately and that error is returned.
+//
+// Parameters:
+//   - filename: Path to the log file to process
+//   - callback: Function to call for each parsed LogEntry
+//
+// Returns:
+//   - An error if the file cannot be opened, read, or if the callback returns an error
+//
+// Example:
+//   err := StreamLogFile("erupe.log", func(entry *LogEntry) error {
+//       if entry.Level == "error" {
+//           fmt.Println(entry.Message)
+//       }
+//       return nil
+//   })
+func StreamLogFile(filename string, callback func(*LogEntry) error) error {
+	file, err := os.Open(filename)
+	if err != nil {
+		return fmt.Errorf("failed to open log file: %w", err)
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+
+	// Increase buffer size for long lines
+	const maxCapacity = 1024 * 1024 // 1MB
+	buf := make([]byte, maxCapacity)
+	scanner.Buffer(buf, maxCapacity)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line == "" || line == "nohup: ignoring input" {
+			continue
+		}
+
+		entry := ParseLogLine(line)
+		if entry != nil {
+			if err := callback(entry); err != nil {
+				return err
+			}
+		}
+	}
+
+	return scanner.Err()
+}
+
+// FormatLogEntry formats a log entry for human-readable display.
+//
+// The output format is: "TIMESTAMP LEVEL [LOGGER] MESSAGE key=value ..."
+//
+// Timestamps are formatted as "2006-01-02 15:04:05.000". Log levels can be colorized
+// for terminal display:
+//   - Errors (error, fatal, panic): Red
+//   - Warnings: Yellow
+//   - Info: Green
+//
+// If the entry contains an error message, it's appended as error="message".
+// Any extra fields in ExtraData are appended as key=value pairs.
+//
+// Parameters:
+//   - entry: The LogEntry to format
+//   - colorize: Whether to add ANSI color codes for terminal display
+//
+// Returns:
+//   - A formatted string representation of the log entry
+//
+// Example:
+//   formatted := FormatLogEntry(entry, true)
+//   fmt.Println(formatted)
+//   // Output: 2025-11-12 23:19:31.546 INFO [main] Starting Erupe
+func FormatLogEntry(entry *LogEntry, colorize bool) string {
+	var sb strings.Builder
+
+	// Format timestamp
+	if !entry.Timestamp.IsZero() {
+		sb.WriteString(entry.Timestamp.Format("2006-01-02 15:04:05.000"))
+		sb.WriteString(" ")
+	}
+
+	// Format level with colors
+	levelStr := strings.ToUpper(entry.Level)
+	if colorize {
+		switch entry.Level {
+		case "error", "fatal", "panic":
+			levelStr = fmt.Sprintf("\033[31m%s\033[0m", levelStr) // Red
+		case "warn":
+			levelStr = fmt.Sprintf("\033[33m%s\033[0m", levelStr) // Yellow
+		case "info":
+			levelStr = fmt.Sprintf("\033[32m%s\033[0m", levelStr) // Green
+		}
+	}
+	sb.WriteString(fmt.Sprintf("%-5s ", levelStr))
+
+	// Format logger
+	if entry.Logger != "" {
+		sb.WriteString(fmt.Sprintf("[%s] ", entry.Logger))
+	}
+
+	// Format message
+	sb.WriteString(entry.Message)
+
+	// Add error if present
+	if entry.Error != "" {
+		sb.WriteString(fmt.Sprintf(" error=%q", entry.Error))
+	}
+
+	// Add extra data
+	if len(entry.ExtraData) > 0 {
+		sb.WriteString(" ")
+		first := true
+		for k, v := range entry.ExtraData {
+			if !first {
+				sb.WriteString(" ")
+			}
+			sb.WriteString(fmt.Sprintf("%s=%v", k, v))
+			first = false
+		}
+	}
+
+	return sb.String()
+}