implement logger with loki support

Cargo.lock

@@ -1807,6 +1807,15 @@ version = "2.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f"
 
+[[package]]
+name = "itertools"
+version = "0.10.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.6"
@@ -1890,11 +1899,24 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
 name = "logger"
 version = "1.0.1"
 dependencies = [
+ "anyhow",
  "serde",
+ "tokio",
  "tracing",
+ "tracing-loki",
  "tracing-subscriber",
 ]
 
+[[package]]
+name = "loki-api"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f56d36f573486ba7f462b62cbae597fef7d5d93665e7047956b457531b8a1ced"
+dependencies = [
+ "prost",
+ "prost-types",
+]
+
 [[package]]
 name = "lru-cache"
 version = "0.1.2"
@@ -2504,6 +2526,38 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "prost"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd"
+dependencies = [
+ "bytes",
+ "prost-derive",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4"
+dependencies = [
+ "anyhow",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13"
+dependencies = [
+ "prost",
+]
+
 [[package]]
 name = "quick-error"
 version = "1.2.3"
@@ -3542,6 +3596,17 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-stream"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af"
+dependencies = [
+ "futures-core",
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-tungstenite"
 version = "0.21.0"
@@ -3714,6 +3779,17 @@ dependencies = [
  "valuable",
 ]
 
+[[package]]
+name = "tracing-log"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2"
+dependencies = [
+ "log",
+ "once_cell",
+ "tracing-core",
+]
+
 [[package]]
 name = "tracing-log"
 version = "0.2.0"
@@ -3725,6 +3801,37 @@ dependencies = [
  "tracing-core",
 ]
 
+[[package]]
+name = "tracing-loki"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49bbc87d08020d7c2a9f4bb0b7d10da5381d3867f8ae57fcc54621b34567e963"
+dependencies = [
+ "loki-api",
+ "reqwest 0.11.27",
+ "serde",
+ "serde_json",
+ "snap",
+ "tokio",
+ "tokio-stream",
+ "tracing",
+ "tracing-core",
+ "tracing-log 0.1.4",
+ "tracing-serde",
+ "tracing-subscriber",
+ "url",
+]
+
+[[package]]
+name = "tracing-serde"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1"
+dependencies = [
+ "serde",
+ "tracing-core",
+]
+
 [[package]]
 name = "tracing-subscriber"
 version = "0.3.18"
@@ -3736,7 +3843,7 @@ dependencies = [
  "smallvec",
  "thread_local",
  "tracing-core",
- "tracing-log",
+ "tracing-log 0.2.0",
 ]
 
 [[package]]

Cargo.toml

@@ -61,6 +61,7 @@ thiserror = "1.0.58"
 # LOGGING
 tracing = "0.1.40"
 tracing-subscriber = "0.3.18"
+tracing-loki = "0.2.4"
 
 # CONFIG
 clap = { version = "4.5.3", features = ["derive"] }

bin/alert_logger/src/main.rs

@@ -5,7 +5,10 @@ use std::{net::SocketAddr, str::FromStr};
 
 use anyhow::Context;
 use axum::{routing::post, Json, Router};
-use monitor_client::entities::{alert::Alert, server::stats::SeverityLevel};
+use logger::LogConfig;
+use monitor_client::entities::{
+  alert::Alert, server::stats::SeverityLevel,
+};
 use serde::Deserialize;
 use termination_signal::tokio::immediate_term_handle;
 
@@ -21,7 +24,10 @@ fn default_port() -> u16 {
 
 async fn app() -> anyhow::Result<()> {
   dotenv::dotenv().ok();
-  logger::init(tracing::Level::INFO);
+  logger::init(LogConfig {
+    stdio: true,
+    ..Default::default()
+  })?;
 
   let Env { port } =
     envy::from_env().context("failed to parse env")?;
@@ -49,7 +55,7 @@ async fn app() -> anyhow::Result<()> {
   let listener = tokio::net::TcpListener::bind(socket_addr)
     .await
     .context("failed to bind tcp listener")?;
-  
+
   axum::serve(listener, app).await.context("server crashed")
 }
 

bin/core/src/main.rs

@@ -5,6 +5,7 @@ use std::{net::SocketAddr, str::FromStr};
 
 use anyhow::Context;
 use axum::Router;
+use logger::LogConfig;
 use termination_signal::tokio::immediate_term_handle;
 use tower_http::{
   cors::{Any, CorsLayer},
@@ -28,7 +29,11 @@ mod ws;
 async fn app() -> anyhow::Result<()> {
   dotenv::dotenv().ok();
   let config = core_config();
-  logger::init(config.log_level);
+  logger::init(LogConfig {
+    stdio: true,
+    level: config.log_level,
+    ..Default::default()
+  })?;
   info!("monitor core version: v{}", env!("CARGO_PKG_VERSION"));
 
   // Spawn monitoring loops

bin/migrator/src/main.rs

@@ -2,6 +2,8 @@
 #[macro_use]
 extern crate tracing;
 
+use logger::LogConfig;
+
 use crate::state::State;
 
 mod legacy;
@@ -10,7 +12,10 @@ mod state;
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
-  logger::init(tracing::Level::INFO);
+  logger::init(LogConfig {
+    stdio: true,
+    ..Default::default()
+  })?;
 
   info!("starting migrator");
 

bin/periphery/src/main.rs

@@ -5,6 +5,7 @@ use std::{net::SocketAddr, str::FromStr};
 
 use anyhow::Context;
 use axum::{middleware, routing::post, Router};
+use logger::LogConfig;
 use termination_signal::tokio::immediate_term_handle;
 
 mod api;
@@ -19,7 +20,10 @@ struct State;
 async fn app() -> anyhow::Result<()> {
   dotenv::dotenv().ok();
   let config = config::periphery_config();
-  logger::init(config.log_level);
+  logger::init(LogConfig {
+    stdio: true,
+    ..Default::default()
+  })?;
 
   info!("version: v{}", env!("CARGO_PKG_VERSION"));
 

bin/tests/src/main.rs

@@ -1,6 +1,7 @@
 #[macro_use]
 extern crate tracing;
 
+use logger::LogConfig;
 use rand::{distributions::Alphanumeric, thread_rng, Rng};
 
 mod core;
@@ -8,7 +9,10 @@ mod core;
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
-  logger::init(logger::LogLevel::Info);
+  logger::init(LogConfig {
+    stdio: true,
+    ..Default::default()
+  })?;
   // periphery::tests().await?;
   core::tests().await?;
 

bin/update_logger/src/main.rs

@@ -1,11 +1,15 @@
 #[macro_use]
 extern crate tracing;
 
+use logger::LogConfig;
 use monitor_client::MonitorClient;
 use termination_signal::tokio::immediate_term_handle;
 
 async fn app() -> anyhow::Result<()> {
-  logger::init(tracing::Level::INFO);
+  logger::init(LogConfig {
+    stdio: true,
+    ..Default::default()
+  })?;
 
   info!("v {}", env!("CARGO_PKG_VERSION"));
 

lib/logger/Cargo.toml

@@ -8,6 +8,9 @@ license.workspace = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+tokio.workspace = true
 serde.workspace = true
+anyhow.workspace = true
 tracing.workspace = true
+tracing-loki.workspace = true
 tracing-subscriber.workspace = true

lib/logger/src/lib.rs

@@ -1,11 +1,74 @@
+use anyhow::Context;
 use serde::{Deserialize, Serialize};
 use tracing::level_filters::LevelFilter;
+use tracing_loki::url::Url;
+use tracing_subscriber::{
+  layer::SubscriberExt, util::SubscriberInitExt,
+};
 
-pub fn init(log_level: impl Into<tracing::Level>) {
-  let log_level: tracing::Level = log_level.into();
-  tracing_subscriber::fmt()
-    .with_max_level(LevelFilter::from(log_level))
-    .init()
+#[derive(Debug, Clone, Default, Deserialize)]
+pub struct LogConfig {
+  /// Whether to log to stdout / stderr with tracing_subscriber::fmt().init(). default: true.
+  #[serde(default = "default_stdio")]
+  pub stdio: bool,
+
+  /// The logging level. default: info
+  #[serde(default)]
+  pub level: LogLevel,
+
+  /// Send tracing logs to loki
+  pub loki_url: Option<String>,
+}
+
+fn default_stdio() -> bool {
+  true
+}
+
+pub fn init(config: LogConfig) -> anyhow::Result<()> {
+  let log_level: tracing::Level = config.level.into();
+
+  match (config.stdio, config.loki_url) {
+    // Both loki and stdio
+    (true, Some(loki_url)) => {
+      let (loki_layer, task) = tracing_loki::builder()
+        .label("host", "mine")?
+        .build_url(Url::parse(&loki_url)?)?;
+      tokio::spawn(task);
+
+      tracing_subscriber::registry()
+        .with(LevelFilter::from(log_level))
+        .with(tracing_subscriber::fmt::Layer::new())
+        .with(loki_layer)
+        .try_init()
+        .context("failed to init logger")
+    }
+
+    // Just stdio
+    (true, None) => tracing_subscriber::registry()
+      .with(LevelFilter::from(log_level))
+      .with(tracing_subscriber::fmt::Layer::new())
+      .try_init()
+      .context("failed to init logger"),
+
+    // Just loki
+    (false, Some(loki_url)) => {
+      let (loki_layer, task) = tracing_loki::builder()
+        .label("host", "mine")?
+        .build_url(Url::parse(&loki_url)?)?;
+      tokio::spawn(task);
+      tracing_subscriber::registry()
+        .with(LevelFilter::from(log_level))
+        .with(loki_layer)
+        .try_init()
+        .context("failed to init logger")
+    }
+
+    // Neither (not recommended)
+    (false, None) => tracing_subscriber::registry()
+      .with(LevelFilter::from(log_level))
+      .try_init()
+      .context("failed to init logger"),
+  }
 }
 
 #[derive(