periphery runs with logs if couldn't connect to docker daemon

bin/periphery/src/api/container.rs

@@ -30,7 +30,12 @@ impl Resolve<super::Args> for InspectContainer {
     self,
     _: &super::Args,
   ) -> anyhow::Result<Container> {
-    docker_client().inspect_container(&self.name).await
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    client.inspect_container(&self.name).await
   }
 }
 
@@ -106,7 +111,12 @@ impl Resolve<super::Args> for GetFullContainerStats {
     self,
     _: &super::Args,
   ) -> anyhow::Result<FullContainerStats> {
-    docker_client().full_container_stats(&self.name).await
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    client.full_container_stats(&self.name).await
   }
 }
 
@@ -279,7 +289,12 @@ impl Resolve<super::Args> for StartAllContainers {
     self,
     _: &super::Args,
   ) -> anyhow::Result<Vec<Log>> {
-    let containers = docker_client()
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    let containers = client
       .list_containers()
       .await
       .context("failed to list all containers on host")?;
@@ -308,7 +323,12 @@ impl Resolve<super::Args> for RestartAllContainers {
     self,
     _: &super::Args,
   ) -> anyhow::Result<Vec<Log>> {
-    let containers = docker_client()
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    let containers = client
       .list_containers()
       .await
       .context("failed to list all containers on host")?;
@@ -337,7 +357,12 @@ impl Resolve<super::Args> for PauseAllContainers {
     self,
     _: &super::Args,
   ) -> anyhow::Result<Vec<Log>> {
-    let containers = docker_client()
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    let containers = client
       .list_containers()
       .await
       .context("failed to list all containers on host")?;
@@ -366,7 +391,12 @@ impl Resolve<super::Args> for UnpauseAllContainers {
     self,
     _: &super::Args,
   ) -> anyhow::Result<Vec<Log>> {
-    let containers = docker_client()
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    let containers = client
       .list_containers()
       .await
       .context("failed to list all containers on host")?;
@@ -395,7 +425,12 @@ impl Resolve<super::Args> for StopAllContainers {
     self,
     _: &super::Args,
   ) -> anyhow::Result<Vec<Log>> {
-    let containers = docker_client()
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    let containers = client
       .list_containers()
       .await
       .context("failed to list all containers on host")?;

bin/periphery/src/api/docker.rs

@@ -1,5 +1,6 @@
 use std::sync::OnceLock;
 
+use anyhow::Context;
 use cache::TimeoutCache;
 use command::run_komodo_command;
 use komodo_client::entities::{
@@ -24,7 +25,12 @@ use crate::docker::{docker_client, docker_login};
 impl Resolve<super::Args> for InspectImage {
   #[instrument(name = "InspectImage", level = "debug")]
   async fn resolve(self, _: &super::Args) -> anyhow::Result<Image> {
-    docker_client().inspect_image(&self.name).await
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    client.inspect_image(&self.name).await
   }
 }
 
@@ -36,7 +42,12 @@ impl Resolve<super::Args> for ImageHistory {
     self,
     _: &super::Args,
   ) -> anyhow::Result<Vec<ImageHistoryResponseItem>> {
-    docker_client().image_history(&self.name).await
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    client.image_history(&self.name).await
   }
 }
 
@@ -125,7 +136,12 @@ impl Resolve<super::Args> for PruneImages {
 impl Resolve<super::Args> for InspectNetwork {
   #[instrument(name = "InspectNetwork", level = "debug")]
   async fn resolve(self, _: &super::Args) -> anyhow::Result<Network> {
-    docker_client().inspect_network(&self.name).await
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    client.inspect_network(&self.name).await
   }
 }
 
@@ -171,7 +187,12 @@ impl Resolve<super::Args> for PruneNetworks {
 impl Resolve<super::Args> for InspectVolume {
   #[instrument(name = "InspectVolume", level = "debug")]
   async fn resolve(self, _: &super::Args) -> anyhow::Result<Volume> {
-    docker_client().inspect_volume(&self.name).await
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    client.inspect_volume(&self.name).await
   }
 }
 

bin/periphery/src/api/mod.rs

@@ -189,19 +189,22 @@ impl Resolve<Args> for PollStatus {
   ) -> anyhow::Result<PollStatusResponse> {
     // Docker lists
     let docker_lists = async {
-      let docker = docker_client();
+      let client = docker_client().load();
+      let Some(client) = client.iter().next() else {
+        return Default::default();
+      };
       let containers =
-        docker.list_containers().await.unwrap_or_default();
+        client.list_containers().await.unwrap_or_default();
       // Todo: handle errors better
       (
         tokio::join!(
-          docker
+          client
             .list_networks(&containers)
             .map(Result::unwrap_or_default),
-          docker
+          client
             .list_images(&containers)
             .map(Result::unwrap_or_default),
-          docker
+          client
             .list_volumes(&containers)
             .map(Result::unwrap_or_default)
         ),

bin/periphery/src/docker/mod.rs

@@ -1,6 +1,7 @@
-use std::sync::OnceLock;
+use std::sync::{Arc, OnceLock};
 
-use anyhow::anyhow;
+use anyhow::{Context, anyhow};
+use arc_swap::ArcSwap;
 use bollard::Docker;
 use command::run_komodo_command;
 use komodo_client::entities::{TerminationSignal, update::Log};
@@ -13,9 +14,41 @@ mod images;
 mod networks;
 mod volumes;
 
-pub fn docker_client() -> &'static DockerClient {
-  static DOCKER_CLIENT: OnceLock<DockerClient> = OnceLock::new();
-  DOCKER_CLIENT.get_or_init(Default::default)
+pub fn docker_client() -> &'static SwappableDockerClient {
+  static DOCKER_CLIENT: OnceLock<SwappableDockerClient> =
+    OnceLock::new();
+  DOCKER_CLIENT.get_or_init(SwappableDockerClient::init)
+}
+
+#[derive(Default)]
+pub struct SwappableDockerClient(ArcSwap<Option<DockerClient>>);
+
+impl SwappableDockerClient {
+  pub fn init() -> Self {
+    let docker = Docker::connect_with_defaults()
+      .context("Failed to connect to docker api. Docker monitoring won't work and will return empty results.")
+      // Only logs on first init, although keeps trying to connect
+      .inspect_err(|e| warn!("{e:#}"))
+      .map(|docker| DockerClient { docker })
+      .ok();
+    Self(ArcSwap::new(Arc::new(docker)))
+  }
+
+  pub fn load(&self) -> arc_swap::Guard<Arc<Option<DockerClient>>> {
+    let res = self.0.load();
+    if res.is_some() {
+      return res;
+    }
+    self.reload();
+    self.0.load()
+  }
+
+  pub fn reload(&self) {
+    let docker = Docker::connect_with_defaults()
+      .map(|docker| DockerClient { docker })
+      .ok();
+    self.0.store(Arc::new(docker));
+  }
 }
 
 pub struct DockerClient {