More work on unifi, including a firmware downloader

images/mongo/Dockerfile

@@ -1,9 +1,9 @@
 FROM danielberteaud/alma:8
 MAINTAINER Daniel Berteaud <dbd@ehtrace.com>
 
-ARG MONGO_MAJOR=3.6
+ARG MONGO_MAJOR=4.0
 
-COPY mongodb.repo /etc/yum.repos.d/
+COPY root/ /
 RUN set -eux &&\
     sed -i -e "s/__MONGO_MAJOR__/${MONGO_MAJOR}/g" /etc/yum.repos.d/mongodb.repo &&\
     microdnf -y --best --nodocs --noplugins --setopt=install_weak_deps=0 update &&\

images/ubnt-firmware-downloader/Dockerfile

@@ -0,0 +1,16 @@
+FROM [[ .docker.repo ]][[ .docker.base_images.alpine.image ]]
+MAINTAINER [[ .docker.maintainer ]]
+
+ENV UBNT_UPDATE_API="https://fw-update.ubnt.com/api/firmware-latest?filter=eq~~product~~unifi-firmware&filter=eq~~channel~~release" \
+    UBNT_FIRMWARE_DIR="/opt/unifi/app/data/firmware/" \
+    UBNT_PLATFORMS=
+
+RUN set -eux &&\
+    apk --no-cache upgrade &&\
+    apk --no-cache add perl-libwww \
+                       perl-lwp-protocol-https \
+                       perl-json \
+                       supercronic
+
+COPY root/ /
+CMD ["ubnt-firmware-downloader.sh"]

images/ubnt-firmware-downloader/root/usr/local/bin/ubnt-firmware-downloader

@@ -0,0 +1,141 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use JSON;
+use LWP::UserAgent;
+use File::Basename;
+use File::Path qw(make_path);
+use Digest::SHA;
+use Data::Dumper;
+
+my $ubnt_api    = $ENV{UBNT_UPDATE_API} || 'https://fw-update.ubnt.com/api/firmware-latest?filter=eq~~product~~unifi-firmware&filter=eq~~channel~~release';
+my $fw_dir      = $ENV{UBNT_FIRMWARE_DIR} || '/opt/unifi/app/data/firmware/';
+my @dl_platform = split(/,/, $ENV{UBNT_PLATFORMS} || "");
+my $fw_cached   = {};
+my $fw_entries  = {};
+my $fw_devices  = {};
+my $fw_latest   = {};
+
+# Ensure the list is empty if UBNT_PLATFORMS was empty
+if (scalar @dl_platform > 0 and $dl_platform[0] eq "") {
+  @dl_platform = ();
+}
+
+if (scalar @dl_platform > 0){
+  print STDERR "Will only try to download firmware for platforms " . join(",", @dl_platform) . "\n";
+}
+
+# If the firmware_meta.json file exists, open it and read
+# already present firmwares
+if (-e "$fw_dir/firmware_meta.json"){
+  open CACHED_FIRMWARES, "<$fw_dir/firmware_meta.json";
+  my $data = <CACHED_FIRMWARES>;
+  my $json = eval {
+    from_json($data);
+  };
+  if (defined $json and defined $json->{cached_firmwares}){
+    foreach my $fw (@{$json->{cached_firmwares}}){
+      print STDERR "file $fw->{path} found\n";
+      if (-e "$fw_dir/$fw->{path}"){
+        $fw_entries->{$fw->{md5}} = $fw;
+      }
+    }
+  }
+  close CACHED_FIRMWARES;
+}
+
+my $ua = LWP::UserAgent->new(timeout => 10);
+$ua->env_proxy;
+
+# Ask ubnt API for the list of firmwares
+my $resp = $ua->get($ubnt_api);
+
+if (not $resp->is_success){
+  die "Couldn't fetch $ubnt_api : " . $resp->status_line . "\n";
+}
+
+my $firmwares = from_json($resp->decoded_content);
+
+FIRMWARE: foreach my $fw (@{$firmwares->{_embedded}->{firmware}}){
+  my $version = $fw->{version};
+  $version =~ s/^v//;
+  $version =~ s/\+/./g;
+  my $file = basename($fw->{_links}->{data}->{href});
+  my $fw_path = "$fw_dir/$fw->{platform}/$version/$file";
+  my $sha256;
+
+  # If the firmware is already cached, we need to check its checksum
+  if (-e "$fw_path"){
+    print STDERR "Found firmware for $fw->{platform} version $version, checking checksum of $fw_path\n";
+    $sha256 = Digest::SHA->new("sha256")->addfile($fw_path)->hexdigest;
+
+    if ($sha256 eq $fw->{sha256_checksum}){
+      print STDERR "Checksum matched, firmware already cached\n";
+
+      $fw_entries->{$fw->{md5}} = {
+        md5     => $fw->{md5},
+        version => $version,
+        size    => $fw->{file_size},
+        path    => "$fw->{platform}/$file"
+      };
+
+      push @{$fw_devices->{$fw->{md5}}}, $fw->{platform};
+      $fw_latest->{$fw->{md5}} = 1;
+
+      # File is OK, just continue with the next firmware
+      next FIRMWARE;
+    } else {
+      print STDERR "Checksum mismatch: got $sha256 while expecting $fw->{sha256_checksum}, downloading again\n";
+    }
+  }
+
+  # If we restrict the list of platform to download firmwares for
+  # check the current firmware matches
+  if (scalar @dl_platform > 0 and not grep { $_ eq $fw->{platform} } @dl_platform){
+    print STDERR "Platform $fw->{platform} is not in the list of downloads, skipping download\n";
+  } else {
+    print STDERR "Downloading firmware from $fw->{_links}->{data}->{href}\n";
+    make_path "$fw_dir/$fw->{platform}/$version/";
+    #$resp = getstore($fw->{_links}->{data}->{href}, $fw_path);
+    $resp = $ua->get($fw->{_links}->{data}->{href}, ":content_file" => $fw_path);
+    if (not $resp->is_success){
+      print STDERR "Error downloading $fw->{_links}->{data}->{href} : " . $resp->status_line ."\n";
+      next FIRMWARE;
+    }
+
+    $sha256 = Digest::SHA->new("sha256")->addfile($fw_path)->hexdigest;
+    if ($sha256 ne $fw->{sha256_checksum}){
+      print STDERR "Checksum mismatch : got $sha256 while expecting $fw->{sha256_checksum}\n";
+      next FIRMWARE;
+    } else {
+      print STDERR "Checksum correctly verified\n";
+      push @{$fw_devices->{$fw->{md5}}}, $fw->{platform};
+      $fw_latest->{$fw->{md5}} = 1;
+      $fw_entries->{$fw->{md5}} = {
+        md5     => $fw->{md5},
+        version => $version,
+        size    => $fw->{file_size},
+        path    => "$fw->{platform}/$file"
+      };
+    }
+  }
+
+  # In anycase, push this platform to the fw_devices hash
+  # so the list of devices will be complete
+  push @{$fw_devices->{$fw->{md5}}}, $fw->{platform};
+}
+
+print STDERR "Finished downloading firmwares, now building the firmware_meta.json file\n";
+foreach my $fw (sort { $fw_entries->{$a}->{version} cmp $fw_entries->{$b}->{version} } keys %{$fw_entries}){
+  my $firmware = $fw_entries->{$fw};
+  # Only override device list for latest firmwares (those returned by the ubnt API)
+  # for the previous one, we do not have fresh info, so just trust what was in the firmware_meta.json file
+  if (defined $fw_latest->{$fw_entries->{$fw}->{md5}}){
+    $firmware->{devices} = $fw_devices->{$fw};
+  }
+  push @{$fw_cached->{cached_firmwares}}, $firmware;
+}
+
+open CACHED_FIRMWARES, ">$fw_dir/firmware_meta.json";
+print CACHED_FIRMWARES to_json($fw_cached);

images/ubnt-firmware-downloader/root/usr/local/bin/ubnt-firmware-downloader.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -euo pipefail
+
+/usr/local/bin/ubnt-firmware-downloader
+# If a cron expression is defined, run a cron daemon
+if [ -n "${UBNT_CRON}" ]; then
+  echo "Running using cron with expression ${UBNT_CRON}"
+  cat <<_EOF > /tmp/crontab
+  ${UBNT_CRON} /usr/local/bin/ubnt-firmware-downloader
+_EOF
+  supercronic /tmp/crontab
+fi
+

images/unifi/Dockerfile

@@ -11,8 +11,8 @@ RUN set -eu &&\
     unzip UniFi.unix.zip &&\
     rm -f UniFi/bin/mongod
 
-FROM danielberteaud/java:11-alpine
-MAINTAINER Daniel Berteaud <dbd@ehtrace.com>
+FROM [[ .docker.repo ]][[ .docker.base_images.java11.image ]]
+MAINTAINER [[ .docker.maintainer ]]
 
 COPY --from=builder /tmp/UniFi /opt/unifi
 RUN set -eu &&\
@@ -27,7 +27,7 @@ RUN set -eu &&\
     ln -s /data/unifi /opt/unifi/data &&\
     ln -s /data/logs /opt/unifi/logs
 
-EXPOSE 8443 8080 3778
+EXPOSE 8443 8080 8843 3778
 USER unifi
 VOLUME /data
 

unifi.nomad.hcl

@@ -71,16 +71,17 @@ job "unifi" {
           grace = "3m"
         }
       }
-
     }
 
     volume "unifi-data" {
       type            = [[ .unifi.controller.volume.type | toJSON ]]
       source          = [[ .unifi.controller.volume.source | toJSON ]]
-      access_mode     = "single-node-writer"
+      access_mode     = "multi-node-multi-writer"
       attachment_mode = "file-system"
     }
 
+[[- if not .unifi.controller.mongo.is_external ]]
+
     volume "unifi-mongo" {
       type            = [[ .unifi.mongo.volume.type | toJSON ]]
       source          = [[ .unifi.mongo.volume.source | toJSON ]]
@@ -88,9 +89,11 @@ job "unifi" {
       attachment_mode = "file-system"
     }
 
+[[- end ]]
+
 [[ template "common/task.wait_for.tpl" dict 
   "ctx" .
-  "wait_for" (coll.Slice (dict "service" "unifi-mongo")) ]]
+  "wait_for" (coll.Slice (dict "service" .unifi.controller.mongo.service_name)) ]]
 
     task "nginx" {
       driver = [[ .unifi.nginx.driver | toJSON ]]
@@ -116,6 +119,32 @@ _EOF
 [[ template "common/resources.tpl" .unifi.nginx.resources ]]
     }
 
+    task "firmware-downloader" {
+      driver = [[ .unifi.fw_dl.driver | toJSON ]]
+      user = 8443
+
+      lifecycle {
+        hook = "poststart"
+        sidecar = true
+      }
+
+      config {
+        image = [[ .unifi.fw_dl.image | toJSON ]]
+      }
+
+      env {
+[[ template "common/env.tpl" .unifi.fw_dl.env ]]
+[[ template "common/proxy_env.tpl" . ]]
+      }
+
+      volume_mount {
+        volume      = "unifi-data"
+        destination = "/data"
+      }
+
+[[ template "common/resources.tpl" .unifi.fw_dl.resources ]]
+    }
+
     task "controller" {
 
       leader = true
@@ -171,6 +200,8 @@ _EOF
 
     }
 
+[[- if not .unifi.controller.mongo.is_external ]]
+
     task "mongo" {
       driver = [[ .unifi.mongo.driver | toJSON ]]
 
@@ -203,7 +234,7 @@ _EOF
 [[ template "common/resources.tpl" .unifi.mongo.resources ]]
 
     }
-
   }
+[[- end ]]
 }
 

variables.yml

@@ -12,8 +12,10 @@ unifi:
       cpu: 200
       memory: 1024
 
-    wait_for:
-      - service: unifi-mongo
+    mongo:
+      address: mongodb://127.0.0.1:27017/unifi
+      service_name: unifi-mongo
+      is_external: False
 
     env: {}
 
@@ -51,7 +53,6 @@ unifi:
       entrypoints:
         - unifi-portal
       middlewares: []
-        # - ip-guests@file
 
   stun:
     traefik:
@@ -66,7 +67,7 @@ unifi:
       memory: 15
 
   mongo:
-    image: danielberteaud/mongo:3.6
+    image: danielberteaud/mongo:4.0-1
     driver: docker
     resources:
       cpu: 100
@@ -74,3 +75,14 @@ unifi:
     volume:
       type: csi
       source: unifi-mongo
+
+  fw_dl:
+    image: danielberteaud/ubnt-firmware-downloader:latest
+    driver: docker
+    resources:
+      cpu: 10
+      memory: 64
+    env:
+      UBNT_FIRMWARE_DIR: /data/unifi/firmware
+      UBNT_PLATFORMS: U7HD,US48PRO
+      UBNT_CRON: 48 22 * * *