Update to 2022-01-25 15:00

roles/elasticsearch/defaults/main.yml

@@ -1,5 +1,17 @@
 ---
 
+# Version of elasticsearch to deploy
+es_version: 7.16.3
+# root directory
+es_root_dir: /opt/elasticsearch
+# URL of the archive
+es_archive_url: https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{ es_version }}-linux-x86_64.tar.gz
+# Expected sha512 of the archive
+es_archive_sha512: d9ad7a510b8bad63788f5081b9431519e0581242499394f7a2c59f6097f8956603b28881e30697c50fe440b0ced7a2eb66afadb0e12bf97126db1d468d3818ff
+# Should ansible handle upgrades or only initial installation ?
+es_manage_upgrade: True
+# User under which the service will run (will be created)
+es_user: elasticsearch
 # Name of the Elasticsearch cluster
 es_cluster_name: elasticsearch
 # Name of this ES node
@@ -8,7 +20,6 @@ es_node_name: "{{ inventory_hostname }}"
 es_port: 9200
 # List of IP/CIDR which will have access to es_port (if iptables_manage == True)
 es_src_ip: []
-# Path where ES will store its data
-es_data_dir: /opt/elasticsearch/data
-# Path where ES will store snapshots for backups (created by pre-backup, removed by post-backup)
-es_backup_dir: /opt/elasticsearch/dumps
+# Define Xms / Xmx
+es_memory: 1g
+

roles/elasticsearch/meta/main.yml

@@ -1,5 +1,4 @@
 ---
 
 dependencies:
-  - role: repo_elasticsearch
   - role: mkdir

roles/elasticsearch/tasks/archive_post.yml

@@ -0,0 +1,10 @@
+---
+
+- name: Compress previous version
+  command: tar cf {{ es_root_dir }}/archives/{{ es_current_version }}.tar.zst ./ --use-compress-program=zstd
+  args:
+    chdir: "{{ es_root_dir }}/archives/{{ es_current_version }}"
+    warn: False
+  environment:
+    ZSTD_CLEVEL: 15
+  tags: es

roles/elasticsearch/tasks/archive_pre.yml

@@ -0,0 +1,15 @@
+---
+
+- name: Create the archive dir
+  file: path={{ es_root_dir }}/archives/{{ es_current_version }} state=directory
+  tags: es
+
+- name: Archive current version
+  synchronize:
+    src: "{{ es_root_dir }}/app"
+    dest: "{{ es_root_dir }}/archives/{{ es_current_version }}/"
+    compress: False
+    delete: True
+  delegate_to: "{{ inventory_hostname }}"
+  tags: es
+

roles/elasticsearch/tasks/backup.yml

@@ -8,11 +8,11 @@
       type: fs
       settings:
         compress: True
-        location: "{{ es_backup_dir }}"
+        location: "{{ es_root_dir }}/backup"
     body_format: json
   register: es_lbkp
   until: es_lbkp.failed == False
-  retries: 10
+  retries: 20
   delay: 10
   tags: es
 

roles/elasticsearch/tasks/cleanup.yml

@@ -0,0 +1,9 @@
+---
+
+- name: Remove tmp and obsolete files
+  file: path={{ item }} state=absent
+  loop:
+    - "{{ es_root_dir }}/tmp/elasticsearch-{{ es_version }}"
+    - "{{ es_root_dir }}/tmp/elasticsearch-{{ es_version }}-linux-x86_64.tar.gz"
+    - "{{ es_root_dir }}/archives/{{ es_current_version }}"
+  tags: es

roles/elasticsearch/tasks/conf.yml

@@ -1,9 +1,10 @@
 ---
 
 - name: Deploy configuration
-  template: src={{ item }}.j2 dest=/etc/elasticsearch/{{ item }} group=elasticsearch mode=660
+  template: src={{ item }}.j2 dest={{ es_root_dir }}/etc/{{ item }} group={{ es_user }} mode=660
   loop:
     - elasticsearch.yml
     - log4j2.properties
+    - jvm.options
   notify: restart elasticsearch
   tags: es

roles/elasticsearch/tasks/directories.yml

@@ -1,14 +1,37 @@
 ---
 
-- name: Ensure the data dir exists
-  file: path={{ es_data_dir }} state=directory
-  tags: es
-
-  # We do it in two steps, so that parent dirs aren't created with restrictive permissions
-- name: Restrict permissions on data dir
-  file: path={{ es_data_dir }} state=directory owner=elasticsearch group=elasticsearch mode=750
-  tags: es
-
-- name: Create backup dir
-  file: path={{ es_backup_dir }} state=directory owner=elasticsearch group=elasticsearch mode=700
+- name: Create directories
+  file: path={{ item.dir }} state=directory owner={{ item.owner | default(omit) }} group={{ item.group | default(omit) }} mode={{ item.mode | default(omit) }}
+  loop:
+    - dir: "{{ es_root_dir }}"
+      owner: "{{ es_user }}"
+      mode: 700
+    - dir: "{{ es_root_dir }}/backup"
+      owner: "{{ es_user }}"
+      group: "{{ es_user }}"
+      mode: 700
+    - dir: "{{ es_root_dir }}/archives"
+      owner: root
+      group: root
+      mode: 700
+    - dir: "{{ es_root_dir }}/meta"
+      owner: root
+      group: root
+      mode: 700
+    - dir: "{{ es_root_dir }}/data"
+      owner: "{{ es_user }}"
+      group: "{{ es_user }}"
+      mode: 700
+    - dir: "{{ es_root_dir }}/logs"
+      owner: "{{ es_user }}"
+      group: "{{ es_user }}"
+      mode: 700
+    - dir: "{{ es_root_dir }}/etc"
+      group: "{{ es_user }}"
+      mode: 770
+    - dir: "{{ es_root_dir }}/tmp"
+      owner: "{{ es_user }}"
+      group: "{{ es_user }}"
+      mode: 700
+    - dir: "{{ es_root_dir }}/app"
   tags: es

roles/elasticsearch/tasks/facts.yml

@@ -0,0 +1,21 @@
+---
+
+# Detect installed version (if any)
+- block:
+    - import_tasks: ../includes/webapps_set_install_mode.yml
+      vars:
+        - root_dir: "{{ es_root_dir }}"
+        - version: "{{ es_version }}"
+    - set_fact: es_install_mode={{ (install_mode == 'upgrade' and not es_manage_upgrade) | ternary('none',install_mode) }}
+    - set_fact: es_current_version={{ current_version | default('') }}
+  tags: es
+
+- name: Check if Seafile is installed
+  stat: path={{ seafile_root_dir | default('/opt/seafile') }}/seafile-server/pro/elasticsearch/
+  register: es_seafile_bundled_es
+  tags: es
+
+- name: Check if seafile is installed
+  stat: path=/etc/systemd/system/seafile.service
+  register: es_seafile_service
+  tags: es

roles/elasticsearch/tasks/install.yml

@@ -1,10 +1,64 @@
 ---
 
-- name: Install needed packages
-  yum:
-    name:
-      - elasticsearch-oss
-      - java-1.8.0-openjdk-headless
+- name: Stop the service during upgrades
+  service: name=elasticsearch state=stopped
+  when: es_install_mode == 'upgrade'
+  tags: es
+
+- name: Stop seafile to free elasticsearch port
+  service: name=seafile state=stopped
+  when: es_seafile_bundled_es.stat.exists and es_seafile_service.stat.exists
+  tags: es
+
+- when: es_install_mode != 'none'
+  block:
+    - name: Download Elasticsearch {{ es_version }}
+      get_url:
+        url: "{{ es_archive_url }}"
+        dest: "{{ es_root_dir }}/tmp"
+        checksum: sha512:{{ es_archive_sha512 }}
+
+    - name: Extract the archive
+      unarchive:
+        src: "{{ es_root_dir }}/tmp/elasticsearch-{{ es_version }}-linux-x86_64.tar.gz"
+        dest: "{{ es_root_dir }}/tmp"
+        remote_src: True
+
+    - name: Move Elasticsearch to its final dir
+      synchronize:
+        src: "{{ es_root_dir }}/tmp/elasticsearch-{{ es_version }}/"
+        dest: "{{ es_root_dir }}/app/"
+        delete: True
+        compress: False
+      delegate_to: "{{ inventory_hostname }}"
+
+    - name: Populate config dir
+      synchronize:
+        src: "{{ es_root_dir }}/tmp/elasticsearch-{{ es_version }}/config/"
+        dest: "{{ es_root_dir }}/etc/"
+        delete: True
+        compress: False
+      delegate_to: "{{ inventory_hostname }}"
+
+    - name: Set permissions
+      shell: |
+        {% if ansible_selinux.status == 'enabled' %}
+        restorecon -R {{ es_root_dir }}
+        {% endif %}
+        chown -R root:root {{ es_root_dir }}/app
+        chown -R :{{ es_user }} {{ es_root_dir }}/etc
+        chown -R {{ es_user }}:{{ es_user }} {{ es_root_dir }}/data/*
+        find {{ es_root_dir }}/etc -type d -exec chmod 770 "{}" \;
+        find {{ es_root_dir }}/etc -type f -exec chmod 660 "{}" \;
+        find {{ es_root_dir }}/app -type d -exec chmod 755 "{}" \;
+        find {{ es_root_dir }}/app -type f -exec chmod 644 "{}" \;
+        find {{ es_root_dir }}/app/jdk/bin -type f -exec chmod 755 "{}" \;
+        chmod 755 {{ es_root_dir }}/app/jdk/lib/jspawnhelper
+        find {{ es_root_dir }}/app/bin -type f -exec chmod 755 "{}" \;
+        find {{ es_root_dir }}/app/modules/x-pack-ml/platform/linux-x86_64/bin/ -type f -exec chmod 755 "{}" \;
+      args:
+        warn: False
+
   tags: es
 
 - name: Deploy pre and post backup script
@@ -14,29 +68,30 @@
     - post
   tags: es
 
-- name: Create systemd unit snippet dir
-  file: path=/etc/systemd/system/elasticsearch.service.d state=directory
+- name: Deploy tmpfile fragment
+  template: src=tmpfiles.conf.j2 dest=/etc/tmpfiles.d/elasticsearch.conf
+  register: es_tmpfiles
   tags: es
 
-- name: Customize systemd unit
-  copy:
-    content: |
-      [Service]
-      ProtectSystem=full
-      PrivateDevices=yes
-      ProtectHome=yes
-      NoNewPrivileges=yes
-      SyslogIdentifier=elasticsearch
-      Restart=on-failure
-      ExecStart=
-      ExecStart=/usr/share/elasticsearch/bin/elasticsearch -p ${PID_DIR}/elasticsearch.pid
-    dest: /etc/systemd/system/elasticsearch.service.d/ansible.conf
+- name: Create tmpfiles
+  command: systemd-tmpfiles --create
+  when: es_tmpfiles.changed
+  tags: es
+
+- name: Deploy systemd unit
+  template: src=elasticsearch.service.j2 dest=/etc/systemd/system/elasticsearch.service
   register: es_unit
   notify: restart elasticsearch
   tags: es
 
+- name: Remove previous systemd unit snippet
+  file: path=/etc/systemd/system/elasticsearch.service.d/ansible.conf state=absent
+  register: es_unit_snip
+  notify: restart elasticsearch
+  tags: es
+
 - name: Reload systemd
   systemd: daemon_reload=True
-  when: es_unit.changed
+  when: es_unit.changed or es_unit_snip.changed
   tags: es
 

roles/elasticsearch/tasks/main.yml

@@ -1,10 +1,18 @@
 ---
 
-- include: install.yml
+- include: remove_pkg.yml
+- include: user.yml
 - include: directories.yml
+- include: facts.yml
+- include: archive_pre.yml
+  when: es_install_mode == 'upgrade'
+- include: install.yml
 - include: conf.yml
 - include: iptables.yml
   when: iptables_manage | default(True)
 - include: services.yml
 - include: backup.yml
-
+- include: write_version.yml
+- include: archive_post.yml
+  when: es_install_mode == 'upgrade'
+- include: cleanup.yml

roles/elasticsearch/tasks/remove_pkg.yml

@@ -0,0 +1,9 @@
+---
+
+- name: Remove ES package
+  package:
+    name:
+      - elasticsearch-oss
+      - elasticsearch
+    state: absent
+  tags: es

roles/elasticsearch/tasks/user.yml

@@ -0,0 +1,9 @@
+---
+
+- name: Create elasticsearch user
+  user:
+    name: "{{ es_user }}"
+    home: "{{ es_root_dir }}"
+    system: True
+    shell: /bin/nologin
+  tags: es

roles/elasticsearch/tasks/write_version.yml

@@ -0,0 +1,5 @@
+---
+
+- name: Write installed version
+  copy: content={{ es_version }} dest={{ es_root_dir }}/meta/ansible_version
+  tags: es

roles/elasticsearch/templates/elasticsearch.service.j2

@@ -0,0 +1,49 @@
+[Unit]
+Description=Elasticsearch
+Documentation=https://www.elastic.co
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=simple
+RuntimeDirectory=elasticsearch
+PrivateTmp=true
+Environment=ES_HOME={{ es_root_dir }}/app
+Environment=ES_PATH_CONF={{ es_root_dir }}/etc
+Environment=ES_TMPDIR={{ es_root_dir }}/tmp
+Environment=PID_DIR=/run/elasticsearch
+WorkingDirectory={{ es_root_dir }}/app
+User={{ es_user }}
+Group={{ es_user }}
+
+ExecStart={{ es_root_dir }}/app/bin/elasticsearch -p ${PID_DIR}/elasticsearch.pid
+
+# StandardOutput is configured to redirect to journalctl since
+# some error messages may be logged in standard output before
+# elasticsearch logging system is initialized. Elasticsearch
+# stores its logs in {{ es_root_dir }}/log/
+StandardOutput=journal
+StandardError=inherit
+
+# Specifies the maximum file descriptor number that can be opened by this process
+LimitNOFILE=65535
+# Specifies the maximum number of processes
+LimitNPROC=4096
+# Specifies the maximum size of virtual memory
+LimitAS=infinity
+# Specifies the maximum file size
+LimitFSIZE=infinity
+# Give some time for the process to stop
+TimeoutStopSec=300
+# SIGTERM signal is used to stop the Java process
+KillSignal=SIGTERM
+# Send the signal only to the JVM rather than its control group
+KillMode=process
+# Java process is never killed
+SendSIGKILL=no
+# When a JVM receives a SIGTERM signal it exits with code 143
+SuccessExitStatus=143
+
+[Install]
+WantedBy=multi-user.target
+

roles/elasticsearch/templates/elasticsearch.yml.j2

@@ -1,11 +1,11 @@
 cluster.name: {{ es_cluster_name }}
+node.name: {{ es_node_name }}
+path.data: {{ es_root_dir }}/data
+path.logs: {{ es_root_dir }}/logs
 network.host: 0.0.0.0
 http.port: {{ es_port }}
-node.name: {{ es_node_name }}
-path.data: {{ es_data_dir }}
-path.logs: /var/log/elasticsearch
-path.repo: [ {{ es_backup_dir }} ]
+path.repo: [ {{ es_root_dir }}/backup ]
 action.auto_create_index: false
-{% if es_major_version is defined and es_major_version is version('7','>=') %}
+{% if es_version is version('7','>=') %}
 discovery.type: single-node
 {% endif %}

roles/elasticsearch/templates/jvm.options.j2

@@ -0,0 +1,46 @@
+-Xms{{ es_memory }}
+-Xmx{{ es_memory }}
+## GC configuration
+8-13:-XX:+UseConcMarkSweepGC
+8-13:-XX:CMSInitiatingOccupancyFraction=75
+8-13:-XX:+UseCMSInitiatingOccupancyOnly
+# G1GC Configuration
+# NOTE: G1 GC is only supported on JDK version 10 or later
+# to use G1GC, uncomment the next two lines and update the version on the
+# following three lines to your version of the JDK
+# 10-13:-XX:-UseConcMarkSweepGC
+# 10-13:-XX:-UseCMSInitiatingOccupancyOnly
+14-:-XX:+UseG1GC
+
+## JVM temporary directory
+-Djava.io.tmpdir=${ES_TMPDIR}
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails; heap dumps
+# are created in the working directory of the JVM unless an alternative path is
+# specified
+-XX:+HeapDumpOnOutOfMemoryError
+# exit right after heap dump on out of memory error. Recommended to also use
+# on java 8 for supported versions (8u92+).
+9-:-XX:+ExitOnOutOfMemoryError
+
+# specify an alternative path for heap dumps; ensure the directory exists and
+# has sufficient space
+-XX:HeapDumpPath={{ es_root_dir }}/data
+
+# specify an alternative path for JVM fatal error logs
+-XX:ErrorFile={{ es_root_dir }}/logs/hs_err_pid%p.log
+
+## JDK 8 GC logging
+8:-XX:+PrintGCDetails
+8:-XX:+PrintGCDateStamps
+8:-XX:+PrintTenuringDistribution
+8:-XX:+PrintGCApplicationStoppedTime
+8:-Xloggc:logs/gc.log
+8:-XX:+UseGCLogFileRotation
+8:-XX:NumberOfGCLogFiles=32
+8:-XX:GCLogFileSize=64m
+
+# JDK 9+ GC logging
+9-:-Xlog:gc*,gc+age=trace,safepoint:file={{ es_root_dir }}/logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m

roles/elasticsearch/templates/post-backup.j2

@@ -1,5 +1,4 @@
 #!/bin/bash -e
 
 curl -X DELETE http://localhost:{{ es_port }}/_snapshot/lbkp/lbkp
-umount /home/lbkp/es
 fstrim -a -v

roles/elasticsearch/templates/pre-backup.j2

@@ -2,6 +2,4 @@
 
 set -eo pipefail
 
-mkdir -p /home/lbkp/es
-mount -o bind,ro {{ es_backup_dir }} /home/lbkp/es
 curl -X PUT http://localhost:{{ es_port }}/_snapshot/lbkp/lbkp?wait_for_completion=true

roles/elasticsearch/templates/tmpfiles.conf.j2

@@ -0,0 +1 @@
+d /run/elasticsearch 700 {{ es_user }} {{ es_user }}

roles/seafile/defaults/main.yml

@@ -11,7 +11,7 @@
 #   MaxUsers = "9"
 #   Mode = "subscription"
 #   etc...
-seafile_version: "{{ seafile_license is defined | ternary('8.0.17','9.0.2') }}"
+seafile_version: "{{ seafile_license is defined | ternary('9.0.4','9.0.2') }}"
 
 # Archive URL and sha1 are only used for the community version
 seafile_archive_url: https://s3.eu-central-1.amazonaws.com/download.seadrive.org/seafile-server_{{ seafile_version }}_x86-64.tar.gz
@@ -31,6 +31,10 @@ seafile_db_seahub: seahub
 # Set to none to disable memcached
 seafile_memcached_server: 127.0.0.1:11211
 
+# Elasticsearch is only used with pro edition
+seafile_es_server: localhost
+seafile_es_port: 9200
+
 # Account under which services will run
 seafile_user: seafile
 seafile_group: "{{ seafile_user }}"
@@ -61,7 +65,7 @@ seafile_admin_pass: seafile
 
 # LDAP integration
 seafile_ldap_auth: "{{ (ad_auth | default(False) or ldap_auth | default(False)) | ternary(True,False) }}"
-seafile_ldap_uri: "{{ ad_auth | default(False) | ternary('ldaps://' + ad_realm | default(samba_realm) | default(ansible_domain) | lower,ldap_uri | default('ldap://ldap' ~ ansible_domain) | regex_replace('^ldap://(.*)','ldaps://\\1')) }}"
+seafile_ldap_uri: "{{ ad_auth | default(False) | ternary('ldaps://' + ad_realm | default(samba_realm) | default(ansible_domain) | lower,ldap_uri | default('ldap://ldap.' ~ ansible_domain) | regex_replace('^ldap://(.*)','ldaps://\\1')) }}"
 seafile_ldap_base: "{{ ad_auth | default(False) | ternary('DC=' + ad_realm | default(samba_realm) | default(ansible_domain) | regex_replace('\\.',',DC='), 'ou=Users,' + ldap_base) }}"
 # seafile_ldap_bind_dn:
 # Note that seafile_ldap_bind_pass should not contain special caracter, as it might break the parser and

roles/seafile/meta/main.yml

@@ -7,3 +7,9 @@ dependencies:
     when: seafile_license is defined and seafile_scan_av == True
   - role: mysql_server
     when: seafile_db_server in ['127.0.0.1', 'localhost']
+  - role: elasticsearch
+    vars:
+      es_version: 7.16.3
+    when:
+      - seafile_license is defined
+      - seafile_es_server in ['127.0.0.1', 'localhost']

roles/seafile/templates/seafevents.conf.j2

@@ -19,6 +19,9 @@ enabled = true
 enabled = true
 interval = 10m
 index_office_pdf = true
+external_es_server = true
+es_host = {{ seafile_es_server }}
+es_port = {{ seafile_es_port }}
 
 [OFFICE CONVERTER]
 enabled = true

roles/seafile/templates/seafile.service.j2

@@ -1,6 +1,6 @@
 [Unit]
 Description=Seafile
-After=network.target mariadb.service
+After=network.target mariadb.service elasticsearch.service
 
 [Service]
 Type=forking