commit 56da6814eb5e3ce18b8ec2979f489e03fb484865
Author: Mark Goddard <mark@stackhpc.com>
Date:   Thu Jul 2 18:17:10 2020 +0100

    Performance: use a single config file for fluentd
    
    Backport note: moved filter directly into common role since Python
    import of kolla_ansible modules from Ansible does not work in Stein.
    
    Currently we generate multiple fluentd configuration files for inputs,
    filters, formatters and outputs.
    These are then included from the main td-agent.conf configuration file.
    With a large number of hosts, this can take a long time to template.
    
    Benchmarking of templating is available at [1].
    
    This change switches to a single fluentd configuration file, with the
    include done locally. For the default template files included with Kolla
    Ansible we use Jinja includes, but this does not work with templates in
    a different directory. We therefore use the Ansible template lookup
    plugin, which has a slightly higher overhead than a jinja include, but
    far lower than generating multiple templates. This should drastically
    improve the performance of this task.
    
    [1] https://github.com/stackhpc/ansible-scaling/blob/master/doc/template.md
    
    Partially-Implements: blueprint performance-improvements
    
    Change-Id: Ia8623be0aa861fea3e54d2c9e1c971dfd8e3afa9
    (cherry picked from commit 56a07702bcea02cf3793d6c830acc687b68244be)

diff --git a/ansible/roles/common/filter_plugins/filters.py b/ansible/roles/common/filter_plugins/filters.py
new file mode 100644
index 0000000..e1546aa
--- /dev/null
+++ b/ansible/roles/common/filter_plugins/filters.py
@@ -0,0 +1,47 @@
+# Copyright (c) 2020 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os.path
+
+
+def customise_fluentd(default_paths, customised_paths):
+    """Return a sorted list of templates for fluentd.
+
+    :param default_paths: Iterable of default template paths.
+    :param customised_paths: Iterable of customised template paths.
+    :returns: A sorted combined list of template paths.
+    """
+
+    def _basename_no_ext(path):
+        """Return the basename of a path, stripping off any extension."""
+        return os.path.splitext(os.path.basename(path))[0]
+
+    customised_file_names = {os.path.basename(f) for f in customised_paths}
+    # Starting with the default paths, remove any that have been overridden,
+    # ignoring the .j2 extension of default paths.
+    result = {f for f in default_paths
+              if _basename_no_ext(f) not in customised_file_names}
+    # Add all customised paths.
+    result.update(customised_paths)
+    # Sort by the basename of the paths.
+    return sorted(result, key=os.path.basename)
+
+
+class FilterModule(object):
+    """Service filters."""
+
+    def filters(self):
+        return {
+            "customise_fluentd": customise_fluentd,
+        }
diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml
index b78bd2c..e1c21b2 100644
--- a/ansible/roles/common/tasks/config.yml
+++ b/ansible/roles/common/tasks/config.yml
@@ -1,30 +1,28 @@
 ---
 - name: Ensuring config directories exist
+  vars:
+    service_name: "{{ item.0.service_name }}"
+    service: "{{ common_services[service_name] }}"
   file:
-    path: "{{ node_config_directory }}/{{ item }}"
+    path: "{{ node_config_directory }}/{{ item.1 }}"
     state: "directory"
     owner: "{{ config_owner_user }}"
     group: "{{ config_owner_group }}"
     mode: "0770"
   become: true
-  with_items:
-    - "kolla-toolbox"
-    - "cron"
-    - "cron/logrotate"
-
-- name: Ensuring fluentd config directories exist
-  file:
-    path: "{{ node_config_directory }}/{{ item }}"
-    state: "directory"
-    mode: "0770"
-  become: true
-  with_items:
-    - "fluentd"
-    - "fluentd/input"
-    - "fluentd/output"
-    - "fluentd/format"
-    - "fluentd/filter"
-  when: enable_fluentd | bool
+  with_subelements:
+    - - service_name: "cron"
+        paths:
+          - "cron"
+          - "cron/logrotate"
+      - service_name: "fluentd"
+        paths:
+          - "fluentd"
+      - service_name: "kolla-toolbox"
+        paths:
+          - "kolla-toolbox"
+    - paths
+  when: service.enabled | bool
 
 - name: Copying over config.json files for services
   template:
@@ -45,114 +43,16 @@
     pattern: "*.conf"
   run_once: True
   register: find_custom_fluentd_inputs
-  when:
-    - enable_fluentd | bool
-
-- name: Copying over fluentd input config files
-  vars:
-    customised_input_files: "{{ find_custom_fluentd_inputs.files | map(attribute='path') | map('basename') | list }}"
-  template:
-    src: "conf/input/{{ item }}.conf.j2"
-    dest: "{{ node_config_directory }}/fluentd/input/{{ item }}.conf"
-    mode: "0660"
-  become: true
-  register: fluentd_input
-  when:
-    - enable_fluentd | bool
-    - item ~ '.conf' not in customised_input_files
-  with_items:
-    - "00-global"
-    - "01-syslog"
-    - "02-mariadb"
-    - "03-rabbitmq"
-    - "04-openstack-wsgi"
-    - "05-libvirt"
-    - "06-zookeeper"
-    - "07-kafka"
-    - "08-opendaylight"
-    - "09-monasca"
-  notify:
-    - Restart fluentd container
-
-- name: Copying over custom fluentd input config files
-  template:
-    src: "{{ item.path }}"
-    dest: "{{ node_config_directory }}/fluentd/input/{{ item.path | basename }}"
-    mode: "0660"
-  register: fluentd_input_custom
-  when:
-    - enable_fluentd | bool
-  with_items: "{{ find_custom_fluentd_inputs.files }}"
-  notify:
-    - Restart fluentd container
+  when: common_services.fluentd.enabled | bool
 
-- name: Determine whether logs should be forwarded directly to Elasticsearch
-  set_fact:
-    log_direct_to_elasticsearch: "{{ ( enable_elasticsearch | bool or
-                ( elasticsearch_address != kolla_internal_vip_address )) and
-                not enable_monasca | bool }}"
-
-- name: Find custom fluentd output config files
+- name: Find custom fluentd filter config files
   local_action:
     module: find
-    path: "{{ node_custom_config }}/fluentd/output"
+    path: "{{ node_custom_config }}/fluentd/filter"
     pattern: "*.conf"
   run_once: True
-  register: find_custom_fluentd_outputs
-  when:
-    - enable_fluentd | bool
-
-- name: Copying over fluentd output config files
-  vars:
-    customised_output_files: "{{ find_custom_fluentd_outputs.files | map(attribute='path') | map('basename') | list }}"
-  template:
-    src: "conf/output/{{ item.name }}.conf.j2"
-    dest: "{{ node_config_directory }}/fluentd/output/{{ item.name }}.conf"
-    mode: "0660"
-  become: true
-  register: fluentd_output
-  when:
-    - enable_fluentd | bool
-    - item.enabled | bool
-    - item.name ~ '.conf' not in customised_output_files
-  with_items:
-    - name: "00-local"
-      enabled: true
-    - name: "01-es"
-      enabled: "{{ log_direct_to_elasticsearch }}"
-    - name: "02-monasca"
-      enabled: "{{ enable_monasca | bool }}"
-  notify:
-    - Restart fluentd container
-
-- name: Removing stale output config files
-  file:
-    path: "{{ node_config_directory }}/fluentd/output/{{ item.name }}.conf"
-    state: "absent"
-  become: true
-  when:
-    - enable_fluentd | bool
-    - item.disable | bool
-  with_items:
-    - name: "02-monasca"
-      disable: "{{ not enable_monasca | bool }}"
-    - name: "01-es"
-      disable: "{{ not log_direct_to_elasticsearch }}"
-  notify:
-    - Restart fluentd container
-
-- name: Copying over custom fluentd output config files
-  template:
-    src: "{{ item.path }}"
-    dest: "{{ node_config_directory }}/fluentd/output/{{ item.path | basename }}"
-    mode: "0660"
-  become: true
-  register: fluentd_output_custom
-  when:
-    - enable_fluentd | bool
-  with_items: "{{ find_custom_fluentd_outputs.files }}"
-  notify:
-    - Restart fluentd container
+  register: find_custom_fluentd_filters
+  when: common_services.fluentd.enabled | bool
 
 - name: Find custom fluentd format config files
   local_action:
@@ -160,93 +60,70 @@
     path: "{{ node_custom_config }}/fluentd/format"
     pattern: "*.conf"
   run_once: True
-  register: find_custom_fluentd_format
-  when:
-    - enable_fluentd | bool
-
-- name: Copying over fluentd format config files
-  vars:
-    customised_format_files: "{{ find_custom_fluentd_format.files | map(attribute='path') | map('basename') | list }}"
-  template:
-    src: "conf/format/{{ item }}.conf.j2"
-    dest: "{{ node_config_directory }}/fluentd/format/{{ item }}.conf"
-    mode: "0660"
-  become: true
-  register: fluentd_format
-  with_items:
-    - "apache_access"
-    - "wsgi_access"
-  when:
-    - enable_fluentd | bool
-    - item ~ '.conf' not in customised_format_files
-  notify:
-    - Restart fluentd container
-
-- name: Copying over custom fluentd format config files
-  template:
-    src: "{{ item.path }}"
-    dest: "{{ node_config_directory }}/fluentd/format/{{ item.path | basename }}"
-    mode: "0660"
-  register: fluentd_format_custom
-  when:
-    - enable_fluentd | bool
-  with_items: "{{ find_custom_fluentd_format.files }}"
-  notify:
-    - Restart fluentd container
+  register: find_custom_fluentd_formats
+  when: common_services.fluentd.enabled | bool
 
-- name: Find custom fluentd filter config files
+- name: Find custom fluentd output config files
   local_action:
     module: find
-    path: "{{ node_custom_config }}/fluentd/filter"
+    path: "{{ node_custom_config }}/fluentd/output"
     pattern: "*.conf"
   run_once: True
-  register: find_custom_fluentd_filters
-  when: enable_fluentd | bool
-
-- name: Copying over fluentd filter config files
-  vars:
-    customised_filter_files: "{{ find_custom_fluentd_filters.files | map(attribute='path') | map('basename') | list }}"
-  template:
-    src: "conf/filter/{{ item.src }}.conf.j2"
-    dest: "{{ node_config_directory }}/fluentd/filter/{{ item.dest }}.conf"
-    mode: "0660"
-  become: true
-  register: fluentd_filter
-  with_items:
-    - src: 00-record_transformer
-      dest: 00-record_transformer
-    - src: "{{ '01-rewrite-0.14' if kolla_base_distro in ['debian', 'ubuntu'] else '01-rewrite-0.12' }}"
-      dest: 01-rewrite
-    - src: 02-parser
-      dest: 02-parser
-  when:
-    - enable_fluentd | bool
-    - item.src ~ '.conf' not in customised_filter_files
-  notify:
-    - Restart fluentd container
-
-- name: Copying over custom fluentd filter config files
-  template:
-    src: "{{ item.path }}"
-    dest: "{{ node_config_directory }}/fluentd/filter/{{ item.path | basename }}"
-    mode: "0660"
-  become: true
-  register: fluentd_filter_custom
-  with_items: "{{ find_custom_fluentd_filters.files }}"
-  when: enable_fluentd | bool
-  notify:
-    - Restart fluentd container
+  register: find_custom_fluentd_outputs
+  when: common_services.fluentd.enabled | bool
 
 - name: Copying over td-agent.conf
+  vars:
+    log_direct_to_elasticsearch: >-
+      {{ ( enable_elasticsearch | bool or
+           ( elasticsearch_address != kolla_internal_vip_address )) and
+         not enable_monasca | bool }}
+    fluentd_version: "{{ '0.14' if kolla_base_distro in ['debian', 'ubuntu'] else '0.12' }}"
+    # Inputs
+    fluentd_input_files: "{{ default_input_files | customise_fluentd(customised_input_files) }}"
+    default_input_files:
+      - "conf/input/00-global.conf.j2"
+      - "conf/input/01-syslog.conf.j2"
+      - "conf/input/02-mariadb.conf.j2"
+      - "conf/input/03-rabbitmq.conf.j2"
+      - "conf/input/04-openstack-wsgi.conf.j2"
+      - "conf/input/05-libvirt.conf.j2"
+      - "conf/input/06-zookeeper.conf.j2"
+      - "conf/input/07-kafka.conf.j2"
+      - "conf/input/09-monasca.conf.j2"
+    customised_input_files: "{{ find_custom_fluentd_inputs.files | map(attribute='path') | list }}"
+    # Filters
+    fluentd_filter_files: "{{ default_filter_files | customise_fluentd(customised_filter_files) }}"
+    default_filter_files:
+      - "conf/filter/00-record_transformer.conf.j2"
+      - "conf/filter/{{ '01-rewrite-0.14' if fluentd_version == '0.14' else '01-rewrite-0.12' }}.conf.j2"
+      - "conf/filter/02-parser.conf.j2"
+    customised_filter_files: "{{ find_custom_fluentd_filters.files | map(attribute='path') | list }}"
+    # Formats
+    fluentd_format_files: "{{ default_format_files | customise_fluentd(customised_format_files) }}"
+    default_format_files:
+      - "conf/format/apache_access.conf.j2"
+      - "conf/format/wsgi_access.conf.j2"
+    customised_format_files: "{{ find_custom_fluentd_formats.files | map(attribute='path') | list }}"
+    # Outputs
+    fluentd_output_files: "{{ default_output_files_enabled | customise_fluentd(customised_output_files) }}"
+    default_output_files_enabled: "{{ default_output_files | selectattr('enabled') | map(attribute='name') | list }}"
+    default_output_files:
+      - name: "conf/output/00-local.conf.j2"
+        enabled: true
+      - name: "conf/output/01-es.conf.j2"
+        enabled: "{{ log_direct_to_elasticsearch }}"
+      - name: "conf/output/02-monasca.conf.j2"
+        enabled: "{{ enable_monasca | bool }}"
+    customised_output_files: "{{ find_custom_fluentd_outputs.files | map(attribute='path') | list }}"
   template:
     src: "td-agent.conf.j2"
-    dest: "{{ node_config_directory }}/{{ item }}/td-agent.conf"
+    dest: "{{ node_config_directory }}/fluentd/td-agent.conf"
     mode: "0660"
   become: true
   register: fluentd_td_agent
-  with_items:
-    - "fluentd"
-  when: enable_fluentd | bool
+  when:
+    - common_services.fluentd.enabled | bool
   notify:
     - Restart fluentd container
 
diff --git a/ansible/roles/common/templates/fluentd.json.j2 b/ansible/roles/common/templates/fluentd.json.j2
index bbb8acf..9706b10 100644
--- a/ansible/roles/common/templates/fluentd.json.j2
+++ b/ansible/roles/common/templates/fluentd.json.j2
@@ -3,14 +3,6 @@
 {% set fluentd_conf = 'td-agent.conf' if kolla_base_distro in ['ubuntu', 'debian'] else 'fluent.conf' %}
 {% set fluentd_user = 'td-agent' if kolla_base_distro in ['ubuntu', 'debian'] else 'fluentd' %}
 
-{%- macro config_directory_permissions(dir) -%}
-        {
-            "path": "{{ fluentd_dir }}/{{ dir }}",
-            "owner": "{{ fluentd_user }}:{{ fluentd_user }}",
-            "perm": "0700"
-        }
-{%- endmacro -%}
-
 {
     "command": "{{ fluentd_cmd }}",
     "config_files": [
@@ -19,38 +11,6 @@
             "dest": "{{ fluentd_dir }}/{{ fluentd_conf }}",
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
-        },
-        {# Copy all configuration files in input/ directory to include #}
-        {# custom input configs. #}
-        {
-            "source": "{{ container_config_directory }}/input",
-            "dest": "{{ fluentd_dir }}/input",
-            "owner": "{{ fluentd_user }}",
-            "perm": "0600"
-        },
-        {# Copy all configuration files in filter/ directory to include #}
-        {# custom filter configs. #}
-        {
-            "source": "{{ container_config_directory }}/filter",
-            "dest": "{{ fluentd_dir }}/filter",
-            "owner": "{{ fluentd_user }}",
-            "perm": "0600"
-        },
-        {# Copy all configuration files in format/ directory to include #}
-        {# custom format configs. #}
-        {
-            "source": "{{ container_config_directory }}/format",
-            "dest": "{{ fluentd_dir }}/format",
-            "owner": "{{ fluentd_user }}",
-            "perm": "0600"
-        },
-        {# Copy all configuration files in output/ directory to include #}
-        {# custom output configs. #}
-        {
-            "source": "{{ container_config_directory }}/output",
-            "dest": "{{ fluentd_dir }}/output",
-            "owner": "{{ fluentd_user }}",
-            "perm": "0600"
         }
     ],
     "permissions": [
@@ -68,12 +28,7 @@
             "path": "/var/lib/fluentd/data",
             "owner": "{{ fluentd_user }}:{{ fluentd_user }}",
             "recurse": true
-        },
-        {# Allow Fluentd to read configuration from folders #}
-        {{ config_directory_permissions("input") }},
-        {{ config_directory_permissions("filter") }},
-        {{ config_directory_permissions("format") }},
-        {{ config_directory_permissions("output") }}
+        }
     ]
 
 }
diff --git a/ansible/roles/common/templates/td-agent.conf.j2 b/ansible/roles/common/templates/td-agent.conf.j2
index 42c8df0..c5c54cd 100644
--- a/ansible/roles/common/templates/td-agent.conf.j2
+++ b/ansible/roles/common/templates/td-agent.conf.j2
@@ -1,4 +1,45 @@
-@include input/*.conf
-@include filter/*.conf
-@include format/*.conf
-@include output/*.conf
+#jinja2: trim_blocks: False
+{# Ansible restricts Jinja includes to the same directory or subdirectory of a
+   template. To support customised configuration outside of this path we use
+   the template lookup plugin. Jinja includes have a lower overhead, so we use
+   those where possible. #}
+
+# Inputs
+{%- for path in fluentd_input_files %}
+# Included from {{ path }}:
+{%- if path.startswith('/') %}
+{{ lookup('template', path) }}
+{%- else %}
+{% include path %}
+{%- endif %}
+{%- endfor %}
+
+# Filters
+{%- for path in fluentd_filter_files %}
+# Included from {{ path }}:
+{%- if path.startswith('/') %}
+{{ lookup('template', path) }}
+{%- else %}
+{% include path %}
+{%- endif %}
+{%- endfor %}
+
+# Formats
+{%- for path in fluentd_format_files %}
+# Included from {{ path }}:
+{%- if path.startswith('/') %}
+{{ lookup('template', path) }}
+{%- else %}
+{% include path %}
+{%- endif %}
+{%- endfor %}
+
+# Outputs
+{%- for path in fluentd_output_files %}
+# Included from {{ path }}:
+{%- if path.startswith('/') %}
+{{ lookup('template', path) }}
+{%- else %}
+{% include path %}
+{%- endif %}
+{%- endfor %}
diff --git a/releasenotes/notes/fluentd-single-config-d5ae95fecbfb6e3e.yaml b/releasenotes/notes/fluentd-single-config-d5ae95fecbfb6e3e.yaml
new file mode 100644
index 0000000..071a2df
--- /dev/null
+++ b/releasenotes/notes/fluentd-single-config-d5ae95fecbfb6e3e.yaml
@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Improves performance of the ``common`` role by generating all fluentd
+    configuration in a single file.