commit c66f6a135f0e01749308499ccc47a92619edc042 Author: Mark Goddard Date: Thu Jul 2 18:17:10 2020 +0100 Performance: use a single config file for fluentd Currently we generate multiple fluentd configuration files for inputs, filters, formatters and outputs. These are then included from the main td-agent.conf configuration file. With a large number of hosts, this can take a long time to template. Benchmarking of templating is available at [1]. This change switches to a single fluentd configuration file, with the include done locally. For the default template files included with Kolla Ansible we use Jinja includes, but this does not work with templates in a different directory. We therefore use the Ansible template lookup plugin, which has a slightly higher overhead than a jinja include, but far lower than generating multiple templates. This should drastically improve the performance of this task. [1] https://github.com/stackhpc/ansible-scaling/blob/master/doc/template.md Partially-Implements: blueprint performance-improvements Change-Id: Ia8623be0aa861fea3e54d2c9e1c971dfd8e3afa9 (cherry picked from commit 56a07702bcea02cf3793d6c830acc687b68244be) diff --git a/ansible/roles/common/filter_plugins/filters.py b/ansible/roles/common/filter_plugins/filters.py new file mode 100644 index 0000000..9cc72bc --- /dev/null +++ b/ansible/roles/common/filter_plugins/filters.py @@ -0,0 +1,22 @@ +# Copyright (c) 2020 StackHPC Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from kolla_ansible import fluentd_filters + + +class FilterModule(object): + """Service filters.""" + + def filters(self): + return fluentd_filters.get_filters() diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml index bf70062..f33310d 100644 --- a/ansible/roles/common/tasks/config.yml +++ b/ansible/roles/common/tasks/config.yml @@ -1,30 +1,28 @@ --- - name: Ensuring config directories exist + vars: + service_name: "{{ item.0.service_name }}" + service: "{{ common_services[service_name] }}" file: - path: "{{ node_config_directory }}/{{ item }}" + path: "{{ node_config_directory }}/{{ item.1 }}" state: "directory" owner: "{{ config_owner_user }}" group: "{{ config_owner_group }}" mode: "0770" become: true - with_items: - - "kolla-toolbox" - - "cron" - - "cron/logrotate" - -- name: Ensuring fluentd config directories exist - file: - path: "{{ node_config_directory }}/{{ item }}" - state: "directory" - mode: "0770" - become: true - with_items: - - "fluentd" - - "fluentd/input" - - "fluentd/output" - - "fluentd/format" - - "fluentd/filter" - when: enable_fluentd | bool + with_subelements: + - - service_name: "cron" + paths: + - "cron" + - "cron/logrotate" + - service_name: "fluentd" + paths: + - "fluentd" + - service_name: "kolla-toolbox" + paths: + - "kolla-toolbox" + - paths + when: service | service_enabled_and_mapped_to_host - name: Ensure fluentd image is present for label check vars: @@ -74,199 +72,86 @@ run_once: True register: find_custom_fluentd_inputs delegate_to: localhost - when: - - enable_fluentd | bool - -- name: Copying over fluentd input config files - vars: - customised_input_files: "{{ find_custom_fluentd_inputs.files | map(attribute='path') | map('basename') | list }}" - template: - src: "conf/input/{{ item }}.conf.j2" - dest: "{{ node_config_directory }}/fluentd/input/{{ item }}.conf" - mode: "0660" - become: true - when: - - enable_fluentd | bool - - item ~ '.conf' not in customised_input_files - with_items: - - "00-global" - - "01-syslog" - - "02-mariadb" - - "03-rabbitmq" - - "04-openstack-wsgi" - - "05-libvirt" - - "06-zookeeper" - - "07-kafka" - - "09-monasca" - notify: - - Restart fluentd container - -- name: Copying over custom fluentd input config files - template: - src: "{{ item.path }}" - dest: "{{ node_config_directory }}/fluentd/input/{{ item.path | basename }}" - mode: "0660" - when: - - enable_fluentd | bool - with_items: "{{ find_custom_fluentd_inputs.files }}" - notify: - - Restart fluentd container + when: common_services.fluentd.enabled | bool -- name: Determine whether logs should be forwarded directly to Elasticsearch - set_fact: - log_direct_to_elasticsearch: "{{ ( enable_elasticsearch | bool or - ( elasticsearch_address != kolla_internal_vip_address )) and - not enable_monasca | bool }}" - -- name: Find custom fluentd output config files +- name: Find custom fluentd filter config files find: - path: "{{ node_custom_config }}/fluentd/output" + path: "{{ node_custom_config }}/fluentd/filter" pattern: "*.conf" run_once: True - register: find_custom_fluentd_outputs + register: find_custom_fluentd_filters delegate_to: localhost - when: - - enable_fluentd | bool - -- name: Copying over fluentd output config files - vars: - customised_output_files: "{{ find_custom_fluentd_outputs.files | map(attribute='path') | map('basename') | list }}" - template: - src: "conf/output/{{ item.name }}.conf.j2" - dest: "{{ node_config_directory }}/fluentd/output/{{ item.name }}.conf" - mode: "0660" - become: true - when: - - enable_fluentd | bool - - item.enabled | bool - - item.name ~ '.conf' not in customised_output_files - with_items: - - name: "00-local" - enabled: true - - name: "01-es" - enabled: "{{ log_direct_to_elasticsearch }}" - - name: "02-monasca" - enabled: "{{ enable_monasca | bool }}" - notify: - - Restart fluentd container - -- name: Removing stale output config files - file: - path: "{{ node_config_directory }}/fluentd/output/{{ item.name }}.conf" - state: "absent" - become: true - when: - - enable_fluentd | bool - - item.disable | bool - with_items: - - name: "02-monasca" - disable: "{{ not enable_monasca | bool }}" - - name: "01-es" - disable: "{{ not log_direct_to_elasticsearch }}" - notify: - - Restart fluentd container - -- name: Copying over custom fluentd output config files - template: - src: "{{ item.path }}" - dest: "{{ node_config_directory }}/fluentd/output/{{ item.path | basename }}" - mode: "0660" - become: true - when: - - enable_fluentd | bool - with_items: "{{ find_custom_fluentd_outputs.files }}" - notify: - - Restart fluentd container + when: common_services.fluentd.enabled | bool - name: Find custom fluentd format config files find: path: "{{ node_custom_config }}/fluentd/format" pattern: "*.conf" run_once: True - register: find_custom_fluentd_format + register: find_custom_fluentd_formats delegate_to: localhost - when: - - enable_fluentd | bool + when: common_services.fluentd.enabled | bool -- name: Copying over fluentd format config files - vars: - customised_format_files: "{{ find_custom_fluentd_format.files | map(attribute='path') | map('basename') | list }}" - template: - src: "conf/format/{{ item }}.conf.j2" - dest: "{{ node_config_directory }}/fluentd/format/{{ item }}.conf" - mode: "0660" - become: true - with_items: - - "apache_access" - - "wsgi_access" - when: - - enable_fluentd | bool - - item ~ '.conf' not in customised_format_files - notify: - - Restart fluentd container - -- name: Copying over custom fluentd format config files - template: - src: "{{ item.path }}" - dest: "{{ node_config_directory }}/fluentd/format/{{ item.path | basename }}" - mode: "0660" - when: - - enable_fluentd | bool - with_items: "{{ find_custom_fluentd_format.files }}" - notify: - - Restart fluentd container - -- name: Find custom fluentd filter config files +- name: Find custom fluentd output config files find: - path: "{{ node_custom_config }}/fluentd/filter" + path: "{{ node_custom_config }}/fluentd/output" pattern: "*.conf" run_once: True - register: find_custom_fluentd_filters + register: find_custom_fluentd_outputs delegate_to: localhost - when: enable_fluentd | bool + when: common_services.fluentd.enabled | bool -- name: Copying over fluentd filter config files +- name: Copying over td-agent.conf vars: - customised_filter_files: "{{ find_custom_fluentd_filters.files | map(attribute='path') | map('basename') | list }}" + log_direct_to_elasticsearch: >- + {{ ( enable_elasticsearch | bool or + ( elasticsearch_address != kolla_internal_vip_address )) and + not enable_monasca | bool }} fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}" - template: - src: "conf/filter/{{ item.src }}.conf.j2" - dest: "{{ node_config_directory }}/fluentd/filter/{{ item.dest }}.conf" - mode: "0660" - become: true - with_items: - - src: 00-record_transformer - dest: 00-record_transformer - - src: "{{ '01-rewrite-0.14' if fluentd_version == '0.14' else '01-rewrite-0.12' }}" - dest: 01-rewrite - - src: 02-parser - dest: 02-parser - when: - - enable_fluentd | bool - - item.src ~ '.conf' not in customised_filter_files - notify: - - Restart fluentd container - -- name: Copying over custom fluentd filter config files - template: - src: "{{ item.path }}" - dest: "{{ node_config_directory }}/fluentd/filter/{{ item.path | basename }}" - mode: "0660" - become: true - with_items: "{{ find_custom_fluentd_filters.files }}" - when: enable_fluentd | bool - notify: - - Restart fluentd container - -- name: Copying over td-agent.conf + # Inputs + fluentd_input_files: "{{ default_input_files | customise_fluentd(customised_input_files) }}" + default_input_files: + - "conf/input/00-global.conf.j2" + - "conf/input/01-syslog.conf.j2" + - "conf/input/02-mariadb.conf.j2" + - "conf/input/03-rabbitmq.conf.j2" + - "conf/input/04-openstack-wsgi.conf.j2" + - "conf/input/05-libvirt.conf.j2" + - "conf/input/06-zookeeper.conf.j2" + - "conf/input/07-kafka.conf.j2" + - "conf/input/09-monasca.conf.j2" + customised_input_files: "{{ find_custom_fluentd_inputs.files | map(attribute='path') | list }}" + # Filters + fluentd_filter_files: "{{ default_filter_files | customise_fluentd(customised_filter_files) }}" + default_filter_files: + - "conf/filter/00-record_transformer.conf.j2" + - "conf/filter/{{ '01-rewrite-0.14' if fluentd_version == '0.14' else '01-rewrite-0.12' }}.conf.j2" + - "conf/filter/02-parser.conf.j2" + customised_filter_files: "{{ find_custom_fluentd_filters.files | map(attribute='path') | list }}" + # Formats + fluentd_format_files: "{{ default_format_files | customise_fluentd(customised_format_files) }}" + default_format_files: + - "conf/format/apache_access.conf.j2" + - "conf/format/wsgi_access.conf.j2" + customised_format_files: "{{ find_custom_fluentd_formats.files | map(attribute='path') | list }}" + # Outputs + fluentd_output_files: "{{ default_output_files_enabled | customise_fluentd(customised_output_files) }}" + default_output_files_enabled: "{{ default_output_files | selectattr('enabled') | map(attribute='name') | list }}" + default_output_files: + - name: "conf/output/00-local.conf.j2" + enabled: true + - name: "conf/output/01-es.conf.j2" + enabled: "{{ log_direct_to_elasticsearch }}" + - name: "conf/output/02-monasca.conf.j2" + enabled: "{{ enable_monasca | bool }}" + customised_output_files: "{{ find_custom_fluentd_outputs.files | map(attribute='path') | list }}" template: src: "td-agent.conf.j2" - dest: "{{ node_config_directory }}/{{ item }}/td-agent.conf" + dest: "{{ node_config_directory }}/fluentd/td-agent.conf" mode: "0660" become: true - with_items: - - "fluentd" - when: enable_fluentd | bool + when: + - common_services.fluentd | service_enabled_and_mapped_to_host notify: - Restart fluentd container diff --git a/ansible/roles/common/templates/fluentd.json.j2 b/ansible/roles/common/templates/fluentd.json.j2 index 94656c2..bd98438 100644 --- a/ansible/roles/common/templates/fluentd.json.j2 +++ b/ansible/roles/common/templates/fluentd.json.j2 @@ -1,15 +1,6 @@ {% set fluentd_user = fluentd_binary %} {% set fluentd_dir = '/etc/' ~ fluentd_binary %} -{%- macro config_directory(dir) -%} - { - "source": "{{ container_config_directory }}/{{ dir }}", - "dest": "{{ fluentd_dir }}/{{ dir }}", - "owner": "{{ fluentd_user }}", - "perm": "0600" - } -{%- endmacro -%} - {% if fluentd_binary == 'fluentd' %} {% set fluentd_conf = 'fluent.conf' %} {% if kolla_base_distro in ['ubuntu', 'debian'] %} @@ -22,14 +13,6 @@ {% set fluentd_cmd = '/usr/sbin/td-agent' %} {% endif %} -{%- macro config_directory_permissions(dir) -%} - { - "path": "{{ fluentd_dir }}/{{ dir }}", - "owner": "{{ fluentd_user }}:{{ fluentd_user }}", - "perm": "0700" - } -{%- endmacro -%} - { "command": "{{ fluentd_cmd }} -o /var/log/kolla/fluentd/fluentd.log", "config_files": [ @@ -38,12 +21,7 @@ "dest": "{{ fluentd_dir }}/{{ fluentd_conf }}", "owner": "{{ fluentd_user }}", "perm": "0600" - }, - {# Copy all files in the following directories #} - {{ config_directory("input") }}, - {{ config_directory("filter") }}, - {{ config_directory("format") }}, - {{ config_directory("output") }} + } ], "permissions": [ { @@ -65,12 +43,7 @@ "path": "/var/lib/fluentd/data", "owner": "{{ fluentd_user }}:{{ fluentd_user }}", "recurse": true - }, - {# Allow Fluentd to read configuration from folders #} - {{ config_directory_permissions("input") }}, - {{ config_directory_permissions("filter") }}, - {{ config_directory_permissions("format") }}, - {{ config_directory_permissions("output") }} + } ] } diff --git a/ansible/roles/common/templates/td-agent.conf.j2 b/ansible/roles/common/templates/td-agent.conf.j2 index 42c8df0..c5c54cd 100644 --- a/ansible/roles/common/templates/td-agent.conf.j2 +++ b/ansible/roles/common/templates/td-agent.conf.j2 @@ -1,4 +1,45 @@ -@include input/*.conf -@include filter/*.conf -@include format/*.conf -@include output/*.conf +#jinja2: trim_blocks: False +{# Ansible restricts Jinja includes to the same directory or subdirectory of a + template. To support customised configuration outside of this path we use + the template lookup plugin. Jinja includes have a lower overhead, so we use + those where possible. #} + +# Inputs +{%- for path in fluentd_input_files %} +# Included from {{ path }}: +{%- if path.startswith('/') %} +{{ lookup('template', path) }} +{%- else %} +{% include path %} +{%- endif %} +{%- endfor %} + +# Filters +{%- for path in fluentd_filter_files %} +# Included from {{ path }}: +{%- if path.startswith('/') %} +{{ lookup('template', path) }} +{%- else %} +{% include path %} +{%- endif %} +{%- endfor %} + +# Formats +{%- for path in fluentd_format_files %} +# Included from {{ path }}: +{%- if path.startswith('/') %} +{{ lookup('template', path) }} +{%- else %} +{% include path %} +{%- endif %} +{%- endfor %} + +# Outputs +{%- for path in fluentd_output_files %} +# Included from {{ path }}: +{%- if path.startswith('/') %} +{{ lookup('template', path) }} +{%- else %} +{% include path %} +{%- endif %} +{%- endfor %} diff --git a/kolla_ansible/fluentd_filters.py b/kolla_ansible/fluentd_filters.py new file mode 100644 index 0000000..6fc2504 --- /dev/null +++ b/kolla_ansible/fluentd_filters.py @@ -0,0 +1,44 @@ +# Copyright (c) 2020 StackHPC Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os.path + + +def customise_fluentd(default_paths, customised_paths): + """Return a sorted list of templates for fluentd. + + :param default_paths: Iterable of default template paths. + :param customised_paths: Iterable of customised template paths. + :returns: A sorted combined list of template paths. + """ + + def _basename_no_ext(path): + """Return the basename of a path, stripping off any extension.""" + return os.path.splitext(os.path.basename(path))[0] + + customised_file_names = {os.path.basename(f) for f in customised_paths} + # Starting with the default paths, remove any that have been overridden, + # ignoring the .j2 extension of default paths. + result = {f for f in default_paths + if _basename_no_ext(f) not in customised_file_names} + # Add all customised paths. + result.update(customised_paths) + # Sort by the basename of the paths. + return sorted(result, key=os.path.basename) + + +def get_filters(): + return { + "customise_fluentd": customise_fluentd, + } diff --git a/kolla_ansible/tests/unit/test_fluentd_filters.py b/kolla_ansible/tests/unit/test_fluentd_filters.py new file mode 100644 index 0000000..353d017 --- /dev/null +++ b/kolla_ansible/tests/unit/test_fluentd_filters.py @@ -0,0 +1,96 @@ +# Copyright (c) 2020 StackHPC Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import unittest + +from kolla_ansible.fluentd_filters import customise_fluentd + + +class TestFilters(unittest.TestCase): + + def test_customise_fluentd_no_files(self): + default_files = [ + ] + customised_files = [ + ] + expected = [ + ] + result = customise_fluentd(default_files, customised_files) + self.assertEqual(expected, result) + + def test_customise_fluentd_no_customised_files(self): + default_files = [ + "foo/bar.conf.j2" + ] + customised_files = [ + ] + expected = [ + "foo/bar.conf.j2" + ] + result = customise_fluentd(default_files, customised_files) + self.assertEqual(expected, result) + + def test_customise_fluentd_no_default_files(self): + default_files = [ + ] + customised_files = [ + "foo/bar.conf" + ] + expected = [ + "foo/bar.conf" + ] + result = customise_fluentd(default_files, customised_files) + self.assertEqual(expected, result) + + def test_customise_fluentd_both(self): + default_files = [ + "foo/bar.conf.j2" + ] + customised_files = [ + "baz/qux.conf" + ] + expected = [ + "foo/bar.conf.j2", + "baz/qux.conf" + ] + result = customise_fluentd(default_files, customised_files) + self.assertEqual(expected, result) + + def test_customise_fluentd_override(self): + default_files = [ + "foo/bar.conf.j2" + ] + customised_files = [ + "baz/bar.conf" + ] + expected = [ + "baz/bar.conf" + ] + result = customise_fluentd(default_files, customised_files) + self.assertEqual(expected, result) + + def test_customise_fluentd_both_with_override(self): + default_files = [ + "foo/bar.conf.j2", + "baz/qux.conf.j2" + ] + customised_files = [ + "baz/bar.conf" + ] + expected = [ + "baz/bar.conf", + "baz/qux.conf.j2" + ] + result = customise_fluentd(default_files, customised_files) + self.assertEqual(expected, result) diff --git a/releasenotes/notes/fluentd-single-config-d5ae95fecbfb6e3e.yaml b/releasenotes/notes/fluentd-single-config-d5ae95fecbfb6e3e.yaml new file mode 100644 index 0000000..071a2df --- /dev/null +++ b/releasenotes/notes/fluentd-single-config-d5ae95fecbfb6e3e.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Improves performance of the ``common`` role by generating all fluentd + configuration in a single file.