From edf06d64683bf22ab5b5cf1876004839bd12db2a Mon Sep 17 00:00:00 2001 From: Diogenes Pelisson Date: Sun, 19 Apr 2020 21:21:00 -0300 Subject: [PATCH] add promtail --- defaults/main.yml | 97 +++++++++++++++++++++++++++++++---- handlers/main.yml | 13 +++++ tasks/configure.yml | 24 +++++++++ tasks/install.yml | 24 +++++---- tasks/main.yml | 7 ++- templates/loki.service.j2 | 32 ++++++++++++ templates/loki.yml.j2 | 64 +++++++++++++---------- templates/promtail.service.j2 | 31 +++++++++++ templates/promtail.yml.j2 | 19 +++++++ 9 files changed, 262 insertions(+), 49 deletions(-) create mode 100644 tasks/configure.yml create mode 100644 templates/promtail.service.j2 create mode 100644 templates/promtail.yml.j2 diff --git a/defaults/main.yml b/defaults/main.yml index 0acfadd..8cadc5e 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -11,26 +11,103 @@ loki_bins: loki_user: loki loki_group: loki loki_config_dir: /etc/loki +loki_storage_dir: /var/lib/loki + +loki_tags: [] + +loki_listen_address: '127.0.0.1' +loki_listen_port: '3100' loki_binary_local_dir: '' loki_target: all -loki_auth_enabled: true +loki_auth_enabled: false loki_server_config: - http_listen_address: 0.0.0.0 - http_listen_port: 80 - grpc_listen_address: 0.0.0.0 - grpc_listen_port: 9095 + http_listen_address: "{{ loki_listen_address }}" + http_listen_port: "{{ loki_listen_port }}" loki_distributor_config: [] loki_querier_config: [] loki_ingester_client_config: [] -loki_ingester_config: [] -loki_storage_config: [] -loki_chunk_store_config: [] -loki_schema_config: [] +loki_ingester_config: + lifecycler: + address: 127.0.0.1 + ring: + kvstore: + store: inmemory + replication_factor: 1 + final_sleep: 0s + chunk_idle_period: 5m + chunk_retain_period: 30s + +loki_storage_config: + boltdb: + directory: "{{ loki_storage_dir }}/index" + filesystem: + directory: "{{ loki_storage_dir }}/chunks" + +loki_chunk_store_config: + max_look_back_period: 0 + +loki_schema_config: + configs: + - from: 2020-01-01 + store: boltdb + object_store: filesystem + schema: v11 + index: + prefix: loki + period: 168h + + chunks: + prefix: loki + period: 168h + row_shards: 16 + loki_limits_config: [] loki_frontend_worker_config: [] -loki_table_manager_config: [] loki_runtime_config: [] +loki_table_manager_config: + chunk_tables_provisioning: + inactive_read_throughput: 0 + inactive_write_throughput: 0 + provisioned_read_throughput: 0 + provisioned_write_throughput: 0 + index_tables_provisioning: + inactive_read_throughput: 0 + inactive_write_throughput: 0 + provisioned_read_throughput: 0 + provisioned_write_throughput: 0 + retention_deletes_enabled: false + retention_period: 0 + + +promtail_client_config: + - url: "http://{{ loki_listen_address }}:{{ loki_listen_port }}/loki/api/v1/push" + +promtail_server_config: + http_listen_port: 9080 + grpc_listen_port: 0 + +promtail_scrape_config: + - job_name: system + pipeline_stages: + static_configs: + - targets: + - localhost + labels: + job: varlogs + host: "{{ ansible_hostname }}" + __path__: /var/log/*.log + + - job_name: journal + journal: + max_age: 12h + labels: + job: systemd-journal + relabel_configs: + - source_labels: ['__journal__systemd_unit'] + target_label: 'unit' + +promtail_target_config: [] \ No newline at end of file diff --git a/handlers/main.yml b/handlers/main.yml index cc960c3..45c4549 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -11,3 +11,16 @@ systemd: name: loki state: reloaded + +- name: restart promtail + become: true + systemd: + daemon_reload: true + name: promtail + state: restarted + +- name: reload promtail + become: true + systemd: + name: promtail + state: reloaded diff --git a/tasks/configure.yml b/tasks/configure.yml new file mode 100644 index 0000000..7cdc08e --- /dev/null +++ b/tasks/configure.yml @@ -0,0 +1,24 @@ +--- +- name: configure loki + template: + src: "loki.yml.j2" + dest: "{{ loki_config_dir }}/loki.yml" + force: true + owner: root + group: "{{ loki_user }}" + mode: 0640 + notify: + - reload loki + when: ('loki' in loki_bins) + +- name: configure promtail + template: + src: "promtail.yml.j2" + dest: "{{ loki_config_dir }}/promtail.yml" + force: true + owner: root + group: "{{ loki_user }}" + mode: 0640 + notify: + - reload promtail + when: ('promtail' in loki_bins) \ No newline at end of file diff --git a/tasks/install.yml b/tasks/install.yml index 760104a..12a3039 100644 --- a/tasks/install.yml +++ b/tasks/install.yml @@ -12,15 +12,19 @@ shell: "/usr/sbin/nologin" group: "{{ loki_group }}" createhome: false - # home: "{{ loki_db_dir }}" + home: "{{ loki_storage_dir }}" -# - name: create loki data directory -# file: -# path: "{{ loki_db_dir }}" -# state: directory -# owner: loki -# group: loki -# mode: 0755 +- name: create loki data directory + file: + path: "{{ item }}" + state: directory + owner: loki + group: loki + mode: 0755 + with_items: + - "{{ loki_storage_dir }}" + - "{{ loki_storage_dir }}/index" + - "{{ loki_storage_dir }}/chunks" - name: create loki configuration directories file: @@ -75,6 +79,4 @@ - restart loki loop: "{{ loki_bins }}" when: - - item == 'loki' or - - item == 'loki-canary' or - - item == 'promtail' + - item == 'loki' diff --git a/tasks/main.yml b/tasks/main.yml index c1b2ee7..3602128 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,5 +1,8 @@ --- -- include: preflight.yml +# - include: preflight.yml -- include: install.yml +# - include: install.yml +# become: true + +- include: configure.yml become: true diff --git a/templates/loki.service.j2 b/templates/loki.service.j2 index e69de29..5d6fa05 100644 --- a/templates/loki.service.j2 +++ b/templates/loki.service.j2 @@ -0,0 +1,32 @@ +{{ ansible_managed | comment }} + +[Unit] +Description=Loki +After=network-online.target + +[Service] +Type=simple +Environment="GOMAXPROCS={{ ansible_processor_vcpus|default(ansible_processor_count) }}" +User={{ loki_user }} +Group={{ loki_group }} +ExecReload=/bin/kill -HUP $MAINPID +ExecStart=/usr/local/bin/loki \ + -config.file {{ loki_config_dir }}/loki.yml \ + +CapabilityBoundingSet=CAP_SET_UID +LimitNOFILE=65000 +LockPersonality=true +NoNewPrivileges=true +MemoryDenyWriteExecute=true +PrivateDevices=true +PrivateTmp=true +ProtectHome=true +RemoveIPC=true +RestrictSUIDSGID=true + + +SyslogIdentifier=loki +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/templates/loki.yml.j2 b/templates/loki.yml.j2 index b7e4b10..5438373 100644 --- a/templates/loki.yml.j2 +++ b/templates/loki.yml.j2 @@ -4,44 +4,56 @@ {% if loki_target != "" %} target: {{ loki_target }} -{% endif %}} +{% endif %} {% if loki_auth_enabled %} auth_enabled: {{ loki_auth_enabled }} -{% endif %}} +{% endif %} {% if loki_server_config != [] %} -server: {{ loki_server_config }} -{% endif %}} +server: + {{ loki_server_config | to_nice_yaml(indent=2) | indent(2, False) }} +{% endif %} {% if loki_distributor_config != [] %} -distributor: {{ loki_distributor_config }} -{% endif %}} +distributor: + {{ loki_distributor_config | to_nice_yaml(indent=2) | indent(2, False) }} +{% endif %} {% if loki_querier_config != [] %} -querier: {{ loki_querier_config }} -{% endif %}} +querier: + {{ loki_querier_config | to_nice_yaml(indent=2) | indent(2, False) }} +{% endif %} {% if loki_ingester_client_config != [] %} -ingester_client: {{ loki_ingester_client_config }} -{% endif %}} +ingester_client: + {{ loki_ingester_client_config | to_nice_yaml(indent=2) | indent(2, False) }} +{% endif %} {% if loki_ingester_config != [] %} -ingester: {{ loki_ingester_config }} -{% endif %}} +ingester: + {{ loki_ingester_config | to_nice_yaml(indent=2) | indent(2, False) }} +{% endif %} {% if loki_storage_config != [] %} -storage_config: {{ loki_storage_config }} -{% endif %}} +storage_config: + {{ loki_storage_config | to_nice_yaml(indent=2) | indent(2, False) }} +{% endif %} {% if loki_chunk_store_config != [] %} -chunk_store_config: {{ loki_chunk_store_config }} -{% endif %}} +chunk_store_config: + {{ loki_chunk_store_config | to_nice_yaml(indent=2) | indent(2, False) }} +{% endif %} {% if loki_schema_config != [] %} -schema_config: {{ loki_schema_config }} -{% endif %}} +schema_config: + {{ loki_schema_config | to_nice_yaml(indent=2) | indent(2, False) }} +{% endif %} {% if loki_limits_config != [] %} -limits_config: {{ loki_limits_config }} -{% endif %}} +limits_config: + {{ loki_limits_config | to_nice_yaml(indent=2) | indent(2, False) }} +{% endif %} {% if loki_frontend_worker_config != [] %} -frontend_worker_config: {{ loki_frontend_worker_config }} -{% endif %}} +frontend_worker_config: + {{ loki_frontend_worker_config | to_nice_yaml(indent=2) | indent(2, False) }} +{% endif %} {% if loki_table_manager_config != [] %} -table_manager_config: {{ loki_table_manager_config }} -{% endif %}} +table_manager: + {{ loki_table_manager_config | to_nice_yaml(indent=2) | indent(2, False) }} +{% endif %} {% if loki_runtime_config != [] %} -runtime_config: {{ loki_runtime_config }} -{% endif %}} +runtime_config: + {{ loki_runtime_config | to_nice_yaml(indent=2) | indent(2, False) }} +{% endif %} diff --git a/templates/promtail.service.j2 b/templates/promtail.service.j2 new file mode 100644 index 0000000..11407e5 --- /dev/null +++ b/templates/promtail.service.j2 @@ -0,0 +1,31 @@ +{{ ansible_managed | comment }} + +[Unit] +Description=Loki +After=network-online.target + +[Service] +Type=simple +Environment="GOMAXPROCS={{ ansible_processor_vcpus|default(ansible_processor_count) }}" +User={{ loki_user }} +Group={{ loki_group }} +ExecReload=/bin/kill -HUP $MAINPID +ExecStart=/usr/local/bin/promtail \ + -config.file {{ loki_config_dir }}/promtail.yml \ + +CapabilityBoundingSet=CAP_SET_UID +LimitNOFILE=65000 +LockPersonality=true +NoNewPrivileges=true +MemoryDenyWriteExecute=true +PrivateDevices=true +PrivateTmp=true +ProtectHome=true +RemoveIPC=true +RestrictSUIDSGID=true + +SyslogIdentifier=loki +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/templates/promtail.yml.j2 b/templates/promtail.yml.j2 new file mode 100644 index 0000000..7d81b0b --- /dev/null +++ b/templates/promtail.yml.j2 @@ -0,0 +1,19 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +{{ ansible_managed | comment }} + +{% if promtail_server_config != [] %} +server: + {{ promtail_server_config | to_nice_yaml(indent=2) | indent(2, False) }} +{% endif %} +{% if promtail_client_config != [] %} +clients: + {{ promtail_client_config | to_nice_yaml(indent=2) | indent(2, False) }} +{% endif %} +{% if promtail_scrape_config != [] %} +scrape_configs: + {{ promtail_scrape_config | to_nice_yaml(indent=2) | indent(2, False) }} +{% endif %} +{% if promtail_target_config != [] %} +target_config: + {{ promtail_target_config | to_nice_yaml(indent=2) | indent(2, False) }} +{% endif %} \ No newline at end of file