From 9ce6c0721a71a0d34cd3a2497a5455f5a099da4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 16 Jun 2024 16:23:51 +0200 Subject: [PATCH 001/107] Create akvorado.py --- modules/akvorado.py | 57 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 modules/akvorado.py diff --git a/modules/akvorado.py b/modules/akvorado.py new file mode 100644 index 00000000..034e579c --- /dev/null +++ b/modules/akvorado.py @@ -0,0 +1,57 @@ +# vim: ts=4: sts=4: sw=4: expandtab +# Copyright 2024 dhtech +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file +import lib +import os +import sqlite3 +import yaml + +DB_FILE = '/etc/ipplan.db' + +def get_prefixes(ipversion): + if os.path.isfile(DB_FILE): + try: + conn = sqlite3.connect(DB_FILE) + db = conn.cursor() + except sqlite3.Error as e: + print "An error occurred:", e.args[0] + sys.exit(2) + else: + print "No database file found: %s" % DB_FILE + sys.exit(3) + + if ipversion == "4": + db.execute( + 'SELECT SUBSTR(name,1, INSTR(name, "@")-1), name, short_name, ipv4_txt' + ' FROM network' + ' WHERE node_id NOT IN (SELECT option.node_id from option where name = "NO-AKV") and name like "%@%" and ipv4_txt is not NULL' + ) + + elif ipversion == "6": + db.execute( + 'SELECT SUBSTR(name,1, INSTR(name, "@")-1), name, short_name, ipv6_txt' + ' FROM network' + ' WHERE node_id NOT IN (SELECT option.node_id from option where name = "NO-AKV") and name like "%@%" and ipv6_txt is not NULL' + ) + else: + raise NetworkTypeNotFoundError('network type must be 4 or 6') + + res = db.fetchall() + conn.close() + if not res: + raise NetworkNotFoundError('network not found') + + return res + + +def generate(host, *args): + + info = {} + info['current_event'] = lib.get_current_event() + info['ipv6_prefixes'] = get_prefixes('6') + info['ipv4_prefixes'] = get_prefixes('4') + print(info) + return {'akverado': info} + From 05487713243f86708b06fe1c166767d8021d7b31 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 17:30:38 +0200 Subject: [PATCH 002/107] Add akvorado module and service files --- modules/akvorado/files/akvorado-console.service | 15 +++++++++++++++ modules/akvorado/files/akvorado-inlet.service | 15 +++++++++++++++ modules/akvorado/files/akvorado-orch.service | 13 +++++++++++++ modules/akvorado/files/kafka.service | 13 +++++++++++++ modules/akvorado/files/zookeeper.service | 13 +++++++++++++ 5 files changed, 69 insertions(+) create mode 100644 modules/akvorado/files/akvorado-console.service create mode 100644 modules/akvorado/files/akvorado-inlet.service create mode 100644 modules/akvorado/files/akvorado-orch.service create mode 100644 modules/akvorado/files/kafka.service create mode 100644 modules/akvorado/files/zookeeper.service diff --git a/modules/akvorado/files/akvorado-console.service b/modules/akvorado/files/akvorado-console.service new file mode 100644 index 00000000..23e2f587 --- /dev/null +++ b/modules/akvorado/files/akvorado-console.service @@ -0,0 +1,15 @@ +[Unit] +Description=Akvorado Console +After=akvorado-orch.service +Requires=akvorado-orch.service + +[Service] +Type=simple +Restart=on-failure +RestartSec=15 +User=akvorado +ExecStart=/usr/local/bin/akvorado console http://127.0.0.1:8080 + +[Install] +WantedBy=multi-user.target + diff --git a/modules/akvorado/files/akvorado-inlet.service b/modules/akvorado/files/akvorado-inlet.service new file mode 100644 index 00000000..1930cb0c --- /dev/null +++ b/modules/akvorado/files/akvorado-inlet.service @@ -0,0 +1,15 @@ +[Unit] +Description=Akvorado Inlet +After=akvorado-orch.service +Requires=akvorado-orch.service + +[Service] +Type=simple +Restart=on-failure +RestartSec=15 +User=akvorado +ExecStart=/usr/local/bin/akvorado inlet http://127.0.0.1:8080 + +[Install] +WantedBy=multi-user.target + diff --git a/modules/akvorado/files/akvorado-orch.service b/modules/akvorado/files/akvorado-orch.service new file mode 100644 index 00000000..23e0f153 --- /dev/null +++ b/modules/akvorado/files/akvorado-orch.service @@ -0,0 +1,13 @@ +[Unit] +Description=Akvorado Orchestrator +After=network.target +[Service] +Type=simple +Restart=on-failure +RestartSec=15 +User=akvorado +ExecStart=/usr/local/bin/akvorado orchestrator /etc/akvorado/akvorado.yaml + +[Install] +WantedBy=multi-user.target + diff --git a/modules/akvorado/files/kafka.service b/modules/akvorado/files/kafka.service new file mode 100644 index 00000000..51df02db --- /dev/null +++ b/modules/akvorado/files/kafka.service @@ -0,0 +1,13 @@ +[Unit] +Requires=zookeeper.service +After=zookeeper.service + +[Service] +Type=simple +User=kafka +ExecStart=/bin/sh -c '/var/lib/kafka/bin/kafka-server-start.sh /var/lib/kafka/config/server.properties > /var/log/kaf +ExecStop=/var/lib/kafka/bin/kafka-server-stop.sh +Restart=on-abnormal + +[Install] +WantedBy=multi-user.target diff --git a/modules/akvorado/files/zookeeper.service b/modules/akvorado/files/zookeeper.service new file mode 100644 index 00000000..62fcd238 --- /dev/null +++ b/modules/akvorado/files/zookeeper.service @@ -0,0 +1,13 @@ +[Unit] +Requires=network.target remote-fs.target +After=network.target remote-fs.target + +[Service] +Type=simple +User=kafka +ExecStart=/var/lib/kafka/bin/zookeeper-server-start.sh /var/lib/kafka/config/zookeeper.properties +ExecStop=/var/lib/kafka/bin/zookeeper-server-stop.sh +Restart=on-abnormal + +[Install] +WantedBy=multi-user.target From 1e33c7f8bbb78a2ef659398745b6f9c49f4d9a50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 16 Jun 2024 17:46:51 +0200 Subject: [PATCH 003/107] Create akvorado.yaml.erb --- modules/akvorado/templates/akvorado.yaml.erb | 303 +++++++++++++++++++ 1 file changed, 303 insertions(+) create mode 100644 modules/akvorado/templates/akvorado.yaml.erb diff --git a/modules/akvorado/templates/akvorado.yaml.erb b/modules/akvorado/templates/akvorado.yaml.erb new file mode 100644 index 00000000..041c3280 --- /dev/null +++ b/modules/akvorado/templates/akvorado.yaml.erb @@ -0,0 +1,303 @@ +--- +reporting: + logging: {} + metrics: {} +http: + listen: :8080 + profiler: true + cache: + type: memory +clickhouse: + servers: + - localhost:9000 + cluster: "" + database: default + username: default + password: "dhtech" + maxopenconns: 10 + dialtimeout: 5s + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + skipmigrations: false + kafka: + topic: flows + brokers: + - localhost:9092 + version: 3.3.1 + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + saslusername: "" + saslpassword: "" + saslmechanism: none + consumers: 4 + groupname: clickhouse + enginesettings: [] + resolutions: + - interval: 0s + ttl: 360h0m0s + - interval: 1m0s + ttl: 168h0m0s + - interval: 5m0s + ttl: 2160h0m0s + - interval: 1h0m0s + ttl: 8640h0m0s + maxpartitions: 50 + systemlogttl: 720h0m0s + prometheusendpoint: /metrics + asns: + 25037: Dreamhack ACME Corporation + networks: + # 2a01:db8:cafe:1::/64: + # name: ipv6-customers + # role: customers + # site: "" + # region: "" + # city: "" + # state: "" + # country: "" + # tenant: "" + # asn: 0 + + <% @ipv4_prefixes.each do |ipv4| -%> + "<%= ipv4['ipv4_txt'] %>" + name: "<%= ipv4['short_name'] %>" + role: "<%= ipv4['location'] %>" + <% end -%> + <% @ipv6_prefixes.each do |ipv6| -%> + "<%= ipv6['ipv6_txt'] %>" + name: "<%= ipv6['short_name'] %>" + role: "<%= ipv6['location'] %>" + <% end -%> + + networksources: {} + networksourcestimeout: 10s + orchestratorurl: http://localhost:8080 +kafka: + topic: flows + brokers: + - localhost:9092 + version: 3.3.1 + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + saslusername: "" + saslpassword: "" + saslmechanism: none + topicconfiguration: + numpartitions: 8 + replicationfactor: 1 + configentries: + cleanup.policy: delete + compression.type: producer + retention.ms: "86400000" + segment.bytes: "1073741824" + configentriesstrictsync: true +geoip: + asndatabase: + - /usr/share/GeoIP/asn.mmdb + geodatabase: + - /usr/share/GeoIP/country.mmdb + optional: true +schema: + disabled: [] + enabled: [] + maintableonly: [] + notmaintableonly: [] + materialize: [] + customdictionaries: {} +inlet: + - reporting: + logging: {} + metrics: {} + http: + listen: :8081 + profiler: true + cache: + type: memory + flow: + inputs: + - decoder: netflow + listen: :2055 + queuesize: 100000 + receivebuffer: 10485760 + timestampsource: udp + type: udp + usesrcaddrforexporteraddr: false + workers: 6 + - decoder: sflow + listen: :6343 + queuesize: 100000 + receivebuffer: 10485760 + timestampsource: udp + type: udp + usesrcaddrforexporteraddr: false + workers: 6 + ratelimit: 0 + metadata: + cacheduration: 30m0s + cacherefresh: 1h0m0s + cachecheckinterval: 2m0s + cachepersistfile: "" + providers: + - agents: {} + communities: + ::/0: + - public + pollerretries: 1 + pollertimeout: 1s + ports: + ::/0: 161 + securityparameters: {} + type: snmp + workers: 10 + maxbatchrequests: 10 + routing: + provider: + collectasns: true + collectaspaths: true + collectcommunities: true + keep: 5m0s + listen: :10179 + rds: [] + ribpeerremovalbatchroutes: 5000 + ribpeerremovalmaxqueue: 10000 + ribpeerremovalmaxtime: 100ms + ribpeerremovalsleepinterval: 500ms + type: bmp + kafka: + topic: flows + brokers: + - localhost:9092 + version: 3.3.1 + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + saslusername: "" + saslpassword: "" + saslmechanism: none + flushinterval: 10s + flushbytes: 104857599 + maxmessagebytes: 1000000 + compressioncodec: zstd + queuesize: 32 + core: + workers: 6 + exporterclassifiers: + - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") + - ClassifyRegion("europe") + - ClassifyTenant("acme") + - ClassifyRole("edge") + interfaceclassifiers: + - | + ClassifyConnectivityRegex(Interface.Description, "^(?i)(transit|pni|ppni|ix):? ", "$1") && + ClassifyProviderRegex(Interface.Description, "^\\S+?\\s(\\S+)", "$1") && + ClassifyExternal() + - ClassifyInternal() + classifiercacheduration: 5m0s + defaultsamplingrate: {} + overridesamplingrate: {} + asnproviders: + - flow + - routing + netproviders: + - flow + - routing + schema: + disabled: [] + enabled: [] + maintableonly: [] + notmaintableonly: [] + materialize: [] + customdictionaries: {} +console: + - reporting: + logging: {} + metrics: {} + http: + listen: :8082 + profiler: true + cache: + db: 0 + password: "" + protocol: tcp + server: localhost:6379 + type: redis + username: "" + defaultvisualizeoptions: + graphtype: stacked + start: 6 hours ago + end: now + filter: InIfBoundary = external + dimensions: + - SrcAS + limit: 10 + homepagetopwidgets: + - src-as + - src-port + - protocol + - src-country + - etype + homepagegraphfilter: InIfBoundary = 'external' + dimensionslimit: 50 + cachettl: 3h0m0s + clickhouse: + servers: + - localhost:9000 + cluster: "" + database: default + username: default + password: "" + maxopenconns: 10 + dialtimeout: 5s + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + auth: + headers: + login: Remote-User + name: Remote-Name + email: Remote-Email + logouturl: X-Logout-URL + defaultuser: + login: __default + name: Default User + email: "" + logouturl: "" + database: + driver: sqlite + dsn: file::memory:?cache=shared + savedfilters: + - description: From Netflix + content: InIfBoundary = external AND SrcAS = AS2906 + - description: From GAFAM + content: InIfBoundary = external AND SrcAS IN (AS15169, AS16509, AS32934, AS6185, AS8075) + - description: From Swedish Armed Forces + content: InIfBoundary = external AND SrcAS = AS9201 + - description: Valve Corporation + content: InIfBoundary = external AND SrcAS = AS32590 + + + schema: + disabled: [] + enabled: [] + maintableonly: [] + notmaintableonly: [] + materialize: [] + customdictionaries: {} From c4f4acbaf41ea7aaf14485287f418e3ff2e99f34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 16 Jun 2024 18:38:28 +0200 Subject: [PATCH 004/107] Create init.pp --- modules/akvorado/manifests/init.pp | 90 ++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 modules/akvorado/manifests/init.pp diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp new file mode 100644 index 00000000..8e62f457 --- /dev/null +++ b/modules/akvorado/manifests/init.pp @@ -0,0 +1,90 @@ +# Copyright 2018 dhtech +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file +# +# == Class: akvorado +# +# Alert manager for prometheus to handle sending alerts +# +# === Parameters +# + +class akvorado { + + #Create user/group for Akvorodo + group { 'akvorado': + ensure => 'present', + } + -> user { 'akvorado': + ensure => 'present', + system => true, + } + #Create directories for akvorado + -> file { '/etc/akvorado': + ensure => 'directory', + owner => 'root', + group => 'akvorado', + mode => '0750', + } + #Copy akvorado to the server + -> file { '/usr/local/bin/akvorado': + ensure => file, + owner => 'root', + group => 'akvorado', + mode => '0550', + links => follow, + source => 'puppet:///data/akvorado-latest', + } + + file { '/etc/akvorado/akvorado.yaml': + ensure => file, + content => template('akvorado/akvorado.yaml.erb'), + notify => Service['akvorado-orch'], + } + #Systemctl config + file { '/etc/systemd/system/akvorado-orch.service': + ensure => present, + source => 'puppet:///modules/akvorado/akvorado-orch.service', + mode => '0644', + owner => 'root', + group => 'root', + notify => [Exec['systemctl-daemon-reload'],Service['akvorado-orch']], + } + file { '/etc/systemd/system/akvorado-inlet.service': + ensure => present, + source => 'puppet:///modules/akvorado/akvorado-inlet.service', + mode => '0644', + owner => 'root', + group => 'root', + notify => [Exec['systemctl-daemon-reload'],Service['akvorado-inlet']], + } + file { '/etc/systemd/system/akvorado-console.service': + ensure => present, + source => 'puppet:///modules/akvorado/akvorado-console.service', + mode => '0644', + owner => 'root', + group => 'root', + notify => [Exec['systemctl-daemon-reload'],Service['akvorado-console']], + } + + -> apache::proxy { 'akvorado': + url => '/', + backend => 'http://localhost:8082/', + } + -> service { 'akvorado-orch': + ensure => running, + } +-> service { 'akvorado-inlet': + ensure => running, + } +-> service { 'akvorado-console': + ensure => running, + } + + + exec { 'systemctl-daemon-reload': + command => '/bin/systemctl daemon-reload', + refreshonly => true, + } +} From 881db288c9f9cfd2f9aab8e970a58b92929d8d88 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 20:44:11 +0200 Subject: [PATCH 005/107] Add kafka and clickhouse installation --- modules/akvorado/manifests/init.pp | 108 +++++++++++++++++++++++++++-- 1 file changed, 104 insertions(+), 4 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 8e62f457..9b511207 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -15,7 +15,7 @@ #Create user/group for Akvorodo group { 'akvorado': ensure => 'present', - } + } -> user { 'akvorado': ensure => 'present', system => true, @@ -36,7 +36,7 @@ links => follow, source => 'puppet:///data/akvorado-latest', } - + file { '/etc/akvorado/akvorado.yaml': ensure => file, content => template('akvorado/akvorado.yaml.erb'), @@ -67,7 +67,7 @@ group => 'root', notify => [Exec['systemctl-daemon-reload'],Service['akvorado-console']], } - + -> apache::proxy { 'akvorado': url => '/', backend => 'http://localhost:8082/', @@ -82,9 +82,109 @@ ensure => running, } - exec { 'systemctl-daemon-reload': command => '/bin/systemctl daemon-reload', refreshonly => true, } + + ##Kafka installation + group { 'kafka': + ensure => 'present', + } + -> user { 'kafka': + ensure => 'present', + system => true, + home => '/var/lib/kafka', + managegome => true, + } + -> file { '/tmp/kafka.tgz': + ensure => file, + links => follow, + source => 'puppet:///data/kafka-latest.tgz', + notify => Exec[ 'untar-kafka' ], + } + -> file { '/var/log/kafka': + ensure => 'directory', + owner => 'kafka', + group => 'kafka', + mode => '0700', + } + -> file { '/etc/systemd/system/kafka.service': + ensure => present, + source => 'puppet:///modules/akvorado/kafka.service', + mode => '0644', + owner => 'root', + group => 'root', + notify => [Exec['systemctl-daemon-reload'],Service['kafka']], + } + -> file_line { 'kafka-enabledeletetopics' + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'delete.topic.enable = true' + line => 'delete.topic.enable' + } + -> file_line { 'kafka-listenlocalhost' + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'listeners=PLAINTEXT://localhost:9092' + match => '#listeners=PLAINTEXT' + } + -> file_line { 'kafka-logdir' + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'log.dirs=/var/log/kafka' + match => 'log.dirs=' + } + exec { 'untar-kafka': + command => '/bin/tar -zxf /tmp/kafka.tgz -C /var/lib/kafka --strip=1', + refreshonly => true, + user => 'kafka', + } + + ##Zookeeper installation + ensure_packages([ + 'apt-transport-https', + 'ca-certificates', + 'curl', + 'gnupg', + ]) + file { 'clickhouse-source-add': + ensure => file, + path => '/etc/apt/sources.list.d/clickhouse.list', + content => "deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main", + notify => Exec['clickhouse-source-key'], + } + file_line { 'clickhouse-listen' + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'clientPortAddress=127.0.0.1', + match => 'clientPortAddress=', + } + exec { 'clickhouse-source-key': + command => '/usr/bin/wget -fsSLhttps://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key -O /usr/share/keyrings/clickhouse-keyring.gpg', + logoutput => 'on_failure', + try_sleep => 1, + refreshonly => true, + notify => Exec['docker-source-update'], + } + exec { 'apt-update': + command => '/usr/bin/apt-get update', + logoutput => 'on_failure', + try_sleep => 1, + refreshonly => true, + require => Package['apt-transport-https'], + } + + package { 'clickhouse': + ensure => installed, + require => [File['clickhouse-source-add'], Exec['clickhouse-source-key'], Exec['apt-update'], File_Line['clickhouse-listen']], + } + -> file { '/etc/systemd/system/clickhouse.service': + ensure => present, + source => 'puppet:///modules/akvorado/clickhouse.service', + mode => '0644', + owner => 'root', + group => 'root', + notify => [Exec['systemctl-daemon-reload'],Service['clickhouse']], + } } From fa7019a25d1e14bfa32c331aecbef8c3ee12b5d3 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 20:50:22 +0200 Subject: [PATCH 006/107] fix cosmetic issues --- modules/akvorado/manifests/init.pp | 44 +++++++++++++++--------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 9b511207..8f046804 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -92,10 +92,10 @@ ensure => 'present', } -> user { 'kafka': - ensure => 'present', - system => true, - home => '/var/lib/kafka', - managegome => true, + ensure => 'present', + system => true, + home => '/var/lib/kafka', + managegome => true, } -> file { '/tmp/kafka.tgz': ensure => file, @@ -124,16 +124,16 @@ line => 'delete.topic.enable' } -> file_line { 'kafka-listenlocalhost' - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'listeners=PLAINTEXT://localhost:9092' - match => '#listeners=PLAINTEXT' + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'listeners=PLAINTEXT://localhost:9092' + match => '#listeners=PLAINTEXT' } -> file_line { 'kafka-logdir' - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'log.dirs=/var/log/kafka' - match => 'log.dirs=' + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'log.dirs=/var/log/kafka' + match => 'log.dirs=' } exec { 'untar-kafka': command => '/bin/tar -zxf /tmp/kafka.tgz -C /var/lib/kafka --strip=1', @@ -143,22 +143,22 @@ ##Zookeeper installation ensure_packages([ - 'apt-transport-https', - 'ca-certificates', - 'curl', - 'gnupg', + 'apt-transport-https', + 'ca-certificates', + 'curl', + 'gnupg', ]) file { 'clickhouse-source-add': ensure => file, path => '/etc/apt/sources.list.d/clickhouse.list', - content => "deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main", + content => 'deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main', notify => Exec['clickhouse-source-key'], } file_line { 'clickhouse-listen' - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'clientPortAddress=127.0.0.1', - match => 'clientPortAddress=', + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'clientPortAddress=127.0.0.1', + match => 'clientPortAddress=', } exec { 'clickhouse-source-key': command => '/usr/bin/wget -fsSLhttps://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key -O /usr/share/keyrings/clickhouse-keyring.gpg', @@ -174,7 +174,7 @@ refreshonly => true, require => Package['apt-transport-https'], } - + package { 'clickhouse': ensure => installed, require => [File['clickhouse-source-add'], Exec['clickhouse-source-key'], Exec['apt-update'], File_Line['clickhouse-listen']], From f11d948499b5b3900fdf333673ead29db45fa6ab Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 20:51:57 +0200 Subject: [PATCH 007/107] fix indent --- modules/akvorado/manifests/init.pp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 8f046804..558c3a76 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -92,10 +92,10 @@ ensure => 'present', } -> user { 'kafka': - ensure => 'present', - system => true, - home => '/var/lib/kafka', - managegome => true, + ensure => 'present', + system => true, + home => '/var/lib/kafka', + managegome => true, } -> file { '/tmp/kafka.tgz': ensure => file, From 6b567eb68d521f418a9f76b5ce5d22d8aaec6a46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 16 Jun 2024 21:40:17 +0200 Subject: [PATCH 008/107] Update akvorado.py --- modules/akvorado.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/modules/akvorado.py b/modules/akvorado.py index 034e579c..721034cf 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -24,26 +24,31 @@ def get_prefixes(ipversion): if ipversion == "4": db.execute( - 'SELECT SUBSTR(name,1, INSTR(name, "@")-1), name, short_name, ipv4_txt' + 'SELECT SUBSTR(name,1, INSTR(name, "@")-1) AS location, name, short_name, ipv4_txt' ' FROM network' - ' WHERE node_id NOT IN (SELECT option.node_id from option where name = "NO-AKV") and name like "%@%" and ipv4_txt is not NULL' + ' WHERE node_id NOT IN (SELECT option.node_id FROM option WHERE name = "no-akv")' + ' AND name LIKE "%@%" AND ipv4_txt IS NOT NULL' ) elif ipversion == "6": db.execute( - 'SELECT SUBSTR(name,1, INSTR(name, "@")-1), name, short_name, ipv6_txt' + 'SELECT SUBSTR(name,1, INSTR(name, "@")-1) AS location, name, short_name, ipv6_txt' ' FROM network' - ' WHERE node_id NOT IN (SELECT option.node_id from option where name = "NO-AKV") and name like "%@%" and ipv6_txt is not NULL' + ' WHERE node_id NOT IN (SELECT option.node_id FROM option WHERE name = "no-akv")' + ' AND name LIKE "%@%" AND ipv6_txt IS NOT NULL' ) else: raise NetworkTypeNotFoundError('network type must be 4 or 6') res = db.fetchall() - conn.close() if not res: raise NetworkNotFoundError('network not found') + + column_names = [description[0] for description in db.description] + conn.close() + rows_dict = [dict(zip(column_names, row)) for row in res] - return res + return rows_dict def generate(host, *args): From c3e200a0e8a12aba14a8eda961e41d2038d77104 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 16 Jun 2024 21:53:26 +0200 Subject: [PATCH 009/107] Update akvorado.py --- modules/akvorado.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/akvorado.py b/modules/akvorado.py index 721034cf..590781a5 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -16,11 +16,11 @@ def get_prefixes(ipversion): conn = sqlite3.connect(DB_FILE) db = conn.cursor() except sqlite3.Error as e: - print "An error occurred:", e.args[0] - sys.exit(2) + print("An error occurred: {}".format(e.args[0])) + exit(2) else: - print "No database file found: %s" % DB_FILE - sys.exit(3) + print("No database file found: {}".format(DB_FILE)) + exit(3) if ipversion == "4": db.execute( From c82b7bd488f85e100e9c2eae23cb81f5014346b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 16 Jun 2024 21:54:55 +0200 Subject: [PATCH 010/107] Update akvorado.py --- modules/akvorado.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/akvorado.py b/modules/akvorado.py index 590781a5..95058f5c 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -43,7 +43,7 @@ def get_prefixes(ipversion): res = db.fetchall() if not res: raise NetworkNotFoundError('network not found') - + column_names = [description[0] for description in db.description] conn.close() rows_dict = [dict(zip(column_names, row)) for row in res] @@ -59,4 +59,3 @@ def generate(host, *args): info['ipv4_prefixes'] = get_prefixes('4') print(info) return {'akverado': info} - From b94eec3982d5ab314d5eddba1ec5315b8f39dcd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 16 Jun 2024 21:56:11 +0200 Subject: [PATCH 011/107] Update akvorado.py --- modules/akvorado.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/akvorado.py b/modules/akvorado.py index 95058f5c..544c521b 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -10,6 +10,7 @@ DB_FILE = '/etc/ipplan.db' + def get_prefixes(ipversion): if os.path.isfile(DB_FILE): try: From 2ea8a88490cdbc14ce9301dc3cb92b035ccd56fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 16 Jun 2024 23:01:33 +0200 Subject: [PATCH 012/107] Update akvorado.py --- modules/akvorado.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/akvorado.py b/modules/akvorado.py index 544c521b..ec6b6b09 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -58,5 +58,4 @@ def generate(host, *args): info['current_event'] = lib.get_current_event() info['ipv6_prefixes'] = get_prefixes('6') info['ipv4_prefixes'] = get_prefixes('4') - print(info) return {'akverado': info} From add344a20795fa7cebf0cd42534e887d18f4b6ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 16 Jun 2024 23:05:02 +0200 Subject: [PATCH 013/107] Update akvorado.py --- modules/akvorado.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado.py b/modules/akvorado.py index ec6b6b09..0825ccc6 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -58,4 +58,4 @@ def generate(host, *args): info['current_event'] = lib.get_current_event() info['ipv6_prefixes'] = get_prefixes('6') info['ipv4_prefixes'] = get_prefixes('4') - return {'akverado': info} + return {'akvorado': info} From f2647031ca17a123928a0ba73846a30faa8ea6e4 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:11:31 +0200 Subject: [PATCH 014/107] add columns --- modules/akvorado/manifests/init.pp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 558c3a76..2ba92536 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -117,19 +117,19 @@ group => 'root', notify => [Exec['systemctl-daemon-reload'],Service['kafka']], } - -> file_line { 'kafka-enabledeletetopics' + -> file_line { 'kafka-enabledeletetopics': ensure => 'present', path => '/var/lib/kafka/config/server.properties', line => 'delete.topic.enable = true' line => 'delete.topic.enable' } - -> file_line { 'kafka-listenlocalhost' + -> file_line { 'kafka-listenlocalhost': ensure => 'present', path => '/var/lib/kafka/config/server.properties', line => 'listeners=PLAINTEXT://localhost:9092' match => '#listeners=PLAINTEXT' } - -> file_line { 'kafka-logdir' + -> file_line { 'kafka-logdir': ensure => 'present', path => '/var/lib/kafka/config/server.properties', line => 'log.dirs=/var/log/kafka' @@ -154,7 +154,7 @@ content => 'deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main', notify => Exec['clickhouse-source-key'], } - file_line { 'clickhouse-listen' + file_line { 'clickhouse-listen': ensure => 'present', path => '/var/lib/kafka/config/server.properties', line => 'clientPortAddress=127.0.0.1', From 3b5855ad1087efbe07403cbc27b2942cc6c75c13 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:13:44 +0200 Subject: [PATCH 015/107] add commas --- modules/akvorado/manifests/init.pp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 2ba92536..eaa450a2 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -120,20 +120,20 @@ -> file_line { 'kafka-enabledeletetopics': ensure => 'present', path => '/var/lib/kafka/config/server.properties', - line => 'delete.topic.enable = true' - line => 'delete.topic.enable' + line => 'delete.topic.enable = true', + match => 'delete.topic.enable', } -> file_line { 'kafka-listenlocalhost': ensure => 'present', path => '/var/lib/kafka/config/server.properties', - line => 'listeners=PLAINTEXT://localhost:9092' - match => '#listeners=PLAINTEXT' + line => 'listeners=PLAINTEXT://localhost:9092', + match => '#listeners=PLAINTEXT', } -> file_line { 'kafka-logdir': ensure => 'present', path => '/var/lib/kafka/config/server.properties', - line => 'log.dirs=/var/log/kafka' - match => 'log.dirs=' + line => 'log.dirs=/var/log/kafka', + match => 'log.dirs=', } exec { 'untar-kafka': command => '/bin/tar -zxf /tmp/kafka.tgz -C /var/lib/kafka --strip=1', From 644d284fa6009decfe8c390dfabe45ffb42c0a57 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:21:00 +0200 Subject: [PATCH 016/107] add parameters to akvorado class --- modules/akvorado/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index eaa450a2..e0eb7c6e 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -10,7 +10,7 @@ # === Parameters # -class akvorado { +class akvorado ($current_event, $ipv4_prefixes, $ipv6_prefixes) { #Create user/group for Akvorodo group { 'akvorado': From e1f09e46a136d8b2eecb5baa270287f1bec85fc3 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:22:49 +0200 Subject: [PATCH 017/107] fix typo --- modules/akvorado/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index e0eb7c6e..d2616ce2 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -95,7 +95,7 @@ ensure => 'present', system => true, home => '/var/lib/kafka', - managegome => true, + managehome => true, } -> file { '/tmp/kafka.tgz': ensure => file, From adde4b4acae6487b7c13580e69ad009cd7b45d2a Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:26:56 +0200 Subject: [PATCH 018/107] fix indent more --- modules/akvorado/manifests/init.pp | 32 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index d2616ce2..8a14bd12 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -115,25 +115,25 @@ mode => '0644', owner => 'root', group => 'root', - notify => [Exec['systemctl-daemon-reload'],Service['kafka']], + notify => [ Exec['systemctl-daemon-reload'], Service['kafka'] ], } -> file_line { 'kafka-enabledeletetopics': ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'delete.topic.enable = true', - match => 'delete.topic.enable', + path => '/var/lib/kafka/config/server.properties', + line => 'delete.topic.enable = true', + match => 'delete.topic.enable', } -> file_line { 'kafka-listenlocalhost': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'listeners=PLAINTEXT://localhost:9092', - match => '#listeners=PLAINTEXT', + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'listeners=PLAINTEXT://localhost:9092', + match => '#listeners=PLAINTEXT', } -> file_line { 'kafka-logdir': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'log.dirs=/var/log/kafka', - match => 'log.dirs=', + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'log.dirs=/var/log/kafka', + match => 'log.dirs=', } exec { 'untar-kafka': command => '/bin/tar -zxf /tmp/kafka.tgz -C /var/lib/kafka --strip=1', @@ -155,10 +155,10 @@ notify => Exec['clickhouse-source-key'], } file_line { 'clickhouse-listen': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'clientPortAddress=127.0.0.1', - match => 'clientPortAddress=', + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'clientPortAddress=127.0.0.1', + match => 'clientPortAddress=', } exec { 'clickhouse-source-key': command => '/usr/bin/wget -fsSLhttps://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key -O /usr/share/keyrings/clickhouse-keyring.gpg', From ee7f97f8decc795e23be2caa2d196dd28fbdcb73 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:28:33 +0200 Subject: [PATCH 019/107] declare kafka service --- modules/akvorado/manifests/init.pp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 8a14bd12..e76fef67 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -75,10 +75,10 @@ -> service { 'akvorado-orch': ensure => running, } --> service { 'akvorado-inlet': + -> service { 'akvorado-inlet': ensure => running, } --> service { 'akvorado-console': + -> service { 'akvorado-console': ensure => running, } @@ -140,6 +140,9 @@ refreshonly => true, user => 'kafka', } + -> service { 'kafka': + ensure => running, + } ##Zookeeper installation ensure_packages([ @@ -187,4 +190,7 @@ group => 'root', notify => [Exec['systemctl-daemon-reload'],Service['clickhouse']], } + -> service { 'kafka': + ensure => running, + } } From d99a4ec8dfa9dfa6e4e96e3309300f46c782a62d Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:31:28 +0200 Subject: [PATCH 020/107] rename duplicate service kafka --- modules/akvorado/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index e76fef67..c102fb2b 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -190,7 +190,7 @@ group => 'root', notify => [Exec['systemctl-daemon-reload'],Service['clickhouse']], } - -> service { 'kafka': + -> service { 'clickhouse': ensure => running, } } From 4698f4817da3601e769466442907cb32fec72f63 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:34:58 +0200 Subject: [PATCH 021/107] fix more stuff --- modules/akvorado/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index c102fb2b..073f0683 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -168,7 +168,7 @@ logoutput => 'on_failure', try_sleep => 1, refreshonly => true, - notify => Exec['docker-source-update'], + notify => Exec['apt-update'], } exec { 'apt-update': command => '/usr/bin/apt-get update', From b58bc5e6659336ba2065b99d33c39b2ff478131e Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:39:01 +0200 Subject: [PATCH 022/107] give wget a bit more space --- modules/akvorado/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 073f0683..17aebfd5 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -164,7 +164,7 @@ match => 'clientPortAddress=', } exec { 'clickhouse-source-key': - command => '/usr/bin/wget -fsSLhttps://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key -O /usr/share/keyrings/clickhouse-keyring.gpg', + command => '/usr/bin/wget -fsSL https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key -O /usr/share/keyrings/clickhouse-keyring.gpg', logoutput => 'on_failure', try_sleep => 1, refreshonly => true, From fc3d0f3fd980036a8d4b58353201cc00696cbd77 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:52:03 +0200 Subject: [PATCH 023/107] fix zookeeper --- modules/akvorado/manifests/init.pp | 31 ++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 17aebfd5..b51e753d 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -117,34 +117,55 @@ group => 'root', notify => [ Exec['systemctl-daemon-reload'], Service['kafka'] ], } + -> file { '/etc/systemd/system/zookeeper.service': + ensure => present, + source => 'puppet:///modules/akvorado/zookeeper.service', + mode => '0644', + owner => 'root', + group => 'root', + notify => [ Exec['systemctl-daemon-reload'], Service['zookeeper'] ], + } -> file_line { 'kafka-enabledeletetopics': ensure => 'present', path => '/var/lib/kafka/config/server.properties', line => 'delete.topic.enable = true', match => 'delete.topic.enable', + notify => Service['kafka'], } -> file_line { 'kafka-listenlocalhost': ensure => 'present', path => '/var/lib/kafka/config/server.properties', line => 'listeners=PLAINTEXT://localhost:9092', match => '#listeners=PLAINTEXT', + notify => Service['kafka'], } -> file_line { 'kafka-logdir': ensure => 'present', path => '/var/lib/kafka/config/server.properties', line => 'log.dirs=/var/log/kafka', match => 'log.dirs=', + notify => Service['kafka'], + } + -> file_line { 'zookeeper-listen': + ensure => 'present', + path => '/var/lib/kafka/config/zookeeper.properties', + line => 'clientPortAddress=127.0.0.1', + match => 'clientPortAddress=', + notify => Service['zookeeper'], } exec { 'untar-kafka': - command => '/bin/tar -zxf /tmp/kafka.tgz -C /var/lib/kafka --strip=1', + command => '/bin/tar -xvf kafka.tgz -C /var/lib/kafka --strip 1', refreshonly => true, user => 'kafka', } -> service { 'kafka': ensure => running, } + -> service { 'zookeeper': + ensure => running, + } - ##Zookeeper installation + ##Clickhouse installation ensure_packages([ 'apt-transport-https', 'ca-certificates', @@ -157,12 +178,6 @@ content => 'deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main', notify => Exec['clickhouse-source-key'], } - file_line { 'clickhouse-listen': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'clientPortAddress=127.0.0.1', - match => 'clientPortAddress=', - } exec { 'clickhouse-source-key': command => '/usr/bin/wget -fsSL https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key -O /usr/share/keyrings/clickhouse-keyring.gpg', logoutput => 'on_failure', From 1d2b937de54cce157a652e3e7a401a41ad3593e6 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:54:43 +0200 Subject: [PATCH 024/107] . --- modules/akvorado/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index b51e753d..dd53455e 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -195,7 +195,7 @@ package { 'clickhouse': ensure => installed, - require => [File['clickhouse-source-add'], Exec['clickhouse-source-key'], Exec['apt-update'], File_Line['clickhouse-listen']], + require => [File['clickhouse-source-add'], Exec['clickhouse-source-key'], Exec['apt-update']], } -> file { '/etc/systemd/system/clickhouse.service': ensure => present, From c321f7e740af6ac4bac922beeeb53202e3c7d530 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:58:07 +0200 Subject: [PATCH 025/107] . --- modules/akvorado/manifests/init.pp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index dd53455e..4653754d 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -193,10 +193,13 @@ require => Package['apt-transport-https'], } - package { 'clickhouse': + package { 'clickhouse-server': ensure => installed, require => [File['clickhouse-source-add'], Exec['clickhouse-source-key'], Exec['apt-update']], } + -> package { 'clickhouse-client': + ensure => installed, + } -> file { '/etc/systemd/system/clickhouse.service': ensure => present, source => 'puppet:///modules/akvorado/clickhouse.service', From 9d59ec6306f2d0245eb3d4d91f03f1d6b3a6b438 Mon Sep 17 00:00:00 2001 From: furest Date: Mon, 17 Jun 2024 00:27:26 +0200 Subject: [PATCH 026/107] . --- modules/akvorado/manifests/init.pp | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 4653754d..350858ad 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -154,7 +154,7 @@ notify => Service['zookeeper'], } exec { 'untar-kafka': - command => '/bin/tar -xvf kafka.tgz -C /var/lib/kafka --strip 1', + command => '/bin/tar -xvf /tmp/kafka.tgz -C /var/lib/kafka --strip 1', refreshonly => true, user => 'kafka', } @@ -200,15 +200,7 @@ -> package { 'clickhouse-client': ensure => installed, } - -> file { '/etc/systemd/system/clickhouse.service': - ensure => present, - source => 'puppet:///modules/akvorado/clickhouse.service', - mode => '0644', - owner => 'root', - group => 'root', - notify => [Exec['systemctl-daemon-reload'],Service['clickhouse']], - } - -> service { 'clickhouse': + -> service { 'clickhouse-server': ensure => running, } } From 23b734231e1ed2931a45d572082e124d45b2015e Mon Sep 17 00:00:00 2001 From: furest Date: Mon, 17 Jun 2024 00:34:25 +0200 Subject: [PATCH 027/107] fix service file --- modules/akvorado/files/kafka.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/files/kafka.service b/modules/akvorado/files/kafka.service index 51df02db..64d7b10e 100644 --- a/modules/akvorado/files/kafka.service +++ b/modules/akvorado/files/kafka.service @@ -5,7 +5,7 @@ After=zookeeper.service [Service] Type=simple User=kafka -ExecStart=/bin/sh -c '/var/lib/kafka/bin/kafka-server-start.sh /var/lib/kafka/config/server.properties > /var/log/kaf +ExecStart=/bin/sh -c '/home/kafka/kafka/bin/kafka-server-start.sh /home/kafka/kafka/config/server.properties > /var/log/kafka/kafka.log 2>&1' ExecStop=/var/lib/kafka/bin/kafka-server-stop.sh Restart=on-abnormal From 5fb19cb0655081603d8669a71605bef976a34e9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Mon, 17 Jun 2024 00:53:58 +0200 Subject: [PATCH 028/107] Update akvorado.py --- modules/akvorado.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/akvorado.py b/modules/akvorado.py index 0825ccc6..f8a21dd9 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -52,6 +52,10 @@ def get_prefixes(ipversion): return rows_dict +def requires(host, *args): + return ['apache(ldap)'] + + def generate(host, *args): info = {} From 2324939b34445c945078d5f1a4b507fef8de89ea Mon Sep 17 00:00:00 2001 From: furest Date: Mon, 17 Jun 2024 01:03:07 +0200 Subject: [PATCH 029/107] fix indent in template --- modules/akvorado/templates/akvorado.yaml.erb | 566 +++++++++---------- 1 file changed, 282 insertions(+), 284 deletions(-) diff --git a/modules/akvorado/templates/akvorado.yaml.erb b/modules/akvorado/templates/akvorado.yaml.erb index 041c3280..be90ee47 100644 --- a/modules/akvorado/templates/akvorado.yaml.erb +++ b/modules/akvorado/templates/akvorado.yaml.erb @@ -1,303 +1,301 @@ --- reporting: + logging: {} + metrics: {} +http: + listen: :8080 + profiler: true + cache: + type: memory +clickhouse: + servers: + - localhost:9000 + cluster: "" + database: default + username: default + password: "dhtech" + maxopenconns: 10 + dialtimeout: 5s + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + skipmigrations: false + kafka: + topic: flows + brokers: + - localhost:9092 + version: 3.3.1 + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + saslusername: "" + saslpassword: "" + saslmechanism: none + consumers: 4 + groupname: clickhouse + enginesettings: [] + resolutions: + - interval: 0s + ttl: 360h0m0s + - interval: 1m0s + ttl: 168h0m0s + - interval: 5m0s + ttl: 2160h0m0s + - interval: 1h0m0s + ttl: 8640h0m0s + maxpartitions: 50 + systemlogttl: 720h0m0s + prometheusendpoint: /metrics + asns: + 25037: Dreamhack ACME Corporation + networks: + # 2a01:db8:cafe:1::/64: + # name: ipv6-customers + # role: customers + # site: "" + # region: "" + # city: "" + # state: "" + # country: "" + # tenant: "" + # asn: 0 +<% @ipv4_prefixes.each do |ipv4| -%> + <%=ipv4['ipv4_txt']%>: + name: "<%= ipv4['short_name'] %>" + role: "<%= ipv4['location'] %>" +<% end -%> +<% @ipv6_prefixes.each do |ipv6| -%> + <%=ipv6['ipv6_txt']%>: + name: "<%= ipv6['short_name'] %>" + role: "<%= ipv6['location'] %>" +<% end -%> + networksources: {} + networksourcestimeout: 10s + orchestratorurl: http://localhost:8080 +kafka: + topic: flows + brokers: + - localhost:9092 + version: 3.3.1 + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + saslusername: "" + saslpassword: "" + saslmechanism: none + topicconfiguration: + numpartitions: 8 + replicationfactor: 1 + configentries: + cleanup.policy: delete + compression.type: producer + retention.ms: "86400000" + segment.bytes: "1073741824" + configentriesstrictsync: true +geoip: + asndatabase: + - /usr/share/GeoIP/asn.mmdb + geodatabase: + - /usr/share/GeoIP/country.mmdb + optional: true +schema: + disabled: [] + enabled: [] + maintableonly: [] + notmaintableonly: [] + materialize: [] + customdictionaries: {} +inlet: + - reporting: logging: {} metrics: {} -http: - listen: :8080 + http: + listen: :8081 profiler: true cache: - type: memory -clickhouse: + type: memory + flow: + inputs: + - decoder: netflow + listen: :2055 + queuesize: 100000 + receivebuffer: 10485760 + timestampsource: udp + type: udp + usesrcaddrforexporteraddr: false + workers: 6 + - decoder: sflow + listen: :6343 + queuesize: 100000 + receivebuffer: 10485760 + timestampsource: udp + type: udp + usesrcaddrforexporteraddr: false + workers: 6 + ratelimit: 0 + metadata: + cacheduration: 30m0s + cacherefresh: 1h0m0s + cachecheckinterval: 2m0s + cachepersistfile: "" + providers: + - agents: {} + communities: + ::/0: + - public + pollerretries: 1 + pollertimeout: 1s + ports: + ::/0: 161 + securityparameters: {} + type: snmp + workers: 10 + maxbatchrequests: 10 + routing: + provider: + collectasns: true + collectaspaths: true + collectcommunities: true + keep: 5m0s + listen: :10179 + rds: [] + ribpeerremovalbatchroutes: 5000 + ribpeerremovalmaxqueue: 10000 + ribpeerremovalmaxtime: 100ms + ribpeerremovalsleepinterval: 500ms + type: bmp + kafka: + topic: flows + brokers: + - localhost:9092 + version: 3.3.1 + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + saslusername: "" + saslpassword: "" + saslmechanism: none + flushinterval: 10s + flushbytes: 104857599 + maxmessagebytes: 1000000 + compressioncodec: zstd + queuesize: 32 + core: + workers: 6 + exporterclassifiers: + - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") + - ClassifyRegion("europe") + - ClassifyTenant("acme") + - ClassifyRole("edge") + interfaceclassifiers: + - | + ClassifyConnectivityRegex(Interface.Description, "^(?i)(transit|pni|ppni|ix):? ", "$1") && + ClassifyProviderRegex(Interface.Description, "^\\S+?\\s(\\S+)", "$1") && + ClassifyExternal() + - ClassifyInternal() + classifiercacheduration: 5m0s + defaultsamplingrate: {} + overridesamplingrate: {} + asnproviders: + - flow + - routing + netproviders: + - flow + - routing + schema: + disabled: [] + enabled: [] + maintableonly: [] + notmaintableonly: [] + materialize: [] + customdictionaries: {} +console: + - reporting: + logging: {} + metrics: {} + http: + listen: :8082 + profiler: true + cache: + db: 0 + password: "" + protocol: tcp + server: localhost:6379 + type: redis + username: "" + defaultvisualizeoptions: + graphtype: stacked + start: 6 hours ago + end: now + filter: InIfBoundary = external + dimensions: + - SrcAS + limit: 10 + homepagetopwidgets: + - src-as + - src-port + - protocol + - src-country + - etype + homepagegraphfilter: InIfBoundary = 'external' + dimensionslimit: 50 + cachettl: 3h0m0s + clickhouse: servers: - - localhost:9000 + - localhost:9000 cluster: "" database: default username: default - password: "dhtech" + password: "" maxopenconns: 10 dialtimeout: 5s tls: - enable: false - verify: true - cafile: "" - certfile: "" - keyfile: "" - skipmigrations: false - kafka: - topic: flows - brokers: - - localhost:9092 - version: 3.3.1 - tls: - enable: false - verify: true - cafile: "" - certfile: "" - keyfile: "" - saslusername: "" - saslpassword: "" - saslmechanism: none - consumers: 4 - groupname: clickhouse - enginesettings: [] - resolutions: - - interval: 0s - ttl: 360h0m0s - - interval: 1m0s - ttl: 168h0m0s - - interval: 5m0s - ttl: 2160h0m0s - - interval: 1h0m0s - ttl: 8640h0m0s - maxpartitions: 50 - systemlogttl: 720h0m0s - prometheusendpoint: /metrics - asns: - 25037: Dreamhack ACME Corporation - networks: - # 2a01:db8:cafe:1::/64: - # name: ipv6-customers - # role: customers - # site: "" - # region: "" - # city: "" - # state: "" - # country: "" - # tenant: "" - # asn: 0 - - <% @ipv4_prefixes.each do |ipv4| -%> - "<%= ipv4['ipv4_txt'] %>" - name: "<%= ipv4['short_name'] %>" - role: "<%= ipv4['location'] %>" - <% end -%> - <% @ipv6_prefixes.each do |ipv6| -%> - "<%= ipv6['ipv6_txt'] %>" - name: "<%= ipv6['short_name'] %>" - role: "<%= ipv6['location'] %>" - <% end -%> - - networksources: {} - networksourcestimeout: 10s - orchestratorurl: http://localhost:8080 -kafka: - topic: flows - brokers: - - localhost:9092 - version: 3.3.1 - tls: - enable: false - verify: true - cafile: "" - certfile: "" - keyfile: "" - saslusername: "" - saslpassword: "" - saslmechanism: none - topicconfiguration: - numpartitions: 8 - replicationfactor: 1 - configentries: - cleanup.policy: delete - compression.type: producer - retention.ms: "86400000" - segment.bytes: "1073741824" - configentriesstrictsync: true -geoip: - asndatabase: - - /usr/share/GeoIP/asn.mmdb - geodatabase: - - /usr/share/GeoIP/country.mmdb - optional: true -schema: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + auth: + headers: + login: Remote-User + name: Remote-Name + email: Remote-Email + logouturl: X-Logout-URL + defaultuser: + login: __default + name: Default User + email: "" + logouturl: "" + database: + driver: sqlite + dsn: file::memory:?cache=shared + savedfilters: + - description: From Netflix + content: InIfBoundary = external AND SrcAS = AS2906 + - description: From GAFAM + content: InIfBoundary = external AND SrcAS IN (AS15169, AS16509, AS32934, AS6185, AS8075) + - description: From Swedish Armed Forces + content: InIfBoundary = external AND SrcAS = AS9201 + - description: Valve Corporation + content: InIfBoundary = external AND SrcAS = AS32590 + + + schema: disabled: [] enabled: [] maintableonly: [] notmaintableonly: [] materialize: [] customdictionaries: {} -inlet: - - reporting: - logging: {} - metrics: {} - http: - listen: :8081 - profiler: true - cache: - type: memory - flow: - inputs: - - decoder: netflow - listen: :2055 - queuesize: 100000 - receivebuffer: 10485760 - timestampsource: udp - type: udp - usesrcaddrforexporteraddr: false - workers: 6 - - decoder: sflow - listen: :6343 - queuesize: 100000 - receivebuffer: 10485760 - timestampsource: udp - type: udp - usesrcaddrforexporteraddr: false - workers: 6 - ratelimit: 0 - metadata: - cacheduration: 30m0s - cacherefresh: 1h0m0s - cachecheckinterval: 2m0s - cachepersistfile: "" - providers: - - agents: {} - communities: - ::/0: - - public - pollerretries: 1 - pollertimeout: 1s - ports: - ::/0: 161 - securityparameters: {} - type: snmp - workers: 10 - maxbatchrequests: 10 - routing: - provider: - collectasns: true - collectaspaths: true - collectcommunities: true - keep: 5m0s - listen: :10179 - rds: [] - ribpeerremovalbatchroutes: 5000 - ribpeerremovalmaxqueue: 10000 - ribpeerremovalmaxtime: 100ms - ribpeerremovalsleepinterval: 500ms - type: bmp - kafka: - topic: flows - brokers: - - localhost:9092 - version: 3.3.1 - tls: - enable: false - verify: true - cafile: "" - certfile: "" - keyfile: "" - saslusername: "" - saslpassword: "" - saslmechanism: none - flushinterval: 10s - flushbytes: 104857599 - maxmessagebytes: 1000000 - compressioncodec: zstd - queuesize: 32 - core: - workers: 6 - exporterclassifiers: - - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") - - ClassifyRegion("europe") - - ClassifyTenant("acme") - - ClassifyRole("edge") - interfaceclassifiers: - - | - ClassifyConnectivityRegex(Interface.Description, "^(?i)(transit|pni|ppni|ix):? ", "$1") && - ClassifyProviderRegex(Interface.Description, "^\\S+?\\s(\\S+)", "$1") && - ClassifyExternal() - - ClassifyInternal() - classifiercacheduration: 5m0s - defaultsamplingrate: {} - overridesamplingrate: {} - asnproviders: - - flow - - routing - netproviders: - - flow - - routing - schema: - disabled: [] - enabled: [] - maintableonly: [] - notmaintableonly: [] - materialize: [] - customdictionaries: {} -console: - - reporting: - logging: {} - metrics: {} - http: - listen: :8082 - profiler: true - cache: - db: 0 - password: "" - protocol: tcp - server: localhost:6379 - type: redis - username: "" - defaultvisualizeoptions: - graphtype: stacked - start: 6 hours ago - end: now - filter: InIfBoundary = external - dimensions: - - SrcAS - limit: 10 - homepagetopwidgets: - - src-as - - src-port - - protocol - - src-country - - etype - homepagegraphfilter: InIfBoundary = 'external' - dimensionslimit: 50 - cachettl: 3h0m0s - clickhouse: - servers: - - localhost:9000 - cluster: "" - database: default - username: default - password: "" - maxopenconns: 10 - dialtimeout: 5s - tls: - enable: false - verify: true - cafile: "" - certfile: "" - keyfile: "" - auth: - headers: - login: Remote-User - name: Remote-Name - email: Remote-Email - logouturl: X-Logout-URL - defaultuser: - login: __default - name: Default User - email: "" - logouturl: "" - database: - driver: sqlite - dsn: file::memory:?cache=shared - savedfilters: - - description: From Netflix - content: InIfBoundary = external AND SrcAS = AS2906 - - description: From GAFAM - content: InIfBoundary = external AND SrcAS IN (AS15169, AS16509, AS32934, AS6185, AS8075) - - description: From Swedish Armed Forces - content: InIfBoundary = external AND SrcAS = AS9201 - - description: Valve Corporation - content: InIfBoundary = external AND SrcAS = AS32590 - - - schema: - disabled: [] - enabled: [] - maintableonly: [] - notmaintableonly: [] - materialize: [] - customdictionaries: {} From 390d6ef67f99c648ce4544741c627a1036a1c9ff Mon Sep 17 00:00:00 2001 From: furest Date: Mon, 17 Jun 2024 01:31:36 +0200 Subject: [PATCH 030/107] fix yaml --- modules/akvorado/templates/akvorado.yaml.erb | 385 +++++++++---------- 1 file changed, 192 insertions(+), 193 deletions(-) diff --git a/modules/akvorado/templates/akvorado.yaml.erb b/modules/akvorado/templates/akvorado.yaml.erb index be90ee47..0660ed63 100644 --- a/modules/akvorado/templates/akvorado.yaml.erb +++ b/modules/akvorado/templates/akvorado.yaml.erb @@ -9,7 +9,7 @@ http: type: memory clickhouse: servers: - - localhost:9000 + - 127.0.0.1:9000 cluster: "" database: default username: default @@ -26,8 +26,8 @@ clickhouse: kafka: topic: flows brokers: - - localhost:9092 - version: 3.3.1 + - 127.0.0.1:9092 + version: 3.7.0 tls: enable: false verify: true @@ -37,21 +37,21 @@ clickhouse: saslusername: "" saslpassword: "" saslmechanism: none - consumers: 4 + consumers: 1 groupname: clickhouse enginesettings: [] resolutions: - interval: 0s - ttl: 360h0m0s + ttl: 360h0m0s - interval: 1m0s - ttl: 168h0m0s + ttl: 168h0m0s - interval: 5m0s - ttl: 2160h0m0s + ttl: 2160h0m0s - interval: 1h0m0s - ttl: 8640h0m0s + ttl: 8640h0m0s maxpartitions: 50 systemlogttl: 720h0m0s - prometheusendpoint: /metrics + prometheusendpoint: "/metrics" asns: 25037: Dreamhack ACME Corporation networks: @@ -77,12 +77,12 @@ clickhouse: <% end -%> networksources: {} networksourcestimeout: 10s - orchestratorurl: http://localhost:8080 + orchestratorurl: "http://localhost:8080" kafka: topic: flows brokers: - - localhost:9092 - version: 3.3.1 + - 127.0.0.1:9092 + version: 3.7.0 tls: enable: false verify: true @@ -106,7 +106,7 @@ geoip: - /usr/share/GeoIP/asn.mmdb geodatabase: - /usr/share/GeoIP/country.mmdb - optional: true + optional: false schema: disabled: [] enabled: [] @@ -116,186 +116,185 @@ schema: customdictionaries: {} inlet: - reporting: - logging: {} - metrics: {} - http: - listen: :8081 - profiler: true - cache: - type: memory - flow: - inputs: - - decoder: netflow - listen: :2055 - queuesize: 100000 - receivebuffer: 10485760 - timestampsource: udp - type: udp - usesrcaddrforexporteraddr: false - workers: 6 - - decoder: sflow - listen: :6343 - queuesize: 100000 - receivebuffer: 10485760 - timestampsource: udp - type: udp - usesrcaddrforexporteraddr: false - workers: 6 - ratelimit: 0 - metadata: - cacheduration: 30m0s - cacherefresh: 1h0m0s - cachecheckinterval: 2m0s - cachepersistfile: "" - providers: - - agents: {} - communities: - ::/0: - - public - pollerretries: 1 - pollertimeout: 1s - ports: - ::/0: 161 - securityparameters: {} - type: snmp - workers: 10 - maxbatchrequests: 10 - routing: - provider: - collectasns: true - collectaspaths: true - collectcommunities: true - keep: 5m0s - listen: :10179 - rds: [] - ribpeerremovalbatchroutes: 5000 - ribpeerremovalmaxqueue: 10000 - ribpeerremovalmaxtime: 100ms - ribpeerremovalsleepinterval: 500ms - type: bmp - kafka: - topic: flows - brokers: - - localhost:9092 - version: 3.3.1 - tls: - enable: false - verify: true - cafile: "" - certfile: "" - keyfile: "" - saslusername: "" - saslpassword: "" - saslmechanism: none - flushinterval: 10s - flushbytes: 104857599 - maxmessagebytes: 1000000 - compressioncodec: zstd - queuesize: 32 - core: - workers: 6 - exporterclassifiers: - - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") - - ClassifyRegion("europe") - - ClassifyTenant("acme") - - ClassifyRole("edge") - interfaceclassifiers: - - | - ClassifyConnectivityRegex(Interface.Description, "^(?i)(transit|pni|ppni|ix):? ", "$1") && - ClassifyProviderRegex(Interface.Description, "^\\S+?\\s(\\S+)", "$1") && - ClassifyExternal() - - ClassifyInternal() - classifiercacheduration: 5m0s - defaultsamplingrate: {} - overridesamplingrate: {} - asnproviders: - - flow - - routing - netproviders: - - flow - - routing - schema: - disabled: [] - enabled: [] - maintableonly: [] - notmaintableonly: [] - materialize: [] - customdictionaries: {} + logging: {} + metrics: {} + http: + listen: :8081 + profiler: true + cache: + type: memory + flow: + inputs: + - decoder: netflow + listen: :2055 + queuesize: 100000 + receivebuffer: 10485760 + timestampsource: udp + type: udp + usesrcaddrforexporteraddr: false + workers: 6 + - decoder: sflow + listen: :6343 + queuesize: 100000 + receivebuffer: 10485760 + timestampsource: udp + type: udp + usesrcaddrforexporteraddr: false + workers: 6 + ratelimit: 0 + metadata: + cacheduration: 30m0s + cacherefresh: 1h0m0s + cachecheckinterval: 2m0s + cachepersistfile: "" + providers: + - agents: {} + communities: + ::/0: + - public + pollerretries: 1 + pollertimeout: 1s + ports: + ::/0: 161 + securityparameters: {} + type: snmp + workers: 1 + maxbatchrequests: 10 + routing: + provider: + collectasns: true + collectaspaths: true + collectcommunities: true + keep: 5m0s + listen: :10179 + rds: [] + ribpeerremovalbatchroutes: 5000 + ribpeerremovalmaxqueue: 10000 + ribpeerremovalmaxtime: 100ms + ribpeerremovalsleepinterval: 500ms + type: bmp + kafka: + topic: flows + brokers: + - 127.0.0.1:9092 + version: 3.7.0 + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + saslusername: "" + saslpassword: "" + saslmechanism: none + flushinterval: 10s + flushbytes: 104857599 + maxmessagebytes: 1000000 + compressioncodec: zstd + queuesize: 32 + core: + workers: 6 + exporterclassifiers: + - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") + - ClassifyRegion("europe") + - ClassifyTenant("acme") + - ClassifyRole("edge") + interfaceclassifiers: + - | + ClassifyConnectivityRegex(Interface.Description, "^(?i)(transit|pni|ppni|ix):? ", "$1") && + ClassifyProviderRegex(Interface.Description, "^\\S+?\\s(\\S+)", "$1") && + ClassifyExternal() + - ClassifyInternal() + classifiercacheduration: 5m0s + defaultsamplingrate: {} + overridesamplingrate: {} + asnproviders: + - flow + - routing + netproviders: + - flow + - routing + schema: + disabled: [] + enabled: [] + maintableonly: [] + notmaintableonly: [] + materialize: [] + customdictionaries: {} console: - reporting: - logging: {} - metrics: {} - http: - listen: :8082 - profiler: true - cache: - db: 0 + logging: {} + metrics: {} + http: + listen: :8082 + profiler: true + cache: + db: 0 + password: "" + protocol: tcp + server: localhost:6379 + type: redis + username: "" + defaultvisualizeoptions: + graphtype: stacked + start: 6 hours ago + end: now + filter: InIfBoundary = external + dimensions: + - SrcAS + limit: 10 + homepagetopwidgets: + - src-as + - src-port + - protocol + - src-country + - etype + homepagegraphfilter: InIfBoundary = 'external' + dimensionslimit: 50 + cachettl: 3h0m0s + clickhouse: + servers: + - 127.0.0.1:9000 + cluster: "" + database: default + username: default password: "" - protocol: tcp - server: localhost:6379 - type: redis - username: "" - defaultvisualizeoptions: - graphtype: stacked - start: 6 hours ago - end: now - filter: InIfBoundary = external - dimensions: - - SrcAS - limit: 10 - homepagetopwidgets: - - src-as - - src-port - - protocol - - src-country - - etype - homepagegraphfilter: InIfBoundary = 'external' - dimensionslimit: 50 - cachettl: 3h0m0s - clickhouse: - servers: - - localhost:9000 - cluster: "" - database: default - username: default - password: "" - maxopenconns: 10 - dialtimeout: 5s - tls: - enable: false - verify: true - cafile: "" - certfile: "" - keyfile: "" - auth: - headers: - login: Remote-User - name: Remote-Name - email: Remote-Email - logouturl: X-Logout-URL - defaultuser: - login: __default - name: Default User - email: "" - logouturl: "" - database: - driver: sqlite - dsn: file::memory:?cache=shared - savedfilters: - - description: From Netflix - content: InIfBoundary = external AND SrcAS = AS2906 - - description: From GAFAM - content: InIfBoundary = external AND SrcAS IN (AS15169, AS16509, AS32934, AS6185, AS8075) - - description: From Swedish Armed Forces - content: InIfBoundary = external AND SrcAS = AS9201 - - description: Valve Corporation - content: InIfBoundary = external AND SrcAS = AS32590 - - - schema: - disabled: [] - enabled: [] - maintableonly: [] - notmaintableonly: [] - materialize: [] - customdictionaries: {} + maxopenconns: 10 + dialtimeout: 5s + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + auth: + headers: + login: Remote-User + name: Remote-Name + email: Remote-Email + logouturl: X-Logout-URL + defaultuser: + login: default + name: Default User + email: "" + logouturl: "" + database: + driver: sqlite + dsn: file::memory:?cache=shared + savedfilters: + - description: From Netflix + content: InIfBoundary = external AND SrcAS = AS2906 + - description: From GAFAM + content: InIfBoundary = external AND SrcAS IN (AS15169, AS16509, AS32934, AS6185, AS8075) + - description: From Swedish Armed Forces + content: InIfBoundary = external AND SrcAS = AS9201 + - description: Valve Corporation + content: InIfBoundary = external AND SrcAS = AS32590 + schema: + disabled: [] + enabled: [] + maintableonly: [] + notmaintableonly: [] + materialize: [] + customdictionaries: {} +demoexporter: [] From 64c362e984b0081113be3c571092ec5f3bb58934 Mon Sep 17 00:00:00 2001 From: furest Date: Mon, 17 Jun 2024 02:00:16 +0200 Subject: [PATCH 031/107] fix service kafka --- modules/akvorado/files/kafka.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/files/kafka.service b/modules/akvorado/files/kafka.service index 64d7b10e..b1b0ebbd 100644 --- a/modules/akvorado/files/kafka.service +++ b/modules/akvorado/files/kafka.service @@ -5,7 +5,7 @@ After=zookeeper.service [Service] Type=simple User=kafka -ExecStart=/bin/sh -c '/home/kafka/kafka/bin/kafka-server-start.sh /home/kafka/kafka/config/server.properties > /var/log/kafka/kafka.log 2>&1' +ExecStart=/bin/sh -c ' /var/lib/kafka/bin/kafka-server-start.sh /var/lib/kafka/config/server.properties > /var/log/kafka/kafka.log 2>&1' ExecStop=/var/lib/kafka/bin/kafka-server-stop.sh Restart=on-abnormal From a2ea3d0da157d78a8a1178fc416820011acc62b4 Mon Sep 17 00:00:00 2001 From: furest Date: Mon, 17 Jun 2024 02:28:41 +0200 Subject: [PATCH 032/107] fixes --- modules/akvorado.py | 1 + modules/akvorado/manifests/init.pp | 2 +- modules/akvorado/templates/akvorado.yaml.erb | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/akvorado.py b/modules/akvorado.py index f8a21dd9..85f03acd 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -37,6 +37,7 @@ def get_prefixes(ipversion): ' FROM network' ' WHERE node_id NOT IN (SELECT option.node_id FROM option WHERE name = "no-akv")' ' AND name LIKE "%@%" AND ipv6_txt IS NOT NULL' + ' AND NOT (name = "BOGAL@DREAMHACK" AND ipv6_txt = "2a05:2240:5000::/48")' ) else: raise NetworkTypeNotFoundError('network type must be 4 or 6') diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 350858ad..1a79e72d 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -103,7 +103,7 @@ source => 'puppet:///data/kafka-latest.tgz', notify => Exec[ 'untar-kafka' ], } - -> file { '/var/log/kafka': + file { '/var/log/kafka': ensure => 'directory', owner => 'kafka', group => 'kafka', diff --git a/modules/akvorado/templates/akvorado.yaml.erb b/modules/akvorado/templates/akvorado.yaml.erb index 0660ed63..dbe728f7 100644 --- a/modules/akvorado/templates/akvorado.yaml.erb +++ b/modules/akvorado/templates/akvorado.yaml.erb @@ -13,7 +13,7 @@ clickhouse: cluster: "" database: default username: default - password: "dhtech" + password: "" maxopenconns: 10 dialtimeout: 5s tls: From 95c733c418b8d89aa06d2f7efe25c8d25b3486c3 Mon Sep 17 00:00:00 2001 From: furest Date: Mon, 17 Jun 2024 02:42:39 +0200 Subject: [PATCH 033/107] Fix order --- modules/akvorado/manifests/init.pp | 47 ++++++++++++++++-------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 1a79e72d..6c871ca2 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -101,9 +101,8 @@ ensure => file, links => follow, source => 'puppet:///data/kafka-latest.tgz', - notify => Exec[ 'untar-kafka' ], } - file { '/var/log/kafka': + -> file { '/var/log/kafka': ensure => 'directory', owner => 'kafka', group => 'kafka', @@ -126,32 +125,36 @@ notify => [ Exec['systemctl-daemon-reload'], Service['zookeeper'] ], } -> file_line { 'kafka-enabledeletetopics': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'delete.topic.enable = true', - match => 'delete.topic.enable', - notify => Service['kafka'], + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'delete.topic.enable = true', + match => 'delete.topic.enable', + notify => Service['kafka'], + require => Exec['untar-kafka'], } -> file_line { 'kafka-listenlocalhost': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'listeners=PLAINTEXT://localhost:9092', - match => '#listeners=PLAINTEXT', - notify => Service['kafka'], + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'listeners=PLAINTEXT://localhost:9092', + match => '#listeners=PLAINTEXT', + notify => Service['kafka'], + require => Exec['untar-kafka'], } -> file_line { 'kafka-logdir': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'log.dirs=/var/log/kafka', - match => 'log.dirs=', - notify => Service['kafka'], + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'log.dirs=/var/log/kafka', + match => 'log.dirs=', + notify => Service['kafka'], + require => Exec['untar-kafka'], } -> file_line { 'zookeeper-listen': - ensure => 'present', - path => '/var/lib/kafka/config/zookeeper.properties', - line => 'clientPortAddress=127.0.0.1', - match => 'clientPortAddress=', - notify => Service['zookeeper'], + ensure => 'present', + path => '/var/lib/kafka/config/zookeeper.properties', + line => 'clientPortAddress=127.0.0.1', + match => 'clientPortAddress=', + notify => Service['zookeeper'], + require => Exec['untar-kafka'], } exec { 'untar-kafka': command => '/bin/tar -xvf /tmp/kafka.tgz -C /var/lib/kafka --strip 1', From fa969bcedb651c992b095f47051d86119cbba4d0 Mon Sep 17 00:00:00 2001 From: furest Date: Sat, 22 Jun 2024 11:08:27 +0200 Subject: [PATCH 034/107] Major improvements and fixes --- modules/akvorado.py | 64 +++++ modules/akvorado/manifests/init.pp | 240 +++++++++++-------- modules/akvorado/templates/akvorado.yaml.erb | 45 +++- 3 files changed, 245 insertions(+), 104 deletions(-) diff --git a/modules/akvorado.py b/modules/akvorado.py index 85f03acd..50f02356 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -10,6 +10,68 @@ DB_FILE = '/etc/ipplan.db' +def get_sflow_clients(): + if os.path.isfile(DB_FILE): + try: + conn = sqlite3.connect(DB_FILE) + db = conn.cursor() + except sqlite3.Error as e: + print("An error occurred: {}".format(e.args[0])) + exit(2) + else: + print("No database file found: {}".format(DB_FILE)) + exit(3) + db.execute( + "SELECT h.name AS hostname, h.ipv4_addr_txt AS ipv4_addr ,h.ipv6_addr_txt AS ipv6_addr, o2.value AS layer " + "FROM host h " + "INNER JOIN option o1 ON h.node_id = o1.node_id " + "INNER JOIN option o2 ON h.node_id = o2.node_id " + "WHERE o1.name='pkg' AND o1.value='sflowclient' " + "AND o2.name='layer'" + ) + res = db.fetchall() + if not res: + return None + + column_names = [description[0] for description in db.description] + conn.close() + rows_dict = [dict(zip(column_names, row)) for row in res] + + return rows_dict + +def get_snmpv2_providers(): + providers = [] + clients = get_sflow_clients() + current_event = lib.get_current_event() + for client in clients: + key = current_event+'-mgmt/snmp:'+client['layer'] + secrets = lib.read_secret(key) + if "community" in secrets: + provider = { + "ipv4": client["ipv4_addr"], + "community": secrets["community"], + } + providers.append(provider) + return providers + +def get_snmpv3_providers(): + providers = [] + clients = get_sflow_clients() + current_event = lib.get_current_event() + for client in clients: + key = current_event+'-mgmt/snmp:'+client['layer'] + secrets = lib.read_secret(key) + if "user" in secrets: + provider = { + "ipv4": client["ipv4_addr"], + "authentication-passphrase": secrets["auth"], + "authentication-protocol": secrets["authtype"].replace(" ","").upper(), + "privacy-passphrase": secrets["priv"], + "privacy-protocol": secrets["privtype"].replace(" ","").replace("128","").upper(), + "user": secrets["user"], + } + providers.append(provider) + return providers def get_prefixes(ipversion): if os.path.isfile(DB_FILE): @@ -60,6 +122,8 @@ def requires(host, *args): def generate(host, *args): info = {} + info['snmpv3_providers'] = get_snmpv3_providers() + info['snmpv2_providers'] = get_snmpv2_providers() info['current_event'] = lib.get_current_event() info['ipv6_prefixes'] = get_prefixes('6') info['ipv4_prefixes'] = get_prefixes('4') diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 6c871ca2..e3d6def5 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -10,84 +10,13 @@ # === Parameters # -class akvorado ($current_event, $ipv4_prefixes, $ipv6_prefixes) { +class akvorado ($current_event, $ipv4_prefixes, $ipv6_prefixes, $snmpv3_providers, $snmpv2_providers) { - #Create user/group for Akvorodo - group { 'akvorado': - ensure => 'present', - } - -> user { 'akvorado': - ensure => 'present', - system => true, - } - #Create directories for akvorado - -> file { '/etc/akvorado': - ensure => 'directory', - owner => 'root', - group => 'akvorado', - mode => '0750', - } - #Copy akvorado to the server - -> file { '/usr/local/bin/akvorado': - ensure => file, - owner => 'root', - group => 'akvorado', - mode => '0550', - links => follow, - source => 'puppet:///data/akvorado-latest', - } - - file { '/etc/akvorado/akvorado.yaml': - ensure => file, - content => template('akvorado/akvorado.yaml.erb'), - notify => Service['akvorado-orch'], - } - #Systemctl config - file { '/etc/systemd/system/akvorado-orch.service': - ensure => present, - source => 'puppet:///modules/akvorado/akvorado-orch.service', - mode => '0644', - owner => 'root', - group => 'root', - notify => [Exec['systemctl-daemon-reload'],Service['akvorado-orch']], - } - file { '/etc/systemd/system/akvorado-inlet.service': - ensure => present, - source => 'puppet:///modules/akvorado/akvorado-inlet.service', - mode => '0644', - owner => 'root', - group => 'root', - notify => [Exec['systemctl-daemon-reload'],Service['akvorado-inlet']], - } - file { '/etc/systemd/system/akvorado-console.service': - ensure => present, - source => 'puppet:///modules/akvorado/akvorado-console.service', - mode => '0644', - owner => 'root', - group => 'root', - notify => [Exec['systemctl-daemon-reload'],Service['akvorado-console']], - } - - -> apache::proxy { 'akvorado': - url => '/', - backend => 'http://localhost:8082/', - } - -> service { 'akvorado-orch': - ensure => running, - } - -> service { 'akvorado-inlet': - ensure => running, - } - -> service { 'akvorado-console': - ensure => running, - } - - exec { 'systemctl-daemon-reload': - command => '/bin/systemctl daemon-reload', - refreshonly => true, - } ##Kafka installation + ensure_packages([ + 'openjdk-17-jre', + ]) group { 'kafka': ensure => 'present', } @@ -97,10 +26,11 @@ home => '/var/lib/kafka', managehome => true, } - -> file { '/tmp/kafka.tgz': - ensure => file, - links => follow, - source => 'puppet:///data/kafka-latest.tgz', + -> file { '/var/lib/kafka/kafka.tgz': + ensure => file, + links => follow, + source => 'puppet:///data/kafka-latest.tgz', + notify => Exec['untar-kafka'] } -> file { '/var/log/kafka': ensure => 'directory', @@ -108,7 +38,18 @@ group => 'kafka', mode => '0700', } - -> file { '/etc/systemd/system/kafka.service': + -> file { '/var/lib/zookeeper-data': + ensure => 'directory', + owner => 'kafka', + group => 'kafka', + mode => '0700', + } + exec { 'untar-kafka': + command => '/bin/tar -xvf /var/lib/kafka/kafka.tgz -C /var/lib/kafka --strip 1', + refreshonly => 'true', + user => 'kafka', + } + file { '/etc/systemd/system/kafka.service': ensure => present, source => 'puppet:///modules/akvorado/kafka.service', mode => '0644', @@ -128,9 +69,7 @@ ensure => 'present', path => '/var/lib/kafka/config/server.properties', line => 'delete.topic.enable = true', - match => 'delete.topic.enable', notify => Service['kafka'], - require => Exec['untar-kafka'], } -> file_line { 'kafka-listenlocalhost': ensure => 'present', @@ -138,34 +77,34 @@ line => 'listeners=PLAINTEXT://localhost:9092', match => '#listeners=PLAINTEXT', notify => Service['kafka'], - require => Exec['untar-kafka'], } -> file_line { 'kafka-logdir': ensure => 'present', path => '/var/lib/kafka/config/server.properties', line => 'log.dirs=/var/log/kafka', - match => 'log.dirs=', + match => 'log.dirs=/tmp/kafka-logs', notify => Service['kafka'], - require => Exec['untar-kafka'], + } + -> file_line { 'zookeeper-datadir': + ensure => 'present', + path => '/var/lib/kafka/config/zookeeper.properties', + line => 'dataDir=/var/lib/zookeeper-data', + match => 'dataDir=/tmp/zookeeper', + notify => Service['zookeeper'], } -> file_line { 'zookeeper-listen': ensure => 'present', path => '/var/lib/kafka/config/zookeeper.properties', line => 'clientPortAddress=127.0.0.1', - match => 'clientPortAddress=', notify => Service['zookeeper'], - require => Exec['untar-kafka'], - } - exec { 'untar-kafka': - command => '/bin/tar -xvf /tmp/kafka.tgz -C /var/lib/kafka --strip 1', - refreshonly => true, - user => 'kafka', } - -> service { 'kafka': + service { 'kafka': ensure => running, + enable => true, } - -> service { 'zookeeper': + service { 'zookeeper': ensure => running, + enable => true, } ##Clickhouse installation @@ -182,7 +121,7 @@ notify => Exec['clickhouse-source-key'], } exec { 'clickhouse-source-key': - command => '/usr/bin/wget -fsSL https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key -O /usr/share/keyrings/clickhouse-keyring.gpg', + command => '/usr/bin/curl -fsSL https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key | gpg --dearmor > /usr/share/keyrings/clickhouse-keyring.gpg', logoutput => 'on_failure', try_sleep => 1, refreshonly => true, @@ -205,5 +144,116 @@ } -> service { 'clickhouse-server': ensure => running, + enable => true, + } + + #Create user/group for Akvorodo + ensure_packages([ + 'redis', + ]) + group { 'akvorado': + ensure => 'present', + } + -> user { 'akvorado': + ensure => 'present', + system => true, + home => '/var/lib/akvorado', + managehome => true, + } + #Create directories for akvorado + -> file { '/etc/akvorado': + ensure => 'directory', + owner => 'root', + group => 'akvorado', + mode => '0750', + } + #Copy akvorado to the server + -> file { '/usr/local/bin/akvorado': + ensure => file, + owner => 'root', + group => 'akvorado', + mode => '0550', + links => follow, + source => 'puppet:///data/akvorado-latest', + notify => [Service['akvorado-orch'],Exec['protobuf-schema']] + } + file { '/etc/akvorado/akvorado.yaml': + ensure => file, + content => template('akvorado/akvorado.yaml.erb'), + notify => Service['akvorado-orch'], + } + #Systemctl config + file { '/etc/systemd/system/akvorado-orch.service': + ensure => present, + source => 'puppet:///modules/akvorado/akvorado-orch.service', + mode => '0644', + owner => 'root', + group => 'root', + notify => [Exec['systemctl-daemon-reload'],Service['akvorado-orch']], + } + file { '/etc/systemd/system/akvorado-inlet.service': + ensure => present, + source => 'puppet:///modules/akvorado/akvorado-inlet.service', + mode => '0644', + owner => 'root', + group => 'root', + notify => [Exec['systemctl-daemon-reload'],Service['akvorado-inlet']], + } + file { '/etc/systemd/system/akvorado-console.service': + ensure => present, + source => 'puppet:///modules/akvorado/akvorado-console.service', + mode => '0644', + owner => 'root', + group => 'root', + notify => [Exec['systemctl-daemon-reload'],Service['akvorado-console']], + } + file { '/usr/share/GeoIP': + ensure => 'directory', + owner => 'root', + group => 'root', + mode => '0755', + } + file { '/usr/share/GeoIP/asn.mmdb': + ensure => present, + source => 'puppet:///data/asn.mmdb', + mode => '0644', + owner => 'root', + group => 'root', + } + file { '/usr/share/GeoIP/country.mmdb': + ensure => present, + source => 'puppet:///data/country.mmdb', + mode => '0644', + owner => 'root', + group => 'root', + } + apache::proxy { 'akvorado': + url => '/', + backend => 'http://localhost:8082/', + } + -> service { 'akvorado-orch': + ensure => running, + enable => true, + } + -> service { 'akvorado-inlet': + ensure => running, + enable => true, + } + -> service { 'akvorado-console': + ensure => running, + enable => true, + } + -> service { 'redis': + ensure => running, + enable => true, + } + exec { 'systemctl-daemon-reload': + command => '/bin/systemctl daemon-reload', + refreshonly => true, + } + exec { 'protobuf-schema': + command => '/usr/bin/curl http://127.0.0.1:8080/api/v0/orchestrator/clickhouse/init.sh | sh', + refreshonly => true, + require => Service['akvorado-orch'] } } diff --git a/modules/akvorado/templates/akvorado.yaml.erb b/modules/akvorado/templates/akvorado.yaml.erb index dbe728f7..21030595 100644 --- a/modules/akvorado/templates/akvorado.yaml.erb +++ b/modules/akvorado/templates/akvorado.yaml.erb @@ -1,4 +1,6 @@ --- +# AUTOGENERATED BY PUPPET +# All manual changes will be overwritten reporting: logging: {} metrics: {} @@ -148,6 +150,32 @@ inlet: cachecheckinterval: 2m0s cachepersistfile: "" providers: +<% unless @snmpv2_providers.empty? -%> + - agents: {} + communities: +<% @snmpv2_providers.each do |provider| -%> + <%=provider['ipv4']%>: <%=provider['community']%> +<% end -%> + ports: + ::/0: 161 + securityparameters: {} + type: snmp +<% end -%> +<% unless @snmpv3_providers.empty? -%> + - agents: {} + ports: + ::/0: 161 + securityparameters: +<% @snmpv3_providers.each do |provider| -%> + <%=provider['ipv4']%>: + user-name: <%=provider['user']%> + authentication-protocol: <%=provider['authentication-protocol']%> + authentication-passphrase: <%=provider['authentication-passphrase']%> + privacy-protocol: <%=provider['privacy-protocol']%> + privacy-passphrase: <%=provider['privacy-passphrase']%> +<% end -%> + type: snmp +<% end -%> - agents: {} communities: ::/0: @@ -197,7 +225,6 @@ inlet: exporterclassifiers: - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") - ClassifyRegion("europe") - - ClassifyTenant("acme") - ClassifyRole("edge") interfaceclassifiers: - | @@ -206,7 +233,7 @@ inlet: ClassifyExternal() - ClassifyInternal() classifiercacheduration: 5m0s - defaultsamplingrate: {} + defaultsamplingrate: 1 overridesamplingrate: {} asnproviders: - flow @@ -239,7 +266,7 @@ console: graphtype: stacked start: 6 hours ago end: now - filter: InIfBoundary = external + filter: "" dimensions: - SrcAS limit: 10 @@ -249,9 +276,9 @@ console: - protocol - src-country - etype - homepagegraphfilter: InIfBoundary = 'external' + homepagegraphfilter: "" dimensionslimit: 50 - cachettl: 3h0m0s + cachettl: 0h10m0s clickhouse: servers: - 127.0.0.1:9000 @@ -269,18 +296,18 @@ console: keyfile: "" auth: headers: - login: Remote-User + login: X-Proxy-REMOTE-USER name: Remote-Name email: Remote-Email logouturl: X-Logout-URL defaultuser: - login: default - name: Default User + login: "" + name: "" email: "" logouturl: "" database: driver: sqlite - dsn: file::memory:?cache=shared + dsn: /var/lib/akvorado/console.sqlite savedfilters: - description: From Netflix content: InIfBoundary = external AND SrcAS = AS2906 From 7c2d41ccd60af90f43bb52794931e6b2bdde691e Mon Sep 17 00:00:00 2001 From: furest Date: Sat, 22 Jun 2024 11:14:56 +0200 Subject: [PATCH 035/107] Fix manifest indent to make circleci happy again --- modules/akvorado/manifests/init.pp | 88 +++++++++++++++--------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index e3d6def5..09e11212 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -27,10 +27,10 @@ managehome => true, } -> file { '/var/lib/kafka/kafka.tgz': - ensure => file, - links => follow, - source => 'puppet:///data/kafka-latest.tgz', - notify => Exec['untar-kafka'] + ensure => file, + links => follow, + source => 'puppet:///data/kafka-latest.tgz', + notify => Exec['untar-kafka'] } -> file { '/var/log/kafka': ensure => 'directory', @@ -66,45 +66,45 @@ notify => [ Exec['systemctl-daemon-reload'], Service['zookeeper'] ], } -> file_line { 'kafka-enabledeletetopics': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'delete.topic.enable = true', - notify => Service['kafka'], + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'delete.topic.enable = true', + notify => Service['kafka'], } -> file_line { 'kafka-listenlocalhost': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'listeners=PLAINTEXT://localhost:9092', - match => '#listeners=PLAINTEXT', - notify => Service['kafka'], + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'listeners=PLAINTEXT://localhost:9092', + match => '#listeners=PLAINTEXT', + notify => Service['kafka'], } -> file_line { 'kafka-logdir': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'log.dirs=/var/log/kafka', - match => 'log.dirs=/tmp/kafka-logs', - notify => Service['kafka'], + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'log.dirs=/var/log/kafka', + match => 'log.dirs=/tmp/kafka-logs', + notify => Service['kafka'], } -> file_line { 'zookeeper-datadir': - ensure => 'present', - path => '/var/lib/kafka/config/zookeeper.properties', - line => 'dataDir=/var/lib/zookeeper-data', - match => 'dataDir=/tmp/zookeeper', - notify => Service['zookeeper'], + ensure => 'present', + path => '/var/lib/kafka/config/zookeeper.properties', + line => 'dataDir=/var/lib/zookeeper-data', + match => 'dataDir=/tmp/zookeeper', + notify => Service['zookeeper'], } -> file_line { 'zookeeper-listen': - ensure => 'present', - path => '/var/lib/kafka/config/zookeeper.properties', - line => 'clientPortAddress=127.0.0.1', - notify => Service['zookeeper'], + ensure => 'present', + path => '/var/lib/kafka/config/zookeeper.properties', + line => 'clientPortAddress=127.0.0.1', + notify => Service['zookeeper'], } service { 'kafka': - ensure => running, - enable => true, + ensure => running, + enable => true, } service { 'zookeeper': - ensure => running, - enable => true, + ensure => running, + enable => true, } ##Clickhouse installation @@ -140,11 +140,11 @@ require => [File['clickhouse-source-add'], Exec['clickhouse-source-key'], Exec['apt-update']], } -> package { 'clickhouse-client': - ensure => installed, + ensure => installed, } -> service { 'clickhouse-server': - ensure => running, - enable => true, + ensure => running, + enable => true, } #Create user/group for Akvorodo @@ -155,8 +155,8 @@ ensure => 'present', } -> user { 'akvorado': - ensure => 'present', - system => true, + ensure => 'present', + system => true, home => '/var/lib/akvorado', managehome => true, } @@ -232,20 +232,20 @@ backend => 'http://localhost:8082/', } -> service { 'akvorado-orch': - ensure => running, - enable => true, + ensure => running, + enable => true, } -> service { 'akvorado-inlet': - ensure => running, - enable => true, + ensure => running, + enable => true, } -> service { 'akvorado-console': - ensure => running, - enable => true, + ensure => running, + enable => true, } -> service { 'redis': - ensure => running, - enable => true, + ensure => running, + enable => true, } exec { 'systemctl-daemon-reload': command => '/bin/systemctl daemon-reload', From 7ab1f8dbfc200a7b6e7828e551fc0336c45fd8e7 Mon Sep 17 00:00:00 2001 From: furest Date: Sat, 22 Jun 2024 11:19:06 +0200 Subject: [PATCH 036/107] Update init.pp --- modules/akvorado/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 09e11212..2c095d2a 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -46,7 +46,7 @@ } exec { 'untar-kafka': command => '/bin/tar -xvf /var/lib/kafka/kafka.tgz -C /var/lib/kafka --strip 1', - refreshonly => 'true', + refreshonly => true, user => 'kafka', } file { '/etc/systemd/system/kafka.service': From c70c64bac64dc7978d731f240f72922066190ed9 Mon Sep 17 00:00:00 2001 From: furest Date: Tue, 25 Jun 2024 22:32:43 +0200 Subject: [PATCH 037/107] Fix proxy + improvements --- modules/akvorado.py | 2 +- modules/akvorado/manifests/init.pp | 39 ++++++++++++++++---- modules/akvorado/templates/akvorado.yaml.erb | 6 +-- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/modules/akvorado.py b/modules/akvorado.py index 50f02356..bc72dd1d 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -27,7 +27,7 @@ def get_sflow_clients(): "INNER JOIN option o1 ON h.node_id = o1.node_id " "INNER JOIN option o2 ON h.node_id = o2.node_id " "WHERE o1.name='pkg' AND o1.value='sflowclient' " - "AND o2.name='layer'" + "AND o2.name='layer';" ) res = db.fetchall() if not res: diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 2c095d2a..e3ad37fc 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -150,13 +150,16 @@ #Create user/group for Akvorodo ensure_packages([ 'redis', - ]) + ],{ + ensure => 'present', + notify => Service['redis'], + }) group { 'akvorado': ensure => 'present', } -> user { 'akvorado': - ensure => 'present', - system => true, + ensure => 'present', + system => true, home => '/var/lib/akvorado', managehome => true, } @@ -227,23 +230,43 @@ owner => 'root', group => 'root', } - apache::proxy { 'akvorado': + apache::proxy { '1_akvorado-orch-api': + url => '/api/v0/orchestrator/', + backend => 'http://localhost:8080/api/v0/orchestrator/', + } + apache::proxy { '2_akvorado-inlet-api': + url => '/api/v0/inlet/', + backend => 'http://localhost:8081/api/v0/inlet/', + } + apache::proxy { '3_akvorado-console': url => '/', backend => 'http://localhost:8082/', } - -> service { 'akvorado-orch': + # By default apache answers with status code 404 when an URL contains an encoded slash (%2F) + # The following allows apache to simply forward the request to the prox backend. + file { '/etc/apache2/conf-available/allow-slashes.conf': + content => 'AllowEncodedSlashes On', + ensure => present, + mode => '0644', + } + -> file { '/etc/apache2/conf-enabled/allow-slashes.conf': + ensure => link, + mode => '0644', + target => '/etc/apache2/conf-available/allow-slashes.conf', + } + service { 'akvorado-orch': ensure => running, enable => true, } - -> service { 'akvorado-inlet': + service { 'akvorado-inlet': ensure => running, enable => true, } - -> service { 'akvorado-console': + service { 'akvorado-console': ensure => running, enable => true, } - -> service { 'redis': + service { 'redis': ensure => running, enable => true, } diff --git a/modules/akvorado/templates/akvorado.yaml.erb b/modules/akvorado/templates/akvorado.yaml.erb index 21030595..3c29da73 100644 --- a/modules/akvorado/templates/akvorado.yaml.erb +++ b/modules/akvorado/templates/akvorado.yaml.erb @@ -297,9 +297,9 @@ console: auth: headers: login: X-Proxy-REMOTE-USER - name: Remote-Name - email: Remote-Email - logouturl: X-Logout-URL + name: "" + email: "" + logouturl: "" defaultuser: login: "" name: "" From 70d1f608debfc69f5b043b7479cef546375d8255 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sat, 12 Oct 2024 23:01:02 +0200 Subject: [PATCH 038/107] change to not scripts --- modules/dnsstatd/manifests/init.pp | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/modules/dnsstatd/manifests/init.pp b/modules/dnsstatd/manifests/init.pp index 18daee91..b7742c42 100644 --- a/modules/dnsstatd/manifests/init.pp +++ b/modules/dnsstatd/manifests/init.pp @@ -15,7 +15,7 @@ class dnsstatd($current_event) { - $secret_db_dnsstatd = vault('postgresql:dnsstatd', {}) + $secret_db_dnsstatd = vault('postgresql:dnsstatd', {}) ensure_packages([ 'python3-netifaces', @@ -36,8 +36,23 @@ provider => 'pip', } + file { '/opt/dnsstatd': + ensure => directory, + mode => '0750', + owner => 'root', + group => 'root', + } + + file { '/opt/dnsstatd/dnsstatd.py': + ensure => present, + source => 'puppet:///repos/dnsstatd/dnsstatd.py', + mode => '0750', + owner => 'root', + group => 'root', + } + if $secret_db_dnsstatd != {} { - file { '/scripts/dnsstatd/config': + file { '/opt/dnsstatd/config': ensure => file, content => template('dnsstatd/config.erb'), mode => '0600', @@ -46,7 +61,6 @@ } supervisor::register { 'dnsstatd': - command => '/scripts/dnsstatd/dnsstatd.py', + command => '/opt/dnsstatd/dnsstatd.py', } - } From 20e6aaf22301f09c5834560b9a9bbe585695a906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 20 Oct 2024 00:52:10 +0200 Subject: [PATCH 039/107] Create puppet-lint.yaml (#397) * Create puppet-lint.yaml * Update puppet-lint.yaml * Update puppet-lint.yaml * Update puppet-lint.yaml * Update puppet-lint.yaml * Delete .circleci directory --- .circleci/config.yml | 20 -------------------- .github/workflows/puppet-lint.yaml | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 20 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/workflows/puppet-lint.yaml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index a2e0796b..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: 2 -jobs: - build: - docker: - - image: quay.io/dhtech/puppet-ci:latest - - working_directory: ~/repo - - steps: - - checkout - - - run: - name: puppet lint - command: | - /root/.rbenv/shims/puppet-lint --no-puppet_url_without_modules-check --fail-on-warnings . - - - run: - name: erb check - command: | - /root/.rbenv/shims/rails-erb-lint check diff --git a/.github/workflows/puppet-lint.yaml b/.github/workflows/puppet-lint.yaml new file mode 100644 index 00000000..3e44f2f2 --- /dev/null +++ b/.github/workflows/puppet-lint.yaml @@ -0,0 +1,24 @@ +name: puppet linting +on: + pull_request: + branches: + - master + - main + - production + paths-ignore: + - "**.md" + +jobs: + puppet-lint: + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: puppet-lint + uses: scottbrenner/puppet-lint-action@v1.0.4 + with: + args: ./ + From 8a3b57cbfa484dba0230756a4e05cad4803f79a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Mon, 21 Oct 2024 21:40:08 +0200 Subject: [PATCH 040/107] Create .puppet-lint.rc (#398) * Create .puppet-lint.rc * Update init.pp --- .puppet-lint.rc | 7 +++++++ modules/hardware/manifests/init.pp | 8 ++++---- modules/system/manifests/init.pp | 2 +- modules/vault/manifests/init.pp | 8 ++++---- 4 files changed, 16 insertions(+), 9 deletions(-) create mode 100644 .puppet-lint.rc diff --git a/.puppet-lint.rc b/.puppet-lint.rc new file mode 100644 index 00000000..ae6aee38 --- /dev/null +++ b/.puppet-lint.rc @@ -0,0 +1,7 @@ +--fail-on-warnings +--relative +--no-class_inherits_from_params_class-check +--no-documentation-check +--no-single_quote_string_with_variables-check +--no-puppet_url_without_modules-check +--no-legacy_facts diff --git a/modules/hardware/manifests/init.pp b/modules/hardware/manifests/init.pp index 77651df3..a0014e36 100644 --- a/modules/hardware/manifests/init.pp +++ b/modules/hardware/manifests/init.pp @@ -18,13 +18,13 @@ if defined('$::productname') and $::productname =~ /VMware/ { # OpenBSD does not use open-vm-tools, see the vmt(4) driver. - if $::operatingsystem != 'OpenBSD' { + if $facts['operatingsystem'] != 'OpenBSD' { package { 'open-vm-tools': ensure => installed } } } else { - if $::manufacturer == 'HP' { + if $facts['manufacturer'] == 'HP' { package { 'gnupg': ensure => installed } @@ -63,8 +63,8 @@ ensure => installed } - } elsif $::productname == 'Wedge-DC-F 20-001331' { - if $::kernelrelease =~ /OpenNetworkLinux/ { + } elsif $facts['productname'] == 'Wedge-DC-F 20-001331' { + if $facts['kernelrelease'] =~ /OpenNetworkLinux/ { service { 'onlpd': ensure => 'stopped', enable => false, diff --git a/modules/system/manifests/init.pp b/modules/system/manifests/init.pp index b616eeb6..8fb8268c 100644 --- a/modules/system/manifests/init.pp +++ b/modules/system/manifests/init.pp @@ -100,7 +100,7 @@ creates => '/scripts/.git/modules', } - if $::kernelrelease =~ /OpenNetworkLinux/ { + if $facts['kernelrelease'] =~ /OpenNetworkLinux/ { package { 'snmpd': ensure => 'purged', } diff --git a/modules/vault/manifests/init.pp b/modules/vault/manifests/init.pp index 8bd74ab9..e21c11e6 100644 --- a/modules/vault/manifests/init.pp +++ b/modules/vault/manifests/init.pp @@ -20,7 +20,7 @@ provider => 'pip', } - if $::kernel == 'Linux' { + if $facts['kernel'] == 'Linux' { file { 'vault': ensure => file, path => '/usr/local/bin/vault', @@ -35,7 +35,7 @@ } } - if $::operatingsystem == 'Debian' and $::operatingsystemmajrelease == '11' { + if $facts['operatingsystem'] == 'Debian' and $facts['operatingsystemmajrelease'] == '11' { file { 'vault-input': ensure => file, path => '/usr/local/bin/vault-input', @@ -65,7 +65,7 @@ path => '/usr/local/bin/vault-auth', } - if $::operatingsystem == 'Debian' and $::operatingsystemmajrelease == '11' { + if $facts['operatingsystem'] == 'Debian' and $facts['operatingsystemmajrelease'] == '11' { file { 'vault-machine': ensure => file, path => '/usr/local/bin/vault-machine', @@ -82,7 +82,7 @@ } } - if $::operatingsystem == 'Debian' and $::operatingsystemmajrelease == '11' { + if $facts['operatingsystem'] == 'Debian' and $facts['operatingsystemmajrelease'] == '11' { file { 'dh-create-service-account': ensure => file, path => '/usr/local/bin/dh-create-service-account', From 7331b59e8c883ea1f370d0f1db2848f7507e4e98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Mon, 21 Oct 2024 23:24:56 +0200 Subject: [PATCH 041/107] Update .puppet-lint.rc (#399) --- .puppet-lint.rc | 1 - 1 file changed, 1 deletion(-) diff --git a/.puppet-lint.rc b/.puppet-lint.rc index ae6aee38..60a11cfc 100644 --- a/.puppet-lint.rc +++ b/.puppet-lint.rc @@ -2,6 +2,5 @@ --relative --no-class_inherits_from_params_class-check --no-documentation-check ---no-single_quote_string_with_variables-check --no-puppet_url_without_modules-check --no-legacy_facts From 9d0e3ddae82ea5c1a22dfe66a143e6f71cb50fbb Mon Sep 17 00:00:00 2001 From: Tisteagle Date: Tue, 29 Oct 2024 22:52:20 +0100 Subject: [PATCH 042/107] Update prometheus-exporter-distconfcheck.erb (#400) --- .../templates/prometheus-exporter-distconfcheck.erb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/rancid/templates/prometheus-exporter-distconfcheck.erb b/modules/rancid/templates/prometheus-exporter-distconfcheck.erb index b1e2a238..155f8cae 100755 --- a/modules/rancid/templates/prometheus-exporter-distconfcheck.erb +++ b/modules/rancid/templates/prometheus-exporter-distconfcheck.erb @@ -24,10 +24,10 @@ def open_db(): conn = sqlite3.connect(db_file) db = conn.cursor() except sqlite3.Error as e: - print >>sys-stderr, "An error accurd:", e.args[0] + print("An error accurd:", e.args[0], file=sys-stderr) sys.exit(1) else: - print >>sys-stderr, "No database file found: %s" % db_file + print("No database file found: %s" % db_file, file=sys-stderr) sys.exit(2) return db @@ -87,7 +87,7 @@ access_switches = get_access_switches(db) try: output_file = open('%s/%s' % (export_dir, export_file), 'w') except: - print >>sys.stderr, "Unable to open export file for writing" + print("Unable to open export file for writing", file=sys.stderr) sys.exit(1) @@ -107,8 +107,8 @@ for filename in rancid_conf_files: table_interfaces = cfg.find_objects_w_child(parentspec='^interface GigabitEthernet', childspec='^ description BORD;') for intf in table_interfaces: if get_circuit_id_access_switch(intf) != get_description_access_switch(intf): - print get_circuit_id_access_switch(intf) - print get_description_access_switch(intf) + print(get_circuit_id_access_switch(intf)) + print(get_description_access_switch(intf)) checks = { "IpDhcpSnooping": bool(cfg.find_objects('^ip dhcp snooping$') and cfg.find_objects('ip dhcp snooping vlan 601')), @@ -130,5 +130,5 @@ for filename in rancid_conf_files: "SpanningTreeInstance": cfg.find_objects('^spanning-tree mst configuration$')[0].has_child_with('^ instance 1 vlan 2-4094$'), } - for k, v in checks.items(): + for k, v in list(checks.items()): output_file.write('dist_check{{device="{}",check="{}"}} {} {}\n'.format(filename, k, int(v), int(time.time()*1000))) From 97c4f695fe7e1a4357d2d7a869b22d60f9e02374 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Tue, 29 Oct 2024 22:59:11 +0100 Subject: [PATCH 043/107] add logging instead of pcap (#396) * add logging instead of pcap * Update init.pp * Update puppet-lint.yaml * Update puppet-lint.yaml --- .github/workflows/puppet-lint.yaml | 2 +- modules/bind/manifests/init.pp | 12 ++++++++++++ modules/bind/templates/named.conf.erb | 20 +++++++++++++++++--- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/.github/workflows/puppet-lint.yaml b/.github/workflows/puppet-lint.yaml index 3e44f2f2..2c479037 100644 --- a/.github/workflows/puppet-lint.yaml +++ b/.github/workflows/puppet-lint.yaml @@ -10,13 +10,13 @@ on: jobs: puppet-lint: - runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 + - name: puppet-lint uses: scottbrenner/puppet-lint-action@v1.0.4 with: diff --git a/modules/bind/manifests/init.pp b/modules/bind/manifests/init.pp index 2e459907..9f9208b4 100644 --- a/modules/bind/manifests/init.pp +++ b/modules/bind/manifests/init.pp @@ -34,6 +34,7 @@ if $::operatingsystem == 'OpenBSD' { $named_user = '_bind' $conf_dir = '/var/named/etc' + $log_dir = '/var/log/named' $conf_cfg = 'etc' $package_name = 'isc-bind' $rc_name = 'isc_named' @@ -53,6 +54,7 @@ else { $named_user = 'bind' $conf_dir = '/etc/bind' + $log_dir = '/var/log/bind' $conf_cfg = '/etc/bind' $package_name = 'bind9' $rc_name = 'bind9' @@ -124,6 +126,16 @@ require => Package[$package_name], } +# Make sure the log directory exists + file { 'logdir': + ensure => 'directory', + owner => $named_user, + group => $named_user, + mode => '0770', + path => $log_dir, + require => Package[$package_name], + } + # Make sure the stats directory exists file { 'statsdir': ensure => 'directory', diff --git a/modules/bind/templates/named.conf.erb b/modules/bind/templates/named.conf.erb index ca7c44b0..63d48a6d 100644 --- a/modules/bind/templates/named.conf.erb +++ b/modules/bind/templates/named.conf.erb @@ -88,9 +88,15 @@ options { <% if @role == 'resolver' -%> logging { - category lame-servers {null;}; - category client {null;}; + // Set up a channel for logging DNS queries to a file + channel query_log { + file "/var/log/bind/query.log" versions 3 size 10m; + severity info; + print-time yes; + }; + + // Configure syslog for general logging channel syslog { syslog daemon; print-time yes; @@ -98,7 +104,15 @@ logging { print-severity yes; severity warning; }; -category default { syslog; }; + + + // Disable logging for specific categories to reduce log clutter + category lame-servers { null; }; + category client { null; }; + // Log DNS queries to the query_log channel + category queries { query_log; }; + // Use syslog for default logging category + category default { syslog; }; }; # root hints provided by dns-root-data package From aca3bb3a1ab1964c5af254c6fe8aa9a06cd9faeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Malmstr=C3=B6m?= Date: Mon, 18 Nov 2024 11:27:25 +0100 Subject: [PATCH 044/107] Wire guard (#401) * New puppet module for wireguard * Update wg0.conf.erb * Update wg0.conf.erb * lint fix * lint fix * Update init.pp --- modules/wireguard.py | 10 +++++ modules/wireguard/manifests/init.pp | 48 ++++++++++++++++++++++++ modules/wireguard/templates/wg0.conf.erb | 13 +++++++ 3 files changed, 71 insertions(+) create mode 100644 modules/wireguard.py create mode 100644 modules/wireguard/manifests/init.pp create mode 100644 modules/wireguard/templates/wg0.conf.erb diff --git a/modules/wireguard.py b/modules/wireguard.py new file mode 100644 index 00000000..8511483d --- /dev/null +++ b/modules/wireguard.py @@ -0,0 +1,10 @@ +# Copyright 2024 dhtech +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file + + +def generate(host, *args): + return {'wireguard': None} + +# vim: ts=4: sts=4: sw=4: expandtab diff --git a/modules/wireguard/manifests/init.pp b/modules/wireguard/manifests/init.pp new file mode 100644 index 00000000..1c411f03 --- /dev/null +++ b/modules/wireguard/manifests/init.pp @@ -0,0 +1,48 @@ +class wireguard { + # Execute 'apt-get update' + exec { 'apt-update': # exec resource named 'apt-update' + command => '/usr/bin/apt-get update' # command this resource will run + } + + # Install wireguard package + package { 'wireguard': + ensure => installed, + require => Exec['apt-update'], # require 'apt-update' before installing + } + + # Create wireguard interface + exec { 'create': + require => Package['wireguard'], + command => '/usr/bin/ip link add dev wg0 type wireguard', + unless => '/usr/bin/ip link show wg0' + } + + +# Set wireguard interface IP + exec { 'set wg interface IP': + require => Package['wireguard'], + command => '/usr/bin/ip address add dev wg0 77.80.200.129/25', + unless => '/usr/bin/ip addr show wg0 | grep 77.80.200.129/25' + } + +# Specify all clients usable IPs 77.80.200.130 - 77.80.200.254 + $clients = [ + { nick => 'felix', ip => '77.80.200.130', key => '5Dk2crqm8A51OQ1blVK701YMZj33U+GONpmLrr0LWkM=' }, + { nick => 'washington', ip => '77.80.200.131', key => 'Z8aCXv4ydIhUEtvH+NJv39mAMGiS8uF8oNgCoIByAFI=' }, + ] + + +# Build the wg0 config file will all clients from previous step + file { 'setConf': + ensure => file, + path => '/etc/wireguard/wg0.conf', + notify => Exec[syncConf], + content => template('wireguard/templates/wg0.conf.erb'), + } + +# Sync changes towards the wg0 interface + exec { 'syncConf': + require => Package['wireguard'], + command => '/usr/bin/wg syncconf wg0 /etc/wireguard/wg0.conf', + } +} \ No newline at end of file diff --git a/modules/wireguard/templates/wg0.conf.erb b/modules/wireguard/templates/wg0.conf.erb new file mode 100644 index 00000000..993d734b --- /dev/null +++ b/modules/wireguard/templates/wg0.conf.erb @@ -0,0 +1,13 @@ +[Interface] +#Just placeholder privkey will be pulled from vault +PrivateKey = UJywHJtV58X11nF6zmouBCmfKfKbH4iggugRA/Th/k8= +ListenPort = 51820 + + +<% @clients.each do |client| -%> +#<%= client['nick'] %> +[Peer] +PublicKey = <%= client['key'] %> +AllowedIPs = <%= client['ip'] %> + +<% end -%> From 7dddfc025e2d7cf94d86b5f5284a387ec34bc55f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Malmstr=C3=B6m?= Date: Mon, 18 Nov 2024 21:39:52 +0100 Subject: [PATCH 045/107] WireGuard (#402) * New puppet module for wireguard * Update wg0.conf.erb * Update wg0.conf.erb * lint fix * lint fix * Update init.pp * Moved client key to SVN Client keys is now read from SVN Server priv/pub key is now generated on sever when server is deployed. * Update init.pp * Update init.pp * Update init.pp * Update init.pp * Changed IP --- modules/wireguard/manifests/init.pp | 32 ++++++++++++++++++------ modules/wireguard/templates/wg0.conf.erb | 15 +++++------ 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/modules/wireguard/manifests/init.pp b/modules/wireguard/manifests/init.pp index 1c411f03..3ece90bb 100644 --- a/modules/wireguard/manifests/init.pp +++ b/modules/wireguard/manifests/init.pp @@ -17,19 +17,35 @@ unless => '/usr/bin/ip link show wg0' } + exec { 'create-privkey': + command => '/usr/bin/wg pubkey < /etc/wireguard/privkey > /etc/wireguard/pubkey', + unless => '/usr/bin/ls /etc/wireguard/privkey' + } + + exec { 'create-pubkey': + command => '/usr/bin/wg genkey > /etc/wireguard/privkey', + unless => '/usr/bin/ls /etc/wireguard/privkey' + } + + + exec { 'add-key': + command => '/usr/bin/wg set wg0 listen-port 51820 private-key /etc/wireguard/privkey', + require => Exec['create-key'], # require 'apt-update' before installing + } + # Set wireguard interface IP exec { 'set wg interface IP': require => Package['wireguard'], - command => '/usr/bin/ip address add dev wg0 77.80.200.129/25', - unless => '/usr/bin/ip addr show wg0 | grep 77.80.200.129/25' + command => '/usr/bin/ip address add dev wg0 77.80.229.133/25', + unless => '/usr/bin/ip addr show wg0 | grep 77.80.229.133/25' } -# Specify all clients usable IPs 77.80.200.130 - 77.80.200.254 - $clients = [ - { nick => 'felix', ip => '77.80.200.130', key => '5Dk2crqm8A51OQ1blVK701YMZj33U+GONpmLrr0LWkM=' }, - { nick => 'washington', ip => '77.80.200.131', key => 'Z8aCXv4ydIhUEtvH+NJv39mAMGiS8uF8oNgCoIByAFI=' }, - ] + file { '/tmp/wireguard/wireguard-clients.yaml': + ensure => directory, + recurse => remote, + source => 'puppet:///svn/$::{current_event}/services/wireguard-clients.yaml', +} # Build the wg0 config file will all clients from previous step @@ -45,4 +61,4 @@ require => Package['wireguard'], command => '/usr/bin/wg syncconf wg0 /etc/wireguard/wg0.conf', } -} \ No newline at end of file +} diff --git a/modules/wireguard/templates/wg0.conf.erb b/modules/wireguard/templates/wg0.conf.erb index 993d734b..26f18296 100644 --- a/modules/wireguard/templates/wg0.conf.erb +++ b/modules/wireguard/templates/wg0.conf.erb @@ -1,13 +1,10 @@ -[Interface] -#Just placeholder privkey will be pulled from vault -PrivateKey = UJywHJtV58X11nF6zmouBCmfKfKbH4iggugRA/Th/k8= -ListenPort = 51820 +<% require 'yaml' %> +<% clients = YAML.load_file('/tmp/wireguard/wireguard-clients.yaml')['clients'] %> - -<% @clients.each do |client| -%> -#<%= client['nick'] %> +<% clients.each do |nick, client| -%> +# <%= nick %> [Peer] -PublicKey = <%= client['key'] %> +PublicKey = <%= client['publickey'] %> AllowedIPs = <%= client['ip'] %> -<% end -%> +<% end -%> \ No newline at end of file From e4012a867ee150794fe1cd292505215d4b197dd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Malmstr=C3=B6m?= Date: Mon, 18 Nov 2024 22:24:28 +0100 Subject: [PATCH 046/107] Update init.pp (#403) --- modules/wireguard/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/wireguard/manifests/init.pp b/modules/wireguard/manifests/init.pp index 3ece90bb..69826136 100644 --- a/modules/wireguard/manifests/init.pp +++ b/modules/wireguard/manifests/init.pp @@ -53,7 +53,7 @@ ensure => file, path => '/etc/wireguard/wg0.conf', notify => Exec[syncConf], - content => template('wireguard/templates/wg0.conf.erb'), + content => template('wireguard/wg0.conf.erb'), } # Sync changes towards the wg0 interface From 3443587d2be2129e0e54744a2ade98d3b9c7da9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Malmstr=C3=B6m?= Date: Mon, 18 Nov 2024 22:58:15 +0100 Subject: [PATCH 047/107] Wire guard (#404) * Update init.pp * Update init.pp fixed a path --- modules/wireguard/manifests/init.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/wireguard/manifests/init.pp b/modules/wireguard/manifests/init.pp index 69826136..103ca5f7 100644 --- a/modules/wireguard/manifests/init.pp +++ b/modules/wireguard/manifests/init.pp @@ -41,8 +41,8 @@ unless => '/usr/bin/ip addr show wg0 | grep 77.80.229.133/25' } - file { '/tmp/wireguard/wireguard-clients.yaml': - ensure => directory, + file { '/etc/wireguard/wireguard-clients.yaml': + ensure => file, recurse => remote, source => 'puppet:///svn/$::{current_event}/services/wireguard-clients.yaml', } From c6022f7e36991db3d732821d4380b5d78d247315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Malmstr=C3=B6m?= Date: Mon, 18 Nov 2024 23:20:27 +0100 Subject: [PATCH 048/107] Update wg0.conf.erb (#405) Fixed a path --- modules/wireguard/templates/wg0.conf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/wireguard/templates/wg0.conf.erb b/modules/wireguard/templates/wg0.conf.erb index 26f18296..374153c9 100644 --- a/modules/wireguard/templates/wg0.conf.erb +++ b/modules/wireguard/templates/wg0.conf.erb @@ -1,5 +1,5 @@ <% require 'yaml' %> -<% clients = YAML.load_file('/tmp/wireguard/wireguard-clients.yaml')['clients'] %> +<% clients = YAML.load_file('/etc/wireguard/wireguard-clients.yaml')['clients'] %> <% clients.each do |nick, client| -%> # <%= nick %> From c1abd88c58fd618dafbd2f507b2906cdbf852099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Malmstr=C3=B6m?= Date: Mon, 18 Nov 2024 23:40:41 +0100 Subject: [PATCH 049/107] Path fix (#406) * Path fix * Update init.pp --- modules/wireguard/manifests/init.pp | 6 +++--- modules/wireguard/templates/wg0.conf.erb | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/wireguard/manifests/init.pp b/modules/wireguard/manifests/init.pp index 103ca5f7..c5a50f76 100644 --- a/modules/wireguard/manifests/init.pp +++ b/modules/wireguard/manifests/init.pp @@ -41,10 +41,10 @@ unless => '/usr/bin/ip addr show wg0 | grep 77.80.229.133/25' } - file { '/etc/wireguard/wireguard-clients.yaml': - ensure => file, + file { '/etc/wireguard/yaml': + ensure => directory, recurse => remote, - source => 'puppet:///svn/$::{current_event}/services/wireguard-clients.yaml', + source => 'puppet:///svn/$::{current_event}/services/wireguard', } diff --git a/modules/wireguard/templates/wg0.conf.erb b/modules/wireguard/templates/wg0.conf.erb index 374153c9..c18d6fad 100644 --- a/modules/wireguard/templates/wg0.conf.erb +++ b/modules/wireguard/templates/wg0.conf.erb @@ -1,5 +1,5 @@ <% require 'yaml' %> -<% clients = YAML.load_file('/etc/wireguard/wireguard-clients.yaml')['clients'] %> +<% clients = YAML.load_file('/etc/wireguard/yaml/wireguard-clients.yaml')['clients'] %> <% clients.each do |nick, client| -%> # <%= nick %> From 49254be3f101df7726043544a9e5294f24facb40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Malmstr=C3=B6m?= Date: Mon, 18 Nov 2024 23:53:15 +0100 Subject: [PATCH 050/107] Update init.pp (#407) --- modules/wireguard/manifests/init.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/wireguard/manifests/init.pp b/modules/wireguard/manifests/init.pp index c5a50f76..755f3a08 100644 --- a/modules/wireguard/manifests/init.pp +++ b/modules/wireguard/manifests/init.pp @@ -54,6 +54,7 @@ path => '/etc/wireguard/wg0.conf', notify => Exec[syncConf], content => template('wireguard/wg0.conf.erb'), + require => file['/etc/wireguard/yaml'], # require that yaml file exists before trying to use it.... } # Sync changes towards the wg0 interface From 21fb1294485622d70ae3101b5343f8bcb33041c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Malmstr=C3=B6m?= Date: Tue, 19 Nov 2024 00:15:43 +0100 Subject: [PATCH 051/107] Wireguard (#408) * Update init.pp * puppet run order Fixed puppet run order Had missed to include current_event var * Update wireguard.py * lint fix * lint fix * lint --- modules/wireguard.py | 10 ++++++++-- modules/wireguard/manifests/init.pp | 13 ++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/modules/wireguard.py b/modules/wireguard.py index 8511483d..81a06a82 100644 --- a/modules/wireguard.py +++ b/modules/wireguard.py @@ -2,9 +2,15 @@ # # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file - +import lib def generate(host, *args): - return {'wireguard': None} + + # Get current event, used to get up-to-date switch conf + current_event = lib.get_current_event() + + info = {} + info['current_event'] = current_event + return {'wireguard': info} # vim: ts=4: sts=4: sw=4: expandtab diff --git a/modules/wireguard/manifests/init.pp b/modules/wireguard/manifests/init.pp index 755f3a08..7a7b6304 100644 --- a/modules/wireguard/manifests/init.pp +++ b/modules/wireguard/manifests/init.pp @@ -1,4 +1,4 @@ -class wireguard { +class wireguard($current_event) { # Execute 'apt-get update' exec { 'apt-update': # exec resource named 'apt-update' command => '/usr/bin/apt-get update' # command this resource will run @@ -20,29 +20,32 @@ exec { 'create-privkey': command => '/usr/bin/wg pubkey < /etc/wireguard/privkey > /etc/wireguard/pubkey', unless => '/usr/bin/ls /etc/wireguard/privkey' + require => Exec['create'], } exec { 'create-pubkey': command => '/usr/bin/wg genkey > /etc/wireguard/privkey', unless => '/usr/bin/ls /etc/wireguard/privkey' + require => Exec['create-privkey'], } exec { 'add-key': command => '/usr/bin/wg set wg0 listen-port 51820 private-key /etc/wireguard/privkey', - require => Exec['create-key'], # require 'apt-update' before installing + require => Exec['create-pubkey'], } # Set wireguard interface IP - exec { 'set wg interface IP': - require => Package['wireguard'], + exec { 'set-IP': + require => Exec['add-key'], command => '/usr/bin/ip address add dev wg0 77.80.229.133/25', unless => '/usr/bin/ip addr show wg0 | grep 77.80.229.133/25' } file { '/etc/wireguard/yaml': ensure => directory, + require => Exec['set-IP'], recurse => remote, source => 'puppet:///svn/$::{current_event}/services/wireguard', } @@ -59,7 +62,7 @@ # Sync changes towards the wg0 interface exec { 'syncConf': - require => Package['wireguard'], + require => file['setConf'], command => '/usr/bin/wg syncconf wg0 /etc/wireguard/wg0.conf', } } From 48938a5f0a6d6a38dd103b725ff7121fd7ebbb3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 16 Jun 2024 16:23:51 +0200 Subject: [PATCH 052/107] Create akvorado.py --- modules/akvorado.py | 57 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 modules/akvorado.py diff --git a/modules/akvorado.py b/modules/akvorado.py new file mode 100644 index 00000000..034e579c --- /dev/null +++ b/modules/akvorado.py @@ -0,0 +1,57 @@ +# vim: ts=4: sts=4: sw=4: expandtab +# Copyright 2024 dhtech +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file +import lib +import os +import sqlite3 +import yaml + +DB_FILE = '/etc/ipplan.db' + +def get_prefixes(ipversion): + if os.path.isfile(DB_FILE): + try: + conn = sqlite3.connect(DB_FILE) + db = conn.cursor() + except sqlite3.Error as e: + print "An error occurred:", e.args[0] + sys.exit(2) + else: + print "No database file found: %s" % DB_FILE + sys.exit(3) + + if ipversion == "4": + db.execute( + 'SELECT SUBSTR(name,1, INSTR(name, "@")-1), name, short_name, ipv4_txt' + ' FROM network' + ' WHERE node_id NOT IN (SELECT option.node_id from option where name = "NO-AKV") and name like "%@%" and ipv4_txt is not NULL' + ) + + elif ipversion == "6": + db.execute( + 'SELECT SUBSTR(name,1, INSTR(name, "@")-1), name, short_name, ipv6_txt' + ' FROM network' + ' WHERE node_id NOT IN (SELECT option.node_id from option where name = "NO-AKV") and name like "%@%" and ipv6_txt is not NULL' + ) + else: + raise NetworkTypeNotFoundError('network type must be 4 or 6') + + res = db.fetchall() + conn.close() + if not res: + raise NetworkNotFoundError('network not found') + + return res + + +def generate(host, *args): + + info = {} + info['current_event'] = lib.get_current_event() + info['ipv6_prefixes'] = get_prefixes('6') + info['ipv4_prefixes'] = get_prefixes('4') + print(info) + return {'akverado': info} + From 5924487817974a56f483c4a58f1f565c17daac29 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 17:30:38 +0200 Subject: [PATCH 053/107] Add akvorado module and service files --- modules/akvorado/files/akvorado-console.service | 15 +++++++++++++++ modules/akvorado/files/akvorado-inlet.service | 15 +++++++++++++++ modules/akvorado/files/akvorado-orch.service | 13 +++++++++++++ modules/akvorado/files/kafka.service | 13 +++++++++++++ modules/akvorado/files/zookeeper.service | 13 +++++++++++++ 5 files changed, 69 insertions(+) create mode 100644 modules/akvorado/files/akvorado-console.service create mode 100644 modules/akvorado/files/akvorado-inlet.service create mode 100644 modules/akvorado/files/akvorado-orch.service create mode 100644 modules/akvorado/files/kafka.service create mode 100644 modules/akvorado/files/zookeeper.service diff --git a/modules/akvorado/files/akvorado-console.service b/modules/akvorado/files/akvorado-console.service new file mode 100644 index 00000000..23e2f587 --- /dev/null +++ b/modules/akvorado/files/akvorado-console.service @@ -0,0 +1,15 @@ +[Unit] +Description=Akvorado Console +After=akvorado-orch.service +Requires=akvorado-orch.service + +[Service] +Type=simple +Restart=on-failure +RestartSec=15 +User=akvorado +ExecStart=/usr/local/bin/akvorado console http://127.0.0.1:8080 + +[Install] +WantedBy=multi-user.target + diff --git a/modules/akvorado/files/akvorado-inlet.service b/modules/akvorado/files/akvorado-inlet.service new file mode 100644 index 00000000..1930cb0c --- /dev/null +++ b/modules/akvorado/files/akvorado-inlet.service @@ -0,0 +1,15 @@ +[Unit] +Description=Akvorado Inlet +After=akvorado-orch.service +Requires=akvorado-orch.service + +[Service] +Type=simple +Restart=on-failure +RestartSec=15 +User=akvorado +ExecStart=/usr/local/bin/akvorado inlet http://127.0.0.1:8080 + +[Install] +WantedBy=multi-user.target + diff --git a/modules/akvorado/files/akvorado-orch.service b/modules/akvorado/files/akvorado-orch.service new file mode 100644 index 00000000..23e0f153 --- /dev/null +++ b/modules/akvorado/files/akvorado-orch.service @@ -0,0 +1,13 @@ +[Unit] +Description=Akvorado Orchestrator +After=network.target +[Service] +Type=simple +Restart=on-failure +RestartSec=15 +User=akvorado +ExecStart=/usr/local/bin/akvorado orchestrator /etc/akvorado/akvorado.yaml + +[Install] +WantedBy=multi-user.target + diff --git a/modules/akvorado/files/kafka.service b/modules/akvorado/files/kafka.service new file mode 100644 index 00000000..51df02db --- /dev/null +++ b/modules/akvorado/files/kafka.service @@ -0,0 +1,13 @@ +[Unit] +Requires=zookeeper.service +After=zookeeper.service + +[Service] +Type=simple +User=kafka +ExecStart=/bin/sh -c '/var/lib/kafka/bin/kafka-server-start.sh /var/lib/kafka/config/server.properties > /var/log/kaf +ExecStop=/var/lib/kafka/bin/kafka-server-stop.sh +Restart=on-abnormal + +[Install] +WantedBy=multi-user.target diff --git a/modules/akvorado/files/zookeeper.service b/modules/akvorado/files/zookeeper.service new file mode 100644 index 00000000..62fcd238 --- /dev/null +++ b/modules/akvorado/files/zookeeper.service @@ -0,0 +1,13 @@ +[Unit] +Requires=network.target remote-fs.target +After=network.target remote-fs.target + +[Service] +Type=simple +User=kafka +ExecStart=/var/lib/kafka/bin/zookeeper-server-start.sh /var/lib/kafka/config/zookeeper.properties +ExecStop=/var/lib/kafka/bin/zookeeper-server-stop.sh +Restart=on-abnormal + +[Install] +WantedBy=multi-user.target From 0356354d0733fdb23ac19e5dc8b2cec7c361be35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 16 Jun 2024 17:46:51 +0200 Subject: [PATCH 054/107] Create akvorado.yaml.erb --- modules/akvorado/templates/akvorado.yaml.erb | 303 +++++++++++++++++++ 1 file changed, 303 insertions(+) create mode 100644 modules/akvorado/templates/akvorado.yaml.erb diff --git a/modules/akvorado/templates/akvorado.yaml.erb b/modules/akvorado/templates/akvorado.yaml.erb new file mode 100644 index 00000000..041c3280 --- /dev/null +++ b/modules/akvorado/templates/akvorado.yaml.erb @@ -0,0 +1,303 @@ +--- +reporting: + logging: {} + metrics: {} +http: + listen: :8080 + profiler: true + cache: + type: memory +clickhouse: + servers: + - localhost:9000 + cluster: "" + database: default + username: default + password: "dhtech" + maxopenconns: 10 + dialtimeout: 5s + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + skipmigrations: false + kafka: + topic: flows + brokers: + - localhost:9092 + version: 3.3.1 + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + saslusername: "" + saslpassword: "" + saslmechanism: none + consumers: 4 + groupname: clickhouse + enginesettings: [] + resolutions: + - interval: 0s + ttl: 360h0m0s + - interval: 1m0s + ttl: 168h0m0s + - interval: 5m0s + ttl: 2160h0m0s + - interval: 1h0m0s + ttl: 8640h0m0s + maxpartitions: 50 + systemlogttl: 720h0m0s + prometheusendpoint: /metrics + asns: + 25037: Dreamhack ACME Corporation + networks: + # 2a01:db8:cafe:1::/64: + # name: ipv6-customers + # role: customers + # site: "" + # region: "" + # city: "" + # state: "" + # country: "" + # tenant: "" + # asn: 0 + + <% @ipv4_prefixes.each do |ipv4| -%> + "<%= ipv4['ipv4_txt'] %>" + name: "<%= ipv4['short_name'] %>" + role: "<%= ipv4['location'] %>" + <% end -%> + <% @ipv6_prefixes.each do |ipv6| -%> + "<%= ipv6['ipv6_txt'] %>" + name: "<%= ipv6['short_name'] %>" + role: "<%= ipv6['location'] %>" + <% end -%> + + networksources: {} + networksourcestimeout: 10s + orchestratorurl: http://localhost:8080 +kafka: + topic: flows + brokers: + - localhost:9092 + version: 3.3.1 + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + saslusername: "" + saslpassword: "" + saslmechanism: none + topicconfiguration: + numpartitions: 8 + replicationfactor: 1 + configentries: + cleanup.policy: delete + compression.type: producer + retention.ms: "86400000" + segment.bytes: "1073741824" + configentriesstrictsync: true +geoip: + asndatabase: + - /usr/share/GeoIP/asn.mmdb + geodatabase: + - /usr/share/GeoIP/country.mmdb + optional: true +schema: + disabled: [] + enabled: [] + maintableonly: [] + notmaintableonly: [] + materialize: [] + customdictionaries: {} +inlet: + - reporting: + logging: {} + metrics: {} + http: + listen: :8081 + profiler: true + cache: + type: memory + flow: + inputs: + - decoder: netflow + listen: :2055 + queuesize: 100000 + receivebuffer: 10485760 + timestampsource: udp + type: udp + usesrcaddrforexporteraddr: false + workers: 6 + - decoder: sflow + listen: :6343 + queuesize: 100000 + receivebuffer: 10485760 + timestampsource: udp + type: udp + usesrcaddrforexporteraddr: false + workers: 6 + ratelimit: 0 + metadata: + cacheduration: 30m0s + cacherefresh: 1h0m0s + cachecheckinterval: 2m0s + cachepersistfile: "" + providers: + - agents: {} + communities: + ::/0: + - public + pollerretries: 1 + pollertimeout: 1s + ports: + ::/0: 161 + securityparameters: {} + type: snmp + workers: 10 + maxbatchrequests: 10 + routing: + provider: + collectasns: true + collectaspaths: true + collectcommunities: true + keep: 5m0s + listen: :10179 + rds: [] + ribpeerremovalbatchroutes: 5000 + ribpeerremovalmaxqueue: 10000 + ribpeerremovalmaxtime: 100ms + ribpeerremovalsleepinterval: 500ms + type: bmp + kafka: + topic: flows + brokers: + - localhost:9092 + version: 3.3.1 + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + saslusername: "" + saslpassword: "" + saslmechanism: none + flushinterval: 10s + flushbytes: 104857599 + maxmessagebytes: 1000000 + compressioncodec: zstd + queuesize: 32 + core: + workers: 6 + exporterclassifiers: + - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") + - ClassifyRegion("europe") + - ClassifyTenant("acme") + - ClassifyRole("edge") + interfaceclassifiers: + - | + ClassifyConnectivityRegex(Interface.Description, "^(?i)(transit|pni|ppni|ix):? ", "$1") && + ClassifyProviderRegex(Interface.Description, "^\\S+?\\s(\\S+)", "$1") && + ClassifyExternal() + - ClassifyInternal() + classifiercacheduration: 5m0s + defaultsamplingrate: {} + overridesamplingrate: {} + asnproviders: + - flow + - routing + netproviders: + - flow + - routing + schema: + disabled: [] + enabled: [] + maintableonly: [] + notmaintableonly: [] + materialize: [] + customdictionaries: {} +console: + - reporting: + logging: {} + metrics: {} + http: + listen: :8082 + profiler: true + cache: + db: 0 + password: "" + protocol: tcp + server: localhost:6379 + type: redis + username: "" + defaultvisualizeoptions: + graphtype: stacked + start: 6 hours ago + end: now + filter: InIfBoundary = external + dimensions: + - SrcAS + limit: 10 + homepagetopwidgets: + - src-as + - src-port + - protocol + - src-country + - etype + homepagegraphfilter: InIfBoundary = 'external' + dimensionslimit: 50 + cachettl: 3h0m0s + clickhouse: + servers: + - localhost:9000 + cluster: "" + database: default + username: default + password: "" + maxopenconns: 10 + dialtimeout: 5s + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + auth: + headers: + login: Remote-User + name: Remote-Name + email: Remote-Email + logouturl: X-Logout-URL + defaultuser: + login: __default + name: Default User + email: "" + logouturl: "" + database: + driver: sqlite + dsn: file::memory:?cache=shared + savedfilters: + - description: From Netflix + content: InIfBoundary = external AND SrcAS = AS2906 + - description: From GAFAM + content: InIfBoundary = external AND SrcAS IN (AS15169, AS16509, AS32934, AS6185, AS8075) + - description: From Swedish Armed Forces + content: InIfBoundary = external AND SrcAS = AS9201 + - description: Valve Corporation + content: InIfBoundary = external AND SrcAS = AS32590 + + + schema: + disabled: [] + enabled: [] + maintableonly: [] + notmaintableonly: [] + materialize: [] + customdictionaries: {} From f4f9acfe66a561b200b734964ce8fb690aa2bc6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 16 Jun 2024 18:38:28 +0200 Subject: [PATCH 055/107] Create init.pp --- modules/akvorado/manifests/init.pp | 90 ++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 modules/akvorado/manifests/init.pp diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp new file mode 100644 index 00000000..8e62f457 --- /dev/null +++ b/modules/akvorado/manifests/init.pp @@ -0,0 +1,90 @@ +# Copyright 2018 dhtech +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file +# +# == Class: akvorado +# +# Alert manager for prometheus to handle sending alerts +# +# === Parameters +# + +class akvorado { + + #Create user/group for Akvorodo + group { 'akvorado': + ensure => 'present', + } + -> user { 'akvorado': + ensure => 'present', + system => true, + } + #Create directories for akvorado + -> file { '/etc/akvorado': + ensure => 'directory', + owner => 'root', + group => 'akvorado', + mode => '0750', + } + #Copy akvorado to the server + -> file { '/usr/local/bin/akvorado': + ensure => file, + owner => 'root', + group => 'akvorado', + mode => '0550', + links => follow, + source => 'puppet:///data/akvorado-latest', + } + + file { '/etc/akvorado/akvorado.yaml': + ensure => file, + content => template('akvorado/akvorado.yaml.erb'), + notify => Service['akvorado-orch'], + } + #Systemctl config + file { '/etc/systemd/system/akvorado-orch.service': + ensure => present, + source => 'puppet:///modules/akvorado/akvorado-orch.service', + mode => '0644', + owner => 'root', + group => 'root', + notify => [Exec['systemctl-daemon-reload'],Service['akvorado-orch']], + } + file { '/etc/systemd/system/akvorado-inlet.service': + ensure => present, + source => 'puppet:///modules/akvorado/akvorado-inlet.service', + mode => '0644', + owner => 'root', + group => 'root', + notify => [Exec['systemctl-daemon-reload'],Service['akvorado-inlet']], + } + file { '/etc/systemd/system/akvorado-console.service': + ensure => present, + source => 'puppet:///modules/akvorado/akvorado-console.service', + mode => '0644', + owner => 'root', + group => 'root', + notify => [Exec['systemctl-daemon-reload'],Service['akvorado-console']], + } + + -> apache::proxy { 'akvorado': + url => '/', + backend => 'http://localhost:8082/', + } + -> service { 'akvorado-orch': + ensure => running, + } +-> service { 'akvorado-inlet': + ensure => running, + } +-> service { 'akvorado-console': + ensure => running, + } + + + exec { 'systemctl-daemon-reload': + command => '/bin/systemctl daemon-reload', + refreshonly => true, + } +} From a265b3191c7cfddb206c6fb25f8f62495149285b Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 20:44:11 +0200 Subject: [PATCH 056/107] Add kafka and clickhouse installation --- modules/akvorado/manifests/init.pp | 108 +++++++++++++++++++++++++++-- 1 file changed, 104 insertions(+), 4 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 8e62f457..9b511207 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -15,7 +15,7 @@ #Create user/group for Akvorodo group { 'akvorado': ensure => 'present', - } + } -> user { 'akvorado': ensure => 'present', system => true, @@ -36,7 +36,7 @@ links => follow, source => 'puppet:///data/akvorado-latest', } - + file { '/etc/akvorado/akvorado.yaml': ensure => file, content => template('akvorado/akvorado.yaml.erb'), @@ -67,7 +67,7 @@ group => 'root', notify => [Exec['systemctl-daemon-reload'],Service['akvorado-console']], } - + -> apache::proxy { 'akvorado': url => '/', backend => 'http://localhost:8082/', @@ -82,9 +82,109 @@ ensure => running, } - exec { 'systemctl-daemon-reload': command => '/bin/systemctl daemon-reload', refreshonly => true, } + + ##Kafka installation + group { 'kafka': + ensure => 'present', + } + -> user { 'kafka': + ensure => 'present', + system => true, + home => '/var/lib/kafka', + managegome => true, + } + -> file { '/tmp/kafka.tgz': + ensure => file, + links => follow, + source => 'puppet:///data/kafka-latest.tgz', + notify => Exec[ 'untar-kafka' ], + } + -> file { '/var/log/kafka': + ensure => 'directory', + owner => 'kafka', + group => 'kafka', + mode => '0700', + } + -> file { '/etc/systemd/system/kafka.service': + ensure => present, + source => 'puppet:///modules/akvorado/kafka.service', + mode => '0644', + owner => 'root', + group => 'root', + notify => [Exec['systemctl-daemon-reload'],Service['kafka']], + } + -> file_line { 'kafka-enabledeletetopics' + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'delete.topic.enable = true' + line => 'delete.topic.enable' + } + -> file_line { 'kafka-listenlocalhost' + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'listeners=PLAINTEXT://localhost:9092' + match => '#listeners=PLAINTEXT' + } + -> file_line { 'kafka-logdir' + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'log.dirs=/var/log/kafka' + match => 'log.dirs=' + } + exec { 'untar-kafka': + command => '/bin/tar -zxf /tmp/kafka.tgz -C /var/lib/kafka --strip=1', + refreshonly => true, + user => 'kafka', + } + + ##Zookeeper installation + ensure_packages([ + 'apt-transport-https', + 'ca-certificates', + 'curl', + 'gnupg', + ]) + file { 'clickhouse-source-add': + ensure => file, + path => '/etc/apt/sources.list.d/clickhouse.list', + content => "deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main", + notify => Exec['clickhouse-source-key'], + } + file_line { 'clickhouse-listen' + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'clientPortAddress=127.0.0.1', + match => 'clientPortAddress=', + } + exec { 'clickhouse-source-key': + command => '/usr/bin/wget -fsSLhttps://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key -O /usr/share/keyrings/clickhouse-keyring.gpg', + logoutput => 'on_failure', + try_sleep => 1, + refreshonly => true, + notify => Exec['docker-source-update'], + } + exec { 'apt-update': + command => '/usr/bin/apt-get update', + logoutput => 'on_failure', + try_sleep => 1, + refreshonly => true, + require => Package['apt-transport-https'], + } + + package { 'clickhouse': + ensure => installed, + require => [File['clickhouse-source-add'], Exec['clickhouse-source-key'], Exec['apt-update'], File_Line['clickhouse-listen']], + } + -> file { '/etc/systemd/system/clickhouse.service': + ensure => present, + source => 'puppet:///modules/akvorado/clickhouse.service', + mode => '0644', + owner => 'root', + group => 'root', + notify => [Exec['systemctl-daemon-reload'],Service['clickhouse']], + } } From 8f82300eb45d39a1f3f19153485032b15dc714bc Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 20:50:22 +0200 Subject: [PATCH 057/107] fix cosmetic issues --- modules/akvorado/manifests/init.pp | 44 +++++++++++++++--------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 9b511207..8f046804 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -92,10 +92,10 @@ ensure => 'present', } -> user { 'kafka': - ensure => 'present', - system => true, - home => '/var/lib/kafka', - managegome => true, + ensure => 'present', + system => true, + home => '/var/lib/kafka', + managegome => true, } -> file { '/tmp/kafka.tgz': ensure => file, @@ -124,16 +124,16 @@ line => 'delete.topic.enable' } -> file_line { 'kafka-listenlocalhost' - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'listeners=PLAINTEXT://localhost:9092' - match => '#listeners=PLAINTEXT' + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'listeners=PLAINTEXT://localhost:9092' + match => '#listeners=PLAINTEXT' } -> file_line { 'kafka-logdir' - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'log.dirs=/var/log/kafka' - match => 'log.dirs=' + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'log.dirs=/var/log/kafka' + match => 'log.dirs=' } exec { 'untar-kafka': command => '/bin/tar -zxf /tmp/kafka.tgz -C /var/lib/kafka --strip=1', @@ -143,22 +143,22 @@ ##Zookeeper installation ensure_packages([ - 'apt-transport-https', - 'ca-certificates', - 'curl', - 'gnupg', + 'apt-transport-https', + 'ca-certificates', + 'curl', + 'gnupg', ]) file { 'clickhouse-source-add': ensure => file, path => '/etc/apt/sources.list.d/clickhouse.list', - content => "deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main", + content => 'deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main', notify => Exec['clickhouse-source-key'], } file_line { 'clickhouse-listen' - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'clientPortAddress=127.0.0.1', - match => 'clientPortAddress=', + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'clientPortAddress=127.0.0.1', + match => 'clientPortAddress=', } exec { 'clickhouse-source-key': command => '/usr/bin/wget -fsSLhttps://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key -O /usr/share/keyrings/clickhouse-keyring.gpg', @@ -174,7 +174,7 @@ refreshonly => true, require => Package['apt-transport-https'], } - + package { 'clickhouse': ensure => installed, require => [File['clickhouse-source-add'], Exec['clickhouse-source-key'], Exec['apt-update'], File_Line['clickhouse-listen']], From 4deb8a92e5b5ee0b1585493a9f9e71dd75bf11ca Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 20:51:57 +0200 Subject: [PATCH 058/107] fix indent --- modules/akvorado/manifests/init.pp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 8f046804..558c3a76 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -92,10 +92,10 @@ ensure => 'present', } -> user { 'kafka': - ensure => 'present', - system => true, - home => '/var/lib/kafka', - managegome => true, + ensure => 'present', + system => true, + home => '/var/lib/kafka', + managegome => true, } -> file { '/tmp/kafka.tgz': ensure => file, From 708ea00716e694eaa2425c0e9f928067a23a1a46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 16 Jun 2024 21:40:17 +0200 Subject: [PATCH 059/107] Update akvorado.py --- modules/akvorado.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/modules/akvorado.py b/modules/akvorado.py index 034e579c..721034cf 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -24,26 +24,31 @@ def get_prefixes(ipversion): if ipversion == "4": db.execute( - 'SELECT SUBSTR(name,1, INSTR(name, "@")-1), name, short_name, ipv4_txt' + 'SELECT SUBSTR(name,1, INSTR(name, "@")-1) AS location, name, short_name, ipv4_txt' ' FROM network' - ' WHERE node_id NOT IN (SELECT option.node_id from option where name = "NO-AKV") and name like "%@%" and ipv4_txt is not NULL' + ' WHERE node_id NOT IN (SELECT option.node_id FROM option WHERE name = "no-akv")' + ' AND name LIKE "%@%" AND ipv4_txt IS NOT NULL' ) elif ipversion == "6": db.execute( - 'SELECT SUBSTR(name,1, INSTR(name, "@")-1), name, short_name, ipv6_txt' + 'SELECT SUBSTR(name,1, INSTR(name, "@")-1) AS location, name, short_name, ipv6_txt' ' FROM network' - ' WHERE node_id NOT IN (SELECT option.node_id from option where name = "NO-AKV") and name like "%@%" and ipv6_txt is not NULL' + ' WHERE node_id NOT IN (SELECT option.node_id FROM option WHERE name = "no-akv")' + ' AND name LIKE "%@%" AND ipv6_txt IS NOT NULL' ) else: raise NetworkTypeNotFoundError('network type must be 4 or 6') res = db.fetchall() - conn.close() if not res: raise NetworkNotFoundError('network not found') + + column_names = [description[0] for description in db.description] + conn.close() + rows_dict = [dict(zip(column_names, row)) for row in res] - return res + return rows_dict def generate(host, *args): From f004e0257343c0931369cfd77081b8272f7dac26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 16 Jun 2024 21:53:26 +0200 Subject: [PATCH 060/107] Update akvorado.py --- modules/akvorado.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/akvorado.py b/modules/akvorado.py index 721034cf..590781a5 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -16,11 +16,11 @@ def get_prefixes(ipversion): conn = sqlite3.connect(DB_FILE) db = conn.cursor() except sqlite3.Error as e: - print "An error occurred:", e.args[0] - sys.exit(2) + print("An error occurred: {}".format(e.args[0])) + exit(2) else: - print "No database file found: %s" % DB_FILE - sys.exit(3) + print("No database file found: {}".format(DB_FILE)) + exit(3) if ipversion == "4": db.execute( From abe9952a283d7648c2b9d5a118e7027492c5dd6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 16 Jun 2024 21:54:55 +0200 Subject: [PATCH 061/107] Update akvorado.py --- modules/akvorado.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/akvorado.py b/modules/akvorado.py index 590781a5..95058f5c 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -43,7 +43,7 @@ def get_prefixes(ipversion): res = db.fetchall() if not res: raise NetworkNotFoundError('network not found') - + column_names = [description[0] for description in db.description] conn.close() rows_dict = [dict(zip(column_names, row)) for row in res] @@ -59,4 +59,3 @@ def generate(host, *args): info['ipv4_prefixes'] = get_prefixes('4') print(info) return {'akverado': info} - From 81f2542be1284a29418db9a7812074063b8bcedd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 16 Jun 2024 21:56:11 +0200 Subject: [PATCH 062/107] Update akvorado.py --- modules/akvorado.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/akvorado.py b/modules/akvorado.py index 95058f5c..544c521b 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -10,6 +10,7 @@ DB_FILE = '/etc/ipplan.db' + def get_prefixes(ipversion): if os.path.isfile(DB_FILE): try: From de7d459e4657155fbf3884c0f6d4cafc66c5b7b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 16 Jun 2024 23:01:33 +0200 Subject: [PATCH 063/107] Update akvorado.py --- modules/akvorado.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/akvorado.py b/modules/akvorado.py index 544c521b..ec6b6b09 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -58,5 +58,4 @@ def generate(host, *args): info['current_event'] = lib.get_current_event() info['ipv6_prefixes'] = get_prefixes('6') info['ipv4_prefixes'] = get_prefixes('4') - print(info) return {'akverado': info} From 79c0e535617b0a60b8d87eb4a9e3a7888db1e7da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 16 Jun 2024 23:05:02 +0200 Subject: [PATCH 064/107] Update akvorado.py --- modules/akvorado.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado.py b/modules/akvorado.py index ec6b6b09..0825ccc6 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -58,4 +58,4 @@ def generate(host, *args): info['current_event'] = lib.get_current_event() info['ipv6_prefixes'] = get_prefixes('6') info['ipv4_prefixes'] = get_prefixes('4') - return {'akverado': info} + return {'akvorado': info} From 6db5de53412d07c9f626effe96ce432009d8e3e5 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:11:31 +0200 Subject: [PATCH 065/107] add columns --- modules/akvorado/manifests/init.pp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 558c3a76..2ba92536 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -117,19 +117,19 @@ group => 'root', notify => [Exec['systemctl-daemon-reload'],Service['kafka']], } - -> file_line { 'kafka-enabledeletetopics' + -> file_line { 'kafka-enabledeletetopics': ensure => 'present', path => '/var/lib/kafka/config/server.properties', line => 'delete.topic.enable = true' line => 'delete.topic.enable' } - -> file_line { 'kafka-listenlocalhost' + -> file_line { 'kafka-listenlocalhost': ensure => 'present', path => '/var/lib/kafka/config/server.properties', line => 'listeners=PLAINTEXT://localhost:9092' match => '#listeners=PLAINTEXT' } - -> file_line { 'kafka-logdir' + -> file_line { 'kafka-logdir': ensure => 'present', path => '/var/lib/kafka/config/server.properties', line => 'log.dirs=/var/log/kafka' @@ -154,7 +154,7 @@ content => 'deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main', notify => Exec['clickhouse-source-key'], } - file_line { 'clickhouse-listen' + file_line { 'clickhouse-listen': ensure => 'present', path => '/var/lib/kafka/config/server.properties', line => 'clientPortAddress=127.0.0.1', From 7b17770a683b7417dd9dacd7f1d1b4c68c21cf99 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:13:44 +0200 Subject: [PATCH 066/107] add commas --- modules/akvorado/manifests/init.pp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 2ba92536..eaa450a2 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -120,20 +120,20 @@ -> file_line { 'kafka-enabledeletetopics': ensure => 'present', path => '/var/lib/kafka/config/server.properties', - line => 'delete.topic.enable = true' - line => 'delete.topic.enable' + line => 'delete.topic.enable = true', + match => 'delete.topic.enable', } -> file_line { 'kafka-listenlocalhost': ensure => 'present', path => '/var/lib/kafka/config/server.properties', - line => 'listeners=PLAINTEXT://localhost:9092' - match => '#listeners=PLAINTEXT' + line => 'listeners=PLAINTEXT://localhost:9092', + match => '#listeners=PLAINTEXT', } -> file_line { 'kafka-logdir': ensure => 'present', path => '/var/lib/kafka/config/server.properties', - line => 'log.dirs=/var/log/kafka' - match => 'log.dirs=' + line => 'log.dirs=/var/log/kafka', + match => 'log.dirs=', } exec { 'untar-kafka': command => '/bin/tar -zxf /tmp/kafka.tgz -C /var/lib/kafka --strip=1', From 9042c2080b7281f6018b6eba644504f736aea04b Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:21:00 +0200 Subject: [PATCH 067/107] add parameters to akvorado class --- modules/akvorado/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index eaa450a2..e0eb7c6e 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -10,7 +10,7 @@ # === Parameters # -class akvorado { +class akvorado ($current_event, $ipv4_prefixes, $ipv6_prefixes) { #Create user/group for Akvorodo group { 'akvorado': From c3d8cf440434a12a0cce121752e833558bf5a484 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:22:49 +0200 Subject: [PATCH 068/107] fix typo --- modules/akvorado/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index e0eb7c6e..d2616ce2 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -95,7 +95,7 @@ ensure => 'present', system => true, home => '/var/lib/kafka', - managegome => true, + managehome => true, } -> file { '/tmp/kafka.tgz': ensure => file, From 788ffa2761af3f25551ec4b7b86b1656b18eccb6 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:26:56 +0200 Subject: [PATCH 069/107] fix indent more --- modules/akvorado/manifests/init.pp | 32 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index d2616ce2..8a14bd12 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -115,25 +115,25 @@ mode => '0644', owner => 'root', group => 'root', - notify => [Exec['systemctl-daemon-reload'],Service['kafka']], + notify => [ Exec['systemctl-daemon-reload'], Service['kafka'] ], } -> file_line { 'kafka-enabledeletetopics': ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'delete.topic.enable = true', - match => 'delete.topic.enable', + path => '/var/lib/kafka/config/server.properties', + line => 'delete.topic.enable = true', + match => 'delete.topic.enable', } -> file_line { 'kafka-listenlocalhost': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'listeners=PLAINTEXT://localhost:9092', - match => '#listeners=PLAINTEXT', + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'listeners=PLAINTEXT://localhost:9092', + match => '#listeners=PLAINTEXT', } -> file_line { 'kafka-logdir': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'log.dirs=/var/log/kafka', - match => 'log.dirs=', + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'log.dirs=/var/log/kafka', + match => 'log.dirs=', } exec { 'untar-kafka': command => '/bin/tar -zxf /tmp/kafka.tgz -C /var/lib/kafka --strip=1', @@ -155,10 +155,10 @@ notify => Exec['clickhouse-source-key'], } file_line { 'clickhouse-listen': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'clientPortAddress=127.0.0.1', - match => 'clientPortAddress=', + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'clientPortAddress=127.0.0.1', + match => 'clientPortAddress=', } exec { 'clickhouse-source-key': command => '/usr/bin/wget -fsSLhttps://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key -O /usr/share/keyrings/clickhouse-keyring.gpg', From 163928973a69228bed7f4ef947a9ed47dbb12206 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:28:33 +0200 Subject: [PATCH 070/107] declare kafka service --- modules/akvorado/manifests/init.pp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 8a14bd12..e76fef67 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -75,10 +75,10 @@ -> service { 'akvorado-orch': ensure => running, } --> service { 'akvorado-inlet': + -> service { 'akvorado-inlet': ensure => running, } --> service { 'akvorado-console': + -> service { 'akvorado-console': ensure => running, } @@ -140,6 +140,9 @@ refreshonly => true, user => 'kafka', } + -> service { 'kafka': + ensure => running, + } ##Zookeeper installation ensure_packages([ @@ -187,4 +190,7 @@ group => 'root', notify => [Exec['systemctl-daemon-reload'],Service['clickhouse']], } + -> service { 'kafka': + ensure => running, + } } From 5165a1b86d006b9947669cb266af997c88840c50 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:31:28 +0200 Subject: [PATCH 071/107] rename duplicate service kafka --- modules/akvorado/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index e76fef67..c102fb2b 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -190,7 +190,7 @@ group => 'root', notify => [Exec['systemctl-daemon-reload'],Service['clickhouse']], } - -> service { 'kafka': + -> service { 'clickhouse': ensure => running, } } From c0bb440b4aba29d9c0b13d30b7899c668eb62c12 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:34:58 +0200 Subject: [PATCH 072/107] fix more stuff --- modules/akvorado/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index c102fb2b..073f0683 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -168,7 +168,7 @@ logoutput => 'on_failure', try_sleep => 1, refreshonly => true, - notify => Exec['docker-source-update'], + notify => Exec['apt-update'], } exec { 'apt-update': command => '/usr/bin/apt-get update', From c8a3dc64fd219ec00193f5e4519dd1d4b1df0000 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:39:01 +0200 Subject: [PATCH 073/107] give wget a bit more space --- modules/akvorado/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 073f0683..17aebfd5 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -164,7 +164,7 @@ match => 'clientPortAddress=', } exec { 'clickhouse-source-key': - command => '/usr/bin/wget -fsSLhttps://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key -O /usr/share/keyrings/clickhouse-keyring.gpg', + command => '/usr/bin/wget -fsSL https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key -O /usr/share/keyrings/clickhouse-keyring.gpg', logoutput => 'on_failure', try_sleep => 1, refreshonly => true, From 5773af8dfdf3b949f6f018c9be952099ec4f99ad Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:52:03 +0200 Subject: [PATCH 074/107] fix zookeeper --- modules/akvorado/manifests/init.pp | 31 ++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 17aebfd5..b51e753d 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -117,34 +117,55 @@ group => 'root', notify => [ Exec['systemctl-daemon-reload'], Service['kafka'] ], } + -> file { '/etc/systemd/system/zookeeper.service': + ensure => present, + source => 'puppet:///modules/akvorado/zookeeper.service', + mode => '0644', + owner => 'root', + group => 'root', + notify => [ Exec['systemctl-daemon-reload'], Service['zookeeper'] ], + } -> file_line { 'kafka-enabledeletetopics': ensure => 'present', path => '/var/lib/kafka/config/server.properties', line => 'delete.topic.enable = true', match => 'delete.topic.enable', + notify => Service['kafka'], } -> file_line { 'kafka-listenlocalhost': ensure => 'present', path => '/var/lib/kafka/config/server.properties', line => 'listeners=PLAINTEXT://localhost:9092', match => '#listeners=PLAINTEXT', + notify => Service['kafka'], } -> file_line { 'kafka-logdir': ensure => 'present', path => '/var/lib/kafka/config/server.properties', line => 'log.dirs=/var/log/kafka', match => 'log.dirs=', + notify => Service['kafka'], + } + -> file_line { 'zookeeper-listen': + ensure => 'present', + path => '/var/lib/kafka/config/zookeeper.properties', + line => 'clientPortAddress=127.0.0.1', + match => 'clientPortAddress=', + notify => Service['zookeeper'], } exec { 'untar-kafka': - command => '/bin/tar -zxf /tmp/kafka.tgz -C /var/lib/kafka --strip=1', + command => '/bin/tar -xvf kafka.tgz -C /var/lib/kafka --strip 1', refreshonly => true, user => 'kafka', } -> service { 'kafka': ensure => running, } + -> service { 'zookeeper': + ensure => running, + } - ##Zookeeper installation + ##Clickhouse installation ensure_packages([ 'apt-transport-https', 'ca-certificates', @@ -157,12 +178,6 @@ content => 'deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main', notify => Exec['clickhouse-source-key'], } - file_line { 'clickhouse-listen': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'clientPortAddress=127.0.0.1', - match => 'clientPortAddress=', - } exec { 'clickhouse-source-key': command => '/usr/bin/wget -fsSL https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key -O /usr/share/keyrings/clickhouse-keyring.gpg', logoutput => 'on_failure', From fbd8acba7851904c3fb8720f0a76a13ed4f2ef4d Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:54:43 +0200 Subject: [PATCH 075/107] . --- modules/akvorado/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index b51e753d..dd53455e 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -195,7 +195,7 @@ package { 'clickhouse': ensure => installed, - require => [File['clickhouse-source-add'], Exec['clickhouse-source-key'], Exec['apt-update'], File_Line['clickhouse-listen']], + require => [File['clickhouse-source-add'], Exec['clickhouse-source-key'], Exec['apt-update']], } -> file { '/etc/systemd/system/clickhouse.service': ensure => present, From 323f9aba066f8f4c287919cbbaca07323189d8a7 Mon Sep 17 00:00:00 2001 From: furest Date: Sun, 16 Jun 2024 23:58:07 +0200 Subject: [PATCH 076/107] . --- modules/akvorado/manifests/init.pp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index dd53455e..4653754d 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -193,10 +193,13 @@ require => Package['apt-transport-https'], } - package { 'clickhouse': + package { 'clickhouse-server': ensure => installed, require => [File['clickhouse-source-add'], Exec['clickhouse-source-key'], Exec['apt-update']], } + -> package { 'clickhouse-client': + ensure => installed, + } -> file { '/etc/systemd/system/clickhouse.service': ensure => present, source => 'puppet:///modules/akvorado/clickhouse.service', From 088133c921946274c77223c40bc017cdbe426e3d Mon Sep 17 00:00:00 2001 From: furest Date: Mon, 17 Jun 2024 00:27:26 +0200 Subject: [PATCH 077/107] . --- modules/akvorado/manifests/init.pp | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 4653754d..350858ad 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -154,7 +154,7 @@ notify => Service['zookeeper'], } exec { 'untar-kafka': - command => '/bin/tar -xvf kafka.tgz -C /var/lib/kafka --strip 1', + command => '/bin/tar -xvf /tmp/kafka.tgz -C /var/lib/kafka --strip 1', refreshonly => true, user => 'kafka', } @@ -200,15 +200,7 @@ -> package { 'clickhouse-client': ensure => installed, } - -> file { '/etc/systemd/system/clickhouse.service': - ensure => present, - source => 'puppet:///modules/akvorado/clickhouse.service', - mode => '0644', - owner => 'root', - group => 'root', - notify => [Exec['systemctl-daemon-reload'],Service['clickhouse']], - } - -> service { 'clickhouse': + -> service { 'clickhouse-server': ensure => running, } } From 0b2360c3ee1115fe5217cbe89ad67682100ea337 Mon Sep 17 00:00:00 2001 From: furest Date: Mon, 17 Jun 2024 00:34:25 +0200 Subject: [PATCH 078/107] fix service file --- modules/akvorado/files/kafka.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/files/kafka.service b/modules/akvorado/files/kafka.service index 51df02db..64d7b10e 100644 --- a/modules/akvorado/files/kafka.service +++ b/modules/akvorado/files/kafka.service @@ -5,7 +5,7 @@ After=zookeeper.service [Service] Type=simple User=kafka -ExecStart=/bin/sh -c '/var/lib/kafka/bin/kafka-server-start.sh /var/lib/kafka/config/server.properties > /var/log/kaf +ExecStart=/bin/sh -c '/home/kafka/kafka/bin/kafka-server-start.sh /home/kafka/kafka/config/server.properties > /var/log/kafka/kafka.log 2>&1' ExecStop=/var/lib/kafka/bin/kafka-server-stop.sh Restart=on-abnormal From 014f0e35b604edbf356d443f7f4a1f794e393c9e Mon Sep 17 00:00:00 2001 From: furest Date: Mon, 17 Jun 2024 01:03:07 +0200 Subject: [PATCH 079/107] fix indent in template --- modules/akvorado/templates/akvorado.yaml.erb | 566 +++++++++---------- 1 file changed, 282 insertions(+), 284 deletions(-) diff --git a/modules/akvorado/templates/akvorado.yaml.erb b/modules/akvorado/templates/akvorado.yaml.erb index 041c3280..be90ee47 100644 --- a/modules/akvorado/templates/akvorado.yaml.erb +++ b/modules/akvorado/templates/akvorado.yaml.erb @@ -1,303 +1,301 @@ --- reporting: + logging: {} + metrics: {} +http: + listen: :8080 + profiler: true + cache: + type: memory +clickhouse: + servers: + - localhost:9000 + cluster: "" + database: default + username: default + password: "dhtech" + maxopenconns: 10 + dialtimeout: 5s + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + skipmigrations: false + kafka: + topic: flows + brokers: + - localhost:9092 + version: 3.3.1 + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + saslusername: "" + saslpassword: "" + saslmechanism: none + consumers: 4 + groupname: clickhouse + enginesettings: [] + resolutions: + - interval: 0s + ttl: 360h0m0s + - interval: 1m0s + ttl: 168h0m0s + - interval: 5m0s + ttl: 2160h0m0s + - interval: 1h0m0s + ttl: 8640h0m0s + maxpartitions: 50 + systemlogttl: 720h0m0s + prometheusendpoint: /metrics + asns: + 25037: Dreamhack ACME Corporation + networks: + # 2a01:db8:cafe:1::/64: + # name: ipv6-customers + # role: customers + # site: "" + # region: "" + # city: "" + # state: "" + # country: "" + # tenant: "" + # asn: 0 +<% @ipv4_prefixes.each do |ipv4| -%> + <%=ipv4['ipv4_txt']%>: + name: "<%= ipv4['short_name'] %>" + role: "<%= ipv4['location'] %>" +<% end -%> +<% @ipv6_prefixes.each do |ipv6| -%> + <%=ipv6['ipv6_txt']%>: + name: "<%= ipv6['short_name'] %>" + role: "<%= ipv6['location'] %>" +<% end -%> + networksources: {} + networksourcestimeout: 10s + orchestratorurl: http://localhost:8080 +kafka: + topic: flows + brokers: + - localhost:9092 + version: 3.3.1 + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + saslusername: "" + saslpassword: "" + saslmechanism: none + topicconfiguration: + numpartitions: 8 + replicationfactor: 1 + configentries: + cleanup.policy: delete + compression.type: producer + retention.ms: "86400000" + segment.bytes: "1073741824" + configentriesstrictsync: true +geoip: + asndatabase: + - /usr/share/GeoIP/asn.mmdb + geodatabase: + - /usr/share/GeoIP/country.mmdb + optional: true +schema: + disabled: [] + enabled: [] + maintableonly: [] + notmaintableonly: [] + materialize: [] + customdictionaries: {} +inlet: + - reporting: logging: {} metrics: {} -http: - listen: :8080 + http: + listen: :8081 profiler: true cache: - type: memory -clickhouse: + type: memory + flow: + inputs: + - decoder: netflow + listen: :2055 + queuesize: 100000 + receivebuffer: 10485760 + timestampsource: udp + type: udp + usesrcaddrforexporteraddr: false + workers: 6 + - decoder: sflow + listen: :6343 + queuesize: 100000 + receivebuffer: 10485760 + timestampsource: udp + type: udp + usesrcaddrforexporteraddr: false + workers: 6 + ratelimit: 0 + metadata: + cacheduration: 30m0s + cacherefresh: 1h0m0s + cachecheckinterval: 2m0s + cachepersistfile: "" + providers: + - agents: {} + communities: + ::/0: + - public + pollerretries: 1 + pollertimeout: 1s + ports: + ::/0: 161 + securityparameters: {} + type: snmp + workers: 10 + maxbatchrequests: 10 + routing: + provider: + collectasns: true + collectaspaths: true + collectcommunities: true + keep: 5m0s + listen: :10179 + rds: [] + ribpeerremovalbatchroutes: 5000 + ribpeerremovalmaxqueue: 10000 + ribpeerremovalmaxtime: 100ms + ribpeerremovalsleepinterval: 500ms + type: bmp + kafka: + topic: flows + brokers: + - localhost:9092 + version: 3.3.1 + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + saslusername: "" + saslpassword: "" + saslmechanism: none + flushinterval: 10s + flushbytes: 104857599 + maxmessagebytes: 1000000 + compressioncodec: zstd + queuesize: 32 + core: + workers: 6 + exporterclassifiers: + - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") + - ClassifyRegion("europe") + - ClassifyTenant("acme") + - ClassifyRole("edge") + interfaceclassifiers: + - | + ClassifyConnectivityRegex(Interface.Description, "^(?i)(transit|pni|ppni|ix):? ", "$1") && + ClassifyProviderRegex(Interface.Description, "^\\S+?\\s(\\S+)", "$1") && + ClassifyExternal() + - ClassifyInternal() + classifiercacheduration: 5m0s + defaultsamplingrate: {} + overridesamplingrate: {} + asnproviders: + - flow + - routing + netproviders: + - flow + - routing + schema: + disabled: [] + enabled: [] + maintableonly: [] + notmaintableonly: [] + materialize: [] + customdictionaries: {} +console: + - reporting: + logging: {} + metrics: {} + http: + listen: :8082 + profiler: true + cache: + db: 0 + password: "" + protocol: tcp + server: localhost:6379 + type: redis + username: "" + defaultvisualizeoptions: + graphtype: stacked + start: 6 hours ago + end: now + filter: InIfBoundary = external + dimensions: + - SrcAS + limit: 10 + homepagetopwidgets: + - src-as + - src-port + - protocol + - src-country + - etype + homepagegraphfilter: InIfBoundary = 'external' + dimensionslimit: 50 + cachettl: 3h0m0s + clickhouse: servers: - - localhost:9000 + - localhost:9000 cluster: "" database: default username: default - password: "dhtech" + password: "" maxopenconns: 10 dialtimeout: 5s tls: - enable: false - verify: true - cafile: "" - certfile: "" - keyfile: "" - skipmigrations: false - kafka: - topic: flows - brokers: - - localhost:9092 - version: 3.3.1 - tls: - enable: false - verify: true - cafile: "" - certfile: "" - keyfile: "" - saslusername: "" - saslpassword: "" - saslmechanism: none - consumers: 4 - groupname: clickhouse - enginesettings: [] - resolutions: - - interval: 0s - ttl: 360h0m0s - - interval: 1m0s - ttl: 168h0m0s - - interval: 5m0s - ttl: 2160h0m0s - - interval: 1h0m0s - ttl: 8640h0m0s - maxpartitions: 50 - systemlogttl: 720h0m0s - prometheusendpoint: /metrics - asns: - 25037: Dreamhack ACME Corporation - networks: - # 2a01:db8:cafe:1::/64: - # name: ipv6-customers - # role: customers - # site: "" - # region: "" - # city: "" - # state: "" - # country: "" - # tenant: "" - # asn: 0 - - <% @ipv4_prefixes.each do |ipv4| -%> - "<%= ipv4['ipv4_txt'] %>" - name: "<%= ipv4['short_name'] %>" - role: "<%= ipv4['location'] %>" - <% end -%> - <% @ipv6_prefixes.each do |ipv6| -%> - "<%= ipv6['ipv6_txt'] %>" - name: "<%= ipv6['short_name'] %>" - role: "<%= ipv6['location'] %>" - <% end -%> - - networksources: {} - networksourcestimeout: 10s - orchestratorurl: http://localhost:8080 -kafka: - topic: flows - brokers: - - localhost:9092 - version: 3.3.1 - tls: - enable: false - verify: true - cafile: "" - certfile: "" - keyfile: "" - saslusername: "" - saslpassword: "" - saslmechanism: none - topicconfiguration: - numpartitions: 8 - replicationfactor: 1 - configentries: - cleanup.policy: delete - compression.type: producer - retention.ms: "86400000" - segment.bytes: "1073741824" - configentriesstrictsync: true -geoip: - asndatabase: - - /usr/share/GeoIP/asn.mmdb - geodatabase: - - /usr/share/GeoIP/country.mmdb - optional: true -schema: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + auth: + headers: + login: Remote-User + name: Remote-Name + email: Remote-Email + logouturl: X-Logout-URL + defaultuser: + login: __default + name: Default User + email: "" + logouturl: "" + database: + driver: sqlite + dsn: file::memory:?cache=shared + savedfilters: + - description: From Netflix + content: InIfBoundary = external AND SrcAS = AS2906 + - description: From GAFAM + content: InIfBoundary = external AND SrcAS IN (AS15169, AS16509, AS32934, AS6185, AS8075) + - description: From Swedish Armed Forces + content: InIfBoundary = external AND SrcAS = AS9201 + - description: Valve Corporation + content: InIfBoundary = external AND SrcAS = AS32590 + + + schema: disabled: [] enabled: [] maintableonly: [] notmaintableonly: [] materialize: [] customdictionaries: {} -inlet: - - reporting: - logging: {} - metrics: {} - http: - listen: :8081 - profiler: true - cache: - type: memory - flow: - inputs: - - decoder: netflow - listen: :2055 - queuesize: 100000 - receivebuffer: 10485760 - timestampsource: udp - type: udp - usesrcaddrforexporteraddr: false - workers: 6 - - decoder: sflow - listen: :6343 - queuesize: 100000 - receivebuffer: 10485760 - timestampsource: udp - type: udp - usesrcaddrforexporteraddr: false - workers: 6 - ratelimit: 0 - metadata: - cacheduration: 30m0s - cacherefresh: 1h0m0s - cachecheckinterval: 2m0s - cachepersistfile: "" - providers: - - agents: {} - communities: - ::/0: - - public - pollerretries: 1 - pollertimeout: 1s - ports: - ::/0: 161 - securityparameters: {} - type: snmp - workers: 10 - maxbatchrequests: 10 - routing: - provider: - collectasns: true - collectaspaths: true - collectcommunities: true - keep: 5m0s - listen: :10179 - rds: [] - ribpeerremovalbatchroutes: 5000 - ribpeerremovalmaxqueue: 10000 - ribpeerremovalmaxtime: 100ms - ribpeerremovalsleepinterval: 500ms - type: bmp - kafka: - topic: flows - brokers: - - localhost:9092 - version: 3.3.1 - tls: - enable: false - verify: true - cafile: "" - certfile: "" - keyfile: "" - saslusername: "" - saslpassword: "" - saslmechanism: none - flushinterval: 10s - flushbytes: 104857599 - maxmessagebytes: 1000000 - compressioncodec: zstd - queuesize: 32 - core: - workers: 6 - exporterclassifiers: - - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") - - ClassifyRegion("europe") - - ClassifyTenant("acme") - - ClassifyRole("edge") - interfaceclassifiers: - - | - ClassifyConnectivityRegex(Interface.Description, "^(?i)(transit|pni|ppni|ix):? ", "$1") && - ClassifyProviderRegex(Interface.Description, "^\\S+?\\s(\\S+)", "$1") && - ClassifyExternal() - - ClassifyInternal() - classifiercacheduration: 5m0s - defaultsamplingrate: {} - overridesamplingrate: {} - asnproviders: - - flow - - routing - netproviders: - - flow - - routing - schema: - disabled: [] - enabled: [] - maintableonly: [] - notmaintableonly: [] - materialize: [] - customdictionaries: {} -console: - - reporting: - logging: {} - metrics: {} - http: - listen: :8082 - profiler: true - cache: - db: 0 - password: "" - protocol: tcp - server: localhost:6379 - type: redis - username: "" - defaultvisualizeoptions: - graphtype: stacked - start: 6 hours ago - end: now - filter: InIfBoundary = external - dimensions: - - SrcAS - limit: 10 - homepagetopwidgets: - - src-as - - src-port - - protocol - - src-country - - etype - homepagegraphfilter: InIfBoundary = 'external' - dimensionslimit: 50 - cachettl: 3h0m0s - clickhouse: - servers: - - localhost:9000 - cluster: "" - database: default - username: default - password: "" - maxopenconns: 10 - dialtimeout: 5s - tls: - enable: false - verify: true - cafile: "" - certfile: "" - keyfile: "" - auth: - headers: - login: Remote-User - name: Remote-Name - email: Remote-Email - logouturl: X-Logout-URL - defaultuser: - login: __default - name: Default User - email: "" - logouturl: "" - database: - driver: sqlite - dsn: file::memory:?cache=shared - savedfilters: - - description: From Netflix - content: InIfBoundary = external AND SrcAS = AS2906 - - description: From GAFAM - content: InIfBoundary = external AND SrcAS IN (AS15169, AS16509, AS32934, AS6185, AS8075) - - description: From Swedish Armed Forces - content: InIfBoundary = external AND SrcAS = AS9201 - - description: Valve Corporation - content: InIfBoundary = external AND SrcAS = AS32590 - - - schema: - disabled: [] - enabled: [] - maintableonly: [] - notmaintableonly: [] - materialize: [] - customdictionaries: {} From 8200c6aab75b7c8ac0bb44e8a1946761be1e3176 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Mon, 17 Jun 2024 00:53:58 +0200 Subject: [PATCH 080/107] Update akvorado.py --- modules/akvorado.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/akvorado.py b/modules/akvorado.py index 0825ccc6..f8a21dd9 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -52,6 +52,10 @@ def get_prefixes(ipversion): return rows_dict +def requires(host, *args): + return ['apache(ldap)'] + + def generate(host, *args): info = {} From 558866b7edb06f855e587e647058fff115ab1c70 Mon Sep 17 00:00:00 2001 From: furest Date: Mon, 17 Jun 2024 01:31:36 +0200 Subject: [PATCH 081/107] fix yaml --- modules/akvorado/templates/akvorado.yaml.erb | 385 +++++++++---------- 1 file changed, 192 insertions(+), 193 deletions(-) diff --git a/modules/akvorado/templates/akvorado.yaml.erb b/modules/akvorado/templates/akvorado.yaml.erb index be90ee47..0660ed63 100644 --- a/modules/akvorado/templates/akvorado.yaml.erb +++ b/modules/akvorado/templates/akvorado.yaml.erb @@ -9,7 +9,7 @@ http: type: memory clickhouse: servers: - - localhost:9000 + - 127.0.0.1:9000 cluster: "" database: default username: default @@ -26,8 +26,8 @@ clickhouse: kafka: topic: flows brokers: - - localhost:9092 - version: 3.3.1 + - 127.0.0.1:9092 + version: 3.7.0 tls: enable: false verify: true @@ -37,21 +37,21 @@ clickhouse: saslusername: "" saslpassword: "" saslmechanism: none - consumers: 4 + consumers: 1 groupname: clickhouse enginesettings: [] resolutions: - interval: 0s - ttl: 360h0m0s + ttl: 360h0m0s - interval: 1m0s - ttl: 168h0m0s + ttl: 168h0m0s - interval: 5m0s - ttl: 2160h0m0s + ttl: 2160h0m0s - interval: 1h0m0s - ttl: 8640h0m0s + ttl: 8640h0m0s maxpartitions: 50 systemlogttl: 720h0m0s - prometheusendpoint: /metrics + prometheusendpoint: "/metrics" asns: 25037: Dreamhack ACME Corporation networks: @@ -77,12 +77,12 @@ clickhouse: <% end -%> networksources: {} networksourcestimeout: 10s - orchestratorurl: http://localhost:8080 + orchestratorurl: "http://localhost:8080" kafka: topic: flows brokers: - - localhost:9092 - version: 3.3.1 + - 127.0.0.1:9092 + version: 3.7.0 tls: enable: false verify: true @@ -106,7 +106,7 @@ geoip: - /usr/share/GeoIP/asn.mmdb geodatabase: - /usr/share/GeoIP/country.mmdb - optional: true + optional: false schema: disabled: [] enabled: [] @@ -116,186 +116,185 @@ schema: customdictionaries: {} inlet: - reporting: - logging: {} - metrics: {} - http: - listen: :8081 - profiler: true - cache: - type: memory - flow: - inputs: - - decoder: netflow - listen: :2055 - queuesize: 100000 - receivebuffer: 10485760 - timestampsource: udp - type: udp - usesrcaddrforexporteraddr: false - workers: 6 - - decoder: sflow - listen: :6343 - queuesize: 100000 - receivebuffer: 10485760 - timestampsource: udp - type: udp - usesrcaddrforexporteraddr: false - workers: 6 - ratelimit: 0 - metadata: - cacheduration: 30m0s - cacherefresh: 1h0m0s - cachecheckinterval: 2m0s - cachepersistfile: "" - providers: - - agents: {} - communities: - ::/0: - - public - pollerretries: 1 - pollertimeout: 1s - ports: - ::/0: 161 - securityparameters: {} - type: snmp - workers: 10 - maxbatchrequests: 10 - routing: - provider: - collectasns: true - collectaspaths: true - collectcommunities: true - keep: 5m0s - listen: :10179 - rds: [] - ribpeerremovalbatchroutes: 5000 - ribpeerremovalmaxqueue: 10000 - ribpeerremovalmaxtime: 100ms - ribpeerremovalsleepinterval: 500ms - type: bmp - kafka: - topic: flows - brokers: - - localhost:9092 - version: 3.3.1 - tls: - enable: false - verify: true - cafile: "" - certfile: "" - keyfile: "" - saslusername: "" - saslpassword: "" - saslmechanism: none - flushinterval: 10s - flushbytes: 104857599 - maxmessagebytes: 1000000 - compressioncodec: zstd - queuesize: 32 - core: - workers: 6 - exporterclassifiers: - - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") - - ClassifyRegion("europe") - - ClassifyTenant("acme") - - ClassifyRole("edge") - interfaceclassifiers: - - | - ClassifyConnectivityRegex(Interface.Description, "^(?i)(transit|pni|ppni|ix):? ", "$1") && - ClassifyProviderRegex(Interface.Description, "^\\S+?\\s(\\S+)", "$1") && - ClassifyExternal() - - ClassifyInternal() - classifiercacheduration: 5m0s - defaultsamplingrate: {} - overridesamplingrate: {} - asnproviders: - - flow - - routing - netproviders: - - flow - - routing - schema: - disabled: [] - enabled: [] - maintableonly: [] - notmaintableonly: [] - materialize: [] - customdictionaries: {} + logging: {} + metrics: {} + http: + listen: :8081 + profiler: true + cache: + type: memory + flow: + inputs: + - decoder: netflow + listen: :2055 + queuesize: 100000 + receivebuffer: 10485760 + timestampsource: udp + type: udp + usesrcaddrforexporteraddr: false + workers: 6 + - decoder: sflow + listen: :6343 + queuesize: 100000 + receivebuffer: 10485760 + timestampsource: udp + type: udp + usesrcaddrforexporteraddr: false + workers: 6 + ratelimit: 0 + metadata: + cacheduration: 30m0s + cacherefresh: 1h0m0s + cachecheckinterval: 2m0s + cachepersistfile: "" + providers: + - agents: {} + communities: + ::/0: + - public + pollerretries: 1 + pollertimeout: 1s + ports: + ::/0: 161 + securityparameters: {} + type: snmp + workers: 1 + maxbatchrequests: 10 + routing: + provider: + collectasns: true + collectaspaths: true + collectcommunities: true + keep: 5m0s + listen: :10179 + rds: [] + ribpeerremovalbatchroutes: 5000 + ribpeerremovalmaxqueue: 10000 + ribpeerremovalmaxtime: 100ms + ribpeerremovalsleepinterval: 500ms + type: bmp + kafka: + topic: flows + brokers: + - 127.0.0.1:9092 + version: 3.7.0 + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + saslusername: "" + saslpassword: "" + saslmechanism: none + flushinterval: 10s + flushbytes: 104857599 + maxmessagebytes: 1000000 + compressioncodec: zstd + queuesize: 32 + core: + workers: 6 + exporterclassifiers: + - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") + - ClassifyRegion("europe") + - ClassifyTenant("acme") + - ClassifyRole("edge") + interfaceclassifiers: + - | + ClassifyConnectivityRegex(Interface.Description, "^(?i)(transit|pni|ppni|ix):? ", "$1") && + ClassifyProviderRegex(Interface.Description, "^\\S+?\\s(\\S+)", "$1") && + ClassifyExternal() + - ClassifyInternal() + classifiercacheduration: 5m0s + defaultsamplingrate: {} + overridesamplingrate: {} + asnproviders: + - flow + - routing + netproviders: + - flow + - routing + schema: + disabled: [] + enabled: [] + maintableonly: [] + notmaintableonly: [] + materialize: [] + customdictionaries: {} console: - reporting: - logging: {} - metrics: {} - http: - listen: :8082 - profiler: true - cache: - db: 0 + logging: {} + metrics: {} + http: + listen: :8082 + profiler: true + cache: + db: 0 + password: "" + protocol: tcp + server: localhost:6379 + type: redis + username: "" + defaultvisualizeoptions: + graphtype: stacked + start: 6 hours ago + end: now + filter: InIfBoundary = external + dimensions: + - SrcAS + limit: 10 + homepagetopwidgets: + - src-as + - src-port + - protocol + - src-country + - etype + homepagegraphfilter: InIfBoundary = 'external' + dimensionslimit: 50 + cachettl: 3h0m0s + clickhouse: + servers: + - 127.0.0.1:9000 + cluster: "" + database: default + username: default password: "" - protocol: tcp - server: localhost:6379 - type: redis - username: "" - defaultvisualizeoptions: - graphtype: stacked - start: 6 hours ago - end: now - filter: InIfBoundary = external - dimensions: - - SrcAS - limit: 10 - homepagetopwidgets: - - src-as - - src-port - - protocol - - src-country - - etype - homepagegraphfilter: InIfBoundary = 'external' - dimensionslimit: 50 - cachettl: 3h0m0s - clickhouse: - servers: - - localhost:9000 - cluster: "" - database: default - username: default - password: "" - maxopenconns: 10 - dialtimeout: 5s - tls: - enable: false - verify: true - cafile: "" - certfile: "" - keyfile: "" - auth: - headers: - login: Remote-User - name: Remote-Name - email: Remote-Email - logouturl: X-Logout-URL - defaultuser: - login: __default - name: Default User - email: "" - logouturl: "" - database: - driver: sqlite - dsn: file::memory:?cache=shared - savedfilters: - - description: From Netflix - content: InIfBoundary = external AND SrcAS = AS2906 - - description: From GAFAM - content: InIfBoundary = external AND SrcAS IN (AS15169, AS16509, AS32934, AS6185, AS8075) - - description: From Swedish Armed Forces - content: InIfBoundary = external AND SrcAS = AS9201 - - description: Valve Corporation - content: InIfBoundary = external AND SrcAS = AS32590 - - - schema: - disabled: [] - enabled: [] - maintableonly: [] - notmaintableonly: [] - materialize: [] - customdictionaries: {} + maxopenconns: 10 + dialtimeout: 5s + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + auth: + headers: + login: Remote-User + name: Remote-Name + email: Remote-Email + logouturl: X-Logout-URL + defaultuser: + login: default + name: Default User + email: "" + logouturl: "" + database: + driver: sqlite + dsn: file::memory:?cache=shared + savedfilters: + - description: From Netflix + content: InIfBoundary = external AND SrcAS = AS2906 + - description: From GAFAM + content: InIfBoundary = external AND SrcAS IN (AS15169, AS16509, AS32934, AS6185, AS8075) + - description: From Swedish Armed Forces + content: InIfBoundary = external AND SrcAS = AS9201 + - description: Valve Corporation + content: InIfBoundary = external AND SrcAS = AS32590 + schema: + disabled: [] + enabled: [] + maintableonly: [] + notmaintableonly: [] + materialize: [] + customdictionaries: {} +demoexporter: [] From c1bd203cc7525c289d9bdec1504bf7eb25159c04 Mon Sep 17 00:00:00 2001 From: furest Date: Mon, 17 Jun 2024 02:00:16 +0200 Subject: [PATCH 082/107] fix service kafka --- modules/akvorado/files/kafka.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/files/kafka.service b/modules/akvorado/files/kafka.service index 64d7b10e..b1b0ebbd 100644 --- a/modules/akvorado/files/kafka.service +++ b/modules/akvorado/files/kafka.service @@ -5,7 +5,7 @@ After=zookeeper.service [Service] Type=simple User=kafka -ExecStart=/bin/sh -c '/home/kafka/kafka/bin/kafka-server-start.sh /home/kafka/kafka/config/server.properties > /var/log/kafka/kafka.log 2>&1' +ExecStart=/bin/sh -c ' /var/lib/kafka/bin/kafka-server-start.sh /var/lib/kafka/config/server.properties > /var/log/kafka/kafka.log 2>&1' ExecStop=/var/lib/kafka/bin/kafka-server-stop.sh Restart=on-abnormal From 53c18fbcc94b9e8d08d5640b0252653aec3d7ddb Mon Sep 17 00:00:00 2001 From: furest Date: Mon, 17 Jun 2024 02:28:41 +0200 Subject: [PATCH 083/107] fixes --- modules/akvorado.py | 1 + modules/akvorado/manifests/init.pp | 2 +- modules/akvorado/templates/akvorado.yaml.erb | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/akvorado.py b/modules/akvorado.py index f8a21dd9..85f03acd 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -37,6 +37,7 @@ def get_prefixes(ipversion): ' FROM network' ' WHERE node_id NOT IN (SELECT option.node_id FROM option WHERE name = "no-akv")' ' AND name LIKE "%@%" AND ipv6_txt IS NOT NULL' + ' AND NOT (name = "BOGAL@DREAMHACK" AND ipv6_txt = "2a05:2240:5000::/48")' ) else: raise NetworkTypeNotFoundError('network type must be 4 or 6') diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 350858ad..1a79e72d 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -103,7 +103,7 @@ source => 'puppet:///data/kafka-latest.tgz', notify => Exec[ 'untar-kafka' ], } - -> file { '/var/log/kafka': + file { '/var/log/kafka': ensure => 'directory', owner => 'kafka', group => 'kafka', diff --git a/modules/akvorado/templates/akvorado.yaml.erb b/modules/akvorado/templates/akvorado.yaml.erb index 0660ed63..dbe728f7 100644 --- a/modules/akvorado/templates/akvorado.yaml.erb +++ b/modules/akvorado/templates/akvorado.yaml.erb @@ -13,7 +13,7 @@ clickhouse: cluster: "" database: default username: default - password: "dhtech" + password: "" maxopenconns: 10 dialtimeout: 5s tls: From 12cecd9a341d06265cc68f298d8d0448f55ceb05 Mon Sep 17 00:00:00 2001 From: furest Date: Mon, 17 Jun 2024 02:42:39 +0200 Subject: [PATCH 084/107] Fix order --- modules/akvorado/manifests/init.pp | 47 ++++++++++++++++-------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 1a79e72d..6c871ca2 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -101,9 +101,8 @@ ensure => file, links => follow, source => 'puppet:///data/kafka-latest.tgz', - notify => Exec[ 'untar-kafka' ], } - file { '/var/log/kafka': + -> file { '/var/log/kafka': ensure => 'directory', owner => 'kafka', group => 'kafka', @@ -126,32 +125,36 @@ notify => [ Exec['systemctl-daemon-reload'], Service['zookeeper'] ], } -> file_line { 'kafka-enabledeletetopics': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'delete.topic.enable = true', - match => 'delete.topic.enable', - notify => Service['kafka'], + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'delete.topic.enable = true', + match => 'delete.topic.enable', + notify => Service['kafka'], + require => Exec['untar-kafka'], } -> file_line { 'kafka-listenlocalhost': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'listeners=PLAINTEXT://localhost:9092', - match => '#listeners=PLAINTEXT', - notify => Service['kafka'], + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'listeners=PLAINTEXT://localhost:9092', + match => '#listeners=PLAINTEXT', + notify => Service['kafka'], + require => Exec['untar-kafka'], } -> file_line { 'kafka-logdir': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'log.dirs=/var/log/kafka', - match => 'log.dirs=', - notify => Service['kafka'], + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'log.dirs=/var/log/kafka', + match => 'log.dirs=', + notify => Service['kafka'], + require => Exec['untar-kafka'], } -> file_line { 'zookeeper-listen': - ensure => 'present', - path => '/var/lib/kafka/config/zookeeper.properties', - line => 'clientPortAddress=127.0.0.1', - match => 'clientPortAddress=', - notify => Service['zookeeper'], + ensure => 'present', + path => '/var/lib/kafka/config/zookeeper.properties', + line => 'clientPortAddress=127.0.0.1', + match => 'clientPortAddress=', + notify => Service['zookeeper'], + require => Exec['untar-kafka'], } exec { 'untar-kafka': command => '/bin/tar -xvf /tmp/kafka.tgz -C /var/lib/kafka --strip 1', From ba34350f9b955a61eefbebd7449e50db385875f9 Mon Sep 17 00:00:00 2001 From: furest Date: Sat, 22 Jun 2024 11:08:27 +0200 Subject: [PATCH 085/107] Major improvements and fixes --- modules/akvorado.py | 64 +++++ modules/akvorado/manifests/init.pp | 240 +++++++++++-------- modules/akvorado/templates/akvorado.yaml.erb | 45 +++- 3 files changed, 245 insertions(+), 104 deletions(-) diff --git a/modules/akvorado.py b/modules/akvorado.py index 85f03acd..50f02356 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -10,6 +10,68 @@ DB_FILE = '/etc/ipplan.db' +def get_sflow_clients(): + if os.path.isfile(DB_FILE): + try: + conn = sqlite3.connect(DB_FILE) + db = conn.cursor() + except sqlite3.Error as e: + print("An error occurred: {}".format(e.args[0])) + exit(2) + else: + print("No database file found: {}".format(DB_FILE)) + exit(3) + db.execute( + "SELECT h.name AS hostname, h.ipv4_addr_txt AS ipv4_addr ,h.ipv6_addr_txt AS ipv6_addr, o2.value AS layer " + "FROM host h " + "INNER JOIN option o1 ON h.node_id = o1.node_id " + "INNER JOIN option o2 ON h.node_id = o2.node_id " + "WHERE o1.name='pkg' AND o1.value='sflowclient' " + "AND o2.name='layer'" + ) + res = db.fetchall() + if not res: + return None + + column_names = [description[0] for description in db.description] + conn.close() + rows_dict = [dict(zip(column_names, row)) for row in res] + + return rows_dict + +def get_snmpv2_providers(): + providers = [] + clients = get_sflow_clients() + current_event = lib.get_current_event() + for client in clients: + key = current_event+'-mgmt/snmp:'+client['layer'] + secrets = lib.read_secret(key) + if "community" in secrets: + provider = { + "ipv4": client["ipv4_addr"], + "community": secrets["community"], + } + providers.append(provider) + return providers + +def get_snmpv3_providers(): + providers = [] + clients = get_sflow_clients() + current_event = lib.get_current_event() + for client in clients: + key = current_event+'-mgmt/snmp:'+client['layer'] + secrets = lib.read_secret(key) + if "user" in secrets: + provider = { + "ipv4": client["ipv4_addr"], + "authentication-passphrase": secrets["auth"], + "authentication-protocol": secrets["authtype"].replace(" ","").upper(), + "privacy-passphrase": secrets["priv"], + "privacy-protocol": secrets["privtype"].replace(" ","").replace("128","").upper(), + "user": secrets["user"], + } + providers.append(provider) + return providers def get_prefixes(ipversion): if os.path.isfile(DB_FILE): @@ -60,6 +122,8 @@ def requires(host, *args): def generate(host, *args): info = {} + info['snmpv3_providers'] = get_snmpv3_providers() + info['snmpv2_providers'] = get_snmpv2_providers() info['current_event'] = lib.get_current_event() info['ipv6_prefixes'] = get_prefixes('6') info['ipv4_prefixes'] = get_prefixes('4') diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 6c871ca2..e3d6def5 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -10,84 +10,13 @@ # === Parameters # -class akvorado ($current_event, $ipv4_prefixes, $ipv6_prefixes) { +class akvorado ($current_event, $ipv4_prefixes, $ipv6_prefixes, $snmpv3_providers, $snmpv2_providers) { - #Create user/group for Akvorodo - group { 'akvorado': - ensure => 'present', - } - -> user { 'akvorado': - ensure => 'present', - system => true, - } - #Create directories for akvorado - -> file { '/etc/akvorado': - ensure => 'directory', - owner => 'root', - group => 'akvorado', - mode => '0750', - } - #Copy akvorado to the server - -> file { '/usr/local/bin/akvorado': - ensure => file, - owner => 'root', - group => 'akvorado', - mode => '0550', - links => follow, - source => 'puppet:///data/akvorado-latest', - } - - file { '/etc/akvorado/akvorado.yaml': - ensure => file, - content => template('akvorado/akvorado.yaml.erb'), - notify => Service['akvorado-orch'], - } - #Systemctl config - file { '/etc/systemd/system/akvorado-orch.service': - ensure => present, - source => 'puppet:///modules/akvorado/akvorado-orch.service', - mode => '0644', - owner => 'root', - group => 'root', - notify => [Exec['systemctl-daemon-reload'],Service['akvorado-orch']], - } - file { '/etc/systemd/system/akvorado-inlet.service': - ensure => present, - source => 'puppet:///modules/akvorado/akvorado-inlet.service', - mode => '0644', - owner => 'root', - group => 'root', - notify => [Exec['systemctl-daemon-reload'],Service['akvorado-inlet']], - } - file { '/etc/systemd/system/akvorado-console.service': - ensure => present, - source => 'puppet:///modules/akvorado/akvorado-console.service', - mode => '0644', - owner => 'root', - group => 'root', - notify => [Exec['systemctl-daemon-reload'],Service['akvorado-console']], - } - - -> apache::proxy { 'akvorado': - url => '/', - backend => 'http://localhost:8082/', - } - -> service { 'akvorado-orch': - ensure => running, - } - -> service { 'akvorado-inlet': - ensure => running, - } - -> service { 'akvorado-console': - ensure => running, - } - - exec { 'systemctl-daemon-reload': - command => '/bin/systemctl daemon-reload', - refreshonly => true, - } ##Kafka installation + ensure_packages([ + 'openjdk-17-jre', + ]) group { 'kafka': ensure => 'present', } @@ -97,10 +26,11 @@ home => '/var/lib/kafka', managehome => true, } - -> file { '/tmp/kafka.tgz': - ensure => file, - links => follow, - source => 'puppet:///data/kafka-latest.tgz', + -> file { '/var/lib/kafka/kafka.tgz': + ensure => file, + links => follow, + source => 'puppet:///data/kafka-latest.tgz', + notify => Exec['untar-kafka'] } -> file { '/var/log/kafka': ensure => 'directory', @@ -108,7 +38,18 @@ group => 'kafka', mode => '0700', } - -> file { '/etc/systemd/system/kafka.service': + -> file { '/var/lib/zookeeper-data': + ensure => 'directory', + owner => 'kafka', + group => 'kafka', + mode => '0700', + } + exec { 'untar-kafka': + command => '/bin/tar -xvf /var/lib/kafka/kafka.tgz -C /var/lib/kafka --strip 1', + refreshonly => 'true', + user => 'kafka', + } + file { '/etc/systemd/system/kafka.service': ensure => present, source => 'puppet:///modules/akvorado/kafka.service', mode => '0644', @@ -128,9 +69,7 @@ ensure => 'present', path => '/var/lib/kafka/config/server.properties', line => 'delete.topic.enable = true', - match => 'delete.topic.enable', notify => Service['kafka'], - require => Exec['untar-kafka'], } -> file_line { 'kafka-listenlocalhost': ensure => 'present', @@ -138,34 +77,34 @@ line => 'listeners=PLAINTEXT://localhost:9092', match => '#listeners=PLAINTEXT', notify => Service['kafka'], - require => Exec['untar-kafka'], } -> file_line { 'kafka-logdir': ensure => 'present', path => '/var/lib/kafka/config/server.properties', line => 'log.dirs=/var/log/kafka', - match => 'log.dirs=', + match => 'log.dirs=/tmp/kafka-logs', notify => Service['kafka'], - require => Exec['untar-kafka'], + } + -> file_line { 'zookeeper-datadir': + ensure => 'present', + path => '/var/lib/kafka/config/zookeeper.properties', + line => 'dataDir=/var/lib/zookeeper-data', + match => 'dataDir=/tmp/zookeeper', + notify => Service['zookeeper'], } -> file_line { 'zookeeper-listen': ensure => 'present', path => '/var/lib/kafka/config/zookeeper.properties', line => 'clientPortAddress=127.0.0.1', - match => 'clientPortAddress=', notify => Service['zookeeper'], - require => Exec['untar-kafka'], - } - exec { 'untar-kafka': - command => '/bin/tar -xvf /tmp/kafka.tgz -C /var/lib/kafka --strip 1', - refreshonly => true, - user => 'kafka', } - -> service { 'kafka': + service { 'kafka': ensure => running, + enable => true, } - -> service { 'zookeeper': + service { 'zookeeper': ensure => running, + enable => true, } ##Clickhouse installation @@ -182,7 +121,7 @@ notify => Exec['clickhouse-source-key'], } exec { 'clickhouse-source-key': - command => '/usr/bin/wget -fsSL https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key -O /usr/share/keyrings/clickhouse-keyring.gpg', + command => '/usr/bin/curl -fsSL https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key | gpg --dearmor > /usr/share/keyrings/clickhouse-keyring.gpg', logoutput => 'on_failure', try_sleep => 1, refreshonly => true, @@ -205,5 +144,116 @@ } -> service { 'clickhouse-server': ensure => running, + enable => true, + } + + #Create user/group for Akvorodo + ensure_packages([ + 'redis', + ]) + group { 'akvorado': + ensure => 'present', + } + -> user { 'akvorado': + ensure => 'present', + system => true, + home => '/var/lib/akvorado', + managehome => true, + } + #Create directories for akvorado + -> file { '/etc/akvorado': + ensure => 'directory', + owner => 'root', + group => 'akvorado', + mode => '0750', + } + #Copy akvorado to the server + -> file { '/usr/local/bin/akvorado': + ensure => file, + owner => 'root', + group => 'akvorado', + mode => '0550', + links => follow, + source => 'puppet:///data/akvorado-latest', + notify => [Service['akvorado-orch'],Exec['protobuf-schema']] + } + file { '/etc/akvorado/akvorado.yaml': + ensure => file, + content => template('akvorado/akvorado.yaml.erb'), + notify => Service['akvorado-orch'], + } + #Systemctl config + file { '/etc/systemd/system/akvorado-orch.service': + ensure => present, + source => 'puppet:///modules/akvorado/akvorado-orch.service', + mode => '0644', + owner => 'root', + group => 'root', + notify => [Exec['systemctl-daemon-reload'],Service['akvorado-orch']], + } + file { '/etc/systemd/system/akvorado-inlet.service': + ensure => present, + source => 'puppet:///modules/akvorado/akvorado-inlet.service', + mode => '0644', + owner => 'root', + group => 'root', + notify => [Exec['systemctl-daemon-reload'],Service['akvorado-inlet']], + } + file { '/etc/systemd/system/akvorado-console.service': + ensure => present, + source => 'puppet:///modules/akvorado/akvorado-console.service', + mode => '0644', + owner => 'root', + group => 'root', + notify => [Exec['systemctl-daemon-reload'],Service['akvorado-console']], + } + file { '/usr/share/GeoIP': + ensure => 'directory', + owner => 'root', + group => 'root', + mode => '0755', + } + file { '/usr/share/GeoIP/asn.mmdb': + ensure => present, + source => 'puppet:///data/asn.mmdb', + mode => '0644', + owner => 'root', + group => 'root', + } + file { '/usr/share/GeoIP/country.mmdb': + ensure => present, + source => 'puppet:///data/country.mmdb', + mode => '0644', + owner => 'root', + group => 'root', + } + apache::proxy { 'akvorado': + url => '/', + backend => 'http://localhost:8082/', + } + -> service { 'akvorado-orch': + ensure => running, + enable => true, + } + -> service { 'akvorado-inlet': + ensure => running, + enable => true, + } + -> service { 'akvorado-console': + ensure => running, + enable => true, + } + -> service { 'redis': + ensure => running, + enable => true, + } + exec { 'systemctl-daemon-reload': + command => '/bin/systemctl daemon-reload', + refreshonly => true, + } + exec { 'protobuf-schema': + command => '/usr/bin/curl http://127.0.0.1:8080/api/v0/orchestrator/clickhouse/init.sh | sh', + refreshonly => true, + require => Service['akvorado-orch'] } } diff --git a/modules/akvorado/templates/akvorado.yaml.erb b/modules/akvorado/templates/akvorado.yaml.erb index dbe728f7..21030595 100644 --- a/modules/akvorado/templates/akvorado.yaml.erb +++ b/modules/akvorado/templates/akvorado.yaml.erb @@ -1,4 +1,6 @@ --- +# AUTOGENERATED BY PUPPET +# All manual changes will be overwritten reporting: logging: {} metrics: {} @@ -148,6 +150,32 @@ inlet: cachecheckinterval: 2m0s cachepersistfile: "" providers: +<% unless @snmpv2_providers.empty? -%> + - agents: {} + communities: +<% @snmpv2_providers.each do |provider| -%> + <%=provider['ipv4']%>: <%=provider['community']%> +<% end -%> + ports: + ::/0: 161 + securityparameters: {} + type: snmp +<% end -%> +<% unless @snmpv3_providers.empty? -%> + - agents: {} + ports: + ::/0: 161 + securityparameters: +<% @snmpv3_providers.each do |provider| -%> + <%=provider['ipv4']%>: + user-name: <%=provider['user']%> + authentication-protocol: <%=provider['authentication-protocol']%> + authentication-passphrase: <%=provider['authentication-passphrase']%> + privacy-protocol: <%=provider['privacy-protocol']%> + privacy-passphrase: <%=provider['privacy-passphrase']%> +<% end -%> + type: snmp +<% end -%> - agents: {} communities: ::/0: @@ -197,7 +225,6 @@ inlet: exporterclassifiers: - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") - ClassifyRegion("europe") - - ClassifyTenant("acme") - ClassifyRole("edge") interfaceclassifiers: - | @@ -206,7 +233,7 @@ inlet: ClassifyExternal() - ClassifyInternal() classifiercacheduration: 5m0s - defaultsamplingrate: {} + defaultsamplingrate: 1 overridesamplingrate: {} asnproviders: - flow @@ -239,7 +266,7 @@ console: graphtype: stacked start: 6 hours ago end: now - filter: InIfBoundary = external + filter: "" dimensions: - SrcAS limit: 10 @@ -249,9 +276,9 @@ console: - protocol - src-country - etype - homepagegraphfilter: InIfBoundary = 'external' + homepagegraphfilter: "" dimensionslimit: 50 - cachettl: 3h0m0s + cachettl: 0h10m0s clickhouse: servers: - 127.0.0.1:9000 @@ -269,18 +296,18 @@ console: keyfile: "" auth: headers: - login: Remote-User + login: X-Proxy-REMOTE-USER name: Remote-Name email: Remote-Email logouturl: X-Logout-URL defaultuser: - login: default - name: Default User + login: "" + name: "" email: "" logouturl: "" database: driver: sqlite - dsn: file::memory:?cache=shared + dsn: /var/lib/akvorado/console.sqlite savedfilters: - description: From Netflix content: InIfBoundary = external AND SrcAS = AS2906 From 29fef7c2ab9ed7279c64fdcb1be8a41dddfd9f32 Mon Sep 17 00:00:00 2001 From: furest Date: Sat, 22 Jun 2024 11:14:56 +0200 Subject: [PATCH 086/107] Fix manifest indent to make circleci happy again --- modules/akvorado/manifests/init.pp | 88 +++++++++++++++--------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index e3d6def5..09e11212 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -27,10 +27,10 @@ managehome => true, } -> file { '/var/lib/kafka/kafka.tgz': - ensure => file, - links => follow, - source => 'puppet:///data/kafka-latest.tgz', - notify => Exec['untar-kafka'] + ensure => file, + links => follow, + source => 'puppet:///data/kafka-latest.tgz', + notify => Exec['untar-kafka'] } -> file { '/var/log/kafka': ensure => 'directory', @@ -66,45 +66,45 @@ notify => [ Exec['systemctl-daemon-reload'], Service['zookeeper'] ], } -> file_line { 'kafka-enabledeletetopics': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'delete.topic.enable = true', - notify => Service['kafka'], + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'delete.topic.enable = true', + notify => Service['kafka'], } -> file_line { 'kafka-listenlocalhost': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'listeners=PLAINTEXT://localhost:9092', - match => '#listeners=PLAINTEXT', - notify => Service['kafka'], + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'listeners=PLAINTEXT://localhost:9092', + match => '#listeners=PLAINTEXT', + notify => Service['kafka'], } -> file_line { 'kafka-logdir': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'log.dirs=/var/log/kafka', - match => 'log.dirs=/tmp/kafka-logs', - notify => Service['kafka'], + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'log.dirs=/var/log/kafka', + match => 'log.dirs=/tmp/kafka-logs', + notify => Service['kafka'], } -> file_line { 'zookeeper-datadir': - ensure => 'present', - path => '/var/lib/kafka/config/zookeeper.properties', - line => 'dataDir=/var/lib/zookeeper-data', - match => 'dataDir=/tmp/zookeeper', - notify => Service['zookeeper'], + ensure => 'present', + path => '/var/lib/kafka/config/zookeeper.properties', + line => 'dataDir=/var/lib/zookeeper-data', + match => 'dataDir=/tmp/zookeeper', + notify => Service['zookeeper'], } -> file_line { 'zookeeper-listen': - ensure => 'present', - path => '/var/lib/kafka/config/zookeeper.properties', - line => 'clientPortAddress=127.0.0.1', - notify => Service['zookeeper'], + ensure => 'present', + path => '/var/lib/kafka/config/zookeeper.properties', + line => 'clientPortAddress=127.0.0.1', + notify => Service['zookeeper'], } service { 'kafka': - ensure => running, - enable => true, + ensure => running, + enable => true, } service { 'zookeeper': - ensure => running, - enable => true, + ensure => running, + enable => true, } ##Clickhouse installation @@ -140,11 +140,11 @@ require => [File['clickhouse-source-add'], Exec['clickhouse-source-key'], Exec['apt-update']], } -> package { 'clickhouse-client': - ensure => installed, + ensure => installed, } -> service { 'clickhouse-server': - ensure => running, - enable => true, + ensure => running, + enable => true, } #Create user/group for Akvorodo @@ -155,8 +155,8 @@ ensure => 'present', } -> user { 'akvorado': - ensure => 'present', - system => true, + ensure => 'present', + system => true, home => '/var/lib/akvorado', managehome => true, } @@ -232,20 +232,20 @@ backend => 'http://localhost:8082/', } -> service { 'akvorado-orch': - ensure => running, - enable => true, + ensure => running, + enable => true, } -> service { 'akvorado-inlet': - ensure => running, - enable => true, + ensure => running, + enable => true, } -> service { 'akvorado-console': - ensure => running, - enable => true, + ensure => running, + enable => true, } -> service { 'redis': - ensure => running, - enable => true, + ensure => running, + enable => true, } exec { 'systemctl-daemon-reload': command => '/bin/systemctl daemon-reload', From 2d9738349b28fde6c6b7ad0573957a7cd4f7585d Mon Sep 17 00:00:00 2001 From: furest Date: Sat, 22 Jun 2024 11:19:06 +0200 Subject: [PATCH 087/107] Update init.pp --- modules/akvorado/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 09e11212..2c095d2a 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -46,7 +46,7 @@ } exec { 'untar-kafka': command => '/bin/tar -xvf /var/lib/kafka/kafka.tgz -C /var/lib/kafka --strip 1', - refreshonly => 'true', + refreshonly => true, user => 'kafka', } file { '/etc/systemd/system/kafka.service': From 11f450fa85570a658df987bbe6acd1facc0c172d Mon Sep 17 00:00:00 2001 From: furest Date: Tue, 25 Jun 2024 22:32:43 +0200 Subject: [PATCH 088/107] Fix proxy + improvements --- modules/akvorado.py | 2 +- modules/akvorado/manifests/init.pp | 39 ++++++++++++++++---- modules/akvorado/templates/akvorado.yaml.erb | 6 +-- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/modules/akvorado.py b/modules/akvorado.py index 50f02356..bc72dd1d 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -27,7 +27,7 @@ def get_sflow_clients(): "INNER JOIN option o1 ON h.node_id = o1.node_id " "INNER JOIN option o2 ON h.node_id = o2.node_id " "WHERE o1.name='pkg' AND o1.value='sflowclient' " - "AND o2.name='layer'" + "AND o2.name='layer';" ) res = db.fetchall() if not res: diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 2c095d2a..e3ad37fc 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -150,13 +150,16 @@ #Create user/group for Akvorodo ensure_packages([ 'redis', - ]) + ],{ + ensure => 'present', + notify => Service['redis'], + }) group { 'akvorado': ensure => 'present', } -> user { 'akvorado': - ensure => 'present', - system => true, + ensure => 'present', + system => true, home => '/var/lib/akvorado', managehome => true, } @@ -227,23 +230,43 @@ owner => 'root', group => 'root', } - apache::proxy { 'akvorado': + apache::proxy { '1_akvorado-orch-api': + url => '/api/v0/orchestrator/', + backend => 'http://localhost:8080/api/v0/orchestrator/', + } + apache::proxy { '2_akvorado-inlet-api': + url => '/api/v0/inlet/', + backend => 'http://localhost:8081/api/v0/inlet/', + } + apache::proxy { '3_akvorado-console': url => '/', backend => 'http://localhost:8082/', } - -> service { 'akvorado-orch': + # By default apache answers with status code 404 when an URL contains an encoded slash (%2F) + # The following allows apache to simply forward the request to the prox backend. + file { '/etc/apache2/conf-available/allow-slashes.conf': + content => 'AllowEncodedSlashes On', + ensure => present, + mode => '0644', + } + -> file { '/etc/apache2/conf-enabled/allow-slashes.conf': + ensure => link, + mode => '0644', + target => '/etc/apache2/conf-available/allow-slashes.conf', + } + service { 'akvorado-orch': ensure => running, enable => true, } - -> service { 'akvorado-inlet': + service { 'akvorado-inlet': ensure => running, enable => true, } - -> service { 'akvorado-console': + service { 'akvorado-console': ensure => running, enable => true, } - -> service { 'redis': + service { 'redis': ensure => running, enable => true, } diff --git a/modules/akvorado/templates/akvorado.yaml.erb b/modules/akvorado/templates/akvorado.yaml.erb index 21030595..3c29da73 100644 --- a/modules/akvorado/templates/akvorado.yaml.erb +++ b/modules/akvorado/templates/akvorado.yaml.erb @@ -297,9 +297,9 @@ console: auth: headers: login: X-Proxy-REMOTE-USER - name: Remote-Name - email: Remote-Email - logouturl: X-Logout-URL + name: "" + email: "" + logouturl: "" defaultuser: login: "" name: "" From b32d1b0171d980400b83c9d25f075b8bcb66fc5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Tue, 19 Nov 2024 21:07:52 +0100 Subject: [PATCH 089/107] allow empty list --- modules/akvorado.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/akvorado.py b/modules/akvorado.py index bc72dd1d..123f8b1b 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -42,6 +42,8 @@ def get_sflow_clients(): def get_snmpv2_providers(): providers = [] clients = get_sflow_clients() + if not clients: + return providers current_event = lib.get_current_event() for client in clients: key = current_event+'-mgmt/snmp:'+client['layer'] @@ -57,6 +59,8 @@ def get_snmpv2_providers(): def get_snmpv3_providers(): providers = [] clients = get_sflow_clients() + if not clients: + return providers current_event = lib.get_current_event() for client in clients: key = current_event+'-mgmt/snmp:'+client['layer'] From 321151cce0e5d9861478be49d1acbf0d7b5f0136 Mon Sep 17 00:00:00 2001 From: furest Date: Tue, 19 Nov 2024 23:13:06 +0100 Subject: [PATCH 090/107] update snmp provider discovery --- modules/akvorado.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/akvorado.py b/modules/akvorado.py index bc72dd1d..6eb19c86 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -44,7 +44,7 @@ def get_snmpv2_providers(): clients = get_sflow_clients() current_event = lib.get_current_event() for client in clients: - key = current_event+'-mgmt/snmp:'+client['layer'] + key = current_event+'-mgmt/snmpv2:'+client['layer'] secrets = lib.read_secret(key) if "community" in secrets: provider = { @@ -59,7 +59,7 @@ def get_snmpv3_providers(): clients = get_sflow_clients() current_event = lib.get_current_event() for client in clients: - key = current_event+'-mgmt/snmp:'+client['layer'] + key = current_event+'-mgmt/snmpv3:'+client['layer'] secrets = lib.read_secret(key) if "user" in secrets: provider = { From 28b0b668e70e54764658741ba1ee5814ecdf75ef Mon Sep 17 00:00:00 2001 From: furest Date: Tue, 19 Nov 2024 23:17:17 +0100 Subject: [PATCH 091/107] fix indent --- modules/akvorado/manifests/init.pp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index e3ad37fc..427da310 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -156,10 +156,10 @@ }) group { 'akvorado': ensure => 'present', - } + } -> user { 'akvorado': - ensure => 'present', - system => true, + ensure => 'present', + system => true, home => '/var/lib/akvorado', managehome => true, } From 0c5dbfe23814205bffc2bfdb4fc4c35c0719c937 Mon Sep 17 00:00:00 2001 From: furest Date: Tue, 19 Nov 2024 23:19:22 +0100 Subject: [PATCH 092/107] fix ensure order --- modules/akvorado/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index 427da310..dcac6d0a 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -245,8 +245,8 @@ # By default apache answers with status code 404 when an URL contains an encoded slash (%2F) # The following allows apache to simply forward the request to the prox backend. file { '/etc/apache2/conf-available/allow-slashes.conf': - content => 'AllowEncodedSlashes On', ensure => present, + content => 'AllowEncodedSlashes On', mode => '0644', } -> file { '/etc/apache2/conf-enabled/allow-slashes.conf': From 7acf72377750469683d711288036101a7dba4030 Mon Sep 17 00:00:00 2001 From: furest Date: Tue, 19 Nov 2024 23:24:57 +0100 Subject: [PATCH 093/107] fix bug --- modules/akvorado.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/akvorado.py b/modules/akvorado.py index 05bdd52a..bde0561f 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -48,6 +48,10 @@ def get_snmpv2_providers(): for client in clients: key = current_event+'-mgmt/snmpv2:'+client['layer'] secrets = lib.read_secret(key) + if not secrets: + return + if not isinstance(secrets, Iterable): + return if "community" in secrets: provider = { "ipv4": client["ipv4_addr"], @@ -65,6 +69,10 @@ def get_snmpv3_providers(): for client in clients: key = current_event+'-mgmt/snmpv3:'+client['layer'] secrets = lib.read_secret(key) + if not secrets: + return + if not isinstance(secrets, Iterable): + return if "user" in secrets: provider = { "ipv4": client["ipv4_addr"], From 951cf8d61bbd28c4a37c372d71d5c5da70ae3998 Mon Sep 17 00:00:00 2001 From: furest Date: Tue, 19 Nov 2024 23:27:53 +0100 Subject: [PATCH 094/107] remove iterable check --- modules/akvorado.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/akvorado.py b/modules/akvorado.py index bde0561f..2e664c0a 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -50,8 +50,6 @@ def get_snmpv2_providers(): secrets = lib.read_secret(key) if not secrets: return - if not isinstance(secrets, Iterable): - return if "community" in secrets: provider = { "ipv4": client["ipv4_addr"], @@ -71,8 +69,6 @@ def get_snmpv3_providers(): secrets = lib.read_secret(key) if not secrets: return - if not isinstance(secrets, Iterable): - return if "user" in secrets: provider = { "ipv4": client["ipv4_addr"], From 367b2f2603706d432a239800eb79ebef3bc5c702 Mon Sep 17 00:00:00 2001 From: furest Date: Tue, 19 Nov 2024 23:33:00 +0100 Subject: [PATCH 095/107] return empty array instead of none --- modules/akvorado.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/akvorado.py b/modules/akvorado.py index 2e664c0a..c0b89096 100644 --- a/modules/akvorado.py +++ b/modules/akvorado.py @@ -49,7 +49,7 @@ def get_snmpv2_providers(): key = current_event+'-mgmt/snmpv2:'+client['layer'] secrets = lib.read_secret(key) if not secrets: - return + return providers if "community" in secrets: provider = { "ipv4": client["ipv4_addr"], @@ -68,7 +68,7 @@ def get_snmpv3_providers(): key = current_event+'-mgmt/snmpv3:'+client['layer'] secrets = lib.read_secret(key) if not secrets: - return + return providers if "user" in secrets: provider = { "ipv4": client["ipv4_addr"], From e218dbee08d13ed66425f7d85b6e8e40c216a670 Mon Sep 17 00:00:00 2001 From: furest Date: Fri, 22 Nov 2024 22:14:11 +0100 Subject: [PATCH 096/107] add encoded slashes support --- modules/akvorado/manifests/init.pp | 5 +++-- modules/apache/manifests/proxy.pp | 2 +- modules/apache/templates/proxy.conf.erb | 5 +++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index dcac6d0a..d0ddef8d 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -239,8 +239,9 @@ backend => 'http://localhost:8081/api/v0/inlet/', } apache::proxy { '3_akvorado-console': - url => '/', - backend => 'http://localhost:8082/', + url => '/', + backend => 'http://localhost:8082/', + allowEncodedSlashes => true, } # By default apache answers with status code 404 when an URL contains an encoded slash (%2F) # The following allows apache to simply forward the request to the prox backend. diff --git a/modules/apache/manifests/proxy.pp b/modules/apache/manifests/proxy.pp index bf5fa09f..00e6408e 100644 --- a/modules/apache/manifests/proxy.pp +++ b/modules/apache/manifests/proxy.pp @@ -3,7 +3,7 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file # -define apache::proxy($url, $backend) { +define apache::proxy($url, $backend, $allowEncodedSlashes = false) { exec { "apache_proxy_reload_${name}": command => '/usr/sbin/apachectl graceful', refreshonly => true, diff --git a/modules/apache/templates/proxy.conf.erb b/modules/apache/templates/proxy.conf.erb index c1b9f6de..78709786 100644 --- a/modules/apache/templates/proxy.conf.erb +++ b/modules/apache/templates/proxy.conf.erb @@ -4,3 +4,8 @@ ProxyPass <%= @url %> <%= @backend %> RequestHeader set X-Proxy-REMOTE-USER %{REMOTE_USER}s ProxyPreserveHost on + +<% if @allowEncodedSlashes == true -%> +AllowEncodedSlashes On +<% end -%> + From d15b3c9d1fceed44b2a9d9a012a352276e5a4dbe Mon Sep 17 00:00:00 2001 From: furest Date: Fri, 22 Nov 2024 22:23:06 +0100 Subject: [PATCH 097/107] Remove previous encoded slashes fix --- modules/akvorado/manifests/init.pp | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index d0ddef8d..eafbf444 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -243,18 +243,6 @@ backend => 'http://localhost:8082/', allowEncodedSlashes => true, } - # By default apache answers with status code 404 when an URL contains an encoded slash (%2F) - # The following allows apache to simply forward the request to the prox backend. - file { '/etc/apache2/conf-available/allow-slashes.conf': - ensure => present, - content => 'AllowEncodedSlashes On', - mode => '0644', - } - -> file { '/etc/apache2/conf-enabled/allow-slashes.conf': - ensure => link, - mode => '0644', - target => '/etc/apache2/conf-available/allow-slashes.conf', - } service { 'akvorado-orch': ensure => running, enable => true, From ddf98ea2573abfdce439a03916ac6a7bfe204bdd Mon Sep 17 00:00:00 2001 From: furest Date: Sat, 23 Nov 2024 00:04:31 +0100 Subject: [PATCH 098/107] make linter happy again --- modules/akvorado/manifests/init.pp | 6 +++--- modules/apache/manifests/proxy.pp | 2 +- modules/apache/templates/proxy.conf.erb | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index eafbf444..cdfae488 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -239,9 +239,9 @@ backend => 'http://localhost:8081/api/v0/inlet/', } apache::proxy { '3_akvorado-console': - url => '/', - backend => 'http://localhost:8082/', - allowEncodedSlashes => true, + url => '/', + backend => 'http://localhost:8082/', + allow_encoded_slashes => true, } service { 'akvorado-orch': ensure => running, diff --git a/modules/apache/manifests/proxy.pp b/modules/apache/manifests/proxy.pp index 00e6408e..16f832dc 100644 --- a/modules/apache/manifests/proxy.pp +++ b/modules/apache/manifests/proxy.pp @@ -3,7 +3,7 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file # -define apache::proxy($url, $backend, $allowEncodedSlashes = false) { +define apache::proxy($url, $backend, $allow_encoded_slashes = false) { exec { "apache_proxy_reload_${name}": command => '/usr/sbin/apachectl graceful', refreshonly => true, diff --git a/modules/apache/templates/proxy.conf.erb b/modules/apache/templates/proxy.conf.erb index 78709786..763f23e7 100644 --- a/modules/apache/templates/proxy.conf.erb +++ b/modules/apache/templates/proxy.conf.erb @@ -5,7 +5,7 @@ ProxyPass <%= @url %> <%= @backend %> RequestHeader set X-Proxy-REMOTE-USER %{REMOTE_USER}s ProxyPreserveHost on -<% if @allowEncodedSlashes == true -%> +<% if @allow_encoded_slashes == true -%> AllowEncodedSlashes On <% end -%> From 47a12c8fc094fc9e8124efaa9133d53c991fc8f4 Mon Sep 17 00:00:00 2001 From: furest Date: Sat, 23 Nov 2024 18:52:34 +0100 Subject: [PATCH 099/107] rename dh as --- modules/akvorado/templates/akvorado.yaml.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/templates/akvorado.yaml.erb b/modules/akvorado/templates/akvorado.yaml.erb index 3c29da73..26efb3a6 100644 --- a/modules/akvorado/templates/akvorado.yaml.erb +++ b/modules/akvorado/templates/akvorado.yaml.erb @@ -55,7 +55,7 @@ clickhouse: systemlogttl: 720h0m0s prometheusendpoint: "/metrics" asns: - 25037: Dreamhack ACME Corporation + 25037: Dreamhack Events networks: # 2a01:db8:cafe:1::/64: # name: ipv6-customers From 03d6fb8fb3761efb38483f971db431d1abbb15c8 Mon Sep 17 00:00:00 2001 From: furest Date: Sat, 23 Nov 2024 18:52:51 +0100 Subject: [PATCH 100/107] change ldap template --- modules/apache/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/apache/manifests/init.pp b/modules/apache/manifests/init.pp index 97c90a8b..f9058da5 100644 --- a/modules/apache/manifests/init.pp +++ b/modules/apache/manifests/init.pp @@ -47,7 +47,7 @@ notify => Service['apache2'], } - if $::fqdn == 'status.event.dreamhack.se' or $::fqdn == 'grafana.event.dreamhack.se' { + if $::fqdn == 'status.event.dreamhack.se' or $::fqdn == 'grafana.event.dreamhack.se' or $::fqdn == 'sflow1.event.dreamhack.se' { file { 'apache-security.conf': ensure => present, path => '/etc/apache2/conf-available/security.conf', From c38b04f844e788f60da77b96ed3b9fe095a0cf5d Mon Sep 17 00:00:00 2001 From: furest Date: Sat, 29 Nov 2025 20:20:55 +0100 Subject: [PATCH 101/107] migrate to akvorado 2.0.3 --- .../akvorado/files/akvorado-outlet.service | 15 +++ modules/akvorado/files/zookeeper.service | 13 --- modules/akvorado/manifests/init.pp | 78 +++++++------- modules/akvorado/templates/akvorado.yaml.erb | 101 +++++++++--------- 4 files changed, 108 insertions(+), 99 deletions(-) create mode 100644 modules/akvorado/files/akvorado-outlet.service delete mode 100644 modules/akvorado/files/zookeeper.service diff --git a/modules/akvorado/files/akvorado-outlet.service b/modules/akvorado/files/akvorado-outlet.service new file mode 100644 index 00000000..639674c4 --- /dev/null +++ b/modules/akvorado/files/akvorado-outlet.service @@ -0,0 +1,15 @@ +[Unit] +Description=Akvorado Outlet +After=akvorado-orch.service +Requires=akvorado-orch.service + +[Service] +Type=simple +Restart=on-failure +RestartSec=15 +User=akvorado +ExecStart=/usr/local/bin/akvorado outlet http://127.0.0.1:8080 + +[Install] +WantedBy=multi-user.target + diff --git a/modules/akvorado/files/zookeeper.service b/modules/akvorado/files/zookeeper.service deleted file mode 100644 index 62fcd238..00000000 --- a/modules/akvorado/files/zookeeper.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Requires=network.target remote-fs.target -After=network.target remote-fs.target - -[Service] -Type=simple -User=kafka -ExecStart=/var/lib/kafka/bin/zookeeper-server-start.sh /var/lib/kafka/config/zookeeper.properties -ExecStop=/var/lib/kafka/bin/zookeeper-server-stop.sh -Restart=on-abnormal - -[Install] -WantedBy=multi-user.target diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index cdfae488..eb256dd2 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -38,12 +38,6 @@ group => 'kafka', mode => '0700', } - -> file { '/var/lib/zookeeper-data': - ensure => 'directory', - owner => 'kafka', - group => 'kafka', - mode => '0700', - } exec { 'untar-kafka': command => '/bin/tar -xvf /var/lib/kafka/kafka.tgz -C /var/lib/kafka --strip 1', refreshonly => true, @@ -57,20 +51,42 @@ group => 'root', notify => [ Exec['systemctl-daemon-reload'], Service['kafka'] ], } - -> file { '/etc/systemd/system/zookeeper.service': - ensure => present, - source => 'puppet:///modules/akvorado/zookeeper.service', - mode => '0644', - owner => 'root', - group => 'root', - notify => [ Exec['systemctl-daemon-reload'], Service['zookeeper'] ], - } -> file_line { 'kafka-enabledeletetopics': ensure => 'present', path => '/var/lib/kafka/config/server.properties', line => 'delete.topic.enable = true', notify => Service['kafka'], } + -> file_line { 'kafka-quorumvoters': + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'controller.quorum.voters=1@localhost:9093', + notify => Service['kafka'], + } + -> file_line { 'kafka-securityprotocolmap': + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'listener.security.protocol.map=CLIENT:PLAINTEXT,CONTROLLER:PLAINTEXT', + notify => Service['kafka'], + } + -> file_line { 'kafka-advertisedlsiteners': + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'advertised.listeners=CLIENT://localhost:9092', + notify => Service['kafka'], + } + -> file_line { 'kafka-controllerlistenernames': + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'controller.listener.names=CONTROLLER', + notify => Service['kafka'], + } + -> file_line { 'kafka-interbrokerlistenername': + ensure => 'present', + path => '/var/lib/kafka/config/server.properties', + line => 'inter.broker.listener.name=CLIENT', + notify => Service['kafka'], + } -> file_line { 'kafka-listenlocalhost': ensure => 'present', path => '/var/lib/kafka/config/server.properties', @@ -85,27 +101,10 @@ match => 'log.dirs=/tmp/kafka-logs', notify => Service['kafka'], } - -> file_line { 'zookeeper-datadir': - ensure => 'present', - path => '/var/lib/kafka/config/zookeeper.properties', - line => 'dataDir=/var/lib/zookeeper-data', - match => 'dataDir=/tmp/zookeeper', - notify => Service['zookeeper'], - } - -> file_line { 'zookeeper-listen': - ensure => 'present', - path => '/var/lib/kafka/config/zookeeper.properties', - line => 'clientPortAddress=127.0.0.1', - notify => Service['zookeeper'], - } service { 'kafka': ensure => running, enable => true, } - service { 'zookeeper': - ensure => running, - enable => true, - } ##Clickhouse installation ensure_packages([ @@ -202,6 +201,14 @@ group => 'root', notify => [Exec['systemctl-daemon-reload'],Service['akvorado-inlet']], } + file { '/etc/systemd/system/akvorado-outlet.service': + ensure => present, + source => 'puppet:///modules/akvorado/akvorado-outlet.service', + mode => '0644', + owner => 'root', + group => 'root', + notify => [Exec['systemctl-daemon-reload'],Service['akvorado-outlet']], + } file { '/etc/systemd/system/akvorado-console.service': ensure => present, source => 'puppet:///modules/akvorado/akvorado-console.service', @@ -251,6 +258,10 @@ ensure => running, enable => true, } + service { 'akvorado-outlet': + ensure => running, + enable => true, + } service { 'akvorado-console': ensure => running, enable => true, @@ -263,9 +274,4 @@ command => '/bin/systemctl daemon-reload', refreshonly => true, } - exec { 'protobuf-schema': - command => '/usr/bin/curl http://127.0.0.1:8080/api/v0/orchestrator/clickhouse/init.sh | sh', - refreshonly => true, - require => Service['akvorado-orch'] - } } diff --git a/modules/akvorado/templates/akvorado.yaml.erb b/modules/akvorado/templates/akvorado.yaml.erb index 26efb3a6..5e5a0030 100644 --- a/modules/akvorado/templates/akvorado.yaml.erb +++ b/modules/akvorado/templates/akvorado.yaml.erb @@ -116,35 +116,8 @@ schema: notmaintableonly: [] materialize: [] customdictionaries: {} -inlet: - - reporting: - logging: {} - metrics: {} - http: - listen: :8081 - profiler: true - cache: - type: memory - flow: - inputs: - - decoder: netflow - listen: :2055 - queuesize: 100000 - receivebuffer: 10485760 - timestampsource: udp - type: udp - usesrcaddrforexporteraddr: false - workers: 6 - - decoder: sflow - listen: :6343 - queuesize: 100000 - receivebuffer: 10485760 - timestampsource: udp - type: udp - usesrcaddrforexporteraddr: false - workers: 6 - ratelimit: 0 - metadata: +outlet: + metadata: cacheduration: 30m0s cacherefresh: 1h0m0s cachecheckinterval: 2m0s @@ -188,7 +161,7 @@ inlet: type: snmp workers: 1 maxbatchrequests: 10 - routing: + routing: provider: collectasns: true collectaspaths: true @@ -201,26 +174,7 @@ inlet: ribpeerremovalmaxtime: 100ms ribpeerremovalsleepinterval: 500ms type: bmp - kafka: - topic: flows - brokers: - - 127.0.0.1:9092 - version: 3.7.0 - tls: - enable: false - verify: true - cafile: "" - certfile: "" - keyfile: "" - saslusername: "" - saslpassword: "" - saslmechanism: none - flushinterval: 10s - flushbytes: 104857599 - maxmessagebytes: 1000000 - compressioncodec: zstd - queuesize: 32 - core: + core: workers: 6 exporterclassifiers: - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") @@ -241,6 +195,53 @@ inlet: netproviders: - flow - routing +inlet: + - reporting: + logging: {} + metrics: {} + http: + listen: :8081 + profiler: true + cache: + type: memory + flow: + inputs: + - decoder: netflow + listen: :2055 + queuesize: 100000 + receivebuffer: 10485760 + timestampsource: udp + type: udp + usesrcaddrforexporteraddr: false + workers: 6 + - decoder: sflow + listen: :6343 + queuesize: 100000 + receivebuffer: 10485760 + timestampsource: udp + type: udp + usesrcaddrforexporteraddr: false + workers: 6 + ratelimit: 0 + kafka: + topic: flows + brokers: + - 127.0.0.1:9092 + version: 3.7.0 + tls: + enable: false + verify: true + cafile: "" + certfile: "" + keyfile: "" + saslusername: "" + saslpassword: "" + saslmechanism: none + flushinterval: 10s + flushbytes: 104857599 + maxmessagebytes: 1000000 + compressioncodec: zstd + queuesize: 32 schema: disabled: [] enabled: [] From ef9891f21ea52e38c1fb64cb6c874a825918907e Mon Sep 17 00:00:00 2001 From: furest Date: Sat, 29 Nov 2025 21:04:24 +0100 Subject: [PATCH 102/107] remove protobuf from dependencies --- modules/akvorado/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index eb256dd2..eb06567f 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -177,7 +177,7 @@ mode => '0550', links => follow, source => 'puppet:///data/akvorado-latest', - notify => [Service['akvorado-orch'],Exec['protobuf-schema']] + notify => [Service['akvorado-orch']] } file { '/etc/akvorado/akvorado.yaml': ensure => file, From b68e6a17d23c0b7b18b8311b838cc4a2b508c61d Mon Sep 17 00:00:00 2001 From: furest Date: Sat, 29 Nov 2025 22:28:38 +0100 Subject: [PATCH 103/107] corrections --- modules/akvorado/manifests/init.pp | 28 ++------------------ modules/akvorado/templates/akvorado.yaml.erb | 24 ++++------------- 2 files changed, 7 insertions(+), 45 deletions(-) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index eb06567f..ee6a8253 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -63,35 +63,11 @@ line => 'controller.quorum.voters=1@localhost:9093', notify => Service['kafka'], } - -> file_line { 'kafka-securityprotocolmap': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'listener.security.protocol.map=CLIENT:PLAINTEXT,CONTROLLER:PLAINTEXT', - notify => Service['kafka'], - } - -> file_line { 'kafka-advertisedlsiteners': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'advertised.listeners=CLIENT://localhost:9092', - notify => Service['kafka'], - } - -> file_line { 'kafka-controllerlistenernames': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'controller.listener.names=CONTROLLER', - notify => Service['kafka'], - } - -> file_line { 'kafka-interbrokerlistenername': - ensure => 'present', - path => '/var/lib/kafka/config/server.properties', - line => 'inter.broker.listener.name=CLIENT', - notify => Service['kafka'], - } -> file_line { 'kafka-listenlocalhost': ensure => 'present', path => '/var/lib/kafka/config/server.properties', - line => 'listeners=PLAINTEXT://localhost:9092', - match => '#listeners=PLAINTEXT', + line => 'listeners=PLAINTEXT://localhost:9092,CONTROLLER://localhost:9093', + match => 'listeners=PLAINTEXT://:9092,CONTROLLER://:9093', notify => Service['kafka'], } -> file_line { 'kafka-logdir': diff --git a/modules/akvorado/templates/akvorado.yaml.erb b/modules/akvorado/templates/akvorado.yaml.erb index 5e5a0030..ebdc39f1 100644 --- a/modules/akvorado/templates/akvorado.yaml.erb +++ b/modules/akvorado/templates/akvorado.yaml.erb @@ -104,20 +104,13 @@ kafka: segment.bytes: "1073741824" configentriesstrictsync: true geoip: - asndatabase: + asn-database: - /usr/share/GeoIP/asn.mmdb - geodatabase: + geo-database: - /usr/share/GeoIP/country.mmdb optional: false -schema: - disabled: [] - enabled: [] - maintableonly: [] - notmaintableonly: [] - materialize: [] - customdictionaries: {} outlet: - metadata: + - metadata: cacheduration: 30m0s cacherefresh: 1h0m0s cachecheckinterval: 2m0s @@ -161,7 +154,7 @@ outlet: type: snmp workers: 1 maxbatchrequests: 10 - routing: + routing: provider: collectasns: true collectaspaths: true @@ -174,7 +167,7 @@ outlet: ribpeerremovalmaxtime: 100ms ribpeerremovalsleepinterval: 500ms type: bmp - core: + core: workers: 6 exporterclassifiers: - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") @@ -242,13 +235,6 @@ inlet: maxmessagebytes: 1000000 compressioncodec: zstd queuesize: 32 - schema: - disabled: [] - enabled: [] - maintableonly: [] - notmaintableonly: [] - materialize: [] - customdictionaries: {} console: - reporting: logging: {} From ea56a2d4daf016425f74fc02a3bb8502d2f4e7c0 Mon Sep 17 00:00:00 2001 From: furest Date: Sat, 29 Nov 2025 22:42:20 +0100 Subject: [PATCH 104/107] define outlet listener --- modules/akvorado/templates/akvorado.yaml.erb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/akvorado/templates/akvorado.yaml.erb b/modules/akvorado/templates/akvorado.yaml.erb index ebdc39f1..6b8ffd6d 100644 --- a/modules/akvorado/templates/akvorado.yaml.erb +++ b/modules/akvorado/templates/akvorado.yaml.erb @@ -29,7 +29,6 @@ clickhouse: topic: flows brokers: - 127.0.0.1:9092 - version: 3.7.0 tls: enable: false verify: true @@ -84,7 +83,6 @@ kafka: topic: flows brokers: - 127.0.0.1:9092 - version: 3.7.0 tls: enable: false verify: true @@ -154,6 +152,11 @@ outlet: type: snmp workers: 1 maxbatchrequests: 10 + http: + listen: :8083 + profiler: true + cache: + type: memory routing: provider: collectasns: true @@ -220,7 +223,6 @@ inlet: topic: flows brokers: - 127.0.0.1:9092 - version: 3.7.0 tls: enable: false verify: true From 60eaebe2806b8da2af9c42215c84478720683926 Mon Sep 17 00:00:00 2001 From: furest Date: Sat, 29 Nov 2025 23:18:33 +0100 Subject: [PATCH 105/107] use cities database --- modules/akvorado/manifests/init.pp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/akvorado/manifests/init.pp b/modules/akvorado/manifests/init.pp index ee6a8253..735645f4 100644 --- a/modules/akvorado/manifests/init.pp +++ b/modules/akvorado/manifests/init.pp @@ -213,6 +213,13 @@ owner => 'root', group => 'root', } + file { '/usr/share/GeoIP/city.mmdb': + ensure => present, + source => 'puppet:///data/city.mmdb', + mode => '0644', + owner => 'root', + group => 'root', + } apache::proxy { '1_akvorado-orch-api': url => '/api/v0/orchestrator/', backend => 'http://localhost:8080/api/v0/orchestrator/', From 60b2152be10550f1268db72f59508b11be4fde41 Mon Sep 17 00:00:00 2001 From: furest Date: Sat, 29 Nov 2025 23:26:05 +0100 Subject: [PATCH 106/107] use cities database 2 --- modules/akvorado/templates/akvorado.yaml.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/akvorado/templates/akvorado.yaml.erb b/modules/akvorado/templates/akvorado.yaml.erb index 6b8ffd6d..2a3a512c 100644 --- a/modules/akvorado/templates/akvorado.yaml.erb +++ b/modules/akvorado/templates/akvorado.yaml.erb @@ -105,7 +105,7 @@ geoip: asn-database: - /usr/share/GeoIP/asn.mmdb geo-database: - - /usr/share/GeoIP/country.mmdb + - /usr/share/GeoIP/city.mmdb optional: false outlet: - metadata: From 01f2fc57eaea55ce12df767599b7ae1ccacd5043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Sun, 30 Nov 2025 01:25:34 +0100 Subject: [PATCH 107/107] more asn (#429) --- modules/akvorado/templates/akvorado.yaml.erb | 23 ++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/modules/akvorado/templates/akvorado.yaml.erb b/modules/akvorado/templates/akvorado.yaml.erb index 2a3a512c..e4b6202e 100644 --- a/modules/akvorado/templates/akvorado.yaml.erb +++ b/modules/akvorado/templates/akvorado.yaml.erb @@ -306,6 +306,29 @@ console: content: InIfBoundary = external AND SrcAS = AS9201 - description: Valve Corporation content: InIfBoundary = external AND SrcAS = AS32590 + - description: Axians + content: InIfBoundary = external AND SrcAS = AS20514 + - description: Edgevana + content: InIfBoundary = external AND SrcAS = AS215724 + - description: Elisa Oyj + content: InIfBoundary = external AND SrcAS = AS6667 + - description: NORDUnet + content: InIfBoundary = external AND SrcAS = AS2603 + - description: 31173 Services + content: InIfBoundary = external AND SrcAS = AS39351 + - description: Riot Games, Inc + content: InIfBoundary = external AND SrcAS = AS6507 + - description: Hi3G Access AB + content: InIfBoundary = external AND SrcAS = AS44034 + - description: Telia + content: InIfBoundary = external AND SrcAS = AS3301 + - description: Arelion + content: InIfBoundary = external AND SrcAS = AS1299 + - description: CDN77 + content: InIfBoundary = external AND SrcAS = AS60068 + - description: Epic games + content: InIfBoundary = external AND SrcAS = AS4356 + schema: disabled: [] enabled: []