indexer: allow multiple indexer journal client instances
And for production worker, add twice the content indexer instances so we may consume messages a bit faster than the current 209 years left [1]
Test Plan
2 more services for production workers:
$ $SWH_PUPPET_ENVIRONMENT_HOME/bin/octocatalog-diff indexer-worker01.euwest.azure.internal.softwareheritage.org
Found host indexer-worker01.euwest.azure.internal.softwareheritage.org
...
diff origin/production/indexer-worker01.euwest.azure.internal.softwareheritage.org current/indexer-worker01.euwest.azure.internal.softwareheritage.org
*******************************************
File[/etc/softwareheritage/indexer/content_fossology_license.yml] =>
parameters =>
content =>
@@ -1,3 +1,3 @@
----
+# File managed by puppet - modifications will be lost
scheduler:
cls: remote
@@ -106,3 +106,2 @@
command_line: nomossa <filepath>
write_batch_size: 1000
-
notify =>
- Service[swh-indexer-journal-client@content_fossology_license.service]
*******************************************
File[/etc/softwareheritage/indexer/content_mimetype.yml] =>
parameters =>
content =>
@@ -1,3 +1,3 @@
----
+# File managed by puppet - modifications will be lost
scheduler:
cls: remote
@@ -106,3 +106,2 @@
debian-package: python3-magic
write_batch_size: 1000
-
notify =>
- Service[swh-indexer-journal-client@content_mimetype.service]
*******************************************
File[/etc/softwareheritage/indexer/extrinsic_metadata.yml] =>
parameters =>
content =>
@@ -1,3 +1,3 @@
----
+# File managed by puppet - modifications will be lost
scheduler:
cls: remote
@@ -106,3 +106,2 @@
version: 0.0.2
configuration: {}
-
notify =>
- Service[swh-indexer-journal-client@extrinsic_metadata.service]
*******************************************
File[/etc/systemd/system/swh-indexer-journal-client@content_fossology_license.service.d/parameters.conf] =>
parameters =>
ensure =>
- file
+ absent
*******************************************
- File[/etc/systemd/system/swh-indexer-journal-client@content_fossology_license.service.d]
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@content_fossology_license_1.service.d/parameters.conf] =>
parameters =>
"content": "# Managed by puppet - modifications will be overwritten\n# In de...
"ensure": "file",
"group": "root",
"mode": "0444",
"notify": [
"Class[Systemd::Systemctl::Daemon_reload]"
],
"owner": "root",
"selinux_ignore_defaults": false,
"show_diff": true
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@content_fossology_license_1.service.d] =>
parameters =>
"ensure": "directory",
"group": "root",
"owner": "root",
"purge": true,
"recurse": true,
"selinux_ignore_defaults": false
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@content_fossology_license_2.service.d/parameters.conf] =>
parameters =>
"content": "# Managed by puppet - modifications will be overwritten\n# In de...
"ensure": "file",
"group": "root",
"mode": "0444",
"notify": [
"Class[Systemd::Systemctl::Daemon_reload]"
],
"owner": "root",
"selinux_ignore_defaults": false,
"show_diff": true
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@content_fossology_license_2.service.d] =>
parameters =>
"ensure": "directory",
"group": "root",
"owner": "root",
"purge": true,
"recurse": true,
"selinux_ignore_defaults": false
*******************************************
File[/etc/systemd/system/swh-indexer-journal-client@content_mimetype.service.d/parameters.conf] =>
parameters =>
ensure =>
- file
+ absent
*******************************************
- File[/etc/systemd/system/swh-indexer-journal-client@content_mimetype.service.d]
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@content_mimetype_1.service.d/parameters.conf] =>
parameters =>
"content": "# Managed by puppet - modifications will be overwritten\n# In de...
"ensure": "file",
"group": "root",
"mode": "0444",
"notify": [
"Class[Systemd::Systemctl::Daemon_reload]"
],
"owner": "root",
"selinux_ignore_defaults": false,
"show_diff": true
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@content_mimetype_1.service.d] =>
parameters =>
"ensure": "directory",
"group": "root",
"owner": "root",
"purge": true,
"recurse": true,
"selinux_ignore_defaults": false
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@content_mimetype_2.service.d/parameters.conf] =>
parameters =>
"content": "# Managed by puppet - modifications will be overwritten\n# In de...
"ensure": "file",
"group": "root",
"mode": "0444",
"notify": [
"Class[Systemd::Systemctl::Daemon_reload]"
],
"owner": "root",
"selinux_ignore_defaults": false,
"show_diff": true
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@content_mimetype_2.service.d] =>
parameters =>
"ensure": "directory",
"group": "root",
"owner": "root",
"purge": true,
"recurse": true,
"selinux_ignore_defaults": false
*******************************************
File[/etc/systemd/system/swh-indexer-journal-client@extrinsic_metadata.service.d/parameters.conf] =>
parameters =>
ensure =>
- file
+ absent
*******************************************
- File[/etc/systemd/system/swh-indexer-journal-client@extrinsic_metadata.service.d]
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@extrinsic_metadata_1.service.d/parameters.conf] =>
parameters =>
"content": "# Managed by puppet - modifications will be overwritten\n# In de...
"ensure": "file",
"group": "root",
"mode": "0444",
"notify": [
"Class[Systemd::Systemctl::Daemon_reload]"
],
"owner": "root",
"selinux_ignore_defaults": false,
"show_diff": true
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@extrinsic_metadata_1.service.d] =>
parameters =>
"ensure": "directory",
"group": "root",
"owner": "root",
"purge": true,
"recurse": true,
"selinux_ignore_defaults": false
*******************************************
Service[swh-indexer-journal-client@content_fossology_license.service] =>
parameters =>
enable =>
- true
ensure =>
- running
+ stopped
*******************************************
+ Service[swh-indexer-journal-client@content_fossology_license_1.service] =>
parameters =>
"enable": true,
"ensure": "running"
*******************************************
+ Service[swh-indexer-journal-client@content_fossology_license_2.service] =>
parameters =>
"enable": true,
"ensure": "running"
*******************************************
Service[swh-indexer-journal-client@content_mimetype.service] =>
parameters =>
enable =>
- true
ensure =>
- running
+ stopped
*******************************************
+ Service[swh-indexer-journal-client@content_mimetype_1.service] =>
parameters =>
"enable": true,
"ensure": "running"
*******************************************
+ Service[swh-indexer-journal-client@content_mimetype_2.service] =>
parameters =>
"enable": true,
"ensure": "running"
*******************************************
Service[swh-indexer-journal-client@extrinsic_metadata.service] =>
parameters =>
enable =>
- true
ensure =>
- running
+ stopped
*******************************************
+ Service[swh-indexer-journal-client@extrinsic_metadata_1.service] =>
parameters =>
"enable": true,
"ensure": "running"
*******************************************
Systemd::Dropin_file[swh-indexer-journal-client@content_fossology_license.service/parameters.conf] =>
parameters =>
content =>
- # Managed by puppet - modifications will be overwritten
# In defined class profile::swh::deploy::indexer_journal_client
[Service]
Environment=SWH_CONFIG_FILENAME=/etc/softwareheritage/indexer/content_fossology_license.yml
Environment=SWH_SENTRY_DSN=https://swh::deploy::indexer::sentry_token@sentry.softwareheritage.org/5
Environment=SWH_SENTRY_ENVIRONMENT=production
Environment=SWH_MAIN_PACKAGE=swh.indexer
Environment=LOGLEVEL=INFO
ensure =>
- present
+ absent
*******************************************
+ Systemd::Dropin_file[swh-indexer-journal-client@content_fossology_license_1.service/parameters.conf] =>
parameters =>
"content": "# Managed by puppet - modifications will be overwritten\n# In de...
"daemon_reload": "lazy",
"ensure": "present",
"filename": "parameters.conf",
"group": "root",
"mode": "0444",
"owner": "root",
"path": "/etc/systemd/system",
"selinux_ignore_defaults": false,
"show_diff": true,
"unit": "swh-indexer-journal-client@content_fossology_license_1.service"
*******************************************
+ Systemd::Dropin_file[swh-indexer-journal-client@content_fossology_license_2.service/parameters.conf] =>
parameters =>
"content": "# Managed by puppet - modifications will be overwritten\n# In de...
"daemon_reload": "lazy",
"ensure": "present",
"filename": "parameters.conf",
"group": "root",
"mode": "0444",
"owner": "root",
"path": "/etc/systemd/system",
"selinux_ignore_defaults": false,
"show_diff": true,
"unit": "swh-indexer-journal-client@content_fossology_license_2.service"
*******************************************
Systemd::Dropin_file[swh-indexer-journal-client@content_mimetype.service/parameters.conf] =>
parameters =>
content =>
- # Managed by puppet - modifications will be overwritten
# In defined class profile::swh::deploy::indexer_journal_client
[Service]
Environment=SWH_CONFIG_FILENAME=/etc/softwareheritage/indexer/content_mimetype.yml
Environment=SWH_SENTRY_DSN=https://swh::deploy::indexer::sentry_token@sentry.softwareheritage.org/5
Environment=SWH_SENTRY_ENVIRONMENT=production
Environment=SWH_MAIN_PACKAGE=swh.indexer
Environment=LOGLEVEL=INFO
ensure =>
- present
+ absent
*******************************************
+ Systemd::Dropin_file[swh-indexer-journal-client@content_mimetype_1.service/parameters.conf] =>
parameters =>
"content": "# Managed by puppet - modifications will be overwritten\n# In de...
"daemon_reload": "lazy",
"ensure": "present",
"filename": "parameters.conf",
"group": "root",
"mode": "0444",
"owner": "root",
"path": "/etc/systemd/system",
"selinux_ignore_defaults": false,
"show_diff": true,
"unit": "swh-indexer-journal-client@content_mimetype_1.service"
*******************************************
+ Systemd::Dropin_file[swh-indexer-journal-client@content_mimetype_2.service/parameters.conf] =>
parameters =>
"content": "# Managed by puppet - modifications will be overwritten\n# In de...
"daemon_reload": "lazy",
"ensure": "present",
"filename": "parameters.conf",
"group": "root",
"mode": "0444",
"owner": "root",
"path": "/etc/systemd/system",
"selinux_ignore_defaults": false,
"show_diff": true,
"unit": "swh-indexer-journal-client@content_mimetype_2.service"
*******************************************
Systemd::Dropin_file[swh-indexer-journal-client@extrinsic_metadata.service/parameters.conf] =>
parameters =>
content =>
- # Managed by puppet - modifications will be overwritten
# In defined class profile::swh::deploy::indexer_journal_client
[Service]
Environment=SWH_CONFIG_FILENAME=/etc/softwareheritage/indexer/extrinsic_metadata.yml
Environment=SWH_SENTRY_DSN=https://swh::deploy::indexer::sentry_token@sentry.softwareheritage.org/5
Environment=SWH_SENTRY_ENVIRONMENT=production
Environment=SWH_MAIN_PACKAGE=swh.indexer
Environment=LOGLEVEL=INFO
ensure =>
- present
+ absent
*******************************************
+ Systemd::Dropin_file[swh-indexer-journal-client@extrinsic_metadata_1.service/parameters.conf] =>
parameters =>
"content": "# Managed by puppet - modifications will be overwritten\n# In de...
"daemon_reload": "lazy",
"ensure": "present",
"filename": "parameters.conf",
"group": "root",
"mode": "0444",
"owner": "root",
"path": "/etc/systemd/system",
"selinux_ignore_defaults": false,
"show_diff": true,
"unit": "swh-indexer-journal-client@extrinsic_metadata_1.service"
*******************************************
*** End octocatalog-diff on indexer-worker01.euwest.azure.internal.softwareheritage.org
For staging, the services stays the same, but the services are renamed _1.service
suffix:
$ $SWH_PUPPET_ENVIRONMENT_HOME/bin/octocatalog-diff worker0.internal.staging.swh.network
...
*******************************************
File[/etc/softwareheritage/indexer/content_fossology_license.yml] =>
parameters =>
content =>
@@ -1,3 +1,3 @@
----
+# File managed by puppet - modifications will be lost
scheduler:
cls: remote
@@ -25,3 +25,2 @@
command_line: nomossa <filepath>
write_batch_size: 1000
-
notify =>
- Service[swh-indexer-journal-client@content_fossology_license.service]
*******************************************
File[/etc/softwareheritage/indexer/content_mimetype.yml] =>
parameters =>
content =>
@@ -1,3 +1,3 @@
----
+# File managed by puppet - modifications will be lost
scheduler:
cls: remote
@@ -25,3 +25,2 @@
debian-package: python3-magic
write_batch_size: 1000
-
notify =>
- Service[swh-indexer-journal-client@content_mimetype.service]
*******************************************
File[/etc/softwareheritage/indexer/extrinsic_metadata.yml] =>
parameters =>
content =>
@@ -1,3 +1,3 @@
----
+# File managed by puppet - modifications will be lost
scheduler:
cls: remote
@@ -25,3 +25,2 @@
version: 0.0.2
configuration: {}
-
notify =>
- Service[swh-indexer-journal-client@extrinsic_metadata.service]
*******************************************
File[/etc/softwareheritage/indexer/origin_intrinsic_metadata.yml] =>
parameters =>
content =>
@@ -1,3 +1,3 @@
----
+# File managed by puppet - modifications will be lost
scheduler:
cls: remote
@@ -25,3 +25,2 @@
version: 0.0.2
configuration: {}
-
notify =>
- Service[swh-indexer-journal-client@origin_intrinsic_metadata.service]
*******************************************
File[/etc/systemd/system/swh-indexer-journal-client@content_fossology_license.service.d/parameters.conf] =>
parameters =>
ensure =>
- file
+ absent
*******************************************
- File[/etc/systemd/system/swh-indexer-journal-client@content_fossology_license.service.d]
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@content_fossology_license_1.service.d/parameters.conf] =>
parameters =>
"content": "# Managed by puppet - modifications will be overwritten\n# In de...
"ensure": "file",
"group": "root",
"mode": "0444",
"notify": [
"Class[Systemd::Systemctl::Daemon_reload]"
],
"owner": "root",
"selinux_ignore_defaults": false,
"show_diff": true
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@content_fossology_license_1.service.d] =>
parameters =>
"ensure": "directory",
"group": "root",
"owner": "root",
"purge": true,
"recurse": true,
"selinux_ignore_defaults": false
*******************************************
File[/etc/systemd/system/swh-indexer-journal-client@content_mimetype.service.d/parameters.conf] =>
parameters =>
ensure =>
- file
+ absent
*******************************************
- File[/etc/systemd/system/swh-indexer-journal-client@content_mimetype.service.d]
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@content_mimetype_1.service.d/parameters.conf] =>
parameters =>
"content": "# Managed by puppet - modifications will be overwritten\n# In de...
"ensure": "file",
"group": "root",
"mode": "0444",
"notify": [
"Class[Systemd::Systemctl::Daemon_reload]"
],
"owner": "root",
"selinux_ignore_defaults": false,
"show_diff": true
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@content_mimetype_1.service.d] =>
parameters =>
"ensure": "directory",
"group": "root",
"owner": "root",
"purge": true,
"recurse": true,
"selinux_ignore_defaults": false
*******************************************
File[/etc/systemd/system/swh-indexer-journal-client@extrinsic_metadata.service.d/parameters.conf] =>
parameters =>
ensure =>
- file
+ absent
*******************************************
- File[/etc/systemd/system/swh-indexer-journal-client@extrinsic_metadata.service.d]
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@extrinsic_metadata_1.service.d/parameters.conf] =>
parameters =>
"content": "# Managed by puppet - modifications will be overwritten\n# In de...
"ensure": "file",
"group": "root",
"mode": "0444",
"notify": [
"Class[Systemd::Systemctl::Daemon_reload]"
],
"owner": "root",
"selinux_ignore_defaults": false,
"show_diff": true
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@extrinsic_metadata_1.service.d] =>
parameters =>
"ensure": "directory",
"group": "root",
"owner": "root",
"purge": true,
"recurse": true,
"selinux_ignore_defaults": false
*******************************************
File[/etc/systemd/system/swh-indexer-journal-client@origin_intrinsic_metadata.service.d/parameters.conf] =>
parameters =>
ensure =>
- file
+ absent
*******************************************
- File[/etc/systemd/system/swh-indexer-journal-client@origin_intrinsic_metadata.service.d]
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@origin_intrinsic_metadata_1.service.d/parameters.conf] =>
parameters =>
"content": "# Managed by puppet - modifications will be overwritten\n# In de...
"ensure": "file",
"group": "root",
"mode": "0444",
"notify": [
"Class[Systemd::Systemctl::Daemon_reload]"
],
"owner": "root",
"selinux_ignore_defaults": false,
"show_diff": true
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@origin_intrinsic_metadata_1.service.d] =>
parameters =>
"ensure": "directory",
"group": "root",
"owner": "root",
"purge": true,
"recurse": true,
"selinux_ignore_defaults": false
*******************************************
Service[swh-indexer-journal-client@content_fossology_license.service] =>
parameters =>
enable =>
- true
ensure =>
- running
+ stopped
*******************************************
+ Service[swh-indexer-journal-client@content_fossology_license_1.service] =>
parameters =>
"enable": true,
"ensure": "running"
*******************************************
Service[swh-indexer-journal-client@content_mimetype.service] =>
parameters =>
enable =>
- true
ensure =>
- running
+ stopped
*******************************************
+ Service[swh-indexer-journal-client@content_mimetype_1.service] =>
parameters =>
"enable": true,
"ensure": "running"
*******************************************
Service[swh-indexer-journal-client@extrinsic_metadata.service] =>
parameters =>
enable =>
- true
ensure =>
- running
+ stopped
*******************************************
+ Service[swh-indexer-journal-client@extrinsic_metadata_1.service] =>
parameters =>
"enable": true,
"ensure": "running"
*******************************************
Service[swh-indexer-journal-client@origin_intrinsic_metadata.service] =>
parameters =>
enable =>
- true
ensure =>
- running
+ stopped
*******************************************
+ Service[swh-indexer-journal-client@origin_intrinsic_metadata_1.service] =>
parameters =>
"enable": true,
"ensure": "running"
*******************************************
Systemd::Dropin_file[swh-indexer-journal-client@content_fossology_license.service/parameters.conf] =>
parameters =>
content =>
- # Managed by puppet - modifications will be overwritten
# In defined class profile::swh::deploy::indexer_journal_client
[Service]
Environment=SWH_CONFIG_FILENAME=/etc/softwareheritage/indexer/content_fossology_license.yml
Environment=SWH_SENTRY_DSN=https://swh::deploy::indexer::sentry_token@sentry.softwareheritage.org/5
Environment=SWH_SENTRY_ENVIRONMENT=staging
Environment=SWH_MAIN_PACKAGE=swh.indexer
Environment=LOGLEVEL=INFO
ensure =>
- present
+ absent
*******************************************
+ Systemd::Dropin_file[swh-indexer-journal-client@content_fossology_license_1.service/parameters.conf] =>
parameters =>
"content": "# Managed by puppet - modifications will be overwritten\n# In de...
"daemon_reload": "lazy",
"ensure": "present",
"filename": "parameters.conf",
"group": "root",
"mode": "0444",
"owner": "root",
"path": "/etc/systemd/system",
"selinux_ignore_defaults": false,
"show_diff": true,
"unit": "swh-indexer-journal-client@content_fossology_license_1.service"
*******************************************
Systemd::Dropin_file[swh-indexer-journal-client@content_mimetype.service/parameters.conf] =>
parameters =>
content =>
- # Managed by puppet - modifications will be overwritten
# In defined class profile::swh::deploy::indexer_journal_client
[Service]
Environment=SWH_CONFIG_FILENAME=/etc/softwareheritage/indexer/content_mimetype.yml
Environment=SWH_SENTRY_DSN=https://swh::deploy::indexer::sentry_token@sentry.softwareheritage.org/5
Environment=SWH_SENTRY_ENVIRONMENT=staging
Environment=SWH_MAIN_PACKAGE=swh.indexer
Environment=LOGLEVEL=INFO
ensure =>
- present
+ absent
*******************************************
+ Systemd::Dropin_file[swh-indexer-journal-client@content_mimetype_1.service/parameters.conf] =>
parameters =>
"content": "# Managed by puppet - modifications will be overwritten\n# In de...
"daemon_reload": "lazy",
"ensure": "present",
"filename": "parameters.conf",
"group": "root",
"mode": "0444",
"owner": "root",
"path": "/etc/systemd/system",
"selinux_ignore_defaults": false,
"show_diff": true,
"unit": "swh-indexer-journal-client@content_mimetype_1.service"
*******************************************
Systemd::Dropin_file[swh-indexer-journal-client@extrinsic_metadata.service/parameters.conf] =>
parameters =>
content =>
- # Managed by puppet - modifications will be overwritten
# In defined class profile::swh::deploy::indexer_journal_client
[Service]
Environment=SWH_CONFIG_FILENAME=/etc/softwareheritage/indexer/extrinsic_metadata.yml
Environment=SWH_SENTRY_DSN=https://swh::deploy::indexer::sentry_token@sentry.softwareheritage.org/5
Environment=SWH_SENTRY_ENVIRONMENT=staging
Environment=SWH_MAIN_PACKAGE=swh.indexer
Environment=LOGLEVEL=INFO
ensure =>
- present
+ absent
*******************************************
+ Systemd::Dropin_file[swh-indexer-journal-client@extrinsic_metadata_1.service/parameters.conf] =>
parameters =>
"content": "# Managed by puppet - modifications will be overwritten\n# In de...
"daemon_reload": "lazy",
"ensure": "present",
"filename": "parameters.conf",
"group": "root",
"mode": "0444",
"owner": "root",
"path": "/etc/systemd/system",
"selinux_ignore_defaults": false,
"show_diff": true,
"unit": "swh-indexer-journal-client@extrinsic_metadata_1.service"
*******************************************
Systemd::Dropin_file[swh-indexer-journal-client@origin_intrinsic_metadata.service/parameters.conf] =>
parameters =>
content =>
- # Managed by puppet - modifications will be overwritten
# In defined class profile::swh::deploy::indexer_journal_client
[Service]
Environment=SWH_CONFIG_FILENAME=/etc/softwareheritage/indexer/origin_intrinsic_metadata.yml
Environment=SWH_SENTRY_DSN=https://swh::deploy::indexer::sentry_token@sentry.softwareheritage.org/5
Environment=SWH_SENTRY_ENVIRONMENT=staging
Environment=SWH_MAIN_PACKAGE=swh.indexer
Environment=LOGLEVEL=INFO
ensure =>
- present
+ absent
*******************************************
+ Systemd::Dropin_file[swh-indexer-journal-client@origin_intrinsic_metadata_1.service/parameters.conf] =>
parameters =>
"content": "# Managed by puppet - modifications will be overwritten\n# In de...
"daemon_reload": "lazy",
"ensure": "present",
"filename": "parameters.conf",
"group": "root",
"mode": "0444",
"owner": "root",
"path": "/etc/systemd/system",
"selinux_ignore_defaults": false,
"show_diff": true,
"unit": "swh-indexer-journal-client@origin_intrinsic_metadata_1.service"
*******************************************
*** End octocatalog-diff on worker0.internal.staging.swh.network
Migrated from D8190 (view on Phabricator)