Staging instance, all changes can be removed at any time

Skip to content

indexer-storage: move journal_writer to the proper indent level

Nicolas Dandrimont requested to merge mr/fix-journal-writers into production

The journal_writer config needs to be under the indexer_storage key, rather than at the toplevel.

Also give a per-pod clientId to all journal writers, instead of a hardcoded value for all pods.

Noticed while working on swh/infra/sysadm-environment#5200 (closed)

helm diff
------------- diff for environment staging namespace swh -------------

--- /tmp/swh-chart.swh.fNFN4Av3/staging-swh.before	2024-01-11 19:50:49.718377187 +0100
+++ /tmp/swh-chart.swh.fNFN4Av3/staging-swh.after	2024-01-11 19:50:50.126382763 +0100
@@ -168,29 +168,28 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
   namespace: swh
   name: indexer-storage-rpc-configuration-template
 data:
   config.yml.template: |
     indexer_storage:
       cls: postgresql
       db: host=db1.internal.staging.swh.network port=5432 user=swh-indexer dbname=swh-indexer password=${POSTGRESQL_PASSWORD}
-    journal_writer:
-      
-      client_id: swh.idx_storage.journal_writer.storage1
-      cls: kafka
-      prefix: swh.journal.indexed
-      producer_config:
-        message.max.bytes: 1000000000
-      brokers:
-      - journal2.internal.staging.swh.network
+      journal_writer:
+        client_id: swh.idx_storage.journal_writer.${HOSTNAME}
+        cls: kafka
+        prefix: swh.journal.indexed
+        producer_config:
+          message.max.bytes: 1000000000
+        brokers:
+        - journal2.internal.staging.swh.network
 ---
 # Source: swh/templates/indexers/configmap-utils.yaml
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: indexer-utils
   namespace: swh
 data:
   pre-stop-idempotent.sh: |
     #!/bin/bash
@@ -4009,21 +4008,21 @@
         db: host=db1.internal.staging.swh.network port=5432 user=swh dbname=swh password=${POSTGRESQL_PASSWORD}
         objstorage:
           cls: remote
           url: http://objstorage-read-write-rpc-ingress
       
         journal_writer:
           cls: kafka
           brokers:
           - journal2.internal.staging.swh.network
           prefix: swh.journal.objects
-          client_id: swh.storage.journal_writer.storage1
+          client_id: swh.storage.journal_writer.${HOSTNAME}
           anonymize: true
           producer_config:
             message.max.bytes: 1000000000
 ---
 # Source: swh/templates/toolbox/configmap.yaml
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: toolbox-indexer-storage-template
   namespace: swh
@@ -5062,21 +5061,21 @@
       app: indexer-storage-rpc
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: indexer-storage-rpc
       annotations:
-        checksum/config: ac4c875d8a03fa39a48eed188b4049cbcabbd435f51707fd069335ffcad9ac39
+        checksum/config: 56ab19e613bf228521fd9f136904bfea0fbbd7edff68e66e66bc98b0ab1e1188
         checksum/database-utils: 931b7f92b50eae0ecb6203c3e0e48f7c363a99ac0064451187dc12c6eaf0a793
     spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/rpc
                 operator: In
                 values:
@@ -13789,21 +13788,21 @@
       app: storage-postgresql-read-write
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: storage-postgresql-read-write
       annotations:
-        checksum/config: f41bf53c1a3df2d84c89b62efa58ba4ca7d74c9d1eb5265c7ad056fd62de5e42
+        checksum/config: 2bc23e8868305ed4ab563a7338240a0d309419477d87bbcdc0738acb0a123712
         checksum/database-utils: 931b7f92b50eae0ecb6203c3e0e48f7c363a99ac0064451187dc12c6eaf0a793
         checksum/config-utils: 0cce256531366ac8ea2a0bde2a8f10937284a0f599f45fabe6babbbc096e179d
     spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/storage
                 operator: In


------------- diff for environment staging namespace swh-cassandra -------------

--- /tmp/swh-chart.swh.fNFN4Av3/staging-swh-cassandra.before	2024-01-11 19:50:49.854379046 +0100
+++ /tmp/swh-chart.swh.fNFN4Av3/staging-swh-cassandra.after	2024-01-11 19:50:50.262384621 +0100
@@ -4017,21 +4017,21 @@
       directory_entries_insert_algo: batch
       objstorage:
         cls: remote
         url: http://objstorage-read-write-rpc-ingress
       
       journal_writer:
         cls: kafka
         brokers:
         - journal2.internal.staging.swh.network
         prefix: swh.journal.objects
-        client_id: swh.storage-cassandra.journal_writer.storage
+        client_id: swh.storage-cassandra.journal_writer.${HOSTNAME}
         anonymize: true
         producer_config:
           message.max.bytes: 1000000000
 ---
 # Source: swh/templates/toolbox/configmap.yaml
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: toolbox-scrubber-storage-template
   namespace: swh-cassandra
@@ -12472,21 +12472,21 @@
       app: storage-cassandra
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: storage-cassandra
       annotations:
-        checksum/config: b0f34bb659c89e7c81f94018039c5c6c358e56f57abcbb876f49c2c8f804b8be
+        checksum/config: 63337dd276105a1988c8e077eac3ea9fe3918d28fcb42cb720e0674a18d21838
         checksum/database-utils: 367f28d8854b3b188d7a45015752b10be8e691f0c3b2a21db02501e655b932f0
         checksum/config-utils: 6293d71ddd4a7f401d1105bc7f7590fcead5a3a0c152fe46d7d407839ea9a1d6
     spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/storage
                 operator: In


------------- diff for environment staging namespace swh-cassandra-next-version -------------

--- /tmp/swh-chart.swh.fNFN4Av3/staging-swh-cassandra-next-version.before	2024-01-11 19:50:49.974380685 +0100
+++ /tmp/swh-chart.swh.fNFN4Av3/staging-swh-cassandra-next-version.after	2024-01-11 19:50:50.386386315 +0100
@@ -3689,21 +3689,21 @@
       directory_entries_insert_algo: batch
       objstorage:
         cls: remote
         url: http://objstorage-read-write-rpc-ingress
       
       journal_writer:
         cls: kafka
         brokers:
         - journal2.internal.staging.swh.network
         prefix: swh.journal.objects
-        client_id: swh.storage-cassandra.journal_writer.storage
+        client_id: swh.storage-cassandra.journal_writer.${HOSTNAME}
         anonymize: true
         producer_config:
           message.max.bytes: 1000000000
 ---
 # Source: swh/templates/utils/config-utils.yaml
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: config-utils
   namespace: swh-cassandra-next-version
@@ -10790,21 +10790,21 @@
       app: storage-cassandra
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: storage-cassandra
       annotations:
-        checksum/config: 5ddeb73cd2cdd437e49835607a9a59ad5d7fc4b4feb26369c6711e407db5f965
+        checksum/config: 2cddfaac605a2f5e9a8689326082009456503eb4d7792e92fa26a9260659cb31
         checksum/database-utils: dd7d83d547731ea7598fa298565f3b1fe41eb657ae93bab75e15b722dbcc71ae
         checksum/config-utils: 00e1b490a5d09e6ab4c9b40bae85197b945f800ba39c757d5c4c37b1be3f0193
     spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/storage
                 operator: In


------------- diff for environment production namespace swh -------------

No differences


------------- diff for environment production namespace swh-cassandra -------------

No differences
Edited by Nicolas Dandrimont

Merge request reports

Loading