Staging instance, all changes can be removed at any time

Skip to content

staging: Migrate services to use the dynamic counters service

Antoine R. Dumont requested to merge migrate-to-counters-dynamic into production
make swh-helm-diff 
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
Switched to branch 'migrate-to-counters-dynamic'
[swh] Generate config in migrate-to-counters-dynamic branch for environment staging...
[swh] Generate config in migrate-to-counters-dynamic branch for environment staging...
[swh] Generate config in migrate-to-counters-dynamic branch for environment staging...
Switched to branch 'production'
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
Switched to branch 'migrate-to-counters-dynamic'
[swh] Generate config in migrate-to-counters-dynamic branch for environment production...
[swh] Generate config in migrate-to-counters-dynamic branch for environment production...
[swh] Generate config in migrate-to-counters-dynamic branch for environment production...


------------- diff for environment staging namespace swh -------------

--- /tmp/swh-chart.swh.HhHwDa13/staging-swh.before      2023-12-12 12:00:13.949222897 +0100
+++ /tmp/swh-chart.swh.HhHwDa13/staging-swh.after       2023-12-12 12:00:14.709224812 +0100
@@ -4600,21 +4600,21 @@
       url: http://scheduler.internal.staging.swh.network
     vault:
       cls: remote
       url: http://vault-rpc-ingress
     indexer_storage:
       cls: remote
       url: http://indexer-storage-rpc-ingress
     counters_backend: swh-counters
     counters:
       cls: remote
-      url: http://counters0.internal.staging.swh.network:5011/
+      url: http://counters-rpc-ingress
     deposit:
       private_api_url: https://deposit-dynamic.internal.staging.swh.network/1/private/
       private_api_user: ${DEPOSIT_USERNAME}
       private_api_password: ${DEPOSIT_PASSWORD}
     add_forge_now:
       email_address: add-forge-now@webapp.staging.swh.network
       gitlab_pipeline:
         token: ${GITLAB_AFN_TOKEN}
         trigger_url: https://gitlab.softwareheritage.org/api/v4/projects/474/trigger/pipeline

@@ -4655,21 +4655,21 @@
           limiter_rate:
             default: 700/m
         swh_save_origin:
           limiter_rate:
             POST: 10/h
             default: 120/h
     content_display_max_size: 5242880
     give:
       public_key: ${GIVE_PUBLIC_KEY}
       token: ${GIVE_PRIVATE_TOKEN}
-    history_counters_url: http://counters0.internal.staging.swh.network:5011/counters_history/history.json
+    history_counters_url: http://counters-rpc-ingress/counters_history/history.json
     keycloak:
       realm_name: SoftwareHeritageStaging
       server_url: https://auth.softwareheritage.org/auth/
     matomo: {}
     search_config:
       metadata_backend: swh-search
     swh_extra_django_apps:
     - swh.web.add_forge_now
     - swh.web.archive_coverage
     - swh.web.badges
@@ -24246,21 +24246,21 @@
       app: web
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: web
       annotations:
-        checksum/config: c86cf64156c3f0ae4cad1bcf379bd04bd190a330c595cd92f7540c6493689808
+        checksum/config: a59de8d2746e93c1b89ec4c6e62d78db404fcb11b9d1ddf9fd6f60d7b4e72074
     spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/web
                 operator: In
                 values:
                 - "true"


------------- diff for environment staging namespace swh-cassandra -------------

--- /tmp/swh-chart.swh.HhHwDa13/staging-swh-cassandra.before    2023-12-12 12:00:14.197223522 +0100
+++ /tmp/swh-chart.swh.HhHwDa13/staging-swh-cassandra.after     2023-12-12 12:00:14.949225417 +0100
@@ -344,21 +344,21 @@
 # Source: swh/templates/counters/journal-client-configmap.yaml
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: counters-journal-client-configuration-template
   namespace: swh-cassandra
 data:
   config.yml.template: |
     counters:
       cls: remote
-      url: http://counters0.internal.staging.swh.network:5011/
+      url: http://counters-rpc-ingress
     journal:
       brokers:
         - journal1.internal.staging.swh.network
         - journal2.internal.staging.swh.network
       group_id: swh.counters.journal_client
       message.max.bytes: 524288000
       object_types:
       - content
       - directory
       - origin
@@ -416,21 +416,21 @@
     trap "rm -f ${tmp_file}" EXIT

     cat >"${tmp_file}" <<EOF
     {
         "cache_file": "${cache_file}",
         "objects": ["content", "origin", "revision"]
         ${static_file_stanza}
     }
     EOF

-    curl -s -XPOST -H 'Content-Type: application/json' http://counters0.internal.staging.swh.network:5011//refresh_history -d @"${tmp_file}"
+    curl -s -XPOST -H 'Content-Type: application/json' http://counters-rpc-ingress/refresh_history -d @"${tmp_file}"

   fetch-static-history.sh: |
     #!/bin/bash

     set -ex

     static_history_path=$1

     url="https://gitlab.softwareheritage.org/swh/devel/swh-counters/-/snippets/1617/raw/main/snippetfile1.txt?inline=false"

@@ -4533,21 +4533,21 @@
       url: http://scheduler.internal.staging.swh.network
     vault:
       cls: remote
       url: http://vault-rpc-ingress
     indexer_storage:
       cls: remote
       url: http://indexer-storage-rpc-ingress
     counters_backend: swh-counters
     counters:
       cls: remote
-      url: http://counters0.internal.staging.swh.network:5011/
+      url: http://counters-rpc-ingress
     deposit:
       private_api_url: https://deposit-dynamic.internal.staging.swh.network/1/private/
       private_api_user: ${DEPOSIT_USERNAME}
       private_api_password: ${DEPOSIT_PASSWORD}
     add_forge_now:
       email_address: add-forge-now@webapp.staging.swh.network
       gitlab_pipeline:
         token: ${GITLAB_AFN_TOKEN}
         trigger_url: https://gitlab.softwareheritage.org/api/v4/projects/474/trigger/pipeline

@@ -4588,21 +4588,21 @@
           limiter_rate:
             default: 700/m
         swh_save_origin:
           limiter_rate:
             POST: 10/h
             default: 120/h
     content_display_max_size: 5242880
     give:
       public_key: ${GIVE_PUBLIC_KEY}
       token: ${GIVE_PRIVATE_TOKEN}
-    history_counters_url: http://counters0.internal.staging.swh.network:5011/counters_history/history.json
+    history_counters_url: http://counters-rpc-ingress/counters_history/history.json
     keycloak:
       realm_name: SoftwareHeritageStaging
       server_url: https://auth.softwareheritage.org/auth/
     matomo: {}
     save_code_now_webhook_secret: ${WEBHOOKS_SECRET}
     search_config:
       metadata_backend: swh-search
     swh_extra_django_apps:
     - swh.web.add_forge_now
     - swh.web.archive_coverage
@@ -15098,21 +15098,21 @@
       app: counters-journal-client
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: counters-journal-client
       annotations:
-        checksum/config: 6597d6214907697304099de2ec36ccb1799ae3bdd8a16945276817815822d13e
+        checksum/config: 8684d54fb0348609ff5a063c50a3614abd36b180af87d773eac1c45f3c015f70
     spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/journal_client
                 operator: In
                 values:
                 - "true"
@@ -15193,21 +15193,21 @@
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: counters-rpc
       annotations:
         checksum/configmap: d974568e5cc0e38a7f87d277d3b43fd143eb758c1b94089ed5755b4d353d802b
-        checksum/script-utils: d03e38335b3ef317dd10c9e7e4065a6fa7a43f8fa7a6c0d10bea4a9cdc6ac3ba
+        checksum/script-utils: 85c87a276d201f60e05738729ca91996c0e04d942f3399676ecef39c610e96eb
     spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/rpc
                 operator: In
                 values:
                 - "true"
@@ -22889,21 +22889,21 @@
       app: web
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: web
       annotations:
-        checksum/config: 8cb91c6e3702f6973acf9df3fa88496f985248a20da5228f472beecaddd05822
+        checksum/config: ac673a4a5fdd310be2aa60f8d95f8c80c68a04225dfa0a1d93f479ae3939ea56
     spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/web
                 operator: In
                 values:
                 - "true"


------------- diff for environment staging namespace swh-cassandra-next-version -------------

--- /tmp/swh-chart.swh.HhHwDa13/staging-swh-cassandra-next-version.before       2023-12-12 12:00:14.441224136 +0100
+++ /tmp/swh-chart.swh.HhHwDa13/staging-swh-cassandra-next-version.after        2023-12-12 12:00:15.189226022 +0100
@@ -344,21 +344,21 @@
 # Source: swh/templates/counters/journal-client-configmap.yaml
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: counters-journal-client-configuration-template
   namespace: swh-cassandra-next-version
 data:
   config.yml.template: |
     counters:
       cls: remote
-      url: http://counters0.internal.staging.swh.network:5011/
+      url: http://counters-rpc-ingress
     journal:
       brokers:
         - journal1.internal.staging.swh.network
         - journal2.internal.staging.swh.network
       group_id: swh.counters.journal_client
       message.max.bytes: 524288000
       object_types:
       - content
       - directory
       - origin
@@ -416,21 +416,21 @@
     trap "rm -f ${tmp_file}" EXIT

     cat >"${tmp_file}" <<EOF
     {
         "cache_file": "${cache_file}",
         "objects": ["content", "origin", "revision"]
         ${static_file_stanza}
     }
     EOF

-    curl -s -XPOST -H 'Content-Type: application/json' http://counters0.internal.staging.swh.network:5011//refresh_history -d @"${tmp_file}"
+    curl -s -XPOST -H 'Content-Type: application/json' http://counters-rpc-ingress/refresh_history -d @"${tmp_file}"

   fetch-static-history.sh: |
     #!/bin/bash

     set -ex

     static_history_path=$1

     url="https://gitlab.softwareheritage.org/swh/devel/swh-counters/-/snippets/1617/raw/main/snippetfile1.txt?inline=false"

@@ -4045,21 +4045,21 @@
       url: http://scheduler.internal.staging.swh.network
     vault:
       cls: remote
       url: http://vault-rpc-ingress-next-version
     indexer_storage:
       cls: remote
       url: http://indexer-storage-rpc-ingress
     counters_backend: swh-counters
     counters:
       cls: remote
-      url: http://counters0.internal.staging.swh.network:5011/
+      url: http://counters-rpc-ingress
     deposit:
       private_api_url: https://deposit-dynamic.internal.staging.swh.network/1/private/
       private_api_user: ${DEPOSIT_USERNAME}
       private_api_password: ${DEPOSIT_PASSWORD}
     add_forge_now:
       email_address: add-forge-now@webapp.staging.swh.network
       gitlab_pipeline:
         token: ${GITLAB_AFN_TOKEN}
         trigger_url: https://gitlab.softwareheritage.org/api/v4/projects/474/trigger/pipeline

@@ -4100,21 +4100,21 @@
           limiter_rate:
             default: 700/m
         swh_save_origin:
           limiter_rate:
             POST: 10/h
             default: 120/h
     content_display_max_size: 5242880
     give:
       public_key: ${GIVE_PUBLIC_KEY}
       token: ${GIVE_PRIVATE_TOKEN}
-    history_counters_url: http://counters0.internal.staging.swh.network:5011/counters_history/history.json
+    history_counters_url: http://counters-rpc-ingress/counters_history/history.json
     keycloak:
       realm_name: SoftwareHeritageStaging
       server_url: https://auth.softwareheritage.org/auth/
     matomo: {}
     save_code_now_webhook_secret: ${WEBHOOKS_SECRET}
     search_config:
       metadata_backend: swh-search
     swh_extra_django_apps:
     - swh.web.add_forge_now
     - swh.web.archive_coverage
@@ -14539,21 +14539,21 @@
       app: counters-journal-client
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: counters-journal-client
       annotations:
-        checksum/config: e3a1e7bbe7ee89f5b9e19210728429e94c74e18c0f069fd3b3e1f04f2e56dcbf
+        checksum/config: a3e21a503756c79ebc2aa0af017caa41a8568f9a186e91dfbb1d2d93045dbc4e
     spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/journal_client
                 operator: In
                 values:
                 - "true"
@@ -14634,21 +14634,21 @@
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: counters-rpc
       annotations:
         checksum/configmap: c09a1ed2efd89f46b09f8353e42e2135afbc1c6aaa9ee9fc8ac33992c359a49d
-        checksum/script-utils: 5c95764c23110e4be51e081ef12725bd48ec8f5ab51dfd443d29e5d918de9a2b
+        checksum/script-utils: 09bfb564fd293c316f1d03286a927e80ecc505e9d3058453c5262321b297a8cb
     spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/rpc
                 operator: In
                 values:
                 - "true"
@@ -21065,21 +21065,21 @@
       app: web
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: web
       annotations:
-        checksum/config: bc2c1e4c6d687d115cabc72e6fa78320c9eff911814c0ad0738b2c2753b17089
+        checksum/config: 1abcbbdf0d471db284dff6bef6b8999ab9a1f5ac88158401be759b6d0c2df0f0
     spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/web
                 operator: In
                 values:
                 - "true"


------------- diff for environment production namespace swh -------------

No differences


------------- diff for environment production namespace swh-cassandra -------------

No differences

Refs. swh/infra/sysadm-environment#5179 (closed)

Edited by Antoine R. Dumont

Merge request reports