Staging instance, all changes can be removed at any time

Skip to content

production: Deploy compressed graph to kubernetes as main instance

Antoine R. Dumont requested to merge mr/production-switch-compressed-graph-to-kube into production

This triggers the change to use the kubernetes compressed graph instance for:

  • web-archive (archive.s.o)
  • provenance
  • vault

This still exposes granet's compressed graph instance through the webapp-postgres web instance.

To ease the maintenance, i've grouped together the graph configuration at the same location.

helm diff 
[swh] Comparing changes between branches production and mr/production-switch-compressed-graph-to-kube (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in mr/production-switch-compressed-graph-to-kube branch for environment staging...
[swh] Generate config in mr/production-switch-compressed-graph-to-kube branch for environment staging...
[swh] Generate config in mr/production-switch-compressed-graph-to-kube branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in mr/production-switch-compressed-graph-to-kube branch for environment production...
[swh] Generate config in mr/production-switch-compressed-graph-to-kube branch for environment production...
[swh] Generate config in mr/production-switch-compressed-graph-to-kube branch for environment production...


------------- diff for environment staging namespace swh -------------

No differences


------------- diff for environment staging namespace swh-cassandra -------------

No differences


------------- diff for environment staging namespace swh-cassandra-next-version -------------

No differences


------------- diff for environment production namespace swh -------------

--- /tmp/swh-chart.swh.5OCsaxv9/production-swh.before	2024-10-23 16:06:20.152604345 +0200
+++ /tmp/swh-chart.swh.5OCsaxv9/production-swh.after	2024-10-23 16:06:20.620604353 +0200
@@ -243,21 +243,21 @@
 # Source: swh/templates/alter/configmap.yaml
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: alter-template
   namespace: swh
 data:
   config.yml.template: |
     graph:
       timeout: null
-      url: http://graph.internal.softwareheritage.org:5009/graph
+      url: http://graph-rpc-20240331-ingress/graph
     recovery_bundles:
       secret_sharing:
         groups:
           legal:
             minimum_required_shares: 1
             recipient_keys:
               Ali: age123hpq9m25xsmx7caqvyv8k3fxaqastc3evyq9q7myur7l9ukj4dsnp7a5v
               Bob: age1mrhte5tlpzpz57gg85nzcefqc5pm5usmakqpuurxux7ry2rmhdgs7r9u68
           sysadmins:
             minimum_required_shares: 1
@@ -2419,21 +2419,21 @@
       cls: remote
       url: http://scheduler-rpc-ingress-swh-cassandra
     vault:
       cls: remote
       url: http://vault-rpc-ingress-swh-cassandra
     graph:
       max_edges:
         anonymous: 1000
         staff: 0
         user: 100000
-      server_url: http://graph-rpc-20240331-ingress/graph/
+      server_url: http://graph.internal.softwareheritage.org:5009/graph/
     indexer_storage:
       cls: remote
       url: http://indexer-storage-read-only-rpc-ingress-swh-cassandra
     counters_backend: swh-counters
     counters:
       cls: remote
       url: http://counters-rpc-ingress-swh-cassandra
     deposit:
       private_api_url: https://deposit.softwareheritage.org/1/private/
       private_api_user: ${DEPOSIT_USERNAME}
@@ -3080,21 +3080,21 @@
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: alter
       annotations:
         # Force a rollout upgrade if the configuration changes
-        checksum/config: 01ff1b3f8ca9de69c60593a28bc043baffa413696d99741901e43913f158499a
+        checksum/config: a12f5ae9e7b268239975f3820e4dcf05e6f6c440fea51b963ac61b6a7c122b9a
     spec:
       securityContext:
         fsGroup: 1000
       
       affinity:
         
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
@@ -6078,21 +6078,21 @@
       app: web-postgresql
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: web-postgresql
       annotations:
-        checksum/config: f21a42eaa7148a923ef943b32ff715cfe05aeb1c90a4c74c489e0ae2f109133f
+        checksum/config: 09e33427c0792e1c30a50df400bfccc5420c9acbdc87c8ca166bdf0c19cee199
         checksum/config-logging: 81fb24577eb1777be8690f58c1e92d701777fe4ff045bb8445feb924947b9f84
         checksum/config-utils: d75ca13b805bce6a8ab59c8e24c938f2283108f6a79134f6e71db86308651dc6
     spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/web
                 operator: In


------------- diff for environment production namespace swh-cassandra -------------

--- /tmp/swh-chart.swh.5OCsaxv9/production-swh-cassandra.before	2024-10-23 16:06:20.464604351 +0200
+++ /tmp/swh-chart.swh.5OCsaxv9/production-swh-cassandra.after	2024-10-23 16:06:20.940604359 +0200
@@ -337,21 +337,21 @@
         cls: azure-prefixed
         name: azure
       - cls: remote
         name: saam
         url: http://objstorage-ro-saam-zfs-rpc-ingress-swh-cassandra
       - cls: remote
         name: banco
         url: http://objstorage-ro-banco-xfs-rpc-ingress-swh-cassandra
       readonly: true
     graph:
-      url: http://graph.internal.softwareheritage.org:5009/graph
+      url: http://graph-rpc-20240331-ingress/graph
     max_bundle_size: 1073741824
     celery:
       task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@rabbitmq.internal.softwareheritage.org:5672/%2f
       task_acks_late: false
       task_modules:
         - swh.vault.cooking_tasks
       task_queues:
       - swh.vault.cooking_tasks.SWHBatchCookingTask
     
       sentry_settings_for_celery_tasks:
@@ -461,21 +461,21 @@
         cls: azure-prefixed
         name: azure
       - cls: remote
         name: saam
         url: http://objstorage-ro-saam-zfs-rpc-ingress-swh-cassandra
       - cls: remote
         name: banco
         url: http://objstorage-ro-banco-xfs-rpc-ingress-swh-cassandra
       readonly: true
     graph:
-      url: http://graph.internal.softwareheritage.org:5009/graph
+      url: http://graph-rpc-20240331-ingress/graph
     max_bundle_size: 1073741824
     celery:
       task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@rabbitmq.internal.softwareheritage.org:5672/%2f
       task_acks_late: true
       task_modules:
         - swh.vault.cooking_tasks
       task_queues:
       - swh.vault.cooking_tasks.SWHCookingTask
     
       sentry_settings_for_celery_tasks:
@@ -7106,21 +7106,21 @@
   namespace: swh-cassandra
   name: provenance-graph-granet-configuration-template
 data:
   config.yml.template: |
     provenance:
       cls: known_swhid_filter
       filter_licenses: false
       provenance:
         cls: graph
         max_edges: 100000
-        url: graph.internal.softwareheritage.org:50091
+        url: graph-grpc-20240331-ingress:80
 ---
 # Source: swh/templates/provenance/configmap.yaml
 apiVersion: v1
 kind: ConfigMap
 metadata:
   namespace: swh-cassandra
   name: provenance-graph-granet-configuration-logging
 data:
   logging-gunicorn.json: |
     {
@@ -9444,21 +9444,21 @@
       cls: remote
       url: http://scheduler-rpc-ingress-swh-cassandra
     vault:
       cls: remote
       url: http://vault-rpc-ingress-swh-cassandra
     graph:
       max_edges:
         anonymous: 1000
         staff: 0
         user: 100000
-      server_url: http://graph.internal.softwareheritage.org:5009/graph/
+      server_url: http://graph-rpc-20240331-ingress/graph/
     indexer_storage:
       cls: remote
       url: http://indexer-storage-read-only-rpc-ingress-swh-cassandra
     counters_backend: swh-counters
     counters:
       cls: remote
       url: http://counters-rpc-ingress-swh-cassandra
     deposit:
       private_api_url: https://deposit.softwareheritage.org/1/private/
       private_api_user: ${DEPOSIT_USERNAME}
@@ -10851,21 +10851,21 @@
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: cooker-batch
       annotations:
         # Force a rollout upgrade if the configuration changes
-        checksum/config: aa56ca2d4ce650222df2c8706431540831d9446a7557c42f21cea7e446755f43
+        checksum/config: e8bf953def8446b28c03cfa9c2921777cc0ff6a025f6e99a117a25f0b929e770
     spec:
       affinity:
         
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/cooker
                 operator: In
                 values:
@@ -11090,21 +11090,21 @@
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: cooker-simple
       annotations:
         # Force a rollout upgrade if the configuration changes
-        checksum/config: aa56ca2d4ce650222df2c8706431540831d9446a7557c42f21cea7e446755f43
+        checksum/config: e8bf953def8446b28c03cfa9c2921777cc0ff6a025f6e99a117a25f0b929e770
     spec:
       affinity:
         
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/cooker
                 operator: In
                 values:
@@ -23027,21 +23027,21 @@
       app: provenance-graph-granet
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: provenance-graph-granet
       annotations:
-        checksum/config: efc6933069d30442c937aeedba25e7a0c2555e1ca65aea52be04b618e9d33d40
+        checksum/config: aed387babd93015157fd884294f2ed7cd99f9747c628794449e050d03e59b056
         checksum/config-logging: ddcd27d991938c46f4fc0ad7ee028cb3005f186b3db022596c9ae94363881e4f
         checksum/config-utils: 13a26f6add17e96ce01550153c77dcd48de60241a3f4db3c93d5467234be2a7f
     spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/rpc
                 operator: In
@@ -26757,21 +26757,21 @@
       app: web-archive
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: web-archive
       annotations:
-        checksum/config: 9f6038d63c8dec98af392ca46c9aaaaa491b261d76bd9d036974ac1588ae073a
+        checksum/config: ed1c0975463dd9284cd20ff8953c98d31a2ded8761d2c86534914f59a06ffa2d
         checksum/config-logging: af7bf52757798a2fcd4c237ed3de9df87c15b7f38419128a8d67d02b8a485097
         checksum/config-utils: 13a26f6add17e96ce01550153c77dcd48de60241a3f4db3c93d5467234be2a7f
     spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/web
                 operator: In

Refs. swh/infra/sysadm-environment#5465

Edited by Antoine R. Dumont

Merge request reports