staging/objstorage: Deploy read-write multiplexer objstorage
This will continue to read from storage1's (read-only) objstorage but write to the new db1's objstorage. This readapts the configuration of the read-write db1 objstorage to use no compression (when storing blobs) since zfs's dataset has already a compression set.
This does not decommission yet the storage1's read-write objstorage (soon to be decommissionned).
Visual graph [1]
[1] swh/infra/sysadm-environment#5260 (comment 167017)
helm diff
[swh] Comparing changes between branches production and deploy-new-objstorage-rpc (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in deploy-new-objstorage-rpc branch for environment staging...
[swh] Generate config in deploy-new-objstorage-rpc branch for environment staging...
[swh] Generate config in deploy-new-objstorage-rpc branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in deploy-new-objstorage-rpc branch for environment production...
[swh] Generate config in deploy-new-objstorage-rpc branch for environment production...
[swh] Generate config in deploy-new-objstorage-rpc branch for environment production...
------------- diff for environment staging namespace swh -------------
--- /tmp/swh-chart.swh.P7QSQfJE/staging-swh.before 2024-03-05 12:13:59.441249995 +0100
+++ /tmp/swh-chart.swh.P7QSQfJE/staging-swh.after 2024-03-05 12:14:00.281248631 +0100
@@ -11128,20 +11128,30 @@
name: objstorage-db1-rw-rpc-ingress
namespace: swh
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
+ name: objstorage-multiplexer-read-write-rpc-ingress
+ namespace: swh
+spec:
+ type: ExternalName
+ externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
name: objstorage-read-only-rpc-ingress
namespace: swh
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
------------- diff for environment staging namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.P7QSQfJE/staging-swh-cassandra.before 2024-03-05 12:13:59.801249410 +0100
+++ /tmp/swh-chart.swh.P7QSQfJE/staging-swh-cassandra.after 2024-03-05 12:14:00.629248066 +0100
@@ -4181,20 +4181,21 @@
# Source: swh/templates/objstorage/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh-cassandra
name: objstorage-db1-read-write-configuration-template
data:
config.yml.template: |
objstorage:
cls: pathslicing
+ compression: none
root: /srv/softwareheritage/objects
slicing: 0:1/1:5
client_max_size: 1073741824
---
# Source: swh/templates/objstorage/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh-cassandra
name: objstorage-multiplexer-read-only-configuration-template
@@ -4207,20 +4208,40 @@
url: http://objstorage-storage1-ro-rpc-ingress
- cls: remote
url: http://objstorage-db1-ro-rpc-ingress
client_max_size: 1073741824
---
# Source: swh/templates/objstorage/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh-cassandra
+ name: objstorage-multiplexer-read-write-configuration-template
+data:
+ config.yml.template: |
+ objstorage:
+ cls: multiplexer
+ objstorages:
+ - cls: remote
+ url: http://objstorage-db1-rw-rpc-ingress
+ - cls: filtered
+ filters_conf:
+ - type: readonly
+ storage_conf:
+ cls: remote
+ url: http://objstorage-storage1-ro-rpc-ingress
+---
+# Source: swh/templates/objstorage/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
name: objstorage-storage1-read-only-configuration-template
data:
config.yml.template: |
objstorage:
cls: pathslicing
root: /srv/softwareheritage/objects
slicing: 0:1/1:5
client_max_size: 1073741824
---
# Source: swh/templates/objstorage/configmap.yaml
@@ -15309,20 +15330,30 @@
name: objstorage-db1-rw-rpc-ingress
namespace: swh-cassandra
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
+ name: objstorage-multiplexer-read-write-rpc-ingress
+ namespace: swh-cassandra
+spec:
+ type: ExternalName
+ externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
name: objstorage-read-only-rpc-ingress
namespace: swh-cassandra
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
@@ -15485,20 +15516,37 @@
app: objstorage-multiplexer-read-only
ports:
- port: 5003
targetPort: 5003
name: rpc
---
# Source: swh/templates/objstorage/service.yaml
apiVersion: v1
kind: Service
metadata:
+ name: objstorage-multiplexer-read-write
+ namespace: swh-cassandra
+ labels:
+ app: objstorage-multiplexer-read-write
+spec:
+ type: ClusterIP
+ selector:
+ app: objstorage-multiplexer-read-write
+ ports:
+ - port: 5003
+ targetPort: 5003
+ name: rpc
+---
+# Source: swh/templates/objstorage/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
name: objstorage-storage1-read-only
namespace: swh-cassandra
labels:
app: objstorage-storage1-read-only
spec:
type: ClusterIP
selector:
app: objstorage-storage1-read-only
ports:
- port: 5003
@@ -23295,21 +23343,21 @@
app: objstorage-db1-read-write
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: objstorage-db1-read-write
annotations:
- checksum/config: ebbca3b234acf3d6ad0a56a61a22552a027d875412f3697f9fdbf633101892c5
+ checksum/config: dcc009da7ad006e7d1802580ae7660d91805b7350f9dbbec41e40bcb9d9177b6
checksum/config-utils: 13a26f6add17e96ce01550153c77dcd48de60241a3f4db3c93d5467234be2a7f
spec:
nodeSelector:
kubernetes.io/hostname: db1
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/objstorage
@@ -23539,20 +23587,145 @@
- name: config-utils
configMap:
name: config-utils
defaultMode: 0555
---
# Source: swh/templates/objstorage/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
namespace: swh-cassandra
+ name: objstorage-multiplexer-read-write
+ labels:
+ app: objstorage-multiplexer-read-write
+spec:
+ revisionHistoryLimit: 2
+ replicas: 2
+ selector:
+ matchLabels:
+ app: objstorage-multiplexer-read-write
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: objstorage-multiplexer-read-write
+ annotations:
+ checksum/config: 5f358f8854bf585d6209efbb0eb0fe3dca962c477ec7cd450c101690ed0d9b1e
+ checksum/config-utils: 13a26f6add17e96ce01550153c77dcd48de60241a3f4db3c93d5467234be2a7f
+ spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/objstorage
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-frontend-rpc
+ initContainers:
+ - name: prepare-configuration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/prepare-configuration.sh
+ env:
+
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ - name: config-utils
+ mountPath: /entrypoints
+ readOnly: true
+ containers:
+ - name: objstorage-multiplexer-read-write
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/objstorage:20240222.1
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 5003
+ name: rpc
+ readinessProbe:
+ httpGet:
+ path: /
+ port: rpc
+ initialDelaySeconds: 15
+ failureThreshold: 30
+ periodSeconds: 5
+ livenessProbe:
+ tcpSocket:
+ port: rpc
+ initialDelaySeconds: 10
+ periodSeconds: 5
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - /opt/swh/entrypoint.sh
+ env:
+ - name: THREADS
+ value: "2"
+ - name: WORKERS
+ value: "4"
+ - name: TIMEOUT
+ value: "60"
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: LOG_LEVEL
+ value: "INFO"
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.objstorage
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: objstorage-sentry-dsn
+ # 'name' secret should exist & include key
+ # if the setting doesn't exist, sentry pushes will be disabled
+ optional: true
+ - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+ value: "true"
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: objstorage-multiplexer-read-write-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: config-utils
+ configMap:
+ name: config-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/objstorage/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: swh-cassandra
name: objstorage-storage1-read-only
labels:
app: objstorage-storage1-read-only
spec:
revisionHistoryLimit: 2
replicas: 2
selector:
matchLabels:
app: objstorage-storage1-read-only
strategy:
@@ -28350,20 +28523,50 @@
service:
name: objstorage-multiplexer-read-only
port:
number: 5003
---
# Source: swh/templates/objstorage/ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
namespace: swh-cassandra
+ name: objstorage-multiplexer-read-write-ingress-default
+ labels:
+ app: objstorage-multiplexer-read-write
+ endpoint-definition: default
+ annotations:
+ nginx.ingress.kubernetes.io/service-upstream: "true"
+ nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16,192.168.100.29/32,192.168.130.0/24,192.168.50.0/24
+ nginx.ingress.kubernetes.io/client-body-buffer-size: 128K
+ nginx.ingress.kubernetes.io/proxy-body-size: 4G
+ nginx.ingress.kubernetes.io/proxy-buffering: "on"
+
+spec:
+ ingressClassName: nginx
+ rules:
+ - host: objstorage-multiplexer-read-write-rpc-ingress
+ http:
+ paths:
+ - path: /
+ pathType: Prefix
+ backend:
+ service:
+ name: objstorage-multiplexer-read-write
+ port:
+ number: 5003
+---
+# Source: swh/templates/objstorage/ingress.yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ namespace: swh-cassandra
name: objstorage-storage1-read-only-ingress-default
labels:
app: objstorage-storage1-read-only
endpoint-definition: default
annotations:
nginx.ingress.kubernetes.io/service-upstream: "true"
nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16,192.168.100.29/32,192.168.130.0/24,192.168.50.0/24
nginx.ingress.kubernetes.io/client-body-buffer-size: 128K
nginx.ingress.kubernetes.io/proxy-body-size: 4G
nginx.ingress.kubernetes.io/proxy-buffering: "on"
------------- diff for environment staging namespace swh-cassandra-next-version -------------
--- /tmp/swh-chart.swh.P7QSQfJE/staging-swh-cassandra-next-version.before 2024-03-05 12:14:00.041249021 +0100
+++ /tmp/swh-chart.swh.P7QSQfJE/staging-swh-cassandra-next-version.after 2024-03-05 12:14:00.861247689 +0100
@@ -11503,20 +11503,30 @@
name: objstorage-db1-rw-rpc-ingress
namespace: swh-cassandra-next-version
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
+ name: objstorage-multiplexer-read-write-rpc-ingress
+ namespace: swh-cassandra-next-version
+spec:
+ type: ExternalName
+ externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
name: objstorage-read-only-rpc-ingress
namespace: swh-cassandra-next-version
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
------------- diff for environment production namespace swh -------------
No differences
------------- diff for environment production namespace swh-cassandra -------------
No differencesEdited by Antoine R. Dumont