swh/scrubber: Support the storage scrubbers deployment
Related to swh/infra/sysadm-environment#5145 (closed)
create 2 scrubbers per object type per environment, 1 for the reference checks and 1 for the hashes.
It will allow to scale differently by type of check.
helm diff
[swh] Comparing changes between branches production and storage-scrubber-staging (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in storage-scrubber-staging branch for environment staging...
[swh] Generate config in storage-scrubber-staging branch for environment staging...
[swh] Generate config in storage-scrubber-staging branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in storage-scrubber-staging branch for environment production...
[swh] Generate config in storage-scrubber-staging branch for environment production...
[swh] Generate config in storage-scrubber-staging branch for environment production...
------------- diff for environment staging namespace swh -------------
--- /tmp/swh-chart.swh.ShLIAK8u/staging-swh.before 2023-11-13 13:52:55.290863918 +0100
+++ /tmp/swh-chart.swh.ShLIAK8u/staging-swh.after 2023-11-13 13:52:56.130867191 +0100
@@ -3295,20 +3295,148 @@
- journal2.internal.staging.swh.network:9094
cls: kafka
group_id: swh-archive-stg-journalchecker
on_eof: restart
prefix: swh.journal.objects
sasl.mechanism: SCRAM-SHA-512
sasl.password: ${BROKER_USER_PASSWORD}
sasl.username: swh-archive-stg
security.protocol: SASL_SSL
---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-directory-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=guest dbname=swh password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-directory-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=guest dbname=swh password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-release-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=guest dbname=swh password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-release-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=guest dbname=swh password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-revision-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=guest dbname=swh password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-revision-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=guest dbname=swh password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-snapshot-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=guest dbname=swh password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-snapshot-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=guest dbname=swh password=${POSTGRESQL_PASSWORD}
+---
# Source: swh/templates/search/journal-client-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: search-journal-client-indexed-configuration-template
namespace: swh
data:
config.yml.template: |
search:
cls: remote
@@ -4107,20 +4235,36 @@
prefix: swh.journal.objects
sasl.mechanism: SCRAM-SHA-512
sasl.password: ${BROKER_USER_PASSWORD}
sasl.username: swh-archive-stg
security.protocol: SASL_SSL
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
+ name: toolbox-scrubber-storage-template
+ namespace: swh
+data:
+ config.yml.template: |
+ storage:
+
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=guest dbname=swh password=${POSTGRESQL_PASSWORD}
+ scrubber:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/toolbox/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
name: toolbox-storage-template
namespace: swh
data:
config.yml.template: |
storage:
cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh dbname=swh password=${POSTGRESQL_PASSWORD}
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
@@ -4249,20 +4393,35 @@
/opt/swh/bin/check-db-version.sh scrubber-journal
migrate-scrubber-journal-db-version.sh: |
#!/bin/bash
set -eu
/opt/swh/bin/migrate-db-version.sh scrubber-journal
+ check-scrubber-storage-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ /opt/swh/bin/check-db-version.sh scrubber-storage
+
+ migrate-scrubber-storage-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ /opt/swh/bin/migrate-db-version.sh scrubber-storage
+
+
check-storage-db-version.sh: |
#!/bin/bash
set -eu
/opt/swh/bin/check-db-version.sh storage
migrate-storage-db-version.sh: |
#!/bin/bash
@@ -4343,21 +4502,23 @@
TEMP_FILE=/tmp/db-version.txt
if [ -z ${MODULE} ]; then
echo The env variable must be defined with the module to check
echo for example "storage"
exit 1
fi
# extracting the postgresql configuration from a full configuration
# possibly with a pipeline
- python /entrypoints/extract-storage-postgresql-config-py
+ set +e
+ python /entrypoints/extract-storage-postgresql-config-py || exit 0
+ set -e
# checking the database status
swh db --config-file=/tmp/config.yml version "${MODULE}" | tee "${TEMP_FILE}"
CODE_VERSION=$(awk -F':' '/code/ {print $2}' ${TEMP_FILE})
DB_VERSION=$(awk -F':' '/^version/ {print $2}' ${TEMP_FILE})
if [ -e "${CODE_VERSION}" ]; then
echo "Unable to find the code version"
exit 1
@@ -4953,21 +5114,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: indexer-storage-rpc
annotations:
checksum/config: b71cc0136e069c1a6ad2041e5e764a0823db7d2b29692156e3358374ca9ba604
- checksum/config-utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
+ checksum/config-utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/rpc
operator: In
values:
- "true"
@@ -10789,21 +10950,20 @@
env:
- name: SCRUBBER_POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-scrubber-postgresql-common-secret
key: postgres-swh-scrubber-password
# 'name' secret must exist & include that ^ key
optional: false
-
- name: BROKER_USER_PASSWORD
valueFrom:
secretKeyRef:
name: swh-archive-broker-secret
key: BROKER_USER_PASSWORD
# 'name' secret must exist & include that ^ key
optional: false
command:
- /bin/bash
args:
@@ -10931,21 +11091,20 @@
env:
- name: SCRUBBER_POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-scrubber-postgresql-common-secret
key: postgres-swh-scrubber-password
# 'name' secret must exist & include that ^ key
optional: false
-
- name: BROKER_USER_PASSWORD
valueFrom:
secretKeyRef:
name: swh-archive-broker-secret
key: BROKER_USER_PASSWORD
# 'name' secret must exist & include that ^ key
optional: false
command:
- /bin/bash
args:
@@ -11073,21 +11232,20 @@
env:
- name: SCRUBBER_POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-scrubber-postgresql-common-secret
key: postgres-swh-scrubber-password
# 'name' secret must exist & include that ^ key
optional: false
-
- name: BROKER_USER_PASSWORD
valueFrom:
secretKeyRef:
name: swh-archive-broker-secret
key: BROKER_USER_PASSWORD
# 'name' secret must exist & include that ^ key
optional: false
command:
- /bin/bash
args:
@@ -11215,21 +11373,20 @@
env:
- name: SCRUBBER_POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-scrubber-postgresql-common-secret
key: postgres-swh-scrubber-password
# 'name' secret must exist & include that ^ key
optional: false
-
- name: BROKER_USER_PASSWORD
valueFrom:
secretKeyRef:
name: swh-archive-broker-secret
key: BROKER_USER_PASSWORD
# 'name' secret must exist & include that ^ key
optional: false
command:
- /bin/bash
args:
@@ -11305,20 +11462,1244 @@
name: scrubber-journalchecker-snapshot-template
defaultMode: 0777
items:
- key: "config.yml.template"
path: "config.yml.template"
- name: database-utils
configMap:
name: database-utils
defaultMode: 0555
---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-directory-hashes
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-directory-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 1
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-directory-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-directory-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: a57b35b73615ca40284037a0848fab89d6825e6cfd904a8da85b4cc054adf6b2
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 100Mi
+ cpu: 100m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-hashes-directory
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-directory-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-directory-references
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-directory-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 1
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-directory-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-directory-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 219681c90ac884a0ac06e33d63e00769ace3f27975b29134f6430fe1e487e2d8
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 100Mi
+ cpu: 100m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-references-directory
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-directory-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-release-hashes
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-release-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 1
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-release-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-release-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 048e24e35996782adb9ccf68d17c32bb5e721da3bc3c2a753c8f1cc8be548743
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-hashes-release
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-release-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-release-references
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-release-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 1
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-release-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-release-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 1b9a31a2523de97f92f9b35c3ecb79be8ccf51601a5b9110d30fde7ccfc3df41
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-references-release
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-release-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-revision-hashes
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-revision-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 1
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-revision-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-revision-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: b87b2295877fa4b509eec89244b8efa979a1a6fbd042445b9adf105065307397
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-hashes-revision
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-revision-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-revision-references
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-revision-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 1
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-revision-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-revision-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: e098ca8b6e7937c1d72b923ae58c803f7bee484bbeae123d3a9e1d02bd73190a
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-references-revision
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-revision-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-snapshot-hashes
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-snapshot-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 1
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-snapshot-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-snapshot-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 781398b1492ce020b8ce791833b6219096fba0026587ecdb3fc7c2d3c0cf4854
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 128Mi
+ cpu: 150m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-hashes-snapshot
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-snapshot-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-snapshot-references
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-snapshot-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 1
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-snapshot-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-snapshot-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 9ad6ee18c50a181b756257731c1af713911c30b3a8102972fdd19957d594999f
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 128Mi
+ cpu: 150m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-references-snapshot
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-snapshot-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
# Source: swh/templates/search/journal-client-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
namespace: swh
name: search-journal-client-indexed
labels:
app: search-journal-client-indexed
spec:
revisionHistoryLimit: 2
@@ -11667,21 +13048,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-content
template:
metadata:
labels:
app: storage-replayer-content
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
+ checksum/config_utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -11791,21 +13172,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-directory
template:
metadata:
labels:
app: storage-replayer-directory
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
+ checksum/config_utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -11915,21 +13296,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-extid
template:
metadata:
labels:
app: storage-replayer-extid
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
+ checksum/config_utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -12039,21 +13420,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-metadata
template:
metadata:
labels:
app: storage-replayer-metadata
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
+ checksum/config_utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -12163,21 +13544,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin
template:
metadata:
labels:
app: storage-replayer-origin
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
+ checksum/config_utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -12287,21 +13668,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin-visit
template:
metadata:
labels:
app: storage-replayer-origin-visit
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
+ checksum/config_utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -12411,21 +13792,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin-visit-status
template:
metadata:
labels:
app: storage-replayer-origin-visit-status
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
+ checksum/config_utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -12535,21 +13916,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-raw-extrinsic-metadata
template:
metadata:
labels:
app: storage-replayer-raw-extrinsic-metadata
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
+ checksum/config_utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -12659,21 +14040,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-release
template:
metadata:
labels:
app: storage-replayer-release
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
+ checksum/config_utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -12783,21 +14164,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-revision
template:
metadata:
labels:
app: storage-replayer-revision
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
+ checksum/config_utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -12907,21 +14288,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-skipped-content
template:
metadata:
labels:
app: storage-replayer-skipped-content
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
+ checksum/config_utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -13031,21 +14412,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-snapshot
template:
metadata:
labels:
app: storage-replayer-snapshot
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
+ checksum/config_utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -13159,147 +14540,204 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: swh-toolbox
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 5813a438f1dc196e606fd15e8c7827cb601b49623c6f76cb6a4b6b4f08504aa2
- checksum/configScript: 2f52c9c95b13a0c755571c6a81681958552538cbf149b458241f7c36a8bbb01f
+ checksum/config: e00d56c355fb587a59bdf0290ab5437a414e5158597e8b876b4707a2e381f278
+ checksum/configScript: 79a755472f8cc1db81583541755341f801850f6b1d9304077657a26e46099c69
spec:
priorityClassName: swh-tools
initContainers:
- name: prepare-configuration-indexer-storage
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-indexer-storage.yml
env:
+
- name: POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-indexer-storage-postgresql-secret
key: postgres-swh-indexer-password
# 'name' secret must exist & include that ^ key
optional: false
+
+
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-indexer-storage-template
mountPath: /etc/swh/configuration-template
- name: prepare-configuration-scheduler
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-scheduler.yml
env:
+
- name: AMQP_PASSWORD
valueFrom:
secretKeyRef:
name: amqp-secrets
key: swhproducer-password
# 'name' secret must exist & include that ^ key
optional: false
+
+
+
- name: POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-scheduler-postgresql-common-secret
key: postgres-swh-scheduler-password
# 'name' secret must exist & include that ^ key
optional: false
+
+
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-scheduler-template
mountPath: /etc/swh/configuration-template
- name: prepare-configuration-scrubber-journal
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-scrubber-journal.yml
env:
+
- name: BROKER_USER_PASSWORD
valueFrom:
secretKeyRef:
name: swh-archive-broker-secret
key: BROKER_USER_PASSWORD
# 'name' secret must exist & include that ^ key
optional: false
+
+
+
- name: SCRUBBER_POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-scrubber-postgresql-common-secret
key: postgres-swh-scrubber-password
# 'name' secret must exist & include that ^ key
optional: false
+
+
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-scrubber-journal-template
mountPath: /etc/swh/configuration-template
+ - name: prepare-configuration-scrubber-storage
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-scrubber-storage.yml
+ env:
+
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+
+
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-scrubber-storage-template
+ mountPath: /etc/swh/configuration-template
- name: prepare-configuration-storage
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-storage.yml
env:
+
- name: POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-postgresql-common-secret
key: postgres-swh-password
# 'name' secret must exist & include that ^ key
optional: false
+
+
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-storage-template
mountPath: /etc/swh/configuration-template
- name: prepare-configuration-vault
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-vault.yml
env:
+
- name: POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-vault-postgresql-secret
key: postgres-swh-vault-password
# 'name' secret must exist & include that ^ key
optional: false
+
+
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-vault-template
mountPath: /etc/swh/configuration-template
containers:
- name: swh-toolbox
image: container-registry.softwareheritage.org/swh/infra/swh-apps/toolbox:20231110.1
imagePullPolicy: IfNotPresent
resources:
@@ -13338,20 +14776,28 @@
path: "config.yml.template"
- name: configuration-scrubber-journal-template
configMap:
name: toolbox-scrubber-journal-template
defaultMode: 0777
items:
- key: "config.yml.template"
path: "config.yml.template"
+ - name: configuration-scrubber-storage-template
+ configMap:
+ name: toolbox-scrubber-storage-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+
- name: configuration-storage-template
configMap:
name: toolbox-storage-template
defaultMode: 0777
items:
- key: "config.yml.template"
path: "config.yml.template"
- name: configuration-vault-template
configMap:
------------- diff for environment staging namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.ShLIAK8u/staging-swh-cassandra.before 2023-11-13 13:52:55.538864872 +0100
+++ /tmp/swh-chart.swh.ShLIAK8u/staging-swh-cassandra.after 2023-11-13 13:52:56.370868196 +0100
@@ -3267,20 +3267,212 @@
swh:
level: "INFO"
celery.task:
level: "INFO"
root:
level: "INFO"
handlers:
- console
---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: scrubber-storagechecker-directory-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+ cls: cassandra
+ hosts:
+ - cassandra1.internal.staging.swh.network
+ - cassandra2.internal.staging.swh.network
+ - cassandra3.internal.staging.swh.network
+ keyspace: swh
+ consistency_level: LOCAL_QUORUM
+ auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-ro
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: scrubber-storagechecker-directory-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+ cls: cassandra
+ hosts:
+ - cassandra1.internal.staging.swh.network
+ - cassandra2.internal.staging.swh.network
+ - cassandra3.internal.staging.swh.network
+ keyspace: swh
+ consistency_level: LOCAL_QUORUM
+ auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-ro
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: scrubber-storagechecker-release-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+ cls: cassandra
+ hosts:
+ - cassandra1.internal.staging.swh.network
+ - cassandra2.internal.staging.swh.network
+ - cassandra3.internal.staging.swh.network
+ keyspace: swh
+ consistency_level: LOCAL_QUORUM
+ auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-ro
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: scrubber-storagechecker-release-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+ cls: cassandra
+ hosts:
+ - cassandra1.internal.staging.swh.network
+ - cassandra2.internal.staging.swh.network
+ - cassandra3.internal.staging.swh.network
+ keyspace: swh
+ consistency_level: LOCAL_QUORUM
+ auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-ro
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: scrubber-storagechecker-revision-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+ cls: cassandra
+ hosts:
+ - cassandra1.internal.staging.swh.network
+ - cassandra2.internal.staging.swh.network
+ - cassandra3.internal.staging.swh.network
+ keyspace: swh
+ consistency_level: LOCAL_QUORUM
+ auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-ro
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: scrubber-storagechecker-revision-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+ cls: cassandra
+ hosts:
+ - cassandra1.internal.staging.swh.network
+ - cassandra2.internal.staging.swh.network
+ - cassandra3.internal.staging.swh.network
+ keyspace: swh
+ consistency_level: LOCAL_QUORUM
+ auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-ro
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: scrubber-storagechecker-snapshot-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+ cls: cassandra
+ hosts:
+ - cassandra1.internal.staging.swh.network
+ - cassandra2.internal.staging.swh.network
+ - cassandra3.internal.staging.swh.network
+ keyspace: swh
+ consistency_level: LOCAL_QUORUM
+ auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-ro
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: scrubber-storagechecker-snapshot-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+ cls: cassandra
+ hosts:
+ - cassandra1.internal.staging.swh.network
+ - cassandra2.internal.staging.swh.network
+ - cassandra3.internal.staging.swh.network
+ keyspace: swh
+ consistency_level: LOCAL_QUORUM
+ auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-ro
+---
# Source: swh/templates/statsd-exporter/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus-statsd-exporter
namespace: swh-cassandra
data:
config.yml: |
defaults:
timer_type: histogram
@@ -3907,20 +4099,136 @@
cls: kafka
brokers:
- journal1.internal.staging.swh.network
- journal2.internal.staging.swh.network
prefix: swh.journal.objects
client_id: swh.storage-cassandra.journal_writer.storage
anonymize: true
producer_config:
message.max.bytes: 1000000000
---
+# Source: swh/templates/toolbox/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: toolbox-scrubber-storage-template
+ namespace: swh-cassandra
+data:
+ config.yml.template: |
+ storage:
+ cls: cassandra
+ hosts:
+ - cassandra1.internal.staging.swh.network
+ - cassandra2.internal.staging.swh.network
+ - cassandra3.internal.staging.swh.network
+ keyspace: swh
+ consistency_level: LOCAL_QUORUM
+ auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-ro
+
+ scrubber:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/toolbox/script-utils-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: toolbox-script-utils
+ namespace: swh-cassandra
+data:
+ register-task-types.sh: |
+ #!/bin/bash
+
+ set -eux
+
+ export SWH_CONFIG_FILENAME=/etc/swh/config-scheduler.yml
+
+ swh scheduler -C $SWH_CONFIG_FILENAME task-type register
+ check-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ TEMP_FILE=/tmp/db-version.txt
+
+ MODULE=$1
+ CODE_VERSION=${2-""}
+ DB_VERSION=${3-""}
+
+ if [ -z ${MODULE} ]; then
+ echo The environment variable must be defined with the module to check
+ echo for example "storage"
+ exit 1
+ fi
+
+ # checking the database status
+ swh db --config-file=/etc/swh/config-${MODULE}.yml version "${MODULE}" | \
+ tee "${TEMP_FILE}"
+
+ CODE_VERSION=$(awk -F':' '/code/ {print $2}' ${TEMP_FILE})
+ DB_VERSION=$(awk -F':' '/^version/ {print $2}' ${TEMP_FILE})
+
+ if [ -e "${CODE_VERSION}" ]; then
+ echo "Unable to find the code version"
+ exit 1
+ fi
+
+ if [ -e "${DB_VERSION}" ]; then
+ echo "Unable to find the code version"
+ exit 1
+ fi
+
+ if [ "$DB_VERSION" -eq "$CODE_VERSION" ]; then
+ echo "Database already configured at the latest version"
+ else
+ echo "Migration required from $DB_VERSION to $CODE_VERSION."
+ fi
+
+ migrate-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ TEMP_FILE=/tmp/db-version.txt
+
+ MODULE=$1
+ CODE_VERSION=${2-""}
+
+ if [ -z "${MODULE}" ]; then
+ echo The environment variable must be defined with the module to check
+ echo for example "storage"
+ exit 1
+ fi
+
+ if [ ! -z "${CODE_VERSION}" ]; then
+ swh db --config-file=/etc/swh/config-${MODULE}.yml upgrade "${MODULE}" \
+ --to-version ${CODE_VERSION}
+ else
+ swh db --config-file=/etc/swh/config-${MODULE}.yml upgrade "${MODULE}"
+ fi
+ check-scrubber-storage-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ /opt/swh/bin/check-db-version.sh scrubber-storage
+
+ migrate-scrubber-storage-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ /opt/swh/bin/migrate-db-version.sh scrubber-storage
+---
# Source: swh/templates/utils/database-utils.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: database-utils
namespace: swh-cassandra
data:
init-keyspace.py: |
from swh.core import config
from swh.storage.cassandra import create_keyspace
@@ -3972,21 +4280,23 @@
TEMP_FILE=/tmp/db-version.txt
if [ -z ${MODULE} ]; then
echo The env variable must be defined with the module to check
echo for example "storage"
exit 1
fi
# extracting the postgresql configuration from a full configuration
# possibly with a pipeline
- python /entrypoints/extract-storage-postgresql-config-py
+ set +e
+ python /entrypoints/extract-storage-postgresql-config-py || exit 0
+ set -e
# checking the database status
swh db --config-file=/tmp/config.yml version "${MODULE}" | tee "${TEMP_FILE}"
CODE_VERSION=$(awk -F':' '/code/ {print $2}' ${TEMP_FILE})
DB_VERSION=$(awk -F':' '/^version/ {print $2}' ${TEMP_FILE})
if [ -e "${CODE_VERSION}" ]; then
echo "Unable to find the code version"
exit 1
@@ -9382,20 +9692,1244 @@
port: 9150
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
httpGet:
path: /metrics
port: 9150
initialDelaySeconds: 5
periodSeconds: 10
---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-directory-hashes
+ namespace: swh-cassandra
+ labels:
+ app: scrubber-storagechecker-directory-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 1
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-directory-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-directory-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 6c938923ac30205da59ce63191604f620a42f1cfcddaefce3c21b84543b3738e
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: CASSANDRA_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: cassandra-swh-ro-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 200Mi
+ cpu: 400m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-cassandra-hashes-directory
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-directory-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-directory-references
+ namespace: swh-cassandra
+ labels:
+ app: scrubber-storagechecker-directory-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 1
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-directory-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-directory-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 0b0695d74ddf47fce9241311f8070c7845d019bb4ff30d883866b4cf518c574b
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: CASSANDRA_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: cassandra-swh-ro-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 200Mi
+ cpu: 400m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-cassandra-references-directory
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-directory-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-release-hashes
+ namespace: swh-cassandra
+ labels:
+ app: scrubber-storagechecker-release-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 1
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-release-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-release-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: bf76a116586bd5ad0cc7cd274829f47fe29d6cd069253a23d440ac8d7d1d7088
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: CASSANDRA_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: cassandra-swh-ro-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-cassandra-hashes-release
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-release-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-release-references
+ namespace: swh-cassandra
+ labels:
+ app: scrubber-storagechecker-release-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 1
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-release-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-release-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 81c4175ab8005cb304108abac9d14c59d276f544c11df49e9713897dd0e33817
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: CASSANDRA_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: cassandra-swh-ro-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-cassandra-references-release
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-release-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-revision-hashes
+ namespace: swh-cassandra
+ labels:
+ app: scrubber-storagechecker-revision-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 1
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-revision-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-revision-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 78c228b1195d5c5de27e2d60db3bc09139998062a1bee2fb46a3eb322773e90f
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: CASSANDRA_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: cassandra-swh-ro-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 200Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-cassandra-hashes-revision
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-revision-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-revision-references
+ namespace: swh-cassandra
+ labels:
+ app: scrubber-storagechecker-revision-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 1
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-revision-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-revision-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: f72d15dec4f7b6739df03a4c6d03d8808e4fe877ee5e31b8e9c814636ed33164
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: CASSANDRA_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: cassandra-swh-ro-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 200Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-cassandra-references-revision
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-revision-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-snapshot-hashes
+ namespace: swh-cassandra
+ labels:
+ app: scrubber-storagechecker-snapshot-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 1
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-snapshot-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-snapshot-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 0a10bc8900e5f6a0c6e0d1bca7cd0c80e3ed30262c8cbd8c84cebc8a460c1e21
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: CASSANDRA_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: cassandra-swh-ro-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 500Mi
+ cpu: 650m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-cassandra-hashes-snapshot
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-snapshot-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-snapshot-references
+ namespace: swh-cassandra
+ labels:
+ app: scrubber-storagechecker-snapshot-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 1
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-snapshot-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-snapshot-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: ae74f264f0d9d9ddecdc1cea15946091f59f4fee6d4e64f4b84b292687b19a64
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: CASSANDRA_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: cassandra-swh-ro-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 500Mi
+ cpu: 650m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-cassandra-references-snapshot
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-snapshot-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
# Source: swh/templates/statsd-exporter/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: prometheus-statsd-exporter
namespace: swh-cassandra
labels:
app: prometheus-statsd-exporter
spec:
replicas: 1
@@ -9439,21 +10973,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-content
template:
metadata:
labels:
app: storage-replayer-content
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9563,21 +11097,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-directory
template:
metadata:
labels:
app: storage-replayer-directory
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9687,21 +11221,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-extid
template:
metadata:
labels:
app: storage-replayer-extid
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9811,21 +11345,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-metadata
template:
metadata:
labels:
app: storage-replayer-metadata
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9935,21 +11469,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin
template:
metadata:
labels:
app: storage-replayer-origin
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10059,21 +11593,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin-visit
template:
metadata:
labels:
app: storage-replayer-origin-visit
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10183,21 +11717,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin-visit-status
template:
metadata:
labels:
app: storage-replayer-origin-visit-status
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10307,21 +11841,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-raw-extrinsic-metadata
template:
metadata:
labels:
app: storage-replayer-raw-extrinsic-metadata
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10431,21 +11965,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-release
template:
metadata:
labels:
app: storage-replayer-release
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10555,21 +12089,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-revision
template:
metadata:
labels:
app: storage-replayer-revision
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10679,21 +12213,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-skipped-content
template:
metadata:
labels:
app: storage-replayer-skipped-content
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10803,21 +12337,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-snapshot
template:
metadata:
labels:
app: storage-replayer-snapshot
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10931,21 +12465,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: storage
annotations:
checksum/config: bce424b10db0b622f5b8050c99eac996142552bf9473bfba9d9e038cb6b61ece
- checksum/config-utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config-utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/storage
operator: In
values:
- "true"
@@ -11054,20 +12588,121 @@
configMap:
name: storage-configuration-template
items:
- key: "config.yml.template"
path: "config.yml.template"
- name: database-utils
configMap:
name: database-utils
defaultMode: 0555
---
+# Source: swh/templates/toolbox/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: swh-toolbox
+ namespace: swh-cassandra
+ labels:
+ app: swh-toolbox
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: swh-toolbox
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: swh-toolbox
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: ac74de660137810ca77a168f8c34c16d01078c5ee1b0db90de7c4339dbc461e9
+ checksum/configScript: e524fc0d85bca4929459b068d030f3d7dd3680cd450d39c51791420840539736
+ spec:
+ priorityClassName: swh-cassandra-tools
+
+ initContainers:
+ - name: prepare-configuration-scrubber-storage
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-scrubber-storage.yml
+ env:
+
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+
+
+
+ - name: CASSANDRA_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: cassandra-swh-ro-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-scrubber-storage-template
+ mountPath: /etc/swh/configuration-template
+ containers:
+ - name: swh-toolbox
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/toolbox:20231110.1
+ imagePullPolicy: IfNotPresent
+ resources:
+ requests:
+ memory: 10Mi
+ cpu: 10m
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - /opt/swh/entrypoint.sh
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: toolbox-script-utils
+ mountPath: /opt/swh/bin
+ readOnly: true
+ volumes:
+ - name: configuration
+ emptyDir: {}
+
+ - name: configuration-scrubber-storage-template
+ configMap:
+ name: toolbox-scrubber-storage-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+
+ - name: toolbox-script-utils
+ configMap:
+ name: toolbox-script-utils
+ defaultMode: 0555
+---
# Source: swh/templates/vault/rpc-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
namespace: swh-cassandra
name: vault-rpc
labels:
app: vault-rpc
spec:
revisionHistoryLimit: 2
------------- diff for environment staging namespace swh-cassandra-next-version -------------
--- /tmp/swh-chart.swh.ShLIAK8u/staging-swh-cassandra-next-version.before 2023-11-13 13:52:55.798865872 +0100
+++ /tmp/swh-chart.swh.ShLIAK8u/staging-swh-cassandra-next-version.after 2023-11-13 13:52:56.574869050 +0100
@@ -3766,21 +3766,23 @@
TEMP_FILE=/tmp/db-version.txt
if [ -z ${MODULE} ]; then
echo The env variable must be defined with the module to check
echo for example "storage"
exit 1
fi
# extracting the postgresql configuration from a full configuration
# possibly with a pipeline
- python /entrypoints/extract-storage-postgresql-config-py
+ set +e
+ python /entrypoints/extract-storage-postgresql-config-py || exit 0
+ set -e
# checking the database status
swh db --config-file=/tmp/config.yml version "${MODULE}" | tee "${TEMP_FILE}"
CODE_VERSION=$(awk -F':' '/code/ {print $2}' ${TEMP_FILE})
DB_VERSION=$(awk -F':' '/^version/ {print $2}' ${TEMP_FILE})
if [ -e "${CODE_VERSION}" ]; then
echo "Unable to find the code version"
exit 1
@@ -8901,21 +8903,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-content
template:
metadata:
labels:
app: storage-replayer-content
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
+ checksum/config_utils: 908f9182febd57f799c59c25abdfbd7cfd832e1f1edc150c536d5f8087dd9486
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9025,21 +9027,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-directory
template:
metadata:
labels:
app: storage-replayer-directory
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
+ checksum/config_utils: 908f9182febd57f799c59c25abdfbd7cfd832e1f1edc150c536d5f8087dd9486
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9149,21 +9151,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-extid
template:
metadata:
labels:
app: storage-replayer-extid
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
+ checksum/config_utils: 908f9182febd57f799c59c25abdfbd7cfd832e1f1edc150c536d5f8087dd9486
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9273,21 +9275,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-metadata
template:
metadata:
labels:
app: storage-replayer-metadata
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
+ checksum/config_utils: 908f9182febd57f799c59c25abdfbd7cfd832e1f1edc150c536d5f8087dd9486
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9397,21 +9399,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin
template:
metadata:
labels:
app: storage-replayer-origin
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
+ checksum/config_utils: 908f9182febd57f799c59c25abdfbd7cfd832e1f1edc150c536d5f8087dd9486
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9521,21 +9523,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin-visit
template:
metadata:
labels:
app: storage-replayer-origin-visit
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
+ checksum/config_utils: 908f9182febd57f799c59c25abdfbd7cfd832e1f1edc150c536d5f8087dd9486
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9645,21 +9647,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin-visit-status
template:
metadata:
labels:
app: storage-replayer-origin-visit-status
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
+ checksum/config_utils: 908f9182febd57f799c59c25abdfbd7cfd832e1f1edc150c536d5f8087dd9486
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9769,21 +9771,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-raw-extrinsic-metadata
template:
metadata:
labels:
app: storage-replayer-raw-extrinsic-metadata
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
+ checksum/config_utils: 908f9182febd57f799c59c25abdfbd7cfd832e1f1edc150c536d5f8087dd9486
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9893,21 +9895,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-release
template:
metadata:
labels:
app: storage-replayer-release
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
+ checksum/config_utils: 908f9182febd57f799c59c25abdfbd7cfd832e1f1edc150c536d5f8087dd9486
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10017,21 +10019,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-revision
template:
metadata:
labels:
app: storage-replayer-revision
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
+ checksum/config_utils: 908f9182febd57f799c59c25abdfbd7cfd832e1f1edc150c536d5f8087dd9486
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10141,21 +10143,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-skipped-content
template:
metadata:
labels:
app: storage-replayer-skipped-content
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
+ checksum/config_utils: 908f9182febd57f799c59c25abdfbd7cfd832e1f1edc150c536d5f8087dd9486
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10265,21 +10267,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-snapshot
template:
metadata:
labels:
app: storage-replayer-snapshot
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
+ checksum/config_utils: 908f9182febd57f799c59c25abdfbd7cfd832e1f1edc150c536d5f8087dd9486
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10393,21 +10395,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: storage
annotations:
checksum/config: 54d5bdf8fcab3e26c7fa46f3b8562521315657c36c880d4ab8f2527152beb09d
- checksum/config-utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
+ checksum/config-utils: 908f9182febd57f799c59c25abdfbd7cfd832e1f1edc150c536d5f8087dd9486
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/storage
operator: In
values:
- "true"
------------- diff for environment production namespace swh -------------
--- /tmp/swh-chart.swh.ShLIAK8u/production-swh.before 2023-11-13 13:52:56.834870139 +0100
+++ /tmp/swh-chart.swh.ShLIAK8u/production-swh.after 2023-11-13 13:52:57.146871445 +0100
@@ -5438,21 +5438,23 @@
TEMP_FILE=/tmp/db-version.txt
if [ -z ${MODULE} ]; then
echo The env variable must be defined with the module to check
echo for example "storage"
exit 1
fi
# extracting the postgresql configuration from a full configuration
# possibly with a pipeline
- python /entrypoints/extract-storage-postgresql-config-py
+ set +e
+ python /entrypoints/extract-storage-postgresql-config-py || exit 0
+ set -e
# checking the database status
swh db --config-file=/tmp/config.yml version "${MODULE}" | tee "${TEMP_FILE}"
CODE_VERSION=$(awk -F':' '/code/ {print $2}' ${TEMP_FILE})
DB_VERSION=$(awk -F':' '/^version/ {print $2}' ${TEMP_FILE})
if [ -e "${CODE_VERSION}" ]; then
echo "Unable to find the code version"
exit 1
@@ -14375,21 +14377,20 @@
env:
- name: SCRUBBER_POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-scrubber-postgresql-common-secret
key: postgres-swh-scrubber-password
# 'name' secret must exist & include that ^ key
optional: false
-
- name: BROKER_USER_PASSWORD
valueFrom:
secretKeyRef:
name: swh-archive-broker-secret
key: BROKER_USER_PASSWORD
# 'name' secret must exist & include that ^ key
optional: false
command:
- /bin/bash
args:
@@ -14517,21 +14518,20 @@
env:
- name: SCRUBBER_POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-scrubber-postgresql-common-secret
key: postgres-swh-scrubber-password
# 'name' secret must exist & include that ^ key
optional: false
-
- name: BROKER_USER_PASSWORD
valueFrom:
secretKeyRef:
name: swh-archive-broker-secret
key: BROKER_USER_PASSWORD
# 'name' secret must exist & include that ^ key
optional: false
command:
- /bin/bash
args:
@@ -14659,21 +14659,20 @@
env:
- name: SCRUBBER_POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-scrubber-postgresql-common-secret
key: postgres-swh-scrubber-password
# 'name' secret must exist & include that ^ key
optional: false
-
- name: BROKER_USER_PASSWORD
valueFrom:
secretKeyRef:
name: swh-archive-broker-secret
key: BROKER_USER_PASSWORD
# 'name' secret must exist & include that ^ key
optional: false
command:
- /bin/bash
args:
@@ -14825,110 +14824,128 @@
initContainers:
- name: prepare-configuration-indexer-storage
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-indexer-storage.yml
env:
+
- name: POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-indexer-storage-postgresql-secret
key: postgres-swh-indexer-password
# 'name' secret must exist & include that ^ key
optional: false
+
+
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-indexer-storage-template
mountPath: /etc/swh/configuration-template
- name: prepare-configuration-scheduler
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-scheduler.yml
env:
+
- name: AMQP_PASSWORD
valueFrom:
secretKeyRef:
name: amqp-secrets
key: swhproducer-password
# 'name' secret must exist & include that ^ key
optional: false
+
+
+
- name: POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-scheduler-postgresql-common-secret
key: postgres-swh-scheduler-password
# 'name' secret must exist & include that ^ key
optional: false
+
+
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-scheduler-template
mountPath: /etc/swh/configuration-template
- name: prepare-configuration-scrubber-journal
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-scrubber-journal.yml
env:
+
- name: BROKER_USER_PASSWORD
valueFrom:
secretKeyRef:
name: swh-archive-broker-secret
key: BROKER_USER_PASSWORD
# 'name' secret must exist & include that ^ key
optional: false
+
+
+
- name: SCRUBBER_POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-scrubber-postgresql-common-secret
key: postgres-swh-scrubber-password
# 'name' secret must exist & include that ^ key
optional: false
+
+
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-scrubber-journal-template
mountPath: /etc/swh/configuration-template
- name: prepare-configuration-storage
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-storage.yml
env:
+
- name: POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-storage-postgresql-common-secret
key: postgres-swh-storage-password
# 'name' secret must exist & include that ^ key
optional: false
+
+
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-storage-template
mountPath: /etc/swh/configuration-template
containers:
- name: swh-toolbox
image: container-registry.softwareheritage.org/swh/infra/swh-apps/toolbox:20231110.1
imagePullPolicy: IfNotPresent
resources:
------------- diff for environment production namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.ShLIAK8u/production-swh-cassandra.before 2023-11-13 13:52:56.922870507 +0100
+++ /tmp/swh-chart.swh.ShLIAK8u/production-swh-cassandra.after 2023-11-13 13:52:57.242871846 +0100
@@ -963,21 +963,23 @@
TEMP_FILE=/tmp/db-version.txt
if [ -z ${MODULE} ]; then
echo The env variable must be defined with the module to check
echo for example "storage"
exit 1
fi
# extracting the postgresql configuration from a full configuration
# possibly with a pipeline
- python /entrypoints/extract-storage-postgresql-config-py
+ set +e
+ python /entrypoints/extract-storage-postgresql-config-py || exit 0
+ set -e
# checking the database status
swh db --config-file=/tmp/config.yml version "${MODULE}" | tee "${TEMP_FILE}"
CODE_VERSION=$(awk -F':' '/code/ {print $2}' ${TEMP_FILE})
DB_VERSION=$(awk -F':' '/^version/ {print $2}' ${TEMP_FILE})
if [ -e "${CODE_VERSION}" ]; then
echo "Unable to find the code version"
exit 1
@@ -1464,21 +1466,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-content
template:
metadata:
labels:
app: storage-replayer-content
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -1575,21 +1577,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-directory
template:
metadata:
labels:
app: storage-replayer-directory
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -1686,21 +1688,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-extid
template:
metadata:
labels:
app: storage-replayer-extid
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -1797,21 +1799,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-metadata
template:
metadata:
labels:
app: storage-replayer-metadata
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -1908,21 +1910,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin
template:
metadata:
labels:
app: storage-replayer-origin
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -2019,21 +2021,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin-visit
template:
metadata:
labels:
app: storage-replayer-origin-visit
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -2130,21 +2132,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin-visit-status
template:
metadata:
labels:
app: storage-replayer-origin-visit-status
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -2241,21 +2243,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-raw-extrinsic-metadata
template:
metadata:
labels:
app: storage-replayer-raw-extrinsic-metadata
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -2352,21 +2354,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-release
template:
metadata:
labels:
app: storage-replayer-release
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -2463,21 +2465,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-revision
template:
metadata:
labels:
app: storage-replayer-revision
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -2574,21 +2576,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-skipped-content
template:
metadata:
labels:
app: storage-replayer-skipped-content
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -2685,21 +2687,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-snapshot
template:
metadata:
labels:
app: storage-replayer-snapshot
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config_utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -2801,21 +2803,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: storage
annotations:
checksum/config: 404b00dbad2af855b6bfca45fa8048c82a0a5658fc1a82d8a93944c58ff3cf22
- checksum/config-utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
+ checksum/config-utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/storage
operator: In
values:
- "true"
Edited by Antoine R. Dumont