Activate the mailmaps -> masking proxy sync job
- A couple of commits to prepare the stack in the local-cluster environment
- Refactor the old syncmailmaps django command config to make it generic
- Migrate the old configurations to be iso
- Activate the new sync job in staging
Sync test in the local cluster:
2024-08-27T09:15:00.896579234Z web-local-archive-syncmaskingmailmaps django-admin sync_masking_mailmaps routine
2024-08-27T09:15:00.901856882Z web-local-archive-syncmaskingmailmaps Tue Aug 27 09:15:00 UTC 2024
2024-08-27T09:15:07.531238590Z web-local-archive-syncmaskingmailmaps 0 mailmaps to disable, 1 mailmaps to set/refresh
2024-08-27T09:15:07.531351365Z web-local-archive-syncmaskingmailmaps Synced 1 mailmaps to the masking proxy databasehelm diff
[swh] Comparing changes between branches production and mailmap-sync (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in mailmap-sync branch for environment staging...
[swh] Generate config in mailmap-sync branch for environment staging...
[swh] Generate config in mailmap-sync branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in mailmap-sync branch for environment production...
[swh] Generate config in mailmap-sync branch for environment production...
[swh] Generate config in mailmap-sync branch for environment production...
------------- diff for environment staging namespace swh -------------
--- /tmp/swh-chart.swh.AfSyZwOF/staging-swh.before 2024-08-29 08:51:54.328641594 +0200
+++ /tmp/swh-chart.swh.AfSyZwOF/staging-swh.after 2024-08-29 08:51:56.668644589 +0200
@@ -5844,52 +5844,52 @@
optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: config-utils
mountPath: /entrypoints
readOnly: true
- name: do-migration
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
env:
- name: SWH_CONFIG_FILENAME
value: /etc/swh/config.yml
command:
- django-admin
args:
- migrate
- --settings=swh.web.settings.production
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: prepare-static
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- cp -r $PWD/.local/share/swh/web/static/ /usr/share/swh/web/static/
volumeMounts:
- name: static
mountPath: /usr/share/swh/web/static
containers:
- name: web-postgresql
resources:
requests:
memory: 250Mi
cpu: 50m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5004
name: webapp
readinessProbe:
httpGet:
path: /api/1/
port: webapp
httpHeaders:
- name: Host
------------- diff for environment staging namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.AfSyZwOF/staging-swh-cassandra.before 2024-08-29 08:51:55.340642890 +0200
+++ /tmp/swh-chart.swh.AfSyZwOF/staging-swh-cassandra.after 2024-08-29 08:51:57.748645972 +0200
@@ -7651,34 +7651,20 @@
},
"root": {
"handlers": [
"console"
],
"level": "INFO"
},
"version": 1
}
---
-# Source: swh/templates/web/configmap-pgservice.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: pgservice-cassandra-configuration-template
- namespace: swh-cassandra
-data:
- pg-service-conf: |
- [syncmailmaps]
- dbname=swh
- host=db1.internal.staging.swh.network
- port=5432
- user=swhmailmap
----
# Source: swh/templates/web/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh-cassandra
name: web-cassandra-configuration-template
data:
config.yml.template: |
instance_name: webapp.staging.swh.network
allowed_hosts:
@@ -8027,20 +8013,48 @@
},
"root": {
"handlers": [
"console"
],
"level": "INFO"
},
"version": 1
}
---
+# Source: swh/templates/web/cronjob-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: web-cassandra-syncmailmaps-config-template
+ namespace: swh-cassandra
+data:
+ pg-service-conf: |
+ [syncmailmaps]
+ dbname=swh
+ host=db1.internal.staging.swh.network
+ port=5432
+ user=swhmailmap
+---
+# Source: swh/templates/web/cronjob-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: web-cassandra-syncmaskingmailmaps-config-template
+ namespace: swh-cassandra
+data:
+ pg-service-conf: |
+ [syncmaskingmailmaps]
+ dbname=swh-masking
+ host=db1.internal.staging.swh.network
+ port=5432
+ user=swhmailmap
+---
# Source: swh/templates/webhooks/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh-cassandra
name: webhooks-origin-visit-status-template
data:
config.yml.template: |
webhooks:
event_retention_period: 7
@@ -21472,52 +21486,52 @@
optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: config-utils
mountPath: /entrypoints
readOnly: true
- name: do-migration
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
env:
- name: SWH_CONFIG_FILENAME
value: /etc/swh/config.yml
command:
- django-admin
args:
- migrate
- --settings=swh.web.settings.production
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: prepare-static
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- cp -r $PWD/.local/share/swh/web/static/ /usr/share/swh/web/static/
volumeMounts:
- name: static
mountPath: /usr/share/swh/web/static
containers:
- name: web-cassandra
resources:
requests:
memory: 512Mi
cpu: 100m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5004
name: webapp
readinessProbe:
httpGet:
path: /api/1/
port: webapp
httpHeaders:
- name: Host
@@ -21734,52 +21748,52 @@
optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: config-utils
mountPath: /entrypoints
readOnly: true
- name: do-migration
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
env:
- name: SWH_CONFIG_FILENAME
value: /etc/swh/config.yml
command:
- django-admin
args:
- migrate
- --settings=swh.web.settings.production
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: prepare-static
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- cp -r $PWD/.local/share/swh/web/static/ /usr/share/swh/web/static/
volumeMounts:
- name: static
mountPath: /usr/share/swh/web/static
containers:
- name: web-webhooks
resources:
requests:
memory: 512Mi
cpu: 100m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5004
name: webapp
readinessProbe:
httpGet:
path: /api/1/
port: webapp
httpHeaders:
- name: Host
@@ -22320,53 +22334,114 @@
- name: configuration
emptyDir: {}
- name: configuration-template
configMap:
name: scheduler-update-metrics-configuration-template
items:
- key: "config.yml.template"
path: "config.yml.template"
restartPolicy: OnFailure
---
-# Source: swh/templates/web/sync-mailmaps-cronjob.yaml
+# Source: swh/templates/web/cronjob.yaml
apiVersion: batch/v1
kind: CronJob
metadata:
- name: web-cassandra-sync-mailmaps-cronjob
+ name: web-cassandra-syncmailmaps-cronjob
namespace: swh-cassandra
spec:
- schedule: "15 * * * *"
+ schedule: "0 */1 * * *"
concurrencyPolicy: Forbid
jobTemplate:
spec:
template:
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/web
operator: In
values:
- "true"
priorityClassName: swh-cassandra-frontend-rpc-workload
initContainers:
- name: prepare-pgservice-configuration
- image: debian:bullseye
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- - eval "cp /etc/swh/config/pg_service.conf /etc/swh/.pg_service.conf"
+ - eval "cat /etc/swh/config/pg_service.conf | envsubst > /etc/swh/.pg_service.conf"
+ env:
+ - name: DEPOSIT_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: password
+ name: deposit-secrets
+ optional: false
+ - name: DEPOSIT_USERNAME
+ valueFrom:
+ secretKeyRef:
+ key: username
+ name: deposit-secrets
+ optional: false
+ - name: DJANGO_SECRET_KEY
+ valueFrom:
+ secretKeyRef:
+ key: webapp-django-secret-key
+ name: swh-webapp-django-secret
+ optional: false
+ - name: GIVE_PRIVATE_TOKEN
+ valueFrom:
+ secretKeyRef:
+ key: private-token
+ name: web-give-secrets
+ optional: false
+ - name: GIVE_PUBLIC_KEY
+ valueFrom:
+ secretKeyRef:
+ key: public-key
+ name: web-give-secrets
+ optional: false
+ - name: INBOUND_EMAIL_SHARED_KEY
+ valueFrom:
+ secretKeyRef:
+ key: web-inbound-email-shared-key
+ name: common-secrets
+ optional: false
+ - name: PGPASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: postgres-syncmailmap-password
+ name: swh-postgresql-syncmailmap-secret
+ optional: false
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: password
+ name: swh-postgresql-swh-web-secret
+ optional: false
+ - name: POSTGRESQL_USERNAME
+ valueFrom:
+ secretKeyRef:
+ key: username
+ name: swh-postgresql-swh-web-secret
+ optional: false
+ - name: WEBHOOKS_SECRET
+ valueFrom:
+ secretKeyRef:
+ key: webhooks-secret
+ name: common-secrets
+ optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: pgservice-configuration-template
mountPath: /etc/swh/config
- name: prepare-web-configuration
image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
imagePullPolicy: IfNotPresent
command:
@@ -22442,38 +22517,295 @@
optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: web-configuration-template
mountPath: /etc/swh/configuration-template
- name: config-utils
mountPath: /entrypoints
readOnly: true
containers:
- - name: web-cassandra-sync-mailmaps
+ - name: web-cassandra-syncmailmaps
resources:
requests:
memory: 512Mi
cpu: 500m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
command:
- /opt/swh/entrypoint.sh
args:
- - sync-mailmaps
+ - django-admin
+ - sync_mailmaps
+ - --perform
- service=syncmailmaps
env:
- name: STATSD_HOST
value: prometheus-statsd-exporter
- name: STATSD_PORT
value: "9125"
- name: STATSD_TAGS
- value: deployment:web-cassandra-sync-mailmaps-cronjob
+ value: deployment:web-cassandra-syncmailmaps-cronjob
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_LOG_LEVEL
+ value: INFO
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.web
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: web-sentry-dsn
+ # if the setting doesn't exist, sentry issue pushes will be disabled
+ optional: false
+ - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+ value: "true"
+ - name: PGSERVICEFILE
+ value: /etc/swh/.pg_service.conf
+ - name: PGPASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: postgres-syncmailmap-password
+ name: swh-postgresql-syncmailmap-secret
+ optional: false
+
+ imagePullPolicy: IfNotPresent
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: config-utils
+ configMap:
+ name: config-utils
+ defaultMode: 0555
+ - name: web-configuration-template
+ configMap:
+ name: web-cassandra-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: pgservice-configuration-template
+ configMap:
+ name: web-cassandra-syncmailmaps-config-template
+ items:
+ - key: "pg-service-conf"
+ path: "pg_service.conf"
+
+ restartPolicy: OnFailure
+---
+# Source: swh/templates/web/cronjob.yaml
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+ name: web-cassandra-syncmaskingmailmaps-cronjob
+ namespace: swh-cassandra
+spec:
+ schedule: "0 */1 * * *"
+ concurrencyPolicy: Forbid
+ jobTemplate:
+ spec:
+ template:
+ spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/web
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-frontend-rpc-workload
+
+ initContainers:
+ - name: prepare-pgservice-configuration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval "cat /etc/swh/config/pg_service.conf | envsubst > /etc/swh/.pg_service.conf"
+ env:
+ - name: DEPOSIT_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: password
+ name: deposit-secrets
+ optional: false
+ - name: DEPOSIT_USERNAME
+ valueFrom:
+ secretKeyRef:
+ key: username
+ name: deposit-secrets
+ optional: false
+ - name: DJANGO_SECRET_KEY
+ valueFrom:
+ secretKeyRef:
+ key: webapp-django-secret-key
+ name: swh-webapp-django-secret
+ optional: false
+ - name: GIVE_PRIVATE_TOKEN
+ valueFrom:
+ secretKeyRef:
+ key: private-token
+ name: web-give-secrets
+ optional: false
+ - name: GIVE_PUBLIC_KEY
+ valueFrom:
+ secretKeyRef:
+ key: public-key
+ name: web-give-secrets
+ optional: false
+ - name: INBOUND_EMAIL_SHARED_KEY
+ valueFrom:
+ secretKeyRef:
+ key: web-inbound-email-shared-key
+ name: common-secrets
+ optional: false
+ - name: PGPASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: postgres-syncmailmap-password
+ name: swh-postgresql-syncmailmap-secret
+ optional: false
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: password
+ name: swh-postgresql-swh-web-secret
+ optional: false
+ - name: POSTGRESQL_USERNAME
+ valueFrom:
+ secretKeyRef:
+ key: username
+ name: swh-postgresql-swh-web-secret
+ optional: false
+ - name: WEBHOOKS_SECRET
+ valueFrom:
+ secretKeyRef:
+ key: webhooks-secret
+ name: common-secrets
+ optional: false
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: pgservice-configuration-template
+ mountPath: /etc/swh/config
+
+ - name: prepare-web-configuration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/prepare-configuration.sh
+ env:
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: web-sentry-dsn
+ # 'name' secret should exist & include key
+ # if the setting doesn't exist, sentry pushes will be disabled
+ optional: true
+ - name: DEPOSIT_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: password
+ name: deposit-secrets
+ optional: false
+ - name: DEPOSIT_USERNAME
+ valueFrom:
+ secretKeyRef:
+ key: username
+ name: deposit-secrets
+ optional: false
+ - name: DJANGO_SECRET_KEY
+ valueFrom:
+ secretKeyRef:
+ key: webapp-django-secret-key
+ name: swh-webapp-django-secret
+ optional: false
+ - name: GIVE_PRIVATE_TOKEN
+ valueFrom:
+ secretKeyRef:
+ key: private-token
+ name: web-give-secrets
+ optional: false
+ - name: GIVE_PUBLIC_KEY
+ valueFrom:
+ secretKeyRef:
+ key: public-key
+ name: web-give-secrets
+ optional: false
+ - name: INBOUND_EMAIL_SHARED_KEY
+ valueFrom:
+ secretKeyRef:
+ key: web-inbound-email-shared-key
+ name: common-secrets
+ optional: false
+ - name: PGPASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: postgres-syncmailmap-password
+ name: swh-postgresql-syncmailmap-secret
+ optional: false
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: password
+ name: swh-postgresql-swh-web-secret
+ optional: false
+ - name: POSTGRESQL_USERNAME
+ valueFrom:
+ secretKeyRef:
+ key: username
+ name: swh-postgresql-swh-web-secret
+ optional: false
+ - name: WEBHOOKS_SECRET
+ valueFrom:
+ secretKeyRef:
+ key: webhooks-secret
+ name: common-secrets
+ optional: false
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: web-configuration-template
+ mountPath: /etc/swh/configuration-template
+ - name: config-utils
+ mountPath: /entrypoints
+ readOnly: true
+ containers:
+ - name: web-cassandra-syncmaskingmailmaps
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - django-admin
+ - sync_masking_mailmaps
+ - --perform
+ - service=syncmaskingmailmaps
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: STATSD_TAGS
+ value: deployment:web-cassandra-syncmaskingmailmaps-cronjob
- name: SWH_CONFIG_FILENAME
value: /etc/swh/config.yml
- name: SWH_LOG_LEVEL
value: INFO
- name: SWH_SENTRY_ENVIRONMENT
value: staging
- name: SWH_MAIN_PACKAGE
value: swh.web
- name: SWH_SENTRY_DSN
valueFrom:
@@ -22505,21 +22837,21 @@
name: config-utils
defaultMode: 0555
- name: web-configuration-template
configMap:
name: web-cassandra-configuration-template
items:
- key: "config.yml.template"
path: "config.yml.template"
- name: pgservice-configuration-template
configMap:
- name: pgservice-cassandra-configuration-template
+ name: web-cassandra-syncmaskingmailmaps-config-template
items:
- key: "pg-service-conf"
path: "pg_service.conf"
restartPolicy: OnFailure
---
# Source: swh/templates/counters/rpc-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
------------- diff for environment staging namespace swh-cassandra-next-version -------------
--- /tmp/swh-chart.swh.AfSyZwOF/staging-swh-cassandra-next-version.before 2024-08-29 08:51:56.304644123 +0200
+++ /tmp/swh-chart.swh.AfSyZwOF/staging-swh-cassandra-next-version.after 2024-08-29 08:51:58.648647124 +0200
@@ -19494,52 +19494,52 @@
optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: config-utils
mountPath: /entrypoints
readOnly: true
- name: do-migration
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
env:
- name: SWH_CONFIG_FILENAME
value: /etc/swh/config.yml
command:
- django-admin
args:
- migrate
- --settings=swh.web.settings.production
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: prepare-static
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- cp -r $PWD/.local/share/swh/web/static/ /usr/share/swh/web/static/
volumeMounts:
- name: static
mountPath: /usr/share/swh/web/static
containers:
- name: web-cassandra
resources:
requests:
memory: 512Mi
cpu: 100m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5004
name: webapp
readinessProbe:
httpGet:
path: /api/1/
port: webapp
httpHeaders:
- name: Host
------------- diff for environment production namespace swh -------------
--- /tmp/swh-chart.swh.AfSyZwOF/production-swh.before 2024-08-29 08:51:59.876648695 +0200
+++ /tmp/swh-chart.swh.AfSyZwOF/production-swh.after 2024-08-29 08:52:01.744651087 +0200
@@ -9092,34 +9092,20 @@
},
"root": {
"handlers": [
"console"
],
"level": "INFO"
},
"version": 1
}
---
-# Source: swh/templates/web/configmap-pgservice.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: pgservice-archive-configuration-template
- namespace: swh
-data:
- pg-service-conf: |
- [syncmailmaps]
- dbname=softwareheritage
- host=postgresql-storage-rw.internal.softwareheritage.org
- port=5432
- user=swhmailmap
----
# Source: swh/templates/web/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh
name: web-app1-configuration-template
data:
config.yml.template: |
instance_name: webapp1.internal.softwareheritage.org
allowed_hosts:
@@ -9753,20 +9739,34 @@
},
"root": {
"handlers": [
"console"
],
"level": "INFO"
},
"version": 1
}
---
+# Source: swh/templates/web/cronjob-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: web-archive-syncmailmaps-config-template
+ namespace: swh
+data:
+ pg-service-conf: |
+ [syncmailmaps]
+ dbname=softwareheritage
+ host=postgresql-storage-rw.internal.softwareheritage.org
+ port=5432
+ user=swhmailmap
+---
# Source: swh/templates/webhooks/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh
name: webhooks-origin-visit-status-template
data:
config.yml.template: |
webhooks:
event_retention_period: 7
@@ -25883,52 +25883,52 @@
optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: config-utils
mountPath: /entrypoints
readOnly: true
- name: do-migration
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
env:
- name: SWH_CONFIG_FILENAME
value: /etc/swh/config.yml
command:
- django-admin
args:
- migrate
- --settings=swh.web.settings.production
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: prepare-static
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- cp -r $PWD/.local/share/swh/web/static/ /usr/share/swh/web/static/
volumeMounts:
- name: static
mountPath: /usr/share/swh/web/static
containers:
- name: web-app1
resources:
requests:
memory: 1Gi
cpu: 500m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5004
name: webapp
readinessProbe:
httpGet:
path: /api/1/
port: webapp
httpHeaders:
- name: Host
@@ -26162,52 +26162,52 @@
optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: config-utils
mountPath: /entrypoints
readOnly: true
- name: do-migration
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
env:
- name: SWH_CONFIG_FILENAME
value: /etc/swh/config.yml
command:
- django-admin
args:
- migrate
- --settings=swh.web.settings.production
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: prepare-static
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- cp -r $PWD/.local/share/swh/web/static/ /usr/share/swh/web/static/
volumeMounts:
- name: static
mountPath: /usr/share/swh/web/static
containers:
- name: web-archive
resources:
requests:
memory: 6Gi
cpu: 500m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5004
name: webapp
readinessProbe:
httpGet:
path: /api/1/
port: webapp
httpHeaders:
- name: Host
@@ -26436,52 +26436,52 @@
optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: config-utils
mountPath: /entrypoints
readOnly: true
- name: do-migration
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
env:
- name: SWH_CONFIG_FILENAME
value: /etc/swh/config.yml
command:
- django-admin
args:
- migrate
- --settings=swh.web.settings.production
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: prepare-static
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- cp -r $PWD/.local/share/swh/web/static/ /usr/share/swh/web/static/
volumeMounts:
- name: static
mountPath: /usr/share/swh/web/static
containers:
- name: web-webhooks
resources:
requests:
memory: 7Gi
cpu: 200m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5004
name: webapp
readinessProbe:
httpGet:
path: /api/1/
port: webapp
httpHeaders:
- name: Host
@@ -27262,25 +27262,25 @@
name: cronjob-create-object-reference-partitions-configuration-template
items:
- key: "config.yml.template"
path: "config.yml.template"
- name: config-utils
configMap:
name: config-utils
defaultMode: 0555
restartPolicy: OnFailure
---
-# Source: swh/templates/web/sync-mailmaps-cronjob.yaml
+# Source: swh/templates/web/cronjob.yaml
apiVersion: batch/v1
kind: CronJob
metadata:
- name: web-archive-sync-mailmaps-cronjob
+ name: web-archive-syncmailmaps-cronjob
namespace: swh
spec:
schedule: "15 * * * *"
concurrencyPolicy: Forbid
jobTemplate:
spec:
template:
spec:
affinity:
nodeAffinity:
@@ -27288,27 +27288,100 @@
nodeSelectorTerms:
- matchExpressions:
- key: swh/web
operator: In
values:
- "true"
priorityClassName: swh-frontend-rpc-workload
initContainers:
- name: prepare-pgservice-configuration
- image: debian:bullseye
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- - eval "cp /etc/swh/config/pg_service.conf /etc/swh/.pg_service.conf"
+ - eval "cat /etc/swh/config/pg_service.conf | envsubst > /etc/swh/.pg_service.conf"
+ env:
+ - name: DEPOSIT_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: password
+ name: deposit-secrets
+ optional: false
+ - name: DEPOSIT_USERNAME
+ valueFrom:
+ secretKeyRef:
+ key: username
+ name: deposit-secrets
+ optional: false
+ - name: DJANGO_SECRET_KEY
+ valueFrom:
+ secretKeyRef:
+ key: webapp-django-secret-key
+ name: swh-webapp-django-secret
+ optional: false
+ - name: DJANGO_SECRET_KEY_FALLBACK_1
+ valueFrom:
+ secretKeyRef:
+ key: webapp-django-secret-key-fallback-1
+ name: swh-webapp-django-secret
+ optional: false
+ - name: DJANGO_SECRET_KEY_FALLBACK_2
+ valueFrom:
+ secretKeyRef:
+ key: webapp-django-secret-key-fallback-2
+ name: swh-webapp-django-secret
+ optional: false
+ - name: GITLAB_AFN_TOKEN
+ valueFrom:
+ secretKeyRef:
+ key: gitlab_afn_token
+ name: common-secrets
+ optional: false
+ - name: GIVE_PRIVATE_TOKEN
+ valueFrom:
+ secretKeyRef:
+ key: private-token
+ name: web-give-secrets
+ optional: false
+ - name: GIVE_PUBLIC_KEY
+ valueFrom:
+ secretKeyRef:
+ key: public-key
+ name: web-give-secrets
+ optional: false
+ - name: INBOUND_EMAIL_SHARED_KEY
+ valueFrom:
+ secretKeyRef:
+ key: web-inbound-email-shared-key
+ name: common-secrets
+ optional: false
+ - name: PGPASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: postgres-syncmailmap-password
+ name: swh-postgresql-syncmailmap-secret
+ optional: false
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: postgres-swh-web-password
+ name: swh-postgresql-web-secrets
+ optional: false
+ - name: WEBHOOKS_SECRET
+ valueFrom:
+ secretKeyRef:
+ key: webhooks-secret
+ name: common-secrets
+ optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: pgservice-configuration-template
mountPath: /etc/swh/config
- name: prepare-web-configuration
image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
imagePullPolicy: IfNotPresent
command:
@@ -27396,38 +27469,40 @@
optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: web-configuration-template
mountPath: /etc/swh/configuration-template
- name: config-utils
mountPath: /entrypoints
readOnly: true
containers:
- - name: web-archive-sync-mailmaps
+ - name: web-archive-syncmailmaps
resources:
requests:
memory: 512Mi
cpu: 500m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
command:
- /opt/swh/entrypoint.sh
args:
- - sync-mailmaps
+ - django-admin
+ - sync_mailmaps
+ - --perform
- service=syncmailmaps
env:
- name: STATSD_HOST
value: prometheus-statsd-exporter
- name: STATSD_PORT
value: "9125"
- name: STATSD_TAGS
- value: deployment:web-archive-sync-mailmaps-cronjob
+ value: deployment:web-archive-syncmailmaps-cronjob
- name: SWH_CONFIG_FILENAME
value: /etc/swh/config.yml
- name: SWH_LOG_LEVEL
value: INFO
- name: SWH_SENTRY_ENVIRONMENT
value: production
- name: SWH_MAIN_PACKAGE
value: swh.web
- name: SWH_SENTRY_DSN
valueFrom:
@@ -27459,21 +27534,21 @@
name: config-utils
defaultMode: 0555
- name: web-configuration-template
configMap:
name: web-archive-configuration-template
items:
- key: "config.yml.template"
path: "config.yml.template"
- name: pgservice-configuration-template
configMap:
- name: pgservice-archive-configuration-template
+ name: web-archive-syncmailmaps-config-template
items:
- key: "pg-service-conf"
path: "pg_service.conf"
restartPolicy: OnFailure
---
# Source: swh/templates/counters/rpc-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
------------- diff for environment production namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.AfSyZwOF/production-swh-cassandra.before 2024-08-29 08:52:00.396649361 +0200
+++ /tmp/swh-chart.swh.AfSyZwOF/production-swh-cassandra.after 2024-08-29 08:52:02.148651604 +0200
@@ -5318,52 +5318,52 @@
optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: config-utils
mountPath: /entrypoints
readOnly: true
- name: do-migration
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
env:
- name: SWH_CONFIG_FILENAME
value: /etc/swh/config.yml
command:
- django-admin
args:
- migrate
- --settings=swh.web.settings.production
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: prepare-static
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- cp -r $PWD/.local/share/swh/web/static/ /usr/share/swh/web/static/
volumeMounts:
- name: static
mountPath: /usr/share/swh/web/static
containers:
- name: web-cassandra
resources:
requests:
memory: 500Mi
cpu: 500m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240822.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/web:20240828.1
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5004
name: webapp
readinessProbe:
httpGet:
path: /api/1/
port: webapp
httpHeaders:
- name: Host
Edited by Vincent Sellier