swh/scrubber: Avoid error because the message is not fully received
so the uncompress is failing
Related to swh/infra/sysadm-environment#5292 (closed)
helm diff
[swh] Comparing changes between branches production and scrubber-max-message-bytes (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in scrubber-max-message-bytes branch for environment staging...
[swh] Generate config in scrubber-max-message-bytes branch for environment staging...
[swh] Generate config in scrubber-max-message-bytes branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in scrubber-max-message-bytes branch for environment production...
[swh] Generate config in scrubber-max-message-bytes branch for environment production...
[swh] Generate config in scrubber-max-message-bytes branch for environment production...
------------- diff for environment staging namespace swh -------------
--- /tmp/swh-chart.swh.PmIi34pX/staging-swh.before 2024-03-22 18:06:49.135676800 +0100
+++ /tmp/swh-chart.swh.PmIi34pX/staging-swh.after 2024-03-22 18:06:50.167680915 +0100
@@ -167,20 +167,21 @@
config.yml.template: |
scrubber:
cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
journal:
brokers:
- journal2.internal.staging.swh.network:9094
batch_size: 100
cls: kafka
group_id: swh-archive-stg-journalchecker
+ message.max.bytes: 524288000
on_eof: restart
prefix: swh.journal.objects
sasl.mechanism: SCRAM-SHA-512
sasl.password: ${BROKER_USER_PASSWORD}
sasl.username: swh-archive-stg
security.protocol: SASL_SSL
---
# Source: swh/templates/scrubber/journal-checker-configmap.yaml
apiVersion: v1
kind: ConfigMap
@@ -191,20 +192,21 @@
config.yml.template: |
scrubber:
cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
journal:
brokers:
- journal2.internal.staging.swh.network:9094
batch_size: 200
cls: kafka
group_id: swh-archive-stg-journalchecker
+ message.max.bytes: 524288000
on_eof: restart
prefix: swh.journal.objects
privileged: true
sasl.mechanism: SCRAM-SHA-512
sasl.password: ${BROKER_USER_PASSWORD}
sasl.username: swh-archive-stg
security.protocol: SASL_SSL
---
# Source: swh/templates/scrubber/journal-checker-configmap.yaml
apiVersion: v1
@@ -215,20 +217,21 @@
data:
config.yml.template: |
scrubber:
cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
journal:
brokers:
- journal2.internal.staging.swh.network:9094
cls: kafka
group_id: swh-archive-stg-journalchecker
+ message.max.bytes: 524288000
on_eof: restart
prefix: swh.journal.objects
privileged: true
sasl.mechanism: SCRAM-SHA-512
sasl.password: ${BROKER_USER_PASSWORD}
sasl.username: swh-archive-stg
security.protocol: SASL_SSL
---
# Source: swh/templates/scrubber/journal-checker-configmap.yaml
apiVersion: v1
@@ -239,20 +242,21 @@
data:
config.yml.template: |
scrubber:
cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
journal:
brokers:
- journal2.internal.staging.swh.network:9094
cls: kafka
group_id: swh-archive-stg-journalchecker
+ message.max.bytes: 524288000
on_eof: restart
prefix: swh.journal.objects
sasl.mechanism: SCRAM-SHA-512
sasl.password: ${BROKER_USER_PASSWORD}
sasl.username: swh-archive-stg
security.protocol: SASL_SSL
---
# Source: swh/templates/scrubber/storage-checker-configmap.yaml
apiVersion: v1
kind: ConfigMap
@@ -1029,20 +1033,21 @@
data:
config.yml.template: |
scrubber:
cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
journal:
brokers:
- journal2.internal.staging.swh.network:9094
cls: kafka
group_id: swh-archive-stg-journalchecker
+ message.max.bytes: 524288000
on_eof: restart
prefix: swh.journal.objects
sasl.mechanism: SCRAM-SHA-512
sasl.password: ${BROKER_USER_PASSWORD}
sasl.username: swh-archive-stg
security.protocol: SASL_SSL
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
@@ -1919,21 +1924,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: scrubber-journalchecker-directory
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 96b76b398969a1ad44a77f55a6a1d8bdce60547b9736e556fb165afb0ce296c1
+ checksum/config: a142a5207d8d5a158b539d935b0b75953ba9650eadfd91cd4fd8700c9d5f5d4b
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/scrubber
operator: In
values:
@@ -2060,21 +2065,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: scrubber-journalchecker-release
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 4c6e2e536c49c92b0e70270dba3e0e2660037f0de751f5cdc8ee7cfcfd1c9c41
+ checksum/config: 45bb62ef95b6282856c3cef6780aa6d0adc52ceb94972461e554a99a004bc93b
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/scrubber
operator: In
values:
@@ -2201,21 +2206,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: scrubber-journalchecker-revision
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: e9564811e4873d9c86843e17271ebbaa26bcbab6ab572ee1a2b848245fd304dd
+ checksum/config: 80725683360fcab3686491dad8409f29decbba0b8f9ac03f9e860510a08aec18
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/scrubber
operator: In
values:
@@ -2342,21 +2347,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: scrubber-journalchecker-snapshot
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: ae0e4237615e8bf5612497b77eec94c04449867148d59572a0a2f463ba7aedae
+ checksum/config: 1decc7785aaeacbc3590c0885518b3f62728abd4ecd3bc5b207060acdc6e1e5b
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/scrubber
operator: In
values:
@@ -4646,21 +4651,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: swh-toolbox
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: e97aeaf7d335204f17859d112f29e02d63964df65690c3445ede85921cac9910
+ checksum/config: f826f7bced54609138a93052d0b80d9634c8fb6640f44c2816664794aedfddad
checksum/configScript: 5f0986a71adce81bcdab1a3886cbd18e67d89370b3a20b342248c5f15be406ff
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/toolbox
operator: In
values:
------------- diff for environment staging namespace swh-cassandra -------------
No differences
------------- diff for environment staging namespace swh-cassandra-next-version -------------
No differences
------------- diff for environment production namespace swh -------------
--- /tmp/swh-chart.swh.PmIi34pX/production-swh.before 2024-03-22 18:06:51.515686290 +0100
+++ /tmp/swh-chart.swh.PmIi34pX/production-swh.after 2024-03-22 18:06:52.191688984 +0100
@@ -1146,20 +1146,21 @@
url: http://objstorage-read-only-rpc-ingress
journal_client:
brokers:
- kafka1.internal.softwareheritage.org:9094
- kafka2.internal.softwareheritage.org:9094
- kafka3.internal.softwareheritage.org:9094
- kafka4.internal.softwareheritage.org:9094
batch_size: 200
cls: kafka
group_id: swh-archive-prod-swh.indexer.journal_client.extrinsic_metadata
+ message.max.bytes: "524288000"
prefix: swh.journal.objects
sasl.mechanism: SCRAM-SHA-512
sasl.password: ${BROKER_USER_PASSWORD}
sasl.username: ${BROKER_USER}
security.protocol: SASL_SSL
tools:
configuration: {}
name: swh-metadata-detector
version: 0.0.2
---
@@ -6555,20 +6556,21 @@
journal_client:
brokers:
- kafka1.internal.softwareheritage.org:9094
- kafka2.internal.softwareheritage.org:9094
- kafka3.internal.softwareheritage.org:9094
- kafka4.internal.softwareheritage.org:9094
auto_offset_reset: earliest
batch_size: 1000
cls: kafka
group_id: swh-archive-prod-winery-content-replayer
+ message.max.bytes: "524288000"
prefix: swh.journal.objects
sasl.mechanism: SCRAM-SHA-512
sasl.password: ${BROKER_USER_PASSWORD}
sasl.username: ${BROKER_USER}
security.protocol: SASL_SSL
replayer:
error_reporter:
db: 0
host: redis-winery-replay.redis
port: 6379
@@ -6728,20 +6730,21 @@
db: host=postgresql-scrubber-rw.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
journal:
brokers:
- kafka1.internal.softwareheritage.org:9094
- kafka2.internal.softwareheritage.org:9094
- kafka3.internal.softwareheritage.org:9094
- kafka4.internal.softwareheritage.org:9094
batch_size: 200
cls: kafka
group_id: swh-archive-prod-journalchecker
+ message.max.bytes: "524288000"
on_eof: restart
prefix: swh.journal.objects
privileged: true
sasl.mechanism: SCRAM-SHA-512
sasl.password: ${BROKER_USER_PASSWORD}
sasl.username: ${BROKER_USER}
security.protocol: SASL_SSL
---
# Source: swh/templates/scrubber/journal-checker-configmap.yaml
apiVersion: v1
@@ -6755,20 +6758,21 @@
cls: postgresql
db: host=postgresql-scrubber-rw.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
journal:
brokers:
- kafka1.internal.softwareheritage.org:9094
- kafka2.internal.softwareheritage.org:9094
- kafka3.internal.softwareheritage.org:9094
- kafka4.internal.softwareheritage.org:9094
cls: kafka
group_id: swh-archive-prod-journalchecker
+ message.max.bytes: "524288000"
on_eof: restart
prefix: swh.journal.objects
privileged: true
sasl.mechanism: SCRAM-SHA-512
sasl.password: ${BROKER_USER_PASSWORD}
sasl.username: ${BROKER_USER}
security.protocol: SASL_SSL
---
# Source: swh/templates/scrubber/journal-checker-configmap.yaml
apiVersion: v1
@@ -6782,20 +6786,21 @@
cls: postgresql
db: host=postgresql-scrubber-rw.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
journal:
brokers:
- kafka1.internal.softwareheritage.org:9094
- kafka2.internal.softwareheritage.org:9094
- kafka3.internal.softwareheritage.org:9094
- kafka4.internal.softwareheritage.org:9094
cls: kafka
group_id: swh-archive-prod-journalchecker
+ message.max.bytes: "524288000"
on_eof: restart
prefix: swh.journal.objects
sasl.mechanism: SCRAM-SHA-512
sasl.password: ${BROKER_USER_PASSWORD}
sasl.username: ${BROKER_USER}
security.protocol: SASL_SSL
---
# Source: swh/templates/scrubber/storage-checker-configmap.yaml
apiVersion: v1
kind: ConfigMap
@@ -7271,20 +7276,21 @@
cls: postgresql
db: host=postgresql-scrubber-rw.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
journal:
brokers:
- kafka1.internal.softwareheritage.org:9094
- kafka2.internal.softwareheritage.org:9094
- kafka3.internal.softwareheritage.org:9094
- kafka4.internal.softwareheritage.org:9094
cls: kafka
group_id: swh-archive-prod-journalchecker
+ message.max.bytes: "524288000"
prefix: swh.journal.objects
sasl.mechanism: SCRAM-SHA-512
sasl.password: ${BROKER_USER_PASSWORD}
sasl.username: ${BROKER_USER}
security.protocol: SASL_SSL
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
@@ -10801,21 +10807,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: indexer-extrinsic
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: e8cf7d6defe3e7763a222bd2a99022bb780fff2a85606b59617b2e86b57ac3cc
+ checksum/config: c7e0c293b656873a094e1a60a9820bf566384b320a373f9fef826cb83e81da50
checksum/config-utils: d75ca13b805bce6a8ab59c8e24c938f2283108f6a79134f6e71db86308651dc6
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/indexer
operator: In
@@ -19944,21 +19950,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: objstorage-replayer-winery
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 6e1dd1ab0b35d960a95bfb31eacd31dfbd41b7330718bb8dba44442521ea5001
+ checksum/config: c7207affa11c5c5a88beff35fd3e3cff90c7db1627aaf5f09c025aa74a279c21
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
@@ -21412,21 +21418,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: scrubber-journalchecker-release
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 69a71f10d6e0e06b61405b98ab8f40ee9125166ededc6399085a278d5b9f7fa8
+ checksum/config: cd0a1cee59b9f8bc72fb93a2d4c607377e88751cb4e85abb0b47a2f592051711
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/scrubber
operator: In
values:
@@ -21560,21 +21566,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: scrubber-journalchecker-revision
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: a7114bfff7438621711958b96aa080caa988b33bad0b01c5244c5be0ca47579b
+ checksum/config: 84513e943956d5146668540e38c8e6b673e84bb969e8a9536dd2d21aece1071d
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/scrubber
operator: In
values:
@@ -21708,21 +21714,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: scrubber-journalchecker-snapshot
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 5559dd70c4b1aa47f2e1577e3734691e2f717ec141418dcb40720f3f5738b02a
+ checksum/config: cd4d24b648de17139c423cc02ae29aa100a22bdfd6a7a26fdca91183bb06e82d
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/scrubber
operator: In
values:
@@ -23877,21 +23883,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: swh-toolbox
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: ad08aa1e482a1681ce6d3bf1f53a4bbd00d15bedffbf39844a779c7cd1d63281
+ checksum/config: 7f5c16452ec35bd838368c66e83ace493b8d65905ae1c1a8df7679703a5f7428
checksum/configScript: 617a3bf1b7cc114666253d98def7fe1fb764db4c48d5398e09cc7f506c5f4cf6
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/toolbox
operator: In
values:
------------- diff for environment production namespace swh-cassandra -------------
No differences
Edited by Vincent Sellier