storages: Add model validation server side as well
Belts and whistles!
Test Plan
octoctalog
$ bin/octocatalog-diff --octocatalog-diff-args --no-truncate-details --to staging storage0.internal.staging.swh.network
Found host storage0.internal.staging.swh.network
Cloning into '/tmp/swh-ocd.7pMGQikp/environments/production/data/private'...
done.
Cloning into '/tmp/swh-ocd.7pMGQikp/environments/staging/data/private'...
done.
*** Running octocatalog-diff on host storage0.internal.staging.swh.network
I, [2020-02-27T10:42:12.080252 #30133] INFO -- : Catalogs compiled for storage0.internal.staging.swh.network
I, [2020-02-27T10:42:13.221729 #30133] INFO -- : Diffs computed for storage0.internal.staging.swh.network
diff origin/production/storage0.internal.staging.swh.network current/storage0.internal.staging.swh.network
*******************************************
File[/etc/softwareheritage/storage/storage.yml] =>
parameters =>
content =>
@@ -1,6 +1,8 @@
---
storage:
- cls: local
- args:
+ cls: pipeline
+ steps:
+ - cls: validate
+ - cls: local
db: host=db0.internal.staging.swh.network port=5432 user=swh dbname=swh password=swh::deploy::storage::db::password
objstorage:
@@ -9,12 +11,12 @@
root: "/srv/softwareheritage/objects"
slicing: 0:1/1:5
-journal_writer:
- cls: kafka
- args:
- brokers:
- - journal0.internal.staging.swh.network
- prefix: swh.journal.objects
- client_id: swh.storage.journal_writer.storage0
- producer_config:
- message.max.bytes: 1000000000
+ journal_writer:
+ cls: kafka
+ args:
+ brokers:
+ - journal0.internal.staging.swh.network
+ prefix: swh.journal.objects
+ client_id: swh.storage.journal_writer.storage0
+ producer_config:
+ message.max.bytes: 1000000000
_
*******************************************
*** End octocatalog-diff on storage0.internal.staging.swh.network
$ bin/octocatalog-diff --octocatalog-diff-args --no-truncate-details --to staging uffizi
Found host uffizi.softwareheritage.org
Cloning into '/tmp/swh-ocd.kVY7XoPT/environments/production/data/private'...
done.
Cloning into '/tmp/swh-ocd.kVY7XoPT/environments/staging/data/private'...
done.
*** Running octocatalog-diff on host uffizi.softwareheritage.org
I, [2020-02-27T10:46:18.673130 #2782] INFO -- : Catalogs compiled for uffizi.softwareheritage.org
I, [2020-02-27T10:46:19.830727 #2782] INFO -- : Diffs computed for uffizi.softwareheritage.org
diff origin/production/uffizi.softwareheritage.org current/uffizi.softwareheritage.org
*******************************************
- File[/etc/softwareheritage/journal]
*******************************************
File[/etc/softwareheritage/storage/storage.yml] =>
parameters =>
content =>
@@ -1,104 +1,107 @@
---
storage:
- cls: local
- args:
- db: host=db.internal.softwareheritage.org user=swhstorage dbname=softwareheritage
- password=swh::deploy::storage::db::password
- objstorage:
- cls: multiplexer
- args:
- objstorages:
- - cls: pathslicing
- args:
- root: "/srv/softwareheritage/objects"
- slicing: 0:2/0:5
- compression: none
- - cls: filtered
- args:
- storage_conf:
- cls: pathslicing
- args:
- root: "/srv/softwareheritage/objects-xfs"
- slicing: 0:1/0:2/2:4/4:6
- compression: gzip
- filters_conf:
- - type: readonly
- - cls: azure-prefixed
- args:
- accounts:
- '0':
- account_name: 0euwestswh
- api_secret_key: swh::azure::credentials::0euwestswh
- container_name: contents
- '1':
- account_name: 1euwestswh
- api_secret_key: swh::azure::credentials::1euwestswh
- container_name: contents
- '2':
- account_name: 2euwestswh
- api_secret_key: swh::azure::credentials::2euwestswh
- container_name: contents
- '3':
- account_name: 3euwestswh
- api_secret_key: swh::azure::credentials::3euwestswh
- container_name: contents
- '4':
- account_name: 4euwestswh
- api_secret_key: swh::azure::credentials::4euwestswh
- container_name: contents
- '5':
- account_name: 5euwestswh
- api_secret_key: swh::azure::credentials::5euwestswh
- container_name: contents
- '6':
- account_name: 6euwestswh
- api_secret_key: swh::azure::credentials::6euwestswh
- container_name: contents
- '7':
- account_name: 7euwestswh
- api_secret_key: swh::azure::credentials::7euwestswh
- container_name: contents
- '8':
- account_name: 8euwestswh
- api_secret_key: swh::azure::credentials::8euwestswh
- container_name: contents
- '9':
- account_name: 9euwestswh
- api_secret_key: swh::azure::credentials::9euwestswh
- container_name: contents
- a:
- account_name: aeuwestswh
- api_secret_key: swh::azure::credentials::aeuwestswh
- container_name: contents
- b:
- account_name: beuwestswh
- api_secret_key: swh::azure::credentials::beuwestswh
- container_name: contents
- c:
- account_name: ceuwestswh
- api_secret_key: swh::azure::credentials::ceuwestswh
- container_name: contents
- d:
- account_name: deuwestswh
- api_secret_key: swh::azure::credentials::deuwestswh
- container_name: contents
- e:
- account_name: eeuwestswh
- api_secret_key: swh::azure::credentials::eeuwestswh
- container_name: contents
- f:
- account_name: feuwestswh
- api_secret_key: swh::azure::credentials::feuwestswh
- container_name: contents
- journal_writer:
- cls: kafka
- args:
- brokers:
- - esnode1.internal.softwareheritage.org
- - esnode2.internal.softwareheritage.org
- - esnode3.internal.softwareheritage.org
- prefix: swh.journal.objects
- client_id: swh.storage.journal_writer.uffizi
- producer_config:
- message.max.bytes: 1000000000
+ cls: pipeline
+ steps:
+ - cls: validate
+ - cls: local
+ args:
+ db: host=db.internal.softwareheritage.org user=swhstorage dbname=softwareheritage
+ password=swh::deploy::storage::db::password
+ objstorage:
+ cls: multiplexer
+ args:
+ objstorages:
+ - cls: pathslicing
+ args:
+ root: "/srv/softwareheritage/objects"
+ slicing: 0:2/0:5
+ compression: none
+ - cls: filtered
+ args:
+ storage_conf:
+ cls: pathslicing
+ args:
+ root: "/srv/softwareheritage/objects-xfs"
+ slicing: 0:1/0:2/2:4/4:6
+ compression: gzip
+ filters_conf:
+ - type: readonly
+ - cls: azure-prefixed
+ args:
+ accounts:
+ '0':
+ account_name: 0euwestswh
+ api_secret_key: swh::azure::credentials::0euwestswh
+ container_name: contents
+ '1':
+ account_name: 1euwestswh
+ api_secret_key: swh::azure::credentials::1euwestswh
+ container_name: contents
+ '2':
+ account_name: 2euwestswh
+ api_secret_key: swh::azure::credentials::2euwestswh
+ container_name: contents
+ '3':
+ account_name: 3euwestswh
+ api_secret_key: swh::azure::credentials::3euwestswh
+ container_name: contents
+ '4':
+ account_name: 4euwestswh
+ api_secret_key: swh::azure::credentials::4euwestswh
+ container_name: contents
+ '5':
+ account_name: 5euwestswh
+ api_secret_key: swh::azure::credentials::5euwestswh
+ container_name: contents
+ '6':
+ account_name: 6euwestswh
+ api_secret_key: swh::azure::credentials::6euwestswh
+ container_name: contents
+ '7':
+ account_name: 7euwestswh
+ api_secret_key: swh::azure::credentials::7euwestswh
+ container_name: contents
+ '8':
+ account_name: 8euwestswh
+ api_secret_key: swh::azure::credentials::8euwestswh
+ container_name: contents
+ '9':
+ account_name: 9euwestswh
+ api_secret_key: swh::azure::credentials::9euwestswh
+ container_name: contents
+ a:
+ account_name: aeuwestswh
+ api_secret_key: swh::azure::credentials::aeuwestswh
+ container_name: contents
+ b:
+ account_name: beuwestswh
+ api_secret_key: swh::azure::credentials::beuwestswh
+ container_name: contents
+ c:
+ account_name: ceuwestswh
+ api_secret_key: swh::azure::credentials::ceuwestswh
+ container_name: contents
+ d:
+ account_name: deuwestswh
+ api_secret_key: swh::azure::credentials::deuwestswh
+ container_name: contents
+ e:
+ account_name: eeuwestswh
+ api_secret_key: swh::azure::credentials::eeuwestswh
+ container_name: contents
+ f:
+ account_name: feuwestswh
+ api_secret_key: swh::azure::credentials::feuwestswh
+ container_name: contents
+ journal_writer:
+ cls: kafka
+ args:
+ brokers:
+ - esnode1.internal.softwareheritage.org
+ - esnode2.internal.softwareheritage.org
+ - esnode3.internal.softwareheritage.org
+ prefix: swh.journal.objects
+ client_id: swh.storage.journal_writer.uffizi
+ producer_config:
+ message.max.bytes: 1000000000
_
*******************************************
- Package[python3-swh.journal]
*******************************************
*** End octocatalog-diff on uffizi.softwareheritage.orgMigrated from D2719 (view on Phabricator)