Refactor the buffer system and add methods for synchronize_communication!

.buildkite/distributed/pipeline.yml

@@ -29,7 +29,7 @@ steps:
 
   - wait
 
-  - label: "🐉 cpu distributed unit tests"
+  - label: "🐉 cpu unit tests"
     key: "distributed_cpu"
     env:
       TEST_GROUP: "distributed"
@@ -44,7 +44,7 @@ steps:
         - exit_status: 1
           limit: 1
 
-  - label: "🐲 gpu distributed unit tests"
+  - label: "🐲 gpu unit tests"
     key: "distributed_gpu"
     env:
       TEST_GROUP: "distributed"
@@ -60,7 +60,7 @@ steps:
         - exit_status: 1
           limit: 1
 
-  - label: "🦾 cpu distributed solvers tests"
+  - label: "🦾 cpu solvers tests"
     key: "distributed_solvers_cpu"
     env:
       TEST_GROUP: "distributed_solvers"
@@ -74,7 +74,7 @@ steps:
         - exit_status: 1
           limit: 1
 
-  - label: "🛸 gpu distributed solvers tests"
+  - label: "🛸 gpu solvers tests"
     key: "distributed_solvers_gpu"
     env:
       TEST_GROUP: "distributed_solvers"
@@ -90,7 +90,40 @@ steps:
         - exit_status: 1
           limit: 1
 
-  - label: "🤺 cpu distributed hydrostatic model tests"
+  - label: "🤿 cpu hydrostatic regression tests"
+    key: "distributed_hydrostatic_regression_cpu"
+    env:
+      TEST_GROUP: "distributed_hydrostatic_regression"
+      TEST_ARCHITECTURE: "CPU"
+    commands:
+      - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'"
+    timeout_in_minutes: 1440
+    agents:
+      slurm_mem: 50G
+      slurm_ntasks: 4
+    retry:
+      automatic:
+        - exit_status: 1
+          limit: 1
+
+  - label: "🥽 gpu hydrostatic regression tests"
+    key: "distributed_hydrostatic_regression_gpu"
+    env:
+      TEST_GROUP: "distributed_hydrostatic_regression"
+      TEST_ARCHITECTURE: "GPU"
+    commands:
+      - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'"
+    timeout_in_minutes: 1440
+    agents:
+      slurm_mem: 100G # Apparently the GPU tests require more memory
+      slurm_ntasks: 4
+      slurm_gpus_per_task: 1
+    retry:
+      automatic:
+        - exit_status: 1
+          limit: 1
+
+  - label: "🤺 cpu hydrostatic model tests"
     key: "distributed_hydrostatic_model_cpu"
     env:
       TEST_GROUP: "distributed_hydrostatic_model"
@@ -106,7 +139,7 @@ steps:
         - exit_status: 1
           limit: 1
 
-  - label: "🦏 gpu distributed hydrostatic model tests"
+  - label: "🦏 gpu hydrostatic model tests"
     key: "distributed_hydrostatic_model_gpu"
     env:
       TEST_GROUP: "distributed_hydrostatic_model"
@@ -123,7 +156,7 @@ steps:
         - exit_status: 1
           limit: 1
 
-  - label: "🦍 cpu distributed nonhydrostatic regression"
+  - label: "🦍 cpu nonhydrostatic regression"
     key: "distributed_nonhydrostatic_regression_cpu"
     env:
       TEST_GROUP: "distributed_nonhydrostatic_regression"
@@ -138,7 +171,7 @@ steps:
         - exit_status: 1
           limit: 1
 
-  - label: "🕺 gpu distributed nonhydrostatic regression"
+  - label: "🕺 gpu nonhydrostatic regression"
     key: "distributed_nonhydrostatic_regression_gpu"
     env:
       TEST_GROUP: "distributed_nonhydrostatic_regression"