Merge pull request #6978 from hakonhagland/fix_rate_comm

Reservoir coupling: Add end-of-sync-step slave-to-master data exchange

Author: totto82

opm/simulators/flow/rescoup/ReservoirCouplingMaster.cpp

@@ -172,6 +172,24 @@ isFirstSubstepOfSyncTimestep() const
     return this->report_step_data_->isFirstSubstepOfSyncTimestep();
 }
 
+template <class Scalar>
+bool
+ReservoirCouplingMaster<Scalar>::
+needsSlaveDataReceive() const
+{
+    assert(this->report_step_data_);
+    return this->report_step_data_->needsSlaveDataReceive();
+}
+
+template <class Scalar>
+void
+ReservoirCouplingMaster<Scalar>::
+setNeedsSlaveDataReceive(bool value)
+{
+    assert(this->report_step_data_);
+    this->report_step_data_->setNeedsSlaveDataReceive(value);
+}
+
 template <class Scalar>
 bool
 ReservoirCouplingMaster<Scalar>::

opm/simulators/flow/rescoup/ReservoirCouplingMaster.hpp

@@ -105,6 +105,17 @@ class ReservoirCouplingMaster {
     void initTimeStepping();
     bool isFirstSubstepOfSyncTimestep() const;
     bool isMasterGroup(const std::string &group_name) const;
+    /// @brief Check if the master needs to receive production data from the slaves.
+    /// @details This flag is used to control reservoir coupling synchronization of
+    ///          summary data sent from the slaves to the master process.
+    ///          The master blocks in timeStepSucceeded() until all slaves have sent
+    ///          their production data.
+    /// @return true if the master needs to receive production data from the slaves, false if not
+    bool needsSlaveDataReceive() const;
+    /// @brief Set whether the master needs to receive production data from the slaves.
+    /// @details See needsSlaveDataReceive() for details.
+    /// @param value true if the master needs to receive production data from the slaves, false if not
+    void setNeedsSlaveDataReceive(bool value);
     ReservoirCoupling::Logger& logger() { return this->logger_; }
     ReservoirCoupling::Logger& logger() const { return this->logger_; }
     void maybeActivate(int report_step);

opm/simulators/flow/rescoup/ReservoirCouplingMasterReportStep.hpp

@@ -154,6 +154,14 @@ class ReservoirCouplingMasterReportStep {
     /// @return true if this is the first substep of a "sync" timestep, false if not
     bool isFirstSubstepOfSyncTimestep() const { return is_first_substep_of_sync_timestep_; }
 
+    /// @brief Check if the master needs to receive slave data at the next
+    ///   timeStepSucceeded() call.
+    /// @details The slave data is needed for correct summary output.
+    bool needsSlaveDataReceive() const { return needs_slave_data_receive_; }
+
+    /// @brief Set/clear the flag for pending slave data receive.
+    void setNeedsSlaveDataReceive(bool value) { needs_slave_data_receive_ = value; }
+
     /// @brief Get the number of slave groups for a specific slave process
     /// @param index Index of the slave process
     /// @return Number of groups managed by the specified slave
@@ -247,6 +255,13 @@ class ReservoirCouplingMasterReportStep {
     /// Flag to track if this is the first substep within a "sync" timestep.
     /// Used to control reservoir coupling synchronization.
     bool is_first_substep_of_sync_timestep_{true};
+
+    /// Flag to indicate that the master needs to receive end-of-sync-step
+    /// production data from slaves.  Set at the start of each sync step,
+    /// cleared after the receive.  The receive happens in timeStepSucceeded()
+    /// on the first converged substep, so that the data is available for
+    /// evalSummaryState() and all subsequent substeps of the same sync step.
+    bool needs_slave_data_receive_{false};
 };
 } // namespace Opm
 #endif // OPM_RESERVOIR_COUPLING_MASTER_REPORT_STEP_HPP

opm/simulators/flow/rescoup/ReservoirCouplingSlave.cpp

@@ -107,6 +107,15 @@ isFirstSubstepOfSyncTimestep() const
     return this->report_step_data_->isFirstSubstepOfSyncTimestep();
 }
 
+template <class Scalar>
+bool
+ReservoirCouplingSlave<Scalar>::
+isLastSubstepOfSyncTimestep() const
+{
+    assert(this->report_step_data_);
+    return this->report_step_data_->isLastSubstepOfSyncTimestep();
+}
+
 template <class Scalar>
 bool
 ReservoirCouplingSlave<Scalar>::
@@ -351,6 +360,15 @@ setFirstSubstepOfSyncTimestep(bool value)
     this->report_step_data_->setFirstSubstepOfSyncTimestep(value);
 }
 
+template <class Scalar>
+void
+ReservoirCouplingSlave<Scalar>::
+setLastSubstepOfSyncTimestep(bool value)
+{
+    assert(this->report_step_data_);
+    this->report_step_data_->setLastSubstepOfSyncTimestep(value);
+}
+
 // ------------------
 // Private methods
 // ------------------

opm/simulators/flow/rescoup/ReservoirCouplingSlave.hpp

@@ -70,6 +70,14 @@ class ReservoirCouplingSlave {
     bool hasMasterProductionTarget(const std::string& gname) const;
     void initTimeStepping();
     bool isFirstSubstepOfSyncTimestep() const;
+    /// @brief Check if this is the last substep within a "sync" timestep.
+    /// @details This flag is used to control reservoir coupling synchronization of
+    ///          summary data sent from the slave to the master process.
+    ///          The slave should send production data to the master at the end of
+    ///          its "sync" timestep, while master is waiting for it in timeStepSucceeded()
+    ///          of the first substep of the sync step.
+    /// @return true if this is the last substep of a "sync" timestep, false if not
+    bool isLastSubstepOfSyncTimestep() const;
     bool isSlaveGroup(const std::string& group_name) const;
     ReservoirCoupling::Logger& logger() { return this->logger_; }
     ReservoirCoupling::Logger& logger() const { return this->logger_; }
@@ -105,6 +113,10 @@ class ReservoirCouplingSlave {
         this->logger_.setDeferredLogger(deferred_logger);
     }
     void setFirstSubstepOfSyncTimestep(bool value);
+    /// @brief Set whether this is the last substep within a "sync" timestep.
+    /// @details See isLastSubstepOfSyncTimestep() for details.
+    /// @param value true if this is the last substep of a "sync" timestep, false if not
+    void setLastSubstepOfSyncTimestep(bool value);
     const std::string& slaveGroupIdxToGroupName(std::size_t group_idx) const {
         return this->slave_group_order_.at(group_idx);
     }

opm/simulators/flow/rescoup/ReservoirCouplingSlaveReportStep.hpp

@@ -98,6 +98,15 @@ class ReservoirCouplingSlaveReportStep {
     /// @return true if this is the first substep of a "sync" timestep, false if not
     bool isFirstSubstepOfSyncTimestep() const { return is_first_substep_of_sync_timestep_; }
 
+    /// @brief Check if this is the last substep within a "sync" timestep.
+    /// @details This flag is used to control reservoir coupling synchronization of
+    ///          summary data sent from the slave to the master process.
+    ///          The slave should send production data to the master at the end of
+    ///          its "sync" timestep, while master is waiting for it in timeStepSucceeded()
+    ///          of the first substep of the sync step.
+    /// @return true if this is the last substep of a "sync" timestep, false if not
+    bool isLastSubstepOfSyncTimestep() const { return is_last_substep_of_sync_timestep_; }
+
     /// @brief Get the logger for reservoir coupling operations
     /// @return Reference to the logger object for this coupling session
     ReservoirCoupling::Logger& logger() const { return this->slave_.logger(); }
@@ -162,6 +171,11 @@ class ReservoirCouplingSlaveReportStep {
     /// @param value true at start of sync timestep, false after first runSubStep_() call
     void setFirstSubstepOfSyncTimestep(bool value) { is_first_substep_of_sync_timestep_ = value; }
 
+    /// @brief Set whether this is the last substep within a "sync" timestep.
+    /// @details See isLastSubstepOfSyncTimestep() for details.
+    /// @param value true if this is the last substep of a "sync" timestep, false if not
+    void setLastSubstepOfSyncTimestep(bool value) { is_last_substep_of_sync_timestep_ = value; }
+
     /// @brief Get the name of this slave process
     /// @return Reference to the name string for this slave
     const std::string& slaveName() const { return this->slave_.getSlaveName(); }
@@ -215,6 +229,10 @@ class ReservoirCouplingSlaveReportStep {
     // Flag to track if this is the first substep within a "sync" timestep.
     // Used to control reservoir coupling synchronization.
     bool is_first_substep_of_sync_timestep_{true};
+    // Flag to track if this is the last substep within a "sync" timestep.
+    // Used to control reservoir coupling synchronization of summary data sent from
+    // the slave to the master process.
+    bool is_last_substep_of_sync_timestep_{false};
 
     // Master-imposed targets and corresponding control modes, received from the master
     // process at the beginning of each sync timestep. Cleared and repopulated on every

opm/simulators/timestepping/AdaptiveTimeStepping.hpp

@@ -156,6 +156,7 @@ class AdaptiveTimeStepping
         double maybeRestrictTimeStepGrowth_(const double dt,
                                             double dt_estimate,
                                             const int restarts) const;
+        void maybeUpdateLastSubstepOfSyncTimestep_(double dt);
         void maybeUpdateTuningAndTimeStep_();
         double maxGrowth_() const;
         double minTimeStepBeforeClosingWells_() const;

opm/simulators/timestepping/AdaptiveTimeStepping_impl.hpp

@@ -682,6 +682,10 @@ runStepReservoirCouplingMaster_()
         // whether master-slave data exchange should occur in beginTimeStep() in the well model.
         // It will be cleared after the first runSubStep_() call.
         reservoirCouplingMaster_().setFirstSubstepOfSyncTimestep(true);
+        // After the first master substep completes, timeStepSucceeded() will
+        // block until slaves finish the sync step and send production data.
+        // This ensures correct summary output for all subsequent substeps.
+        reservoirCouplingMaster_().setNeedsSlaveDataReceive(true);
         SubStepIteration<Solver> substepIteration{*this, substep_timer, current_step_length, final_step};
         const auto sub_steps_report = substepIteration.run();
         report += sub_steps_report;
@@ -806,6 +810,7 @@ run()
             detail::logTimer(this->substep_timer_);
         }
 
+        maybeUpdateLastSubstepOfSyncTimestep_(dt);  // Needed for reservoir coupling
         auto substep_report = runSubStep_();
         markFirstSubStepAsFinished_();  // Needed for reservoir coupling
 
@@ -1154,6 +1159,28 @@ maybeRestrictTimeStepGrowth_(const double dt, double dt_estimate, const int rest
     return dt_estimate;
 }
 
+
+template<class TypeTag>
+template<class Solver>
+void
+AdaptiveTimeStepping<TypeTag>::SubStepIteration<Solver>::
+maybeUpdateLastSubstepOfSyncTimestep_(double dt)
+{
+#ifdef RESERVOIR_COUPLING_ENABLED
+    // For reservoir coupling slaves: predict if this substep will complete
+    // the sync timestep.  If so, timeStepSucceeded() will send production
+    // data to the master (which is blocking on receive after its first substep).
+    // This is used for summary data synchronization between slaves and master.
+    if (isReservoirCouplingSlave_()) {
+        const bool is_last = ReservoirCoupling::Seconds::compare_gt_or_eq(
+            this->substep_timer_.simulationTimeElapsed() + dt,
+            this->substep_timer_.totalTime()
+        );
+        reservoirCouplingSlave_().setLastSubstepOfSyncTimestep(is_last);
+    }
+#endif
+}
+
 // The maybeUpdateTuning_() lambda callback is defined in SimulatorFullyImplicitBlackoil::runStep()
 // It has to be called for each substep since TUNING might have been changed for next sub step due
 // to ACTIONX (via NEXTSTEP) or WCYCLE keywords.

opm/simulators/wells/BlackoilWellModel.hpp

@@ -434,6 +434,7 @@ template<class Scalar> class WellContributions;
 
             void receiveGroupConstraintsFromMaster();
             void sendMasterGroupConstraintsToSlaves();
+            void rescoupSyncSummaryData();
 
             /// \brief Setup RAII guard for reservoir coupling logger
             ///

opm/simulators/wells/BlackoilWellModel_impl.hpp

@@ -642,6 +642,34 @@ namespace Opm {
         constraint_receiver.receiveGroupConstraintsFromMaster();
     }
 
+    template<typename TypeTag>
+    void
+    BlackoilWellModel<TypeTag>::
+    rescoupSyncSummaryData()
+    {
+        // Reservoir coupling: exchange production data between slaves and master.
+        //
+        // Master side: after its first substep, the master blocks here until all
+        // slaves have completed the sync step and sent their production data.
+        // This ensures evalSummaryState() (called next in endTimeStep) and all
+        // subsequent master substeps have correct slave production rates.
+        //
+        // Slave side: on the last substep of the sync step, the slave sends its
+        // production data to the master.  The master is already waiting at this
+        // point (blocked on MPI_Recv from its first substep's timeStepSucceeded).
+        if (this->isReservoirCouplingMaster()) {
+            if (this->reservoirCouplingMaster().needsSlaveDataReceive()) {
+                this->receiveSlaveGroupData();
+                this->reservoirCouplingMaster().setNeedsSlaveDataReceive(false);
+            }
+        }
+        if (this->isReservoirCouplingSlave()) {
+            if (this->reservoirCouplingSlave().isLastSubstepOfSyncTimestep()) {
+                this->sendSlaveGroupDataToMaster();
+            }
+        }
+    }
+
 #endif // RESERVOIR_COUPLING_ENABLED
 
     template<typename TypeTag>
@@ -797,6 +825,9 @@ namespace Opm {
 
         this->groupStateHelper().updateNONEProductionGroups();
 
+#ifdef RESERVOIR_COUPLING_ENABLED
+        this->rescoupSyncSummaryData();
+#endif
         this->commitWGState();
 
         //reporting output temperatures