Fix integer overflows and parallelize the output writers for large meshes

Common/include/CConfig.hpp

@@ -747,6 +747,7 @@ class CConfig {
   su2double Cauchy_Eps;               /*!< \brief Epsilon used for the convergence. */
   bool Restart,                       /*!< \brief Restart solution (for direct, adjoint, and linearized problems).*/
   Wrt_Restart_Compact,                /*!< \brief Write compact restart files with minimum nr. of variables. */
+  Wrt_Output_Double_Precision,        /*!< \brief Write the fields of the volume and surface files in double. */
   Read_Binary_Restart,                /*!< \brief Read binary SU2 native restart files.*/
   Wrt_Restart_Overwrite,              /*!< \brief Overwrite restart files or append iteration number.*/
   Wrt_Surface_Overwrite,              /*!< \brief Overwrite surface output files or append iteration number.*/
@@ -5682,6 +5683,12 @@ class CConfig {
    */
   bool GetWrt_Restart_Compact(void) const { return Wrt_Restart_Compact; }
 
+  /*!
+   * \brief Flag for whether the fields of the volume and surface files are written in double precision.
+   * \return <code>TRUE</code> means that double precision is used.
+   */
+  bool GetWrt_Output_Double_Precision(void) const { return Wrt_Output_Double_Precision; }
+
   /*!
    * \brief Flag for whether restart solution files are overwritten.
    * \return Flag for overwriting. If Flag=false, iteration nr is appended to filename

Common/include/parallelization/mpi_structure.hpp

@@ -488,6 +488,7 @@ class CMediMPIWrapper : public CBaseMPIWrapper {
 #define MPI_MAX 10
 #define MPI_INT 11
 #define MPI_PROD 12
+#define MPI_BOR 13
 #define MPI_STATUS_IGNORE nullptr
 
 /*!

Common/src/CConfig.cpp

@@ -1263,6 +1263,10 @@ void CConfig::SetConfig_Options() {
   addBoolOption("RESTART_SOL", Restart, false);
   /*!\brief WRT_RESTART_COMPACT \n DESCRIPTION: Minimize the size of restart files \n Options: NO, YES \ingroup Config */
   addBoolOption("WRT_RESTART_COMPACT", Wrt_Restart_Compact, true);
+  /*!\brief WRT_OUTPUT_DOUBLE_PRECISION \n DESCRIPTION: Write the fields of the volume and surface files (CGNS and
+   Paraview XML) in double precision instead of single, the coordinates are always written in double precision by the
+   CGNS writer. \n Options: NO, YES \ingroup Config */
+  addBoolOption("WRT_OUTPUT_DOUBLE_PRECISION", Wrt_Output_Double_Precision, false);
   /*!\brief BINARY_RESTART \n DESCRIPTION: Read binary SU2 native restart files. \n Options: YES, NO \ingroup Config */
   addBoolOption("READ_BINARY_RESTART", Read_Binary_Restart, true);
   /*!\brief WRT_RESTART_OVERWRITE \n DESCRIPTION: overwrite restart files or append iteration number. \n Options: YES, NO \ingroup Config */

SU2_CFD/include/output/filewriter/CCGNSFileWriter.hpp

@@ -32,13 +32,38 @@
 #include <basetsd.h>
 #endif
 #include "cgnslib.h"
+#ifdef HAVE_MPI
+#include "pcgnslib.h"
 #endif
+#endif
+
+#include <cstdint>
+#include <limits>
 
 #include "CFileWriter.hpp"
 
+class CConfig;
+class CGeometry;
+class CFVMDataSorter;
+
 class CCGNSFileWriter final : public CFileWriter {
  private:
-  const bool isSurface; /*!< \brief True if surface file. */
+  const bool isSurface;             /*!< \brief True if surface file. */
+  const bool doublePrecisionFields; /*!< \brief True to write the fields in double precision instead of single. */
+
+  /*!
+   * \brief Boundary elements of one marker owned by this rank, written as a boundary section of a volume file.
+   */
+  struct BoundaryMarker {
+    string name;                /*!< \brief Marker tag. */
+    unsigned short kindBC;      /*!< \brief SU2 boundary condition kind. */
+    vector<unsigned long> conn; /*!< \brief VTK type followed by the (1-based) output ids of the nodes, per element. */
+  };
+  vector<BoundaryMarker> boundaryMarkers; /*!< \brief Markers written as boundaries of a volume file. */
+
+  vector<string> surfaceMarkers; /*!< \brief Markers written as one zone each in a surface file. */
+  CConfig* config = nullptr;     /*!< \brief Config, to sort the surface data of each marker. */
+  CGeometry* geometry = nullptr; /*!< \brief Geometry, to sort the surface data of each marker. */
 
 #ifdef HAVE_CGNS
   int cgnsFileID; /*!< \brief CGNS file identifier. */
@@ -54,15 +79,10 @@ class CCGNSFileWriter final : public CFileWriter {
   cgsize_t GlobalPoint;       /*!< \brief Total number of points. */
   cgsize_t GlobalElem;        /*!< \brief Total number of elements. */
 
-  typedef float dataPrecision;            /*!< \brief Define data precision of output (float or double). */
-  const DataType_t dataType = RealSingle; /*!< \brief Datatype of fields can be RealSingle or RealDouble. */
-
-  vector<cgsize_t> sendBufferConnectivity; /*!< \brief Send buffer for connectivity data. */
-  vector<cgsize_t> recvBufferConnectivity; /*!< \brief Receive buffer for connectivity data. */
-  vector<dataPrecision> recvBufferField;   /*!< \brief Send buffer for field data. */
-  vector<dataPrecision> sendBufferField;   /*!< \brief Receive buffer for field data. */
-
   cgsize_t cumulative; /*!< \brief Cumulative number of elements written. */
+
+  /*--- Max connectivity entries per section, so that readers using 32-bit sizes can read it. ---*/
+  static constexpr cgsize_t maxSectionEntries = std::numeric_limits<int32_t>::max();
 #endif
  public:
   /*!
@@ -75,28 +95,72 @@ class CCGNSFileWriter final : public CFileWriter {
    * \param[in] valDataSorter - The parallel sorted data to write.
    * \param[in] isSurf - True if it is a surface file.
    */
-  CCGNSFileWriter(CParallelDataSorter* valDataSorter, bool isSurf = false);
+  CCGNSFileWriter(CParallelDataSorter* valDataSorter, bool isSurf = false, bool doublePrecision = false);
 
   /*!
    * \brief Write sorted data to file in CGNS file format.
    * \param[in] val_filename - The name of the file.
    */
   void WriteData(string val_filename) override ;
 
+  /*!
+   * \brief Add the boundaries to a volume file: one boundary section, BC and family per marker, named as the marker.
+   * \param[in] valConfig - Definition of the problem.
+   * \param[in] valGeometry - Geometrical definition of the problem.
+   * \param[in] volumeSorter - The volume data sorter, to find the boundary elements owned by this rank.
+   */
+  void SetBoundaryMarkers(CConfig* valConfig, CGeometry* valGeometry, const CFVMDataSorter* volumeSorter);
+
+  /*!
+   * \brief Write a surface file with one zone per plotted marker, named as the marker. The data of the surface
+   *        sorter is sorted again for each marker when the file is written.
+   * \param[in] valConfig - Definition of the problem.
+   * \param[in] valGeometry - Geometrical definition of the problem.
+   */
+  void SetSurfaceMarkers(CConfig* valConfig, CGeometry* valGeometry);
+
  private:
 #ifdef HAVE_CGNS
   /*!
-   * \brief Initialize CGNS mesh file.
+   * \brief Create the CGNS file and its base.
+   * \param[in] val_filename - The name of the file.
    */
   void InitializeMeshFile(const string& val_filename);
 
   /*!
-   * \brief Write i-th coordinate to file in CGNS file format.
+   * \brief Write a zone with the data currently held by the data sorter.
+   * \param[in] zoneName - Name of the zone.
+   */
+  void WriteZone(const string& zoneName);
+
+  /*!
+   * \brief Create a zone for the data currently held by the data sorter.
+   * \param[in] zoneName - Name of the zone.
+   */
+  void InitializeZone(const string& zoneName);
+
+  /*!
+   * \brief Write the boundary sections, BCs and families of the markers set with SetBoundaryMarkers.
+   */
+  void WriteBoundaries();
+
+  /*!
+   * \brief Write i-th coordinate to file in CGNS file format. Coordinates are always written in double precision,
+   *        the fields follow the precision requested by the user.
    * \param[in] iField - the output field ID.
    * \param[in] FieldName - Field name in the CGNS.
    */
   void WriteField(int iField, const string& FieldName);
 
+  /*!
+   * \brief Write i-th coordinate or field to file with the given data type.
+   * \param[in] iField - the output field ID.
+   * \param[in] FieldName - Field name in the CGNS.
+   * \param[in] dataType - CGNS data type matching T, i.e. RealSingle for float and RealDouble for double.
+   */
+  template <class T>
+  void WriteFieldOfType(int iField, const string& FieldName, DataType_t dataType);
+
   /*!
    * \brief Write connectivity to file for GEO_TYPE in CGNS file format.
    * \param[in] type - GEO_TYPE.
@@ -109,6 +173,27 @@ class CCGNSFileWriter final : public CFileWriter {
    */
   void InitializeFields();
 
+  /*!
+   * \brief Create an element section, collectively when built with MPI.
+   * \param[in] name - Name of the section.
+   * \param[in] type - CGNS element type of the section.
+   * \param[in] start - First element of the section.
+   * \param[in] end - Last element of the section.
+   * \param[out] section - Index of the section.
+   * \returns CGNS error code.
+   */
+  int SectionWrite(const string& name, ElementType_t type, cgsize_t start, cgsize_t end, int* section);
+
+  /*!
+   * \brief Write the connectivity of the elements [start, end] of a section, the elements of this rank.
+   * \param[in] section - Index of the section.
+   * \param[in] start - First element written by this rank.
+   * \param[in] end - Last element written by this rank.
+   * \param[in] elements - Connectivity, nullptr if this rank writes no element of the section.
+   * \returns CGNS error code.
+   */
+  int ElementsWriteData(int section, cgsize_t start, cgsize_t end, const cgsize_t* elements);
+
   /*!
    * \brief Call a generic CGNS function.
    * \param[in] ier - error value.
@@ -117,6 +202,12 @@ class CCGNSFileWriter final : public CFileWriter {
     if (ier) cg_error_exit();
   }
 
+  /*!
+   * \brief Return the CGNS boundary condition type of an SU2 boundary condition kind.
+   * \param[in] kindBC - SU2 boundary condition kind.
+   */
+  static BCType_t GetCGNSBCType(unsigned short kindBC);
+
   /*!
    * \brief Return the CGNS element type (ElementType_t).
    * \param[in] elementType - GEO_TYPE.

SU2_CFD/include/output/filewriter/CFileWriter.hpp

@@ -168,12 +168,19 @@ class CFileWriter{
    */
   bool WriteMPIString(const std::string& str, unsigned short processor);
 
+  /*!
+   * \brief Write a string of each rank to the file, one after the other in rank order.
+   * \param[in] str - The string of this rank.
+   * \return <code>TRUE</code> if the writing was successful.
+   */
+  bool WriteMPIStringAll(const std::string& str);
+
   /*!
    * \brief Open a file to write using MPI I/O. Already existing file is deleted.
    * \param[in] val_filename - The name of the file
    * \return Boolean indicating whether the opening was successful.
    */
-  bool OpenMPIFile(string val_filename);
+  bool OpenMPIFile(string val_filename, bool append = false);
 
   /*!
    * \brief Close a file using MPI I/O.

SU2_CFD/include/output/filewriter/CParallelDataSorter.hpp

@@ -99,8 +99,8 @@ class CParallelDataSorter{
   int *nPoint_Recv;                    //!< Number of points this processor receives from other processors
   int *nElem_Send;                     //!< Number of elements this processor has to send to other processors
   int *nElem_Cum;                      //!< Cumulative number of elements
-  int *nElemConn_Send;                 //!< Number of element connectivity this processor has to send to other processors
-  int *nElemConn_Cum;                  //!< Cumulative number of element connectivity entries
+  unsigned long *nElemConn_Send;       //!< Number of element connectivity this processor has to send to other processors
+  unsigned long *nElemConn_Cum;        //!< Cumulative number of element connectivity entries
   unsigned long *Index;                //!< Index each point has in the send buffer
   passivedouble *connSend;             //!< Send buffer holding the data that will be send to other processors
   passivedouble *dataBuffer;           //!< Buffer holding the sorted, partitioned data as passivedouble types

SU2_CFD/include/output/filewriter/CParaviewVTMFileWriter.hpp

@@ -124,7 +124,8 @@ class CParaviewVTMFileWriter final: public CFileWriter{
    * \param[in] dataSorter - Datasorter object containing the actual data. Note, data must be sorted.
    */
   //void AddDataset(string name, string file, CParallelDataSorter* dataSorter);
-  void AddDataset(const string& foldername, string name, const string& file, CParallelDataSorter* dataSorter);
+  void AddDataset(const string& foldername, string name, const string& file, CParallelDataSorter* dataSorter,
+                  bool doublePrecision);
 
   /*!
    * \brief Start a new block

SU2_CFD/include/output/filewriter/CParaviewXMLFileWriter.hpp

@@ -38,7 +38,9 @@ class CParaviewXMLFileWriter final: public CFileWriter{
    */
   enum class VTKDatatype {
     FLOAT32,
+    FLOAT64,
     INT32,
+    INT64,
     UINT8
   };
 
@@ -47,6 +49,11 @@ class CParaviewXMLFileWriter final: public CFileWriter{
    */
   bool bigEndian;
 
+  /*!
+   * \brief True to write the coordinates and fields in double precision instead of single.
+   */
+  bool doublePrecision = false;
+
   /*!
    * \brief The current data offset that is used to find data in the binary blob at the end of the file
    */
@@ -70,7 +77,7 @@ class CParaviewXMLFileWriter final: public CFileWriter{
    * \brief Construct a file writer using field names and the data sorter.
    * \param[in] valDataSorter - The parallel sorted data to write
    */
-  CParaviewXMLFileWriter(CParallelDataSorter* valDataSorter);
+  CParaviewXMLFileWriter(CParallelDataSorter* valDataSorter, bool valDoublePrecision = false);
 
   /*!
    * \brief Destructor
@@ -112,13 +119,21 @@ class CParaviewXMLFileWriter final: public CFileWriter{
    */
   inline void GetTypeInfo(const VTKDatatype type, string &typeStr, unsigned long &typeSize) const {
     switch (type) {
+      case VTKDatatype::FLOAT64:
+        typeStr = "\"Float64\"";
+        typeSize = sizeof(double);
+        break;
       case VTKDatatype::FLOAT32:
         typeStr = "\"Float32\"";
         typeSize = sizeof(float);
         break;
       case VTKDatatype::INT32:
         typeStr = "\"Int32\"";
-        typeSize = sizeof(int);
+        typeSize = sizeof(int32_t);
+        break;
+      case VTKDatatype::INT64:
+        typeStr = "\"Int64\"";
+        typeSize = sizeof(int64_t);
         break;
       case VTKDatatype::UINT8:
         typeStr = "\"UInt8\"";

SU2_CFD/src/output/COutput.cpp

@@ -556,7 +556,7 @@ void COutput::WriteToFile(CConfig *config, CGeometry *geometry, OUTPUT_TYPE form
       volumeDataSorter->SortConnectivity(config, geometry, true);
 
       LogOutputFiles("Paraview");
-      fileWriter = new CParaviewXMLFileWriter(volumeDataSorter);
+      fileWriter = new CParaviewXMLFileWriter(volumeDataSorter, config->GetWrt_Output_Double_Precision());
 
       break;
 
@@ -683,7 +683,7 @@ void COutput::WriteToFile(CConfig *config, CGeometry *geometry, OUTPUT_TYPE form
       surfaceDataSorter->SortOutputData();
 
       LogOutputFiles("Paraview surface");
-      fileWriter = new CParaviewXMLFileWriter(surfaceDataSorter);
+      fileWriter = new CParaviewXMLFileWriter(surfaceDataSorter, config->GetWrt_Output_Double_Precision());
 
       break;
 
@@ -760,7 +760,11 @@ void COutput::WriteToFile(CConfig *config, CGeometry *geometry, OUTPUT_TYPE form
       volumeDataSorter->SortConnectivity(config, geometry, true);
 
       LogOutputFiles("CGNS");
-      fileWriter = new CCGNSFileWriter(volumeDataSorter);
+      fileWriter = new CCGNSFileWriter(volumeDataSorter, false, config->GetWrt_Output_Double_Precision());
+
+      /*--- Add the boundaries, named as the markers (the finite volume sorter knows which elements are halos). ---*/
+      if (const auto* fvmSorter = dynamic_cast<const CFVMDataSorter*>(volumeDataSorter))
+        static_cast<CCGNSFileWriter*>(fileWriter)->SetBoundaryMarkers(config, geometry, fvmSorter);
 
       break;
 
@@ -779,7 +783,10 @@ void COutput::WriteToFile(CConfig *config, CGeometry *geometry, OUTPUT_TYPE form
       surfaceDataSorter->SortOutputData();
 
       LogOutputFiles("CGNS surface");
-      fileWriter = new CCGNSFileWriter(surfaceDataSorter, true);
+      fileWriter = new CCGNSFileWriter(surfaceDataSorter, true, config->GetWrt_Output_Double_Precision());
+
+      /*--- One zone per plotted marker, named as the marker. ---*/
+      static_cast<CCGNSFileWriter*>(fileWriter)->SetSurfaceMarkers(config, geometry);
 
       break;