ElmerFoamFSI  2.0
ElmerFoamFSI is fluid-solid interaction simulation application built up from OpenFOAM CFD and Elmer CSM coupled through the IMPACT multiphysics software integration infrastructure.
 All Classes Namespaces Files Functions Variables Typedefs Enumerator Macros Groups Pages
int COM::parallelProgram ( int  argc,
char **  argv 
)

Register the Data Items

write initial mesh data out to a VTK file

Get the handle for the run function

Get the handle for the step function

write mesh data out to a VTK file at each timestep

Definition at line 47 of file OFModuleDriverPar.C.

References Usage().

47  {
48 
49  // instantiating IRAD's communicatorObject
50  // this constructor calls MPI_init internally and
51  // sets up everthing with MPI
52  std::cout <<"OFModuleDriver:Main: Setting up parallel communicator..." << std::endl;
53  COM::CommType communicator(&argc,&argv);
54  MPI_Comm masterComm = communicator.GetCommunicator();
55  int rank = communicator.Rank();
56  int nproc = communicator.Size();
57  std::cout << "OFModuleDriver:Main:Rank #"
58  << rank
59  << ", I see "
60  << nproc
61  << " proccesses."
62  << std::endl;
63 
64  //int main(int argc, char *argv[])
65  COM_init( &argc, &argv);
66 
67  // split the communicator to a single process and the rest
68  // a single process for driver jobs and the rest for
69  // computations
70  COM::CommType newCommunicator;
71  int color;
72  if(rank == -1) color=0;
73  else color=1;
74  /* Not splitting for now
75  communicator.Split(color, rank, newCommunicator);
76  MPI_Comm newComm = newCommunicator.GetCommunicator();
77  std::cout << "OFModuleDriver:Main:Rank #" << rank
78  << ", Default communicator is split to -> "
79  << newComm
80  << std::endl;
81  */
82  MPI_Comm newComm = communicator.GetCommunicator();
83 
84  // flags for operation modes
85  // default mode of operation, runs the original OpenFOAM functionality
86  bool regression = true;
87  // prescribe a solid surface displacement and run the fluid solver only
88  bool prescribedDisplacement = false;
89  // run in parallel mode
90  bool runParallel = false;
91  // run test cycle
92  bool dryRun = false;
93 
94 
95  // we read any driver specific flags passed
96  // the strings are then removed from argv and argc is decremented
97  // to preserve the input command line for OpenFoam
98  std::string arg;
99  std::stringstream ss;
100  if (argc > 1) {
101  for (int i=1; i<argc; ++i) {
102  ss.clear();
103  ss.str("");
104  ss << argv[i];
105  if (ss.str() == "--driverRegression") {
106  regression = true;
107  } else if (ss.str() == "--driverPrescribedDisplacement") {
108  regression = false;
109  prescribedDisplacement = true;
110  } else if (ss.str() == "--parallel") {
111  runParallel = true;
112  } else if (ss.str() == "--dryrun") {
113  dryRun = true;
114  } else {
115  Usage(argv[0]);
116  exit(2);
117  }
118  }
119  } else {
120  Usage(argv[0]);
121  exit(2);
122  }
123 
124  // changing default communicator and testing it
125  COM_set_default_communicator(newComm);
126  std::cout << "OFModuleDriver:Main:Rank #" << rank
127  << ", Default communicator for COM is now "
128  << newComm
129  << std::endl;
130  MPI_Comm test_comm;
131  test_comm = COM_get_default_communicator();
132  if (test_comm != newComm)
133  std::cout << "OFModuleDriver:Main:Rank #"
134  << rank
135  << ", default communicator is not set to "
136  << newComm
137  << " properly."
138  << std::endl;
139 
140  // loading parallel openfoam module
141  // all processes load fluid module
142  COM_LOAD_MODULE_STATIC_DYNAMIC( OpenFoamFSIPar, "OFModule");
143 
144  // loading SimIn on all processes
145  COM_LOAD_MODULE_STATIC_DYNAMIC( SimOUT, "OUT");
146 
147  // getting number of processes
148  if (rank==0) {
149  int* nProcReg;
150  COM_get_array("OFModule.nproc", 0, &nProcReg);
151  std::cout << "The communicator registered in OFModule uses "
152  << *nProcReg << " prcesses" << std::endl;
153  }
154 
155  // initializing the openfoam using fluid processes
156  //if (color == 1) {
157  // getting the rank
158  //int fluidProcessRank = newCommunicator.Rank();
159 
160  // get the handle for the initialize function and call it
161  int init_handle = COM_get_function_handle("OFModule.InitFoam");
162  if(init_handle <= 0) { // fail
163  std::cout << "OFModuleDriver:Main:Rank #" << rank
164  << ", could not get handle for init function." << std::endl;
165  exit(-1);
166  }
167 
168  // making a dummy argc/argv for OpenFOAM.
169  // For now, no options passed from the command
170  // line will be used by the driver
171  int myArgc = 1;
172  char *myArgv[2];
173  // passing parallel triggers openFoam to run in parallel mode
174  myArgv[0] = argv[0];
175  if (runParallel) {
176  myArgc = 2;
177  myArgv[1] = const_cast<char *>("-parallel");
178  } else {
179  myArgv[1] = NULL;
180  }
181  // setting verbose level
182  int verb=3;
183  // call the funciton
184  COM_call_function(init_handle, &myArgc, &myArgv, &verb);
185  // make a barrier
186  //newCommunicator.Barrier();
187 
188  // getting information about what was registered in this window by another process
189  if (rank==0){
190  int numDataItems=0;
191  std::string output;
192  COM_get_dataitems("OFModule", &numDataItems, output);
193  std::istringstream Istr(output);
194  std::vector<std::string> dataItemNames;
195 
196  for (int i=0; i<numDataItems; ++i) {
197  std::string name;
198  Istr >> name;
199  dataItemNames.push_back(name);
200  std::cout << "Rank #" << rank
201  << ", OFModuleDriver:main: DataItem # " << i << ": " << name << std::endl;
202  }
203  }
204 
205  // how to a pane in this window registered by another proccess
206  if (rank==0){
207  int nPane;
208  int* paneList;
209  int paneRank = 1;
210  COM_get_panes("OFModule", &nPane, &paneList, paneRank);
211  std::cout << "Rank #" << rank << ", I see "<<nPane<<" pane in the process rank #"
212  << paneRank << std::endl;
213  std::cout << "Here is the list:" << std::endl;
214  for (int i=0; i<nPane; ++i)
215  std::cout <<"Pane ID # " << i+1 << "=" << paneList[i] << std::endl;
216  }
217 
218  // List of panes for this process: each process creates a pane starting from 100
219  int numPanes;
220  int* paneList;
221  COM_get_panes("OFModule", &numPanes, &paneList);
222  std::cout << "Rank #" << rank
223  << ", OFModuleDriver:main: Number of Panes " << numPanes << std::endl;
224  for (int i=0; i<numPanes; ++i)
225  std::cout << "Rank #" << rank
226  << ", OFModuleDriver:main: Pane ID # " << i+1 << "=" << paneList[i] << std::endl;
227  // only one pane per process currently
228  int pane = paneList[0];
229 
230  // getting node coordinates
231  double* Coord;
232  COM_get_array("OFModule.nc", pane, &Coord);
233  // check for expected number of nodes
234  int numNodes = 0;
235  COM_get_size("OFModule.nc", pane, &numNodes);
236  // get connectivity tables for panes
237  // :q4: is quad element
238  int numConn;
239  std::string stringNames;
240  COM_get_connectivities("OFModule", pane, &numConn, stringNames);
241  std::istringstream ConnISS(stringNames);
242  std::vector<std::string> connNames;
243  for (int i=0; i<numConn; ++i) {
244  std::string name;
245  ConnISS >> name;
246  connNames.push_back(name);
247  std::cout << "Rank #" << rank
248  << ", OFModuleDriver:main: Connectivity Table # " << i+1 << ": " << name << std::endl;
249  }
250  // number of nodes per element
251  char getDataItemLoc;
252  COM_Type getDataItemType;
253  int numElementNodes;
254  std::string getDataItemUnits;
255  std::string fullConnName("OFModule."+connNames[0]);
256  COM_get_dataitem(fullConnName, &getDataItemLoc, &getDataItemType,
257  &numElementNodes, &getDataItemUnits);
258  std::cout << "Rank #" << rank
259  << ", OFModuleDriver:main: getDataItemLoc " << getDataItemLoc << std::endl;
260  std::cout << "Rank #" << rank
261  << ", OFModuleDriver:main: getDataItemType " << getDataItemType << std::endl;
262  std::cout << "Rank #" << rank
263  << ", OFModuleDriver:main: numElementNodes " << numElementNodes << std::endl;
264  std::cout << "Rank #" << rank
265  << ", OFModuleDriver:main: getDataItemUnits " << getDataItemUnits << std::endl;
266 
267  // get connectivities
268  int* Conn;
269  int numElem;
270  COM_get_array(fullConnName.c_str(), pane, &Conn);
271  COM_get_size(fullConnName, pane, &numElem);
272 
273  // put elements into a vector so we can build the solver agent
274  std::vector<unsigned int> connVector;
275  for (int i=0; i<numElem; ++i) {
276  for (int j=0; j<numElementNodes; ++j) {
277  connVector.push_back((Conn[i*numElementNodes+j]));
278  }
279  }
280 
281  // get non-mesh data items
282  int arrayLength;
283  std::string name("OFModule.time");
284  COM_get_dataitem(name, &getDataItemLoc, &getDataItemType,
285  &arrayLength, &getDataItemUnits);
286  double* time;
287  COM_get_array(name.c_str(), pane, &time);
288  name = "OFModule.endTime";
289  COM_get_dataitem(name, &getDataItemLoc, &getDataItemType,
290  &arrayLength, &getDataItemUnits);
291  double* endTime;
292  COM_get_array(name.c_str(), pane, &endTime);
293 
294 
295  // make a solverAgent to store our data
296  SolverUtils::FEM::SolverAgent myAgent;
297 
299  // set up solution meta data
300  myAgent.Solution().Meta().AddField("time", 's', 1, 8, "s");
301  myAgent.Solution().Meta().AddField("endTime", 's', 1, 8, "s");
302  //myAgent.Solution().Meta().AddField("initStatus", 's', 1, 4, "");
303  //myAgent.Solution().Meta().AddField("runStatus", 's', 1, 4, "");
304  //myAgent.Solution().Meta().AddField("dt", 's', 1, 8, "s");
305  //myAgent.Solution().Meta().AddField("displacement", 'n', 3, 8, "m");
306  //myAgent.Solution().Meta().AddField("traction", 'c', 1, 8, "N");
307  myAgent.Mesh().nc.init(numNodes, Coord);
308  // this assumes that our elements are quads...we can generalize this
309  myAgent.Mesh().con.AddElements(numElem, numElementNodes, connVector);
310 
311  // get pressures
312  name = "OFModule.pressure";
313  COM_get_dataitem(name, &getDataItemLoc, &getDataItemType,
314  &arrayLength, &getDataItemUnits);
315  std::cout << "Rank #" << rank << " ,OFModuleDriver:main: Pressure Get DataItem" << std::endl;
316  std::cout << "Rank #" << rank << " ,OFModuleDriver:main: getDataItemLoc: " << getDataItemLoc << std::endl;
317  std::cout << "Rank #" << rank << " ,OFModuleDriver:main: getDataItemType: " << getDataItemType << std::endl;
318  std::cout << "Rank #" << rank << " ,OFModuleDriver:main: arrayLength: " << arrayLength << std::endl;
319  std::cout << "Rank #" << rank << " ,OFModuleDriver:main: getDataItemUnits: " << getDataItemUnits << std::endl;
320  char myDataItemLoc;
321  // translate element (e) to cell (c)
322  if (getDataItemLoc == 'e' || getDataItemLoc == 'E') {
323  myDataItemLoc = 'c';
324  } else {
325  std::cout << "OFModuleDriver:main: Unknown Data Item Location" << std::endl;
326  exit(1);
327  }
328  int myDataItemType;
329  if (getDataItemType == COM_DOUBLE) {
330  myDataItemType = 8;
331  } else {
332  std::cout << "OFModuleDriver:main: Unknown Data Item Type" << std::endl;
333  exit(1);
334  }
335  myAgent.Solution().Meta().AddField("surfacePressure", myDataItemLoc, arrayLength,
336  myDataItemType, getDataItemUnits);
337  double* surfacePressure;
338  COM_get_array(name.c_str(), pane, &surfacePressure);
339 
340  // get tractions
341  name = "OFModule.traction";
342  COM_get_dataitem(name, &getDataItemLoc, &getDataItemType,
343  &arrayLength, &getDataItemUnits);
344  std::cout << "Rank #" << rank << " ,OFModuleDriver:main: Traction Get DataItem" << std::endl;
345  std::cout << "Rank #" << rank << " ,OFModuleDriver:main: getDataItemLoc: " << getDataItemLoc << std::endl;
346  std::cout << "Rank #" << rank << " ,OFModuleDriver:main: getDataItemType: " << getDataItemType << std::endl;
347  std::cout << "Rank #" << rank << " ,OFModuleDriver:main: arrayLength: " << arrayLength << std::endl;
348  std::cout << "Rank #" << rank << " ,OFModuleDriver:main: getDataItemUnits: " << getDataItemUnits << std::endl;
349  // translate element (e) to cell (c)
350  if (getDataItemLoc == 'e' || getDataItemLoc == 'E') {
351  myDataItemLoc = 'c';
352  } else {
353  std::cout << "OFModuleDriver:main: Unknown Data Item Location" << std::endl;
354  exit(1);
355  }
356  if (getDataItemType == COM_DOUBLE) {
357  myDataItemType = 8;
358  } else {
359  std::cout << "OFModuleDriver:main: Unknown Data Item Type" << std::endl;
360  exit(1);
361  }
362  myAgent.Solution().Meta().AddField("surfaceTraction", myDataItemLoc, arrayLength,
363  myDataItemType, getDataItemUnits);
364  double* surfaceTraction;
365  COM_get_array(name.c_str(), pane, &surfaceTraction);
366 
367  // get solid displacements
368  name = "OFModule.solidDisplacement";
369  COM_get_dataitem(name, &getDataItemLoc, &getDataItemType,
370  &arrayLength, &getDataItemUnits);
371  std::cout << "Rank #" << rank << " ,OFModuleDriver:main: Solid Displacement Get DataItem" << std::endl;
372  std::cout << "Rank #" << rank << " ,OFModuleDriver:main: getDataItemLoc: " << getDataItemLoc << std::endl;
373  std::cout << "Rank #" << rank << " ,OFModuleDriver:main: getDataItemType: " << getDataItemType << std::endl;
374  std::cout << "Rank #" << rank << " ,OFModuleDriver:main: arrayLength: " << arrayLength << std::endl;
375  std::cout << "Rank #" << rank << " ,OFModuleDriver:main: getDataItemUnits: " << getDataItemUnits << std::endl;
376  // translate element (e) to cell (c)
377  if (getDataItemLoc == 'e' || getDataItemLoc == 'E') {
378  myDataItemLoc = 'c';
379  } else if (getDataItemLoc == 'n' || getDataItemLoc == 'N') {
380  myDataItemLoc = 'n';
381  } else {
382  std::cout << "OFModuleDriver:main: Unknown Data Item Location" << std::endl;
383  exit(1);
384  }
385  if (getDataItemType == COM_DOUBLE) {
386  myDataItemType = 8;
387  } else {
388  std::cout << "OFModuleDriver:main: Unknown Data Item Type" << std::endl;
389  exit(1);
390  }
391  myAgent.Solution().Meta().AddField("solidDisplacement", myDataItemLoc, arrayLength,
392  myDataItemType, getDataItemUnits);
393  double* solidDisplacement;
394  COM_get_array("OFModule.solidDisplacement", pane, &solidDisplacement);
395 
396  // create buffers for the actual data
397  myAgent.CreateSoln();
398  unsigned int nnodes = myAgent.Mesh().nc.NNodes();
399  unsigned int nelem = myAgent.Mesh().con.Nelem();
400  // reset the buffers to be our own local buffers
401  myAgent.Solution().SetFieldBuffer("time", time);
402  myAgent.Solution().SetFieldBuffer("endTime", endTime);
403  myAgent.Solution().SetFieldBuffer("surfacePressure", surfacePressure);
404  myAgent.Solution().SetFieldBuffer("surfaceTraction", surfaceTraction);
405  myAgent.Solution().SetFieldBuffer("solidDisplacement", solidDisplacement);
406 
408  std::ofstream Ouf;
409  ss.clear();
410  ss.str("");
411  ss << time[0]
412  << "_proc_"
413  << rank;
414  std::string filename;
415  filename = "fsi_" + ss.str() +".vtk";
416  Ouf.open(filename.c_str());
417  if(!Ouf){
418  std::cerr << "OFModuleDriver:main: OFModuleDriver::DumpSolution:Error: Could not open output file, "
419  << filename << "." << std::endl;
420  return -1;
421  }
422  std::cout << "Rank #" << rank << ", OFModuleDriver:main: WriteVTKToStream time " << time[0] << std::endl;
423  SolverUtils::WriteVTKToStream("OFModule", myAgent, Ouf);
424  Ouf.close();
425 
426  /*
427  // gathering total number of elements in root
428  int numElemTot = 0;
429  communicator.Reduce(numElem, numElemTot, IRAD::Comm::DTINT, IRAD::Comm::SUMOP, 0);
430  std::cout << "Rank #"<<rank<<", number of elements = "<<numElem<<std::endl;
431  std::cout << "Rank #"<<rank<<", total number of elements = "<<numElemTot<<std::endl;
432  */
433 
434  // setting a barrier here
435  std::cout << "Rank #"<<rank<<"...Reaching to barrier"<<std::endl;
436  communicator.Barrier();
437 
438  // gathering all pressures in root
439  std::vector<double> surfPresVec;
440  std::vector<double> surfPresTotVec;
441  std::vector<int> nSendAllVec;
442  int gatherErr;
443  for (int iElem=0; iElem<numElem; ++iElem)
444  surfPresVec.push_back(surfacePressure[iElem]);
445  std::cout << "Rank #"<<rank<<", surfPresVec size = "<<surfPresVec.size()<<std::endl;
446  gatherErr = communicator.Gatherv(surfPresVec, surfPresTotVec, nSendAllVec);
447  if (gatherErr)
448  std::cout << "Error in gathering information from processes." << std::endl;
449  std::cout << "Rank #" << rank
450  << ", size of surfPresTotVec = " << surfPresTotVec.size()
451  << std::endl;
452 
453  // setting a barrier here
454  std::cout << "Rank #"<<rank<<"...Reaching to barrier"<<std::endl;
455  communicator.Barrier();
456 
457  // scattering surface displacements
458  // first gathering number of nodes for each process
459  std::vector<int> numNodesVec;
460  std::vector<int> numNodesTotVec;
461  std::vector<int> nNodesAllVec;
462  numNodesVec.push_back(numNodes);
463  //std::cout << "Rank #"<<rank<<", numNodesVec[rank] = "<<numNodesVec[0]<<std::endl;
464  gatherErr = communicator.Gatherv(numNodesVec, numNodesTotVec, nNodesAllVec);
465  if (gatherErr)
466  std::cout << "Error in gathering information from processes." << std::endl;
467  if (rank == 0){
468  for (int iProc=0; iProc<numNodesTotVec.size(); iProc++)
469  std::cout << "Rank #"<<rank
470  <<", numNodesTotVec["<<iProc<<"] = "<<numNodesTotVec[iProc]<<std::endl;
471  }
472  std::vector<double> surfDispVec;
473  std::vector<double> surfDispVecTot;
474  std::vector<int> nRecvAllVec;
475  int scatterErr;
476  int numNodesTot = 0;
477  if (rank == 0){
478  for (int iProc=0; iProc<numNodesTotVec.size(); iProc++){
479  numNodesTot+= numNodesTotVec[iProc];
480  nRecvAllVec.push_back(3*numNodesTotVec[iProc]);
481  }
482  surfDispVecTot.resize(3*numNodesTot, -0.1);
483  std::cout<<"Rank #"<<rank<<", numNodesTot = "<<numNodesTot<<std::endl;
484  std::cout<<"Rank #"<<rank<<", surfDispVecTot = "<<surfDispVecTot.size()<<std::endl;
485  }
486  scatterErr = communicator.Scatterv(surfDispVecTot, nRecvAllVec, surfDispVec);
487  if (scatterErr)
488  std::cout << "Error in scattering information to processes." << std::endl;
489  std::cout << "Rank #" << rank
490  << ", size surfDispVec = " << surfDispVec.size()
491  << ", first element of which = " << surfDispVec[0]
492  << std::endl;
493 
494 
495  if (!dryRun) {
497  int run_handle = COM_get_function_handle("OFModule.RunFoam");
498  if(run_handle <= 0) { // fail
499  std::cout << "OFModuleDriver:main: Could not get handle for run function." << std::endl;
500  exit(-1);
501  }
502 
503  int step_handle;
505  if (regression) {
506  step_handle = COM_get_function_handle("OFModule.StepFoam");
507  std::cout << "OFModuleDriver:main: StepFoam will be running shortly." << std::endl;
508  if(step_handle <= 0) { // fail
509  std::cout << "OFModuleDriver:main: Could not get handle for step function." << std::endl;
510  exit(-1);
511  }
512  } else if (prescribedDisplacement) {
513  step_handle = COM_get_function_handle("OFModule.StepFluid");
514  std::cout << "OFModuleDriver:main: StepFluid will be running shortly." << std::endl;
515  if(step_handle <= 0) { // fail
516  std::cout << "OFModuleDriver:main: Could not get handle for step fluid alone function." << std::endl;
517  exit(-1);
518  }
519  }
520 
521 
522  std::cout << "Rank #" << rank
523  << ", OFModuleDriver:main: In Driver:" << " time=" << time[0]
524  << " endTime=" << endTime[0] << std::endl;
525 
526  while(time[0] <= endTime[0]) {
527  //COM_call_function(run_handle);
528  // step solution
529  if (prescribedDisplacement) {
530  double xmax = .6;
531  double xmin = .26;
532  double alpha = 3.141592/(2.0*(xmax-xmin));
533  double beta = 2*3.141592/endTime[0];
534  for(int i = 0;i < numNodes;i++){
535  double xpos = Coord[i*3] - xmin;
536  if(xpos > 0){
537  xpos *= alpha;
538  solidDisplacement[i*3] = solidDisplacement[i*3+2] = 0.0;
539  solidDisplacement[i*3+1] = 0.001*std::sin(xpos)*std::sin(beta*time[0]);
540  }
541  }
542  }
543  COM_call_function(step_handle);
545  std::ofstream Ouf;
546  ss.clear();
547  ss.str("");
548  ss << time[0]
549  << "_proc_"
550  << rank;
551  std::string filename;
552  filename = "fsi_" + ss.str() +".vtk";
553  Ouf.open(filename.c_str());
554  if(!Ouf){
555  std::cerr << "OFModuleDriver::DumpSolution:Error: Could not open output file, "
556  << filename << "." << std::endl;
557  return -1;
558  }
559  std::cout << "Rank #" << rank
560  << ", OFModuleDriver:main: WriteVTKToStream time "
561  << time[0] << std::endl;
562  SolverUtils::WriteVTKToStream("OFModule", myAgent, Ouf);
563  Ouf.close();
564  }
565  }
566  //}
567  // Setting COM to default COMM
568  //COM_set_default_communicator(masterComm);
569 
570  // set a barrier
571  communicator.Barrier();
572 
573  /*
574  // Creating a Pane_communicator object
575  COM::COM_base *rbase = COM_get_com();
576  COM::Window* winObj = NULL;
577  int winHndl = COM_get_window_handle("OFModule");
578  winObj = rbase->get_window_object(winHndl);
579  MAP::Pane_communicator *pComm = new MAP::Pane_communicator(winObj, newComm);
580  std::cout << "Total number of panes = " << pComm->total_npanes() << std::endl;
581  */
582 
583 
584  // writing fluid window to hdf
585  int OUT_set = COM_get_function_handle( "OUT.set_option");
586  int OUT_write = COM_get_function_handle( "OUT.write_dataitem");
587  int OUT_write_ctrl = COM_get_function_handle( "OUT.write_rocin_control_file");
588  const char *win_out= "OFModule";
589  std::string win_out_pre( win_out);
590  win_out_pre.append(".");
591  int OUT_all = COM_get_dataitem_handle((win_out_pre+"all").c_str());
592  ss.clear();
593  ss.str("");
594  ss << "_proc_"
595  << rank;
596  std::string fileOut;
597  std::string fname, ctrl_fname;
598  fileOut = "OFModule_window" + ss.str();
599  fname = (std::string)win_out + "_0_";
600  ctrl_fname = (std::string)win_out + "_in_0.txt";
601  COM_call_function( OUT_set, "format", "HDF4");
602  COM_call_function( OUT_write, (fileOut + ".hdf").c_str(), &OUT_all, win_out,
603  "0000");
604  COM_call_function( OUT_write_ctrl, win_out, fileOut.c_str(), ctrl_fname.c_str());
605 
606 
607 
608  // Finalizing
609  // unloading SimIn on all processes
610  COM_UNLOAD_MODULE_STATIC_DYNAMIC( SimOUT, "OUT");
611  // unloading openFoam module
612  COM_UNLOAD_MODULE_STATIC_DYNAMIC(OpenFoamFSIPar, "OFModule");
613  // finalize comm
614  COM_finalize();
615  // finalizing MPI
616  communicator.Finalize();
617  return(0);
618  }
IRAD::Comm::CommunicatorObject CommType
void Usage(char *exec)

Here is the call graph for this function: