int COM::parallelProgram	(	int	argc,
		char *	argv[]
	)

Definition at line 400 of file SolverModuleDriverParallel.C.

References SolverModuleDriver::Disp, SolverModuleDriver::finalize(), SolverModuleDriver::init(), SolverModuleDriver::isChangeLoad(), SolverModuleDriver::isFSISim(), SolverModuleDriver::isStreamer(), SolverModuleDriver::Loads, SolverModuleDriver::myPaneId, SolverModuleDriver::nDisp, SolverModuleDriver::nLoads, SolverModuleDriver::nNodes, SolverModuleDriver::outfile, SolverModuleDriver::run(), SolverModuleDriver::setColor(), SolverModuleDriver::setComm(), SolverModuleDriver::setRank(), and SolverModuleDriver::updateSolution().

Referenced by main().

                                               {
         int parErr;
         // instantiating IRAD's communicatorObject
         // this constructor calls MPI_init internally and
         // sets up everthing with MPI
         std::cout <<"ElmerModuleDriver:Main: Setting up parallel communicator..." << std::endl;
         CommType communicator(&argc,&argv);
         MPI_Comm masterComm = communicator.GetCommunicator();
         int rank = communicator.Rank();
         int nproc = communicator.Size();
         int color = 0;
         std::cout << "ElmerModuleDriver:Main:Rank #"
                   << rank
                   << ", I see "
                   << nproc
                   << " proccesses."
                   << std::endl;
         // initializing COM 
         COM_init( &argc, &argv);

         // instantiating solver driver object
         SolverModuleDriver driverObject;
         driverObject.setRank(rank);
         driverObject.myPaneId=100+rank;

         // checking streamer process
         if (driverObject.isStreamer())
            std::cout << "Hi, I am a streamer ..." << std::endl;

         // setting up output files for parallel debugging
         std::string outfile_name;
         std::stringstream ss; 
         std::string str_rank;
         ss << rank;
         ss >> str_rank;
         outfile_name = "out" + str_rank + ".dat";
         driverObject.outfile.open(outfile_name.c_str());
         
         // setting color for the driver object
         driverObject.setColor(color);
         driverObject.outfile << "color = " << color << std::endl;
         driverObject.outfile << "rank = " << rank << std::endl;
         CommType newCommunicator;
         communicator.Split(color, rank, newCommunicator);
         driverObject.outfile << "communicator.Size() = " << communicator.Size() << std::endl;
         driverObject.outfile << "newCommunicator.Size() = " << newCommunicator.Size() << std::endl;
         driverObject.setComm(newCommunicator.GetCommunicator());
         MPI_Comm newComm;
         newComm = newCommunicator.GetCommunicator();
         COM_set_default_communicator(newComm);

         // calling initializer, for FSI problems mesh and solution
         // related datastructures will be automatically registered
         driverObject.init(argc, argv);

         // get information about what was registered in this window
         int numDataItems=0;
         std::string output;
         COM_get_dataitems("ELMModule", &numDataItems, output);
         std::istringstream Istr(output);
         std::vector<std::string> dataItemNames;  
         for (int i=0; i<numDataItems; ++i) {
           std::string name;
           Istr >> name;
           dataItemNames.push_back(name);
           std::cout << "Rank #" << rank
                     << ", ElmerModuleDriver:main: DataItem # " << i << ": " << name << std::endl;
         }
         // list of panes for this process: each process creates a single pane
         // with paneId = 100 + rank
         int numPanes;
         int* paneList;
         COM_get_panes("ELMModule", &numPanes, &paneList);
         std::cout << "Rank #" << rank
                   << ", ELMModuleDriver:main: Number of Panes " << numPanes << std::endl;
         for (int i=0; i<numPanes; ++i)
           std::cout << "Rank #" << rank
                     << ", ELMModuleDriver:main: Pane ID # " << i+1 << "=" << paneList[i] << std::endl;
         
         // updating solution
         driverObject.updateSolution();

         // dumping intial conditions to vtk
         //if (driverObject.isFSISim())
         //   driverObject.vtkDump(0); 
         
         // scattering FSI loads to the structures solver processes
         // only for FSI problesm
         if(driverObject.isFSISim() && driverObject.isChangeLoad()){
            std::vector<int> nLoadsVec;
            std::vector<int> nLoadsTotVec;
            std::vector<int> nRecvAllVec;
            nLoadsVec.push_back(driverObject.nNodes*driverObject.nLoads);
            std::cout << "Rank #"<<rank
                      <<", nLoadsVec["<<rank<<"] = "<<nLoadsVec[rank]<<std::endl;
            parErr = newCommunicator.Gatherv(nLoadsVec, nLoadsTotVec, nRecvAllVec);
            if (parErr)
               std::cout << "Error in gathering information from processes." << std::endl;
            if (rank == 0){
               for (int iProc=0; iProc<nLoadsTotVec.size(); iProc++)
                  std::cout << "Rank #"<<rank
                            <<", nLoadsTotVec["<<iProc<<"] = "<<nLoadsTotVec[iProc]<<std::endl;
            }
            // defining new loads
            std::vector<double> loadsVec;
            std::vector<double> loadsTotVec;
            int nLoadsTot = 0;
            if (rank == 0){
              for (int iProc=0; iProc < nLoadsTotVec.size(); iProc++){
                 nLoadsTot += nLoadsTotVec[iProc];
              }
              loadsTotVec.resize(nLoadsTot,0.0);
              for (int iLoad=1; iLoad < nLoadsTot; iLoad+=3)
                  loadsTotVec[iLoad] = 1.0;
              std::cout << "Size of loadsTotVec = " << loadsTotVec.size() << std::endl;
            }
            // parallel scattering
            parErr = newCommunicator.Scatterv(loadsTotVec, nLoadsTotVec, loadsVec);
            if (parErr)
               std::cout << "Error in scattering information to processes." << std::endl;
            std::cout << "Rank #" << rank
                      << ", number of load components I received = " << loadsVec.size()
                      << std::endl;
            // applying loads for each process
            for (int iLoad=0; iLoad<(driverObject.nLoads*driverObject.nNodes); ++iLoad){
               driverObject.Loads[iLoad] = loadsVec[iLoad];
               std::cout << "Rank #" << rank
                         << ", driverObject.Loads ["<<iLoad<<"] = " 
                         << driverObject.Loads[iLoad] << std::endl;
            }
            // setting a barrier here
            std::cout << "Rank #"<<rank<<"...Reaching to barrier"<<std::endl;
            newCommunicator.Barrier();
         }

         // calling run step
         driverObject.run();
         newCommunicator.Barrier();
         
         // gathering all displacements in root
         // only for FSI problesm
         if(driverObject.isFSISim() && driverObject.isChangeLoad()){
            std::vector<double> dispVec;
            std::vector<double> dispTotVec;
            std::vector<int> nSendAllVec;
            for (int iDisp=0; iDisp<(driverObject.nDisp*driverObject.nNodes); ++iDisp){
               dispVec.push_back(driverObject.Disp[iDisp]);
               std::cout << "Rank #" << rank
                         << ", dispVec ["<<iDisp<<"] = " << dispVec[iDisp] << std::endl;
            }
            std::cout << "Rank #" << rank
                      << ", dispVec size = " << dispVec.size() << std::endl;
            // parallel gathering
            parErr =  newCommunicator.Gatherv(dispVec, dispTotVec, nSendAllVec);
            if (parErr)
               std::cout << "Error in gathering information from processes." << std::endl;
            std::cout << "Rank #" << rank
                      << ", size of dispTotVec = " << dispTotVec.size()
                      << std::endl;         
            // setting a barrier here
            std::cout << "Rank #"<<rank<<"...Reaching to barrier"<<std::endl;
            newCommunicator.Barrier();
            // checking gather data vector
            if (rank == 0){
               double maxDisp = 0.0;
               double minDisp = 0.0;
               for (int iDisp=0; iDisp<dispTotVec.size(); iDisp++){
                  maxDisp = std::max(dispTotVec[iDisp], maxDisp);
                  minDisp = std::min(dispTotVec[iDisp], minDisp);
                  std::cout << "Rank #0, dispTotVec [" << iDisp << "] = " 
                            << dispTotVec[iDisp] << std::endl;
               }
               outfile_name = "maxMinDisp.dat";
               std::ofstream testFile;
               testFile.open(outfile_name.c_str());
               testFile << maxDisp << std::endl << minDisp << std::endl;
               testFile.close();
            }
            // writing window to HDF
            int OUT_set = COM_get_function_handle( "OUT.set_option");
            int OUT_write = COM_get_function_handle( "OUT.write_dataitem");
            const char *win_out= "ELMModule";
            std::string win_out_pre(win_out);
            win_out_pre.append(".");
            int OUT_all = COM_get_dataitem_handle((win_out_pre+"all").c_str());
            ss.clear();
            ss.str("");
            ss << "_proc_"
               << rank;
            std::string fileOut;
            fileOut = "ELMModule_window" + ss.str();
            COM_call_function( OUT_set, "format", "HDF4");
            COM_call_function( OUT_write, (fileOut + ".hdf").c_str(), &OUT_all, win_out,
                            "0000");
         }
                  
         // calling finalize
         driverObject.finalize();
  
         // closing log file for the driver object
         driverObject.outfile.close();

         // finalize comm
         COM_finalize();

         // finalizing communicator
         communicator.Finalize();
      return 0;
    }

Here is the call graph for this function:

Here is the caller graph for this function: