download the original source code.
  1 /*
  2    Example 5big
  3 
  4    Interface:    Linear-Algebraic (IJ)
  5 
  6    Compile with: make ex5big
  7 
  8    Sample run:   mpirun -np 4 ex5big
  9 
 10    Description:  This example is a slight modification of Example 5 that
 11                  illustrates the 64-bit integer support in hypre needed to run
 12                  problems with more than 2B unknowns.
 13 
 14                  Specifically, the changes compared to Example 5 are as follows:
 15 
 16                  1) All integer arguments to HYPRE functions should be declared
 17                     of type HYPRE_Int.
 18 
 19                  2) Variables of type HYPRE_Int are 64-bit integers, so they
 20                     should be printed in the %lld format (not %d).
 21 
 22                  To enable the 64-bit integer support, you need to build hypre
 23                  with the --enable-bigint option of the configure script.  We
 24                  recommend comparing this example with Example 5.
 25 */
 26 
 27 #include <math.h>
 28 #include "_hypre_utilities.h"
 29 #include "HYPRE_krylov.h"
 30 #include "HYPRE.h"
 31 #include "HYPRE_parcsr_ls.h"
 32 
 33 int hypre_FlexGMRESModifyPCAMGExample(void *precond_data, int iterations,
 34                                       double rel_residual_norm);
 35 
 36 
 37 int main (int argc, char *argv[])
 38 {
 39    HYPRE_Int i;
 40    int myid, num_procs;
 41    int N, n;
 42 
 43    HYPRE_Int ilower, iupper;
 44    HYPRE_Int local_size, extra;
 45 
 46    int solver_id;
 47    int print_system;
 48 
 49    double h, h2;
 50 
 51    HYPRE_IJMatrix A;
 52    HYPRE_ParCSRMatrix parcsr_A;
 53    HYPRE_IJVector b;
 54    HYPRE_ParVector par_b;
 55    HYPRE_IJVector x;
 56    HYPRE_ParVector par_x;
 57 
 58    HYPRE_Solver solver, precond;
 59 
 60    /* Initialize MPI */
 61    MPI_Init(&argc, &argv);
 62    MPI_Comm_rank(MPI_COMM_WORLD, &myid);
 63    MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
 64 
 65    /* Default problem parameters */
 66    n = 33;
 67    solver_id = 0;
 68    print_system = 0;
 69 
 70 
 71    /* Parse command line */
 72    {
 73       int arg_index = 0;
 74       int print_usage = 0;
 75 
 76       while (arg_index < argc)
 77       {
 78          if ( strcmp(argv[arg_index], "-n") == 0 )
 79          {
 80             arg_index++;
 81             n = atoi(argv[arg_index++]);
 82          }
 83          else if ( strcmp(argv[arg_index], "-solver") == 0 )
 84          {
 85             arg_index++;
 86             solver_id = atoi(argv[arg_index++]);
 87          }
 88          else if ( strcmp(argv[arg_index], "-print_system") == 0 )
 89          {
 90             arg_index++;
 91             print_system = 1;
 92          }
 93          else if ( strcmp(argv[arg_index], "-help") == 0 )
 94          {
 95             print_usage = 1;
 96             break;
 97          }
 98          else
 99          {
100             arg_index++;
101          }
102       }
103 
104       if ((print_usage) && (myid == 0))
105       {
106          printf("\n");
107          printf("Usage: %s [<options>]\n", argv[0]);
108          printf("\n");
109          printf("  -n <n>              : problem size in each direction (default: 33)\n");
110          printf("  -solver <ID>        : solver ID\n");
111          printf("                        0  - AMG (default) \n");
112          printf("                        1  - AMG-PCG\n");
113          printf("                        8  - ParaSails-PCG\n");
114          printf("                        50 - PCG\n");
115          printf("                        61 - AMG-FlexGMRES\n");
116          printf("  -print_system       : print the matrix and rhs\n");
117          printf("\n");
118       }
119 
120       if (print_usage)
121       {
122          MPI_Finalize();
123          return (0);
124       }
125    }
126 
127    /* Preliminaries: want at least one processor per row */
128    if (n*n < num_procs) n = sqrt(num_procs) + 1;
129    N = n*n; /* global number of rows */
130    h = 1.0/(n+1); /* mesh size*/
131    h2 = h*h;
132 
133    /* Each processor knows only of its own rows - the range is denoted by ilower
134       and upper.  Here we partition the rows. We account for the fact that
135       N may not divide evenly by the number of processors. */
136    local_size = N/num_procs;
137    extra = N - local_size*num_procs;
138 
139    ilower = local_size*myid;
140    ilower += hypre_min(myid, extra);
141 
142    iupper = local_size*(myid+1);
143    iupper += hypre_min(myid+1, extra);
144    iupper = iupper - 1;
145 
146    /* How many rows do I have? */
147    local_size = iupper - ilower + 1;
148 
149    /* Create the matrix.
150       Note that this is a square matrix, so we indicate the row partition
151       size twice (since number of rows = number of cols) */
152    HYPRE_IJMatrixCreate(MPI_COMM_WORLD, ilower, iupper, ilower, iupper, &A);
153 
154    /* Choose a parallel csr format storage (see the User's Manual) */
155    HYPRE_IJMatrixSetObjectType(A, HYPRE_PARCSR);
156 
157    /* Initialize before setting coefficients */
158    HYPRE_IJMatrixInitialize(A);
159 
160    /* Now go through my local rows and set the matrix entries.
161       Each row has at most 5 entries. For example, if n=3:
162 
163       A = [M -I 0; -I M -I; 0 -I M]
164       M = [4 -1 0; -1 4 -1; 0 -1 4]
165 
166       Note that here we are setting one row at a time, though
167       one could set all the rows together (see the User's Manual).
168    */
169    {
170       HYPRE_Int nnz;
171       double values[5];
172       HYPRE_Int cols[5];
173 
174       for (i = ilower; i <= iupper; i++)
175       {
176          nnz = 0;
177 
178          /* The left identity block:position i-n */
179          if ((i-n)>=0)
180          {
181             cols[nnz] = i-n;
182             values[nnz] = -1.0;
183             nnz++;
184          }
185 
186          /* The left -1: position i-1 */
187          if (i%n)
188          {
189             cols[nnz] = i-1;
190             values[nnz] = -1.0;
191             nnz++;
192          }
193 
194          /* Set the diagonal: position i */
195          cols[nnz] = i;
196          values[nnz] = 4.0;
197          nnz++;
198 
199          /* The right -1: position i+1 */
200          if ((i+1)%n)
201          {
202             cols[nnz] = i+1;
203             values[nnz] = -1.0;
204             nnz++;
205          }
206 
207          /* The right identity block:position i+n */
208          if ((i+n)< N)
209          {
210             cols[nnz] = i+n;
211             values[nnz] = -1.0;
212             nnz++;
213          }
214 
215          /* Set the values for row i */
216          HYPRE_IJMatrixSetValues(A, 1, &nnz, &i, cols, values);
217       }
218    }
219 
220    /* Assemble after setting the coefficients */
221    HYPRE_IJMatrixAssemble(A);
222 
223    /* Note: for the testing of small problems, one may wish to read
224       in a matrix in IJ format (for the format, see the output files
225       from the -print_system option).
226       In this case, one would use the following routine:
227       HYPRE_IJMatrixRead( <filename>, MPI_COMM_WORLD,
228                           HYPRE_PARCSR, &A );
229       <filename>  = IJ.A.out to read in what has been printed out
230       by -print_system (processor numbers are omitted).
231       A call to HYPRE_IJMatrixRead is an *alternative* to the
232       following sequence of HYPRE_IJMatrix calls:
233       Create, SetObjectType, Initialize, SetValues, and Assemble
234    */
235 
236 
237    /* Get the parcsr matrix object to use */
238    HYPRE_IJMatrixGetObject(A, (void**) &parcsr_A);
239 
240 
241    /* Create the rhs and solution */
242    HYPRE_IJVectorCreate(MPI_COMM_WORLD, ilower, iupper,&b);
243    HYPRE_IJVectorSetObjectType(b, HYPRE_PARCSR);
244    HYPRE_IJVectorInitialize(b);
245 
246    HYPRE_IJVectorCreate(MPI_COMM_WORLD, ilower, iupper,&x);
247    HYPRE_IJVectorSetObjectType(x, HYPRE_PARCSR);
248    HYPRE_IJVectorInitialize(x);
249 
250    /* Set the rhs values to h^2 and the solution to zero */
251    {
252       double *rhs_values, *x_values;
253       HYPRE_Int *rows;
254 
255       rhs_values = calloc(local_size, sizeof(double));
256       x_values = calloc(local_size, sizeof(double));
257       rows = calloc(local_size, sizeof(HYPRE_Int));
258 
259       for (i=0; i<local_size; i++)
260       {
261          rhs_values[i] = h2;
262          x_values[i] = 0.0;
263          rows[i] = ilower + i;
264       }
265 
266       HYPRE_IJVectorSetValues(b, local_size, rows, rhs_values);
267       HYPRE_IJVectorSetValues(x, local_size, rows, x_values);
268 
269       free(x_values);
270       free(rhs_values);
271       free(rows);
272    }
273 
274 
275    HYPRE_IJVectorAssemble(b);
276    /*  As with the matrix, for testing purposes, one may wish to read in a rhs:
277        HYPRE_IJVectorRead( <filename>, MPI_COMM_WORLD,
278                                  HYPRE_PARCSR, &b );
279        as an alternative to the
280        following sequence of HYPRE_IJVectors calls:
281        Create, SetObjectType, Initialize, SetValues, and Assemble
282    */
283    HYPRE_IJVectorGetObject(b, (void **) &par_b);
284 
285    HYPRE_IJVectorAssemble(x);
286    HYPRE_IJVectorGetObject(x, (void **) &par_x);
287 
288 
289   /*  Print out the system  - files names will be IJ.out.A.XXXXX
290        and IJ.out.b.XXXXX, where XXXXX = processor id */
291    if (print_system)
292    {
293       HYPRE_IJMatrixPrint(A, "IJ.out.A");
294       HYPRE_IJVectorPrint(b, "IJ.out.b");
295    }
296 
297 
298    /* Choose a solver and solve the system */
299 
300    /* AMG */
301    if (solver_id == 0)
302    {
303       HYPRE_Int num_iterations;
304       double final_res_norm;
305 
306       /* Create solver */
307       HYPRE_BoomerAMGCreate(&solver);
308 
309       /* Set some parameters (See Reference Manual for more parameters) */
310       HYPRE_BoomerAMGSetPrintLevel(solver, 3);  /* print solve info + parameters */
311       HYPRE_BoomerAMGSetCoarsenType(solver, 6); /* Falgout coarsening */
312       HYPRE_BoomerAMGSetRelaxType(solver, 3);   /* G-S/Jacobi hybrid relaxation */
313       HYPRE_BoomerAMGSetNumSweeps(solver, 1);   /* Sweeeps on each level */
314       HYPRE_BoomerAMGSetMaxLevels(solver, 20);  /* maximum number of levels */
315       HYPRE_BoomerAMGSetTol(solver, 1e-7);      /* conv. tolerance */
316 
317       /* Now setup and solve! */
318       HYPRE_BoomerAMGSetup(solver, parcsr_A, par_b, par_x);
319       HYPRE_BoomerAMGSolve(solver, parcsr_A, par_b, par_x);
320 
321       /* Run info - needed logging turned on */
322       HYPRE_BoomerAMGGetNumIterations(solver, &num_iterations);
323       HYPRE_BoomerAMGGetFinalRelativeResidualNorm(solver, &final_res_norm);
324       if (myid == 0)
325       {
326          printf("\n");
327          printf("Iterations = %lld\n", num_iterations);
328          printf("Final Relative Residual Norm = %e\n", final_res_norm);
329          printf("\n");
330       }
331 
332       /* Destroy solver */
333       HYPRE_BoomerAMGDestroy(solver);
334    }
335    /* PCG */
336    else if (solver_id == 50)
337    {
338       HYPRE_Int num_iterations;
339       double final_res_norm;
340 
341       /* Create solver */
342       HYPRE_ParCSRPCGCreate(MPI_COMM_WORLD, &solver);
343 
344       /* Set some parameters (See Reference Manual for more parameters) */
345       HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
346       HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
347       HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
348       HYPRE_PCGSetPrintLevel(solver, 2); /* prints out the iteration info */
349       HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
350 
351       /* Now setup and solve! */
352       HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
353       HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
354 
355       /* Run info - needed logging turned on */
356       HYPRE_PCGGetNumIterations(solver, &num_iterations);
357       HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
358       if (myid == 0)
359       {
360          printf("\n");
361          printf("Iterations = %lld\n", num_iterations);
362          printf("Final Relative Residual Norm = %e\n", final_res_norm);
363          printf("\n");
364       }
365 
366       /* Destroy solver */
367       HYPRE_ParCSRPCGDestroy(solver);
368    }
369    /* PCG with AMG preconditioner */
370    else if (solver_id == 1)
371    {
372       HYPRE_Int num_iterations;
373       double final_res_norm;
374 
375       /* Create solver */
376       HYPRE_ParCSRPCGCreate(MPI_COMM_WORLD, &solver);
377 
378       /* Set some parameters (See Reference Manual for more parameters) */
379       HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
380       HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
381       HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
382       HYPRE_PCGSetPrintLevel(solver, 2); /* print solve info */
383       HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
384 
385       /* Now set up the AMG preconditioner and specify any parameters */
386       HYPRE_BoomerAMGCreate(&precond);
387       HYPRE_BoomerAMGSetPrintLevel(precond, 1); /* print amg solution info */
388       HYPRE_BoomerAMGSetCoarsenType(precond, 6);
389       HYPRE_BoomerAMGSetRelaxType(precond, 6); /* Sym G.S./Jacobi hybrid */
390       HYPRE_BoomerAMGSetNumSweeps(precond, 1);
391       HYPRE_BoomerAMGSetTol(precond, 0.0); /* conv. tolerance zero */
392       HYPRE_BoomerAMGSetMaxIter(precond, 1); /* do only one iteration! */
393 
394       /* Set the PCG preconditioner */
395       HYPRE_PCGSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
396                           (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup, precond);
397 
398       /* Now setup and solve! */
399       HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
400       HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
401 
402       /* Run info - needed logging turned on */
403       HYPRE_PCGGetNumIterations(solver, &num_iterations);
404       HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
405       if (myid == 0)
406       {
407          printf("\n");
408          printf("Iterations = %lld\n", num_iterations);
409          printf("Final Relative Residual Norm = %e\n", final_res_norm);
410          printf("\n");
411       }
412 
413       /* Destroy solver and preconditioner */
414       HYPRE_ParCSRPCGDestroy(solver);
415       HYPRE_BoomerAMGDestroy(precond);
416    }
417    /* PCG with Parasails Preconditioner */
418    else if (solver_id == 8)
419    {
420       HYPRE_Int num_iterations;
421       double final_res_norm;
422 
423       int      sai_max_levels = 1;
424       double   sai_threshold = 0.1;
425       double   sai_filter = 0.05;
426       int      sai_sym = 1;
427 
428       /* Create solver */
429       HYPRE_ParCSRPCGCreate(MPI_COMM_WORLD, &solver);
430 
431       /* Set some parameters (See Reference Manual for more parameters) */
432       HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
433       HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
434       HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
435       HYPRE_PCGSetPrintLevel(solver, 2); /* print solve info */
436       HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
437 
438       /* Now set up the ParaSails preconditioner and specify any parameters */
439       HYPRE_ParaSailsCreate(MPI_COMM_WORLD, &precond);
440 
441       /* Set some parameters (See Reference Manual for more parameters) */
442       HYPRE_ParaSailsSetParams(precond, sai_threshold, sai_max_levels);
443       HYPRE_ParaSailsSetFilter(precond, sai_filter);
444       HYPRE_ParaSailsSetSym(precond, sai_sym);
445       HYPRE_ParaSailsSetLogging(precond, 3);
446 
447       /* Set the PCG preconditioner */
448       HYPRE_PCGSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSolve,
449                           (HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSetup, precond);
450 
451       /* Now setup and solve! */
452       HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
453       HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
454 
455 
456       /* Run info - needed logging turned on */
457       HYPRE_PCGGetNumIterations(solver, &num_iterations);
458       HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
459       if (myid == 0)
460       {
461          printf("\n");
462          printf("Iterations = %lld\n", num_iterations);
463          printf("Final Relative Residual Norm = %e\n", final_res_norm);
464          printf("\n");
465       }
466 
467       /* Destory solver and preconditioner */
468       HYPRE_ParCSRPCGDestroy(solver);
469       HYPRE_ParaSailsDestroy(precond);
470    }
471    /* Flexible GMRES with  AMG Preconditioner */
472    else if (solver_id == 61)
473    {
474       HYPRE_Int num_iterations;
475       double final_res_norm;
476       int    restart = 30;
477       int    modify = 1;
478 
479 
480       /* Create solver */
481       HYPRE_ParCSRFlexGMRESCreate(MPI_COMM_WORLD, &solver);
482 
483       /* Set some parameters (See Reference Manual for more parameters) */
484       HYPRE_FlexGMRESSetKDim(solver, restart);
485       HYPRE_FlexGMRESSetMaxIter(solver, 1000); /* max iterations */
486       HYPRE_FlexGMRESSetTol(solver, 1e-7); /* conv. tolerance */
487       HYPRE_FlexGMRESSetPrintLevel(solver, 2); /* print solve info */
488       HYPRE_FlexGMRESSetLogging(solver, 1); /* needed to get run info later */
489 
490 
491       /* Now set up the AMG preconditioner and specify any parameters */
492       HYPRE_BoomerAMGCreate(&precond);
493       HYPRE_BoomerAMGSetPrintLevel(precond, 1); /* print amg solution info */
494       HYPRE_BoomerAMGSetCoarsenType(precond, 6);
495       HYPRE_BoomerAMGSetRelaxType(precond, 6); /* Sym G.S./Jacobi hybrid */
496       HYPRE_BoomerAMGSetNumSweeps(precond, 1);
497       HYPRE_BoomerAMGSetTol(precond, 0.0); /* conv. tolerance zero */
498       HYPRE_BoomerAMGSetMaxIter(precond, 1); /* do only one iteration! */
499 
500       /* Set the FlexGMRES preconditioner */
501       HYPRE_FlexGMRESSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
502                           (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup, precond);
503 
504 
505       if (modify)
506       /* this is an optional call  - if you don't call it, hypre_FlexGMRESModifyPCDefault
507          is used - which does nothing.  Otherwise, you can define your own, similar to
508          the one used here */
509          HYPRE_FlexGMRESSetModifyPC( solver,
510                                      (HYPRE_PtrToModifyPCFcn) hypre_FlexGMRESModifyPCAMGExample);
511 
512 
513       /* Now setup and solve! */
514       HYPRE_ParCSRFlexGMRESSetup(solver, parcsr_A, par_b, par_x);
515       HYPRE_ParCSRFlexGMRESSolve(solver, parcsr_A, par_b, par_x);
516 
517       /* Run info - needed logging turned on */
518       HYPRE_FlexGMRESGetNumIterations(solver, &num_iterations);
519       HYPRE_FlexGMRESGetFinalRelativeResidualNorm(solver, &final_res_norm);
520       if (myid == 0)
521       {
522          printf("\n");
523          printf("Iterations = %lld\n", num_iterations);
524          printf("Final Relative Residual Norm = %e\n", final_res_norm);
525          printf("\n");
526       }
527 
528       /* Destory solver and preconditioner */
529       HYPRE_ParCSRFlexGMRESDestroy(solver);
530       HYPRE_BoomerAMGDestroy(precond);
531 
532    }
533    else
534    {
535       if (myid ==0) printf("Invalid solver id specified.\n");
536    }
537 
538    /* Clean up */
539    HYPRE_IJMatrixDestroy(A);
540    HYPRE_IJVectorDestroy(b);
541    HYPRE_IJVectorDestroy(x);
542 
543    /* Finalize MPI*/
544    MPI_Finalize();
545 
546    return(0);
547 }
548 
549 /*--------------------------------------------------------------------------
550    hypre_FlexGMRESModifyPCAMGExample -
551 
552     This is an example (not recommended)
553    of how we can modify things about AMG that
554    affect the solve phase based on how FlexGMRES is doing...For
555    another preconditioner it may make sense to modify the tolerance..
556 
557  *--------------------------------------------------------------------------*/
558 
559 int hypre_FlexGMRESModifyPCAMGExample(void *precond_data, int iterations,
560                                    double rel_residual_norm)
561 {
562 
563 
564    if (rel_residual_norm > .1)
565    {
566       HYPRE_BoomerAMGSetNumSweeps(precond_data, 10);
567    }
568    else
569    {
570       HYPRE_BoomerAMGSetNumSweeps(precond_data, 1);
571    }
572 
573 
574    return 0;
575 }


syntax highlighted by Code2HTML, v. 0.9.1