Actual source code: ex9.c
2: static char help[] = "The solution of 2 different linear systems with different linear solvers.\n\
3: Also, this example illustrates the repeated\n\
4: solution of linear systems, while reusing matrix, vector, and solver data\n\
5: structures throughout the process. Note the various stages of event logging.\n\n";
7: /*T
8: Concepts: KSP^repeatedly solving linear systems;
9: Concepts: PetscLog^profiling multiple stages of code;
10: Concepts: PetscLog^user-defined event profiling;
11: Processors: n
12: T*/
14: /*
15: Include "petscksp.h" so that we can use KSP solvers. Note that this file
16: automatically includes:
17: petsc.h - base PETSc routines petscvec.h - vectors
18: petscsys.h - system routines petscmat.h - matrices
19: petscis.h - index sets petscksp.h - Krylov subspace methods
20: petscviewer.h - viewers petscpc.h - preconditioners
21: */
22: #include petscksp.h
24: /*
25: Declare user-defined routines
26: */
32: int main(int argc,char **args)
33: {
34: Vec x1,b1,x2,b2; /* solution and RHS vectors for systems #1 and #2 */
35: Vec u; /* exact solution vector */
36: Mat C1,C2; /* matrices for systems #1 and #2 */
37: KSP ksp1,ksp2; /* KSP contexts for systems #1 and #2 */
38: PetscInt ntimes = 3; /* number of times to solve the linear systems */
39: PetscEvent CHECK_ERROR; /* event number for error checking */
40: PetscInt ldim,low,high,iglobal,Istart,Iend,Istart2,Iend2;
41: PetscInt I,J,i,j,m = 3,n = 2,its,t;
43: int stages[3];
44: PetscTruth flg;
45: PetscScalar v;
46: PetscMPIInt rank,size;
48: PetscInitialize(&argc,&args,(char *)0,help);
49: PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);
50: PetscOptionsGetInt(PETSC_NULL,"-t",&ntimes,PETSC_NULL);
51: MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
52: MPI_Comm_size(PETSC_COMM_WORLD,&size);
53: n = 2*size;
55: /*
56: Register various stages for profiling
57: */
58: PetscLogStageRegister(&stages[0],"Prelim setup");
59: PetscLogStageRegister(&stages[1],"Linear System 1");
60: PetscLogStageRegister(&stages[2],"Linear System 2");
62: /*
63: Register a user-defined event for profiling (error checking).
64: */
65: CHECK_ERROR = 0;
66: PetscLogEventRegister(&CHECK_ERROR,"Check Error",KSP_COOKIE);
68: /* - - - - - - - - - - - - Stage 0: - - - - - - - - - - - - - -
69: Preliminary Setup
70: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
72: PetscLogStagePush(stages[0]);
74: /*
75: Create data structures for first linear system.
76: - Create parallel matrix, specifying only its global dimensions.
77: When using MatCreate(), the matrix format can be specified at
78: runtime. Also, the parallel partitioning of the matrix is
79: determined by PETSc at runtime.
80: - Create parallel vectors.
81: - When using VecSetSizes(), we specify only the vector's global
82: dimension; the parallel partitioning is determined at runtime.
83: - Note: We form 1 vector from scratch and then duplicate as needed.
84: */
85: MatCreate(PETSC_COMM_WORLD,&C1);
86: MatSetSizes(C1,PETSC_DECIDE,PETSC_DECIDE,m*n,m*n);
87: MatSetFromOptions(C1);
88: MatGetOwnershipRange(C1,&Istart,&Iend);
89: VecCreate(PETSC_COMM_WORLD,&u);
90: VecSetSizes(u,PETSC_DECIDE,m*n);
91: VecSetFromOptions(u);
92: VecDuplicate(u,&b1);
93: VecDuplicate(u,&x1);
95: /*
96: Create first linear solver context.
97: Set runtime options (e.g., -pc_type <type>).
98: Note that the first linear system uses the default option
99: names, while the second linear systme uses a different
100: options prefix.
101: */
102: KSPCreate(PETSC_COMM_WORLD,&ksp1);
103: KSPSetFromOptions(ksp1);
105: /*
106: Set user-defined monitoring routine for first linear system.
107: */
108: PetscOptionsHasName(PETSC_NULL,"-my_ksp_monitor",&flg);
109: if (flg) {KSPSetMonitor(ksp1,MyKSPMonitor,PETSC_NULL,0);}
111: /*
112: Create data structures for second linear system.
113: */
114: MatCreate(PETSC_COMM_WORLD,&C2);
115: MatSetSizes(C2,PETSC_DECIDE,PETSC_DECIDE,m*n,m*n);
116: MatSetFromOptions(C2);
117: MatGetOwnershipRange(C2,&Istart2,&Iend2);
118: VecDuplicate(u,&b2);
119: VecDuplicate(u,&x2);
121: /*
122: Create second linear solver context
123: */
124: KSPCreate(PETSC_COMM_WORLD,&ksp2);
126: /*
127: Set different options prefix for second linear system.
128: Set runtime options (e.g., -s2_pc_type <type>)
129: */
130: KSPAppendOptionsPrefix(ksp2,"s2_");
131: KSPSetFromOptions(ksp2);
133: /*
134: Assemble exact solution vector in parallel. Note that each
135: processor needs to set only its local part of the vector.
136: */
137: VecGetLocalSize(u,&ldim);
138: VecGetOwnershipRange(u,&low,&high);
139: for (i=0; i<ldim; i++) {
140: iglobal = i + low;
141: v = (PetscScalar)(i + 100*rank);
142: VecSetValues(u,1,&iglobal,&v,ADD_VALUES);
143: }
144: VecAssemblyBegin(u);
145: VecAssemblyEnd(u);
147: /*
148: Log the number of flops for computing vector entries
149: */
150: PetscLogFlops(2*ldim);
152: /*
153: End curent profiling stage
154: */
155: PetscLogStagePop();
157: /* --------------------------------------------------------------
158: Linear solver loop:
159: Solve 2 different linear systems several times in succession
160: -------------------------------------------------------------- */
162: for (t=0; t<ntimes; t++) {
164: /* - - - - - - - - - - - - Stage 1: - - - - - - - - - - - - - -
165: Assemble and solve first linear system
166: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
168: /*
169: Begin profiling stage #1
170: */
171: PetscLogStagePush(stages[1]);
173: /*
174: Initialize all matrix entries to zero. MatZeroEntries() retains
175: the nonzero structure of the matrix for sparse formats.
176: */
177: if (t > 0) {MatZeroEntries(C1);}
179: /*
180: Set matrix entries in parallel. Also, log the number of flops
181: for computing matrix entries.
182: - Each processor needs to insert only elements that it owns
183: locally (but any non-local elements will be sent to the
184: appropriate processor during matrix assembly).
185: - Always specify global row and columns of matrix entries.
186: */
187: for (I=Istart; I<Iend; I++) {
188: v = -1.0; i = I/n; j = I - i*n;
189: if (i>0) {J = I - n; MatSetValues(C1,1,&I,1,&J,&v,ADD_VALUES);}
190: if (i<m-1) {J = I + n; MatSetValues(C1,1,&I,1,&J,&v,ADD_VALUES);}
191: if (j>0) {J = I - 1; MatSetValues(C1,1,&I,1,&J,&v,ADD_VALUES);}
192: if (j<n-1) {J = I + 1; MatSetValues(C1,1,&I,1,&J,&v,ADD_VALUES);}
193: v = 4.0; MatSetValues(C1,1,&I,1,&I,&v,ADD_VALUES);
194: }
195: for (I=Istart; I<Iend; I++) { /* Make matrix nonsymmetric */
196: v = -1.0*(t+0.5); i = I/n;
197: if (i>0) {J = I - n; MatSetValues(C1,1,&I,1,&J,&v,ADD_VALUES);}
198: }
199: PetscLogFlops(2*(Istart-Iend));
201: /*
202: Assemble matrix, using the 2-step process:
203: MatAssemblyBegin(), MatAssemblyEnd()
204: Computations can be done while messages are in transition
205: by placing code between these two statements.
206: */
207: MatAssemblyBegin(C1,MAT_FINAL_ASSEMBLY);
208: MatAssemblyEnd(C1,MAT_FINAL_ASSEMBLY);
210: /*
211: Indicate same nonzero structure of successive linear system matrices
212: */
213: MatSetOption(C1,MAT_NO_NEW_NONZERO_LOCATIONS);
215: /*
216: Compute right-hand-side vector
217: */
218: MatMult(C1,u,b1);
220: /*
221: Set operators. Here the matrix that defines the linear system
222: also serves as the preconditioning matrix.
223: - The flag SAME_NONZERO_PATTERN indicates that the
224: preconditioning matrix has identical nonzero structure
225: as during the last linear solve (although the values of
226: the entries have changed). Thus, we can save some
227: work in setting up the preconditioner (e.g., no need to
228: redo symbolic factorization for ILU/ICC preconditioners).
229: - If the nonzero structure of the matrix is different during
230: the second linear solve, then the flag DIFFERENT_NONZERO_PATTERN
231: must be used instead. If you are unsure whether the
232: matrix structure has changed or not, use the flag
233: DIFFERENT_NONZERO_PATTERN.
234: - Caution: If you specify SAME_NONZERO_PATTERN, PETSc
235: believes your assertion and does not check the structure
236: of the matrix. If you erroneously claim that the structure
237: is the same when it actually is not, the new preconditioner
238: will not function correctly. Thus, use this optimization
239: feature with caution!
240: */
241: KSPSetOperators(ksp1,C1,C1,SAME_NONZERO_PATTERN);
243: /*
244: Use the previous solution of linear system #1 as the initial
245: guess for the next solve of linear system #1. The user MUST
246: call KSPSetInitialGuessNonzero() in indicate use of an initial
247: guess vector; otherwise, an initial guess of zero is used.
248: */
249: if (t>0) {
250: KSPSetInitialGuessNonzero(ksp1,PETSC_TRUE);
251: }
253: /*
254: Solve the first linear system. Here we explicitly call
255: KSPSetUp() for more detailed performance monitoring of
256: certain preconditioners, such as ICC and ILU. This call
257: is optional, ase KSPSetUp() will automatically be called
258: within KSPSolve() if it hasn't been called already.
259: */
260: KSPSetUp(ksp1);
261: KSPSolve(ksp1,b1,x1);
262: KSPGetIterationNumber(ksp1,&its);
264: /*
265: Check error of solution to first linear system
266: */
267: CheckError(u,x1,b1,its,CHECK_ERROR);
269: /* - - - - - - - - - - - - Stage 2: - - - - - - - - - - - - - -
270: Assemble and solve second linear system
271: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
273: /*
274: Conclude profiling stage #1; begin profiling stage #2
275: */
276: PetscLogStagePop();
277: PetscLogStagePush(stages[2]);
279: /*
280: Initialize all matrix entries to zero
281: */
282: if (t > 0) {MatZeroEntries(C2);}
284: /*
285: Assemble matrix in parallel. Also, log the number of flops
286: for computing matrix entries.
287: - To illustrate the features of parallel matrix assembly, we
288: intentionally set the values differently from the way in
289: which the matrix is distributed across the processors. Each
290: entry that is not owned locally will be sent to the appropriate
291: processor during MatAssemblyBegin() and MatAssemblyEnd().
292: - For best efficiency the user should strive to set as many
293: entries locally as possible.
294: */
295: for (i=0; i<m; i++) {
296: for (j=2*rank; j<2*rank+2; j++) {
297: v = -1.0; I = j + n*i;
298: if (i>0) {J = I - n; MatSetValues(C2,1,&I,1,&J,&v,ADD_VALUES);}
299: if (i<m-1) {J = I + n; MatSetValues(C2,1,&I,1,&J,&v,ADD_VALUES);}
300: if (j>0) {J = I - 1; MatSetValues(C2,1,&I,1,&J,&v,ADD_VALUES);}
301: if (j<n-1) {J = I + 1; MatSetValues(C2,1,&I,1,&J,&v,ADD_VALUES);}
302: v = 6.0 + t*0.5; MatSetValues(C2,1,&I,1,&I,&v,ADD_VALUES);
303: }
304: }
305: for (I=Istart2; I<Iend2; I++) { /* Make matrix nonsymmetric */
306: v = -1.0*(t+0.5); i = I/n;
307: if (i>0) {J = I - n; MatSetValues(C2,1,&I,1,&J,&v,ADD_VALUES);}
308: }
309: MatAssemblyBegin(C2,MAT_FINAL_ASSEMBLY);
310: MatAssemblyEnd(C2,MAT_FINAL_ASSEMBLY);
311: PetscLogFlops(2*(Istart-Iend));
313: /*
314: Indicate same nonzero structure of successive linear system matrices
315: */
316: MatSetOption(C2,MAT_NO_NEW_NONZERO_LOCATIONS);
318: /*
319: Compute right-hand-side vector
320: */
321: MatMult(C2,u,b2);
323: /*
324: Set operators. Here the matrix that defines the linear system
325: also serves as the preconditioning matrix. Indicate same nonzero
326: structure of successive preconditioner matrices by setting flag
327: SAME_NONZERO_PATTERN.
328: */
329: KSPSetOperators(ksp2,C2,C2,SAME_NONZERO_PATTERN);
331: /*
332: Solve the second linear system
333: */
334: KSPSetUp(ksp2);
335: KSPSolve(ksp2,b2,x2);
336: KSPGetIterationNumber(ksp2,&its);
338: /*
339: Check error of solution to second linear system
340: */
341: CheckError(u,x2,b2,its,CHECK_ERROR);
343: /*
344: Conclude profiling stage #2
345: */
346: PetscLogStagePop();
347: }
348: /* --------------------------------------------------------------
349: End of linear solver loop
350: -------------------------------------------------------------- */
352: /*
353: Free work space. All PETSc objects should be destroyed when they
354: are no longer needed.
355: */
356: KSPDestroy(ksp1); KSPDestroy(ksp2);
357: VecDestroy(x1); VecDestroy(x2);
358: VecDestroy(b1); VecDestroy(b2);
359: MatDestroy(C1); MatDestroy(C2);
360: VecDestroy(u);
362: PetscFinalize();
363: return 0;
364: }
367: /* ------------------------------------------------------------- */
368: /*
369: CheckError - Checks the error of the solution.
371: Input Parameters:
372: u - exact solution
373: x - approximate solution
374: b - work vector
375: its - number of iterations for convergence
376: CHECK_ERROR - the event number for error checking
377: (for use with profiling)
379: Notes:
380: In order to profile this section of code separately from the
381: rest of the program, we register it as an "event" with
382: PetscLogEventRegister() in the main program. Then, we indicate
383: the start and end of this event by respectively calling
384: PetscLogEventBegin(CHECK_ERROR,u,x,b,0);
385: PetscLogEventEnd(CHECK_ERROR,u,x,b,0);
386: Here, we specify the objects most closely associated with
387: the event (the vectors u,x,b). Such information is optional;
388: we could instead just use 0 instead for all objects.
389: */
390: PetscErrorCode CheckError(Vec u,Vec x,Vec b,PetscInt its,PetscEvent CHECK_ERROR)
391: {
392: PetscScalar none = -1.0;
393: PetscReal norm;
396: PetscLogEventBegin(CHECK_ERROR,u,x,b,0);
398: /*
399: Compute error of the solution, using b as a work vector.
400: */
401: VecCopy(x,b);
402: VecAXPY(b,none,u);
403: VecNorm(b,NORM_2,&norm);
404: PetscPrintf(PETSC_COMM_WORLD,"Norm of error %A, Iterations %D\n",norm,its);
405: PetscLogEventEnd(CHECK_ERROR,u,x,b,0);
406: return 0;
407: }
408: /* ------------------------------------------------------------- */
411: /*
412: MyKSPMonitor - This is a user-defined routine for monitoring
413: the KSP iterative solvers.
415: Input Parameters:
416: ksp - iterative context
417: n - iteration number
418: rnorm - 2-norm (preconditioned) residual value (may be estimated)
419: dummy - optional user-defined monitor context (unused here)
420: */
421: PetscErrorCode MyKSPMonitor(KSP ksp,PetscInt n,PetscReal rnorm,void *dummy)
422: {
423: Vec x;
426: /*
427: Build the solution vector
428: */
429: KSPBuildSolution(ksp,PETSC_NULL,&x);
431: /*
432: Write the solution vector and residual norm to stdout.
433: - PetscPrintf() handles output for multiprocessor jobs
434: by printing from only one processor in the communicator.
435: - The parallel viewer PETSC_VIEWER_STDOUT_WORLD handles
436: data from multiple processors so that the output
437: is not jumbled.
438: */
439: PetscPrintf(PETSC_COMM_WORLD,"iteration %D solution vector:\n",n);
440: VecView(x,PETSC_VIEWER_STDOUT_WORLD);
441: PetscPrintf(PETSC_COMM_WORLD,"iteration %D KSP Residual norm %14.12e \n",n,rnorm);
442: return 0;
443: }