Actual source code: bjacobi.c
1: /*$Id: bjacobi.c,v 1.160 2001/08/07 03:03:35 balay Exp $*/
2: /*
3: Defines a block Jacobi preconditioner.
4: */
5: #include src/mat/matimpl.h
6: #include src/sles/pc/pcimpl.h
7: #include src/sles/pc/impls/bjacobi/bjacobi.h
9: static int PCSetUp_BJacobi_Singleblock(PC,Mat,Mat);
10: static int PCSetUp_BJacobi_Multiblock(PC,Mat,Mat);
12: static int PCSetUp_BJacobi(PC pc)
13: {
14: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
15: Mat mat = pc->mat,pmat = pc->pmat;
16: int ierr,N,M,start,i,rank,size,sum,end;
17: int bs,i_start=-1,i_end=-1;
18: char *pprefix,*mprefix;
21: MPI_Comm_rank(pc->comm,&rank);
22: MPI_Comm_size(pc->comm,&size);
23: MatGetLocalSize(pc->pmat,&M,&N);
24: MatGetBlockSize(pc->pmat,&bs);
26: /* ----------
27: Determines the number of blocks assigned to each processor
28: */
30: /* local block count given */
31: if (jac->n_local > 0 && jac->n < 0) {
32: MPI_Allreduce(&jac->n_local,&jac->n,1,MPI_INT,MPI_SUM,pc->comm);
33: if (jac->l_lens) { /* check that user set these correctly */
34: sum = 0;
35: for (i=0; i<jac->n_local; i++) {
36: if (jac->l_lens[i]/bs*bs !=jac->l_lens[i]) {
37: SETERRQ(PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
38: }
39: sum += jac->l_lens[i];
40: }
41: if (sum != M) SETERRQ(PETSC_ERR_ARG_SIZ,"Local lens sent incorrectly");
42: }
43: } else if (jac->n > 0 && jac->n_local < 0) { /* global block count given */
44: /* global blocks given: determine which ones are local */
45: if (jac->g_lens) {
46: /* check if the g_lens is has valid entries */
47: for (i=0; i<jac->n; i++) {
48: if (!jac->g_lens[i]) SETERRQ(PETSC_ERR_ARG_SIZ,"Zero block not allowed");
49: if (jac->g_lens[i]/bs*bs != jac->g_lens[i]) {
50: SETERRQ(PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
51: }
52: }
53: if (size == 1) {
54: jac->n_local = jac->n;
55: ierr = PetscMalloc(jac->n_local*sizeof(int),&jac->l_lens);
56: ierr = PetscMemcpy(jac->l_lens,jac->g_lens,jac->n_local*sizeof(int));
57: /* check that user set these correctly */
58: sum = 0;
59: for (i=0; i<jac->n_local; i++) sum += jac->l_lens[i];
60: if (sum != M) SETERRQ(PETSC_ERR_ARG_SIZ,"Global lens sent incorrectly");
61: } else {
62: MatGetOwnershipRange(pc->pmat,&start,&end);
63: /* loop over blocks determing first one owned by me */
64: sum = 0;
65: for (i=0; i<jac->n+1; i++) {
66: if (sum == start) { i_start = i; goto start_1;}
67: if (i < jac->n) sum += jac->g_lens[i];
68: }
69: SETERRQ(PETSC_ERR_ARG_SIZ,"Block sizesn
70: used in PCBJacobiSetTotalBlocks()n
71: are not compatible with parallel matrix layout");
72: start_1:
73: for (i=i_start; i<jac->n+1; i++) {
74: if (sum == end) { i_end = i; goto end_1; }
75: if (i < jac->n) sum += jac->g_lens[i];
76: }
77: SETERRQ(PETSC_ERR_ARG_SIZ,"Block sizesn
78: used in PCBJacobiSetTotalBlocks()n
79: are not compatible with parallel matrix layout");
80: end_1:
81: jac->n_local = i_end - i_start;
82: ierr = PetscMalloc(jac->n_local*sizeof(int),&jac->l_lens);
83: ierr = PetscMemcpy(jac->l_lens,jac->g_lens+i_start,jac->n_local*sizeof(int));
84: }
85: } else { /* no global blocks given, determine then using default layout */
86: jac->n_local = jac->n/size + ((jac->n % size) > rank);
87: ierr = PetscMalloc(jac->n_local*sizeof(int),&jac->l_lens);
88: for (i=0; i<jac->n_local; i++) {
89: jac->l_lens[i] = ((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i))*bs;
90: if (!jac->l_lens[i]) SETERRQ(PETSC_ERR_ARG_SIZ,"Too many blocks given");
91: }
92: }
93: } else if (jac->n < 0 && jac->n_local < 0) { /* no blocks given */
94: jac->n = size;
95: jac->n_local = 1;
96: ierr = PetscMalloc(sizeof(int),&jac->l_lens);
97: jac->l_lens[0] = M;
98: }
100: MPI_Comm_size(pc->comm,&size);
101: if (size == 1) {
102: mat = pc->mat;
103: pmat = pc->pmat;
104: } else {
105: PetscTruth iscopy;
106: MatReuse scall;
107: int (*f)(Mat,PetscTruth*,MatReuse,Mat*);
109: if (jac->use_true_local) {
110: scall = MAT_INITIAL_MATRIX;
111: if (pc->setupcalled) {
112: if (pc->flag == SAME_NONZERO_PATTERN) {
113: if (jac->tp_mat) {
114: scall = MAT_REUSE_MATRIX;
115: mat = jac->tp_mat;
116: }
117: } else {
118: if (jac->tp_mat) {
119: MatDestroy(jac->tp_mat);
120: }
121: }
122: }
123: PetscObjectQueryFunction((PetscObject)pc->mat,"MatGetDiagonalBlock_C",(void (**)(void))&f);
124: if (!f) {
125: SETERRQ(PETSC_ERR_SUP,"This matrix does not support getting diagonal block");
126: }
127: (*f)(pc->mat,&iscopy,scall,&mat);
128: /* make submatrix have same prefix as entire matrix */
129: PetscObjectGetOptionsPrefix((PetscObject)pc->mat,&mprefix);
130: PetscObjectSetOptionsPrefix((PetscObject)mat,mprefix);
131: if (iscopy) {
132: jac->tp_mat = mat;
133: }
134: }
135: if (pc->pmat != pc->mat || !jac->use_true_local) {
136: scall = MAT_INITIAL_MATRIX;
137: if (pc->setupcalled) {
138: if (pc->flag == SAME_NONZERO_PATTERN) {
139: if (jac->tp_pmat) {
140: scall = MAT_REUSE_MATRIX;
141: pmat = jac->tp_pmat;
142: }
143: } else {
144: if (jac->tp_pmat) {
145: MatDestroy(jac->tp_pmat);
146: }
147: }
148: }
149: PetscObjectQueryFunction((PetscObject)pc->pmat,"MatGetDiagonalBlock_C",(void (**)(void))&f);
150: if (!f) {
151: SETERRQ(PETSC_ERR_SUP,"This matrix does not support getting diagonal block");
152: }
153: (*f)(pc->pmat,&iscopy,scall,&pmat);
154: /* make submatrix have same prefix as entire matrix */
155: PetscObjectGetOptionsPrefix((PetscObject)pc->pmat,&pprefix);
156: PetscObjectSetOptionsPrefix((PetscObject)pmat,pprefix);
157: if (iscopy) {
158: jac->tp_pmat = pmat;
159: }
160: } else {
161: pmat = mat;
162: }
163: }
165: /* ------
166: Setup code depends on the number of blocks
167: */
168: if (jac->n_local == 1) {
169: PCSetUp_BJacobi_Singleblock(pc,mat,pmat);
170: } else {
171: PCSetUp_BJacobi_Multiblock(pc,mat,pmat);
172: }
173: return(0);
174: }
176: /* Default destroy, if it has never been setup */
177: static int PCDestroy_BJacobi(PC pc)
178: {
179: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
183: if (jac->g_lens) {PetscFree(jac->g_lens);}
184: if (jac->l_lens) {PetscFree(jac->l_lens);}
185: PetscFree(jac);
186: return(0);
187: }
189: static int PCSetFromOptions_BJacobi(PC pc)
190: {
191: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
192: int blocks,ierr;
193: PetscTruth flg;
196: PetscOptionsHead("Block Jacobi options");
197: PetscOptionsInt("-pc_bjacobi_blocks","Total number of blocks","PCBJacobiSetTotalBlocks",jac->n,&blocks,&flg);
198: if (flg) {
199: PCBJacobiSetTotalBlocks(pc,blocks,PETSC_NULL);
200: }
201: PetscOptionsName("-pc_bjacobi_truelocal","Use the true matrix, not preconditioner matrix to define matrix vector product in sub-problems","PCBJacobiSetUseTrueLocal",&flg);
202: if (flg) {
203: PCBJacobiSetUseTrueLocal(pc);
204: }
205: PetscOptionsTail();
206: return(0);
207: }
209: static int PCView_BJacobi(PC pc,PetscViewer viewer)
210: {
211: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
212: int rank,ierr,i;
213: PetscTruth isascii,isstring;
214: PetscViewer sviewer;
217: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&isascii);
218: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_STRING,&isstring);
219: if (isascii) {
220: if (jac->use_true_local) {
221: PetscViewerASCIIPrintf(viewer," block Jacobi: using true local matrix, number of blocks = %dn",jac->n);
222: }
223: PetscViewerASCIIPrintf(viewer," block Jacobi: number of blocks = %dn",jac->n);
224: MPI_Comm_rank(pc->comm,&rank);
225: if (jac->same_local_solves) {
226: PetscViewerASCIIPrintf(viewer," Local solve is same for all blocks, in the following KSP and PC objects:n");
227: PetscViewerGetSingleton(viewer,&sviewer);
228: if (!rank && jac->sles) {
229: PetscViewerASCIIPushTab(viewer);
230: SLESView(jac->sles[0],sviewer);
231: PetscViewerASCIIPopTab(viewer);
232: }
233: PetscViewerRestoreSingleton(viewer,&sviewer);
234: } else {
236: PetscViewerASCIIPrintf(viewer," Local solve info for each block is in the following KSP and PC objects:n");
237: PetscViewerASCIISynchronizedPrintf(viewer,"Proc %d: number of local blocks = %d, first local block number = %dn",
238: rank,jac->n_local,jac->first_local);
239: PetscViewerASCIIPushTab(viewer);
240: for (i=0; i<jac->n_local; i++) {
241: PetscViewerASCIISynchronizedPrintf(viewer,"Proc %d: local block number %dn",rank,i);
242: PetscViewerGetSingleton(viewer,&sviewer);
243: SLESView(jac->sles[i],sviewer);
244: if (i != jac->n_local-1) {
245: PetscViewerASCIISynchronizedPrintf(viewer,"- - - - - - - - - - - - - - - - - -n");
246: }
247: PetscViewerRestoreSingleton(viewer,&sviewer);
248: }
249: PetscViewerASCIIPopTab(viewer);
250: PetscViewerFlush(viewer);
251: }
252: } else if (isstring) {
253: PetscViewerStringSPrintf(viewer," blks=%d",jac->n);
254: PetscViewerGetSingleton(viewer,&sviewer);
255: if (jac->sles) {SLESView(jac->sles[0],sviewer);}
256: PetscViewerRestoreSingleton(viewer,&sviewer);
257: } else {
258: SETERRQ1(1,"Viewer type %s not supported for block Jacobi",((PetscObject)viewer)->type_name);
259: }
260: return(0);
261: }
263: /* -------------------------------------------------------------------------------------*/
265: EXTERN_C_BEGIN
266: int PCBJacobiSetUseTrueLocal_BJacobi(PC pc)
267: {
268: PC_BJacobi *jac;
271: jac = (PC_BJacobi*)pc->data;
272: jac->use_true_local = PETSC_TRUE;
273: return(0);
274: }
275: EXTERN_C_END
277: EXTERN_C_BEGIN
278: int PCBJacobiGetSubSLES_BJacobi(PC pc,int *n_local,int *first_local,SLES **sles)
279: {
280: PC_BJacobi *jac = (PC_BJacobi*)pc->data;;
283: if (!pc->setupcalled) SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"Must call SLESSetUp() or PCSetUp() first");
285: if (n_local) *n_local = jac->n_local;
286: if (first_local) *first_local = jac->first_local;
287: *sles = jac->sles;
288: jac->same_local_solves = PETSC_FALSE; /* Assume that local solves are now different;
289: not necessarily true though! This flag is
290: used only for PCView_BJacobi() */
291: return(0);
292: }
293: EXTERN_C_END
295: EXTERN_C_BEGIN
296: int PCBJacobiSetTotalBlocks_BJacobi(PC pc,int blocks,int *lens)
297: {
298: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
299: int ierr;
303: jac->n = blocks;
304: if (!lens) {
305: jac->g_lens = 0;
306: } else {
307: PetscMalloc(blocks*sizeof(int),&jac->g_lens);
308: PetscLogObjectMemory(pc,blocks*sizeof(int));
309: PetscMemcpy(jac->g_lens,lens,blocks*sizeof(int));
310: }
311: return(0);
312: }
313: EXTERN_C_END
315: EXTERN_C_BEGIN
316: int PCBJacobiSetLocalBlocks_BJacobi(PC pc,int blocks,int *lens)
317: {
318: PC_BJacobi *jac;
319: int ierr;
322: jac = (PC_BJacobi*)pc->data;
324: jac->n_local = blocks;
325: if (!lens) {
326: jac->l_lens = 0;
327: } else {
328: PetscMalloc(blocks*sizeof(int),&jac->l_lens);
329: PetscLogObjectMemory(pc,blocks*sizeof(int));
330: PetscMemcpy(jac->l_lens,lens,blocks*sizeof(int));
331: }
332: return(0);
333: }
334: EXTERN_C_END
336: /* -------------------------------------------------------------------------------------*/
338: /*@
339: PCBJacobiSetUseTrueLocal - Sets a flag to indicate that the block
340: problem is associated with the linear system matrix instead of the
341: default (where it is associated with the preconditioning matrix).
342: That is, if the local system is solved iteratively then it iterates
343: on the block from the matrix using the block from the preconditioner
344: as the preconditioner for the local block.
346: Collective on PC
348: Input Parameters:
349: . pc - the preconditioner context
351: Options Database Key:
352: . -pc_bjacobi_truelocal - Activates PCBJacobiSetUseTrueLocal()
354: Notes:
355: For the common case in which the preconditioning and linear
356: system matrices are identical, this routine is unnecessary.
358: Level: intermediate
360: .keywords: block, Jacobi, set, true, local, flag
362: .seealso: PCSetOperators(), PCBJacobiSetLocalBlocks()
363: @*/
364: int PCBJacobiSetUseTrueLocal(PC pc)
365: {
366: int ierr,(*f)(PC);
370: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetUseTrueLocal_C",(void (**)(void))&f);
371: if (f) {
372: (*f)(pc);
373: }
375: return(0);
376: }
378: /*@C
379: PCBJacobiGetSubSLES - Gets the local SLES contexts for all blocks on
380: this processor.
381:
382: Note Collective
384: Input Parameter:
385: . pc - the preconditioner context
387: Output Parameters:
388: + n_local - the number of blocks on this processor, or PETSC_NULL
389: . first_local - the global number of the first block on this processor, or PETSC_NULL
390: - sles - the array of SLES contexts
392: Notes:
393: After PCBJacobiGetSubSLES() the array of SLES contexts is not to be freed.
394:
395: Currently for some matrix implementations only 1 block per processor
396: is supported.
397:
398: You must call SLESSetUp() or PCSetUp() before calling PCBJacobiGetSubSLES().
400: Level: advanced
402: .keywords: block, Jacobi, get, sub, SLES, context
404: .seealso: PCBJacobiGetSubSLES()
405: @*/
406: int PCBJacobiGetSubSLES(PC pc,int *n_local,int *first_local,SLES **sles)
407: {
408: int ierr,(*f)(PC,int *,int *,SLES **);
412: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetSubSLES_C",(void (**)(void))&f);
413: if (f) {
414: (*f)(pc,n_local,first_local,sles);
415: } else {
416: SETERRQ(1,"Cannot get subsolvers for this preconditioner");
417: }
418: return(0);
419: }
421: /*@
422: PCBJacobiSetTotalBlocks - Sets the global number of blocks for the block
423: Jacobi preconditioner.
425: Collective on PC
427: Input Parameters:
428: + pc - the preconditioner context
429: . blocks - the number of blocks
430: - lens - [optional] integer array containing the size of each block
432: Options Database Key:
433: . -pc_bjacobi_blocks <blocks> - Sets the number of global blocks
435: Notes:
436: Currently only a limited number of blocking configurations are supported.
437: All processors sharing the PC must call this routine with the same data.
439: Level: intermediate
441: .keywords: set, number, Jacobi, global, total, blocks
443: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiSetLocalBlocks()
444: @*/
445: int PCBJacobiSetTotalBlocks(PC pc,int blocks,int *lens)
446: {
447: int ierr,(*f)(PC,int,int *);
451: if (blocks <= 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have positive blocks");
452: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetTotalBlocks_C",(void (**)(void))&f);
453: if (f) {
454: (*f)(pc,blocks,lens);
455: }
456: return(0);
457: }
458:
459: /*@
460: PCBJacobiSetLocalBlocks - Sets the local number of blocks for the block
461: Jacobi preconditioner.
463: Not Collective
465: Input Parameters:
466: + pc - the preconditioner context
467: . blocks - the number of blocks
468: - lens - [optional] integer array containing size of each block
470: Note:
471: Currently only a limited number of blocking configurations are supported.
473: Level: intermediate
475: .keywords: PC, set, number, Jacobi, local, blocks
477: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiSetTotalBlocks()
478: @*/
479: int PCBJacobiSetLocalBlocks(PC pc,int blocks,int *lens)
480: {
481: int ierr,(*f)(PC,int,int *);
485: if (blocks < 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have nonegative blocks");
486: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetLocalBlocks_C",(void (**)(void))&f);
487: if (f) {
488: (*f)(pc,blocks,lens);
489: }
490: return(0);
491: }
493: /* -----------------------------------------------------------------------------------*/
495: EXTERN_C_BEGIN
496: int PCCreate_BJacobi(PC pc)
497: {
498: int rank,ierr;
499: PC_BJacobi *jac;
502: PetscNew(PC_BJacobi,&jac);
503: PetscLogObjectMemory(pc,sizeof(PC_BJacobi));
504: MPI_Comm_rank(pc->comm,&rank);
505: pc->ops->apply = 0;
506: pc->ops->applytranspose = 0;
507: pc->ops->setup = PCSetUp_BJacobi;
508: pc->ops->destroy = PCDestroy_BJacobi;
509: pc->ops->setfromoptions = PCSetFromOptions_BJacobi;
510: pc->ops->view = PCView_BJacobi;
511: pc->ops->applyrichardson = 0;
513: pc->data = (void*)jac;
514: jac->n = -1;
515: jac->n_local = -1;
516: jac->first_local = rank;
517: jac->sles = 0;
518: jac->use_true_local = PETSC_FALSE;
519: jac->same_local_solves = PETSC_TRUE;
520: jac->g_lens = 0;
521: jac->l_lens = 0;
522: jac->tp_mat = 0;
523: jac->tp_pmat = 0;
525: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetUseTrueLocal_C",
526: "PCBJacobiSetUseTrueLocal_BJacobi",
527: PCBJacobiSetUseTrueLocal_BJacobi);
528: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetSubSLES_C","PCBJacobiGetSubSLES_BJacobi",
529: PCBJacobiGetSubSLES_BJacobi);
530: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetTotalBlocks_C","PCBJacobiSetTotalBlocks_BJacobi",
531: PCBJacobiSetTotalBlocks_BJacobi);
532: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetLocalBlocks_C","PCBJacobiSetLocalBlocks_BJacobi",
533: PCBJacobiSetLocalBlocks_BJacobi);
535: return(0);
536: }
537: EXTERN_C_END
539: /* --------------------------------------------------------------------------------------------*/
540: /*
541: These are for a single block per processor; works for AIJ, BAIJ; Seq and MPI
542: */
543: int PCDestroy_BJacobi_Singleblock(PC pc)
544: {
545: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
546: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
547: int ierr;
550: /*
551: If the on processor block had to be generated via a MatGetDiagonalBlock()
552: that creates a copy (for example MPIBDiag matrices do), this frees the space
553: */
554: if (jac->tp_mat) {
555: MatDestroy(jac->tp_mat);
556: }
557: if (jac->tp_pmat) {
558: MatDestroy(jac->tp_pmat);
559: }
561: SLESDestroy(jac->sles[0]);
562: PetscFree(jac->sles);
563: VecDestroy(bjac->x);
564: VecDestroy(bjac->y);
565: if (jac->l_lens) {PetscFree(jac->l_lens);}
566: if (jac->g_lens) {PetscFree(jac->g_lens);}
567: PetscFree(bjac);
568: PetscFree(jac);
569: return(0);
570: }
572: int PCSetUpOnBlocks_BJacobi_Singleblock(PC pc)
573: {
574: int ierr;
575: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
576: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
579: SLESSetUp(jac->sles[0],bjac->x,bjac->y);
580: return(0);
581: }
583: int PCApply_BJacobi_Singleblock(PC pc,Vec x,Vec y)
584: {
585: int ierr,its;
586: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
587: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
588: PetscScalar *x_array,*y_array;
591: /*
592: The VecPlaceArray() is to avoid having to copy the
593: y vector into the bjac->x vector. The reason for
594: the bjac->x vector is that we need a sequential vector
595: for the sequential solve.
596: */
597: VecGetArray(x,&x_array);
598: VecGetArray(y,&y_array);
599: VecPlaceArray(bjac->x,x_array);
600: VecPlaceArray(bjac->y,y_array);
601: SLESSolve(jac->sles[0],bjac->x,bjac->y,&its);
602: VecRestoreArray(x,&x_array);
603: VecRestoreArray(y,&y_array);
604: return(0);
605: }
607: int PCApplySymmetricLeft_BJacobi_Singleblock(PC pc,Vec x,Vec y)
608: {
609: int ierr;
610: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
611: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
612: PetscScalar *x_array,*y_array;
613: PC subpc;
616: /*
617: The VecPlaceArray() is to avoid having to copy the
618: y vector into the bjac->x vector. The reason for
619: the bjac->x vector is that we need a sequential vector
620: for the sequential solve.
621: */
622: VecGetArray(x,&x_array);
623: VecGetArray(y,&y_array);
624: VecPlaceArray(bjac->x,x_array);
625: VecPlaceArray(bjac->y,y_array);
627: /* apply the symmetric left portion of the inner PC operator */
628: /* note this by-passes the inner SLES and its options completely */
630: SLESGetPC(jac->sles[0],&subpc);
631: PCApplySymmetricLeft(subpc,bjac->x,bjac->y);
633: VecRestoreArray(x,&x_array);
634: VecRestoreArray(y,&y_array);
635: return(0);
636: }
638: int PCApplySymmetricRight_BJacobi_Singleblock(PC pc,Vec x,Vec y)
639: {
640: int ierr;
641: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
642: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
643: PetscScalar *x_array,*y_array;
644: PC subpc;
647: /*
648: The VecPlaceArray() is to avoid having to copy the
649: y vector into the bjac->x vector. The reason for
650: the bjac->x vector is that we need a sequential vector
651: for the sequential solve.
652: */
653: VecGetArray(x,&x_array);
654: VecGetArray(y,&y_array);
655: VecPlaceArray(bjac->x,x_array);
656: VecPlaceArray(bjac->y,y_array);
658: /* apply the symmetric right portion of the inner PC operator */
659: /* note this by-passes the inner SLES and its options completely */
661: SLESGetPC(jac->sles[0],&subpc);
662: PCApplySymmetricRight(subpc,bjac->x,bjac->y);
664: VecRestoreArray(x,&x_array);
665: VecRestoreArray(y,&y_array);
666: return(0);
667: }
669: int PCApplyTranspose_BJacobi_Singleblock(PC pc,Vec x,Vec y)
670: {
671: int ierr,its;
672: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
673: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
674: PetscScalar *x_array,*y_array;
677: /*
678: The VecPlaceArray() is to avoid having to copy the
679: y vector into the bjac->x vector. The reason for
680: the bjac->x vector is that we need a sequential vector
681: for the sequential solve.
682: */
683: VecGetArray(x,&x_array);
684: VecGetArray(y,&y_array);
685: VecPlaceArray(bjac->x,x_array);
686: VecPlaceArray(bjac->y,y_array);
687: SLESSolveTranspose(jac->sles[0],bjac->x,bjac->y,&its);
688: VecRestoreArray(x,&x_array);
689: VecRestoreArray(y,&y_array);
690: return(0);
691: }
693: static int PCSetUp_BJacobi_Singleblock(PC pc,Mat mat,Mat pmat)
694: {
695: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
696: int ierr,m;
697: SLES sles;
698: Vec x,y;
699: PC_BJacobi_Singleblock *bjac;
700: KSP subksp;
701: PC subpc;
705: /* set default direct solver with no Krylov method */
706: if (!pc->setupcalled) {
707: char *prefix;
708: SLESCreate(PETSC_COMM_SELF,&sles);
709: PetscLogObjectParent(pc,sles);
710: SLESGetKSP(sles,&subksp);
711: KSPSetType(subksp,KSPPREONLY);
712: SLESGetPC(sles,&subpc);
713: PCSetType(subpc,PCILU);
714: PCGetOptionsPrefix(pc,&prefix);
715: SLESSetOptionsPrefix(sles,prefix);
716: SLESAppendOptionsPrefix(sles,"sub_");
717: SLESSetFromOptions(sles);
718: /*
719: The reason we need to generate these vectors is to serve
720: as the right-hand side and solution vector for the solve on the
721: block. We do not need to allocate space for the vectors since
722: that is provided via VecPlaceArray() just before the call to
723: SLESSolve() on the block.
724: */
725: MatGetSize(pmat,&m,&m);
726: VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&x);
727: VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&y);
728: PetscLogObjectParent(pc,x);
729: PetscLogObjectParent(pc,y);
731: pc->ops->destroy = PCDestroy_BJacobi_Singleblock;
732: pc->ops->apply = PCApply_BJacobi_Singleblock;
733: pc->ops->applysymmetricleft = PCApplySymmetricLeft_BJacobi_Singleblock;
734: pc->ops->applysymmetricright = PCApplySymmetricRight_BJacobi_Singleblock;
735: pc->ops->applytranspose = PCApplyTranspose_BJacobi_Singleblock;
736: pc->ops->setuponblocks = PCSetUpOnBlocks_BJacobi_Singleblock;
738: PetscMalloc(sizeof(PC_BJacobi_Singleblock),&bjac);
739: PetscLogObjectMemory(pc,sizeof(PC_BJacobi_Singleblock));
740: bjac->x = x;
741: bjac->y = y;
743: PetscMalloc(sizeof(SLES),&jac->sles);
744: jac->sles[0] = sles;
745: jac->data = (void*)bjac;
746: } else {
747: sles = jac->sles[0];
748: bjac = (PC_BJacobi_Singleblock *)jac->data;
749: }
750: if (jac->use_true_local) {
751: SLESSetOperators(sles,mat,pmat,pc->flag);
752: } else {
753: SLESSetOperators(sles,pmat,pmat,pc->flag);
754: }
755: return(0);
756: }
758: /* ---------------------------------------------------------------------------------------------*/
760: int PCDestroy_BJacobi_Multiblock(PC pc)
761: {
762: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
763: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
764: int i,ierr;
767: MatDestroyMatrices(jac->n_local,&bjac->pmat);
768: if (jac->use_true_local) {
769: MatDestroyMatrices(jac->n_local,&bjac->mat);
770: }
772: /*
773: If the on processor block had to be generated via a MatGetDiagonalBlock()
774: that creates a copy (for example MPIBDiag matrices do), this frees the space
775: */
776: if (jac->tp_mat) {
777: MatDestroy(jac->tp_mat);
778: }
779: if (jac->tp_pmat) {
780: MatDestroy(jac->tp_pmat);
781: }
783: for (i=0; i<jac->n_local; i++) {
784: SLESDestroy(jac->sles[i]);
785: VecDestroy(bjac->x[i]);
786: VecDestroy(bjac->y[i]);
787: ISDestroy(bjac->is[i]);
788: }
789: PetscFree(jac->sles);
790: PetscFree(bjac->x);
791: PetscFree(bjac->starts);
792: PetscFree(bjac->is);
793: PetscFree(bjac);
794: if (jac->l_lens) {PetscFree(jac->l_lens);}
795: if (jac->g_lens) {PetscFree(jac->g_lens);}
796: PetscFree(jac);
797: return(0);
798: }
800: int PCSetUpOnBlocks_BJacobi_Multiblock(PC pc)
801: {
802: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
803: int ierr,i,n_local = jac->n_local;
804: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
807: for (i=0; i<n_local; i++) {
808: SLESSetUp(jac->sles[i],bjac->x[i],bjac->y[i]);
809: }
810: return(0);
811: }
813: /*
814: Preconditioner for block Jacobi
815: */
816: int PCApply_BJacobi_Multiblock(PC pc,Vec x,Vec y)
817: {
818: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
819: int ierr,its,i,n_local = jac->n_local;
820: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
821: PetscScalar *xin,*yin;
822: static PetscTruth flag = PETSC_TRUE;
823: static int SUBSlesSolve;
826: if (flag) {
827: PetscLogEventRegister(&SUBSlesSolve,"SubSlesSolve",SLES_COOKIE);
828: flag = PETSC_FALSE;
829: }
830: VecGetArray(x,&xin);
831: VecGetArray(y,&yin);
832: for (i=0; i<n_local; i++) {
833: /*
834: To avoid copying the subvector from x into a workspace we instead
835: make the workspace vector array point to the subpart of the array of
836: the global vector.
837: */
838: VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
839: VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);
841: PetscLogEventBegin(SUBSlesSolve,jac->sles[i],bjac->x[i],bjac->y[i],0);
842: SLESSolve(jac->sles[i],bjac->x[i],bjac->y[i],&its);
843: PetscLogEventEnd(SUBSlesSolve,jac->sles[i],bjac->x[i],bjac->y[i],0);
844: }
845: VecRestoreArray(x,&xin);
846: VecRestoreArray(y,&yin);
847: return(0);
848: }
850: /*
851: Preconditioner for block Jacobi
852: */
853: int PCApplyTranspose_BJacobi_Multiblock(PC pc,Vec x,Vec y)
854: {
855: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
856: int ierr,its,i,n_local = jac->n_local;
857: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
858: PetscScalar *xin,*yin;
859: static PetscTruth flag = PETSC_TRUE;
860: static int SUBSlesSolve;
863: if (flag) {
864: PetscLogEventRegister(&SUBSlesSolve,"SubSlesSolveTranspose",SLES_COOKIE);
865: flag = PETSC_FALSE;
866: }
867: VecGetArray(x,&xin);
868: VecGetArray(y,&yin);
869: for (i=0; i<n_local; i++) {
870: /*
871: To avoid copying the subvector from x into a workspace we instead
872: make the workspace vector array point to the subpart of the array of
873: the global vector.
874: */
875: VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
876: VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);
878: PetscLogEventBegin(SUBSlesSolve,jac->sles[i],bjac->x[i],bjac->y[i],0);
879: SLESSolveTranspose(jac->sles[i],bjac->x[i],bjac->y[i],&its);
880: PetscLogEventEnd(SUBSlesSolve,jac->sles[i],bjac->x[i],bjac->y[i],0);
881: }
882: VecRestoreArray(x,&xin);
883: VecRestoreArray(y,&yin);
884: return(0);
885: }
887: static int PCSetUp_BJacobi_Multiblock(PC pc,Mat mat,Mat pmat)
888: {
889: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
890: int ierr,m,n_local,N,M,start,i;
891: char *prefix,*pprefix,*mprefix;
892: SLES sles;
893: Vec x,y;
894: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
895: KSP subksp;
896: PC subpc;
897: IS is;
898: MatReuse scall = MAT_REUSE_MATRIX;
901: MatGetLocalSize(pc->pmat,&M,&N);
903: n_local = jac->n_local;
905: if (jac->use_true_local) {
906: if (mat->type != pmat->type) SETERRQ(PETSC_ERR_ARG_INCOMP,"Matrices not of same type");
907: }
909: /* set default direct solver with no Krylov method */
910: if (!pc->setupcalled) {
911: scall = MAT_INITIAL_MATRIX;
912: pc->ops->destroy = PCDestroy_BJacobi_Multiblock;
913: pc->ops->apply = PCApply_BJacobi_Multiblock;
914: pc->ops->applytranspose= PCApplyTranspose_BJacobi_Multiblock;
915: pc->ops->setuponblocks = PCSetUpOnBlocks_BJacobi_Multiblock;
917: PetscMalloc(sizeof(PC_BJacobi_Multiblock),&bjac);
918: PetscLogObjectMemory(pc,sizeof(PC_BJacobi_Multiblock));
919: PetscMalloc(n_local*sizeof(SLES),&jac->sles);
920: PetscLogObjectMemory(pc,sizeof(n_local*sizeof(SLES)));
921: PetscMalloc(2*n_local*sizeof(Vec),&bjac->x);
922: PetscLogObjectMemory(pc,sizeof(2*n_local*sizeof(Vec)));
923: bjac->y = bjac->x + n_local;
924: PetscMalloc(n_local*sizeof(PetscScalar),&bjac->starts);
925: PetscLogObjectMemory(pc,sizeof(n_local*sizeof(PetscScalar)));
926:
927: jac->data = (void*)bjac;
928: PetscMalloc(n_local*sizeof(IS),&bjac->is);
929: PetscLogObjectMemory(pc,sizeof(n_local*sizeof(IS)));
931: start = 0;
932: for (i=0; i<n_local; i++) {
933: SLESCreate(PETSC_COMM_SELF,&sles);
934: PetscLogObjectParent(pc,sles);
935: SLESGetKSP(sles,&subksp);
936: KSPSetType(subksp,KSPPREONLY);
937: SLESGetPC(sles,&subpc);
938: PCSetType(subpc,PCILU);
939: PCGetOptionsPrefix(pc,&prefix);
940: SLESSetOptionsPrefix(sles,prefix);
941: SLESAppendOptionsPrefix(sles,"sub_");
942: SLESSetFromOptions(sles);
944: m = jac->l_lens[i];
946: /*
947: The reason we need to generate these vectors is to serve
948: as the right-hand side and solution vector for the solve on the
949: block. We do not need to allocate space for the vectors since
950: that is provided via VecPlaceArray() just before the call to
951: SLESSolve() on the block.
953: */
954: VecCreateSeq(PETSC_COMM_SELF,m,&x);
955: VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&y);
956: PetscLogObjectParent(pc,x);
957: PetscLogObjectParent(pc,y);
958: bjac->x[i] = x;
959: bjac->y[i] = y;
960: bjac->starts[i] = start;
961: jac->sles[i] = sles;
963: ISCreateStride(PETSC_COMM_SELF,m,start,1,&is);
964: bjac->is[i] = is;
965: PetscLogObjectParent(pc,is);
967: start += m;
968: }
969: } else {
970: bjac = (PC_BJacobi_Multiblock*)jac->data;
971: /*
972: Destroy the blocks from the previous iteration
973: */
974: if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
975: MatDestroyMatrices(n_local,&bjac->pmat);
976: if (jac->use_true_local) {
977: MatDestroyMatrices(n_local,&bjac->mat);
978: }
979: scall = MAT_INITIAL_MATRIX;
980: }
981: }
983: MatGetSubMatrices(pmat,n_local,bjac->is,bjac->is,scall,&bjac->pmat);
984: if (jac->use_true_local) {
985: PetscObjectGetOptionsPrefix((PetscObject)mat,&mprefix);
986: MatGetSubMatrices(mat,n_local,bjac->is,bjac->is,scall,&bjac->mat);
987: }
988: /* Return control to the user so that the submatrices can be modified (e.g., to apply
989: different boundary conditions for the submatrices than for the global problem) */
990: PCModifySubMatrices(pc,n_local,bjac->is,bjac->is,bjac->pmat,pc->modifysubmatricesP);
992: PetscObjectGetOptionsPrefix((PetscObject)pmat,&pprefix);
993: for (i=0; i<n_local; i++) {
994: PetscLogObjectParent(pc,bjac->pmat[i]);
995: PetscObjectSetOptionsPrefix((PetscObject)bjac->pmat[i],pprefix);
996: if (jac->use_true_local) {
997: PetscLogObjectParent(pc,bjac->mat[i]);
998: PetscObjectSetOptionsPrefix((PetscObject)bjac->mat[i],mprefix);
999: SLESSetOperators(jac->sles[i],bjac->mat[i],bjac->pmat[i],pc->flag);
1000: } else {
1001: SLESSetOperators(jac->sles[i],bjac->pmat[i],bjac->pmat[i],pc->flag);
1002: }
1003: }
1005: return(0);
1006: }