Actual source code: bjacobi.c
1: #define PETSCKSP_DLL
3: /*
4: Defines a block Jacobi preconditioner.
5: */
6: #include src/mat/matimpl.h
7: #include src/ksp/pc/pcimpl.h
8: #include src/ksp/pc/impls/bjacobi/bjacobi.h
10: static PetscErrorCode PCSetUp_BJacobi_Singleblock(PC,Mat,Mat);
11: static PetscErrorCode PCSetUp_BJacobi_Multiblock(PC,Mat,Mat);
15: static PetscErrorCode PCSetUp_BJacobi(PC pc)
16: {
17: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
18: Mat mat = pc->mat,pmat = pc->pmat;
19: PetscErrorCode ierr,(*f)(Mat,PetscTruth*,MatReuse,Mat*);
20: PetscInt N,M,start,i,sum,end;
21: PetscInt bs,i_start=-1,i_end=-1;
22: PetscMPIInt rank,size;
23: const char *pprefix,*mprefix;
26: MPI_Comm_rank(pc->comm,&rank);
27: MPI_Comm_size(pc->comm,&size);
28: MatGetLocalSize(pc->pmat,&M,&N);
29: MatGetBlockSize(pc->pmat,&bs);
31: /* ----------
32: Determines the number of blocks assigned to each processor
33: */
35: /* local block count given */
36: if (jac->n_local > 0 && jac->n < 0) {
37: MPI_Allreduce(&jac->n_local,&jac->n,1,MPIU_INT,MPI_SUM,pc->comm);
38: if (jac->l_lens) { /* check that user set these correctly */
39: sum = 0;
40: for (i=0; i<jac->n_local; i++) {
41: if (jac->l_lens[i]/bs*bs !=jac->l_lens[i]) {
42: SETERRQ(PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
43: }
44: sum += jac->l_lens[i];
45: }
46: if (sum != M) SETERRQ(PETSC_ERR_ARG_SIZ,"Local lens sent incorrectly");
47: } else {
48: PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
49: for (i=0; i<jac->n_local; i++) {
50: jac->l_lens[i] = bs*((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i));
51: }
52: }
53: } else if (jac->n > 0 && jac->n_local < 0) { /* global block count given */
54: /* global blocks given: determine which ones are local */
55: if (jac->g_lens) {
56: /* check if the g_lens is has valid entries */
57: for (i=0; i<jac->n; i++) {
58: if (!jac->g_lens[i]) SETERRQ(PETSC_ERR_ARG_SIZ,"Zero block not allowed");
59: if (jac->g_lens[i]/bs*bs != jac->g_lens[i]) {
60: SETERRQ(PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
61: }
62: }
63: if (size == 1) {
64: jac->n_local = jac->n;
65: PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
66: PetscMemcpy(jac->l_lens,jac->g_lens,jac->n_local*sizeof(PetscInt));
67: /* check that user set these correctly */
68: sum = 0;
69: for (i=0; i<jac->n_local; i++) sum += jac->l_lens[i];
70: if (sum != M) SETERRQ(PETSC_ERR_ARG_SIZ,"Global lens sent incorrectly");
71: } else {
72: MatGetOwnershipRange(pc->pmat,&start,&end);
73: /* loop over blocks determing first one owned by me */
74: sum = 0;
75: for (i=0; i<jac->n+1; i++) {
76: if (sum == start) { i_start = i; goto start_1;}
77: if (i < jac->n) sum += jac->g_lens[i];
78: }
79: SETERRQ(PETSC_ERR_ARG_SIZ,"Block sizes\n\
80: used in PCBJacobiSetTotalBlocks()\n\
81: are not compatible with parallel matrix layout");
82: start_1:
83: for (i=i_start; i<jac->n+1; i++) {
84: if (sum == end) { i_end = i; goto end_1; }
85: if (i < jac->n) sum += jac->g_lens[i];
86: }
87: SETERRQ(PETSC_ERR_ARG_SIZ,"Block sizes\n\
88: used in PCBJacobiSetTotalBlocks()\n\
89: are not compatible with parallel matrix layout");
90: end_1:
91: jac->n_local = i_end - i_start;
92: PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
93: PetscMemcpy(jac->l_lens,jac->g_lens+i_start,jac->n_local*sizeof(PetscInt));
94: }
95: } else { /* no global blocks given, determine then using default layout */
96: jac->n_local = jac->n/size + ((jac->n % size) > rank);
97: PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
98: for (i=0; i<jac->n_local; i++) {
99: jac->l_lens[i] = ((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i))*bs;
100: if (!jac->l_lens[i]) SETERRQ(PETSC_ERR_ARG_SIZ,"Too many blocks given");
101: }
102: }
103: } else if (jac->n < 0 && jac->n_local < 0) { /* no blocks given */
104: jac->n = size;
105: jac->n_local = 1;
106: PetscMalloc(sizeof(PetscInt),&jac->l_lens);
107: jac->l_lens[0] = M;
108: }
110: MPI_Comm_size(pc->comm,&size);
111: PetscObjectQueryFunction((PetscObject)pc->mat,"MatGetDiagonalBlock_C",(void (**)(void))&f);
112: if (size == 1 && !f) {
113: mat = pc->mat;
114: pmat = pc->pmat;
115: } else {
116: PetscTruth iscopy;
117: MatReuse scall;
119: if (jac->use_true_local) {
120: scall = MAT_INITIAL_MATRIX;
121: if (pc->setupcalled) {
122: if (pc->flag == SAME_NONZERO_PATTERN) {
123: if (jac->tp_mat) {
124: scall = MAT_REUSE_MATRIX;
125: mat = jac->tp_mat;
126: }
127: } else {
128: if (jac->tp_mat) {
129: MatDestroy(jac->tp_mat);
130: }
131: }
132: }
133: if (!f) {
134: SETERRQ(PETSC_ERR_SUP,"This matrix does not support getting diagonal block");
135: }
136: (*f)(pc->mat,&iscopy,scall,&mat);
137: /* make submatrix have same prefix as entire matrix */
138: PetscObjectGetOptionsPrefix((PetscObject)pc->mat,&mprefix);
139: PetscObjectSetOptionsPrefix((PetscObject)mat,mprefix);
140: if (iscopy) {
141: jac->tp_mat = mat;
142: }
143: }
144: if (pc->pmat != pc->mat || !jac->use_true_local) {
145: scall = MAT_INITIAL_MATRIX;
146: if (pc->setupcalled) {
147: if (pc->flag == SAME_NONZERO_PATTERN) {
148: if (jac->tp_pmat) {
149: scall = MAT_REUSE_MATRIX;
150: pmat = jac->tp_pmat;
151: }
152: } else {
153: if (jac->tp_pmat) {
154: MatDestroy(jac->tp_pmat);
155: }
156: }
157: }
158: PetscObjectQueryFunction((PetscObject)pc->pmat,"MatGetDiagonalBlock_C",(void (**)(void))&f);
159: if (!f) {
160: SETERRQ(PETSC_ERR_SUP,"This matrix does not support getting diagonal block");
161: }
162: (*f)(pc->pmat,&iscopy,scall,&pmat);
163: /* make submatrix have same prefix as entire matrix */
164: PetscObjectGetOptionsPrefix((PetscObject)pc->pmat,&pprefix);
165: PetscObjectSetOptionsPrefix((PetscObject)pmat,pprefix);
166: if (iscopy) {
167: jac->tp_pmat = pmat;
168: }
169: } else {
170: pmat = mat;
171: }
172: }
174: /* ------
175: Setup code depends on the number of blocks
176: */
177: if (jac->n_local == 1) {
178: PCSetUp_BJacobi_Singleblock(pc,mat,pmat);
179: } else {
180: PCSetUp_BJacobi_Multiblock(pc,mat,pmat);
181: }
182: return(0);
183: }
185: /* Default destroy, if it has never been setup */
188: static PetscErrorCode PCDestroy_BJacobi(PC pc)
189: {
190: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
194: if (jac->g_lens) {PetscFree(jac->g_lens);}
195: if (jac->l_lens) {PetscFree(jac->l_lens);}
196: PetscFree(jac);
197: return(0);
198: }
202: static PetscErrorCode PCSetFromOptions_BJacobi(PC pc)
203: {
204: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
206: PetscInt blocks;
207: PetscTruth flg;
210: PetscOptionsHead("Block Jacobi options");
211: PetscOptionsInt("-pc_bjacobi_blocks","Total number of blocks","PCBJacobiSetTotalBlocks",jac->n,&blocks,&flg);
212: if (flg) {
213: PCBJacobiSetTotalBlocks(pc,blocks,PETSC_NULL);
214: }
215: PetscOptionsName("-pc_bjacobi_truelocal","Use the true matrix, not preconditioner matrix to define matrix vector product in sub-problems","PCBJacobiSetUseTrueLocal",&flg);
216: if (flg) {
217: PCBJacobiSetUseTrueLocal(pc);
218: }
219: PetscOptionsTail();
220: return(0);
221: }
225: static PetscErrorCode PCView_BJacobi(PC pc,PetscViewer viewer)
226: {
227: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
229: PetscMPIInt rank;
230: PetscInt i;
231: PetscTruth iascii,isstring;
232: PetscViewer sviewer;
235: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
236: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_STRING,&isstring);
237: if (iascii) {
238: if (jac->use_true_local) {
239: PetscViewerASCIIPrintf(viewer," block Jacobi: using true local matrix, number of blocks = %D\n",jac->n);
240: }
241: PetscViewerASCIIPrintf(viewer," block Jacobi: number of blocks = %D\n",jac->n);
242: MPI_Comm_rank(pc->comm,&rank);
243: if (jac->same_local_solves) {
244: PetscViewerASCIIPrintf(viewer," Local solve is same for all blocks, in the following KSP and PC objects:\n");
245: PetscViewerGetSingleton(viewer,&sviewer);
246: if (!rank && jac->ksp) {
247: PetscViewerASCIIPushTab(viewer);
248: KSPView(jac->ksp[0],sviewer);
249: PetscViewerASCIIPopTab(viewer);
250: }
251: PetscViewerRestoreSingleton(viewer,&sviewer);
252: } else {
253: PetscInt n_global;
254: MPI_Allreduce(&jac->n_local,&n_global,1,MPIU_INT,MPI_MAX,pc->comm);
255: PetscViewerASCIIPrintf(viewer," Local solve info for each block is in the following KSP and PC objects:\n");
256: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] number of local blocks = %D, first local block number = %D\n",
257: rank,jac->n_local,jac->first_local);
258: PetscViewerASCIIPushTab(viewer);
259: for (i=0; i<n_global; i++) {
260: PetscViewerGetSingleton(viewer,&sviewer);
261: if (i < jac->n_local) {
262: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] local block number %D\n",rank,i);
263: KSPView(jac->ksp[i],sviewer);
264: PetscViewerASCIISynchronizedPrintf(viewer,"- - - - - - - - - - - - - - - - - -\n");
265: }
266: PetscViewerRestoreSingleton(viewer,&sviewer);
267: }
268: PetscViewerASCIIPopTab(viewer);
269: PetscViewerFlush(viewer);
270: }
271: } else if (isstring) {
272: PetscViewerStringSPrintf(viewer," blks=%D",jac->n);
273: PetscViewerGetSingleton(viewer,&sviewer);
274: if (jac->ksp) {KSPView(jac->ksp[0],sviewer);}
275: PetscViewerRestoreSingleton(viewer,&sviewer);
276: } else {
277: SETERRQ1(PETSC_ERR_SUP,"Viewer type %s not supported for block Jacobi",((PetscObject)viewer)->type_name);
278: }
279: return(0);
280: }
282: /* -------------------------------------------------------------------------------------*/
287: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiSetUseTrueLocal_BJacobi(PC pc)
288: {
289: PC_BJacobi *jac;
292: jac = (PC_BJacobi*)pc->data;
293: jac->use_true_local = PETSC_TRUE;
294: return(0);
295: }
301: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiGetSubKSP_BJacobi(PC pc,PetscInt *n_local,PetscInt *first_local,KSP **ksp)
302: {
303: PC_BJacobi *jac = (PC_BJacobi*)pc->data;;
306: if (!pc->setupcalled) SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"Must call KSPSetUp() or PCSetUp() first");
308: if (n_local) *n_local = jac->n_local;
309: if (first_local) *first_local = jac->first_local;
310: *ksp = jac->ksp;
311: jac->same_local_solves = PETSC_FALSE; /* Assume that local solves are now different;
312: not necessarily true though! This flag is
313: used only for PCView_BJacobi() */
314: return(0);
315: }
321: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiSetTotalBlocks_BJacobi(PC pc,PetscInt blocks,PetscInt *lens)
322: {
323: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
328: if (pc->setupcalled > 0) SETERRQ(PETSC_ERR_ORDER,"Cannot set number of blocks after PCSetUp()/KSPSetUp() has been called");
329: jac->n = blocks;
330: if (!lens) {
331: jac->g_lens = 0;
332: } else {
333: PetscMalloc(blocks*sizeof(PetscInt),&jac->g_lens);
334: PetscLogObjectMemory(pc,blocks*sizeof(PetscInt));
335: PetscMemcpy(jac->g_lens,lens,blocks*sizeof(PetscInt));
336: }
337: return(0);
338: }
344: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiGetTotalBlocks_BJacobi(PC pc, PetscInt *blocks, const PetscInt *lens[])
345: {
346: PC_BJacobi *jac = (PC_BJacobi*) pc->data;
349: *blocks = jac->n;
350: if (lens) *lens = jac->g_lens;
351: return(0);
352: }
358: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiSetLocalBlocks_BJacobi(PC pc,PetscInt blocks,const PetscInt lens[])
359: {
360: PC_BJacobi *jac;
364: jac = (PC_BJacobi*)pc->data;
366: jac->n_local = blocks;
367: if (!lens) {
368: jac->l_lens = 0;
369: } else {
370: PetscMalloc(blocks*sizeof(PetscInt),&jac->l_lens);
371: PetscLogObjectMemory(pc,blocks*sizeof(PetscInt));
372: PetscMemcpy(jac->l_lens,lens,blocks*sizeof(PetscInt));
373: }
374: return(0);
375: }
381: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiGetLocalBlocks_BJacobi(PC pc, PetscInt *blocks, const PetscInt *lens[])
382: {
383: PC_BJacobi *jac = (PC_BJacobi*) pc->data;
386: *blocks = jac->n_local;
387: if (lens) *lens = jac->l_lens;
388: return(0);
389: }
392: /* -------------------------------------------------------------------------------------*/
396: /*@
397: PCBJacobiSetUseTrueLocal - Sets a flag to indicate that the block
398: problem is associated with the linear system matrix instead of the
399: default (where it is associated with the preconditioning matrix).
400: That is, if the local system is solved iteratively then it iterates
401: on the block from the matrix using the block from the preconditioner
402: as the preconditioner for the local block.
404: Collective on PC
406: Input Parameters:
407: . pc - the preconditioner context
409: Options Database Key:
410: . -pc_bjacobi_truelocal - Activates PCBJacobiSetUseTrueLocal()
412: Notes:
413: For the common case in which the preconditioning and linear
414: system matrices are identical, this routine is unnecessary.
416: Level: intermediate
418: .keywords: block, Jacobi, set, true, local, flag
420: .seealso: PCSetOperators(), PCBJacobiSetLocalBlocks()
421: @*/
422: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiSetUseTrueLocal(PC pc)
423: {
424: PetscErrorCode ierr,(*f)(PC);
428: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetUseTrueLocal_C",(void (**)(void))&f);
429: if (f) {
430: (*f)(pc);
431: }
433: return(0);
434: }
438: /*@C
439: PCBJacobiGetSubKSP - Gets the local KSP contexts for all blocks on
440: this processor.
441:
442: Note Collective
444: Input Parameter:
445: . pc - the preconditioner context
447: Output Parameters:
448: + n_local - the number of blocks on this processor, or PETSC_NULL
449: . first_local - the global number of the first block on this processor, or PETSC_NULL
450: - ksp - the array of KSP contexts
452: Notes:
453: After PCBJacobiGetSubKSP() the array of KSP contexts is not to be freed.
454:
455: Currently for some matrix implementations only 1 block per processor
456: is supported.
457:
458: You must call KSPSetUp() or PCSetUp() before calling PCBJacobiGetSubKSP().
460: Level: advanced
462: .keywords: block, Jacobi, get, sub, KSP, context
464: .seealso: PCBJacobiGetSubKSP()
465: @*/
466: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiGetSubKSP(PC pc,PetscInt *n_local,PetscInt *first_local,KSP *ksp[])
467: {
468: PetscErrorCode ierr,(*f)(PC,PetscInt *,PetscInt *,KSP **);
472: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetSubKSP_C",(void (**)(void))&f);
473: if (f) {
474: (*f)(pc,n_local,first_local,ksp);
475: } else {
476: SETERRQ(PETSC_ERR_ARG_WRONG,"Cannot get subsolvers for this preconditioner");
477: }
478: return(0);
479: }
483: /*@
484: PCBJacobiSetTotalBlocks - Sets the global number of blocks for the block
485: Jacobi preconditioner.
487: Collective on PC
489: Input Parameters:
490: + pc - the preconditioner context
491: . blocks - the number of blocks
492: - lens - [optional] integer array containing the size of each block
494: Options Database Key:
495: . -pc_bjacobi_blocks <blocks> - Sets the number of global blocks
497: Notes:
498: Currently only a limited number of blocking configurations are supported.
499: All processors sharing the PC must call this routine with the same data.
501: Level: intermediate
503: .keywords: set, number, Jacobi, global, total, blocks
505: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiSetLocalBlocks()
506: @*/
507: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiSetTotalBlocks(PC pc,PetscInt blocks,const PetscInt lens[])
508: {
509: PetscErrorCode ierr,(*f)(PC,PetscInt,const PetscInt[]);
513: if (blocks <= 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have positive blocks");
514: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetTotalBlocks_C",(void (**)(void))&f);
515: if (f) {
516: (*f)(pc,blocks,lens);
517: }
518: return(0);
519: }
523: /*@C
524: PCBJacobiGetTotalBlocks - Gets the global number of blocks for the block
525: Jacobi preconditioner.
527: Collective on PC
529: Input Parameter:
530: . pc - the preconditioner context
532: Output parameters:
533: + blocks - the number of blocks
534: - lens - integer array containing the size of each block
536: Level: intermediate
538: .keywords: get, number, Jacobi, global, total, blocks
540: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiGetLocalBlocks()
541: @*/
542: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiGetTotalBlocks(PC pc, PetscInt *blocks, const PetscInt *lens[])
543: {
544: PetscErrorCode ierr,(*f)(PC,PetscInt*, const PetscInt *[]);
549: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetTotalBlocks_C",(void (**)(void))&f);
550: if (f) {
551: (*f)(pc,blocks,lens);
552: }
553: return(0);
554: }
555:
558: /*@
559: PCBJacobiSetLocalBlocks - Sets the local number of blocks for the block
560: Jacobi preconditioner.
562: Not Collective
564: Input Parameters:
565: + pc - the preconditioner context
566: . blocks - the number of blocks
567: - lens - [optional] integer array containing size of each block
569: Note:
570: Currently only a limited number of blocking configurations are supported.
572: Level: intermediate
574: .keywords: PC, set, number, Jacobi, local, blocks
576: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiSetTotalBlocks()
577: @*/
578: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiSetLocalBlocks(PC pc,PetscInt blocks,const PetscInt lens[])
579: {
580: PetscErrorCode ierr,(*f)(PC,PetscInt,const PetscInt []);
584: if (blocks < 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have nonegative blocks");
585: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetLocalBlocks_C",(void (**)(void))&f);
586: if (f) {
587: (*f)(pc,blocks,lens);
588: }
589: return(0);
590: }
591:
594: /*@C
595: PCBJacobiGetLocalBlocks - Gets the local number of blocks for the block
596: Jacobi preconditioner.
598: Not Collective
600: Input Parameters:
601: + pc - the preconditioner context
602: . blocks - the number of blocks
603: - lens - [optional] integer array containing size of each block
605: Note:
606: Currently only a limited number of blocking configurations are supported.
608: Level: intermediate
610: .keywords: PC, get, number, Jacobi, local, blocks
612: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiGetTotalBlocks()
613: @*/
614: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiGetLocalBlocks(PC pc, PetscInt *blocks, const PetscInt *lens[])
615: {
616: PetscErrorCode ierr,(*f)(PC,PetscInt*, const PetscInt *[]);
621: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetLocalBlocks_C",(void (**)(void))&f);
622: if (f) {
623: (*f)(pc,blocks,lens);
624: }
625: return(0);
626: }
628: /* -----------------------------------------------------------------------------------*/
630: /*MC
631: PCBJACOBI - Use block Jacobi preconditioning, each block is (approximately) solved with
632: its own KSP object.
634: Options Database Keys:
635: . -pc_bjacobi_truelocal - Activates PCBJacobiSetUseTrueLocal()
637: Notes: Each processor can have one or more blocks, but a block cannot be shared by more
638: than one processor. Defaults to one block per processor.
640: To set options on the solvers for each block append -sub_ to all the KSP, KSP, and PC
641: options database keys. For example, -sub_pc_type ilu -sub_pc_ilu_levels 1 -sub_ksp_type preonly
642:
643: To set the options on the solvers seperate for each block call PCBJacobiGetSubKSP()
644: and set the options directly on the resulting KSP object (you can access its PC
645: KSPGetPC())
647: Level: beginner
649: Concepts: block Jacobi
651: .seealso: PCCreate(), PCSetType(), PCType (for list of available types), PC,
652: PCASM, PCBJacobiSetUseTrueLocal(), PCBJacobiGetSubKSP(), PCBJacobiSetTotalBlocks(),
653: PCBJacobiSetLocalBlocks(), PCSetModifySubmatrices()
654: M*/
659: PetscErrorCode PETSCKSP_DLLEXPORT PCCreate_BJacobi(PC pc)
660: {
662: PetscMPIInt rank;
663: PC_BJacobi *jac;
666: PetscNew(PC_BJacobi,&jac);
667: PetscLogObjectMemory(pc,sizeof(PC_BJacobi));
668: MPI_Comm_rank(pc->comm,&rank);
669: pc->ops->apply = 0;
670: pc->ops->applytranspose = 0;
671: pc->ops->setup = PCSetUp_BJacobi;
672: pc->ops->destroy = PCDestroy_BJacobi;
673: pc->ops->setfromoptions = PCSetFromOptions_BJacobi;
674: pc->ops->view = PCView_BJacobi;
675: pc->ops->applyrichardson = 0;
677: pc->data = (void*)jac;
678: jac->n = -1;
679: jac->n_local = -1;
680: jac->first_local = rank;
681: jac->ksp = 0;
682: jac->use_true_local = PETSC_FALSE;
683: jac->same_local_solves = PETSC_TRUE;
684: jac->g_lens = 0;
685: jac->l_lens = 0;
686: jac->tp_mat = 0;
687: jac->tp_pmat = 0;
689: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetUseTrueLocal_C",
690: "PCBJacobiSetUseTrueLocal_BJacobi",
691: PCBJacobiSetUseTrueLocal_BJacobi);
692: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetSubKSP_C","PCBJacobiGetSubKSP_BJacobi",
693: PCBJacobiGetSubKSP_BJacobi);
694: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetTotalBlocks_C","PCBJacobiSetTotalBlocks_BJacobi",
695: PCBJacobiSetTotalBlocks_BJacobi);
696: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetTotalBlocks_C","PCBJacobiGetTotalBlocks_BJacobi",
697: PCBJacobiGetTotalBlocks_BJacobi);
698: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetLocalBlocks_C","PCBJacobiSetLocalBlocks_BJacobi",
699: PCBJacobiSetLocalBlocks_BJacobi);
700: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetLocalBlocks_C","PCBJacobiGetLocalBlocks_BJacobi",
701: PCBJacobiGetLocalBlocks_BJacobi);
703: return(0);
704: }
707: /* --------------------------------------------------------------------------------------------*/
708: /*
709: These are for a single block per processor; works for AIJ, BAIJ; Seq and MPI
710: */
713: PetscErrorCode PCDestroy_BJacobi_Singleblock(PC pc)
714: {
715: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
716: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
717: PetscErrorCode ierr;
720: /*
721: If the on processor block had to be generated via a MatGetDiagonalBlock()
722: that creates a copy (for example MPIBDiag matrices do), this frees the space
723: */
724: if (jac->tp_mat) {
725: MatDestroy(jac->tp_mat);
726: }
727: if (jac->tp_pmat) {
728: MatDestroy(jac->tp_pmat);
729: }
731: KSPDestroy(jac->ksp[0]);
732: PetscFree(jac->ksp);
733: VecDestroy(bjac->x);
734: VecDestroy(bjac->y);
735: if (jac->l_lens) {PetscFree(jac->l_lens);}
736: if (jac->g_lens) {PetscFree(jac->g_lens);}
737: PetscFree(bjac);
738: PetscFree(jac);
739: return(0);
740: }
744: PetscErrorCode PCSetUpOnBlocks_BJacobi_Singleblock(PC pc)
745: {
747: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
750: KSPSetUp(jac->ksp[0]);
751: return(0);
752: }
756: PetscErrorCode PCApply_BJacobi_Singleblock(PC pc,Vec x,Vec y)
757: {
758: PetscErrorCode ierr;
759: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
760: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
761: PetscScalar *x_array,*y_array;
764: /*
765: The VecPlaceArray() is to avoid having to copy the
766: y vector into the bjac->x vector. The reason for
767: the bjac->x vector is that we need a sequential vector
768: for the sequential solve.
769: */
770: VecGetArray(x,&x_array);
771: VecGetArray(y,&y_array);
772: VecPlaceArray(bjac->x,x_array);
773: VecPlaceArray(bjac->y,y_array);
774: KSPSolve(jac->ksp[0],bjac->x,bjac->y);
775: VecResetArray(bjac->x);
776: VecResetArray(bjac->y);
777: VecRestoreArray(x,&x_array);
778: VecRestoreArray(y,&y_array);
779: return(0);
780: }
784: PetscErrorCode PCApplySymmetricLeft_BJacobi_Singleblock(PC pc,Vec x,Vec y)
785: {
786: PetscErrorCode ierr;
787: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
788: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
789: PetscScalar *x_array,*y_array;
790: PC subpc;
793: /*
794: The VecPlaceArray() is to avoid having to copy the
795: y vector into the bjac->x vector. The reason for
796: the bjac->x vector is that we need a sequential vector
797: for the sequential solve.
798: */
799: VecGetArray(x,&x_array);
800: VecGetArray(y,&y_array);
801: VecPlaceArray(bjac->x,x_array);
802: VecPlaceArray(bjac->y,y_array);
804: /* apply the symmetric left portion of the inner PC operator */
805: /* note this by-passes the inner KSP and its options completely */
807: KSPGetPC(jac->ksp[0],&subpc);
808: PCApplySymmetricLeft(subpc,bjac->x,bjac->y);
809: VecResetArray(bjac->x);
810: VecResetArray(bjac->y);
812: VecRestoreArray(x,&x_array);
813: VecRestoreArray(y,&y_array);
814: return(0);
815: }
819: PetscErrorCode PCApplySymmetricRight_BJacobi_Singleblock(PC pc,Vec x,Vec y)
820: {
821: PetscErrorCode ierr;
822: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
823: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
824: PetscScalar *x_array,*y_array;
825: PC subpc;
828: /*
829: The VecPlaceArray() is to avoid having to copy the
830: y vector into the bjac->x vector. The reason for
831: the bjac->x vector is that we need a sequential vector
832: for the sequential solve.
833: */
834: VecGetArray(x,&x_array);
835: VecGetArray(y,&y_array);
836: VecPlaceArray(bjac->x,x_array);
837: VecPlaceArray(bjac->y,y_array);
839: /* apply the symmetric right portion of the inner PC operator */
840: /* note this by-passes the inner KSP and its options completely */
842: KSPGetPC(jac->ksp[0],&subpc);
843: PCApplySymmetricRight(subpc,bjac->x,bjac->y);
845: VecRestoreArray(x,&x_array);
846: VecRestoreArray(y,&y_array);
847: return(0);
848: }
852: PetscErrorCode PCApplyTranspose_BJacobi_Singleblock(PC pc,Vec x,Vec y)
853: {
854: PetscErrorCode ierr;
855: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
856: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
857: PetscScalar *x_array,*y_array;
860: /*
861: The VecPlaceArray() is to avoid having to copy the
862: y vector into the bjac->x vector. The reason for
863: the bjac->x vector is that we need a sequential vector
864: for the sequential solve.
865: */
866: VecGetArray(x,&x_array);
867: VecGetArray(y,&y_array);
868: VecPlaceArray(bjac->x,x_array);
869: VecPlaceArray(bjac->y,y_array);
870: KSPSolveTranspose(jac->ksp[0],bjac->x,bjac->y);
871: VecResetArray(bjac->x);
872: VecResetArray(bjac->y);
873: VecRestoreArray(x,&x_array);
874: VecRestoreArray(y,&y_array);
875: return(0);
876: }
880: static PetscErrorCode PCSetUp_BJacobi_Singleblock(PC pc,Mat mat,Mat pmat)
881: {
882: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
883: PetscErrorCode ierr;
884: PetscInt m;
885: KSP ksp;
886: Vec x,y;
887: PC_BJacobi_Singleblock *bjac;
888: PC subpc;
889: PetscTruth wasSetup;
893: /* set default direct solver with no Krylov method */
894: if (!pc->setupcalled) {
895: const char *prefix;
896: wasSetup = PETSC_FALSE;
897: KSPCreate(PETSC_COMM_SELF,&ksp);
898: PetscLogObjectParent(pc,ksp);
899: KSPSetType(ksp,KSPPREONLY);
900: KSPGetPC(ksp,&subpc);
901: PCGetOptionsPrefix(pc,&prefix);
902: KSPSetOptionsPrefix(ksp,prefix);
903: KSPAppendOptionsPrefix(ksp,"sub_");
904: /*
905: The reason we need to generate these vectors is to serve
906: as the right-hand side and solution vector for the solve on the
907: block. We do not need to allocate space for the vectors since
908: that is provided via VecPlaceArray() just before the call to
909: KSPSolve() on the block.
910: */
911: MatGetSize(pmat,&m,&m);
912: VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&x);
913: VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&y);
914: PetscLogObjectParent(pc,x);
915: PetscLogObjectParent(pc,y);
917: pc->ops->destroy = PCDestroy_BJacobi_Singleblock;
918: pc->ops->apply = PCApply_BJacobi_Singleblock;
919: pc->ops->applysymmetricleft = PCApplySymmetricLeft_BJacobi_Singleblock;
920: pc->ops->applysymmetricright = PCApplySymmetricRight_BJacobi_Singleblock;
921: pc->ops->applytranspose = PCApplyTranspose_BJacobi_Singleblock;
922: pc->ops->setuponblocks = PCSetUpOnBlocks_BJacobi_Singleblock;
924: PetscMalloc(sizeof(PC_BJacobi_Singleblock),&bjac);
925: PetscLogObjectMemory(pc,sizeof(PC_BJacobi_Singleblock));
926: bjac->x = x;
927: bjac->y = y;
929: PetscMalloc(sizeof(KSP),&jac->ksp);
930: jac->ksp[0] = ksp;
931: jac->data = (void*)bjac;
932: } else {
933: wasSetup = PETSC_TRUE;
934: ksp = jac->ksp[0];
935: bjac = (PC_BJacobi_Singleblock *)jac->data;
936: }
937: if (jac->use_true_local) {
938: KSPSetOperators(ksp,mat,pmat,pc->flag);
939: } else {
940: KSPSetOperators(ksp,pmat,pmat,pc->flag);
941: }
942: if (!wasSetup) {
943: KSPSetFromOptions(ksp);
944: }
945: return(0);
946: }
948: /* ---------------------------------------------------------------------------------------------*/
952: PetscErrorCode PCDestroy_BJacobi_Multiblock(PC pc)
953: {
954: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
955: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
956: PetscErrorCode ierr;
957: PetscInt i;
960: MatDestroyMatrices(jac->n_local,&bjac->pmat);
961: if (jac->use_true_local) {
962: MatDestroyMatrices(jac->n_local,&bjac->mat);
963: }
965: /*
966: If the on processor block had to be generated via a MatGetDiagonalBlock()
967: that creates a copy (for example MPIBDiag matrices do), this frees the space
968: */
969: if (jac->tp_mat) {
970: MatDestroy(jac->tp_mat);
971: }
972: if (jac->tp_pmat) {
973: MatDestroy(jac->tp_pmat);
974: }
976: for (i=0; i<jac->n_local; i++) {
977: KSPDestroy(jac->ksp[i]);
978: VecDestroy(bjac->x[i]);
979: VecDestroy(bjac->y[i]);
980: ISDestroy(bjac->is[i]);
981: }
982: PetscFree(jac->ksp);
983: PetscFree(bjac->x);
984: PetscFree(bjac->starts);
985: PetscFree(bjac->is);
986: PetscFree(bjac);
987: if (jac->l_lens) {PetscFree(jac->l_lens);}
988: if (jac->g_lens) {PetscFree(jac->g_lens);}
989: PetscFree(jac);
990: return(0);
991: }
995: PetscErrorCode PCSetUpOnBlocks_BJacobi_Multiblock(PC pc)
996: {
997: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
999: PetscInt i,n_local = jac->n_local;
1002: for (i=0; i<n_local; i++) {
1003: KSPSetUp(jac->ksp[i]);
1004: }
1005: return(0);
1006: }
1008: /*
1009: Preconditioner for block Jacobi
1010: */
1013: PetscErrorCode PCApply_BJacobi_Multiblock(PC pc,Vec x,Vec y)
1014: {
1015: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
1016: PetscErrorCode ierr;
1017: PetscInt i,n_local = jac->n_local;
1018: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
1019: PetscScalar *xin,*yin;
1020: static PetscTruth flag = PETSC_TRUE;
1021: static PetscEvent SUBKspSolve;
1024: if (flag) {
1025: PetscLogEventRegister(&SUBKspSolve,"SubKspSolve",KSP_COOKIE);
1026: flag = PETSC_FALSE;
1027: }
1028: VecGetArray(x,&xin);
1029: VecGetArray(y,&yin);
1030: for (i=0; i<n_local; i++) {
1031: /*
1032: To avoid copying the subvector from x into a workspace we instead
1033: make the workspace vector array point to the subpart of the array of
1034: the global vector.
1035: */
1036: VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
1037: VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);
1039: PetscLogEventBegin(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1040: KSPSolve(jac->ksp[i],bjac->x[i],bjac->y[i]);
1041: PetscLogEventEnd(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1043: VecResetArray(bjac->x[i]);
1044: VecResetArray(bjac->y[i]);
1045: }
1046: VecRestoreArray(x,&xin);
1047: VecRestoreArray(y,&yin);
1048: return(0);
1049: }
1051: /*
1052: Preconditioner for block Jacobi
1053: */
1056: PetscErrorCode PCApplyTranspose_BJacobi_Multiblock(PC pc,Vec x,Vec y)
1057: {
1058: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
1059: PetscErrorCode ierr;
1060: PetscInt i,n_local = jac->n_local;
1061: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
1062: PetscScalar *xin,*yin;
1063: static PetscTruth flag = PETSC_TRUE;
1064: static PetscEvent SUBKspSolve;
1067: if (flag) {
1068: PetscLogEventRegister(&SUBKspSolve,"SubKspSolveTranspose",KSP_COOKIE);
1069: flag = PETSC_FALSE;
1070: }
1071: VecGetArray(x,&xin);
1072: VecGetArray(y,&yin);
1073: for (i=0; i<n_local; i++) {
1074: /*
1075: To avoid copying the subvector from x into a workspace we instead
1076: make the workspace vector array point to the subpart of the array of
1077: the global vector.
1078: */
1079: VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
1080: VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);
1082: PetscLogEventBegin(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1083: KSPSolveTranspose(jac->ksp[i],bjac->x[i],bjac->y[i]);
1084: PetscLogEventEnd(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1085: }
1086: VecRestoreArray(x,&xin);
1087: VecRestoreArray(y,&yin);
1088: return(0);
1089: }
1093: static PetscErrorCode PCSetUp_BJacobi_Multiblock(PC pc,Mat mat,Mat pmat)
1094: {
1095: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
1096: PetscErrorCode ierr;
1097: PetscInt m,n_local,N,M,start,i;
1098: const char *prefix,*pprefix,*mprefix;
1099: KSP ksp;
1100: Vec x,y;
1101: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
1102: PC subpc;
1103: IS is;
1104: MatReuse scall = MAT_REUSE_MATRIX;
1107: MatGetLocalSize(pc->pmat,&M,&N);
1109: n_local = jac->n_local;
1111: if (jac->use_true_local) {
1112: if (mat->type != pmat->type) SETERRQ(PETSC_ERR_ARG_INCOMP,"Matrices not of same type");
1113: }
1115: /* set default direct solver with no Krylov method */
1116: if (!pc->setupcalled) {
1117: scall = MAT_INITIAL_MATRIX;
1118: pc->ops->destroy = PCDestroy_BJacobi_Multiblock;
1119: pc->ops->apply = PCApply_BJacobi_Multiblock;
1120: pc->ops->applytranspose= PCApplyTranspose_BJacobi_Multiblock;
1121: pc->ops->setuponblocks = PCSetUpOnBlocks_BJacobi_Multiblock;
1123: PetscMalloc(sizeof(PC_BJacobi_Multiblock),&bjac);
1124: PetscLogObjectMemory(pc,sizeof(PC_BJacobi_Multiblock));
1125: PetscMalloc(n_local*sizeof(KSP),&jac->ksp);
1126: PetscLogObjectMemory(pc,sizeof(n_local*sizeof(KSP)));
1127: PetscMalloc(2*n_local*sizeof(Vec),&bjac->x);
1128: PetscLogObjectMemory(pc,sizeof(2*n_local*sizeof(Vec)));
1129: bjac->y = bjac->x + n_local;
1130: PetscMalloc(n_local*sizeof(PetscScalar),&bjac->starts);
1131: PetscLogObjectMemory(pc,sizeof(n_local*sizeof(PetscScalar)));
1132:
1133: jac->data = (void*)bjac;
1134: PetscMalloc(n_local*sizeof(IS),&bjac->is);
1135: PetscLogObjectMemory(pc,sizeof(n_local*sizeof(IS)));
1137: start = 0;
1138: for (i=0; i<n_local; i++) {
1139: KSPCreate(PETSC_COMM_SELF,&ksp);
1140: PetscLogObjectParent(pc,ksp);
1141: KSPSetType(ksp,KSPPREONLY);
1142: KSPGetPC(ksp,&subpc);
1143: PCGetOptionsPrefix(pc,&prefix);
1144: KSPSetOptionsPrefix(ksp,prefix);
1145: KSPAppendOptionsPrefix(ksp,"sub_");
1147: m = jac->l_lens[i];
1149: /*
1150: The reason we need to generate these vectors is to serve
1151: as the right-hand side and solution vector for the solve on the
1152: block. We do not need to allocate space for the vectors since
1153: that is provided via VecPlaceArray() just before the call to
1154: KSPSolve() on the block.
1156: */
1157: VecCreateSeq(PETSC_COMM_SELF,m,&x);
1158: VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&y);
1159: PetscLogObjectParent(pc,x);
1160: PetscLogObjectParent(pc,y);
1161: bjac->x[i] = x;
1162: bjac->y[i] = y;
1163: bjac->starts[i] = start;
1164: jac->ksp[i] = ksp;
1166: ISCreateStride(PETSC_COMM_SELF,m,start,1,&is);
1167: bjac->is[i] = is;
1168: PetscLogObjectParent(pc,is);
1170: start += m;
1171: }
1172: } else {
1173: bjac = (PC_BJacobi_Multiblock*)jac->data;
1174: /*
1175: Destroy the blocks from the previous iteration
1176: */
1177: if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
1178: MatDestroyMatrices(n_local,&bjac->pmat);
1179: if (jac->use_true_local) {
1180: MatDestroyMatrices(n_local,&bjac->mat);
1181: }
1182: scall = MAT_INITIAL_MATRIX;
1183: }
1184: }
1186: MatGetSubMatrices(pmat,n_local,bjac->is,bjac->is,scall,&bjac->pmat);
1187: if (jac->use_true_local) {
1188: PetscObjectGetOptionsPrefix((PetscObject)mat,&mprefix);
1189: MatGetSubMatrices(mat,n_local,bjac->is,bjac->is,scall,&bjac->mat);
1190: }
1191: /* Return control to the user so that the submatrices can be modified (e.g., to apply
1192: different boundary conditions for the submatrices than for the global problem) */
1193: PCModifySubMatrices(pc,n_local,bjac->is,bjac->is,bjac->pmat,pc->modifysubmatricesP);
1195: PetscObjectGetOptionsPrefix((PetscObject)pmat,&pprefix);
1196: for (i=0; i<n_local; i++) {
1197: PetscLogObjectParent(pc,bjac->pmat[i]);
1198: PetscObjectSetOptionsPrefix((PetscObject)bjac->pmat[i],pprefix);
1199: if (jac->use_true_local) {
1200: PetscLogObjectParent(pc,bjac->mat[i]);
1201: PetscObjectSetOptionsPrefix((PetscObject)bjac->mat[i],mprefix);
1202: KSPSetOperators(jac->ksp[i],bjac->mat[i],bjac->pmat[i],pc->flag);
1203: } else {
1204: KSPSetOperators(jac->ksp[i],bjac->pmat[i],bjac->pmat[i],pc->flag);
1205: }
1206: KSPSetFromOptions(jac->ksp[i]);
1207: }
1209: return(0);
1210: }