Actual source code: bjacobi.c

  1: /*$Id: bjacobi.c,v 1.160 2001/08/07 03:03:35 balay Exp $*/
  2: /*
  3:    Defines a block Jacobi preconditioner.
  4: */
 5:  #include src/mat/matimpl.h
 6:  #include src/sles/pc/pcimpl.h
 7:  #include src/sles/pc/impls/bjacobi/bjacobi.h

  9: static int PCSetUp_BJacobi_Singleblock(PC,Mat,Mat);
 10: static int PCSetUp_BJacobi_Multiblock(PC,Mat,Mat);

 12: static int PCSetUp_BJacobi(PC pc)
 13: {
 14:   PC_BJacobi      *jac = (PC_BJacobi*)pc->data;
 15:   Mat             mat = pc->mat,pmat = pc->pmat;
 16:   int             ierr,N,M,start,i,rank,size,sum,end;
 17:   int             bs,i_start=-1,i_end=-1;
 18:   char            *pprefix,*mprefix;

 21:   MPI_Comm_rank(pc->comm,&rank);
 22:   MPI_Comm_size(pc->comm,&size);
 23:   MatGetLocalSize(pc->pmat,&M,&N);
 24:   MatGetBlockSize(pc->pmat,&bs);

 26:   /* ----------
 27:       Determines the number of blocks assigned to each processor 
 28:   */

 30:   /*   local block count  given */
 31:   if (jac->n_local > 0 && jac->n < 0) {
 32:     MPI_Allreduce(&jac->n_local,&jac->n,1,MPI_INT,MPI_SUM,pc->comm);
 33:     if (jac->l_lens) { /* check that user set these correctly */
 34:       sum = 0;
 35:       for (i=0; i<jac->n_local; i++) {
 36:         if (jac->l_lens[i]/bs*bs !=jac->l_lens[i]) {
 37:           SETERRQ(PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
 38:         }
 39:         sum += jac->l_lens[i];
 40:       }
 41:       if (sum != M) SETERRQ(PETSC_ERR_ARG_SIZ,"Local lens sent incorrectly");
 42:     }
 43:   } else if (jac->n > 0 && jac->n_local < 0) { /* global block count given */
 44:     /* global blocks given: determine which ones are local */
 45:     if (jac->g_lens) {
 46:       /* check if the g_lens is has valid entries */
 47:       for (i=0; i<jac->n; i++) {
 48:         if (!jac->g_lens[i]) SETERRQ(PETSC_ERR_ARG_SIZ,"Zero block not allowed");
 49:         if (jac->g_lens[i]/bs*bs != jac->g_lens[i]) {
 50:           SETERRQ(PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
 51:         }
 52:       }
 53:       if (size == 1) {
 54:         jac->n_local = jac->n;
 55:         ierr         = PetscMalloc(jac->n_local*sizeof(int),&jac->l_lens);
 56:         ierr         = PetscMemcpy(jac->l_lens,jac->g_lens,jac->n_local*sizeof(int));
 57:         /* check that user set these correctly */
 58:         sum = 0;
 59:         for (i=0; i<jac->n_local; i++) sum += jac->l_lens[i];
 60:         if (sum != M) SETERRQ(PETSC_ERR_ARG_SIZ,"Global lens sent incorrectly");
 61:       } else {
 62:         MatGetOwnershipRange(pc->pmat,&start,&end);
 63:         /* loop over blocks determing first one owned by me */
 64:         sum = 0;
 65:         for (i=0; i<jac->n+1; i++) {
 66:           if (sum == start) { i_start = i; goto start_1;}
 67:           if (i < jac->n) sum += jac->g_lens[i];
 68:         }
 69:         SETERRQ(PETSC_ERR_ARG_SIZ,"Block sizesn
 70:                    used in PCBJacobiSetTotalBlocks()n
 71:                    are not compatible with parallel matrix layout");
 72:  start_1:
 73:         for (i=i_start; i<jac->n+1; i++) {
 74:           if (sum == end) { i_end = i; goto end_1; }
 75:           if (i < jac->n) sum += jac->g_lens[i];
 76:         }
 77:         SETERRQ(PETSC_ERR_ARG_SIZ,"Block sizesn
 78:                       used in PCBJacobiSetTotalBlocks()n
 79:                       are not compatible with parallel matrix layout");
 80:  end_1:
 81:         jac->n_local = i_end - i_start;
 82:         ierr         = PetscMalloc(jac->n_local*sizeof(int),&jac->l_lens);
 83:         ierr         = PetscMemcpy(jac->l_lens,jac->g_lens+i_start,jac->n_local*sizeof(int));
 84:       }
 85:     } else { /* no global blocks given, determine then using default layout */
 86:       jac->n_local = jac->n/size + ((jac->n % size) > rank);
 87:       ierr         = PetscMalloc(jac->n_local*sizeof(int),&jac->l_lens);
 88:       for (i=0; i<jac->n_local; i++) {
 89:         jac->l_lens[i] = ((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i))*bs;
 90:         if (!jac->l_lens[i]) SETERRQ(PETSC_ERR_ARG_SIZ,"Too many blocks given");
 91:       }
 92:     }
 93:   } else if (jac->n < 0 && jac->n_local < 0) { /* no blocks given */
 94:     jac->n         = size;
 95:     jac->n_local   = 1;
 96:     ierr           = PetscMalloc(sizeof(int),&jac->l_lens);
 97:     jac->l_lens[0] = M;
 98:   }

100:   MPI_Comm_size(pc->comm,&size);
101:   if (size == 1) {
102:     mat  = pc->mat;
103:     pmat = pc->pmat;
104:   } else {
105:     PetscTruth iscopy;
106:     MatReuse   scall;
107:     int        (*f)(Mat,PetscTruth*,MatReuse,Mat*);

109:     if (jac->use_true_local) {
110:       scall = MAT_INITIAL_MATRIX;
111:       if (pc->setupcalled) {
112:         if (pc->flag == SAME_NONZERO_PATTERN) {
113:           if (jac->tp_mat) {
114:             scall = MAT_REUSE_MATRIX;
115:             mat   = jac->tp_mat;
116:           }
117:         } else {
118:           if (jac->tp_mat)  {
119:             MatDestroy(jac->tp_mat);
120:           }
121:         }
122:       }
123:       PetscObjectQueryFunction((PetscObject)pc->mat,"MatGetDiagonalBlock_C",(void (**)(void))&f);
124:       if (!f) {
125:         SETERRQ(PETSC_ERR_SUP,"This matrix does not support getting diagonal block");
126:       }
127:       (*f)(pc->mat,&iscopy,scall,&mat);
128:       /* make submatrix have same prefix as entire matrix */
129:       PetscObjectGetOptionsPrefix((PetscObject)pc->mat,&mprefix);
130:       PetscObjectSetOptionsPrefix((PetscObject)mat,mprefix);
131:       if (iscopy) {
132:         jac->tp_mat = mat;
133:       }
134:     }
135:     if (pc->pmat != pc->mat || !jac->use_true_local) {
136:       scall = MAT_INITIAL_MATRIX;
137:       if (pc->setupcalled) {
138:         if (pc->flag == SAME_NONZERO_PATTERN) {
139:           if (jac->tp_pmat) {
140:             scall = MAT_REUSE_MATRIX;
141:             pmat   = jac->tp_pmat;
142:           }
143:         } else {
144:           if (jac->tp_pmat)  {
145:             MatDestroy(jac->tp_pmat);
146:           }
147:         }
148:       }
149:       PetscObjectQueryFunction((PetscObject)pc->pmat,"MatGetDiagonalBlock_C",(void (**)(void))&f);
150:       if (!f) {
151:         SETERRQ(PETSC_ERR_SUP,"This matrix does not support getting diagonal block");
152:       }
153:       (*f)(pc->pmat,&iscopy,scall,&pmat);
154:       /* make submatrix have same prefix as entire matrix */
155:       PetscObjectGetOptionsPrefix((PetscObject)pc->pmat,&pprefix);
156:       PetscObjectSetOptionsPrefix((PetscObject)pmat,pprefix);
157:       if (iscopy) {
158:         jac->tp_pmat = pmat;
159:       }
160:     } else {
161:       pmat = mat;
162:     }
163:   }

165:   /* ------
166:      Setup code depends on the number of blocks 
167:   */
168:   if (jac->n_local == 1) {
169:     PCSetUp_BJacobi_Singleblock(pc,mat,pmat);
170:   } else {
171:     PCSetUp_BJacobi_Multiblock(pc,mat,pmat);
172:   }
173:   return(0);
174: }

176: /* Default destroy, if it has never been setup */
177: static int PCDestroy_BJacobi(PC pc)
178: {
179:   PC_BJacobi *jac = (PC_BJacobi*)pc->data;

183:   if (jac->g_lens) {PetscFree(jac->g_lens);}
184:   if (jac->l_lens) {PetscFree(jac->l_lens);}
185:   PetscFree(jac);
186:   return(0);
187: }

189: static int PCSetFromOptions_BJacobi(PC pc)
190: {
191:   PC_BJacobi *jac = (PC_BJacobi*)pc->data;
192:   int        blocks,ierr;
193:   PetscTruth flg;

196:   PetscOptionsHead("Block Jacobi options");
197:     PetscOptionsInt("-pc_bjacobi_blocks","Total number of blocks","PCBJacobiSetTotalBlocks",jac->n,&blocks,&flg);
198:     if (flg) {
199:       PCBJacobiSetTotalBlocks(pc,blocks,PETSC_NULL);
200:     }
201:     PetscOptionsName("-pc_bjacobi_truelocal","Use the true matrix, not preconditioner matrix to define matrix vector product in sub-problems","PCBJacobiSetUseTrueLocal",&flg);
202:     if (flg) {
203:       PCBJacobiSetUseTrueLocal(pc);
204:     }
205:   PetscOptionsTail();
206:   return(0);
207: }

209: static int PCView_BJacobi(PC pc,PetscViewer viewer)
210: {
211:   PC_BJacobi  *jac = (PC_BJacobi*)pc->data;
212:   int         rank,ierr,i;
213:   PetscTruth  isascii,isstring;
214:   PetscViewer sviewer;

217:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&isascii);
218:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_STRING,&isstring);
219:   if (isascii) {
220:     if (jac->use_true_local) {
221:       PetscViewerASCIIPrintf(viewer,"  block Jacobi: using true local matrix, number of blocks = %dn",jac->n);
222:     }
223:     PetscViewerASCIIPrintf(viewer,"  block Jacobi: number of blocks = %dn",jac->n);
224:     MPI_Comm_rank(pc->comm,&rank);
225:     if (jac->same_local_solves) {
226:       PetscViewerASCIIPrintf(viewer,"  Local solve is same for all blocks, in the following KSP and PC objects:n");
227:       PetscViewerGetSingleton(viewer,&sviewer);
228:       if (!rank && jac->sles) {
229:         PetscViewerASCIIPushTab(viewer);
230:         SLESView(jac->sles[0],sviewer);
231:         PetscViewerASCIIPopTab(viewer);
232:       }
233:       PetscViewerRestoreSingleton(viewer,&sviewer);
234:     } else {

236:       PetscViewerASCIIPrintf(viewer,"  Local solve info for each block is in the following KSP and PC objects:n");
237:       PetscViewerASCIISynchronizedPrintf(viewer,"Proc %d: number of local blocks = %d, first local block number = %dn",
238:                    rank,jac->n_local,jac->first_local);
239:       PetscViewerASCIIPushTab(viewer);
240:       for (i=0; i<jac->n_local; i++) {
241:         PetscViewerASCIISynchronizedPrintf(viewer,"Proc %d: local block number %dn",rank,i);
242:         PetscViewerGetSingleton(viewer,&sviewer);
243:         SLESView(jac->sles[i],sviewer);
244:         if (i != jac->n_local-1) {
245:           PetscViewerASCIISynchronizedPrintf(viewer,"- - - - - - - - - - - - - - - - - -n");
246:         }
247:         PetscViewerRestoreSingleton(viewer,&sviewer);
248:       }
249:       PetscViewerASCIIPopTab(viewer);
250:       PetscViewerFlush(viewer);
251:     }
252:   } else if (isstring) {
253:     PetscViewerStringSPrintf(viewer," blks=%d",jac->n);
254:     PetscViewerGetSingleton(viewer,&sviewer);
255:     if (jac->sles) {SLESView(jac->sles[0],sviewer);}
256:     PetscViewerRestoreSingleton(viewer,&sviewer);
257:   } else {
258:     SETERRQ1(1,"Viewer type %s not supported for block Jacobi",((PetscObject)viewer)->type_name);
259:   }
260:   return(0);
261: }

263: /* -------------------------------------------------------------------------------------*/

265: EXTERN_C_BEGIN
266: int PCBJacobiSetUseTrueLocal_BJacobi(PC pc)
267: {
268:   PC_BJacobi   *jac;

271:   jac                 = (PC_BJacobi*)pc->data;
272:   jac->use_true_local = PETSC_TRUE;
273:   return(0);
274: }
275: EXTERN_C_END

277: EXTERN_C_BEGIN
278: int PCBJacobiGetSubSLES_BJacobi(PC pc,int *n_local,int *first_local,SLES **sles)
279: {
280:   PC_BJacobi   *jac = (PC_BJacobi*)pc->data;;

283:   if (!pc->setupcalled) SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"Must call SLESSetUp() or PCSetUp() first");

285:   if (n_local)     *n_local     = jac->n_local;
286:   if (first_local) *first_local = jac->first_local;
287:   *sles                         = jac->sles;
288:   jac->same_local_solves        = PETSC_FALSE; /* Assume that local solves are now different;
289:                                                   not necessarily true though!  This flag is 
290:                                                   used only for PCView_BJacobi() */
291:   return(0);
292: }
293: EXTERN_C_END

295: EXTERN_C_BEGIN
296: int PCBJacobiSetTotalBlocks_BJacobi(PC pc,int blocks,int *lens)
297: {
298:   PC_BJacobi *jac = (PC_BJacobi*)pc->data;
299:   int        ierr;


303:   jac->n = blocks;
304:   if (!lens) {
305:     jac->g_lens = 0;
306:   } else {
307:     PetscMalloc(blocks*sizeof(int),&jac->g_lens);
308:     PetscLogObjectMemory(pc,blocks*sizeof(int));
309:     PetscMemcpy(jac->g_lens,lens,blocks*sizeof(int));
310:   }
311:   return(0);
312: }
313: EXTERN_C_END

315: EXTERN_C_BEGIN
316: int PCBJacobiSetLocalBlocks_BJacobi(PC pc,int blocks,int *lens)
317: {
318:   PC_BJacobi *jac;
319:   int        ierr;

322:   jac = (PC_BJacobi*)pc->data;

324:   jac->n_local = blocks;
325:   if (!lens) {
326:     jac->l_lens = 0;
327:   } else {
328:     PetscMalloc(blocks*sizeof(int),&jac->l_lens);
329:     PetscLogObjectMemory(pc,blocks*sizeof(int));
330:     PetscMemcpy(jac->l_lens,lens,blocks*sizeof(int));
331:   }
332:   return(0);
333: }
334: EXTERN_C_END

336: /* -------------------------------------------------------------------------------------*/

338: /*@
339:    PCBJacobiSetUseTrueLocal - Sets a flag to indicate that the block 
340:    problem is associated with the linear system matrix instead of the
341:    default (where it is associated with the preconditioning matrix).
342:    That is, if the local system is solved iteratively then it iterates
343:    on the block from the matrix using the block from the preconditioner
344:    as the preconditioner for the local block.

346:    Collective on PC

348:    Input Parameters:
349: .  pc - the preconditioner context

351:    Options Database Key:
352: .  -pc_bjacobi_truelocal - Activates PCBJacobiSetUseTrueLocal()

354:    Notes:
355:    For the common case in which the preconditioning and linear 
356:    system matrices are identical, this routine is unnecessary.

358:    Level: intermediate

360: .keywords:  block, Jacobi, set, true, local, flag

362: .seealso: PCSetOperators(), PCBJacobiSetLocalBlocks()
363: @*/
364: int PCBJacobiSetUseTrueLocal(PC pc)
365: {
366:   int ierr,(*f)(PC);

370:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetUseTrueLocal_C",(void (**)(void))&f);
371:   if (f) {
372:     (*f)(pc);
373:   }

375:   return(0);
376: }

378: /*@C
379:    PCBJacobiGetSubSLES - Gets the local SLES contexts for all blocks on
380:    this processor.
381:    
382:    Note Collective

384:    Input Parameter:
385: .  pc - the preconditioner context

387:    Output Parameters:
388: +  n_local - the number of blocks on this processor, or PETSC_NULL
389: .  first_local - the global number of the first block on this processor, or PETSC_NULL
390: -  sles - the array of SLES contexts

392:    Notes:  
393:    After PCBJacobiGetSubSLES() the array of SLES contexts is not to be freed.
394:    
395:    Currently for some matrix implementations only 1 block per processor 
396:    is supported.
397:    
398:    You must call SLESSetUp() or PCSetUp() before calling PCBJacobiGetSubSLES().

400:    Level: advanced

402: .keywords:  block, Jacobi, get, sub, SLES, context

404: .seealso: PCBJacobiGetSubSLES()
405: @*/
406: int PCBJacobiGetSubSLES(PC pc,int *n_local,int *first_local,SLES **sles)
407: {
408:   int ierr,(*f)(PC,int *,int *,SLES **);

412:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetSubSLES_C",(void (**)(void))&f);
413:   if (f) {
414:     (*f)(pc,n_local,first_local,sles);
415:   } else {
416:     SETERRQ(1,"Cannot get subsolvers for this preconditioner");
417:   }
418:   return(0);
419: }

421: /*@
422:    PCBJacobiSetTotalBlocks - Sets the global number of blocks for the block
423:    Jacobi preconditioner.

425:    Collective on PC

427:    Input Parameters:
428: +  pc - the preconditioner context
429: .  blocks - the number of blocks
430: -  lens - [optional] integer array containing the size of each block

432:    Options Database Key:
433: .  -pc_bjacobi_blocks <blocks> - Sets the number of global blocks

435:    Notes:  
436:    Currently only a limited number of blocking configurations are supported.
437:    All processors sharing the PC must call this routine with the same data.

439:    Level: intermediate

441: .keywords:  set, number, Jacobi, global, total, blocks

443: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiSetLocalBlocks()
444: @*/
445: int PCBJacobiSetTotalBlocks(PC pc,int blocks,int *lens)
446: {
447:   int ierr,(*f)(PC,int,int *);

451:   if (blocks <= 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have positive blocks");
452:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetTotalBlocks_C",(void (**)(void))&f);
453:   if (f) {
454:     (*f)(pc,blocks,lens);
455:   }
456:   return(0);
457: }
458: 
459: /*@
460:    PCBJacobiSetLocalBlocks - Sets the local number of blocks for the block
461:    Jacobi preconditioner.

463:    Not Collective

465:    Input Parameters:
466: +  pc - the preconditioner context
467: .  blocks - the number of blocks
468: -  lens - [optional] integer array containing size of each block

470:    Note:  
471:    Currently only a limited number of blocking configurations are supported.

473:    Level: intermediate

475: .keywords: PC, set, number, Jacobi, local, blocks

477: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiSetTotalBlocks()
478: @*/
479: int PCBJacobiSetLocalBlocks(PC pc,int blocks,int *lens)
480: {
481:   int ierr,(*f)(PC,int,int *);

485:   if (blocks < 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have nonegative blocks");
486:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetLocalBlocks_C",(void (**)(void))&f);
487:   if (f) {
488:     (*f)(pc,blocks,lens);
489:   }
490:   return(0);
491: }

493: /* -----------------------------------------------------------------------------------*/

495: EXTERN_C_BEGIN
496: int PCCreate_BJacobi(PC pc)
497: {
498:   int          rank,ierr;
499:   PC_BJacobi   *jac;

502:   PetscNew(PC_BJacobi,&jac);
503:   PetscLogObjectMemory(pc,sizeof(PC_BJacobi));
504:   MPI_Comm_rank(pc->comm,&rank);
505:   pc->ops->apply              = 0;
506:   pc->ops->applytranspose     = 0;
507:   pc->ops->setup              = PCSetUp_BJacobi;
508:   pc->ops->destroy            = PCDestroy_BJacobi;
509:   pc->ops->setfromoptions     = PCSetFromOptions_BJacobi;
510:   pc->ops->view               = PCView_BJacobi;
511:   pc->ops->applyrichardson    = 0;

513:   pc->data               = (void*)jac;
514:   jac->n                 = -1;
515:   jac->n_local           = -1;
516:   jac->first_local       = rank;
517:   jac->sles              = 0;
518:   jac->use_true_local    = PETSC_FALSE;
519:   jac->same_local_solves = PETSC_TRUE;
520:   jac->g_lens            = 0;
521:   jac->l_lens            = 0;
522:   jac->tp_mat            = 0;
523:   jac->tp_pmat           = 0;

525:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetUseTrueLocal_C",
526:                     "PCBJacobiSetUseTrueLocal_BJacobi",
527:                     PCBJacobiSetUseTrueLocal_BJacobi);
528:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetSubSLES_C","PCBJacobiGetSubSLES_BJacobi",
529:                     PCBJacobiGetSubSLES_BJacobi);
530:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetTotalBlocks_C","PCBJacobiSetTotalBlocks_BJacobi",
531:                     PCBJacobiSetTotalBlocks_BJacobi);
532:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetLocalBlocks_C","PCBJacobiSetLocalBlocks_BJacobi",
533:                     PCBJacobiSetLocalBlocks_BJacobi);

535:   return(0);
536: }
537: EXTERN_C_END

539: /* --------------------------------------------------------------------------------------------*/
540: /*
541:         These are for a single block per processor; works for AIJ, BAIJ; Seq and MPI
542: */
543: int PCDestroy_BJacobi_Singleblock(PC pc)
544: {
545:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
546:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
547:   int                    ierr;

550:   /*
551:         If the on processor block had to be generated via a MatGetDiagonalBlock()
552:      that creates a copy (for example MPIBDiag matrices do), this frees the space
553:   */
554:   if (jac->tp_mat) {
555:     MatDestroy(jac->tp_mat);
556:   }
557:   if (jac->tp_pmat) {
558:     MatDestroy(jac->tp_pmat);
559:   }

561:   SLESDestroy(jac->sles[0]);
562:   PetscFree(jac->sles);
563:   VecDestroy(bjac->x);
564:   VecDestroy(bjac->y);
565:   if (jac->l_lens) {PetscFree(jac->l_lens);}
566:   if (jac->g_lens) {PetscFree(jac->g_lens);}
567:   PetscFree(bjac);
568:   PetscFree(jac);
569:   return(0);
570: }

572: int PCSetUpOnBlocks_BJacobi_Singleblock(PC pc)
573: {
574:   int                    ierr;
575:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
576:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;

579:   SLESSetUp(jac->sles[0],bjac->x,bjac->y);
580:   return(0);
581: }

583: int PCApply_BJacobi_Singleblock(PC pc,Vec x,Vec y)
584: {
585:   int                    ierr,its;
586:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
587:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
588:   PetscScalar            *x_array,*y_array;

591:   /* 
592:       The VecPlaceArray() is to avoid having to copy the 
593:     y vector into the bjac->x vector. The reason for 
594:     the bjac->x vector is that we need a sequential vector
595:     for the sequential solve.
596:   */
597:   VecGetArray(x,&x_array);
598:   VecGetArray(y,&y_array);
599:   VecPlaceArray(bjac->x,x_array);
600:   VecPlaceArray(bjac->y,y_array);
601:   SLESSolve(jac->sles[0],bjac->x,bjac->y,&its);
602:   VecRestoreArray(x,&x_array);
603:   VecRestoreArray(y,&y_array);
604:   return(0);
605: }

607: int PCApplySymmetricLeft_BJacobi_Singleblock(PC pc,Vec x,Vec y)
608: {
609:   int                    ierr;
610:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
611:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
612:   PetscScalar            *x_array,*y_array;
613:   PC                     subpc;

616:   /* 
617:       The VecPlaceArray() is to avoid having to copy the 
618:     y vector into the bjac->x vector. The reason for 
619:     the bjac->x vector is that we need a sequential vector
620:     for the sequential solve.
621:   */
622:   VecGetArray(x,&x_array);
623:   VecGetArray(y,&y_array);
624:   VecPlaceArray(bjac->x,x_array);
625:   VecPlaceArray(bjac->y,y_array);

627:   /* apply the symmetric left portion of the inner PC operator */
628:   /* note this by-passes the inner SLES and its options completely */

630:   SLESGetPC(jac->sles[0],&subpc);
631:   PCApplySymmetricLeft(subpc,bjac->x,bjac->y);

633:   VecRestoreArray(x,&x_array);
634:   VecRestoreArray(y,&y_array);
635:   return(0);
636: }

638: int PCApplySymmetricRight_BJacobi_Singleblock(PC pc,Vec x,Vec y)
639: {
640:   int                    ierr;
641:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
642:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
643:   PetscScalar            *x_array,*y_array;
644:   PC                     subpc;

647:   /* 
648:       The VecPlaceArray() is to avoid having to copy the 
649:     y vector into the bjac->x vector. The reason for 
650:     the bjac->x vector is that we need a sequential vector
651:     for the sequential solve.
652:   */
653:   VecGetArray(x,&x_array);
654:   VecGetArray(y,&y_array);
655:   VecPlaceArray(bjac->x,x_array);
656:   VecPlaceArray(bjac->y,y_array);

658:   /* apply the symmetric right portion of the inner PC operator */
659:   /* note this by-passes the inner SLES and its options completely */

661:   SLESGetPC(jac->sles[0],&subpc);
662:   PCApplySymmetricRight(subpc,bjac->x,bjac->y);

664:   VecRestoreArray(x,&x_array);
665:   VecRestoreArray(y,&y_array);
666:   return(0);
667: }

669: int PCApplyTranspose_BJacobi_Singleblock(PC pc,Vec x,Vec y)
670: {
671:   int                    ierr,its;
672:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
673:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
674:   PetscScalar            *x_array,*y_array;

677:   /* 
678:       The VecPlaceArray() is to avoid having to copy the 
679:     y vector into the bjac->x vector. The reason for 
680:     the bjac->x vector is that we need a sequential vector
681:     for the sequential solve.
682:   */
683:   VecGetArray(x,&x_array);
684:   VecGetArray(y,&y_array);
685:   VecPlaceArray(bjac->x,x_array);
686:   VecPlaceArray(bjac->y,y_array);
687:   SLESSolveTranspose(jac->sles[0],bjac->x,bjac->y,&its);
688:   VecRestoreArray(x,&x_array);
689:   VecRestoreArray(y,&y_array);
690:   return(0);
691: }

693: static int PCSetUp_BJacobi_Singleblock(PC pc,Mat mat,Mat pmat)
694: {
695:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
696:   int                    ierr,m;
697:   SLES                   sles;
698:   Vec                    x,y;
699:   PC_BJacobi_Singleblock *bjac;
700:   KSP                    subksp;
701:   PC                     subpc;


705:   /* set default direct solver with no Krylov method */
706:   if (!pc->setupcalled) {
707:     char *prefix;
708:     SLESCreate(PETSC_COMM_SELF,&sles);
709:     PetscLogObjectParent(pc,sles);
710:     SLESGetKSP(sles,&subksp);
711:     KSPSetType(subksp,KSPPREONLY);
712:     SLESGetPC(sles,&subpc);
713:     PCSetType(subpc,PCILU);
714:     PCGetOptionsPrefix(pc,&prefix);
715:     SLESSetOptionsPrefix(sles,prefix);
716:     SLESAppendOptionsPrefix(sles,"sub_");
717:     SLESSetFromOptions(sles);
718:     /*
719:       The reason we need to generate these vectors is to serve 
720:       as the right-hand side and solution vector for the solve on the 
721:       block. We do not need to allocate space for the vectors since
722:       that is provided via VecPlaceArray() just before the call to 
723:       SLESSolve() on the block.
724:     */
725:     MatGetSize(pmat,&m,&m);
726:     VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&x);
727:     VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&y);
728:     PetscLogObjectParent(pc,x);
729:     PetscLogObjectParent(pc,y);

731:     pc->ops->destroy             = PCDestroy_BJacobi_Singleblock;
732:     pc->ops->apply               = PCApply_BJacobi_Singleblock;
733:     pc->ops->applysymmetricleft  = PCApplySymmetricLeft_BJacobi_Singleblock;
734:     pc->ops->applysymmetricright = PCApplySymmetricRight_BJacobi_Singleblock;
735:     pc->ops->applytranspose      = PCApplyTranspose_BJacobi_Singleblock;
736:     pc->ops->setuponblocks       = PCSetUpOnBlocks_BJacobi_Singleblock;

738:     PetscMalloc(sizeof(PC_BJacobi_Singleblock),&bjac);
739:     PetscLogObjectMemory(pc,sizeof(PC_BJacobi_Singleblock));
740:     bjac->x      = x;
741:     bjac->y      = y;

743:     PetscMalloc(sizeof(SLES),&jac->sles);
744:     jac->sles[0] = sles;
745:     jac->data    = (void*)bjac;
746:   } else {
747:     sles = jac->sles[0];
748:     bjac = (PC_BJacobi_Singleblock *)jac->data;
749:   }
750:   if (jac->use_true_local) {
751:     SLESSetOperators(sles,mat,pmat,pc->flag);
752:   }  else {
753:     SLESSetOperators(sles,pmat,pmat,pc->flag);
754:   }
755:   return(0);
756: }

758: /* ---------------------------------------------------------------------------------------------*/

760: int PCDestroy_BJacobi_Multiblock(PC pc)
761: {
762:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
763:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
764:   int                   i,ierr;

767:   MatDestroyMatrices(jac->n_local,&bjac->pmat);
768:   if (jac->use_true_local) {
769:     MatDestroyMatrices(jac->n_local,&bjac->mat);
770:   }

772:   /*
773:         If the on processor block had to be generated via a MatGetDiagonalBlock()
774:      that creates a copy (for example MPIBDiag matrices do), this frees the space
775:   */
776:   if (jac->tp_mat) {
777:     MatDestroy(jac->tp_mat);
778:   }
779:   if (jac->tp_pmat) {
780:     MatDestroy(jac->tp_pmat);
781:   }

783:   for (i=0; i<jac->n_local; i++) {
784:     SLESDestroy(jac->sles[i]);
785:     VecDestroy(bjac->x[i]);
786:     VecDestroy(bjac->y[i]);
787:     ISDestroy(bjac->is[i]);
788:   }
789:   PetscFree(jac->sles);
790:   PetscFree(bjac->x);
791:   PetscFree(bjac->starts);
792:   PetscFree(bjac->is);
793:   PetscFree(bjac);
794:   if (jac->l_lens) {PetscFree(jac->l_lens);}
795:   if (jac->g_lens) {PetscFree(jac->g_lens);}
796:   PetscFree(jac);
797:   return(0);
798: }

800: int PCSetUpOnBlocks_BJacobi_Multiblock(PC pc)
801: {
802:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
803:   int                   ierr,i,n_local = jac->n_local;
804:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;

807:   for (i=0; i<n_local; i++) {
808:     SLESSetUp(jac->sles[i],bjac->x[i],bjac->y[i]);
809:   }
810:   return(0);
811: }

813: /*
814:       Preconditioner for block Jacobi 
815: */
816: int PCApply_BJacobi_Multiblock(PC pc,Vec x,Vec y)
817: {
818:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
819:   int                   ierr,its,i,n_local = jac->n_local;
820:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
821:   PetscScalar           *xin,*yin;
822:   static PetscTruth     flag = PETSC_TRUE;
823:   static int            SUBSlesSolve;

826:   if (flag) {
827:     PetscLogEventRegister(&SUBSlesSolve,"SubSlesSolve",SLES_COOKIE);
828:     flag = PETSC_FALSE;
829:   }
830:   VecGetArray(x,&xin);
831:   VecGetArray(y,&yin);
832:   for (i=0; i<n_local; i++) {
833:     /* 
834:        To avoid copying the subvector from x into a workspace we instead 
835:        make the workspace vector array point to the subpart of the array of
836:        the global vector.
837:     */
838:     VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
839:     VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);

841:     PetscLogEventBegin(SUBSlesSolve,jac->sles[i],bjac->x[i],bjac->y[i],0);
842:     SLESSolve(jac->sles[i],bjac->x[i],bjac->y[i],&its);
843:     PetscLogEventEnd(SUBSlesSolve,jac->sles[i],bjac->x[i],bjac->y[i],0);
844:   }
845:   VecRestoreArray(x,&xin);
846:   VecRestoreArray(y,&yin);
847:   return(0);
848: }

850: /*
851:       Preconditioner for block Jacobi 
852: */
853: int PCApplyTranspose_BJacobi_Multiblock(PC pc,Vec x,Vec y)
854: {
855:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
856:   int                   ierr,its,i,n_local = jac->n_local;
857:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
858:   PetscScalar           *xin,*yin;
859:   static PetscTruth     flag = PETSC_TRUE;
860:   static int            SUBSlesSolve;

863:   if (flag) {
864:     PetscLogEventRegister(&SUBSlesSolve,"SubSlesSolveTranspose",SLES_COOKIE);
865:     flag = PETSC_FALSE;
866:   }
867:   VecGetArray(x,&xin);
868:   VecGetArray(y,&yin);
869:   for (i=0; i<n_local; i++) {
870:     /* 
871:        To avoid copying the subvector from x into a workspace we instead 
872:        make the workspace vector array point to the subpart of the array of
873:        the global vector.
874:     */
875:     VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
876:     VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);

878:     PetscLogEventBegin(SUBSlesSolve,jac->sles[i],bjac->x[i],bjac->y[i],0);
879:     SLESSolveTranspose(jac->sles[i],bjac->x[i],bjac->y[i],&its);
880:     PetscLogEventEnd(SUBSlesSolve,jac->sles[i],bjac->x[i],bjac->y[i],0);
881:   }
882:   VecRestoreArray(x,&xin);
883:   VecRestoreArray(y,&yin);
884:   return(0);
885: }

887: static int PCSetUp_BJacobi_Multiblock(PC pc,Mat mat,Mat pmat)
888: {
889:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
890:   int                    ierr,m,n_local,N,M,start,i;
891:   char                   *prefix,*pprefix,*mprefix;
892:   SLES                   sles;
893:   Vec                    x,y;
894:   PC_BJacobi_Multiblock  *bjac = (PC_BJacobi_Multiblock*)jac->data;
895:   KSP                    subksp;
896:   PC                     subpc;
897:   IS                     is;
898:   MatReuse               scall = MAT_REUSE_MATRIX;

901:   MatGetLocalSize(pc->pmat,&M,&N);

903:   n_local = jac->n_local;

905:   if (jac->use_true_local) {
906:     if (mat->type != pmat->type) SETERRQ(PETSC_ERR_ARG_INCOMP,"Matrices not of same type");
907:   }

909:   /* set default direct solver with no Krylov method */
910:   if (!pc->setupcalled) {
911:     scall                  = MAT_INITIAL_MATRIX;
912:     pc->ops->destroy       = PCDestroy_BJacobi_Multiblock;
913:     pc->ops->apply         = PCApply_BJacobi_Multiblock;
914:     pc->ops->applytranspose= PCApplyTranspose_BJacobi_Multiblock;
915:     pc->ops->setuponblocks = PCSetUpOnBlocks_BJacobi_Multiblock;

917:     PetscMalloc(sizeof(PC_BJacobi_Multiblock),&bjac);
918:     PetscLogObjectMemory(pc,sizeof(PC_BJacobi_Multiblock));
919:     PetscMalloc(n_local*sizeof(SLES),&jac->sles);
920:     PetscLogObjectMemory(pc,sizeof(n_local*sizeof(SLES)));
921:     PetscMalloc(2*n_local*sizeof(Vec),&bjac->x);
922:     PetscLogObjectMemory(pc,sizeof(2*n_local*sizeof(Vec)));
923:     bjac->y      = bjac->x + n_local;
924:     PetscMalloc(n_local*sizeof(PetscScalar),&bjac->starts);
925:     PetscLogObjectMemory(pc,sizeof(n_local*sizeof(PetscScalar)));
926: 
927:     jac->data    = (void*)bjac;
928:     PetscMalloc(n_local*sizeof(IS),&bjac->is);
929:     PetscLogObjectMemory(pc,sizeof(n_local*sizeof(IS)));

931:     start = 0;
932:     for (i=0; i<n_local; i++) {
933:       SLESCreate(PETSC_COMM_SELF,&sles);
934:       PetscLogObjectParent(pc,sles);
935:       SLESGetKSP(sles,&subksp);
936:       KSPSetType(subksp,KSPPREONLY);
937:       SLESGetPC(sles,&subpc);
938:       PCSetType(subpc,PCILU);
939:       PCGetOptionsPrefix(pc,&prefix);
940:       SLESSetOptionsPrefix(sles,prefix);
941:       SLESAppendOptionsPrefix(sles,"sub_");
942:       SLESSetFromOptions(sles);

944:       m = jac->l_lens[i];

946:       /*
947:       The reason we need to generate these vectors is to serve 
948:       as the right-hand side and solution vector for the solve on the 
949:       block. We do not need to allocate space for the vectors since
950:       that is provided via VecPlaceArray() just before the call to 
951:       SLESSolve() on the block.

953:       */
954:       VecCreateSeq(PETSC_COMM_SELF,m,&x);
955:       VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&y);
956:       PetscLogObjectParent(pc,x);
957:       PetscLogObjectParent(pc,y);
958:       bjac->x[i]      = x;
959:       bjac->y[i]      = y;
960:       bjac->starts[i] = start;
961:       jac->sles[i]    = sles;

963:       ISCreateStride(PETSC_COMM_SELF,m,start,1,&is);
964:       bjac->is[i] = is;
965:       PetscLogObjectParent(pc,is);

967:       start += m;
968:     }
969:   } else {
970:     bjac = (PC_BJacobi_Multiblock*)jac->data;
971:     /* 
972:        Destroy the blocks from the previous iteration
973:     */
974:     if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
975:       MatDestroyMatrices(n_local,&bjac->pmat);
976:       if (jac->use_true_local) {
977:         MatDestroyMatrices(n_local,&bjac->mat);
978:       }
979:       scall = MAT_INITIAL_MATRIX;
980:     }
981:   }

983:   MatGetSubMatrices(pmat,n_local,bjac->is,bjac->is,scall,&bjac->pmat);
984:   if (jac->use_true_local) {
985:     PetscObjectGetOptionsPrefix((PetscObject)mat,&mprefix);
986:     MatGetSubMatrices(mat,n_local,bjac->is,bjac->is,scall,&bjac->mat);
987:   }
988:   /* Return control to the user so that the submatrices can be modified (e.g., to apply
989:      different boundary conditions for the submatrices than for the global problem) */
990:   PCModifySubMatrices(pc,n_local,bjac->is,bjac->is,bjac->pmat,pc->modifysubmatricesP);

992:   PetscObjectGetOptionsPrefix((PetscObject)pmat,&pprefix);
993:   for (i=0; i<n_local; i++) {
994:     PetscLogObjectParent(pc,bjac->pmat[i]);
995:     PetscObjectSetOptionsPrefix((PetscObject)bjac->pmat[i],pprefix);
996:     if (jac->use_true_local) {
997:       PetscLogObjectParent(pc,bjac->mat[i]);
998:       PetscObjectSetOptionsPrefix((PetscObject)bjac->mat[i],mprefix);
999:       SLESSetOperators(jac->sles[i],bjac->mat[i],bjac->pmat[i],pc->flag);
1000:     } else {
1001:       SLESSetOperators(jac->sles[i],bjac->pmat[i],bjac->pmat[i],pc->flag);
1002:     }
1003:   }

1005:   return(0);
1006: }