Actual source code: fdda.c

  1: /*$Id: fdda.c,v 1.75 2001/08/07 21:31:51 bsmith Exp $*/
  2: 
 3:  #include src/dm/da/daimpl.h
 4:  #include petscmat.h


  7: EXTERN int DAGetColoring1d_MPIAIJ(DA,ISColoringType,ISColoring *);
  8: EXTERN int DAGetColoring2d_MPIAIJ(DA,ISColoringType,ISColoring *);
  9: EXTERN int DAGetColoring2d_5pt_MPIAIJ(DA,ISColoringType,ISColoring *);
 10: EXTERN int DAGetColoring3d_MPIAIJ(DA,ISColoringType,ISColoring *);

 12: /*@C
 13:     DAGetColoring - Gets the coloring required for computing the Jacobian via
 14:     finite differences on a function defined using a stencil on the DA.

 16:     Collective on DA

 18:     Input Parameter:
 19: +   da - the distributed array
 20: -   ctype - IS_COLORING_LOCAL or IS_COLORING_GHOSTED

 22:     Output Parameters:
 23: .   coloring - matrix coloring for use in computing Jacobians (or PETSC_NULL if not needed)

 25:     Level: advanced

 27:     Notes: These compute the graph coloring of the graph of A^{T}A. The coloring used 
 28:    for efficient (parallel or thread based) triangular solves etc is NOT yet 
 29:    available. 


 32: .seealso ISColoringView(), ISColoringGetIS(), MatFDColoringCreate(), ISColoringType, ISColoring

 34: @*/
 35: int DAGetColoring(DA da,ISColoringType ctype,ISColoring *coloring)
 36: {
 37:   int        ierr,dim;

 40:   /*
 41:                                   m
 42:           ------------------------------------------------------
 43:          |                                                     |
 44:          |                                                     |
 45:          |               ----------------------                |
 46:          |               |                    |                |
 47:       n  |           yn  |                    |                |
 48:          |               |                    |                |
 49:          |               .---------------------                |
 50:          |             (xs,ys)     xn                          |
 51:          |            .                                        |
 52:          |         (gxs,gys)                                   |
 53:          |                                                     |
 54:           -----------------------------------------------------
 55:   */

 57:   /*     
 58:          nc - number of components per grid point 
 59:          col - number of colors needed in one direction for single component problem
 60:   
 61:   */
 62:   DAGetInfo(da,&dim,0,0,0,0,0,0,0,0,0,0);

 64:   /*
 65:      We do not provide a getcoloring function in the DA operations because 
 66:    the basic DA does not know about matrices. We think of DA as being more 
 67:    more low-level then matrices.
 68:   */
 69:   if (dim == 1) {
 70:     DAGetColoring1d_MPIAIJ(da,ctype,coloring);
 71:   } else if (dim == 2) {
 72:      DAGetColoring2d_MPIAIJ(da,ctype,coloring);
 73:   } else if (dim == 3) {
 74:      DAGetColoring3d_MPIAIJ(da,ctype,coloring);
 75:   } else {
 76:       SETERRQ1(1,"Not done for %d dimension, send us mail petsc-maint@mcs.anl.gov for code",dim);
 77:   }
 78:   return(0);
 79: }

 81: /* ---------------------------------------------------------------------------------*/

 83: int DAGetColoring2d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
 84: {
 85:   int                    ierr,xs,ys,nx,ny,*colors,i,j,ii,gxs,gys,gnx,gny;
 86:   int                    m,n,M,N,dim,w,s,k,nc,col,size;
 87:   MPI_Comm               comm;
 88:   DAPeriodicType         wrap;
 89:   DAStencilType          st;

 92:   /*     
 93:          nc - number of components per grid point 
 94:          col - number of colors needed in one direction for single component problem
 95:   
 96:   */
 97:   DAGetInfo(da,&dim,&m,&n,0,&M,&N,0,&w,&s,&wrap,&st);
 98:   nc     = w;
 99:   col    = 2*s + 1;
100:   if (DAXPeriodic(wrap) && (m % col)){
101:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisiblen
102:                  by 2*stencil_width + 1n");
103:   }
104:   if (DAYPeriodic(wrap) && (n % col)){
105:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisiblen
106:                  by 2*stencil_width + 1n");
107:   }
108:   DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
109:   DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
110:   PetscObjectGetComm((PetscObject)da,&comm);
111:   MPI_Comm_size(comm,&size);

113:   /* create the coloring */
114:   if (st == DA_STENCIL_STAR && s == 1 && !DAXPeriodic(wrap) && !DAYPeriodic(wrap)) {
115:     DAGetColoring2d_5pt_MPIAIJ(da,ctype,coloring);
116:   } else if (ctype == IS_COLORING_LOCAL) {
117:     if (!da->localcoloring) {
118:     PetscMalloc(nc*nx*ny*sizeof(int),&colors);
119:     ii = 0;
120:     for (j=ys; j<ys+ny; j++) {
121:         for (i=xs; i<xs+nx; i++) {
122:           for (k=0; k<nc; k++) {
123:             colors[ii++] = k + nc*((i % col) + col*(j % col));
124:           }
125:         }
126:       }
127:       ISColoringCreate(comm,nc*nx*ny,colors,&da->localcoloring);
128:     }
129:     *coloring = da->localcoloring;
130:   } else if (ctype == IS_COLORING_GHOSTED) {
131:     if (!da->ghostedcoloring) {
132:       PetscMalloc(nc*gnx*gny*sizeof(int),&colors);
133:       ii = 0;
134:       for (j=gys; j<gys+gny; j++) {
135:         for (i=gxs; i<gxs+gnx; i++) {
136:           for (k=0; k<nc; k++) {
137:             /* the complicated stuff is to handle periodic boundaries */
138:             colors[ii++] = k + nc*(    (((i < 0) ? m+i:((i >= m) ? i-m:i)) % col) +
139:                                      col*(((j < 0) ? n+j:((j >= n) ? j-n:j)) % col));
140:           }
141:         }
142:       }
143:       ISColoringCreate(comm,nc*gnx*gny,colors,&da->ghostedcoloring);
144:       ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
145:     }
146:     *coloring = da->ghostedcoloring;
147:   } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
148:   ISColoringReference(*coloring);
149:   return(0);
150: }

152: /* ---------------------------------------------------------------------------------*/

154: int DAGetColoring3d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
155: {
156:   int                    ierr,xs,ys,nx,ny,*colors,i,j,gxs,gys,gnx,gny;
157:   int                    m,n,p,dim,s,k,nc,col,size,zs,gzs,ii,l,nz,gnz,M,N,P;
158:   MPI_Comm               comm;
159:   DAPeriodicType         wrap;
160:   DAStencilType          st;

163:   /*     
164:          nc - number of components per grid point 
165:          col - number of colors needed in one direction for single component problem
166:   
167:   */
168:   DAGetInfo(da,&dim,&m,&n,&p,&M,&N,&P,&nc,&s,&wrap,&st);
169:   col    = 2*s + 1;
170:   if (DAXPeriodic(wrap) && (m % col)){
171:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisiblen
172:                  by 2*stencil_width + 1n");
173:   }
174:   if (DAYPeriodic(wrap) && (n % col)){
175:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisiblen
176:                  by 2*stencil_width + 1n");
177:   }
178:   if (DAZPeriodic(wrap) && (p % col)){
179:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisiblen
180:                  by 2*stencil_width + 1n");
181:   }

183:   DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
184:   DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
185:   PetscObjectGetComm((PetscObject)da,&comm);
186:   MPI_Comm_size(comm,&size);

188:   /* create the coloring */
189:   if (ctype == IS_COLORING_LOCAL) {
190:     if (!da->localcoloring) {
191:       PetscMalloc(nc*nx*ny*nz*sizeof(int),&colors);
192:       ii = 0;
193:       for (k=zs; k<zs+nz; k++) {
194:         for (j=ys; j<ys+ny; j++) {
195:           for (i=xs; i<xs+nx; i++) {
196:             for (l=0; l<nc; l++) {
197:               colors[ii++] = l + nc*((i % col) + col*(j % col) + col*col*(k % col));
198:             }
199:           }
200:         }
201:       }
202:       ISColoringCreate(comm,nc*nx*ny*nz,colors,&da->localcoloring);
203:     }
204:     *coloring = da->localcoloring;
205:   } else if (ctype == IS_COLORING_GHOSTED) {
206:     if (!da->ghostedcoloring) {
207:       PetscMalloc(nc*gnx*gny*gnz*sizeof(int),&colors);
208:       ii = 0;
209:       for (k=gzs; k<gzs+gnz; k++) {
210:         for (j=gys; j<gys+gny; j++) {
211:           for (i=gxs; i<gxs+gnx; i++) {
212:             for (l=0; l<nc; l++) {
213:               /* the complicated stuff is to handle periodic boundaries */
214:               colors[ii++] = l + nc*(        (((i < 0) ? m+i:((i >= m) ? i-m:i)) % col) +
215:                                            col*(((j < 0) ? n+j:((j >= n) ? j-n:j)) % col) +
216:                                      col*col*(((k < 0) ? p+k:((k >= p) ? k-p:k)) % col));
217:             }
218:           }
219:         }
220:       }
221:       ISColoringCreate(comm,nc*gnx*gny*gnz,colors,&da->ghostedcoloring);
222:       ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
223:     }
224:     *coloring = da->ghostedcoloring;
225:   } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
226:   ISColoringReference(*coloring);
227:   return(0);
228: }

230: /* ---------------------------------------------------------------------------------*/

232: int DAGetColoring1d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
233: {
234:   int                    ierr,xs,nx,*colors,i,i1,gxs,gnx,l;
235:   int                    m,M,dim,s,nc,col,size;
236:   MPI_Comm               comm;
237:   DAPeriodicType         wrap;

240:   /*     
241:          nc - number of components per grid point 
242:          col - number of colors needed in one direction for single component problem
243:   
244:   */
245:   DAGetInfo(da,&dim,&m,0,0,&M,0,0,&nc,&s,&wrap,0);
246:   col    = 2*s + 1;

248:   if (DAXPeriodic(wrap) && (m % col)) {
249:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points is divisiblen
250:                  by 2*stencil_width + 1n");
251:   }

253:   DAGetCorners(da,&xs,0,0,&nx,0,0);
254:   DAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);
255:   PetscObjectGetComm((PetscObject)da,&comm);
256:   MPI_Comm_size(comm,&size);

258:   /* create the coloring */
259:   if (ctype == IS_COLORING_LOCAL) {
260:     if (!da->localcoloring) {
261:       PetscMalloc(nc*nx*sizeof(int),&colors);
262:       i1 = 0;
263:       for (i=xs; i<xs+nx; i++) {
264:         for (l=0; l<nc; l++) {
265:           colors[i1++] = l + nc*(i % col);
266:         }
267:       }
268:       ISColoringCreate(comm,nc*nx,colors,&da->localcoloring);
269:     }
270:     *coloring = da->localcoloring;
271:   } else if (ctype == IS_COLORING_GHOSTED) {
272:     if (!da->ghostedcoloring) {
273:       PetscMalloc(nc*gnx*sizeof(int),&colors);
274:       i1 = 0;
275:       for (i=gxs; i<gxs+gnx; i++) {
276:         for (l=0; l<nc; l++) {
277:           /* the complicated stuff is to handle periodic boundaries */
278:           colors[i1++] = l + nc*((((i < 0) ? m+i:((i >= m) ? i-m:i)) % col));
279:         }
280:       }
281:       ISColoringCreate(comm,nc*gnx,colors,&da->ghostedcoloring);
282:       ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
283:     }
284:     *coloring = da->ghostedcoloring;
285:   } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
286:   ISColoringReference(*coloring);
287:   return(0);
288: }

290: int DAGetColoring2d_5pt_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
291: {
292:   int      ierr,xs,ys,nx,ny,*colors,i,j,ii,gxs,gys,gnx,gny;
293:   int      m,n,dim,w,s,k,nc;
294:   MPI_Comm comm;

297:   /*     
298:          nc - number of components per grid point 
299:          col - number of colors needed in one direction for single component problem
300:   
301:   */
302:   ierr   = DAGetInfo(da,&dim,&m,&n,0,0,0,0,&w,&s,0,0);
303:   nc     = w;
304:   ierr   = DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
305:   ierr   = DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
306:   ierr   = PetscObjectGetComm((PetscObject)da,&comm);

308:   /* create the coloring */
309:   if (ctype == IS_COLORING_LOCAL) {
310:     if (!da->localcoloring) {
311:       PetscMalloc(nc*nx*ny*sizeof(int),&colors);
312:       ii = 0;
313:       for (j=ys; j<ys+ny; j++) {
314:         for (i=xs; i<xs+nx; i++) {
315:           for (k=0; k<nc; k++) {
316:             colors[ii++] = k + nc*((3*j+i) % 5);
317:           }
318:         }
319:       }
320:       ISColoringCreate(comm,nc*nx*ny,colors,&da->localcoloring);
321:     }
322:     *coloring = da->localcoloring;
323:   } else if (ctype == IS_COLORING_GHOSTED) {
324:     if (!da->ghostedcoloring) {
325:       PetscMalloc(nc*gnx*gny*sizeof(int),&colors);
326:       ii = 0;
327:       for (j=gys; j<gys+gny; j++) {
328:         for (i=gxs; i<gxs+gnx; i++) {
329:           for (k=0; k<nc; k++) {
330:             colors[ii++] = k + nc*((3*j+i) % 5);
331:           }
332:         }
333:       }
334:       ISColoringCreate(comm,nc*gnx*gny,colors,&da->ghostedcoloring);
335:       ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
336:     }
337:     *coloring = da->ghostedcoloring;
338:   } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
339:   return(0);
340: }

342: /* =========================================================================== */
343: EXTERN int DAGetMatrix1d_MPIAIJ(DA,Mat *);
344: EXTERN int DAGetMatrix2d_MPIAIJ(DA,Mat *);
345: EXTERN int DAGetMatrix3d_MPIAIJ(DA,Mat *);
346: EXTERN int DAGetMatrix3d_MPIBAIJ(DA,Mat *);

348: /*@C
349:     DAGetMatrix - Gets the nonzero structure required for computing the Jacobian via
350:     finite differences on a function defined using a stencil on the DA.

352:     Collective on DA

354:     Input Parameter:
355: +   da - the distributed array
356: -   mtype - either MATMPIAIJ or MATMPIBAIJ

358:     Output Parameters:
359: .   J  - matrix with the correct nonzero structure
360:         (obviously without the correct Jacobian values)

362:     Level: advanced

364: .seealso ISColoringView(), ISColoringGetIS(), MatFDColoringCreate()

366: @*/
367: int DAGetMatrix(DA da,MatType mtype,Mat *J)
368: {
369:   int        ierr,dim;
370:   PetscTruth aij;

373:   /*
374:                                   m
375:           ------------------------------------------------------
376:          |                                                     |
377:          |                                                     |
378:          |               ----------------------                |
379:          |               |                    |                |
380:       n  |           yn  |                    |                |
381:          |               |                    |                |
382:          |               .---------------------                |
383:          |             (xs,ys)     xn                          |
384:          |            .                                        |
385:          |         (gxs,gys)                                   |
386:          |                                                     |
387:           -----------------------------------------------------
388:   */

390:   /*     
391:          nc - number of components per grid point 
392:          col - number of colors needed in one direction for single component problem
393:   
394:   */
395:   DAGetInfo(da,&dim,0,0,0,0,0,0,0,0,0,0);

397:   /*
398:      We do not provide a getcoloring function in the DA operations because 
399:    the basic DA does not know about matrices. We think of DA as being more 
400:    more low-level then matrices.
401:   */
402:   PetscStrcmp(MATMPIAIJ,mtype,&aij);
403:   if (aij) {
404:     if (dim == 1) {
405:       DAGetMatrix1d_MPIAIJ(da,J);
406:     } else if (dim == 2) {
407:        DAGetMatrix2d_MPIAIJ(da,J);
408:     } else if (dim == 3) {
409:        DAGetMatrix3d_MPIAIJ(da,J);
410:     }
411:   } else {
412:     if (dim == 3) {
413:        DAGetMatrix3d_MPIBAIJ(da,J);
414:   } else {
415:       SETERRQ1(1,"Not done for %d dimension, send us mail petsc-maint@mcs.anl.gov for code",dim);
416:     }
417:   }
418:   return(0);
419: }

421: /* ---------------------------------------------------------------------------------*/

423: int DAGetMatrix2d_MPIAIJ(DA da,Mat *J)
424: {
425:   int                    ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
426:   int                    m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p;
427:   int                    lstart,lend,pstart,pend,*dnz,*onz,size;
428:   int                    dims[2],starts[2];
429:   MPI_Comm               comm;
430:   PetscScalar            *values;
431:   DAPeriodicType         wrap;
432:   ISLocalToGlobalMapping ltog;
433:   DAStencilType          st;

436:   /*     
437:          nc - number of components per grid point 
438:          col - number of colors needed in one direction for single component problem
439:   
440:   */
441:   DAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&wrap,&st);
442:   col = 2*s + 1;
443:   if (DAXPeriodic(wrap) && (m % col)){
444:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisiblen
445:                  by 2*stencil_width + 1n");
446:   }
447:   if (DAYPeriodic(wrap) && (n % col)){
448:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisiblen
449:                  by 2*stencil_width + 1n");
450:   }
451:   DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
452:   DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
453:   PetscObjectGetComm((PetscObject)da,&comm);
454:   MPI_Comm_size(comm,&size);

456:   /* create empty Jacobian matrix */
457:   MatCreate(comm,nc*nx*ny,nc*nx*ny,PETSC_DECIDE,PETSC_DECIDE,J);

459:   PetscMalloc(col*col*nc*nc*sizeof(PetscScalar),&values);
460:   PetscMemzero(values,col*col*nc*nc*sizeof(PetscScalar));
461:   PetscMalloc(nc*sizeof(int),&rows);
462:   PetscMalloc(col*col*nc*nc*sizeof(int),&cols);
463:   DAGetISLocalToGlobalMapping(da,&ltog);
464: 
465:   /* determine the matrix preallocation information */
466:   MatPreallocateInitialize(comm,nc*nx*ny,nc*nx*ny,dnz,onz);
467:   for (i=xs; i<xs+nx; i++) {

469:     pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
470:     pend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));

472:     for (j=ys; j<ys+ny; j++) {
473:       slot = i - gxs + gnx*(j - gys);

475:       lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
476:       lend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));

478:       cnt  = 0;
479:       for (k=0; k<nc; k++) {
480:         for (l=lstart; l<lend+1; l++) {
481:           for (p=pstart; p<pend+1; p++) {
482:             if ((st == DA_STENCIL_BOX) || (!l || !p)) {  /* entries on star have either l = 0 or p = 0 */
483:               cols[cnt++]  = k + nc*(slot + gnx*l + p);
484:             }
485:           }
486:         }
487:         rows[k] = k + nc*(slot);
488:       }
489:       MatPreallocateSetLocal(ltog,nc,rows,cnt,cols,dnz,onz);
490:     }
491:   }
492:   /* set matrix type and preallocation information */
493:   if (size > 1) {
494:     MatSetType(*J,MATMPIAIJ);
495:   } else {
496:     MatSetType(*J,MATSEQAIJ);
497:   }
498:   MatSeqAIJSetPreallocation(*J,0,dnz);
499:   MatSeqBAIJSetPreallocation(*J,nc,0,dnz);
500:   MatMPIAIJSetPreallocation(*J,0,dnz,0,onz);
501:   MatMPIBAIJSetPreallocation(*J,nc,0,dnz,0,onz);
502:   MatPreallocateFinalize(dnz,onz);
503:   MatSetLocalToGlobalMapping(*J,ltog);
504:   DAGetGhostCorners(da,&starts[0],&starts[1],PETSC_IGNORE,&dims[0],&dims[1],PETSC_IGNORE);
505:   MatSetStencil(*J,2,dims,starts,nc);

507:   /*
508:     For each node in the grid: we get the neighbors in the local (on processor ordering
509:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
510:     PETSc ordering.
511:   */
512:   for (i=xs; i<xs+nx; i++) {
513: 
514:     pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
515:     pend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
516: 
517:     for (j=ys; j<ys+ny; j++) {
518:       slot = i - gxs + gnx*(j - gys);
519: 
520:       lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
521:       lend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));

523:       cnt  = 0;
524:       for (k=0; k<nc; k++) {
525:         for (l=lstart; l<lend+1; l++) {
526:           for (p=pstart; p<pend+1; p++) {
527:             if ((st == DA_STENCIL_BOX) || (!l || !p)) {  /* entries on star have either l = 0 or p = 0 */
528:               cols[cnt++]  = k + nc*(slot + gnx*l + p);
529:             }
530:           }
531:         }
532:         rows[k]      = k + nc*(slot);
533:       }
534:       MatSetValuesLocal(*J,nc,rows,cnt,cols,values,INSERT_VALUES);
535:     }
536:   }
537:   PetscFree(values);
538:   PetscFree(rows);
539:   PetscFree(cols);
540:   MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);
541:   MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);
542:   return(0);
543: }

545: /* ---------------------------------------------------------------------------------*/

547: int DAGetMatrix3d_MPIAIJ(DA da,Mat *J)
548: {
549:   int                    ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
550:   int                    m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p,*dnz,*onz;
551:   int                    istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk,size;
552:   int                    dims[3],starts[3];
553:   MPI_Comm               comm;
554:   PetscScalar            *values;
555:   DAPeriodicType         wrap;
556:   ISLocalToGlobalMapping ltog;
557:   DAStencilType          st;

560:   /*     
561:          nc - number of components per grid point 
562:          col - number of colors needed in one direction for single component problem
563:   
564:   */
565:   DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
566:   col    = 2*s + 1;
567:   if (DAXPeriodic(wrap) && (m % col)){
568:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisiblen
569:                  by 2*stencil_width + 1n");
570:   }
571:   if (DAYPeriodic(wrap) && (n % col)){
572:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisiblen
573:                  by 2*stencil_width + 1n");
574:   }
575:   if (DAZPeriodic(wrap) && (p % col)){
576:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisiblen
577:                  by 2*stencil_width + 1n");
578:   }

580:   DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
581:   DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
582:   PetscObjectGetComm((PetscObject)da,&comm);
583:   MPI_Comm_size(comm,&size);


586:   /* create the matrix */
587:   /* create empty Jacobian matrix */
588:   MatCreate(comm,nc*nx*ny*nz,nc*nx*ny*nz,PETSC_DECIDE,PETSC_DECIDE,J);
589:   PetscMalloc(col*col*col*nc*nc*nc*sizeof(PetscScalar),&values);
590:   PetscMemzero(values,col*col*col*nc*nc*nc*sizeof(PetscScalar));
591:   PetscMalloc(nc*sizeof(int),&rows);
592:   PetscMalloc(col*col*col*nc*sizeof(int),&cols);
593:   DAGetISLocalToGlobalMapping(da,&ltog);

595:   /* determine the matrix preallocation information */
596:   MatPreallocateInitialize(comm,nc*nx*ny*nz,nc*nx*ny*nz,dnz,onz);
597:   for (i=xs; i<xs+nx; i++) {
598:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
599:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
600:     for (j=ys; j<ys+ny; j++) {
601:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
602:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
603:       for (k=zs; k<zs+nz; k++) {
604:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
605:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
606: 
607:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
608: 
609:         cnt  = 0;
610:         for (l=0; l<nc; l++) {
611:           for (ii=istart; ii<iend+1; ii++) {
612:             for (jj=jstart; jj<jend+1; jj++) {
613:               for (kk=kstart; kk<kend+1; kk++) {
614:                 if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
615:                   cols[cnt++]  = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
616:                 }
617:               }
618:             }
619:           }
620:           rows[l] = l + nc*(slot);
621:         }
622:         MatPreallocateSetLocal(ltog,nc,rows,cnt,cols,dnz,onz);
623:       }
624:     }
625:   }
626:   /* set matrix type and preallocation */
627:   if (size > 1) {
628:     MatSetType(*J,MATMPIAIJ);
629:   } else {
630:     MatSetType(*J,MATSEQAIJ);
631:   }
632:   MatSeqAIJSetPreallocation(*J,0,dnz);
633:   MatSeqBAIJSetPreallocation(*J,nc,0,dnz);
634:   MatMPIAIJSetPreallocation(*J,0,dnz,0,onz);
635:   MatMPIBAIJSetPreallocation(*J,nc,0,dnz,0,onz);
636:   MatPreallocateFinalize(dnz,onz);
637:   MatSetLocalToGlobalMapping(*J,ltog);
638:   DAGetGhostCorners(da,&starts[0],&starts[1],&starts[2],&dims[0],&dims[1],&dims[2]);
639:   MatSetStencil(*J,3,dims,starts,nc);

641:   /*
642:     For each node in the grid: we get the neighbors in the local (on processor ordering
643:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
644:     PETSc ordering.
645:   */
646:   for (i=xs; i<xs+nx; i++) {
647:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
648:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
649:     for (j=ys; j<ys+ny; j++) {
650:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
651:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
652:       for (k=zs; k<zs+nz; k++) {
653:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
654:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
655: 
656:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
657: 
658:         cnt  = 0;
659:         for (l=0; l<nc; l++) {
660:           for (ii=istart; ii<iend+1; ii++) {
661:             for (jj=jstart; jj<jend+1; jj++) {
662:               for (kk=kstart; kk<kend+1; kk++) {
663:                 if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
664:                   cols[cnt++]  = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
665:                 }
666:               }
667:             }
668:           }
669:           rows[l]      = l + nc*(slot);
670:         }
671:         MatSetValuesLocal(*J,nc,rows,cnt,cols,values,INSERT_VALUES);
672:       }
673:     }
674:   }
675:   PetscFree(values);
676:   PetscFree(rows);
677:   PetscFree(cols);
678:   MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);
679:   MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);
680:   return(0);
681: }

683: /* ---------------------------------------------------------------------------------*/

685: int DAGetMatrix1d_MPIAIJ(DA da,Mat *J)
686: {
687:   int                    ierr,xs,nx,i,i1,slot,gxs,gnx;
688:   int                    m,dim,s,*cols,nc,*rows,col,cnt,l;
689:   int                    istart,iend,size;
690:   int                    dims[1],starts[1];
691:   MPI_Comm               comm;
692:   PetscScalar            *values;
693:   DAPeriodicType         wrap;
694:   ISLocalToGlobalMapping ltog;

697:   /*     
698:          nc - number of components per grid point 
699:          col - number of colors needed in one direction for single component problem
700:   
701:   */
702:   DAGetInfo(da,&dim,&m,0,0,0,0,0,&nc,&s,&wrap,0);
703:   col    = 2*s + 1;

705:   if (DAXPeriodic(wrap) && (m % col)) {
706:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points is divisiblen
707:                  by 2*stencil_width + 1n");
708:   }


711:   DAGetCorners(da,&xs,0,0,&nx,0,0);
712:   DAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);
713:   PetscObjectGetComm((PetscObject)da,&comm);
714:   MPI_Comm_size(comm,&size);

716:   /* create empty Jacobian matrix */
717: 
718:   ierr    = MatCreate(comm,nc*nx,nc*nx,PETSC_DECIDE,PETSC_DECIDE,J);
719:   if (size > 1) {
720:     MatSetType(*J,MATMPIAIJ);
721:   } else {
722:     MatSetType(*J,MATSEQAIJ);
723:   }
724:   MatSeqAIJSetPreallocation(*J,col*nc,0);
725:   MatSeqBAIJSetPreallocation(*J,nc,col,0);
726:   MatMPIAIJSetPreallocation(*J,col*nc,0,0,0);
727:   MatMPIBAIJSetPreallocation(*J,nc,col,0,0,0);
728:   DAGetGhostCorners(da,&starts[0],PETSC_IGNORE,PETSC_IGNORE,&dims[0],PETSC_IGNORE,PETSC_IGNORE);
729:   MatSetStencil(*J,1,dims,starts,nc);
730: 
731:   PetscMalloc(col*nc*nc*sizeof(PetscScalar),&values);
732:   PetscMemzero(values,col*nc*nc*sizeof(PetscScalar));
733:   PetscMalloc(nc*sizeof(int),&rows);
734:   PetscMalloc(col*nc*sizeof(int),&cols);
735: 
736:   DAGetISLocalToGlobalMapping(da,&ltog);
737:   MatSetLocalToGlobalMapping(*J,ltog);
738: 
739:   /*
740:     For each node in the grid: we get the neighbors in the local (on processor ordering
741:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
742:     PETSc ordering.
743:   */
744:   for (i=xs; i<xs+nx; i++) {
745:     istart = PetscMax(-s,gxs - i);
746:     iend   = PetscMin(s,gxs + gnx - i - 1);
747:     slot   = i - gxs;
748: 
749:     cnt  = 0;
750:     for (l=0; l<nc; l++) {
751:       for (i1=istart; i1<iend+1; i1++) {
752:         cols[cnt++] = l + nc*(slot + i1);
753:       }
754:       rows[l]      = l + nc*(slot);
755:     }
756:     MatSetValuesLocal(*J,nc,rows,cnt,cols,values,INSERT_VALUES);
757:   }
758:   PetscFree(values);
759:   PetscFree(rows);
760:   PetscFree(cols);
761:   MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);
762:   MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);
763:   return(0);
764: }

766: int DAGetMatrix3d_MPIBAIJ(DA da,Mat *J)
767: {
768:   int                    ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
769:   int                    m,n,dim,s,*cols,k,nc,col,cnt,p,*dnz,*onz;
770:   int                    istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
771:   MPI_Comm               comm;
772:   PetscScalar            *values;
773:   DAPeriodicType         wrap;
774:   DAStencilType          st;
775:   ISLocalToGlobalMapping ltog;

778:   /*     
779:          nc - number of components per grid point 
780:          col - number of colors needed in one direction for single component problem
781:   
782:   */
783:   DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
784:   if (wrap != DA_NONPERIODIC) SETERRQ(PETSC_ERR_SUP,"Currently no support for periodic");
785:   col    = 2*s + 1;

787:   DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
788:   DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
789:   PetscObjectGetComm((PetscObject)da,&comm);

791:   /* create the matrix */
792:   ierr  = PetscMalloc(col*col*col*nc*nc*sizeof(PetscScalar),&values);
793:   ierr  = PetscMemzero(values,col*col*col*nc*nc*sizeof(PetscScalar));
794:   ierr  = PetscMalloc(col*col*col*sizeof(int),&cols);

796:   DAGetISLocalToGlobalMappingBlck(da,&ltog);

798:   /* determine the matrix preallocation information */
799:   MatPreallocateInitialize(comm,nx*ny*nz,nx*ny*nz,dnz,onz);
800:   for (i=xs; i<xs+nx; i++) {
801:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
802:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
803:     for (j=ys; j<ys+ny; j++) {
804:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
805:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
806:       for (k=zs; k<zs+nz; k++) {
807:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
808:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));

810:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);

812:         /* Find block columns in block row */
813:         cnt  = 0;
814:         if (st == DA_STENCIL_BOX) {   /* if using BOX stencil */
815:           for (ii=istart; ii<iend+1; ii++) {
816:             for (jj=jstart; jj<jend+1; jj++) {
817:               for (kk=kstart; kk<kend+1; kk++) {
818:                 cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk;
819:               }
820:             }
821:           }
822:         } else {  /* Star stencil */
823:           cnt  = 0;
824:           for (ii=istart; ii<iend+1; ii++) {
825:             if (ii) {
826:               /* jj and kk must be zero */
827:               /* cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk; */
828:               cols[cnt++]  = slot + ii;
829:             } else {
830:               for (jj=jstart; jj<jend+1; jj++) {
831:                 if (jj) {
832:                   /* ii and kk must be zero */
833:                   cols[cnt++]  = slot + gnx*jj;
834:                 } else {
835:                   /* ii and jj must be zero */
836:                   for (kk=kstart; kk<kend+1; kk++) {
837:                     cols[cnt++]  = slot + gnx*gny*kk;
838:                   }
839:                 }
840:               }
841:             }
842:           }
843:         }
844:         MatPreallocateSetLocal(ltog,1,&slot,cnt,cols,dnz,onz);
845:       }
846:     }
847:   }

849:   /* create empty Jacobian matrix */
850:   MatCreateMPIBAIJ(comm,nc,nc*nx*ny*nz,nc*nx*ny*nz,PETSC_DECIDE,PETSC_DECIDE,0,dnz,0,onz,J);

852:   MatPreallocateFinalize(dnz,onz);
853:   MatSetLocalToGlobalMappingBlock(*J,ltog);

855:   /*
856:     For each node in the grid: we get the neighbors in the local (on processor ordering
857:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
858:     PETSc ordering.
859:   */

861:   for (i=xs; i<xs+nx; i++) {
862:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
863:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
864:     for (j=ys; j<ys+ny; j++) {
865:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
866:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
867:       for (k=zs; k<zs+nz; k++) {
868:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
869:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
870: 
871:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
872: 
873:         cnt  = 0;
874:         if (st == DA_STENCIL_BOX) {   /* if using BOX stencil */
875:           for (ii=istart; ii<iend+1; ii++) {
876:             for (jj=jstart; jj<jend+1; jj++) {
877:               for (kk=kstart; kk<kend+1; kk++) {
878:                 cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk;
879:               }
880:             }
881:           }
882:         } else {  /* Star stencil */
883:           cnt  = 0;
884:           for (ii=istart; ii<iend+1; ii++) {
885:             if (ii) {
886:               /* jj and kk must be zero */
887:               cols[cnt++]  = slot + ii;
888:             } else {
889:               for (jj=jstart; jj<jend+1; jj++) {
890:                 if (jj) {
891:                   /* ii and kk must be zero */
892:                   cols[cnt++]  = slot + gnx*jj;
893:                 } else {
894:                   /* ii and jj must be zero */
895:                   for (kk=kstart; kk<kend+1; kk++) {
896:                     cols[cnt++]  = slot + gnx*gny*kk;
897:                   }
898:                 }
899:               }
900:             }
901:           }
902:         }
903:         MatSetValuesBlockedLocal(*J,1,&slot,cnt,cols,values,INSERT_VALUES);
904:       }
905:     }
906:   }
907:   PetscFree(values);
908:   PetscFree(cols);
909:   MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);
910:   MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);
911:   return(0);
912: }