Actual source code: fdda.c
1: /*$Id: fdda.c,v 1.75 2001/08/07 21:31:51 bsmith Exp $*/
2:
3: #include src/dm/da/daimpl.h
4: #include petscmat.h
7: EXTERN int DAGetColoring1d_MPIAIJ(DA,ISColoringType,ISColoring *);
8: EXTERN int DAGetColoring2d_MPIAIJ(DA,ISColoringType,ISColoring *);
9: EXTERN int DAGetColoring2d_5pt_MPIAIJ(DA,ISColoringType,ISColoring *);
10: EXTERN int DAGetColoring3d_MPIAIJ(DA,ISColoringType,ISColoring *);
12: /*@C
13: DAGetColoring - Gets the coloring required for computing the Jacobian via
14: finite differences on a function defined using a stencil on the DA.
16: Collective on DA
18: Input Parameter:
19: + da - the distributed array
20: - ctype - IS_COLORING_LOCAL or IS_COLORING_GHOSTED
22: Output Parameters:
23: . coloring - matrix coloring for use in computing Jacobians (or PETSC_NULL if not needed)
25: Level: advanced
27: Notes: These compute the graph coloring of the graph of A^{T}A. The coloring used
28: for efficient (parallel or thread based) triangular solves etc is NOT yet
29: available.
32: .seealso ISColoringView(), ISColoringGetIS(), MatFDColoringCreate(), ISColoringType, ISColoring
34: @*/
35: int DAGetColoring(DA da,ISColoringType ctype,ISColoring *coloring)
36: {
37: int ierr,dim;
40: /*
41: m
42: ------------------------------------------------------
43: | |
44: | |
45: | ---------------------- |
46: | | | |
47: n | yn | | |
48: | | | |
49: | .--------------------- |
50: | (xs,ys) xn |
51: | . |
52: | (gxs,gys) |
53: | |
54: -----------------------------------------------------
55: */
57: /*
58: nc - number of components per grid point
59: col - number of colors needed in one direction for single component problem
60:
61: */
62: DAGetInfo(da,&dim,0,0,0,0,0,0,0,0,0,0);
64: /*
65: We do not provide a getcoloring function in the DA operations because
66: the basic DA does not know about matrices. We think of DA as being more
67: more low-level then matrices.
68: */
69: if (dim == 1) {
70: DAGetColoring1d_MPIAIJ(da,ctype,coloring);
71: } else if (dim == 2) {
72: DAGetColoring2d_MPIAIJ(da,ctype,coloring);
73: } else if (dim == 3) {
74: DAGetColoring3d_MPIAIJ(da,ctype,coloring);
75: } else {
76: SETERRQ1(1,"Not done for %d dimension, send us mail petsc-maint@mcs.anl.gov for code",dim);
77: }
78: return(0);
79: }
81: /* ---------------------------------------------------------------------------------*/
83: int DAGetColoring2d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
84: {
85: int ierr,xs,ys,nx,ny,*colors,i,j,ii,gxs,gys,gnx,gny;
86: int m,n,M,N,dim,w,s,k,nc,col,size;
87: MPI_Comm comm;
88: DAPeriodicType wrap;
89: DAStencilType st;
92: /*
93: nc - number of components per grid point
94: col - number of colors needed in one direction for single component problem
95:
96: */
97: DAGetInfo(da,&dim,&m,&n,0,&M,&N,0,&w,&s,&wrap,&st);
98: nc = w;
99: col = 2*s + 1;
100: if (DAXPeriodic(wrap) && (m % col)){
101: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisiblen
102: by 2*stencil_width + 1n");
103: }
104: if (DAYPeriodic(wrap) && (n % col)){
105: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisiblen
106: by 2*stencil_width + 1n");
107: }
108: DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
109: DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
110: PetscObjectGetComm((PetscObject)da,&comm);
111: MPI_Comm_size(comm,&size);
113: /* create the coloring */
114: if (st == DA_STENCIL_STAR && s == 1 && !DAXPeriodic(wrap) && !DAYPeriodic(wrap)) {
115: DAGetColoring2d_5pt_MPIAIJ(da,ctype,coloring);
116: } else if (ctype == IS_COLORING_LOCAL) {
117: if (!da->localcoloring) {
118: PetscMalloc(nc*nx*ny*sizeof(int),&colors);
119: ii = 0;
120: for (j=ys; j<ys+ny; j++) {
121: for (i=xs; i<xs+nx; i++) {
122: for (k=0; k<nc; k++) {
123: colors[ii++] = k + nc*((i % col) + col*(j % col));
124: }
125: }
126: }
127: ISColoringCreate(comm,nc*nx*ny,colors,&da->localcoloring);
128: }
129: *coloring = da->localcoloring;
130: } else if (ctype == IS_COLORING_GHOSTED) {
131: if (!da->ghostedcoloring) {
132: PetscMalloc(nc*gnx*gny*sizeof(int),&colors);
133: ii = 0;
134: for (j=gys; j<gys+gny; j++) {
135: for (i=gxs; i<gxs+gnx; i++) {
136: for (k=0; k<nc; k++) {
137: /* the complicated stuff is to handle periodic boundaries */
138: colors[ii++] = k + nc*( (((i < 0) ? m+i:((i >= m) ? i-m:i)) % col) +
139: col*(((j < 0) ? n+j:((j >= n) ? j-n:j)) % col));
140: }
141: }
142: }
143: ISColoringCreate(comm,nc*gnx*gny,colors,&da->ghostedcoloring);
144: ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
145: }
146: *coloring = da->ghostedcoloring;
147: } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
148: ISColoringReference(*coloring);
149: return(0);
150: }
152: /* ---------------------------------------------------------------------------------*/
154: int DAGetColoring3d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
155: {
156: int ierr,xs,ys,nx,ny,*colors,i,j,gxs,gys,gnx,gny;
157: int m,n,p,dim,s,k,nc,col,size,zs,gzs,ii,l,nz,gnz,M,N,P;
158: MPI_Comm comm;
159: DAPeriodicType wrap;
160: DAStencilType st;
163: /*
164: nc - number of components per grid point
165: col - number of colors needed in one direction for single component problem
166:
167: */
168: DAGetInfo(da,&dim,&m,&n,&p,&M,&N,&P,&nc,&s,&wrap,&st);
169: col = 2*s + 1;
170: if (DAXPeriodic(wrap) && (m % col)){
171: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisiblen
172: by 2*stencil_width + 1n");
173: }
174: if (DAYPeriodic(wrap) && (n % col)){
175: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisiblen
176: by 2*stencil_width + 1n");
177: }
178: if (DAZPeriodic(wrap) && (p % col)){
179: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisiblen
180: by 2*stencil_width + 1n");
181: }
183: DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
184: DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
185: PetscObjectGetComm((PetscObject)da,&comm);
186: MPI_Comm_size(comm,&size);
188: /* create the coloring */
189: if (ctype == IS_COLORING_LOCAL) {
190: if (!da->localcoloring) {
191: PetscMalloc(nc*nx*ny*nz*sizeof(int),&colors);
192: ii = 0;
193: for (k=zs; k<zs+nz; k++) {
194: for (j=ys; j<ys+ny; j++) {
195: for (i=xs; i<xs+nx; i++) {
196: for (l=0; l<nc; l++) {
197: colors[ii++] = l + nc*((i % col) + col*(j % col) + col*col*(k % col));
198: }
199: }
200: }
201: }
202: ISColoringCreate(comm,nc*nx*ny*nz,colors,&da->localcoloring);
203: }
204: *coloring = da->localcoloring;
205: } else if (ctype == IS_COLORING_GHOSTED) {
206: if (!da->ghostedcoloring) {
207: PetscMalloc(nc*gnx*gny*gnz*sizeof(int),&colors);
208: ii = 0;
209: for (k=gzs; k<gzs+gnz; k++) {
210: for (j=gys; j<gys+gny; j++) {
211: for (i=gxs; i<gxs+gnx; i++) {
212: for (l=0; l<nc; l++) {
213: /* the complicated stuff is to handle periodic boundaries */
214: colors[ii++] = l + nc*( (((i < 0) ? m+i:((i >= m) ? i-m:i)) % col) +
215: col*(((j < 0) ? n+j:((j >= n) ? j-n:j)) % col) +
216: col*col*(((k < 0) ? p+k:((k >= p) ? k-p:k)) % col));
217: }
218: }
219: }
220: }
221: ISColoringCreate(comm,nc*gnx*gny*gnz,colors,&da->ghostedcoloring);
222: ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
223: }
224: *coloring = da->ghostedcoloring;
225: } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
226: ISColoringReference(*coloring);
227: return(0);
228: }
230: /* ---------------------------------------------------------------------------------*/
232: int DAGetColoring1d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
233: {
234: int ierr,xs,nx,*colors,i,i1,gxs,gnx,l;
235: int m,M,dim,s,nc,col,size;
236: MPI_Comm comm;
237: DAPeriodicType wrap;
240: /*
241: nc - number of components per grid point
242: col - number of colors needed in one direction for single component problem
243:
244: */
245: DAGetInfo(da,&dim,&m,0,0,&M,0,0,&nc,&s,&wrap,0);
246: col = 2*s + 1;
248: if (DAXPeriodic(wrap) && (m % col)) {
249: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points is divisiblen
250: by 2*stencil_width + 1n");
251: }
253: DAGetCorners(da,&xs,0,0,&nx,0,0);
254: DAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);
255: PetscObjectGetComm((PetscObject)da,&comm);
256: MPI_Comm_size(comm,&size);
258: /* create the coloring */
259: if (ctype == IS_COLORING_LOCAL) {
260: if (!da->localcoloring) {
261: PetscMalloc(nc*nx*sizeof(int),&colors);
262: i1 = 0;
263: for (i=xs; i<xs+nx; i++) {
264: for (l=0; l<nc; l++) {
265: colors[i1++] = l + nc*(i % col);
266: }
267: }
268: ISColoringCreate(comm,nc*nx,colors,&da->localcoloring);
269: }
270: *coloring = da->localcoloring;
271: } else if (ctype == IS_COLORING_GHOSTED) {
272: if (!da->ghostedcoloring) {
273: PetscMalloc(nc*gnx*sizeof(int),&colors);
274: i1 = 0;
275: for (i=gxs; i<gxs+gnx; i++) {
276: for (l=0; l<nc; l++) {
277: /* the complicated stuff is to handle periodic boundaries */
278: colors[i1++] = l + nc*((((i < 0) ? m+i:((i >= m) ? i-m:i)) % col));
279: }
280: }
281: ISColoringCreate(comm,nc*gnx,colors,&da->ghostedcoloring);
282: ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
283: }
284: *coloring = da->ghostedcoloring;
285: } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
286: ISColoringReference(*coloring);
287: return(0);
288: }
290: int DAGetColoring2d_5pt_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
291: {
292: int ierr,xs,ys,nx,ny,*colors,i,j,ii,gxs,gys,gnx,gny;
293: int m,n,dim,w,s,k,nc;
294: MPI_Comm comm;
297: /*
298: nc - number of components per grid point
299: col - number of colors needed in one direction for single component problem
300:
301: */
302: ierr = DAGetInfo(da,&dim,&m,&n,0,0,0,0,&w,&s,0,0);
303: nc = w;
304: ierr = DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
305: ierr = DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
306: ierr = PetscObjectGetComm((PetscObject)da,&comm);
308: /* create the coloring */
309: if (ctype == IS_COLORING_LOCAL) {
310: if (!da->localcoloring) {
311: PetscMalloc(nc*nx*ny*sizeof(int),&colors);
312: ii = 0;
313: for (j=ys; j<ys+ny; j++) {
314: for (i=xs; i<xs+nx; i++) {
315: for (k=0; k<nc; k++) {
316: colors[ii++] = k + nc*((3*j+i) % 5);
317: }
318: }
319: }
320: ISColoringCreate(comm,nc*nx*ny,colors,&da->localcoloring);
321: }
322: *coloring = da->localcoloring;
323: } else if (ctype == IS_COLORING_GHOSTED) {
324: if (!da->ghostedcoloring) {
325: PetscMalloc(nc*gnx*gny*sizeof(int),&colors);
326: ii = 0;
327: for (j=gys; j<gys+gny; j++) {
328: for (i=gxs; i<gxs+gnx; i++) {
329: for (k=0; k<nc; k++) {
330: colors[ii++] = k + nc*((3*j+i) % 5);
331: }
332: }
333: }
334: ISColoringCreate(comm,nc*gnx*gny,colors,&da->ghostedcoloring);
335: ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
336: }
337: *coloring = da->ghostedcoloring;
338: } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
339: return(0);
340: }
342: /* =========================================================================== */
343: EXTERN int DAGetMatrix1d_MPIAIJ(DA,Mat *);
344: EXTERN int DAGetMatrix2d_MPIAIJ(DA,Mat *);
345: EXTERN int DAGetMatrix3d_MPIAIJ(DA,Mat *);
346: EXTERN int DAGetMatrix3d_MPIBAIJ(DA,Mat *);
348: /*@C
349: DAGetMatrix - Gets the nonzero structure required for computing the Jacobian via
350: finite differences on a function defined using a stencil on the DA.
352: Collective on DA
354: Input Parameter:
355: + da - the distributed array
356: - mtype - either MATMPIAIJ or MATMPIBAIJ
358: Output Parameters:
359: . J - matrix with the correct nonzero structure
360: (obviously without the correct Jacobian values)
362: Level: advanced
364: .seealso ISColoringView(), ISColoringGetIS(), MatFDColoringCreate()
366: @*/
367: int DAGetMatrix(DA da,MatType mtype,Mat *J)
368: {
369: int ierr,dim;
370: PetscTruth aij;
373: /*
374: m
375: ------------------------------------------------------
376: | |
377: | |
378: | ---------------------- |
379: | | | |
380: n | yn | | |
381: | | | |
382: | .--------------------- |
383: | (xs,ys) xn |
384: | . |
385: | (gxs,gys) |
386: | |
387: -----------------------------------------------------
388: */
390: /*
391: nc - number of components per grid point
392: col - number of colors needed in one direction for single component problem
393:
394: */
395: DAGetInfo(da,&dim,0,0,0,0,0,0,0,0,0,0);
397: /*
398: We do not provide a getcoloring function in the DA operations because
399: the basic DA does not know about matrices. We think of DA as being more
400: more low-level then matrices.
401: */
402: PetscStrcmp(MATMPIAIJ,mtype,&aij);
403: if (aij) {
404: if (dim == 1) {
405: DAGetMatrix1d_MPIAIJ(da,J);
406: } else if (dim == 2) {
407: DAGetMatrix2d_MPIAIJ(da,J);
408: } else if (dim == 3) {
409: DAGetMatrix3d_MPIAIJ(da,J);
410: }
411: } else {
412: if (dim == 3) {
413: DAGetMatrix3d_MPIBAIJ(da,J);
414: } else {
415: SETERRQ1(1,"Not done for %d dimension, send us mail petsc-maint@mcs.anl.gov for code",dim);
416: }
417: }
418: return(0);
419: }
421: /* ---------------------------------------------------------------------------------*/
423: int DAGetMatrix2d_MPIAIJ(DA da,Mat *J)
424: {
425: int ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
426: int m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p;
427: int lstart,lend,pstart,pend,*dnz,*onz,size;
428: int dims[2],starts[2];
429: MPI_Comm comm;
430: PetscScalar *values;
431: DAPeriodicType wrap;
432: ISLocalToGlobalMapping ltog;
433: DAStencilType st;
436: /*
437: nc - number of components per grid point
438: col - number of colors needed in one direction for single component problem
439:
440: */
441: DAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&wrap,&st);
442: col = 2*s + 1;
443: if (DAXPeriodic(wrap) && (m % col)){
444: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisiblen
445: by 2*stencil_width + 1n");
446: }
447: if (DAYPeriodic(wrap) && (n % col)){
448: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisiblen
449: by 2*stencil_width + 1n");
450: }
451: DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
452: DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
453: PetscObjectGetComm((PetscObject)da,&comm);
454: MPI_Comm_size(comm,&size);
456: /* create empty Jacobian matrix */
457: MatCreate(comm,nc*nx*ny,nc*nx*ny,PETSC_DECIDE,PETSC_DECIDE,J);
459: PetscMalloc(col*col*nc*nc*sizeof(PetscScalar),&values);
460: PetscMemzero(values,col*col*nc*nc*sizeof(PetscScalar));
461: PetscMalloc(nc*sizeof(int),&rows);
462: PetscMalloc(col*col*nc*nc*sizeof(int),&cols);
463: DAGetISLocalToGlobalMapping(da,<og);
464:
465: /* determine the matrix preallocation information */
466: MatPreallocateInitialize(comm,nc*nx*ny,nc*nx*ny,dnz,onz);
467: for (i=xs; i<xs+nx; i++) {
469: pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
470: pend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
472: for (j=ys; j<ys+ny; j++) {
473: slot = i - gxs + gnx*(j - gys);
475: lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
476: lend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
478: cnt = 0;
479: for (k=0; k<nc; k++) {
480: for (l=lstart; l<lend+1; l++) {
481: for (p=pstart; p<pend+1; p++) {
482: if ((st == DA_STENCIL_BOX) || (!l || !p)) { /* entries on star have either l = 0 or p = 0 */
483: cols[cnt++] = k + nc*(slot + gnx*l + p);
484: }
485: }
486: }
487: rows[k] = k + nc*(slot);
488: }
489: MatPreallocateSetLocal(ltog,nc,rows,cnt,cols,dnz,onz);
490: }
491: }
492: /* set matrix type and preallocation information */
493: if (size > 1) {
494: MatSetType(*J,MATMPIAIJ);
495: } else {
496: MatSetType(*J,MATSEQAIJ);
497: }
498: MatSeqAIJSetPreallocation(*J,0,dnz);
499: MatSeqBAIJSetPreallocation(*J,nc,0,dnz);
500: MatMPIAIJSetPreallocation(*J,0,dnz,0,onz);
501: MatMPIBAIJSetPreallocation(*J,nc,0,dnz,0,onz);
502: MatPreallocateFinalize(dnz,onz);
503: MatSetLocalToGlobalMapping(*J,ltog);
504: DAGetGhostCorners(da,&starts[0],&starts[1],PETSC_IGNORE,&dims[0],&dims[1],PETSC_IGNORE);
505: MatSetStencil(*J,2,dims,starts,nc);
507: /*
508: For each node in the grid: we get the neighbors in the local (on processor ordering
509: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
510: PETSc ordering.
511: */
512: for (i=xs; i<xs+nx; i++) {
513:
514: pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
515: pend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
516:
517: for (j=ys; j<ys+ny; j++) {
518: slot = i - gxs + gnx*(j - gys);
519:
520: lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
521: lend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
523: cnt = 0;
524: for (k=0; k<nc; k++) {
525: for (l=lstart; l<lend+1; l++) {
526: for (p=pstart; p<pend+1; p++) {
527: if ((st == DA_STENCIL_BOX) || (!l || !p)) { /* entries on star have either l = 0 or p = 0 */
528: cols[cnt++] = k + nc*(slot + gnx*l + p);
529: }
530: }
531: }
532: rows[k] = k + nc*(slot);
533: }
534: MatSetValuesLocal(*J,nc,rows,cnt,cols,values,INSERT_VALUES);
535: }
536: }
537: PetscFree(values);
538: PetscFree(rows);
539: PetscFree(cols);
540: MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);
541: MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);
542: return(0);
543: }
545: /* ---------------------------------------------------------------------------------*/
547: int DAGetMatrix3d_MPIAIJ(DA da,Mat *J)
548: {
549: int ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
550: int m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p,*dnz,*onz;
551: int istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk,size;
552: int dims[3],starts[3];
553: MPI_Comm comm;
554: PetscScalar *values;
555: DAPeriodicType wrap;
556: ISLocalToGlobalMapping ltog;
557: DAStencilType st;
560: /*
561: nc - number of components per grid point
562: col - number of colors needed in one direction for single component problem
563:
564: */
565: DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
566: col = 2*s + 1;
567: if (DAXPeriodic(wrap) && (m % col)){
568: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisiblen
569: by 2*stencil_width + 1n");
570: }
571: if (DAYPeriodic(wrap) && (n % col)){
572: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisiblen
573: by 2*stencil_width + 1n");
574: }
575: if (DAZPeriodic(wrap) && (p % col)){
576: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisiblen
577: by 2*stencil_width + 1n");
578: }
580: DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
581: DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
582: PetscObjectGetComm((PetscObject)da,&comm);
583: MPI_Comm_size(comm,&size);
586: /* create the matrix */
587: /* create empty Jacobian matrix */
588: MatCreate(comm,nc*nx*ny*nz,nc*nx*ny*nz,PETSC_DECIDE,PETSC_DECIDE,J);
589: PetscMalloc(col*col*col*nc*nc*nc*sizeof(PetscScalar),&values);
590: PetscMemzero(values,col*col*col*nc*nc*nc*sizeof(PetscScalar));
591: PetscMalloc(nc*sizeof(int),&rows);
592: PetscMalloc(col*col*col*nc*sizeof(int),&cols);
593: DAGetISLocalToGlobalMapping(da,<og);
595: /* determine the matrix preallocation information */
596: MatPreallocateInitialize(comm,nc*nx*ny*nz,nc*nx*ny*nz,dnz,onz);
597: for (i=xs; i<xs+nx; i++) {
598: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
599: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
600: for (j=ys; j<ys+ny; j++) {
601: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
602: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
603: for (k=zs; k<zs+nz; k++) {
604: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
605: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
606:
607: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
608:
609: cnt = 0;
610: for (l=0; l<nc; l++) {
611: for (ii=istart; ii<iend+1; ii++) {
612: for (jj=jstart; jj<jend+1; jj++) {
613: for (kk=kstart; kk<kend+1; kk++) {
614: if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
615: cols[cnt++] = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
616: }
617: }
618: }
619: }
620: rows[l] = l + nc*(slot);
621: }
622: MatPreallocateSetLocal(ltog,nc,rows,cnt,cols,dnz,onz);
623: }
624: }
625: }
626: /* set matrix type and preallocation */
627: if (size > 1) {
628: MatSetType(*J,MATMPIAIJ);
629: } else {
630: MatSetType(*J,MATSEQAIJ);
631: }
632: MatSeqAIJSetPreallocation(*J,0,dnz);
633: MatSeqBAIJSetPreallocation(*J,nc,0,dnz);
634: MatMPIAIJSetPreallocation(*J,0,dnz,0,onz);
635: MatMPIBAIJSetPreallocation(*J,nc,0,dnz,0,onz);
636: MatPreallocateFinalize(dnz,onz);
637: MatSetLocalToGlobalMapping(*J,ltog);
638: DAGetGhostCorners(da,&starts[0],&starts[1],&starts[2],&dims[0],&dims[1],&dims[2]);
639: MatSetStencil(*J,3,dims,starts,nc);
641: /*
642: For each node in the grid: we get the neighbors in the local (on processor ordering
643: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
644: PETSc ordering.
645: */
646: for (i=xs; i<xs+nx; i++) {
647: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
648: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
649: for (j=ys; j<ys+ny; j++) {
650: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
651: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
652: for (k=zs; k<zs+nz; k++) {
653: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
654: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
655:
656: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
657:
658: cnt = 0;
659: for (l=0; l<nc; l++) {
660: for (ii=istart; ii<iend+1; ii++) {
661: for (jj=jstart; jj<jend+1; jj++) {
662: for (kk=kstart; kk<kend+1; kk++) {
663: if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
664: cols[cnt++] = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
665: }
666: }
667: }
668: }
669: rows[l] = l + nc*(slot);
670: }
671: MatSetValuesLocal(*J,nc,rows,cnt,cols,values,INSERT_VALUES);
672: }
673: }
674: }
675: PetscFree(values);
676: PetscFree(rows);
677: PetscFree(cols);
678: MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);
679: MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);
680: return(0);
681: }
683: /* ---------------------------------------------------------------------------------*/
685: int DAGetMatrix1d_MPIAIJ(DA da,Mat *J)
686: {
687: int ierr,xs,nx,i,i1,slot,gxs,gnx;
688: int m,dim,s,*cols,nc,*rows,col,cnt,l;
689: int istart,iend,size;
690: int dims[1],starts[1];
691: MPI_Comm comm;
692: PetscScalar *values;
693: DAPeriodicType wrap;
694: ISLocalToGlobalMapping ltog;
697: /*
698: nc - number of components per grid point
699: col - number of colors needed in one direction for single component problem
700:
701: */
702: DAGetInfo(da,&dim,&m,0,0,0,0,0,&nc,&s,&wrap,0);
703: col = 2*s + 1;
705: if (DAXPeriodic(wrap) && (m % col)) {
706: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points is divisiblen
707: by 2*stencil_width + 1n");
708: }
711: DAGetCorners(da,&xs,0,0,&nx,0,0);
712: DAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);
713: PetscObjectGetComm((PetscObject)da,&comm);
714: MPI_Comm_size(comm,&size);
716: /* create empty Jacobian matrix */
717:
718: ierr = MatCreate(comm,nc*nx,nc*nx,PETSC_DECIDE,PETSC_DECIDE,J);
719: if (size > 1) {
720: MatSetType(*J,MATMPIAIJ);
721: } else {
722: MatSetType(*J,MATSEQAIJ);
723: }
724: MatSeqAIJSetPreallocation(*J,col*nc,0);
725: MatSeqBAIJSetPreallocation(*J,nc,col,0);
726: MatMPIAIJSetPreallocation(*J,col*nc,0,0,0);
727: MatMPIBAIJSetPreallocation(*J,nc,col,0,0,0);
728: DAGetGhostCorners(da,&starts[0],PETSC_IGNORE,PETSC_IGNORE,&dims[0],PETSC_IGNORE,PETSC_IGNORE);
729: MatSetStencil(*J,1,dims,starts,nc);
730:
731: PetscMalloc(col*nc*nc*sizeof(PetscScalar),&values);
732: PetscMemzero(values,col*nc*nc*sizeof(PetscScalar));
733: PetscMalloc(nc*sizeof(int),&rows);
734: PetscMalloc(col*nc*sizeof(int),&cols);
735:
736: DAGetISLocalToGlobalMapping(da,<og);
737: MatSetLocalToGlobalMapping(*J,ltog);
738:
739: /*
740: For each node in the grid: we get the neighbors in the local (on processor ordering
741: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
742: PETSc ordering.
743: */
744: for (i=xs; i<xs+nx; i++) {
745: istart = PetscMax(-s,gxs - i);
746: iend = PetscMin(s,gxs + gnx - i - 1);
747: slot = i - gxs;
748:
749: cnt = 0;
750: for (l=0; l<nc; l++) {
751: for (i1=istart; i1<iend+1; i1++) {
752: cols[cnt++] = l + nc*(slot + i1);
753: }
754: rows[l] = l + nc*(slot);
755: }
756: MatSetValuesLocal(*J,nc,rows,cnt,cols,values,INSERT_VALUES);
757: }
758: PetscFree(values);
759: PetscFree(rows);
760: PetscFree(cols);
761: MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);
762: MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);
763: return(0);
764: }
766: int DAGetMatrix3d_MPIBAIJ(DA da,Mat *J)
767: {
768: int ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
769: int m,n,dim,s,*cols,k,nc,col,cnt,p,*dnz,*onz;
770: int istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
771: MPI_Comm comm;
772: PetscScalar *values;
773: DAPeriodicType wrap;
774: DAStencilType st;
775: ISLocalToGlobalMapping ltog;
778: /*
779: nc - number of components per grid point
780: col - number of colors needed in one direction for single component problem
781:
782: */
783: DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
784: if (wrap != DA_NONPERIODIC) SETERRQ(PETSC_ERR_SUP,"Currently no support for periodic");
785: col = 2*s + 1;
787: DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
788: DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
789: PetscObjectGetComm((PetscObject)da,&comm);
791: /* create the matrix */
792: ierr = PetscMalloc(col*col*col*nc*nc*sizeof(PetscScalar),&values);
793: ierr = PetscMemzero(values,col*col*col*nc*nc*sizeof(PetscScalar));
794: ierr = PetscMalloc(col*col*col*sizeof(int),&cols);
796: DAGetISLocalToGlobalMappingBlck(da,<og);
798: /* determine the matrix preallocation information */
799: MatPreallocateInitialize(comm,nx*ny*nz,nx*ny*nz,dnz,onz);
800: for (i=xs; i<xs+nx; i++) {
801: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
802: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
803: for (j=ys; j<ys+ny; j++) {
804: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
805: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
806: for (k=zs; k<zs+nz; k++) {
807: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
808: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
810: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
812: /* Find block columns in block row */
813: cnt = 0;
814: if (st == DA_STENCIL_BOX) { /* if using BOX stencil */
815: for (ii=istart; ii<iend+1; ii++) {
816: for (jj=jstart; jj<jend+1; jj++) {
817: for (kk=kstart; kk<kend+1; kk++) {
818: cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
819: }
820: }
821: }
822: } else { /* Star stencil */
823: cnt = 0;
824: for (ii=istart; ii<iend+1; ii++) {
825: if (ii) {
826: /* jj and kk must be zero */
827: /* cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk; */
828: cols[cnt++] = slot + ii;
829: } else {
830: for (jj=jstart; jj<jend+1; jj++) {
831: if (jj) {
832: /* ii and kk must be zero */
833: cols[cnt++] = slot + gnx*jj;
834: } else {
835: /* ii and jj must be zero */
836: for (kk=kstart; kk<kend+1; kk++) {
837: cols[cnt++] = slot + gnx*gny*kk;
838: }
839: }
840: }
841: }
842: }
843: }
844: MatPreallocateSetLocal(ltog,1,&slot,cnt,cols,dnz,onz);
845: }
846: }
847: }
849: /* create empty Jacobian matrix */
850: MatCreateMPIBAIJ(comm,nc,nc*nx*ny*nz,nc*nx*ny*nz,PETSC_DECIDE,PETSC_DECIDE,0,dnz,0,onz,J);
852: MatPreallocateFinalize(dnz,onz);
853: MatSetLocalToGlobalMappingBlock(*J,ltog);
855: /*
856: For each node in the grid: we get the neighbors in the local (on processor ordering
857: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
858: PETSc ordering.
859: */
861: for (i=xs; i<xs+nx; i++) {
862: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
863: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
864: for (j=ys; j<ys+ny; j++) {
865: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
866: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
867: for (k=zs; k<zs+nz; k++) {
868: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
869: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
870:
871: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
872:
873: cnt = 0;
874: if (st == DA_STENCIL_BOX) { /* if using BOX stencil */
875: for (ii=istart; ii<iend+1; ii++) {
876: for (jj=jstart; jj<jend+1; jj++) {
877: for (kk=kstart; kk<kend+1; kk++) {
878: cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
879: }
880: }
881: }
882: } else { /* Star stencil */
883: cnt = 0;
884: for (ii=istart; ii<iend+1; ii++) {
885: if (ii) {
886: /* jj and kk must be zero */
887: cols[cnt++] = slot + ii;
888: } else {
889: for (jj=jstart; jj<jend+1; jj++) {
890: if (jj) {
891: /* ii and kk must be zero */
892: cols[cnt++] = slot + gnx*jj;
893: } else {
894: /* ii and jj must be zero */
895: for (kk=kstart; kk<kend+1; kk++) {
896: cols[cnt++] = slot + gnx*gny*kk;
897: }
898: }
899: }
900: }
901: }
902: }
903: MatSetValuesBlockedLocal(*J,1,&slot,cnt,cols,values,INSERT_VALUES);
904: }
905: }
906: }
907: PetscFree(values);
908: PetscFree(cols);
909: MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);
910: MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);
911: return(0);
912: }