Actual source code: mlSerial.c
1: #ifdef PETSC_RCS_HEADER
2: static char vcid[] = "$Id: mlSerial.c,v 1.12 2000/10/22 20:27:52 knepley Exp $";
3: #endif
4: /*
5: Defines the serial multilevel preconditioner routines
6: */
7: #include src/sles/pc/pcimpl.h
8: #include ml.h
10: PetscTruth PCMultiLevelDoQR_Private(PC pc, int numRows, int numCols)
11: {
12: /* This functions decides whether to QR factorize the matrix U of an SVD.
13: This is a memory saving device when B = UDV is long and skinny.
14: */
15: PC_Multilevel *ml = (PC_Multilevel *) pc->data;
18: if (numRows > (int) numCols*ml->DoQRFactor) {
19: PetscFunctionReturn(PETSC_TRUE);
20: } else {
21: PetscFunctionReturn(PETSC_FALSE);
22: }
23: }
25: int PCMultiLevelPartitionMesh_Private(PC pc, int level, int **mesh, int *numParts, int *colPartition)
26: {
27: PC_Multilevel *ml = (PC_Multilevel *) pc->data;
28: PetscTruth change;
29: int numCols, freeCols;
30: int numAdj, newNumAdj, numFound;
31: int *offsets;
32: int *adj;
33: int *sizes;
34: int part, col, newCol, adjCol, newAdjCol;
35: int ierr;
38: PC_MLLogEventBegin(PC_ML_ReducePartitionMesh, pc, 0, 0, 0);
39: if (ml->useMath == PETSC_FALSE) {
40: offsets = mesh[MESH_OFFSETS];
41: adj = mesh[MESH_ADJ];
42: /* Set initial number of partitions */
43: numCols = ml->numVertices[level];
44: *numParts = numCols/ml->partSize;
45: /* If the graph is small enough, just return the whole thing */
46: if (*numParts == 0) {
47: *numParts = 1;
48: PetscMemzero(colPartition, numCols * sizeof(int));
49: PC_MLLogEventEnd(PC_ML_ReducePartitionMesh, pc, 0, 0, 0);
50: return(0);
51: }
52: /* Get connectivity information */
53: /* Initialize partition and partition sizes */
54: PetscMalloc((*numParts) * sizeof(int), &sizes);
55: for(col = 0; col < numCols; col++) colPartition[col] = -1;
56: /* Pick numParts starting points -- Conforms to the Mathematica */
57: for(part = 1; part <= *numParts; part++) {
58: colPartition[ml->partSize*part-1] = part-1;
59: sizes[part-1] = 1;
60: }
61: /* Start enlarging partitions */
62: freeCols = numCols - (*numParts);
63: change = PETSC_TRUE;
64: while(freeCols > 0)
65: {
66: /* Quit if nothing more can be done */
67: if (change == PETSC_FALSE)
68: break;
69: change = PETSC_FALSE;
70: /* Add a node to each partition if possible */
71: for(part = 0; part < (*numParts); part++)
72: {
73: /* Look for the column adjacent to the most columns in this partition */
74: newCol = -1;
75: numAdj = 0;
76: for(col = 0, numFound = 0; (col < numCols) && (numFound < sizes[part]); col++)
77: {
78: if (colPartition[col] == part)
79: {
80: numFound++;
81: /* Look for adjacent columns ignoring self edge */
82: for(adjCol = offsets[col]+1; adjCol < offsets[col+1]; adjCol++)
83: {
84: /* Ignore columns already in a partition */
85: if (colPartition[adj[adjCol]] >= 0)
86: continue;
87: newNumAdj = 0;
88: /* Count the number of columns adjacent to this one that are already in the partition */
89: for(newAdjCol = offsets[adj[adjCol]]+1; newAdjCol < offsets[adj[adjCol]+1]; newAdjCol++)
90: if (colPartition[adj[newAdjCol]] == part)
91: newNumAdj++;
92: if ((newNumAdj > numAdj) || ((newNumAdj == numAdj) && (adj[adjCol] < newCol)))
93: {
94: newCol = adj[adjCol];
95: numAdj = newNumAdj;
96: }
97: }
98: }
99: }
100: if (newCol >= 0)
101: {
102: colPartition[newCol] = part;
103: sizes[part]++;
104: freeCols--;
105: change = PETSC_TRUE;
106: }
107: }
108: }
109: /* Take care of extra columns */
110: if (change == PETSC_FALSE) {
111: for(col = 0; col < numCols; col++)
112: if (colPartition[col] < 0)
113: colPartition[col] = (*numParts)++;
114: }
115: PetscFree(sizes);
116: } else {
117: #ifdef PETSC_HAVE_MATHEMATICA
118: ViewerMathematicaPartitionMesh(ml->mathViewer, ml->numElements[level], ml->numVertices[level],
119: ml->vertices, mesh, numParts, colPartition);
120:
121: #else
122: SETERRQ(PETSC_ERR_SUP, " ");
123: #endif
124: }
125: PC_MLLogEventEnd(PC_ML_ReducePartitionMesh, pc, 0, 0, 0);
127: return(0);
128: }
130: int PCMultiLevelPartitionCols_Private(PC pc, int level, int numParts, int numCols, int *colPartition, int *globalCols)
131: {
132: PC_Multilevel *ml = (PC_Multilevel *) pc->data;
133: int numLocCols;
134: int part, col;
135: int ierr;
138: PetscMalloc(numParts * sizeof(int), &ml->numPartitionCols[level]);
139: PetscLogObjectMemory(pc, numParts * sizeof(int));
140: PetscMemzero(ml->numPartitionCols[level], numParts * sizeof(int));
141: for(col = 0; col < numCols; col++) {
142: ml->numPartitionCols[level][colPartition[col]]++;
143: }
144: PetscMalloc(numParts * sizeof(int *), &ml->colPartition[level]);
145: PetscLogObjectMemory(pc, numParts * sizeof(int *));
146: for(part = 0; part < numParts; part++) {
147: numLocCols = ml->numPartitionCols[level][part];
148: PetscMalloc(numLocCols * sizeof(int), &ml->colPartition[level][part]);
149: PetscLogObjectMemory(pc, numLocCols * sizeof(int));
150: ml->numPartitionCols[level][part] = 0;
151: }
152: for(col = 0; col < numCols; col++) {
153: ml->colPartition[level][colPartition[col]][ml->numPartitionCols[level][colPartition[col]]++] = globalCols[col];
154: }
155: for(part = 0, col = 0; part < numParts; part++) {
156: col += ml->numPartitionCols[level][part];
157: }
158: if (col != numCols) {
159: int rank;
161: MPI_Comm_rank(pc->comm, &rank);
162: PetscPrintf(PETSC_COMM_SELF, "[%d] %d does not equal the number of columns %dn", rank, col, numCols);
163: for(col = 0; col < numCols; col++) {
164: PetscPrintf(PETSC_COMM_SELF, "[%d] part: %d col: %dn", rank, part, colPartition[col]);
165: }
166: SETERRQ(PETSC_ERR_PLIB, "Invalid column partition");
167: }
169: return(0);
170: }
172: int PCMultiLevelPartitionRows_Private(PC pc, int level, int numParts, int numCols, int numRows, int *colPartition,
173: int *rowPartition, int *globalCols, Mat grad)
174: {
175: PC_Multilevel *ml = (PC_Multilevel *) pc->data;
176: int numLocRows, numLocCols, ncols, numSlots;
177: int *cols;
178: PetscScalar *vals;
179: int part, col, locCol, row;
180: int ierr;
183: /* Allocation */
184: numSlots = numParts + NUM_PART_ROW_DIV - 1;
185: PetscMalloc(numSlots * sizeof(int), &ml->numPartitionRows[level]);
186: PetscMalloc(NUM_PART_ROW_DIV * sizeof(int **), &ml->rowPartition[level]);
187: PetscMalloc(numParts * sizeof(int *), &ml->rowPartition[level][PART_ROW_INT]);
188: PetscMalloc(1 * sizeof(int *), &ml->rowPartition[level][PART_ROW_BD]);
189: PetscMalloc(1 * sizeof(int *), &ml->rowPartition[level][PART_ROW_RES]);
190: PetscLogObjectMemory(pc, numSlots*sizeof(int) + NUM_PART_ROW_DIV*sizeof(int **) + numSlots*sizeof(int *));
192: /* Create row partition */
193: for(row = 0; row < numRows; row++) {
194: /* Find partitions adjacent to this row */
195: MatGetRow(grad, row, &ncols, &cols, &vals);
196: PetscMemzero(ml->numPartitionRows[level], numParts * sizeof(int));
197: for(col = 0; col < ncols; col++) {
198: /* Could invert this mapping explicitly or do bisection */
199: for(locCol = 0; locCol < numCols; locCol++)
200: if (globalCols[locCol] == cols[col])
201: break;
202: if (locCol == numCols) SETERRQ(PETSC_ERR_ARG_CORRUPT, "Invalid boundary gradient");
203: ml->numPartitionRows[level][colPartition[locCol]] = 1;
204: }
205: MatRestoreRow(grad, row, &ncols, &cols, &vals);
207: /* Classify row */
208: if (ncols == 0)
209: {
210: /* This is caused by rows with nonzeros only in full rank partitions, so that
211: B_Gamma (I - D^{-1} D) has a null row. */
212: rowPartition[row] = 0;
213: continue;
214: }
215: for(part = 0, numLocCols = 0; part < numParts; part++) {
216: if (ml->numPartitionRows[level][part] > 0) {
217: numLocCols++;
218: rowPartition[row] = part;
219: }
220: }
221: if (numLocCols == 2) {
222: rowPartition[row] = numParts + PART_ROW_BD - 1;
223: } else if (numLocCols > 2) {
224: rowPartition[row] = numParts + PART_ROW_RES - 1;
225: } else if (numLocCols < 1) {
226: SETERRQ(PETSC_ERR_ARG_CORRUPT, "Invalid boundary gradient");
227: }
228: }
230: /* Copy rows */
231: PetscMemzero(ml->numPartitionRows[level], (numParts + NUM_PART_ROW_DIV - 1) * sizeof(int));
232: for(row = 0; row < numRows; row++) {
233: ml->numPartitionRows[level][rowPartition[row]]++;
234: }
235: for(part = 0; part < numParts; part++) {
236: numLocRows = ml->numPartitionRows[level][part];
237: if (numLocRows) {
238: PetscMalloc(numLocRows * sizeof(int), &ml->rowPartition[level][PART_ROW_INT][part]);
239: PetscLogObjectMemory(pc, numLocRows * sizeof(int));
240: }
241: ml->numPartitionRows[level][part] = 0;
242: }
243: numLocRows = ml->numPartitionRows[level][numParts];
244: if (numLocRows) {
245: PetscMalloc(numLocRows * sizeof(int), &ml->rowPartition[level][PART_ROW_BD][0]);
246: PetscLogObjectMemory(pc, numLocRows * sizeof(int));
247: }
248: ml->numPartitionRows[level][numParts] = 0;
249: numLocRows = ml->numPartitionRows[level][numParts+1];
250: if (numLocRows) {
251: PetscMalloc(numLocRows * sizeof(int), &ml->rowPartition[level][PART_ROW_RES][0]);
252: PetscLogObjectMemory(pc, numLocRows * sizeof(int));
253: }
254: ml->numPartitionRows[level][numParts+1] = 0;
255: for(row = 0; row < numRows; row++) {
256: if (rowPartition[row] < numParts) {
257: ml->rowPartition[level][PART_ROW_INT][rowPartition[row]][ml->numPartitionRows[level][rowPartition[row]]++] = row;
258: } else if (rowPartition[row] == numParts) {
259: ml->rowPartition[level][PART_ROW_BD][0][ml->numPartitionRows[level][rowPartition[row]]++] = row;
260: } else if (rowPartition[row] > numParts) {
261: ml->rowPartition[level][PART_ROW_RES][0][ml->numPartitionRows[level][rowPartition[row]]++] = row;
262: }
263: }
265: return(0);
266: }
268: int PCMultiLevelFactorPartitions_Private(PC pc, int level, int numParts, Mat grad)
269: {
270: PC_Multilevel *ml = (PC_Multilevel *) pc->data;
271: int ncols;
272: int *cols;
273: PetscScalar *vals;
274: PetscReal *B;
275: int maxSize;
276: int numRows, numCols;
277: int part, row, col, locCol;
278: int ierr;
281: PC_MLLogEventBegin(PC_ML_ReduceFactor, pc, grad, 0, 0);
282: /* Allocation */
283: for(part = 0, maxSize = 0; part < numParts; part++) {
284: numRows = ml->numPartitionRows[level][part];
285: numCols = ml->numPartitionCols[level][part];
286: maxSize = PetscMax(maxSize, numRows*numCols);
287: }
288: if (maxSize == 0) {
289: PetscFunctionReturn(2);
290: /* SETERRQ(1, "Empty interior (increase partition size)"); */
291: }
292: PetscMalloc(numParts * sizeof(PetscReal **), &ml->factors[level]);
293: PetscLogObjectMemory(pc, numParts * sizeof(PetscReal **));
294: PetscMalloc(maxSize * sizeof(PetscReal), &B);
295:
296: for(part = 0; part < numParts; part++) {
297: /* Allocation */
298: numRows = ml->numPartitionRows[level][part];
299: numCols = ml->numPartitionCols[level][part];
300: PetscMalloc(NUM_FACT_DIV * sizeof(PetscReal *), &ml->factors[level][part]);
301: PetscMalloc(numCols * sizeof(PetscReal), &ml->factors[level][part][FACT_DINV]);
302: PetscMalloc(numCols*numCols * sizeof(PetscReal), &ml->factors[level][part][FACT_V]);
303: PetscLogObjectMemory(pc, NUM_FACT_DIV * sizeof(double *) + (numCols + numCols*numCols)*sizeof(PetscReal));
305: if (numRows > 0) {
306: /* If numCols < numRows, LAPACK will leave a bogus values in the end of D^{-1} */
307: if (numCols > numRows) {
308: PetscMemzero(ml->factors[level][part][FACT_DINV], numCols * sizeof(PetscReal));
309: }
311: if (PCMultiLevelDoQR_Private(pc, numRows, numCols) == PETSC_TRUE) {
312: /* Allocation */
313: PetscMalloc(numRows*numCols * sizeof(PetscReal), &ml->factors[level][part][FACT_QR]);
314: PetscMalloc(numCols * sizeof(PetscReal), &ml->factors[level][part][FACT_TAU]);
315: PetscMalloc(numCols*numCols * sizeof(PetscReal), &ml->factors[level][part][FACT_U]);
316: PetscLogObjectMemory(pc, (numRows*numCols + numCols + numCols*numCols)*sizeof(PetscReal));
318: /* Extract the interior gradient for this partition */
319: PetscMemzero(ml->factors[level][part][FACT_QR], numRows*numCols * sizeof(PetscReal));
320: for(row = 0; row < numRows; row++) {
321: MatGetRow(grad, ml->rowPartition[level][PART_ROW_INT][part][row], &ncols, &cols, &vals);
322: for(col = 0; col < ncols; col++) {
323: for(locCol = 0; locCol < numCols; locCol++) {
324: if (cols[col] == ml->colPartition[level][part][locCol]) break;
325: }
326: if (locCol < numCols) {
327: ml->factors[level][part][FACT_QR][numRows*locCol+row] = vals[col];
328: }
329: }
330: MatRestoreRow(grad, ml->rowPartition[level][PART_ROW_INT][part][row], &ncols, &cols, &vals);
331: }
333: /* Do QR of B */
334: LAgeqrf_(&numRows, &numCols, ml->factors[level][part][FACT_QR],
335: &numRows, ml->factors[level][part][FACT_TAU], ml->svdWork, &ml->svdWorkLen, &ierr);
336:
338: /* Put R in B */
339: PetscMemzero(B, numCols*numCols * sizeof(PetscReal));
340: for(col = 0; col < numCols; col++)
341: for(row = 0; row <= col; row++)
342: B[col*numCols+row] = ml->factors[level][part][FACT_QR][col*numRows+row];
344: /* Do SVD of R */
345: #if defined(PETSC_MISSING_LAPACK_GESVD)
346: SETERRQ(PETSC_ERR_SUP,"GESVD - Lapack routine is unavilable.");
347: #else
348: LAgesvd_("A", "A", &numCols, &numCols, B, &numCols,
349: ml->factors[level][part][FACT_DINV], ml->factors[level][part][FACT_U], &numCols,
350: ml->factors[level][part][FACT_V], &numCols, ml->svdWork, &ml->svdWorkLen, &ierr);
351: #endif
352: } else {
353: /* Allocation */
354: PetscMalloc(numRows*numRows * sizeof(PetscReal), &ml->factors[level][part][FACT_U]);
355: PetscLogObjectMemory(pc, numRows*numRows * sizeof(PetscReal));
357: /* Extract the interior gradient for this partition */
358: PetscMemzero(B, numRows*numCols * sizeof(PetscReal));
359: for(row = 0; row < numRows; row++) {
360: MatGetRow(grad, ml->rowPartition[level][PART_ROW_INT][part][row], &ncols, &cols, &vals);
361: for(col = 0; col < ncols; col++) {
362: for(locCol = 0; locCol < numCols; locCol++) {
363: if (cols[col] == ml->colPartition[level][part][locCol]) break;
364: }
365: if (locCol < numCols) {
366: B[numRows*locCol+row] = vals[col];
367: }
368: }
369: MatRestoreRow(grad, ml->rowPartition[level][PART_ROW_INT][part][row], &ncols, &cols, &vals);
370: }
372: /* Factor the gradient */
373: #if defined(PETSC_MISSING_LAPACK_GESVD)
374: SETERRQ(PETSC_ERR_SUP,"GESVD - Lapack routine is unavilable.");
375: #else
376: LAgesvd_("A", "A", &numRows, &numCols, B, &numRows, ml->factors[level][part][FACT_DINV],
377: ml->factors[level][part][FACT_U], &numRows, ml->factors[level][part][FACT_V], &numCols,
378: ml->svdWork, &ml->svdWorkLen, &ierr);
379: #endif
380:
381: }
383: /* Invert the singular values */
384: for(col = 0; col < numCols; col++) {
385: if (ml->factors[level][part][FACT_DINV][col] > ml->zeroTol)
386: ml->factors[level][part][FACT_DINV][col] = 1.0/ml->factors[level][part][FACT_DINV][col];
387: }
388: } else {
389: /* Create null singular values for D^{-1} and the identity for V */
390: PetscMemzero(ml->factors[level][part][FACT_DINV], numCols * sizeof(PetscReal));
391: PetscMemzero(ml->factors[level][part][FACT_V], numCols*numCols * sizeof(PetscReal));
392: for(col = 0; col < numCols; col++)
393: ml->factors[level][part][FACT_V][col*numCols+col] = 1.0;
394: }
395: }
397: /* Cleanup */
398: PetscFree(B);
399: PC_MLLogEventEnd(PC_ML_ReduceFactor, pc, grad, 0, 0);
400: return(0);
401: }
403: int PCMultiLevelExtractBoundaryGradient_Private(PC pc, int level, int numParts, int colsRemain, int *newCols,
404: PetscScalar *newVals, Mat grad)
405: {
406: PC_Multilevel *ml = (PC_Multilevel *) pc->data;
407: int ncols;
408: int *cols;
409: PetscScalar *vals;
410: PetscScalar *temp;
411: int *rowLens;
412: int numBdRows, numResRows, rowsRemain, numCols, numVals, prevCols;
413: int part, row, newRow, col, locCol = 0;
414: int ierr;
417: PC_MLLogEventBegin(PC_ML_ReduceBdGradExtract, pc, grad, 0, 0);
418: /* Initialization */
419: numBdRows = ml->numPartitionRows[level][numParts];
420: numResRows = ml->numPartitionRows[level][numParts+1];
421: rowsRemain = numBdRows + numResRows;
423: /* Get the nonzero structure */
424: VecGetArray(ml->bdReduceVecs[level], &temp);
425: rowLens = (int *) temp;
426: for(row = 0; row < numBdRows; row++)
427: {
428: MatGetRow(grad, ml->rowPartition[level][PART_ROW_BD][0][row], &ncols, &cols, PETSC_NULL);
429: for(col = 0, numVals = 0; col < ncols; col++)
430: {
431: for(part = 0, prevCols = 0; part < numParts; part++)
432: {
433: numCols = ml->numPartitionCols[level][part];
434: for(locCol = 0; locCol < numCols; locCol++)
435: if (cols[col] == ml->colPartition[level][part][locCol])
436: break;
437: if (locCol < numCols)
438: break;
439: prevCols += numCols;
440: }
441: if (part < numParts)
442: numVals++;
443: }
444: if (numVals == 0) SETERRQ(PETSC_ERR_ARG_CORRUPT, "Null boundary row in boundary gradient");
445: rowLens[row] = numVals;
446: MatRestoreRow(grad, ml->rowPartition[level][PART_ROW_BD][0][row], &ncols, &cols, PETSC_NULL);
447: }
448: for(row = 0; row < numResRows; row++)
449: {
450: MatGetRow(grad, ml->rowPartition[level][PART_ROW_RES][0][row], &ncols, &cols, PETSC_NULL);
451: for(col = 0, numVals = 0; col < ncols; col++)
452: {
453: for(part = 0, prevCols = 0; part < numParts; part++)
454: {
455: numCols = ml->numPartitionCols[level][part];
456: for(locCol = 0; locCol < numCols; locCol++)
457: if (cols[col] == ml->colPartition[level][part][locCol])
458: break;
459: if (locCol < numCols)
460: break;
461: prevCols += numCols;
462: }
463: if (part < numParts)
464: numVals++;
465: }
466: if (numVals == 0) SETERRQ(PETSC_ERR_ARG_CORRUPT, "Null residual row in boundary gradient");
467: rowLens[row+numBdRows] = numVals;
468: MatRestoreRow(grad, ml->rowPartition[level][PART_ROW_RES][0][row], &ncols, &cols, PETSC_NULL);
469: }
471: /* Create the boundary gradient B_Gamma */
472: MatCreateSeqAIJ(PETSC_COMM_SELF, rowsRemain, colsRemain, 0, rowLens, &ml->grads[level]);
473: MatSetOption(ml->grads[level], MAT_NEW_NONZERO_ALLOCATION_ERR);
474: VecRestoreArray(ml->bdReduceVecs[level], &temp);
476: /* Extract boundary rows */
477: for(row = 0; row < numBdRows; row++) {
478: MatGetRow(grad, ml->rowPartition[level][PART_ROW_BD][0][row], &ncols, &cols, &vals);
479: for(col = 0, numVals = 0; col < ncols; col++)
480: {
481: for(part = 0, prevCols = 0; part < numParts; part++)
482: {
483: numCols = ml->numPartitionCols[level][part];
484: for(locCol = 0; locCol < numCols; locCol++)
485: if (cols[col] == ml->colPartition[level][part][locCol])
486: break;
487: if (locCol < numCols)
488: break;
489: prevCols += numCols;
490: }
491: if (part < numParts)
492: {
493: newCols[numVals] = locCol + prevCols;
494: newVals[numVals] = vals[col];
495: numVals++;
496: }
497: }
498: MatSetValues(ml->grads[level], 1, &row, numVals, newCols, newVals, INSERT_VALUES);
499: MatRestoreRow(grad, ml->rowPartition[level][PART_ROW_BD][0][row], &ncols, &cols, &vals);
500: }
502: /* Extract residual rows */
503: for(row = 0, newRow = numBdRows; row < numResRows; row++, newRow++)
504: {
505: MatGetRow(grad, ml->rowPartition[level][PART_ROW_RES][0][row], &ncols, &cols, &vals);
506: for(col = 0, numVals = 0; col < ncols; col++)
507: {
508: for(part = 0, prevCols = 0; part < numParts; part++)
509: {
510: numCols = ml->numPartitionCols[level][part];
511: for(locCol = 0; locCol < numCols; locCol++)
512: if (cols[col] == ml->colPartition[level][part][locCol])
513: break;
514: if (locCol < numCols)
515: break;
516: prevCols += numCols;
517: }
518: if (part < numParts)
519: {
520: newCols[numVals] = locCol + prevCols;
521: newVals[numVals] = vals[col];
522: numVals++;
523: }
524: }
525: MatSetValues(ml->grads[level], 1, &newRow, numVals, newCols, newVals, INSERT_VALUES);
526: MatRestoreRow(grad, ml->rowPartition[level][PART_ROW_RES][0][row], &ncols, &cols, &vals);
527: }
529: MatAssemblyBegin(ml->grads[level], MAT_FINAL_ASSEMBLY);
530: MatAssemblyEnd(ml->grads[level], MAT_FINAL_ASSEMBLY);
531: PC_MLLogEventEnd(PC_ML_ReduceBdGradExtract, pc, grad, 0, 0);
532: return(0);
533: }
535: int PCMultiLevelShrinkMesh_Private(PC pc, int level, int numNewCols, int *colPartition, int numElements, int **oldMesh,
536: int *numNewElements, int ***newMesh)
537: {
538: PC_Multilevel *ml = (PC_Multilevel *) pc->data;
539: int numCols = ml->numVertices[level];
540: int *adjCols;
541: int maxSize;
542: int col, oldCol, newCol, adjCol, count;
543: int elem;
544: int ierr;
547: PC_MLLogEventBegin(PC_ML_ReduceShrinkMesh, pc, 0, 0, 0);
548: ml->numMeshes++;
550: /* Allocation */
551: maxSize = PetscMax(PetscMin(oldMesh[MESH_OFFSETS][numCols], numNewCols*numNewCols + 1), 1);
552: PetscMalloc(NUM_MESH_DIV * sizeof(int *), newMesh);
553: PetscMalloc((numNewCols+1) * sizeof(int), &(*newMesh)[MESH_OFFSETS]);
554: PetscMalloc(maxSize * sizeof(int), &(*newMesh)[MESH_ADJ]);
555: PetscMalloc((numNewCols+1) * sizeof(int), &adjCols);
556: PetscLogObjectMemory(pc, NUM_MESH_DIV * sizeof(int *) + (numNewCols+1 + maxSize)*sizeof(int));
557: /* Shrink adjacency lists */
558: (*newMesh)[MESH_OFFSETS][0] = 0;
559: for(col = 0, count = 0; col < numNewCols; col++) {
560: PetscMemzero(adjCols, numNewCols * sizeof(int));
561: /* Put in self edge */
562: (*newMesh)[MESH_ADJ][count++] = col;
563: adjCols[col] = 1;
564: /* Get all columns in this partition */
565: for(oldCol = 0; oldCol < numCols; oldCol++) {
566: if (colPartition[oldCol] == col) {
567: for(adjCol = oldMesh[MESH_OFFSETS][oldCol]; adjCol < oldMesh[MESH_OFFSETS][oldCol+1]; adjCol++) {
568: /* Check for adjacent column */
569: newCol = colPartition[oldMesh[MESH_ADJ][adjCol]];
570: /* Null partitions have number -1 */
571: if ((newCol < 0) || (adjCols[newCol]))
572: continue;
573: (*newMesh)[MESH_ADJ][count++] = newCol;
574: adjCols[newCol] = 1;
575: }
576: }
577: }
578: (*newMesh)[MESH_OFFSETS][col+1] = count;
579: }
580: PetscFree(adjCols);
582: /* Eliminate redundant elements */
583: maxSize = PetscMax(PetscMin(numElements, PetscMax(numCols-3, 0)*2 + 1), 1);
584: PetscMalloc(maxSize*3 * sizeof(int), &(*newMesh)[MESH_ELEM]);
585: PetscLogObjectMemory(pc, maxSize*3 * sizeof(int));
586: for(elem = 0, *numNewElements = 0; elem < numElements; elem++) {
587: if ((colPartition[oldMesh[MESH_ELEM][elem*3]-1] != colPartition[oldMesh[MESH_ELEM][elem*3+1]-1]) &&
588: (colPartition[oldMesh[MESH_ELEM][elem*3+1]-1] != colPartition[oldMesh[MESH_ELEM][elem*3+2]-1]) &&
589: (colPartition[oldMesh[MESH_ELEM][elem*3+2]-1] != colPartition[oldMesh[MESH_ELEM][elem*3]-1]) &&
590: (colPartition[oldMesh[MESH_ELEM][elem*3]-1] >= 0) &&
591: (colPartition[oldMesh[MESH_ELEM][elem*3+1]-1] >= 0) &&
592: (colPartition[oldMesh[MESH_ELEM][elem*3+2]-1] >= 0))
593: {
594: (*newMesh)[MESH_ELEM][(*numNewElements)*3] = colPartition[oldMesh[MESH_ELEM][elem*3]-1] + 1;
595: (*newMesh)[MESH_ELEM][(*numNewElements)*3+1] = colPartition[oldMesh[MESH_ELEM][elem*3+1]-1] + 1;
596: (*newMesh)[MESH_ELEM][(*numNewElements)*3+2] = colPartition[oldMesh[MESH_ELEM][elem*3+2]-1] + 1;
597: (*numNewElements)++;
598: }
599: }
600: PC_MLLogEventEnd(PC_ML_ReduceShrinkMesh, pc, 0, 0, 0);
602: return(0);
603: }
605: int PCMultiLevelRowPartitionLocalToGlobal_Private(PC pc, int level)
606: {
607: PC_Multilevel *ml = (PC_Multilevel *) pc->data;
608: PetscScalar *bdArray;
609: int *prevIndices;
610: int *rowIndices;
611: int numParts, numIntRows, numBdRows, numResRows;
612: int part, row;
613: int ierr;
616: if (level <= 0)
617: return(0);
618: PC_MLLogEventBegin(PC_ML_ReduceBdGradRowPartLocalToGlobal, pc, 0, 0, 0);
619: VecGetArray(ml->bdReduceVecs[level-1], &bdArray);
621: /* Load unreduced rows from last level into a work vector */
622: prevIndices = (int *) bdArray;
623: numParts = ml->numPartitions[level-1];
624: numBdRows = ml->numPartitionRows[level-1][numParts];
625: PetscMemcpy(prevIndices, ml->rowPartition[level-1][PART_ROW_BD][0], numBdRows * sizeof(int));
626: numResRows = ml->numPartitionRows[level-1][numParts+1];
627: PetscMemcpy(prevIndices+numBdRows, ml->rowPartition[level-1][PART_ROW_RES][0], numResRows * sizeof(int));
628: /* Interior edges */
629: numParts = ml->numPartitions[level];
630: for(part = 0; part < numParts; part++)
631: {
632: rowIndices = ml->rowPartition[level][PART_ROW_INT][part];
633: numIntRows = ml->numPartitionRows[level][part];
634: for(row = 0; row < numIntRows; row++)
635: rowIndices[row] = prevIndices[rowIndices[row]];
636: }
637: /* Boundary edges */
638: rowIndices = ml->rowPartition[level][PART_ROW_BD][0];
639: numBdRows = ml->numPartitionRows[level][numParts];
640: for(row = 0; row < numBdRows; row++)
641: rowIndices[row] = prevIndices[rowIndices[row]];
642: /* Residual edges */
643: rowIndices = ml->rowPartition[level][PART_ROW_RES][0];
644: numResRows = ml->numPartitionRows[level][numParts+1];
645: for(row = 0; row < numResRows; row++)
646: rowIndices[row] = prevIndices[rowIndices[row]];
648: VecRestoreArray(ml->bdReduceVecs[level-1], &bdArray);
649: PC_MLLogEventEnd(PC_ML_ReduceBdGradRowPartLocalToGlobal, pc, 0, 0, 0);
651: return(0);
652: }
654: /*@C PCMultiLevelReduce
655: This function creates a multilevel factorization of the gradient matrix
656: which may be used for preconditioning a linear solve. This is called by
657: PCSetUp() when the PC_MULTILEVEL type is selected.
659: Input Parameters:
660: . pc - The preconditioner context
662: Level: intermediate
664: .keywords multilevel
665: .seealso
666: @*/
667: int PCMultiLevelReduce(PC pc)
668: {
669: PC_Multilevel *ml;
670: PetscTruth adj;
671: Mat grad;
672: int ncols;
673: int *cols;
674: int *newCols;
675: PetscScalar *vals;
676: PetscScalar *newVals;
677: PetscScalar *colReduceArray;
678: PetscScalar *colReduceArray2;
679: int *colPartition;
680: int *rowPartition;
681: int *globalCols;
682: int *localCols;
683: int *globalPart;
684: int rowsRemain, colsRemain, newColsRemain;
685: int numParts, numNewParts, numCols, numNullCols, numVals;
686: int numBdRows, numResRows;
687: int rankDef;
688: PetscScalar val;
689: int size, sizeMax;
690: int invalidNull;
691: int level, part, newPart, row, col, col2, locCol, nullCol, vCol, loop, anyAgain;
692: #define NEW_BD_GRAD
693: #ifdef NEW_BD_GRAD
694: int *firstCol, *colPart;
695: #ifdef NEW_BD_GRAD_CHECK
696: int oldPart, oldLocCol;
697: #endif
698: #endif
699: int ierr, ierr2;
703: ml = (PC_Multilevel *) pc->data;
705: /* Create P and Z such that
707: P^T B Z = / D
708: 0 /
710: which is almost and SVD
711: */
712: if (ml->useMath == PETSC_FALSE) {
713: /* Initialization */
714: /* MatConvert(ml->locB, MATSAME, &grad); */
715: grad = ml->locB;
716: colsRemain = ml->numCols;
717: rowsRemain = ml->numRows;
718: ml->rank = 0;
719: sizeMax = PetscMax(ml->numRows, ml->numCols);
720: ml->svdWorkLen = PetscMax(3*PetscMin(ml->numRows,ml->numCols) + PetscMax(ml->numRows,ml->numCols),
721: 5*PetscMin(ml->numRows,ml->numCols) - 4);
723: /* Allocation */
724: PetscMalloc(ml->maxLevels * sizeof(int), &ml->numPartitions);
725: PetscMalloc(ml->maxLevels * sizeof(int *), &ml->numPartitionCols);
726: PetscMalloc(ml->maxLevels * sizeof(int **), &ml->colPartition);
727: PetscMalloc(ml->maxLevels * sizeof(int *), &ml->numPartitionRows);
728: PetscMalloc(ml->maxLevels * sizeof(int ***), &ml->rowPartition);
729: PetscMalloc(ml->maxLevels * sizeof(double ***), &ml->factors);
730: PetscMalloc(ml->maxLevels * sizeof(Mat), &ml->grads);
731: PetscMalloc(ml->maxLevels * sizeof(Vec), &ml->bdReduceVecs);
732: PetscMalloc(ml->maxLevels * sizeof(Vec), &ml->colReduceVecs);
733: PetscMalloc(ml->maxLevels * sizeof(Vec), &ml->colReduceVecs2);
734: PetscMalloc(ml->svdWorkLen * sizeof(double), &ml->svdWork);
735: PetscMalloc(ml->numCols * sizeof(int), &ml->range);
736: PetscLogObjectMemory(pc, ml->maxLevels*(sizeof(int) + sizeof(int *) + sizeof(int **) + sizeof(int *) + sizeof(int ***) + sizeof(double ***) +
737: sizeof(Mat) + sizeof(Vec)*3) + ml->svdWorkLen * sizeof(double) + ml->numCols * sizeof(int));
738: PetscMalloc(ml->numCols * sizeof(int), &colPartition);
739: PetscMalloc(ml->numCols * sizeof(int), &globalCols);
740: PetscMalloc(sizeMax * sizeof(int), &rowPartition);
742: for(col = 0; col < ml->numCols; col++) {
743: ml->range[col] = -1;
744: globalCols[col] = col;
745: }
747: /* Loop until the graph has ml->QRthresh nodes or all rows have been reduced */
748: for(level = 0, anyAgain = 0; (colsRemain > ml->QRthresh) && (rowsRemain > 0); level++) {
749: if (level >= ml->maxLevels) SETERRQ(PETSC_ERR_ARG_SIZ, "Exceeded maximum level");
751: loop = 2;
752: while(loop) {
753: /* Partition the columns (nodes): Give a list with the partition number of each column */
754: PCMultiLevelPartitionMesh_Private(pc, level, ml->meshes[level], &numParts, colPartition);
755: ml->numPartitions[level] = numParts;
757: /* Get the global columns in each partition */
758: PC_MLLogEventBegin(PC_ML_ReducePartitionRowCol, pc, 0, 0, 0);
759: PCMultiLevelPartitionCols_Private(pc, level, numParts, colsRemain, colPartition, globalCols);
760:
762: /* Partition rows (edges): Give the local rows in each partition, then boundary and residual rows */
763: PCMultiLevelPartitionRows_Private(pc, level, numParts, colsRemain, rowsRemain, colPartition,
764: rowPartition, globalCols, grad);
765:
766: PC_MLLogEventEnd(PC_ML_ReducePartitionRowCol, pc, 0, 0, 0);
768: /* Factor the interior of each partition */
769: loop = PCMultiLevelFactorPartitions_Private(pc, level, numParts, grad);
771: if (loop == 2) {
772: /* Free memory from bad partition */
773: for(part = 0; part < numParts; part++) {
774: if (ml->numPartitionCols[level][part] > 0)
775: {PetscFree(ml->colPartition[level][part]); }
776: if (ml->numPartitionRows[level][part] > 0)
777: {PetscFree(ml->rowPartition[level][PART_ROW_INT][part]); }
778: }
779: if (ml->numPartitionRows[level][ml->numPartitions[level]] > 0)
780: {PetscFree(ml->rowPartition[level][PART_ROW_BD][0]); }
781: if (ml->numPartitionRows[level][ml->numPartitions[level]+1] > 0)
782: {PetscFree(ml->rowPartition[level][PART_ROW_RES][0]); }
783: PetscFree(ml->rowPartition[level][PART_ROW_INT]);
784: PetscFree(ml->rowPartition[level][PART_ROW_BD]);
785: PetscFree(ml->rowPartition[level][PART_ROW_RES]);
786: PetscFree(ml->colPartition[level]);
787: PetscFree(ml->rowPartition[level]);
788: PetscFree(ml->numPartitionCols[level]);
789: PetscFree(ml->numPartitionRows[level]);
790: /* Increase partition size */
791: PetscLogInfo(pc, "PCMultiLevelReduce: Increasing partition size from %d to %dn", ml->partSize, ml->partSize*2);
792: if (ml->partSize > ml->numCols) SETERRQ(PETSC_ERR_PLIB, "Partition has grown too large.");
793: ml->partSize *= 2;
794: } else if (loop != 0) {
795: CHKERRQ(loop);
796: }
797: }
799: /* QR factor boundary matrices */
801: /* Create the boundary gradient B_Gamma and workspace for applying it */
802: PC_MLLogEventBegin(PC_ML_ReduceBdGrad, pc, 0, 0, 0);
803: numBdRows = ml->numPartitionRows[level][numParts];
804: numResRows = ml->numPartitionRows[level][numParts+1];
805: rowsRemain = numBdRows + numResRows;
806: VecCreateSeq(PETSC_COMM_SELF, rowsRemain, &ml->bdReduceVecs[level]);
807: VecCreateSeq(PETSC_COMM_SELF, colsRemain, &ml->colReduceVecs[level]);
808: VecDuplicate(ml->colReduceVecs[level], &ml->colReduceVecs2[level]);
810: /* Workspace for matrix reduction */
811: VecGetArray(ml->colReduceVecs[0], &colReduceArray);
812: VecGetArray(ml->colReduceVecs2[0], &colReduceArray2);
813: newCols = (int *) colReduceArray;
814: newVals = ml->svdWork;
816: /* Extract the boundary gradient B_Gamma in CSR format */
817: PCMultiLevelExtractBoundaryGradient_Private(pc, level, numParts, colsRemain, newCols, newVals, grad);
818:
820: /* Convert the local (per level) numbering to the global (original) numbering */
821: PCMultiLevelRowPartitionLocalToGlobal_Private(pc, level);
823: /* Determine the columns active at the next level and fix up colPartition[] */
824: globalPart = (int *) colReduceArray2;
825: localCols = rowPartition;
826: for(part = 0; part < numParts; part++) globalCols[part] = -1;
827: for(part = 0, newPart = 0, newColsRemain = 0, numNewParts = numParts; part < numParts; part++, newPart++) {
828: for(col = 0, rankDef = 0; col < ml->numPartitionCols[level][part]; col++) {
829: if (ml->factors[level][part][FACT_DINV][col] < ml->zeroTol) {
830: newColsRemain++;
831: rankDef++;
833: if (rankDef == 1) {
834: globalCols[newPart] = ml->colPartition[level][part][col];
835: globalPart[newPart] = part;
836: localCols[newPart] = col;
837: } else {
838: globalCols[numNewParts] = ml->colPartition[level][part][col];
839: globalPart[numNewParts] = part;
840: localCols[numNewParts] = col;
842: /* Put the column in its own partition */
843: numCols = ml->numPartitionCols[level][part] - (col+1);
844: for(col2 = colsRemain-1; numCols >= 0; col2--) {
845: if (colPartition[col2] == newPart) {
846: numCols--;
847: if (numCols < 0)
848: colPartition[col2] = numNewParts++;
849: }
850: }
851: }
852: } else {
853: ml->rank++;
854: #ifdef PETSC_USE_BOPT_g
855: if (ml->range[ml->colPartition[level][part][col]] != -1) {
856: PetscLogInfo(pc, "Level %d partition %d col %d: column %d has already been reducedn",
857: level, part, col, ml->colPartition[level][part][col]);
858: SETERRQ(PETSC_ERR_ARG_CORRUPT, "Invalid reduction");
859: }
860: if ((ml->rowPartition[level][PART_ROW_INT][part][col] < 0) ||
861: (ml->rowPartition[level][PART_ROW_INT][part][col] > ml->numRows)) {
862: SETERRQ(PETSC_ERR_ARG_CORRUPT, "Invalid row index in range");
863: }
864: #endif
865: ml->range[ml->colPartition[level][part][col]] = ml->rowPartition[level][PART_ROW_INT][part][col];
866: }
867: }
868: #ifdef PETSC_USE_BOPT_g
869: PetscLogInfo(pc, "Lvl: %d Partition: %d has rank deficiency %dn", level, part, rankDef);
870: #endif
871: if (rankDef == 0) {
872: /* Eliminate this partition */
873: for(col = 0; col < colsRemain; col++) {
874: if (colPartition[col] == newPart) {
875: colPartition[col] = -1;
876: } else if (colPartition[col] > newPart) {
877: colPartition[col]--;
878: }
879: }
880: /* Shift everything down */
881: size = PetscMax(numNewParts, numParts) - part;
882: PetscMemmove(&globalCols[newPart], &globalCols[newPart+1], size * sizeof(int));
883: PetscMemmove(&globalPart[newPart], &globalPart[newPart+1], size * sizeof(int));
884: PetscMemmove(&localCols[newPart], &localCols[newPart+1], size * sizeof(int));
885: newPart--;
886: numNewParts--;
887: }
888: }
890: #ifdef PETSC_USE_BOPT_g
891: for(col = 0, row = 0; col < ml->numCols; col++) {
892: if (ml->range[col] == -1) {
893: row++;
894: } else if (ml->range[col] < 0) {
895: SETERRQ(PETSC_ERR_ARG_CORRUPT, "Invalid range space");
896: }
897: }
898: if (row != newColsRemain) SETERRQ(PETSC_ERR_ARG_CORRUPT, "Invalid range space");
900: for(vCol = 0; vCol < newColsRemain; vCol++) {
901: if (globalCols[vCol] < 0) {
902: PetscPrintf(PETSC_COMM_SELF, "Invalid column in reduced boundary gradientn");
903: PetscPrintf(PETSC_COMM_SELF, "Lvl: %d newRows: %d newCols: %dn", level, rowsRemain, newColsRemain);
904: for(col = 0; col < newColsRemain; col++) {
905: PetscPrintf(PETSC_COMM_SELF, " globalCols[%d]: %dn", col, globalCols[col]);
906: }
907: SETERRQ(PETSC_ERR_ARG_CORRUPT, "Negative column");
908: }
909: }
910: #endif
911: PC_MLLogEventEnd(PC_ML_ReduceBdGrad, pc, 0, 0, 0);
913: /* Transform B_Gamma to B_Gamma V (I - D^{-1} D) */
914: PC_MLLogEventBegin(PC_ML_CreateBdGrad, pc, 0, 0, 0);
915: if (level > 0) {
916: MatDestroy(grad);
917: }
918: MatCreateSeqAIJ(PETSC_COMM_SELF, rowsRemain, ml->numCols, newColsRemain, PETSC_NULL, &grad);
919: if (ierr) {
920: PetscPrintf(PETSC_COMM_SELF, "Interface Gradient failure:");
921: PetscPrintf(PETSC_COMM_SELF, " level: %d", level);
922: PetscPrintf(PETSC_COMM_SELF, " rows: %d cols: %dn", rowsRemain, ml->numCols);
923: PetscPrintf(PETSC_COMM_SELF, " nonzeroes per row: %dn", newColsRemain);
924:
925: }
926: #ifdef NEW_BD_GRAD
927: PetscMalloc((numParts+1) * sizeof(int), &firstCol);
928: PetscMalloc(ml->numCols * sizeof(int), &colPart);
929: for(part = 1, firstCol[0] = 0; part <= numParts; part++) {
930: firstCol[part] = firstCol[part-1] + ml->numPartitionCols[level][part-1];
931: for(col = 0; col < ml->numPartitionCols[level][part-1]; col++) {
932: colPart[col+firstCol[part-1]] = part-1;
933: }
934: }
935: #endif
936: for(row = 0; row < rowsRemain; row++) {
937: MatGetRow(ml->grads[level], row, &ncols, &cols, &vals);
938: /* Sparse VecDot with the active columns of V */
939: for(vCol = 0, numVals = 0; vCol < newColsRemain; vCol++) {
940: /* Calculate the dot product */
941: for(col = 0, adj = PETSC_FALSE, val = 0.0; col < ncols; col++) {
942: /* Get the local column number */
943: #ifndef NEW_BD_GRAD
944: locCol = cols[col];
945: for(part = 0; part < PetscMin(numParts, globalPart[vCol]+2); part++) {
946: numCols = ml->numPartitionCols[level][part];
947: if (locCol >= numCols) {
948: locCol -= numCols;
949: continue;
950: }
951: break;
952: }
953: #else
954: part = colPart[cols[col]];
955: locCol = cols[col] - firstCol[part];
956: numCols = ml->numPartitionCols[level][part];
957: #ifdef NEW_BD_GRAD_CHECK
958: /***** CHECK THE RESULT ***/
959: oldLocCol = cols[col];
960: for(oldPart = 0; oldPart < PetscMin(numParts, globalPart[vCol]+2); oldPart++) {
961: numCols = ml->numPartitionCols[level][oldPart];
962: if (oldLocCol >= numCols) {
963: oldLocCol -= numCols;
964: continue;
965: }
966: break;
967: }
968: if ((oldPart == globalPart[vCol]) && (oldPart != part)) SETERRQ2(PETSC_ERR_PLIB, "Invalid partition %d should be %d", part, oldPart);
969: if ((oldPart == globalPart[vCol]) && (oldLocCol != locCol)) SETERRQ2(PETSC_ERR_PLIB, "Invalid local column %d should be %d", locCol, oldLocCol);
970: #endif
971: #endif
972: /* Multiply by the correct element of V (I - D^{-1} D) */
973: if (part == globalPart[vCol]) {
974: val += vals[col]*ml->factors[level][part][FACT_V][locCol*numCols+localCols[vCol]];
975: adj = PETSC_TRUE;
976: }
977: }
978: /* Store the dot product in B_Gamma V (I - D^{-1} D) */
979: if (adj == PETSC_TRUE) {
980: newCols[numVals] = globalCols[vCol];
981: newVals[numVals] = val;
982: numVals++;
983: }
984: }
985: /* This is an overestimate */
986: PetscLogFlops(numVals*ncols);
987: #if 0
988: #ifdef PETSC_USE_BOPT_g
989: if (numVals == 0) PetscLogInfo(pc, "Null row %d in reduced boundary gradient matrixn", row);
990: #endif
991: #endif
992: MatSetValues(grad, 1, &row, numVals, newCols, newVals, INSERT_VALUES);
993: MatRestoreRow(ml->grads[level], row, &ncols, &cols, &vals);
994: }
995: MatAssemblyBegin(grad, MAT_FINAL_ASSEMBLY);
996: MatAssemblyEnd(grad, MAT_FINAL_ASSEMBLY);
997: #ifdef NEW_BD_GRAD
998: PetscFree(firstCol);
999: PetscFree(colPart);
1000: #endif
1001: PC_MLLogEventEnd(PC_ML_CreateBdGrad, pc, 0, 0, 0);
1003: /* Construct the coarse graph */
1004: ml->numVertices[level+1] = newColsRemain;
1005: PCMultiLevelShrinkMesh_Private(pc, level, newColsRemain, colPartition, ml->numElements[level],
1006: ml->meshes[level], &ml->numElements[level+1], &ml->meshes[level+1]);
1007:
1009: /* Update variables */
1010: colsRemain = newColsRemain;
1011: VecRestoreArray(ml->colReduceVecs[0], &colReduceArray);
1012: VecRestoreArray(ml->colReduceVecs2[0], &colReduceArray2);
1014: /* Visualize factorization */
1015: PCMultiLevelReduceView(pc, level, colsRemain, colPartition,
1016: ((colsRemain > ml->QRthresh) && (rowsRemain > 0)), &anyAgain);
1017:
1018: }
1019: while(anyAgain) {
1020: PCMultiLevelReduceView(pc, -level, 0, colPartition, 0, &anyAgain);
1021: }
1022: ml->numLevels = level;
1024: /* Cleanup */
1025: PetscFree(colPartition);
1026: PetscFree(rowPartition);
1027: PetscFree(globalCols);
1028: if (ml->numLevels > 0) {
1029: MatDestroy(grad);
1030: }
1032: /* Workspace allocation */
1033: ml->interiorWorkLen = 1;
1034: for(level = 0; level < ml->numLevels; level++)
1035: for(part = 0; part < ml->numPartitions[level]; part++)
1036: ml->interiorWorkLen = PetscMax(ml->interiorWorkLen, ml->numPartitionRows[level][part]);
1037: PetscMalloc(ml->interiorWorkLen * sizeof(double), &ml->interiorWork);
1038: PetscMalloc(ml->interiorWorkLen * sizeof(double), &ml->interiorWork2);
1039: PetscLogObjectMemory(pc, ml->interiorWorkLen*2 * sizeof(double));
1041: /* Get the null space rows and compress the range */
1042: ml->globalRank = ml->rank;
1043: ml->numLocNullCols = ml->numCols - ml->rank;
1044: PetscMalloc(PetscMax(ml->numLocNullCols, 1) * sizeof(int), &ml->nullCols);
1045: PetscLogObjectMemory(pc, PetscMax(ml->numLocNullCols, 1) * sizeof(int));
1046: ml->nullCols[0] = -1;
1047: for(col = 0, nullCol = 0, numNullCols = 0, ierr2 = 0; nullCol < ml->numCols; col++, nullCol++) {
1048: if (ml->range[col] < 0) {
1049: if (numNullCols == ml->numLocNullCols) {
1050: int rank;
1052: MPI_Comm_rank(pc->comm, &rank);
1053: PetscPrintf(PETSC_COMM_SELF, "[%d]Range %d, Null %d:n", rank, ml->rank, ml->numLocNullCols);
1054: for(col = 0; col < ml->numCols; col++) PetscPrintf(PETSC_COMM_SELF, " %d", ml->range[col]);
1055: PetscPrintf(PETSC_COMM_SELF, "n");
1056: ierr2 = 1;
1057: break;
1058: }
1059: ml->nullCols[numNullCols++] = nullCol;
1060: PetscMemmove(&ml->range[col], &ml->range[col+1], (ml->numCols - (col+1)) * sizeof(int));
1061: col--;
1062: }
1063: }
1064: MPI_Allreduce(&ierr2, &invalidNull, 1, MPI_INT, MPI_LOR, pc->comm);
1065: if (invalidNull) SETERRQ(PETSC_ERR_ARG_CORRUPT, "Invalid null space");
1066: if (numNullCols != ml->numLocNullCols) SETERRQ(PETSC_ERR_ARG_CORRUPT, "Invalid number of null space columns");
1067: if (ml->numLocNullCols + ml->rank != ml->numCols) SETERRQ(PETSC_ERR_ARG_CORRUPT, "Invalid space decomposition");
1069: #ifdef PETSC_USE_BOPT_g
1070: PCValidQ_Multilevel(pc);
1071: #endif
1072: } else {
1073: Mesh mesh;
1075: /* Cleanup */
1076: GridGetMesh(ml->grid, &mesh);
1077: MeshDestroyLocalCSR(mesh, ml->meshes[0][MESH_OFFSETS], ml->meshes[0][MESH_ADJ], ml->vertices);
1078: #ifdef PETSC_HAVE_MATHEMATICA
1079: ViewerMathematicaReduce(ml->mathViewer, pc, ml->QRthresh);
1080: PetscFree(ml->meshes[0][MESH_ELEM]);
1081: PetscFree(ml->meshes[0]);
1082: PetscFree(ml->meshes);
1083: PetscFree(ml->numElements);
1084: PetscFree(ml->numVertices);
1085: /* PCConvert_Multilevel(pc); */
1086: #else
1087: SETERRQ(PETSC_ERR_SUP, " ");
1088: #endif
1089: }
1090: return(0);
1091: }
1093: int PCMultiLevelFilterVertexCoords_Tutte(PC pc, int numCols, double *vertices, int **mesh)
1094: {
1095: PetscRandom gen;
1096: double residual = 1.0;
1097: double tol = 1.0e-8;
1098: int iter = 0;
1099: int maxIter = 10000;
1100: int xminNode, xmaxNode, yminNode, ymaxNode;
1101: double xmin, xmax, ymin, ymax;
1102: PetscReal x, y;
1103: int node, neighbor;
1104: int ierr;
1107: if (numCols < 3) return(0);
1108: /* Fix boundary */
1109: PetscRandomCreate(PETSC_COMM_SELF, RANDOM_DEFAULT, &gen);
1110: PetscRandomSetInterval(gen, 0, numCols);
1111: xmin = xmax = vertices[0];
1112: ymin = ymax = vertices[1];
1113: xminNode = xmaxNode = yminNode = ymaxNode = 0;
1114: for(node = 0; node < numCols; node++) {
1115: if (vertices[node*2] < xmin) {
1116: xmin = vertices[node*2];
1117: xminNode = node;
1118: }
1119: if (vertices[node*2] > xmax) {
1120: xmax = vertices[node*2];
1121: xmaxNode = node;
1122: }
1123: if (vertices[node*2+1] < ymin) {
1124: ymin = vertices[node*2+1];
1125: yminNode = node;
1126: }
1127: if (vertices[node*2+1] > ymax) {
1128: ymax = vertices[node*2+1];
1129: ymaxNode = node;
1130: }
1131: }
1132: if ((xmaxNode == yminNode) && (xminNode == ymaxNode)) {
1133: while((xmaxNode == yminNode) || (xmaxNode == xminNode)) {
1134: PetscRandomGetValue(gen, &x);
1135: xmaxNode = (int) floor(x);
1136: }
1137: }
1138: if ((xmaxNode == ymaxNode) && (xminNode == yminNode)) {
1139: while((xmaxNode == ymaxNode) || (xmaxNode == xminNode)) {
1140: PetscRandomGetValue(gen, &x);
1141: xmaxNode = (int) floor(x);
1142: }
1143: }
1144: PetscRandomDestroy(gen);
1145: /* First pass: Jacobi Relaxation */
1146: while((residual > tol) && (iter < maxIter)) {
1147: for(node = 0; node < numCols; node++) {
1148: if ((node == xminNode) || (node == xmaxNode) || (node == yminNode) || (node == ymaxNode)) continue;
1149: vertices[node*2] = 0.0;
1150: vertices[node*2+1] = 0.0;
1151: for(neighbor = mesh[MESH_OFFSETS][node]; neighbor < mesh[MESH_OFFSETS][node+1]; neighbor++) {
1152: if (mesh[MESH_ADJ][neighbor] != node) {
1153: vertices[node*2] += vertices[mesh[MESH_ADJ][neighbor]*2];
1154: vertices[node*2+1] += vertices[mesh[MESH_ADJ][neighbor]*2+1];
1155: }
1156: }
1157: /* Remember to get rid of self edge */
1158: vertices[node*2] /= mesh[MESH_OFFSETS][node+1] - mesh[MESH_OFFSETS][node] - 1;
1159: vertices[node*2+1] /= mesh[MESH_OFFSETS][node+1] - mesh[MESH_OFFSETS][node] - 1;
1160: }
1161: for(node = 0, residual = 0.0; node < numCols; node++) {
1162: if ((node == xminNode) || (node == xmaxNode) || (node == yminNode) || (node == ymaxNode)) continue;
1163: x = 0.0;
1164: y = 0.0;
1165: for(neighbor = mesh[MESH_OFFSETS][node]; neighbor < mesh[MESH_OFFSETS][node+1]; neighbor++) {
1166: if (mesh[MESH_ADJ][neighbor] != node) {
1167: x += vertices[mesh[MESH_ADJ][neighbor]*2];
1168: y += vertices[mesh[MESH_ADJ][neighbor]*2+1];
1169: }
1170: }
1171: x /= mesh[MESH_OFFSETS][node+1] - mesh[MESH_OFFSETS][node] - 1;
1172: y /= mesh[MESH_OFFSETS][node+1] - mesh[MESH_OFFSETS][node] - 1;
1173: residual += PetscAbsReal(x - vertices[node*2]) + PetscAbsReal(y - vertices[node*2+1]);
1174: }
1175: residual /= numCols;
1176: iter++;
1177: }
1178: return(0);
1179: }
1181: int PCMultiLevelCalcVertexCoords_Private(PC pc, int numCols, int numNewCols, int *colPartition)
1182: {
1183: PC_Multilevel *ml = (PC_Multilevel *) pc->data;
1184: double *vertices;
1185: int *counts;
1186: int node;
1187: int ierr;
1190: if (numNewCols > 0) {
1191: PetscMalloc(numNewCols*2 * sizeof(double), &vertices);
1192: PetscMalloc(numNewCols * sizeof(int), &counts);
1193: PetscMemzero(vertices, numNewCols*2 * sizeof(double));
1194: PetscMemzero(counts, numNewCols * sizeof(int));
1195: for(node = 0; node < numCols; node++) {
1196: if (colPartition[node] >= 0) {
1197: vertices[colPartition[node]*2] += ml->vertices[node*2];
1198: vertices[colPartition[node]*2+1] += ml->vertices[node*2+1];
1199: counts[colPartition[node]]++;
1200: }
1201: }
1202: for(node = 0; node < numNewCols; node++) {
1203: vertices[node*2] /= counts[node];
1204: vertices[node*2+1] /= counts[node];
1205: }
1206: PetscFree(ml->vertices);
1207: PetscFree(counts);
1208: ml->vertices = vertices;
1209: }
1210: return(0);
1211: }
1213: int PCMultiLevelView_Mesh(PC pc, int numCols, int numElements, int **mesh)
1214: {
1215: PC_Multilevel *ml = (PC_Multilevel *) pc->data;
1216: PetscViewer viewer = ml->factorizationViewer;
1217: Mesh smallMesh;
1218: int *faces;
1219: int face;
1220: int ierr;
1223: if (numElements > 0) {
1224: PetscMalloc(numElements*3 * sizeof(int), &faces);
1225: PetscMemcpy(faces, mesh[MESH_ELEM], numElements*3 * sizeof(int));
1226: for(face = 0; face < numElements*3; face++) {
1227: faces[face]--;
1228: }
1229: }
1230: MeshCreateTriangular2DCSR(pc->comm, numCols, numElements, ml->vertices, mesh[MESH_OFFSETS], mesh[MESH_ADJ], faces, &smallMesh);
1231:
1232: if (numElements > 0) {
1233: PetscFree(faces);
1234: }
1235: MeshView(smallMesh, viewer);
1236: MeshDestroy(smallMesh);
1237: return(0);
1238: }
1240: int PCMultiLevelView_Private(PC pc, PetscViewer viewer, int level, int numCols, int numElements, int **mesh)
1241: {
1242: PetscDraw draw;
1243: char title[256];
1244: int maxLevel;
1245: int ierr;
1248: PetscViewerDrawClear(viewer);
1249: PetscViewerDrawGetDraw(viewer, 0, &draw);
1250: MPI_Allreduce(&level, &maxLevel, 1, MPI_INT, MPI_MAX, pc->comm);
1251: sprintf(title, "ML Mesh, Level %d", maxLevel);
1252: PetscDrawSetTitle(draw, title);
1253: PCMultiLevelView_Mesh(pc, numCols, numElements, mesh);
1254: return(0);
1255: }
1257: int PCMultiLevelReduceView(PC pc, int level, int numNewCols, int *colPartition, int again, int *anyAgain)
1258: {
1259: PC_Multilevel *ml = (PC_Multilevel *) pc->data;
1260: PetscViewer viewer = ml->factorizationViewer;
1261: int **newMesh;
1262: int numCols, numNewElements;
1263: int ierr;
1266: if (viewer != PETSC_NULL) {
1267: if (level < 0) {
1268: PCMultiLevelView_Private(pc, viewer, -1, numNewCols, 0, ml->meshes[-level]);
1269: } else {
1270: numCols = ml->numVertices[level];
1271: numNewElements = ml->numElements[level+1];
1272: newMesh = ml->meshes[level+1];
1273: PCMultiLevelCalcVertexCoords_Private(pc, numCols, numNewCols, colPartition);
1274: PCMultiLevelFilterVertexCoords_Tutte(pc, numNewCols, ml->vertices, newMesh);
1275: PCMultiLevelView_Private(pc, viewer, level+1, numNewCols, numNewElements, newMesh);
1276: }
1277: PetscViewerFlush(viewer);
1278: MPI_Allreduce(&again, anyAgain, 1, MPI_INT, MPI_LOR, pc->comm);
1279: }
1280: return(0);
1281: }