自学教程：C++ DMLocalToGlobalEnd函数代码示例

/**   Form the time step function for the mass term   @param Vec in:       (y^(n+1))   out @param Vec out:  My^(n+1) - M y^(n)    if timestep = 1, backward euler                                              will do the BDF2**/PetscErrorCode FormMassTimeStepFunction(User user, Algebra algebra, Vec in, Vec out, PetscBool rebuild){  PetscErrorCode  ierr;  PetscMPIInt     rank;  Vec            inLocal;  PetscFunctionBegin;  ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);  ierr = VecSet(out, 0.0);CHKERRQ(ierr);  ierr = DMGetLocalVector(user->dm, &inLocal);CHKERRQ(ierr);  ierr = DMGlobalToLocalBegin(user->dm, in, INSERT_VALUES, inLocal);CHKERRQ(ierr);  ierr = DMGlobalToLocalEnd(user->dm, in, INSERT_VALUES, inLocal);CHKERRQ(ierr);  ierr = ApplyBC(user->dm, user->current_time, inLocal, user);CHKERRQ(ierr);  ierr = CaculateLocalMassFunction(user->dm, inLocal, out, user);CHKERRQ(ierr);  ierr = DMLocalToGlobalBegin(user->dm, inLocal, INSERT_VALUES, in);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(user->dm, inLocal, INSERT_VALUES, in);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(user->dm, &inLocal);CHKERRQ(ierr);  PetscFunctionReturn(0);}

PetscErrorCode SetCoordinates3d(DM da){  PetscErrorCode ierr;  PetscInt       i,j,mstart,m,nstart,n,pstart,p,k;  Vec            gc,global;  DMDACoor3d     ***coors;  DM             cda;  PetscFunctionBeginUser;  ierr = DMDASetUniformCoordinates(da,0.0,1.0,0.0,1.0,0.0,1.0);CHKERRQ(ierr);  ierr = DMGetCoordinateDM(da,&cda);CHKERRQ(ierr);  ierr = DMGetCoordinatesLocal(da,&gc);CHKERRQ(ierr);  ierr = DMDAVecGetArray(cda,gc,&coors);CHKERRQ(ierr);  ierr = DMDAGetCorners(cda,&mstart,&nstart,&pstart,&m,&n,&p);CHKERRQ(ierr);  for (i=mstart; i<mstart+m; i++) {    for (j=nstart; j<nstart+n; j++) {      for (k=pstart; k<pstart+p; k++) {        if (i % 2) {          coors[k][j][i].x = coors[k][j][i-1].x + .1*(coors[k][j][i+1].x - coors[k][j][i-1].x);        }        if (j % 2) {          coors[k][j][i].y = coors[k][j-1][i].y + .3*(coors[k][j+1][i].y - coors[k][j-1][i].y);        }        if (k % 2) {          coors[k][j][i].z = coors[k-1][j][i].z + .4*(coors[k+1][j][i].z - coors[k-1][j][i].z);        }      }    }  }  ierr = DMDAVecRestoreArray(cda,gc,&coors);CHKERRQ(ierr);  ierr = DMGetCoordinates(da,&global);CHKERRQ(ierr);  ierr = DMLocalToGlobalBegin(cda,gc,INSERT_VALUES,global);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(cda,gc,INSERT_VALUES,global);CHKERRQ(ierr);  PetscFunctionReturn(0);}

PetscErrorCode SetCoordinates1d(DM da){  PetscErrorCode ierr;  PetscInt       i,start,m;  Vec            gc,global;  PetscScalar    *coors;  DM             cda;  PetscFunctionBeginUser;  ierr = DMDASetUniformCoordinates(da,0.0,1.0,0.0,1.0,0.0,1.0);CHKERRQ(ierr);  ierr = DMGetCoordinateDM(da,&cda);CHKERRQ(ierr);  ierr = DMGetCoordinatesLocal(da,&gc);CHKERRQ(ierr);  ierr = DMDAVecGetArray(cda,gc,&coors);CHKERRQ(ierr);  ierr = DMDAGetCorners(cda,&start,0,0,&m,0,0);CHKERRQ(ierr);  for (i=start; i<start+m; i++) {    if (i % 2) {      coors[i] = coors[i-1] + .1*(coors[i+1] - coors[i-1]);    }  }  ierr = DMDAVecRestoreArray(cda,gc,&coors);CHKERRQ(ierr);  ierr = DMGetCoordinates(da,&global);CHKERRQ(ierr);  ierr = DMLocalToGlobalBegin(cda,gc,INSERT_VALUES,global);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(cda,gc,INSERT_VALUES,global);CHKERRQ(ierr);  PetscFunctionReturn(0);}

/*      Evaluates /integral_{-1}^{1} f*v_i  where v_i is the ith basis polynomial via the GLL nodes and weights, since the v_i      basis function is zero at all nodes except the ith one the integral is simply the weight_i * f(node_i)*/PetscErrorCode ComputeRhs(DM da,PetscGLL *gll,Vec b){  PetscErrorCode ierr;  PetscInt       i,j,xs,xn,n = gll->n;  PetscScalar    *bb,*xx;  PetscReal      xd;  Vec            blocal,xlocal;  PetscFunctionBegin;  ierr = DMDAGetCorners(da,&xs,NULL,NULL,&xn,NULL,NULL);CHKERRQ(ierr);  xs   = xs/(n-1);  xn   = xn/(n-1);  ierr = DMGetLocalVector(da,&blocal);CHKERRQ(ierr);  ierr = VecZeroEntries(blocal);CHKERRQ(ierr);  ierr = DMDAVecGetArray(da,blocal,&bb);CHKERRQ(ierr);  ierr = DMGetCoordinatesLocal(da,&xlocal);CHKERRQ(ierr);  ierr = DMDAVecGetArray(da,xlocal,&xx);CHKERRQ(ierr);  /* loop over local spectral elements */  for (j=xs; j<xs+xn; j++) {    /* loop over GLL points in each element */    for (i=0; i<n; i++) {      xd              = xx[j*(n-1) + i];      bb[j*(n-1) + i] += -gll->weights[i]*(-20.*PETSC_PI*xd*PetscSinReal(5.*PETSC_PI*xd) + (2. - (5.*PETSC_PI)*(5.*PETSC_PI)*(xd*xd - 1.))*PetscCosReal(5.*PETSC_PI*xd));    }  }  ierr = DMDAVecRestoreArray(da,xlocal,&xx);CHKERRQ(ierr);  ierr = DMDAVecRestoreArray(da,blocal,&bb);CHKERRQ(ierr);  ierr = VecZeroEntries(b);CHKERRQ(ierr);  ierr = DMLocalToGlobalBegin(da,blocal,ADD_VALUES,b);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(da,blocal,ADD_VALUES,b);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(da,&blocal);CHKERRQ(ierr);  PetscFunctionReturn(0);}

/*@C  DMPlexProjectField - This projects the given function of the fields into the function space provided.  Input Parameters:+ dm      - The DM. U       - The input field vector. funcs   - The functions to evaluate, one per field- mode    - The insertion mode for values  Output Parameter:. X       - The output vector  Level: developer.seealso: DMPlexProjectFunction(), DMPlexComputeL2Diff()@*/PetscErrorCode DMPlexProjectField(DM dm, Vec U, void (**funcs)(const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscReal [], PetscScalar []), InsertMode mode, Vec X){  Vec            localX, localU;  PetscErrorCode ierr;  PetscFunctionBegin;  PetscValidHeaderSpecific(dm, DM_CLASSID, 1);  ierr = DMGetLocalVector(dm, &localX);CHKERRQ(ierr);  ierr = DMGetLocalVector(dm, &localU);CHKERRQ(ierr);  ierr = DMGlobalToLocalBegin(dm, U, INSERT_VALUES, localU);CHKERRQ(ierr);  ierr = DMGlobalToLocalEnd(dm, U, INSERT_VALUES, localU);CHKERRQ(ierr);  ierr = DMPlexProjectFieldLocal(dm, localU, funcs, mode, localX);CHKERRQ(ierr);  ierr = DMLocalToGlobalBegin(dm, localX, mode, X);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(dm, localX, mode, X);CHKERRQ(ierr);  if (mode == INSERT_VALUES || mode == INSERT_ALL_VALUES || mode == INSERT_BC_VALUES) {    Mat cMat;    ierr = DMGetDefaultConstraints(dm, NULL, &cMat);CHKERRQ(ierr);    if (cMat) {      ierr = DMGlobalToLocalSolve(dm, localX, X);CHKERRQ(ierr);    }  }  ierr = DMRestoreLocalVector(dm, &localX);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(dm, &localU);CHKERRQ(ierr);  PetscFunctionReturn(0);}

/*  We integrate over each cell  (i, j+1)----(i+1, j+1)      | /         |      |  /        |      |   /       |      |    /      |      |     /     |      |      /    |      |       /   |  (i,   j)----(i+1, j)*/PetscErrorCode ComputeRHS(KSP ksp, Vec b, void *ctx){  UserContext    *user = (UserContext*)ctx;  PetscScalar    phi   = user->phi;  PetscScalar    *array;  PetscInt       ne,nc,i;  const PetscInt *e;  PetscErrorCode ierr;  Vec            blocal;  DM             da;  PetscFunctionBeginUser;  ierr = KSPGetDM(ksp,&da);CHKERRQ(ierr);  /* access a local vector with room for the ghost points */  ierr = DMGetLocalVector(da,&blocal);CHKERRQ(ierr);  ierr = VecGetArray(blocal, (PetscScalar**) &array);CHKERRQ(ierr);  /* access the list of elements on this processor and loop over them */  ierr = DMDAGetElements(da,&ne,&nc,&e);CHKERRQ(ierr);  for (i=0; i<ne; i++) {    /* this is nonsense, but set each nodal value to phi (will actually do integration over element */    array[e[3*i]]   = phi;    array[e[3*i+1]] = phi;    array[e[3*i+2]] = phi;  }  ierr = VecRestoreArray(blocal, (PetscScalar**) &array);CHKERRQ(ierr);  ierr = DMDARestoreElements(da,&ne,&nc,&e);CHKERRQ(ierr);  /* add our partial sums over all processors into b */  ierr = DMLocalToGlobalBegin(da,blocal,ADD_VALUES,b);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(da,blocal, ADD_VALUES,b);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(da,&blocal);CHKERRQ(ierr);  PetscFunctionReturn(0);}

static PetscErrorCode SNESComputeFunction_DMLocal(SNES snes,Vec X,Vec F,void *ctx){  PetscErrorCode ierr;  DM             dm;  DMSNES_Local   *dmlocalsnes = (DMSNES_Local*)ctx;  Vec            Xloc,Floc;  PetscFunctionBegin;  PetscValidHeaderSpecific(snes,SNES_CLASSID,1);  PetscValidHeaderSpecific(X,VEC_CLASSID,2);  PetscValidHeaderSpecific(F,VEC_CLASSID,3);  ierr = SNESGetDM(snes,&dm);CHKERRQ(ierr);  ierr = DMGetLocalVector(dm,&Xloc);CHKERRQ(ierr);  ierr = DMGetLocalVector(dm,&Floc);CHKERRQ(ierr);  ierr = VecZeroEntries(Xloc);CHKERRQ(ierr);  ierr = VecZeroEntries(Floc);CHKERRQ(ierr);  ierr = DMGlobalToLocalBegin(dm,X,INSERT_VALUES,Xloc);CHKERRQ(ierr);  ierr = DMGlobalToLocalEnd(dm,X,INSERT_VALUES,Xloc);CHKERRQ(ierr);  CHKMEMQ;  ierr = (*dmlocalsnes->residuallocal)(dm,Xloc,Floc,dmlocalsnes->residuallocalctx);CHKERRQ(ierr);  CHKMEMQ;  ierr = VecZeroEntries(F);CHKERRQ(ierr);  ierr = DMLocalToGlobalBegin(dm,Floc,ADD_VALUES,F);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(dm,Floc,ADD_VALUES,F);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(dm,&Floc);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(dm,&Xloc);CHKERRQ(ierr);  PetscFunctionReturn(0);}

/*    FormInitialGuess - Forms initial approximation.   Input Parameters:   user - user-defined application context   X - vector   Output Parameter:   X - vector */PetscErrorCode FormInitialGuess(AppCtx *user,Vec X){  PetscInt       i,j,row,mx,my,xs,ys,xm,ym,gxm,gym,gxs,gys;  PetscErrorCode ierr;  PetscReal      one = 1.0,lambda,temp1,temp,hx,hy,hxdhy,hydhx,sc;  PetscScalar    *x;  Vec            localX = user->localX;  mx = user->mx;            my = user->my;            lambda = user->param;  hx = one/(PetscReal)(mx-1);  hy = one/(PetscReal)(my-1);  sc = hx*hy*lambda;        hxdhy = hx/hy;            hydhx = hy/hx;  temp1 = lambda/(lambda + one);  /*     Get a pointer to vector data.       - For default PETSc vectors,VecGetArray() returns a pointer to         the data array.  Otherwise, the routine is implementation dependent.       - You MUST call VecRestoreArray() when you no longer need access to         the array.  */  ierr = VecGetArray(localX,&x);CHKERRQ(ierr);  /*     Get local grid boundaries (for 2-dimensional DMDA):       xs, ys   - starting grid indices (no ghost points)       xm, ym   - widths of local grid (no ghost points)       gxs, gys - starting grid indices (including ghost points)       gxm, gym - widths of local grid (including ghost points)  */  ierr = DMDAGetCorners(user->da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL);CHKERRQ(ierr);  ierr = DMDAGetGhostCorners(user->da,&gxs,&gys,PETSC_NULL,&gxm,&gym,PETSC_NULL);CHKERRQ(ierr);  /*     Compute initial guess over the locally owned part of the grid  */  for (j=ys; j<ys+ym; j++) {    temp = (PetscReal)(PetscMin(j,my-j-1))*hy;    for (i=xs; i<xs+xm; i++) {      row = i - gxs + (j - gys)*gxm;       if (i == 0 || j == 0 || i == mx-1 || j == my-1) {        x[row] = 0.0;         continue;      }      x[row] = temp1*PetscSqrtReal(PetscMin((PetscReal)(PetscMin(i,mx-i-1))*hx,temp));     }  }  /*     Restore vector  */  ierr = VecRestoreArray(localX,&x);CHKERRQ(ierr);  /*     Insert values into global vector  */  ierr = DMLocalToGlobalBegin(user->da,localX,INSERT_VALUES,X);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(user->da,localX,INSERT_VALUES,X);CHKERRQ(ierr);  return 0;} 

PetscErrorCode OpForcing(Op op,DM dm,Vec F) {  PetscErrorCode ierr;  Vec X,Floc;  DM dmx;  const PetscScalar *x;  PetscScalar *f;  const PetscReal *B,*D,*w3;  PetscReal L[3];  PetscInt nelem,ne = op->ne,P,Q,P3,Q3;  PetscFunctionBegin;  ierr = PetscLogEventBegin(OP_Forcing,dm,F,0,0);CHKERRQ(ierr);  ierr = DMFEGetTensorEval(dm,&P,&Q,&B,&D,NULL,NULL,&w3);CHKERRQ(ierr);  P3 = P*P*P;  Q3 = Q*Q*Q;  ierr = DMFEGetUniformCoordinates(dm,L);CHKERRQ(ierr);  ierr = DMGetLocalVector(dm,&Floc);CHKERRQ(ierr);  ierr = DMGetCoordinateDM(dm,&dmx);CHKERRQ(ierr);  ierr = DMGetCoordinatesLocal(dm,&X);CHKERRQ(ierr);  ierr = DMFEGetNumElements(dm,&nelem);CHKERRQ(ierr);  ierr = VecGetArrayRead(X,&x);CHKERRQ(ierr);  ierr = VecZeroEntries(Floc);CHKERRQ(ierr);  ierr = VecGetArray(Floc,&f);CHKERRQ(ierr);  for (PetscInt e=0; e<nelem; e+=ne) {    PetscScalar fe[op->dof*P3*ne]_align,fq[op->dof][Q3][ne]_align,xe[3*P3*ne]_align,xq[3][Q3][ne]_align,dx[3][3][Q3][ne]_align,wdxdet[Q3][ne]_align;    ierr = DMFEExtractElements(dmx,x,e,ne,xe);CHKERRQ(ierr);    ierr = PetscMemzero(xq,sizeof xq);CHKERRQ(ierr);    ierr = TensorContract(op->Tensor3,B,B,B,TENSOR_EVAL,xe,xq[0][0]);CHKERRQ(ierr);    ierr = PetscMemzero(dx,sizeof dx);CHKERRQ(ierr);    ierr = TensorContract(op->Tensor3,D,B,B,TENSOR_EVAL,xe,dx[0][0][0]);CHKERRQ(ierr);    ierr = TensorContract(op->Tensor3,B,D,B,TENSOR_EVAL,xe,dx[1][0][0]);CHKERRQ(ierr);    ierr = TensorContract(op->Tensor3,B,B,D,TENSOR_EVAL,xe,dx[2][0][0]);CHKERRQ(ierr);    ierr = PointwiseJacobianInvert(ne,Q*Q*Q,w3,dx,wdxdet);CHKERRQ(ierr);    for (PetscInt i=0; i<Q3; i++) {      for (PetscInt l=0; l<ne; l++) {        PetscReal xx[] = {xq[0][i][l],xq[1][i][l],xq[2][i][l]};        PetscScalar fql[op->dof];        ierr = (op->PointwiseForcing)(op,xx,L,fql);CHKERRQ(ierr);        for (PetscInt d=0; d<op->dof; d++) fq[d][i][l] = wdxdet[i][l] * fql[d];      }    }    ierr = PetscMemzero(fe,sizeof fe);CHKERRQ(ierr);    ierr = TensorContract(op->TensorDOF,B,B,B,TENSOR_TRANSPOSE,fq[0][0],fe);CHKERRQ(ierr);    ierr = DMFESetElements(dm,f,e,ne,ADD_VALUES,DOMAIN_INTERIOR,fe);CHKERRQ(ierr);  }  ierr = VecRestoreArrayRead(X,&x);CHKERRQ(ierr);  ierr = VecRestoreArray(Floc,&f);CHKERRQ(ierr);  ierr = VecZeroEntries(F);CHKERRQ(ierr);  ierr = DMLocalToGlobalBegin(dm,Floc,ADD_VALUES,F);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(dm,Floc,ADD_VALUES,F);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(dm,&Floc);CHKERRQ(ierr);  ierr = PetscLogEventEnd(OP_Forcing,dm,F,0,0);CHKERRQ(ierr);  PetscFunctionReturn(0);}

PetscErrorCode DMLocalToGlobalEnd_Network(DM dm, Vec l, InsertMode mode, Vec g){  PetscErrorCode ierr;  DM_Network     *network = (DM_Network*) dm->data;  PetscFunctionBegin;  ierr = DMLocalToGlobalEnd(network->plex,l,mode,g);CHKERRQ(ierr);  PetscFunctionReturn(0);}

static PetscErrorCode SNESComputeFunction_DMDA(SNES snes,Vec X,Vec F,void *ctx){  PetscErrorCode ierr;  DM             dm;  DMSNES_DA      *dmdasnes = (DMSNES_DA*)ctx;  DMDALocalInfo  info;  Vec            Xloc;  void           *x,*f;  PetscFunctionBegin;  PetscValidHeaderSpecific(snes,SNES_CLASSID,1);  PetscValidHeaderSpecific(X,VEC_CLASSID,2);  PetscValidHeaderSpecific(F,VEC_CLASSID,3);  if (!dmdasnes->residuallocal) SETERRQ(PetscObjectComm((PetscObject)snes),PETSC_ERR_PLIB,"Corrupt context");  ierr = SNESGetDM(snes,&dm);CHKERRQ(ierr);  ierr = DMGetLocalVector(dm,&Xloc);CHKERRQ(ierr);  ierr = DMGlobalToLocalBegin(dm,X,INSERT_VALUES,Xloc);CHKERRQ(ierr);  ierr = DMGlobalToLocalEnd(dm,X,INSERT_VALUES,Xloc);CHKERRQ(ierr);  ierr = DMDAGetLocalInfo(dm,&info);CHKERRQ(ierr);  ierr = DMDAVecGetArray(dm,Xloc,&x);CHKERRQ(ierr);  switch (dmdasnes->residuallocalimode) {  case INSERT_VALUES: {    ierr = DMDAVecGetArray(dm,F,&f);CHKERRQ(ierr);    ierr = PetscLogEventBegin(SNES_FunctionEval,snes,X,F,0);CHKERRQ(ierr);    CHKMEMQ;    ierr = (*dmdasnes->residuallocal)(&info,x,f,dmdasnes->residuallocalctx);CHKERRQ(ierr);    CHKMEMQ;    ierr = PetscLogEventEnd(SNES_FunctionEval,snes,X,F,0);CHKERRQ(ierr);    ierr = DMDAVecRestoreArray(dm,F,&f);CHKERRQ(ierr);  } break;  case ADD_VALUES: {    Vec Floc;    ierr = DMGetLocalVector(dm,&Floc);CHKERRQ(ierr);    ierr = VecZeroEntries(Floc);CHKERRQ(ierr);    ierr = DMDAVecGetArray(dm,Floc,&f);CHKERRQ(ierr);    ierr = PetscLogEventBegin(SNES_FunctionEval,snes,X,F,0);CHKERRQ(ierr);    CHKMEMQ;    ierr = (*dmdasnes->residuallocal)(&info,x,f,dmdasnes->residuallocalctx);CHKERRQ(ierr);    CHKMEMQ;    ierr = PetscLogEventEnd(SNES_FunctionEval,snes,X,F,0);CHKERRQ(ierr);    ierr = DMDAVecRestoreArray(dm,Floc,&f);CHKERRQ(ierr);    ierr = VecZeroEntries(F);CHKERRQ(ierr);    ierr = DMLocalToGlobalBegin(dm,Floc,ADD_VALUES,F);CHKERRQ(ierr);    ierr = DMLocalToGlobalEnd(dm,Floc,ADD_VALUES,F);CHKERRQ(ierr);    ierr = DMRestoreLocalVector(dm,&Floc);CHKERRQ(ierr);  } break;  default: SETERRQ1(PetscObjectComm((PetscObject)snes),PETSC_ERR_ARG_INCOMP,"Cannot use imode=%d",(int)dmdasnes->residuallocalimode);  }  ierr = DMDAVecRestoreArray(dm,Xloc,&x);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(dm,&Xloc);CHKERRQ(ierr);  if (snes->domainerror) {    ierr = VecSetInf(F);CHKERRQ(ierr);  }  PetscFunctionReturn(0);}

PetscErrorCode FormResidual(TS ts,PetscReal t,Vec U,Vec Udot,Vec R,void *user){  PetscFunctionBegin;  PetscErrorCode ierr;  DMDALocalInfo    info;  DM               dm;  Vec              localU,localUdot,localR; // local versions  Field          **h,**hdot,**r;  /* get the da from the snes */  ierr = TSGetDM(ts, &dm);CHKERRQ(ierr);  /* handle the vec U */  ierr = DMGetLocalVector(dm,&localU);CHKERRQ(ierr);  ierr = DMGlobalToLocalBegin(dm,U,INSERT_VALUES,localU);CHKERRQ(ierr);  ierr = DMGlobalToLocalEnd(dm,U,INSERT_VALUES,localU);CHKERRQ(ierr);  /* handle the vec Udot */  ierr = DMGetLocalVector(dm,&localUdot);CHKERRQ(ierr);  ierr = DMGlobalToLocalBegin(dm,Udot,INSERT_VALUES,localUdot);CHKERRQ(ierr);  ierr = DMGlobalToLocalEnd(dm,Udot,INSERT_VALUES,localUdot);CHKERRQ(ierr);  /* handle the vec R */  ierr = DMGetLocalVector(dm,&localR);CHKERRQ(ierr);  ierr = VecZeroEntries(localR);CHKERRQ(ierr);  /* Get the arrays from the vectors */  ierr = DMIGAVecGetArray(dm,localU,&h);CHKERRQ(ierr);  ierr = DMIGAVecGetArray(dm,localUdot,&hdot);CHKERRQ(ierr);  ierr = DMIGAVecGetArray(dm,localR,&r);CHKERRQ(ierr);  /* Grab the local info and call the local residual routine */  ierr = DMIGAGetLocalInfo(dm,&info);CHKERRQ(ierr);CHKERRQ(ierr);  ierr = FormResidualLocal(&info,t,h,hdot,r,(AppCtx *) user);CHKERRQ(ierr);  /* Restore the arrays */  ierr = DMIGAVecRestoreArray(dm,localR,&r);CHKERRQ(ierr);  ierr = DMIGAVecRestoreArray(dm,localUdot,&hdot);CHKERRQ(ierr);  ierr = DMIGAVecRestoreArray(dm,localU,&h);CHKERRQ(ierr);  /* Add contributions back to global R */  ierr = VecZeroEntries(R);CHKERRQ(ierr);  ierr = DMLocalToGlobalBegin(dm,localR,ADD_VALUES,R);CHKERRQ(ierr); // this one adds the values  ierr = DMLocalToGlobalEnd(dm,localR,ADD_VALUES,R);CHKERRQ(ierr);  /* Restore the local vectors */  ierr = DMRestoreLocalVector(dm,&localU);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(dm,&localUdot);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(dm,&localR);CHKERRQ(ierr);  PetscFunctionReturn(0);}

PETSC_INTERN PetscErrorCode DMStagSetUniformCoordinatesExplicit_1d(DM dm,PetscReal xmin,PetscReal xmax){  PetscErrorCode ierr;  DM_Stag        *stagCoord;  DM             dmCoord;  Vec            coordLocal,coord;  PetscReal      h,min;  PetscScalar    **arr;  PetscInt       ind,start,n,nExtra,s;  PetscInt       ileft,ielement;  PetscFunctionBegin;  ierr = DMGetCoordinateDM(dm, &dmCoord);CHKERRQ(ierr);  stagCoord = (DM_Stag*) dmCoord->data;  for (s=0; s<2; ++s) {    if (stagCoord->dof[s] !=0 && stagCoord->dof[s] != 1) SETERRQ2(PetscObjectComm((PetscObject)dm),PETSC_ERR_PLIB,"Coordinate DM in 1 dimensions must have 0 or 1 dof on each stratum, but stratum %d has %d dof",s,stagCoord->dof[s]);  }  ierr = DMGetLocalVector(dmCoord,&coordLocal);CHKERRQ(ierr);  ierr = DMStagVecGetArrayDOF(dmCoord,coordLocal,&arr);CHKERRQ(ierr);  if (stagCoord->dof[0]) {    ierr = DMStagGetLocationSlot(dmCoord,DMSTAG_LEFT,0,&ileft);CHKERRQ(ierr);  }  if (stagCoord->dof[1]) {    ierr = DMStagGetLocationSlot(dmCoord,DMSTAG_ELEMENT,0,&ielement);CHKERRQ(ierr);  }  ierr = DMStagGetCorners(dmCoord,&start,NULL,NULL,&n,NULL,NULL,&nExtra,NULL,NULL);CHKERRQ(ierr);  min = xmin;  h = (xmax-xmin)/stagCoord->N[0];  for(ind=start; ind<start + n + nExtra; ++ind) {    if (stagCoord->dof[0]) {      const PetscReal off = 0.0;        arr[ind][ileft] = min + ((PetscReal)ind + off) * h;    }    if (stagCoord->dof[1]) {      const PetscReal off = 0.5;        arr[ind][ielement] = min + ((PetscReal)ind + off) * h;    }  }  ierr = DMStagVecRestoreArrayDOF(dmCoord,coordLocal,&arr);CHKERRQ(ierr);  ierr = DMCreateGlobalVector(dmCoord,&coord);CHKERRQ(ierr);  ierr = DMLocalToGlobalBegin(dmCoord,coordLocal,INSERT_VALUES,coord);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(dmCoord,coordLocal,INSERT_VALUES,coord);CHKERRQ(ierr);  ierr = DMSetCoordinates(dm,coord);CHKERRQ(ierr);  ierr = PetscLogObjectParent((PetscObject)dm,(PetscObject)coord);CHKERRQ(ierr);  ierr = VecDestroy(&coord);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(dmCoord,&coordLocal);CHKERRQ(ierr);  PetscFunctionReturn(0);}

/*  This function should eventually replace:    DMDAComputeFunction() and DMDAComputeFunction1() */static PetscErrorCode TSComputeIFunction_DMDA(TS ts,PetscReal ptime,Vec X,Vec Xdot,Vec F,void *ctx){  PetscErrorCode ierr;  DM             dm;  DMTS_DA        *dmdats = (DMTS_DA*)ctx;  DMDALocalInfo  info;  Vec            Xloc;  void           *x,*f,*xdot;  PetscFunctionBegin;  PetscValidHeaderSpecific(ts,TS_CLASSID,1);  PetscValidHeaderSpecific(X,VEC_CLASSID,2);  PetscValidHeaderSpecific(F,VEC_CLASSID,3);  if (!dmdats->ifunctionlocal) SETERRQ(PetscObjectComm((PetscObject)ts),PETSC_ERR_PLIB,"Corrupt context");  ierr = TSGetDM(ts,&dm);CHKERRQ(ierr);  ierr = DMGetLocalVector(dm,&Xloc);CHKERRQ(ierr);  ierr = DMGlobalToLocalBegin(dm,X,INSERT_VALUES,Xloc);CHKERRQ(ierr);  ierr = DMGlobalToLocalEnd(dm,X,INSERT_VALUES,Xloc);CHKERRQ(ierr);  ierr = DMDAGetLocalInfo(dm,&info);CHKERRQ(ierr);  ierr = DMDAVecGetArray(dm,Xloc,&x);CHKERRQ(ierr);  ierr = DMDAVecGetArray(dm,Xdot,&xdot);CHKERRQ(ierr);  switch (dmdats->ifunctionlocalimode) {  case INSERT_VALUES: {    ierr = DMDAVecGetArray(dm,F,&f);CHKERRQ(ierr);    CHKMEMQ;    ierr = (*dmdats->ifunctionlocal)(&info,ptime,x,xdot,f,dmdats->ifunctionlocalctx);CHKERRQ(ierr);    CHKMEMQ;    ierr = DMDAVecRestoreArray(dm,F,&f);CHKERRQ(ierr);  } break;  case ADD_VALUES: {    Vec Floc;    ierr = DMGetLocalVector(dm,&Floc);CHKERRQ(ierr);    ierr = VecZeroEntries(Floc);CHKERRQ(ierr);    ierr = DMDAVecGetArray(dm,Floc,&f);CHKERRQ(ierr);    CHKMEMQ;    ierr = (*dmdats->ifunctionlocal)(&info,ptime,x,xdot,f,dmdats->ifunctionlocalctx);CHKERRQ(ierr);    CHKMEMQ;    ierr = DMDAVecRestoreArray(dm,Floc,&f);CHKERRQ(ierr);    ierr = VecZeroEntries(F);CHKERRQ(ierr);    ierr = DMLocalToGlobalBegin(dm,Floc,ADD_VALUES,F);CHKERRQ(ierr);    ierr = DMLocalToGlobalEnd(dm,Floc,ADD_VALUES,F);CHKERRQ(ierr);    ierr = DMRestoreLocalVector(dm,&Floc);CHKERRQ(ierr);  } break;  default: SETERRQ1(PetscObjectComm((PetscObject)ts),PETSC_ERR_ARG_INCOMP,"Cannot use imode=%d",(int)dmdats->ifunctionlocalimode);  }  ierr = DMDAVecRestoreArray(dm,Xloc,&x);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(dm,&Xloc);CHKERRQ(ierr);  ierr = DMDAVecRestoreArray(dm,Xdot,&xdot);CHKERRQ(ierr);  PetscFunctionReturn(0);}

PetscErrorCode SNESNASMComputeFinalJacobian_Private(SNES snes, Vec Xfinal){  Vec            X = Xfinal;  SNES_NASM      *nasm = (SNES_NASM*)snes->data;  SNES           subsnes;  PetscInt       i,lag = 1;  PetscErrorCode ierr;  Vec            Xlloc,Xl,Fl,F;  VecScatter     oscat,gscat;  DM             dm,subdm;  MatStructure   flg = DIFFERENT_NONZERO_PATTERN;  PetscFunctionBegin;  if (nasm->fjtype == 2) X = nasm->xinit;  F = snes->vec_func;  if (snes->normschedule == SNES_NORM_NONE) {ierr = SNESComputeFunction(snes,X,F);CHKERRQ(ierr);}  ierr = SNESComputeJacobian(snes,X,&snes->jacobian,&snes->jacobian_pre,&flg);CHKERRQ(ierr);  ierr = SNESGetDM(snes,&dm);CHKERRQ(ierr);  if (nasm->eventrestrictinterp) {ierr = PetscLogEventBegin(nasm->eventrestrictinterp,snes,0,0,0);CHKERRQ(ierr);}  if (nasm->fjtype != 1) {    for (i=0; i<nasm->n; i++) {      Xlloc = nasm->xl[i];      gscat = nasm->gscatter[i];      oscat = nasm->oscatter[i];      ierr = VecScatterBegin(gscat,X,Xlloc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);    }  }  if (nasm->eventrestrictinterp) {ierr = PetscLogEventEnd(nasm->eventrestrictinterp,snes,0,0,0);CHKERRQ(ierr);}  for (i=0; i<nasm->n; i++) {    Fl      = nasm->subsnes[i]->vec_func;    Xl      = nasm->x[i];    Xlloc   = nasm->xl[i];    subsnes = nasm->subsnes[i];    oscat   = nasm->oscatter[i];    gscat   = nasm->gscatter[i];    if (nasm->fjtype != 1) {ierr = VecScatterEnd(gscat,X,Xlloc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);}    ierr = SNESGetDM(subsnes,&subdm);CHKERRQ(ierr);    ierr = DMSubDomainRestrict(dm,oscat,gscat,subdm);CHKERRQ(ierr);    if (nasm->fjtype != 1) {      ierr = DMLocalToGlobalBegin(subdm,Xlloc,INSERT_VALUES,Xl);CHKERRQ(ierr);      ierr = DMLocalToGlobalEnd(subdm,Xlloc,INSERT_VALUES,Xl);CHKERRQ(ierr);    }    if (subsnes->lagjacobian == -1)    subsnes->lagjacobian = -2;    else if (subsnes->lagjacobian > 1) lag = subsnes->lagjacobian;    ierr = SNESComputeFunction(subsnes,Xl,Fl);CHKERRQ(ierr);    ierr = SNESComputeJacobian(subsnes,Xl,&subsnes->jacobian,&subsnes->jacobian_pre,&flg);CHKERRQ(ierr);    if (lag > 1) subsnes->lagjacobian = lag;    ierr = KSPSetOperators(subsnes->ksp,subsnes->jacobian,subsnes->jacobian_pre,flg);CHKERRQ(ierr);  }  PetscFunctionReturn(0);}

PetscErrorCode CreatePressureNullSpace(DM dm, AppCtx *user, MatNullSpace *nullSpace){  Vec            vec, localVec;  PetscErrorCode ierr;  PetscFunctionBeginUser;  ierr = DMGetGlobalVector(dm, &vec);CHKERRQ(ierr);  ierr = DMGetLocalVector(dm, &localVec);CHKERRQ(ierr);  ierr = VecSet(vec,  0.0);CHKERRQ(ierr);  /* Put a constant in for all pressures     Could change this to project the constant function onto the pressure space (when that is finished) */  {    PetscSection section;    PetscInt     pStart, pEnd, p;    PetscScalar  *a;    ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);    ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);    ierr = VecGetArray(localVec, &a);CHKERRQ(ierr);    for (p = pStart; p < pEnd; ++p) {      PetscInt fDim, off, d;      ierr = PetscSectionGetFieldDof(section, p, 1, &fDim);CHKERRQ(ierr);      ierr = PetscSectionGetFieldOffset(section, p, 1, &off);CHKERRQ(ierr);      for (d = 0; d < fDim; ++d) a[off+d] = 1.0;    }    ierr = VecRestoreArray(localVec, &a);CHKERRQ(ierr);  }  ierr = DMLocalToGlobalBegin(dm, localVec, INSERT_VALUES, vec);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(dm, localVec, INSERT_VALUES, vec);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(dm, &localVec);CHKERRQ(ierr);  ierr = VecNormalize(vec, NULL);CHKERRQ(ierr);  if (user->debug) {    ierr = PetscPrintf(PetscObjectComm((PetscObject)dm), "Pressure Null Space/n");CHKERRQ(ierr);    ierr = VecView(vec, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);  }  ierr = MatNullSpaceCreate(PetscObjectComm((PetscObject)dm), PETSC_FALSE, 1, &vec, nullSpace);CHKERRQ(ierr);  ierr = DMRestoreGlobalVector(dm, &vec);CHKERRQ(ierr);  /* New style for field null spaces */  {    PetscObject  pressure;    MatNullSpace nullSpacePres;    ierr = DMGetField(dm, 1, &pressure);CHKERRQ(ierr);    ierr = MatNullSpaceCreate(PetscObjectComm(pressure), PETSC_TRUE, 0, NULL, &nullSpacePres);CHKERRQ(ierr);    ierr = PetscObjectCompose(pressure, "nullspace", (PetscObject) nullSpacePres);CHKERRQ(ierr);    ierr = MatNullSpaceDestroy(&nullSpacePres);CHKERRQ(ierr);  }  PetscFunctionReturn(0);}

PetscErrorCode SetInitialValues(DM networkdm,Vec X,void* appctx){  PetscErrorCode ierr;  VERTEXDATA     bus;  GEN            gen;  PetscInt       v, vStart, vEnd, offset;  PetscBool      ghostvtex;  Vec            localX;  PetscScalar    *xarr;  PetscInt       key,numComps,j,offsetd;  DMNetworkComponentGenericDataType *arr;    PetscFunctionBegin;  ierr = DMNetworkGetVertexRange(networkdm,&vStart, &vEnd);CHKERRQ(ierr);  ierr = DMGetLocalVector(networkdm,&localX);CHKERRQ(ierr);  ierr = VecSet(X,0.0);CHKERRQ(ierr);  ierr = DMGlobalToLocalBegin(networkdm,X,INSERT_VALUES,localX);CHKERRQ(ierr);  ierr = DMGlobalToLocalEnd(networkdm,X,INSERT_VALUES,localX);CHKERRQ(ierr);  ierr = VecGetArray(localX,&xarr);CHKERRQ(ierr);  ierr = DMNetworkGetComponentDataArray(networkdm,&arr);CHKERRQ(ierr);  for (v = vStart; v < vEnd; v++) {    ierr = DMNetworkIsGhostVertex(networkdm,v,&ghostvtex);CHKERRQ(ierr);    if (ghostvtex) continue;       ierr = DMNetworkGetVariableOffset(networkdm,v,&offset);CHKERRQ(ierr);    ierr = DMNetworkGetNumComponents(networkdm,v,&numComps);CHKERRQ(ierr);    for (j=0; j < numComps; j++) {      ierr = DMNetworkGetComponentTypeOffset(networkdm,v,j,&key,&offsetd);CHKERRQ(ierr);      if (key == 1) {	bus = (VERTEXDATA)(arr+offsetd);	xarr[offset] = bus->va*PETSC_PI/180.0;	xarr[offset+1] = bus->vm;      } else if(key == 2) {	gen = (GEN)(arr+offsetd);	if (!gen->status) continue;	xarr[offset+1] = gen->vs; 	break;      }    }  }  ierr = VecRestoreArray(localX,&xarr);CHKERRQ(ierr);  ierr = DMLocalToGlobalBegin(networkdm,localX,ADD_VALUES,X);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(networkdm,localX,ADD_VALUES,X);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(networkdm,&localX);CHKERRQ(ierr);  PetscFunctionReturn(0);}

PetscErrorCode FormFunction(SNES snes,Vec X,Vec F,void *ptr){  AppCtx         *user = (AppCtx *) ptr;  PetscInt       i, j, row, mx, my, xs, ys, xm, ym, Xs, Ys, Xm, Ym;  PetscErrorCode ierr;  double         two = 2.0, one = 1.0, lambda,hx, hy, hxdhy, hydhx,sc;  PetscScalar    u, uxx, uyy, *x,*f;  Vec            localX = user->fine.localX, localF = user->fine.localF;   mx = user->fine.mx;       my = user->fine.my;       lambda = user->param;  hx = one/(double)(mx-1);  hy = one/(double)(my-1);  sc = hx*hy*lambda;        hxdhy = hx/hy;            hydhx = hy/hx;  /* Get ghost points */  ierr = DMGlobalToLocalBegin(user->fine.da,X,INSERT_VALUES,localX);CHKERRQ(ierr);  ierr = DMGlobalToLocalEnd(user->fine.da,X,INSERT_VALUES,localX);CHKERRQ(ierr);  ierr = DMDAGetCorners(user->fine.da,&xs,&ys,0,&xm,&ym,0);CHKERRQ(ierr);  ierr = DMDAGetGhostCorners(user->fine.da,&Xs,&Ys,0,&Xm,&Ym,0);CHKERRQ(ierr);  ierr = VecGetArray(localX,&x);CHKERRQ(ierr);  ierr = VecGetArray(localF,&f);CHKERRQ(ierr);  /* Evaluate function */  for (j=ys; j<ys+ym; j++) {    row = (j - Ys)*Xm + xs - Xs - 1;     for (i=xs; i<xs+xm; i++) {      row++;      if (i > 0 && i < mx-1 && j > 0 && j < my-1) {        u = x[row];        uxx = (two*u - x[row-1] - x[row+1])*hydhx;        uyy = (two*u - x[row-Xm] - x[row+Xm])*hxdhy;        f[row] = uxx + uyy - sc*exp(u);      } else if ((i > 0 && i < mx-1) || (j > 0 && j < my-1)){        f[row] = .5*two*(hydhx + hxdhy)*x[row];      } else {        f[row] = .25*two*(hydhx + hxdhy)*x[row];      }    }  }  ierr = VecRestoreArray(localX,&x);CHKERRQ(ierr);  ierr = VecRestoreArray(localF,&f);CHKERRQ(ierr);  /* Insert values into global vector */  ierr = DMLocalToGlobalBegin(user->fine.da,localF,INSERT_VALUES,F);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(user->fine.da,localF,INSERT_VALUES,F);CHKERRQ(ierr);  ierr = PetscLogFlops(11.0*ym*xm);CHKERRQ(ierr);  return 0; } 

/*@  DMPlexTSComputeRHSFunctionFVM - Form the local forcing F from the local input X using pointwise functions specified by the user  Input Parameters:+ dm - The mesh. t - The time. locX  - Local solution- user - The user context  Output Parameter:. F  - Global output vector  Level: developer.seealso: DMPlexComputeJacobianActionFEM()@*/PetscErrorCode DMPlexTSComputeRHSFunctionFVM(DM dm, PetscReal time, Vec locX, Vec F, void *user){  Vec            locF;  PetscInt       cStart, cEnd, cEndInterior;  PetscErrorCode ierr;  PetscFunctionBegin;  ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);  ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);  cEnd = cEndInterior < 0 ? cEnd : cEndInterior;  ierr = DMGetLocalVector(dm, &locF);CHKERRQ(ierr);  ierr = VecZeroEntries(locF);CHKERRQ(ierr);  ierr = DMPlexComputeResidual_Internal(dm, cStart, cEnd, time, locX, NULL, locF, user);CHKERRQ(ierr);  ierr = DMLocalToGlobalBegin(dm, locF, INSERT_VALUES, F);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(dm, locF, INSERT_VALUES, F);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(dm, &locF);CHKERRQ(ierr);  PetscFunctionReturn(0);}

PetscErrorCode FormFunctionMatlab(SNES snes,Vec X,Vec F,void *ptr){  AppCtx         *user = (AppCtx*)ptr;  PetscErrorCode ierr;  PetscInt       Mx,My;  PetscReal      lambda,hx,hy;  Vec            localX,localF;  MPI_Comm       comm;  DM             da;  PetscFunctionBeginUser;  ierr = SNESGetDM(snes,&da);CHKERRQ(ierr);  ierr = DMGetLocalVector(da,&localX);CHKERRQ(ierr);  ierr = DMGetLocalVector(da,&localF);CHKERRQ(ierr);  ierr = PetscObjectSetName((PetscObject)localX,"localX");CHKERRQ(ierr);  ierr = PetscObjectSetName((PetscObject)localF,"localF");CHKERRQ(ierr);  ierr = DMDAGetInfo(da,PETSC_IGNORE,&Mx,&My,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE);CHKERRQ(ierr);  lambda = user->param;  hx     = 1.0/(PetscReal)(Mx-1);  hy     = 1.0/(PetscReal)(My-1);  ierr = PetscObjectGetComm((PetscObject)snes,&comm);CHKERRQ(ierr);  /*     Scatter ghost points to local vector,using the 2-step process        DMGlobalToLocalBegin(),DMGlobalToLocalEnd().     By placing code between these two statements, computations can be     done while messages are in transition.  */  ierr = DMGlobalToLocalBegin(da,X,INSERT_VALUES,localX);CHKERRQ(ierr);  ierr = DMGlobalToLocalEnd(da,X,INSERT_VALUES,localX);CHKERRQ(ierr);  ierr = PetscMatlabEnginePut(PETSC_MATLAB_ENGINE_(comm),(PetscObject)localX);CHKERRQ(ierr);  ierr = PetscMatlabEngineEvaluate(PETSC_MATLAB_ENGINE_(comm),"localF=ex5m(localX,%18.16e,%18.16e,%18.16e)",hx,hy,lambda);CHKERRQ(ierr);  ierr = PetscMatlabEngineGet(PETSC_MATLAB_ENGINE_(comm),(PetscObject)localF);CHKERRQ(ierr);  /*     Insert values into global vector  */  ierr = DMLocalToGlobalBegin(da,localF,INSERT_VALUES,F);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(da,localF,INSERT_VALUES,F);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(da,&localX);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(da,&localF);CHKERRQ(ierr);  PetscFunctionReturn(0);}

static PetscErrorCode TestFields(DM dmstag){  PetscErrorCode ierr;  Vec            vecLocal,vecGlobal;  PetscReal      norm2;  PetscFunctionBeginUser;  ierr = DMCreateLocalVector(dmstag,&vecLocal);CHKERRQ(ierr);  ierr = DMCreateGlobalVector(dmstag,&vecGlobal);CHKERRQ(ierr);  ierr = VecSet(vecLocal,1.0);CHKERRQ(ierr);  ierr = DMLocalToGlobalBegin(dmstag,vecLocal,INSERT_VALUES,vecGlobal);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd  (dmstag,vecLocal,INSERT_VALUES,vecGlobal);CHKERRQ(ierr);  ierr = VecSet(vecGlobal,2.0);CHKERRQ(ierr);  ierr = DMGlobalToLocalBegin(dmstag,vecGlobal,INSERT_VALUES,vecLocal);CHKERRQ(ierr);  ierr = DMGlobalToLocalEnd  (dmstag,vecGlobal,INSERT_VALUES,vecLocal);CHKERRQ(ierr);  ierr = VecNorm(vecGlobal,NORM_2,&norm2);CHKERRQ(ierr);  ierr = PetscPrintf(PETSC_COMM_WORLD,"2 Norm of test vector: %g/n",(double)norm2);CHKERRQ(ierr);  ierr = VecDestroy(&vecLocal);CHKERRQ(ierr);  ierr = VecDestroy(&vecGlobal);CHKERRQ(ierr);  PetscFunctionReturn(0);}

/*     Matrix free operation of 1d Laplacian and Grad for GLL spectral elements*/PetscErrorCode MatMult_Laplacian(Mat A,Vec x,Vec y){  AppCtx            *appctx;  PetscErrorCode    ierr;  PetscReal         **temp,vv;  PetscInt          i,j,xs,xn;  Vec               xlocal,ylocal;  const PetscScalar *xl;  PetscScalar       *yl;  PetscBLASInt      _One = 1,n;  PetscScalar       _DOne = 1;    ierr = MatShellGetContext(A,&appctx);CHKERRQ(ierr);  ierr = DMGetLocalVector(appctx->da,&xlocal);CHKERRQ(ierr);  ierr = DMGlobalToLocalBegin(appctx->da,x,INSERT_VALUES,xlocal);CHKERRQ(ierr);  ierr = DMGlobalToLocalEnd(appctx->da,x,INSERT_VALUES,xlocal);CHKERRQ(ierr);  ierr = DMGetLocalVector(appctx->da,&ylocal);CHKERRQ(ierr);  ierr = VecSet(ylocal,0.0);CHKERRQ(ierr);  ierr = PetscGLLElementLaplacianCreate(&appctx->SEMop.gll,&temp);CHKERRQ(ierr);  for (i=0; i<appctx->param.N; i++) {    vv =-appctx->param.mu*2.0/appctx->param.Le;    for (j=0; j<appctx->param.N; j++) temp[i][j]=temp[i][j]*vv;  }  ierr = DMDAVecGetArrayRead(appctx->da,xlocal,(void*)&xl);CHKERRQ(ierr);  ierr = DMDAVecGetArray(appctx->da,ylocal,&yl);CHKERRQ(ierr);  ierr = DMDAGetCorners(appctx->da,&xs,NULL,NULL,&xn,NULL,NULL);CHKERRQ(ierr);  ierr = PetscBLASIntCast(appctx->param.N,&n);CHKERRQ(ierr);  for (j=xs; j<xs+xn; j += appctx->param.N-1) {    PetscStackCallBLAS("BLASgemv",BLASgemv_("N",&n,&n,&_DOne,&temp[0][0],&n,&xl[j],&_One,&_DOne,&yl[j],&_One));  }  ierr = DMDAVecRestoreArrayRead(appctx->da,xlocal,(void*)&xl);CHKERRQ(ierr);  ierr = DMDAVecRestoreArray(appctx->da,ylocal,&yl);CHKERRQ(ierr);  ierr = PetscGLLElementLaplacianDestroy(&appctx->SEMop.gll,&temp);CHKERRQ(ierr);  ierr = VecSet(y,0.0);CHKERRQ(ierr);  ierr = DMLocalToGlobalBegin(appctx->da,ylocal,ADD_VALUES,y);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(appctx->da,ylocal,ADD_VALUES,y);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(appctx->da,&xlocal);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(appctx->da,&ylocal);CHKERRQ(ierr);  ierr = VecPointwiseDivide(y,y,appctx->SEMop.mass);CHKERRQ(ierr);  return 0;}

PetscErrorCode MatMult_GlobalToLocalNormal(Mat CtC, Vec x, Vec y){  DM                    dm;  Vec                   local, mask;  projectConstraintsCtx *ctx;  PetscErrorCode        ierr;  PetscFunctionBegin;  ierr = MatShellGetContext(CtC,&ctx);CHKERRQ(ierr);  dm   = ctx->dm;  mask = ctx->mask;  ierr = DMGetLocalVector(dm,&local);CHKERRQ(ierr);  ierr = DMGlobalToLocalBegin(dm,x,INSERT_VALUES,local);CHKERRQ(ierr);  ierr = DMGlobalToLocalEnd(dm,x,INSERT_VALUES,local);CHKERRQ(ierr);  if (mask) {ierr = VecPointwiseMult(local,mask,local);CHKERRQ(ierr);}  ierr = VecSet(y,0.);CHKERRQ(ierr);  ierr = DMLocalToGlobalBegin(dm,local,ADD_VALUES,y);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(dm,local,ADD_VALUES,y);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(dm,&local);CHKERRQ(ierr);  PetscFunctionReturn(0);}

/*@  DMPlexTSComputeRHSFunctionFVM - Form the local forcing F from the local input X using pointwise functions specified by the user  Input Parameters:+ dm - The mesh. t - The time. locX  - Local solution- user - The user context  Output Parameter:. F  - Global output vector  Level: developer.seealso: DMPlexComputeJacobianActionFEM()@*/PetscErrorCode DMPlexTSComputeRHSFunctionFVM(DM dm, PetscReal time, Vec locX, Vec F, void *user){  Vec            locF;  PetscInt       cStart, cEnd, cEndInterior;  DM             plex;  PetscErrorCode ierr;  PetscFunctionBegin;  ierr = DMTSConvertPlex(dm,&plex,PETSC_TRUE);CHKERRQ(ierr);  ierr = DMPlexGetHeightStratum(plex, 0, &cStart, &cEnd);CHKERRQ(ierr);  ierr = DMPlexGetHybridBounds(plex, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);  cEnd = cEndInterior < 0 ? cEnd : cEndInterior;  ierr = DMGetLocalVector(plex, &locF);CHKERRQ(ierr);  ierr = VecZeroEntries(locF);CHKERRQ(ierr);  ierr = DMPlexComputeResidual_Internal(plex, cStart, cEnd, time, locX, NULL, time, locF, user);CHKERRQ(ierr);  ierr = DMLocalToGlobalBegin(plex, locF, ADD_VALUES, F);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(plex, locF, ADD_VALUES, F);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(plex, &locF);CHKERRQ(ierr);  ierr = DMDestroy(&plex);CHKERRQ(ierr);  PetscFunctionReturn(0);}

PetscErrorCode SetInitialCondition(DM dm, Vec X, User user){  DM                dmCell;  const PetscScalar *cellgeom;  PetscScalar       *x;  PetscInt          cStart, cEnd, cEndInterior = user->cEndInterior, c;  PetscErrorCode    ierr;  PetscFunctionBeginUser;  ierr = VecGetDM(user->cellgeom, &dmCell);CHKERRQ(ierr);  ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);  ierr = VecGetArrayRead(user->cellgeom, &cellgeom);CHKERRQ(ierr);  ierr = VecGetArray(X, &x);CHKERRQ(ierr);  for (c = cStart; c < cEndInterior; ++c) {    const CellGeom *cg;    PetscScalar    *xc;    ierr = DMPlexPointLocalRead(dmCell,c,cellgeom,&cg);CHKERRQ(ierr);    ierr = DMPlexPointGlobalRef(dm,c,x,&xc);CHKERRQ(ierr);    if (xc) {      ierr = InitialCondition(0.0, cg->centroid, xc, user);CHKERRQ(ierr);    }  }  ierr = VecRestoreArrayRead(user->cellgeom, &cellgeom);CHKERRQ(ierr);  ierr = VecRestoreArray(X, &x);CHKERRQ(ierr);  { //Apply the Boundary condition for the intial condition    Vec             XLocal;    ierr = DMGetLocalVector(user->dm, &XLocal);CHKERRQ(ierr);    ierr = VecSet(XLocal, 0);CHKERRQ(ierr);    ierr = DMGlobalToLocalBegin(user->dm, X, INSERT_VALUES, XLocal);CHKERRQ(ierr);    ierr = DMGlobalToLocalEnd(user->dm, X, INSERT_VALUES, XLocal);CHKERRQ(ierr);    ierr = ApplyBC(user->dm, user->current_time, XLocal, user);CHKERRQ(ierr);    ierr = DMLocalToGlobalBegin(user->dm, XLocal, INSERT_VALUES, X);CHKERRQ(ierr);    ierr = DMLocalToGlobalEnd(user->dm, XLocal, INSERT_VALUES, X);CHKERRQ(ierr);  }  PetscFunctionReturn(0);}

PetscErrorCode FormInitialGuess1(AppCtx *user,Vec X){  PetscInt       i, j, row, mx, my, xs, ys, xm, ym, Xm, Ym, Xs, Ys;  PetscErrorCode ierr;  double         one = 1.0, lambda, temp1, temp, hx, hy;  /* double hxdhy, hydhx,sc; */  PetscScalar    *x;  Vec            localX = user->fine.localX;  mx = user->fine.mx;       my = user->fine.my;            lambda = user->param;  hx = one/(double)(mx-1);  hy = one/(double)(my-1);  /* sc = hx*hy*lambda;        hxdhy = hx/hy;            hydhx = hy/hx; */  temp1 = lambda/(lambda + one);  /* Get ghost points */  ierr = DMDAGetCorners(user->fine.da,&xs,&ys,0,&xm,&ym,0);CHKERRQ(ierr);  ierr = DMDAGetGhostCorners(user->fine.da,&Xs,&Ys,0,&Xm,&Ym,0);CHKERRQ(ierr);  ierr = VecGetArray(localX,&x);CHKERRQ(ierr);  /* Compute initial guess */  for (j=ys; j<ys+ym; j++) {    temp = (double)(PetscMin(j,my-j-1))*hy;    for (i=xs; i<xs+xm; i++) {      row = i - Xs + (j - Ys)*Xm;       if (i == 0 || j == 0 || i == mx-1 || j == my-1 ) {        x[row] = 0.0;         continue;      }      x[row] = temp1*PetscSqrtReal( PetscMin( (double)(PetscMin(i,mx-i-1))*hx,temp) );     }  }  ierr = VecRestoreArray(localX,&x);CHKERRQ(ierr);  /* Insert values into global vector */  ierr = DMLocalToGlobalBegin(user->fine.da,localX,INSERT_VALUES,X);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(user->fine.da,localX,INSERT_VALUES,X);CHKERRQ(ierr);  return 0;}

int main(int argc,char **argv){  PetscInt       M = 6,N = 5,P = 4, m = PETSC_DECIDE,n = PETSC_DECIDE,p = PETSC_DECIDE,i,j,k,is,js,ks,in,jen,kn;  PetscErrorCode ierr;  DM             da;  Vec            local,global;  PetscScalar    ****l;  ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;  /* Create distributed array and get vectors */  ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_BOX,M,N,P,m,n,p,2,1,NULL,NULL,NULL,&da);CHKERRQ(ierr);  ierr = DMSetFromOptions(da);CHKERRQ(ierr);  ierr = DMSetUp(da);CHKERRQ(ierr);  ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);  ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);  ierr = DMDAGetCorners(da,&is,&js,&ks,&in,&jen,&kn);CHKERRQ(ierr);  ierr = DMDAVecGetArrayDOF(da,local,&l);CHKERRQ(ierr);  for (i=is; i<is+in; i++) {    for (j=js; j<js+jen; j++) {      for (k=ks; k<ks+kn; k++) {        l[k][j][i][0] = 2*(i + j*M + k*M*N);        l[k][j][i][1] = 2*(i + j*M + k*M*N) + 1;      }    }  }  ierr = DMDAVecRestoreArrayDOF(da,local,&l);CHKERRQ(ierr);  ierr = DMLocalToGlobalBegin(da,local,ADD_VALUES,global);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(da,local,ADD_VALUES,global);CHKERRQ(ierr);  ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);  /* Free memory */  ierr = VecDestroy(&local);CHKERRQ(ierr);  ierr = VecDestroy(&global);CHKERRQ(ierr);  ierr = DMDestroy(&da);CHKERRQ(ierr);  ierr = PetscFinalize();  return ierr;}

PetscErrorCode ComputePredictor(DM da, UserContext *user){  Vec            uOldLocal, uLocal,uOld;  PetscScalar    *pOld;  PetscScalar    *p;  PetscErrorCode ierr;  PetscFunctionBeginUser;  ierr = DMGetGlobalVector(da, &uOld);CHKERRQ(ierr);  ierr = DMGetLocalVector(da, &uOldLocal);CHKERRQ(ierr);  ierr = DMGetLocalVector(da, &uLocal);CHKERRQ(ierr);  ierr = DMGlobalToLocalBegin(da, uOld, INSERT_VALUES, uOldLocal);CHKERRQ(ierr);  ierr = DMGlobalToLocalEnd(da, uOld, INSERT_VALUES, uOldLocal);CHKERRQ(ierr);  ierr = VecGetArray(uOldLocal, &pOld);CHKERRQ(ierr);  ierr = VecGetArray(uLocal,    &p);CHKERRQ(ierr);  /* Source terms are all zero right now */  ierr = CalculateElementVelocity(da, user);CHKERRQ(ierr);  ierr = TaylorGalerkinStepI(da, user);CHKERRQ(ierr);  ierr = TaylorGalerkinStepIIMomentum(da, user);CHKERRQ(ierr);  ierr = TaylorGalerkinStepIIMassEnergy(da, user);CHKERRQ(ierr);  /* Solve equation (9) for /delta(/rho/vu) and (/rho/vu)^* */  /* Solve equation (13) for /delta/rho and /rho^* */  /* Solve equation (15) for /delta(/rho e_t) and (/rho e_t)^* */  /* Apply artifical dissipation */  /* Determine the smoothed explicit pressure, /tilde P and temperature /tilde T using the equation of state */  ierr = VecRestoreArray(uOldLocal, &pOld);CHKERRQ(ierr);  ierr = VecRestoreArray(uLocal,    &p);CHKERRQ(ierr);#if 0  ierr = DMLocalToGlobalBegin(da, uLocal, ADD_VALUES,u);CHKERRQ(ierr);  ierr = DMLocalToGlobalEnd(da, uLocal, ADD_VALUES,u);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(da, &uOldLocal);CHKERRQ(ierr);  ierr = DMRestoreLocalVector(da, &uLocal);CHKERRQ(ierr);#endif  PetscFunctionReturn(0);}