自学教程：C++ ADIOI_Free函数代码示例

int convert_mpi_pvfs2_dtype(MPI_Datatype *mpi_dtype,			    PVFS_Request *pvfs_dtype){    int num_int = -1, num_addr = -1, num_dtype = -1,	combiner = -1, i = -1, ret = -1, leaf = -1;    int *arr_int = NULL;    MPI_Aint *arr_addr = NULL;    MPI_Datatype *arr_dtype = NULL;    PVFS_Request *old_pvfs_dtype = NULL;    PVFS_Request *old_pvfs_dtype_arr = NULL;    int arr_count = -1;    PVFS_size *pvfs_arr_disp = NULL;    int *pvfs_arr_len = NULL;    MPI_Type_get_envelope(*mpi_dtype,			  &num_int,			  &num_addr,			  &num_dtype,			  &combiner);    /* Depending on type of datatype do the following     * operations */    if (combiner == MPI_COMBINER_NAMED)    {	convert_named(mpi_dtype, pvfs_dtype, combiner);	return 1;    }    /* Allocate space for the arrays necessary for     * MPI_Type_get_contents */    if ((arr_int = ADIOI_Malloc(sizeof(int)*num_int)) == NULL)    {	fprintf(stderr, "Failed to allocate array_int/n");	return -1;    }    if ((arr_addr = ADIOI_Malloc(sizeof(int)*num_addr)) == NULL)    {	ADIOI_Free(arr_int);	fprintf(stderr, "Failed to allocate array_addr/n");	return -1;    }    if ((arr_dtype = ADIOI_Malloc(sizeof(MPI_Datatype)*num_dtype)) == NULL)    {	ADIOI_Free(arr_int);	ADIOI_Free(arr_addr);	fprintf(stderr, "Failed to allocate array_dtypes/n");	return -1;    }    MPI_Type_get_contents(*mpi_dtype,			  num_int,			  num_addr,			  num_dtype,			  arr_int,			  arr_addr,			  arr_dtype);    /* If it's not a predefined datatype, it is either a     * derived datatype or a structured datatype */    if (combiner != MPI_COMBINER_STRUCT)    {	if ((old_pvfs_dtype = ADIOI_Malloc(sizeof(PVFS_Request))) == NULL)	    fprintf(stderr, "convert_mpi_pvfs2_dtype: "		    "Failed to allocate PVFS_Request/n");	switch (combiner)	{	    case MPI_COMBINER_CONTIGUOUS:		leaf = convert_mpi_pvfs2_dtype(&arr_dtype[0], old_pvfs_dtype);		ret = PVFS_Request_contiguous(arr_int[0],					      *old_pvfs_dtype, pvfs_dtype);		break;	    case MPI_COMBINER_VECTOR:		leaf = convert_mpi_pvfs2_dtype(&arr_dtype[0], old_pvfs_dtype);		ret = PVFS_Request_vector(arr_int[0], arr_int[1],					  arr_int[2], *old_pvfs_dtype,					  pvfs_dtype);		break;	    case MPI_COMBINER_HVECTOR:		leaf = convert_mpi_pvfs2_dtype(&arr_dtype[0], old_pvfs_dtype);		ret = PVFS_Request_hvector(arr_int[0], arr_int[1],					   arr_addr[0], *old_pvfs_dtype,					   pvfs_dtype);		break;		/* Both INDEXED and HINDEXED types require PVFS_size		 * address arrays.  Therefore, we need to copy and		 * convert the data from MPI_get_contents() into		 * a PVFS_size buffer */	    case MPI_COMBINER_INDEXED:		leaf = convert_mpi_pvfs2_dtype(&arr_dtype[0], old_pvfs_dtype);		if ((pvfs_arr_disp =			    ADIOI_Malloc(arr_int[0]*sizeof(PVFS_size))) == 0)		{		    fprintf(stderr, "convert_mpi_pvfs2_dtype: "			    "Failed to allocate pvfs_arr_disp/n");		}		for (i = 0; i < arr_int[0]; i++)		{//.........这里部分代码省略.........

void ADIOI_NFS_ReadStrided(ADIO_File fd, void *buf, int count,                       MPI_Datatype datatype, int file_ptr_type,                       ADIO_Offset offset, ADIO_Status *status, int                       *error_code){/* offset is in units of etype relative to the filetype. */    ADIOI_Flatlist_node *flat_buf, *flat_file;    ADIO_Offset i_offset, new_brd_size, brd_size, size;    int i, j, k, err, err_flag, st_index=0;    MPI_Count num, bufsize;    int n_etypes_in_filetype;    ADIO_Offset n_filetypes, etype_in_filetype, st_n_filetypes, size_in_filetype;    ADIO_Offset abs_off_in_filetype=0, new_frd_size, frd_size=0, st_frd_size;    MPI_Count filetype_size, etype_size, buftype_size, partial_read;    MPI_Aint filetype_extent, buftype_extent;     int buf_count, buftype_is_contig, filetype_is_contig;    ADIO_Offset userbuf_off, req_len, sum;    ADIO_Offset off, req_off, disp, end_offset=0, readbuf_off, start_off;    char *readbuf, *tmp_buf, *value;    int info_flag;    unsigned max_bufsize, readbuf_len;    static char myname[] = "ADIOI_NFS_READSTRIDED";    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);    MPI_Type_size_x(fd->filetype, &filetype_size);    if ( ! filetype_size ) {#ifdef HAVE_STATUS_SET_BYTES	MPIR_Status_set_bytes(status, datatype, 0);#endif	*error_code = MPI_SUCCESS; 	return;    }    MPI_Type_extent(fd->filetype, &filetype_extent);    MPI_Type_size_x(datatype, &buftype_size);    MPI_Type_extent(datatype, &buftype_extent);    etype_size = fd->etype_size;    ADIOI_Assert((buftype_size * count) == ((ADIO_Offset)(MPI_Count)buftype_size * (ADIO_Offset)count));    bufsize = buftype_size * count;/* get max_bufsize from the info object. */    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));    ADIOI_Info_get(fd->info, "ind_rd_buffer_size", MPI_MAX_INFO_VAL, value,                  &info_flag);    max_bufsize = atoi(value);    ADIOI_Free(value);    if (!buftype_is_contig && filetype_is_contig) {/* noncontiguous in memory, contiguous in file. */	flat_buf = ADIOI_Flatten_and_find(datatype);        off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind :                  fd->disp + etype_size * offset;	start_off = off;	end_offset = off + bufsize - 1;        readbuf_off = off;        readbuf = (char *) ADIOI_Malloc(max_bufsize);        readbuf_len = (unsigned) (MPL_MIN(max_bufsize, end_offset-readbuf_off+1));/* if atomicity is true, lock (exclusive) the region to be accessed */        if (fd->atomicity)            ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);#ifdef ADIOI_MPE_LOGGING        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );#endif	lseek(fd->fd_sys, readbuf_off, SEEK_SET);#ifdef ADIOI_MPE_LOGGING        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );#endif        if (!(fd->atomicity)) ADIOI_READ_LOCK(fd, readbuf_off, SEEK_SET, readbuf_len);#ifdef ADIOI_MPE_LOGGING        MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );#endif        err = read(fd->fd_sys, readbuf, readbuf_len);#ifdef ADIOI_MPE_LOGGING        MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );#endif        if (!(fd->atomicity)) ADIOI_UNLOCK(fd, readbuf_off, SEEK_SET, readbuf_len);        if (err == -1) err_flag = 1;        for (j=0; j<count; j++)             for (i=0; i<flat_buf->count; i++) {                userbuf_off = (ADIO_Offset)j*buftype_extent + flat_buf->indices[i];		req_off = off;		req_len = flat_buf->blocklens[i];		ADIOI_BUFFERED_READ                off += flat_buf->blocklens[i];            }        if (fd->atomicity)//.........这里部分代码省略.........

//.........这里部分代码省略.......... buf - initial address of buffer (choice).N fortran@*/int MPI_File_read_ordered_begin(MPI_File fh, void *buf, int count,				MPI_Datatype datatype){    int error_code,  nprocs, myrank;    MPI_Count datatype_size;    int source, dest;    ADIO_Offset shared_fp, incr;    ADIO_File adio_fh;    static char myname[] = "MPI_FILE_READ_ORDERED_BEGIN";    void *xbuf=NULL, *e32_buf=NULL;    MPIU_THREAD_CS_ENTER(ALLFUNC,);    adio_fh = MPIO_File_resolve(fh);    /* --BEGIN ERROR HANDLING-- */    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);    if (adio_fh->split_coll_count)    {	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,					  myname, __LINE__, MPI_ERR_IO, 					  "**iosplitcoll", 0);	error_code = MPIO_Err_return_file(adio_fh, error_code);	goto fn_exit;    }    /* --END ERROR HANDLING-- */    adio_fh->split_coll_count = 1;    MPI_Type_size_x(datatype, &datatype_size);    /* --BEGIN ERROR HANDLING-- */    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);    MPIO_CHECK_FS_SUPPORTS_SHARED(adio_fh, myname, error_code);    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);    /* --END ERROR HANDLING-- */    ADIOI_TEST_DEFERRED(adio_fh, myname, &error_code);    MPI_Comm_size(adio_fh->comm, &nprocs);    MPI_Comm_rank(adio_fh->comm, &myrank);    incr = (count*datatype_size)/adio_fh->etype_size;    /* Use a message as a 'token' to order the operations */    source = myrank - 1;    dest   = myrank + 1;    if (source < 0) source = MPI_PROC_NULL;    if (dest >= nprocs) dest = MPI_PROC_NULL;    MPI_Recv(NULL, 0, MPI_BYTE, source, 0, adio_fh->comm, MPI_STATUS_IGNORE);    ADIO_Get_shared_fp(adio_fh, incr, &shared_fp, &error_code);    /* --BEGIN ERROR HANDLING-- */    if (error_code != MPI_SUCCESS)    {	error_code = MPIO_Err_return_file(adio_fh, error_code);	goto fn_exit;    }    /* --END ERROR HANDLING-- */    MPI_Send(NULL, 0, MPI_BYTE, dest, 0, adio_fh->comm);    xbuf = buf;    if (adio_fh->is_external32)    {        MPI_Aint e32_size = 0;        error_code = MPIU_datatype_full_size(datatype, &e32_size);        if (error_code != MPI_SUCCESS)            goto fn_exit;        e32_buf = ADIOI_Malloc(e32_size*count);	xbuf = e32_buf;    }    ADIO_ReadStridedColl(adio_fh, xbuf, count, datatype, ADIO_EXPLICIT_OFFSET,			 shared_fp, &adio_fh->split_status, &error_code);    /* --BEGIN ERROR HANDLING-- */    if (error_code != MPI_SUCCESS)	error_code = MPIO_Err_return_file(adio_fh, error_code);    /* --END ERROR HANDLING-- */    if (e32_buf != NULL) {        error_code = MPIU_read_external32_conversion_fn(xbuf, datatype,                count, e32_buf);	ADIOI_Free(e32_buf);    }fn_exit:    MPIU_THREAD_CS_EXIT(ALLFUNC,);    return error_code;}

void ADIOI_NFS_WriteStrided(ADIO_File fd, const void *buf, int count,                       MPI_Datatype datatype, int file_ptr_type,                       ADIO_Offset offset, ADIO_Status *status, int                       *error_code){/* offset is in units of etype relative to the filetype. */    ADIOI_Flatlist_node *flat_buf, *flat_file;    int i, j, k, err=-1, bwr_size, fwr_size=0, st_index=0;    int bufsize, num, size, sum, n_etypes_in_filetype, size_in_filetype;    int n_filetypes, etype_in_filetype;    ADIO_Offset abs_off_in_filetype=0;    int req_len;    MPI_Count filetype_size, etype_size, buftype_size;    MPI_Aint filetype_extent, buftype_extent;     int buf_count, buftype_is_contig, filetype_is_contig;    ADIO_Offset userbuf_off;    ADIO_Offset off, req_off, disp, end_offset=0, writebuf_off, start_off;    char *writebuf=NULL, *value;    int st_fwr_size, st_n_filetypes, writebuf_len, write_sz;    int new_bwr_size, new_fwr_size, err_flag=0, info_flag, max_bufsize;    static char myname[] = "ADIOI_NFS_WRITESTRIDED";    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);    MPI_Type_size_x(fd->filetype, &filetype_size);    if ( ! filetype_size ) {#ifdef HAVE_STATUS_SET_BYTES	MPIR_Status_set_bytes(status, datatype, 0);#endif	*error_code = MPI_SUCCESS; 	return;    }    MPI_Type_extent(fd->filetype, &filetype_extent);    MPI_Type_size_x(datatype, &buftype_size);    MPI_Type_extent(datatype, &buftype_extent);    etype_size = fd->etype_size;    bufsize = buftype_size * count;/* get max_bufsize from the info object. */    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));    ADIOI_Info_get(fd->info, "ind_wr_buffer_size", MPI_MAX_INFO_VAL, value,                  &info_flag);    max_bufsize = atoi(value);    ADIOI_Free(value);    if (!buftype_is_contig && filetype_is_contig) {/* noncontiguous in memory, contiguous in file. */	ADIOI_Flatten_datatype(datatype);	flat_buf = ADIOI_Flatlist;	while (flat_buf->type != datatype) flat_buf = flat_buf->next;        off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind :                  fd->disp + etype_size * offset;        start_off = off;	end_offset = off + bufsize - 1;        writebuf_off = off;        writebuf = (char *) ADIOI_Malloc(max_bufsize);        writebuf_len = (int) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));/* if atomicity is true, lock the region to be accessed */        if (fd->atomicity)             ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);        for (j=0; j<count; j++)             for (i=0; i<flat_buf->count; i++) {                userbuf_off = j*buftype_extent + flat_buf->indices[i];		req_off = off;		req_len = flat_buf->blocklens[i];		ADIOI_BUFFERED_WRITE_WITHOUT_READ                off += flat_buf->blocklens[i];            }        /* write the buffer out finally */#ifdef ADIOI_MPE_LOGGING        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );#endif	lseek(fd->fd_sys, writebuf_off, SEEK_SET); #ifdef ADIOI_MPE_LOGGING        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );#endif	if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len);#ifdef ADIOI_MPE_LOGGING        MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );#endif	err = write(fd->fd_sys, writebuf, writebuf_len); #ifdef ADIOI_MPE_LOGGING        MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );#endif        if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len);        if (err == -1) err_flag = 1;         if (fd->atomicity) //.........这里部分代码省略.........

void ADIOI_NFS_WriteStrided(ADIO_File fd, void *buf, int count,                       MPI_Datatype datatype, int file_ptr_type,                       ADIO_Offset offset, ADIO_Status *status, int                       *error_code){/* offset is in units of etype relative to the filetype. */    ADIOI_Flatlist_node *flat_buf, *flat_file;    int i, j, k, err=-1, bwr_size, fwr_size=0, st_index=0;    int bufsize, num, size, sum, n_etypes_in_filetype, size_in_filetype;    int n_filetypes, etype_in_filetype;    ADIO_Offset abs_off_in_filetype=0;    int filetype_size, etype_size, buftype_size, req_len;    MPI_Aint filetype_extent, buftype_extent;     int buf_count, buftype_is_contig, filetype_is_contig;    ADIO_Offset userbuf_off;    ADIO_Offset off, req_off, disp, end_offset, writebuf_off, start_off;    char *writebuf, *value;    int flag, st_fwr_size, st_n_filetypes, writebuf_len, write_sz;    int new_bwr_size, new_fwr_size, err_flag=0, info_flag, max_bufsize;#ifndef PRINT_ERR_MSG    static char myname[] = "ADIOI_NFS_WRITESTRIDED";#endif    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);    MPI_Type_size(fd->filetype, &filetype_size);    MPI_Type_extent(fd->filetype, &filetype_extent);    MPI_Type_size(datatype, &buftype_size);    MPI_Type_extent(datatype, &buftype_extent);    etype_size = fd->etype_size;    bufsize = buftype_size * count;/* get max_bufsize from the info object. */    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));    MPI_Info_get(fd->info, "ind_wr_buffer_size", MPI_MAX_INFO_VAL, value,                  &info_flag);    max_bufsize = atoi(value);    ADIOI_Free(value);    if (!buftype_is_contig && filetype_is_contig) {/* noncontiguous in memory, contiguous in file. */	ADIOI_Flatten_datatype(datatype);	flat_buf = ADIOI_Flatlist;	while (flat_buf->type != datatype) flat_buf = flat_buf->next;        off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind :                  fd->disp + etype_size * offset;        start_off = off;	end_offset = off + bufsize - 1;        writebuf_off = off;        writebuf = (char *) ADIOI_Malloc(max_bufsize);        writebuf_len = (int) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));/* if atomicity is true, lock the region to be accessed */        if (fd->atomicity)             ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);        for (j=0; j<count; j++)             for (i=0; i<flat_buf->count; i++) {                userbuf_off = j*buftype_extent + flat_buf->indices[i];		req_off = off;		req_len = flat_buf->blocklens[i];		ADIOI_BUFFERED_WRITE_WITHOUT_READ                off += flat_buf->blocklens[i];            }        /* write the buffer out finally */	lseek(fd->fd_sys, writebuf_off, SEEK_SET); 	if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len);	err = write(fd->fd_sys, writebuf, writebuf_len);         if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len);        if (err == -1) err_flag = 1;         if (fd->atomicity)             ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);	ADIOI_Free(writebuf); /* malloced in the buffered_write macro */        if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;#ifdef PRINT_ERR_MSG        *error_code = (err_flag) ? MPI_ERR_UNKNOWN : MPI_SUCCESS;#else	if (err_flag) {	    *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,			      myname, "I/O Error", "%s", strerror(errno));	    ADIOI_Error(fd, *error_code, myname);	}	else *error_code = MPI_SUCCESS;#endif    }    else {  /* noncontiguous in file *///.........这里部分代码省略.........

void ADIOI_Get_eof_offset(ADIO_File fd, ADIO_Offset *eof_offset){    unsigned filetype_size;    int error_code, filetype_is_contig, etype_size;    ADIO_Offset fsize, disp, sum=0, size_in_file, n_filetypes, rem;    int flag, i;    ADIO_Fcntl_t *fcntl_struct;    MPI_Aint filetype_extent;    ADIOI_Flatlist_node *flat_file;    /* find the eof in bytes */    fcntl_struct = (ADIO_Fcntl_t *) ADIOI_Malloc(sizeof(ADIO_Fcntl_t));    ADIO_Fcntl(fd, ADIO_FCNTL_GET_FSIZE, fcntl_struct, &error_code);    fsize = fcntl_struct->fsize;    ADIOI_Free(fcntl_struct);	    /* Find the offset in etype units corresponding to eof.       The eof could lie in a hole in the current view, or in the        middle of an etype. In that case the offset will be the offset       corresponding to the start of the next etype in the current view.*/    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);    etype_size = fd->etype_size;    if (filetype_is_contig) 	*eof_offset = (fsize - fd->disp + etype_size - 1)/etype_size;    /* ceiling division in case fsize is not a multiple of etype_size;*/    else {	/* filetype already flattened in ADIO_Open */	flat_file = ADIOI_Flatlist;	while (flat_file->type != fd->filetype) 	    flat_file = flat_file->next;		MPI_Type_size(fd->filetype, (int*)&filetype_size);	MPI_Type_extent(fd->filetype, &filetype_extent);	disp = fd->disp;	n_filetypes = -1;	flag = 0;	while (!flag) {	    sum = 0;	    n_filetypes++;	    for (i=0; i<flat_file->count; i++) {		sum += flat_file->blocklens[i];		if (disp + flat_file->indices[i] + 		    n_filetypes* ADIOI_AINT_CAST_TO_OFFSET filetype_extent + 		       flat_file->blocklens[i] >= fsize) {		    if (disp + flat_file->indices[i] + 			   n_filetypes * ADIOI_AINT_CAST_TO_OFFSET filetype_extent >= fsize)			sum -= flat_file->blocklens[i];		    else {			rem = (disp + flat_file->indices[i] + 				n_filetypes* ADIOI_AINT_CAST_TO_OFFSET filetype_extent				+ flat_file->blocklens[i] - fsize);			sum -= rem;		    }		    flag = 1;		    break;		}	    }	}	size_in_file = n_filetypes*(ADIO_Offset)filetype_size + sum;	*eof_offset = (size_in_file+etype_size-1)/etype_size; /* ceiling division */    }}

//.........这里部分代码省略.........            send_count_arr[tmp_agg_idx].count = flat_file_p->count;            send_count_arr[tmp_agg_idx].fp_ind = disp_off_sz_ext_typesz[0];            send_count_arr[tmp_agg_idx].disp = disp_off_sz_ext_typesz[1];            send_count_arr[tmp_agg_idx].byte_off = disp_off_sz_ext_typesz[2];            send_count_arr[tmp_agg_idx].sz = disp_off_sz_ext_typesz[3];            send_count_arr[tmp_agg_idx].ext = disp_off_sz_ext_typesz[4];            send_count_arr[tmp_agg_idx].type_sz = disp_off_sz_ext_typesz[5];        }    }#ifdef DEBUG2    fprintf(stderr, "my own flattened memtype: ");    ADIOI_Print_flatlist_node(flat_mem_p);    fprintf(stderr, "my own flattened filetype: ");    ADIOI_Print_flatlist_node(flat_file_p);#endif    if (fd->hints->cb_alltoall != ADIOI_HINT_DISABLE) {        ret = MPI_Alltoall(send_count_arr, sizeof(amount_and_extra_data_t),                           MPI_BYTE,                           recv_count_arr, sizeof(amount_and_extra_data_t), MPI_BYTE, fd->comm);        if (ret != MPI_SUCCESS) {            fprintf(stderr, "ADIOI_Exchange_file_views: MPI_Alltoall failed " "with error %d", ret);            return;        }    } else {#ifdef MPI_STATUSES_IGNORE        statuses = MPI_STATUSES_IGNORE;#else        statuses = (MPI_Status *) ADIOI_Malloc(1 + nprocs * sizeof(MPI_Status));#endif        if (fd->is_agg) {            MPI_Waitall(nprocs, recv_req_arr, statuses);            ADIOI_Free(recv_req_arr);        }        MPI_Waitall(fd->hints->cb_nodes, send_req_arr, statuses);#ifndef MPI_STATUSES_IGNORE        ADIOI_Free(statuses);#endif        ADIOI_Free(send_req_arr);    }#ifdef DEBUG2    if (fd->hints->cb_alltoall != ADIOI_HINT_DISABLE) {        fprintf(stderr, "send_count_arr:");        for (i = 0; i < nprocs; i++) {            fprintf(stderr, "[%d]=%d ", i, send_count_arr[i].count);        }        fprintf(stderr, "/n");        fprintf(stderr, "recv_count_arr:");        for (i = 0; i < nprocs; i++) {            fprintf(stderr, "[%d]=%d ", i, recv_count_arr[i].count);        }        fprintf(stderr, "/n");    } else {        fprintf(stderr, "send_count_arr:");        for (i = 0; i < fd->hints->cb_nodes; i++) {            fprintf(stderr, "[%d]=%d ", i, send_count_arr[i].count);        }        fprintf(stderr, "/n");        if (fd->is_agg) {            fprintf(stderr, "recv_count_arr:");            for (i = 0; i < nprocs; i++) {                fprintf(stderr, "[%d]=%d ", i, recv_count_arr[i].count);            }            fprintf(stderr, "/n");        }

//.........这里部分代码省略.........		MPI_MAX_INFO_VAL, value, &flag);	if (flag)	    start_iodev=atoll(value);    }    if ((str_factor > 0) || (str_unit > 0) || (start_iodev >= 0))	set_layout = 1;    /* if hints were set, we need to delay creation of any lustre objects.     * However, if we open the file with O_LOV_DELAY_CREATE and don't call the     * follow-up ioctl, subsequent writes will fail */    if (myrank == 0 && set_layout)	amode = amode | O_LOV_DELAY_CREATE;    fd->fd_sys = open(fd->filename, amode, perm);    if (fd->fd_sys == -1) goto fn_exit;    /* we can only set these hints on new files */    /* It was strange and buggy to open the file in the hint path.  Instead,     * we'll apply the file tunings at open time */    if ((amode & O_CREAT) && set_layout ) {	/* if user has specified striping info, first aggregator tries to set	 * it */	if (myrank == fd->hints->ranklist[0] || fd->comm == MPI_COMM_SELF) {	    lum->lmm_magic = LOV_USER_MAGIC;	    lum->lmm_pattern = 0;	    /* crude check for overflow of lustre internal datatypes.		 * Silently cap to large value if user provides a value		 * larger than lustre supports */	    if (str_unit > UINT_MAX)	            lum->lmm_stripe_size = UINT_MAX;	    else	            lum->lmm_stripe_size = str_unit;	    if (str_factor > USHRT_MAX)	            lum->lmm_stripe_count = USHRT_MAX;	    else	            lum->lmm_stripe_count = str_factor;	    if (start_iodev > USHRT_MAX)	             lum->lmm_stripe_offset = USHRT_MAX;	    else	            lum->lmm_stripe_offset = start_iodev;	    err = ioctl(fd->fd_sys, LL_IOC_LOV_SETSTRIPE, lum);	    if (err == -1 && errno != EEXIST) {		fprintf(stderr, "Failure to set stripe info %s /n", strerror(errno));		/* not a fatal error, but user might care to know */	    }	} /* End of striping parameters validation */    }    /* Pascal Deveze reports that, even though we pass a     * "GETSTRIPE" (read) flag to the ioctl, if some of the values of this     * struct are uninitialzed, the call can give an error.  zero it out in case     * there are other members that must be initialized and in case     * lov_user_md struct changes in future */    memset(lum, 0, lumlen);    lum->lmm_magic = LOV_USER_MAGIC;    err = ioctl(fd->fd_sys, LL_IOC_LOV_GETSTRIPE, (void *)lum);    if (!err) {	fd->hints->striping_unit = lum->lmm_stripe_size;	MPL_snprintf(value, value_sz, "%d", lum->lmm_stripe_size);	ADIOI_Info_set(fd->info, "striping_unit", value);	fd->hints->striping_factor = lum->lmm_stripe_count;	MPL_snprintf(value, value_sz, "%d", lum->lmm_stripe_count);	ADIOI_Info_set(fd->info, "striping_factor", value);	fd->hints->start_iodevice = lum->lmm_stripe_offset;	MPL_snprintf(value, value_sz, "%d", lum->lmm_stripe_offset);	ADIOI_Info_set(fd->info, "romio_lustre_start_iodevice", value);    }    if (fd->access_mode & ADIO_APPEND)	fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END);    fd->fd_direct = -1;    if (fd->direct_write || fd->direct_read) {	fd->fd_direct = open(fd->filename, amode_direct, perm);	if (fd->fd_direct != -1) {	    fd->d_mem = fd->d_miniosz = (1<<12);	} else {	    perror("cannot open file with O_Direct");	    fd->direct_write = fd->direct_read = 0;	}    }fn_exit:    ADIOI_Free(lum);    ADIOI_Free(value);    /* --BEGIN ERROR HANDLING-- */    if (fd->fd_sys == -1 || ((fd->fd_direct == -1) && 		(fd->direct_write || fd->direct_read))) {	*error_code = ADIOI_Err_create_code(myname, fd->filename, errno);    }    /* --END ERROR HANDLING-- */    else *error_code = MPI_SUCCESS;}

void ADIOI_PIOFS_WriteStrided(ADIO_File fd, void *buf, int count,                       MPI_Datatype datatype, int file_ptr_type,                       ADIO_Offset offset, ADIO_Status *status, int                       *error_code){/* Since PIOFS does not support file locking, can't do buffered writes   as on Unix *//* offset is in units of etype relative to the filetype. */    ADIOI_Flatlist_node *flat_buf, *flat_file;    struct iovec *iov;    int i, j, k, err=-1, bwr_size, fwr_size=0, st_index=0;    int bufsize, num, size, sum, n_etypes_in_filetype, size_in_filetype;    int n_filetypes, etype_in_filetype;    ADIO_Offset abs_off_in_filetype=0;    int filetype_size, etype_size, buftype_size;    MPI_Aint filetype_extent, buftype_extent, indx;    int buf_count, buftype_is_contig, filetype_is_contig;    ADIO_Offset off, disp;    int flag, new_bwr_size, new_fwr_size, err_flag=0;#ifndef PRINT_ERR_MSG    static char myname[] = "ADIOI_PIOFS_WRITESTRIDED";#endif    if (fd->atomicity) {	FPRINTF(stderr, "ROMIO cannot guarantee atomicity of noncontiguous accesses in atomic mode, as PIOFS doesn't support file locking. Use nonatomic mode and its associated semantics./n");	MPI_Abort(MPI_COMM_WORLD, 1);    }    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);    MPI_Type_size(fd->filetype, &filetype_size);    if ( ! filetype_size ) {	*error_code = MPI_SUCCESS; 	return;    }    MPI_Type_extent(fd->filetype, &filetype_extent);    MPI_Type_size(datatype, &buftype_size);    MPI_Type_extent(datatype, &buftype_extent);    etype_size = fd->etype_size;        bufsize = buftype_size * count;    if (!buftype_is_contig && filetype_is_contig) {/* noncontiguous in memory, contiguous in file. use writev */	ADIOI_Flatten_datatype(datatype);	flat_buf = ADIOI_Flatlist;	while (flat_buf->type != datatype) flat_buf = flat_buf->next;/* There is a limit of 16 on the number of iovecs for readv/writev! */	iov = (struct iovec *) ADIOI_Malloc(16*sizeof(struct iovec));	if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {	    off = fd->disp + etype_size * offset;	    llseek(fd->fd_sys, off, SEEK_SET);	}	else off = llseek(fd->fd_sys, fd->fp_ind, SEEK_SET);	k = 0;	for (j=0; j<count; j++) 	    for (i=0; i<flat_buf->count; i++) {		iov[k].iov_base = ((char *) buf) + j*buftype_extent +		    flat_buf->indices[i]; 		iov[k].iov_len = flat_buf->blocklens[i];		/*FPRINTF(stderr, "%d %d/n", iov[k].iov_base, iov[k].iov_len);*/		off += flat_buf->blocklens[i];		k = (k+1)%16;		if (!k) {		    err = writev(fd->fd_sys, iov, 16);		    if (err == -1) err_flag = 1;		}	    }	if (k) {	    err = writev(fd->fd_sys, iov, k);	    if (err == -1) err_flag = 1;	}	if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;	ADIOI_Free(iov);	if (err_flag) {#ifdef MPICH2	    *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io",		"**io %s", strerror(errno));#elif defined(PRINT_ERR_MSG) 	    *error_code =  MPI_ERR_UNKNOWN;#else /* MPICH-1 */	    *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,			      myname, "I/O Error", "%s", strerror(errno));	    ADIOI_Error(fd, *error_code, myname);#endif//.........这里部分代码省略.........

void ADIOI_GPFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code){/* if fd->info is null, create a new info object.   Initialize fd->info to default values.   Initialize fd->hints to default values.   Examine the info object passed by the user. If it contains values that   ROMIO understands, override the default. */    MPI_Info info;    char *value;    int flag, intval, nprocs=0, nprocs_is_valid = 0;    static char myname[] = "ADIOI_GPFS_SETINFO";    int did_anything = 0;    if (fd->info == MPI_INFO_NULL) MPI_Info_create(&(fd->info));    info = fd->info;    /* Note that fd->hints is allocated at file open time; thus it is     * not necessary to allocate it, or check for allocation, here.     */    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));    ADIOI_Assert ((value != NULL));    /* initialize info and hints to default values if they haven't been     * previously initialized     */    if (!fd->hints->initialized) {	ad_get_env_vars();	ad_gpfs_get_env_vars();	did_anything = 1;	/* buffer size for collective I/O */	ADIOI_Info_set(info, "cb_buffer_size", ADIOI_GPFS_CB_BUFFER_SIZE_DFLT);	fd->hints->cb_buffer_size = atoi(ADIOI_GPFS_CB_BUFFER_SIZE_DFLT);	/* default is to let romio automatically decide when to use	 * collective buffering	 */	ADIOI_Info_set(info, "romio_cb_read", "enable");	fd->hints->cb_read = ADIOI_HINT_ENABLE;	ADIOI_Info_set(info, "romio_cb_write", "enable");	fd->hints->cb_write = ADIOI_HINT_ENABLE;   	if ( fd->hints->cb_config_list != NULL ) ADIOI_Free (fd->hints->cb_config_list);	fd->hints->cb_config_list = NULL;	/* number of processes that perform I/O in collective I/O */	MPI_Comm_size(fd->comm, &nprocs);	nprocs_is_valid = 1;	MPL_snprintf(value, MPI_MAX_INFO_VAL+1, "%d", nprocs);	ADIOI_Info_set(info, "cb_nodes", value);	fd->hints->cb_nodes = -1;	/* hint indicating that no indep. I/O will be performed on this file */	ADIOI_Info_set(info, "romio_no_indep_rw", "false");	fd->hints->no_indep_rw = 0;	/* gpfs is not implementing file realms (ADIOI_IOStridedColl),	   initialize to disabled it. 	   */	/* hint instructing the use of persistent file realms */	ADIOI_Info_set(info, "romio_cb_pfr", "disable");	fd->hints->cb_pfr = ADIOI_HINT_DISABLE;	/* hint guiding the assignment of persistent file realms */	ADIOI_Info_set(info, "romio_cb_fr_types", "aar");	fd->hints->cb_fr_type = ADIOI_FR_AAR;	/* hint to align file realms with a certain byte value */	ADIOI_Info_set(info, "romio_cb_fr_alignment", "1");	fd->hints->cb_fr_alignment = 1;	/* hint to set a threshold percentage for a datatype's size/extent at	 * which data sieving should be done in collective I/O */	ADIOI_Info_set(info, "romio_cb_ds_threshold", "0");	fd->hints->cb_ds_threshold = 0;	/* hint to switch between point-to-point or all-to-all for two-phase */	ADIOI_Info_set(info, "romio_cb_alltoall", "automatic");	fd->hints->cb_alltoall = ADIOI_HINT_AUTO;	 /* deferred_open derived from no_indep_rw and cb_{read,write} */	fd->hints->deferred_open = 0;	/* buffer size for data sieving in independent reads */	ADIOI_Info_set(info, "ind_rd_buffer_size", ADIOI_GPFS_IND_RD_BUFFER_SIZE_DFLT);	fd->hints->ind_rd_buffer_size = atoi(ADIOI_GPFS_IND_RD_BUFFER_SIZE_DFLT);	/* buffer size for data sieving in independent writes */	ADIOI_Info_set(info, "ind_wr_buffer_size", ADIOI_GPFS_IND_WR_BUFFER_SIZE_DFLT);	fd->hints->ind_wr_buffer_size = atoi(ADIOI_GPFS_IND_WR_BUFFER_SIZE_DFLT);    ADIOI_Info_set(info, "romio_ds_read", "automatic");    fd->hints->ds_read = ADIOI_HINT_AUTO;    ADIOI_Info_set(info, "romio_ds_write", "automatic");    fd->hints->ds_write = ADIOI_HINT_AUTO;//.........这里部分代码省略.........

int MPIOI_File_write_all_begin(MPI_File fh,			       MPI_Offset offset,			       int file_ptr_type,			       const void *buf,			       int count,			       MPI_Datatype datatype,			       char *myname){    int error_code;    MPI_Count datatype_size;    ADIO_File adio_fh;    void *e32buf=NULL;    const void *xbuf=NULL;    MPIU_THREAD_CS_ENTER(ALLFUNC,);    adio_fh = MPIO_File_resolve(fh);    /* --BEGIN ERROR HANDLING-- */    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);    MPIO_CHECK_NOT_SEQUENTIAL_MODE(adio_fh, myname, error_code);    if (file_ptr_type == ADIO_EXPLICIT_OFFSET && offset < 0)    {	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,					  myname, __LINE__, MPI_ERR_ARG,					  "**iobadoffset", 0);	error_code = MPIO_Err_return_file(adio_fh, error_code);	goto fn_exit;    }    if (adio_fh->split_coll_count)    {	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,					  myname, __LINE__, MPI_ERR_IO, 					  "**iosplitcoll", 0);	error_code = MPIO_Err_return_file(adio_fh, error_code);	goto fn_exit;    }    /* --END ERROR HANDLING-- */    adio_fh->split_coll_count = 1;    MPI_Type_size_x(datatype, &datatype_size);    /* --BEGIN ERROR HANDLING-- */    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);    /* --END ERROR HANDLING-- */    xbuf = buf;    if (adio_fh->is_external32) {	error_code = MPIU_external32_buffer_setup(buf, count, datatype, &e32buf);	if (error_code != MPI_SUCCESS) 	    goto fn_exit;	xbuf = e32buf;    }    adio_fh->split_datatype = datatype;    ADIO_WriteStridedColl(adio_fh, xbuf, count, datatype, file_ptr_type,			  offset, &adio_fh->split_status, &error_code);    /* --BEGIN ERROR HANDLING-- */    if (error_code != MPI_SUCCESS)	error_code = MPIO_Err_return_file(adio_fh, error_code);    /* --END ERROR HANDLING-- */fn_exit:    if ( e32buf != NULL) ADIOI_Free(e32buf);    MPIU_THREAD_CS_EXIT(ALLFUNC,);    return error_code;}

/*@    MPI_File_close - Closes a fileInput Parameters:. fh - file handle (handle).N fortran@*/int MPI_File_close(MPI_File *fh){    int error_code;    ADIO_File adio_fh;    static char myname[] = "MPI_FILE_CLOSE";#ifdef MPI_hpux    int fl_xmpi;    HPMP_IO_WSTART(fl_xmpi, BLKMPIFILECLOSE, TRDTBLOCK, *adio_fh);#endif /* MPI_hpux */    MPIU_THREAD_CS_ENTER(ALLFUNC,);    adio_fh = MPIO_File_resolve(*fh);    /* --BEGIN ERROR HANDLING-- */    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);    /* --END ERROR HANDLING-- */    if (ADIO_Feature(adio_fh, ADIO_SHARED_FP))    {	ADIOI_Free((adio_fh)->shared_fp_fname);	/* POSIX semantics say a deleted file remains available until all	 * processes close the file.  But since when was NFS posix-compliant?	 */	if (!ADIO_Feature(adio_fh, ADIO_UNLINK_AFTER_CLOSE)) {		MPI_Barrier((adio_fh)->comm);	}	if ((adio_fh)->shared_fp_fd != ADIO_FILE_NULL) {	    MPI_File *fh_shared = &(adio_fh->shared_fp_fd);	    ADIO_Close((adio_fh)->shared_fp_fd, &error_code);            MPIO_File_free(fh_shared);	    /* --BEGIN ERROR HANDLING-- */	    if (error_code != MPI_SUCCESS) goto fn_fail;	    /* --END ERROR HANDLING-- */	}    }    /* Because ROMIO expects the MPI library to provide error handler management     * routines but it doesn't ever participate in MPI_File_close, we have to     * somehow inform the MPI library that we no longer hold a reference to any     * user defined error handler.  We do this by setting the errhandler at this     * point to MPI_ERRORS_RETURN. */    error_code = PMPI_File_set_errhandler(*fh, MPI_ERRORS_RETURN);    if (error_code != MPI_SUCCESS) goto fn_fail;    ADIO_Close(adio_fh, &error_code);    MPIO_File_free(fh);    /* --BEGIN ERROR HANDLING-- */    if (error_code != MPI_SUCCESS) goto fn_fail;    /* --END ERROR HANDLING-- */#ifdef MPI_hpux    HPMP_IO_WEND(fl_xmpi);#endif /* MPI_hpux */fn_exit:    MPIU_THREAD_CS_EXIT(ALLFUNC,);    return error_code;fn_fail:    /* --BEGIN ERROR HANDLING-- */    error_code = MPIO_Err_return_file(adio_fh, error_code);    goto fn_exit;    /* --END ERROR HANDLING-- */}

//.........这里部分代码省略.........    case ADIO_FCNTL_SET_DISKSPACE:	/* will be called by one process only */#ifdef SPPUX	/* SPPUX has no prealloc64. therefore, use prealloc           if size < (2GB - 1), otherwise use long method. */        if (fcntl_struct->diskspace <= 2147483647) {	    err = prealloc(fd->fd_sys, (off_t) fcntl_struct->diskspace);	    if (err && (errno != ENOTEMPTY)) {#ifdef PRINT_ERR_MSG    	        *error_code = MPI_ERR_UNKNOWN;#else		*error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,			      myname, "I/O Error", "%s", strerror(errno));		ADIOI_Error(fd, *error_code, myname);#endif	        return;	    }	}    	if ((fcntl_struct->diskspace > 2147483647) || 	    (err && (errno == ENOTEMPTY))) {#endif        	/* Explicitly write to allocate space. Since there could be	   holes in the file, I need to read up to the current file	   size, write it back, and then write beyond that depending	   on how much preallocation is needed.           read/write in sizes of no more than ADIOI_PREALLOC_BUFSZ */	    curr_fsize = lseek64(fd->fd_sys, 0, SEEK_END);	    alloc_size = fcntl_struct->diskspace;	    size = ADIOI_MIN(curr_fsize, alloc_size);	    	    ntimes = (size + ADIOI_PREALLOC_BUFSZ - 1)/ADIOI_PREALLOC_BUFSZ;	    buf = (char *) ADIOI_Malloc(ADIOI_PREALLOC_BUFSZ);	    done = 0;	    for (i=0; i<ntimes; i++) {		len = ADIOI_MIN(size-done, ADIOI_PREALLOC_BUFSZ);		ADIO_ReadContig(fd, buf, len, MPI_BYTE, ADIO_EXPLICIT_OFFSET,                       done, &status, error_code);		if (*error_code != MPI_SUCCESS) {#ifdef PRINT_ERR_MSG		    FPRINTF(stderr, "ADIOI_HFS_Fcntl: To preallocate disk space, ROMIO needs to read the file and write it back, but is unable to read the file. Please give the file read permission and open it with MPI_MODE_RDWR./n");		    MPI_Abort(MPI_COMM_WORLD, 1);#else		    *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_PREALLOC_PERM,			      myname, (char *) 0, (char *) 0);		    ADIOI_Error(fd, *error_code, myname);		    return;  #endif		}		ADIO_WriteContig(fd, buf, len, MPI_BYTE, ADIO_EXPLICIT_OFFSET,                         done,  &status, error_code);		if (*error_code != MPI_SUCCESS) return;		done += len;	    }	    if (alloc_size > curr_fsize) {		memset(buf, 0, ADIOI_PREALLOC_BUFSZ); 		size = alloc_size - curr_fsize;		ntimes = (size + ADIOI_PREALLOC_BUFSZ - 1)/ADIOI_PREALLOC_BUFSZ;		for (i=0; i<ntimes; i++) {		    len = ADIOI_MIN(alloc_size-done, ADIOI_PREALLOC_BUFSZ);		    ADIO_WriteContig(fd, buf, len, MPI_BYTE, ADIO_EXPLICIT_OFFSET, 				     done, &status, error_code);		    if (*error_code != MPI_SUCCESS) return;		    done += len;  		}	    }	    ADIOI_Free(buf);#ifdef SPPUX	}#endif	*error_code = MPI_SUCCESS;	break;    case ADIO_FCNTL_SET_IOMODE:        /* for implementing PFS I/O modes. will not occur in MPI-IO           implementation.*/	if (fd->iomode != fcntl_struct->iomode) {	    fd->iomode = fcntl_struct->iomode;	    MPI_Barrier(MPI_COMM_WORLD);	}	*error_code = MPI_SUCCESS;	break;    case ADIO_FCNTL_SET_ATOMICITY:	fd->atomicity = (fcntl_struct->atomicity == 0) ? 0 : 1;	*error_code = MPI_SUCCESS;	break;    default:	FPRINTF(stderr, "Unknown flag passed to ADIOI_HFS_Fcntl/n");	MPI_Abort(MPI_COMM_WORLD, 1);    }}

//.........这里部分代码省略.........	off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind :            fd->disp + (ADIO_Offset)etype_size * offset;	start_off = off;	end_offset = start_off + bufsize - 1;        /* write stripe size buffer each time */	writebuf = (char *) ADIOI_Malloc(ADIOI_MIN(bufsize, stripe_size));        writebuf_off = 0;        writebuf_len = 0;        /* if atomicity is true, lock the region to be accessed */	if (fd->atomicity)	    ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, bufsize);	for (j = 0; j < count; j++) {	    for (i = 0; i < flat_buf->count; i++) {                userbuf_off = (ADIO_Offset)j * (ADIO_Offset)buftype_extent +                    flat_buf->indices[i];		req_off = off;		req_len = flat_buf->blocklens[i];		ADIOI_BUFFERED_WRITE_WITHOUT_READ		off += flat_buf->blocklens[i];	    }        }	/* write the buffer out finally */	ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE,			 ADIO_EXPLICIT_OFFSET, writebuf_off, &status1,			 error_code);	if (fd->atomicity)	    ADIOI_UNLOCK(fd, start_off, SEEK_SET, bufsize);	if (*error_code != MPI_SUCCESS) {            ADIOI_Free(writebuf);	    return;        }	ADIOI_Free(writebuf);	if (file_ptr_type == ADIO_INDIVIDUAL)	    fd->fp_ind = off;    } else {        /* noncontiguous in file */        /* filetype already flattened in ADIO_Open */	flat_file = ADIOI_Flatlist;	while (flat_file->type != fd->filetype)	    flat_file = flat_file->next;	disp = fd->disp;	if (file_ptr_type == ADIO_INDIVIDUAL) {            /* Wei-keng reworked type processing to be a bit more efficient */            offset       = fd->fp_ind - disp;            n_filetypes  = (offset - flat_file->indices[0]) / filetype_extent;            offset      -= (ADIO_Offset)n_filetypes * filetype_extent;            /* now offset is local to this extent */            /* find the block where offset is located, skip blocklens[i]==0 */            for (i=0; i<flat_file->count; i++) {                ADIO_Offset dist;                if (flat_file->blocklens[i] == 0) continue;                dist = flat_file->indices[i] + flat_file->blocklens[i] - offset;                /* fwr_size is from offset to the end of block i */                if (dist == 0) {                    i++;                    offset   = flat_file->indices[i];                    fwr_size = flat_file->blocklens[i];			break;		    }

//.........这里部分代码省略.........	    if (!nprocs_is_valid) {		/* if hints were already initialized, we might not		 * have already gotten this?		 */		MPI_Comm_size(fd->comm, &nprocs);		nprocs_is_valid = 1;	    }	    if (intval < nprocs) {		MPI_Info_set(info, "cb_nodes", value);		fd->hints->cb_nodes = intval;	    }	}	MPI_Info_get(users_info, "ind_wr_buffer_size", MPI_MAX_INFO_VAL, 		     value, &flag);	if (flag && ((intval = atoi(value)) > 0)) {	    MPI_Info_set(info, "ind_wr_buffer_size", value);	    fd->hints->ind_wr_buffer_size = intval;	}	MPI_Info_get(users_info, "ind_rd_buffer_size", MPI_MAX_INFO_VAL, 		     value, &flag);	if (flag && ((intval = atoi(value)) > 0)) {	    MPI_Info_set(info, "ind_rd_buffer_size", value);	    fd->hints->ind_rd_buffer_size = intval;	}	MPI_Info_get(users_info, "cb_config_list", MPI_MAX_INFO_VAL,		     value, &flag);	if (flag) {	    if (fd->hints->cb_config_list == NULL) {		/* only set cb_config_list if it isn't already set.		 * Note that since we set it below, this ensures that		 * the cb_config_list hint will be set at file open time		 * either by the user or to the default		 */	    	MPI_Info_set(info, "cb_config_list", value);		fd->hints->cb_config_list = ADIOI_Malloc((strlen(value)+1) * sizeof(char));		if (fd->hints->cb_config_list == NULL) {		    /* NEED TO HANDLE ENOMEM */		}		strcpy(fd->hints->cb_config_list, value);	    }	    /* if it has been set already, we ignore it the second time. 	     * otherwise we would get an error if someone used the same	     * info value with a cb_config_list value in it in a couple	     * of calls, which would be irritating. */	}    }    /* handle cb_config_list default value here; avoids an extra     * free/alloc and insures it is always set     */    if (fd->hints->cb_config_list == NULL) {	MPI_Info_set(info, "cb_config_list", ADIOI_CB_CONFIG_LIST_DFLT);	fd->hints->cb_config_list = ADIOI_Malloc((strlen(ADIOI_CB_CONFIG_LIST_DFLT)+1) * sizeof(char));	if (fd->hints->cb_config_list == NULL) {	    /* NEED TO HANDLE ENOMEM */	}	strcpy(fd->hints->cb_config_list, ADIOI_CB_CONFIG_LIST_DFLT);    }    /* deferred_open won't be set by callers, but if the user doesn't     * explicitly disable collecitve buffering (two-phase) and does hint that     * io w/o independent io is going on, we'll set this internal hint as a     * convenience */    if ( ( (fd->hints->cb_read != ADIOI_HINT_DISABLE) /			    && (fd->hints->cb_write != ADIOI_HINT_DISABLE)/			    && fd->hints->no_indep_rw ) ) {	    fd->hints->deferred_open = 1;    } else {	    /* setting romio_no_indep_rw enable and romio_cb_{read,write}	     * disable at the same time doesn't make sense. honor	     * romio_cb_{read,write} and force the no_indep_rw hint to	     * 'disable' */	    MPI_Info_set(info, "romio_no_indep_rw", "false");	    fd->hints->no_indep_rw = 0;	    fd->hints->deferred_open = 0;    }    if ((fd->file_system == ADIO_PIOFS) || (fd->file_system == ADIO_PVFS)) {    /* no data sieving for writes in PIOFS and PVFS, because they do not       support file locking */       	MPI_Info_get(info, "ind_wr_buffer_size", MPI_MAX_INFO_VAL,		     value, &flag);	if (flag) {	    /* get rid of this value if it is set */	    MPI_Info_delete(info, "ind_wr_buffer_size");	}	/* note: leave ind_wr_buffer_size alone; used for other cases	 * as well. -- Rob Ross, 04/22/2003	 */	MPI_Info_set(info, "romio_ds_write", "disable");	fd->hints->ds_write = ADIOI_HINT_DISABLE;    }    ADIOI_Free(value);    *error_code = MPI_SUCCESS;}

/*@    MPI_File_write_ordered - Collective write using shared file pointerInput Parameters:. fh - file handle (handle). buf - initial address of buffer (choice). count - number of elements in buffer (nonnegative integer). datatype - datatype of each buffer element (handle)Output Parameters:. status - status object (Status).N fortran@*/int MPI_File_write_ordered(MPI_File fh, ROMIO_CONST void *buf, int count,			   MPI_Datatype datatype, MPI_Status *status){    int error_code, nprocs, myrank;    ADIO_Offset incr;    MPI_Count datatype_size;    int source, dest;    static char myname[] = "MPI_FILE_WRITE_ORDERED";    ADIO_Offset shared_fp;    ADIO_File adio_fh;    void *e32buf=NULL;    const void *xbuf;    MPIU_THREAD_CS_ENTER(ALLFUNC,);    adio_fh = MPIO_File_resolve(fh);    /* --BEGIN ERROR HANDLING-- */    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);    /* --END ERROR HANDLING-- */    MPI_Type_size_x(datatype, &datatype_size);    /* --BEGIN ERROR HANDLING-- */    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);    MPIO_CHECK_FS_SUPPORTS_SHARED(adio_fh, myname, error_code);    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);    /* --END ERROR HANDLING-- */    ADIOI_TEST_DEFERRED(adio_fh, myname, &error_code);    MPI_Comm_size(adio_fh->comm, &nprocs);    MPI_Comm_rank(adio_fh->comm, &myrank);    incr = (count*datatype_size)/adio_fh->etype_size;    /* Use a message as a 'token' to order the operations */    source = myrank - 1;    dest   = myrank + 1;    if (source < 0) source = MPI_PROC_NULL;    if (dest >= nprocs) dest = MPI_PROC_NULL;    MPI_Recv(NULL, 0, MPI_BYTE, source, 0, adio_fh->comm, MPI_STATUS_IGNORE);    ADIO_Get_shared_fp(adio_fh, incr, &shared_fp, &error_code);    /* --BEGIN ERROR HANDLING-- */    if (error_code != MPI_SUCCESS) {	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL,					  myname, __LINE__, MPI_ERR_INTERN,					  "**iosharedfailed", 0);	error_code = MPIO_Err_return_file(adio_fh, error_code);	goto fn_exit;    }    /* --END ERROR HANDLING-- */    MPI_Send(NULL, 0, MPI_BYTE, dest, 0, adio_fh->comm);    xbuf = buf;    if (adio_fh->is_external32) {	error_code = MPIU_external32_buffer_setup(buf, count, datatype, &e32buf);	if (error_code != MPI_SUCCESS)	    goto fn_exit;	xbuf = e32buf;    }    ADIO_WriteStridedColl(adio_fh, xbuf, count, datatype, ADIO_EXPLICIT_OFFSET,                          shared_fp, status, &error_code);    /* --BEGIN ERROR HANDLING-- */    if (error_code != MPI_SUCCESS)	error_code = MPIO_Err_return_file(adio_fh, error_code);    /* --END ERROR HANDLING-- */fn_exit:    if (e32buf != NULL) ADIOI_Free(e32buf);    MPIU_THREAD_CS_EXIT(ALLFUNC,);    /* FIXME: Check for error code from WriteStridedColl? */    return error_code;}

void ADIOI_PFS_Open(ADIO_File fd, int *error_code){    int perm, amode, old_mask, np_comm, np_total, err, flag;    char *value;    struct sattr attr;    static char myname[] = "ADIOI_PFS_OPEN";    if (fd->perm == ADIO_PERM_NULL) {	old_mask = umask(022);	umask(old_mask);	perm = old_mask ^ 0666;    }    else perm = fd->perm;    amode = 0;    if (fd->access_mode & ADIO_CREATE)	amode = amode | O_CREAT;    if (fd->access_mode & ADIO_RDONLY)	amode = amode | O_RDONLY;    if (fd->access_mode & ADIO_WRONLY)	amode = amode | O_WRONLY;    if (fd->access_mode & ADIO_RDWR)	amode = amode | O_RDWR;    if (fd->access_mode & ADIO_EXCL)	amode = amode | O_EXCL;    MPI_Comm_size(MPI_COMM_WORLD, &np_total);    MPI_Comm_size(fd->comm, &np_comm);    if (np_total == np_comm) 	fd->fd_sys = _gopen(fd->filename, amode, M_ASYNC, perm);    else fd->fd_sys = open(fd->filename, amode, perm);    fd->fd_direct = -1;    if (fd->fd_sys != -1) {	value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));        /* if user has asked for pfs server buffering to be turned on,           it will be set to true in fd->info in the earlier call           to ADIOI_PFS_SetInfo. Turn it on now, since we now have a            valid file descriptor. */	ADIOI_Info_get(fd->info, "pfs_svr_buf", MPI_MAX_INFO_VAL, 		     value, &flag);	if (flag && (!strcmp(value, "true"))) {	    err = fcntl(fd->fd_sys, F_PFS_SVR_BUF, TRUE);	    if (err) ADIOI_Info_set(fd->info, "pfs_svr_buf", "false");	}        /* get file striping information and set it in info */	err = fcntl(fd->fd_sys, F_GETSATTR, &attr);	if (!err) {	    MPL_snprintf(value, MPI_MAX_INFO_VAL+1, "%d", attr.s_sunitsize);	    ADIOI_Info_set(fd->info, "striping_unit", value);	    MPL_snprintf(value, MPI_MAX_INFO_VAL+1, "%d", attr.s_sfactor);	    ADIOI_Info_set(fd->info, "striping_factor", value);	    MPL_snprintf(value, MPI_MAX_INFO_VAL+1, "%d", attr.s_start_sdir);	    ADIOI_Info_set(fd->info, "start_iodevice", value);	}	ADIOI_Free(value);	if (fd->access_mode & ADIO_APPEND) 	    fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END);    }    if (fd->fd_sys == -1) {	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,					   myname, __LINE__, MPI_ERR_IO,					   "**io",					   "**io %s", strerror(errno));    }    else *error_code = MPI_SUCCESS;}

/* If successful, error_code is set to MPI_SUCCESS.  Otherwise an error * code is created and returned in error_code. */static void ADIOI_Exch_and_write(ADIO_File fd, void *buf, MPI_Datatype				 datatype, int nprocs,				 int myrank,				 ADIOI_Access				 *others_req, ADIO_Offset *offset_list,				 ADIO_Offset *len_list, int contig_access_count,				 ADIO_Offset min_st_offset, ADIO_Offset fd_size,				 ADIO_Offset *fd_start, ADIO_Offset *fd_end,				 int *buf_idx, int *error_code){/* Send data to appropriate processes and write in sizes of no more   than coll_bufsize.   The idea is to reduce the amount of extra memory required for   collective I/O. If all data were written all at once, which is much   easier, it would require temp space more than the size of user_buf,   which is often unacceptable. For example, to write a distributed   array to a file, where each local array is 8Mbytes, requiring   at least another 8Mbytes of temp space is unacceptable. */    /* Not convinced end_loc-st_loc couldn't be > int, so make these offsets*/    ADIO_Offset size=0;    int hole, i, j, m, ntimes, max_ntimes, buftype_is_contig;    ADIO_Offset st_loc=-1, end_loc=-1, off, done, req_off;    char *write_buf=NULL;    int *curr_offlen_ptr, *count, *send_size, req_len, *recv_size;    int *partial_recv, *sent_to_proc, *start_pos, flag;    int *send_buf_idx, *curr_to_proc, *done_to_proc;    MPI_Status status;    ADIOI_Flatlist_node *flat_buf=NULL;    MPI_Aint buftype_extent;    int info_flag, coll_bufsize;    char *value;    static char myname[] = "ADIOI_EXCH_AND_WRITE";    *error_code = MPI_SUCCESS;  /* changed below if error */    /* only I/O errors are currently reported *//* calculate the number of writes of size coll_bufsize   to be done by each process and the max among all processes.   That gives the no. of communication phases as well. */    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));    ADIOI_Info_get(fd->info, "cb_buffer_size", MPI_MAX_INFO_VAL, value,                 &info_flag);    coll_bufsize = atoi(value);    ADIOI_Free(value);    for (i=0; i < nprocs; i++) {	if (others_req[i].count) {	    st_loc = others_req[i].offsets[0];	    end_loc = others_req[i].offsets[0];	    break;	}    }    for (i=0; i < nprocs; i++)	for (j=0; j < others_req[i].count; j++) {	    st_loc = ADIOI_MIN(st_loc, others_req[i].offsets[j]);	    end_loc = ADIOI_MAX(end_loc, (others_req[i].offsets[j]				       + others_req[i].lens[j] - 1));	}/* ntimes=ceiling_div(end_loc - st_loc + 1, coll_bufsize)*/    ntimes = (int) ((end_loc - st_loc + coll_bufsize)/coll_bufsize);    if ((st_loc==-1) && (end_loc==-1)) {	ntimes = 0; /* this process does no writing. */    }    MPI_Allreduce(&ntimes, &max_ntimes, 1, MPI_INT, MPI_MAX,		  fd->comm);    write_buf = fd->io_buf;    curr_offlen_ptr = (int *) ADIOI_Calloc(nprocs, sizeof(int));    /* its use is explained below. calloc initializes to 0. */    count = (int *) ADIOI_Malloc(nprocs*sizeof(int));    /* to store count of how many off-len pairs per proc are satisfied       in an iteration. */    partial_recv = (int *) ADIOI_Calloc(nprocs, sizeof(int));    /* if only a portion of the last off-len pair is recd. from a process       in a particular iteration, the length recd. is stored here.       calloc initializes to 0. */    send_size = (int *) ADIOI_Malloc(nprocs*sizeof(int));    /* total size of data to be sent to each proc. in an iteration.       Of size nprocs so that I can use MPI_Alltoall later. */    recv_size = (int *) ADIOI_Malloc(nprocs*sizeof(int));    /* total size of data to be recd. from each proc. in an iteration.*/    sent_to_proc = (int *) ADIOI_Calloc(nprocs, sizeof(int));    /* amount of data sent to each proc so far. Used in//.........这里部分代码省略.........

/* this used to be implemented in every file system as an fcntl, but the code * is identical for all file systems without a real "preallocate" system call. * This naive approach will get the job done, but not in a terribly efficient * manner. */void ADIOI_GEN_Prealloc(ADIO_File fd, ADIO_Offset diskspace, int *error_code) {	ADIO_Offset curr_fsize, alloc_size, size, len, done;	ADIO_Status status;	int i, ntimes;	char *buf;	ADIO_Fcntl_t *fcntl_struct;	static char myname[] = "ADIOI_GEN_PREALLOC";	/* will be called by one process only */	/* On file systems with no preallocation function, we have to            explicitly write            to allocate space. Since there could be holes in the file,            we need to read up to the current file size, write it back,            and then write beyond that depending on how much            preallocation is needed.           read/write in sizes of no more than ADIOI_PREALLOC_BUFSZ */	/*curr_fsize = fd->fp_ind; */	fcntl_struct = (ADIO_Fcntl_t *) ADIOI_Malloc(sizeof(ADIO_Fcntl_t));	ADIO_Fcntl(fd, ADIO_FCNTL_GET_FSIZE, fcntl_struct, error_code);	curr_fsize = fcntl_struct->fsize; /* don't rely on fd->fp_ind: might be					    working on a pre-existing file */	alloc_size = diskspace;	size = ADIOI_MIN(curr_fsize, alloc_size);		ntimes = (size + ADIOI_PREALLOC_BUFSZ - 1)/ADIOI_PREALLOC_BUFSZ;	buf = (char *) ADIOI_Malloc(ADIOI_PREALLOC_BUFSZ);	done = 0;	for (i=0; i<ntimes; i++) {	    len = ADIOI_MIN(size-done, ADIOI_PREALLOC_BUFSZ);	    ADIO_ReadContig(fd, buf, len, MPI_BYTE, ADIO_EXPLICIT_OFFSET, done,			    &status, error_code);	    if (*error_code != MPI_SUCCESS) {		*error_code = MPIO_Err_create_code(MPI_SUCCESS,						   MPIR_ERR_RECOVERABLE,						   myname, __LINE__,						   MPI_ERR_IO, 						   "**iopreallocrdwr",						   0);                return;  	    }	    ADIO_WriteContig(fd, buf, len, MPI_BYTE, ADIO_EXPLICIT_OFFSET,                              done, &status, error_code);	    if (*error_code != MPI_SUCCESS) return;	    done += len;	}	if (alloc_size > curr_fsize) {	    memset(buf, 0, ADIOI_PREALLOC_BUFSZ); 	    size = alloc_size - curr_fsize;	    ntimes = (size + ADIOI_PREALLOC_BUFSZ - 1)/ADIOI_PREALLOC_BUFSZ;	    for (i=0; i<ntimes; i++) {		len = ADIOI_MIN(alloc_size-done, ADIOI_PREALLOC_BUFSZ);		ADIO_WriteContig(fd, buf, len, MPI_BYTE, ADIO_EXPLICIT_OFFSET, 				 done, &status, error_code);		if (*error_code != MPI_SUCCESS) return;		done += len;  	    }	}	ADIOI_Free(fcntl_struct);	ADIOI_Free(buf);	*error_code = MPI_SUCCESS;}

void ADIOI_GEN_WriteStridedColl(ADIO_File fd, const void *buf, int count,                       MPI_Datatype datatype, int file_ptr_type,                       ADIO_Offset offset, ADIO_Status *status, int                       *error_code){/* Uses a generalized version of the extended two-phase method described   in "An Extended Two-Phase Method for Accessing Sections of   Out-of-Core Arrays", Rajeev Thakur and Alok Choudhary,   Scientific Programming, (5)4:301--317, Winter 1996.   http://www.mcs.anl.gov/home/thakur/ext2ph.ps */    ADIOI_Access *my_req;    /* array of nprocs access structures, one for each other process in       whose file domain this process's request lies */    ADIOI_Access *others_req;    /* array of nprocs access structures, one for each other process       whose request lies in this process's file domain. */    int i, filetype_is_contig, nprocs, nprocs_for_coll, myrank;    int contig_access_count=0, interleave_count = 0, buftype_is_contig;    int *count_my_req_per_proc, count_my_req_procs, count_others_req_procs;    ADIO_Offset orig_fp, start_offset, end_offset, fd_size, min_st_offset, off;    ADIO_Offset *offset_list = NULL, *st_offsets = NULL, *fd_start = NULL,	*fd_end = NULL, *end_offsets = NULL;    int *buf_idx = NULL;    ADIO_Offset *len_list = NULL;    int old_error, tmp_error;    if (fd->hints->cb_pfr != ADIOI_HINT_DISABLE) {        /* Cast away const'ness as the below function is used for read         * and write */	ADIOI_IOStridedColl (fd, (char *) buf, count, ADIOI_WRITE, datatype,			file_ptr_type, offset, status, error_code);	return;    }    MPI_Comm_size(fd->comm, &nprocs);    MPI_Comm_rank(fd->comm, &myrank);/* the number of processes that actually perform I/O, nprocs_for_coll, * is stored in the hints off the ADIO_File structure */    nprocs_for_coll = fd->hints->cb_nodes;    orig_fp = fd->fp_ind;    /* only check for interleaving if cb_write isn't disabled */    if (fd->hints->cb_write != ADIOI_HINT_DISABLE) {	/* For this process's request, calculate the list of offsets and	   lengths in the file and determine the start and end offsets. */	/* Note: end_offset points to the last byte-offset that will be accessed.	   e.g., if start_offset=0 and 100 bytes to be read, end_offset=99*/	ADIOI_Calc_my_off_len(fd, count, datatype, file_ptr_type, offset,			      &offset_list, &len_list, &start_offset,			      &end_offset, &contig_access_count);	/* each process communicates its start and end offsets to other	   processes. The result is an array each of start and end offsets stored	   in order of process rank. */	st_offsets = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));	end_offsets = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));	MPI_Allgather(&start_offset, 1, ADIO_OFFSET, st_offsets, 1,		      ADIO_OFFSET, fd->comm);	MPI_Allgather(&end_offset, 1, ADIO_OFFSET, end_offsets, 1,		      ADIO_OFFSET, fd->comm);	/* are the accesses of different processes interleaved? */	for (i=1; i<nprocs; i++)	    if ((st_offsets[i] < end_offsets[i-1]) &&                (st_offsets[i] <= end_offsets[i]))                interleave_count++;	/* This is a rudimentary check for interleaving, but should suffice	   for the moment. */    }    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);    if (fd->hints->cb_write == ADIOI_HINT_DISABLE ||	(!interleave_count && (fd->hints->cb_write == ADIOI_HINT_AUTO)))    {	/* use independent accesses */	if (fd->hints->cb_write != ADIOI_HINT_DISABLE) {	    ADIOI_Free(offset_list);	    ADIOI_Free(len_list);	    ADIOI_Free(st_offsets);	    ADIOI_Free(end_offsets);	}	fd->fp_ind = orig_fp;        ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);        if (buftype_is_contig && filetype_is_contig) {            if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {                off = fd->disp + (ADIO_Offset)(fd->etype_size) * offset;                ADIO_WriteContig(fd, buf, count, datatype,				 ADIO_EXPLICIT_OFFSET,//.........这里部分代码省略.........

void ADIOI_XFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code){    char *value, * c;    int flag;    static char xfs_initialized = 0;    if (fd->info == MPI_INFO_NULL) MPI_Info_create(&(fd->info));    ADIOI_Info_set(fd->info, "direct_read", "false");    ADIOI_Info_set(fd->info, "direct_write", "false");    fd->direct_read = fd->direct_write = 0;	if (!xfs_initialized) {		xfs_initialized = 1;		c = getenv("MPIO_DIRECT_READ_CHUNK_SIZE");		if (c) {			int io;			io = atoi(c);			if (io <= 0) {				fprintf(stderr,"MPI: Ignoring an invalid setting for MPIO_DIRECT_READ_CHUNK_SIZE./n""     It must be set to a positive integer value./n");			} else {				xfs_direct_read_chunk_size = io;			}		} else {			xfs_direct_read_chunk_size = 0;		}		c = getenv("MPIO_DIRECT_WRITE_CHUNK_SIZE");		if (c) {			int io;			io = atoi(c);			if (io <= 0) {				fprintf(stderr,"MPI: Ignoring an invalid setting for MPIO_DIRECT_WRITE_CHUNK_SIZE./n""     It must be set to a positive integer value./n");			} else {				xfs_direct_write_chunk_size = io;			}		} else {			xfs_direct_write_chunk_size = 0;		}	}	if (!fd->hints->initialized) {		fd->hints->fs_hints.xfs.read_chunk_sz =			xfs_direct_read_chunk_size;		fd->hints->fs_hints.xfs.write_chunk_sz =			xfs_direct_write_chunk_size;	}    /* has user specified values for keys "direct_read" and "direct write"? */    if (users_info != MPI_INFO_NULL) {	value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));	ADIOI_Info_get(users_info, "direct_read", MPI_MAX_INFO_VAL, 			 value, &flag);	if (flag && !strcmp(value, "true")) {	    ADIOI_Info_set(fd->info, "direct_read", "true");	    fd->direct_read = 1;	}	ADIOI_Info_get(users_info, "direct_write", MPI_MAX_INFO_VAL, 			 value, &flag);	if (flag && !strcmp(value, "true")) {	    ADIOI_Info_set(fd->info, "direct_write", "true");	    fd->direct_write = 1;	}	ADIOI_Free(value);    }        /* set the values for collective I/O and data sieving parameters */    ADIOI_GEN_SetInfo(fd, users_info, error_code);    /* Environment variables override MPI_Info hints */    if (ADIOI_Direct_read) fd->direct_read = 1;    if (ADIOI_Direct_write) fd->direct_write = 1;    /* environment variables checked in ADIO_Init */    *error_code = MPI_SUCCESS;}

/* Sets error_code to MPI_SUCCESS if successful, or creates an error code * in the case of error. */static void ADIOI_W_Exchange_data(ADIO_File fd, void *buf, char *write_buf,				  ADIOI_Flatlist_node *flat_buf, ADIO_Offset				  *offset_list, ADIO_Offset *len_list, int *send_size,				  int *recv_size, ADIO_Offset off, int size,				  int *count, int *start_pos,				  int *partial_recv,				  int *sent_to_proc, int nprocs,				  int myrank, int				  buftype_is_contig, int contig_access_count,				  ADIO_Offset min_st_offset,				  ADIO_Offset fd_size,				  ADIO_Offset *fd_start, ADIO_Offset *fd_end,				  ADIOI_Access *others_req,				  int *send_buf_idx, int *curr_to_proc,				  int *done_to_proc, int *hole, int iter,				  MPI_Aint buftype_extent, int *buf_idx,				  int *error_code){    int i, j, k, *tmp_len, nprocs_recv, nprocs_send, err;    char **send_buf = NULL;    MPI_Request *requests, *send_req;    MPI_Datatype *recv_types;    MPI_Status *statuses, status;    int *srt_len=NULL, sum;    ADIO_Offset *srt_off=NULL;    static char myname[] = "ADIOI_W_EXCHANGE_DATA";/* exchange recv_size info so that each process knows how much to   send to whom. */    MPI_Alltoall(recv_size, 1, MPI_INT, send_size, 1, MPI_INT, fd->comm);    /* create derived datatypes for recv */    nprocs_recv = 0;    for (i=0; i<nprocs; i++) if (recv_size[i]) nprocs_recv++;    recv_types = (MPI_Datatype *)	ADIOI_Malloc((nprocs_recv+1)*sizeof(MPI_Datatype));/* +1 to avoid a 0-size malloc */    tmp_len = (int *) ADIOI_Malloc(nprocs*sizeof(int));    j = 0;    for (i=0; i<nprocs; i++) {	if (recv_size[i]) {/* take care if the last off-len pair is a partial recv */	    if (partial_recv[i]) {		k = start_pos[i] + count[i] - 1;		tmp_len[i] = others_req[i].lens[k];		others_req[i].lens[k] = partial_recv[i];	    }	    ADIOI_Type_create_hindexed_x(count[i],		     &(others_req[i].lens[start_pos[i]]),	             &(others_req[i].mem_ptrs[start_pos[i]]),			 MPI_BYTE, recv_types+j);	    /* absolute displacements; use MPI_BOTTOM in recv */	    MPI_Type_commit(recv_types+j);	    j++;	}    }    /* To avoid a read-modify-write, check if there are holes in the       data to be written. For this, merge the (sorted) offset lists       others_req using a heap-merge. */    sum = 0;    for (i=0; i<nprocs; i++) sum += count[i];    /* valgrind-detcted optimization: if there is no work on this process we do     * not need to search for holes */    if (sum) {        srt_off = (ADIO_Offset *) ADIOI_Malloc(sum*sizeof(ADIO_Offset));        srt_len = (int *) ADIOI_Malloc(sum*sizeof(int));        ADIOI_Heap_merge(others_req, count, srt_off, srt_len, start_pos,                         nprocs, nprocs_recv, sum);    }/* for partial recvs, restore original lengths */    for (i=0; i<nprocs; i++)        if (partial_recv[i]) {            k = start_pos[i] + count[i] - 1;            others_req[i].lens[k] = tmp_len[i];        }    ADIOI_Free(tmp_len);    /* check if there are any holes. If yes, must do read-modify-write.     * holes can be in three places.  'middle' is what you'd expect: the     * processes are operating on noncontigous data.  But holes can also show     * up at the beginning or end of the file domain (see John Bent ROMIO REQ     * #835). Missing these holes would result in us writing more data than     * recieved by everyone else. */    *hole = 0;    if (sum) {        if (off != srt_off[0]) /* hole at the front */            *hole = 1;        else { /* coalesce the sorted offset-length pairs *///.........这里部分代码省略.........

void ADIOI_Heap_free(heap_t *heap) {    ADIOI_Free(heap->nodes);}

//.........这里部分代码省略.........                        req_len = others_req[i].lens[j] -			    partial_send[i];			partial_send[i] = 0;			/* modify the off-len pair to reflect this change */			others_req[i].offsets[j] = req_off;			others_req[i].lens[j] = req_len;		    }		    else {			req_off = others_req[i].offsets[j];                        req_len = others_req[i].lens[j];		    }		    if (req_off < real_off + real_size) {			count[i]++;      ADIOI_Assert((((ADIO_Offset)(MPIR_Upint)read_buf)+req_off-real_off) == (ADIO_Offset)(MPIR_Upint)(read_buf+req_off-real_off));			MPI_Address(read_buf+req_off-real_off,                                &(others_req[i].mem_ptrs[j]));      ADIOI_Assert((real_off + real_size - req_off) == (int)(real_off + real_size - req_off));			send_size[i] += (int)(ADIOI_MIN(real_off + real_size - req_off,                                       (ADIO_Offset)(unsigned)req_len)); 			if (real_off+real_size-req_off < (ADIO_Offset)(unsigned)req_len) {			    partial_send[i] = (int) (real_off + real_size - req_off);			    if ((j+1 < others_req[i].count) &&                                  (others_req[i].offsets[j+1] <                                      real_off+real_size)) { 				/* this is the case illustrated in the				   figure above. */				for_next_iter = ADIOI_MAX(for_next_iter,					  real_off + real_size - others_req[i].offsets[j+1]); 				/* max because it must cover requests 				   from different processes */			    }			    break;			}		    }		    else break;		}		curr_offlen_ptr[i] = j;	    }	}	flag = 0;	for (i=0; i<nprocs; i++)	    if (count[i]) flag = 1;	if (flag) {      ADIOI_Assert(size == (int)size);	    ADIO_ReadContig(fd, read_buf+for_curr_iter, (int)size, MPI_BYTE,			    ADIO_EXPLICIT_OFFSET, off, &status, error_code);	    if (*error_code != MPI_SUCCESS) return;	}		for_curr_iter = for_next_iter;		ADIOI_R_Exchange_data(fd, buf, flat_buf, offset_list, len_list,			    send_size, recv_size, count,        			    start_pos, partial_send, recd_from_proc, nprocs,			    myrank, 			    buftype_is_contig, contig_access_count,			    min_st_offset, fd_size, fd_start, fd_end,			    others_req,                             m, buftype_extent, buf_idx); 	if (for_next_iter) {	    tmp_buf = (char *) ADIOI_Malloc(for_next_iter);      ADIOI_Assert((((ADIO_Offset)(MPIR_Upint)read_buf)+real_size-for_next_iter) == (ADIO_Offset)(MPIR_Upint)(read_buf+real_size-for_next_iter));      ADIOI_Assert((for_next_iter+coll_bufsize) == (size_t)(for_next_iter+coll_bufsize));	    memcpy(tmp_buf, read_buf+real_size-for_next_iter, for_next_iter);	    ADIOI_Free(fd->io_buf);	    fd->io_buf = (char *) ADIOI_Malloc(for_next_iter+coll_bufsize);	    memcpy(fd->io_buf, tmp_buf, for_next_iter);	    read_buf = fd->io_buf;	    ADIOI_Free(tmp_buf);	}	off += size;	done += size;    }    for (i=0; i<nprocs; i++) count[i] = send_size[i] = 0;    for (m=ntimes; m<max_ntimes; m++) /* nothing to send, but check for recv. */	ADIOI_R_Exchange_data(fd, buf, flat_buf, offset_list, len_list,			    send_size, recv_size, count, 			    start_pos, partial_send, recd_from_proc, nprocs,			    myrank, 			    buftype_is_contig, contig_access_count,			    min_st_offset, fd_size, fd_start, fd_end,			    others_req, m,                            buftype_extent, buf_idx);     ADIOI_Free(curr_offlen_ptr);    ADIOI_Free(count);    ADIOI_Free(partial_send);    ADIOI_Free(send_size);    ADIOI_Free(recv_size);    ADIOI_Free(recd_from_proc);    ADIOI_Free(start_pos);}

void ADIOI_BEEGFS_SetInfo( ADIO_File fd, MPI_Info users_info, int *error_code ){    char *value, *pathname, *dname, *slash;    int flag, stripe_val[2], numtargets = 0, chunksize = 0;    struct BeegfsIoctl_MkFileWithStripeHints_Arg createFileArg;    int err, myrank, fd_pdir, perm, old_mask;    static char myname[] = "ADIOI_BEEGFS_SETINFO";    /* set error code to success */    *error_code = MPI_SUCCESS;    value = ( char * )ADIOI_Malloc( ( MPI_MAX_INFO_VAL + 1 ) * sizeof( char ) );    MPI_Comm_rank( fd->comm, &myrank );    /* set hints */    if( ( fd->info ) == MPI_INFO_NULL ) {	MPI_Info_create( &( fd->info ) );	ADIOI_Info_set( fd->info, "striping_unit", "0" );	ADIOI_Info_set( fd->info, "striping_factor", "0" );	/* set users infos */	if( users_info != MPI_INFO_NULL ) {	    /* striping information */	    ADIOI_Info_get( users_info, "striping_unit", MPI_MAX_INFO_VAL, value, &flag );	    if( flag )		chunksize = atoi( value );	    ADIOI_Info_get( users_info, "striping_factor", MPI_MAX_INFO_VAL, value, &flag );	    if( flag )		numtargets = atoi( value );	    /* check stripe info consistency */	    if( myrank == 0 ) {		stripe_val[0] = numtargets;		stripe_val[1] = chunksize;	    }	    MPI_Bcast( stripe_val, 2, MPI_INT, 0, fd->comm );	    if( stripe_val[0] != numtargets || stripe_val[1] != chunksize ) {		FPRINTF( stderr, "ADIOI_BEEGFS_SetInfo: All keys"			         "-striping_factor:striping_unit "			         "need to be identical across all processes/n" );		MPI_Abort( MPI_COMM_WORLD, 1 );	    }	    /* if user has specified striping info, process 0 tries to set it */	    if( myrank == 0 && ( fd->access_mode & ADIO_CREATE ) && numtargets && chunksize ) {		/* open the parent dir to get/set striping info */		pathname = ADIOI_Strdup( fd->filename );		dname = strrchr( pathname, '/' );		if( dname != NULL ) {		    *dname = '/0'; // replace / with nul-character		    fd_pdir = open( pathname, O_RDONLY );		    if( fd_pdir == -1 ) {			FPRINTF( stderr, "Error opening %s: %s/n", pathname, strerror( errno ) );		    }		}		else {		    /* current dir relative path */		    fd_pdir = open( ".", O_RDONLY );		    if( fd_pdir == -1 ) {			FPRINTF( stderr, "Error opening .: %s/n", strerror( errno ) );		    }		}		ADIOI_Free( pathname );		if( fd->perm == ADIO_PERM_NULL ) {		    old_mask = umask( 022 );		    umask( old_mask );		    perm = old_mask ^ 0666;		}		else perm = fd->perm;		/* set create hints depending on e10 hints previously set */		slash = strrchr( fd->filename, '/' );		if( slash != NULL )		    slash += 1;		else		    slash = fd->filename;		createFileArg.filename = slash;		createFileArg.mode = perm;		createFileArg.numtargets = numtargets;		createFileArg.chunksize = chunksize;		/* create the hint file */		err = ioctl( fd_pdir, BEEGFS_IOC_MKFILE_STRIPEHINTS, &createFileArg );		if( err ) {		    FPRINTF( stderr, "BEEGFS_IOC_MKFILE_STRIPEHINTS: %s. ", strerror( errno ) );		    if( errno == EEXIST ) {			/* ignore user striping and use current file info */			FPRINTF( stderr, "[rank:%d] Failure to set stripe info for %s!/n", myrank, fd->filename );		    }		}		/* close the parent dir file descriptor */		close( fd_pdir );	    } /* End of striping parameters validation */	}//.........这里部分代码省略.........

void ADIOI_GEN_ReadStridedColl(ADIO_File fd, void *buf, int count,			       MPI_Datatype datatype, int file_ptr_type,			       ADIO_Offset offset, ADIO_Status *status, int			       *error_code){/* Uses a generalized version of the extended two-phase method described   in "An Extended Two-Phase Method for Accessing Sections of    Out-of-Core Arrays", Rajeev Thakur and Alok Choudhary,   Scientific Programming, (5)4:301--317, Winter 1996.    http://www.mcs.anl.gov/home/thakur/ext2ph.ps */    ADIOI_Access *my_req;     /* array of nprocs structures, one for each other process in       whose file domain this process's request lies */        ADIOI_Access *others_req;    /* array of nprocs structures, one for each other process       whose request lies in this process's file domain. */    int i, filetype_is_contig, nprocs, nprocs_for_coll, myrank;    int contig_access_count=0, interleave_count = 0, buftype_is_contig;    int *count_my_req_per_proc, count_my_req_procs, count_others_req_procs;    ADIO_Offset start_offset, end_offset, orig_fp, fd_size, min_st_offset, off;    ADIO_Offset *offset_list = NULL, *st_offsets = NULL, *fd_start = NULL,	*fd_end = NULL, *end_offsets = NULL;    ADIO_Offset *len_list = NULL;    int *buf_idx = NULL;#ifdef HAVE_STATUS_SET_BYTES    MPI_Count bufsize, size;#endif    if (fd->hints->cb_pfr != ADIOI_HINT_DISABLE) {        ADIOI_IOStridedColl (fd, buf, count, ADIOI_READ, datatype, 			file_ptr_type, offset, status, error_code);        return;    }    MPI_Comm_size(fd->comm, &nprocs);    MPI_Comm_rank(fd->comm, &myrank);    /* number of aggregators, cb_nodes, is stored in the hints */    nprocs_for_coll = fd->hints->cb_nodes;    orig_fp = fd->fp_ind;    /* only check for interleaving if cb_read isn't disabled */    if (fd->hints->cb_read != ADIOI_HINT_DISABLE) {    /* For this process's request, calculate the list of offsets and       lengths in the file and determine the start and end offsets. */    /* Note: end_offset points to the last byte-offset that will be accessed.       e.g., if start_offset=0 and 100 bytes to be read, end_offset=99*/	ADIOI_Calc_my_off_len(fd, count, datatype, file_ptr_type, offset,			      &offset_list, &len_list, &start_offset,			      &end_offset, &contig_access_count);     #ifdef RDCOLL_DEBUG    for (i=0; i<contig_access_count; i++) {	      DBG_FPRINTF(stderr, "rank %d  off %lld  len %lld/n", 			      myrank, offset_list[i], len_list[i]);	      }#endif	/* each process communicates its start and end offsets to other 	   processes. The result is an array each of start and end offsets	   stored in order of process rank. */     	st_offsets = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));	end_offsets = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));	MPI_Allgather(&start_offset, 1, ADIO_OFFSET, st_offsets, 1,		      ADIO_OFFSET, fd->comm);	MPI_Allgather(&end_offset, 1, ADIO_OFFSET, end_offsets, 1,		      ADIO_OFFSET, fd->comm);	/* are the accesses of different processes interleaved? */	for (i=1; i<nprocs; i++)	    if ((st_offsets[i] < end_offsets[i-1]) &&                 (st_offsets[i] <= end_offsets[i]))                interleave_count++;	/* This is a rudimentary check for interleaving, but should suffice	   for the moment. */    }    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);    if (fd->hints->cb_read == ADIOI_HINT_DISABLE	|| (!interleave_count && (fd->hints->cb_read == ADIOI_HINT_AUTO)))     {	/* don't do aggregation */	if (fd->hints->cb_read != ADIOI_HINT_DISABLE) {	    ADIOI_Free(offset_list);	    ADIOI_Free(len_list);	    ADIOI_Free(st_offsets);	    ADIOI_Free(end_offsets);	}	fd->fp_ind = orig_fp;//.........这里部分代码省略.........

/* * ADIOI_Sync_thread_start - start the synchronisation routine */void *ADIOI_Sync_thread_start(void *ptr) {    ADIOI_Sync_thread_t t = (ADIOI_Sync_thread_t)ptr;    ADIOI_Atomic_queue_t q = (ADIOI_Atomic_queue_t)t->sub_;    ADIOI_Sync_req_t r;    size_t wr_count;    MPI_Count datatype_size;    char *buf;    ADIO_Offset bytes_xfered, len, buf_size, offset, off;    int type, count, fflags, error_code;    ADIO_Request *req;    MPI_Datatype datatype;    /* get sync buffer size */    t->fd_;    buf_size = t->fd_->hints->ind_wr_buffer_size;    buf = (char *)ADIOI_Malloc(buf_size);    for(;;) {	/* get a new sync request */#ifndef _USE_PTHREAD_MUTEX_	if ((r = ADIOI_Atomic_queue_front(q)) == NULL)	    continue;#else	r = ADIOI_Atomic_queue_front(q);#endif	/* pop sync request */	ADIOI_Atomic_queue_pop(q);	/* get request type */	ADIOI_Sync_req_get_key(r, ADIOI_SYNC_TYPE, &type);	/* check for shutdown type */	if (type == ADIOI_THREAD_SHUTDOWN) {	    break;	}	/* if sync type get all the fields */	ADIOI_Sync_req_get_key(r, ADIOI_SYNC_ALL, &offset,		&datatype, &count, &req, &error_code, &fflags);	/* init I/O req */	MPI_Type_size_x(datatype, &datatype_size);	len = (ADIO_Offset)datatype_size * (ADIO_Offset)count;	bytes_xfered = 0;	off = offset;	/* satisfy sync req */	while (bytes_xfered < len) {	    wr_count = (size_t)ADIOI_MIN(buf_size, len - bytes_xfered);#ifdef ADIOI_MPE_LOGGING	    MPE_Log_event(ADIOI_MPE_thread_read_a, 0, NULL);#endif	    /* read data from cache file */	    pread(t->fd_->cache_fd->fd_sys, buf, wr_count, offset);#ifdef ADIOI_MPE_LOGGING	    MPE_Log_event(ADIOI_MPE_thread_read_b, 0, NULL);	    MPE_Log_event(ADIOI_MPE_thread_write_a, 0, NULL);#endif	    /* write data to global file */	    pwrite(t->fd_->fd_sys, buf, wr_count, offset);#ifdef ADIOI_MPE_LOGGING	    MPE_Log_event(ADIOI_MPE_thread_write_b, 0, NULL);#endif	    /* update offset */	    bytes_xfered += (ADIO_Offset)wr_count;	    offset += (ADIO_Offset)wr_count;	}	/* unlock extent locked in ADIO_WriteContig() */	if (t->fd_->hints->e10_cache_coherent == ADIOI_HINT_ENABLE)	    ADIOI_UNLOCK(t->fd_, off, SEEK_SET, len);	/*  ---Begin Error Handling--- */	/*  --- End Error Handling --- */	/* complete Grequest */	MPI_Grequest_complete(*req);    }    ADIOI_Free(buf);    pthread_exit(NULL);}

//.........这里部分代码省略........./* post recvs. if buftype_is_contig, data can be directly recd. into   user buf at location given by buf_idx. else use recv_buf. */#ifdef AGGREGATION_PROFILE    MPE_Log_event (5032, 0, NULL);#endif    if (buftype_is_contig) {	j = 0;	for (i=0; i < nprocs; i++) 	    if (recv_size[i]) {		MPI_Irecv(((char *) buf) + buf_idx[i], recv_size[i], 		  MPI_BYTE, i, myrank+i+100*iter, fd->comm, requests+j);		j++;		buf_idx[i] += recv_size[i];	    }    }    else {/* allocate memory for recv_buf and post receives */	recv_buf = (char **) ADIOI_Malloc(nprocs * sizeof(char*));	for (i=0; i < nprocs; i++) 	    if (recv_size[i]) recv_buf[i] =                                   (char *) ADIOI_Malloc(recv_size[i]);	    j = 0;	    for (i=0; i < nprocs; i++) 		if (recv_size[i]) {		    MPI_Irecv(recv_buf[i], recv_size[i], MPI_BYTE, i, 			      myrank+i+100*iter, fd->comm, requests+j);		    j++;#ifdef RDCOLL_DEBUG		    DBG_FPRINTF(stderr, "node %d, recv_size %d, tag %d /n", 		       myrank, recv_size[i], myrank+i+100*iter); #endif		}    }/* create derived datatypes and send data */    j = 0;    for (i=0; i<nprocs; i++) {	if (send_size[i]) {/* take care if the last off-len pair is a partial send */	    if (partial_send[i]) {		k = start_pos[i] + count[i] - 1;		tmp = others_req[i].lens[k];		others_req[i].lens[k] = partial_send[i];	    }	    ADIOI_Type_create_hindexed_x(count[i],		  &(others_req[i].lens[start_pos[i]]),	            &(others_req[i].mem_ptrs[start_pos[i]]), 			 MPI_BYTE, &send_type);	    /* absolute displacement; use MPI_BOTTOM in send */	    MPI_Type_commit(&send_type);	    MPI_Isend(MPI_BOTTOM, 1, send_type, i, myrank+i+100*iter,		      fd->comm, requests+nprocs_recv+j);	    MPI_Type_free(&send_type);	    if (partial_send[i]) others_req[i].lens[k] = tmp;	    j++;	}    }    statuses = (MPI_Status *) ADIOI_Malloc((nprocs_send+nprocs_recv+1) * /                                     sizeof(MPI_Status));      /* +1 to avoid a 0-size malloc */    /* wait on the receives */    if (nprocs_recv) {#ifdef NEEDS_MPI_TEST	j = 0;	while (!j) MPI_Testall(nprocs_recv, requests, &j, statuses);#else	MPI_Waitall(nprocs_recv, requests, statuses);#endif	/* if noncontiguous, to the copies from the recv buffers */	if (!buftype_is_contig) 	    ADIOI_Fill_user_buffer(fd, buf, flat_buf, recv_buf,				   offset_list, len_list, (unsigned*)recv_size, 				   requests, statuses, recd_from_proc, 				   nprocs, contig_access_count,				   min_st_offset, fd_size, fd_start, fd_end,				   buftype_extent);    }    /* wait on the sends*/    MPI_Waitall(nprocs_send, requests+nprocs_recv, statuses+nprocs_recv);    ADIOI_Free(statuses);    ADIOI_Free(requests);    if (!buftype_is_contig) {	for (i=0; i < nprocs; i++) 	    if (recv_size[i]) ADIOI_Free(recv_buf[i]);	ADIOI_Free(recv_buf);    }#ifdef AGGREGATION_PROFILE    MPE_Log_event (5033, 0, NULL);#endif}

void ADIOI_PIOFS_Open(ADIO_File fd, int *error_code){    int amode, perm, old_mask, err;    piofs_fstat_t piofs_fstat;    char *value;#ifndef PRINT_ERR_MSG    static char myname[] = "ADIOI_PIOFS_OPEN";#endif    if (fd->perm == ADIO_PERM_NULL) {	old_mask = umask(022);	umask(old_mask);	perm = old_mask ^ 0666;    }    else perm = fd->perm;    amode = 0;    if (fd->access_mode & ADIO_CREATE)	amode = amode | O_CREAT;    if (fd->access_mode & ADIO_RDONLY)	amode = amode | O_RDONLY;    if (fd->access_mode & ADIO_WRONLY)	amode = amode | O_WRONLY;    if (fd->access_mode & ADIO_RDWR)	amode = amode | O_RDWR;    if (fd->access_mode & ADIO_EXCL)	amode = amode | O_EXCL;#ifdef PROFILE    MPE_Log_event(1, 0, "start open");#endif    fd->fd_sys = open(fd->filename, amode, perm);#ifdef PROFILE    MPE_Log_event(2, 0, "end open");#endif    llseek(fd->fd_sys, 0, SEEK_SET);/* required to initiate use of 64-bit offset */    if (fd->fd_sys != -1) {	value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));        /* get file striping information and set it in info */	err = piofsioctl(fd->fd_sys, PIOFS_FSTAT, &piofs_fstat);	if (!err) {	    sprintf(value, "%d", piofs_fstat.st_bsu);	    MPI_Info_set(fd->info, "striping_unit", value);	    sprintf(value, "%d", piofs_fstat.st_cells);	    MPI_Info_set(fd->info, "striping_factor", value);	    sprintf(value, "%d", piofs_fstat.st_base_node);	    MPI_Info_set(fd->info, "start_iodevice", value);	}	ADIOI_Free(value);	if (fd->access_mode & ADIO_APPEND)	    fd->fp_ind = fd->fp_sys_posn = llseek(fd->fd_sys, 0, SEEK_END);    }#ifdef PRINT_ERR_MSG    *error_code = (fd->fd_sys == -1) ? MPI_ERR_UNKNOWN : MPI_SUCCESS;#else    if (fd->fd_sys == -1) {	*error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,			      myname, "I/O Error", "%s", strerror(errno));	ADIOI_Error(ADIO_FILE_NULL, *error_code, myname);	        }    else *error_code = MPI_SUCCESS;#endif}

//.........这里部分代码省略.........		if ((remainingDataOffsetToSendPerProc[i] +			    remainingDataAmountToSendPerProc[i]) <= currentRoundFDEnd)		    dataSizeSentThisRoundPerProc[i] = remainingDataAmountToSendPerProc[i];		else		    dataSizeSentThisRoundPerProc[i] =			(currentRoundFDEnd - remainingDataOffsetToSendPerProc[i]) +1;	    }	    else if (((remainingDataOffsetToSendPerProc[i]+			    remainingDataAmountToSendPerProc[i]) >=			currentRoundFDStart) &&		    ((remainingDataOffsetToSendPerProc[i]+		      remainingDataAmountToSendPerProc[i]) <= currentRoundFDEnd)) {		if ((remainingDataOffsetToSendPerProc[i]) >= currentRoundFDStart)		    dataSizeSentThisRoundPerProc[i] = remainingDataAmountToSendPerProc[i];		else		    dataSizeSentThisRoundPerProc[i] =			(remainingDataOffsetToSendPerProc[i]-currentRoundFDStart) +1;	    }	    else		dataSizeSentThisRoundPerProc[i] = 0;	}	/* the target procs get the data from the source aggs */	for (i = 0; i < numSourceAggsThisRound; i++) {	    MPI_Irecv(&((char*)buf)[recvBufferOffsetsThisRound[i]],		    bufferAmountsToGetThisRound[i],MPI_BYTE,		    sourceAggsForMyData[sourceAggIndexesForMyDataThisRound[i]],0,fd->comm,		    &mpiRecvDataFromSourceAggsRequest[i]);	}	/* the source aggs send the data to the target procs */	int numTargetProcsSentThisRound = 0;	for (i=0;i<numTargetProcs;i++) {	    int currentWBOffset = 0;	    for (j=0;j<i;j++)		currentWBOffset += dataSizeSentThisRoundPerProc[j];	    /* only send to target procs that will recv > 0 count data */	    if (dataSizeSentThisRoundPerProc[i] > 0) {		MPI_Isend(&((char*)read_buf)[currentWBOffset],			dataSizeSentThisRoundPerProc[i],			MPI_BYTE,targetProcsForMyData[i],0,			fd->comm,&mpiSendDataToTargetProcRequest[numTargetProcsSentThisRound]);		numTargetProcsSentThisRound++;		remainingDataAmountToSendPerProc[i] -= dataSizeSentThisRoundPerProc[i];		remainingDataOffsetToSendPerProc[i] += dataSizeSentThisRoundPerProc[i];	    }	}	/* wait for the target procs to get their data */	for (i = 0; i < numSourceAggsThisRound; i++) {	    MPI_Waitany(numSourceAggsThisRound,mpiRecvDataFromSourceAggsRequest,		    &irecv,&mpiWaitAnyStatusFromSourceProcs);	}	nextRoundFDStart = currentRoundFDStart + coll_bufsize;        /* clean up the MPI_Isend MPI_Requests */        for (i=0;i<numTargetProcsSentThisRound;i++) {          MPI_Waitany(numTargetProcsSentThisRound,mpiSendDataToTargetProcRequest,		  &isend,&mpiIsendStatusForData);        }	MPI_Barrier(fd->comm); /* need to sync up the source aggs which did the isend with the target procs which did the irecvs to give the target procs time to get the data before overwriting with next round readcontig */    } /* for-loop roundIter */    if (useIOBuffer) { /* thread reader cleanup */	if ( !pthread_equal(io_thread, pthread_self()) ) {	    pthread_join(io_thread, &thread_ret);	    *error_code = *(int *)thread_ret;	}    }    if (iAmUsedAgg) {	ADIOI_Free(targetProcsForMyData);	ADIOI_Free(remainingDataAmountToSendPerProc);	ADIOI_Free(remainingDataOffsetToSendPerProc);    }    ADIOI_Free(sourceAggsForMyData);    ADIOI_Free(sourceAggsForMyDataFDStart);    ADIOI_Free(sourceAggsForMyDataFDEnd);    ADIOI_Free(mpiRecvDataFromSourceAggsRequest);    ADIOI_Free(mpiSendDataToTargetProcRequest);    ADIOI_Free(dataSizeSentThisRoundPerProc);    ADIOI_Free(sourceAggIndexesForMyDataThisRound);    ADIOI_Free(recvBufferOffsetsThisRound);    ADIOI_Free(bufferAmountsToGetThisRound);    /* TODO: is Barrier here needed? */    MPI_Barrier(fd->comm);    return;}