Skip to content

Commit 9d984c4

Browse files
JulianKunkelJulianKunkel
authored
Merge pull request #492 from adrianjhpc/master
Fixing issue raised in issue 491 by previously added Lustre pool functionality
2 parents 5d8cb1a + b370c21 commit 9d984c4

File tree

2 files changed

+76
-43
lines changed

2 files changed

+76
-43
lines changed

‎src/aiori-POSIX.c

+71-40
Original file line numberDiff line numberDiff line change
@@ -113,17 +113,16 @@ option_help * POSIX_options(aiori_mod_opt_t ** init_backend_options, aiori_mod_o
113113
}else{
114114
memset(o, 0, sizeof(posix_options_t));
115115
o->direct_io = 0;
116-
o->lustre_stripe_count = -1;
117-
#ifdef HAVE_LUSTRE_USER
118-
if(LOV_USER_MAGIC != LOV_USER_MAGIC_V1){
119-
strcpy(o->lustre_pool, "");
120-
}
121-
#endif /* HAVE_LUSTRE_USER */
116+
o->lustre_set_striping = 0;
117+
o->lustre_stripe_count = 0;
118+
o->lustre_stripe_size = 0;
119+
o->lustre_pool = NULL;
120+
o->lustre_set_pool = 0;
122121
o->lustre_start_ost = -1;
123122
o->beegfs_numTargets = -1;
124123
o->beegfs_chunkSize = -1;
125124
}
126-
125+
127126
*init_backend_options = (aiori_mod_opt_t*) o;
128127

129128
option_help h [] = {
@@ -144,20 +143,23 @@ option_help * POSIX_options(aiori_mod_opt_t ** init_backend_options, aiori_mod_o
144143
{0, "posix.gpfs.createsharing", " Enable efficient file creation in a shared directory", OPTION_FLAG, 'd', & o->gpfs_createsharing},
145144
#endif
146145
#endif // HAVE_GPFS_FCNTL_H
147-
#ifdef HAVE_LUSTRE_USER
146+
#if defined(HAVE_LUSTRE_USER) || defined(HAVE_LUSTRE_LUSTREAPI)
148147
{0, "posix.lustre.stripecount", "", OPTION_OPTIONAL_ARGUMENT, 'd', & o->lustre_stripe_count},
149148
{0, "posix.lustre.stripesize", "", OPTION_OPTIONAL_ARGUMENT, 'd', & o->lustre_stripe_size},
150-
{0, "posix.lustre.pool", "", OPTION_OPTIONAL_ARGUMENT, 'd', & o->lustre_pool},
151149
{0, "posix.lustre.startost", "", OPTION_OPTIONAL_ARGUMENT, 'd', & o->lustre_start_ost},
152150
{0, "posix.lustre.ignorelocks", "", OPTION_FLAG, 'd', & o->lustre_ignore_locks},
153151
#endif /* HAVE_LUSTRE_USER */
152+
#ifdef HAVE_LUSTRE_LUSTREAPI
153+
{0, "posix.lustre.pool", "Name for Lustre pool to use", OPTION_OPTIONAL_ARGUMENT, 's', & o->lustre_pool},
154+
#endif
154155
#ifdef HAVE_GPU_DIRECT
155156
{0, "gpuDirect", "allocate I/O buffers on the GPU", OPTION_FLAG, 'd', & o->gpuDirect},
156157
#endif
157158
LAST_OPTION
158159
};
159160
option_help * help = malloc(sizeof(h));
160161
memcpy(help, h, sizeof(h));
162+
161163
return help;
162164
}
163165

@@ -204,26 +206,24 @@ int POSIX_check_params(aiori_mod_opt_t * param){
204206
posix_options_t * o = (posix_options_t*) param;
205207
if (o->beegfs_chunkSize != -1 && (!ISPOWEROFTWO(o->beegfs_chunkSize) || o->beegfs_chunkSize < (1<<16)))
206208
ERR("beegfsChunkSize must be a power of two and >64k");
207-
if(o->lustre_stripe_count != -1 || o->lustre_stripe_size != 0 || o->lustre_pool != ""){
209+
if(o->lustre_stripe_count != 0 || o->lustre_stripe_size != 0 || (o->lustre_pool)){
210+
#if defined(HAVE_LUSTRE_USER) || defined(HAVE_LUSTRE_LUSTREAPI)
208211
o->lustre_set_striping = 1;
209-
/* Setting a lustre pool via the API is only support for Lustre API versions higher than V1 */
210-
/* This code checks that the user hasn't just specified a pool with no stripe size or stripe count */
211-
/* on the V1 API because in that scenario we cannot do anything. If that is the case disable trying */
212-
/* to set the stripe. If stripe count and/or stripe size are set the operation can continue but the */
213-
/* pool will not be applied. */
214-
#ifdef HAVE_LUSTRE_USER
215-
if(o->lustre_pool != "" && LOV_USER_MAGIC == LOV_USER_MAGIC_V1){
216-
WARN("Lustre pool specified, but the Lustre User API on this system does not support that.");
217-
if(o->lustre_stripe_count == -1 && o->lustre_stripe_size == 0){
218-
o->lustre_set_striping = 0;
219-
ERR("Setting Lustre pool is not supported for the version of the Lustre API used on this system");
220-
}
221-
}else{
222-
if(o->lustre_pool != "" && o->lustre_stripe_count <= 0 ){
223-
o->lustre_stripe_count = 1;
224-
}
225-
}
226-
#endif /* HAVE_LUSTRE_USER */
212+
#else
213+
WARN("Lustre striping options set but not available");
214+
o->lustre_set_striping = 0;
215+
#endif
216+
}
217+
if((o->lustre_pool)){
218+
if(o->lustre_stripe_count == 0 || o->lustre_stripe_size == 0){
219+
ERR("Lustre pool specified but no stripe count or stripe size specified");
220+
}
221+
#ifdef HAVE_LUSTRE_LUSTREAPI
222+
o->lustre_set_pool = 1;
223+
#else
224+
WARN("Lustre pool option set but the Lustre API is not available to do this");
225+
o->lustre_set_pool = 0;
226+
#endif
227227
}
228228
if(o->gpuDirect && ! o->direct_io){
229229
ERR("GPUDirect required direct I/O to be used!");
@@ -504,6 +504,7 @@ void lustre_disable_file_locks(const int fd) {
504504
aiori_fd_t *POSIX_Create(char *testFileName, int flags, aiori_mod_opt_t * param)
505505
{
506506
int fd_oflag = O_BINARY;
507+
int fd_mode;
507508
int mode = 0664;
508509
posix_fd * pfd = safeMalloc(sizeof(posix_fd));
509510
posix_options_t * o = (posix_options_t*) param;
@@ -514,16 +515,15 @@ aiori_fd_t *POSIX_Create(char *testFileName, int flags, aiori_mod_opt_t * param)
514515
if(hints->dryRun)
515516
return (aiori_fd_t*) 0;
516517

517-
#ifdef HAVE_LUSTRE_USER
518+
#if defined(HAVE_LUSTRE_USER) || defined(HAVE_LUSTRE_LUSTREAPI)
518519
/* Add a #define for FASYNC if not available, as it forms part of
519520
* the Lustre O_LOV_DELAY_CREATE definition. */
520521
#ifndef FASYNC
521522
#define FASYNC 00020000 /* fcntl, for BSD compatibility */
522523
#endif
523-
if (o->lustre_set_striping) {
524-
/* In the single-shared-file case, task 0 has to create the
525-
file with the Lustre striping options before any other
526-
processes open the file */
524+
if (o->lustre_set_striping || o->lustre_set_pool) {
525+
#ifdef HAVE_LUSTRE_LUSTREAPI
526+
527527
if (!hints->filePerProc && rank != 0) {
528528
MPI_CHECK(MPI_Barrier(testComm), "barrier error");
529529
fd_oflag |= O_RDWR;
@@ -533,19 +533,46 @@ aiori_fd_t *POSIX_Create(char *testFileName, int flags, aiori_mod_opt_t * param)
533533
testFileName, fd_oflag, mode, strerror(errno));
534534
}
535535
} else {
536-
struct lov_user_md opts = { 0 };
537536

537+
fd_oflag |= O_CREAT | O_EXCL | O_RDWR;
538+
fd_mode = S_IRWXU;
539+
if(o->lustre_set_pool){
540+
pfd->fd = llapi_file_open_pool(testFileName, fd_oflag, fd_mode, o->lustre_stripe_size,
541+
o->lustre_start_ost, o->lustre_stripe_count, 0,
542+
o->lustre_pool);
543+
}else{
544+
pfd->fd = llapi_file_open(testFileName, fd_oflag, fd_mode, o->lustre_stripe_size,
545+
o->lustre_start_ost, o->lustre_stripe_count, 0);
546+
}
547+
548+
if (pfd->fd < 0)
549+
ERRF("Unable to open '%s': %s\n",
550+
testFileName, strerror(errno));
551+
552+
if (!hints->filePerProc)
553+
MPI_CHECK(MPI_Barrier(testComm),
554+
"barrier error");
555+
556+
}
557+
558+
#else
559+
if (!hints->filePerProc && rank != 0) {
560+
MPI_CHECK(MPI_Barrier(testComm), "barrier error");
561+
fd_oflag |= O_RDWR;
562+
pfd->fd = open64(testFileName, fd_oflag, mode);
563+
if (pfd->fd < 0){
564+
ERRF("open64(\"%s\", %d, %#o) failed. Error: %s",
565+
testFileName, fd_oflag, mode, strerror(errno));
566+
}
567+
} else {
568+
struct lov_user_md opts = { 0 };
538569
/* Setup Lustre IOCTL striping pattern structure */
539570
opts.lmm_magic = LOV_USER_MAGIC;
540571
opts.lmm_stripe_size = o->lustre_stripe_size;
541572
opts.lmm_stripe_offset = o->lustre_start_ost;
542573
opts.lmm_stripe_count = o->lustre_stripe_count;
543-
#if(LOV_USER_MAGIC != LOV_USER_MAGIC_V1)
544-
opts.lmm_pool_name = o->lustre_pool;
545-
#endif
546574
/* File needs to be opened O_EXCL because we cannot set
547575
* Lustre striping information on a pre-existing file.*/
548-
549576
fd_oflag |= O_CREAT | O_EXCL | O_RDWR | O_LOV_DELAY_CREATE;
550577
pfd->fd = open64(testFileName, fd_oflag, mode);
551578
if (pfd->fd < 0) {
@@ -558,12 +585,16 @@ aiori_fd_t *POSIX_Create(char *testFileName, int flags, aiori_mod_opt_t * param)
558585
ERRF("Error on ioctl for '%s' (%d): %s\n",
559586
testFileName, pfd->fd, errmsg);
560587
}
588+
561589
if (!hints->filePerProc)
562590
MPI_CHECK(MPI_Barrier(testComm),
563-
"barrier error");
591+
"barrier error");
592+
564593
}
594+
#endif
595+
565596
} else {
566-
#endif /* HAVE_LUSTRE_USER */
597+
#endif /* HAVE_LUSTRE_USER || HAVE_LUSTRE_LUSTREAPI */
567598

568599
fd_oflag |= O_CREAT | O_RDWR;
569600

‎src/aiori-POSIX.h

+5-3
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@
1212
#else
1313
#define LUSTRE_POOL_NAME_MAX 15
1414
#endif /* HAVE_LUSTRE_USER */
15-
15+
#ifdef HAVE_LUSTRE_LUSTREAPI
16+
#include <lustre/lustreapi.h>
17+
#endif /* HAVE_LUSTRE_LUSTREAPI */
1618

1719
/************************** O P T I O N S *****************************/
1820
typedef struct{
@@ -22,8 +24,8 @@ typedef struct{
2224
/* Lustre variables */
2325
int lustre_set_striping; /* flag that we need to set lustre striping */
2426
int lustre_stripe_count;
25-
int lustre_set_pool;
26-
char lustre_pool[LUSTRE_POOL_NAME_MAX];
27+
int lustre_set_pool; /* flag that we need to set a lustre pool */
28+
char * lustre_pool;
2729
int lustre_stripe_size;
2830
int lustre_start_ost;
2931
int lustre_ignore_locks;

0 commit comments

Comments
 (0)