Numerous changes to file-modes, small build-tweaks, and a tweak to aiori-S3.c

(Only rank-0 should create the bucket, if it doesn't already exist.)

Prepping this for a push (as an experimental S3 build) to github.

Author: jti-lanl

COPYRIGHT

@@ -1,3 +1,5 @@
+[See also NOTES.txt]
+
 Building
 --------
 

ChangeLog

@@ -0,0 +1,82 @@
+[This is some help for building with experimental S3 support.]
+
+
+--- BUILDING
+
+module load openmpi-gnu
+
+./bootstrap
+
+
+# configure must be able to find libaws4c, libxml2, libcurl, libcrypto, etc.
+# On some machines, the libxml2 include files are in:
+#
+#     /usr/include/libxml2
+#
+# which is not a standard include location, so we must define CPPFLAGS
+# to put that path in the include path. Do this:
+#
+#     setenv CPPFLAGS -I/usr/include/libxml2
+#     [or add "CPPFLAGS=-I/usr/include/libxml2" to the end of the configure command]
+#
+# Use the --with-aws4c=DIR to point to the path where your libaws4c library
+# sits.
+#
+# Use the --prefix=iorInstallDir to point to the path where you want IOR
+# to install all the files it builds during the "make" process.
+
+./configure --prefix=iorInstallDir --with-S3 [ --with-aws4c=DIR ]
+
+
+# Here is an example configure-command invocation, for building with the
+# S3-targeting extensions, which require access to additional libraries
+# mentioned above.  This will probably not work verbatim on your
+# installation, because libraries will be installed in different locations,
+# but it could help you to understand what is needed.
+
+./configure --prefix=`pwd`/installed \
+            --with-S3 \
+            --with-aws4c=`pwd`/aws4c \
+            LDFLAGS="-L/usr/lib64 -L`pwd`/libxml2-2.9.1/installed/lib" \
+            CFLAGS="-I`pwd`/libxml2-2.9.1/installed/include/libxml2"
+
+
+# 14-May-2015:
+# To change the target of the experiment, there is an #if block from line
+# 284-309. The "if" portion is activated by putting a 1 as the argument
+# instead of a 0. In that case, the experiment will use the four ECS nodes
+# directly, splitting the load up between all four.
+#
+# If the #if argument is 0, then the "#else" portion is executed. In this
+# case you can use the load balancer, haproxy.ccstar.lanl.gov, by using
+# the IP 10.143.0.1:80. If you want to use one of the ECS nodes directly
+# us the IP 10.140.0.[15-17]:9020.
+#
+# To specify the bucket where the experiment file(s) will go, you need
+# to set that with the "const char* bucket_name declaration. There are
+# a couple options at lines 207-208.
+
+
+make
+make install
+
+
+-- RUNNING  (various options ...)
+
+# llogin -np 4
+msub -IX -l nodes=4:ppn=4,walltime=11:00:00
+
+
+
+# For debugging, run on 1 node, -vvvv turns on detailed curl debugging
+mpirun -np 1 MY_INSTALL_DIR/bin/ior -a S3 -o test_`date +"%Y%m%d_%H%M%S"` -vvvv -t1k -b1k
+
+# this defaults the number of processors
+mpirun MY_INSTALL_DIRbin/ior -a S3 -C -o test_`date +"%Y%m%d_%H%M%S"`
+
+# this does one parallel run, putting a heavy load on the server [assumes bash]
+mpirun -npernode 8 MY_INSTALL_DIR/bin/ior -a S3_EMC -C -o test_`date +"%Y%m%d_%H%M%S"` \
+ -b $(( 128 * 1024 * 1024 )) \
+ -t $(( 128 * 1024 * 1024 )) \
+ -i 1
+

META

@@ -112,17 +112,81 @@ AM_COND_IF([USE_POSIX_AIORI],[
         AC_DEFINE([USE_POSIX_AIORI], [], [Build POSIX backend AIORI])
 ])
 
-# Amazon S3 support
+
+
+
+# aws4c is needed for the S3 backend (see --with-S3, below).
+# Version 0.5.2 of aws4c is available at https://github.com/jti-lanl/aws4c.git
+# Install it something like this:
+#
+#   cd $my_install_dir
+#   git clone https://github.com/jti-lanl/aws4c.git
+#   cd aws4c
+#   make
+#
+# Then:
+#   --with-S3 --with-aws4c=$my_install_dir/aws4c
+
+aws4c_dir=
+AC_ARG_WITH([aws4c],
+        [AS_HELP_STRING([--with-aws4c=DIR],
+           [aws4c library is needed for Amazon S3 backend])],
+        [aws4c_dir="$withval"])
+AM_CONDITIONAL([AWS4C_DIR], [test x$aws4c_dir != x])
+
+# AC_SUBST([AWS4C_DIR],[$aws4c_dir])
+AM_COND_IF([AWS4C_DIR],[
+   AC_SUBST([AWS4C_CPPFLAGS],[-I$aws4c_dir])
+   AC_SUBST([AWS4C_LDFLAGS], [-L$aws4c_dir])
+])
+
+
+# Amazon S3 support  [see also:  --with-aws4c]
 AC_ARG_WITH([S3],
         [AS_HELP_STRING([--with-S3],
-           [support IO with Amazon S3 backend @<:@default=yes@:>@])],
+           [support IO with Amazon S3 backend @<:@default=no@:>@])],
         [],
-        [with_S3=yes])
+        [with_S3=no])
 AM_CONDITIONAL([USE_S3_AIORI], [test x$with_S3 = xyes])
 AM_COND_IF([USE_S3_AIORI],[
         AC_DEFINE([USE_S3_AIORI], [], [Build Amazon-S3 backend AIORI])
 ])
 
+err=0
+AS_IF([test "x$with_S3" != xno], [
+            AC_MSG_NOTICE([beginning of S3-related checks])
+
+            # save user's values, while we use AC_CHECK_HEADERS with $AWS4C_DIR
+            ORIG_CPPFLAGS=$CPPFLAGS
+            ORIG_LDFLAGS=$LDFLAGS
+
+            CPPFLAGS="$CPPFLAGS $AWS4C_CPPFLAGS"
+            LDFLAGS=" $LDFLAGS  $AWS4C_LDFLAGS"
+
+            AC_CHECK_HEADERS([aws4c.h], [], [err=1])
+            AC_CHECK_HEADERS([libxml/parser.h], [], [err=1])
+
+            # Autotools thinks searching for a library means I want it added to LIBS
+            ORIG_LIBS=$LIBS
+            AC_CHECK_LIB([curl], [curl_easy_init], [], [err=1])
+            AC_CHECK_LIB([xml2], [xmlDocGetRootElement], [], [err=1])
+            AC_CHECK_LIB([aws4c], [s3_get], [], [err=1], [-lcurl -lxml2 -lcrypto])
+            LIBS=$ORIG_LIBS
+
+            AC_MSG_NOTICE([end of S3-related checks])
+            if test "$err" == 1; then
+                AC_MSG_FAILURE([S3 support is missing.  dnl
+Make sure you have access to libaws4c, libcurl, libxml2, and libcrypto.  dnl
+Consider --with-aws4c=, CPPFLAGS, LDFLAGS, etc])
+            fi
+
+            # restore user's values
+            CPPFLAGS=$ORIG_CPPFLAGS
+            LDFLAGS=$ORIG_LDFLAGS
+])
+
+
+
 
 
 

Makefile.am

@@ -52,12 +52,11 @@ endif
 
 
 if USE_S3_AIORI
-# TBD: Find the aws4c and libxml installations programmatically
-AWS4C  = /users/jti/projects/ecs_hobo/tools/aws4c-0.5
 ior_SOURCES  += aiori-S3.c
-ior_CPPFLAGS += -I/usr/include/libxml2
-ior_CPPFLAGS += -I$(AWS4C)
-ior_LDFLAGS  += -L$(AWS4C)
+if AWS4C_DIR
+  ior_CPPFLAGS += $(AWS4C_CPPFLAGS)
+  ior_LDFLAGS  += $(AWS4C_LDFLAGS)
+endif
 ior_LDADD    += -lcurl
 ior_LDADD    += -lxml2
 ior_LDADD    += -laws4c

README

@@ -205,6 +205,7 @@ CURLcode           rc;
 
 /* Any objects we create or delete will be under this bucket */
 const char* bucket_name = "ior";
+//const char* bucket_name = "brettk";
 
 
 /***************************** F U N C T I O N S ******************************/
@@ -259,9 +260,9 @@ s3_connect( IOR_param_t* param ) {
 	// NOTE: These inits could be done in init_IORParam_t(), in ior.c, but
 	//       would require conditional compilation, there.
 
+	aws_set_debug(param->verbose >= 4);
 	aws_read_config(getenv("USER"));  // requires ~/.awsAuth
 	aws_reuse_connections(1);
-	aws_set_debug(param->verbose >= 4);
 
 	// initalize IOBufs.  These are basically dynamically-extensible
 	// linked-lists.  "growth size" controls the increment of new memory
@@ -272,24 +273,59 @@ s3_connect( IOR_param_t* param ) {
 	param->etags = aws_iobuf_new();
 	aws_iobuf_growth_size(param->etags, 1024*1024*8);
 
+   // WARNING: if you have http_proxy set in your environment, you may need
+   //          to override it here.  TBD: add a command-line variable to
+   //          allow you to define a proxy.
+   //
 	// our hosts are currently 10.140.0.15 - 10.140 0.18
 	// TBD: Try DNS-round-robin server at vi-lb.ccstar.lanl.gov
-	snprintf(buff, BUFF_SIZE, "10.140.0.%d:9020", 15 + (rank % 4));
-	s3_set_host(buff);
+   // TBD: try HAProxy round-robin at 10.143.0.1
+
+#if 1
+   //   snprintf(buff, BUFF_SIZE, "10.140.0.%d:9020", 15 + (rank % 4));
+   //   s3_set_proxy(buff);
+   //
+   //   snprintf(buff, BUFF_SIZE, "10.140.0.%d", 15 + (rank % 4));
+   //	s3_set_host(buff);
+
+   snprintf(buff, BUFF_SIZE, "10.140.0.%d:9020", 15 + (rank % 4));
+   s3_set_host(buff);
+
+#else
+/*
+ * If you just want to go to one if the ECS nodes, put that IP
+ * address in here directly with port 9020.
+ *
+ */
+//   s3_set_host("10.140.0.15:9020");
+
+/*
+ * If you want to go to haproxy.ccstar.lanl.gov, this is its IP
+ * address.
+ *
+ */
+//   s3_set_proxy("10.143.0.1:80");
+//   s3_set_host( "10.143.0.1:80");
+#endif
 
 	// make sure test-bucket exists
 	s3_set_bucket((char*)bucket_name);
-   AWS4C_CHECK( s3_head(param->io_buf, "") );
-   if ( param->io_buf->code == 404 ) {					// "404 Not Found"
-      printf("  bucket '%s' doesn't exist\n", bucket_name);
 
-      AWS4C_CHECK( s3_put(param->io_buf, "") );	/* creates URL as bucket + obj */
-      AWS4C_CHECK_OK(     param->io_buf );		// assure "200 OK"
-		printf("created bucket '%s'\n", bucket_name);
-	}
-   else {														// assure "200 OK"
-      AWS4C_CHECK_OK( param->io_buf );
-	}
+   if (rank == 0) {
+      AWS4C_CHECK( s3_head(param->io_buf, "") );
+      if ( param->io_buf->code == 404 ) {					// "404 Not Found"
+         printf("  bucket '%s' doesn't exist\n", bucket_name);
+
+         AWS4C_CHECK( s3_put(param->io_buf, "") );	/* creates URL as bucket + obj */
+         AWS4C_CHECK_OK(     param->io_buf );		// assure "200 OK"
+         printf("created bucket '%s'\n", bucket_name);
+      }
+      else {														// assure "200 OK"
+         AWS4C_CHECK_OK( param->io_buf );
+      }
+   }
+   MPI_CHECK(MPI_Barrier(param->testComm), "barrier error");
+
 
 	// Maybe allow EMC extensions to S3
 	s3_enable_EMC_extensions(param->curl_flags & IOR_CURL_S3_EMC_EXT);
@@ -821,7 +857,6 @@ S3_Xfer_internal(int          access,
 		// we're "extending" rather than "appending".  That means the
 		// buffer represents empty storage, which will be filled by the
 		// libcurl writefunction, invoked via aws4c.
-
 		aws_iobuf_reset(param->io_buf);
 		aws_iobuf_extend_static(param->io_buf, data_ptr, remaining);
 		AWS4C_CHECK( s3_get(param->io_buf, file) );