S3: integrate nicely into XRootD CMAKE build system - define a new sy…

…stemd unit xrootd-s3@s3 which runs xrootd as uid/gid=0 with nfs-uid/nfs-gid as specified and allows the S3 plug-in to change the file system ID on the fly as required - stremaling the logging and correct the fsid scopes

src/XrdS3/CMakeLists.txt

@@ -0,0 +1,10 @@
+set(XS3_SYSTEMD_SERVICE_FILE "${CMAKE_CURRENT_SOURCE_DIR}/systemd/[email protected]")
+
+install(FILES ${XS3_SYSTEMD_SERVICE_FILE}
+        DESTINATION /lib/systemd/system)
+
+set(XS3_CONFIG_FILE "${CMAKE_CURRENT_SOURCE_DIR}/config/xrootd-s3.cfg")
+
+install(FILES ${XS3_CONFIG_FILE}
+        DESTINATION /etc/xrootd/)
+

src/XrdS3/XrdS3.cc

@@ -34,6 +34,8 @@
 #include "XrdOuc/XrdOucStream.hh"
 #include "XrdOuc/XrdOucString.hh"
 #include "XrdS3ErrorResponse.hh"
+#include "XrdS3Log.hh"
+#include "XrdS3ScopedFsId.hh"
 #include "XrdVersion.hh"
 //------------------------------------------------------------------------------
 
@@ -61,19 +63,24 @@ int NotFoundHandler(XrdS3Req &req) {
 //!
 //------------------------------------------------------------------------------
 S3Handler::S3Handler(XrdSysError *log, const char *config, XrdOucEnv *myEnv)
-    : mLog(log->logger(), "S3_"), mApi(), mRouter(&mLog, NotFoundHandler) {
+    : mErr(log->logger(), "S3_"), mApi(), mRouter(mLog, NotFoundHandler) {
+
+  mLog.Init(&mErr);
+
+  S3::ScopedFsId::Validate(); // verify we can switch filesystem IDs !
+
   if (!ParseConfig(config, *myEnv)) {
     throw std::runtime_error("Failed to configure the HTTP S3 handler.");
   }
 
-  ctx.log = &mLog;
+  ctx.log = &mErr;
 
-  mApi = S3Api(mConfig.config_dir, mConfig.region, mConfig.service,
+  mApi = S3Api(mLog, mConfig.config_dir, mConfig.region, mConfig.service,
                mConfig.multipart_upload_dir);
 
   ConfigureRouter();
 
-  mLog.Say("Finished configuring S3 Handler");
+  mLog.Log(S3::ALL, "Handler", "finished configuring S3 Handler");
 }
 
 //------------------------------------------------------------------------------
@@ -517,7 +524,7 @@ void S3Handler::ConfigureRouter() {
 //!
 //------------------------------------------------------------------------------
 bool S3Handler::ParseConfig(const char *config, XrdOucEnv &env) {
-  XrdOucStream Config(&mLog, getenv("XRDINSTANCE"), &env, "=====> ");
+  XrdOucStream Config(&mErr, getenv("XRDINSTANCE"), &env, "=====> ");
 
   auto fd = open(config, O_RDONLY);
 

src/XrdS3/XrdS3.hh

@@ -35,13 +35,12 @@
 #include "XrdS3Crypt.hh"
 #include "XrdS3Router.hh"
 #include "XrdS3Utils.hh"
+#include "XrdS3Log.hh"
 #include "XrdSys/XrdSysError.hh"
 //------------------------------------------------------------------------------
 
 
 namespace S3 {
-
-
 //------------------------------------------------------------------------------
 //! \brief S3Handler is a class that implements the XRootD HTTP extension
 //! handler for S3.
@@ -70,8 +69,8 @@ class S3Handler : public XrdHttpExtHandler {
     std::string multipart_upload_dir;
   } mConfig;
 
-  XrdSysError mLog;
-
+  XrdSysError mErr;
+  S3Log mLog;
   S3Api mApi;
   S3Router mRouter;
 

src/XrdS3/XrdS3Api.cc

@@ -31,6 +31,7 @@
 #include "XrdS3Auth.hh"
 #include "XrdS3ErrorResponse.hh"
 #include "XrdS3Response.hh"
+#include "XrdS3Log.hh"
 //------------------------------------------------------------------------------
 
 namespace S3 {
@@ -501,6 +502,8 @@ int S3Api::CopyObjectHandler(XrdS3Req& req) {
 int S3Api::PutObjectHandler(XrdS3Req& req) {
   VALIDATE_REQUEST(Action::PutObject)
 
+  mLog->Log(S3::DEBUG, "Api", "Action::PutObject");
+
   auto chunked = false;
   unsigned long length = 0;
   auto it = req.lowercase_headers.find("content-length");
@@ -523,6 +526,8 @@ int S3Api::PutObjectHandler(XrdS3Req& req) {
     }
   }
 
+  mLog->Log(S3::DEBUG, "Api", "GetObject");
+
   S3ObjectStore::Object obj;
   err = objectStore.GetObject(bucket, req.object, obj);
   if (err == S3Error::None) {
@@ -538,6 +543,8 @@ int S3Api::PutObjectHandler(XrdS3Req& req) {
         ValidatePreconditions(etag, last_modified, req.lowercase_headers))
   }
 
+  mLog->Log(S3::DEBUG, "Api", "ObjectStore::PutObject");
+
   std::map<std::string, std::string> headers;
   RET_ON_ERROR(objectStore.PutObject(req, bucket, length, chunked, headers))
 

src/XrdS3/XrdS3Api.hh

@@ -33,6 +33,7 @@
 #include "XrdS3Auth.hh"
 #include "XrdS3ObjectStore.hh"
 #include "XrdS3Req.hh"
+#include "XrdS3Log.hh"
 
 namespace S3 {
 
@@ -46,9 +47,9 @@ namespace S3 {
 class S3Api {
  public:
   S3Api() = default;
-  S3Api(const std::string &config_path, const std::string &region, const std::string &service,
+  S3Api(S3Log& log, const std::string &config_path, const std::string &region, const std::string &service,
         const std::string &mtpu_path)
-      : objectStore(config_path, mtpu_path), auth(config_path, region, service) {}
+    : mLog(&log),objectStore(config_path, mtpu_path), auth(config_path, region, service) {}
 
   ~S3Api() = default;
 
@@ -310,6 +311,7 @@ class S3Api {
   }
 
  private:
+  S3Log* mLog;
   S3ObjectStore objectStore;
   S3Auth auth;
 };

src/XrdS3/XrdS3Log.hh

@@ -0,0 +1,48 @@
+
+
+#include <mutex>
+#include <stdio.h>
+#include <stdarg.h>
+
+#include "XrdSys/XrdSysError.hh"
+
+#pragma once
+namespace S3 {
+  //------------------------------------------------------------------------------
+  //! \brief enum defining logging levels
+  //------------------------------------------------------------------------------
+  enum LogMask {
+                DEBUG = 0x01,
+                INFO = 0x02,
+                WARN = 0x04,
+                ERROR = 0x08,
+                ALL = 0xff
+  };
+
+  class S3Log {
+  public:
+    S3Log(XrdSysError& mErr) : mLog(&mErr) {}
+    S3Log() {}
+    virtual ~S3Log() {}
+
+    void Init(XrdSysError* log) { mLog = log; }
+
+    std::string
+    Log(S3::LogMask mask, const char* unit, const char* msg, ...)
+    {
+      std::lock_guard<std::mutex> guard(logMutex); 
+
+      va_list args;
+      va_start(args, msg);
+      vsnprintf(logBuffer, sizeof(logBuffer), msg, args);
+      va_end(args);
+      mLog->Log( (int) mask, unit, logBuffer );
+      return std::string(logBuffer);
+    }
+
+  private:
+    XrdSysError* mLog;
+    char logBuffer[65535]; 
+    std::mutex logMutex; 
+  };
+}

src/XrdS3/XrdS3ObjectStore.cc

@@ -50,7 +50,7 @@ namespace S3 {
 //! \param config Path to the configuration file
 //! \param mtpu Path to the MTPU directory
 //------------------------------------------------------------------------------
-S3ObjectStore::S3ObjectStore(const std::string &config, const std::string &mtpu)
+  S3ObjectStore::S3ObjectStore(const std::string &config, const std::string &mtpu)
     : config_path(config), mtpu_path(mtpu) {
   user_map = config_path / "users";
 
@@ -205,7 +205,7 @@ S3Error S3ObjectStore::CreateBucket(S3Auth &auth, S3Auth::Bucket bucket,
   int mkdir_retc=0;
   {
     // Create the backend directory with the users filesystem id
-    ScopedFsId(bucket.owner.uid,bucket.owner.gid);
+    ScopedFsId scop (bucket.owner.uid,bucket.owner.gid);
     mkdir_retc = XrdPosix_Mkdir(bucket.path.c_str(), S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
   }
   if (mkdir_retc) {
@@ -246,7 +246,7 @@ S3Error S3ObjectStore::DeleteBucket(S3Auth &auth,
 
   {
     // Check the backend directory with the users filesystem id
-    ScopedFsId(bucket.owner.uid,bucket.owner.gid);
+    ScopedFsId scope(bucket.owner.uid,bucket.owner.gid);
 
     if (!S3Utils::IsDirEmpty(bucket.path)) {
       return S3Error::BucketNotEmpty;
@@ -306,7 +306,7 @@ uid_t uid, gid_t gid) {
   struct stat buf;
 
   // Do the backend operations with the users filesystem id
-  ScopedFsId(uid, gid);
+  ScopedFsId scope(uid, gid);
   if (XrdPosix_Stat(p.c_str(), &buf) || S_ISDIR(buf.st_mode)) {
     return S3Error::NoSuchKey;
   }
@@ -747,10 +747,14 @@ struct FileUploadResult {
 FileUploadResult FileUploader(XrdS3Req &req, bool chunked, size_t size,
                               std::filesystem::path &path) {
   auto fd = XrdPosix_Open(path.c_str(), O_CREAT | O_EXCL | O_WRONLY,
-                          S_IRWXU | S_IRWXG);
+                          S_IRWXU | S_IRGRP | S_IXGRP);
 
   if (fd < 0) {
-    return FileUploadResult{S3Error::InternalError, {}, {}, {}};
+    if ( errno == 13) {
+      return FileUploadResult{S3Error::AccessDenied, {}, {}, {}};
+    } else {
+      return FileUploadResult{S3Error::InternalError, {}, {}, {}};
+    }
   }
 
   // TODO: Implement handling of different checksum types.
@@ -931,6 +935,7 @@ S3Error S3ObjectStore::UploadPart(XrdS3Req &req, const std::string &upload_id,
 S3Error S3ObjectStore::PutObject(XrdS3Req &req, const S3Auth::Bucket &bucket,
                                  unsigned long size, bool chunked,
                                  Headers &headers) {
+  ScopedFsId scope(bucket.owner.uid,bucket.owner.gid);
   auto final_path = bucket.path / req.object;
 
   struct stat buf;
@@ -945,7 +950,6 @@ S3Error S3ObjectStore::PutObject(XrdS3Req &req, const S3Auth::Bucket &bucket,
 
   auto err = S3Utils::makePath((char *)final_path.parent_path().c_str(),
                                S_IRWXU | S_IRWXG);
-
   if (err == ENOTDIR) {
     return S3Error::ObjectExistInObjectPath;
   } else if (err != 0) {
@@ -1504,7 +1508,7 @@ S3Error S3ObjectStore::CompleteMultipartUpload(
 
   {
     // Check if the final file exists in the backend and is a directory
-    ScopedFsId(bucket.owner.uid,bucket.owner.gid);
+    ScopedFsId scope(bucket.owner.uid,bucket.owner.gid);
 
     if (!XrdPosix_Stat(final_path.c_str(), &buf)) {
       if (S_ISDIR(buf.st_mode)) {
@@ -1524,7 +1528,7 @@ S3Error S3ObjectStore::CompleteMultipartUpload(
   int fd = 0;
   {
     // The temp file has to created using the filesystem id of the owner
-    ScopedFsId(bucket.owner.uid,bucket.owner.gid);
+    ScopedFsId scope(bucket.owner.uid,bucket.owner.gid);
     fd = XrdPosix_Open(tmp_path.c_str(), O_CREAT | O_EXCL | O_WRONLY);
   }
 
@@ -1569,7 +1573,7 @@ S3Error S3ObjectStore::CompleteMultipartUpload(
       ssize_t len = opt_len;
       while ((i = optimized_obj.Read(len, &ptr)) > 0) {
         if (len < i) {
-          ScopedFsId(bucket.owner.uid,bucket.owner.gid);
+          ScopedFsId scope(bucket.owner.uid,bucket.owner.gid);
           XrdPosix_Close(fd);
           XrdPosix_Unlink(tmp_path.c_str());
           S3Utils::RmPath(final_path.parent_path(), bucket.path);
@@ -1585,7 +1589,7 @@ S3Error S3ObjectStore::CompleteMultipartUpload(
 
       while ((i = obj.Read(len, &ptr)) > 0) {
         if (len < i) {
-          ScopedFsId(bucket.owner.uid,bucket.owner.gid);
+          ScopedFsId scope(bucket.owner.uid,bucket.owner.gid);
           XrdPosix_Close(fd);
           XrdPosix_Unlink(tmp_path.c_str());
           S3Utils::RmPath(final_path.parent_path(), bucket.path);
@@ -1616,7 +1620,7 @@ S3Error S3ObjectStore::CompleteMultipartUpload(
 
   {
     // Rename using the owner filesystem id
-    ScopedFsId(bucket.owner.uid,bucket.owner.gid);
+    ScopedFsId scope(bucket.owner.uid,bucket.owner.gid);
     XrdPosix_Rename(tmp_path.c_str(), final_path.c_str());
   }
 

src/XrdS3/XrdS3ObjectStore.hh

@@ -35,6 +35,7 @@
 #include "XrdS3Auth.hh"
 #include "XrdS3ErrorResponse.hh"
 #include "XrdS3Req.hh"
+#include "XrdS3Log.hh"
 
 namespace S3 {
 

src/XrdS3/XrdS3Router.cc

@@ -24,6 +24,7 @@
 
 //------------------------------------------------------------------------------
 #include "XrdS3Router.hh"
+#include "XrdS3Log.hh"
 //------------------------------------------------------------------------------
 
 namespace S3 {
@@ -150,7 +151,7 @@ bool S3Route::MatchMap(
 //! \param route the route
 //------------------------------------------------------------------------------
 void S3Router::AddRoute(S3Route &route) {
-  mLog.Say("Registered route:", route.GetName().c_str());
+  mLog->Log(S3::ALL, "Router", "registered route: %s", route.GetName().c_str());
   routes.push_back(route);
 }
 
@@ -162,11 +163,13 @@ void S3Router::AddRoute(S3Route &route) {
 int S3Router::ProcessReq(XrdS3Req &req) {
   for (const auto &route : routes) {
     if (route.Match(req)) {
-      mLog.Say("Found matching route for req:", route.GetName().c_str());
-      return route.Handler()(req);
+      mLog->Log(S3::DEBUG, "Router", "found matching route for req: %s", route.GetName().c_str());
+      int rc = route.Handler()(req);
+      mLog->Log(S3::INFO, "Router", "request returned: %d", rc);
+      return rc;
     }
   }
-  mLog.Say("Unable to find matching route for request...");
+  mLog->Log(S3::ERROR, "Router", "unable to find matching route for req: %s.", req.uri_path.c_str());
   return not_found_handler(req);
 }
 

src/XrdS3/XrdS3Router.hh

@@ -34,6 +34,7 @@
 //------------------------------------------------------------------------------
 #include "XrdS3Action.hh"
 #include "XrdS3Req.hh"
+#include "XrdS3Log.hh"
 #include "XrdSys/XrdSysError.hh"
 //------------------------------------------------------------------------------
 
@@ -91,8 +92,8 @@ class S3Route {
 //------------------------------------------------------------------------------
 class S3Router {
  public:
-  explicit S3Router(XrdSysError *log, HandlerFunc fn)
-      : mLog(log->logger(), "S3Router_"), not_found_handler(std::move(fn)){};
+  explicit S3Router(S3Log &log, HandlerFunc fn)
+    : mLog(&log), not_found_handler(std::move(fn)){};
 
   ~S3Router() = default;
 
@@ -101,7 +102,7 @@ class S3Router {
   int ProcessReq(XrdS3Req &req);
 
  private:
-  XrdSysError mLog;
+  S3Log* mLog;
   std::vector<S3Route> routes;
 
   HandlerFunc not_found_handler;

src/XrdS3/XrdS3ScopedFsId.hh

@@ -94,6 +94,12 @@ public:
     return ok;
   }
 
+  static void Validate() {
+    ScopedFsId scope(geteuid()+1, geteuid()+1);
+    if (!scope.IsOk()) {
+      throw std::runtime_error("XrdS3 misses the capability to set the filesystem IDs on the fly!");
+    }
+  }
 private:
   int fsuid;
   int fsgid;
@@ -126,4 +132,4 @@ public:
 };
 
 } // namespace S3
-#endif 
+#endif