From fc674aa5c12c1dd1530acf4ab17cfe760dfb3eb4 Mon Sep 17 00:00:00 2001
From: Fons Rademakers <Fons.Rademakers@cern.ch>
Date: Mon, 25 Feb 2013 15:14:20 +0000
Subject: [PATCH] From Fabio Hernandez: The class TS3WebFile was modified to
support also reading files hosted by Google using the S3 protocol. Its TFile
plugin was also modified to reflect this. The class TS3HTTPRequest is new.
git-svn-id: http://root.cern.ch/svn/root/trunk@48687 27541ba8-7e3a-0410-8455-c3a389f83636
---
etc/plugins/TFile/P130_TAS3File.C | 5 -
etc/plugins/TFile/P140_TGSFile.C | 5 -
etc/plugins/TFile/P150_TS3WebFile.C | 11 +
io/io/src/TFile.cxx | 2 +-
net/doc/v600/index.html | 14 +-
net/net/Module.mk | 2 +
net/net/inc/LinkDef.h | 4 +-
net/net/inc/TAS3File.h | 73 -------
net/net/inc/TGSFile.h | 73 -------
net/net/inc/THTTPMessage.h | 114 ----------
net/net/inc/TS3HTTPRequest.h | 158 ++++++++++++++
net/net/inc/TS3WebFile.h | 34 +--
net/net/src/TAS3File.cxx | 265 ------------------------
net/net/src/TGSFile.cxx | 309 ----------------------------
net/net/src/THTTPMessage.cxx | 262 -----------------------
net/net/src/TS3HTTPRequest.cxx | 240 +++++++++++++++++++++
net/net/src/TS3WebFile.cxx | 174 +++++++++-------
net/net/src/TWebFile.cxx | 10 +
18 files changed, 553 insertions(+), 1202 deletions(-)
delete mode 100644 etc/plugins/TFile/P130_TAS3File.C
delete mode 100644 etc/plugins/TFile/P140_TGSFile.C
create mode 100644 etc/plugins/TFile/P150_TS3WebFile.C
delete mode 100644 net/net/inc/TAS3File.h
delete mode 100644 net/net/inc/TGSFile.h
delete mode 100644 net/net/inc/THTTPMessage.h
create mode 100644 net/net/inc/TS3HTTPRequest.h
delete mode 100644 net/net/src/TAS3File.cxx
delete mode 100644 net/net/src/TGSFile.cxx
delete mode 100644 net/net/src/THTTPMessage.cxx
create mode 100644 net/net/src/TS3HTTPRequest.cxx
diff --git a/etc/plugins/TFile/P130_TAS3File.C b/etc/plugins/TFile/P130_TAS3File.C
deleted file mode 100644
index bdee5cde30e..00000000000
--- a/etc/plugins/TFile/P130_TAS3File.C
+++ /dev/null
@@ -1,5 +0,0 @@
-void P130_TAS3File()
-{
- gPluginMgr->AddHandler("TFile", "^as3:", "TAS3File",
- "Net", "TAS3File(const char*,Option_t*)");
-}
diff --git a/etc/plugins/TFile/P140_TGSFile.C b/etc/plugins/TFile/P140_TGSFile.C
deleted file mode 100644
index 9803c5a0476..00000000000
--- a/etc/plugins/TFile/P140_TGSFile.C
+++ /dev/null
@@ -1,5 +0,0 @@
-void P140_TGSFile()
-{
- gPluginMgr->AddHandler("TFile", "^gs:", "TGSFile",
- "Net", "TGSFile(const char*,Option_t*)");
-}
diff --git a/etc/plugins/TFile/P150_TS3WebFile.C b/etc/plugins/TFile/P150_TS3WebFile.C
new file mode 100644
index 00000000000..85f28d17b8b
--- /dev/null
+++ b/etc/plugins/TFile/P150_TS3WebFile.C
@@ -0,0 +1,11 @@
+void P150_TS3WebFile()
+{
+ gPluginMgr->AddHandler("TFile", "^[a]?s3:", "TS3WebFile",
+ "Net", "TS3WebFile(const char*,Option_t*)");
+ gPluginMgr->AddHandler("TFile", "^s3http[s]?:", "TS3WebFile",
+ "Net", "TS3WebFile(const char*,Option_t*)");
+ gPluginMgr->AddHandler("TFile", "^gs:", "TS3WebFile",
+ "Net", "TS3WebFile(const char*,Option_t*)");
+ gPluginMgr->AddHandler("TFile", "^gshttp[s]?:", "TS3WebFile",
+ "Net", "TS3WebFile(const char*,Option_t*)");
+}
diff --git a/io/io/src/TFile.cxx b/io/io/src/TFile.cxx
index fb4f84edd31..3a19890259a 100644
--- a/io/io/src/TFile.cxx
+++ b/io/io/src/TFile.cxx
@@ -4498,7 +4498,7 @@ TFile::EFileType TFile::GetType(const char *name, Option_t *option, TString *pre
//
// Adjust the type according to findings
type = (localFile) ? kLocal : type;
- } else if (TPMERegexp("^(http[s]?|s3http[s]?|[a]?s3|gs){1}:", "i").Match(name)) {
+ } else if (TPMERegexp("^(http[s]?|s3http[s]?|[a]?s3|gs|gshttp[s]?){1}:", "i").Match(name)) {
//
// Web file
type = kWeb;
diff --git a/net/doc/v600/index.html b/net/doc/v600/index.html
index 76dd35c1f00..dc042cf4ff9 100644
--- a/net/doc/v600/index.html
+++ b/net/doc/v600/index.html
@@ -3,16 +3,15 @@
<a name="net"></a>
<h3>Networking Libraries</h3>
<p>
-The enclosed patch includes a new class and some modifications to existing
-classes. The new class TS3WebFile is an extension of TWebFile and belongs
-to the net module. I deliberately did not modify the existing TAS3File.
-I think that the name TS3WebFile reflects better the fact that this solution
+A new class TS3WebFile has been introduced. The new class TS3WebFile is an
+extension of TWebFile and belongs to the net module.
+The name TS3WebFile reflects better the fact that this solution
is intended to be generic to several S3 servers and not limited to Amazon's,
in addition to the fact that it actually extends the capabilities of TWebFile.
</p>
<p>
Compared to the current support of S3 in ROOT (basically the class TAS3File),
-the modifications in this patch include the improvements below:
+the modifications include the improvements below:
</p>
<ul>
<li>
@@ -66,4 +65,7 @@ Limitations:
is not supported but can be added if this is considered useful.
</li>
</ul>
-
+<p>
+The TAS3File class will be removed and should not have been used directly by
+users anyway as it was only accessed via the plugin manager in TFile::Open().
+</p>
diff --git a/net/net/Module.mk b/net/net/Module.mk
index 0be2586c9ba..676ab28e662 100644
--- a/net/net/Module.mk
+++ b/net/net/Module.mk
@@ -25,10 +25,12 @@ NETNOCRYPTO := -DR__NO_CRYPTO
NETH := $(filter-out $(MODDIRI)/TAS3File.h,$(NETH))
NETH := $(filter-out $(MODDIRI)/TGSFile.h,$(NETH))
NETH := $(filter-out $(MODDIRI)/TS3WebFile.h,$(NETH))
+NETH := $(filter-out $(MODDIRI)/TS3HTTPRequest.h,$(NETH))
NETH := $(filter-out $(MODDIRI)/THTTPMessage.h,$(NETH))
NETS := $(filter-out $(MODDIRS)/TAS3File.cxx,$(NETS))
NETS := $(filter-out $(MODDIRS)/TGSFile.cxx,$(NETS))
NETS := $(filter-out $(MODDIRS)/TS3WebFile.cxx,$(NETS))
+NETS := $(filter-out $(MODDIRS)/TS3HTTPRequest.cxx,$(NETS))
NETS := $(filter-out $(MODDIRS)/THTTPMessage.cxx,$(NETS))
else
NETNOCRYPTO :=
diff --git a/net/net/inc/LinkDef.h b/net/net/inc/LinkDef.h
index 87945e615c9..9e10fe1e53c 100644
--- a/net/net/inc/LinkDef.h
+++ b/net/net/inc/LinkDef.h
@@ -53,9 +53,7 @@
#pragma link C++ class TApplicationServer;
#pragma link C++ class TUDPSocket;
#ifndef R__NO_CRYPTO
-#pragma link C++ class THTTPMessage+;
-#pragma link C++ class TAS3File+;
-#pragma link C++ class TGSFile+;
+#pragma link C++ class TS3HTTPRequest+;
#pragma link C++ class TS3WebFile+;
#endif
#ifdef R__SSL
diff --git a/net/net/inc/TAS3File.h b/net/net/inc/TAS3File.h
deleted file mode 100644
index 2b1ae33febb..00000000000
--- a/net/net/inc/TAS3File.h
+++ /dev/null
@@ -1,73 +0,0 @@
-// @(#)root/net:$Id$
-// Author: Marcelo Sousa 23/08/2011
-
-/*************************************************************************
- * Copyright (C) 1995-2011, Rene Brun and Fons Rademakers. *
- * All rights reserved. *
- * *
- * For the licensing terms see $ROOTSYS/LICENSE. *
- * For the list of contributors see $ROOTSYS/README/CREDITS. *
- *************************************************************************/
-
-#ifndef ROOT_TAS3File
-#define ROOT_TAS3File
-
-//////////////////////////////////////////////////////////////////////////
-// //
-// TAS3File //
-// //
-// A TAS3File is a normal TWebFile but it reads data from the //
-// Amazon S3 server. As a derived TWebFile class TAS3File it is a //
-// read only file. The HTTP requests are generated by THTTPMessage //
-// objects with the auth_prefix set as AWS. The user id and secret pass //
-// required to sign the requests are passed through the environment //
-// variables S3_ACCESS_ID and S3_ACCESS_KEY. //
-// For more information check: //
-// http://aws.amazon.com/documentation/s3/ //
-// http://awsdocs.s3.amazonaws.com/S3/latest/s3-qrc.pdf //
-// //
-//////////////////////////////////////////////////////////////////////////
-
-#ifndef ROOT_TWebFile
-#include "TWebFile.h"
-#endif
-
-#ifndef ROOT_TUrl
-#include "TUrl.h"
-#endif
-
-#ifndef ROOT_TString
-#include "TString.h"
-#endif
-
-class TAS3File: public TWebFile {
-
-private:
- TAS3File();
-
-protected:
- TString fAuthPrefix; //Authentication prefix for Amazon S3
- TString fAccessId; //User id
- TString fAccessKey; //Secret key
- TUrl fServer; //Server url
- TString fBucket; //Bucket name
-
- Int_t GetHead();
- Bool_t ReadBuffer10(char *buf, Int_t len);
-
-public:
- TAS3File(const char *url, Option_t *opt="");
- virtual ~TAS3File() { }
-
- Bool_t ReadBuffer(char *buf, Int_t len);
- Bool_t ReadBuffer(char *buf, Long64_t pos, Int_t len) { return TWebFile::ReadBuffer(buf, pos, len); }
- TString GetAuthPrefix() const { return fAuthPrefix; }
- TString GetAccessId() const { return fAccessId; }
- TString GetAccessKey() const { return fAccessKey; }
- TUrl GetUrl() const { return fServer; }
- TString GetBucket() const { return fBucket; }
-
- ClassDef(TAS3File, 0) // Read a ROOT file from the Amazon S3 cloud
-};
-
-#endif
diff --git a/net/net/inc/TGSFile.h b/net/net/inc/TGSFile.h
deleted file mode 100644
index 9064438b2a8..00000000000
--- a/net/net/inc/TGSFile.h
+++ /dev/null
@@ -1,73 +0,0 @@
-// @(#)root/net:$Id$
-// Author: Marcelo Sousa 23/08/2011
-
-/*************************************************************************
- * Copyright (C) 1995-2011, Rene Brun and Fons Rademakers. *
- * All rights reserved. *
- * *
- * For the licensing terms see $ROOTSYS/LICENSE. *
- * For the list of contributors see $ROOTSYS/README/CREDITS. *
- *************************************************************************/
-
-#ifndef ROOT_TGSFile
-#define ROOT_TGSFile
-
-//////////////////////////////////////////////////////////////////////////
-// //
-// TGSFile //
-// //
-// A TGSFile is a normal TWebFile but it reads data from the //
-// Google Storage server. As a derived TWebFile class TGSFile it is //
-// a read only file. The HTTP requests are generated by THTTPMessage //
-// objects with the auth_prefix set as GOOG1. The user id and secret //
-// pass required to sign the requests are passed through the //
-// environment variables GT_ACCESS_ID and GT_ACCESS_KEY. //
-// For more information check: //
-// http://code.google.com/apis/storage/docs/getting-started.html //
-// //
-//////////////////////////////////////////////////////////////////////////
-
-#ifndef ROOT_TWebFile
-#include "TWebFile.h"
-#endif
-
-#ifndef ROOT_TUrl
-#include "TUrl.h"
-#endif
-
-#ifndef ROOT_TString
-#include "TString.h"
-#endif
-
-class TGSFile: public TWebFile {
-
-private:
- TGSFile();
-
-protected:
- TString fAuthPrefix; //Authentication prefix for Google Storage
- TString fAccessId; //User id
- TString fAccessKey; //Secret key
- TUrl fServer; //Server url
- TString fBucket; //Bucket name
-
- Int_t GetHead();
- Bool_t ReadBuffer10(char *buf, Int_t len);
- //Bool_t ReadBuffers10(char *buf, Long64_t *pos, Int_t *len, Int_t nbuf);
-
-public:
- TGSFile(const char *url, Option_t *opt="");
- virtual ~TGSFile() { }
-
- Bool_t ReadBuffer(char *buf, Int_t len);
- Bool_t ReadBuffer(char *buf, Long64_t pos, Int_t len) { return TWebFile::ReadBuffer(buf, pos, len); }
- TString GetAuthPrefix() const { return fAuthPrefix; }
- TString GetAccessId() const { return fAccessId; }
- TString GetAccessKey() const { return fAccessKey; }
- TUrl GetUrl() const { return fServer; }
- TString GetBucket() const { return fBucket; }
-
- ClassDef(TGSFile, 0) // Read a ROOT file from the Google Storage cloud
-};
-
-#endif
diff --git a/net/net/inc/THTTPMessage.h b/net/net/inc/THTTPMessage.h
deleted file mode 100644
index c85190d3ea5..00000000000
--- a/net/net/inc/THTTPMessage.h
+++ /dev/null
@@ -1,114 +0,0 @@
-// @(#)root/net:$Id$
-// Author: Marcelo Sousa 23/08/2011
-
-/*************************************************************************
- * Copyright (C) 1995-2011, Rene Brun and Fons Rademakers. *
- * All rights reserved. *
- * *
- * For the licensing terms see $ROOTSYS/LICENSE. *
- * For the list of contributors see $ROOTSYS/README/CREDITS. *
- *************************************************************************/
-
-#ifndef ROOT_THTTPMessage
-#define ROOT_THTTPMessage
-
-//////////////////////////////////////////////////////////////////////////
-// //
-// THTTPMessage //
-// //
-// A THTTPMessage object represents a generic HTTP request for the //
-// Amazon S3 and the Google Storage services. It can easily be extended //
-// to other API's. It assumes that each request is signed with the //
-// client id and an encripted key, Base64(HMAC + SHA1 (HTTP Request)) //
-// which is based on a secret key provided in the constructor. //
-// For more information about the authentication : //
-// Google Storage: //
-// http://code.google.com/apis/storage/docs/reference/v1/developer-guidev1.html#authentication //
-// Amazon S3: //
-// http://awsdocs.s3.amazonaws.com/S3/latest/s3-qrc.pdf //
-// At the moment THTTPMessage is used for derived classes of TWebFile //
-// (read only) files supporting HEAD and GET requests. //
-// //
-//////////////////////////////////////////////////////////////////////////
-
-#ifndef ROOT_TObject
-#include "TObject.h"
-#endif
-
-#ifndef ROOT_TString
-#include "TString.h"
-#endif
-
-enum EHTTP_Verb {
- kGET,
- kPOST,
- kPUT,
- kDELETE,
- kHEAD,
- kCOPY
-};
-
-class THTTPMessage : public TObject{
-
-private:
- enum EHTTP_Verb fVerb; //HTTP Verb
- TString fPath; //Given path to be parsed
- TString fBucket; //Bucket associated with the file
- TString fHost; //Server name
- TString fDate; //Date
- TString fAuthPrefix; //Authentication prefix to distinguish between GT and AWS3
- TString fAccessId; //User id
- TString fAccessIdKey; //Secret key
- Bool_t fHasRange; //GET request with range
- Long64_t fOffset; //Offset
- Long64_t *fInitByte; //Init positions for the range
- Int_t *fLen; //Range length
- Int_t fNumBuf; //Number of buffers
- Int_t fCurrentBuf; //For requests > 8000 we need to generate several requests
- Int_t fLength; //Request length
- TString fSignature; //Message signature
-
-protected:
- TString Sign();
-
-public:
- THTTPMessage(EHTTP_Verb mverb, TString mpath, TString mbucket, TString mhost,
- TString maprefix, TString maid, TString maidkey);
- THTTPMessage(EHTTP_Verb mverb, TString mpath, TString mbucket, TString mhost,
- TString maprefix, TString maid, TString maidkey, Long64_t offset, Long64_t *pos, Int_t *len, Int_t nbuf);
- THTTPMessage() : fInitByte(0), fLen(0) { }
- virtual ~THTTPMessage() { }
- THTTPMessage &operator=(const THTTPMessage& rhs);
-
- EHTTP_Verb GetHTTPVerb() const { return fVerb; }
- TString GetPath() const { return fPath; }
- TString GetBucket() const { return fBucket; }
- TString GetHost() const { return fHost; }
- TString GetDatime() const { return fDate; }
- TString GetAuthPrefix() const { return fAuthPrefix; }
- TString GetAccessId() const { return fAccessId; }
- TString GetAccessIdKey() const { return fAccessIdKey; }
- Long64_t GetOffset() const { return fOffset; }
- Long64_t* GetInitByte() const { return fInitByte; }
- Int_t* GetRangeLength() const { return fLen; }
- Int_t GetCurrentBuffer() const { return fCurrentBuf; }
- Int_t GetNumBuffers() const { return fNumBuf; }
- Int_t GetLength() const { return fLength; }
- TString GetSignature() const { return fSignature; }
-
- Bool_t HasRange() const { return fHasRange; }
-
- TString DatimeToTString() const;
- TString HTTPVerbToTString() const;
-
- TString CreateHead() const;
- TString CreateHost() const;
- TString CreateDate() const;
- TString CreateAuth() const;
-
- TString GetRequest(Bool_t appendCRLF=kTRUE);
-
- ClassDef(THTTPMessage, 0) // Create generic HTTP request for Amazon S3 and Google Storage services
-};
-
-#endif
diff --git a/net/net/inc/TS3HTTPRequest.h b/net/net/inc/TS3HTTPRequest.h
new file mode 100644
index 00000000000..d2a0f95efdf
--- /dev/null
+++ b/net/net/inc/TS3HTTPRequest.h
@@ -0,0 +1,158 @@
+// @(#)root/net:$Id$
+// Author: Fabio Hernandez 30/01/2013
+// based on an initial version by Marcelo Sousa (class THTTPMessage)
+
+/*************************************************************************
+ * Copyright (C) 1995-2011, Rene Brun and Fons Rademakers. *
+ * All rights reserved. *
+ * *
+ * For the licensing terms see $ROOTSYS/LICENSE. *
+ * For the list of contributors see $ROOTSYS/README/CREDITS. *
+ *************************************************************************/
+
+#ifndef ROOT_TS3HTTPRequest
+#define ROOT_TS3HTTPRequest
+
+//////////////////////////////////////////////////////////////////////////
+// //
+// TS3HTTPRequest //
+// //
+// An object of this class represents an HTTP request extended to be //
+// compatible with Amazon's S3 protocol. //
+// Specifically, such a request contains an 'Authorization' header with //
+// information used by the S3 server for authenticating this request. //
+// The authentication information is computed based on a pair of access //
+// key and secret key which are both provided to the user by the S3 //
+// service provider (e.g. Amazon, Google, etc.). //
+// The secret key is used to compute a signature of selected fields in //
+// the request. The algorithm for computing the signature is documented //
+// in: //
+// //
+// Google storage: //
+// http://code.google.com/apis/storage/docs/reference/v1/developer-guidev1.html#authentication
+// //
+// Amazon: //
+// http://docs.aws.amazon.com/AmazonS3/latest/dev/S3_Authentication2.html
+// //
+//////////////////////////////////////////////////////////////////////////
+
+#ifndef ROOT_TObject
+#include "TObject.h"
+#endif
+
+#ifndef ROOT_TString
+#include "TString.h"
+#endif
+
+
+
+class TS3HTTPRequest : public TObject {
+
+public:
+
+ enum EHTTPVerb { kGET, kPOST, kPUT, kDELETE, kHEAD, kCOPY };
+ enum EAuthType { kNoAuth, kAmazon, kGoogle };
+
+private:
+ EHTTPVerb fVerb; // HTTP Verb
+ EAuthType fAuthType; // Authentication type
+ TString fHost; // Host name
+ TString fBucket; // Bucket name
+ TString fObjectKey; // Object key
+ TString fTimeStamp; // Request time stamp
+ TString fAccessKey; // Access key (for authentication)
+ TString fSecretKey; // Secret key (for authentication)
+
+
+protected:
+ TString HTTPVerbToTString(EHTTPVerb httpVerb) const;
+ TString MakeRequestLine(TS3HTTPRequest::EHTTPVerb httpVerb) const;
+ TString MakeAuthHeader(TS3HTTPRequest::EHTTPVerb httpVerb) const;
+ TString ComputeSignature(TS3HTTPRequest::EHTTPVerb httpVerb) const;
+ TString MakeAuthPrefix() const;
+ TString MakeHostHeader() const;
+ TString MakeDateHeader() const;
+ TS3HTTPRequest& SetTimeStamp();
+
+public:
+
+ TS3HTTPRequest();
+ TS3HTTPRequest(EHTTPVerb httpVerb, const TString& host,
+ const TString& bucket, const TString& objectKey,
+ EAuthType authType, const TString& accessKey,
+ const TString& secretKey);
+ TS3HTTPRequest(const TS3HTTPRequest& m);
+ virtual ~TS3HTTPRequest() { }
+
+ EHTTPVerb GetHTTPVerb() const { return fVerb; }
+ const TString& GetHost() const { return fHost; }
+ const TString& GetBucket() const { return fBucket; }
+ const TString& GetObjectKey() const { return fObjectKey; }
+ const TString& GetTimeStamp() const { return fTimeStamp; }
+ const TString& GetAccessKey() const { return fAccessKey; }
+ const TString& GetSecretKey() const { return fSecretKey; }
+ TString GetAuthType() const { return fAuthType; }
+ TString GetRequest(TS3HTTPRequest::EHTTPVerb httpVerb, Bool_t appendCRLF=kTRUE);
+
+ TS3HTTPRequest& SetHost(const TString& host);
+ TS3HTTPRequest& SetBucket(const TString& bucket);
+ TS3HTTPRequest& SetObjectKey(const TString& objectKey);
+ TS3HTTPRequest& SetAccessKey(const TString& accessKey);
+ TS3HTTPRequest& SetSecretKey(const TString& secretKey);
+ TS3HTTPRequest& SetAuthKeys(const TString& accessKey, const TString& secretKey);
+ TS3HTTPRequest& SetAuthType(TS3HTTPRequest::EAuthType authType);
+
+ ClassDef(TS3HTTPRequest, 0) // Create generic HTTP request for Amazon S3 and Google Storage services
+};
+
+
+//////////////////////////////////////////////////////////////////////////
+// //
+// Inlines //
+// //
+//////////////////////////////////////////////////////////////////////////
+
+inline TS3HTTPRequest& TS3HTTPRequest::SetHost(const TString& host)
+{
+ fHost = host;
+ return *this;
+}
+
+inline TS3HTTPRequest& TS3HTTPRequest::SetBucket(const TString& bucket)
+{
+ fBucket = bucket;
+ return *this;
+}
+
+inline TS3HTTPRequest& TS3HTTPRequest::SetObjectKey(const TString& objectKey)
+{
+ fObjectKey = objectKey;
+ return *this;
+}
+
+inline TS3HTTPRequest& TS3HTTPRequest::SetAuthKeys(const TString& accessKey, const TString& secretKey)
+{
+ fAccessKey = accessKey;
+ fSecretKey = secretKey;
+ return *this;
+}
+
+inline TS3HTTPRequest& TS3HTTPRequest::SetAuthType(TS3HTTPRequest::EAuthType authType)
+{
+ fAuthType = authType;
+ return *this;
+}
+
+inline TS3HTTPRequest& TS3HTTPRequest::SetAccessKey(const TString& accessKey)
+{
+ fAccessKey = accessKey;
+ return *this;
+}
+
+inline TS3HTTPRequest& TS3HTTPRequest::SetSecretKey(const TString& secretKey)
+{
+ fSecretKey = secretKey;
+ return *this;
+}
+
+#endif
diff --git a/net/net/inc/TS3WebFile.h b/net/net/inc/TS3WebFile.h
index b887e50f323..95947ed53aa 100644
--- a/net/net/inc/TS3WebFile.h
+++ b/net/net/inc/TS3WebFile.h
@@ -37,7 +37,7 @@
// initializing an object of this class by two means: //
// a) by using the environmental variables S3_ACCESS_KEY and //
// S3_SECRET_KEY, or //
-// b) by specifying them when opening each file. //
+// b) by specifying them as an argument when opening each file. //
// //
// The first method is convenient if all the S3 files you want to //
// access are hosted by a single provider. The second one is more //
@@ -46,7 +46,7 @@
// this class for details on the syntax. //
// //
// For generating and signing the HTTP request, this class uses //
-// THTTPMessage. //
+// TS3HTTPRequest. //
// //
// For more information on the details of S3 protocol please refer to: //
// "Amazon Simple Storage Service Developer Guide": //
@@ -68,13 +68,18 @@
#include "TString.h"
#endif
+#ifndef ROOT_TS3HTTPRequest
+#include "TS3HTTPRequest.h"
+#endif
+
class TS3WebFile: public TWebFile {
private:
TS3WebFile();
- Bool_t ParseOptions(const TString& options);
- Bool_t SetCredentialsFromEnv(const char* accessKeyEnv, const char* secretKeyEnv);
+ Bool_t ParseOptions(Option_t* options, TString& accessKey, TString& secretKey);
+ Bool_t GetCredentialsFromEnv(const char* accessKeyEnv, const char* secretKeyEnv,
+ TString& outAccessKey, TString& outSecretKey);
protected:
// Super-class methods extended by this class
@@ -83,16 +88,12 @@ protected:
virtual void ProcessHttpHeader(const TString& headerLine);
// Modifiers of data members (to be used mainly by subclasses)
- void SetAccessKey(const TString& accessKey) { fAccessKey = accessKey; }
- void SetSecretKey(const TString& secretKey) { fSecretKey = secretKey; }
+ void SetAccessKey(const TString& accessKey) { fS3Request.SetAccessKey(accessKey); }
+ void SetSecretKey(const TString& secretKey) { fS3Request.SetSecretKey(secretKey); }
// Data members
- static const TString fgAuthPrefix; // Authentication prefix
- TString fAccessKey; // Access key ID
- TString fSecretKey; // Secret access key
- TString fBucket; // Bucket name
- TString fObjectKey; // S3 object key (i.e. the file path within the bucket)
- Bool_t fUseMultiRange; // Is the S3 server capable of serving multirange requests?
+ TS3HTTPRequest fS3Request; // S3 HTTP request
+ Bool_t fUseMultiRange; // Is the S3 server capable of serving multirange requests?
public:
// Constructors & Destructor
@@ -100,12 +101,11 @@ public:
virtual ~TS3WebFile() {}
// Selectors
- const TString& GetAuthPrefix() const { return fgAuthPrefix; }
- const TString& GetAccessKey() const { return fAccessKey; }
- const TString& GetSecretKey() const { return fSecretKey; }
+ const TString& GetAccessKey() const { return fS3Request.GetAccessKey(); }
+ const TString& GetSecretKey() const { return fS3Request.GetSecretKey(); }
+ const TString& GetBucket() const { return fS3Request.GetBucket(); }
+ const TString& GetObjectKey() const { return fS3Request.GetObjectKey(); }
const TUrl& GetUrl() const { return fUrl; }
- const TString& GetBucket() const { return fBucket; }
- const TString& GetObjectKey() const { return fObjectKey; }
// Modifiers
virtual Bool_t ReadBuffers(char* buf, Long64_t* pos, Int_t* len, Int_t nbuf);
diff --git a/net/net/src/TAS3File.cxx b/net/net/src/TAS3File.cxx
deleted file mode 100644
index 7de0ce9eaa8..00000000000
--- a/net/net/src/TAS3File.cxx
+++ /dev/null
@@ -1,265 +0,0 @@
-// @(#)root/net:$Id$
-// Author: Marcelo Sousa 23/08/2011
-
-/*************************************************************************
- * Copyright (C) 1995-2011, Rene Brun and Fons Rademakers. *
- * All rights reserved. *
- * *
- * For the licensing terms see $ROOTSYS/LICENSE. *
- * For the list of contributors see $ROOTSYS/README/CREDITS. *
- *************************************************************************/
-
-//////////////////////////////////////////////////////////////////////////
-// //
-// TAS3File //
-// //
-// A TAS3File is a normal TWebFile but it reads data from the //
-// Amazon S3 server. As a derived TWebFile class TAS3File it is a //
-// read only file. The HTTP requests are generated by THTTPMessage //
-// objects with the auth_prefix set as AWS. The user id and secret pass //
-// required to sign the requests are passed through the environment //
-// variables S3_ACCESS_ID and S3_ACCESS_KEY. //
-// For more information check: //
-// http://aws.amazon.com/documentation/s3/ //
-// http://awsdocs.s3.amazonaws.com/S3/latest/s3-qrc.pdf //
-// //
-//////////////////////////////////////////////////////////////////////////
-
-#include "TAS3File.h"
-#include "THTTPMessage.h"
-#include "TSocket.h"
-#include "TROOT.h"
-
-#include <errno.h>
-#include <stdlib.h>
-
-ClassImp(TAS3File)
-
-//______________________________________________________________________________
-TAS3File::TAS3File(const char *path, Option_t *) : TWebFile(path, "IO")
-{
- // For TAS3File to properly work you need to set up
- // environment variables S3_ACCESS_ID and S3_ACCESS_KEY
- // The format of the path is: server/bucket/file
- // Example: f = new TAS3File("as3://s3-eu-west-1.amazonaws.com/roots3/hsimple.root")
-
- TString tpath = path;
-
- Int_t begPath = 6, slash = 0, i = 0;
-
- if (tpath.BeginsWith("as3://") == kFALSE) {
- Error("TAS3File", "invalid path %s", path);
- goto zombie;
- }
-
- while (i < 2 && begPath < tpath.Length()) {
- slash = tpath.Index('/', begPath);
-
- if (slash == kNPOS) {
- Error("TAS3File","invalid path %s", path);
- goto zombie;
- }
-
- switch(i){
- case 0:
- fServer = TUrl(TString(tpath(begPath,slash)));
- break;
- case 1:
- fBucket = tpath(begPath,slash-begPath);
- fRealName = "/" + tpath(slash+1, tpath.Length()-(slash+1));
- }
- i++;
- begPath = slash+1;
- }
-
- fAuthPrefix = "AWS";
- fAccessId = gSystem->Getenv("S3_ACCESS_ID");
- fAccessKey = gSystem->Getenv("S3_ACCESS_KEY");
- if (fAccessId == "" || fAccessKey == "") {
- if (fAccessId == "") Error("TAS3File", "shell variable S3_ACCESS_ID not set");
- if (fAccessKey == "") Error("TAS3File", "shell variable S3_ACCESS_KEY not set");
- goto zombie;
- }
-
- Init(kFALSE);
- return;
-
-zombie:
- MakeZombie();
- gDirectory = gROOT;
-}
-
-//______________________________________________________________________________
-Int_t TAS3File::GetHead()
-{
- // Clone of TWebFile::GetHead except it uses THTTPMessage to generate
- // the HTTP request.
-
- THTTPMessage s3head = THTTPMessage(kHEAD, fRealName, GetBucket(),
- GetUrl().GetHost(), GetAuthPrefix(),
- GetAccessId(), GetAccessKey());
-
- TString msg = s3head.GetRequest();
-
- TUrl connurl;
-
- fUrl = fServer;
-
- if (fProxy.IsValid())
- connurl = fProxy;
- else
- connurl = fUrl;
-
- TSocket *s = 0;
- for (Int_t i = 0; i < 5; i++) {
- s = new TSocket(connurl.GetHost(), connurl.GetPort());
- if (!s->IsValid()) {
- delete s;
- if (gSystem->GetErrno() == EADDRINUSE || gSystem->GetErrno() == EISCONN) {
- s = 0;
- gSystem->Sleep(i*10);
- } else {
- Error("GetHead", "cannot connect to host %s (errno=%d)", fUrl.GetHost(),
- gSystem->GetErrno());
- return -1;
- }
- } else
- break;
- }
- if (!s)
- return -1;
-
- if (s->SendRaw(msg.Data(), msg.Length()) == -1) {
- Error("GetHead", "error sending command to host %s", fUrl.GetHost());
- delete s;
- return -1;
- }
-
- char line[8192];
- Int_t n, ret = 0, redirect = 0;
-
- while ((n = GetLine(s, line, sizeof(line))) >= 0) {
- if (n == 0) {
- if (gDebug > 0)
- Info("GetHead", "got all headers");
- delete s;
- if (fBasicUrlOrg != "" && !redirect) {
- // set back to original url in case of temp redirect
- SetMsgReadBuffer10();
- fMsgGetHead = "";
- }
- if (ret < 0)
- return ret;
- if (redirect)
- return GetHead();
- return 0;
- }
-
- if (gDebug > 0)
- Info("GetHead", "header: %s", line);
-
- TString res = line;
- if (res.BeginsWith("HTTP/1.")) {
- if (res.BeginsWith("HTTP/1.1")) {
- if (!fHTTP11) {
- fMsgGetHead = "";
- fMsgReadBuffer10 = "";
- }
- fHTTP11 = kTRUE;
- }
- TString scode = res(9, 3);
- Int_t code = scode.Atoi();
- if (code >= 500) {
- if (code == 500)
- fHasModRoot = kTRUE;
- else {
- ret = -1;
- TString mess = res(13, 1000);
- Error("GetHead", "%s: %s (%d)", fBasicUrl.Data(), mess.Data(), code);
- }
- } else if (code >= 400) {
- if (code == 400)
- ret = -3; // command not supported
- else if (code == 404)
- ret = -2; // file does not exist
- else {
- ret = -1;
- TString mess = res(13, 1000);
- Error("GetHead", "%s: %s (%d)", fBasicUrl.Data(), mess.Data(), code);
- }
- } else if (code >= 300) {
- if (code == 301 || code == 303)
- redirect = 1; // permanent redirect
- else if (code == 302 || code == 307)
- redirect = 2; // temp redirect
- else {
- ret = -1;
- TString mess = res(13, 1000);
- Error("GetHead", "%s: %s (%d)", fBasicUrl.Data(), mess.Data(), code);
- }
- } else if (code > 200) {
- ret = -1;
- TString mess = res(13, 1000);
- Error("GetHead", "%s: %s (%d)", fBasicUrl.Data(), mess.Data(), code);
- }
- } else if (res.BeginsWith("Content-Length:")) {
- TString slen = res(16, 1000);
- fSize = slen.Atoll();
- } else if (res.BeginsWith("Location:") && redirect) {
- TString redir = res(10, 1000);
- if (redirect == 2) // temp redirect
- SetMsgReadBuffer10(redir, kTRUE);
- else // permanent redirect
- SetMsgReadBuffer10(redir, kFALSE);
- fMsgGetHead = "";
- }
- }
-
- delete s;
- return ret;
-}
-
-//______________________________________________________________________________
-Bool_t TAS3File::ReadBuffer(char *buf, Int_t len)
-{
- return ReadBuffer10(buf,len);
-}
-
-//______________________________________________________________________________
-Bool_t TAS3File::ReadBuffer10(char *buf, Int_t len)
-{
- // Read specified byte range from Amazon S3.
- // This routine connects to the Amazon S3 server, sends the
- // request created by THTTPMessage and returns the buffer.
- // Returns kTRUE in case of error.
-
- const Int_t nbuf = 1;
- Long64_t pos[nbuf];
- pos[nbuf-1] = fOffset;
-
- Int_t lens[nbuf];
- lens[nbuf-1] = len;
-
- THTTPMessage s3get = THTTPMessage(kGET, fRealName, GetBucket(),
- GetUrl().GetHost(), GetAuthPrefix(),
- GetAccessId(), GetAccessKey(),
- 0, pos, lens, nbuf);
- TString msg = s3get.GetRequest();
-
- Int_t n = GetFromWeb10(buf, len, msg);
- if (n == -1)
- return kTRUE;
- // The -2 error condition typically only happens when
- // GetHead() failed because not implemented, in the first call to
- // ReadBuffer() in Init(), it is not checked in ReadBuffers10().
- if (n == -2) {
- Error("ReadBuffer10", "%s does not exist", fBasicUrl.Data());
- MakeZombie();
- gDirectory = gROOT;
- return kTRUE;
- }
-
- fOffset += len;
-
- return kFALSE;
-}
diff --git a/net/net/src/TGSFile.cxx b/net/net/src/TGSFile.cxx
deleted file mode 100644
index 2df43ef6f47..00000000000
--- a/net/net/src/TGSFile.cxx
+++ /dev/null
@@ -1,309 +0,0 @@
-// @(#)root/net:$Id$
-// Author: Marcelo Sousa 23/08/2011
-
-/*************************************************************************
- * Copyright (C) 1995-2011, Rene Brun and Fons Rademakers. *
- * All rights reserved. *
- * *
- * For the licensing terms see $ROOTSYS/LICENSE. *
- * For the list of contributors see $ROOTSYS/README/CREDITS. *
- *************************************************************************/
-
-//////////////////////////////////////////////////////////////////////////
-// //
-// TGSFile //
-// //
-// A TGSFile is a normal TWebFile but it reads data from the //
-// Google Storage server. As a derived TWebFile class TGSFile it is //
-// a read only file. The HTTP requests are generated by THTTPMessage //
-// objects with the auth_prefix set as GOOG1. The user id and secret //
-// pass required to sign the requests are passed through the //
-// environment variables GT_ACCESS_ID and GT_ACCESS_KEY. //
-// For more information check: //
-// http://code.google.com/apis/storage/docs/getting-started.html //
-// //
-//////////////////////////////////////////////////////////////////////////
-
-#include "TGSFile.h"
-#include "THTTPMessage.h"
-#include "TSocket.h"
-#include "TROOT.h"
-
-#include <errno.h>
-#include <stdlib.h>
-
-ClassImp(TGSFile)
-
-//______________________________________________________________________________
-TGSFile::TGSFile(const char *path, Option_t *) : TWebFile(path, "IO")
-{
- // For TGSFile to properly work you need to set up
- // environment variables GT_ACCESS_ID and GT_ACCESS_KEY.
- // The format of the path is: server/bucket/file, e.g.
- // f = new TGSFile("gs://commondatastorage.googleapis.com/roots3/hsimple.root")
-
- TString tpath = path;
-
- Int_t begPath = 5, slash = 0, i = 0;
-
- if (tpath.BeginsWith("gs://") == kFALSE) {
- Error("TGSFile", "invalid path %s", path);
- goto zombie;
- }
-
- while (i < 2 && begPath < tpath.Length()) {
- slash = tpath.Index('/', begPath);
-
- if (slash == kNPOS) {
- Error("TGSFile","invalid path %s", path);
- goto zombie;
- }
-
- switch(i){
- case 0:
- fServer = TUrl(TString(tpath(begPath,slash)));
- break;
- case 1:
- fBucket = tpath(begPath,slash-begPath);
- fRealName = "/" + tpath(slash+1, tpath.Length()-(slash+1));
- }
- i++;
- begPath = slash+1;
- }
-
- fAuthPrefix = "GOOG1";
- fAccessId = gSystem->Getenv("GT_ACCESS_ID");
- fAccessKey = gSystem->Getenv("GT_ACCESS_KEY");
- if (fAccessId == "" || fAccessKey == "") {
- if (fAccessId == "") Error("TGSFile", "shell variable GT_ACCESS_ID not set");
- if (fAccessKey == "") Error("TGSFile", "shell variable GT_ACCESS_KEY not set");
- goto zombie;
- }
-
- Init(kFALSE);
- return;
-
-zombie:
- MakeZombie();
- gDirectory = gROOT;
-}
-
-//______________________________________________________________________________
-Int_t TGSFile::GetHead()
-{
- // Clone of TWebFile::GetHead except it uses THTTPMessage to generate
- // the HTTP request.
-
- THTTPMessage s3head = THTTPMessage(kHEAD, fRealName, GetBucket(),
- GetUrl().GetHost(), GetAuthPrefix(),
- GetAccessId(), GetAccessKey());
-
- TString msg = s3head.GetRequest();
-
- TUrl connurl;
-
- fUrl = fServer;
-
- if (fProxy.IsValid())
- connurl = fProxy;
- else
- connurl = fUrl;
-
- TSocket *s = 0;
- for (Int_t i = 0; i < 5; i++) {
- s = new TSocket(connurl.GetHost(), connurl.GetPort());
- if (!s->IsValid()) {
- delete s;
- if (gSystem->GetErrno() == EADDRINUSE || gSystem->GetErrno() == EISCONN) {
- s = 0;
- gSystem->Sleep(i*10);
- } else {
- Error("GetHead", "cannot connect to host %s (errno=%d)", fUrl.GetHost(),
- gSystem->GetErrno());
- return -1;
- }
- } else
- break;
- }
- if (!s)
- return -1;
-
- if (s->SendRaw(msg.Data(), msg.Length()) == -1) {
- Error("GetHead", "error sending command to host %s", fUrl.GetHost());
- delete s;
- return -1;
- }
-
- char line[8192];
- Int_t n, ret = 0, redirect = 0;
-
- while ((n = GetLine(s, line, sizeof(line))) >= 0) {
- if (n == 0) {
- if (gDebug > 0)
- Info("GetHead", "got all headers");
- delete s;
- if (fBasicUrlOrg != "" && !redirect) {
- // set back to original url in case of temp redirect
- SetMsgReadBuffer10();
- fMsgGetHead = "";
- }
- if (ret < 0)
- return ret;
- if (redirect)
- return GetHead();
- return 0;
- }
-
- if (gDebug > 0)
- Info("GetHead", "header: %s", line);
-
- TString res = line;
- if (res.BeginsWith("HTTP/1.")) {
- if (res.BeginsWith("HTTP/1.1")) {
- if (!fHTTP11) {
- fMsgGetHead = "";
- fMsgReadBuffer10 = "";
- }
- fHTTP11 = kTRUE;
- }
- TString scode = res(9, 3);
- Int_t code = scode.Atoi();
- if (code >= 500) {
- if (code == 500)
- fHasModRoot = kTRUE;
- else {
- ret = -1;
- TString mess = res(13, 1000);
- Error("GetHead", "%s: %s (%d)", fBasicUrl.Data(), mess.Data(), code);
- }
- } else if (code >= 400) {
- if (code == 400)
- ret = -3; // command not supported
- else if (code == 404)
- ret = -2; // file does not exist
- else {
- ret = -1;
- TString mess = res(13, 1000);
- Error("GetHead", "%s: %s (%d)", fBasicUrl.Data(), mess.Data(), code);
- }
- } else if (code >= 300) {
- if (code == 301 || code == 303)
- redirect = 1; // permanent redirect
- else if (code == 302 || code == 307)
- redirect = 2; // temp redirect
- else {
- ret = -1;
- TString mess = res(13, 1000);
- Error("GetHead", "%s: %s (%d)", fBasicUrl.Data(), mess.Data(), code);
- }
- } else if (code > 200) {
- ret = -1;
- TString mess = res(13, 1000);
- Error("GetHead", "%s: %s (%d)", fBasicUrl.Data(), mess.Data(), code);
- }
- } else if (res.BeginsWith("Content-Length:")) {
- TString slen = res(16, 1000);
- fSize = slen.Atoll();
- } else if (res.BeginsWith("Location:") && redirect) {
- TString redir = res(10, 1000);
- if (redirect == 2) // temp redirect
- SetMsgReadBuffer10(redir, kTRUE);
- else // permanent redirect
- SetMsgReadBuffer10(redir, kFALSE);
- fMsgGetHead = "";
- }
- }
-
- delete s;
- return ret;
-}
-
-#if 0
-// Currently not supported on the Google Storage cloud side because
-// the chunked responses are not "standard"
-//______________________________________________________________________________
-Bool_t TGSFile::ReadBuffers10(char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
-{
- // Read specified byte range from Google Storage.
- // This routine connects to the Google Storage server, sends the
- // request created by THTTPMessage and returns the buffer.
- // Returns kTRUE in case of error.
-
- THTTPMessage gsget = THTTPMessage(kGET, fRealName, GetBucket(),
- GetUrl().GetHost(), GetAuthPrefix(),
- GetAccessId(), GetAccessKey(),
- fOffset, pos, len, nbuf);
- TString msg = gsget.GetRequest();
- Int_t size = gsget.GetLength();
- Int_t curbuf = gsget.GetCurrentBuffer();
-
- //printf("Num of buffers %d, current buffer %d size %d\n",nbuf,curbuf,size);
-
- Int_t n = GetFromWeb10(buf, size, msg);
- if (n == -1)
- return kTRUE;
- // The -2 error condition typically only happens when
- // GetHead() failed because not implemented, in the first call to
- // ReadBuffer() in Init(), it is not checked in ReadBuffers10().
- if (n == -2) {
- Error("ReadBuffer10", "%s does not exist", fBasicUrl.Data());
- MakeZombie();
- gDirectory = gROOT;
- return kTRUE;
- }
-
- fOffset += size;
-
- if (nbuf == curbuf) {
- return kFALSE;
- } else {
- return ReadBuffers10(&buf[size], &pos[curbuf], &len[curbuf], nbuf-curbuf);
- }
-}
-#endif
-
-//______________________________________________________________________________
-Bool_t TGSFile::ReadBuffer(char *buf, Int_t len)
-{
- return ReadBuffer10(buf,len);
-}
-
-//______________________________________________________________________________
-Bool_t TGSFile::ReadBuffer10(char *buf, Int_t len)
-{
- // Read specified byte range from Google Storage.
- // This routine connects to the Google Storage server, sends the
- // request created by THTTPMessage and returns the buffer.
- // Returns kTRUE in case of error.
-
- const Int_t nbuf = 1;
- Long64_t pos[nbuf];
- pos[nbuf-1] = fOffset;
-
- Int_t lens[nbuf];
- lens[nbuf-1] = len;
-
- THTTPMessage gsget = THTTPMessage(kGET, fRealName, GetBucket(),
- GetUrl().GetHost(), GetAuthPrefix(),
- GetAccessId(), GetAccessKey(),
- 0, pos, lens, nbuf);
- TString msg = gsget.GetRequest();
-
- Int_t n = GetFromWeb10(buf, len, msg);
- if (n == -1)
- return kTRUE;
- // The -2 error condition typically only happens when
- // GetHead() failed because not implemented, in the first call to
- // ReadBuffer() in Init(), it is not checked in ReadBuffers10().
- if (n == -2) {
- Error("ReadBuffer10", "%s does not exist", fBasicUrl.Data());
- MakeZombie();
- gDirectory = gROOT;
- return kTRUE;
- }
-
- fOffset += len;
-
- return kFALSE;
-}
-
diff --git a/net/net/src/THTTPMessage.cxx b/net/net/src/THTTPMessage.cxx
deleted file mode 100644
index 320e547f2e6..00000000000
--- a/net/net/src/THTTPMessage.cxx
+++ /dev/null
@@ -1,262 +0,0 @@
-// @(#)root/net:$Id$
-// Author: Marcelo Sousa 23/08/2011
-
-/*************************************************************************
- * Copyright (C) 1995-2011, Rene Brun and Fons Rademakers. *
- * All rights reserved. *
- * *
- * For the licensing terms see $ROOTSYS/LICENSE. *
- * For the list of contributors see $ROOTSYS/README/CREDITS. *
- *************************************************************************/
-
-//////////////////////////////////////////////////////////////////////////
-// //
-// THTTPMessage //
-// //
-// A THTTPMessage object represents a generic HTTP request for the //
-// Amazon S3 and the Google Storage services. It can easily be extended //
-// to other API's. It assumes that each request is signed with the //
-// client id and an encripted key, Base64(HMAC + SHA1 (HTTP Request)) //
-// which is based on a secret key provided in the constructor. //
-// For more information about the authentication : //
-// Google Storage: //
-// http://code.google.com/apis/storage/docs/reference/v1/developer-guidev1.html#authentication //
-// Amazon S3: //
-// http://awsdocs.s3.amazonaws.com/S3/latest/s3-qrc.pdf //
-// At the moment THTTPMessage is used for derived classes of TWebFile //
-// (read only) files supporting HEAD and GET requests. //
-// //
-//////////////////////////////////////////////////////////////////////////
-
-#include "THTTPMessage.h"
-#include "TBase64.h"
-#if defined(MAC_OS_X_VERSION_10_7)
-#include <CommonCrypto/CommonHMAC.h>
-#define SHA_DIGEST_LENGTH 20
-#else
-#include <openssl/sha.h>
-#include <openssl/hmac.h>
-#include <openssl/evp.h>
-#include <openssl/bio.h>
-#include <openssl/buffer.h>
-#endif
-
-#include <stdio.h>
-#include <time.h>
-#include <string.h>
-
-ClassImp(THTTPMessage)
-
-//______________________________________________________________________________
-THTTPMessage::THTTPMessage(EHTTP_Verb mverb, TString mpath, TString mbucket, TString mhost,
- TString maprefix, TString maid, TString maidkey)
-{
- // THTTPMessage for HTTP requests without the Range attribute.
-
- fVerb = mverb;
- fPath = mpath;
- fBucket = mbucket;
- fHost = mhost;
- fDate = DatimeToTString();
- fAuthPrefix = maprefix;
- fAccessId = maid;
- fAccessIdKey = maidkey;
- fHasRange = kFALSE;
- fInitByte = 0;
- fOffset = 0;
- fLen = 0;
- fNumBuf = 0;
- fCurrentBuf = 0;
- fLength = 0;
-
- fSignature = Sign();
-}
-
-//______________________________________________________________________________
-THTTPMessage::THTTPMessage(EHTTP_Verb mverb, TString mpath, TString mbucket, TString mhost,
- TString maprefix, TString maid, TString maidkey, Long64_t offset,
- Long64_t *pos, Int_t *len, Int_t nbuf)
-{
- // THTTPMessage for HTTP Get Requests with Range.
-
- fVerb = mverb;
- fPath = mpath;
- fBucket = mbucket;
- fHost = mhost;
- fDate = DatimeToTString();
- fAuthPrefix = maprefix;
- fAccessId = maid;
- fAccessIdKey = maidkey;
- fHasRange = kTRUE;
- fInitByte = pos;
- fOffset = offset;
- fLen = len;
- fNumBuf = nbuf;
- fCurrentBuf = 0;
- fLength = 0;
-
- fSignature = Sign();
-}
-
-//______________________________________________________________________________
-THTTPMessage &THTTPMessage::operator=(const THTTPMessage &rhs)
-{
- // Copy ctor.
-
- if (this != &rhs){
- TObject::operator=(rhs);
- fVerb = rhs.fVerb;
- fPath = rhs.fPath;
- fBucket = rhs.fBucket;
- fHost = rhs.fHost;
- fDate = rhs.fDate;
- fHasRange = rhs.fHasRange;
- fInitByte = rhs.fInitByte;
- fOffset = rhs.fOffset;
- fLen = rhs.fLen;
- fNumBuf = rhs.fNumBuf;
- fCurrentBuf = rhs.fCurrentBuf;
- fAuthPrefix = rhs.fAuthPrefix;
- fAccessId = rhs.fAccessId;
- fAccessIdKey = rhs.fAccessIdKey;
- fSignature = rhs.fSignature;
- fLength = rhs.fLength;
- }
- return *this;
-}
-
-//______________________________________________________________________________
-TString THTTPMessage::Sign()
-{
- // Message Signature according to:
- // http://awsdocs.s3.amazonaws.com/S3/latest/s3-qrc.pdf
- // and
- // http://code.google.com/apis/storage/docs/reference/v1/developer-guidev1.html#authentication
-
- TString sign;
- sign += HTTPVerbToTString() + "\n";
- sign += "\n"; // GetContentMD5()
- sign += "\n"; // GetContentType()
- sign += DatimeToTString() + "\n";
-
- if (GetAuthPrefix() == "GOOG1") {
- sign += "x-goog-api-version:1\n";
- }
-
- sign += "/" + GetBucket() + GetPath();
- char digest[SHA_DIGEST_LENGTH] = {0};
- TString key = GetAccessIdKey();
-
-#if defined(MAC_OS_X_VERSION_10_7)
- CCHmac(kCCHmacAlgSHA1, key.Data(), key.Length() , (unsigned char *) sign.Data(), sign.Length(), (unsigned char *) digest);
-#else
- unsigned int *sd = NULL;
- HMAC(EVP_sha1(), key.Data(), key.Length() , (unsigned char *) sign.Data(), sign.Length(), (unsigned char *) digest, sd);
-#endif
-
- return TBase64::Encode((const char *) digest, SHA_DIGEST_LENGTH);
-}
-
-//______________________________________________________________________________
-TString THTTPMessage::HTTPVerbToTString() const
-{
- EHTTP_Verb mverb = GetHTTPVerb();
- switch(mverb){
- case kGET: return TString("GET");
- case kPOST: return TString("POST");
- case kPUT: return TString("PUT");
- case kDELETE: return TString("DELETE");
- case kHEAD: return TString("HEAD");
- case kCOPY: return TString("COPY");
- default: return TString("");
- }
-}
-
-//______________________________________________________________________________
-TString THTTPMessage::DatimeToTString() const
-{
- // Generates a Date TString according to:
- // http://code.google.com/apis/storage/docs/reference-headers.html#date
-
- time_t now = time(NULL);
- {
- // TODO: remove this block which was introduced to turnaround the
- // time skew problem of UDS
- // now -= 16*60;
- }
- char result[128];
- struct tm date_format;
- strftime(result, sizeof(result), "%a, %d %b %Y %H:%M:%S GMT", gmtime_r(&now, &date_format));
- return TString(result);
-}
-
-//______________________________________________________________________________
-TString THTTPMessage::CreateHead() const
-{
- // Returns the first line of a HTTP request for this object. Note that since
- // we don't use the virtual host syntax which is supported by Amazon, we
- // must include the bucket name in thr resource. For example, we don't use
- // http://mybucket.s3.amazonaws.com/path/to/my/file but instead
- // http://s3.amazonaws.com/mybucket/path/to/my/file so the HTTP request
- // will be of the form "GET /mybucket/path/to/my/file HTTP/1.1"
- // Also note that the path must include the leading '/'.
-
- return TString::Format("%s /%s%s HTTP/1.1",
- (const char*)HTTPVerbToTString(), (const char*)GetBucket(), (const char*)GetPath());
-}
-
-//______________________________________________________________________________
-TString THTTPMessage::CreateHost() const
-{
- // Returns the 'Host' header to include in the HTTP request.
-
- return "Host: " + GetHost();
-}
-
-//______________________________________________________________________________
-TString THTTPMessage::CreateDate() const
-{
- return "Date: " + GetDatime();
-}
-
-//______________________________________________________________________________
-TString THTTPMessage::CreateAuth() const
-{
- if (GetAuthPrefix() == "AWS") {
- return "Authorization: " + GetAuthPrefix() + " " + GetAccessId() + ":" + GetSignature();
- } else {
- return "x-goog-api-version: 1\r\nAuthorization: " + GetAuthPrefix() + " " +
- GetAccessId() + ":" + GetSignature();
- }
-}
-
-//______________________________________________________________________________
-TString THTTPMessage::GetRequest(Bool_t appendCRLF)
-{
- // Returns the HTTP request
-
- TString msg = TString::Format("%s\r\n%s\r\n%s\r\n%s\r\n",
- (const char*)CreateHead(), (const char*)CreateHost(),
- (const char*)CreateDate(), (const char*)CreateAuth());
-
- if (HasRange()) {
- Int_t n = 0;
- msg += "Range: bytes=";
- for (Int_t i = 0; i < fNumBuf; i++) {
- if (n) msg += ",";
- msg += fInitByte[i] + fOffset;
- msg += "-";
- msg += fInitByte[i] + fOffset + fLen[i] - 1;
- fLength += fLen[i];
- n += fLen[i];
- fCurrentBuf++;
- if (msg.Length() > 8000) {
- break;
- }
- }
- msg += "\r\n";
- }
- if (appendCRLF)
- msg += "\r\n";
- return msg;
-}
diff --git a/net/net/src/TS3HTTPRequest.cxx b/net/net/src/TS3HTTPRequest.cxx
new file mode 100644
index 00000000000..c8c29a17bcf
--- /dev/null
+++ b/net/net/src/TS3HTTPRequest.cxx
@@ -0,0 +1,240 @@
+// @(#)root/net:$Id$
+// Author: Fabio Hernandez 30/01/2013
+// based on an initial version by Marcelo Sousa (class THTTPMessage)
+
+/*************************************************************************
+ * Copyright (C) 1995-2011, Rene Brun and Fons Rademakers. *
+ * All rights reserved. *
+ * *
+ * For the licensing terms see $ROOTSYS/LICENSE. *
+ * For the list of contributors see $ROOTSYS/README/CREDITS. *
+ *************************************************************************/
+
+//////////////////////////////////////////////////////////////////////////
+// //
+// TS3HTTPRequest //
+// //
+// An object of this class represents an HTTP request extended to be //
+// compatible with Amazon's S3 protocol. //
+// Specifically, such a request contains an 'Authorization' header with //
+// information used by the S3 server for authenticating this request. //
+// The authentication information is computed based on a pair of access //
+// key and secret key which are both provided to the user by the S3 //
+// service provider (e.g. Amazon, Google, etc.). //
+// The secret key is used to compute a signature of selected fields in //
+// the request. The algorithm for computing the signature is documented //
+// in: //
+// //
+// Google storage: //
+// http://code.google.com/apis/storage/docs/reference/v1/developer-guidev1.html#authentication
+// //
+// Amazon: //
+// http://docs.aws.amazon.com/AmazonS3/latest/dev/S3_Authentication2.html
+// //
+//////////////////////////////////////////////////////////////////////////
+
+#include "TS3HTTPRequest.h"
+#include "TBase64.h"
+#if defined(MAC_OS_X_VERSION_10_7)
+#include <CommonCrypto/CommonHMAC.h>
+#define SHA_DIGEST_LENGTH 20
+#else
+#include <openssl/sha.h>
+#include <openssl/hmac.h>
+#include <openssl/evp.h>
+#include <openssl/bio.h>
+#include <openssl/buffer.h>
+#endif
+
+#include <stdio.h>
+#include <time.h>
+#include <string.h>
+
+ClassImp(TS3HTTPRequest)
+
+//______________________________________________________________________________
+TS3HTTPRequest::TS3HTTPRequest()
+ : fAuthType(kNoAuth), fHost("NoHost")
+{
+}
+
+//______________________________________________________________________________
+TS3HTTPRequest::TS3HTTPRequest(EHTTPVerb httpVerb, const TString& host,
+ const TString& bucket, const TString& objectKey, EAuthType authType,
+ const TString& accessKey, const TString& secretKey)
+{
+ // Default constructor
+
+ fVerb = httpVerb;
+ fHost = host;
+ fBucket = bucket;
+ fObjectKey = objectKey;
+ fAuthType = authType;
+ fAccessKey = accessKey;
+ fSecretKey = secretKey;
+}
+
+//______________________________________________________________________________
+TS3HTTPRequest::TS3HTTPRequest(const TS3HTTPRequest& r)
+ : TObject(r)
+{
+ // Copy constructor
+
+ fVerb = r.fVerb;
+ fHost = r.fHost;
+ fBucket = r.fBucket;
+ fObjectKey = r.fObjectKey;
+ fAuthType = r.fAuthType;
+ fAccessKey = r.fAccessKey;
+ fSecretKey = r.fSecretKey;
+ fTimeStamp = r.fTimeStamp;
+}
+
+//______________________________________________________________________________
+TString TS3HTTPRequest::ComputeSignature(TS3HTTPRequest::EHTTPVerb httpVerb) const
+{
+ // Returns this request's signature
+
+ // Please note, the order of the fields used for computing
+ // the signature is important. Make sure that the changes you
+ // make are compatible with the reference documentation.
+ //
+ // Refs:
+ // AMAZON http://awsdocs.s3.amazonaws.com/S3/latest/s3-qrc.pdf
+ // GOOGLE: http://code.google.com/apis/storage/docs/reference/v1/developer-guidev1.html#authentication
+
+ TString toSign = TString::Format("%s\n\n\n%s\n", // empty Content-MD5 and Content-Type
+ (const char*)HTTPVerbToTString(httpVerb),
+ (const char*)fTimeStamp);
+ if (fAuthType == kGoogle) {
+ // Must use API version 1. Google Storage API v2 only
+ // accepts OAuth authentication.
+ // This header is not strictly needed but if used for computing
+ // the signature, the request must contain it as a header
+ // (see method MakeAuthHeader)
+ // Ref: https://developers.google.com/storage/docs/reference/v1/apiversion1
+ toSign += "x-goog-api-version:1\n"; // Lowercase, no spaces around ':'
+ }
+
+ toSign += "/" + fBucket + fObjectKey;
+
+ unsigned char digest[SHA_DIGEST_LENGTH] = {0};
+#if defined(MAC_OS_X_VERSION_10_7)
+ CCHmac(kCCHmacAlgSHA1, fSecretKey.Data(), fSecretKey.Length() , (unsigned char *)toSign.Data(), toSign.Length(), digest);
+#else
+ unsigned int *sd = NULL;
+ HMAC(EVP_sha1(), fSecretKey.Data(), fSecretKey.Length() , (unsigned char *)toSign.Data(), toSign.Length(), digest, sd);
+#endif
+
+ return TBase64::Encode((const char *)digest, SHA_DIGEST_LENGTH);
+}
+
+//______________________________________________________________________________
+TString TS3HTTPRequest::HTTPVerbToTString(TS3HTTPRequest::EHTTPVerb httpVerb) const
+{
+ switch (httpVerb) {
+ case kGET: return TString("GET");
+ case kPOST: return TString("POST");
+ case kPUT: return TString("PUT");
+ case kDELETE: return TString("DELETE");
+ case kHEAD: return TString("HEAD");
+ case kCOPY: return TString("COPY");
+ default: return TString("");
+ }
+}
+
+//______________________________________________________________________________
+TS3HTTPRequest& TS3HTTPRequest::SetTimeStamp()
+{
+ // Sets this request's time stamp according to:
+ // http://code.google.com/apis/storage/docs/reference-headers.html#date
+
+ time_t now = time(NULL);
+ char result[128];
+ struct tm dateFormat;
+ strftime(result, sizeof(result), "%a, %d %b %Y %H:%M:%S GMT",
+ gmtime_r(&now, &dateFormat));
+ fTimeStamp = result;
+ return *this;
+}
+
+//______________________________________________________________________________
+TString TS3HTTPRequest::MakeRequestLine(TS3HTTPRequest::EHTTPVerb httpVerb) const
+{
+ // Returns the first line of a HTTP request for this object. Note that since
+ // we don't use the virtual host syntax which is supported by Amazon, we
+ // must include the bucket name in thr resource. For example, we don't use
+ // http://mybucket.s3.amazonaws.com/path/to/my/file but instead
+ // http://s3.amazonaws.com/mybucket/path/to/my/file so the HTTP request
+ // will be of the form "GET /mybucket/path/to/my/file HTTP/1.1"
+ // Also note that the path must include the leading '/'.
+
+ return TString::Format("%s /%s%s HTTP/1.1",
+ (const char*)HTTPVerbToTString(httpVerb),
+ (const char*)fBucket,
+ (const char*)fObjectKey);
+}
+
+//______________________________________________________________________________
+TString TS3HTTPRequest::MakeHostHeader() const
+{
+ // Returns the 'Host' header to include in the HTTP request.
+
+ return "Host: " + fHost;
+}
+
+//______________________________________________________________________________
+TString TS3HTTPRequest::MakeDateHeader() const
+{
+ // Returns the date header for this HTTP request
+
+ return "Date: " + fTimeStamp;
+}
+
+//______________________________________________________________________________
+TString TS3HTTPRequest::MakeAuthPrefix() const
+{
+ // Returns the authentication prefix
+
+ switch (fAuthType) {
+ case kNoAuth: return "";
+ case kGoogle: return "GOOG1";
+ case kAmazon:
+ default: return "AWS";
+ }
+}
+
+//______________________________________________________________________________
+TString TS3HTTPRequest::MakeAuthHeader(TS3HTTPRequest::EHTTPVerb httpVerb) const
+{
+ // Returns the authentication header for this HTTP request
+
+ if (fAuthType == kNoAuth)
+ return "";
+
+ return TString::Format("Authorization: %s %s:%s%s",
+ (const char*)MakeAuthPrefix(),
+ (const char*)fAccessKey,
+ (const char*)ComputeSignature(httpVerb),
+ (fAuthType == kGoogle) ? "\r\nx-goog-api-version: 1" : "");
+}
+
+//______________________________________________________________________________
+TString TS3HTTPRequest::GetRequest(TS3HTTPRequest::EHTTPVerb httpVerb, Bool_t appendCRLF)
+{
+ // Returns the HTTP request ready to be sent to the server
+
+ // Set time stamp before computing this request's signature. The signature
+ // includes the date.
+ SetTimeStamp();
+ TString request = TString::Format("%s\r\n%s\r\n%s\r\n",
+ (const char*)MakeRequestLine(httpVerb),
+ (const char*)MakeHostHeader(),
+ (const char*)MakeDateHeader());
+ TString authHeader = MakeAuthHeader(httpVerb);
+ if (!authHeader.IsNull())
+ request += authHeader + "\r\n";
+ if (appendCRLF)
+ request += "\r\n";
+ return request;
+}
diff --git a/net/net/src/TS3WebFile.cxx b/net/net/src/TS3WebFile.cxx
index 83697c2d310..718520bc5a4 100644
--- a/net/net/src/TS3WebFile.cxx
+++ b/net/net/src/TS3WebFile.cxx
@@ -43,7 +43,7 @@
// this class for details on the syntax. //
// //
// For generating and signing the HTTP request, this class uses //
-// THTTPMessage. //
+// TS3HTTPRequest. //
// //
// For more information on the details of S3 protocol please refer to: //
// "Amazon Simple Storage Service Developer Guide": //
@@ -55,7 +55,6 @@
#include "TS3WebFile.h"
#include "TROOT.h"
-#include "THTTPMessage.h"
#include "TError.h"
#include "TSystem.h"
#include "TPRegexp.h"
@@ -64,8 +63,6 @@
ClassImp(TS3WebFile)
-const TString TS3WebFile::fgAuthPrefix = "AWS";
-
//_____________________________________________________________________________
TS3WebFile::TS3WebFile(const char* path, Option_t* options)
: TWebFile(path, "IO")
@@ -78,23 +75,38 @@ TS3WebFile::TS3WebFile(const char* path, Option_t* options)
// s3https://host.example.com/bucket/path/to/my/file
// as3://host.example.com/bucket/path/to/my/file
//
- // The 'as3' schema is accepted for backwards compatibility but its usage is
+ // For files hosted by Google Storage, use the following forms:
+ //
+ // gs://storage.googleapis.com/bucket/path/to/my/file
+ // gshttp://storage.googleapis.com/bucket/path/to/my/file
+ // gsthttps://storage.googleapis.com/bucket/path/to/my/file
+ //
+ // The 'as3' scheme is accepted for backwards compatibility but its usage is
// deprecated.
//
// The recommended way to create an instance of this class is through
// TFile::Open, for instance:
//
- // TFile* f = TFile::Open("s3://host.example.com/bucket/path/to/my/file")
+ // TFile* f1 = TFile::Open("s3://host.example.com/bucket/path/to/my/file")
+ // TFile* f2 = TFile::Open("gs://storage.googleapis.com/bucket/path/to/my/file")
//
- // The specified schema (i.e. s3, s3http or s3https) determines the underlying
- // protocol to use for downloading the file contents, namely HTTP or HTTPS. The
- // 's3' and 's3https' schemas imply using HTTPS as the transport protocol.
- // The 's3http' and 'as3' schema imply using HTTP as the transport protocol.
+ // The specified scheme (i.e. s3, s3http, s3https, ...) determines the underlying
+ // transport protocol to use for downloading the file contents, namely HTTP or HTTPS.
+ // The 's3', 's3https', 'gs' and 'gshttps' schemes imply using HTTPS as the transport
+ // protocol. The 's3http', 'as3' and 'gshttp' schemes imply using HTTP as the transport
+ // protocol.
//
// The 'options' argument can contain 'NOPROXY' if you want to bypass
- // the HTTP proxy when retrieving this file's contents. In addition, you can
- // provide the access key and secret key to be used for authentication purposes
- // for this file by using a string of the form "AUTH=myAccessKey:mySecretkey".
+ // the HTTP proxy when retrieving this file's contents. As for any TWebFile-derived
+ // object, the URL of the web proxy can be specified by setting an environmental
+ // variable 'http_proxy'. If this variable is set, we ask that proxy to route our
+ // requests HTTP(S) requests to the file server.
+ //
+ // In addition, you can also use the 'options' argument to provide the access key
+ // and secret key to be used for authentication purposes for this file by using a
+ // string of the form "AUTH=myAccessKey:mySecretkey". This may be useful to
+ // open several files hosted by different providers in the same program/macro,
+ // where the environemntal variables solution is not convenient (see below).
//
// If you need to specify both NOPROXY and AUTH separate them by ' '
// (blank), for instance:
@@ -111,17 +123,25 @@ TS3WebFile::TS3WebFile(const char* path, Option_t* options)
// S3_ACCESS_KEY and S3_SECRET_KEY (if set) are expected to contain
// the access key id and the secret access key, respectively. You have
// been provided with these credentials by your S3 service provider.
+ //
+ // If neither the AUTH information is provided in the 'options' argument
+ // nor the environmental variables are set, we try to open the file
+ // without providing any authentication information to the server. This
+ // is useful when the file is set an access control that allows for
+ // any unidentified user to read the file.
- // Make sure this is a valid S3 path. We accept 'as3' as a schema, for
+ // Make sure this is a valid S3 path. We accept 'as3' as a scheme, for
// backwards compatibility
Bool_t doMakeZombie = kFALSE;
TString errorMsg;
- TPMERegexp rex("^(s3|s3http[s]?|as3){1}://([^/]+)/([^/]+)/([^/].*)", "i");
+ TString accessKey;
+ TString secretKey;
+ TPMERegexp rex("^([a]?s3|s3http[s]?|gs|gshttp[s]?){1}://([^/]+)/([^/]+)/([^/].*)", "i");
if (rex.Match(TString(path)) != 5) {
errorMsg = TString::Format("invalid S3 path '%s'", path);
doMakeZombie = kTRUE;
}
- else if (!ParseOptions(options)) {
+ else if (!ParseOptions(options, accessKey, secretKey)) {
errorMsg = TString::Format("could not parse options '%s'", options);
doMakeZombie = kTRUE;
}
@@ -136,34 +156,42 @@ TS3WebFile::TS3WebFile(const char* path, Option_t* options)
// Set this S3 object's URL, the bucket name this file is located in
// and the object key
- fBucket = rex[3]; // bucket name
- fObjectKey.Form("/%s", (const char*)rex[4]); // include '/' as the starting character in object key
-
+ fS3Request.SetBucket(rex[3]);
+ fS3Request.SetObjectKey(TString::Format("/%s", (const char*)rex[4]));
+
// Initialize super-classes data members (fUrl is a data member of
// super-super class TFile)
TString protocol = "https";
- if (rex[1].EqualTo("http", TString::kIgnoreCase) ||
+ if (rex[1].EndsWith("http", TString::kIgnoreCase) ||
rex[1].EqualTo("as3", TString::kIgnoreCase))
protocol = "http";
fUrl.SetUrl(TString::Format("%s://%s/%s/%s", (const char*)protocol,
- (const char*)rex[2],
- (const char*)rex[3],
- (const char*)rex[4]));
+ (const char*)rex[2], (const char*)rex[3], (const char*)rex[4]));
// Set S3-specific data members. If the access and secret keys are not
// provided in the 'options' argument we look in the environmental
// variables.
- if (fAccessKey.IsNull()) {
- const char* kAccessKeyEnv = "S3_ACCESS_KEY";
- const char* kSecretKeyEnv = "S3_SECRET_KEY";
- if (!SetCredentialsFromEnv(kAccessKeyEnv, kSecretKeyEnv)) {
- Error("TS3WebFile", "could not find authentication info in "\
- "'options' argument and at least one of the environment variables '%s' or '%s' is not set",
- kAccessKeyEnv, kSecretKeyEnv);
- MakeZombie();
- gDirectory = gROOT;
- return;
- }
+ const char* kAccessKeyEnv = "S3_ACCESS_KEY";
+ const char* kSecretKeyEnv = "S3_SECRET_KEY";
+ if (accessKey.IsNull())
+ GetCredentialsFromEnv(kAccessKeyEnv, kSecretKeyEnv, accessKey, secretKey);
+
+ // Initialize the S3 HTTP request
+ fS3Request.SetHost(fUrl.GetHost());
+ if (accessKey.IsNull() || secretKey.IsNull()) {
+ // We have no authentication information, neither in the options
+ // nor in the enviromental variables. So may be this is a
+ // world-readable file, so let's continue and see if
+ // we can open it.
+ fS3Request.SetAuthType(TS3HTTPRequest::kNoAuth);
+ } else {
+ // Set the authentication information we need to use
+ // for this file
+ fS3Request.SetAuthKeys(accessKey, secretKey);
+ if (rex[1].BeginsWith("gs"))
+ fS3Request.SetAuthType(TS3HTTPRequest::kGoogle);
+ else
+ fS3Request.SetAuthType(TS3HTTPRequest::kAmazon);
}
// Assume this server does not serve multi-range HTTP GET requests. We
@@ -173,11 +201,20 @@ TS3WebFile::TS3WebFile(const char* path, Option_t* options)
// Call super-class initializer
TWebFile::Init(kFALSE);
+
+ // Were there some errors opening this file?
+ if (IsZombie() && (accessKey.IsNull() || secretKey.IsNull())) {
+ // We could not open the file and we have no authentication information
+ // so inform the user so that he can check.
+ Error("TS3WebFile", "could not find authentication info in "\
+ "'options' argument and at least one of the environment variables '%s' or '%s' is not set",
+ kAccessKeyEnv, kSecretKeyEnv);
+ }
}
//_____________________________________________________________________________
-Bool_t TS3WebFile::ParseOptions(const TString& options)
+Bool_t TS3WebFile::ParseOptions(Option_t* options, TString& accessKey, TString& secretKey)
{
// Extracts the S3 authentication key pair (access key and secret key)
// from the options. The authentication credentials can be specified in
@@ -186,24 +223,27 @@ Bool_t TS3WebFile::ParseOptions(const TString& options)
// options, for instance "NOPROXY" for not using the HTTP proxy for
// accessing this file's contents.
// For instance:
- // "NOPROXY AUTH=F38XYZABCDeFgH4D0E1F:V+frt4re7J1euSNFnmaf8wwmI4AAAE7kzxZ/TTM+"
+ // "NOPROXY AUTH=F38XYZABCDeFgHiJkLm:V+frt4re7J1euSNFnmaf8wwmI401234E7kzxZ/TTM+"
- if (options.IsNull())
+ TString optStr = (const char*)options;
+ if (optStr.IsNull())
return kTRUE;
fNoProxy = kFALSE;
- if (options.Contains("NOPROXY", TString::kIgnoreCase))
+ if (optStr.Contains("NOPROXY", TString::kIgnoreCase))
fNoProxy = kTRUE;
CheckProxy();
// Look in the options string for the authentication information.
TPMERegexp rex("(^AUTH=|^.* AUTH=)([a-z0-9]+):([a-z0-9+/]+)[\\s]*.*$", "i");
- if (rex.Match(options) < 4) {
+ if (rex.Match(optStr) < 4) {
Error("ParseOptions", "expecting options of the form \"AUTH=myAccessKey:mySecretKey\"");
return kFALSE;
}
- fAccessKey = rex[2];
- fSecretKey = rex[3];
+ accessKey = rex[2];
+ secretKey = rex[3];
+ if (gDebug > 0)
+ Info("ParseOptions", "using authentication information from 'options' argument");
return kTRUE;
}
@@ -212,13 +252,9 @@ Bool_t TS3WebFile::ParseOptions(const TString& options)
Int_t TS3WebFile::GetHead()
{
// Overwrites TWebFile::GetHead() for retrieving the HTTP headers of this
- // file. Uses THTTPMessage to generate an HTTP HEAD request which includes
+ // file. Uses TS3HTTPRequest to generate an HTTP HEAD request which includes
// the authorization header expected by the S3 server.
-
- THTTPMessage s3head = THTTPMessage(kHEAD, fObjectKey, fBucket,
- fUrl.GetHost(), fgAuthPrefix,
- fAccessKey, fSecretKey);
- fMsgGetHead = s3head.GetRequest();
+ fMsgGetHead = fS3Request.GetRequest(TS3HTTPRequest::kHEAD);
return TWebFile::GetHead();
}
@@ -233,10 +269,7 @@ void TS3WebFile::SetMsgReadBuffer10(const char* redirectLocation, Bool_t tempRed
// key.
TWebFile::SetMsgReadBuffer10(redirectLocation, tempRedirect);
- THTTPMessage s3get = THTTPMessage(kGET, fObjectKey, fBucket,
- fUrl.GetHost(), fgAuthPrefix,
- fAccessKey, fSecretKey);
- fMsgReadBuffer10 = s3get.GetRequest(kFALSE) + "Range: bytes=";
+ fMsgReadBuffer10 = fS3Request.GetRequest(TS3HTTPRequest::kGET, kFALSE) + "Range: bytes=";
return;
}
@@ -257,12 +290,9 @@ Bool_t TS3WebFile::ReadBuffers(char* buf, Long64_t* pos, Int_t* len, Int_t nbuf)
// Send multiple GET requests with a single range of bytes
// Adapted from original version by Wang Lu
for (Int_t i=0, offset=0; i < nbuf; i++) {
- THTTPMessage s3get = THTTPMessage(kGET, fObjectKey, fBucket,
- fUrl.GetHost(), fgAuthPrefix,
- fAccessKey, fSecretKey);
- TString rangeHeader = TString::Format("Range: bytes=%lld-%lld\r\n\r\n",
- pos[i], pos[i] + len[i] - 1);
- TString s3Request = s3get.GetRequest(kFALSE) + rangeHeader;
+ TString rangeHeader = TString::Format("Range: bytes=%lld-%lld\r\n\r\n",
+ pos[i], pos[i] + len[i] - 1);
+ TString s3Request = fS3Request.GetRequest(TS3HTTPRequest::kGET, kFALSE) + rangeHeader;
if (GetFromWeb10(&buf[offset], len[i], s3Request) == -1)
return kTRUE;
offset += len[i];
@@ -296,30 +326,36 @@ void TS3WebFile::ProcessHttpHeader(const TString& headerLine)
fUseMultiRange = multirangeServers.Contains(serverId, TString::kIgnoreCase) ? kTRUE : kFALSE;
}
+
//_____________________________________________________________________________
-Bool_t TS3WebFile::SetCredentialsFromEnv(const char* accessKeyEnv, const char* secretKeyEnv)
+Bool_t TS3WebFile::GetCredentialsFromEnv(const char* accessKeyEnv, const char* secretKeyEnv,
+ TString& outAccessKey, TString& outSecretKey)
{
// Sets the access and secret keys from the environmental variables, if
// they are both set.
// Look first in the recommended environmental variables. Both variables
// must be set.
- TString accessKey = gSystem->Getenv(accessKeyEnv);
- TString secretKey = gSystem->Getenv(secretKeyEnv);
- if (!accessKey.IsNull() && !secretKey.IsNull()) {
- fAccessKey = accessKey;
- fSecretKey = secretKey;
+ TString accKey = gSystem->Getenv(accessKeyEnv);
+ TString secKey = gSystem->Getenv(secretKeyEnv);
+ if (!accKey.IsNull() && !secKey.IsNull()) {
+ outAccessKey = accKey;
+ outSecretKey = secKey;
+ if (gDebug > 0)
+ Info("GetCredentialsFromEnv", "using authentication information from environmental variables '%s' and '%s'",
+ accessKeyEnv, secretKeyEnv);
return kTRUE;
}
- // Look now in the legacy environmental variables
- accessKey = gSystem->Getenv("S3_ACCESS_ID"); // Legacy access key
- secretKey = gSystem->Getenv("S3_ACCESS_KEY"); // Legacy secret key
- if (!accessKey.IsNull() && !secretKey.IsNull()) {
+ // Look now in the legacy environmental variables, for keeping backwards
+ // compatibility.
+ accKey = gSystem->Getenv("S3_ACCESS_ID"); // Legacy access key
+ secKey = gSystem->Getenv("S3_ACCESS_KEY"); // Legacy secret key
+ if (!accKey.IsNull() && !secKey.IsNull()) {
Warning("SetAuthKeys", "usage of S3_ACCESS_ID and S3_ACCESS_KEY environmental variables is deprecated.");
Warning("SetAuthKeys", "please use S3_ACCESS_KEY and S3_SECRET_KEY environmental variables.");
- fAccessKey = accessKey;
- fSecretKey = secretKey;
+ outAccessKey = accKey;
+ outSecretKey = secKey;
return kTRUE;
}
diff --git a/net/net/src/TWebFile.cxx b/net/net/src/TWebFile.cxx
index 2ac4b917c02..bd812bc6355 100644
--- a/net/net/src/TWebFile.cxx
+++ b/net/net/src/TWebFile.cxx
@@ -354,6 +354,8 @@ void TWebFile::CheckProxy()
return;
}
fProxy = p;
+ if (gDebug > 0)
+ Info("CheckProxy", "using HTTP proxy %s", fProxy.GetUrl());
}
}
@@ -644,6 +646,9 @@ Int_t TWebFile::GetFromWeb10(char *buf, Int_t len, const TString &msg)
return -1;
}
+ if (gDebug > 0)
+ Info("GetFromWeb10", "sending HTTP request:\n%s", msg.Data());
+
if (fSocket->SendRaw(msg.Data(), msg.Length()) == -1) {
Error("GetFromWeb10", "error sending command to host %s", fUrl.GetHost());
return -1;
@@ -925,6 +930,11 @@ Int_t TWebFile::GetHead()
if (!s)
return -1;
+ if (gDebug > 0) {
+ Info("GetHead", "connected to host %s", connurl.GetHost());
+ Info("GetHead", "sending HTTP request:\n%s", msg.Data());
+ }
+
if (s->SendRaw(msg.Data(), msg.Length()) == -1) {
Error("GetHead", "error sending command to host %s", fUrl.GetHost());
delete s;
--
GitLab