diff --git a/Makefile b/Makefile index 1dd11680aca5f50a74b3884d35b2f18aeee84a63..ad610a0c4efa7a65458ca0e52844819f1c9f2106 100644 --- a/Makefile +++ b/Makefile @@ -174,6 +174,9 @@ endif ifeq ($(BUILDDCAP),yes) MODULES += io/dcache endif +ifeq ($(BUILDDAVIX),yes) +MODULES += net/davix +endif ifeq ($(BUILDGFAL),yes) MODULES += io/gfal endif @@ -298,12 +301,12 @@ ifneq ($(findstring $(MAKECMDGOALS),distclean maintainer-clean),) MODULES += core/unix core/winnt graf2d/x11 graf2d/x11ttf \ graf3d/gl graf3d/ftgl graf3d/glew io/rfio io/castor \ montecarlo/pythia6 montecarlo/pythia8 misc/table \ - sql/mysql sql/pgsql sql/sqlite sql/sapdb net/srputils graf3d/x3d \ + sql/mysql sql/pgsql sql/sqlite sql/sapdb net/srputils \ rootx net/rootd io/dcache io/chirp hist/hbook graf2d/asimage \ net/ldap net/krb5auth net/rpdutils net/globusauth \ bindings/pyroot bindings/ruby io/gfal misc/minicern \ graf2d/qt gui/qtroot gui/qtgsi net/netx net/alien \ - proof/proofd proof/proofx proof/pq2 \ + proof/proofd proof/proofx proof/pq2 graf3d/x3d net/davix \ sql/oracle io/xmlparser math/mathmore \ tmva math/genetic io/hdfs graf2d/fitsio roofit/roofitcore \ roofit/roofit roofit/roostats roofit/histfactory \ @@ -1339,6 +1342,7 @@ showbuild: @echo "OSTHREADLIB = $(OSTHREADLIB)" @echo "SHIFTLIB = $(SHIFTLIB)" @echo "DCAPLIB = $(DCAPLIB)" + @echo "DAVIXLIB = $(DAVIXLIB)" @echo "GFALLIB = $(GFALLIB)" @echo "MYSQLINCDIR = $(MYSQLINCDIR)" @echo "ORACLEINCDIR = $(ORACLEINCDIR)" diff --git a/build/unix/installDavix.sh b/build/unix/installDavix.sh new file mode 100755 index 0000000000000000000000000000000000000000..99bec49bbf1d01481e8b570ecb64198f5ec64992 --- /dev/null +++ b/build/unix/installDavix.sh @@ -0,0 +1,381 @@ +#!/bin/bash + +# +# Script to install a given version of Davix +# created from the xrootd similar script +# +# Syntax: +# ./installDavix.sh [<installdir>] [-h|--help] [-d|--debug] [-o|--optimized] +# [-v <version>|--version=<version>] +# [-t <tarball>|--tarball=<tarball>] +# [-b <where-to-build>|--builddir=<where-to-build>] +# [--dvxopts="<opts-to-davix>"] +# [--vers-subdir[=<version-root>]] [--no-vers-subdir] +# [-j <concurrent-build-jobs>|--jobs=<concurrent-build-jobs>] +# [-k|--keep] [--bzip2] +# +# See printhelp for a description of the options. +# + +printhelp() +{ + echo " " + echo " Script to install a given version of Davix" + echo " " + echo " Syntax:" + echo " ./installDavix.sh [<installdir>] [-h|--help] [-d|--debug] [-o|--optimized]" + echo " [-v <version>|--version=<version>]" + echo " [-t <tarball>|--tarball=<tarball>]" + echo " [-b <where-to-build>|--builddir=<where-to-build>]" + echo " [--dvxopts=\"<opts-to-davix>\"]" + echo " [-j <concurrent-build-jobs>|--jobs=<concurrent-build-jobs>]" + echo " [--vers-subdir[=<version-root>]] [--no-vers-subdir]" + echo " [-k|--keep] [--bzip2]" + echo " " + echo " where" + echo " <installdir>: the directory where the bin, lib, include/davix, share and" + echo " man directories will appear (see also --vers-subdir)" + echo " The default is ." + echo " -b <where-to-build>, --builddir=<where-to-build>" + echo " directory where to build; default /tmp/davix-<version>" + echo " -d,--debug build in debug mode (no optimization)" + echo " -h, --help print this help screen" + echo " -o,--optimized build in optimized mode without any debug symbol" + echo " -t <tarball>, --tarball=<tarball>" + echo " full local path to source tarball" + echo " -v <version>, --version=<version>" + echo " version in the form x.j.w[-hash-or-tag] ;" + echo " current default 0.2.7-3" + echo " --dvxopts=<opts-to-davix>" + echo " additional configuration options to davix" + echo " (see davix web site)" + echo " --no-vers-subdir install in <installdir> instead of <installdir>/davix-<version>" + echo " (or <installdir>/<version-root><version>, see --vers-subdir" + echo " --vers-subdir[=<version-root>]" + echo " install in <installdir>/<version-root><version> instead of" + echo " <installdir>/davix-<version> or <installdir>. Has priority" + echo " over --no-vers-subdir. Default <version-root>=davix-." + echo " This option is on by default." + echo " -j <jobs>, --jobs=<jobs>" + echo " number of build jobs to run simultaneously when bulding;" + echo " default is <number-of-cores> + 1." + echo " -k, --keep" + echo " keep the build directory" + echo " --bzip2" + echo " use bzip2 to manage the tarball (when extension is .b2z)" + echo " " + echo " When relevant, the script uses 'wget' ('curl' on MacOS X) to retrieve" + echo " the tarball" +} + +cleanbuilddir() +{ + if test ! "x$KEEP" = "xyes"; then + if test ! "x$BUILDDIR" = "x" && test -d $BUILDDIR ; then + rm -rf $BUILDDIR + fi + fi +} + +DBGOPT="-DCMAKE_BUILD_TYPE=RelWithDebInfo" +TGTDIR="." +VERS="" +TARBALL="" +BUILDDIR="" +XRDOPTS="" +VSUBDIR="davix-" +MAKEMJ="" +KEEP="" +UNZIP="gunzip" +TUNZIP="xzf" + +# +# Parse long options first +other_args= +short_opts= +is_short="no" +for i in $@ ; do + opt="" + case $i in + --*) opt=`echo "$i" | sed 's/--//'` ;; + -*) if test "x$short_opts" = "x" ; then + short_opts="$i" ; + else + short_opts="$short_opts $i" ; + fi; is_short="yes" ;; + *) if test "x$is_short" = "xyes" ; then + if test "x$short_opts" = "x" ; then + short_opts="$i" ; + else + short_opts="$short_opts $i" ; + fi; + is_short="no" + else + if test "x$other_args" = "x" ; then + other_args="$i"; + else + other_args="$other_args $i"; + fi; + fi; + esac + if test ! "x$opt" = "x" ; then + case "$opt" in + *=*) oarg=`echo "$opt" | sed 's/[-_a-zA-Z0-9]*=//'`;; + *) oarg= ;; + esac ; + case $opt in + builddir=*) BUILDDIR="$oarg" ;; + debug) DBGOPT="-DCMAKE_BUILD_TYPE=Debug" ;; + help) printhelp ; exit ;; + jobs) MAKEMJ="-j$OPTARG" ;; + no-vers-subdir) VSUBDIR="" ;; + optimized) DBGOPT="-DCMAKE_BUILD_TYPE=Release" ;; + tarball=*) TARBALL="$oarg" ;; + version=*) VERS="$oarg" ;; + vers-subdir) VSUBDIR="davix-" ;; + vers-subdir=*) VSUBDIR="$oarg" ;; + xrdopts=*) XRDOPTS="$oarg" ;; + keep) KEEP="yes" ;; + bzip2) UNZIP="bunzip2" ; TUNZIP="xjf" ;; + esac + fi +done + +if test ! "x$short_opts" = "x" ; then + while getopts b:j:t:v:dhok i $short_opts ; do + case $i in + b) BUILDDIR="$OPTARG" ;; + d) DBGOPT="-DCMAKE_BUILD_TYPE=Debug" ;; + h) printhelp ; exit ;; + j) MAKEMJ="-j$OPTARG" ;; + o) DBGOPT="-DCMAKE_BUILD_TYPE=Release" ;; + t) TARBALL="$OPTARG" ;; + v) VERS="$OPTARG" ;; + k) KEEP="yes" ;; + \?) printhelp; exit 1 ;; + esac + if test ! "x$OPTARG" = "x" ; then + noa= + for a in $other_args ; do + if test ! "x$OPTARG" = "x$a" ; then + if test "x$noa" = "x" ; then + noa="$a" + else + noa="$noa $a" + fi + fi + done + other_args=$noa + fi + done +fi + +# Fill empty fields with any non-prefixed argument +if test ! "x$other_args" = "x" ; then + TGTDIR="$other_args" +fi + +XMK=make + +WRKDIR=$PWD + +if test "x$TGTDIR" = "x" ; then + echo " Install dir undefined!" + printhelp + exit 1 +else + tgtd="$TGTDIR" + TGTDIR=`(cd $tgtd && pwd)` + if [ "$?" -ne "0" ]; then + echo "Install dir $tgtd does not exist, please create it first." + exit 1 + fi +fi + +if test "x$VERS" = "x" ; then + VERS="0.2.7" +fi +echo "Version: $VERS" + +if test ! "x$VSUBDIR" = "x" ; then + TGTDIR="$TGTDIR/$VSUBDIR$VERS" +fi +echo "Installing in: $TGTDIR" + +retrieve="yes" +if test ! "x$TARBALL" = "x" && test -f $TARBALL ; then + retrieve="no" + TGTBALL=$TARBALL +fi +if test "x$retrieve" = "xyes" ; then + if test "x$TARBALL" = "x" ; then + TARBALL="http://grid-deployment.web.cern.ch/grid-deployment/dms/lcgutil/tar/davix/davix-$VERS.tar.gz" + TGTBALL="davix-$VERS.tar.gz" + else + TGTBALL=`basename $TARBALL` + fi +fi +if test "x$retrieve" = "xyes" ; then + echo "Retrieving source from tarball $TARBALL" +else + echo "Building tarball $TARBALL" +fi + +# Build dir +if test "x$BUILDDIR" = "x"; then + BUILDDIR="/tmp/davix-$VERS-$RANDOM" +fi +if test ! -d $BUILDDIR ; then + mkdir -p $BUILDDIR + if test ! -d $BUILDDIR ; then + echo "Could not create build dir $BUILDDIR, exiting..." + exit 1 + fi +else + # Builddir already exists, exit + echo "Build dir $BUILDDIR already exists, exiting..." + exit 1 +fi +echo "Build dir: $BUILDDIR" + +cd $BUILDDIR + +# Retrieving source +ARCH=`uname -s` +if test "x$retrieve" = "xyes" ; then + if test "x$ARCH" = "xDarwin" ; then + curl $TARBALL -o $TGTBALL + else + wget $TARBALL -O $TGTBALL + fi + if test ! -f $TGTBALL ; then + echo "Tarball retrieval failed!" + cd $WRKDIR + cleanbuilddir + exit 1 + fi +fi + +# Untar tarball +if test "x$ARCH" = "xSunOS" ; then + XMK="gmake" + $UNZIP -c $TGTBALL > "$TGTBALL.tar" + tar xf "$TGTBALL.tar" + rm -f "$TGTBALL.tar" +else + tar $TUNZIP $TGTBALL +fi +if test ! -d davix-$VERS ; then + echo "Could not find source sub-directory davix-$VERS" + cd $WRKDIR + cleanbuilddir + exit 1 +fi +cd davix-$VERS + +# CMake or old {make,configure} ? +if test -f CMakeLists.txt ; then + + # CMake: check if there + XCMK=`which cmake 2> /dev/null` + echo "XCMK = '$XCMK'" + if test "x$XCMK" = "x" || test ! -f $XCMK ; then + echo " " + echo "To build davix cmake is required: " + echo "you can get it from http://cmake.org/cmake/resources/software.html" + echo "or from the software manager of your system" + echo " " + cd $WRKDIR + cleanbuilddir + exit 1 + fi + + # Check that we can build this version + #if test ! -r VERSION_INFO ; then + # echo "VERSION_INFO file not found: this davix version is probably too old and cannot be built by this script" + # cd $WRKDIR + # cleanbuilddir + # exit 1 + #fi + + # Create build directory + mkdir build + cd build + + # Configure + $XCMK -DCMAKE_INSTALL_PREFIX=$TGTDIR $DBGOPT $XRDOPTS .. + + # Get the '-j' setting if not specified + if test "x$MAKEMJ" = "x" ; then + MJ=`grep -c bogomips /proc/cpuinfo 2> /dev/null` + [ "$?" != 0 ] && MJ=`sysctl hw.ncpu | cut -b10 2> /dev/null` + let MJ++ + MAKEMJ="-j$MJ" + fi + + # Build + $XMK $MAKEMJ + if [ "$?" != "0" ] ; then + echo "Problems running $XMK $MAKEMJ ..." + cd $WRKDIR + cleanbuilddir + exit "$?" + fi + + # Install + $XMK install + if [ "$?" != "0" ] ; then + echo "Problems running $XMK install ..." + cd $WRKDIR + cleanbuilddir + exit "$?" + fi + +else + + # Old {configure,make} + + # Check that we can build this version + if test ! -r configure.classic ; then + echo "configure.classic file not found: this davix version cannot be built by this script" + cd $WRKDIR + cleanbuilddir + exit 1 + fi + + # Configure options + if test "x$DBGOPT" = "xRelease" ; then + DBGOPT="" + else + DBGOPT="--build=debug" + fi + CFGOPT="--disable-krb4 --no-arch-subdirs --disable-mon --enable-krb5" + + # Configure + ./configure.classic --prefix=$TGTDIR $DBGOPT $CFGOPT $XRDOPTS + if [ "$?" != "0" ] ; then + echo "Problems running configure.classic ..." + cd $WRKDIR + cleanbuilddir + exit "$?" + fi + + # Make + $XMK + if [ "$?" != "0" ] ; then + echo "Problems running $XMK ..." + cd $WRKDIR + cleanbuilddir + exit "$?" + fi + + # Install + $XMK install + +fi + +# Go back where we started +cd $WRKDIR +cleanbuilddir + +exit diff --git a/cmake/modules/RootConfiguration.cmake b/cmake/modules/RootConfiguration.cmake index 6b066a328a1b7701354a06cc8340b145e843d14f..44f2764bed6798e8867d80b4c1ea4dac1537ffcd 100644 --- a/cmake/modules/RootConfiguration.cmake +++ b/cmake/modules/RootConfiguration.cmake @@ -130,6 +130,12 @@ set(castorlib ${CASTOR_LIBRARY}) set(castorincdir ${CASTOR_INCLUDE_DIR}) set(castorcflags) + +set(builddavix ${value${davix}}) +set(davixlibdir ${DAVIX_LIBRARY_DIR}) +set(davixlib ${DAVIX_LIBRARY}) +set(davixincdir ${DAVIX_INCLUDE_DIR}) + set(builddcap ${value${dcap}}) set(dcaplibdir ${DCAP_LIBRARY_DIR}) set(dcaplib ${DCAP_LIBRARY}) diff --git a/config/Makefile.depend b/config/Makefile.depend index dc678856055735aab0a6905a4a5b5fad11fcf86e..45e40900652e74952c708e2fc343ee8cbb188071 100644 --- a/config/Makefile.depend +++ b/config/Makefile.depend @@ -53,6 +53,7 @@ PHYSICSLIBDEPM = $(MATRIXLIB) $(MATHCORELIB) PYTHIA6LIBDEPM = $(EGLIB) $(GRAFLIB) $(VMCLIB) $(PHYSICSLIB) PYTHIA8LIBDEPM = $(EGLIB) $(GRAFLIB) $(VMCLIB) $(PHYSICSLIB) XMLLIBDEPM = $(IOLIB) +DAVIXLIBDEPM = $(NETLIB) $(IOLIB) $(THREADLIB) TABLELIBDEPM = $(TREELIB) $(GPADLIB) $(G3DLIB) $(GRAFLIB) $(HISTLIB) \ $(IOLIB) $(MATHCORELIB) MLPLIBDEPM = $(HISTLIB) $(MATRIXLIB) $(TREELIB) $(GRAFLIB) \ @@ -198,6 +199,7 @@ GQTLIBDEP = $(GQTLIBDEPM) RUBYLIBDEP = $(RUBYLIBDEPM) GUIBLDLIBDEP = $(GUIBLDLIBDEPM) XMLLIBDEP = $(XMLLIBDEPM) +DAVIXLIBDEP = $(DAVIXLIBDEPM) FOAMLIBDEP = $(FOAMLIBDEPM) ALIENLIBDEP = $(ALIENLIBDEPM) ROOFITCORELIBDEP = $(ROOFITCORELIBDEPM) @@ -343,6 +345,7 @@ RUBYLIBEXTRA = lib/libHist.lib lib/libMathCore.lib GUIBLDLIBEXTRA = lib/libGui.lib lib/libGraf.lib lib/libMathCore.lib GUIHTMLLIBEXTRA = lib/libGui.lib lib/libGraf.lib lib/libNet.lib XMLLIBEXTRA = lib/libRIO.lib +DAVIXLIBEXTRA = lib/libNet.lib lib/libRIO.lib lib/libThread.lib FOAMLIBEXTRA = lib/libHist.lib lib/libMathCore.lib ALIENLIBEXTRA = lib/libXMLIO.lib lib/libNetx.lib lib/libTree.lib \ lib/libProof.lib lib/libProofPlayer.lib \ @@ -485,6 +488,7 @@ RUBYLIBEXTRA = -Llib -lHist -lMathCore GUIBLDLIBEXTRA = -Llib -lGui -lGraf -lMathCore GUIHTMLLIBEXTRA = -Llib -lGui -lGraf -lNet XMLLIBEXTRA = -Llib -lRIO +DAVIXLIBEXTRA = -Llib -lNet -lRIO -lThread FOAMLIBEXTRA = -Llib -lHist -lMathCore ALIENLIBEXTRA = -Llib -lXMLIO -lNetx -lTree -lProof -lProofPlayer \ -lNet -lRIO diff --git a/config/Makefile.in b/config/Makefile.in index 197936763cf6f4265f3ef0b758179516a7ffcabb..23150e667cee63abe29885c2ebe3822008e63239 100644 --- a/config/Makefile.in +++ b/config/Makefile.in @@ -192,6 +192,11 @@ CASTORCLILIB := @castorlib@ CASTORINCDIR := $(filter-out /usr/include, @castorincdir@) CASTORCFLAGS := @castorcflags@ +BUILDDAVIX := @builddavix@ +DAVIXLIBDIR := @davixlibdir@ +DAVIXCLILIB := @davixlib@ +DAVIXINCDIR := $(filter-out /usr/include, @davixincdir@) + BUILDDCAP := @builddcap@ DCAPLIBDIR := @dcaplibdir@ DCAPLIB := @dcaplib@ diff --git a/config/rootrc.in b/config/rootrc.in index 9581de693ad23f9b651566c72be2fdb16c8fdd85..2de893900faddaea5fb0ca0ff3be3197bfcadf0f 100644 --- a/config/rootrc.in +++ b/config/rootrc.in @@ -636,3 +636,43 @@ XNet.ParStreamsPerPhyConn: 0 # This overrides the default specified above for a generic application. # Color 5 is yellow. Rint.Canvas.HighLightColor: 5 + + +# Parameters that influence the behavior of TDavixFile/TDavixSystem. These +# classes give a comprehensive client side support for HTTP and WebDAV, +# with all the goodies (session caching, flexible authentication, support +# for different dialects like S3, and others) + +# Verbosity level of the external Davix library +# Davix.Debug: 0 + +# Path to the X.509 user proxy +# Davix.GSI.UserProxy: /my/path/my_proxy + +# Paths to the X.509 user certificate and key, in PEM format +# Davix.GSI.UserCert: /my/path/my_cert +# Davix.GSI.UserKey: /my/path/my_key + +# Path to the directory containing the CAs +# Davix.GSI.CAdir: /etc/grid-security/certificates + +# Enable/disable the Check on the peer's certificate +# Davix.GSI.CACheck: y + +# Sets the HTTP client in Grid mode, to talk to Grid HTTP SEs +# without having to set additional parameters +# Grid mode enable the following action: +# - Auto-detect and use the available voms proxy certificate +# - Add the standard grid security CAs as valid CAs +# - Use the grid specific extensions if needed. +# Davix.GSI.GridMode: y + +# Sets the Amazon S3 authentication parameters +# Davix.S3.SecretKey: secret +# Davix.S3.AccessKey: token + +# NOTE: The authentication of TDavixFile/TDavixSystem can be influenced +# through some well known environment variables: +# X509_USER_CERT, X509_USER_KEY, X509_USER_PROXY +# They have the usual meaning for the X509 Grid things. The gEnv vars +# have higher priority over the these envvars. diff --git a/configure b/configure index e271c42300947f9c8a0ace1264f7127a80531f2a..845f8cd154104855c6ccdeadf4606c8062797b30 100755 --- a/configure +++ b/configure @@ -60,6 +60,7 @@ options=" \ enable_castor \ enable_chirp \ enable_cocoa \ + enable_davix \ enable_dcache \ enable_explicitlink \ enable_fink \ @@ -239,6 +240,7 @@ TABLE \ XMLDIR \ CASROOT \ VC \ +DAVIX \ " configenvvars= for c in $envvars ; do @@ -1623,6 +1625,7 @@ enable/disable options, prefix with either --enable- or --disable- castor CASTOR support, requires libshift from CASTOR >= 1.5.2 chirp Chirp filesystem support, requires cctools >= 3.2.2 cocoa Use native Cocoa/Quartz graphics backend (MacOS X only) + davix DavIx library for HTTP/WEBDAV access dcache dCache support, requires libdcap from DESY explicitlink Explicitly link with all dependent libraries fink Use libraries from an extra MacOS X package manager (Fink, MacPorts, ...) (default: enabled) @@ -1700,6 +1703,8 @@ with options, prefix with --with-, enables corresponding support chirp-incdir Chirp support, location of chirp_reli.h chirp-libdir Chirp support, location of libchirp_client cppunit Cppunit testing, directory of the cppunit installation + davix-incdir DavIx support, location of davix.hpp + davix-libdir DavIx support, location of libdavix dcap-incdir dCache support, location of dcap.h dcap-libdir dCache support, location of libdcap dnssd-libdir Bonjour support, location of libdns_sd @@ -2071,6 +2076,8 @@ if test $# -gt 0 ; then --with-chirp-incdir=*) chirpincdir=$optarg ; enable_chirp="yes" ;; --with-chirp-libdir=*) chirplibdir=$optarg ; enable_chirp="yes" ;; --with-cppunit=*) cppunit=$optarg ;; + --with-davix-incdir=*) davixincdir=$optarg ; enable_davix="yes" ;; + --with-davix-libdir=*) davixlibdir=$optarg ; enable_davix="yes" ;; --with-dcap-incdir=*) dcapincdir=$optarg ; enable_dcache="yes" ;; --with-dcap-libdir=*) dcaplibdir=$optarg ; enable_dcache="yes" ;; --with-dnssd-incdir=*) dnssdincdir=$optarg ; enable_bonjour="yes" ;; @@ -4175,6 +4182,40 @@ fi check_explicit "$enable_dcache" "$enable_dcache_explicit" \ "Explicitly required dCache dependencies not fulfilled" +###################################################################### +# +### echo %%% DavIx Support - Third party libraries +# +if test ! "x$enable_davix" = "xno" ; then + check_header "davix.hpp" "$davixincdir" \ + $DAVIX ${DAVIX:+$DAVIX/include/davix} \ + /usr/local/include/davix \ + /usr/include/davix + davixincdir=$found_dir + + check_library "libdavix" "$enable_shared" "$davixlibdir" \ + $DAVIX ${DAVIX:+$DAVIX/lib} ${DAVIX:+$DAVIX/lib64} \ + /usr/lib /usr/lib64 /usr/local/lib64 + davixlib=$found_lib + davixlibdir=$found_dir + + if test ! "x$davixlib" = "x" ; then + check_symbol davix_logger "$davixlib" "$davixlibdir" + if test $found_symbol -eq 0 ; then + result "### Need davix_posix_stat in libdavix" + davixlib= + davixlibdir= + davixincdir= + fi + fi + + if test "x$davixincdir" = "x" || test "x$davixlib" = "x"; then + enable_davix="no" + fi +fi +check_explicit "$enable_davix" "$enable_davix_explicit" \ + "Explicitly required DavIx dependencies not fulfilled" + ###################################################################### # ### echo %%% Chirp Support - Third party libraries @@ -6601,6 +6642,7 @@ if test "x$show_pkglist" = "xyes" ; then test "x$enable_castor" = "xyes" && pl="$pl root-plugin-io-castor" test "x$enable_chirp" = "xyes" && pl="$pl root-plugin-io-chirp" test "x$enable_dcache" = "xyes" && pl="$pl root-plugin-io-dcache" + test "x$enable_davix" = "xyes" && pl="$pl root-plugin-io-davix" test "x$enable_gfal" = "xyes" && pl="$pl root-plugin-io-gfal" test "x$enable_hdfs" = "xyes" && pl="$pl root-plugin-io-hdfs" test "x$enable_rfio" = "xyes" && pl="$pl root-plugin-io-rfio" @@ -7070,11 +7112,14 @@ sed -e "s|@aclocaldir@|$aclocaldir|" \ -e "s|@chirplibdir@|$chirplibdir|" \ -e "s|@cppunit@|$cppunit|" \ -e "s|@datadir@|$datadir|" \ + -e "s|@davixincdir@|$davixincdir|" \ -e "s|@dcapincdir@|$dcapincdir|" \ -e "s|@comerrlib@|$comerrlib|" \ -e "s|@comerrlibdir@|$comerrlibdir|" \ -e "s|@cryptolib@|$cryptolib|" \ -e "s|@cryptolibdir@|$cryptolibdir|" \ + -e "s|@davixlib@|$davixlib|" \ + -e "s|@davixlibdir@|$davixlibdir|" \ -e "s|@dcaplib@|$dcaplib|" \ -e "s|@dcaplibdir@|$dcaplibdir|" \ -e "s|@docdir@|$docdir|" \ @@ -7276,6 +7321,7 @@ sed -e "s|@srcdir@|$ac_srcdir|" \ -e "s|@buildrfio@|$enable_rfio|" \ -e "s|@buildcastor@|$enable_castor|" \ -e "s|@builddcap@|$enable_dcache|" \ + -e "s|@builddavix@|$enable_davix|" \ -e "s|@buildfftw3@|$enable_fftw3|" \ -e "s|@buildgviz@|$enable_gviz|" \ -e "s|@buildgfal@|$enable_gfal|" \ diff --git a/etc/plugins/TFile/P130_TDavixFile.C b/etc/plugins/TFile/P130_TDavixFile.C new file mode 100644 index 0000000000000000000000000000000000000000..c86432d5dcfe85d01a9b24b1412fdf3246292fa7 --- /dev/null +++ b/etc/plugins/TFile/P130_TDavixFile.C @@ -0,0 +1,20 @@ +void P130_TDavixFile() +{ + TString configfeatures = gROOT->GetConfigFeatures(); + + // only if ROOT was compiled with davix enabled we configure a handler + if ( configfeatures.Contains("davix") ) { + + gPluginMgr->AddHandler("TFile", "^http[s]?:", "TDavixFile", + "RDAVIX", "TDavixFile(const char*, Option_t *, const char *,Int_t)"); + + gPluginMgr->AddHandler("TFile", "^dav[s]?:", "TDavixFile", + "RDAVIX", "TDavixFile(const char*, Option_t *, const char *,Int_t)"); + + gPluginMgr->AddHandler("TFile", "^s3[s]?:", "TDavixFile", + "RDAVIX", "TDavixFile(const char*, Option_t *, const char *,Int_t)"); + + } else { + Error("P130_TDavixFile","Please fix your ROOT config to be able to load libdavix.so"); + } +} diff --git a/etc/plugins/TSystem/P045_TDavixSystem.C b/etc/plugins/TSystem/P045_TDavixSystem.C new file mode 100644 index 0000000000000000000000000000000000000000..b8712f50f9e378d6fe843b71a1fa95577d1f5db9 --- /dev/null +++ b/etc/plugins/TSystem/P045_TDavixSystem.C @@ -0,0 +1,20 @@ +void P045_TDavixSystem() +{ + TString configfeatures = gROOT->GetConfigFeatures(); + + // only if ROOT was compiled with davix enabled we configure a handler + if ( configfeatures.Contains("davix") ) { + + gPluginMgr->AddHandler("TSystem", "^http[s]?:", "TDavixSystem", + "RDAVIX", "TDavixSystem()"); + + gPluginMgr->AddHandler("TSystem", "^dav[s]?:", "TDavixSystem", + "RDAVIX", "TDavixSystem()"); + + gPluginMgr->AddHandler("TSystem", "^s3[s]?:", "TDavixSystem", + "RDAVIX", "TDavixSystem()"); + + } else { + Error("P045_TDavixSystem","Please fix your ROOT config to be able to load libdavix.so"); + } +} diff --git a/etc/system.plugins-ios b/etc/system.plugins-ios index bfc8d94421a06ba64daa5c15f231577dbd3ef612..eb754d3e9e50c74f02d120945e72d00584e14a60 100644 --- a/etc/system.plugins-ios +++ b/etc/system.plugins-ios @@ -29,7 +29,7 @@ Plugin.TBrowserImp: TRootBrowser TRootBrowser Gui "NewBrowser(TBrowser *, const Plugin.TChain: ^proof TProofChain Proof "TProofChain(TChain *, Bool_t)" Plugin.TDataProgressDialog: * TDataProgressDialog PeacGui "TDataProgressDialog(TProof*,const char*,Int_t,Long64_t)" Plugin.TDataSetManager: ^file TDataSetManagerFile Proof "TDataSetManagerFile(const char *,const char *,const char *)" -Plugin.TFile: ^http: TWebFile Net "TWebFile(const char*,Option_t*)" +Plugin.TFile: ^http: TWebFile RDAVIX "TDavixFile(const char*,Option_t*)" +Plugin.TFile: ^rfio: TRFIOFile RFIO "TRFIOFile(const char*,Option_t*,const char*,Int_t)" +Plugin.TFile: ^castor: TCastorFile RCastor "TCastorFile(const char*,Option_t*,const char*,Int_t,Int_t)" +Plugin.TFile: ^dcache: TDCacheFile DCache "TDCacheFile(const char*,Option_t*,const char*,Int_t)" @@ -99,7 +99,7 @@ Plugin.TSystem: ^rfio: TRFIOSystem RFIO "TRFIOSystem()" +Plugin.TSystem: ^gsidcap: TDCacheSystem DCache "TDCacheSystem()" +Plugin.TSystem: ^alien: TAlienSystem RAliEn "TAlienSystem()" +Plugin.TSystem: ^root: TXNetSystem Netx "TXNetSystem(const char *,Bool_t)" -+Plugin.TSystem: ^http: TWebSystem Net "TWebSystem()" ++Plugin.TSystem: ^http: TWebSystem RDAVIX "TDavixSystem()" +Plugin.TSystem: ^hdfs: THDFSSystem HDFS "THDFSSystem()" +Plugin.TSystem: ^rootd: TNetSystem Net "TNetSystem(const char *,Bool_t)" +Plugin.TSystem: ^rootup: TNetSystem Net "TNetSystem(const char *,Bool_t)" diff --git a/net/CMakeLists.txt b/net/CMakeLists.txt index 0ae71021e61711427722bd918b08a7e3cdcf9fff..6aab949b9a299e5fb6a44c912bcd56baa747cb5f 100644 --- a/net/CMakeLists.txt +++ b/net/CMakeLists.txt @@ -30,4 +30,8 @@ endif() if(globus) add_subdirectory(globusauth) -endif() \ No newline at end of file +endif() + +if(davix) + add_subdirectory(davix) +endif() diff --git a/net/davix/Module.mk b/net/davix/Module.mk new file mode 100644 index 0000000000000000000000000000000000000000..03a7d2d23c1b9e8b4a39b25bd6043e9f58f163db --- /dev/null +++ b/net/davix/Module.mk @@ -0,0 +1,73 @@ +# Module.mk for DAVIX module +# +# Author: Tigran Mkrtchyan <tigran.mkrtchyan@desy.de> + +MODNAME := davix +MODDIR := $(ROOT_SRCDIR)/net/$(MODNAME) +MODDIRS := $(MODDIR)/src +MODDIRI := $(MODDIR)/inc + +DAVIXDIR := $(MODDIR) +DAVIXDIRS := $(DAVIXDIR)/src +DAVIXDIRI := $(DAVIXDIR)/inc + +##### libRDAVIX ##### +DAVIXL := $(MODDIRI)/LinkDef.h +DAVIXDS := $(call stripsrc,$(MODDIRS)/G__DAVIX.cxx) +DAVIXDO := $(DAVIXDS:.cxx=.o) +DAVIXDH := $(DAVIXDS:.cxx=.h) + +DAVIXH := $(filter-out $(MODDIRI)/LinkDef%,$(wildcard $(MODDIRI)/*.h)) +DAVIXS := $(filter-out $(MODDIRS)/G__%,$(wildcard $(MODDIRS)/*.cxx)) +DAVIXO := $(call stripsrc,$(DAVIXS:.cxx=.o)) + +DAVIXDEP := $(DAVIXO:.o=.d) $(DAVIXDO:.o=.d) + +DAVIXLIB := $(LPATH)/libRDAVIX.$(SOEXT) +DAVIXMAP := $(DAVIXLIB:.$(SOEXT)=.rootmap) + +# used in the main Makefile +ALLHDRS += $(patsubst $(MODDIRI)/%.h,include/%.h,$(DAVIXH)) +ALLLIBS += $(DAVIXLIB) +ALLMAPS += $(DAVIXMAP) + +# include all dependency files +INCLUDEFILES += $(DAVIXDEP) + +##### local rules ##### +.PHONY: all-$(MODNAME) clean-$(MODNAME) distclean-$(MODNAME) + +include/%.h: $(DAVIXDIRI)/%.h + cp $< $@ + +$(DAVIXLIB): $(DAVIXO) $(DAVIXDO) $(ORDER_) $(MAINLIBS) $(DAVIXLIBDEP) + @$(MAKELIB) $(PLATFORM) $(LD) "$(LDFLAGS)" \ + "$(SOFLAGS)" libRDAVIX.$(SOEXT) $@ "$(DAVIXO) $(DAVIXDO)" \ + "$(DAVIXLIBEXTRA) $(DAVIXLIBDIR) $(DAVIXCLILIB)" + +$(call pcmrule,DAVIX) + $(noop) + +$(DAVIXDS): $(DAVIXH) $(DAVIXL) $(ROOTCINTTMPDEP) $(call pcmdep,DAVIX) + $(MAKEDIR) + @echo "Generating dictionary $@..." + $(ROOTCINTTMP) -f $@ $(call dictModule,DAVIX) -c $(DAVIXH) $(DAVIXL) + +$(DAVIXMAP): $(RLIBMAP) $(MAKEFILEDEP) $(DAVIXL) + $(RLIBMAP) -o $@ -l $(DAVIXLIB) \ + -d $(DAVIXLIBDEPM) -c $(DAVIXL) + +all-$(MODNAME): $(DAVIXLIB) $(DAVIXMAP) + +clean-$(MODNAME): + @rm -f $(DAVIXO) $(DAVIXDO) + +clean:: clean-$(MODNAME) + +distclean-$(MODNAME): clean-$(MODNAME) + @rm -f $(DAVIXDEP) $(DAVIXDS) $(DAVIXDH) $(DAVIXLIB) $(DAVIXMAP) + +distclean:: distclean-$(MODNAME) + +##### extra rules ###### +$(DAVIXO) $(DAVIXDO): CXXFLAGS += $(DAVIXINCDIR:%=-I%) diff --git a/net/davix/inc/LinkDef.h b/net/davix/inc/LinkDef.h new file mode 100644 index 0000000000000000000000000000000000000000..0aab1b4e841d9b84ddb85ac79759dce89cce0dfd --- /dev/null +++ b/net/davix/inc/LinkDef.h @@ -0,0 +1,10 @@ +#ifdef __CINT__ + +#pragma link off all globals; +#pragma link off all classes; +#pragma link off all functions; + +#pragma link C++ class TDavixFile+; +#pragma link C++ class TDavixSystem+; + +#endif diff --git a/net/davix/inc/TDavixFile.h b/net/davix/inc/TDavixFile.h new file mode 100644 index 0000000000000000000000000000000000000000..12a5e8099a70f61b3b94c92689aeb5c1907b4799 --- /dev/null +++ b/net/davix/inc/TDavixFile.h @@ -0,0 +1,111 @@ +#ifndef ROOT_TDavixFile +#define ROOT_TDavixFile + + + + +////////////////////////////////////////////////////////////////////////// +// // +// TDavixFile // +// // +// A TDavixFile is like a normal TFile except that it uses // +// libdavix to read/write remote files. // +// It supports HTTP and HTTPS in a number of dialects and options // +// e.g. S3 is one of them // +// Other caracteristics come from the full support of Davix, // +// e.g. full redirection support in any circumstance // +// // +// Authors: Adrien Devresse (CERN IT/SDC) // +// Tigran Mkrtchyan (DESY) // +// // +// Checks, refactoring and ROOT5 porting: // +// Fabrizio Furano (CERN IT/SDC) // +// // +// September 2013 // +// // +////////////////////////////////////////////////////////////////////////// + +// +// Parameters that influence the behavior of TDavixFile/TDavixSystem. The names should be self-explanatory +// +//Davix.Debug +//Davix.GSI.UserProxy +//Davix.GSI.UserCert +//Davix.GSI.UserKey + +//Davix.GSI.CAdir +//Davix.GSI.CACheck +//Davix.GSI.GridMode +// +// Environment variables: +// X509_USER_CERT, X509_USER_KEY, X509_USER_PROXY ... usual meaning for the X509 Grid things. gEnv vars have higher priority. + +#include "TFile.h" +#include "TUrl.h" +#include "TSystem.h" +#include "TMutex.h" + +class TDavixFileInternal; +struct Davix_fd; + + + + + +class TDavixFile : public TFile { +private: + TDavixFileInternal* d_ptr; + + void Init(Bool_t init); + Long64_t DavixReadBuffer(Davix_fd *fd, char *buf, Int_t len); + Long64_t DavixPReadBuffer(Davix_fd *fd, char *buf, Long64_t pos, Int_t len); + Long64_t DavixReadBuffers(Davix_fd *fd, char *buf, Long64_t *pos, Int_t *len, Int_t nbuf); + Long64_t DavixWriteBuffer(Davix_fd *fd, const char *buf, Int_t len); + Int_t DavixStat(struct stat *st) const; + + // perfStats + Double_t eventStart(); + void eventStop(Double_t t, Long64_t len); + +public: + /// + /// Open function for TDavixFile + /// + /// TDavixFile supports several options : + /// + /// - GRID_MODE=yes : enable the grid authentication and CA support + /// - CA_CHECK=no : remove all the certificate authority check, this option can create a security vulnerability + /// - S3_SECKEY=string : Amazon S3 secret token + /// - S3_ACCKEY=string : Amazon S3 access token + /// + /// Several parameters can be used if separated with whitespace + + TDavixFile(const char* url, Option_t *option="", const char *ftitle="", Int_t compress=1); + + ~TDavixFile(); + + // TFile interface. + virtual Long64_t GetSize() const; + virtual void Seek(Long64_t offset, ERelativeTo pos = kBeg); + virtual Bool_t ReadBuffer(char *buf, Int_t len); + virtual Bool_t ReadBuffer(char *buf, Long64_t pos, Int_t len); + virtual Bool_t ReadBuffers(char *buf, Long64_t *pos, Int_t *len, Int_t nbuf); + virtual Bool_t ReadBufferAsync(Long64_t offs, Int_t len); + virtual Bool_t WriteBuffer(const char *buffer, Int_t bufferLength); + + + // TDavixFile options + /// Enable or disable certificate authority check + void setCACheck(Bool_t check); + + /// Enable the grid mode + /// The grid Mode configure automatically all grid-CA path, VOMS authentication + /// and grid related extension for a grid analysis usage + void enableGridMode(); + + ClassDef(TDavixFile, 0) +}; + + + +#endif diff --git a/net/davix/inc/TDavixFileInternal.h b/net/davix/inc/TDavixFileInternal.h new file mode 100644 index 0000000000000000000000000000000000000000..f9e406319d06ae2c4a04e2ad9f38537aa35f4eb6 --- /dev/null +++ b/net/davix/inc/TDavixFileInternal.h @@ -0,0 +1,134 @@ + +#ifndef TDAVIXFILEINTERNAL_H +#define TDAVIXFILEINTERNAL_H + +#include "TUrl.h" +#include "TMutex.h" + +#include <vector> +#include <iterator> +#include <algorithm> +#include <errno.h> +#include <stdlib.h> +#include <unistd.h> +#include <fcntl.h> +#include <sys/stat.h> +#include <sstream> +#include <string> +#include <cstring> + + + +////////////////////////////////////////////////////////////////////////// +// // +// TDavixFileInternal // +// // +// // +// Support class, common to TDavixFile and TDavixSystem // +// // +// Authors: Adrien Devresse (CERN IT/SDC) // +// Fabrizio Furano (CERN IT/SDC) // +// // +// September 2013 // +// // +////////////////////////////////////////////////////////////////////////// + +namespace Davix { +class Context; +class RequestParams; +class DavPosix; +} + + +class TDavixFileInternal { + friend class TDavixFile; + friend class TDavixSystem; + + TDavixFileInternal(const TUrl & mUrl, Option_t* mopt) : + positionLock(), + openLock(), + davixContext(getDavixInstance()), + davixParam(NULL), + davixPosix(NULL), + davixFd(NULL), + fUrl(mUrl), + opt(mopt), + oflags(0), + dirdVec(){ + + } + + TDavixFileInternal(const char* url, Option_t* mopt) : + positionLock(), + openLock(), + davixContext(getDavixInstance()), + davixParam(NULL), + davixPosix(NULL), + davixFd(NULL), + fUrl(url), + opt(mopt), + oflags(0), + dirdVec(){ + + } + + ~TDavixFileInternal(); + + inline Davix_fd * getDavixFileInstance() { + // singleton init + if (davixFd == NULL) { + TLockGuard l(&(openLock)); + if (davixFd == NULL) { + davixFd = this->Open(); + } + } + return davixFd; + } + + Davix_fd * Open(); + + void Close(); + + void enableGridMode(); + + void setS3Auth(const std::string & key, const std::string & token); + + void parseConfig(); + + void parseParams(Option_t* option); + + void init(); + + bool isMyDird(void* fd); + + void addDird(void* fd); + + void removeDird(void* fd); + + + + TMutex positionLock; + TMutex openLock; + + // DAVIX + Davix::Context *davixContext; + Davix::RequestParams *davixParam; + Davix::DavPosix *davixPosix; + Davix_fd *davixFd; + TUrl fUrl; + Option_t* opt; + int oflags; + std::vector<void*> dirdVec; + +public: + + Int_t DavixStat(const char *url, struct stat *st); + + static Davix::Context* getDavixInstance(); + +}; + + + +#endif /* TDAVIXFILEINTERNAL_H */ + diff --git a/net/davix/inc/TDavixSystem.h b/net/davix/inc/TDavixSystem.h new file mode 100644 index 0000000000000000000000000000000000000000..f990e8961f5a8cf34701b1b3735beadd9dff81d6 --- /dev/null +++ b/net/davix/inc/TDavixSystem.h @@ -0,0 +1,60 @@ + +#ifndef TDAVIXSYSTEM_H +#define TDAVIXSYSTEM_H + + +////////////////////////////////////////////////////////////////////////// +// // +// TDavixSystem // +// // +// A TSystem specialization for HTTP and WebDAV // +// It supports HTTP and HTTPS in a number of dialects and options // +// e.g. S3 is one of them // +// Other caracteristics come from the full support of Davix, // +// e.g. full redirection support in any circumstance // +// // +// Authors: Adrien Devresse (CERN IT/SDC) // +// Fabrizio Furano (CERN IT/SDC) // +// // +// September 2013 // +// // +////////////////////////////////////////////////////////////////////////// + + + +#include "TUrl.h" +#include "TSystem.h" +#include "TMutex.h" + + +class TDavixFileInternal; +struct Davix_fd; + + +class TDavixSystem : public TSystem { +private: + TDavixFileInternal* d_ptr; + +public: + + TDavixSystem(); + TDavixSystem(const char *url); + + virtual ~TDavixSystem(); + + virtual void FreeDirectory(void *dirp); + virtual const char *GetDirEntry(void *dirp); + virtual Bool_t ConsistentWith(const char *path, void *dirptr); + + virtual Int_t GetPathInfo(const char* path, FileStat_t &buf); + virtual Bool_t IsPathLocal(const char *path); + virtual Int_t Locate(const char* path, TString &endurl); + virtual Int_t MakeDirectory(const char* dir); + virtual void *OpenDirectory(const char* dir); + virtual int Unlink(const char *path); + + ClassDef(TDavixSystem, 0); +}; + +#endif /* TDAVIXSYSTEM_H */ + diff --git a/net/davix/src/TDavixFile.cxx b/net/davix/src/TDavixFile.cxx new file mode 100644 index 0000000000000000000000000000000000000000..5507c6bdbad5574820dacdbb40198cb0c03813b8 --- /dev/null +++ b/net/davix/src/TDavixFile.cxx @@ -0,0 +1,651 @@ +/************************************************************************* + * Copyright (C) 2013, Tigran Mkrtchyan <tigran.mkrtchyan@desy.de>. * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + + +////////////////////////////////////////////////////////////////////////// +// // +// TDavixFile // +// // +// A TDavixFile is like a normal TFile except that it uses // +// libdavix to read/write remote files. // +// It supports HTTP and HTTPS in a number of dialects and options // +// e.g. S3 is one of them // +// Other caracteristics come from the full support of Davix, // +// e.g. full redirection support in any circumstance // +// // +// Authors: Adrien Devresse (CERN IT/SDC) // +// Tigran Mkrtchyan (DESY) // +// // +// Checks and ROOT5 porting: // +// Fabrizio Furano (CERN IT/SDC) // +// // +// September 2013 // +// // +////////////////////////////////////////////////////////////////////////// + + +#include "TDavixFile.h" +#include "TROOT.h" +#include "TSocket.h" +#include "Bytes.h" +#include "TError.h" +#include "TSystem.h" +#include "TEnv.h" +#include "TBase64.h" +#include "TVirtualPerfStats.h" +#include "TDavixFileInternal.h" +#include "TSocket.h" + +#include <errno.h> +#include <stdlib.h> +#include <unistd.h> +#include <fcntl.h> +#include <davix.hpp> +#include <sstream> +#include <string> +#include <cstring> + + +static const std::string VERSION = "0.0.1"; + +static const std::string gUserAgent = "ROOT/" + std::string(gROOT->GetVersion()) + +" TDavixFile/" + VERSION + " davix/" + Davix::version(); + +// The prefix that is used to find the variables in the gEnv +#define ENVPFX "Davix." + +ClassImp(TDavixFile) + +using namespace Davix; + +const char* grid_mode_opt = "grid_mode=yes"; +const char* ca_check_opt = "ca_check=no"; +const char* s3_seckey_opt = "s3seckey="; +const char* s3_acckey_opt = "s3acckey="; +const char* open_mode_read = "READ"; +const char* open_mode_create = "CREATE"; +const char* open_mode_new = "NEW"; +const char* open_mode_update = "UPDATE"; + +static TMutex createLock; +static Context* davix_context_s = NULL; + + +bool isno(const char *str) { + if (!str) return false; + + if (!strcmp(str, "n") || !strcmp(str, "no") || !strcmp(str, "0") || !strcmp(str, "false")) return true; + + return false; + +} + +int configure_open_flag(const std::string & str, int old_flag){ + if( strcasecmp(str.c_str(), open_mode_read) == 0) + old_flag |= O_RDONLY; + if( (strcasecmp(str.c_str(), open_mode_create) == 0) + || (strcasecmp(str.c_str(), open_mode_new) == 0)){ + old_flag |= (O_CREAT|O_WRONLY|O_TRUNC); + } + if( (strcasecmp(str.c_str(), open_mode_update) == 0)){ + old_flag |= (O_RDWR); + } + return old_flag; +} + + +//____________________________________________________________________________ + +static void ConfigureDavixLogLevel() { + Int_t log_level = (gEnv) ? gEnv->GetValue("Davix.Debug", 0) : 0; + + switch (log_level) { + case 0: + davix_set_log_level(0); + break; + case 1: + davix_set_log_level(DAVIX_LOG_WARNING); + break; + case 2: + davix_set_log_level(DAVIX_LOG_VERBOSE); + break; + case 3: + davix_set_log_level(DAVIX_LOG_DEBUG); + break; + default: + davix_set_log_level(DAVIX_LOG_ALL); + break; + } +} + +/////////////////////////////////////////////////////////////////// +// Authn implementation, Locate and get VOMS cred if exist + +static void TDavixFile_http_get_ucert(std::string& ucert, std::string& ukey) { + char default_proxy[64]; + const char *genvvar = 0, *genvvar1 = 0; + // The gEnv has higher priority, let's look for a proxy cert + genvvar = gEnv->GetValue("Davix.GSI.UserProxy", (const char*) NULL); + if (genvvar) { + ucert = ukey = genvvar; + if (gDebug > 0) + Info("TDavixFile_http_get_ucert", "Found proxy in gEnv"); + return; + } + + // Try explicit environment for proxy + if (getenv("X509_USER_PROXY")) { + if (gDebug > 0) + Info("TDavixFile_http_get_ucert", "Found proxy in X509_USER_PROXY"); + ucert = ukey = getenv("X509_USER_PROXY"); + return; + } + + // Try with default location + snprintf(default_proxy, sizeof (default_proxy), "/tmp/x509up_u%d", + geteuid()); + + if (access(default_proxy, R_OK) == 0) { + if (gDebug > 0) + Info("TDavixFile_http_get_ucert", "Found proxy in /tmp"); + ucert = ukey = default_proxy; + return; + } + + // It seems we got no proxy, let's try to gather the keys + genvvar = gEnv->GetValue("Davix.GSI.UserCert", (const char*) NULL); + genvvar1 = gEnv->GetValue("Davix.GSI.UserKey", (const char*) NULL); + if (genvvar || genvvar1) { + if (gDebug > 0) + Info("TDavixFile_http_get_ucert", "Found cert and key in gEnv"); + + ucert = genvvar; + ukey = genvvar1; + return; + } + + // try with X509_* environment + if (getenv("X509_USER_CERT")) + ucert = getenv("X509_USER_CERT"); + if (getenv("X509_USER_KEY")) + ukey = getenv("X509_USER_KEY"); + + if ((ucert.size() > 0) || (ukey.size() > 0)) { + if (gDebug > 0) + Info("TDavixFile_http_get_ucert", "Found cert and key in gEnv"); } + return; + +} + +static int TDavixFile_http_authn_cert_X509(void* userdata, const Davix::SessionInfo & info, + Davix::X509Credential * cert, Davix::DavixError** err) { + (void) userdata; // keep quiete compilation warnings + (void) info; + std::string ucert, ukey; + TDavixFile_http_get_ucert(ucert, ukey); + + if (ucert.empty() || ukey.empty()) { + Davix::DavixError::setupError(err, "TDavixFile", + Davix::StatusCode::AuthentificationError, + "Could not set the user's proxy or certificate"); + return -1; + } + return cert->loadFromFilePEM(ukey, ucert, "", err); +} +///////////////////////////////////////////////////////////////////////////////////////////// + + +TDavixFileInternal::~TDavixFileInternal() { + delete davixPosix; + delete davixParam; +} + +Context* TDavixFileInternal::getDavixInstance(){ + if(davix_context_s == NULL){ + TLockGuard guard(&createLock); + if(davix_context_s == NULL){ + davix_context_s = new Context(); + } + } + return davix_context_s; +} + +Davix_fd * TDavixFileInternal::Open() { + DavixError *davixErr = NULL; + Davix_fd *fd = davixPosix->open(davixParam, fUrl.GetUrl(), oflags, &davixErr); + if (fd == NULL) { + Error("DavixOpen", "failed to open file with davix: %s (%d)", + davixErr->getErrMsg().c_str(), davixErr->getStatus()); + DavixError::clearError(&davixErr); + }else{ + // setup ROOT style read + davixPosix->fadvise(fd, 0, 300, Davix::AdviseRandom); + } + + return fd; +} + +void TDavixFileInternal::Close() { + DavixError *davixErr = NULL; + if (davixFd != NULL && davixPosix->close(davixFd, &davixErr)) { + Error("DavixClose", "failed to close file with davix: %s (%d)", + davixErr->getErrMsg().c_str(), davixErr->getStatus()); + DavixError::clearError(&davixErr); + } +} + +void TDavixFileInternal::enableGridMode() { + const char * env_var = NULL; + + if (gDebug > 1) + Info("enableGridMode", " grid mode enabled !"); + + env_var = gEnv->GetValue("Davix.GSI.CAdir", (const char*) NULL); + if (!env_var){ + env_var = (((char*) getenv("X509_CERT_DIR")) ? env_var : "/etc/grid-security/certificates/"); + davixParam->addCertificateAuthorityPath(env_var); + if (gDebug > 0) + Info("enableGridMode", "Setting CAdir to %s", env_var); + } + +} + +void TDavixFileInternal::setS3Auth(const std::string & key, const std::string & token) { + if (gDebug > 1) + Info("setS3Auth", " Aws S3 tokens configured"); + davixParam->setAwsAuthorizationKeys(key, token); + davixParam->setProtocol(RequestProtocol::AwsS3); +} + +void TDavixFileInternal::parseConfig(){ + const char * env_var = NULL, *env_var2 = NULL; + // default opts + davixParam->setTransparentRedirectionSupport(true); + davixParam->setClientCertCallbackX509(&TDavixFile_http_authn_cert_X509, NULL); + + // setup CADIR + env_var = gEnv->GetValue("Davix.GSI.CAdir", (const char*) NULL); + if (env_var) { + davixParam->addCertificateAuthorityPath(env_var); + if (gDebug > 0) + Info("parseConfig", "Add Setting CA dir: %s", env_var); + } + // CA Check + bool ca_check_local = !isno( gEnv->GetValue("Davix.GSI.CACheck", (const char*)"y") ); + davixParam->setSSLCAcheck(ca_check_local); + if (gDebug > 0) + Info("parseConfig", "Setting CAcheck to %s",((ca_check_local)?("true"):("false"))); + + // S3 Auth + if( ((env_var = gEnv->GetValue("Davix.S3.SecretKey", getenv("S3_SECRET_KEY"))) != NULL) + && ((env_var2 = gEnv->GetValue("Davix.S3.AccessKey", getenv("S3_ACCESS_KEY"))) != NULL) ){ + Info("parseConfig", "Setting S3 SecretKey and AccessKey. Access Key : %s ", env_var2); + davixParam->setAwsAuthorizationKeys(env_var, env_var2); + } + + env_var = gEnv->GetValue("Davix.GSI.GridMode", (const char*) NULL); + if (env_var && !isno(env_var)) + enableGridMode(); +} + +void TDavixFileInternal::parseParams(Option_t* option) { + // intput params + std::stringstream ss(option); + std::string item; + std::vector<std::string> parsed_options; + // parameters + std::string s3seckey, s3acckey; + + while (std::getline(ss, item, ' ')) { + parsed_options.push_back(item); + } + + for (std::vector<std::string>::iterator it = parsed_options.begin(); it < parsed_options.end(); ++it) { + // grid mode option + if ((strcasecmp(it->c_str(), grid_mode_opt)) == 0) { + enableGridMode(); + } + // ca check option + if ((strcasecmp(it->c_str(), ca_check_opt)) == 0) { + davixParam->setSSLCAcheck(false); + } + // s3 sec key + if (strncasecmp(it->c_str(), s3_seckey_opt, strlen(s3_seckey_opt)) == 0) { + s3seckey = std::string(it->c_str() + strlen(s3_seckey_opt)); + } + // s3 access key + if (strncasecmp(it->c_str(), s3_acckey_opt, strlen(s3_acckey_opt)) == 0) { + s3acckey = std::string(it->c_str() + strlen(s3_acckey_opt)); + } + // open mods + oflags= configure_open_flag(*it, oflags); + } + + if (s3seckey.size() > 0) { + setS3Auth(s3seckey, s3acckey); + } + + if(oflags == 0) // default open mode + oflags = O_RDONLY; +} + +void TDavixFileInternal::init(){ + davixPosix = new DavPosix(davixContext); + davixParam = new RequestParams(); + davixParam->setUserAgent(gUserAgent); + ConfigureDavixLogLevel(); + parseConfig(); + parseParams(opt); +} + + + + + +Int_t TDavixFileInternal::DavixStat(const char *url, struct stat *st) { + DavixError *davixErr = NULL; + + if (davixPosix->stat(davixParam, url, st, &davixErr) < 0) { + + Error("DavixStat", "failed to stat the file with davix: %s (%d)", + davixErr->getErrMsg().c_str(), davixErr->getStatus()); + DavixError::clearError(&davixErr); + return 0; + } + return 1; +} + + + +///////////////////////////////////////////////////////////////////////////////////////////// + + + +//______________________________________________________________________________ + +TDavixFile::TDavixFile(const char* url, Option_t *opt, const char *ftitle, Int_t compress) : TFile(url, "WEB"), +d_ptr(new TDavixFileInternal(fUrl, opt)) { + (void) ftitle; + (void) compress; + Init(kFALSE); +} + + +//______________________________________________________________________________ + +TDavixFile::~TDavixFile() { + d_ptr->Close(); + delete d_ptr; +} + + +//______________________________________________________________________________ + +void TDavixFile::Init(Bool_t init) { + (void) init; + d_ptr->init(); + TFile::Init(kFALSE); + fOffset = 0; + fD = -2; // so TFile::IsOpen() will return true when in TFile::~TFi */ +} + + +//______________________________________________________________________________ + +void TDavixFile::Seek(Long64_t offset, ERelativeTo pos) { + // Set position from where to start reading. + + TLockGuard guard(&(d_ptr->positionLock)); + switch (pos) { + case kBeg: + fOffset = offset + fArchiveOffset; + break; + case kCur: + fOffset += offset; + break; + case kEnd: + // this option is not used currently in the ROOT code + if (fArchiveOffset) + Error("Seek", "seeking from end in archive is not (yet) supported"); + fOffset = fEND - offset; // is fEND really EOF or logical EOF? + break; + } + + if (gDebug > 1) + Info("Seek", " move cursor to %lld" + , fOffset); +} + +//______________________________________________________________________________ + +Bool_t TDavixFile::ReadBuffer(char *buf, Int_t len) { + // Read specified byte range from remote file via HTTP. + // Returns kTRUE in case of error. + TLockGuard guard(&(d_ptr->positionLock)); + Davix_fd *fd; + if ((fd = d_ptr->getDavixFileInstance()) == NULL) + return kTRUE; + Long64_t ret = DavixReadBuffer(fd, buf, len); + if (ret < 0) + return kTRUE; + + if (gDebug > 1) + Info("ReadBuffer", "%lld bytes of data read sequentially" + " (%d requested)", ret, len); + + return kFALSE; +} + +//______________________________________________________________________________ + +Bool_t TDavixFile::ReadBuffer(char *buf, Long64_t pos, Int_t len) { + + Davix_fd *fd; + if ((fd = d_ptr->getDavixFileInstance()) == NULL) + return kTRUE; + + Long64_t ret = DavixPReadBuffer(fd, buf, pos, len); + if (ret < 0) + return kTRUE; + + + if (gDebug > 1) + Info("ReadBuffer", "%lld bytes of data read from offset" + " %lld (%d requested)", ret, pos, len); + return kFALSE; +} + +Bool_t TDavixFile::ReadBufferAsync(Long64_t offs, Int_t len){ + + Davix_fd *fd; + if ((fd = d_ptr->getDavixFileInstance()) == NULL) + return kFALSE; + + d_ptr->davixPosix->fadvise(fd, static_cast<dav_off_t>(offs), static_cast<dav_size_t>(len), Davix::AdviseRandom); + + if (gDebug > 1) + Info("ReadBufferAsync", "%d bytes of data prefected from offset" + " %lld ", len, offs); + return kFALSE; +} + +//______________________________________________________________________________ + +Bool_t TDavixFile::ReadBuffers(char *buf, Long64_t *pos, Int_t *len, Int_t nbuf) { + Davix_fd *fd; + if ((fd = d_ptr->getDavixFileInstance()) == NULL) + return kTRUE; + + Long64_t ret = DavixReadBuffers(fd, buf, pos, len, nbuf); + if (ret < 0) + return kTRUE; + + if (gDebug > 1) + Info("ReadBuffers", "%lld bytes of data read from a list of %d buffers", + ret, nbuf); + + return kFALSE; +} + +//_____________________________________________________________________________ +Bool_t TDavixFile::WriteBuffer(const char *buf, Int_t len) +{ + + Davix_fd *fd; + if ((fd = d_ptr->getDavixFileInstance()) == NULL) + return kTRUE; + + Long64_t ret = DavixWriteBuffer(fd, buf, len); + if (ret < 0) + return kTRUE; + + if (gDebug > 1) + Info("WriteBuffer", "%lld bytes of data write" + " %d requested", ret, len); + return kFALSE; +} + +void TDavixFile::setCACheck(Bool_t check) { + d_ptr->davixParam->setSSLCAcheck((bool)check); +} + +void TDavixFile::enableGridMode() { + d_ptr->enableGridMode(); +} + +bool TDavixFileInternal::isMyDird(void* fd){ + TLockGuard l(&(openLock)); + std::vector<void*>::iterator f = std::find(dirdVec.begin(), dirdVec.end(), fd); + return (f != dirdVec.end()); +} + +void TDavixFileInternal::addDird(void* fd){ + TLockGuard l(&(openLock)); + dirdVec.push_back(fd); +} + +void TDavixFileInternal::removeDird(void* fd){ + TLockGuard l(&(openLock)); + std::vector<void*>::iterator f = std::find(dirdVec.begin(), dirdVec.end(), fd); + if(f != dirdVec.end()) + dirdVec.erase(f); +} + + +//______________________________________________________________________________ + +Long64_t TDavixFile::GetSize() const { + struct stat st; + Int_t ret = d_ptr->DavixStat(fUrl.GetUrl(), &st); + if (ret) { + if (gDebug > 1) + Info("GetSize", "file size requested: %ld", + st.st_size); + return st.st_size; + } + return -1; +} + +//______________________________________________________________________________ + +Double_t TDavixFile::eventStart() { + if (gPerfStats) + return TTimeStamp(); + return 0; +} + +//______________________________________________________________________________ + +void TDavixFile::eventStop(Double_t t_start, Long64_t len) { + // set TFile state info + fBytesRead += len; + fReadCalls += 1; + + if (gPerfStats) + gPerfStats->FileReadEvent(this, (Int_t) len, t_start); +} + +Long64_t TDavixFile::DavixReadBuffer(Davix_fd *fd, char *buf, Int_t len) { + DavixError *davixErr = NULL; + Double_t start_time = eventStart(); + + Long64_t ret = d_ptr->davixPosix->pread(fd, buf, len, fOffset, &davixErr); + if (ret < 0) { + Error("DavixReadBuffer", "failed to read data with davix: %s (%d)", + davixErr->getErrMsg().c_str(), davixErr->getStatus()); + DavixError::clearError(&davixErr); + } else { + fOffset += ret; + eventStop(start_time, ret); + } + + return ret; +} + +Long64_t TDavixFile::DavixWriteBuffer(Davix_fd *fd, const char *buf, Int_t len) { + DavixError *davixErr = NULL; + Double_t start_time = eventStart(); + + Long64_t ret = d_ptr->davixPosix->pwrite(fd, buf, len, fOffset, &davixErr); + if (ret < 0) { + Error("DavixWriteBuffer", "failed to write data with davix: %s (%d)", + davixErr->getErrMsg().c_str(), davixErr->getStatus()); + DavixError::clearError(&davixErr); + } else { + fOffset += ret; + eventStop(start_time, ret); + } + + return ret; +} + +Long64_t TDavixFile::DavixPReadBuffer(Davix_fd *fd, char *buf, Long64_t pos, Int_t len) { + DavixError *davixErr = NULL; + Double_t start_time = eventStart(); + + Long64_t ret = d_ptr->davixPosix->pread(fd, buf, len, pos, &davixErr); + if (ret < 0) { + Error("DavixPReadBuffer", "failed to read data with davix: %s (%d)", + davixErr->getErrMsg().c_str(), davixErr->getStatus()); + DavixError::clearError(&davixErr); + } else { + eventStop(start_time, ret); + } + + + return ret; +} + +Long64_t TDavixFile::DavixReadBuffers(Davix_fd *fd, char *buf, Long64_t *pos, Int_t *len, Int_t nbuf) { + DavixError *davixErr = NULL; + Double_t start_time = eventStart(); + DavIOVecInput in[nbuf]; + DavIOVecOuput out[nbuf]; + + int lastPos = 0; + for (Int_t i = 0; i < nbuf; ++i) { + in[i].diov_buffer = &buf[lastPos]; + in[i].diov_offset = pos[i]; + in[i].diov_size = len[i]; + lastPos += len[i]; + } + + Long64_t ret = d_ptr->davixPosix->preadVec(fd, in, out, nbuf, &davixErr); + if (ret < 0) { + Error("DavixReadBuffers", "failed to read data with davix: %s (%d)", + davixErr->getErrMsg().c_str(), davixErr->getStatus()); + DavixError::clearError(&davixErr); + } else { + eventStop(start_time, ret); + } + + return ret; +} + + + diff --git a/net/davix/src/TDavixSystem.cxx b/net/davix/src/TDavixSystem.cxx new file mode 100644 index 0000000000000000000000000000000000000000..18dde9c498a206071adfdc9061ad0b1082672172 --- /dev/null +++ b/net/davix/src/TDavixSystem.cxx @@ -0,0 +1,182 @@ + + +////////////////////////////////////////////////////////////////////////// +// // +// TDavixSystem // +// // +// A TSystem specialization for HTTP and WebDAV // +// It supports HTTP and HTTPS in a number of dialects and options // +// e.g. S3 is one of them // +// Other caracteristics come from the full support of Davix, // +// e.g. full redirection support in any circumstance // +// // +// Authors: Adrien Devresse (CERN IT/SDC) // +// Fabrizio Furano (CERN IT/SDC) // +// // +// September 2013 // +// // +////////////////////////////////////////////////////////////////////////// + + + +#include "TDavixSystem.h" +#include "TROOT.h" +#include "TSocket.h" +#include "Bytes.h" +#include "TError.h" +#include "TSystem.h" +#include "TEnv.h" +#include "TBase64.h" +#include "TVirtualPerfStats.h" +#include "TDavixFileInternal.h" +#include "TSocket.h" + +#include <errno.h> +#include <stdlib.h> +#include <unistd.h> +#include <fcntl.h> +#include <davix.hpp> +#include <sstream> +#include <string> +#include <cstring> + + +extern const std::string VERSION; +extern const std::string gUserAgent; + +// The prefix that is used to find the variables in the gEnv +#define ENVPFX "Davix." + +ClassImp(TDavixSystem) + +using namespace Davix; + +extern const char* grid_mode_opt; +extern const char* ca_check_opt; +extern const char* s3_seckey_opt; +extern const char* s3_acckey_opt; + +TDavixSystem::TDavixSystem(const char *url) : + TSystem(url), + d_ptr(new TDavixFileInternal(url, "WEB")) + { + d_ptr->init(); + SetTitle("WebDAV system administration"); +} + +TDavixSystem::TDavixSystem() : + TSystem(), + d_ptr(new TDavixFileInternal("", "WEB")) + { + d_ptr->init(); + SetTitle("WebDAV system administration"); +} + +TDavixSystem::~TDavixSystem() { + SafeDelete(d_ptr); +} + +void TDavixSystem::FreeDirectory(void *dirp) { + d_ptr->davixPosix->closedir( static_cast<DAVIX_DIR*>(dirp), NULL); + d_ptr->removeDird(dirp); +} + +const char *TDavixSystem::GetDirEntry(void *dirp) { + struct dirent * dir; + DavixError* davixErr=NULL; + if( ((dir = d_ptr->davixPosix->readdir(static_cast<DAVIX_DIR*>(dirp), &davixErr) ) == NULL) + && ( davixErr != NULL ) ){ + Error("DavixReaddir", "failed to readdir the directory: %s (%d)", + davixErr->getErrMsg().c_str(), davixErr->getStatus()); + DavixError::clearError(&davixErr); + } + return (dir)?(dir->d_name):NULL; +} + +void *TDavixSystem::OpenDirectory(const char* dir) { + DavixError* davixErr=NULL; + DAVIX_DIR* d; + if( (d = d_ptr->davixPosix->opendir(d_ptr->davixParam, dir, &davixErr) ) == NULL){ + Error("DavixOpendir", "failed to opendir the directory: %s (%d)", + davixErr->getErrMsg().c_str(), davixErr->getStatus()); + DavixError::clearError(&davixErr); + }else{ + d_ptr->addDird(d); + } + return d; +} + +Bool_t TDavixSystem::ConsistentWith(const char * /*path*/, void *dirptr){ + return (Bool_t) d_ptr->isMyDird(dirptr); +} + +Int_t TDavixSystem::GetPathInfo(const char* path, FileStat_t &buf) { + struct stat st; + + if (!d_ptr->DavixStat(path, &st)) return 1; + buf.fDev = 0; + buf.fIno = 0; + buf.fMode = st.st_mode; // protection (combination of EFileModeMask bits) + + + buf.fUid = st.st_uid; // user id of owner + buf.fGid = st.st_gid; // group id of owner + buf.fSize = st.st_size; // total size in bytes + buf.fMtime = st.st_mtime; // modification date + buf.fIsLink = kFALSE; // symbolic link + buf.fUrl = path; // end point url of file + + return 0; +} + +Bool_t TDavixSystem::IsPathLocal(const char *path) { + (void) path; + return kFALSE; +} + +Int_t TDavixSystem::Locate(const char* path, TString &endurl) { + DavixError* davixErr=NULL; + ssize_t ret; + ReplicaVec vecRep; + DavFile f(*d_ptr->davixContext, Uri(path)); + if( (ret = f.getAllReplicas(d_ptr->davixParam, + vecRep, + &davixErr)) < 0){ + Error("DavixLocate", "failed to Locate file: %s (%d)", + davixErr->getErrMsg().c_str(), davixErr->getStatus()); + DavixError::clearError(&davixErr); + return 1; + } + if(vecRep.size() > 0){ + endurl = vecRep[0].getString().c_str(); + }else{ + endurl = path; + } + if (gDebug > 0) + Info("DavixLocate", "Davix Locate %s to %s", path, endurl.Data()); + + return 0; +} + +Int_t TDavixSystem::MakeDirectory(const char* dir) { + DavixError* davixErr=NULL; + int ret; + if( (ret = d_ptr->davixPosix->mkdir(d_ptr->davixParam, dir, 0755, &davixErr) ) < 0){ + Error("DavixMkdir", "failed to create the directory: %s (%d)", + davixErr->getErrMsg().c_str(), davixErr->getStatus()); + DavixError::clearError(&davixErr); + } + return ret; +} + +int TDavixSystem::Unlink(const char *path) { + DavixError* davixErr=NULL; + int ret; + if( (ret = d_ptr->davixPosix->unlink(d_ptr->davixParam, path, &davixErr) ) < 0){ + Error("DavixUnlink", "failed to unlink the file: %s (%d)", + davixErr->getErrMsg().c_str(), davixErr->getStatus()); + DavixError::clearError(&davixErr); + } + return ret; +} +