From 0b96b866d54723ae1329524f3aed75e3b72455d3 Mon Sep 17 00:00:00 2001
From: Jakob Blomer <jblomer@cern.ch>
Date: Sat, 12 Oct 2019 17:53:18 +0200
Subject: [PATCH] [rawfile, davix] add support multi-range HTTP queries
---
io/io/inc/ROOT/RRawFile.hxx | 7 +++++--
io/io/src/RRawFile.cxx | 14 +++++++++++---
net/davix/inc/ROOT/RRawFileDavix.hxx | 1 +
net/davix/src/RRawFileDavix.cxx | 22 ++++++++++++++++++++++
net/davix/test/RRawFileDavix.cxx | 24 ++++++++++++++++++++++++
5 files changed, 63 insertions(+), 5 deletions(-)
diff --git a/io/io/inc/ROOT/RRawFile.hxx b/io/io/inc/ROOT/RRawFile.hxx
index f7f35b893fc..1cf7849c9c8 100644
--- a/io/io/inc/ROOT/RRawFile.hxx
+++ b/io/io/inc/ROOT/RRawFile.hxx
@@ -133,6 +133,9 @@ protected:
/// Derived classes with mmap support must be able to unmap the memory area handed out by Map()
virtual void DoUnmap(void *region, size_t nbytes);
+ /// By default implemented as a loop of ReadAt calls but can be overwritten, e.g. XRootD or DAVIX implementations
+ virtual void DoReadV(RIOVec *ioVec, unsigned int nReq);
+
public:
RRawFile(std::string_view url, ROptions options);
RRawFile(const RRawFile &) = delete;
@@ -161,8 +164,8 @@ public:
/// Returns the size of the file
std::uint64_t GetSize();
- /// By default implemented as a loop of ReadAt calls but can be overwritten, e.g. XRootD or DAVIX implementations
- virtual void ReadV(RIOVec *ioVec, unsigned int nReq);
+ /// Opens the file if necessary and calls DoReadV
+ void ReadV(RIOVec *ioVec, unsigned int nReq);
/// Memory mapping according to POSIX standard; in particular, new mappings of the same range replace older ones.
/// Mappings need to be aligned at page boundaries, therefore the real offset can be smaller than the desired value.
diff --git a/io/io/src/RRawFile.cxx b/io/io/src/RRawFile.cxx
index 375aecf8886..66f335f00e7 100644
--- a/io/io/src/RRawFile.cxx
+++ b/io/io/src/RRawFile.cxx
@@ -98,6 +98,13 @@ void *ROOT::Experimental::Detail::RRawFile::DoMap(size_t /* nbytes */, std::uint
throw std::runtime_error("Memory mapping unsupported");
}
+void ROOT::Experimental::Detail::RRawFile::DoReadV(RIOVec *ioVec, unsigned int nReq)
+{
+ for (unsigned i = 0; i < nReq; ++i) {
+ ioVec[i].fOutBytes = ReadAt(ioVec[i].fBuffer, ioVec[i].fSize, ioVec[i].fOffset);
+ }
+}
+
void ROOT::Experimental::Detail::RRawFile::DoUnmap(void * /* region */, size_t /* nbytes */)
{
throw std::runtime_error("Memory mapping unsupported");
@@ -195,9 +202,10 @@ size_t ROOT::Experimental::Detail::RRawFile::ReadAt(void *buffer, size_t nbytes,
void ROOT::Experimental::Detail::RRawFile::ReadV(RIOVec *ioVec, unsigned int nReq)
{
- for (unsigned i = 0; i < nReq; ++i) {
- ioVec[i].fOutBytes = ReadAt(ioVec[i].fBuffer, ioVec[i].fSize, ioVec[i].fOffset);
- }
+ if (!fIsOpen)
+ DoOpen();
+ fIsOpen = true;
+ DoReadV(ioVec, nReq);
}
bool ROOT::Experimental::Detail::RRawFile::Readln(std::string &line)
diff --git a/net/davix/inc/ROOT/RRawFileDavix.hxx b/net/davix/inc/ROOT/RRawFileDavix.hxx
index c1d42adcd7c..d413c851d19 100644
--- a/net/davix/inc/ROOT/RRawFileDavix.hxx
+++ b/net/davix/inc/ROOT/RRawFileDavix.hxx
@@ -41,6 +41,7 @@ private:
protected:
void DoOpen() final;
size_t DoReadAt(void *buffer, size_t nbytes, std::uint64_t offset) final;
+ void DoReadV(RIOVec *ioVec, unsigned int nReq) final;
std::uint64_t DoGetSize() final;
public:
diff --git a/net/davix/src/RRawFileDavix.cxx b/net/davix/src/RRawFileDavix.cxx
index 677a0b531ce..81e20e79850 100644
--- a/net/davix/src/RRawFileDavix.cxx
+++ b/net/davix/src/RRawFileDavix.cxx
@@ -88,3 +88,25 @@ size_t ROOT::Experimental::Detail::RRawFileDavix::DoReadAt(void *buffer, size_t
}
return static_cast<size_t>(retval);
}
+
+void ROOT::Experimental::Detail::RRawFileDavix::DoReadV(RIOVec *ioVec, unsigned int nReq)
+{
+ Davix::DavixError *davixErr = NULL;
+ Davix::DavIOVecInput in[nReq];
+ Davix::DavIOVecOuput out[nReq];
+
+ for (unsigned int i = 0; i < nReq; ++i) {
+ in[i].diov_buffer = ioVec[i].fBuffer;
+ in[i].diov_offset = ioVec[i].fOffset;
+ in[i].diov_size = ioVec[i].fSize;
+ }
+
+ auto ret = fFileDes->pos.preadVec(fFileDes->fd, in, out, nReq, &davixErr);
+ if (ret < 0) {
+ throw std::runtime_error("Cannot do vector read from '" + fUrl + "', error: " + davixErr->getErrMsg());
+ }
+
+ for (unsigned int i = 0; i < nReq; ++i) {
+ ioVec[i].fOutBytes = out[i].diov_size;
+ }
+}
diff --git a/net/davix/test/RRawFileDavix.cxx b/net/davix/test/RRawFileDavix.cxx
index 2b84cd2c722..ec376e7a185 100644
--- a/net/davix/test/RRawFileDavix.cxx
+++ b/net/davix/test/RRawFileDavix.cxx
@@ -38,3 +38,27 @@ TEST(RRawFileDavix, Eof)
EXPECT_EQ(3u, nbytes);
EXPECT_STREQ("ld\n", tail);
}
+
+
+TEST(RRawFileDavix, ReadV)
+{
+ RRawFile::ROptions options;
+ options.fBlockSize = 0;
+ std::unique_ptr<RRawFileDavix> f(new RRawFileDavix("http://root.cern.ch/files/davix.test", options));
+
+ char buffer[2];
+ buffer[0] = buffer[1] = 0;
+ RRawFile::RIOVec iovec[2];
+ iovec[0].fBuffer = &buffer[0];
+ iovec[0].fOffset = 0;
+ iovec[0].fSize = 1;
+ iovec[1].fBuffer = &buffer[1];
+ iovec[1].fOffset = 11;
+ iovec[1].fSize = 1;
+ f->ReadV(iovec, 2);
+
+ EXPECT_EQ(1U, iovec[0].fOutBytes);
+ EXPECT_EQ(1U, iovec[1].fOutBytes);
+ EXPECT_EQ('H', buffer[0]);
+ EXPECT_EQ('d', buffer[1]);
+}
--
GitLab