Skip to content
Snippets Groups Projects
Commit 0b96b866 authored by Jakob Blomer's avatar Jakob Blomer
Browse files

[rawfile, davix] add support multi-range HTTP queries

parent 07e47d55
No related branches found
No related tags found
No related merge requests found
...@@ -133,6 +133,9 @@ protected: ...@@ -133,6 +133,9 @@ protected:
/// Derived classes with mmap support must be able to unmap the memory area handed out by Map() /// Derived classes with mmap support must be able to unmap the memory area handed out by Map()
virtual void DoUnmap(void *region, size_t nbytes); virtual void DoUnmap(void *region, size_t nbytes);
/// By default implemented as a loop of ReadAt calls but can be overwritten, e.g. XRootD or DAVIX implementations
virtual void DoReadV(RIOVec *ioVec, unsigned int nReq);
public: public:
RRawFile(std::string_view url, ROptions options); RRawFile(std::string_view url, ROptions options);
RRawFile(const RRawFile &) = delete; RRawFile(const RRawFile &) = delete;
...@@ -161,8 +164,8 @@ public: ...@@ -161,8 +164,8 @@ public:
/// Returns the size of the file /// Returns the size of the file
std::uint64_t GetSize(); std::uint64_t GetSize();
/// By default implemented as a loop of ReadAt calls but can be overwritten, e.g. XRootD or DAVIX implementations /// Opens the file if necessary and calls DoReadV
virtual void ReadV(RIOVec *ioVec, unsigned int nReq); void ReadV(RIOVec *ioVec, unsigned int nReq);
/// Memory mapping according to POSIX standard; in particular, new mappings of the same range replace older ones. /// Memory mapping according to POSIX standard; in particular, new mappings of the same range replace older ones.
/// Mappings need to be aligned at page boundaries, therefore the real offset can be smaller than the desired value. /// Mappings need to be aligned at page boundaries, therefore the real offset can be smaller than the desired value.
......
...@@ -98,6 +98,13 @@ void *ROOT::Experimental::Detail::RRawFile::DoMap(size_t /* nbytes */, std::uint ...@@ -98,6 +98,13 @@ void *ROOT::Experimental::Detail::RRawFile::DoMap(size_t /* nbytes */, std::uint
throw std::runtime_error("Memory mapping unsupported"); throw std::runtime_error("Memory mapping unsupported");
} }
void ROOT::Experimental::Detail::RRawFile::DoReadV(RIOVec *ioVec, unsigned int nReq)
{
for (unsigned i = 0; i < nReq; ++i) {
ioVec[i].fOutBytes = ReadAt(ioVec[i].fBuffer, ioVec[i].fSize, ioVec[i].fOffset);
}
}
void ROOT::Experimental::Detail::RRawFile::DoUnmap(void * /* region */, size_t /* nbytes */) void ROOT::Experimental::Detail::RRawFile::DoUnmap(void * /* region */, size_t /* nbytes */)
{ {
throw std::runtime_error("Memory mapping unsupported"); throw std::runtime_error("Memory mapping unsupported");
...@@ -195,9 +202,10 @@ size_t ROOT::Experimental::Detail::RRawFile::ReadAt(void *buffer, size_t nbytes, ...@@ -195,9 +202,10 @@ size_t ROOT::Experimental::Detail::RRawFile::ReadAt(void *buffer, size_t nbytes,
void ROOT::Experimental::Detail::RRawFile::ReadV(RIOVec *ioVec, unsigned int nReq) void ROOT::Experimental::Detail::RRawFile::ReadV(RIOVec *ioVec, unsigned int nReq)
{ {
for (unsigned i = 0; i < nReq; ++i) { if (!fIsOpen)
ioVec[i].fOutBytes = ReadAt(ioVec[i].fBuffer, ioVec[i].fSize, ioVec[i].fOffset); DoOpen();
} fIsOpen = true;
DoReadV(ioVec, nReq);
} }
bool ROOT::Experimental::Detail::RRawFile::Readln(std::string &line) bool ROOT::Experimental::Detail::RRawFile::Readln(std::string &line)
......
...@@ -41,6 +41,7 @@ private: ...@@ -41,6 +41,7 @@ private:
protected: protected:
void DoOpen() final; void DoOpen() final;
size_t DoReadAt(void *buffer, size_t nbytes, std::uint64_t offset) final; size_t DoReadAt(void *buffer, size_t nbytes, std::uint64_t offset) final;
void DoReadV(RIOVec *ioVec, unsigned int nReq) final;
std::uint64_t DoGetSize() final; std::uint64_t DoGetSize() final;
public: public:
......
...@@ -88,3 +88,25 @@ size_t ROOT::Experimental::Detail::RRawFileDavix::DoReadAt(void *buffer, size_t ...@@ -88,3 +88,25 @@ size_t ROOT::Experimental::Detail::RRawFileDavix::DoReadAt(void *buffer, size_t
} }
return static_cast<size_t>(retval); return static_cast<size_t>(retval);
} }
void ROOT::Experimental::Detail::RRawFileDavix::DoReadV(RIOVec *ioVec, unsigned int nReq)
{
Davix::DavixError *davixErr = NULL;
Davix::DavIOVecInput in[nReq];
Davix::DavIOVecOuput out[nReq];
for (unsigned int i = 0; i < nReq; ++i) {
in[i].diov_buffer = ioVec[i].fBuffer;
in[i].diov_offset = ioVec[i].fOffset;
in[i].diov_size = ioVec[i].fSize;
}
auto ret = fFileDes->pos.preadVec(fFileDes->fd, in, out, nReq, &davixErr);
if (ret < 0) {
throw std::runtime_error("Cannot do vector read from '" + fUrl + "', error: " + davixErr->getErrMsg());
}
for (unsigned int i = 0; i < nReq; ++i) {
ioVec[i].fOutBytes = out[i].diov_size;
}
}
...@@ -38,3 +38,27 @@ TEST(RRawFileDavix, Eof) ...@@ -38,3 +38,27 @@ TEST(RRawFileDavix, Eof)
EXPECT_EQ(3u, nbytes); EXPECT_EQ(3u, nbytes);
EXPECT_STREQ("ld\n", tail); EXPECT_STREQ("ld\n", tail);
} }
TEST(RRawFileDavix, ReadV)
{
RRawFile::ROptions options;
options.fBlockSize = 0;
std::unique_ptr<RRawFileDavix> f(new RRawFileDavix("http://root.cern.ch/files/davix.test", options));
char buffer[2];
buffer[0] = buffer[1] = 0;
RRawFile::RIOVec iovec[2];
iovec[0].fBuffer = &buffer[0];
iovec[0].fOffset = 0;
iovec[0].fSize = 1;
iovec[1].fBuffer = &buffer[1];
iovec[1].fOffset = 11;
iovec[1].fSize = 1;
f->ReadV(iovec, 2);
EXPECT_EQ(1U, iovec[0].fOutBytes);
EXPECT_EQ(1U, iovec[1].fOutBytes);
EXPECT_EQ('H', buffer[0]);
EXPECT_EQ('d', buffer[1]);
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment