From eef558ed37c04a0f9bab8e1753442feb00a73fb4 Mon Sep 17 00:00:00 2001
From: Philippe Canal <pcanal@fnal.gov>
Date: Thu, 9 Jun 2011 17:18:42 +0000
Subject: [PATCH] From Brian and Philippe:
Reduce the memory used by a TTree in half.
Refactor the code reading and writing the TBasket data.
A single transient buffer holding the compressed data is
now managed by TTree (and could be made thread local)
rather than having one per TBranch.
git-svn-id: http://root.cern.ch/svn/root/trunk@39642 27541ba8-7e3a-0410-8455-c3a389f83636
---
tree/tree/inc/TBasket.h | 16 +-
tree/tree/inc/TTree.h | 3 +
tree/tree/src/TBasket.cxx | 426 ++++++++++++++++++++++----------------
tree/tree/src/TTree.cxx | 21 ++
4 files changed, 278 insertions(+), 188 deletions(-)
diff --git a/tree/tree/inc/TBasket.h b/tree/tree/inc/TBasket.h
index 932b5df561c..c699b1dcb51 100644
--- a/tree/tree/inc/TBasket.h
+++ b/tree/tree/inc/TBasket.h
@@ -40,7 +40,14 @@ class TBasket : public TKey {
private:
TBasket(const TBasket&); // TBasket objects are not copiable.
TBasket& operator=(const TBasket&); // TBasket objects are not copiable.
-
+
+ // Internal corner cases for ReadBasketBuffers
+ Int_t ReadBasketBuffersUnzip(char*, Int_t, Bool_t, TFile*);
+ Int_t ReadBasketBuffersUncompressedCase();
+
+ // Helper for managing the compressed buffer.
+ void InitializeCompressedBuffer(Int_t len, TFile* file);
+
protected:
Int_t fBufferSize; //fBuffer length in bytes
Int_t fNevBufSize; //Length in Int_t of fEntryOffset OR fixed length of each entry if fEntryOffset is null!
@@ -50,9 +57,10 @@ protected:
Int_t *fDisplacement; //![fNevBuf] Displacement of entries in fBuffer(TKey)
Int_t *fEntryOffset; //[fNevBuf] Offset of entries in fBuffer(TKey)
TBranch *fBranch; //Pointer to the basket support branch
- Int_t fCompressedSize; //!Size of the allocated memroy in fCompressedBuffer
- char *fCompressedBuffer;//!Temporary place holder for the compressed buffer if needed.
-
+ TBuffer *fCompressedBufferRef; //! Compressed buffer.
+ Bool_t fOwnsCompressedBuffer; //! Whether or not we own the compressed buffer.
+ Int_t fLastWriteBufferSize; //! Size of the buffer last time we wrote it to disk
+
public:
TBasket();
diff --git a/tree/tree/inc/TTree.h b/tree/tree/inc/TTree.h
index f0ec9e8da52..cb62ea98a09 100644
--- a/tree/tree/inc/TTree.h
+++ b/tree/tree/inc/TTree.h
@@ -139,6 +139,7 @@ protected:
TList *fClones; //! List of cloned trees which share our addresses
TBranchRef *fBranchRef; // Branch supporting the TRefTable (if any)
UInt_t fFriendLockStatus; //! Record which method is locking the friend recursion
+ TBuffer *fTransientBuffer; //! Pointer to the current transient buffer.
static Int_t fgBranchStyle; // Old/New branch style
static Long64_t fgMaxTreeSize; // Maximum size of a file containg a Tree
@@ -408,6 +409,7 @@ public:
TTreeFormula *GetSelect() { return GetPlayer()->GetSelect(); }
virtual Long64_t GetSelectedRows() { return GetPlayer()->GetSelectedRows(); }
virtual Int_t GetTimerInterval() const { return fTimerInterval; }
+ TBuffer* GetTransientBuffer(Int_t size);
virtual Long64_t GetTotBytes() const { return fTotBytes; }
virtual TTree *GetTree() const { return const_cast<TTree*>(this); }
virtual TVirtualIndex *GetTreeIndex() const { return fTreeIndex; }
@@ -536,6 +538,7 @@ public:
virtual Int_t Write(const char *name=0, Int_t option=0, Int_t bufsize=0);
virtual Int_t Write(const char *name=0, Int_t option=0, Int_t bufsize=0) const;
+
ClassDef(TTree,19) //Tree descriptor (the main ROOT I/O class)
};
diff --git a/tree/tree/src/TBasket.cxx b/tree/tree/src/TBasket.cxx
index 94cb4f334f4..8148fb41f2b 100644
--- a/tree/tree/src/TBasket.cxx
+++ b/tree/tree/src/TBasket.cxx
@@ -20,6 +20,20 @@
#include "TVirtualPerfStats.h"
#include "TTimeStamp.h"
+// TODO: Copied from TBranch.cxx
+#if (__GNUC__ >= 3) || defined(__INTEL_COMPILER)
+#if !defined(R__unlikely)
+ #define R__unlikely(expr) __builtin_expect(!!(expr), 0)
+#endif
+#if !defined(R__likely)
+ #define R__likely(expr) __builtin_expect(!!(expr), 1)
+#endif
+#else
+ #define R__unlikely(expr) expr
+ #define R__likely(expr) expr
+#endif
+
+
extern "C" void R__zip (Int_t cxlevel, Int_t *nin, char *bufin, Int_t *lout, char *bufout, Int_t *nout);
extern "C" void R__unzip(Int_t *nin, UChar_t *bufin, Int_t *lout, char *bufout, Int_t *nout);
extern "C" int R__unzip_header(Int_t *nin, UChar_t *bufin, Int_t *lout);
@@ -37,7 +51,7 @@ ClassImp(TBasket)
//
//_______________________________________________________________________
-TBasket::TBasket() : fCompressedSize(0),fCompressedBuffer(0)
+TBasket::TBasket() : fCompressedBufferRef(0), fLastWriteBufferSize(0)
{
// Default contructor.
@@ -54,10 +68,9 @@ TBasket::TBasket() : fCompressedSize(0),fCompressedBuffer(0)
}
//_______________________________________________________________________
-TBasket::TBasket(TDirectory *motherDir) : TKey(motherDir),fCompressedSize(0),fCompressedBuffer(0)
+TBasket::TBasket(TDirectory *motherDir) : TKey(motherDir),fCompressedBufferRef(0), fLastWriteBufferSize(0)
{
// Constructor used during reading.
-
fDisplacement = 0;
fEntryOffset = 0;
fBufferRef = 0;
@@ -72,7 +85,7 @@ TBasket::TBasket(TDirectory *motherDir) : TKey(motherDir),fCompressedSize(0),fCo
//_______________________________________________________________________
TBasket::TBasket(const char *name, const char *title, TBranch *branch) :
- TKey(branch->GetDirectory()),fCompressedSize(0),fCompressedBuffer(0)
+ TKey(branch->GetDirectory()), fLastWriteBufferSize(0)
{
// Basket normal constructor, used during writing.
@@ -93,7 +106,14 @@ TBasket::TBasket(const char *name, const char *title, TBranch *branch) :
}
fHeaderOnly = kTRUE;
fLast = 0; // Must initialize before calling Streamer()
-
+ if (branch && branch->GetTree()) {
+ fCompressedBufferRef = branch->GetTree()->GetTransientBuffer(fBufferSize);
+ fOwnsCompressedBuffer = kFALSE;
+ if (!fCompressedBufferRef) {
+ fCompressedBufferRef = new TBufferFile(TBuffer::kRead, fBufferSize);
+ fOwnsCompressedBuffer = kTRUE;
+ }
+ }
Streamer(*fBufferRef);
fKeylen = fBufferRef->Length();
fObjlen = fBufferSize - fKeylen;
@@ -115,12 +135,16 @@ TBasket::~TBasket()
if (fDisplacement) delete [] fDisplacement;
if (fEntryOffset) delete [] fEntryOffset;
- if (fBuffer == fCompressedBuffer) fBuffer = 0;
- if (fCompressedBuffer) delete [] fCompressedBuffer;
+ if (fBufferRef) delete fBufferRef;
+ fBufferRef = 0;
+ fBuffer = 0;
fDisplacement= 0;
fEntryOffset = 0;
- fCompressedSize = 0;
- fCompressedBuffer = 0;
+ // Note we only delete the compressed buffer if we own it
+ if (fCompressedBufferRef && fOwnsCompressedBuffer) {
+ delete fCompressedBufferRef;
+ fCompressedBufferRef = 0;
+ }
}
//_______________________________________________________________________
@@ -174,19 +198,17 @@ void TBasket::DeleteEntryOffset()
Int_t TBasket::DropBuffers()
{
// Drop buffers of this basket if it is not the current basket.
-
if (!fBuffer && !fBufferRef) return 0;
if (fDisplacement) delete [] fDisplacement;
if (fEntryOffset) delete [] fEntryOffset;
if (fBufferRef) delete fBufferRef;
- if (fCompressedBuffer) delete [] fCompressedBuffer;
+ if (fCompressedBufferRef && fOwnsCompressedBuffer) delete fCompressedBufferRef;
fBufferRef = 0;
+ fCompressedBufferRef = 0;
fBuffer = 0;
fDisplacement= 0;
fEntryOffset = 0;
- fCompressedSize = 0;
- fCompressedBuffer = 0;
fBranch->GetTree()->IncrementTotalBuffers(-fBufferSize);
return fBufferSize;
}
@@ -284,6 +306,91 @@ void TBasket::MoveEntries(Int_t dentries)
fNevBuf -= dentries;
}
+#define OLD_CASE_EXPRESSION fObjlen==fNbytes-fKeylen && GetBranch()->GetCompressionLevel()!=0 && file->GetVersion()<=30401
+//_______________________________________________________________________
+Int_t TBasket::ReadBasketBuffersUncompressedCase()
+{
+ // By-passing buffer unzipping has been requested and is
+ // possible (only 1 entry in this basket).
+ fBuffer = fBufferRef->Buffer();
+
+ // Make sure that the buffer is set at the END of the data
+ fBufferRef->SetBufferOffset(fNbytes);
+
+ // Indicate that this buffer is weird.
+ fBufferRef->SetBit(TBufferFile::kNotDecompressed);
+
+ // Usage of this mode assume the existance of only ONE
+ // entry in this basket.
+ delete [] fEntryOffset; fEntryOffset = 0;
+ delete [] fDisplacement; fDisplacement = 0;
+
+ fBranch->GetTree()->IncrementTotalBuffers(fBufferSize);
+ return 0;
+}
+
+//_______________________________________________________________________
+Int_t TBasket::ReadBasketBuffersUnzip(char* buffer, Int_t size, Bool_t mustFree, TFile* file)
+{
+ // We always create the TBuffer for the basket but it hold the buffer from the cache.
+ if (fBufferRef) {
+ fBufferRef->SetBuffer(buffer, size, mustFree);
+ fBufferRef->SetReadMode();
+ fBufferRef->Reset();
+ } else {
+ fBufferRef = new TBufferFile(TBuffer::kRead, size, buffer, mustFree);
+ }
+ fBufferRef->SetParent(file);
+
+ Streamer(*fBufferRef);
+
+ if (IsZombie()) {
+ return -1;
+ }
+
+ Bool_t oldCase = OLD_CASE_EXPRESSION;
+
+ if ((fObjlen > fNbytes-fKeylen || oldCase) && TestBit(TBufferFile::kNotDecompressed) && (fNevBuf==1)) {
+ return TBasket::ReadBasketBuffersUncompressedCase();
+ }
+
+ fBuffer = fBufferRef->Buffer();
+ return fObjlen+fKeylen;
+}
+
+//_______________________________________________________________________
+static inline TBuffer* R__initializeReadBasketBuffer(TBuffer* bufferRef, Int_t len, TFile* file)
+{
+ // Initialize a buffer for reading if it is not already initialized
+
+ TBuffer* result;
+ if (R__likely(bufferRef)) {
+ bufferRef->SetReadMode();
+ Int_t curBufferSize = bufferRef->BufferSize();
+ if (curBufferSize < len) {
+ // Experience shows that giving 5% "wiggle-room" decreases churn.
+ bufferRef->Expand(Int_t(len*1.05));
+ }
+ bufferRef->Reset();
+ result = bufferRef;
+ } else {
+ result = new TBufferFile(TBuffer::kRead, len);
+ }
+ result->SetParent(file);
+ return result;
+}
+
+//_______________________________________________________________________
+void TBasket::InitializeCompressedBuffer(Int_t len, TFile* file)
+{
+ // Initialize the compressed buffer; either from the TTree or create a local one.
+ Bool_t compressedBufferExists = fCompressedBufferRef != NULL;
+ fCompressedBufferRef = R__initializeReadBasketBuffer(fCompressedBufferRef, len, file);
+ if (!compressedBufferExists) {
+ fOwnsCompressedBuffer = kTRUE;
+ }
+}
+
//_______________________________________________________________________
Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file)
{
@@ -301,182 +408,119 @@ Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file)
// There is a lot of code duplication but it was necesary to assure
// the expected behavior when there is no cache.
-
if(!fBranch->GetDirectory()) {
return -1;
- }
- Int_t badread= 0;
+ }
+
+ Bool_t oldCase;
+ char *rawUncompressedBuffer, *rawCompressedBuffer;
+ Int_t uncompressedBufferLen;
+ // See if the cache has already unzipped the buffer for us.
TFileCacheRead *pf = file->GetCacheRead();
- char *buffer = 0;
- Bool_t free = kTRUE; // Must we free this buffer or does it make part of the cache?
- Int_t res = -1;
+ if (pf) {
+ Int_t res = -1;
+ Bool_t free = kTRUE;
+ char *buffer;
+ res = pf->GetUnzipBuffer(&buffer, pos, len, &free);
+ if (R__unlikely(res >= 0)) {
+ len = ReadBasketBuffersUnzip(buffer, res, free, file);
+ // Note that in the kNotDecompressed case, the above function will return 0;
+ // In such a case, we should stop processing
+ if (len <= 0) return -len;
+ goto AfterBuffer;
+ }
+ }
- if (pf) res = pf->GetUnzipBuffer(&buffer, pos, len, &free);
+ // Initialize the buffer to hold the compressed data.
+ InitializeCompressedBuffer(len, file);
+ if (!fCompressedBufferRef) {
+ Error("ReadBasketBuffers", "Unable to allocate buffer.");
+ return 1;
+ }
+ rawCompressedBuffer = fCompressedBufferRef->Buffer();
- if (res >= 0) {
+ // Read from the file and unstream the header information.
+ if (file->ReadBuffer(rawCompressedBuffer,pos,len)) {
+ return 1;
+ }
+ Streamer(*fCompressedBufferRef);
+ if (IsZombie()) {
+ return 1;
+ }
- // We always create the TBuffer for the basket but it will be a shell only,
- // since we pass the pointer to the low level buffer
- if (fBufferRef) {
- fBufferRef->SetBuffer(buffer, res, free);
- fBufferRef->SetReadMode();
- fBufferRef->Reset();
- } else {
- fBufferRef = new TBufferFile(TBuffer::kRead, res, buffer, free);
+ // Initialize buffer to hold the uncompressed data
+ // Note that in previous versions we didn't allocate buffers until we verified
+ // the zip headers; this is no longer beforehand as the buffer lifetime is scoped
+ // to the TBranch.
+ uncompressedBufferLen = len > fObjlen+fKeylen ? len : fObjlen+fKeylen;
+ fBufferRef = R__initializeReadBasketBuffer(fBufferRef, uncompressedBufferLen, file);
+ rawUncompressedBuffer = fBufferRef->Buffer();
+ fBuffer = rawUncompressedBuffer;
+
+ oldCase = OLD_CASE_EXPRESSION;
+ // Case where ROOT thinks the buffer is compressed. Copy over the key and uncompress the object
+ if (fObjlen > fNbytes-fKeylen || oldCase) {
+ if (R__unlikely(TestBit(TBufferFile::kNotDecompressed) && (fNevBuf==1))) {
+ return ReadBasketBuffersUncompressedCase();
}
- fBufferRef->SetParent(file);
- Streamer(*fBufferRef);
-
- if (IsZombie()) {
- badread = 1;
- return badread;
+ // Optional monitor for zip time profiling.
+ Double_t start;
+ if (R__unlikely(gPerfStats)) {
+ start = TTimeStamp();
}
- Bool_t oldCase = fObjlen==fNbytes-fKeylen
- && GetBranch()->GetCompressionLevel()!=0
- && file->GetVersion()<=30401;
- if (fObjlen > fNbytes-fKeylen || oldCase) {
- if (TestBit(TBufferFile::kNotDecompressed) && (fNevBuf==1)) {
- // By-passing buffer unzipping has been requested and is
- // possible (only 1 entry in this basket).
- fBuffer = fBufferRef->Buffer();
-
- // Make sure that the buffer is set at the END of the data
- fBufferRef->SetBufferOffset(fNbytes);
-
- // Indicate that this buffer is weird.
- fBufferRef->SetBit(TBufferFile::kNotDecompressed);
-
- // Usage of this mode assume the existance of only ONE
- // entry in this basket.
- delete [] fEntryOffset; fEntryOffset = 0;
- delete [] fDisplacement; fDisplacement = 0;
-
- fBranch->GetTree()->IncrementTotalBuffers(fBufferSize);
- return badread;
+ memcpy(rawUncompressedBuffer, rawCompressedBuffer, fKeylen);
+ char *rawUncompressedObjectBuffer = rawUncompressedBuffer+fKeylen;
+ UChar_t *rawCompressedObjectBuffer = (UChar_t*)rawCompressedBuffer+fKeylen;
+ Int_t nin, nbuf;
+ Int_t nout = 0, noutot = 0, nintot = 0;
+
+ // Unzip all the compressed objects in the compressed object buffer.
+ while (1) {
+ // Check the header for errors.
+ if (R__unlikely(R__unzip_header(&nin, rawCompressedObjectBuffer, &nbuf) != 0)) {
+ Error("ReadBasketBuffers", "Inconsistency found in header (nin=%d, nbuf=%d)", nin, nbuf);
+ break;
}
- }
-
- fBuffer = fBufferRef->Buffer();
- len = fObjlen+fKeylen;
- }
- else{
- if (fBufferRef) {
- fBufferRef->SetReadMode();
- if (fBufferRef->BufferSize() < len) {
- fBufferRef->Expand(len);
+ if (R__unlikely(oldCase && (nin > fObjlen || nbuf > fObjlen))) {
+ //buffer was very likely not compressed in an old version
+ memcpy(rawUncompressedBuffer+fKeylen, rawCompressedObjectBuffer+fKeylen, fObjlen);
+ goto AfterBuffer;
}
- fBufferRef->Reset();
- } else {
- fBufferRef = new TBufferFile(TBuffer::kRead, len);
- }
- fBufferRef->SetParent(file);
- buffer = fBufferRef->Buffer();
- if (file->ReadBuffer(buffer,pos,len)) {
- badread = 1;
- return badread;
+ R__unzip(&nin, rawCompressedObjectBuffer, &nbuf, rawUncompressedObjectBuffer, &nout);
+ if (!nout) break;
+ noutot += nout;
+ nintot += nin;
+ if (noutot >= fObjlen) break;
+ rawCompressedObjectBuffer += nin;
+ rawUncompressedObjectBuffer += nout;
}
- Streamer(*fBufferRef);
-
- if (IsZombie()) {
- badread = 1;
- return badread;
+ // Make sure the uncompressed numbers are consistent with header.
+ if (R__unlikely(noutot != fObjlen)) {
+ Error("ReadBasketBuffers", "fNbytes = %d, fKeylen = %d, fObjlen = %d, noutot = %d, nout=%d, nin=%d, nbuf=%d", fNbytes,fKeylen,fObjlen, noutot,nout,nin,nbuf);
+ fBranch->GetTree()->IncrementTotalBuffers(fBufferSize);
+ return 1;
}
-
- Bool_t oldCase = fObjlen==fNbytes-fKeylen
- && GetBranch()->GetCompressionLevel()!=0
- && file->GetVersion()<=30401;
- if (fObjlen > fNbytes-fKeylen || oldCase) {
- if (TestBit(TBufferFile::kNotDecompressed) && (fNevBuf==1)) {
- // By-passing buffer unzipping has been requested and is
- // possible (only 1 entry in this basket).
- fBuffer = fBufferRef->Buffer();
-
- // Make sure that the buffer is set at the END of the data
- fBufferRef->SetBufferOffset(fNbytes);
-
- // Indicate that this buffer is weird.
- fBufferRef->SetBit(TBufferFile::kNotDecompressed);
-
- // Usage of this mode assume the existance of only ONE
- // entry in this basket.
- delete [] fEntryOffset; fEntryOffset = 0;
- delete [] fDisplacement; fDisplacement = 0;
-
- fBranch->GetTree()->IncrementTotalBuffers(fBufferSize);
- return badread;
- }
-
- // For monitoring the cost of the unzipping.
- Double_t start = 0;
- if (gPerfStats != 0) start = TTimeStamp();
-
- if ((fObjlen+fKeylen) > fCompressedSize) {
- /* early consistency check before potentially large memory is being allocated */
- Int_t nin, nbuf;
- UChar_t *bufcur = (UChar_t *)&buffer[fKeylen];
- if (R__unzip_header(&nin, bufcur, &nbuf)!=0) {
- Error("ReadBasketBuffers", "Inconsistency found in header (nin=%d, nbuf=%d)", nin, nbuf);
- badread = 1;
- return badread;
- }
- if (fCompressedSize) delete [] fCompressedBuffer;
- fCompressedSize = fObjlen+fKeylen;
- fCompressedBuffer = new char[fCompressedSize];
- }
- fBuffer = fCompressedBuffer;
- memcpy(fBuffer,buffer,fKeylen);
- char *objbuf = fBuffer + fKeylen;
- UChar_t *bufcur = (UChar_t *)&buffer[fKeylen];
- Int_t nin, nbuf;
- Int_t nout = 0;
- Int_t noutot = 0;
- while (1) {
- Int_t hc = R__unzip_header(&nin, bufcur, &nbuf);
- if (hc!=0) break;
- if (oldCase && (nin > fObjlen || nbuf > fObjlen)) {
- //buffer was very likely not compressed in an old version
- delete [] fBuffer;
- fBuffer = fBufferRef->Buffer();
- goto AfterBuffer;
- }
- R__unzip(&nin, bufcur, &nbuf, objbuf, &nout);
- if (!nout) break;
- noutot += nout;
- if (noutot >= fObjlen) break;
- bufcur += nin;
- objbuf += nout;
- }
- if (noutot != fObjlen) {
- Error("ReadBasketBuffers", "fNbytes = %d, fKeylen = %d, fObjlen = %d, noutot = %d, nout=%d, nin=%d, nbuf=%d", fNbytes,fKeylen,fObjlen, noutot,nout,nin,nbuf);
- badread = 1;
- }
- // Switch the 2 buffers
- char *temp = fBufferRef->Buffer();
- Int_t templen = fBufferRef->BufferSize();
- fBufferRef->ResetBit(TBuffer::kIsOwner);
- fBufferRef->SetBuffer(fBuffer, fCompressedSize, kTRUE); // Do adopt the buffer.
- fCompressedBuffer = temp;
- fCompressedSize = templen;
- len = fObjlen+fKeylen;
- if (gPerfStats != 0) {
- gPerfStats->FileUnzipEvent(file,pos,start,fCompressedSize,fObjlen);
- }
- } else {
- fBuffer = fBufferRef->Buffer();
+ len = fObjlen+fKeylen;
+ if (R__unlikely(gPerfStats)) {
+ gPerfStats->FileUnzipEvent(file,pos,start,nintot,fObjlen);
}
+ } else {
+ // Nothing is compressed - copy over wholesale.
+ memcpy(rawUncompressedBuffer, rawCompressedBuffer, len);
}
- AfterBuffer:
- fBranch->GetTree()->IncrementTotalBuffers(fBufferSize);
+AfterBuffer:
- // read offsets table
- if (badread || !fBranch->GetEntryOffsetLen()) {
- return badread;
+ fBranch->GetTree()->IncrementTotalBuffers(fBufferSize);
+
+ // Read offsets table if needed.
+ if (!fBranch->GetEntryOffsetLen()) {
+ return 0;
}
delete [] fEntryOffset;
fEntryOffset = 0;
@@ -486,19 +530,20 @@ Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file)
fEntryOffset = new Int_t[fNevBuf+1];
fEntryOffset[0] = fKeylen;
Warning("ReadBasketBuffers","basket:%s has fNevBuf=%d but fEntryOffset=0, pos=%lld, len=%d, fNbytes=%d, fObjlen=%d, trying to repair",GetName(),fNevBuf,pos,len,fNbytes,fObjlen);
- return badread;
+ return 0;
}
+ // Read the array of diplacement if any.
delete [] fDisplacement;
- fDisplacement = 0;
+ fDisplacement = 0;
if (fBufferRef->Length() != len) {
// There is more data in the buffer! It is the displacement
// array. If len is less than TBuffer::kMinimalSize the actual
// size of the buffer is too large, so we can not use the
// fBufferRef->BufferSize()
fBufferRef->ReadArray(fDisplacement);
- }
+ }
- return badread;
+ return 0;
}
//_______________________________________________________________________
@@ -531,13 +576,14 @@ void TBasket::Reset()
Int_t curSize = fBufferRef->BufferSize();
// fBufferLen at this point is already reset, so use indirect measurements
Int_t curLen = (GetObjlen() + GetKeylen());
+ Long_t newSize = -1;
if (curSize > 2*curLen)
{
Long_t curBsize = fBranch->GetBasketSize();
if (curSize > 2*curBsize ) {
Long_t avgSize = (Long_t)(fBranch->GetTotBytes() / (1+fBranch->GetWriteBasket())); // Average number of bytes per basket so far
if (curSize > 2*avgSize) {
- Long_t newSize = curBsize;
+ newSize = curBsize;
if (curLen > newSize) {
newSize = curLen;
}
@@ -545,10 +591,22 @@ void TBasket::Reset()
newSize = avgSize;
}
newSize = newSize + 512 - newSize%512; // Wiggle room and alignment (512 is same as in OptimizeBaskets)
- fBufferRef->Expand(newSize,kFALSE); // Expand without copying the existing data.
}
}
}
+ /*
+ Philippe has asked us to keep this turned off until we finish memory fragmentation studies.
+ // If fBufferRef grew since we last saw it, shrink it to 105% of the occupied size
+ if (curSize > fLastWriteBufferSize) {
+ if (newSize == -1) {
+ newSize = Int_t(1.05*Float_t(fBufferRef->Length()));
+ }
+ fLastWriteBufferSize = newSize;
+ }
+ */
+ if (newSize != -1) {
+ fBufferRef->Expand(newSize,kFALSE); // Expand without copying the existing data.
+ }
TKey::Reset();
@@ -779,8 +837,8 @@ Int_t TBasket::WriteBuffer()
return -1;
}
fMotherDir = file; // fBranch->GetDirectory();
-
- if (fBufferRef->TestBit(TBufferFile::kNotDecompressed)) {
+
+ if (R__unlikely(fBufferRef->TestBit(TBufferFile::kNotDecompressed))) {
// Read the basket information that was saved inside the buffer.
Bool_t writing = fBufferRef->IsWriting();
fBufferRef->SetReadMode();
@@ -830,12 +888,13 @@ Int_t TBasket::WriteBuffer()
//if (cxlevel == 2) cxlevel--; RB: I cannot remember why we had this!
Int_t nbuffers = fObjlen/kMAXBUF;
Int_t buflen = fKeylen + fObjlen + 28; //add 28 bytes in case object is placed in a deleted gap
- if (buflen > fCompressedSize) {
- if (fCompressedSize) delete [] fCompressedBuffer;
- fCompressedSize = buflen;
- fCompressedBuffer = new char[fCompressedSize];
+ InitializeCompressedBuffer(buflen, file);
+ if (!fCompressedBufferRef) {
+ Warning("WriteBuffer", "Unable to allocate the compressed buffer");
+ return -1;
}
- fBuffer = fCompressedBuffer;
+ fCompressedBufferRef->SetWriteMode();
+ fBuffer = fCompressedBufferRef->Buffer();
char *objbuf = fBufferRef->Buffer() + fKeylen;
char *bufcur = &fBuffer[fKeylen];
noutot = 0;
@@ -851,9 +910,8 @@ Int_t TBasket::WriteBuffer()
// buffer is larger than the input. In this case, we write the original uncompressed buffer
if (nout == 0 || nout >= fObjlen) {
nout = fObjlen;
- // We use do delete fBuffer here, we no longer want to since
- // the buffer (held by fCompressedBuffer) might be re-used later.
- // delete [] fBuffer;
+ // We used to delete fBuffer here, we no longer want to since
+ // the buffer (held by fCompressedBufferRef) might be re-used later.
fBuffer = fBufferRef->Buffer();
Create(fObjlen,file);
fBufferRef->SetBufferOffset(0);
diff --git a/tree/tree/src/TTree.cxx b/tree/tree/src/TTree.cxx
index 1ca2797eb20..34e630e300f 100644
--- a/tree/tree/src/TTree.cxx
+++ b/tree/tree/src/TTree.cxx
@@ -635,6 +635,7 @@ TTree::TTree()
, fClones(0)
, fBranchRef(0)
, fFriendLockStatus(0)
+, fTransientBuffer(0)
{
// Default constructor and I/O constructor.
//
@@ -701,6 +702,7 @@ TTree::TTree(const char* name, const char* title, Int_t splitlevel /* = 99 */)
, fClones(0)
, fBranchRef(0)
, fFriendLockStatus(0)
+, fTransientBuffer(0)
{
// Normal tree constructor.
//
@@ -843,6 +845,25 @@ TTree::~TTree()
// Must be done after the destruction of friends.
// Note: We do *not* own our directory.
fDirectory = 0;
+
+ if (fTransientBuffer) {
+ delete fTransientBuffer;
+ fTransientBuffer = 0;
+ }
+}
+
+//______________________________________________________________________________
+TBuffer* TTree::GetTransientBuffer(Int_t size)
+{
+ // Returns the transient buffer currently used by this TTree for reading/writing baskets
+ if (fTransientBuffer) {
+ if (fTransientBuffer->BufferSize() < size) {
+ fTransientBuffer->Expand(size);
+ }
+ return fTransientBuffer;
+ }
+ fTransientBuffer = new TBufferFile(TBuffer::kRead, size);
+ return fTransientBuffer;
}
//______________________________________________________________________________
--
GitLab